feat(sanitize): in-place, length-preserving log de-identification
Adds internal/sanitize: rewrites the customer-identifying spans that internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with same-length neutral fillers, in place, without changing the file format. - Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP -> "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed. - Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes, and all embedded timestamps preserved; untouched zip entries copied raw; member payload length unchanged so tar headers stay byte-identical; only the .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output. - privacy.FindSpans is the one matcher shared by detection and redaction; fillers are recognised by isRedactionFiller so a re-scan / second pass is a no-op. New privacy FPs fixed along the way: syslog selectors (local7.info), "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer domains. - Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and unreadable nested archives are reported in Result.SkippedBinary, never edited. - Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить и скачать копию" button in the Customer-data panel, and logpile -sanitize <file> (restores mtime/atime). Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C .sds yields the identical hardware inventory; re-scan is clean. ADL-067, bible-local/docs/log-sanitization.md. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
e74e01ad05
commit
a63bb17438
@@ -25,14 +25,26 @@ var (
|
||||
"golang.org", "go.dev", "x.org", "sourceforge.net", "xyz.com", "yandex.net",
|
||||
"haxx.se", "curl.se", "python.org", "perl.org", "isc.org", "sourceware.org",
|
||||
"ipxe.org", "gnupg.org", "gnutls.org", "openldap.org", "netfilter.org",
|
||||
"github.com", "githubusercontent.com", "typoland.com", "schemas.dell.com",
|
||||
"googleapis.com", "gstatic.com", "jquery.com", "jsdelivr.net", "unpkg.com",
|
||||
"cloudflare.com", "cloudflare.net", "certificate.fi",
|
||||
"inspur.com", "inspurcloud.com", "inservice-iq.com", "ieisystem.com", "kaytus.com",
|
||||
"jd.com", "jd.local", "jdcloud.com", "in-addr.arpa", "ip6.arpa", "arpa",
|
||||
}
|
||||
|
||||
allowlistedValues = map[string]struct{}{
|
||||
"asia/shanghai": {},
|
||||
"etc/utc": {},
|
||||
"utc": {},
|
||||
"asia/shanghai": {},
|
||||
"etc/utc": {},
|
||||
"etc/universal": {},
|
||||
"utc": {},
|
||||
// neutral decoy zones the sanitizer writes in place of a real timezone
|
||||
"pacific/tarawa": {},
|
||||
"atlantic/azores": {},
|
||||
"antarctica/troll": {},
|
||||
"antarctica/vostok": {},
|
||||
"antarctica/mcmurdo": {},
|
||||
"pacific/guadalcanal": {},
|
||||
"pacific/bougainville": {},
|
||||
"to be filled by o.e.m.": {},
|
||||
"default string": {},
|
||||
"unknown": {},
|
||||
@@ -56,11 +68,15 @@ var (
|
||||
// not customer data from the source.
|
||||
allowlistedFilenameParts = []string{
|
||||
"_tencent", "_jingdong", "_pdd", "_baidu", "_kuaishou", "_tianyiyun",
|
||||
"_jd.", "syslog_jd", "snmptrapcfg", ".json_bak", "ntp_auto",
|
||||
"raw_export.json", "parser_fields.json", "collect.log",
|
||||
"_jd.", "syslog_jd", "snmptrapcfg", ".json_bak", ".conf_bak", "_bak",
|
||||
"ntp_auto", "raw_export.json", "parser_fields.json", "collect.log",
|
||||
}
|
||||
)
|
||||
|
||||
// IsAllowlistedFile reports whether a member path is a vendor factory template
|
||||
// or a LOGPile-derived artifact that should not be scanned or redacted.
|
||||
func IsAllowlistedFile(name string) bool { return isAllowlistedFilename(name) }
|
||||
|
||||
func isAllowlistedDomain(domain string) bool {
|
||||
d := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(domain), "."))
|
||||
for _, z := range allowlistedZones {
|
||||
@@ -76,7 +92,28 @@ func isAllowlistedValue(v string) bool {
|
||||
if _, ok := allowlistedValues[lv]; ok {
|
||||
return true
|
||||
}
|
||||
return isAllowlistedDomain(v)
|
||||
return isRedactionFiller(lv) || isAllowlistedDomain(v)
|
||||
}
|
||||
|
||||
// isRedactionFiller recognises the internal/sanitize output so a re-scan or a
|
||||
// second sanitize pass of an already-cleaned file finds nothing: an all-'x'
|
||||
// hostname filler ("xxxxx.xxxx.xx", "x@xxxx.xxxx.xx", "xxxxxx/xxxxxx") or an
|
||||
// all-zero IP / offset filler ("00.000.000.00", "000", "-000").
|
||||
func isRedactionFiller(s string) bool {
|
||||
if len(s) < 2 {
|
||||
return false
|
||||
}
|
||||
hasFill := false
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case r == 'x' || r == '0':
|
||||
hasFill = true
|
||||
case r == '.' || r == '-' || r == '@' || r == ':' || r == '/' || r == '+':
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return hasFill
|
||||
}
|
||||
|
||||
func isAllowlistedFilename(p string) bool {
|
||||
|
||||
@@ -28,6 +28,7 @@ var nonSensitiveExact = map[string]struct{}{
|
||||
"4.2.2.2": {}, "4.2.2.1": {}, "9.9.9.9": {}, "1.2.3.4": {}, "2.4.6.8": {},
|
||||
"208.67.222.222": {}, "208.67.220.220": {},
|
||||
"77.88.8.8": {}, "77.88.8.1": {}, "77.88.8.88": {}, // Yandex public DNS
|
||||
"100.2.74.41": {}, // Kaytus/Inspur upnp/config.json factory default
|
||||
}
|
||||
|
||||
// isSensitiveIP reports whether s is a routable address that could identify the
|
||||
|
||||
+128
-59
@@ -22,12 +22,30 @@ const (
|
||||
catCollector = "collector"
|
||||
catCert = "cert"
|
||||
catFRULocation = "fru_location"
|
||||
catHostname = "hostname"
|
||||
catDHCP = "dhcp"
|
||||
catNSUpdate = "nsupdate"
|
||||
catMgmtSubdomain = "mgmt_subdomain"
|
||||
)
|
||||
|
||||
// Span is a matched sensitive substring of one line: line[Start:End]. Detection
|
||||
// (Scan) and redaction (internal/sanitize) both consume the same spans so they
|
||||
// can never disagree about what counts as customer data.
|
||||
type Span struct {
|
||||
Start, End int
|
||||
Category string
|
||||
}
|
||||
|
||||
// FindSpans returns every sensitive span in a single line. certFile enables the
|
||||
// TLS-certificate rules (only meaningful for *.pem / *.csr members).
|
||||
func FindSpans(line string, certFile bool) []Span {
|
||||
ms := findMatches(line, certFile)
|
||||
out := make([]Span, 0, len(ms))
|
||||
for _, m := range ms {
|
||||
out = append(out, Span{Start: m.start, End: m.end, Category: m.category})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tableRule is a simple line-regexp rule. group is the submatch index used as
|
||||
// the reported token (0 = whole match).
|
||||
type tableRule struct {
|
||||
@@ -38,6 +56,14 @@ type tableRule struct {
|
||||
hint string
|
||||
}
|
||||
|
||||
type matchSpan struct {
|
||||
start, end int
|
||||
category string
|
||||
severity string
|
||||
hint string
|
||||
match string
|
||||
}
|
||||
|
||||
var (
|
||||
// Match any dotted name ending in a 2-24 char alpha label; plausibleFQDN
|
||||
// then decides whether that final label is a real TLD.
|
||||
@@ -55,7 +81,10 @@ var (
|
||||
reGoRuntime = regexp.MustCompile(`/go-mod/|\bgoroutine \d|\.go:\d+`)
|
||||
// A dotted quad sitting on a line that is talking about a version or a spec
|
||||
// clause, not a host.
|
||||
reVersionContext = regexp.MustCompile(`(?i)(\bversion|\bver\.?\b|_ver\b|fw[_ ]?ver|\brevision|vbios|firmware|\bbios\b|microcode|ucode|\bbuild\b|x\.?org|xserver|ieee|\bstd\b|clause|section|\brfc\b)`)
|
||||
reVersionContext = regexp.MustCompile(`(?i)(version|\bver\.?\b|_ver\b|fw[_ ]?ver|\brevision|vbios|firmware|\bbios\b|microcode|ucode|\bbuild\b|x\.?org|xserver|ieee|\bstd\b|clause|section|\brfc\b)`)
|
||||
// syslog selector "local7.info", "mail.err" - a facility.severity pair, not a
|
||||
// domain.
|
||||
reSyslogSelector = regexp.MustCompile(`(?i)^(?:\*|local[0-7]|auth|authpriv|cron|daemon|ftp|kern|lpr|mail|news|security|syslog|user|uucp)\.(?:\*|emerg|panic|alert|crit|err|error|warn|warning|notice|info|debug|none)$`)
|
||||
// A TLD label written Title-case ("OS.It", "Rodolfo.Cn") is prose or mojibake,
|
||||
// not a real domain (real ones are lower- or all-upper-case).
|
||||
reTitlecaseTail = regexp.MustCompile(`\.[A-Z][a-z]+$`)
|
||||
@@ -88,13 +117,23 @@ var (
|
||||
}
|
||||
)
|
||||
|
||||
func scanLine(path, line string, ln int, certFile bool, emit func(models.PrivacyFinding)) {
|
||||
var local []models.PrivacyFinding
|
||||
specific := map[string]struct{}{} // matches from categories more precise than a bare FQDN
|
||||
// findMatches is the single matcher shared by Scan and FindSpans. It returns
|
||||
// every accepted span on the line, before the Scan-only "already covered by a
|
||||
// more specific category" dedupe.
|
||||
func findMatches(line string, certFile bool) []matchSpan {
|
||||
var out []matchSpan
|
||||
specific := map[string]struct{}{}
|
||||
noiseLine := reKernelTimestamp.MatchString(line) || reGoRuntime.MatchString(line)
|
||||
|
||||
add := func(cat, sev, match, hint string) {
|
||||
match = strings.Trim(strings.TrimSpace(match), `"',;:`)
|
||||
add := func(cat, sev, hint string, s, e int) {
|
||||
if s < 0 || e > len(line) || s >= e {
|
||||
return
|
||||
}
|
||||
s, e = trimSpan(line, s, e)
|
||||
if s >= e {
|
||||
return
|
||||
}
|
||||
match := line[s:e]
|
||||
if len(match) < 2 || !strings.ContainsFunc(match, isAlnum) || isAllowlistedValue(match) {
|
||||
return
|
||||
}
|
||||
@@ -107,26 +146,28 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy
|
||||
if cat != catDomain && cat != catEmail && cat != catPublicIP {
|
||||
specific[strings.ToLower(match)] = struct{}{}
|
||||
}
|
||||
local = append(local, models.PrivacyFinding{
|
||||
Category: cat, Severity: sev, Path: path, Line: ln,
|
||||
Match: match, Excerpt: excerpt(line), Hint: hint,
|
||||
})
|
||||
out = append(out, matchSpan{start: s, end: e, category: cat, severity: sev, hint: hint, match: match})
|
||||
}
|
||||
|
||||
groupSpan := func(m []int, group int) (int, int) {
|
||||
if 2*group+1 >= len(m) {
|
||||
return -1, -1
|
||||
}
|
||||
return m[2*group], m[2*group+1]
|
||||
}
|
||||
|
||||
for _, r := range tableRules {
|
||||
for _, m := range r.re.FindAllStringSubmatch(line, -1) {
|
||||
if r.group < len(m) {
|
||||
add(r.category, r.severity, m[r.group], r.hint)
|
||||
}
|
||||
for _, m := range r.re.FindAllStringSubmatchIndex(line, -1) {
|
||||
s, e := groupSpan(m, r.group)
|
||||
add(r.category, r.severity, r.hint, s, e)
|
||||
}
|
||||
}
|
||||
|
||||
if certFile {
|
||||
for _, r := range certLineRules {
|
||||
for _, m := range r.re.FindAllStringSubmatch(line, -1) {
|
||||
if r.group < len(m) {
|
||||
add(r.category, r.severity, m[r.group], r.hint)
|
||||
}
|
||||
for _, m := range r.re.FindAllStringSubmatchIndex(line, -1) {
|
||||
s, e := groupSpan(m, r.group)
|
||||
add(r.category, r.severity, r.hint, s, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -138,13 +179,13 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy
|
||||
continue
|
||||
}
|
||||
fqdn := line[s:e]
|
||||
if isAllowlistedDomain(fqdn) || !plausibleFQDN(fqdn) || reTitlecaseTail.MatchString(fqdn) {
|
||||
if isAllowlistedDomain(fqdn) || !plausibleFQDN(fqdn) || reTitlecaseTail.MatchString(fqdn) || reSyslogSelector.MatchString(fqdn) {
|
||||
continue
|
||||
}
|
||||
labels := strings.Split(strings.ToLower(fqdn), ".")
|
||||
cat, sev, hint := catDomain, severityLow, "domain / FQDN in the logs"
|
||||
if len(labels) >= 3 {
|
||||
sev = severityMedium // a real subdomain, not a stray two-word token
|
||||
sev = severityMedium
|
||||
}
|
||||
for _, lbl := range labels {
|
||||
if _, ok := mgmtLabels[lbl]; ok {
|
||||
@@ -152,54 +193,75 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy
|
||||
break
|
||||
}
|
||||
}
|
||||
add(cat, sev, fqdn, hint)
|
||||
add(cat, sev, hint, s, e)
|
||||
}
|
||||
}
|
||||
if !noiseLine {
|
||||
for _, m := range reEmail.FindAllStringSubmatch(line, -1) {
|
||||
host := m[1][strings.IndexByte(m[1], '@')+1:]
|
||||
|
||||
for _, m := range reEmail.FindAllStringSubmatchIndex(line, -1) {
|
||||
s, e := m[2], m[3]
|
||||
addr := line[s:e]
|
||||
host := addr[strings.IndexByte(addr, '@')+1:]
|
||||
if isAllowlistedDomain(host) || !looksLikeMailHost(host) {
|
||||
continue
|
||||
}
|
||||
add(catEmail, severityMedium, m[1], "e-mail address")
|
||||
add(catEmail, severityMedium, "e-mail address", s, e)
|
||||
}
|
||||
}
|
||||
for _, m := range reSyslogTarget.FindAllStringSubmatch(line, -1) {
|
||||
add(catCollector, severityMedium, m[1], "remote syslog target")
|
||||
|
||||
for _, m := range reSyslogTarget.FindAllStringSubmatchIndex(line, -1) {
|
||||
add(catCollector, severityMedium, "remote syslog target", m[2], m[3])
|
||||
}
|
||||
|
||||
commentLine := strings.HasPrefix(strings.TrimSpace(line), "#") || strings.HasPrefix(strings.TrimSpace(line), ";")
|
||||
if !noiseLine && !commentLine && !reVersionContext.MatchString(line) {
|
||||
for _, loc := range reIPv4.FindAllStringIndex(line, -1) {
|
||||
if inDottedNumberRun(line, loc[0], loc[1]) {
|
||||
continue // part of a longer version string like "18:6.1.4.5"
|
||||
continue
|
||||
}
|
||||
ip := line[loc[0]:loc[1]]
|
||||
if strings.HasSuffix(ip, ".0") || strings.HasPrefix(ip, "0.") || strings.HasPrefix(ip, "1.") {
|
||||
continue // network address, or a 0./1. version-style quad
|
||||
continue
|
||||
}
|
||||
if !isSensitiveIP(ip) {
|
||||
continue
|
||||
}
|
||||
add(catPublicIP, severityMedium, ip, "public IP reveals provider / site")
|
||||
add(catPublicIP, severityMedium, "public IP reveals provider / site", loc[0], loc[1])
|
||||
}
|
||||
}
|
||||
if ll := strings.ToLower(line); strings.Contains(ll, "timezone") || strings.Contains(ll, "zoneinfo") || strings.Contains(ll, "localtime") || strings.Contains(ll, "tz=") {
|
||||
for _, m := range reTZName.FindAllStringSubmatch(line, -1) {
|
||||
if isAllowlistedValue(m[1]) {
|
||||
continue
|
||||
}
|
||||
add(catTimezone, severityMedium, m[1], "timezone reveals region - set Etc/UTC")
|
||||
}
|
||||
}
|
||||
for _, m := range reTZAbbr.FindAllStringSubmatch(line, -1) {
|
||||
add(catTimezone, severityLow, m[1], "localized timestamp reveals region")
|
||||
}
|
||||
|
||||
for _, f := range local {
|
||||
if (f.Category == catDomain || f.Category == catEmail) && matchCoveredBySpecific(f.Match, specific) {
|
||||
continue
|
||||
if ll := strings.ToLower(line); strings.Contains(ll, "timezone") || strings.Contains(ll, "zoneinfo") || strings.Contains(ll, "localtime") || strings.Contains(ll, "tz=") {
|
||||
for _, m := range reTZName.FindAllStringSubmatchIndex(line, -1) {
|
||||
s, e := m[2], m[3]
|
||||
if isAllowlistedValue(line[s:e]) {
|
||||
continue
|
||||
}
|
||||
add(catTimezone, severityMedium, "timezone reveals region - set Etc/UTC", s, e)
|
||||
}
|
||||
emit(f)
|
||||
}
|
||||
for _, m := range reTZAbbr.FindAllStringSubmatchIndex(line, -1) {
|
||||
add(catTimezone, severityLow, "localized timestamp reveals region", m[2], m[3])
|
||||
}
|
||||
|
||||
// Scan-only dedupe: drop a bare domain/email span when the same token is
|
||||
// already covered by a more specific category on this line.
|
||||
if len(specific) > 0 {
|
||||
filtered := out[:0]
|
||||
for _, m := range out {
|
||||
if (m.category == catDomain || m.category == catEmail) && matchCoveredBySpecific(m.match, specific) {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, m)
|
||||
}
|
||||
out = filtered
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scanLine(path, line string, ln int, certFile bool, emit func(models.PrivacyFinding)) {
|
||||
for _, m := range findMatches(line, certFile) {
|
||||
emit(models.PrivacyFinding{
|
||||
Category: m.category, Severity: m.severity, Path: path, Line: ln,
|
||||
Match: m.match, Excerpt: excerpt(line), Hint: m.hint,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -207,6 +269,19 @@ func isAlnum(r rune) bool {
|
||||
return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9')
|
||||
}
|
||||
|
||||
// trimSpan narrows [s,e) past leading/trailing quoting and separator bytes,
|
||||
// matching the old strings.Trim(TrimSpace(...), "\"',;:") behaviour.
|
||||
func trimSpan(line string, s, e int) (int, int) {
|
||||
const cut = " \t\"',;:"
|
||||
for s < e && strings.IndexByte(cut, line[s]) >= 0 {
|
||||
s++
|
||||
}
|
||||
for e > s && strings.IndexByte(cut, line[e-1]) >= 0 {
|
||||
e--
|
||||
}
|
||||
return s, e
|
||||
}
|
||||
|
||||
// cleanTokenBoundary reports whether line[s:e] stands as its own token - not
|
||||
// glued to surrounding identifier characters ("auth.backend.gssapi.store-creds",
|
||||
// "mountall.sh [start]" is fine, "96.00.CF.00" is not) - so it reads as a
|
||||
@@ -216,7 +291,6 @@ func cleanTokenBoundary(line string, s, e int) bool {
|
||||
switch c := line[s-1]; c {
|
||||
case ' ', '\t', '"', '\'', '(', '<', '=', ',', '@':
|
||||
case '/', ':':
|
||||
// ok (URL / "server = host:port"), but not a bare path segment
|
||||
default:
|
||||
return false
|
||||
}
|
||||
@@ -225,7 +299,7 @@ func cleanTokenBoundary(line string, s, e int) bool {
|
||||
switch line[e] {
|
||||
case ' ', '\t', '"', '\'', ')', '>', ',', ';', ':', '/', '\\', '?', '!':
|
||||
default:
|
||||
return false // trailing '.', '-', letter, digit -> mid-identifier
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
@@ -272,32 +346,27 @@ func usefulFRUValue(v string) bool {
|
||||
}
|
||||
}
|
||||
if digits == len(v) {
|
||||
return false // pure number: date/manufacturing code
|
||||
return false
|
||||
}
|
||||
if upperAlnum == len(v) && digits >= 4 {
|
||||
return false // looks like a serial number (7J..., 21D634070)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// looksLikeMailHost rejects the many "local@identifier.token" strings that are
|
||||
// not e-mail: OData/Redfish JSON annotations (Members@odata.count), SSH
|
||||
// cipher/kex names (aes256-gcm@openssh.com is handled by the domain allowlist,
|
||||
// but the shape is the same).
|
||||
// cipher/kex names, systemd units ("serial-getty@ttyAMA0.service").
|
||||
func looksLikeMailHost(host string) bool {
|
||||
h := strings.ToLower(host)
|
||||
if strings.Contains(h, "odata") || strings.Contains(h, "redfish") || strings.Contains(h, "message.") {
|
||||
return false
|
||||
}
|
||||
// systemd templated units ("serial-getty@ttyAMA0.service"), Go toolchain
|
||||
// pseudo-versions ("toolchain@v0.0.1-go1.25.0.linux") and the like all fail
|
||||
// the real-TLD check.
|
||||
return plausibleFQDN(h)
|
||||
}
|
||||
|
||||
// matchCoveredBySpecific reports whether a bare FQDN/e-mail finding is already
|
||||
// represented by a more precise finding on the same line (e.g. the resolv.conf
|
||||
// "domain corp.acme.ru" line yields both a resolv and a domain hit).
|
||||
// matchCoveredBySpecific reports whether a bare FQDN/e-mail match is already
|
||||
// represented by a more precise finding on the same line.
|
||||
func matchCoveredBySpecific(match string, specific map[string]struct{}) bool {
|
||||
m := strings.ToLower(match)
|
||||
if _, ok := specific[m]; ok {
|
||||
|
||||
Reference in New Issue
Block a user