From 2be215fca1fd60cd37b171fdf0554bfe25662f15 Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Wed, 2 Sep 2026 16:45:35 +0300 Subject: [PATCH] fix(privacy): calibrate against the RMA log corpus (130k files) Cross-checked internal/privacy against the batch companion-app report over project/rma. Fixes for the dominant false-positive classes: - Real-TLD gate (tld.go): FQDN/e-mail must end in a curated TLD or a pseudo-TLD with a >=3-char label; two-letter file/code suffixes (.sh .so .md .id .service ...) are a hard denylist. Kills "0.linux" (45996 hits), "mountall.sh", "libc.so", "@odata.id", "serial-getty@ttyAMA0.service". - Clean-token boundary + Title-case reject: "auth.backend.gssapi.store-creds", "OS.It" are code, not hosts. - Kernel ring-buffer ("[ 8.07][ T1] ...") and Go stack-trace lines skipped. - resolv domain/search values must contain a dot ("domain 53" -> out). - IPv4: skip comment lines, version/spec lines (X.Org, IEEE Std, l0fw_ver), "0."/"1."/".0" quads; allowlist Yandex resolvers + RFC3849 2001:db8::/32. - fru_location: drop all-digit / serial-like / field-name-echo values. - Drop the hostname rule (zero real hits, only "bmc-state-manager" noise). - domain category: high -> low, medium at 3+ labels. Real customer signal now comes from resolv/nsupdate/ad_ldap/mgmt, which the corpus confirms catches every actual customer (netwell.local, tcsbank.ru). - Allowlist smartmontools.org, openib.org, apache.org, freebsd.org, golang.org, ipxe.org, nvidia.com and other FOSS/vendor infra; skip LOGPile's own raw_export.json / parser_fields.json / collect.log members. Co-Authored-By: Claude Sonnet 5 --- bible-local/docs/privacy-scan.md | 31 +++-- internal/privacy/allowlist.go | 12 +- internal/privacy/customer.go | 6 + internal/privacy/ip.go | 4 +- internal/privacy/privacy_test.go | 49 +++++++- internal/privacy/rules.go | 194 +++++++++++++++++++++---------- internal/privacy/tld.go | 75 ++++++++++++ 7 files changed, 296 insertions(+), 75 deletions(-) create mode 100644 internal/privacy/tld.go diff --git a/bible-local/docs/privacy-scan.md b/bible-local/docs/privacy-scan.md index 10024ae..0015df0 100644 --- a/bible-local/docs/privacy-scan.md +++ b/bible-local/docs/privacy-scan.md @@ -47,19 +47,35 @@ via `parser.PrivacyScanEnabled()`. a `resolv` finding, not also a `domain` one). - A match that parses as an IP but is not sensitive (see IP policy) is dropped regardless of the rule that produced it. -- Noise guards: matches shorter than 2 chars or with no alphanumeric are - dropped; `fru_location` values that echo the field name (`Base Board Asset - Tag`, `P1-DIMMA1_AssetTag`) are dropped; IPv4 inside a longer dotted-number - run (`18:6.1.4.5`) is not an address; e-mail on a kernel ring-buffer line - (`[ 8.07] ...`) is driver/copyright text, not customer contact. +- Noise guards (calibrated against a 130k-file RMA corpus, see ADL-066): + - **Real-TLD gate** (`tld.go`): a bare FQDN / e-mail host must end in a + curated TLD (common gTLDs + regional + major-western ccTLDs) or a + pseudo-TLD (`.local`/`.lan`/... with a >=3-char label). Two-letter + file-extension / code-identifier suffixes (`.sh`, `.so`, `.md`, `.id`, + `.service`, ...) are a denylist that always loses. Kills `0.linux`, + `mountall.sh`, `libc.so`, `serial-getty@ttyAMA0.service`, `@odata.id`. + - **Clean-token boundary**: an FQDN glued to surrounding identifier chars + (`auth.backend.gssapi.store-creds`) or Title-cased (`OS.It`) is a code + fragment, not a host. + - Kernel ring-buffer lines (`[ 8.07]...`, `[ 8.07][ T1]...`) and Go + stack-trace lines (`/go-mod/`, `file.go:123`) are skipped for FQDN/email. + - `resolv` `domain`/`search` values must contain a dot. + - IPv4: skipped on `#`/`;` comment lines and version/spec-clause lines + (`X.Org 1.21.1.7`, `IEEE Std ... 8.4.2.97`, `l0fw_ver:[13.16.30.16]`); + inside a longer run (`18:6.1.4.5`, `v1.2.3.4`); `0.` / `1.` / `.0` quads; + Yandex/Google/... public resolvers and RFC 3849 `2001:db8::/32`. + - `fru_location` values that echo the field name, are all-digit, or look + like the chassis serial are dropped; the `hostname` rule was removed + (zero real hits, only `bmc-state-manager`-style noise). - Customer guess: a single-hit low-confidence domain is not reported at all - (better "unidentified" than guessing `nvidia.com` from a driver comment). + (better "unidentified" than guessing `nvidia.com` from a driver comment or + `smartmontools.org` from a SMART copyright line). ## Rule catalogue (`rules.go`) | Category | Severity | Matches | Remediation target | |----------|----------|---------|--------------------| -| `domain` | high | bare FQDN with a known TLD | replace with `example.local` | +| `domain` | low / medium | bare FQDN with a real TLD (medium at 3+ labels) | replace with `example.local` | | `resolv` | high | `domain` / `search` / `nameserver` lines | `resolv.conf` | | `ad_ldap` | high | `racdomain=`, `adfilterdcN=`, `rolegroupN{name,domain}=`, `binddn`, `bindpw` | `activedir.conf`, `ldap.conf` | | `mgmt_subdomain` | high | `*.mgmt/oob/ipmi/drac/idrac/ilo/bmc.*` | management-network subdomain | @@ -71,7 +87,6 @@ via `parser.PrivacyScanEnabled()`. | `collector` | medium | `SyslogHostname=`, `@ip:port`, JSON `"Destination"` | syslog / SNMP trap receiver | | `fru_location` | medium | `Asset Tag`, `Product Location`, `Chassis Location`, `Board Extra` | FRU fields | | `dhcp` | medium | `option domain-name` | customer DHCP lease | -| `hostname` | low | `sn-`/`bmc-`/`ilo-`/`idrac-`/`drac-`/`srv-` prefixed names | customer naming scheme | ## IP policy (`ip.go`) diff --git a/internal/privacy/allowlist.go b/internal/privacy/allowlist.go index cb2b491..7dc5888 100644 --- a/internal/privacy/allowlist.go +++ b/internal/privacy/allowlist.go @@ -13,13 +13,18 @@ var ( allowlistedZones = []string{ "example.com", "example.net", "example.org", "example.local", "example.edu", "foobar.edu", "issue.net", - "localhost", "localdomain", "local.lan", + "localhost", "localdomain", "local.lan", "localhost.localdomain", + "host.name", "other.host.name", "some.host.name", "other.domain", "some.domain", "pool.ntp.org", "ntp.org", "nist.gov", "windows.com", "microsoft.com", "dmtf.org", "iana.org", "openssl.org", "openssh.com", "openssh.org", "libssh.org", "rsyslog.com", "adiscon.com", "redhat.com", "kernel.org", "megarac.com", "ami.com", "commond.com", "oasis-open.org", "w3.org", "xmlsoap.org", "purl.org", "ietf.org", "nvidia.com", "mellanox.com", "gnu.org", "debian.org", "ubuntu.com", + "smartmontools.org", "openib.org", "apache.org", "freebsd.org", "freedesktop.org", + "golang.org", "go.dev", "x.org", "sourceforge.net", "xyz.com", "yandex.net", + "haxx.se", "curl.se", "python.org", "perl.org", "isc.org", "sourceware.org", + "ipxe.org", "gnupg.org", "gnutls.org", "openldap.org", "netfilter.org", "inspur.com", "inspurcloud.com", "inservice-iq.com", "ieisystem.com", "kaytus.com", "jd.com", "jd.local", "jdcloud.com", "in-addr.arpa", "ip6.arpa", "arpa", } @@ -46,10 +51,13 @@ var ( "[empty]": {}, } - // Archive members that are vendor factory templates, not the active config. + // Archive members that are vendor factory templates or LOGPile's own derived + // export artifacts (operator-entered target host, Redfish annotation keys) - + // not customer data from the source. allowlistedFilenameParts = []string{ "_tencent", "_jingdong", "_pdd", "_baidu", "_kuaishou", "_tianyiyun", "_jd.", "syslog_jd", "snmptrapcfg", ".json_bak", "ntp_auto", + "raw_export.json", "parser_fields.json", "collect.log", } ) diff --git a/internal/privacy/customer.go b/internal/privacy/customer.go index 04caffb..dd580b5 100644 --- a/internal/privacy/customer.go +++ b/internal/privacy/customer.go @@ -147,6 +147,12 @@ func registrableDomain(host string) string { if host == "" || strings.ContainsAny(host, ":/ ") { return "" } + if at := strings.IndexByte(host, '@'); at >= 0 { + host = host[at+1:] + } + if !plausibleFQDN(host) { + return "" + } if strings.Count(host, ".") == 3 { allDigits := true for _, r := range host { diff --git a/internal/privacy/ip.go b/internal/privacy/ip.go index 438b424..9d19065 100644 --- a/internal/privacy/ip.go +++ b/internal/privacy/ip.go @@ -12,6 +12,7 @@ var nonSensitiveNets = func() []*net.IPNet { "198.18.0.0/15", // RFC 2544 benchmarking "100.64.0.0/10", // RFC 6598 CGNAT "192.88.99.0/24", // RFC 7526 6to4 relay anycast + "2001:db8::/32", // RFC 3849 documentation } out := make([]*net.IPNet, 0, len(cidrs)) for _, c := range cidrs { @@ -24,8 +25,9 @@ var nonSensitiveNets = func() []*net.IPNet { var nonSensitiveExact = map[string]struct{}{ "8.8.8.8": {}, "8.8.4.4": {}, "1.1.1.1": {}, "1.0.0.1": {}, - "4.2.2.2": {}, "4.2.2.1": {}, "9.9.9.9": {}, "1.2.3.4": {}, + "4.2.2.2": {}, "4.2.2.1": {}, "9.9.9.9": {}, "1.2.3.4": {}, "2.4.6.8": {}, "208.67.222.222": {}, "208.67.220.220": {}, + "77.88.8.8": {}, "77.88.8.1": {}, "77.88.8.88": {}, // Yandex public DNS } // isSensitiveIP reports whether s is a routable address that could identify the diff --git a/internal/privacy/privacy_test.go b/internal/privacy/privacy_test.go index 0abe1b1..0a1b8cb 100644 --- a/internal/privacy/privacy_test.go +++ b/internal/privacy/privacy_test.go @@ -1,6 +1,9 @@ package privacy -import "testing" +import ( + "strings" + "testing" +) // Fixtures use "acme.ru" as a stand-in customer domain. The RFC 2606 // "example.*" names are on the allowlist (they are the sanitization target), @@ -148,6 +151,50 @@ func TestScan_RealResolvStillCaughtInNoisyFile(t *testing.T) { } } +func TestScan_OSConfigNoise(t *testing.T) { + // Everything here is stock OS / BEE-SP / Redfish text - zero customer data. + noise := strings.Join([]string{ + `Process: 3950 ExecStartPre=/usr/bin/nvidia-fabricmanager-start.sh --mode`, + `echo "Usage: mountall.sh [start|stop]" >&2`, + `. /lib/init/vars.sh`, + `auth.backend.gssapi.store-creds = "disable"`, + `server.network-backend = "writev"`, + `ssl.ca-file = "/conf/server.pem"`, + `# ALL EXCEPT in.fingerd: other.host.name, .other.domain`, + `/lib/arm-linux-gnueabihf/libc.so.6`, + `"@odata.id": "/redfish/v1/Systems/1"`, + `X.Org X Server 1.21.1.7`, + `ME FW Version 6.1.4.75`, + `96.00.CF.00.03 VBIOS`, + `#option dns 129.219.13.81`, + `toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sema.go:9`, + `serial-getty@ttyAMA0.service`, + `Copyright (C) 2002-22, www.smartmontools.org`, + `OpenIB.org BSD license (FreeBSD Variant)`, + }, "\n") + rep := Scan([]File{{Path: "onekeylog/log/sollog/SOLHostCapture.log", Content: []byte(noise)}}) + if rep != nil && len(rep.Findings) > 0 { + t.Fatalf("OS config noise flagged: %+v", rep.Findings) + } +} + +func TestScan_ResolvNeedsDottedValue(t *testing.T) { + rep := Scan([]File{{Path: "resolv.conf", Content: []byte( + "domain 53\nsearch nameserver\ndomain corp.acme.ru\n")}}) + if rep == nil { + t.Fatal("nil") + } + var got []string + for _, f := range rep.Findings { + if f.Category == catResolv { + got = append(got, f.Match) + } + } + if len(got) != 1 || got[0] != "corp.acme.ru" { + t.Fatalf("resolv matches = %v, want [corp.acme.ru]", got) + } +} + func TestSummary(t *testing.T) { rep := Scan([]File{{Path: "resolv.conf", Content: []byte("domain acme.ru\n")}}) if rep.Summary.Total != len(rep.Findings) || rep.Summary.Total == 0 { diff --git a/internal/privacy/rules.go b/internal/privacy/rules.go index 0ce43ef..566207f 100644 --- a/internal/privacy/rules.go +++ b/internal/privacy/rules.go @@ -39,23 +39,29 @@ type tableRule struct { } var ( - // A conservative TLD set keeps the bare-FQDN rule from matching things like - // "foo.bar" in prose or "index.json" in paths. - fqdnTLD = `(?:ru|su|by|kz|ua|com|net|org|local|io|dev|cloud|info|biz|eu|de|uk|fr|nl|cn|jp|kr|us|gov|edu|mil|co|tech|online)` - - reFQDN = regexp.MustCompile(`(?i)\b((?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+` + fqdnTLD + `)\b`) - reMgmtSubdomain = regexp.MustCompile(`(?i)\b([a-z0-9-]+(?:\.[a-z0-9-]+)*\.(?:mgmt|oob|ipmi|drac|idrac|ilo|bmc)\.[a-z0-9.-]+)\b`) - reIPv4 = regexp.MustCompile(`\b((?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3})\b`) - reEmail = regexp.MustCompile(`(?i)\b([a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,})\b`) - reSyslogTarget = regexp.MustCompile(`@((?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}:[0-9]{1,5})`) - reTZName = regexp.MustCompile(`\b((?:Africa|America|Antarctica|Asia|Atlantic|Australia|Europe|Indian|Pacific)/[A-Za-z_]+(?:/[A-Za-z_]+)?)\b`) - reTZAbbr = regexp.MustCompile(`\b(MSK|MSD|EEST|EET|MDST|CEST|CET|WEST|WET)\b\s+20\d\d`) - // Kernel ring-buffer line: "[ 12.345678] ...". These carry driver/copyright - // strings (emails, versions), never customer identity. - reKernelTimestamp = regexp.MustCompile(`^\[\s*\d+\.\d+\]\s`) + // Match any dotted name ending in a 2-24 char alpha label; plausibleFQDN + // then decides whether that final label is a real TLD. + reFQDN = regexp.MustCompile(`(?i)\b((?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,24})\b`) + mgmtLabels = map[string]struct{}{"mgmt": {}, "oob": {}, "ipmi": {}, "drac": {}, "idrac": {}, "ilo": {}, "bmc": {}} + reIPv4 = regexp.MustCompile(`\b((?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3})\b`) + reEmail = regexp.MustCompile(`(?i)\b([a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,})\b`) + reSyslogTarget = regexp.MustCompile(`@((?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}:[0-9]{1,5})`) + reTZName = regexp.MustCompile(`\b((?:Africa|America|Antarctica|Asia|Atlantic|Australia|Europe|Indian|Pacific)/[A-Za-z_]+(?:/[A-Za-z_]+)?)\b`) + reTZAbbr = regexp.MustCompile(`\b(MSK|MSD|EEST|EET|MDST|CEST|CET|WEST|WET)\b\s+20\d\d`) + // Kernel ring-buffer line: "[ 12.345678] ..." or "[ 12.34][ T1] ...". + // These carry driver/copyright strings (emails, versions), never customer id. + reKernelTimestamp = regexp.MustCompile(`^\[\s*\d+\.\d+\]`) + // Go panic / stack-trace lines: module-cache paths and "file.go:123" frames. + reGoRuntime = regexp.MustCompile(`/go-mod/|\bgoroutine \d|\.go:\d+`) + // A dotted quad sitting on a line that is talking about a version or a spec + // clause, not a host. + reVersionContext = regexp.MustCompile(`(?i)(\bversion|\bver\.?\b|_ver\b|fw[_ ]?ver|\brevision|vbios|firmware|\bbios\b|microcode|ucode|\bbuild\b|x\.?org|xserver|ieee|\bstd\b|clause|section|\brfc\b)`) + // A TLD label written Title-case ("OS.It", "Rodolfo.Cn") is prose or mojibake, + // not a real domain (real ones are lower- or all-upper-case). + reTitlecaseTail = regexp.MustCompile(`\.[A-Z][a-z]+$`) tableRules = []tableRule{ - {catResolv, severityHigh, regexp.MustCompile(`(?i)^\s*(?:domain|search)\s+(\S+)`), 1, + {catResolv, severityHigh, regexp.MustCompile(`(?i)^\s*(?:domain|search)\s+([a-z0-9][a-z0-9.\-]*\.[a-z0-9\-]+)`), 1, "resolv.conf DNS suffix - replace with example.local"}, {catADLDAP, severityHigh, regexp.MustCompile(`(?i)\b(racdomain|adfilterdc[0-9]|rolegroup[0-9](?:name|domain))\s*=\s*(\S+)`), 2, "activedir.conf - AD domain / DC address / role-group name"}, @@ -73,8 +79,6 @@ var ( "DDNS update - host FQDN + BMC record"}, {catDHCP, severityMedium, regexp.MustCompile(`(?i)option\s+domain-name\s+"?([^";]+)`), 1, "domain-name handed out by customer DHCP"}, - {catHostname, severityLow, regexp.MustCompile(`(?i)\b((?:sn|bmc|ilo|idrac|drac|srv)-[a-z0-9][a-z0-9-]{2,})\b`), 1, - "hostname follows a customer naming scheme"}, } certLineRules = []tableRule{ @@ -87,7 +91,7 @@ var ( func scanLine(path, line string, ln int, certFile bool, emit func(models.PrivacyFinding)) { var local []models.PrivacyFinding specific := map[string]struct{}{} // matches from categories more precise than a bare FQDN - kernelLine := reKernelTimestamp.MatchString(line) + noiseLine := reKernelTimestamp.MatchString(line) || reGoRuntime.MatchString(line) add := func(cat, sev, match, hint string) { match = strings.Trim(strings.TrimSpace(match), `"',;:`) @@ -97,9 +101,7 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy if ip := net.ParseIP(match); ip != nil && !isSensitiveIP(match) { return } - // BIOS/SMBIOS echoes the field name as the value ("Base Board Asset Tag", - // "P1-DIMMA1_AssetTag") - not a customer site tag. - if cat == catFRULocation && strings.Contains(strings.ToLower(strings.ReplaceAll(match, " ", "")), "assettag") { + if cat == catFRULocation && !usefulFRUValue(match) { return } if cat != catDomain && cat != catEmail && cat != catPublicIP { @@ -129,16 +131,31 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy } } - for _, m := range reMgmtSubdomain.FindAllStringSubmatch(line, -1) { - add(catMgmtSubdomain, severityHigh, m[1], "management-network subdomain") - } - for _, m := range reFQDN.FindAllStringSubmatch(line, -1) { - if isAllowlistedDomain(m[1]) { - continue + if !noiseLine { + for _, idx := range reFQDN.FindAllStringSubmatchIndex(line, -1) { + s, e := idx[2], idx[3] + if !cleanTokenBoundary(line, s, e) { + continue + } + fqdn := line[s:e] + if isAllowlistedDomain(fqdn) || !plausibleFQDN(fqdn) || reTitlecaseTail.MatchString(fqdn) { + continue + } + labels := strings.Split(strings.ToLower(fqdn), ".") + cat, sev, hint := catDomain, severityLow, "domain / FQDN in the logs" + if len(labels) >= 3 { + sev = severityMedium // a real subdomain, not a stray two-word token + } + for _, lbl := range labels { + if _, ok := mgmtLabels[lbl]; ok { + cat, sev, hint = catMgmtSubdomain, severityHigh, "management-network subdomain" + break + } + } + add(cat, sev, fqdn, hint) } - add(catDomain, severityHigh, m[1], "domain / FQDN reveals the customer") } - if !kernelLine { + if !noiseLine { for _, m := range reEmail.FindAllStringSubmatch(line, -1) { host := m[1][strings.IndexByte(m[1], '@')+1:] if isAllowlistedDomain(host) || !looksLikeMailHost(host) { @@ -150,17 +167,23 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy for _, m := range reSyslogTarget.FindAllStringSubmatch(line, -1) { add(catCollector, severityMedium, m[1], "remote syslog target") } - for _, loc := range reIPv4.FindAllStringIndex(line, -1) { - if inDottedNumberRun(line, loc[0], loc[1]) { - continue // part of a longer version string like "18:6.1.4.5" + commentLine := strings.HasPrefix(strings.TrimSpace(line), "#") || strings.HasPrefix(strings.TrimSpace(line), ";") + if !noiseLine && !commentLine && !reVersionContext.MatchString(line) { + for _, loc := range reIPv4.FindAllStringIndex(line, -1) { + if inDottedNumberRun(line, loc[0], loc[1]) { + continue // part of a longer version string like "18:6.1.4.5" + } + ip := line[loc[0]:loc[1]] + if strings.HasSuffix(ip, ".0") || strings.HasPrefix(ip, "0.") || strings.HasPrefix(ip, "1.") { + continue // network address, or a 0./1. version-style quad + } + if !isSensitiveIP(ip) { + continue + } + add(catPublicIP, severityMedium, ip, "public IP reveals provider / site") } - ip := line[loc[0]:loc[1]] - if !isSensitiveIP(ip) { - continue - } - add(catPublicIP, severityMedium, ip, "public IP reveals provider / site") } - if strings.Contains(strings.ToLower(line), "timezone") || strings.Contains(line, "/") { + if ll := strings.ToLower(line); strings.Contains(ll, "timezone") || strings.Contains(ll, "zoneinfo") || strings.Contains(ll, "localtime") || strings.Contains(ll, "tz=") { for _, m := range reTZName.FindAllStringSubmatch(line, -1) { if isAllowlistedValue(m[1]) { continue @@ -184,24 +207,79 @@ func isAlnum(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') } -// inDottedNumberRun reports whether line[start:end] (a dotted quad) is embedded -// in a longer numeric run - a version string like "18:6.1.4.5" or "1.2.3.4.5" - -// rather than a standalone address. -func inDottedNumberRun(line string, start, end int) bool { - if start > 0 { - switch line[start-1] { - case '.', ':': - if start >= 2 && line[start-2] >= '0' && line[start-2] <= '9' { - return true - } +// cleanTokenBoundary reports whether line[s:e] stands as its own token - not +// glued to surrounding identifier characters ("auth.backend.gssapi.store-creds", +// "mountall.sh [start]" is fine, "96.00.CF.00" is not) - so it reads as a +// hostname rather than a fragment of code, a path tail, or a version string. +func cleanTokenBoundary(line string, s, e int) bool { + if s > 0 { + switch c := line[s-1]; c { + case ' ', '\t', '"', '\'', '(', '<', '=', ',', '@': + case '/', ':': + // ok (URL / "server = host:port"), but not a bare path segment + default: + return false } } - if end < len(line) && line[end] == '.' { + if e < len(line) { + switch line[e] { + case ' ', '\t', '"', '\'', ')', '>', ',', ';', ':', '/', '\\', '?', '!': + default: + return false // trailing '.', '-', letter, digit -> mid-identifier + } + } + return true +} + +// inDottedNumberRun reports whether line[start:end] (a dotted quad) is embedded +// in a longer version-like run - "18:6.1.4.5", "v1.2.3.4", "1.2.3.4.5", +// "go1.25.0.linux" - rather than a standalone address. +func inDottedNumberRun(line string, start, end int) bool { + if start > 0 { + switch c := line[start-1]; { + case c == '.', c == '-': + return true + case c == ':' && start >= 2 && line[start-2] >= '0' && line[start-2] <= '9': + return true + case c == 'v' || c == 'V': + return true + } + } + if end < len(line) && (line[end] == '.' || line[end] == '-') { return true } return false } +// usefulFRUValue drops FRU asset/location values that carry no site information: +// the field name echoed back, an all-digit manufacturing code, or a string that +// is just the chassis serial (already in the archive name). +func usefulFRUValue(v string) bool { + lv := strings.ToLower(strings.ReplaceAll(v, " ", "")) + if strings.Contains(lv, "assettag") || strings.Contains(lv, "serialnumber") { + return false + } + if len(v) < 4 { + return false + } + digits, upperAlnum := 0, 0 + for _, r := range v { + if r >= '0' && r <= '9' { + digits++ + } + if (r >= '0' && r <= '9') || (r >= 'A' && r <= 'Z') { + upperAlnum++ + } + } + if digits == len(v) { + return false // pure number: date/manufacturing code + } + if upperAlnum == len(v) && digits >= 4 { + return false // looks like a serial number (7J..., 21D634070) + } + return true +} + // looksLikeMailHost rejects the many "local@identifier.token" strings that are // not e-mail: OData/Redfish JSON annotations (Members@odata.count), SSH // cipher/kex names (aes256-gcm@openssh.com is handled by the domain allowlist, @@ -211,20 +289,10 @@ func looksLikeMailHost(host string) bool { if strings.Contains(h, "odata") || strings.Contains(h, "redfish") || strings.Contains(h, "message.") { return false } - dot := strings.LastIndexByte(h, '.') - if dot < 0 { - return false - } - tld := h[dot+1:] - if len(tld) < 2 || len(tld) > 24 { - return false - } - for _, r := range tld { - if r < 'a' || r > 'z' { - return false - } - } - return true + // systemd templated units ("serial-getty@ttyAMA0.service"), Go toolchain + // pseudo-versions ("toolchain@v0.0.1-go1.25.0.linux") and the like all fail + // the real-TLD check. + return plausibleFQDN(h) } // matchCoveredBySpecific reports whether a bare FQDN/e-mail finding is already diff --git a/internal/privacy/tld.go b/internal/privacy/tld.go new file mode 100644 index 0000000..a3dc99a --- /dev/null +++ b/internal/privacy/tld.go @@ -0,0 +1,75 @@ +package privacy + +import "strings" + +// Top-level domains a customer domain in these logs actually ends in: the common +// gTLDs plus the ccTLDs for our region and the major western countries. This is +// deliberately tight, not the full IANA list - a domain under some rare ccTLD +// (.sh, .so, .cf, .lc ...) simply is not flagged by the bare-FQDN rule, which is +// an acceptable miss: those two-letter suffixes are almost always a filename +// extension or a code identifier ("mountall.sh", "libc.so", "@odata.id"), and +// the real customer signal comes from resolv.conf / nsupdate / AD config anyway. +var knownTLD = func() map[string]struct{} { + list := strings.Fields(` +com net org edu gov mil int info biz pro asia mobi tel coop aero museum travel +app dev cloud tech xyz io co ai me tv cc +ru su by ua kz md ge am az uz kg tj tm +uk de fr nl be lu it es pt pl cz sk at ch se no fi dk ie eu is +ee lv lt ro bg hr si rs hu gr tr +us ca mx br ar cn jp kr in sg hk tw au nz il ae sa za +`) + m := make(map[string]struct{}, len(list)) + for _, t := range list { + m[t] = struct{}{} + } + return m +}() + +// Extensions / code-identifier suffixes that shadow a ccTLD. A dotted name +// ending in one of these is a filename or identifier, never a domain. +var shadowSuffix = func() map[string]struct{} { + list := strings.Fields(` +sh bash zsh ksh so py pyc pl pm rb js mjs cjs jsx ts tsx go rs kt +c cc cpp cxx h hpp hh hxx mm md rst txt text log out err trace +conf config cfg cnf ini toml yaml yml json xml html htm xhtml svg css scss +sql csv tsv pem crt cer key csr der p12 pfx bak old orig tmp temp swp lock pid +sock service socket target timer mount automount path slice scope device dump core +bin dat db img iso gz xz bz2 zip tar tgz rar rpm deb apk whl jar war +class obj lib la lo ko dll dylib exe map lst asm +id ctx ver rev sig hash sum cache idx pack ref meta prop props store network backend +`) + m := make(map[string]struct{}, len(list)) + for _, t := range list { + m[t] = struct{}{} + } + return m +}() + +// Non-registrable suffixes that still identify a customer's internal zone. +var pseudoTLD = map[string]struct{}{ + "local": {}, "lan": {}, "home": {}, "corp": {}, "internal": {}, "intranet": {}, +} + +// plausibleFQDN reports whether host is a real domain worth flagging. +func plausibleFQDN(host string) bool { + host = strings.ToLower(strings.Trim(strings.TrimSpace(host), ".*")) + if strings.Contains(host, "odata") || strings.Contains(host, "redfish") { + return false + } + labels := strings.Split(host, ".") + if len(labels) < 2 { + return false + } + tld := labels[len(labels)-1] + sld := labels[len(labels)-2] + if _, ok := shadowSuffix[tld]; ok { + return false + } + if _, ok := knownTLD[tld]; ok { + return true + } + if _, ok := pseudoTLD[tld]; ok { + return len(sld) >= 3 + } + return false +}