fix(privacy): calibrate against the RMA log corpus (130k files)
Cross-checked internal/privacy against the batch companion-app report over project/rma. Fixes for the dominant false-positive classes: - Real-TLD gate (tld.go): FQDN/e-mail must end in a curated TLD or a pseudo-TLD with a >=3-char label; two-letter file/code suffixes (.sh .so .md .id .service ...) are a hard denylist. Kills "0.linux" (45996 hits), "mountall.sh", "libc.so", "@odata.id", "serial-getty@ttyAMA0.service". - Clean-token boundary + Title-case reject: "auth.backend.gssapi.store-creds", "OS.It" are code, not hosts. - Kernel ring-buffer ("[ 8.07][ T1] ...") and Go stack-trace lines skipped. - resolv domain/search values must contain a dot ("domain 53" -> out). - IPv4: skip comment lines, version/spec lines (X.Org, IEEE Std, l0fw_ver), "0."/"1."/".0" quads; allowlist Yandex resolvers + RFC3849 2001:db8::/32. - fru_location: drop all-digit / serial-like / field-name-echo values. - Drop the hostname rule (zero real hits, only "bmc-state-manager" noise). - domain category: high -> low, medium at 3+ labels. Real customer signal now comes from resolv/nsupdate/ad_ldap/mgmt, which the corpus confirms catches every actual customer (netwell.local, tcsbank.ru). - Allowlist smartmontools.org, openib.org, apache.org, freebsd.org, golang.org, ipxe.org, nvidia.com and other FOSS/vendor infra; skip LOGPile's own raw_export.json / parser_fields.json / collect.log members. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
551d8e450d
commit
2be215fca1
+131
-63
@@ -39,23 +39,29 @@ type tableRule struct {
|
||||
}
|
||||
|
||||
var (
|
||||
// A conservative TLD set keeps the bare-FQDN rule from matching things like
|
||||
// "foo.bar" in prose or "index.json" in paths.
|
||||
fqdnTLD = `(?:ru|su|by|kz|ua|com|net|org|local|io|dev|cloud|info|biz|eu|de|uk|fr|nl|cn|jp|kr|us|gov|edu|mil|co|tech|online)`
|
||||
|
||||
reFQDN = regexp.MustCompile(`(?i)\b((?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+` + fqdnTLD + `)\b`)
|
||||
reMgmtSubdomain = regexp.MustCompile(`(?i)\b([a-z0-9-]+(?:\.[a-z0-9-]+)*\.(?:mgmt|oob|ipmi|drac|idrac|ilo|bmc)\.[a-z0-9.-]+)\b`)
|
||||
reIPv4 = regexp.MustCompile(`\b((?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3})\b`)
|
||||
reEmail = regexp.MustCompile(`(?i)\b([a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,})\b`)
|
||||
reSyslogTarget = regexp.MustCompile(`@((?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}:[0-9]{1,5})`)
|
||||
reTZName = regexp.MustCompile(`\b((?:Africa|America|Antarctica|Asia|Atlantic|Australia|Europe|Indian|Pacific)/[A-Za-z_]+(?:/[A-Za-z_]+)?)\b`)
|
||||
reTZAbbr = regexp.MustCompile(`\b(MSK|MSD|EEST|EET|MDST|CEST|CET|WEST|WET)\b\s+20\d\d`)
|
||||
// Kernel ring-buffer line: "[ 12.345678] ...". These carry driver/copyright
|
||||
// strings (emails, versions), never customer identity.
|
||||
reKernelTimestamp = regexp.MustCompile(`^\[\s*\d+\.\d+\]\s`)
|
||||
// Match any dotted name ending in a 2-24 char alpha label; plausibleFQDN
|
||||
// then decides whether that final label is a real TLD.
|
||||
reFQDN = regexp.MustCompile(`(?i)\b((?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,24})\b`)
|
||||
mgmtLabels = map[string]struct{}{"mgmt": {}, "oob": {}, "ipmi": {}, "drac": {}, "idrac": {}, "ilo": {}, "bmc": {}}
|
||||
reIPv4 = regexp.MustCompile(`\b((?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3})\b`)
|
||||
reEmail = regexp.MustCompile(`(?i)\b([a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,})\b`)
|
||||
reSyslogTarget = regexp.MustCompile(`@((?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}:[0-9]{1,5})`)
|
||||
reTZName = regexp.MustCompile(`\b((?:Africa|America|Antarctica|Asia|Atlantic|Australia|Europe|Indian|Pacific)/[A-Za-z_]+(?:/[A-Za-z_]+)?)\b`)
|
||||
reTZAbbr = regexp.MustCompile(`\b(MSK|MSD|EEST|EET|MDST|CEST|CET|WEST|WET)\b\s+20\d\d`)
|
||||
// Kernel ring-buffer line: "[ 12.345678] ..." or "[ 12.34][ T1] ...".
|
||||
// These carry driver/copyright strings (emails, versions), never customer id.
|
||||
reKernelTimestamp = regexp.MustCompile(`^\[\s*\d+\.\d+\]`)
|
||||
// Go panic / stack-trace lines: module-cache paths and "file.go:123" frames.
|
||||
reGoRuntime = regexp.MustCompile(`/go-mod/|\bgoroutine \d|\.go:\d+`)
|
||||
// A dotted quad sitting on a line that is talking about a version or a spec
|
||||
// clause, not a host.
|
||||
reVersionContext = regexp.MustCompile(`(?i)(\bversion|\bver\.?\b|_ver\b|fw[_ ]?ver|\brevision|vbios|firmware|\bbios\b|microcode|ucode|\bbuild\b|x\.?org|xserver|ieee|\bstd\b|clause|section|\brfc\b)`)
|
||||
// A TLD label written Title-case ("OS.It", "Rodolfo.Cn") is prose or mojibake,
|
||||
// not a real domain (real ones are lower- or all-upper-case).
|
||||
reTitlecaseTail = regexp.MustCompile(`\.[A-Z][a-z]+$`)
|
||||
|
||||
tableRules = []tableRule{
|
||||
{catResolv, severityHigh, regexp.MustCompile(`(?i)^\s*(?:domain|search)\s+(\S+)`), 1,
|
||||
{catResolv, severityHigh, regexp.MustCompile(`(?i)^\s*(?:domain|search)\s+([a-z0-9][a-z0-9.\-]*\.[a-z0-9\-]+)`), 1,
|
||||
"resolv.conf DNS suffix - replace with example.local"},
|
||||
{catADLDAP, severityHigh, regexp.MustCompile(`(?i)\b(racdomain|adfilterdc[0-9]|rolegroup[0-9](?:name|domain))\s*=\s*(\S+)`), 2,
|
||||
"activedir.conf - AD domain / DC address / role-group name"},
|
||||
@@ -73,8 +79,6 @@ var (
|
||||
"DDNS update - host FQDN + BMC record"},
|
||||
{catDHCP, severityMedium, regexp.MustCompile(`(?i)option\s+domain-name\s+"?([^";]+)`), 1,
|
||||
"domain-name handed out by customer DHCP"},
|
||||
{catHostname, severityLow, regexp.MustCompile(`(?i)\b((?:sn|bmc|ilo|idrac|drac|srv)-[a-z0-9][a-z0-9-]{2,})\b`), 1,
|
||||
"hostname follows a customer naming scheme"},
|
||||
}
|
||||
|
||||
certLineRules = []tableRule{
|
||||
@@ -87,7 +91,7 @@ var (
|
||||
func scanLine(path, line string, ln int, certFile bool, emit func(models.PrivacyFinding)) {
|
||||
var local []models.PrivacyFinding
|
||||
specific := map[string]struct{}{} // matches from categories more precise than a bare FQDN
|
||||
kernelLine := reKernelTimestamp.MatchString(line)
|
||||
noiseLine := reKernelTimestamp.MatchString(line) || reGoRuntime.MatchString(line)
|
||||
|
||||
add := func(cat, sev, match, hint string) {
|
||||
match = strings.Trim(strings.TrimSpace(match), `"',;:`)
|
||||
@@ -97,9 +101,7 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy
|
||||
if ip := net.ParseIP(match); ip != nil && !isSensitiveIP(match) {
|
||||
return
|
||||
}
|
||||
// BIOS/SMBIOS echoes the field name as the value ("Base Board Asset Tag",
|
||||
// "P1-DIMMA1_AssetTag") - not a customer site tag.
|
||||
if cat == catFRULocation && strings.Contains(strings.ToLower(strings.ReplaceAll(match, " ", "")), "assettag") {
|
||||
if cat == catFRULocation && !usefulFRUValue(match) {
|
||||
return
|
||||
}
|
||||
if cat != catDomain && cat != catEmail && cat != catPublicIP {
|
||||
@@ -129,16 +131,31 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy
|
||||
}
|
||||
}
|
||||
|
||||
for _, m := range reMgmtSubdomain.FindAllStringSubmatch(line, -1) {
|
||||
add(catMgmtSubdomain, severityHigh, m[1], "management-network subdomain")
|
||||
}
|
||||
for _, m := range reFQDN.FindAllStringSubmatch(line, -1) {
|
||||
if isAllowlistedDomain(m[1]) {
|
||||
continue
|
||||
if !noiseLine {
|
||||
for _, idx := range reFQDN.FindAllStringSubmatchIndex(line, -1) {
|
||||
s, e := idx[2], idx[3]
|
||||
if !cleanTokenBoundary(line, s, e) {
|
||||
continue
|
||||
}
|
||||
fqdn := line[s:e]
|
||||
if isAllowlistedDomain(fqdn) || !plausibleFQDN(fqdn) || reTitlecaseTail.MatchString(fqdn) {
|
||||
continue
|
||||
}
|
||||
labels := strings.Split(strings.ToLower(fqdn), ".")
|
||||
cat, sev, hint := catDomain, severityLow, "domain / FQDN in the logs"
|
||||
if len(labels) >= 3 {
|
||||
sev = severityMedium // a real subdomain, not a stray two-word token
|
||||
}
|
||||
for _, lbl := range labels {
|
||||
if _, ok := mgmtLabels[lbl]; ok {
|
||||
cat, sev, hint = catMgmtSubdomain, severityHigh, "management-network subdomain"
|
||||
break
|
||||
}
|
||||
}
|
||||
add(cat, sev, fqdn, hint)
|
||||
}
|
||||
add(catDomain, severityHigh, m[1], "domain / FQDN reveals the customer")
|
||||
}
|
||||
if !kernelLine {
|
||||
if !noiseLine {
|
||||
for _, m := range reEmail.FindAllStringSubmatch(line, -1) {
|
||||
host := m[1][strings.IndexByte(m[1], '@')+1:]
|
||||
if isAllowlistedDomain(host) || !looksLikeMailHost(host) {
|
||||
@@ -150,17 +167,23 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy
|
||||
for _, m := range reSyslogTarget.FindAllStringSubmatch(line, -1) {
|
||||
add(catCollector, severityMedium, m[1], "remote syslog target")
|
||||
}
|
||||
for _, loc := range reIPv4.FindAllStringIndex(line, -1) {
|
||||
if inDottedNumberRun(line, loc[0], loc[1]) {
|
||||
continue // part of a longer version string like "18:6.1.4.5"
|
||||
commentLine := strings.HasPrefix(strings.TrimSpace(line), "#") || strings.HasPrefix(strings.TrimSpace(line), ";")
|
||||
if !noiseLine && !commentLine && !reVersionContext.MatchString(line) {
|
||||
for _, loc := range reIPv4.FindAllStringIndex(line, -1) {
|
||||
if inDottedNumberRun(line, loc[0], loc[1]) {
|
||||
continue // part of a longer version string like "18:6.1.4.5"
|
||||
}
|
||||
ip := line[loc[0]:loc[1]]
|
||||
if strings.HasSuffix(ip, ".0") || strings.HasPrefix(ip, "0.") || strings.HasPrefix(ip, "1.") {
|
||||
continue // network address, or a 0./1. version-style quad
|
||||
}
|
||||
if !isSensitiveIP(ip) {
|
||||
continue
|
||||
}
|
||||
add(catPublicIP, severityMedium, ip, "public IP reveals provider / site")
|
||||
}
|
||||
ip := line[loc[0]:loc[1]]
|
||||
if !isSensitiveIP(ip) {
|
||||
continue
|
||||
}
|
||||
add(catPublicIP, severityMedium, ip, "public IP reveals provider / site")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(line), "timezone") || strings.Contains(line, "/") {
|
||||
if ll := strings.ToLower(line); strings.Contains(ll, "timezone") || strings.Contains(ll, "zoneinfo") || strings.Contains(ll, "localtime") || strings.Contains(ll, "tz=") {
|
||||
for _, m := range reTZName.FindAllStringSubmatch(line, -1) {
|
||||
if isAllowlistedValue(m[1]) {
|
||||
continue
|
||||
@@ -184,24 +207,79 @@ func isAlnum(r rune) bool {
|
||||
return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9')
|
||||
}
|
||||
|
||||
// inDottedNumberRun reports whether line[start:end] (a dotted quad) is embedded
|
||||
// in a longer numeric run - a version string like "18:6.1.4.5" or "1.2.3.4.5" -
|
||||
// rather than a standalone address.
|
||||
func inDottedNumberRun(line string, start, end int) bool {
|
||||
if start > 0 {
|
||||
switch line[start-1] {
|
||||
case '.', ':':
|
||||
if start >= 2 && line[start-2] >= '0' && line[start-2] <= '9' {
|
||||
return true
|
||||
}
|
||||
// cleanTokenBoundary reports whether line[s:e] stands as its own token - not
|
||||
// glued to surrounding identifier characters ("auth.backend.gssapi.store-creds",
|
||||
// "mountall.sh [start]" is fine, "96.00.CF.00" is not) - so it reads as a
|
||||
// hostname rather than a fragment of code, a path tail, or a version string.
|
||||
func cleanTokenBoundary(line string, s, e int) bool {
|
||||
if s > 0 {
|
||||
switch c := line[s-1]; c {
|
||||
case ' ', '\t', '"', '\'', '(', '<', '=', ',', '@':
|
||||
case '/', ':':
|
||||
// ok (URL / "server = host:port"), but not a bare path segment
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
if end < len(line) && line[end] == '.' {
|
||||
if e < len(line) {
|
||||
switch line[e] {
|
||||
case ' ', '\t', '"', '\'', ')', '>', ',', ';', ':', '/', '\\', '?', '!':
|
||||
default:
|
||||
return false // trailing '.', '-', letter, digit -> mid-identifier
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// inDottedNumberRun reports whether line[start:end] (a dotted quad) is embedded
|
||||
// in a longer version-like run - "18:6.1.4.5", "v1.2.3.4", "1.2.3.4.5",
|
||||
// "go1.25.0.linux" - rather than a standalone address.
|
||||
func inDottedNumberRun(line string, start, end int) bool {
|
||||
if start > 0 {
|
||||
switch c := line[start-1]; {
|
||||
case c == '.', c == '-':
|
||||
return true
|
||||
case c == ':' && start >= 2 && line[start-2] >= '0' && line[start-2] <= '9':
|
||||
return true
|
||||
case c == 'v' || c == 'V':
|
||||
return true
|
||||
}
|
||||
}
|
||||
if end < len(line) && (line[end] == '.' || line[end] == '-') {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// usefulFRUValue drops FRU asset/location values that carry no site information:
|
||||
// the field name echoed back, an all-digit manufacturing code, or a string that
|
||||
// is just the chassis serial (already in the archive name).
|
||||
func usefulFRUValue(v string) bool {
|
||||
lv := strings.ToLower(strings.ReplaceAll(v, " ", ""))
|
||||
if strings.Contains(lv, "assettag") || strings.Contains(lv, "serialnumber") {
|
||||
return false
|
||||
}
|
||||
if len(v) < 4 {
|
||||
return false
|
||||
}
|
||||
digits, upperAlnum := 0, 0
|
||||
for _, r := range v {
|
||||
if r >= '0' && r <= '9' {
|
||||
digits++
|
||||
}
|
||||
if (r >= '0' && r <= '9') || (r >= 'A' && r <= 'Z') {
|
||||
upperAlnum++
|
||||
}
|
||||
}
|
||||
if digits == len(v) {
|
||||
return false // pure number: date/manufacturing code
|
||||
}
|
||||
if upperAlnum == len(v) && digits >= 4 {
|
||||
return false // looks like a serial number (7J..., 21D634070)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// looksLikeMailHost rejects the many "local@identifier.token" strings that are
|
||||
// not e-mail: OData/Redfish JSON annotations (Members@odata.count), SSH
|
||||
// cipher/kex names (aes256-gcm@openssh.com is handled by the domain allowlist,
|
||||
@@ -211,20 +289,10 @@ func looksLikeMailHost(host string) bool {
|
||||
if strings.Contains(h, "odata") || strings.Contains(h, "redfish") || strings.Contains(h, "message.") {
|
||||
return false
|
||||
}
|
||||
dot := strings.LastIndexByte(h, '.')
|
||||
if dot < 0 {
|
||||
return false
|
||||
}
|
||||
tld := h[dot+1:]
|
||||
if len(tld) < 2 || len(tld) > 24 {
|
||||
return false
|
||||
}
|
||||
for _, r := range tld {
|
||||
if r < 'a' || r > 'z' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
// systemd templated units ("serial-getty@ttyAMA0.service"), Go toolchain
|
||||
// pseudo-versions ("toolchain@v0.0.1-go1.25.0.linux") and the like all fail
|
||||
// the real-TLD check.
|
||||
return plausibleFQDN(h)
|
||||
}
|
||||
|
||||
// matchCoveredBySpecific reports whether a bare FQDN/e-mail finding is already
|
||||
|
||||
Reference in New Issue
Block a user