fix(privacy): calibrate against the RMA log corpus (130k files)
Cross-checked internal/privacy against the batch companion-app report over project/rma. Fixes for the dominant false-positive classes: - Real-TLD gate (tld.go): FQDN/e-mail must end in a curated TLD or a pseudo-TLD with a >=3-char label; two-letter file/code suffixes (.sh .so .md .id .service ...) are a hard denylist. Kills "0.linux" (45996 hits), "mountall.sh", "libc.so", "@odata.id", "serial-getty@ttyAMA0.service". - Clean-token boundary + Title-case reject: "auth.backend.gssapi.store-creds", "OS.It" are code, not hosts. - Kernel ring-buffer ("[ 8.07][ T1] ...") and Go stack-trace lines skipped. - resolv domain/search values must contain a dot ("domain 53" -> out). - IPv4: skip comment lines, version/spec lines (X.Org, IEEE Std, l0fw_ver), "0."/"1."/".0" quads; allowlist Yandex resolvers + RFC3849 2001:db8::/32. - fru_location: drop all-digit / serial-like / field-name-echo values. - Drop the hostname rule (zero real hits, only "bmc-state-manager" noise). - domain category: high -> low, medium at 3+ labels. Real customer signal now comes from resolv/nsupdate/ad_ldap/mgmt, which the corpus confirms catches every actual customer (netwell.local, tcsbank.ru). - Allowlist smartmontools.org, openib.org, apache.org, freebsd.org, golang.org, ipxe.org, nvidia.com and other FOSS/vendor infra; skip LOGPile's own raw_export.json / parser_fields.json / collect.log members. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
551d8e450d
commit
2be215fca1
@@ -0,0 +1,75 @@
|
||||
package privacy
|
||||
|
||||
import "strings"
|
||||
|
||||
// Top-level domains a customer domain in these logs actually ends in: the common
|
||||
// gTLDs plus the ccTLDs for our region and the major western countries. This is
|
||||
// deliberately tight, not the full IANA list - a domain under some rare ccTLD
|
||||
// (.sh, .so, .cf, .lc ...) simply is not flagged by the bare-FQDN rule, which is
|
||||
// an acceptable miss: those two-letter suffixes are almost always a filename
|
||||
// extension or a code identifier ("mountall.sh", "libc.so", "@odata.id"), and
|
||||
// the real customer signal comes from resolv.conf / nsupdate / AD config anyway.
|
||||
var knownTLD = func() map[string]struct{} {
|
||||
list := strings.Fields(`
|
||||
com net org edu gov mil int info biz pro asia mobi tel coop aero museum travel
|
||||
app dev cloud tech xyz io co ai me tv cc
|
||||
ru su by ua kz md ge am az uz kg tj tm
|
||||
uk de fr nl be lu it es pt pl cz sk at ch se no fi dk ie eu is
|
||||
ee lv lt ro bg hr si rs hu gr tr
|
||||
us ca mx br ar cn jp kr in sg hk tw au nz il ae sa za
|
||||
`)
|
||||
m := make(map[string]struct{}, len(list))
|
||||
for _, t := range list {
|
||||
m[t] = struct{}{}
|
||||
}
|
||||
return m
|
||||
}()
|
||||
|
||||
// Extensions / code-identifier suffixes that shadow a ccTLD. A dotted name
|
||||
// ending in one of these is a filename or identifier, never a domain.
|
||||
var shadowSuffix = func() map[string]struct{} {
|
||||
list := strings.Fields(`
|
||||
sh bash zsh ksh so py pyc pl pm rb js mjs cjs jsx ts tsx go rs kt
|
||||
c cc cpp cxx h hpp hh hxx mm md rst txt text log out err trace
|
||||
conf config cfg cnf ini toml yaml yml json xml html htm xhtml svg css scss
|
||||
sql csv tsv pem crt cer key csr der p12 pfx bak old orig tmp temp swp lock pid
|
||||
sock service socket target timer mount automount path slice scope device dump core
|
||||
bin dat db img iso gz xz bz2 zip tar tgz rar rpm deb apk whl jar war
|
||||
class obj lib la lo ko dll dylib exe map lst asm
|
||||
id ctx ver rev sig hash sum cache idx pack ref meta prop props store network backend
|
||||
`)
|
||||
m := make(map[string]struct{}, len(list))
|
||||
for _, t := range list {
|
||||
m[t] = struct{}{}
|
||||
}
|
||||
return m
|
||||
}()
|
||||
|
||||
// Non-registrable suffixes that still identify a customer's internal zone.
|
||||
var pseudoTLD = map[string]struct{}{
|
||||
"local": {}, "lan": {}, "home": {}, "corp": {}, "internal": {}, "intranet": {},
|
||||
}
|
||||
|
||||
// plausibleFQDN reports whether host is a real domain worth flagging.
|
||||
func plausibleFQDN(host string) bool {
|
||||
host = strings.ToLower(strings.Trim(strings.TrimSpace(host), ".*"))
|
||||
if strings.Contains(host, "odata") || strings.Contains(host, "redfish") {
|
||||
return false
|
||||
}
|
||||
labels := strings.Split(host, ".")
|
||||
if len(labels) < 2 {
|
||||
return false
|
||||
}
|
||||
tld := labels[len(labels)-1]
|
||||
sld := labels[len(labels)-2]
|
||||
if _, ok := shadowSuffix[tld]; ok {
|
||||
return false
|
||||
}
|
||||
if _, ok := knownTLD[tld]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := pseudoTLD[tld]; ok {
|
||||
return len(sld) >= 3
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user