fix(privacy): calibrate against the RMA log corpus (130k files)
Cross-checked internal/privacy against the batch companion-app report over project/rma. Fixes for the dominant false-positive classes: - Real-TLD gate (tld.go): FQDN/e-mail must end in a curated TLD or a pseudo-TLD with a >=3-char label; two-letter file/code suffixes (.sh .so .md .id .service ...) are a hard denylist. Kills "0.linux" (45996 hits), "mountall.sh", "libc.so", "@odata.id", "serial-getty@ttyAMA0.service". - Clean-token boundary + Title-case reject: "auth.backend.gssapi.store-creds", "OS.It" are code, not hosts. - Kernel ring-buffer ("[ 8.07][ T1] ...") and Go stack-trace lines skipped. - resolv domain/search values must contain a dot ("domain 53" -> out). - IPv4: skip comment lines, version/spec lines (X.Org, IEEE Std, l0fw_ver), "0."/"1."/".0" quads; allowlist Yandex resolvers + RFC3849 2001:db8::/32. - fru_location: drop all-digit / serial-like / field-name-echo values. - Drop the hostname rule (zero real hits, only "bmc-state-manager" noise). - domain category: high -> low, medium at 3+ labels. Real customer signal now comes from resolv/nsupdate/ad_ldap/mgmt, which the corpus confirms catches every actual customer (netwell.local, tcsbank.ru). - Allowlist smartmontools.org, openib.org, apache.org, freebsd.org, golang.org, ipxe.org, nvidia.com and other FOSS/vendor infra; skip LOGPile's own raw_export.json / parser_fields.json / collect.log members. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
551d8e450d
commit
2be215fca1
@@ -13,13 +13,18 @@ var (
|
||||
allowlistedZones = []string{
|
||||
"example.com", "example.net", "example.org", "example.local", "example.edu",
|
||||
"foobar.edu", "issue.net",
|
||||
"localhost", "localdomain", "local.lan",
|
||||
"localhost", "localdomain", "local.lan", "localhost.localdomain",
|
||||
"host.name", "other.host.name", "some.host.name", "other.domain", "some.domain",
|
||||
"pool.ntp.org", "ntp.org", "nist.gov", "windows.com", "microsoft.com",
|
||||
"dmtf.org", "iana.org", "openssl.org", "openssh.com", "openssh.org",
|
||||
"libssh.org", "rsyslog.com", "adiscon.com", "redhat.com", "kernel.org",
|
||||
"megarac.com", "ami.com", "commond.com",
|
||||
"oasis-open.org", "w3.org", "xmlsoap.org", "purl.org", "ietf.org",
|
||||
"nvidia.com", "mellanox.com", "gnu.org", "debian.org", "ubuntu.com",
|
||||
"smartmontools.org", "openib.org", "apache.org", "freebsd.org", "freedesktop.org",
|
||||
"golang.org", "go.dev", "x.org", "sourceforge.net", "xyz.com", "yandex.net",
|
||||
"haxx.se", "curl.se", "python.org", "perl.org", "isc.org", "sourceware.org",
|
||||
"ipxe.org", "gnupg.org", "gnutls.org", "openldap.org", "netfilter.org",
|
||||
"inspur.com", "inspurcloud.com", "inservice-iq.com", "ieisystem.com", "kaytus.com",
|
||||
"jd.com", "jd.local", "jdcloud.com", "in-addr.arpa", "ip6.arpa", "arpa",
|
||||
}
|
||||
@@ -46,10 +51,13 @@ var (
|
||||
"[empty]": {},
|
||||
}
|
||||
|
||||
// Archive members that are vendor factory templates, not the active config.
|
||||
// Archive members that are vendor factory templates or LOGPile's own derived
|
||||
// export artifacts (operator-entered target host, Redfish annotation keys) -
|
||||
// not customer data from the source.
|
||||
allowlistedFilenameParts = []string{
|
||||
"_tencent", "_jingdong", "_pdd", "_baidu", "_kuaishou", "_tianyiyun",
|
||||
"_jd.", "syslog_jd", "snmptrapcfg", ".json_bak", "ntp_auto",
|
||||
"raw_export.json", "parser_fields.json", "collect.log",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user