Mikhail ChusavitinandClaude Sonnet 5
2be215fca1
Cross-checked internal/privacy against the batch companion-app report over project/rma. Fixes for the dominant false-positive classes: - Real-TLD gate (tld.go): FQDN/e-mail must end in a curated TLD or a pseudo-TLD with a >=3-char label; two-letter file/code suffixes (.sh .so .md .id .service ...) are a hard denylist. Kills "0.linux" (45996 hits), "mountall.sh", "libc.so", "@odata.id", "serial-getty@ttyAMA0.service". - Clean-token boundary + Title-case reject: "auth.backend.gssapi.store-creds", "OS.It" are code, not hosts. - Kernel ring-buffer ("[ 8.07][ T1] ...") and Go stack-trace lines skipped. - resolv domain/search values must contain a dot ("domain 53" -> out). - IPv4: skip comment lines, version/spec lines (X.Org, IEEE Std, l0fw_ver), "0."/"1."/".0" quads; allowlist Yandex resolvers + RFC3849 2001:db8::/32. - fru_location: drop all-digit / serial-like / field-name-echo values. - Drop the hostname rule (zero real hits, only "bmc-state-manager" noise). - domain category: high -> low, medium at 3+ labels. Real customer signal now comes from resolv/nsupdate/ad_ldap/mgmt, which the corpus confirms catches every actual customer (netwell.local, tcsbank.ru). - Allowlist smartmontools.org, openib.org, apache.org, freebsd.org, golang.org, ipxe.org, nvidia.com and other FOSS/vendor infra; skip LOGPile's own raw_export.json / parser_fields.json / collect.log members. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
99 lines
3.2 KiB
Go
99 lines
3.2 KiB
Go
package privacy
|
|
|
|
import (
|
|
"path"
|
|
"strings"
|
|
)
|
|
|
|
// Public infrastructure, documentation, and vendor-default values that are not
|
|
// customer leaks. These are reference data (RFC 2606 / 5737 names, well-known
|
|
// NTP pools, standards-body domains, factory defaults), not vendor-detection
|
|
// logic.
|
|
var (
|
|
allowlistedZones = []string{
|
|
"example.com", "example.net", "example.org", "example.local", "example.edu",
|
|
"foobar.edu", "issue.net",
|
|
"localhost", "localdomain", "local.lan", "localhost.localdomain",
|
|
"host.name", "other.host.name", "some.host.name", "other.domain", "some.domain",
|
|
"pool.ntp.org", "ntp.org", "nist.gov", "windows.com", "microsoft.com",
|
|
"dmtf.org", "iana.org", "openssl.org", "openssh.com", "openssh.org",
|
|
"libssh.org", "rsyslog.com", "adiscon.com", "redhat.com", "kernel.org",
|
|
"megarac.com", "ami.com", "commond.com",
|
|
"oasis-open.org", "w3.org", "xmlsoap.org", "purl.org", "ietf.org",
|
|
"nvidia.com", "mellanox.com", "gnu.org", "debian.org", "ubuntu.com",
|
|
"smartmontools.org", "openib.org", "apache.org", "freebsd.org", "freedesktop.org",
|
|
"golang.org", "go.dev", "x.org", "sourceforge.net", "xyz.com", "yandex.net",
|
|
"haxx.se", "curl.se", "python.org", "perl.org", "isc.org", "sourceware.org",
|
|
"ipxe.org", "gnupg.org", "gnutls.org", "openldap.org", "netfilter.org",
|
|
"inspur.com", "inspurcloud.com", "inservice-iq.com", "ieisystem.com", "kaytus.com",
|
|
"jd.com", "jd.local", "jdcloud.com", "in-addr.arpa", "ip6.arpa", "arpa",
|
|
}
|
|
|
|
allowlistedValues = map[string]struct{}{
|
|
"asia/shanghai": {},
|
|
"etc/utc": {},
|
|
"utc": {},
|
|
"to be filled by o.e.m.": {},
|
|
"default string": {},
|
|
"unknown": {},
|
|
"n/a": {},
|
|
"none": {},
|
|
"null": {},
|
|
"0": {},
|
|
"0.0.0.0": {},
|
|
"not specified": {},
|
|
"not available": {},
|
|
"not present": {},
|
|
"unspecified": {},
|
|
"no asset tag": {},
|
|
"no asset information": {},
|
|
"empty": {},
|
|
"[empty]": {},
|
|
}
|
|
|
|
// Archive members that are vendor factory templates or LOGPile's own derived
|
|
// export artifacts (operator-entered target host, Redfish annotation keys) -
|
|
// not customer data from the source.
|
|
allowlistedFilenameParts = []string{
|
|
"_tencent", "_jingdong", "_pdd", "_baidu", "_kuaishou", "_tianyiyun",
|
|
"_jd.", "syslog_jd", "snmptrapcfg", ".json_bak", "ntp_auto",
|
|
"raw_export.json", "parser_fields.json", "collect.log",
|
|
}
|
|
)
|
|
|
|
func isAllowlistedDomain(domain string) bool {
|
|
d := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(domain), "."))
|
|
for _, z := range allowlistedZones {
|
|
if d == z || strings.HasSuffix(d, "."+z) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isAllowlistedValue(v string) bool {
|
|
lv := strings.ToLower(strings.TrimSpace(v))
|
|
if _, ok := allowlistedValues[lv]; ok {
|
|
return true
|
|
}
|
|
return isAllowlistedDomain(v)
|
|
}
|
|
|
|
func isAllowlistedFilename(p string) bool {
|
|
lp := strings.ToLower(p)
|
|
for _, part := range allowlistedFilenameParts {
|
|
if strings.Contains(lp, part) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isCertFilename(p string) bool {
|
|
switch strings.ToLower(path.Ext(p)) {
|
|
case ".pem", ".csr", ".crt", ".cer":
|
|
return true
|
|
}
|
|
return false
|
|
}
|