fix(privacy): calibrate against the RMA log corpus (130k files)
Cross-checked internal/privacy against the batch companion-app report over project/rma. Fixes for the dominant false-positive classes: - Real-TLD gate (tld.go): FQDN/e-mail must end in a curated TLD or a pseudo-TLD with a >=3-char label; two-letter file/code suffixes (.sh .so .md .id .service ...) are a hard denylist. Kills "0.linux" (45996 hits), "mountall.sh", "libc.so", "@odata.id", "serial-getty@ttyAMA0.service". - Clean-token boundary + Title-case reject: "auth.backend.gssapi.store-creds", "OS.It" are code, not hosts. - Kernel ring-buffer ("[ 8.07][ T1] ...") and Go stack-trace lines skipped. - resolv domain/search values must contain a dot ("domain 53" -> out). - IPv4: skip comment lines, version/spec lines (X.Org, IEEE Std, l0fw_ver), "0."/"1."/".0" quads; allowlist Yandex resolvers + RFC3849 2001:db8::/32. - fru_location: drop all-digit / serial-like / field-name-echo values. - Drop the hostname rule (zero real hits, only "bmc-state-manager" noise). - domain category: high -> low, medium at 3+ labels. Real customer signal now comes from resolv/nsupdate/ad_ldap/mgmt, which the corpus confirms catches every actual customer (netwell.local, tcsbank.ru). - Allowlist smartmontools.org, openib.org, apache.org, freebsd.org, golang.org, ipxe.org, nvidia.com and other FOSS/vendor infra; skip LOGPile's own raw_export.json / parser_fields.json / collect.log members. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
551d8e450d
commit
2be215fca1
@@ -1,6 +1,9 @@
|
||||
package privacy
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Fixtures use "acme.ru" as a stand-in customer domain. The RFC 2606
|
||||
// "example.*" names are on the allowlist (they are the sanitization target),
|
||||
@@ -148,6 +151,50 @@ func TestScan_RealResolvStillCaughtInNoisyFile(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScan_OSConfigNoise(t *testing.T) {
|
||||
// Everything here is stock OS / BEE-SP / Redfish text - zero customer data.
|
||||
noise := strings.Join([]string{
|
||||
`Process: 3950 ExecStartPre=/usr/bin/nvidia-fabricmanager-start.sh --mode`,
|
||||
`echo "Usage: mountall.sh [start|stop]" >&2`,
|
||||
`. /lib/init/vars.sh`,
|
||||
`auth.backend.gssapi.store-creds = "disable"`,
|
||||
`server.network-backend = "writev"`,
|
||||
`ssl.ca-file = "/conf/server.pem"`,
|
||||
`# ALL EXCEPT in.fingerd: other.host.name, .other.domain`,
|
||||
`/lib/arm-linux-gnueabihf/libc.so.6`,
|
||||
`"@odata.id": "/redfish/v1/Systems/1"`,
|
||||
`X.Org X Server 1.21.1.7`,
|
||||
`ME FW Version 6.1.4.75`,
|
||||
`96.00.CF.00.03 VBIOS`,
|
||||
`#option dns 129.219.13.81`,
|
||||
`toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sema.go:9`,
|
||||
`serial-getty@ttyAMA0.service`,
|
||||
`Copyright (C) 2002-22, www.smartmontools.org`,
|
||||
`OpenIB.org BSD license (FreeBSD Variant)`,
|
||||
}, "\n")
|
||||
rep := Scan([]File{{Path: "onekeylog/log/sollog/SOLHostCapture.log", Content: []byte(noise)}})
|
||||
if rep != nil && len(rep.Findings) > 0 {
|
||||
t.Fatalf("OS config noise flagged: %+v", rep.Findings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScan_ResolvNeedsDottedValue(t *testing.T) {
|
||||
rep := Scan([]File{{Path: "resolv.conf", Content: []byte(
|
||||
"domain 53\nsearch nameserver\ndomain corp.acme.ru\n")}})
|
||||
if rep == nil {
|
||||
t.Fatal("nil")
|
||||
}
|
||||
var got []string
|
||||
for _, f := range rep.Findings {
|
||||
if f.Category == catResolv {
|
||||
got = append(got, f.Match)
|
||||
}
|
||||
}
|
||||
if len(got) != 1 || got[0] != "corp.acme.ru" {
|
||||
t.Fatalf("resolv matches = %v, want [corp.acme.ru]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummary(t *testing.T) {
|
||||
rep := Scan([]File{{Path: "resolv.conf", Content: []byte("domain acme.ru\n")}})
|
||||
if rep.Summary.Total != len(rep.Findings) || rep.Summary.Total == 0 {
|
||||
|
||||
Reference in New Issue
Block a user