Files
logpile/internal/privacy/privacy_test.go
T
Mikhail ChusavitinandClaude Sonnet 5 2be215fca1 fix(privacy): calibrate against the RMA log corpus (130k files)
Cross-checked internal/privacy against the batch companion-app report over
project/rma. Fixes for the dominant false-positive classes:

- Real-TLD gate (tld.go): FQDN/e-mail must end in a curated TLD or a
  pseudo-TLD with a >=3-char label; two-letter file/code suffixes
  (.sh .so .md .id .service ...) are a hard denylist. Kills "0.linux"
  (45996 hits), "mountall.sh", "libc.so", "@odata.id",
  "serial-getty@ttyAMA0.service".
- Clean-token boundary + Title-case reject: "auth.backend.gssapi.store-creds",
  "OS.It" are code, not hosts.
- Kernel ring-buffer ("[ 8.07][ T1] ...") and Go stack-trace lines skipped.
- resolv domain/search values must contain a dot ("domain 53" -> out).
- IPv4: skip comment lines, version/spec lines (X.Org, IEEE Std, l0fw_ver),
  "0."/"1."/".0" quads; allowlist Yandex resolvers + RFC3849 2001:db8::/32.
- fru_location: drop all-digit / serial-like / field-name-echo values.
- Drop the hostname rule (zero real hits, only "bmc-state-manager" noise).
- domain category: high -> low, medium at 3+ labels. Real customer signal
  now comes from resolv/nsupdate/ad_ldap/mgmt, which the corpus confirms
  catches every actual customer (netwell.local, tcsbank.ru).
- Allowlist smartmontools.org, openib.org, apache.org, freebsd.org,
  golang.org, ipxe.org, nvidia.com and other FOSS/vendor infra; skip
  LOGPile's own raw_export.json / parser_fields.json / collect.log members.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-02 16:45:35 +03:00

207 lines
6.8 KiB
Go

package privacy
import (
"strings"
"testing"
)
// Fixtures use "acme.ru" as a stand-in customer domain. The RFC 2606
// "example.*" names are on the allowlist (they are the sanitization target),
// so they cannot double as the customer placeholder in detection tests.
func TestScan_DetectsAndClassifies(t *testing.T) {
files := []File{
{Path: "onekeylog/configuration/conf/resolv.conf", Content: []byte(
"domain corp.acme.ru\nsearch corp.acme.ru\nnameserver 10.10.0.1\n")},
{Path: "onekeylog/configuration/conf/activedir.conf", Content: []byte(
"racdomain=corp.acme.ru\nadfilterdc1=10.10.0.5\nrolegroup1name=cab-gr-CI00363277-x86bmc\n")},
{Path: "onekeylog/configuration/conf/BMC1/misccfg.ini", Content: []byte(
"TimeZone=Europe/Moscow\nSELTimeUTCOffset=180\n")},
{Path: "onekeylog/log/audit.log", Content: []byte(
"2026-08-24T11:47:34+03:00 login admin from ntp01.acme.ru ok\n" +
"contact ops@acme.ru for access\n" +
"outbound 45.32.10.7 established\n" +
"internal 192.168.31.4 and 10.1.2.3 and doc 203.0.113.9\n")},
{Path: "onekeylog/runningdata/rundatainfo.log", Content: []byte(
"Mon Aug 24 11:58:48 MSK 2026\n")},
// Vendor factory template - must be skipped entirely.
{Path: "onekeylog/configuration/conf/ntp_auto.conf", Content: []byte(
"server ntp.should-not-match.ru\n")},
{Path: "onekeylog/bin/blob", Content: []byte{0x00, 0x01, 0x02, 0xff}},
}
rep := Scan(files)
if rep == nil {
t.Fatal("nil report")
}
if rep.FilesScanned != 5 {
t.Fatalf("FilesScanned = %d, want 5 (binary + template skipped)", rep.FilesScanned)
}
has := func(cat, match string) bool {
for _, f := range rep.Findings {
if f.Category == cat && f.Match == match {
return true
}
}
return false
}
mustHave := []struct{ cat, match string }{
{catResolv, "corp.acme.ru"},
{catADLDAP, "corp.acme.ru"},
{catADLDAP, "cab-gr-CI00363277-x86bmc"},
{catTimezone, "Europe/Moscow"},
{catTimezone, "180"},
{catDomain, "ntp01.acme.ru"},
{catEmail, "ops@acme.ru"},
{catPublicIP, "45.32.10.7"},
{catTimezone, "MSK"},
}
for _, w := range mustHave {
if !has(w.cat, w.match) {
t.Errorf("missing finding %s / %q", w.cat, w.match)
}
}
mustNotHave := []string{"192.168.31.4", "10.1.2.3", "203.0.113.9", "ntp.should-not-match.ru"}
for _, f := range rep.Findings {
for _, bad := range mustNotHave {
if f.Match == bad {
t.Errorf("unexpected finding for %q (%s)", bad, f.Category)
}
}
}
if len(rep.Customers) == 0 || rep.Customers[0].Domain != "acme.ru" {
t.Fatalf("customer guess = %+v, want acme.ru first", rep.Customers)
}
if rep.Customers[0].Confidence != "high" {
t.Errorf("confidence = %s, want high", rep.Customers[0].Confidence)
}
}
func TestScan_NothingToScan(t *testing.T) {
if Scan(nil) != nil {
t.Fatal("want nil for no files")
}
if Scan([]File{{Path: "x", Content: []byte{0}}}) != nil {
t.Fatal("want nil when only binary files")
}
}
func TestScan_Dedupe(t *testing.T) {
rep := Scan([]File{{Path: "a.log", Content: []byte(
"host is srv-prod.acme.net\nhost is srv-prod.acme.net again\n")}})
if rep == nil {
t.Fatal("nil report")
}
n := 0
for _, f := range rep.Findings {
if f.Category == catDomain && f.Match == "srv-prod.acme.net" {
n++
}
}
if n != 1 {
t.Fatalf("domain finding counted %d times, want 1", n)
}
}
func TestScan_AllowlistedDomainNotFlagged(t *testing.T) {
rep := Scan([]File{{Path: "ntp.conf", Content: []byte(
"server 0.pool.ntp.org\nserver time.nist.gov\ncontact admin@example.com\n")}})
if rep != nil && len(rep.Findings) > 0 {
t.Fatalf("allowlisted infra flagged: %+v", rep.Findings)
}
}
func TestScan_NvidiaBugReportNoise(t *testing.T) {
// dmidecode + dmesg boilerplate that must not be flagged.
rep := Scan([]File{{Path: "nvidia-bug-report.log", Content: []byte(
"driver bug via the NVIDIA Linux forum (see forums.developer.nvidia.com)\n" +
"or by sending email to 'linux-bugs@nvidia.com'.\n" +
" Asset Tag: Base Board Asset Tag\n" +
" P1-DIMMA1_AssetTag (Date:24/31)\n" +
" ME FW Version\n 18:6.1.4.5\n" +
"[ 8.078174] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <giometti@linux.it>\n")}})
if rep != nil && len(rep.Findings) > 0 {
t.Fatalf("nvidia boilerplate flagged: %+v", rep.Findings)
}
if rep != nil && len(rep.Customers) > 0 {
t.Fatalf("nvidia boilerplate produced a customer guess: %+v", rep.Customers)
}
}
func TestScan_RealResolvStillCaughtInNoisyFile(t *testing.T) {
rep := Scan([]File{{Path: "nvidia-bug-report.log", Content: []byte(
"[ 8.078174] pps_core: <giometti@linux.it>\n" +
"--- /etc/resolv.conf ---\ndomain corp.acme.ru\nnameserver 10.0.0.1\n")}})
if rep == nil {
t.Fatal("nil report")
}
found := false
for _, f := range rep.Findings {
if f.Category == catResolv && f.Match == "corp.acme.ru" {
found = true
}
}
if !found {
t.Fatalf("real resolv.conf leak missed: %+v", rep.Findings)
}
if len(rep.Customers) == 0 || rep.Customers[0].Domain != "acme.ru" {
t.Fatalf("customer guess = %+v, want acme.ru", rep.Customers)
}
}
func TestScan_OSConfigNoise(t *testing.T) {
// Everything here is stock OS / BEE-SP / Redfish text - zero customer data.
noise := strings.Join([]string{
`Process: 3950 ExecStartPre=/usr/bin/nvidia-fabricmanager-start.sh --mode`,
`echo "Usage: mountall.sh [start|stop]" >&2`,
`. /lib/init/vars.sh`,
`auth.backend.gssapi.store-creds = "disable"`,
`server.network-backend = "writev"`,
`ssl.ca-file = "/conf/server.pem"`,
`# ALL EXCEPT in.fingerd: other.host.name, .other.domain`,
`/lib/arm-linux-gnueabihf/libc.so.6`,
`"@odata.id": "/redfish/v1/Systems/1"`,
`X.Org X Server 1.21.1.7`,
`ME FW Version 6.1.4.75`,
`96.00.CF.00.03 VBIOS`,
`#option dns 129.219.13.81`,
`toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sema.go:9`,
`serial-getty@ttyAMA0.service`,
`Copyright (C) 2002-22, www.smartmontools.org`,
`OpenIB.org BSD license (FreeBSD Variant)`,
}, "\n")
rep := Scan([]File{{Path: "onekeylog/log/sollog/SOLHostCapture.log", Content: []byte(noise)}})
if rep != nil && len(rep.Findings) > 0 {
t.Fatalf("OS config noise flagged: %+v", rep.Findings)
}
}
func TestScan_ResolvNeedsDottedValue(t *testing.T) {
rep := Scan([]File{{Path: "resolv.conf", Content: []byte(
"domain 53\nsearch nameserver\ndomain corp.acme.ru\n")}})
if rep == nil {
t.Fatal("nil")
}
var got []string
for _, f := range rep.Findings {
if f.Category == catResolv {
got = append(got, f.Match)
}
}
if len(got) != 1 || got[0] != "corp.acme.ru" {
t.Fatalf("resolv matches = %v, want [corp.acme.ru]", got)
}
}
func TestSummary(t *testing.T) {
rep := Scan([]File{{Path: "resolv.conf", Content: []byte("domain acme.ru\n")}})
if rep.Summary.Total != len(rep.Findings) || rep.Summary.Total == 0 {
t.Fatalf("summary total mismatch: %+v", rep.Summary)
}
if rep.Summary.High == 0 {
t.Fatalf("expected a high finding, got %+v", rep.Summary)
}
}