feat(privacy): scan ingested sources for customer-identifying data
Detection-only scan (internal/privacy) attached to every AnalysisResult: a customer-domain guess plus a findings list (category, file, line, match, hint), ported from the KB grep playbook. Runs on archive uploads and the serialized Redfish tree; gated by LOGPILE_PRIVACY_SCAN (default on). Surfaced at GET /api/privacy-scan, in the "Customer data" UI panel, and as privacy_report.json in the raw-export bundle. IP policy keeps RFC1918 and example ranges out of findings; allowlist covers standards-body and vendor infrastructure domains. No customer tokens in the repo. See ADL-066 and bible-local/docs/privacy-scan.md. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
3311bafd8e
commit
4a4910f207
@@ -0,0 +1,81 @@
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Public infrastructure, documentation, and vendor-default values that are not
|
||||
// customer leaks. These are reference data (RFC 2606 / 5737 names, well-known
|
||||
// NTP pools, standards-body domains, factory defaults), not vendor-detection
|
||||
// logic.
|
||||
var (
|
||||
allowlistedZones = []string{
|
||||
"example.com", "example.net", "example.org", "example.local", "example.edu",
|
||||
"foobar.edu", "issue.net",
|
||||
"localhost", "localdomain", "local.lan",
|
||||
"pool.ntp.org", "ntp.org", "nist.gov", "windows.com", "microsoft.com",
|
||||
"dmtf.org", "iana.org", "openssl.org", "openssh.com", "openssh.org",
|
||||
"libssh.org", "rsyslog.com", "adiscon.com", "redhat.com", "kernel.org",
|
||||
"megarac.com", "ami.com", "commond.com",
|
||||
"oasis-open.org", "w3.org", "xmlsoap.org", "purl.org", "ietf.org",
|
||||
"inspur.com", "inspurcloud.com", "inservice-iq.com", "kaytus.com",
|
||||
"jd.com", "jd.local", "jdcloud.com", "in-addr.arpa", "ip6.arpa", "arpa",
|
||||
}
|
||||
|
||||
allowlistedValues = map[string]struct{}{
|
||||
"asia/shanghai": {},
|
||||
"etc/utc": {},
|
||||
"utc": {},
|
||||
"to be filled by o.e.m.": {},
|
||||
"default string": {},
|
||||
"unknown": {},
|
||||
"n/a": {},
|
||||
"none": {},
|
||||
"null": {},
|
||||
"0": {},
|
||||
"0.0.0.0": {},
|
||||
}
|
||||
|
||||
// Archive members that are vendor factory templates, not the active config.
|
||||
allowlistedFilenameParts = []string{
|
||||
"_tencent", "_jingdong", "_pdd", "_baidu", "_kuaishou", "_tianyiyun",
|
||||
"_jd.", "syslog_jd", "snmptrapcfg", ".json_bak", "ntp_auto",
|
||||
}
|
||||
)
|
||||
|
||||
func isAllowlistedDomain(domain string) bool {
|
||||
d := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(domain), "."))
|
||||
for _, z := range allowlistedZones {
|
||||
if d == z || strings.HasSuffix(d, "."+z) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isAllowlistedValue(v string) bool {
|
||||
lv := strings.ToLower(strings.TrimSpace(v))
|
||||
if _, ok := allowlistedValues[lv]; ok {
|
||||
return true
|
||||
}
|
||||
return isAllowlistedDomain(v)
|
||||
}
|
||||
|
||||
func isAllowlistedFilename(p string) bool {
|
||||
lp := strings.ToLower(p)
|
||||
for _, part := range allowlistedFilenameParts {
|
||||
if strings.Contains(lp, part) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isCertFilename(p string) bool {
|
||||
switch strings.ToLower(path.Ext(p)) {
|
||||
case ".pem", ".csr", ".crt", ".cer":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
)
|
||||
|
||||
// Multi-label public suffixes handled explicitly; every other final label is
|
||||
// treated as the suffix. This is a heuristic, not a full PSL - good enough to
|
||||
// collapse "sn-x.mgmt.corp.example.co.uk" to "example.co.uk".
|
||||
var multiLabelSuffixes = []string{
|
||||
"co.uk", "org.uk", "gov.uk", "ac.uk", "me.uk",
|
||||
"com.cn", "net.cn", "org.cn", "gov.cn", "edu.cn",
|
||||
"com.au", "net.au", "org.au", "co.jp", "co.kr", "com.br", "com.tr", "com.ua",
|
||||
}
|
||||
|
||||
// Categories whose findings feed the customer guess, and whether a hit in that
|
||||
// category is strong evidence on its own.
|
||||
var customerCategories = map[string]bool{
|
||||
catDomain: false,
|
||||
catResolv: true,
|
||||
catADLDAP: true,
|
||||
catCert: true,
|
||||
catNSUpdate: true,
|
||||
catMgmtSubdomain: true,
|
||||
catDHCP: false,
|
||||
catEmail: false,
|
||||
}
|
||||
|
||||
type customerTally struct {
|
||||
domain string
|
||||
hits int
|
||||
paths map[string]struct{}
|
||||
strong bool
|
||||
evidence []string
|
||||
evidenceSeen map[string]struct{}
|
||||
}
|
||||
|
||||
func guessCustomers(findings []models.PrivacyFinding) []models.CustomerGuess {
|
||||
tallies := map[string]*customerTally{}
|
||||
|
||||
for _, f := range findings {
|
||||
strongCat, ok := customerCategories[f.Category]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
host := f.Match
|
||||
if f.Category == catEmail {
|
||||
if at := strings.IndexByte(host, '@'); at >= 0 {
|
||||
host = host[at+1:]
|
||||
}
|
||||
}
|
||||
reg := registrableDomain(host)
|
||||
if reg == "" || isAllowlistedDomain(reg) {
|
||||
continue
|
||||
}
|
||||
t := tallies[reg]
|
||||
if t == nil {
|
||||
t = &customerTally{domain: reg, paths: map[string]struct{}{}, evidenceSeen: map[string]struct{}{}}
|
||||
tallies[reg] = t
|
||||
}
|
||||
t.hits++
|
||||
if f.Path != "" {
|
||||
t.paths[f.Path] = struct{}{}
|
||||
}
|
||||
if strongCat {
|
||||
t.strong = true
|
||||
}
|
||||
if len(t.evidence) < 3 && f.Excerpt != "" {
|
||||
line := evidenceLine(f)
|
||||
if _, dup := t.evidenceSeen[line]; !dup {
|
||||
t.evidenceSeen[line] = struct{}{}
|
||||
t.evidence = append(t.evidence, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
guesses := make([]models.CustomerGuess, 0, len(tallies))
|
||||
for _, t := range tallies {
|
||||
guesses = append(guesses, models.CustomerGuess{
|
||||
Domain: t.domain,
|
||||
Confidence: confidence(t),
|
||||
Hits: t.hits,
|
||||
Evidence: t.evidence,
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(guesses, func(i, j int) bool {
|
||||
si, sj := score(tallies[guesses[i].Domain]), score(tallies[guesses[j].Domain])
|
||||
if si != sj {
|
||||
return si > sj
|
||||
}
|
||||
return guesses[i].Domain < guesses[j].Domain
|
||||
})
|
||||
|
||||
// Keep the strongest few; drop weak single-hit noise unless it is all we have.
|
||||
out := make([]models.CustomerGuess, 0, 3)
|
||||
for _, g := range guesses {
|
||||
if len(out) >= 3 {
|
||||
break
|
||||
}
|
||||
if g.Hits < 2 && g.Confidence == "low" && len(out) > 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, g)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func score(t *customerTally) int {
|
||||
s := t.hits + 3*len(t.paths)
|
||||
if t.strong {
|
||||
s += 10
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func confidence(t *customerTally) string {
|
||||
switch {
|
||||
case t.strong && len(t.paths) >= 2:
|
||||
return "high"
|
||||
case t.strong || len(t.paths) >= 2:
|
||||
return "medium"
|
||||
default:
|
||||
return "low"
|
||||
}
|
||||
}
|
||||
|
||||
func evidenceLine(f models.PrivacyFinding) string {
|
||||
loc := f.Path
|
||||
if f.Line > 0 {
|
||||
loc += ":" + strconv.Itoa(f.Line)
|
||||
}
|
||||
return loc + ": " + f.Excerpt
|
||||
}
|
||||
|
||||
// registrableDomain returns the registrable ("eTLD+1") portion of a host name,
|
||||
// or "" if host is an IP, has no dot, or is a bare TLD.
|
||||
func registrableDomain(host string) string {
|
||||
host = strings.ToLower(strings.Trim(strings.TrimSpace(host), ".*"))
|
||||
host = strings.TrimPrefix(host, "*.")
|
||||
if host == "" || strings.ContainsAny(host, ":/ ") {
|
||||
return ""
|
||||
}
|
||||
if strings.Count(host, ".") == 3 {
|
||||
allDigits := true
|
||||
for _, r := range host {
|
||||
if r != '.' && (r < '0' || r > '9') {
|
||||
allDigits = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allDigits {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
labels := strings.Split(host, ".")
|
||||
if len(labels) < 2 {
|
||||
return ""
|
||||
}
|
||||
for _, suf := range multiLabelSuffixes {
|
||||
if strings.HasSuffix(host, "."+suf) {
|
||||
sufLabels := strings.Count(suf, ".") + 1
|
||||
if len(labels) > sufLabels {
|
||||
return strings.Join(labels[len(labels)-sufLabels-1:], ".")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return strings.Join(labels[len(labels)-2:], ".")
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package privacy
|
||||
|
||||
import "net"
|
||||
|
||||
// Documentation, benchmarking, and well-known example addresses that carry no
|
||||
// site information even though they are globally routable.
|
||||
var nonSensitiveNets = func() []*net.IPNet {
|
||||
cidrs := []string{
|
||||
"192.0.2.0/24", // RFC 5737 TEST-NET-1
|
||||
"198.51.100.0/24", // RFC 5737 TEST-NET-2
|
||||
"203.0.113.0/24", // RFC 5737 TEST-NET-3
|
||||
"198.18.0.0/15", // RFC 2544 benchmarking
|
||||
"100.64.0.0/10", // RFC 6598 CGNAT
|
||||
"192.88.99.0/24", // RFC 7526 6to4 relay anycast
|
||||
}
|
||||
out := make([]*net.IPNet, 0, len(cidrs))
|
||||
for _, c := range cidrs {
|
||||
if _, n, err := net.ParseCIDR(c); err == nil {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}()
|
||||
|
||||
var nonSensitiveExact = map[string]struct{}{
|
||||
"8.8.8.8": {}, "8.8.4.4": {}, "1.1.1.1": {}, "1.0.0.1": {},
|
||||
"4.2.2.2": {}, "4.2.2.1": {}, "9.9.9.9": {}, "1.2.3.4": {},
|
||||
"208.67.222.222": {}, "208.67.220.220": {},
|
||||
}
|
||||
|
||||
// isSensitiveIP reports whether s is a routable address that could identify the
|
||||
// customer's provider or site. Private (RFC1918/ULA), loopback, link-local,
|
||||
// multicast, and the example/benchmark ranges above are not sensitive.
|
||||
func isSensitiveIP(s string) bool {
|
||||
ip := net.ParseIP(s)
|
||||
if ip == nil {
|
||||
return false
|
||||
}
|
||||
if _, ok := nonSensitiveExact[s]; ok {
|
||||
return false
|
||||
}
|
||||
if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() ||
|
||||
ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
|
||||
ip.IsMulticast() || ip.IsInterfaceLocalMulticast() {
|
||||
return false
|
||||
}
|
||||
if !ip.IsGlobalUnicast() {
|
||||
return false
|
||||
}
|
||||
if v4 := ip.To4(); v4 != nil && (v4[0] == 0 || v4[0] == 255 || v4[0] >= 240) {
|
||||
return false
|
||||
}
|
||||
for _, n := range nonSensitiveNets {
|
||||
if n.Contains(ip) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
// Package privacy scans ingested source files for customer-identifying and
|
||||
// site-identifying data: DNS suffix, AD domain, NTP/DNS/syslog host names,
|
||||
// timezone, e-mail, public IPs, TLS cert CN/SAN, FRU location fields.
|
||||
//
|
||||
// Detection only. Nothing here modifies the source. The rule catalogue is a
|
||||
// port of the KB grep playbook; see bible-local/docs/privacy-scan.md.
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
)
|
||||
|
||||
// File is one unit of scanned content (an extracted archive member, or a
|
||||
// serialized Redfish tree).
|
||||
type File struct {
|
||||
Path string
|
||||
Content []byte
|
||||
}
|
||||
|
||||
const (
|
||||
maxFindings = 1000
|
||||
maxExcerptRunes = 200
|
||||
maxScanLineBytes = 8192
|
||||
)
|
||||
|
||||
// Scan runs every rule over every text file and returns the aggregated report.
|
||||
// Returns nil when there is nothing to scan (no text files).
|
||||
func Scan(files []File) *models.PrivacyScan {
|
||||
scanned := 0
|
||||
seen := make(map[string]struct{})
|
||||
var findings []models.PrivacyFinding
|
||||
|
||||
emit := func(f models.PrivacyFinding) {
|
||||
if len(findings) >= maxFindings {
|
||||
return
|
||||
}
|
||||
key := f.Category + "|" + f.Match + "|" + f.Path
|
||||
if _, ok := seen[key]; ok {
|
||||
return
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
findings = append(findings, f)
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
if isAllowlistedFilename(file.Path) {
|
||||
continue
|
||||
}
|
||||
if !looksLikeText(file.Content) {
|
||||
continue
|
||||
}
|
||||
scanned++
|
||||
scanFileLines(file, emit)
|
||||
}
|
||||
|
||||
if scanned == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
sortFindings(findings)
|
||||
|
||||
report := &models.PrivacyScan{
|
||||
FilesScanned: scanned,
|
||||
Findings: findings,
|
||||
Customers: guessCustomers(findings),
|
||||
Summary: summarize(findings),
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
func scanFileLines(file File, emit func(models.PrivacyFinding)) {
|
||||
certFile := isCertFilename(file.Path)
|
||||
sc := bufio.NewScanner(bytes.NewReader(file.Content))
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
ln := 0
|
||||
for sc.Scan() {
|
||||
ln++
|
||||
line := sc.Text()
|
||||
if len(line) > maxScanLineBytes {
|
||||
line = line[:maxScanLineBytes]
|
||||
}
|
||||
scanLine(file.Path, line, ln, certFile, emit)
|
||||
}
|
||||
}
|
||||
|
||||
func looksLikeText(b []byte) bool {
|
||||
if len(b) == 0 {
|
||||
return false
|
||||
}
|
||||
head := b
|
||||
if len(head) > 8192 {
|
||||
head = head[:8192]
|
||||
}
|
||||
if bytes.IndexByte(head, 0) >= 0 {
|
||||
return false
|
||||
}
|
||||
// Tolerate a truncated multi-byte rune at the sampled boundary.
|
||||
if !utf8.Valid(head) && !utf8.Valid(trimIncompleteRune(head)) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func trimIncompleteRune(b []byte) []byte {
|
||||
for i := 0; i < 3 && i < len(b); i++ {
|
||||
if utf8.RuneStart(b[len(b)-1-i]) {
|
||||
return b[:len(b)-1-i]
|
||||
}
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func excerpt(line string) string {
|
||||
line = strings.Map(func(r rune) rune {
|
||||
if r == 0 || (r < 0x20 && r != '\t') {
|
||||
return ' '
|
||||
}
|
||||
return r
|
||||
}, line)
|
||||
line = strings.Join(strings.Fields(line), " ")
|
||||
line = strings.TrimSpace(line)
|
||||
if utf8.RuneCountInString(line) <= maxExcerptRunes {
|
||||
return line
|
||||
}
|
||||
r := []rune(line)
|
||||
return string(r[:maxExcerptRunes]) + "..."
|
||||
}
|
||||
|
||||
func severityRank(s string) int {
|
||||
switch s {
|
||||
case severityHigh:
|
||||
return 0
|
||||
case severityMedium:
|
||||
return 1
|
||||
default:
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
func sortFindings(f []models.PrivacyFinding) {
|
||||
sort.SliceStable(f, func(i, j int) bool {
|
||||
if f[i].Severity != f[j].Severity {
|
||||
return severityRank(f[i].Severity) < severityRank(f[j].Severity)
|
||||
}
|
||||
if f[i].Category != f[j].Category {
|
||||
return f[i].Category < f[j].Category
|
||||
}
|
||||
if f[i].Path != f[j].Path {
|
||||
return f[i].Path < f[j].Path
|
||||
}
|
||||
return f[i].Match < f[j].Match
|
||||
})
|
||||
}
|
||||
|
||||
func summarize(f []models.PrivacyFinding) models.PrivacySummary {
|
||||
s := models.PrivacySummary{Total: len(f), ByCategory: map[string]int{}}
|
||||
for _, x := range f {
|
||||
switch x.Severity {
|
||||
case severityHigh:
|
||||
s.High++
|
||||
case severityMedium:
|
||||
s.Medium++
|
||||
default:
|
||||
s.Low++
|
||||
}
|
||||
s.ByCategory[x.Category]++
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package privacy
|
||||
|
||||
import "testing"
|
||||
|
||||
// Fixtures use "acme.ru" as a stand-in customer domain. The RFC 2606
|
||||
// "example.*" names are on the allowlist (they are the sanitization target),
|
||||
// so they cannot double as the customer placeholder in detection tests.
|
||||
func TestScan_DetectsAndClassifies(t *testing.T) {
|
||||
files := []File{
|
||||
{Path: "onekeylog/configuration/conf/resolv.conf", Content: []byte(
|
||||
"domain corp.acme.ru\nsearch corp.acme.ru\nnameserver 10.10.0.1\n")},
|
||||
{Path: "onekeylog/configuration/conf/activedir.conf", Content: []byte(
|
||||
"racdomain=corp.acme.ru\nadfilterdc1=10.10.0.5\nrolegroup1name=cab-gr-CI00363277-x86bmc\n")},
|
||||
{Path: "onekeylog/configuration/conf/BMC1/misccfg.ini", Content: []byte(
|
||||
"TimeZone=Europe/Moscow\nSELTimeUTCOffset=180\n")},
|
||||
{Path: "onekeylog/log/audit.log", Content: []byte(
|
||||
"2026-08-24T11:47:34+03:00 login admin from ntp01.acme.ru ok\n" +
|
||||
"contact ops@acme.ru for access\n" +
|
||||
"outbound 45.32.10.7 established\n" +
|
||||
"internal 192.168.31.4 and 10.1.2.3 and doc 203.0.113.9\n")},
|
||||
{Path: "onekeylog/runningdata/rundatainfo.log", Content: []byte(
|
||||
"Mon Aug 24 11:58:48 MSK 2026\n")},
|
||||
// Vendor factory template - must be skipped entirely.
|
||||
{Path: "onekeylog/configuration/conf/ntp_auto.conf", Content: []byte(
|
||||
"server ntp.should-not-match.ru\n")},
|
||||
{Path: "onekeylog/bin/blob", Content: []byte{0x00, 0x01, 0x02, 0xff}},
|
||||
}
|
||||
|
||||
rep := Scan(files)
|
||||
if rep == nil {
|
||||
t.Fatal("nil report")
|
||||
}
|
||||
if rep.FilesScanned != 5 {
|
||||
t.Fatalf("FilesScanned = %d, want 5 (binary + template skipped)", rep.FilesScanned)
|
||||
}
|
||||
|
||||
has := func(cat, match string) bool {
|
||||
for _, f := range rep.Findings {
|
||||
if f.Category == cat && f.Match == match {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
mustHave := []struct{ cat, match string }{
|
||||
{catResolv, "corp.acme.ru"},
|
||||
{catADLDAP, "corp.acme.ru"},
|
||||
{catADLDAP, "cab-gr-CI00363277-x86bmc"},
|
||||
{catTimezone, "Europe/Moscow"},
|
||||
{catTimezone, "180"},
|
||||
{catDomain, "ntp01.acme.ru"},
|
||||
{catEmail, "ops@acme.ru"},
|
||||
{catPublicIP, "45.32.10.7"},
|
||||
{catTimezone, "MSK"},
|
||||
}
|
||||
for _, w := range mustHave {
|
||||
if !has(w.cat, w.match) {
|
||||
t.Errorf("missing finding %s / %q", w.cat, w.match)
|
||||
}
|
||||
}
|
||||
|
||||
mustNotHave := []string{"192.168.31.4", "10.1.2.3", "203.0.113.9", "ntp.should-not-match.ru"}
|
||||
for _, f := range rep.Findings {
|
||||
for _, bad := range mustNotHave {
|
||||
if f.Match == bad {
|
||||
t.Errorf("unexpected finding for %q (%s)", bad, f.Category)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(rep.Customers) == 0 || rep.Customers[0].Domain != "acme.ru" {
|
||||
t.Fatalf("customer guess = %+v, want acme.ru first", rep.Customers)
|
||||
}
|
||||
if rep.Customers[0].Confidence != "high" {
|
||||
t.Errorf("confidence = %s, want high", rep.Customers[0].Confidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScan_NothingToScan(t *testing.T) {
|
||||
if Scan(nil) != nil {
|
||||
t.Fatal("want nil for no files")
|
||||
}
|
||||
if Scan([]File{{Path: "x", Content: []byte{0}}}) != nil {
|
||||
t.Fatal("want nil when only binary files")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScan_Dedupe(t *testing.T) {
|
||||
rep := Scan([]File{{Path: "a.log", Content: []byte(
|
||||
"host is srv-prod.acme.net\nhost is srv-prod.acme.net again\n")}})
|
||||
if rep == nil {
|
||||
t.Fatal("nil report")
|
||||
}
|
||||
n := 0
|
||||
for _, f := range rep.Findings {
|
||||
if f.Category == catDomain && f.Match == "srv-prod.acme.net" {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("domain finding counted %d times, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScan_AllowlistedDomainNotFlagged(t *testing.T) {
|
||||
rep := Scan([]File{{Path: "ntp.conf", Content: []byte(
|
||||
"server 0.pool.ntp.org\nserver time.nist.gov\ncontact admin@example.com\n")}})
|
||||
if rep != nil && len(rep.Findings) > 0 {
|
||||
t.Fatalf("allowlisted infra flagged: %+v", rep.Findings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummary(t *testing.T) {
|
||||
rep := Scan([]File{{Path: "resolv.conf", Content: []byte("domain acme.ru\n")}})
|
||||
if rep.Summary.Total != len(rep.Findings) || rep.Summary.Total == 0 {
|
||||
t.Fatalf("summary total mismatch: %+v", rep.Summary)
|
||||
}
|
||||
if rep.Summary.High == 0 {
|
||||
t.Fatalf("expected a high finding, got %+v", rep.Summary)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"net"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
)
|
||||
|
||||
const (
|
||||
severityHigh = "high"
|
||||
severityMedium = "medium"
|
||||
severityLow = "low"
|
||||
|
||||
catDomain = "domain"
|
||||
catResolv = "resolv"
|
||||
catADLDAP = "ad_ldap"
|
||||
catTimezone = "timezone"
|
||||
catEmail = "email"
|
||||
catPublicIP = "public_ip"
|
||||
catCollector = "collector"
|
||||
catCert = "cert"
|
||||
catFRULocation = "fru_location"
|
||||
catHostname = "hostname"
|
||||
catDHCP = "dhcp"
|
||||
catNSUpdate = "nsupdate"
|
||||
catMgmtSubdomain = "mgmt_subdomain"
|
||||
)
|
||||
|
||||
// tableRule is a simple line-regexp rule. group is the submatch index used as
|
||||
// the reported token (0 = whole match).
|
||||
type tableRule struct {
|
||||
category string
|
||||
severity string
|
||||
re *regexp.Regexp
|
||||
group int
|
||||
hint string
|
||||
}
|
||||
|
||||
var (
|
||||
// A conservative TLD set keeps the bare-FQDN rule from matching things like
|
||||
// "foo.bar" in prose or "index.json" in paths.
|
||||
fqdnTLD = `(?:ru|su|by|kz|ua|com|net|org|local|io|dev|cloud|info|biz|eu|de|uk|fr|nl|cn|jp|kr|us|gov|edu|mil|co|tech|online)`
|
||||
|
||||
reFQDN = regexp.MustCompile(`(?i)\b((?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+` + fqdnTLD + `)\b`)
|
||||
reMgmtSubdomain = regexp.MustCompile(`(?i)\b([a-z0-9-]+(?:\.[a-z0-9-]+)*\.(?:mgmt|oob|ipmi|drac|idrac|ilo|bmc)\.[a-z0-9.-]+)\b`)
|
||||
reIPv4 = regexp.MustCompile(`\b((?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3})\b`)
|
||||
reEmail = regexp.MustCompile(`(?i)\b([a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,})\b`)
|
||||
reSyslogTarget = regexp.MustCompile(`@((?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}:[0-9]{1,5})`)
|
||||
reTZName = regexp.MustCompile(`\b((?:Africa|America|Antarctica|Asia|Atlantic|Australia|Europe|Indian|Pacific)/[A-Za-z_]+(?:/[A-Za-z_]+)?)\b`)
|
||||
reTZAbbr = regexp.MustCompile(`\b(MSK|MSD|EEST|EET|MDST|CEST|CET|WEST|WET)\b\s+20\d\d`)
|
||||
|
||||
tableRules = []tableRule{
|
||||
{catResolv, severityHigh, regexp.MustCompile(`(?i)^\s*(?:domain|search)\s+(\S+)`), 1,
|
||||
"resolv.conf DNS suffix - replace with example.local"},
|
||||
{catADLDAP, severityHigh, regexp.MustCompile(`(?i)\b(racdomain|adfilterdc[0-9]|rolegroup[0-9](?:name|domain))\s*=\s*(\S+)`), 2,
|
||||
"activedir.conf - AD domain / DC address / role-group name"},
|
||||
{catADLDAP, severityHigh, regexp.MustCompile(`(?i)\b(binddn|bindpw)\s*=\s*(\S+)`), 2,
|
||||
"ldap.conf - directory service-account bind"},
|
||||
{catTimezone, severityMedium, regexp.MustCompile(`(?i)\b(TimeZone|SELTimeUTCOffset)\s*=\s*(\S+)`), 2,
|
||||
"timezone reveals region - set Etc/UTC / offset 0"},
|
||||
{catCollector, severityMedium, regexp.MustCompile(`(?i)\b(SyslogHostname)\s*=\s*(\S+)`), 2,
|
||||
"customer syslog collector"},
|
||||
{catCollector, severityMedium, regexp.MustCompile(`(?i)"Destination"\s*:\s*"([^"]+)"`), 1,
|
||||
"SNMP trap / event destination"},
|
||||
{catFRULocation, severityMedium, regexp.MustCompile(`(?i)\b(Asset Tag|Product Location|Chassis Location|Board Extra)\b\s*[:=]\s*(.+)`), 2,
|
||||
"FRU site / inventory field"},
|
||||
{catNSUpdate, severityHigh, regexp.MustCompile(`(?i)\bupdate\s+(?:add|delete)\s+(\S+\.\S+)\s+.*\b(?:A|AAAA|PTR|CNAME)\b`), 1,
|
||||
"DDNS update - host FQDN + BMC record"},
|
||||
{catDHCP, severityMedium, regexp.MustCompile(`(?i)option\s+domain-name\s+"?([^";]+)`), 1,
|
||||
"domain-name handed out by customer DHCP"},
|
||||
{catHostname, severityLow, regexp.MustCompile(`(?i)\b((?:sn|bmc|ilo|idrac|drac|srv)-[a-z0-9][a-z0-9-]{2,})\b`), 1,
|
||||
"hostname follows a customer naming scheme"},
|
||||
}
|
||||
|
||||
certLineRules = []tableRule{
|
||||
{catCert, severityHigh, regexp.MustCompile(`(?i)^\s*(?:Subject|Issuer):\s*(.+)`), 1, "TLS cert subject/issuer"},
|
||||
{catCert, severityHigh, regexp.MustCompile(`(?i)\bCN\s*=\s*([^,/]+)`), 1, "TLS cert common name"},
|
||||
{catCert, severityHigh, regexp.MustCompile(`(?i)\bDNS:\s*([a-z0-9.\-*]+)`), 1, "TLS cert SAN"},
|
||||
}
|
||||
)
|
||||
|
||||
func scanLine(path, line string, ln int, certFile bool, emit func(models.PrivacyFinding)) {
|
||||
var local []models.PrivacyFinding
|
||||
specific := map[string]struct{}{} // matches from categories more precise than a bare FQDN
|
||||
|
||||
add := func(cat, sev, match, hint string) {
|
||||
match = strings.Trim(strings.TrimSpace(match), `"',;`)
|
||||
if match == "" || isAllowlistedValue(match) {
|
||||
return
|
||||
}
|
||||
if ip := net.ParseIP(match); ip != nil && !isSensitiveIP(match) {
|
||||
return
|
||||
}
|
||||
if cat != catDomain && cat != catEmail && cat != catPublicIP {
|
||||
specific[strings.ToLower(match)] = struct{}{}
|
||||
}
|
||||
local = append(local, models.PrivacyFinding{
|
||||
Category: cat, Severity: sev, Path: path, Line: ln,
|
||||
Match: match, Excerpt: excerpt(line), Hint: hint,
|
||||
})
|
||||
}
|
||||
|
||||
for _, r := range tableRules {
|
||||
for _, m := range r.re.FindAllStringSubmatch(line, -1) {
|
||||
if r.group < len(m) {
|
||||
add(r.category, r.severity, m[r.group], r.hint)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if certFile {
|
||||
for _, r := range certLineRules {
|
||||
for _, m := range r.re.FindAllStringSubmatch(line, -1) {
|
||||
if r.group < len(m) {
|
||||
add(r.category, r.severity, m[r.group], r.hint)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, m := range reMgmtSubdomain.FindAllStringSubmatch(line, -1) {
|
||||
add(catMgmtSubdomain, severityHigh, m[1], "management-network subdomain")
|
||||
}
|
||||
for _, m := range reFQDN.FindAllStringSubmatch(line, -1) {
|
||||
if isAllowlistedDomain(m[1]) {
|
||||
continue
|
||||
}
|
||||
add(catDomain, severityHigh, m[1], "domain / FQDN reveals the customer")
|
||||
}
|
||||
for _, m := range reEmail.FindAllStringSubmatch(line, -1) {
|
||||
host := m[1][strings.IndexByte(m[1], '@')+1:]
|
||||
if isAllowlistedDomain(host) || !looksLikeMailHost(host) {
|
||||
continue
|
||||
}
|
||||
add(catEmail, severityMedium, m[1], "e-mail address")
|
||||
}
|
||||
for _, m := range reSyslogTarget.FindAllStringSubmatch(line, -1) {
|
||||
add(catCollector, severityMedium, m[1], "remote syslog target")
|
||||
}
|
||||
for _, m := range reIPv4.FindAllStringSubmatch(line, -1) {
|
||||
if !isSensitiveIP(m[1]) {
|
||||
continue
|
||||
}
|
||||
add(catPublicIP, severityMedium, m[1], "public IP reveals provider / site")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(line), "timezone") || strings.Contains(line, "/") {
|
||||
for _, m := range reTZName.FindAllStringSubmatch(line, -1) {
|
||||
if isAllowlistedValue(m[1]) {
|
||||
continue
|
||||
}
|
||||
add(catTimezone, severityMedium, m[1], "timezone reveals region - set Etc/UTC")
|
||||
}
|
||||
}
|
||||
for _, m := range reTZAbbr.FindAllStringSubmatch(line, -1) {
|
||||
add(catTimezone, severityLow, m[1], "localized timestamp reveals region")
|
||||
}
|
||||
|
||||
for _, f := range local {
|
||||
if (f.Category == catDomain || f.Category == catEmail) && matchCoveredBySpecific(f.Match, specific) {
|
||||
continue
|
||||
}
|
||||
emit(f)
|
||||
}
|
||||
}
|
||||
|
||||
// looksLikeMailHost rejects the many "local@identifier.token" strings that are
|
||||
// not e-mail: OData/Redfish JSON annotations (Members@odata.count), SSH
|
||||
// cipher/kex names (aes256-gcm@openssh.com is handled by the domain allowlist,
|
||||
// but the shape is the same).
|
||||
func looksLikeMailHost(host string) bool {
|
||||
h := strings.ToLower(host)
|
||||
if strings.Contains(h, "odata") || strings.Contains(h, "redfish") || strings.Contains(h, "message.") {
|
||||
return false
|
||||
}
|
||||
dot := strings.LastIndexByte(h, '.')
|
||||
if dot < 0 {
|
||||
return false
|
||||
}
|
||||
tld := h[dot+1:]
|
||||
if len(tld) < 2 || len(tld) > 24 {
|
||||
return false
|
||||
}
|
||||
for _, r := range tld {
|
||||
if r < 'a' || r > 'z' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// matchCoveredBySpecific reports whether a bare FQDN/e-mail finding is already
|
||||
// represented by a more precise finding on the same line (e.g. the resolv.conf
|
||||
// "domain corp.acme.ru" line yields both a resolv and a domain hit).
|
||||
func matchCoveredBySpecific(match string, specific map[string]struct{}) bool {
|
||||
m := strings.ToLower(match)
|
||||
if _, ok := specific[m]; ok {
|
||||
return true
|
||||
}
|
||||
if at := strings.IndexByte(m, '@'); at >= 0 {
|
||||
if _, ok := specific[m[at+1:]]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
)
|
||||
|
||||
func TestIsSensitiveIP(t *testing.T) {
|
||||
cases := []struct {
|
||||
ip string
|
||||
want bool
|
||||
}{
|
||||
{"45.32.10.7", true},
|
||||
{"93.184.216.34", true},
|
||||
{"2606:2800:220:1:248:1893:25c8:1946", true},
|
||||
{"10.1.2.3", false},
|
||||
{"172.16.5.5", false},
|
||||
{"192.168.31.4", false},
|
||||
{"127.0.0.1", false},
|
||||
{"169.254.1.1", false},
|
||||
{"0.0.0.0", false},
|
||||
{"255.255.255.255", false},
|
||||
{"224.0.0.1", false},
|
||||
{"203.0.113.9", false},
|
||||
{"198.51.100.1", false},
|
||||
{"192.0.2.7", false},
|
||||
{"8.8.8.8", false},
|
||||
{"1.1.1.1", false},
|
||||
{"100.64.0.1", false},
|
||||
{"not-an-ip", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := isSensitiveIP(c.ip); got != c.want {
|
||||
t.Errorf("isSensitiveIP(%q) = %v, want %v", c.ip, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsAllowlistedDomain(t *testing.T) {
|
||||
yes := []string{"example.com", "host.example.local", "pool.ntp.org", "0.pool.ntp.org", "redhat.com", "a.b.jd.com"}
|
||||
no := []string{"corp.acme.ru", "tcs.example-bank.com", "sigma.internal.io"}
|
||||
for _, d := range yes {
|
||||
if !isAllowlistedDomain(d) {
|
||||
t.Errorf("%q should be allowlisted", d)
|
||||
}
|
||||
}
|
||||
for _, d := range no {
|
||||
if isAllowlistedDomain(d) {
|
||||
t.Errorf("%q should not be allowlisted", d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistrableDomain(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"sn-x.mgmt.corp.example.local": "example.local",
|
||||
"ntp01.acme.ru": "acme.ru",
|
||||
"a.b.c.example.co.uk": "example.co.uk",
|
||||
"10.20.30.40": "",
|
||||
"localhost": "",
|
||||
"com": "",
|
||||
"*.wildcard.acme.ru": "acme.ru",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := registrableDomain(in); got != want {
|
||||
t.Errorf("registrableDomain(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuessCustomers_RanksStrongEvidence(t *testing.T) {
|
||||
findings := []models.PrivacyFinding{
|
||||
{Category: catResolv, Path: "resolv.conf", Line: 1, Match: "corp.acme.ru", Excerpt: "domain corp.acme.ru"},
|
||||
{Category: catADLDAP, Path: "activedir.conf", Line: 3, Match: "corp.acme.ru", Excerpt: "racdomain=corp.acme.ru"},
|
||||
{Category: catDomain, Path: "audit.log", Line: 9, Match: "noise.other.com", Excerpt: "x noise.other.com"},
|
||||
}
|
||||
got := guessCustomers(findings)
|
||||
if len(got) == 0 || got[0].Domain != "acme.ru" {
|
||||
t.Fatalf("got %+v, want acme.ru first", got)
|
||||
}
|
||||
if got[0].Confidence != "high" {
|
||||
t.Errorf("confidence = %s, want high", got[0].Confidence)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user