feat(privacy): detect an already-sanitized source

Scan now reports PrivacyScan.Sanitized {detected, markers, strong, files,
evidence}. SanitizationMarkers recognises a value slot filled with one
repeated placeholder + separators (xxxxx.xxxx.xx, x@xxxx.xxxx.xx,
000.00.00.0, a decoy timezone) - it matches the shape, not the literal "x",
so evolving the redaction mechanism still trips it.

detected requires corroboration: strong>=2, or strong>=1 && markers>=3, or
markers>=4. A single filler-looking token is reported (markers:1) but never
asserted as sanitized, so a partial future pass or a coincidence does not
read as "done". 0.0.0.0 / 000 / UTC / Etc/UTC are too plausibly intentional
and do not count.

UI: the Customer-data panel shows "файл уже обезличен" and hides the
sanitize button when detected.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jDYM1nnoZZ3vFz23DDaV1

internal/models/models.go

@@ -34,6 +34,18 @@ type PrivacyScan struct {
 	Customers    []CustomerGuess  `json:"customers,omitempty"`
 	Findings     []PrivacyFinding `json:"findings,omitempty"`
 	Summary      PrivacySummary   `json:"summary"`
+	Sanitized    *SanitizedReport `json:"sanitized,omitempty"` // filler markers left by internal/sanitize
+}
+
+// SanitizedReport says whether the source already looks de-identified. Detected
+// requires corroboration - a single filler-looking token is not enough, so
+// evolving the redaction mechanism does not trip false positives.
+type SanitizedReport struct {
+	Detected bool     `json:"detected"`
+	Markers  int      `json:"markers"`
+	Strong   int      `json:"strong"`
+	Files    int      `json:"files"`
+	Evidence []string `json:"evidence,omitempty"`
 }
 
 // CustomerGuess is a registrable domain that most likely identifies the customer,

internal/privacy/privacy.go

@@ -48,6 +48,7 @@ func Scan(files []File) *models.PrivacyScan {
 		findings = append(findings, f)
 	}
 
+	san := newSanTally()
 	for _, file := range files {
 		if isAllowlistedFilename(file.Path) {
 			continue
@@ -56,7 +57,7 @@ func Scan(files []File) *models.PrivacyScan {
 			continue
 		}
 		scanned++
-		scanFileLines(file, emit)
+		scanFileLines(file, emit, san)
 	}
 
 	if scanned == 0 {
@@ -70,11 +71,12 @@ func Scan(files []File) *models.PrivacyScan {
 		Findings:     findings,
 		Customers:    guessCustomers(findings),
 		Summary:      summarize(findings),
+		Sanitized:    san.report(),
 	}
 	return report
 }
 
-func scanFileLines(file File, emit func(models.PrivacyFinding)) {
+func scanFileLines(file File, emit func(models.PrivacyFinding), san *sanTally) {
 	certFile := isCertFilename(file.Path)
 	sc := bufio.NewScanner(bytes.NewReader(file.Content))
 	sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
@@ -86,6 +88,9 @@ func scanFileLines(file File, emit func(models.PrivacyFinding)) {
 			line = line[:maxScanLineBytes]
 		}
 		scanLine(file.Path, line, ln, certFile, emit)
+		for _, m := range SanitizationMarkers(line) {
+			san.add(file.Path, ln, line, m)
+		}
 	}
 }
 

internal/privacy/sanitized.go

@@ -0,0 +1,187 @@
+package privacy
+
+import (
+	"regexp"
+	"strconv"
+	"strings"
+
+	"git.mchus.pro/mchus/logpile/internal/models"
+)
+
+// A dotted quad, permissive about leading zeros so it also matches the
+// zero-padded fillers ("000.00.00.0") that reIPv4 deliberately rejects.
+var reLooseQuad = regexp.MustCompile(`\b\d{1,3}(?:\.\d{1,3}){3}\b`)
+
+// SanitizationMarker is one redaction-filler token found sitting where a real
+// value (hostname, IP, e-mail, timezone) would be. It is the fingerprint left
+// by internal/sanitize.
+type SanitizationMarker struct {
+	Value  string
+	Strong bool
+}
+
+// Timezone names internal/sanitize substitutes for a real zone. Only the
+// implausible ones count - a BMC genuinely in Antarctica, or an operator who
+// deliberately wrote "UTC", must not read as "sanitized".
+var decoyZones = map[string]struct{}{
+	"antarctica/troll": {}, "antarctica/vostok": {}, "antarctica/mcmurdo": {},
+	"pacific/tarawa": {}, "pacific/guadalcanal": {}, "pacific/bougainville": {},
+	"atlantic/azores": {}, "etc/universal": {}, "us/pacific": {},
+	"brazil/east": {}, "canada/yukon": {},
+}
+
+// SanitizationMarkers returns the redaction-filler tokens on one line. Robust to
+// future filler characters: it recognises "a hostname / IP / e-mail / timezone
+// slot filled entirely with a single repeated placeholder plus separators",
+// not the literal string "x".
+func SanitizationMarkers(line string) []SanitizationMarker {
+	var out []SanitizationMarker
+	seen := map[string]struct{}{}
+	push := func(v string, strong bool) {
+		v = strings.Trim(strings.TrimSpace(v), `"',;:`)
+		if v == "" {
+			return
+		}
+		if _, ok := seen[v]; ok {
+			return
+		}
+		seen[v] = struct{}{}
+		out = append(out, SanitizationMarker{Value: v, Strong: strong})
+	}
+
+	// A filler in a known config-key value position.
+	for _, r := range tableRules {
+		switch r.category {
+		case catResolv, catADLDAP, catNSUpdate, catTimezone, catCollector, catDHCP:
+		default:
+			continue
+		}
+		for _, m := range r.re.FindAllStringSubmatch(line, -1) {
+			if r.group >= len(m) {
+				continue
+			}
+			val := m[r.group]
+			if isGenericFiller(val) || isDecoyZone(val) {
+				strong := r.category == catResolv || r.category == catADLDAP || r.category == catNSUpdate
+				push(val, strong && isGenericFiller(val))
+			}
+		}
+	}
+	// A bare FQDN / e-mail made entirely of filler.
+	for _, v := range reFQDN.FindAllString(line, -1) {
+		if isGenericFiller(v) && strings.Contains(v, ".") {
+			push(v, strings.Count(v, ".") >= 2 && len(v) >= 8)
+		}
+	}
+	for _, v := range reEmail.FindAllString(line, -1) {
+		if isGenericFiller(v) {
+			push(v, true)
+		}
+	}
+	// A zero-filled IP ("000.00.00.0"); "0.0.0.0" is a real value, not a marker.
+	for _, v := range reLooseQuad.FindAllString(line, -1) {
+		if isGenericFiller(v) {
+			push(v, true)
+		}
+	}
+	// A decoy timezone name anywhere on the line.
+	for _, v := range reTZName.FindAllString(line, -1) {
+		if isDecoyZone(v) {
+			push(v, false)
+		}
+	}
+	return out
+}
+
+func isDecoyZone(s string) bool {
+	_, ok := decoyZones[strings.ToLower(strings.TrimSpace(s))]
+	return ok
+}
+
+// isGenericFiller reports whether s is a value slot filled with one repeated
+// placeholder character. Letter fills ("xxxxx.xxxx.xx") always qualify; digit
+// fills qualify only in the IP shape with a wide group ("00.000.000.00"), so
+// "0.0.0.0" and "000" do not.
+func isGenericFiller(s string) bool {
+	parts := strings.FieldsFunc(s, func(r rune) bool {
+		return r == '.' || r == '-' || r == '_' || r == '@' || r == ':' || r == '/' || r == '+'
+	})
+	if len(parts) < 2 {
+		return false
+	}
+	var fill rune
+	total, wide := 0, false
+	for _, p := range parts {
+		for _, r := range p {
+			if !isAlnum(r) {
+				return false
+			}
+			if fill == 0 {
+				fill = r
+			} else if r != fill {
+				return false
+			}
+		}
+		total += len(p)
+		if len(p) >= 2 {
+			wide = true
+		}
+	}
+	if total < 4 {
+		return false
+	}
+	if fill >= '0' && fill <= '9' {
+		return len(parts) == 4 && wide // IP shape with a redacted (0-padded) octet
+	}
+	return true
+}
+
+// sanTally accumulates markers while Scan walks the files.
+type sanTally struct {
+	strong, total int
+	files         map[string]struct{}
+	seen          map[string]struct{}
+	evidence      []string
+}
+
+func newSanTally() *sanTally {
+	return &sanTally{files: map[string]struct{}{}, seen: map[string]struct{}{}}
+}
+
+func (t *sanTally) add(path string, ln int, line string, m SanitizationMarker) {
+	key := m.Value + "|" + path
+	if _, ok := t.seen[key]; ok {
+		return
+	}
+	t.seen[key] = struct{}{}
+	t.total++
+	if m.Strong {
+		t.strong++
+	}
+	if path != "" {
+		t.files[path] = struct{}{}
+	}
+	if len(t.evidence) < 3 {
+		loc := path
+		if ln > 0 {
+			loc += ":" + strconv.Itoa(ln)
+		}
+		t.evidence = append(t.evidence, loc+": "+excerpt(line))
+	}
+}
+
+func (t *sanTally) report() *models.SanitizedReport {
+	if t.total == 0 {
+		return nil
+	}
+	// One stray filler-looking token proves nothing (coincidence, or a partial
+	// future redaction pass). Require corroboration.
+	detected := t.strong >= 2 || (t.strong >= 1 && t.total >= 3) || t.total >= 4
+	return &models.SanitizedReport{
+		Detected: detected,
+		Markers:  t.total,
+		Strong:   t.strong,
+		Files:    len(t.files),
+		Evidence: t.evidence,
+	}
+}

internal/privacy/sanitized_test.go

@@ -0,0 +1,69 @@
+package privacy
+
+import "testing"
+
+func TestIsGenericFiller(t *testing.T) {
+	yes := []string{"xxxxx.xxxx.xx", "x@xxxx.xxxx.xx", "xxxxxx/xxxxxx", "000.00.00.0", "00.00.00.00"}
+	no := []string{"0.0.0.0", "000", "corp.acme.ru", "192.0.2.0", "10.20.30.40", "example.local", "x", "xx"}
+	for _, s := range yes {
+		if !isGenericFiller(s) {
+			t.Errorf("isGenericFiller(%q) = false, want true", s)
+		}
+	}
+	for _, s := range no {
+		if isGenericFiller(s) {
+			t.Errorf("isGenericFiller(%q) = true, want false", s)
+		}
+	}
+}
+
+func TestSanitizationMarkers_Line(t *testing.T) {
+	strong := 0
+	for _, m := range SanitizationMarkers("domain xxxxxxx.xxxxx") {
+		if m.Strong {
+			strong++
+		}
+	}
+	if strong == 0 {
+		t.Fatal("redacted resolv domain not detected as a strong marker")
+	}
+	if len(SanitizationMarkers("domain corp.acme.ru")) != 0 {
+		t.Fatal("real domain flagged as a sanitization marker")
+	}
+	if len(SanitizationMarkers("gateway 0.0.0.0 unset")) != 0 {
+		t.Fatal("0.0.0.0 flagged as a marker")
+	}
+}
+
+func TestScan_SanitizedDetectionNeedsCorroboration(t *testing.T) {
+	// One lone filler-looking token: reported, but not "detected".
+	rep := Scan([]File{{Path: "a.log", Content: []byte("host is xxxxx.xxxx.xx today\nnothing else\n")}})
+	if rep == nil || rep.Sanitized == nil {
+		t.Fatal("expected a sanitized report")
+	}
+	if rep.Sanitized.Detected {
+		t.Fatalf("a single marker must not read as sanitized: %+v", rep.Sanitized)
+	}
+	if rep.Sanitized.Markers != 1 {
+		t.Fatalf("markers = %d, want 1", rep.Sanitized.Markers)
+	}
+
+	// Two strong markers in two files -> detected.
+	rep2 := Scan([]File{
+		{Path: "resolv.conf", Content: []byte("domain xxxxxxx.xxxxx\n")},
+		{Path: "nsupdate_temp", Content: []byte("update add xxxxx.xxxxxxx.xx 0 A 10.0.0.1\n")},
+	})
+	if rep2 == nil || rep2.Sanitized == nil || !rep2.Sanitized.Detected {
+		t.Fatalf("two-file sanitized dump not detected: %+v", rep2.Sanitized)
+	}
+}
+
+func TestScan_UnsanitizedHasNoSanitizedReport(t *testing.T) {
+	rep := Scan([]File{{Path: "resolv.conf", Content: []byte("domain corp.acme.ru\nnameserver 10.0.0.1\n")}})
+	if rep == nil {
+		t.Fatal("nil report")
+	}
+	if rep.Sanitized != nil {
+		t.Fatalf("clean dump got a sanitized report: %+v", rep.Sanitized)
+	}
+}