feat(sanitize): in-place, length-preserving log de-identification

Adds internal/sanitize: rewrites the customer-identifying spans that
internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with
same-length neutral fillers, in place, without changing the file format.

- Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP ->
  "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid
  neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed.
- Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes,
  and all embedded timestamps preserved; untouched zip entries copied raw;
  member payload length unchanged so tar headers stay byte-identical; only the
  .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output.
- privacy.FindSpans is the one matcher shared by detection and redaction;
  fillers are recognised by isRedactionFiller so a re-scan / second pass is a
  no-op. New privacy FPs fixed along the way: syslog selectors (local7.info),
  "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer
  domains.
- Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and
  unreadable nested archives are reported in Result.SkippedBinary, never edited.
- Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить
  и скачать копию" button in the Customer-data panel, and
  logpile -sanitize <file> (restores mtime/atime).

Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C
.sds yields the identical hardware inventory; re-scan is clean. ADL-067,
bible-local/docs/log-sanitization.md.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

internal/privacy/rules.go

@@ -22,12 +22,30 @@ const (
 	catCollector     = "collector"
 	catCert          = "cert"
 	catFRULocation   = "fru_location"
-	catHostname      = "hostname"
 	catDHCP          = "dhcp"
 	catNSUpdate      = "nsupdate"
 	catMgmtSubdomain = "mgmt_subdomain"
 )
 
+// Span is a matched sensitive substring of one line: line[Start:End]. Detection
+// (Scan) and redaction (internal/sanitize) both consume the same spans so they
+// can never disagree about what counts as customer data.
+type Span struct {
+	Start, End int
+	Category   string
+}
+
+// FindSpans returns every sensitive span in a single line. certFile enables the
+// TLS-certificate rules (only meaningful for *.pem / *.csr members).
+func FindSpans(line string, certFile bool) []Span {
+	ms := findMatches(line, certFile)
+	out := make([]Span, 0, len(ms))
+	for _, m := range ms {
+		out = append(out, Span{Start: m.start, End: m.end, Category: m.category})
+	}
+	return out
+}
+
 // tableRule is a simple line-regexp rule. group is the submatch index used as
 // the reported token (0 = whole match).
 type tableRule struct {
@@ -38,6 +56,14 @@ type tableRule struct {
 	hint     string
 }
 
+type matchSpan struct {
+	start, end int
+	category   string
+	severity   string
+	hint       string
+	match      string
+}
+
 var (
 	// Match any dotted name ending in a 2-24 char alpha label; plausibleFQDN
 	// then decides whether that final label is a real TLD.
@@ -55,7 +81,10 @@ var (
 	reGoRuntime = regexp.MustCompile(`/go-mod/|\bgoroutine \d|\.go:\d+`)
 	// A dotted quad sitting on a line that is talking about a version or a spec
 	// clause, not a host.
-	reVersionContext = regexp.MustCompile(`(?i)(\bversion|\bver\.?\b|_ver\b|fw[_ ]?ver|\brevision|vbios|firmware|\bbios\b|microcode|ucode|\bbuild\b|x\.?org|xserver|ieee|\bstd\b|clause|section|\brfc\b)`)
+	reVersionContext = regexp.MustCompile(`(?i)(version|\bver\.?\b|_ver\b|fw[_ ]?ver|\brevision|vbios|firmware|\bbios\b|microcode|ucode|\bbuild\b|x\.?org|xserver|ieee|\bstd\b|clause|section|\brfc\b)`)
+	// syslog selector "local7.info", "mail.err" - a facility.severity pair, not a
+	// domain.
+	reSyslogSelector = regexp.MustCompile(`(?i)^(?:\*|local[0-7]|auth|authpriv|cron|daemon|ftp|kern|lpr|mail|news|security|syslog|user|uucp)\.(?:\*|emerg|panic|alert|crit|err|error|warn|warning|notice|info|debug|none)$`)
 	// A TLD label written Title-case ("OS.It", "Rodolfo.Cn") is prose or mojibake,
 	// not a real domain (real ones are lower- or all-upper-case).
 	reTitlecaseTail = regexp.MustCompile(`\.[A-Z][a-z]+$`)
@@ -88,13 +117,23 @@ var (
 	}
 )
 
-func scanLine(path, line string, ln int, certFile bool, emit func(models.PrivacyFinding)) {
-	var local []models.PrivacyFinding
-	specific := map[string]struct{}{} // matches from categories more precise than a bare FQDN
+// findMatches is the single matcher shared by Scan and FindSpans. It returns
+// every accepted span on the line, before the Scan-only "already covered by a
+// more specific category" dedupe.
+func findMatches(line string, certFile bool) []matchSpan {
+	var out []matchSpan
+	specific := map[string]struct{}{}
 	noiseLine := reKernelTimestamp.MatchString(line) || reGoRuntime.MatchString(line)
 
-	add := func(cat, sev, match, hint string) {
-		match = strings.Trim(strings.TrimSpace(match), `"',;:`)
+	add := func(cat, sev, hint string, s, e int) {
+		if s < 0 || e > len(line) || s >= e {
+			return
+		}
+		s, e = trimSpan(line, s, e)
+		if s >= e {
+			return
+		}
+		match := line[s:e]
 		if len(match) < 2 || !strings.ContainsFunc(match, isAlnum) || isAllowlistedValue(match) {
 			return
 		}
@@ -107,26 +146,28 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy
 		if cat != catDomain && cat != catEmail && cat != catPublicIP {
 			specific[strings.ToLower(match)] = struct{}{}
 		}
-		local = append(local, models.PrivacyFinding{
-			Category: cat, Severity: sev, Path: path, Line: ln,
-			Match: match, Excerpt: excerpt(line), Hint: hint,
-		})
+		out = append(out, matchSpan{start: s, end: e, category: cat, severity: sev, hint: hint, match: match})
+	}
+
+	groupSpan := func(m []int, group int) (int, int) {
+		if 2*group+1 >= len(m) {
+			return -1, -1
+		}
+		return m[2*group], m[2*group+1]
 	}
 
 	for _, r := range tableRules {
-		for _, m := range r.re.FindAllStringSubmatch(line, -1) {
-			if r.group < len(m) {
-				add(r.category, r.severity, m[r.group], r.hint)
-			}
+		for _, m := range r.re.FindAllStringSubmatchIndex(line, -1) {
+			s, e := groupSpan(m, r.group)
+			add(r.category, r.severity, r.hint, s, e)
 		}
 	}
 
 	if certFile {
 		for _, r := range certLineRules {
-			for _, m := range r.re.FindAllStringSubmatch(line, -1) {
-				if r.group < len(m) {
-					add(r.category, r.severity, m[r.group], r.hint)
-				}
+			for _, m := range r.re.FindAllStringSubmatchIndex(line, -1) {
+				s, e := groupSpan(m, r.group)
+				add(r.category, r.severity, r.hint, s, e)
 			}
 		}
 	}
@@ -138,13 +179,13 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy
 				continue
 			}
 			fqdn := line[s:e]
-			if isAllowlistedDomain(fqdn) || !plausibleFQDN(fqdn) || reTitlecaseTail.MatchString(fqdn) {
+			if isAllowlistedDomain(fqdn) || !plausibleFQDN(fqdn) || reTitlecaseTail.MatchString(fqdn) || reSyslogSelector.MatchString(fqdn) {
 				continue
 			}
 			labels := strings.Split(strings.ToLower(fqdn), ".")
 			cat, sev, hint := catDomain, severityLow, "domain / FQDN in the logs"
 			if len(labels) >= 3 {
-				sev = severityMedium // a real subdomain, not a stray two-word token
+				sev = severityMedium
 			}
 			for _, lbl := range labels {
 				if _, ok := mgmtLabels[lbl]; ok {
@@ -152,54 +193,75 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy
 					break
 				}
 			}
-			add(cat, sev, fqdn, hint)
+			add(cat, sev, hint, s, e)
 		}
-	}
-	if !noiseLine {
-		for _, m := range reEmail.FindAllStringSubmatch(line, -1) {
-			host := m[1][strings.IndexByte(m[1], '@')+1:]
+
+		for _, m := range reEmail.FindAllStringSubmatchIndex(line, -1) {
+			s, e := m[2], m[3]
+			addr := line[s:e]
+			host := addr[strings.IndexByte(addr, '@')+1:]
 			if isAllowlistedDomain(host) || !looksLikeMailHost(host) {
 				continue
 			}
-			add(catEmail, severityMedium, m[1], "e-mail address")
+			add(catEmail, severityMedium, "e-mail address", s, e)
 		}
 	}
-	for _, m := range reSyslogTarget.FindAllStringSubmatch(line, -1) {
-		add(catCollector, severityMedium, m[1], "remote syslog target")
+
+	for _, m := range reSyslogTarget.FindAllStringSubmatchIndex(line, -1) {
+		add(catCollector, severityMedium, "remote syslog target", m[2], m[3])
 	}
+
 	commentLine := strings.HasPrefix(strings.TrimSpace(line), "#") || strings.HasPrefix(strings.TrimSpace(line), ";")
 	if !noiseLine && !commentLine && !reVersionContext.MatchString(line) {
 		for _, loc := range reIPv4.FindAllStringIndex(line, -1) {
 			if inDottedNumberRun(line, loc[0], loc[1]) {
-				continue // part of a longer version string like "18:6.1.4.5"
+				continue
 			}
 			ip := line[loc[0]:loc[1]]
 			if strings.HasSuffix(ip, ".0") || strings.HasPrefix(ip, "0.") || strings.HasPrefix(ip, "1.") {
-				continue // network address, or a 0./1. version-style quad
+				continue
 			}
 			if !isSensitiveIP(ip) {
 				continue
 			}
-			add(catPublicIP, severityMedium, ip, "public IP reveals provider / site")
+			add(catPublicIP, severityMedium, "public IP reveals provider / site", loc[0], loc[1])
 		}
 	}
-	if ll := strings.ToLower(line); strings.Contains(ll, "timezone") || strings.Contains(ll, "zoneinfo") || strings.Contains(ll, "localtime") || strings.Contains(ll, "tz=") {
-		for _, m := range reTZName.FindAllStringSubmatch(line, -1) {
-			if isAllowlistedValue(m[1]) {
-				continue
-			}
-			add(catTimezone, severityMedium, m[1], "timezone reveals region - set Etc/UTC")
-		}
-	}
-	for _, m := range reTZAbbr.FindAllStringSubmatch(line, -1) {
-		add(catTimezone, severityLow, m[1], "localized timestamp reveals region")
-	}
 
-	for _, f := range local {
-		if (f.Category == catDomain || f.Category == catEmail) && matchCoveredBySpecific(f.Match, specific) {
-			continue
+	if ll := strings.ToLower(line); strings.Contains(ll, "timezone") || strings.Contains(ll, "zoneinfo") || strings.Contains(ll, "localtime") || strings.Contains(ll, "tz=") {
+		for _, m := range reTZName.FindAllStringSubmatchIndex(line, -1) {
+			s, e := m[2], m[3]
+			if isAllowlistedValue(line[s:e]) {
+				continue
+			}
+			add(catTimezone, severityMedium, "timezone reveals region - set Etc/UTC", s, e)
 		}
-		emit(f)
+	}
+	for _, m := range reTZAbbr.FindAllStringSubmatchIndex(line, -1) {
+		add(catTimezone, severityLow, "localized timestamp reveals region", m[2], m[3])
+	}
+
+	// Scan-only dedupe: drop a bare domain/email span when the same token is
+	// already covered by a more specific category on this line.
+	if len(specific) > 0 {
+		filtered := out[:0]
+		for _, m := range out {
+			if (m.category == catDomain || m.category == catEmail) && matchCoveredBySpecific(m.match, specific) {
+				continue
+			}
+			filtered = append(filtered, m)
+		}
+		out = filtered
+	}
+	return out
+}
+
+func scanLine(path, line string, ln int, certFile bool, emit func(models.PrivacyFinding)) {
+	for _, m := range findMatches(line, certFile) {
+		emit(models.PrivacyFinding{
+			Category: m.category, Severity: m.severity, Path: path, Line: ln,
+			Match: m.match, Excerpt: excerpt(line), Hint: m.hint,
+		})
 	}
 }
 
@@ -207,6 +269,19 @@ func isAlnum(r rune) bool {
 	return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9')
 }
 
+// trimSpan narrows [s,e) past leading/trailing quoting and separator bytes,
+// matching the old strings.Trim(TrimSpace(...), "\"',;:") behaviour.
+func trimSpan(line string, s, e int) (int, int) {
+	const cut = " \t\"',;:"
+	for s < e && strings.IndexByte(cut, line[s]) >= 0 {
+		s++
+	}
+	for e > s && strings.IndexByte(cut, line[e-1]) >= 0 {
+		e--
+	}
+	return s, e
+}
+
 // cleanTokenBoundary reports whether line[s:e] stands as its own token - not
 // glued to surrounding identifier characters ("auth.backend.gssapi.store-creds",
 // "mountall.sh [start]" is fine, "96.00.CF.00" is not) - so it reads as a
@@ -216,7 +291,6 @@ func cleanTokenBoundary(line string, s, e int) bool {
 		switch c := line[s-1]; c {
 		case ' ', '\t', '"', '\'', '(', '<', '=', ',', '@':
 		case '/', ':':
-			// ok (URL / "server = host:port"), but not a bare path segment
 		default:
 			return false
 		}
@@ -225,7 +299,7 @@ func cleanTokenBoundary(line string, s, e int) bool {
 		switch line[e] {
 		case ' ', '\t', '"', '\'', ')', '>', ',', ';', ':', '/', '\\', '?', '!':
 		default:
-			return false // trailing '.', '-', letter, digit -> mid-identifier
+			return false
 		}
 	}
 	return true
@@ -272,32 +346,27 @@ func usefulFRUValue(v string) bool {
 		}
 	}
 	if digits == len(v) {
-		return false // pure number: date/manufacturing code
+		return false
 	}
 	if upperAlnum == len(v) && digits >= 4 {
-		return false // looks like a serial number (7J..., 21D634070)
+		return false
 	}
 	return true
 }
 
 // looksLikeMailHost rejects the many "local@identifier.token" strings that are
 // not e-mail: OData/Redfish JSON annotations (Members@odata.count), SSH
-// cipher/kex names (aes256-gcm@openssh.com is handled by the domain allowlist,
-// but the shape is the same).
+// cipher/kex names, systemd units ("serial-getty@ttyAMA0.service").
 func looksLikeMailHost(host string) bool {
 	h := strings.ToLower(host)
 	if strings.Contains(h, "odata") || strings.Contains(h, "redfish") || strings.Contains(h, "message.") {
 		return false
 	}
-	// systemd templated units ("serial-getty@ttyAMA0.service"), Go toolchain
-	// pseudo-versions ("toolchain@v0.0.1-go1.25.0.linux") and the like all fail
-	// the real-TLD check.
 	return plausibleFQDN(h)
 }
 
-// matchCoveredBySpecific reports whether a bare FQDN/e-mail finding is already
-// represented by a more precise finding on the same line (e.g. the resolv.conf
-// "domain corp.acme.ru" line yields both a resolv and a domain hit).
+// matchCoveredBySpecific reports whether a bare FQDN/e-mail match is already
+// represented by a more precise finding on the same line.
 func matchCoveredBySpecific(match string, specific map[string]struct{}) bool {
 	m := strings.ToLower(match)
 	if _, ok := specific[m]; ok {