feat(sanitize): in-place, length-preserving log de-identification

Adds internal/sanitize: rewrites the customer-identifying spans that
internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with
same-length neutral fillers, in place, without changing the file format.

- Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP ->
  "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid
  neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed.
- Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes,
  and all embedded timestamps preserved; untouched zip entries copied raw;
  member payload length unchanged so tar headers stay byte-identical; only the
  .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output.
- privacy.FindSpans is the one matcher shared by detection and redaction;
  fillers are recognised by isRedactionFiller so a re-scan / second pass is a
  no-op. New privacy FPs fixed along the way: syslog selectors (local7.info),
  "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer
  domains.
- Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and
  unreadable nested archives are reported in Result.SkippedBinary, never edited.
- Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить
  и скачать копию" button in the Customer-data panel, and
  logpile -sanitize <file> (restores mtime/atime).

Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C
.sds yields the identical hardware inventory; re-scan is clean. ADL-067,
bible-local/docs/log-sanitization.md.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-09-02 18:05:27 +03:00
co-authored by Claude Sonnet 5
parent e74e01ad05
commit a63bb17438
25 changed files with 1742 additions and 66 deletions
+43 -6
View File
@@ -25,14 +25,26 @@ var (
"golang.org", "go.dev", "x.org", "sourceforge.net", "xyz.com", "yandex.net",
"haxx.se", "curl.se", "python.org", "perl.org", "isc.org", "sourceware.org",
"ipxe.org", "gnupg.org", "gnutls.org", "openldap.org", "netfilter.org",
"github.com", "githubusercontent.com", "typoland.com", "schemas.dell.com",
"googleapis.com", "gstatic.com", "jquery.com", "jsdelivr.net", "unpkg.com",
"cloudflare.com", "cloudflare.net", "certificate.fi",
"inspur.com", "inspurcloud.com", "inservice-iq.com", "ieisystem.com", "kaytus.com",
"jd.com", "jd.local", "jdcloud.com", "in-addr.arpa", "ip6.arpa", "arpa",
}
allowlistedValues = map[string]struct{}{
"asia/shanghai": {},
"etc/utc": {},
"utc": {},
"asia/shanghai": {},
"etc/utc": {},
"etc/universal": {},
"utc": {},
// neutral decoy zones the sanitizer writes in place of a real timezone
"pacific/tarawa": {},
"atlantic/azores": {},
"antarctica/troll": {},
"antarctica/vostok": {},
"antarctica/mcmurdo": {},
"pacific/guadalcanal": {},
"pacific/bougainville": {},
"to be filled by o.e.m.": {},
"default string": {},
"unknown": {},
@@ -56,11 +68,15 @@ var (
// not customer data from the source.
allowlistedFilenameParts = []string{
"_tencent", "_jingdong", "_pdd", "_baidu", "_kuaishou", "_tianyiyun",
"_jd.", "syslog_jd", "snmptrapcfg", ".json_bak", "ntp_auto",
"raw_export.json", "parser_fields.json", "collect.log",
"_jd.", "syslog_jd", "snmptrapcfg", ".json_bak", ".conf_bak", "_bak",
"ntp_auto", "raw_export.json", "parser_fields.json", "collect.log",
}
)
// IsAllowlistedFile reports whether a member path is a vendor factory template
// or a LOGPile-derived artifact that should not be scanned or redacted.
func IsAllowlistedFile(name string) bool { return isAllowlistedFilename(name) }
func isAllowlistedDomain(domain string) bool {
d := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(domain), "."))
for _, z := range allowlistedZones {
@@ -76,7 +92,28 @@ func isAllowlistedValue(v string) bool {
if _, ok := allowlistedValues[lv]; ok {
return true
}
return isAllowlistedDomain(v)
return isRedactionFiller(lv) || isAllowlistedDomain(v)
}
// isRedactionFiller recognises the internal/sanitize output so a re-scan or a
// second sanitize pass of an already-cleaned file finds nothing: an all-'x'
// hostname filler ("xxxxx.xxxx.xx", "x@xxxx.xxxx.xx", "xxxxxx/xxxxxx") or an
// all-zero IP / offset filler ("00.000.000.00", "000", "-000").
func isRedactionFiller(s string) bool {
if len(s) < 2 {
return false
}
hasFill := false
for _, r := range s {
switch {
case r == 'x' || r == '0':
hasFill = true
case r == '.' || r == '-' || r == '@' || r == ':' || r == '/' || r == '+':
default:
return false
}
}
return hasFill
}
func isAllowlistedFilename(p string) bool {