feat(sanitize): in-place, length-preserving log de-identification
Adds internal/sanitize: rewrites the customer-identifying spans that internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with same-length neutral fillers, in place, without changing the file format. - Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP -> "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed. - Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes, and all embedded timestamps preserved; untouched zip entries copied raw; member payload length unchanged so tar headers stay byte-identical; only the .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output. - privacy.FindSpans is the one matcher shared by detection and redaction; fillers are recognised by isRedactionFiller so a re-scan / second pass is a no-op. New privacy FPs fixed along the way: syslog selectors (local7.info), "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer domains. - Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and unreadable nested archives are reported in Result.SkippedBinary, never edited. - Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить и скачать копию" button in the Customer-data panel, and logpile -sanitize <file> (restores mtime/atime). Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C .sds yields the identical hardware inventory; re-scan is clean. ADL-067, bible-local/docs/log-sanitization.md. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e74e01ad05
commit
a63bb17438
@@ -25,14 +25,26 @@ var (
|
||||
"golang.org", "go.dev", "x.org", "sourceforge.net", "xyz.com", "yandex.net",
|
||||
"haxx.se", "curl.se", "python.org", "perl.org", "isc.org", "sourceware.org",
|
||||
"ipxe.org", "gnupg.org", "gnutls.org", "openldap.org", "netfilter.org",
|
||||
"github.com", "githubusercontent.com", "typoland.com", "schemas.dell.com",
|
||||
"googleapis.com", "gstatic.com", "jquery.com", "jsdelivr.net", "unpkg.com",
|
||||
"cloudflare.com", "cloudflare.net", "certificate.fi",
|
||||
"inspur.com", "inspurcloud.com", "inservice-iq.com", "ieisystem.com", "kaytus.com",
|
||||
"jd.com", "jd.local", "jdcloud.com", "in-addr.arpa", "ip6.arpa", "arpa",
|
||||
}
|
||||
|
||||
allowlistedValues = map[string]struct{}{
|
||||
"asia/shanghai": {},
|
||||
"etc/utc": {},
|
||||
"utc": {},
|
||||
"asia/shanghai": {},
|
||||
"etc/utc": {},
|
||||
"etc/universal": {},
|
||||
"utc": {},
|
||||
// neutral decoy zones the sanitizer writes in place of a real timezone
|
||||
"pacific/tarawa": {},
|
||||
"atlantic/azores": {},
|
||||
"antarctica/troll": {},
|
||||
"antarctica/vostok": {},
|
||||
"antarctica/mcmurdo": {},
|
||||
"pacific/guadalcanal": {},
|
||||
"pacific/bougainville": {},
|
||||
"to be filled by o.e.m.": {},
|
||||
"default string": {},
|
||||
"unknown": {},
|
||||
@@ -56,11 +68,15 @@ var (
|
||||
// not customer data from the source.
|
||||
allowlistedFilenameParts = []string{
|
||||
"_tencent", "_jingdong", "_pdd", "_baidu", "_kuaishou", "_tianyiyun",
|
||||
"_jd.", "syslog_jd", "snmptrapcfg", ".json_bak", "ntp_auto",
|
||||
"raw_export.json", "parser_fields.json", "collect.log",
|
||||
"_jd.", "syslog_jd", "snmptrapcfg", ".json_bak", ".conf_bak", "_bak",
|
||||
"ntp_auto", "raw_export.json", "parser_fields.json", "collect.log",
|
||||
}
|
||||
)
|
||||
|
||||
// IsAllowlistedFile reports whether a member path is a vendor factory template
|
||||
// or a LOGPile-derived artifact that should not be scanned or redacted.
|
||||
func IsAllowlistedFile(name string) bool { return isAllowlistedFilename(name) }
|
||||
|
||||
func isAllowlistedDomain(domain string) bool {
|
||||
d := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(domain), "."))
|
||||
for _, z := range allowlistedZones {
|
||||
@@ -76,7 +92,28 @@ func isAllowlistedValue(v string) bool {
|
||||
if _, ok := allowlistedValues[lv]; ok {
|
||||
return true
|
||||
}
|
||||
return isAllowlistedDomain(v)
|
||||
return isRedactionFiller(lv) || isAllowlistedDomain(v)
|
||||
}
|
||||
|
||||
// isRedactionFiller recognises the internal/sanitize output so a re-scan or a
|
||||
// second sanitize pass of an already-cleaned file finds nothing: an all-'x'
|
||||
// hostname filler ("xxxxx.xxxx.xx", "x@xxxx.xxxx.xx", "xxxxxx/xxxxxx") or an
|
||||
// all-zero IP / offset filler ("00.000.000.00", "000", "-000").
|
||||
func isRedactionFiller(s string) bool {
|
||||
if len(s) < 2 {
|
||||
return false
|
||||
}
|
||||
hasFill := false
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case r == 'x' || r == '0':
|
||||
hasFill = true
|
||||
case r == '.' || r == '-' || r == '@' || r == ':' || r == '/' || r == '+':
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return hasFill
|
||||
}
|
||||
|
||||
func isAllowlistedFilename(p string) bool {
|
||||
|
||||
Reference in New Issue
Block a user