feat(sanitize): in-place, length-preserving log de-identification
Adds internal/sanitize: rewrites the customer-identifying spans that internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with same-length neutral fillers, in place, without changing the file format. - Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP -> "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed. - Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes, and all embedded timestamps preserved; untouched zip entries copied raw; member payload length unchanged so tar headers stay byte-identical; only the .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output. - privacy.FindSpans is the one matcher shared by detection and redaction; fillers are recognised by isRedactionFiller so a re-scan / second pass is a no-op. New privacy FPs fixed along the way: syslog selectors (local7.info), "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer domains. - Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and unreadable nested archives are reported in Result.SkippedBinary, never edited. - Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить и скачать копию" button in the Customer-data panel, and logpile -sanitize <file> (restores mtime/atime). Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C .sds yields the identical hardware inventory; re-scan is clean. ADL-067, bible-local/docs/log-sanitization.md. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e74e01ad05
commit
a63bb17438
@@ -0,0 +1,62 @@
|
||||
package sanitize
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNeutralZonesAreValidAndSameLength(t *testing.T) {
|
||||
// If the runtime has no zoneinfo at all, skip the loadability half.
|
||||
_, tzErr := time.LoadLocation("Europe/Moscow")
|
||||
haveTZDB := tzErr == nil
|
||||
|
||||
for n, zone := range neutralZoneByLen {
|
||||
if len(zone) != n {
|
||||
t.Errorf("neutralZoneByLen[%d] = %q has length %d", n, zone, len(zone))
|
||||
}
|
||||
if haveTZDB {
|
||||
if _, err := time.LoadLocation(zone); err != nil {
|
||||
t.Errorf("neutral zone %q does not load: %v", zone, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTZFiller(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"Europe/Moscow": "Etc/Universal",
|
||||
"Asia/Yekaterinburg": "Antarctica/McMurdo",
|
||||
"180": "000",
|
||||
"-300": "-000",
|
||||
"MSK": "UTC",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := tzFiller(in); got != want {
|
||||
t.Errorf("tzFiller(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
if got := tzFiller(in); len(got) != len(in) {
|
||||
t.Errorf("tzFiller(%q) length %d != %d", in, len(got), len(in))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedact_SkipsAllowlistedTemplateFile(t *testing.T) {
|
||||
// _tianyiyun is a vendor factory template - must not be scanned/redacted.
|
||||
nb, ch, _, err := rewriteBytes("onekeylog/configuration/conf/syslog_tianyiyun.conf_bak", []byte("server ntp.acme.ru\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ch) != 0 || string(nb) != "server ntp.acme.ru\n" {
|
||||
t.Fatalf("template file was modified: %q %+v", nb, ch)
|
||||
}
|
||||
}
|
||||
|
||||
func TestXFill(t *testing.T) {
|
||||
if got := xFill("a1-b2.c3_d4@e5:f6"); got != "xx-xx.xx_xx@xx:xx" {
|
||||
t.Fatalf("xFill = %q", got)
|
||||
}
|
||||
if !strings.HasPrefix(xFill("corp.acme.ru"), "xxxx.") {
|
||||
t.Fatalf("xFill domain: %q", xFill("corp.acme.ru"))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user