feat(sanitize): in-place, length-preserving log de-identification
Adds internal/sanitize: rewrites the customer-identifying spans that internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with same-length neutral fillers, in place, without changing the file format. - Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP -> "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed. - Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes, and all embedded timestamps preserved; untouched zip entries copied raw; member payload length unchanged so tar headers stay byte-identical; only the .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output. - privacy.FindSpans is the one matcher shared by detection and redaction; fillers are recognised by isRedactionFiller so a re-scan / second pass is a no-op. New privacy FPs fixed along the way: syslog selectors (local7.info), "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer domains. - Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and unreadable nested archives are reported in Result.SkippedBinary, never edited. - Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить и скачать копию" button in the Customer-data panel, and logpile -sanitize <file> (restores mtime/atime). Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C .sds yields the identical hardware inventory; re-scan is clean. ADL-067, bible-local/docs/log-sanitization.md. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e74e01ad05
commit
a63bb17438
@@ -0,0 +1,123 @@
|
||||
// Package sanitize produces a de-identified copy of an uploaded diagnostic file.
|
||||
//
|
||||
// It rewrites the customer-identifying spans that internal/privacy detects
|
||||
// (domains, FQDNs, e-mails, AD config, public IPs, timezone) with same-length
|
||||
// neutral fillers, in place, without changing the file format: archive
|
||||
// structure, entry names, modes and embedded timestamps are preserved; only the
|
||||
// redacted byte ranges - and, for compressed containers, the compression layer
|
||||
// - differ. Detection only ever reports; this is the matching redactor. See
|
||||
// bible-local/docs/log-sanitization.md.
|
||||
package sanitize
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Change is one aggregated group of replacements for the preview UI.
|
||||
type Change struct {
|
||||
Path string `json:"path"`
|
||||
Category string `json:"category"`
|
||||
Count int `json:"count"`
|
||||
SampleBefore string `json:"sample_before"`
|
||||
SampleAfter string `json:"sample_after"`
|
||||
}
|
||||
|
||||
// Result is the outcome of sanitizing one file.
|
||||
type Result struct {
|
||||
Data []byte `json:"-"`
|
||||
Changes []Change `json:"changes"`
|
||||
SkippedBinary []string `json:"skipped_binary"`
|
||||
TotalReplaced int `json:"total_replaced"`
|
||||
}
|
||||
|
||||
// Sanitize de-identifies data (named filename so the format can be resolved by
|
||||
// extension, same set as the parser accepts) and returns the rebuilt file plus
|
||||
// a summary of what changed.
|
||||
func Sanitize(filename string, data []byte) (*Result, error) {
|
||||
if len(data) == 0 {
|
||||
return nil, fmt.Errorf("empty input")
|
||||
}
|
||||
if len(data) > maxInputBytes {
|
||||
return nil, fmt.Errorf("file too large to sanitize in memory: %d bytes (limit %d)", len(data), maxInputBytes)
|
||||
}
|
||||
|
||||
out, changes, skipped, err := rewriteBytes(filename, data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := &Result{
|
||||
Data: out,
|
||||
TotalReplaced: len(changes),
|
||||
Changes: aggregate(changes),
|
||||
SkippedBinary: dedupeStrings(skipped),
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func aggregate(raw []change) []Change {
|
||||
type key struct{ path, cat string }
|
||||
m := map[key]*Change{}
|
||||
order := []key{}
|
||||
for _, c := range raw {
|
||||
k := key{c.path, c.category}
|
||||
g := m[k]
|
||||
if g == nil {
|
||||
g = &Change{Path: c.path, Category: c.category, SampleBefore: c.before, SampleAfter: c.after}
|
||||
m[k] = g
|
||||
order = append(order, k)
|
||||
}
|
||||
g.Count++
|
||||
}
|
||||
out := make([]Change, 0, len(order))
|
||||
for _, k := range order {
|
||||
out = append(out, *m[k])
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
if out[i].Category != out[j].Category {
|
||||
return out[i].Category < out[j].Category
|
||||
}
|
||||
return out[i].Path < out[j].Path
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func dedupeStrings(in []string) []string {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]string, 0, len(in))
|
||||
for _, s := range in {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[s]; ok {
|
||||
continue
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// CanSanitize reports whether a file with this name is a format the sanitizer
|
||||
// can rebuild.
|
||||
func CanSanitize(filename string) bool {
|
||||
switch strings.ToLower(ext(filename)) {
|
||||
case ".tar", ".sds", ".gz", ".tgz", ".zip", ".txt", ".log":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ext(name string) string {
|
||||
if i := strings.LastIndexByte(name, '.'); i >= 0 {
|
||||
return name[i:]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user