// Package sanitize produces a de-identified copy of an uploaded diagnostic file. // // It rewrites the customer-identifying spans that internal/privacy detects // (domains, FQDNs, e-mails, AD config, public IPs, timezone) with same-length // neutral fillers, in place, without changing the file format: archive // structure, entry names, modes and embedded timestamps are preserved; only the // redacted byte ranges - and, for compressed containers, the compression layer // - differ. Detection only ever reports; this is the matching redactor. See // bible-local/docs/log-sanitization.md. package sanitize import ( "fmt" "sort" "strings" ) // Change is one aggregated group of replacements for the preview UI. type Change struct { Path string `json:"path"` Category string `json:"category"` Count int `json:"count"` SampleBefore string `json:"sample_before"` SampleAfter string `json:"sample_after"` } // Result is the outcome of sanitizing one file. type Result struct { Data []byte `json:"-"` Changes []Change `json:"changes"` SkippedBinary []string `json:"skipped_binary"` TotalReplaced int `json:"total_replaced"` } // Sanitize de-identifies data (named filename so the format can be resolved by // extension, same set as the parser accepts) and returns the rebuilt file plus // a summary of what changed. func Sanitize(filename string, data []byte) (*Result, error) { if len(data) == 0 { return nil, fmt.Errorf("empty input") } if len(data) > maxInputBytes { return nil, fmt.Errorf("file too large to sanitize in memory: %d bytes (limit %d)", len(data), maxInputBytes) } out, changes, skipped, err := rewriteBytes(filename, data) if err != nil { return nil, err } res := &Result{ Data: out, TotalReplaced: len(changes), Changes: aggregate(changes), SkippedBinary: dedupeStrings(skipped), } return res, nil } func aggregate(raw []change) []Change { type key struct{ path, cat string } m := map[key]*Change{} order := []key{} for _, c := range raw { k := key{c.path, c.category} g := m[k] if g == nil { g = &Change{Path: c.path, Category: c.category, SampleBefore: c.before, SampleAfter: c.after} m[k] = g order = append(order, k) } g.Count++ } out := make([]Change, 0, len(order)) for _, k := range order { out = append(out, *m[k]) } sort.SliceStable(out, func(i, j int) bool { if out[i].Category != out[j].Category { return out[i].Category < out[j].Category } return out[i].Path < out[j].Path }) return out } func dedupeStrings(in []string) []string { if len(in) == 0 { return nil } seen := map[string]struct{}{} out := make([]string, 0, len(in)) for _, s := range in { s = strings.TrimSpace(s) if s == "" { continue } if _, ok := seen[s]; ok { continue } seen[s] = struct{}{} out = append(out, s) } sort.Strings(out) return out } // CanSanitize reports whether a file with this name is a format the sanitizer // can rebuild. func CanSanitize(filename string) bool { switch strings.ToLower(ext(filename)) { case ".tar", ".sds", ".gz", ".tgz", ".zip", ".txt", ".log": return true } return false } func ext(name string) string { if i := strings.LastIndexByte(name, '.'); i >= 0 { return name[i:] } return "" }