feat(sanitize): in-place, length-preserving log de-identification

Adds internal/sanitize: rewrites the customer-identifying spans that
internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with
same-length neutral fillers, in place, without changing the file format.

- Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP ->
  "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid
  neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed.
- Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes,
  and all embedded timestamps preserved; untouched zip entries copied raw;
  member payload length unchanged so tar headers stay byte-identical; only the
  .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output.
- privacy.FindSpans is the one matcher shared by detection and redaction;
  fillers are recognised by isRedactionFiller so a re-scan / second pass is a
  no-op. New privacy FPs fixed along the way: syslog selectors (local7.info),
  "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer
  domains.
- Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and
  unreadable nested archives are reported in Result.SkippedBinary, never edited.
- Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить
  и скачать копию" button in the Customer-data panel, and
  logpile -sanitize <file> (restores mtime/atime).

Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C
.sds yields the identical hardware inventory; re-scan is clean. ADL-067,
bible-local/docs/log-sanitization.md.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-09-02 18:05:27 +03:00
co-authored by Claude Sonnet 5
parent e74e01ad05
commit a63bb17438
25 changed files with 1742 additions and 66 deletions
+10
View File
@@ -28,6 +28,8 @@ func main() {
file := flag.String("file", "", "Pre-load archive file")
showVersion := flag.Bool("version", false, "Show version")
noBrowser := flag.Bool("no-browser", false, "Don't open browser automatically")
sanitizeIn := flag.String("sanitize", "", "De-identify customer data in this archive/log and exit (no server)")
sanitizeOut := flag.String("sanitize-out", "", "Write the sanitized file here (default: overwrite -sanitize input)")
flag.Parse()
if *showVersion {
@@ -35,6 +37,14 @@ func main() {
os.Exit(0)
}
if *sanitizeIn != "" {
if err := runSanitize(*sanitizeIn, *sanitizeOut); err != nil {
fmt.Fprintln(os.Stderr, "sanitize:", err)
os.Exit(1)
}
os.Exit(0)
}
// Set embedded web files
server.WebFS = web.FS
+56
View File
@@ -0,0 +1,56 @@
package main
import (
"fmt"
"os"
"git.mchus.pro/mchus/logpile/internal/sanitize"
)
// runSanitize de-identifies inPath and writes the result to outPath (or back to
// inPath). The output file's mtime/atime are restored to the original; ctime
// (inode change time) cannot be restored portably.
func runSanitize(inPath, outPath string) error {
info, err := os.Stat(inPath)
if err != nil {
return err
}
data, err := os.ReadFile(inPath)
if err != nil {
return err
}
if !sanitize.CanSanitize(inPath) {
return fmt.Errorf("unsupported file format for sanitize: %s", inPath)
}
res, err := sanitize.Sanitize(inPath, data)
if err != nil {
return err
}
target := outPath
if target == "" {
target = inPath
}
if err := os.WriteFile(target, res.Data, info.Mode().Perm()); err != nil {
return err
}
mt := info.ModTime()
if err := os.Chtimes(target, mt, mt); err != nil {
fmt.Fprintln(os.Stderr, "warning: could not restore file timestamp:", err)
}
fmt.Printf("sanitized %s -> %s\n", inPath, target)
fmt.Printf(" %d replacement(s), %d byte(s) in, %d byte(s) out\n", res.TotalReplaced, len(data), len(res.Data))
for _, c := range res.Changes {
fmt.Printf(" [%-14s] x%-4d %s\n", c.Category, c.Count, c.Path)
}
if len(res.SkippedBinary) > 0 {
fmt.Printf("\n %d member(s) hold customer data but could not be edited - sanitize by hand:\n", len(res.SkippedBinary))
for _, s := range res.SkippedBinary {
fmt.Printf(" - %s\n", s)
}
}
fmt.Println("\n note: mtime/atime restored; ctime (inode change time) reflects the edit.")
return nil
}