feat(sanitize): in-place, length-preserving log de-identification
Adds internal/sanitize: rewrites the customer-identifying spans that internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with same-length neutral fillers, in place, without changing the file format. - Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP -> "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed. - Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes, and all embedded timestamps preserved; untouched zip entries copied raw; member payload length unchanged so tar headers stay byte-identical; only the .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output. - privacy.FindSpans is the one matcher shared by detection and redaction; fillers are recognised by isRedactionFiller so a re-scan / second pass is a no-op. New privacy FPs fixed along the way: syslog selectors (local7.info), "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer domains. - Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and unreadable nested archives are reported in Result.SkippedBinary, never edited. - Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить и скачать копию" button in the Customer-data panel, and logpile -sanitize <file> (restores mtime/atime). Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C .sds yields the identical hardware inventory; re-scan is clean. ADL-067, bible-local/docs/log-sanitization.md. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e74e01ad05
commit
a63bb17438
@@ -28,6 +28,8 @@ func main() {
|
||||
file := flag.String("file", "", "Pre-load archive file")
|
||||
showVersion := flag.Bool("version", false, "Show version")
|
||||
noBrowser := flag.Bool("no-browser", false, "Don't open browser automatically")
|
||||
sanitizeIn := flag.String("sanitize", "", "De-identify customer data in this archive/log and exit (no server)")
|
||||
sanitizeOut := flag.String("sanitize-out", "", "Write the sanitized file here (default: overwrite -sanitize input)")
|
||||
flag.Parse()
|
||||
|
||||
if *showVersion {
|
||||
@@ -35,6 +37,14 @@ func main() {
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
if *sanitizeIn != "" {
|
||||
if err := runSanitize(*sanitizeIn, *sanitizeOut); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "sanitize:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Set embedded web files
|
||||
server.WebFS = web.FS
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/sanitize"
|
||||
)
|
||||
|
||||
// runSanitize de-identifies inPath and writes the result to outPath (or back to
|
||||
// inPath). The output file's mtime/atime are restored to the original; ctime
|
||||
// (inode change time) cannot be restored portably.
|
||||
func runSanitize(inPath, outPath string) error {
|
||||
info, err := os.Stat(inPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := os.ReadFile(inPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !sanitize.CanSanitize(inPath) {
|
||||
return fmt.Errorf("unsupported file format for sanitize: %s", inPath)
|
||||
}
|
||||
|
||||
res, err := sanitize.Sanitize(inPath, data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
target := outPath
|
||||
if target == "" {
|
||||
target = inPath
|
||||
}
|
||||
if err := os.WriteFile(target, res.Data, info.Mode().Perm()); err != nil {
|
||||
return err
|
||||
}
|
||||
mt := info.ModTime()
|
||||
if err := os.Chtimes(target, mt, mt); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "warning: could not restore file timestamp:", err)
|
||||
}
|
||||
|
||||
fmt.Printf("sanitized %s -> %s\n", inPath, target)
|
||||
fmt.Printf(" %d replacement(s), %d byte(s) in, %d byte(s) out\n", res.TotalReplaced, len(data), len(res.Data))
|
||||
for _, c := range res.Changes {
|
||||
fmt.Printf(" [%-14s] x%-4d %s\n", c.Category, c.Count, c.Path)
|
||||
}
|
||||
if len(res.SkippedBinary) > 0 {
|
||||
fmt.Printf("\n %d member(s) hold customer data but could not be edited - sanitize by hand:\n", len(res.SkippedBinary))
|
||||
for _, s := range res.SkippedBinary {
|
||||
fmt.Printf(" - %s\n", s)
|
||||
}
|
||||
}
|
||||
fmt.Println("\n note: mtime/atime restored; ctime (inode change time) reflects the edit.")
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user