feat(sanitize): in-place, length-preserving log de-identification

Adds internal/sanitize: rewrites the customer-identifying spans that
internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with
same-length neutral fillers, in place, without changing the file format.

- Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP ->
  "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid
  neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed.
- Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes,
  and all embedded timestamps preserved; untouched zip entries copied raw;
  member payload length unchanged so tar headers stay byte-identical; only the
  .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output.
- privacy.FindSpans is the one matcher shared by detection and redaction;
  fillers are recognised by isRedactionFiller so a re-scan / second pass is a
  no-op. New privacy FPs fixed along the way: syslog selectors (local7.info),
  "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer
  domains.
- Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and
  unreadable nested archives are reported in Result.SkippedBinary, never edited.
- Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить
  и скачать копию" button in the Customer-data panel, and
  logpile -sanitize <file> (restores mtime/atime).

Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C
.sds yields the identical hardware inventory; re-scan is clean. ADL-067,
bible-local/docs/log-sanitization.md.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-09-02 18:05:27 +03:00
co-authored by Claude Sonnet 5
parent e74e01ad05
commit a63bb17438
25 changed files with 1742 additions and 66 deletions
+114
View File
@@ -0,0 +1,114 @@
package server
import (
"archive/tar"
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.mchus.pro/mchus/logpile/internal/models"
)
func leakDumpBytes(t *testing.T) []byte {
t.Helper()
var tarBuf bytes.Buffer
tw := tar.NewWriter(&tarBuf)
body := []byte("domain corp.acme.ru\nracdomain=corp.acme.ru\nTimeZone=Europe/Moscow\n")
tw.WriteHeader(&tar.Header{Name: "onekeylog/configuration/conf/resolv.conf", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})
tw.Write(body)
tw.Close()
var gz bytes.Buffer
gw := gzip.NewWriter(&gz)
gw.Name = "dump.tar"
gw.Write(tarBuf.Bytes())
gw.Close()
return gz.Bytes()
}
func serverWithUpload(t *testing.T, filename string, data []byte) *Server {
t.Helper()
s := &Server{}
s.SetResult(&models.AnalysisResult{Filename: filename})
s.SetRawExport(&RawExportPackage{
Source: RawExportSource{
Kind: "file_bytes",
Filename: filename,
MIMEType: "application/gzip",
Encoding: "base64",
Data: base64.StdEncoding.EncodeToString(data),
},
})
return s
}
func TestHandleSanitize_PreviewAndDownload(t *testing.T) {
s := serverWithUpload(t, "dump.tar.gz", leakDumpBytes(t))
rec := httptest.NewRecorder()
s.handleSanitize(rec, httptest.NewRequest("POST", "/api/sanitize", nil))
if rec.Code != http.StatusOK {
t.Fatalf("preview status %d: %s", rec.Code, rec.Body)
}
var preview struct {
TotalReplaced int `json:"total_replaced"`
Changes []struct {
Category string `json:"category"`
} `json:"changes"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &preview); err != nil {
t.Fatal(err)
}
if preview.TotalReplaced == 0 {
t.Fatal("nothing redacted")
}
dl := httptest.NewRecorder()
s.handleSanitizeDownload(dl, httptest.NewRequest("GET", "/api/sanitize/download", nil))
if dl.Code != http.StatusOK {
t.Fatalf("download status %d", dl.Code)
}
if cd := dl.Header().Get("Content-Disposition"); !strings.Contains(cd, `filename="dump.tar.gz"`) {
t.Fatalf("bad Content-Disposition: %q", cd)
}
gr, err := gzip.NewReader(bytes.NewReader(dl.Body.Bytes()))
if err != nil {
t.Fatalf("output not gzip: %v", err)
}
inner, _ := io.ReadAll(gr)
tr := tar.NewReader(bytes.NewReader(inner))
h, err := tr.Next()
if err != nil || h.Name != "onekeylog/configuration/conf/resolv.conf" {
t.Fatalf("inner tar broken: %v %v", h, err)
}
clean, _ := io.ReadAll(tr)
if bytes.Contains(clean, []byte("acme.ru")) || bytes.Contains(clean, []byte("Europe/Moscow")) {
t.Fatalf("leak survived sanitize:\n%s", clean)
}
}
func TestHandleSanitize_RejectsNonFileSource(t *testing.T) {
s := &Server{}
s.SetResult(&models.AnalysisResult{})
s.SetRawExport(&RawExportPackage{Source: RawExportSource{Kind: "live_redfish"}})
rec := httptest.NewRecorder()
s.handleSanitize(rec, httptest.NewRequest("POST", "/api/sanitize", nil))
if rec.Code != http.StatusUnprocessableEntity {
t.Fatalf("status %d, want 422", rec.Code)
}
}
func TestHandleSanitizeDownload_NotReady(t *testing.T) {
s := &Server{}
rec := httptest.NewRecorder()
s.handleSanitizeDownload(rec, httptest.NewRequest("GET", "/api/sanitize/download", nil))
if rec.Code != http.StatusNotFound {
t.Fatalf("status %d, want 404", rec.Code)
}
}