feat(sanitize): in-place, length-preserving log de-identification
Adds internal/sanitize: rewrites the customer-identifying spans that internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with same-length neutral fillers, in place, without changing the file format. - Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP -> "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed. - Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes, and all embedded timestamps preserved; untouched zip entries copied raw; member payload length unchanged so tar headers stay byte-identical; only the .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output. - privacy.FindSpans is the one matcher shared by detection and redaction; fillers are recognised by isRedactionFiller so a re-scan / second pass is a no-op. New privacy FPs fixed along the way: syslog selectors (local7.info), "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer domains. - Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and unreadable nested archives are reported in Result.SkippedBinary, never edited. - Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить и скачать копию" button in the Customer-data panel, and logpile -sanitize <file> (restores mtime/atime). Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C .sds yields the identical hardware inventory; re-scan is clean. ADL-067, bible-local/docs/log-sanitization.md. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e74e01ad05
commit
a63bb17438
@@ -0,0 +1,114 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
)
|
||||
|
||||
func leakDumpBytes(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var tarBuf bytes.Buffer
|
||||
tw := tar.NewWriter(&tarBuf)
|
||||
body := []byte("domain corp.acme.ru\nracdomain=corp.acme.ru\nTimeZone=Europe/Moscow\n")
|
||||
tw.WriteHeader(&tar.Header{Name: "onekeylog/configuration/conf/resolv.conf", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})
|
||||
tw.Write(body)
|
||||
tw.Close()
|
||||
var gz bytes.Buffer
|
||||
gw := gzip.NewWriter(&gz)
|
||||
gw.Name = "dump.tar"
|
||||
gw.Write(tarBuf.Bytes())
|
||||
gw.Close()
|
||||
return gz.Bytes()
|
||||
}
|
||||
|
||||
func serverWithUpload(t *testing.T, filename string, data []byte) *Server {
|
||||
t.Helper()
|
||||
s := &Server{}
|
||||
s.SetResult(&models.AnalysisResult{Filename: filename})
|
||||
s.SetRawExport(&RawExportPackage{
|
||||
Source: RawExportSource{
|
||||
Kind: "file_bytes",
|
||||
Filename: filename,
|
||||
MIMEType: "application/gzip",
|
||||
Encoding: "base64",
|
||||
Data: base64.StdEncoding.EncodeToString(data),
|
||||
},
|
||||
})
|
||||
return s
|
||||
}
|
||||
|
||||
func TestHandleSanitize_PreviewAndDownload(t *testing.T) {
|
||||
s := serverWithUpload(t, "dump.tar.gz", leakDumpBytes(t))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
s.handleSanitize(rec, httptest.NewRequest("POST", "/api/sanitize", nil))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("preview status %d: %s", rec.Code, rec.Body)
|
||||
}
|
||||
var preview struct {
|
||||
TotalReplaced int `json:"total_replaced"`
|
||||
Changes []struct {
|
||||
Category string `json:"category"`
|
||||
} `json:"changes"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &preview); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if preview.TotalReplaced == 0 {
|
||||
t.Fatal("nothing redacted")
|
||||
}
|
||||
|
||||
dl := httptest.NewRecorder()
|
||||
s.handleSanitizeDownload(dl, httptest.NewRequest("GET", "/api/sanitize/download", nil))
|
||||
if dl.Code != http.StatusOK {
|
||||
t.Fatalf("download status %d", dl.Code)
|
||||
}
|
||||
if cd := dl.Header().Get("Content-Disposition"); !strings.Contains(cd, `filename="dump.tar.gz"`) {
|
||||
t.Fatalf("bad Content-Disposition: %q", cd)
|
||||
}
|
||||
gr, err := gzip.NewReader(bytes.NewReader(dl.Body.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("output not gzip: %v", err)
|
||||
}
|
||||
inner, _ := io.ReadAll(gr)
|
||||
tr := tar.NewReader(bytes.NewReader(inner))
|
||||
h, err := tr.Next()
|
||||
if err != nil || h.Name != "onekeylog/configuration/conf/resolv.conf" {
|
||||
t.Fatalf("inner tar broken: %v %v", h, err)
|
||||
}
|
||||
clean, _ := io.ReadAll(tr)
|
||||
if bytes.Contains(clean, []byte("acme.ru")) || bytes.Contains(clean, []byte("Europe/Moscow")) {
|
||||
t.Fatalf("leak survived sanitize:\n%s", clean)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleSanitize_RejectsNonFileSource(t *testing.T) {
|
||||
s := &Server{}
|
||||
s.SetResult(&models.AnalysisResult{})
|
||||
s.SetRawExport(&RawExportPackage{Source: RawExportSource{Kind: "live_redfish"}})
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
s.handleSanitize(rec, httptest.NewRequest("POST", "/api/sanitize", nil))
|
||||
if rec.Code != http.StatusUnprocessableEntity {
|
||||
t.Fatalf("status %d, want 422", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleSanitizeDownload_NotReady(t *testing.T) {
|
||||
s := &Server{}
|
||||
rec := httptest.NewRecorder()
|
||||
s.handleSanitizeDownload(rec, httptest.NewRequest("GET", "/api/sanitize/download", nil))
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("status %d, want 404", rec.Code)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user