Mikhail ChusavitinandClaude Sonnet 5
a63bb17438
Adds internal/sanitize: rewrites the customer-identifying spans that internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with same-length neutral fillers, in place, without changing the file format. - Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP -> "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed. - Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes, and all embedded timestamps preserved; untouched zip entries copied raw; member payload length unchanged so tar headers stay byte-identical; only the .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output. - privacy.FindSpans is the one matcher shared by detection and redaction; fillers are recognised by isRedactionFiller so a re-scan / second pass is a no-op. New privacy FPs fixed along the way: syslog selectors (local7.info), "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer domains. - Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and unreadable nested archives are reported in Result.SkippedBinary, never edited. - Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить и скачать копию" button in the Customer-data panel, and logpile -sanitize <file> (restores mtime/atime). Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C .sds yields the identical hardware inventory; re-scan is clean. ADL-067, bible-local/docs/log-sanitization.md. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
115 lines
3.2 KiB
Go
115 lines
3.2 KiB
Go
package server
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.mchus.pro/mchus/logpile/internal/models"
|
|
)
|
|
|
|
func leakDumpBytes(t *testing.T) []byte {
|
|
t.Helper()
|
|
var tarBuf bytes.Buffer
|
|
tw := tar.NewWriter(&tarBuf)
|
|
body := []byte("domain corp.acme.ru\nracdomain=corp.acme.ru\nTimeZone=Europe/Moscow\n")
|
|
tw.WriteHeader(&tar.Header{Name: "onekeylog/configuration/conf/resolv.conf", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})
|
|
tw.Write(body)
|
|
tw.Close()
|
|
var gz bytes.Buffer
|
|
gw := gzip.NewWriter(&gz)
|
|
gw.Name = "dump.tar"
|
|
gw.Write(tarBuf.Bytes())
|
|
gw.Close()
|
|
return gz.Bytes()
|
|
}
|
|
|
|
func serverWithUpload(t *testing.T, filename string, data []byte) *Server {
|
|
t.Helper()
|
|
s := &Server{}
|
|
s.SetResult(&models.AnalysisResult{Filename: filename})
|
|
s.SetRawExport(&RawExportPackage{
|
|
Source: RawExportSource{
|
|
Kind: "file_bytes",
|
|
Filename: filename,
|
|
MIMEType: "application/gzip",
|
|
Encoding: "base64",
|
|
Data: base64.StdEncoding.EncodeToString(data),
|
|
},
|
|
})
|
|
return s
|
|
}
|
|
|
|
func TestHandleSanitize_PreviewAndDownload(t *testing.T) {
|
|
s := serverWithUpload(t, "dump.tar.gz", leakDumpBytes(t))
|
|
|
|
rec := httptest.NewRecorder()
|
|
s.handleSanitize(rec, httptest.NewRequest("POST", "/api/sanitize", nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("preview status %d: %s", rec.Code, rec.Body)
|
|
}
|
|
var preview struct {
|
|
TotalReplaced int `json:"total_replaced"`
|
|
Changes []struct {
|
|
Category string `json:"category"`
|
|
} `json:"changes"`
|
|
}
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &preview); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if preview.TotalReplaced == 0 {
|
|
t.Fatal("nothing redacted")
|
|
}
|
|
|
|
dl := httptest.NewRecorder()
|
|
s.handleSanitizeDownload(dl, httptest.NewRequest("GET", "/api/sanitize/download", nil))
|
|
if dl.Code != http.StatusOK {
|
|
t.Fatalf("download status %d", dl.Code)
|
|
}
|
|
if cd := dl.Header().Get("Content-Disposition"); !strings.Contains(cd, `filename="dump.tar.gz"`) {
|
|
t.Fatalf("bad Content-Disposition: %q", cd)
|
|
}
|
|
gr, err := gzip.NewReader(bytes.NewReader(dl.Body.Bytes()))
|
|
if err != nil {
|
|
t.Fatalf("output not gzip: %v", err)
|
|
}
|
|
inner, _ := io.ReadAll(gr)
|
|
tr := tar.NewReader(bytes.NewReader(inner))
|
|
h, err := tr.Next()
|
|
if err != nil || h.Name != "onekeylog/configuration/conf/resolv.conf" {
|
|
t.Fatalf("inner tar broken: %v %v", h, err)
|
|
}
|
|
clean, _ := io.ReadAll(tr)
|
|
if bytes.Contains(clean, []byte("acme.ru")) || bytes.Contains(clean, []byte("Europe/Moscow")) {
|
|
t.Fatalf("leak survived sanitize:\n%s", clean)
|
|
}
|
|
}
|
|
|
|
func TestHandleSanitize_RejectsNonFileSource(t *testing.T) {
|
|
s := &Server{}
|
|
s.SetResult(&models.AnalysisResult{})
|
|
s.SetRawExport(&RawExportPackage{Source: RawExportSource{Kind: "live_redfish"}})
|
|
|
|
rec := httptest.NewRecorder()
|
|
s.handleSanitize(rec, httptest.NewRequest("POST", "/api/sanitize", nil))
|
|
if rec.Code != http.StatusUnprocessableEntity {
|
|
t.Fatalf("status %d, want 422", rec.Code)
|
|
}
|
|
}
|
|
|
|
func TestHandleSanitizeDownload_NotReady(t *testing.T) {
|
|
s := &Server{}
|
|
rec := httptest.NewRecorder()
|
|
s.handleSanitizeDownload(rec, httptest.NewRequest("GET", "/api/sanitize/download", nil))
|
|
if rec.Code != http.StatusNotFound {
|
|
t.Fatalf("status %d, want 404", rec.Code)
|
|
}
|
|
}
|