feat(privacy): scan ingested sources for customer-identifying data

Detection-only scan (internal/privacy) attached to every AnalysisResult:
a customer-domain guess plus a findings list (category, file, line, match,
hint), ported from the KB grep playbook. Runs on archive uploads and the
serialized Redfish tree; gated by LOGPILE_PRIVACY_SCAN (default on).

Surfaced at GET /api/privacy-scan, in the "Customer data" UI panel, and as
privacy_report.json in the raw-export bundle. IP policy keeps RFC1918 and
example ranges out of findings; allowlist covers standards-body and vendor
infrastructure domains. No customer tokens in the repo. See ADL-066 and
bible-local/docs/privacy-scan.md.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-09-02 15:24:32 +03:00
co-authored by Claude Sonnet 5
parent 3311bafd8e
commit 4a4910f207
23 changed files with 1466 additions and 33 deletions
+9
View File
@@ -1199,6 +1199,15 @@ func extractFirmwareComponentAndModel(deviceName string) (component, model strin
return deviceName, "-"
}
func (s *Server) handleGetPrivacyScan(w http.ResponseWriter, r *http.Request) {
result := s.GetResult()
if result == nil || result.PrivacyScan == nil {
jsonResponse(w, map[string]any{"loaded": false})
return
}
jsonResponse(w, result.PrivacyScan)
}
func (s *Server) handleGetStatus(w http.ResponseWriter, r *http.Request) {
result := s.GetResult()
if result == nil {
+101
View File
@@ -0,0 +1,101 @@
package server
import (
"archive/zip"
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"git.mchus.pro/mchus/logpile/internal/models"
)
func samplePrivacyScan() *models.PrivacyScan {
return &models.PrivacyScan{
FilesScanned: 3,
Customers: []models.CustomerGuess{{Domain: "acme.ru", Confidence: "high", Hits: 4}},
Findings: []models.PrivacyFinding{
{Category: "resolv", Severity: "high", Path: "resolv.conf", Line: 1, Match: "corp.acme.ru"},
},
Summary: models.PrivacySummary{Total: 1, High: 1, ByCategory: map[string]int{"resolv": 1}},
}
}
func TestHandleGetPrivacyScan_NotLoaded(t *testing.T) {
srv := &Server{}
req := httptest.NewRequest("GET", "/api/privacy-scan", nil)
w := httptest.NewRecorder()
srv.handleGetPrivacyScan(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d", w.Code)
}
var body map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatal(err)
}
if body["loaded"] != false {
t.Fatalf("want loaded:false, got %v", body)
}
}
func TestHandleGetPrivacyScan_Loaded(t *testing.T) {
srv := &Server{}
srv.SetResult(&models.AnalysisResult{PrivacyScan: samplePrivacyScan()})
req := httptest.NewRequest("GET", "/api/privacy-scan", nil)
w := httptest.NewRecorder()
srv.handleGetPrivacyScan(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d", w.Code)
}
var got models.PrivacyScan
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
if len(got.Customers) != 1 || got.Customers[0].Domain != "acme.ru" {
t.Fatalf("unexpected payload: %+v", got)
}
}
func TestBuildRawExportBundle_IncludesPrivacyReport(t *testing.T) {
pkg := newRawExportFromUploadedFile("dump.tar.gz", "application/gzip", []byte("x"), &models.AnalysisResult{})
result := &models.AnalysisResult{PrivacyScan: samplePrivacyScan()}
raw, err := buildRawExportBundle(pkg, result, "test")
if err != nil {
t.Fatal(err)
}
zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
if err != nil {
t.Fatal(err)
}
found := false
for _, f := range zr.File {
if f.Name == rawExportBundlePrivacyFile {
found = true
}
}
if !found {
t.Fatalf("%s missing from bundle", rawExportBundlePrivacyFile)
}
}
func TestBuildRawExportBundle_NoPrivacyReportWhenNil(t *testing.T) {
pkg := newRawExportFromUploadedFile("dump.tar.gz", "application/gzip", []byte("x"), &models.AnalysisResult{})
raw, err := buildRawExportBundle(pkg, &models.AnalysisResult{}, "test")
if err != nil {
t.Fatal(err)
}
zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
if err != nil {
t.Fatal(err)
}
for _, f := range zr.File {
if f.Name == rawExportBundlePrivacyFile {
t.Fatalf("%s should be absent when PrivacyScan is nil", rawExportBundlePrivacyFile)
}
}
}
+15
View File
@@ -19,6 +19,7 @@ const (
rawExportBundlePackageFile = "raw_export.json"
rawExportBundleLogFile = "collect.log"
rawExportBundleFieldsFile = "parser_fields.json"
rawExportBundlePrivacyFile = "privacy_report.json"
)
type RawExportPackage struct {
@@ -150,6 +151,20 @@ func buildRawExportBundle(pkg *RawExportPackage, result *models.AnalysisResult,
return nil, err
}
if result != nil && result.PrivacyScan != nil {
pf, err := zw.Create(rawExportBundlePrivacyFile)
if err != nil {
return nil, err
}
privacyJSON, err := json.MarshalIndent(result.PrivacyScan, "", " ")
if err != nil {
return nil, err
}
if _, err := pf.Write(privacyJSON); err != nil {
return nil, err
}
}
if err := zw.Close(); err != nil {
return nil, err
}
+1
View File
@@ -88,6 +88,7 @@ func (s *Server) setupRoutes() {
s.mux.HandleFunc("GET /api/serials", s.handleGetSerials)
s.mux.HandleFunc("GET /api/firmware", s.handleGetFirmware)
s.mux.HandleFunc("GET /api/parse-errors", s.handleGetParseErrors)
s.mux.HandleFunc("GET /api/privacy-scan", s.handleGetPrivacyScan)
s.mux.HandleFunc("GET /api/export/csv", s.handleExportCSV)
s.mux.HandleFunc("GET /api/export/json", s.handleExportJSON)
s.mux.HandleFunc("GET /api/export/reanimator", s.handleExportReanimator)