Mikhail ChusavitinandClaude Sonnet 5
a63bb17438
Adds internal/sanitize: rewrites the customer-identifying spans that internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with same-length neutral fillers, in place, without changing the file format. - Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP -> "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed. - Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes, and all embedded timestamps preserved; untouched zip entries copied raw; member payload length unchanged so tar headers stay byte-identical; only the .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output. - privacy.FindSpans is the one matcher shared by detection and redaction; fillers are recognised by isRedactionFiller so a re-scan / second pass is a no-op. New privacy FPs fixed along the way: syslog selectors (local7.info), "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer domains. - Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and unreadable nested archives are reported in Result.SkippedBinary, never edited. - Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить и скачать копию" button in the Customer-data panel, and logpile -sanitize <file> (restores mtime/atime). Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C .sds yields the identical hardware inventory; re-scan is clean. ADL-067, bible-local/docs/log-sanitization.md. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
80 lines
2.4 KiB
Go
80 lines
2.4 KiB
Go
package server
|
|
|
|
import (
|
|
"encoding/base64"
|
|
"fmt"
|
|
"net/http"
|
|
"path/filepath"
|
|
"strconv"
|
|
|
|
"git.mchus.pro/mchus/logpile/internal/sanitize"
|
|
)
|
|
|
|
// sanitizeArtifact holds the de-identified copy of the current upload, ready for
|
|
// download. Kept in memory only, like convertOutput.
|
|
type sanitizeArtifact struct {
|
|
Data []byte
|
|
Filename string
|
|
}
|
|
|
|
func (s *Server) setSanitizeArtifact(a *sanitizeArtifact) {
|
|
s.mu.Lock()
|
|
s.sanitizeResult = a
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
func (s *Server) getSanitizeArtifact() *sanitizeArtifact {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
return s.sanitizeResult
|
|
}
|
|
|
|
// handleSanitize builds a de-identified copy of the retained original upload and
|
|
// returns the change preview. The file itself is fetched from GET /api/sanitize/download.
|
|
func (s *Server) handleSanitize(w http.ResponseWriter, r *http.Request) {
|
|
pkg := s.GetRawExport()
|
|
if pkg == nil || pkg.Source.Kind != "file_bytes" {
|
|
jsonError(w, "sanitize is only available for an uploaded archive or log file", http.StatusUnprocessableEntity)
|
|
return
|
|
}
|
|
if !sanitize.CanSanitize(pkg.Source.Filename) {
|
|
jsonError(w, "this file format cannot be sanitized in place", http.StatusUnprocessableEntity)
|
|
return
|
|
}
|
|
data, err := base64.StdEncoding.DecodeString(pkg.Source.Data)
|
|
if err != nil {
|
|
jsonError(w, "cannot read the original file bytes", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
res, err := sanitize.Sanitize(pkg.Source.Filename, data)
|
|
if err != nil {
|
|
jsonError(w, "sanitize failed: "+err.Error(), http.StatusUnprocessableEntity)
|
|
return
|
|
}
|
|
|
|
s.setSanitizeArtifact(&sanitizeArtifact{Data: res.Data, Filename: pkg.Source.Filename})
|
|
|
|
jsonResponse(w, map[string]any{
|
|
"filename": filepath.Base(pkg.Source.Filename),
|
|
"input_size": len(data),
|
|
"output_size": len(res.Data),
|
|
"total_replaced": res.TotalReplaced,
|
|
"changes": res.Changes,
|
|
"skipped_binary": res.SkippedBinary,
|
|
})
|
|
}
|
|
|
|
func (s *Server) handleSanitizeDownload(w http.ResponseWriter, r *http.Request) {
|
|
art := s.getSanitizeArtifact()
|
|
if art == nil {
|
|
jsonError(w, "no sanitized file ready; run POST /api/sanitize first", http.StatusNotFound)
|
|
return
|
|
}
|
|
// Always octet-stream so the browser saves rather than renders it.
|
|
w.Header().Set("Content-Type", "application/octet-stream")
|
|
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filepath.Base(art.Filename)))
|
|
w.Header().Set("Content-Length", strconv.Itoa(len(art.Data)))
|
|
_, _ = w.Write(art.Data)
|
|
}
|