feat(sanitize): in-place, length-preserving log de-identification
Adds internal/sanitize: rewrites the customer-identifying spans that internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with same-length neutral fillers, in place, without changing the file format. - Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP -> "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed. - Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes, and all embedded timestamps preserved; untouched zip entries copied raw; member payload length unchanged so tar headers stay byte-identical; only the .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output. - privacy.FindSpans is the one matcher shared by detection and redaction; fillers are recognised by isRedactionFiller so a re-scan / second pass is a no-op. New privacy FPs fixed along the way: syslog selectors (local7.info), "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer domains. - Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and unreadable nested archives are reported in Result.SkippedBinary, never edited. - Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить и скачать копию" button in the Customer-data panel, and logpile -sanitize <file> (restores mtime/atime). Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C .sds yields the identical hardware inventory; re-scan is clean. ADL-067, bible-local/docs/log-sanitization.md. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
e74e01ad05
commit
a63bb17438
@@ -0,0 +1,348 @@
|
||||
package sanitize
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/privacy"
|
||||
)
|
||||
|
||||
const (
|
||||
maxInputBytes = 800 << 20 // whole file, read into memory and rebuilt
|
||||
maxMemberBytes = 1 << 30 // single decompressed member
|
||||
)
|
||||
|
||||
// rewriteBytes de-identifies data according to its format (by filename
|
||||
// extension), recursing into nested archives. It returns the rebuilt bytes plus
|
||||
// the replacements made and any binary members that held customer data but
|
||||
// could not be edited safely.
|
||||
func rewriteBytes(name string, data []byte) (out []byte, changes []change, skippedBinary []string, err error) {
|
||||
switch strings.ToLower(path.Ext(name)) {
|
||||
case ".tar", ".sds":
|
||||
return rewriteTar(name, data)
|
||||
case ".gz", ".tgz":
|
||||
return rewriteGz(name, data)
|
||||
case ".zip":
|
||||
return rewriteZip(name, data)
|
||||
case ".ahs":
|
||||
// Proprietary HPE container - not editable in place. Report if it holds
|
||||
// customer data.
|
||||
if scanHasFindings(data) {
|
||||
return data, nil, []string{name + " (HPE AHS container - sanitize manually)"}, nil
|
||||
}
|
||||
return data, nil, nil, nil
|
||||
case ".txt", ".log":
|
||||
if privacy.IsAllowlistedFile(name) {
|
||||
return data, nil, nil, nil
|
||||
}
|
||||
nb, ch := redactText(data, isCertName(name))
|
||||
return nb, prefixPath(ch, path.Base(name)), nil, nil
|
||||
default:
|
||||
return rewriteMember(name, data, false)
|
||||
}
|
||||
}
|
||||
|
||||
// rewriteMember handles one file inside an archive: recurse if it is itself an
|
||||
// archive, redact if it is text, otherwise leave it (and flag it when a binary
|
||||
// member carries customer data). Returned changes/skips are already qualified
|
||||
// with name, so callers append them verbatim.
|
||||
func rewriteMember(name string, data []byte, isDirOrSpecial bool) ([]byte, []change, []string, error) {
|
||||
if isDirOrSpecial || len(data) == 0 || privacy.IsAllowlistedFile(name) {
|
||||
return data, nil, nil, nil
|
||||
}
|
||||
if isNestedArchive(name) || looksLikeTar(data) {
|
||||
nb, ch, skip, err := rewriteBytes(name, data)
|
||||
if err == nil {
|
||||
return nb, prefixPath(ch, name), prefixNames(skip, name), nil
|
||||
}
|
||||
// A truncated / mis-named "archive" must not fail the whole job: treat it
|
||||
// as a plain file if it is text, otherwise copy it and flag it.
|
||||
if looksLikeText(data) {
|
||||
nb, ch := redactText(data, isCertName(name))
|
||||
return nb, prefixPath(ch, name), nil, nil
|
||||
}
|
||||
if len(data) <= 8<<20 && binaryHasLeak(data) {
|
||||
return data, nil, []string{name + " (unreadable as an archive; holds customer data - sanitize manually)"}, nil
|
||||
}
|
||||
return data, nil, nil, nil
|
||||
}
|
||||
if looksLikeText(data) {
|
||||
nb, ch := redactText(data, isCertName(name))
|
||||
return nb, prefixPath(ch, name), nil, nil
|
||||
}
|
||||
// Binary member: never edit it (checksums / structure), but tell the
|
||||
// operator if a printable run inside it holds customer data.
|
||||
if len(data) <= 8<<20 && binaryHasLeak(data) {
|
||||
return data, nil, []string{name + " (binary - sanitize manually)"}, nil
|
||||
}
|
||||
return data, nil, nil, nil
|
||||
}
|
||||
|
||||
func rewriteTar(name string, data []byte) ([]byte, []change, []string, error) {
|
||||
tr := tar.NewReader(bytes.NewReader(data))
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
var changes []change
|
||||
var skipped []string
|
||||
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("%s: tar read: %w", name, err)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(tr, maxMemberBytes+1))
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("%s: read %s: %w", name, hdr.Name, err)
|
||||
}
|
||||
special := !hdr.FileInfo().Mode().IsRegular()
|
||||
newBody, ch, skip, err := rewriteMember(hdr.Name, body, special || int64(len(body)) > maxMemberBytes)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
h := *hdr // reuse every header field verbatim (name, mode, uid/gid, mtime/atime/ctime, pax, format)
|
||||
h.Size = int64(len(newBody))
|
||||
if err := tw.WriteHeader(&h); err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("%s: write header %s: %w", name, hdr.Name, err)
|
||||
}
|
||||
if _, err := tw.Write(newBody); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
changes = append(changes, ch...)
|
||||
skipped = append(skipped, skip...)
|
||||
}
|
||||
if len(changes) == 0 {
|
||||
return data, nil, skipped, nil // nothing redacted -> byte-identical
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return buf.Bytes(), changes, skipped, nil
|
||||
}
|
||||
|
||||
func rewriteGz(name string, data []byte) ([]byte, []change, []string, error) {
|
||||
gzr, err := gzip.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("%s: gzip: %w", name, err)
|
||||
}
|
||||
hdr := gzr.Header
|
||||
decompressed, err := io.ReadAll(io.LimitReader(gzr, maxMemberBytes+1))
|
||||
gzr.Close()
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("%s: gunzip: %w", name, err)
|
||||
}
|
||||
if int64(len(decompressed)) > maxMemberBytes {
|
||||
return data, nil, []string{name + " (too large to sanitize)"}, nil
|
||||
}
|
||||
|
||||
innerName := strings.TrimSuffix(hdr.Name, ".gz")
|
||||
if innerName == "" {
|
||||
innerName = strings.TrimSuffix(path.Base(name), ".gz")
|
||||
}
|
||||
|
||||
var newInner []byte
|
||||
var ch []change
|
||||
var skip []string
|
||||
switch {
|
||||
case looksLikeTar(decompressed):
|
||||
newInner, ch, skip, err = rewriteTar(innerName, decompressed)
|
||||
if err != nil {
|
||||
return data, nil, []string{name + " (unreadable inner tar - copied as-is)"}, nil
|
||||
}
|
||||
case privacy.IsAllowlistedFile(innerName):
|
||||
newInner = decompressed
|
||||
case looksLikeText(decompressed):
|
||||
newInner, ch = redactText(decompressed, isCertName(innerName))
|
||||
ch = prefixPath(ch, innerName)
|
||||
case len(decompressed) <= 8<<20 && binaryHasLeak(decompressed):
|
||||
newInner, skip = decompressed, []string{innerName + " (binary - sanitize manually)"}
|
||||
default:
|
||||
newInner = decompressed
|
||||
}
|
||||
if len(ch) == 0 {
|
||||
return data, nil, skip, nil // nothing redacted -> byte-identical
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
gzw, _ := gzip.NewWriterLevel(&buf, gzip.BestCompression)
|
||||
gzw.Name = hdr.Name
|
||||
gzw.Comment = hdr.Comment
|
||||
gzw.ModTime = hdr.ModTime
|
||||
gzw.OS = hdr.OS
|
||||
gzw.Extra = hdr.Extra
|
||||
if _, err := gzw.Write(newInner); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
if err := gzw.Close(); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return buf.Bytes(), ch, skip, nil
|
||||
}
|
||||
|
||||
func rewriteZip(name string, data []byte) ([]byte, []change, []string, error) {
|
||||
zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("%s: zip: %w", name, err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
if zr.Comment != "" {
|
||||
_ = zw.SetComment(zr.Comment)
|
||||
}
|
||||
var changes []change
|
||||
var skipped []string
|
||||
|
||||
for _, f := range zr.File {
|
||||
if f.FileInfo().IsDir() {
|
||||
if err := zw.Copy(f); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err == nil {
|
||||
var body []byte
|
||||
body, err = io.ReadAll(io.LimitReader(rc, maxMemberBytes+1))
|
||||
rc.Close()
|
||||
if err == nil {
|
||||
newBody, ch, skip, mErr := rewriteMember(f.Name, body, int64(len(body)) > maxMemberBytes)
|
||||
if mErr == nil {
|
||||
if bytes.Equal(newBody, body) {
|
||||
if err := zw.Copy(f); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
} else {
|
||||
fh := f.FileHeader
|
||||
fh.CRC32, fh.CompressedSize, fh.CompressedSize64 = 0, 0, 0
|
||||
fh.UncompressedSize, fh.UncompressedSize64 = 0, 0
|
||||
w, cErr := zw.CreateHeader(&fh)
|
||||
if cErr != nil {
|
||||
return nil, nil, nil, cErr
|
||||
}
|
||||
if _, wErr := w.Write(newBody); wErr != nil {
|
||||
return nil, nil, nil, wErr
|
||||
}
|
||||
}
|
||||
changes = append(changes, ch...)
|
||||
skipped = append(skipped, skip...)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
// Unreadable / unprocessable entry: copy it raw, flag it.
|
||||
if err := zw.Copy(f); err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("%s: copy %s: %w", name, f.Name, err)
|
||||
}
|
||||
skipped = append(skipped, f.Name+" (unreadable zip entry - copied as-is)")
|
||||
}
|
||||
if len(changes) == 0 {
|
||||
return data, nil, skipped, nil // nothing redacted -> byte-identical
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return buf.Bytes(), changes, skipped, nil
|
||||
}
|
||||
|
||||
func isNestedArchive(name string) bool {
|
||||
switch strings.ToLower(path.Ext(name)) {
|
||||
case ".gz", ".tgz", ".tar", ".zip", ".sds":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isCertName(name string) bool {
|
||||
switch strings.ToLower(path.Ext(name)) {
|
||||
case ".pem", ".csr", ".crt", ".cer":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func looksLikeTar(b []byte) bool {
|
||||
if len(b) < 512 {
|
||||
return false
|
||||
}
|
||||
_, err := tar.NewReader(bytes.NewReader(b)).Next()
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// looksLikeText mirrors the privacy scanner's heuristic: no NUL byte in the
|
||||
// first 8 KiB.
|
||||
func looksLikeText(b []byte) bool {
|
||||
if len(b) == 0 {
|
||||
return false
|
||||
}
|
||||
head := b
|
||||
if len(head) > 8192 {
|
||||
head = head[:8192]
|
||||
}
|
||||
return bytes.IndexByte(head, 0) < 0
|
||||
}
|
||||
|
||||
func scanHasFindings(b []byte) bool {
|
||||
rep := privacy.Scan([]privacy.File{{Path: "member", Content: b}})
|
||||
return rep != nil && rep.Summary.Total > 0
|
||||
}
|
||||
|
||||
// binaryHasLeak reports whether any printable ASCII run inside a binary member
|
||||
// contains a redactable span (privacy.Scan itself skips non-text files, so it
|
||||
// cannot see strings embedded in FRU.bin / redis-dump.rdb / SDR.dat).
|
||||
func binaryHasLeak(b []byte) bool {
|
||||
start := -1
|
||||
check := func(run []byte) bool {
|
||||
if len(run) < 6 {
|
||||
return false
|
||||
}
|
||||
for _, sp := range privacy.FindSpans(string(run), false) {
|
||||
if redactableCat[sp.Category] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
for i, c := range b {
|
||||
printable := c >= 0x20 && c < 0x7f
|
||||
if printable && start < 0 {
|
||||
start = i
|
||||
}
|
||||
if !printable && start >= 0 {
|
||||
if check(b[start:i]) {
|
||||
return true
|
||||
}
|
||||
start = -1
|
||||
}
|
||||
}
|
||||
if start >= 0 && check(b[start:]) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func prefixPath(ch []change, parent string) []change {
|
||||
for i := range ch {
|
||||
if ch[i].path == "" {
|
||||
ch[i].path = parent
|
||||
} else {
|
||||
ch[i].path = parent + "::" + ch[i].path
|
||||
}
|
||||
}
|
||||
return ch
|
||||
}
|
||||
|
||||
func prefixNames(names []string, parent string) []string {
|
||||
out := make([]string, len(names))
|
||||
for i, n := range names {
|
||||
out[i] = parent + "::" + n
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package sanitize
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/privacy"
|
||||
)
|
||||
|
||||
// Span categories that get rewritten. fru_location is intentionally excluded
|
||||
// (often a serial / manufacturing code, low value, and may live in a binary
|
||||
// FRU area); it is only reported by the scan, never edited.
|
||||
var redactableCat = map[string]bool{
|
||||
"domain": true, "mgmt_subdomain": true, "resolv": true, "ad_ldap": true,
|
||||
"email": true, "nsupdate": true, "collector": true, "dhcp": true,
|
||||
"cert": true, "public_ip": true, "timezone": true,
|
||||
}
|
||||
|
||||
type change struct {
|
||||
path string
|
||||
category string
|
||||
before string
|
||||
after string
|
||||
}
|
||||
|
||||
// redactText rewrites every redactable span in content, keeping each
|
||||
// replacement byte-for-byte the same length as the original so the total
|
||||
// content length never changes. Line terminators are preserved exactly.
|
||||
func redactText(content []byte, certFile bool) ([]byte, []change) {
|
||||
var out bytes.Buffer
|
||||
out.Grow(len(content))
|
||||
var changes []change
|
||||
|
||||
for _, s := range splitKeepEOL(content) {
|
||||
spans := redactableSpans(s.line, certFile)
|
||||
if len(spans) == 0 {
|
||||
out.WriteString(s.line)
|
||||
out.Write(s.eol)
|
||||
continue
|
||||
}
|
||||
prev := 0
|
||||
for _, sp := range spans {
|
||||
out.WriteString(s.line[prev:sp.Start])
|
||||
orig := s.line[sp.Start:sp.End]
|
||||
repl := fillerFor(sp.Category, orig)
|
||||
if len(repl) != len(orig) {
|
||||
repl = xFill(orig)
|
||||
}
|
||||
out.WriteString(repl)
|
||||
changes = append(changes, change{category: sp.Category, before: orig, after: repl})
|
||||
prev = sp.End
|
||||
}
|
||||
out.WriteString(s.line[prev:])
|
||||
out.Write(s.eol)
|
||||
}
|
||||
return out.Bytes(), changes
|
||||
}
|
||||
|
||||
// redactableSpans returns the redactable spans of one line, sorted by start and
|
||||
// with overlaps merged (so a nested domain inside an e-mail is redacted once).
|
||||
func redactableSpans(line string, certFile bool) []privacy.Span {
|
||||
raw := privacy.FindSpans(line, certFile)
|
||||
kept := raw[:0]
|
||||
for _, sp := range raw {
|
||||
if redactableCat[sp.Category] {
|
||||
kept = append(kept, sp)
|
||||
}
|
||||
}
|
||||
if len(kept) < 2 {
|
||||
return kept
|
||||
}
|
||||
sortSpans(kept)
|
||||
merged := kept[:1]
|
||||
for _, sp := range kept[1:] {
|
||||
last := &merged[len(merged)-1]
|
||||
if sp.Start <= last.End {
|
||||
if sp.End > last.End {
|
||||
last.End = sp.End
|
||||
}
|
||||
continue
|
||||
}
|
||||
merged = append(merged, sp)
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func sortSpans(s []privacy.Span) {
|
||||
for i := 1; i < len(s); i++ {
|
||||
for j := i; j > 0 && s[j-1].Start > s[j].Start; j-- {
|
||||
s[j-1], s[j] = s[j], s[j-1]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fillerFor returns a same-length neutral replacement for one matched token.
|
||||
func fillerFor(category, orig string) string {
|
||||
switch category {
|
||||
case "public_ip":
|
||||
return digitZero(orig) // 93.184.216.34 -> 00.000.000.00
|
||||
case "timezone":
|
||||
return tzFiller(orig)
|
||||
default:
|
||||
return xFill(orig) // sigma.sbrf.ru -> xxxxx.xxxx.xx
|
||||
}
|
||||
}
|
||||
|
||||
// xFill replaces every ASCII letter/digit with 'x', keeping punctuation
|
||||
// (dots, hyphens, '@', ':', '*', '_') in place.
|
||||
func xFill(s string) string {
|
||||
b := []byte(s)
|
||||
for i, c := range b {
|
||||
if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') {
|
||||
b[i] = 'x'
|
||||
}
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// digitZero replaces every digit with '0', keeping dots.
|
||||
func digitZero(s string) string {
|
||||
b := []byte(s)
|
||||
for i, c := range b {
|
||||
if c >= '0' && c <= '9' {
|
||||
b[i] = '0'
|
||||
}
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// tzFiller neutralises a timezone value: a numeric UTC offset becomes zeros, a
|
||||
// Region/City name becomes a same-length valid IANA zone, an abbreviation
|
||||
// becomes "UTC" (len 3) or an x-fill.
|
||||
func tzFiller(orig string) string {
|
||||
if isOffset(orig) {
|
||||
return digitZero(orig) // 180 -> 000, -300 -> -000
|
||||
}
|
||||
if strings.Contains(orig, "/") {
|
||||
if z, ok := neutralZoneByLen[len(orig)]; ok {
|
||||
return z
|
||||
}
|
||||
return keepSlashXFill(orig)
|
||||
}
|
||||
if len(orig) == 3 {
|
||||
return "UTC"
|
||||
}
|
||||
return xFill(orig)
|
||||
}
|
||||
|
||||
func isOffset(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
for i, c := range s {
|
||||
if c >= '0' && c <= '9' {
|
||||
continue
|
||||
}
|
||||
if (c == '+' || c == '-') && i == 0 {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func keepSlashXFill(s string) string {
|
||||
b := []byte(xFill(s))
|
||||
for i, c := range []byte(s) {
|
||||
if c == '/' {
|
||||
b[i] = '/'
|
||||
}
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// neutralZoneByLen maps a timezone-name length to a same-length, valid IANA
|
||||
// zone that carries no regional information (UTC/Etc/* where the length allows,
|
||||
// otherwise a fixed far-away decoy). Every value here is on the privacy
|
||||
// allowlist so a re-scan of the sanitized file stays clean.
|
||||
var neutralZoneByLen = map[int]string{
|
||||
3: "UTC",
|
||||
4: "Zulu",
|
||||
7: "Etc/UTC",
|
||||
8: "Etc/GMT0",
|
||||
9: "Universal",
|
||||
10: "US/Pacific",
|
||||
11: "Brazil/East",
|
||||
12: "Canada/Yukon",
|
||||
13: "Etc/Universal",
|
||||
14: "Pacific/Tarawa",
|
||||
15: "Atlantic/Azores",
|
||||
16: "Antarctica/Troll",
|
||||
17: "Antarctica/Vostok",
|
||||
18: "Antarctica/McMurdo",
|
||||
19: "Pacific/Guadalcanal",
|
||||
20: "Pacific/Bougainville",
|
||||
}
|
||||
|
||||
type eolSeg struct {
|
||||
line string
|
||||
eol []byte
|
||||
}
|
||||
|
||||
// splitKeepEOL splits content into lines while preserving each original line
|
||||
// terminator ("\n", "\r\n", or none for a final unterminated line).
|
||||
func splitKeepEOL(b []byte) []eolSeg {
|
||||
var segs []eolSeg
|
||||
i := 0
|
||||
for i < len(b) {
|
||||
j := bytes.IndexByte(b[i:], '\n')
|
||||
if j < 0 {
|
||||
segs = append(segs, eolSeg{line: string(b[i:])})
|
||||
return segs
|
||||
}
|
||||
nl := i + j
|
||||
lineEnd := nl
|
||||
if lineEnd > i && b[lineEnd-1] == '\r' {
|
||||
lineEnd--
|
||||
}
|
||||
segs = append(segs, eolSeg{line: string(b[i:lineEnd]), eol: append([]byte(nil), b[lineEnd:nl+1]...)})
|
||||
i = nl + 1
|
||||
}
|
||||
return segs
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package sanitize
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNeutralZonesAreValidAndSameLength(t *testing.T) {
|
||||
// If the runtime has no zoneinfo at all, skip the loadability half.
|
||||
_, tzErr := time.LoadLocation("Europe/Moscow")
|
||||
haveTZDB := tzErr == nil
|
||||
|
||||
for n, zone := range neutralZoneByLen {
|
||||
if len(zone) != n {
|
||||
t.Errorf("neutralZoneByLen[%d] = %q has length %d", n, zone, len(zone))
|
||||
}
|
||||
if haveTZDB {
|
||||
if _, err := time.LoadLocation(zone); err != nil {
|
||||
t.Errorf("neutral zone %q does not load: %v", zone, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTZFiller(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"Europe/Moscow": "Etc/Universal",
|
||||
"Asia/Yekaterinburg": "Antarctica/McMurdo",
|
||||
"180": "000",
|
||||
"-300": "-000",
|
||||
"MSK": "UTC",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := tzFiller(in); got != want {
|
||||
t.Errorf("tzFiller(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
if got := tzFiller(in); len(got) != len(in) {
|
||||
t.Errorf("tzFiller(%q) length %d != %d", in, len(got), len(in))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedact_SkipsAllowlistedTemplateFile(t *testing.T) {
|
||||
// _tianyiyun is a vendor factory template - must not be scanned/redacted.
|
||||
nb, ch, _, err := rewriteBytes("onekeylog/configuration/conf/syslog_tianyiyun.conf_bak", []byte("server ntp.acme.ru\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ch) != 0 || string(nb) != "server ntp.acme.ru\n" {
|
||||
t.Fatalf("template file was modified: %q %+v", nb, ch)
|
||||
}
|
||||
}
|
||||
|
||||
func TestXFill(t *testing.T) {
|
||||
if got := xFill("a1-b2.c3_d4@e5:f6"); got != "xx-xx.xx_xx@xx:xx" {
|
||||
t.Fatalf("xFill = %q", got)
|
||||
}
|
||||
if !strings.HasPrefix(xFill("corp.acme.ru"), "xxxx.") {
|
||||
t.Fatalf("xFill domain: %q", xFill("corp.acme.ru"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
// Package sanitize produces a de-identified copy of an uploaded diagnostic file.
|
||||
//
|
||||
// It rewrites the customer-identifying spans that internal/privacy detects
|
||||
// (domains, FQDNs, e-mails, AD config, public IPs, timezone) with same-length
|
||||
// neutral fillers, in place, without changing the file format: archive
|
||||
// structure, entry names, modes and embedded timestamps are preserved; only the
|
||||
// redacted byte ranges - and, for compressed containers, the compression layer
|
||||
// - differ. Detection only ever reports; this is the matching redactor. See
|
||||
// bible-local/docs/log-sanitization.md.
|
||||
package sanitize
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Change is one aggregated group of replacements for the preview UI.
|
||||
type Change struct {
|
||||
Path string `json:"path"`
|
||||
Category string `json:"category"`
|
||||
Count int `json:"count"`
|
||||
SampleBefore string `json:"sample_before"`
|
||||
SampleAfter string `json:"sample_after"`
|
||||
}
|
||||
|
||||
// Result is the outcome of sanitizing one file.
|
||||
type Result struct {
|
||||
Data []byte `json:"-"`
|
||||
Changes []Change `json:"changes"`
|
||||
SkippedBinary []string `json:"skipped_binary"`
|
||||
TotalReplaced int `json:"total_replaced"`
|
||||
}
|
||||
|
||||
// Sanitize de-identifies data (named filename so the format can be resolved by
|
||||
// extension, same set as the parser accepts) and returns the rebuilt file plus
|
||||
// a summary of what changed.
|
||||
func Sanitize(filename string, data []byte) (*Result, error) {
|
||||
if len(data) == 0 {
|
||||
return nil, fmt.Errorf("empty input")
|
||||
}
|
||||
if len(data) > maxInputBytes {
|
||||
return nil, fmt.Errorf("file too large to sanitize in memory: %d bytes (limit %d)", len(data), maxInputBytes)
|
||||
}
|
||||
|
||||
out, changes, skipped, err := rewriteBytes(filename, data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := &Result{
|
||||
Data: out,
|
||||
TotalReplaced: len(changes),
|
||||
Changes: aggregate(changes),
|
||||
SkippedBinary: dedupeStrings(skipped),
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func aggregate(raw []change) []Change {
|
||||
type key struct{ path, cat string }
|
||||
m := map[key]*Change{}
|
||||
order := []key{}
|
||||
for _, c := range raw {
|
||||
k := key{c.path, c.category}
|
||||
g := m[k]
|
||||
if g == nil {
|
||||
g = &Change{Path: c.path, Category: c.category, SampleBefore: c.before, SampleAfter: c.after}
|
||||
m[k] = g
|
||||
order = append(order, k)
|
||||
}
|
||||
g.Count++
|
||||
}
|
||||
out := make([]Change, 0, len(order))
|
||||
for _, k := range order {
|
||||
out = append(out, *m[k])
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
if out[i].Category != out[j].Category {
|
||||
return out[i].Category < out[j].Category
|
||||
}
|
||||
return out[i].Path < out[j].Path
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func dedupeStrings(in []string) []string {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]string, 0, len(in))
|
||||
for _, s := range in {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[s]; ok {
|
||||
continue
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// CanSanitize reports whether a file with this name is a format the sanitizer
|
||||
// can rebuild.
|
||||
func CanSanitize(filename string) bool {
|
||||
switch strings.ToLower(ext(filename)) {
|
||||
case ".tar", ".sds", ".gz", ".tgz", ".zip", ".txt", ".log":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ext(name string) string {
|
||||
if i := strings.LastIndexByte(name, '.'); i >= 0 {
|
||||
return name[i:]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
package sanitize
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/privacy"
|
||||
)
|
||||
|
||||
const leakLog = "" +
|
||||
"domain corp.acme.ru\n" +
|
||||
"nameserver 10.0.0.1\n" +
|
||||
"racdomain=corp.acme.ru\n" +
|
||||
"server ntp01.acme.ru\n" +
|
||||
"contact ops@corp.acme.ru\n" +
|
||||
"outbound 93.184.216.34 established\n" +
|
||||
"TimeZone=Europe/Moscow\n" +
|
||||
"SELTimeUTCOffset=180\n" +
|
||||
"pool at pool.ntp.org keep\n"
|
||||
|
||||
func scanClean(t *testing.T, name string, data []byte) {
|
||||
t.Helper()
|
||||
rep := privacy.Scan([]privacy.File{{Path: name, Content: data}})
|
||||
if rep != nil && rep.Summary.Total > 0 {
|
||||
var b strings.Builder
|
||||
for _, f := range rep.Findings {
|
||||
b.WriteString("\n " + f.Category + " " + f.Match)
|
||||
}
|
||||
t.Fatalf("%s still has %d findings:%s", name, rep.Summary.Total, b.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactText_LengthPreservedAndClean(t *testing.T) {
|
||||
in := []byte(leakLog)
|
||||
out, changes := redactText(in, false)
|
||||
|
||||
if len(out) != len(in) {
|
||||
t.Fatalf("length changed: %d -> %d", len(in), len(out))
|
||||
}
|
||||
if len(changes) == 0 {
|
||||
t.Fatal("no changes made")
|
||||
}
|
||||
scanClean(t, "out.log", out)
|
||||
|
||||
// idempotent
|
||||
out2, ch2 := redactText(out, false)
|
||||
if !bytes.Equal(out, out2) || len(ch2) != 0 {
|
||||
t.Fatalf("not idempotent: %d more changes\n%q", len(ch2), out2)
|
||||
}
|
||||
|
||||
s := string(out)
|
||||
if strings.Contains(s, "acme.ru") || strings.Contains(s, "93.184.216.34") || strings.Contains(s, "Europe/Moscow") {
|
||||
t.Fatalf("leak survived:\n%s", s)
|
||||
}
|
||||
if !strings.Contains(s, "pool.ntp.org") {
|
||||
t.Fatal("allowlisted pool.ntp.org was redacted")
|
||||
}
|
||||
if !strings.Contains(s, "10.0.0.1") {
|
||||
t.Fatal("private IP was redacted")
|
||||
}
|
||||
if !strings.Contains(s, "TimeZone=Etc/Universal") {
|
||||
t.Fatalf("timezone not neutralised to a same-length zone:\n%s", s)
|
||||
}
|
||||
if !strings.Contains(s, "SELTimeUTCOffset=000") {
|
||||
t.Fatalf("offset not zeroed:\n%s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitize_PlainLogNoLeak_ByteIdentical(t *testing.T) {
|
||||
in := []byte("just a boring log line\nnothing here\n")
|
||||
res, err := Sanitize("x.log", in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(res.Data, in) {
|
||||
t.Fatalf("clean input was modified:\n%q", res.Data)
|
||||
}
|
||||
if res.TotalReplaced != 0 {
|
||||
t.Fatalf("changes on clean input: %+v", res.Changes)
|
||||
}
|
||||
}
|
||||
|
||||
func buildTar(t *testing.T, members map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
mt := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)
|
||||
// deterministic order
|
||||
for _, name := range []string{"onekeylog/clean.txt", "onekeylog/configuration/conf/resolv.conf"} {
|
||||
body, ok := members[name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if err := tw.WriteHeader(&tar.Header{
|
||||
Name: name, Mode: 0o644, Size: int64(len(body)), ModTime: mt, Typeflag: tar.TypeReg, Format: tar.FormatGNU,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tw.Write([]byte(body))
|
||||
}
|
||||
tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func tarList(t *testing.T, data []byte) map[string]tar.Header {
|
||||
t.Helper()
|
||||
m := map[string]tar.Header{}
|
||||
tr := tar.NewReader(bytes.NewReader(data))
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("re-read tar: %v", err)
|
||||
}
|
||||
body, _ := io.ReadAll(tr)
|
||||
h.Size = int64(len(body))
|
||||
m[h.Name] = *h
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestSanitize_Tar_PreservesListingAndCleanMember(t *testing.T) {
|
||||
members := map[string]string{
|
||||
"onekeylog/clean.txt": "cpu model Xeon Gold\nmemory 512GB\n",
|
||||
"onekeylog/configuration/conf/resolv.conf": leakLog,
|
||||
}
|
||||
in := buildTar(t, members)
|
||||
res, err := Sanitize("dump.tar", in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
before, after := tarList(t, in), tarList(t, res.Data)
|
||||
if len(before) != len(after) {
|
||||
t.Fatalf("member count changed: %d -> %d", len(before), len(after))
|
||||
}
|
||||
for name, hb := range before {
|
||||
ha, ok := after[name]
|
||||
if !ok {
|
||||
t.Fatalf("member %s vanished", name)
|
||||
}
|
||||
if hb.Size != ha.Size || hb.Mode != ha.Mode || !hb.ModTime.Equal(ha.ModTime) || hb.Typeflag != ha.Typeflag {
|
||||
t.Fatalf("member %s header changed: %+v -> %+v", name, hb, ha)
|
||||
}
|
||||
}
|
||||
|
||||
// clean member byte-identical
|
||||
if memberBytes(t, res.Data, "onekeylog/clean.txt") != members["onekeylog/clean.txt"] {
|
||||
t.Fatal("clean member was altered")
|
||||
}
|
||||
// leak member cleaned
|
||||
scanClean(t, "resolv.conf", []byte(memberBytes(t, res.Data, "onekeylog/configuration/conf/resolv.conf")))
|
||||
if res.TotalReplaced == 0 {
|
||||
t.Fatal("nothing redacted")
|
||||
}
|
||||
}
|
||||
|
||||
func memberBytes(t *testing.T, tarData []byte, name string) string {
|
||||
t.Helper()
|
||||
tr := tar.NewReader(bytes.NewReader(tarData))
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
t.Fatalf("member %s not found", name)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h.Name == name {
|
||||
b, _ := io.ReadAll(tr)
|
||||
return string(b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitize_TarGz_PreservesGzipHeaderAndInnerListing(t *testing.T) {
|
||||
inner := buildTar(t, map[string]string{
|
||||
"onekeylog/clean.txt": "board serial ABC123\n",
|
||||
"onekeylog/configuration/conf/resolv.conf": leakLog,
|
||||
})
|
||||
var gz bytes.Buffer
|
||||
gw, _ := gzip.NewWriterLevel(&gz, gzip.BestCompression)
|
||||
gw.Name = "dump.tar"
|
||||
gw.ModTime = time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
gw.Write(inner)
|
||||
gw.Close()
|
||||
|
||||
res, err := Sanitize("dump.tar.gz", gz.Bytes())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
gr, err := gzip.NewReader(bytes.NewReader(res.Data))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gr.Name != "dump.tar" || !gr.ModTime.Equal(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) {
|
||||
t.Fatalf("gzip header not preserved: name=%q mtime=%v", gr.Name, gr.ModTime)
|
||||
}
|
||||
out, _ := io.ReadAll(gr)
|
||||
|
||||
before, after := tarList(t, inner), tarList(t, out)
|
||||
for name, hb := range before {
|
||||
if ha, ok := after[name]; !ok || !hb.ModTime.Equal(ha.ModTime) || hb.Size != ha.Size {
|
||||
t.Fatalf("inner member %s changed", name)
|
||||
}
|
||||
}
|
||||
scanClean(t, "inner-resolv", []byte(memberBytes(t, out, "onekeylog/configuration/conf/resolv.conf")))
|
||||
}
|
||||
|
||||
func TestSanitize_Zip_UntouchedEntryIdentical(t *testing.T) {
|
||||
var zb bytes.Buffer
|
||||
zw := zip.NewWriter(&zb)
|
||||
mod := time.Date(2025, 7, 8, 9, 10, 0, 0, time.UTC)
|
||||
for _, e := range []struct{ name, body string }{
|
||||
{"clean.txt", "hardware inventory only\n"},
|
||||
{"host/resolv.conf", leakLog},
|
||||
} {
|
||||
w, _ := zw.CreateHeader(&zip.FileHeader{Name: e.name, Method: zip.Deflate, Modified: mod})
|
||||
w.Write([]byte(e.body))
|
||||
}
|
||||
zw.Close()
|
||||
|
||||
res, err := Sanitize("bundle.zip", zb.Bytes())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
zr, err := zip.NewReader(bytes.NewReader(res.Data), int64(len(res.Data)))
|
||||
if err != nil {
|
||||
t.Fatalf("output zip invalid: %v", err)
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, f := range zr.File {
|
||||
if !f.Modified.Equal(mod) {
|
||||
t.Fatalf("%s Modified changed: %v", f.Name, f.Modified)
|
||||
}
|
||||
rc, _ := f.Open()
|
||||
b, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
got[f.Name] = string(b)
|
||||
}
|
||||
if got["clean.txt"] != "hardware inventory only\n" {
|
||||
t.Fatalf("clean entry altered: %q", got["clean.txt"])
|
||||
}
|
||||
scanClean(t, "zip-resolv", []byte(got["host/resolv.conf"]))
|
||||
}
|
||||
|
||||
func TestSanitize_BinaryMemberFlagged(t *testing.T) {
|
||||
fru := append([]byte{0x01, 0x00, 0x00, 0x00}, []byte("corp.acme.ru\x00padding")...)
|
||||
in := func() []byte {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
tw.WriteHeader(&tar.Header{Name: "onekeylog/FRU.bin", Mode: 0o644, Size: int64(len(fru)), Typeflag: tar.TypeReg})
|
||||
tw.Write(fru)
|
||||
tw.Close()
|
||||
return buf.Bytes()
|
||||
}()
|
||||
|
||||
res, err := Sanitize("d.tar", in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(res.SkippedBinary) != 1 || !strings.Contains(res.SkippedBinary[0], "FRU.bin") {
|
||||
t.Fatalf("binary member not flagged: %+v", res.SkippedBinary)
|
||||
}
|
||||
if memberBytes(t, res.Data, "onekeylog/FRU.bin") != string(fru) {
|
||||
t.Fatal("binary member was modified")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user