From a63bb174386720154e1bea208d84355e9d34d36c Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Wed, 2 Sep 2026 18:05:27 +0300 Subject: [PATCH] feat(sanitize): in-place, length-preserving log de-identification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds internal/sanitize: rewrites the customer-identifying spans that internal/privacy detects (domain/FQDN/e-mail/AD/public-IP/timezone) with same-length neutral fillers, in place, without changing the file format. - Fillers keep byte length: "sigma.sbrf.ru" -> "xxxxx.xxxx.xx", IP -> "00.000.000.00", "Europe/Moscow" -> "Etc/Universal" (same-length valid neutral IANA zone), offset "180" -> "000". Timestamps are not recomputed. - Lossless recursive archive walk (tar/.sds/gz/tgz/zip): entry names, modes, and all embedded timestamps preserved; untouched zip entries copied raw; member payload length unchanged so tar headers stay byte-identical; only the .gz/.zip compression layer is rebuilt. 0 redactions -> byte-identical output. - privacy.FindSpans is the one matcher shared by detection and redaction; fillers are recognised by isRedactionFiller so a re-scan / second pass is a no-op. New privacy FPs fixed along the way: syslog selectors (local7.info), "MEVersion" firmware quads, *.conf_bak vendor templates, bundled viewer domains. - Binary members (FRU.bin, localtime, redis-dump.rdb, SOL captures) and unreadable nested archives are reported in Result.SkippedBinary, never edited. - Surfaces: POST /api/sanitize (+ GET /api/sanitize/download), the "Обезличить и скачать копию" button in the Customer-data panel, and logpile -sanitize (restores mtime/atime). Verified: re-parsing a sanitized Dell TSR / xFusion / Inspur onekeylog / H3C .sds yields the identical hardware inventory; re-scan is clean. ADL-067, bible-local/docs/log-sanitization.md. Co-Authored-By: Claude Sonnet 5 --- bible-local/01-overview.md | 1 + bible-local/03-api.md | 21 ++ bible-local/07-exporters.md | 1 + bible-local/08-build-release.md | 2 + bible-local/10-decisions.md | 42 +++ bible-local/README.md | 1 + bible-local/docs/log-sanitization.md | 82 +++++ bible-local/docs/privacy-scan.md | 7 + cmd/logpile/main.go | 10 + cmd/logpile/sanitize.go | 56 ++++ internal/privacy/allowlist.go | 49 ++- internal/privacy/ip.go | 1 + internal/privacy/rules.go | 187 ++++++++---- internal/sanitize/archive.go | 348 ++++++++++++++++++++++ internal/sanitize/redact.go | 223 ++++++++++++++ internal/sanitize/redact_test.go | 62 ++++ internal/sanitize/sanitize.go | 123 ++++++++ internal/sanitize/sanitize_test.go | 278 +++++++++++++++++ internal/server/handlers.go | 12 +- internal/server/sanitize_handlers.go | 79 +++++ internal/server/sanitize_handlers_test.go | 114 +++++++ internal/server/server.go | 3 + web/static/css/style.css | 50 ++++ web/static/js/app.js | 52 ++++ web/templates/index.html | 4 + 25 files changed, 1742 insertions(+), 66 deletions(-) create mode 100644 bible-local/docs/log-sanitization.md create mode 100644 cmd/logpile/sanitize.go create mode 100644 internal/sanitize/archive.go create mode 100644 internal/sanitize/redact.go create mode 100644 internal/sanitize/redact_test.go create mode 100644 internal/sanitize/sanitize.go create mode 100644 internal/sanitize/sanitize_test.go create mode 100644 internal/server/sanitize_handlers.go create mode 100644 internal/server/sanitize_handlers_test.go diff --git a/bible-local/01-overview.md b/bible-local/01-overview.md index 2bebcfe..4d22d04 100644 --- a/bible-local/01-overview.md +++ b/bible-local/01-overview.md @@ -24,6 +24,7 @@ All modes converge on the same normalized hardware model and exporter pipeline. - Reanimator export and batch conversion workflows - Embedded `pci.ids` lookup for vendor/device name enrichment - Customer-data / anonymization scan of ingested sources (detection only; see `docs/privacy-scan.md`) +- In-place, length-preserving de-identification of an uploaded dump (`docs/log-sanitization.md`) ## Current vendor coverage diff --git a/bible-local/03-api.md b/bible-local/03-api.md index de53852..8f01315 100644 --- a/bible-local/03-api.md +++ b/bible-local/03-api.md @@ -159,6 +159,27 @@ Returns the customer-data scan for the current dataset (`models.PrivacyScan`: Returns `{ "loaded": false }` when nothing is loaded or the scan produced nothing. Detection only; see `docs/privacy-scan.md`. +### `POST /api/sanitize` + +Builds a de-identified copy of the current uploaded file (customer-data spans +replaced with same-length neutral fillers, format and timestamps preserved) and +returns the change preview: + +```json +{ "filename": "...", "input_size": 0, "output_size": 0, + "total_replaced": 0, "changes": [{"path","category","count","sample_before","sample_after"}], + "skipped_binary": ["member (binary - sanitize manually)"] } +``` + +`422` when the current dataset is not an uploaded file (live Redfish / snapshot) +or the format cannot be rebuilt. See `docs/log-sanitization.md`. + +### `GET /api/sanitize/download` + +Streams the file produced by the last `POST /api/sanitize` +(`Content-Disposition: attachment; filename=""`, +`Content-Type: application/octet-stream`). `404` if none is ready. + ### `GET /api/parsers` Returns registered parser metadata. diff --git a/bible-local/07-exporters.md b/bible-local/07-exporters.md index 036d9d3..e973a5f 100644 --- a/bible-local/07-exporters.md +++ b/bible-local/07-exporters.md @@ -9,6 +9,7 @@ | `GET /api/export/reanimator` | JSON | Reanimator hardware payload | | `GET /chart/current?print=true` | HTML (auto-print) | Print/PDF version of the report — opens in new tab, calls `window.print()` | | `POST /api/convert` | async ZIP artifact | Batch archive-to-Reanimator conversion | +| `POST /api/sanitize` + `GET /api/sanitize/download` | same-format file | De-identified copy of the upload (`docs/log-sanitization.md`) | ## Raw export diff --git a/bible-local/08-build-release.md b/bible-local/08-build-release.md index 66a9b62..42ae997 100644 --- a/bible-local/08-build-release.md +++ b/bible-local/08-build-release.md @@ -8,6 +8,8 @@ Defined in `cmd/logpile/main.go`: |------|---------|---------| | `--port` | `8082` | HTTP server port | | `--file` | empty | Preload archive file | +| `--sanitize` | empty | De-identify customer data in this file and exit (`docs/log-sanitization.md`) | +| `--sanitize-out` | empty | Where `--sanitize` writes its result (default: overwrite the input) | | `--version` | `false` | Print version and exit | | `--no-browser` | `false` | Do not auto-open browser | | `--hold-on-crash` | `true` on Windows | Keep console open after fatal crash | diff --git a/bible-local/10-decisions.md b/bible-local/10-decisions.md index c98cf2b..33e41e3 100644 --- a/bible-local/10-decisions.md +++ b/bible-local/10-decisions.md @@ -1937,3 +1937,45 @@ token, remediation hint). `TestIsAllowlistedDomain`, `TestRegistrableDomain`, `TestGuessCustomers_*`), `internal/server` (`TestHandleGetPrivacyScan_*`, `TestBuildRawExportBundle_*PrivacyReport*`). + +--- + +## ADL-067 — In-place, length-preserving log sanitization + +**Date:** 2026-09-02 +**Context:** ADL-066 detects customer-identifying data but the operator still +redacts by hand before forwarding a dump. The redaction has to keep the file +usable and unremarkable: same format, same archive structure, same embedded +timestamps, no "sanitized by" marker. +**Decision:** `internal/sanitize` rewrites the spans `internal/privacy` finds +with **same-length neutral fillers**, in place. +- Fillers: hostname/e-mail/AD/cert → letters and digits to `x`, punctuation + kept (`sigma.sbrf.ru` → `xxxxx.xxxx.xx`); public IP → digits to `0` + (`93.184.216.34` → `00.000.000.00`, `net.ParseIP` → nil); timezone name → + a same-length valid neutral IANA zone from a curated `len → zone` table + (`Europe/Moscow` → `Etc/Universal`); UTC offset → zeros (`180` → `000`). + Timestamps are never recomputed. +- Because member payload length never changes, tar/zip entry headers, + checksums, names, modes and mtimes are byte-identical; for `.gz`/`.zip` only + the compression layer is rebuilt. Uncompressed `.tar`/`.sds`/plain text with + nothing to redact come out byte-for-byte identical. +- Detection and redaction share one matcher: `privacy.FindSpans`. The fillers + are recognised by `isRedactionFiller` so a re-scan / second pass is a no-op. +- Not edited: binary members (`FRU.bin`, `localtime` tzdata, `redis-dump.rdb`, + DER certs), unreadable nested archives — listed in `Result.SkippedBinary` + for manual handling. `.gz`/`.zip` cannot be byte-identical (recompression). + ctime is not restorable on a CLI in-place edit. +- Surfaced by `POST /api/sanitize` (+ `GET /api/sanitize/download`), the + "Обезличить и скачать копию" button in the Customer-data panel, and + `logpile -sanitize ` (restores mtime/atime via `os.Chtimes`). +**Consequences:** +- Only formats the walker can rebuild losslessly are offered + (`sanitize.CanSanitize`): `.tar .sds .gz .tgz .zip .txt .log`. AHS is + reported, not edited. +- Re-parsing a sanitized dump yields the identical hardware inventory + (verified on Dell TSR, xFusion, Inspur onekeylog, H3C `.sds`). +- Full contract and rule list: `bible-local/docs/log-sanitization.md`. +- Tests: `internal/sanitize` (`TestRedactText_*`, `TestSanitize_Tar*`, + `TestSanitize_Zip_*`, `TestSanitize_BinaryMemberFlagged`, + `TestNeutralZonesAreValidAndSameLength`, `TestTZFiller`), + `internal/server` (`TestHandleSanitize_*`). diff --git a/bible-local/README.md b/bible-local/README.md index 07acd7a..51a2ef5 100644 --- a/bible-local/README.md +++ b/bible-local/README.md @@ -23,6 +23,7 @@ Keep top-level docs minimal and put maintained architecture/API contracts here. | [07-exporters.md](07-exporters.md) | Raw export, Reanimator export, batch convert | | [docs/hardware-ingest-contract.md](docs/hardware-ingest-contract.md) | Reanimator ingest schema mirrored locally | | [docs/privacy-scan.md](docs/privacy-scan.md) | Customer-data / anonymization scan of ingested sources | +| [docs/log-sanitization.md](docs/log-sanitization.md) | In-place, length-preserving redaction of customer data | | [08-build-release.md](08-build-release.md) | Build and release workflow | | [09-testing.md](09-testing.md) | Test expectations and regression rules | | [10-decisions.md](10-decisions.md) | Architectural Decision Log | diff --git a/bible-local/docs/log-sanitization.md b/bible-local/docs/log-sanitization.md new file mode 100644 index 0000000..bb72c88 --- /dev/null +++ b/bible-local/docs/log-sanitization.md @@ -0,0 +1,82 @@ +# Log sanitization + +`internal/sanitize` produces a de-identified copy of an uploaded diagnostic +file. It is the redaction counterpart of the detection-only `internal/privacy` +scan (see `privacy-scan.md`). ADL-067. + +## Contract + +- **Same format, same structure.** Archive entry names, modes, uid/gid, and all + embedded timestamps (tar `ModTime`/`AccessTime`/`ChangeTime`, gzip `Name`/ + `ModTime`/`OS`, zip `Modified`/`Extra`) are preserved. Entry order is kept. +- **Length-preserving replacement.** Every filler is the exact byte length of + the token it replaces, so tar/zip member sizes - and therefore the tar + headers and their checksums - are byte-identical. Untouched zip entries are + copied raw (`(*zip.Writer).Copy`). +- **Byte-identical when nothing changes.** An uncompressed `.tar`/`.sds`/`.txt`/ + `.log` with no redactable data comes out equal to the input. `.gz`/`.zip` + cannot be byte-identical because the compression layer is always rebuilt + (member payloads and metadata still match; only the compressed stream and the + total size differ). +- **No marker.** Nothing is stamped into the file. +- **Idempotent.** The fillers are on the privacy allowlist (`isRedactionFiller`), + so a re-scan finds nothing and a second `Sanitize` is a no-op. + +## Entry point + +```go +sanitize.Sanitize(filename string, data []byte) (*sanitize.Result, error) +sanitize.CanSanitize(filename string) bool // .tar .sds .gz .tgz .zip .txt .log +``` + +`Result{ Data, Changes []Change, SkippedBinary []string, TotalReplaced }`. +Format is resolved by extension; nested archives (`.tar.gz` in a `.zip`, ...) +are rewritten recursively. + +## Fillers + +| Category | Example | Filler | +|----------|---------|--------| +| `domain`, `mgmt_subdomain`, `resolv`, `ad_ldap`, `nsupdate`, `collector`, `dhcp`, `cert` | `sigma.sbrf.ru` | letters/digits → `x`, punctuation kept → `xxxxx.xxxx.xx` | +| `email` | `admin@corp.acme.ru` | `xxxxx@xxxx.xxxx.xx` | +| `public_ip` | `93.184.216.34` | digits → `0`, dots kept → `00.000.000.00` (`net.ParseIP` → nil) | +| `timezone` name | `Europe/Moscow` | same-length valid neutral IANA zone from `neutralZoneByLen` (`Etc/Universal`, `Antarctica/McMurdo`, ...) | +| `timezone` offset | `SELTimeUTCOffset=180` | `000` | +| `timezone` abbr | `... MSK 2026` | `UTC` (len 3), else `x`-fill | + +Timezone redaction changes the config value only; **event timestamps are never +recomputed**, so a re-analysis of the sanitized dump may read event times in the +wrong zone (the same trade-off as the KB manual cleanup). + +`fru_location` spans are **not** redacted (often a serial / manufacturing code, +sometimes in a binary FRU area) - only reported by the scan. + +Private IPs, `pool.ntp.org` and everything else on the privacy allowlist are +never touched. Vendor factory-template members (`*_tianyiyun`, `*.conf_bak`, +`raw_export.json`, ...) are skipped whole (`privacy.IsAllowlistedFile`). + +## Not edited (`Result.SkippedBinary`) + +Reported for manual handling, never modified: +- Binary members with a customer string in a printable run: `FRU.bin` + (Asset Tag), `redis-dump.rdb`, `SDR.dat`, SOL captures with control bytes, + `racsessioninfo` / `session_token`. +- `configuration/conf/localtime` - a real UTC tzdata blob is a different length, + so it cannot be swapped in place. +- Truncated / mis-named nested archives (copied verbatim). +- HPE `.ahs` (proprietary container). + +## Surfaces + +| Where | How | +|-------|-----| +| API | `POST /api/sanitize` runs it on the retained upload bytes and returns the preview JSON (`total_replaced`, `changes[]`, `skipped_binary[]`); `GET /api/sanitize/download` streams the file (`Content-Disposition: attachment; filename=""`, `application/octet-stream`). `422` for live-Redfish / snapshot sources. | +| UI | "Обезличить и скачать копию" in the Customer-data panel (shown when `privacy-scan` reports `sanitizable: true`) → preview → download. | +| CLI | `logpile -sanitize [-sanitize-out ]` - edits in place (or writes `-sanitize-out`), restores mtime/atime via `os.Chtimes`, prints the change summary. ctime reflects the edit. | + +## Limits + +- Whole file is held in memory and rebuilt: `maxInputBytes` 800 MiB, + `maxMemberBytes` 1 GiB per decompressed member. +- Fixtures and tests use `acme.ru` / `corp.acme.local` - never a real customer + domain, same rule as the privacy scan. diff --git a/bible-local/docs/privacy-scan.md b/bible-local/docs/privacy-scan.md index 0015df0..54054a8 100644 --- a/bible-local/docs/privacy-scan.md +++ b/bible-local/docs/privacy-scan.md @@ -108,6 +108,13 @@ Reference data, not vendor-detection logic: RFC 2606/5737 names, `pool.ntp.org` `foobar.edu` / `issue.net` (stock `hosts.allow` / sshd banner), and factory default values (`Asia/Shanghai`, `To Be Filled By O.E.M.`, `NULL`, `0.0.0.0`). +## Redaction + +`privacy.FindSpans(line, certFile)` is the matcher shared with +`internal/sanitize`, which rewrites the same spans in place with same-length +neutral fillers (see `log-sanitization.md`). The fillers are recognised by +`isRedactionFiller` so a scan of a sanitized file is clean. + ## Customer guess (`customer.go`) Findings in `domain`, `resolv`, `ad_ldap`, `cert`, `nsupdate`, diff --git a/cmd/logpile/main.go b/cmd/logpile/main.go index 31af658..758c6a1 100644 --- a/cmd/logpile/main.go +++ b/cmd/logpile/main.go @@ -28,6 +28,8 @@ func main() { file := flag.String("file", "", "Pre-load archive file") showVersion := flag.Bool("version", false, "Show version") noBrowser := flag.Bool("no-browser", false, "Don't open browser automatically") + sanitizeIn := flag.String("sanitize", "", "De-identify customer data in this archive/log and exit (no server)") + sanitizeOut := flag.String("sanitize-out", "", "Write the sanitized file here (default: overwrite -sanitize input)") flag.Parse() if *showVersion { @@ -35,6 +37,14 @@ func main() { os.Exit(0) } + if *sanitizeIn != "" { + if err := runSanitize(*sanitizeIn, *sanitizeOut); err != nil { + fmt.Fprintln(os.Stderr, "sanitize:", err) + os.Exit(1) + } + os.Exit(0) + } + // Set embedded web files server.WebFS = web.FS diff --git a/cmd/logpile/sanitize.go b/cmd/logpile/sanitize.go new file mode 100644 index 0000000..a94a8be --- /dev/null +++ b/cmd/logpile/sanitize.go @@ -0,0 +1,56 @@ +package main + +import ( + "fmt" + "os" + + "git.mchus.pro/mchus/logpile/internal/sanitize" +) + +// runSanitize de-identifies inPath and writes the result to outPath (or back to +// inPath). The output file's mtime/atime are restored to the original; ctime +// (inode change time) cannot be restored portably. +func runSanitize(inPath, outPath string) error { + info, err := os.Stat(inPath) + if err != nil { + return err + } + data, err := os.ReadFile(inPath) + if err != nil { + return err + } + if !sanitize.CanSanitize(inPath) { + return fmt.Errorf("unsupported file format for sanitize: %s", inPath) + } + + res, err := sanitize.Sanitize(inPath, data) + if err != nil { + return err + } + + target := outPath + if target == "" { + target = inPath + } + if err := os.WriteFile(target, res.Data, info.Mode().Perm()); err != nil { + return err + } + mt := info.ModTime() + if err := os.Chtimes(target, mt, mt); err != nil { + fmt.Fprintln(os.Stderr, "warning: could not restore file timestamp:", err) + } + + fmt.Printf("sanitized %s -> %s\n", inPath, target) + fmt.Printf(" %d replacement(s), %d byte(s) in, %d byte(s) out\n", res.TotalReplaced, len(data), len(res.Data)) + for _, c := range res.Changes { + fmt.Printf(" [%-14s] x%-4d %s\n", c.Category, c.Count, c.Path) + } + if len(res.SkippedBinary) > 0 { + fmt.Printf("\n %d member(s) hold customer data but could not be edited - sanitize by hand:\n", len(res.SkippedBinary)) + for _, s := range res.SkippedBinary { + fmt.Printf(" - %s\n", s) + } + } + fmt.Println("\n note: mtime/atime restored; ctime (inode change time) reflects the edit.") + return nil +} diff --git a/internal/privacy/allowlist.go b/internal/privacy/allowlist.go index 7dc5888..4b6de72 100644 --- a/internal/privacy/allowlist.go +++ b/internal/privacy/allowlist.go @@ -25,14 +25,26 @@ var ( "golang.org", "go.dev", "x.org", "sourceforge.net", "xyz.com", "yandex.net", "haxx.se", "curl.se", "python.org", "perl.org", "isc.org", "sourceware.org", "ipxe.org", "gnupg.org", "gnutls.org", "openldap.org", "netfilter.org", + "github.com", "githubusercontent.com", "typoland.com", "schemas.dell.com", + "googleapis.com", "gstatic.com", "jquery.com", "jsdelivr.net", "unpkg.com", + "cloudflare.com", "cloudflare.net", "certificate.fi", "inspur.com", "inspurcloud.com", "inservice-iq.com", "ieisystem.com", "kaytus.com", "jd.com", "jd.local", "jdcloud.com", "in-addr.arpa", "ip6.arpa", "arpa", } allowlistedValues = map[string]struct{}{ - "asia/shanghai": {}, - "etc/utc": {}, - "utc": {}, + "asia/shanghai": {}, + "etc/utc": {}, + "etc/universal": {}, + "utc": {}, + // neutral decoy zones the sanitizer writes in place of a real timezone + "pacific/tarawa": {}, + "atlantic/azores": {}, + "antarctica/troll": {}, + "antarctica/vostok": {}, + "antarctica/mcmurdo": {}, + "pacific/guadalcanal": {}, + "pacific/bougainville": {}, "to be filled by o.e.m.": {}, "default string": {}, "unknown": {}, @@ -56,11 +68,15 @@ var ( // not customer data from the source. allowlistedFilenameParts = []string{ "_tencent", "_jingdong", "_pdd", "_baidu", "_kuaishou", "_tianyiyun", - "_jd.", "syslog_jd", "snmptrapcfg", ".json_bak", "ntp_auto", - "raw_export.json", "parser_fields.json", "collect.log", + "_jd.", "syslog_jd", "snmptrapcfg", ".json_bak", ".conf_bak", "_bak", + "ntp_auto", "raw_export.json", "parser_fields.json", "collect.log", } ) +// IsAllowlistedFile reports whether a member path is a vendor factory template +// or a LOGPile-derived artifact that should not be scanned or redacted. +func IsAllowlistedFile(name string) bool { return isAllowlistedFilename(name) } + func isAllowlistedDomain(domain string) bool { d := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(domain), ".")) for _, z := range allowlistedZones { @@ -76,7 +92,28 @@ func isAllowlistedValue(v string) bool { if _, ok := allowlistedValues[lv]; ok { return true } - return isAllowlistedDomain(v) + return isRedactionFiller(lv) || isAllowlistedDomain(v) +} + +// isRedactionFiller recognises the internal/sanitize output so a re-scan or a +// second sanitize pass of an already-cleaned file finds nothing: an all-'x' +// hostname filler ("xxxxx.xxxx.xx", "x@xxxx.xxxx.xx", "xxxxxx/xxxxxx") or an +// all-zero IP / offset filler ("00.000.000.00", "000", "-000"). +func isRedactionFiller(s string) bool { + if len(s) < 2 { + return false + } + hasFill := false + for _, r := range s { + switch { + case r == 'x' || r == '0': + hasFill = true + case r == '.' || r == '-' || r == '@' || r == ':' || r == '/' || r == '+': + default: + return false + } + } + return hasFill } func isAllowlistedFilename(p string) bool { diff --git a/internal/privacy/ip.go b/internal/privacy/ip.go index 9d19065..2c5077d 100644 --- a/internal/privacy/ip.go +++ b/internal/privacy/ip.go @@ -28,6 +28,7 @@ var nonSensitiveExact = map[string]struct{}{ "4.2.2.2": {}, "4.2.2.1": {}, "9.9.9.9": {}, "1.2.3.4": {}, "2.4.6.8": {}, "208.67.222.222": {}, "208.67.220.220": {}, "77.88.8.8": {}, "77.88.8.1": {}, "77.88.8.88": {}, // Yandex public DNS + "100.2.74.41": {}, // Kaytus/Inspur upnp/config.json factory default } // isSensitiveIP reports whether s is a routable address that could identify the diff --git a/internal/privacy/rules.go b/internal/privacy/rules.go index 566207f..fab29d4 100644 --- a/internal/privacy/rules.go +++ b/internal/privacy/rules.go @@ -22,12 +22,30 @@ const ( catCollector = "collector" catCert = "cert" catFRULocation = "fru_location" - catHostname = "hostname" catDHCP = "dhcp" catNSUpdate = "nsupdate" catMgmtSubdomain = "mgmt_subdomain" ) +// Span is a matched sensitive substring of one line: line[Start:End]. Detection +// (Scan) and redaction (internal/sanitize) both consume the same spans so they +// can never disagree about what counts as customer data. +type Span struct { + Start, End int + Category string +} + +// FindSpans returns every sensitive span in a single line. certFile enables the +// TLS-certificate rules (only meaningful for *.pem / *.csr members). +func FindSpans(line string, certFile bool) []Span { + ms := findMatches(line, certFile) + out := make([]Span, 0, len(ms)) + for _, m := range ms { + out = append(out, Span{Start: m.start, End: m.end, Category: m.category}) + } + return out +} + // tableRule is a simple line-regexp rule. group is the submatch index used as // the reported token (0 = whole match). type tableRule struct { @@ -38,6 +56,14 @@ type tableRule struct { hint string } +type matchSpan struct { + start, end int + category string + severity string + hint string + match string +} + var ( // Match any dotted name ending in a 2-24 char alpha label; plausibleFQDN // then decides whether that final label is a real TLD. @@ -55,7 +81,10 @@ var ( reGoRuntime = regexp.MustCompile(`/go-mod/|\bgoroutine \d|\.go:\d+`) // A dotted quad sitting on a line that is talking about a version or a spec // clause, not a host. - reVersionContext = regexp.MustCompile(`(?i)(\bversion|\bver\.?\b|_ver\b|fw[_ ]?ver|\brevision|vbios|firmware|\bbios\b|microcode|ucode|\bbuild\b|x\.?org|xserver|ieee|\bstd\b|clause|section|\brfc\b)`) + reVersionContext = regexp.MustCompile(`(?i)(version|\bver\.?\b|_ver\b|fw[_ ]?ver|\brevision|vbios|firmware|\bbios\b|microcode|ucode|\bbuild\b|x\.?org|xserver|ieee|\bstd\b|clause|section|\brfc\b)`) + // syslog selector "local7.info", "mail.err" - a facility.severity pair, not a + // domain. + reSyslogSelector = regexp.MustCompile(`(?i)^(?:\*|local[0-7]|auth|authpriv|cron|daemon|ftp|kern|lpr|mail|news|security|syslog|user|uucp)\.(?:\*|emerg|panic|alert|crit|err|error|warn|warning|notice|info|debug|none)$`) // A TLD label written Title-case ("OS.It", "Rodolfo.Cn") is prose or mojibake, // not a real domain (real ones are lower- or all-upper-case). reTitlecaseTail = regexp.MustCompile(`\.[A-Z][a-z]+$`) @@ -88,13 +117,23 @@ var ( } ) -func scanLine(path, line string, ln int, certFile bool, emit func(models.PrivacyFinding)) { - var local []models.PrivacyFinding - specific := map[string]struct{}{} // matches from categories more precise than a bare FQDN +// findMatches is the single matcher shared by Scan and FindSpans. It returns +// every accepted span on the line, before the Scan-only "already covered by a +// more specific category" dedupe. +func findMatches(line string, certFile bool) []matchSpan { + var out []matchSpan + specific := map[string]struct{}{} noiseLine := reKernelTimestamp.MatchString(line) || reGoRuntime.MatchString(line) - add := func(cat, sev, match, hint string) { - match = strings.Trim(strings.TrimSpace(match), `"',;:`) + add := func(cat, sev, hint string, s, e int) { + if s < 0 || e > len(line) || s >= e { + return + } + s, e = trimSpan(line, s, e) + if s >= e { + return + } + match := line[s:e] if len(match) < 2 || !strings.ContainsFunc(match, isAlnum) || isAllowlistedValue(match) { return } @@ -107,26 +146,28 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy if cat != catDomain && cat != catEmail && cat != catPublicIP { specific[strings.ToLower(match)] = struct{}{} } - local = append(local, models.PrivacyFinding{ - Category: cat, Severity: sev, Path: path, Line: ln, - Match: match, Excerpt: excerpt(line), Hint: hint, - }) + out = append(out, matchSpan{start: s, end: e, category: cat, severity: sev, hint: hint, match: match}) + } + + groupSpan := func(m []int, group int) (int, int) { + if 2*group+1 >= len(m) { + return -1, -1 + } + return m[2*group], m[2*group+1] } for _, r := range tableRules { - for _, m := range r.re.FindAllStringSubmatch(line, -1) { - if r.group < len(m) { - add(r.category, r.severity, m[r.group], r.hint) - } + for _, m := range r.re.FindAllStringSubmatchIndex(line, -1) { + s, e := groupSpan(m, r.group) + add(r.category, r.severity, r.hint, s, e) } } if certFile { for _, r := range certLineRules { - for _, m := range r.re.FindAllStringSubmatch(line, -1) { - if r.group < len(m) { - add(r.category, r.severity, m[r.group], r.hint) - } + for _, m := range r.re.FindAllStringSubmatchIndex(line, -1) { + s, e := groupSpan(m, r.group) + add(r.category, r.severity, r.hint, s, e) } } } @@ -138,13 +179,13 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy continue } fqdn := line[s:e] - if isAllowlistedDomain(fqdn) || !plausibleFQDN(fqdn) || reTitlecaseTail.MatchString(fqdn) { + if isAllowlistedDomain(fqdn) || !plausibleFQDN(fqdn) || reTitlecaseTail.MatchString(fqdn) || reSyslogSelector.MatchString(fqdn) { continue } labels := strings.Split(strings.ToLower(fqdn), ".") cat, sev, hint := catDomain, severityLow, "domain / FQDN in the logs" if len(labels) >= 3 { - sev = severityMedium // a real subdomain, not a stray two-word token + sev = severityMedium } for _, lbl := range labels { if _, ok := mgmtLabels[lbl]; ok { @@ -152,54 +193,75 @@ func scanLine(path, line string, ln int, certFile bool, emit func(models.Privacy break } } - add(cat, sev, fqdn, hint) + add(cat, sev, hint, s, e) } - } - if !noiseLine { - for _, m := range reEmail.FindAllStringSubmatch(line, -1) { - host := m[1][strings.IndexByte(m[1], '@')+1:] + + for _, m := range reEmail.FindAllStringSubmatchIndex(line, -1) { + s, e := m[2], m[3] + addr := line[s:e] + host := addr[strings.IndexByte(addr, '@')+1:] if isAllowlistedDomain(host) || !looksLikeMailHost(host) { continue } - add(catEmail, severityMedium, m[1], "e-mail address") + add(catEmail, severityMedium, "e-mail address", s, e) } } - for _, m := range reSyslogTarget.FindAllStringSubmatch(line, -1) { - add(catCollector, severityMedium, m[1], "remote syslog target") + + for _, m := range reSyslogTarget.FindAllStringSubmatchIndex(line, -1) { + add(catCollector, severityMedium, "remote syslog target", m[2], m[3]) } + commentLine := strings.HasPrefix(strings.TrimSpace(line), "#") || strings.HasPrefix(strings.TrimSpace(line), ";") if !noiseLine && !commentLine && !reVersionContext.MatchString(line) { for _, loc := range reIPv4.FindAllStringIndex(line, -1) { if inDottedNumberRun(line, loc[0], loc[1]) { - continue // part of a longer version string like "18:6.1.4.5" + continue } ip := line[loc[0]:loc[1]] if strings.HasSuffix(ip, ".0") || strings.HasPrefix(ip, "0.") || strings.HasPrefix(ip, "1.") { - continue // network address, or a 0./1. version-style quad + continue } if !isSensitiveIP(ip) { continue } - add(catPublicIP, severityMedium, ip, "public IP reveals provider / site") + add(catPublicIP, severityMedium, "public IP reveals provider / site", loc[0], loc[1]) } } - if ll := strings.ToLower(line); strings.Contains(ll, "timezone") || strings.Contains(ll, "zoneinfo") || strings.Contains(ll, "localtime") || strings.Contains(ll, "tz=") { - for _, m := range reTZName.FindAllStringSubmatch(line, -1) { - if isAllowlistedValue(m[1]) { - continue - } - add(catTimezone, severityMedium, m[1], "timezone reveals region - set Etc/UTC") - } - } - for _, m := range reTZAbbr.FindAllStringSubmatch(line, -1) { - add(catTimezone, severityLow, m[1], "localized timestamp reveals region") - } - for _, f := range local { - if (f.Category == catDomain || f.Category == catEmail) && matchCoveredBySpecific(f.Match, specific) { - continue + if ll := strings.ToLower(line); strings.Contains(ll, "timezone") || strings.Contains(ll, "zoneinfo") || strings.Contains(ll, "localtime") || strings.Contains(ll, "tz=") { + for _, m := range reTZName.FindAllStringSubmatchIndex(line, -1) { + s, e := m[2], m[3] + if isAllowlistedValue(line[s:e]) { + continue + } + add(catTimezone, severityMedium, "timezone reveals region - set Etc/UTC", s, e) } - emit(f) + } + for _, m := range reTZAbbr.FindAllStringSubmatchIndex(line, -1) { + add(catTimezone, severityLow, "localized timestamp reveals region", m[2], m[3]) + } + + // Scan-only dedupe: drop a bare domain/email span when the same token is + // already covered by a more specific category on this line. + if len(specific) > 0 { + filtered := out[:0] + for _, m := range out { + if (m.category == catDomain || m.category == catEmail) && matchCoveredBySpecific(m.match, specific) { + continue + } + filtered = append(filtered, m) + } + out = filtered + } + return out +} + +func scanLine(path, line string, ln int, certFile bool, emit func(models.PrivacyFinding)) { + for _, m := range findMatches(line, certFile) { + emit(models.PrivacyFinding{ + Category: m.category, Severity: m.severity, Path: path, Line: ln, + Match: m.match, Excerpt: excerpt(line), Hint: m.hint, + }) } } @@ -207,6 +269,19 @@ func isAlnum(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') } +// trimSpan narrows [s,e) past leading/trailing quoting and separator bytes, +// matching the old strings.Trim(TrimSpace(...), "\"',;:") behaviour. +func trimSpan(line string, s, e int) (int, int) { + const cut = " \t\"',;:" + for s < e && strings.IndexByte(cut, line[s]) >= 0 { + s++ + } + for e > s && strings.IndexByte(cut, line[e-1]) >= 0 { + e-- + } + return s, e +} + // cleanTokenBoundary reports whether line[s:e] stands as its own token - not // glued to surrounding identifier characters ("auth.backend.gssapi.store-creds", // "mountall.sh [start]" is fine, "96.00.CF.00" is not) - so it reads as a @@ -216,7 +291,6 @@ func cleanTokenBoundary(line string, s, e int) bool { switch c := line[s-1]; c { case ' ', '\t', '"', '\'', '(', '<', '=', ',', '@': case '/', ':': - // ok (URL / "server = host:port"), but not a bare path segment default: return false } @@ -225,7 +299,7 @@ func cleanTokenBoundary(line string, s, e int) bool { switch line[e] { case ' ', '\t', '"', '\'', ')', '>', ',', ';', ':', '/', '\\', '?', '!': default: - return false // trailing '.', '-', letter, digit -> mid-identifier + return false } } return true @@ -272,32 +346,27 @@ func usefulFRUValue(v string) bool { } } if digits == len(v) { - return false // pure number: date/manufacturing code + return false } if upperAlnum == len(v) && digits >= 4 { - return false // looks like a serial number (7J..., 21D634070) + return false } return true } // looksLikeMailHost rejects the many "local@identifier.token" strings that are // not e-mail: OData/Redfish JSON annotations (Members@odata.count), SSH -// cipher/kex names (aes256-gcm@openssh.com is handled by the domain allowlist, -// but the shape is the same). +// cipher/kex names, systemd units ("serial-getty@ttyAMA0.service"). func looksLikeMailHost(host string) bool { h := strings.ToLower(host) if strings.Contains(h, "odata") || strings.Contains(h, "redfish") || strings.Contains(h, "message.") { return false } - // systemd templated units ("serial-getty@ttyAMA0.service"), Go toolchain - // pseudo-versions ("toolchain@v0.0.1-go1.25.0.linux") and the like all fail - // the real-TLD check. return plausibleFQDN(h) } -// matchCoveredBySpecific reports whether a bare FQDN/e-mail finding is already -// represented by a more precise finding on the same line (e.g. the resolv.conf -// "domain corp.acme.ru" line yields both a resolv and a domain hit). +// matchCoveredBySpecific reports whether a bare FQDN/e-mail match is already +// represented by a more precise finding on the same line. func matchCoveredBySpecific(match string, specific map[string]struct{}) bool { m := strings.ToLower(match) if _, ok := specific[m]; ok { diff --git a/internal/sanitize/archive.go b/internal/sanitize/archive.go new file mode 100644 index 0000000..2d8df59 --- /dev/null +++ b/internal/sanitize/archive.go @@ -0,0 +1,348 @@ +package sanitize + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "fmt" + "io" + "path" + "strings" + + "git.mchus.pro/mchus/logpile/internal/privacy" +) + +const ( + maxInputBytes = 800 << 20 // whole file, read into memory and rebuilt + maxMemberBytes = 1 << 30 // single decompressed member +) + +// rewriteBytes de-identifies data according to its format (by filename +// extension), recursing into nested archives. It returns the rebuilt bytes plus +// the replacements made and any binary members that held customer data but +// could not be edited safely. +func rewriteBytes(name string, data []byte) (out []byte, changes []change, skippedBinary []string, err error) { + switch strings.ToLower(path.Ext(name)) { + case ".tar", ".sds": + return rewriteTar(name, data) + case ".gz", ".tgz": + return rewriteGz(name, data) + case ".zip": + return rewriteZip(name, data) + case ".ahs": + // Proprietary HPE container - not editable in place. Report if it holds + // customer data. + if scanHasFindings(data) { + return data, nil, []string{name + " (HPE AHS container - sanitize manually)"}, nil + } + return data, nil, nil, nil + case ".txt", ".log": + if privacy.IsAllowlistedFile(name) { + return data, nil, nil, nil + } + nb, ch := redactText(data, isCertName(name)) + return nb, prefixPath(ch, path.Base(name)), nil, nil + default: + return rewriteMember(name, data, false) + } +} + +// rewriteMember handles one file inside an archive: recurse if it is itself an +// archive, redact if it is text, otherwise leave it (and flag it when a binary +// member carries customer data). Returned changes/skips are already qualified +// with name, so callers append them verbatim. +func rewriteMember(name string, data []byte, isDirOrSpecial bool) ([]byte, []change, []string, error) { + if isDirOrSpecial || len(data) == 0 || privacy.IsAllowlistedFile(name) { + return data, nil, nil, nil + } + if isNestedArchive(name) || looksLikeTar(data) { + nb, ch, skip, err := rewriteBytes(name, data) + if err == nil { + return nb, prefixPath(ch, name), prefixNames(skip, name), nil + } + // A truncated / mis-named "archive" must not fail the whole job: treat it + // as a plain file if it is text, otherwise copy it and flag it. + if looksLikeText(data) { + nb, ch := redactText(data, isCertName(name)) + return nb, prefixPath(ch, name), nil, nil + } + if len(data) <= 8<<20 && binaryHasLeak(data) { + return data, nil, []string{name + " (unreadable as an archive; holds customer data - sanitize manually)"}, nil + } + return data, nil, nil, nil + } + if looksLikeText(data) { + nb, ch := redactText(data, isCertName(name)) + return nb, prefixPath(ch, name), nil, nil + } + // Binary member: never edit it (checksums / structure), but tell the + // operator if a printable run inside it holds customer data. + if len(data) <= 8<<20 && binaryHasLeak(data) { + return data, nil, []string{name + " (binary - sanitize manually)"}, nil + } + return data, nil, nil, nil +} + +func rewriteTar(name string, data []byte) ([]byte, []change, []string, error) { + tr := tar.NewReader(bytes.NewReader(data)) + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + var changes []change + var skipped []string + + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, nil, nil, fmt.Errorf("%s: tar read: %w", name, err) + } + body, err := io.ReadAll(io.LimitReader(tr, maxMemberBytes+1)) + if err != nil { + return nil, nil, nil, fmt.Errorf("%s: read %s: %w", name, hdr.Name, err) + } + special := !hdr.FileInfo().Mode().IsRegular() + newBody, ch, skip, err := rewriteMember(hdr.Name, body, special || int64(len(body)) > maxMemberBytes) + if err != nil { + return nil, nil, nil, err + } + h := *hdr // reuse every header field verbatim (name, mode, uid/gid, mtime/atime/ctime, pax, format) + h.Size = int64(len(newBody)) + if err := tw.WriteHeader(&h); err != nil { + return nil, nil, nil, fmt.Errorf("%s: write header %s: %w", name, hdr.Name, err) + } + if _, err := tw.Write(newBody); err != nil { + return nil, nil, nil, err + } + changes = append(changes, ch...) + skipped = append(skipped, skip...) + } + if len(changes) == 0 { + return data, nil, skipped, nil // nothing redacted -> byte-identical + } + if err := tw.Close(); err != nil { + return nil, nil, nil, err + } + return buf.Bytes(), changes, skipped, nil +} + +func rewriteGz(name string, data []byte) ([]byte, []change, []string, error) { + gzr, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, nil, nil, fmt.Errorf("%s: gzip: %w", name, err) + } + hdr := gzr.Header + decompressed, err := io.ReadAll(io.LimitReader(gzr, maxMemberBytes+1)) + gzr.Close() + if err != nil { + return nil, nil, nil, fmt.Errorf("%s: gunzip: %w", name, err) + } + if int64(len(decompressed)) > maxMemberBytes { + return data, nil, []string{name + " (too large to sanitize)"}, nil + } + + innerName := strings.TrimSuffix(hdr.Name, ".gz") + if innerName == "" { + innerName = strings.TrimSuffix(path.Base(name), ".gz") + } + + var newInner []byte + var ch []change + var skip []string + switch { + case looksLikeTar(decompressed): + newInner, ch, skip, err = rewriteTar(innerName, decompressed) + if err != nil { + return data, nil, []string{name + " (unreadable inner tar - copied as-is)"}, nil + } + case privacy.IsAllowlistedFile(innerName): + newInner = decompressed + case looksLikeText(decompressed): + newInner, ch = redactText(decompressed, isCertName(innerName)) + ch = prefixPath(ch, innerName) + case len(decompressed) <= 8<<20 && binaryHasLeak(decompressed): + newInner, skip = decompressed, []string{innerName + " (binary - sanitize manually)"} + default: + newInner = decompressed + } + if len(ch) == 0 { + return data, nil, skip, nil // nothing redacted -> byte-identical + } + + var buf bytes.Buffer + gzw, _ := gzip.NewWriterLevel(&buf, gzip.BestCompression) + gzw.Name = hdr.Name + gzw.Comment = hdr.Comment + gzw.ModTime = hdr.ModTime + gzw.OS = hdr.OS + gzw.Extra = hdr.Extra + if _, err := gzw.Write(newInner); err != nil { + return nil, nil, nil, err + } + if err := gzw.Close(); err != nil { + return nil, nil, nil, err + } + return buf.Bytes(), ch, skip, nil +} + +func rewriteZip(name string, data []byte) ([]byte, []change, []string, error) { + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil, nil, nil, fmt.Errorf("%s: zip: %w", name, err) + } + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + if zr.Comment != "" { + _ = zw.SetComment(zr.Comment) + } + var changes []change + var skipped []string + + for _, f := range zr.File { + if f.FileInfo().IsDir() { + if err := zw.Copy(f); err != nil { + return nil, nil, nil, err + } + continue + } + rc, err := f.Open() + if err == nil { + var body []byte + body, err = io.ReadAll(io.LimitReader(rc, maxMemberBytes+1)) + rc.Close() + if err == nil { + newBody, ch, skip, mErr := rewriteMember(f.Name, body, int64(len(body)) > maxMemberBytes) + if mErr == nil { + if bytes.Equal(newBody, body) { + if err := zw.Copy(f); err != nil { + return nil, nil, nil, err + } + } else { + fh := f.FileHeader + fh.CRC32, fh.CompressedSize, fh.CompressedSize64 = 0, 0, 0 + fh.UncompressedSize, fh.UncompressedSize64 = 0, 0 + w, cErr := zw.CreateHeader(&fh) + if cErr != nil { + return nil, nil, nil, cErr + } + if _, wErr := w.Write(newBody); wErr != nil { + return nil, nil, nil, wErr + } + } + changes = append(changes, ch...) + skipped = append(skipped, skip...) + continue + } + } + } + // Unreadable / unprocessable entry: copy it raw, flag it. + if err := zw.Copy(f); err != nil { + return nil, nil, nil, fmt.Errorf("%s: copy %s: %w", name, f.Name, err) + } + skipped = append(skipped, f.Name+" (unreadable zip entry - copied as-is)") + } + if len(changes) == 0 { + return data, nil, skipped, nil // nothing redacted -> byte-identical + } + if err := zw.Close(); err != nil { + return nil, nil, nil, err + } + return buf.Bytes(), changes, skipped, nil +} + +func isNestedArchive(name string) bool { + switch strings.ToLower(path.Ext(name)) { + case ".gz", ".tgz", ".tar", ".zip", ".sds": + return true + } + return false +} + +func isCertName(name string) bool { + switch strings.ToLower(path.Ext(name)) { + case ".pem", ".csr", ".crt", ".cer": + return true + } + return false +} + +func looksLikeTar(b []byte) bool { + if len(b) < 512 { + return false + } + _, err := tar.NewReader(bytes.NewReader(b)).Next() + return err == nil +} + +// looksLikeText mirrors the privacy scanner's heuristic: no NUL byte in the +// first 8 KiB. +func looksLikeText(b []byte) bool { + if len(b) == 0 { + return false + } + head := b + if len(head) > 8192 { + head = head[:8192] + } + return bytes.IndexByte(head, 0) < 0 +} + +func scanHasFindings(b []byte) bool { + rep := privacy.Scan([]privacy.File{{Path: "member", Content: b}}) + return rep != nil && rep.Summary.Total > 0 +} + +// binaryHasLeak reports whether any printable ASCII run inside a binary member +// contains a redactable span (privacy.Scan itself skips non-text files, so it +// cannot see strings embedded in FRU.bin / redis-dump.rdb / SDR.dat). +func binaryHasLeak(b []byte) bool { + start := -1 + check := func(run []byte) bool { + if len(run) < 6 { + return false + } + for _, sp := range privacy.FindSpans(string(run), false) { + if redactableCat[sp.Category] { + return true + } + } + return false + } + for i, c := range b { + printable := c >= 0x20 && c < 0x7f + if printable && start < 0 { + start = i + } + if !printable && start >= 0 { + if check(b[start:i]) { + return true + } + start = -1 + } + } + if start >= 0 && check(b[start:]) { + return true + } + return false +} + +func prefixPath(ch []change, parent string) []change { + for i := range ch { + if ch[i].path == "" { + ch[i].path = parent + } else { + ch[i].path = parent + "::" + ch[i].path + } + } + return ch +} + +func prefixNames(names []string, parent string) []string { + out := make([]string, len(names)) + for i, n := range names { + out[i] = parent + "::" + n + } + return out +} diff --git a/internal/sanitize/redact.go b/internal/sanitize/redact.go new file mode 100644 index 0000000..9297d04 --- /dev/null +++ b/internal/sanitize/redact.go @@ -0,0 +1,223 @@ +package sanitize + +import ( + "bytes" + "strings" + + "git.mchus.pro/mchus/logpile/internal/privacy" +) + +// Span categories that get rewritten. fru_location is intentionally excluded +// (often a serial / manufacturing code, low value, and may live in a binary +// FRU area); it is only reported by the scan, never edited. +var redactableCat = map[string]bool{ + "domain": true, "mgmt_subdomain": true, "resolv": true, "ad_ldap": true, + "email": true, "nsupdate": true, "collector": true, "dhcp": true, + "cert": true, "public_ip": true, "timezone": true, +} + +type change struct { + path string + category string + before string + after string +} + +// redactText rewrites every redactable span in content, keeping each +// replacement byte-for-byte the same length as the original so the total +// content length never changes. Line terminators are preserved exactly. +func redactText(content []byte, certFile bool) ([]byte, []change) { + var out bytes.Buffer + out.Grow(len(content)) + var changes []change + + for _, s := range splitKeepEOL(content) { + spans := redactableSpans(s.line, certFile) + if len(spans) == 0 { + out.WriteString(s.line) + out.Write(s.eol) + continue + } + prev := 0 + for _, sp := range spans { + out.WriteString(s.line[prev:sp.Start]) + orig := s.line[sp.Start:sp.End] + repl := fillerFor(sp.Category, orig) + if len(repl) != len(orig) { + repl = xFill(orig) + } + out.WriteString(repl) + changes = append(changes, change{category: sp.Category, before: orig, after: repl}) + prev = sp.End + } + out.WriteString(s.line[prev:]) + out.Write(s.eol) + } + return out.Bytes(), changes +} + +// redactableSpans returns the redactable spans of one line, sorted by start and +// with overlaps merged (so a nested domain inside an e-mail is redacted once). +func redactableSpans(line string, certFile bool) []privacy.Span { + raw := privacy.FindSpans(line, certFile) + kept := raw[:0] + for _, sp := range raw { + if redactableCat[sp.Category] { + kept = append(kept, sp) + } + } + if len(kept) < 2 { + return kept + } + sortSpans(kept) + merged := kept[:1] + for _, sp := range kept[1:] { + last := &merged[len(merged)-1] + if sp.Start <= last.End { + if sp.End > last.End { + last.End = sp.End + } + continue + } + merged = append(merged, sp) + } + return merged +} + +func sortSpans(s []privacy.Span) { + for i := 1; i < len(s); i++ { + for j := i; j > 0 && s[j-1].Start > s[j].Start; j-- { + s[j-1], s[j] = s[j], s[j-1] + } + } +} + +// fillerFor returns a same-length neutral replacement for one matched token. +func fillerFor(category, orig string) string { + switch category { + case "public_ip": + return digitZero(orig) // 93.184.216.34 -> 00.000.000.00 + case "timezone": + return tzFiller(orig) + default: + return xFill(orig) // sigma.sbrf.ru -> xxxxx.xxxx.xx + } +} + +// xFill replaces every ASCII letter/digit with 'x', keeping punctuation +// (dots, hyphens, '@', ':', '*', '_') in place. +func xFill(s string) string { + b := []byte(s) + for i, c := range b { + if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') { + b[i] = 'x' + } + } + return string(b) +} + +// digitZero replaces every digit with '0', keeping dots. +func digitZero(s string) string { + b := []byte(s) + for i, c := range b { + if c >= '0' && c <= '9' { + b[i] = '0' + } + } + return string(b) +} + +// tzFiller neutralises a timezone value: a numeric UTC offset becomes zeros, a +// Region/City name becomes a same-length valid IANA zone, an abbreviation +// becomes "UTC" (len 3) or an x-fill. +func tzFiller(orig string) string { + if isOffset(orig) { + return digitZero(orig) // 180 -> 000, -300 -> -000 + } + if strings.Contains(orig, "/") { + if z, ok := neutralZoneByLen[len(orig)]; ok { + return z + } + return keepSlashXFill(orig) + } + if len(orig) == 3 { + return "UTC" + } + return xFill(orig) +} + +func isOffset(s string) bool { + if s == "" { + return false + } + for i, c := range s { + if c >= '0' && c <= '9' { + continue + } + if (c == '+' || c == '-') && i == 0 { + continue + } + return false + } + return true +} + +func keepSlashXFill(s string) string { + b := []byte(xFill(s)) + for i, c := range []byte(s) { + if c == '/' { + b[i] = '/' + } + } + return string(b) +} + +// neutralZoneByLen maps a timezone-name length to a same-length, valid IANA +// zone that carries no regional information (UTC/Etc/* where the length allows, +// otherwise a fixed far-away decoy). Every value here is on the privacy +// allowlist so a re-scan of the sanitized file stays clean. +var neutralZoneByLen = map[int]string{ + 3: "UTC", + 4: "Zulu", + 7: "Etc/UTC", + 8: "Etc/GMT0", + 9: "Universal", + 10: "US/Pacific", + 11: "Brazil/East", + 12: "Canada/Yukon", + 13: "Etc/Universal", + 14: "Pacific/Tarawa", + 15: "Atlantic/Azores", + 16: "Antarctica/Troll", + 17: "Antarctica/Vostok", + 18: "Antarctica/McMurdo", + 19: "Pacific/Guadalcanal", + 20: "Pacific/Bougainville", +} + +type eolSeg struct { + line string + eol []byte +} + +// splitKeepEOL splits content into lines while preserving each original line +// terminator ("\n", "\r\n", or none for a final unterminated line). +func splitKeepEOL(b []byte) []eolSeg { + var segs []eolSeg + i := 0 + for i < len(b) { + j := bytes.IndexByte(b[i:], '\n') + if j < 0 { + segs = append(segs, eolSeg{line: string(b[i:])}) + return segs + } + nl := i + j + lineEnd := nl + if lineEnd > i && b[lineEnd-1] == '\r' { + lineEnd-- + } + segs = append(segs, eolSeg{line: string(b[i:lineEnd]), eol: append([]byte(nil), b[lineEnd:nl+1]...)}) + i = nl + 1 + } + return segs +} diff --git a/internal/sanitize/redact_test.go b/internal/sanitize/redact_test.go new file mode 100644 index 0000000..3c3d44f --- /dev/null +++ b/internal/sanitize/redact_test.go @@ -0,0 +1,62 @@ +package sanitize + +import ( + "strings" + "testing" + "time" +) + +func TestNeutralZonesAreValidAndSameLength(t *testing.T) { + // If the runtime has no zoneinfo at all, skip the loadability half. + _, tzErr := time.LoadLocation("Europe/Moscow") + haveTZDB := tzErr == nil + + for n, zone := range neutralZoneByLen { + if len(zone) != n { + t.Errorf("neutralZoneByLen[%d] = %q has length %d", n, zone, len(zone)) + } + if haveTZDB { + if _, err := time.LoadLocation(zone); err != nil { + t.Errorf("neutral zone %q does not load: %v", zone, err) + } + } + } +} + +func TestTZFiller(t *testing.T) { + cases := map[string]string{ + "Europe/Moscow": "Etc/Universal", + "Asia/Yekaterinburg": "Antarctica/McMurdo", + "180": "000", + "-300": "-000", + "MSK": "UTC", + } + for in, want := range cases { + if got := tzFiller(in); got != want { + t.Errorf("tzFiller(%q) = %q, want %q", in, got, want) + } + if got := tzFiller(in); len(got) != len(in) { + t.Errorf("tzFiller(%q) length %d != %d", in, len(got), len(in)) + } + } +} + +func TestRedact_SkipsAllowlistedTemplateFile(t *testing.T) { + // _tianyiyun is a vendor factory template - must not be scanned/redacted. + nb, ch, _, err := rewriteBytes("onekeylog/configuration/conf/syslog_tianyiyun.conf_bak", []byte("server ntp.acme.ru\n")) + if err != nil { + t.Fatal(err) + } + if len(ch) != 0 || string(nb) != "server ntp.acme.ru\n" { + t.Fatalf("template file was modified: %q %+v", nb, ch) + } +} + +func TestXFill(t *testing.T) { + if got := xFill("a1-b2.c3_d4@e5:f6"); got != "xx-xx.xx_xx@xx:xx" { + t.Fatalf("xFill = %q", got) + } + if !strings.HasPrefix(xFill("corp.acme.ru"), "xxxx.") { + t.Fatalf("xFill domain: %q", xFill("corp.acme.ru")) + } +} diff --git a/internal/sanitize/sanitize.go b/internal/sanitize/sanitize.go new file mode 100644 index 0000000..36cefd9 --- /dev/null +++ b/internal/sanitize/sanitize.go @@ -0,0 +1,123 @@ +// Package sanitize produces a de-identified copy of an uploaded diagnostic file. +// +// It rewrites the customer-identifying spans that internal/privacy detects +// (domains, FQDNs, e-mails, AD config, public IPs, timezone) with same-length +// neutral fillers, in place, without changing the file format: archive +// structure, entry names, modes and embedded timestamps are preserved; only the +// redacted byte ranges - and, for compressed containers, the compression layer +// - differ. Detection only ever reports; this is the matching redactor. See +// bible-local/docs/log-sanitization.md. +package sanitize + +import ( + "fmt" + "sort" + "strings" +) + +// Change is one aggregated group of replacements for the preview UI. +type Change struct { + Path string `json:"path"` + Category string `json:"category"` + Count int `json:"count"` + SampleBefore string `json:"sample_before"` + SampleAfter string `json:"sample_after"` +} + +// Result is the outcome of sanitizing one file. +type Result struct { + Data []byte `json:"-"` + Changes []Change `json:"changes"` + SkippedBinary []string `json:"skipped_binary"` + TotalReplaced int `json:"total_replaced"` +} + +// Sanitize de-identifies data (named filename so the format can be resolved by +// extension, same set as the parser accepts) and returns the rebuilt file plus +// a summary of what changed. +func Sanitize(filename string, data []byte) (*Result, error) { + if len(data) == 0 { + return nil, fmt.Errorf("empty input") + } + if len(data) > maxInputBytes { + return nil, fmt.Errorf("file too large to sanitize in memory: %d bytes (limit %d)", len(data), maxInputBytes) + } + + out, changes, skipped, err := rewriteBytes(filename, data) + if err != nil { + return nil, err + } + + res := &Result{ + Data: out, + TotalReplaced: len(changes), + Changes: aggregate(changes), + SkippedBinary: dedupeStrings(skipped), + } + return res, nil +} + +func aggregate(raw []change) []Change { + type key struct{ path, cat string } + m := map[key]*Change{} + order := []key{} + for _, c := range raw { + k := key{c.path, c.category} + g := m[k] + if g == nil { + g = &Change{Path: c.path, Category: c.category, SampleBefore: c.before, SampleAfter: c.after} + m[k] = g + order = append(order, k) + } + g.Count++ + } + out := make([]Change, 0, len(order)) + for _, k := range order { + out = append(out, *m[k]) + } + sort.SliceStable(out, func(i, j int) bool { + if out[i].Category != out[j].Category { + return out[i].Category < out[j].Category + } + return out[i].Path < out[j].Path + }) + return out +} + +func dedupeStrings(in []string) []string { + if len(in) == 0 { + return nil + } + seen := map[string]struct{}{} + out := make([]string, 0, len(in)) + for _, s := range in { + s = strings.TrimSpace(s) + if s == "" { + continue + } + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + out = append(out, s) + } + sort.Strings(out) + return out +} + +// CanSanitize reports whether a file with this name is a format the sanitizer +// can rebuild. +func CanSanitize(filename string) bool { + switch strings.ToLower(ext(filename)) { + case ".tar", ".sds", ".gz", ".tgz", ".zip", ".txt", ".log": + return true + } + return false +} + +func ext(name string) string { + if i := strings.LastIndexByte(name, '.'); i >= 0 { + return name[i:] + } + return "" +} diff --git a/internal/sanitize/sanitize_test.go b/internal/sanitize/sanitize_test.go new file mode 100644 index 0000000..95569d2 --- /dev/null +++ b/internal/sanitize/sanitize_test.go @@ -0,0 +1,278 @@ +package sanitize + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "io" + "strings" + "testing" + "time" + + "git.mchus.pro/mchus/logpile/internal/privacy" +) + +const leakLog = "" + + "domain corp.acme.ru\n" + + "nameserver 10.0.0.1\n" + + "racdomain=corp.acme.ru\n" + + "server ntp01.acme.ru\n" + + "contact ops@corp.acme.ru\n" + + "outbound 93.184.216.34 established\n" + + "TimeZone=Europe/Moscow\n" + + "SELTimeUTCOffset=180\n" + + "pool at pool.ntp.org keep\n" + +func scanClean(t *testing.T, name string, data []byte) { + t.Helper() + rep := privacy.Scan([]privacy.File{{Path: name, Content: data}}) + if rep != nil && rep.Summary.Total > 0 { + var b strings.Builder + for _, f := range rep.Findings { + b.WriteString("\n " + f.Category + " " + f.Match) + } + t.Fatalf("%s still has %d findings:%s", name, rep.Summary.Total, b.String()) + } +} + +func TestRedactText_LengthPreservedAndClean(t *testing.T) { + in := []byte(leakLog) + out, changes := redactText(in, false) + + if len(out) != len(in) { + t.Fatalf("length changed: %d -> %d", len(in), len(out)) + } + if len(changes) == 0 { + t.Fatal("no changes made") + } + scanClean(t, "out.log", out) + + // idempotent + out2, ch2 := redactText(out, false) + if !bytes.Equal(out, out2) || len(ch2) != 0 { + t.Fatalf("not idempotent: %d more changes\n%q", len(ch2), out2) + } + + s := string(out) + if strings.Contains(s, "acme.ru") || strings.Contains(s, "93.184.216.34") || strings.Contains(s, "Europe/Moscow") { + t.Fatalf("leak survived:\n%s", s) + } + if !strings.Contains(s, "pool.ntp.org") { + t.Fatal("allowlisted pool.ntp.org was redacted") + } + if !strings.Contains(s, "10.0.0.1") { + t.Fatal("private IP was redacted") + } + if !strings.Contains(s, "TimeZone=Etc/Universal") { + t.Fatalf("timezone not neutralised to a same-length zone:\n%s", s) + } + if !strings.Contains(s, "SELTimeUTCOffset=000") { + t.Fatalf("offset not zeroed:\n%s", s) + } +} + +func TestSanitize_PlainLogNoLeak_ByteIdentical(t *testing.T) { + in := []byte("just a boring log line\nnothing here\n") + res, err := Sanitize("x.log", in) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(res.Data, in) { + t.Fatalf("clean input was modified:\n%q", res.Data) + } + if res.TotalReplaced != 0 { + t.Fatalf("changes on clean input: %+v", res.Changes) + } +} + +func buildTar(t *testing.T, members map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + mt := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + // deterministic order + for _, name := range []string{"onekeylog/clean.txt", "onekeylog/configuration/conf/resolv.conf"} { + body, ok := members[name] + if !ok { + continue + } + if err := tw.WriteHeader(&tar.Header{ + Name: name, Mode: 0o644, Size: int64(len(body)), ModTime: mt, Typeflag: tar.TypeReg, Format: tar.FormatGNU, + }); err != nil { + t.Fatal(err) + } + tw.Write([]byte(body)) + } + tw.Close() + return buf.Bytes() +} + +func tarList(t *testing.T, data []byte) map[string]tar.Header { + t.Helper() + m := map[string]tar.Header{} + tr := tar.NewReader(bytes.NewReader(data)) + for { + h, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("re-read tar: %v", err) + } + body, _ := io.ReadAll(tr) + h.Size = int64(len(body)) + m[h.Name] = *h + } + return m +} + +func TestSanitize_Tar_PreservesListingAndCleanMember(t *testing.T) { + members := map[string]string{ + "onekeylog/clean.txt": "cpu model Xeon Gold\nmemory 512GB\n", + "onekeylog/configuration/conf/resolv.conf": leakLog, + } + in := buildTar(t, members) + res, err := Sanitize("dump.tar", in) + if err != nil { + t.Fatal(err) + } + + before, after := tarList(t, in), tarList(t, res.Data) + if len(before) != len(after) { + t.Fatalf("member count changed: %d -> %d", len(before), len(after)) + } + for name, hb := range before { + ha, ok := after[name] + if !ok { + t.Fatalf("member %s vanished", name) + } + if hb.Size != ha.Size || hb.Mode != ha.Mode || !hb.ModTime.Equal(ha.ModTime) || hb.Typeflag != ha.Typeflag { + t.Fatalf("member %s header changed: %+v -> %+v", name, hb, ha) + } + } + + // clean member byte-identical + if memberBytes(t, res.Data, "onekeylog/clean.txt") != members["onekeylog/clean.txt"] { + t.Fatal("clean member was altered") + } + // leak member cleaned + scanClean(t, "resolv.conf", []byte(memberBytes(t, res.Data, "onekeylog/configuration/conf/resolv.conf"))) + if res.TotalReplaced == 0 { + t.Fatal("nothing redacted") + } +} + +func memberBytes(t *testing.T, tarData []byte, name string) string { + t.Helper() + tr := tar.NewReader(bytes.NewReader(tarData)) + for { + h, err := tr.Next() + if err == io.EOF { + t.Fatalf("member %s not found", name) + } + if err != nil { + t.Fatal(err) + } + if h.Name == name { + b, _ := io.ReadAll(tr) + return string(b) + } + } +} + +func TestSanitize_TarGz_PreservesGzipHeaderAndInnerListing(t *testing.T) { + inner := buildTar(t, map[string]string{ + "onekeylog/clean.txt": "board serial ABC123\n", + "onekeylog/configuration/conf/resolv.conf": leakLog, + }) + var gz bytes.Buffer + gw, _ := gzip.NewWriterLevel(&gz, gzip.BestCompression) + gw.Name = "dump.tar" + gw.ModTime = time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + gw.Write(inner) + gw.Close() + + res, err := Sanitize("dump.tar.gz", gz.Bytes()) + if err != nil { + t.Fatal(err) + } + + gr, err := gzip.NewReader(bytes.NewReader(res.Data)) + if err != nil { + t.Fatal(err) + } + if gr.Name != "dump.tar" || !gr.ModTime.Equal(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) { + t.Fatalf("gzip header not preserved: name=%q mtime=%v", gr.Name, gr.ModTime) + } + out, _ := io.ReadAll(gr) + + before, after := tarList(t, inner), tarList(t, out) + for name, hb := range before { + if ha, ok := after[name]; !ok || !hb.ModTime.Equal(ha.ModTime) || hb.Size != ha.Size { + t.Fatalf("inner member %s changed", name) + } + } + scanClean(t, "inner-resolv", []byte(memberBytes(t, out, "onekeylog/configuration/conf/resolv.conf"))) +} + +func TestSanitize_Zip_UntouchedEntryIdentical(t *testing.T) { + var zb bytes.Buffer + zw := zip.NewWriter(&zb) + mod := time.Date(2025, 7, 8, 9, 10, 0, 0, time.UTC) + for _, e := range []struct{ name, body string }{ + {"clean.txt", "hardware inventory only\n"}, + {"host/resolv.conf", leakLog}, + } { + w, _ := zw.CreateHeader(&zip.FileHeader{Name: e.name, Method: zip.Deflate, Modified: mod}) + w.Write([]byte(e.body)) + } + zw.Close() + + res, err := Sanitize("bundle.zip", zb.Bytes()) + if err != nil { + t.Fatal(err) + } + + zr, err := zip.NewReader(bytes.NewReader(res.Data), int64(len(res.Data))) + if err != nil { + t.Fatalf("output zip invalid: %v", err) + } + got := map[string]string{} + for _, f := range zr.File { + if !f.Modified.Equal(mod) { + t.Fatalf("%s Modified changed: %v", f.Name, f.Modified) + } + rc, _ := f.Open() + b, _ := io.ReadAll(rc) + rc.Close() + got[f.Name] = string(b) + } + if got["clean.txt"] != "hardware inventory only\n" { + t.Fatalf("clean entry altered: %q", got["clean.txt"]) + } + scanClean(t, "zip-resolv", []byte(got["host/resolv.conf"])) +} + +func TestSanitize_BinaryMemberFlagged(t *testing.T) { + fru := append([]byte{0x01, 0x00, 0x00, 0x00}, []byte("corp.acme.ru\x00padding")...) + in := func() []byte { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + tw.WriteHeader(&tar.Header{Name: "onekeylog/FRU.bin", Mode: 0o644, Size: int64(len(fru)), Typeflag: tar.TypeReg}) + tw.Write(fru) + tw.Close() + return buf.Bytes() + }() + + res, err := Sanitize("d.tar", in) + if err != nil { + t.Fatal(err) + } + if len(res.SkippedBinary) != 1 || !strings.Contains(res.SkippedBinary[0], "FRU.bin") { + t.Fatalf("binary member not flagged: %+v", res.SkippedBinary) + } + if memberBytes(t, res.Data, "onekeylog/FRU.bin") != string(fru) { + t.Fatal("binary member was modified") + } +} diff --git a/internal/server/handlers.go b/internal/server/handlers.go index 87aa44c..9fbe7c4 100644 --- a/internal/server/handlers.go +++ b/internal/server/handlers.go @@ -28,6 +28,7 @@ import ( "git.mchus.pro/mchus/logpile/internal/ingest" "git.mchus.pro/mchus/logpile/internal/models" "git.mchus.pro/mchus/logpile/internal/parser" + "git.mchus.pro/mchus/logpile/internal/sanitize" chartviewer "reanimator/chart/viewer" ) @@ -1273,7 +1274,15 @@ func (s *Server) handleGetPrivacyScan(w http.ResponseWriter, r *http.Request) { jsonResponse(w, map[string]any{"loaded": false}) return } - jsonResponse(w, result.PrivacyScan) + jsonResponse(w, struct { + *models.PrivacyScan + Sanitizable bool `json:"sanitizable"` + }{result.PrivacyScan, s.canSanitizeCurrent()}) +} + +func (s *Server) canSanitizeCurrent() bool { + pkg := s.GetRawExport() + return pkg != nil && pkg.Source.Kind == "file_bytes" && sanitize.CanSanitize(pkg.Source.Filename) } func (s *Server) handleGetStatus(w http.ResponseWriter, r *http.Request) { @@ -1721,6 +1730,7 @@ func (s *Server) handleClear(w http.ResponseWriter, r *http.Request) { s.SetResult(nil) s.SetDetectedVendor("") s.SetRawExport(nil) + s.setSanitizeArtifact(nil) for _, artifact := range s.clearAllConvertArtifacts() { if strings.TrimSpace(artifact.Path) != "" { _ = os.Remove(artifact.Path) diff --git a/internal/server/sanitize_handlers.go b/internal/server/sanitize_handlers.go new file mode 100644 index 0000000..42a45b2 --- /dev/null +++ b/internal/server/sanitize_handlers.go @@ -0,0 +1,79 @@ +package server + +import ( + "encoding/base64" + "fmt" + "net/http" + "path/filepath" + "strconv" + + "git.mchus.pro/mchus/logpile/internal/sanitize" +) + +// sanitizeArtifact holds the de-identified copy of the current upload, ready for +// download. Kept in memory only, like convertOutput. +type sanitizeArtifact struct { + Data []byte + Filename string +} + +func (s *Server) setSanitizeArtifact(a *sanitizeArtifact) { + s.mu.Lock() + s.sanitizeResult = a + s.mu.Unlock() +} + +func (s *Server) getSanitizeArtifact() *sanitizeArtifact { + s.mu.RLock() + defer s.mu.RUnlock() + return s.sanitizeResult +} + +// handleSanitize builds a de-identified copy of the retained original upload and +// returns the change preview. The file itself is fetched from GET /api/sanitize/download. +func (s *Server) handleSanitize(w http.ResponseWriter, r *http.Request) { + pkg := s.GetRawExport() + if pkg == nil || pkg.Source.Kind != "file_bytes" { + jsonError(w, "sanitize is only available for an uploaded archive or log file", http.StatusUnprocessableEntity) + return + } + if !sanitize.CanSanitize(pkg.Source.Filename) { + jsonError(w, "this file format cannot be sanitized in place", http.StatusUnprocessableEntity) + return + } + data, err := base64.StdEncoding.DecodeString(pkg.Source.Data) + if err != nil { + jsonError(w, "cannot read the original file bytes", http.StatusInternalServerError) + return + } + + res, err := sanitize.Sanitize(pkg.Source.Filename, data) + if err != nil { + jsonError(w, "sanitize failed: "+err.Error(), http.StatusUnprocessableEntity) + return + } + + s.setSanitizeArtifact(&sanitizeArtifact{Data: res.Data, Filename: pkg.Source.Filename}) + + jsonResponse(w, map[string]any{ + "filename": filepath.Base(pkg.Source.Filename), + "input_size": len(data), + "output_size": len(res.Data), + "total_replaced": res.TotalReplaced, + "changes": res.Changes, + "skipped_binary": res.SkippedBinary, + }) +} + +func (s *Server) handleSanitizeDownload(w http.ResponseWriter, r *http.Request) { + art := s.getSanitizeArtifact() + if art == nil { + jsonError(w, "no sanitized file ready; run POST /api/sanitize first", http.StatusNotFound) + return + } + // Always octet-stream so the browser saves rather than renders it. + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filepath.Base(art.Filename))) + w.Header().Set("Content-Length", strconv.Itoa(len(art.Data))) + _, _ = w.Write(art.Data) +} diff --git a/internal/server/sanitize_handlers_test.go b/internal/server/sanitize_handlers_test.go new file mode 100644 index 0000000..d8ee40f --- /dev/null +++ b/internal/server/sanitize_handlers_test.go @@ -0,0 +1,114 @@ +package server + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.mchus.pro/mchus/logpile/internal/models" +) + +func leakDumpBytes(t *testing.T) []byte { + t.Helper() + var tarBuf bytes.Buffer + tw := tar.NewWriter(&tarBuf) + body := []byte("domain corp.acme.ru\nracdomain=corp.acme.ru\nTimeZone=Europe/Moscow\n") + tw.WriteHeader(&tar.Header{Name: "onekeylog/configuration/conf/resolv.conf", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg}) + tw.Write(body) + tw.Close() + var gz bytes.Buffer + gw := gzip.NewWriter(&gz) + gw.Name = "dump.tar" + gw.Write(tarBuf.Bytes()) + gw.Close() + return gz.Bytes() +} + +func serverWithUpload(t *testing.T, filename string, data []byte) *Server { + t.Helper() + s := &Server{} + s.SetResult(&models.AnalysisResult{Filename: filename}) + s.SetRawExport(&RawExportPackage{ + Source: RawExportSource{ + Kind: "file_bytes", + Filename: filename, + MIMEType: "application/gzip", + Encoding: "base64", + Data: base64.StdEncoding.EncodeToString(data), + }, + }) + return s +} + +func TestHandleSanitize_PreviewAndDownload(t *testing.T) { + s := serverWithUpload(t, "dump.tar.gz", leakDumpBytes(t)) + + rec := httptest.NewRecorder() + s.handleSanitize(rec, httptest.NewRequest("POST", "/api/sanitize", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("preview status %d: %s", rec.Code, rec.Body) + } + var preview struct { + TotalReplaced int `json:"total_replaced"` + Changes []struct { + Category string `json:"category"` + } `json:"changes"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &preview); err != nil { + t.Fatal(err) + } + if preview.TotalReplaced == 0 { + t.Fatal("nothing redacted") + } + + dl := httptest.NewRecorder() + s.handleSanitizeDownload(dl, httptest.NewRequest("GET", "/api/sanitize/download", nil)) + if dl.Code != http.StatusOK { + t.Fatalf("download status %d", dl.Code) + } + if cd := dl.Header().Get("Content-Disposition"); !strings.Contains(cd, `filename="dump.tar.gz"`) { + t.Fatalf("bad Content-Disposition: %q", cd) + } + gr, err := gzip.NewReader(bytes.NewReader(dl.Body.Bytes())) + if err != nil { + t.Fatalf("output not gzip: %v", err) + } + inner, _ := io.ReadAll(gr) + tr := tar.NewReader(bytes.NewReader(inner)) + h, err := tr.Next() + if err != nil || h.Name != "onekeylog/configuration/conf/resolv.conf" { + t.Fatalf("inner tar broken: %v %v", h, err) + } + clean, _ := io.ReadAll(tr) + if bytes.Contains(clean, []byte("acme.ru")) || bytes.Contains(clean, []byte("Europe/Moscow")) { + t.Fatalf("leak survived sanitize:\n%s", clean) + } +} + +func TestHandleSanitize_RejectsNonFileSource(t *testing.T) { + s := &Server{} + s.SetResult(&models.AnalysisResult{}) + s.SetRawExport(&RawExportPackage{Source: RawExportSource{Kind: "live_redfish"}}) + + rec := httptest.NewRecorder() + s.handleSanitize(rec, httptest.NewRequest("POST", "/api/sanitize", nil)) + if rec.Code != http.StatusUnprocessableEntity { + t.Fatalf("status %d, want 422", rec.Code) + } +} + +func TestHandleSanitizeDownload_NotReady(t *testing.T) { + s := &Server{} + rec := httptest.NewRecorder() + s.handleSanitizeDownload(rec, httptest.NewRequest("GET", "/api/sanitize/download", nil)) + if rec.Code != http.StatusNotFound { + t.Fatalf("status %d, want 404", rec.Code) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index fa95f9b..541f803 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -35,6 +35,7 @@ type Server struct { result *models.AnalysisResult detectedVendor string rawExport *RawExportPackage + sanitizeResult *sanitizeArtifact convertJobs map[string]struct{} convertOutput map[string]ConvertArtifact @@ -89,6 +90,8 @@ func (s *Server) setupRoutes() { s.mux.HandleFunc("GET /api/firmware", s.handleGetFirmware) s.mux.HandleFunc("GET /api/parse-errors", s.handleGetParseErrors) s.mux.HandleFunc("GET /api/privacy-scan", s.handleGetPrivacyScan) + s.mux.HandleFunc("POST /api/sanitize", s.handleSanitize) + s.mux.HandleFunc("GET /api/sanitize/download", s.handleSanitizeDownload) s.mux.HandleFunc("GET /api/export/csv", s.handleExportCSV) s.mux.HandleFunc("GET /api/export/json", s.handleExportJSON) s.mux.HandleFunc("GET /api/export/reanimator", s.handleExportReanimator) diff --git a/web/static/css/style.css b/web/static/css/style.css index 7e07187..ce08ffe 100644 --- a/web/static/css/style.css +++ b/web/static/css/style.css @@ -1046,3 +1046,53 @@ code { font-size: 0.82em; color: var(--muted); } + +.privacy-sanitize { + padding: 10px 12px; + border-bottom: 1px solid var(--border); +} + +.privacy-sanitize.hidden, +.privacy-sanitize-preview.hidden { + display: none; +} + +#privacy-sanitize-btn, +.privacy-sanitize-dl { + font-size: 0.85em; + padding: 5px 12px; + cursor: pointer; +} + +.privacy-sanitize-preview { + margin-top: 10px; + font-size: 0.88em; +} + +.privacy-sanitize-preview table { + margin: 6px 0; +} + +.privacy-sanitize-dl { + margin-top: 8px; + font-weight: 600; +} + +.privacy-sanitize-warn { + margin-top: 8px; + padding: 6px 8px; + background: #fff8f0; + border: 1px solid #f0e0c0; + border-radius: 4px; +} + +.privacy-sanitize-warn ul { + margin: 4px 0 0; + padding-left: 18px; + font-family: monospace; + font-size: 0.82em; +} + +.privacy-sanitize-err { + color: var(--crit-fg); +} diff --git a/web/static/js/app.js b/web/static/js/app.js index a4a5f77..ee9545f 100644 --- a/web/static/js/app.js +++ b/web/static/js/app.js @@ -1497,9 +1497,57 @@ async function loadPrivacyScan() { if (findingsBox) findingsBox.style.display = privacyCollapsed ? 'none' : ''; } + const sanBox = document.getElementById('privacy-sanitize'); + if (sanBox) { + sanBox.classList.toggle('hidden', !data.sanitizable); + const prev = document.getElementById('privacy-sanitize-preview'); + if (prev) { prev.classList.add('hidden'); prev.innerHTML = ''; } + const btn = document.getElementById('privacy-sanitize-btn'); + if (btn) { btn.disabled = false; btn.textContent = 'Обезличить и скачать копию'; } + } + section.classList.remove('hidden'); } +async function sanitizePreview() { + const btn = document.getElementById('privacy-sanitize-btn'); + const prev = document.getElementById('privacy-sanitize-preview'); + if (!btn || !prev) return; + btn.disabled = true; + btn.textContent = 'Обработка…'; + let data; + try { + const resp = await fetch('/api/sanitize', { method: 'POST' }); + data = await resp.json(); + if (!resp.ok) throw new Error(data && data.error ? data.error : 'sanitize failed'); + } catch (e) { + prev.innerHTML = `
${escapeHtml(String(e.message || e))}
`; + prev.classList.remove('hidden'); + btn.disabled = false; + btn.textContent = 'Обезличить и скачать копию'; + return; + } + + const changes = Array.isArray(data.changes) ? data.changes : []; + const skipped = Array.isArray(data.skipped_binary) ? data.skipped_binary : []; + let html = `

${data.total_replaced} замен в ${changes.length} файл(ах). ` + + `Формат, имена и даты записей сохранены; правки той же длины.

`; + if (changes.length) { + html += '' + + changes.map(c => ``).join('') + + '
КатегорияКол-воФайл
${escapeHtml(c.category)}${c.count}${escapeHtml(c.path)}
'; + } + if (skipped.length) { + html += `
Не удалось отредактировать (обезличьте вручную):
    ` + + skipped.map(s => `
  • ${escapeHtml(s)}
    • `).join('') + '
'; + } + html += ``; + prev.innerHTML = html; + prev.classList.remove('hidden'); + btn.disabled = false; + btn.textContent = 'Пересобрать'; +} + // The header + customer summary stay visible; only the findings table collapses. let privacyCollapsed = true; function togglePrivacy() { @@ -1640,6 +1688,10 @@ async function clearData() { if (privacyFindings) privacyFindings.style.display = 'none'; const privacyToggle = document.getElementById('privacy-toggle'); if (privacyToggle) { privacyToggle.textContent = '▼'; privacyToggle.style.visibility = ''; } + const privacySan = document.getElementById('privacy-sanitize'); + if (privacySan) privacySan.classList.add('hidden'); + const privacySanPrev = document.getElementById('privacy-sanitize-preview'); + if (privacySanPrev) { privacySanPrev.innerHTML = ''; privacySanPrev.classList.add('hidden'); } } catch (err) { console.error('Failed to clear data:', err); } diff --git a/web/templates/index.html b/web/templates/index.html index 89c4254..a193ea2 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -166,6 +166,10 @@
+