feat(privacy): detect an already-sanitized source
Scan now reports PrivacyScan.Sanitized {detected, markers, strong, files,
evidence}. SanitizationMarkers recognises a value slot filled with one
repeated placeholder + separators (xxxxx.xxxx.xx, x@xxxx.xxxx.xx,
000.00.00.0, a decoy timezone) - it matches the shape, not the literal "x",
so evolving the redaction mechanism still trips it.
detected requires corroboration: strong>=2, or strong>=1 && markers>=3, or
markers>=4. A single filler-looking token is reported (markers:1) but never
asserted as sanitized, so a partial future pass or a coincidence does not
read as "done". 0.0.0.0 / 000 / UTC / Etc/UTC are too plausibly intentional
and do not count.
UI: the Customer-data panel shows "файл уже обезличен" and hides the
sanitize button when detected.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jDYM1nnoZZ3vFz23DDaV1
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
a63bb17438
commit
f38fb2de69
@@ -0,0 +1,187 @@
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
)
|
||||
|
||||
// A dotted quad, permissive about leading zeros so it also matches the
|
||||
// zero-padded fillers ("000.00.00.0") that reIPv4 deliberately rejects.
|
||||
var reLooseQuad = regexp.MustCompile(`\b\d{1,3}(?:\.\d{1,3}){3}\b`)
|
||||
|
||||
// SanitizationMarker is one redaction-filler token found sitting where a real
|
||||
// value (hostname, IP, e-mail, timezone) would be. It is the fingerprint left
|
||||
// by internal/sanitize.
|
||||
type SanitizationMarker struct {
|
||||
Value string
|
||||
Strong bool
|
||||
}
|
||||
|
||||
// Timezone names internal/sanitize substitutes for a real zone. Only the
|
||||
// implausible ones count - a BMC genuinely in Antarctica, or an operator who
|
||||
// deliberately wrote "UTC", must not read as "sanitized".
|
||||
var decoyZones = map[string]struct{}{
|
||||
"antarctica/troll": {}, "antarctica/vostok": {}, "antarctica/mcmurdo": {},
|
||||
"pacific/tarawa": {}, "pacific/guadalcanal": {}, "pacific/bougainville": {},
|
||||
"atlantic/azores": {}, "etc/universal": {}, "us/pacific": {},
|
||||
"brazil/east": {}, "canada/yukon": {},
|
||||
}
|
||||
|
||||
// SanitizationMarkers returns the redaction-filler tokens on one line. Robust to
|
||||
// future filler characters: it recognises "a hostname / IP / e-mail / timezone
|
||||
// slot filled entirely with a single repeated placeholder plus separators",
|
||||
// not the literal string "x".
|
||||
func SanitizationMarkers(line string) []SanitizationMarker {
|
||||
var out []SanitizationMarker
|
||||
seen := map[string]struct{}{}
|
||||
push := func(v string, strong bool) {
|
||||
v = strings.Trim(strings.TrimSpace(v), `"',;:`)
|
||||
if v == "" {
|
||||
return
|
||||
}
|
||||
if _, ok := seen[v]; ok {
|
||||
return
|
||||
}
|
||||
seen[v] = struct{}{}
|
||||
out = append(out, SanitizationMarker{Value: v, Strong: strong})
|
||||
}
|
||||
|
||||
// A filler in a known config-key value position.
|
||||
for _, r := range tableRules {
|
||||
switch r.category {
|
||||
case catResolv, catADLDAP, catNSUpdate, catTimezone, catCollector, catDHCP:
|
||||
default:
|
||||
continue
|
||||
}
|
||||
for _, m := range r.re.FindAllStringSubmatch(line, -1) {
|
||||
if r.group >= len(m) {
|
||||
continue
|
||||
}
|
||||
val := m[r.group]
|
||||
if isGenericFiller(val) || isDecoyZone(val) {
|
||||
strong := r.category == catResolv || r.category == catADLDAP || r.category == catNSUpdate
|
||||
push(val, strong && isGenericFiller(val))
|
||||
}
|
||||
}
|
||||
}
|
||||
// A bare FQDN / e-mail made entirely of filler.
|
||||
for _, v := range reFQDN.FindAllString(line, -1) {
|
||||
if isGenericFiller(v) && strings.Contains(v, ".") {
|
||||
push(v, strings.Count(v, ".") >= 2 && len(v) >= 8)
|
||||
}
|
||||
}
|
||||
for _, v := range reEmail.FindAllString(line, -1) {
|
||||
if isGenericFiller(v) {
|
||||
push(v, true)
|
||||
}
|
||||
}
|
||||
// A zero-filled IP ("000.00.00.0"); "0.0.0.0" is a real value, not a marker.
|
||||
for _, v := range reLooseQuad.FindAllString(line, -1) {
|
||||
if isGenericFiller(v) {
|
||||
push(v, true)
|
||||
}
|
||||
}
|
||||
// A decoy timezone name anywhere on the line.
|
||||
for _, v := range reTZName.FindAllString(line, -1) {
|
||||
if isDecoyZone(v) {
|
||||
push(v, false)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isDecoyZone(s string) bool {
|
||||
_, ok := decoyZones[strings.ToLower(strings.TrimSpace(s))]
|
||||
return ok
|
||||
}
|
||||
|
||||
// isGenericFiller reports whether s is a value slot filled with one repeated
|
||||
// placeholder character. Letter fills ("xxxxx.xxxx.xx") always qualify; digit
|
||||
// fills qualify only in the IP shape with a wide group ("00.000.000.00"), so
|
||||
// "0.0.0.0" and "000" do not.
|
||||
func isGenericFiller(s string) bool {
|
||||
parts := strings.FieldsFunc(s, func(r rune) bool {
|
||||
return r == '.' || r == '-' || r == '_' || r == '@' || r == ':' || r == '/' || r == '+'
|
||||
})
|
||||
if len(parts) < 2 {
|
||||
return false
|
||||
}
|
||||
var fill rune
|
||||
total, wide := 0, false
|
||||
for _, p := range parts {
|
||||
for _, r := range p {
|
||||
if !isAlnum(r) {
|
||||
return false
|
||||
}
|
||||
if fill == 0 {
|
||||
fill = r
|
||||
} else if r != fill {
|
||||
return false
|
||||
}
|
||||
}
|
||||
total += len(p)
|
||||
if len(p) >= 2 {
|
||||
wide = true
|
||||
}
|
||||
}
|
||||
if total < 4 {
|
||||
return false
|
||||
}
|
||||
if fill >= '0' && fill <= '9' {
|
||||
return len(parts) == 4 && wide // IP shape with a redacted (0-padded) octet
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// sanTally accumulates markers while Scan walks the files.
|
||||
type sanTally struct {
|
||||
strong, total int
|
||||
files map[string]struct{}
|
||||
seen map[string]struct{}
|
||||
evidence []string
|
||||
}
|
||||
|
||||
func newSanTally() *sanTally {
|
||||
return &sanTally{files: map[string]struct{}{}, seen: map[string]struct{}{}}
|
||||
}
|
||||
|
||||
func (t *sanTally) add(path string, ln int, line string, m SanitizationMarker) {
|
||||
key := m.Value + "|" + path
|
||||
if _, ok := t.seen[key]; ok {
|
||||
return
|
||||
}
|
||||
t.seen[key] = struct{}{}
|
||||
t.total++
|
||||
if m.Strong {
|
||||
t.strong++
|
||||
}
|
||||
if path != "" {
|
||||
t.files[path] = struct{}{}
|
||||
}
|
||||
if len(t.evidence) < 3 {
|
||||
loc := path
|
||||
if ln > 0 {
|
||||
loc += ":" + strconv.Itoa(ln)
|
||||
}
|
||||
t.evidence = append(t.evidence, loc+": "+excerpt(line))
|
||||
}
|
||||
}
|
||||
|
||||
func (t *sanTally) report() *models.SanitizedReport {
|
||||
if t.total == 0 {
|
||||
return nil
|
||||
}
|
||||
// One stray filler-looking token proves nothing (coincidence, or a partial
|
||||
// future redaction pass). Require corroboration.
|
||||
detected := t.strong >= 2 || (t.strong >= 1 && t.total >= 3) || t.total >= 4
|
||||
return &models.SanitizedReport{
|
||||
Detected: detected,
|
||||
Markers: t.total,
|
||||
Strong: t.strong,
|
||||
Files: len(t.files),
|
||||
Evidence: t.evidence,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user