Files
logpile/internal/privacy/sanitized.go
T
Mikhail ChusavitinandClaude Sonnet 5 475318fc2b refactor: extract parsers and Redfish replay/analysis to external module
logpile no longer carries its own archive/vendor parsers or Redfish
raw-payload replay/analysis engine. Both now live in
git.mchus.pro/reanimator/parser/v13 (already used by the sibling `core`
project), reached through parser.NewService().AnalyzeArchivePayload /
AnalyzeRedfishRawPayloads. logpile keeps live BMC/Redfish collection,
the DB, exporter, server and UI.

- Deleted internal/parser/ (interface, registry, archive extraction,
  all vendor packages), internal/collector/redfish_replay*.go (replay
  engine, confirmed near-identical to the module's redfish/ package),
  internal/ingest/ (pure duplicate of the module's Service),
  internal/models/ (repointed every import to the module's models
  package instead), and internal/rar5 (only used by the deleted
  archive extractor). Removed the now-unused third_party/pciids
  submodule.
- internal/collector/redfish.go: Collect() now calls
  parser.NewService().AnalyzeRedfishRawPayloads(...) at the one place
  it used to call the local ReplayRedfishFromRawPayloads (an existing
  "unified tunnel" between live collection and raw import). A
  deadcode-driven pass then removed ~150 doc-to-model conversion
  helpers in redfish.go that duplicated the module's redfish/ package
  and were already unreachable before this change; kept the small
  generic dedup/scoring helpers the live crawler still calls.
- internal/collector/redfishprofile/ stays whole: Collect() depends on
  it directly for acquisition-time tuning. Its analysis-only methods
  (ApplyAnalysisProfiles/PostAnalyze) are now dead but left in place
  rather than splitting the package's single Profile interface.
- pciids moved with the module (now a public package there); logpile's
  live collector and exporter import it from there instead of keeping
  a local copy.
- go.mod: new require on git.mchus.pro/reanimator/parser/v13 v13.0.0
  (plain module dependency, not a submodule like reanimator/chart,
  since this module is independently versioned and shared with core).
- bible-local docs rewritten for the new boundary; see ADL-080.

See also the companion change in git.mchus.pro/reanimator/parser
(tagged v13.0.0): new top-level exports
(IsSupportedArchiveFilename/SupportedArchiveExtensions/
ParseInDefaultArchiveLocation/PrivacyScanEnabled) and pciids moved to
a public package, both needed to complete this migration.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-08 12:59:39 +03:00

188 lines
5.0 KiB
Go

package privacy
import (
"regexp"
"strconv"
"strings"
"git.mchus.pro/reanimator/parser/v13/models"
)
// A dotted quad, permissive about leading zeros so it also matches the
// zero-padded fillers ("000.00.00.0") that reIPv4 deliberately rejects.
var reLooseQuad = regexp.MustCompile(`\b\d{1,3}(?:\.\d{1,3}){3}\b`)
// SanitizationMarker is one redaction-filler token found sitting where a real
// value (hostname, IP, e-mail, timezone) would be. It is the fingerprint left
// by internal/sanitize.
type SanitizationMarker struct {
Value string
Strong bool
}
// Timezone names internal/sanitize substitutes for a real zone. Only the
// implausible ones count - a BMC genuinely in Antarctica, or an operator who
// deliberately wrote "UTC", must not read as "sanitized".
var decoyZones = map[string]struct{}{
"antarctica/troll": {}, "antarctica/vostok": {}, "antarctica/mcmurdo": {},
"pacific/tarawa": {}, "pacific/guadalcanal": {}, "pacific/bougainville": {},
"atlantic/azores": {}, "etc/universal": {}, "us/pacific": {},
"brazil/east": {}, "canada/yukon": {},
}
// SanitizationMarkers returns the redaction-filler tokens on one line. Robust to
// future filler characters: it recognises "a hostname / IP / e-mail / timezone
// slot filled entirely with a single repeated placeholder plus separators",
// not the literal string "x".
func SanitizationMarkers(line string) []SanitizationMarker {
var out []SanitizationMarker
seen := map[string]struct{}{}
push := func(v string, strong bool) {
v = strings.Trim(strings.TrimSpace(v), `"',;:`)
if v == "" {
return
}
if _, ok := seen[v]; ok {
return
}
seen[v] = struct{}{}
out = append(out, SanitizationMarker{Value: v, Strong: strong})
}
// A filler in a known config-key value position.
for _, r := range tableRules {
switch r.category {
case catResolv, catADLDAP, catNSUpdate, catTimezone, catCollector, catDHCP:
default:
continue
}
for _, m := range r.re.FindAllStringSubmatch(line, -1) {
if r.group >= len(m) {
continue
}
val := m[r.group]
if isGenericFiller(val) || isDecoyZone(val) {
strong := r.category == catResolv || r.category == catADLDAP || r.category == catNSUpdate
push(val, strong && isGenericFiller(val))
}
}
}
// A bare FQDN / e-mail made entirely of filler.
for _, v := range reFQDN.FindAllString(line, -1) {
if isGenericFiller(v) && strings.Contains(v, ".") {
push(v, strings.Count(v, ".") >= 2 && len(v) >= 8)
}
}
for _, v := range reEmail.FindAllString(line, -1) {
if isGenericFiller(v) {
push(v, true)
}
}
// A zero-filled IP ("000.00.00.0"); "0.0.0.0" is a real value, not a marker.
for _, v := range reLooseQuad.FindAllString(line, -1) {
if isGenericFiller(v) {
push(v, true)
}
}
// A decoy timezone name anywhere on the line.
for _, v := range reTZName.FindAllString(line, -1) {
if isDecoyZone(v) {
push(v, false)
}
}
return out
}
func isDecoyZone(s string) bool {
_, ok := decoyZones[strings.ToLower(strings.TrimSpace(s))]
return ok
}
// isGenericFiller reports whether s is a value slot filled with one repeated
// placeholder character. Letter fills ("xxxxx.xxxx.xx") always qualify; digit
// fills qualify only in the IP shape with a wide group ("00.000.000.00"), so
// "0.0.0.0" and "000" do not.
func isGenericFiller(s string) bool {
parts := strings.FieldsFunc(s, func(r rune) bool {
return r == '.' || r == '-' || r == '_' || r == '@' || r == ':' || r == '/' || r == '+'
})
if len(parts) < 2 {
return false
}
var fill rune
total, wide := 0, false
for _, p := range parts {
for _, r := range p {
if !isAlnum(r) {
return false
}
if fill == 0 {
fill = r
} else if r != fill {
return false
}
}
total += len(p)
if len(p) >= 2 {
wide = true
}
}
if total < 4 {
return false
}
if fill >= '0' && fill <= '9' {
return len(parts) == 4 && wide // IP shape with a redacted (0-padded) octet
}
return true
}
// sanTally accumulates markers while Scan walks the files.
type sanTally struct {
strong, total int
files map[string]struct{}
seen map[string]struct{}
evidence []string
}
func newSanTally() *sanTally {
return &sanTally{files: map[string]struct{}{}, seen: map[string]struct{}{}}
}
func (t *sanTally) add(path string, ln int, line string, m SanitizationMarker) {
key := m.Value + "|" + path
if _, ok := t.seen[key]; ok {
return
}
t.seen[key] = struct{}{}
t.total++
if m.Strong {
t.strong++
}
if path != "" {
t.files[path] = struct{}{}
}
if len(t.evidence) < 3 {
loc := path
if ln > 0 {
loc += ":" + strconv.Itoa(ln)
}
t.evidence = append(t.evidence, loc+": "+excerpt(line))
}
}
func (t *sanTally) report() *models.SanitizedReport {
if t.total == 0 {
return nil
}
// One stray filler-looking token proves nothing (coincidence, or a partial
// future redaction pass). Require corroboration.
detected := t.strong >= 2 || (t.strong >= 1 && t.total >= 3) || t.total >= 4
return &models.SanitizedReport{
Detected: detected,
Markers: t.total,
Strong: t.strong,
Files: len(t.files),
Evidence: t.evidence,
}
}