logpile no longer carries its own archive/vendor parsers or Redfish raw-payload replay/analysis engine. Both now live in git.mchus.pro/reanimator/parser/v13 (already used by the sibling `core` project), reached through parser.NewService().AnalyzeArchivePayload / AnalyzeRedfishRawPayloads. logpile keeps live BMC/Redfish collection, the DB, exporter, server and UI. - Deleted internal/parser/ (interface, registry, archive extraction, all vendor packages), internal/collector/redfish_replay*.go (replay engine, confirmed near-identical to the module's redfish/ package), internal/ingest/ (pure duplicate of the module's Service), internal/models/ (repointed every import to the module's models package instead), and internal/rar5 (only used by the deleted archive extractor). Removed the now-unused third_party/pciids submodule. - internal/collector/redfish.go: Collect() now calls parser.NewService().AnalyzeRedfishRawPayloads(...) at the one place it used to call the local ReplayRedfishFromRawPayloads (an existing "unified tunnel" between live collection and raw import). A deadcode-driven pass then removed ~150 doc-to-model conversion helpers in redfish.go that duplicated the module's redfish/ package and were already unreachable before this change; kept the small generic dedup/scoring helpers the live crawler still calls. - internal/collector/redfishprofile/ stays whole: Collect() depends on it directly for acquisition-time tuning. Its analysis-only methods (ApplyAnalysisProfiles/PostAnalyze) are now dead but left in place rather than splitting the package's single Profile interface. - pciids moved with the module (now a public package there); logpile's live collector and exporter import it from there instead of keeping a local copy. - go.mod: new require on git.mchus.pro/reanimator/parser/v13 v13.0.0 (plain module dependency, not a submodule like reanimator/chart, since this module is independently versioned and shared with core). - bible-local docs rewritten for the new boundary; see ADL-080. See also the companion change in git.mchus.pro/reanimator/parser (tagged v13.0.0): new top-level exports (IsSupportedArchiveFilename/SupportedArchiveExtensions/ ParseInDefaultArchiveLocation/PrivacyScanEnabled) and pciids moved to a public package, both needed to complete this migration. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
188 lines
5.0 KiB
Go
188 lines
5.0 KiB
Go
package privacy
|
|
|
|
import (
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"git.mchus.pro/reanimator/parser/v13/models"
|
|
)
|
|
|
|
// A dotted quad, permissive about leading zeros so it also matches the
|
|
// zero-padded fillers ("000.00.00.0") that reIPv4 deliberately rejects.
|
|
var reLooseQuad = regexp.MustCompile(`\b\d{1,3}(?:\.\d{1,3}){3}\b`)
|
|
|
|
// SanitizationMarker is one redaction-filler token found sitting where a real
|
|
// value (hostname, IP, e-mail, timezone) would be. It is the fingerprint left
|
|
// by internal/sanitize.
|
|
type SanitizationMarker struct {
|
|
Value string
|
|
Strong bool
|
|
}
|
|
|
|
// Timezone names internal/sanitize substitutes for a real zone. Only the
|
|
// implausible ones count - a BMC genuinely in Antarctica, or an operator who
|
|
// deliberately wrote "UTC", must not read as "sanitized".
|
|
var decoyZones = map[string]struct{}{
|
|
"antarctica/troll": {}, "antarctica/vostok": {}, "antarctica/mcmurdo": {},
|
|
"pacific/tarawa": {}, "pacific/guadalcanal": {}, "pacific/bougainville": {},
|
|
"atlantic/azores": {}, "etc/universal": {}, "us/pacific": {},
|
|
"brazil/east": {}, "canada/yukon": {},
|
|
}
|
|
|
|
// SanitizationMarkers returns the redaction-filler tokens on one line. Robust to
|
|
// future filler characters: it recognises "a hostname / IP / e-mail / timezone
|
|
// slot filled entirely with a single repeated placeholder plus separators",
|
|
// not the literal string "x".
|
|
func SanitizationMarkers(line string) []SanitizationMarker {
|
|
var out []SanitizationMarker
|
|
seen := map[string]struct{}{}
|
|
push := func(v string, strong bool) {
|
|
v = strings.Trim(strings.TrimSpace(v), `"',;:`)
|
|
if v == "" {
|
|
return
|
|
}
|
|
if _, ok := seen[v]; ok {
|
|
return
|
|
}
|
|
seen[v] = struct{}{}
|
|
out = append(out, SanitizationMarker{Value: v, Strong: strong})
|
|
}
|
|
|
|
// A filler in a known config-key value position.
|
|
for _, r := range tableRules {
|
|
switch r.category {
|
|
case catResolv, catADLDAP, catNSUpdate, catTimezone, catCollector, catDHCP:
|
|
default:
|
|
continue
|
|
}
|
|
for _, m := range r.re.FindAllStringSubmatch(line, -1) {
|
|
if r.group >= len(m) {
|
|
continue
|
|
}
|
|
val := m[r.group]
|
|
if isGenericFiller(val) || isDecoyZone(val) {
|
|
strong := r.category == catResolv || r.category == catADLDAP || r.category == catNSUpdate
|
|
push(val, strong && isGenericFiller(val))
|
|
}
|
|
}
|
|
}
|
|
// A bare FQDN / e-mail made entirely of filler.
|
|
for _, v := range reFQDN.FindAllString(line, -1) {
|
|
if isGenericFiller(v) && strings.Contains(v, ".") {
|
|
push(v, strings.Count(v, ".") >= 2 && len(v) >= 8)
|
|
}
|
|
}
|
|
for _, v := range reEmail.FindAllString(line, -1) {
|
|
if isGenericFiller(v) {
|
|
push(v, true)
|
|
}
|
|
}
|
|
// A zero-filled IP ("000.00.00.0"); "0.0.0.0" is a real value, not a marker.
|
|
for _, v := range reLooseQuad.FindAllString(line, -1) {
|
|
if isGenericFiller(v) {
|
|
push(v, true)
|
|
}
|
|
}
|
|
// A decoy timezone name anywhere on the line.
|
|
for _, v := range reTZName.FindAllString(line, -1) {
|
|
if isDecoyZone(v) {
|
|
push(v, false)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func isDecoyZone(s string) bool {
|
|
_, ok := decoyZones[strings.ToLower(strings.TrimSpace(s))]
|
|
return ok
|
|
}
|
|
|
|
// isGenericFiller reports whether s is a value slot filled with one repeated
|
|
// placeholder character. Letter fills ("xxxxx.xxxx.xx") always qualify; digit
|
|
// fills qualify only in the IP shape with a wide group ("00.000.000.00"), so
|
|
// "0.0.0.0" and "000" do not.
|
|
func isGenericFiller(s string) bool {
|
|
parts := strings.FieldsFunc(s, func(r rune) bool {
|
|
return r == '.' || r == '-' || r == '_' || r == '@' || r == ':' || r == '/' || r == '+'
|
|
})
|
|
if len(parts) < 2 {
|
|
return false
|
|
}
|
|
var fill rune
|
|
total, wide := 0, false
|
|
for _, p := range parts {
|
|
for _, r := range p {
|
|
if !isAlnum(r) {
|
|
return false
|
|
}
|
|
if fill == 0 {
|
|
fill = r
|
|
} else if r != fill {
|
|
return false
|
|
}
|
|
}
|
|
total += len(p)
|
|
if len(p) >= 2 {
|
|
wide = true
|
|
}
|
|
}
|
|
if total < 4 {
|
|
return false
|
|
}
|
|
if fill >= '0' && fill <= '9' {
|
|
return len(parts) == 4 && wide // IP shape with a redacted (0-padded) octet
|
|
}
|
|
return true
|
|
}
|
|
|
|
// sanTally accumulates markers while Scan walks the files.
|
|
type sanTally struct {
|
|
strong, total int
|
|
files map[string]struct{}
|
|
seen map[string]struct{}
|
|
evidence []string
|
|
}
|
|
|
|
func newSanTally() *sanTally {
|
|
return &sanTally{files: map[string]struct{}{}, seen: map[string]struct{}{}}
|
|
}
|
|
|
|
func (t *sanTally) add(path string, ln int, line string, m SanitizationMarker) {
|
|
key := m.Value + "|" + path
|
|
if _, ok := t.seen[key]; ok {
|
|
return
|
|
}
|
|
t.seen[key] = struct{}{}
|
|
t.total++
|
|
if m.Strong {
|
|
t.strong++
|
|
}
|
|
if path != "" {
|
|
t.files[path] = struct{}{}
|
|
}
|
|
if len(t.evidence) < 3 {
|
|
loc := path
|
|
if ln > 0 {
|
|
loc += ":" + strconv.Itoa(ln)
|
|
}
|
|
t.evidence = append(t.evidence, loc+": "+excerpt(line))
|
|
}
|
|
}
|
|
|
|
func (t *sanTally) report() *models.SanitizedReport {
|
|
if t.total == 0 {
|
|
return nil
|
|
}
|
|
// One stray filler-looking token proves nothing (coincidence, or a partial
|
|
// future redaction pass). Require corroboration.
|
|
detected := t.strong >= 2 || (t.strong >= 1 && t.total >= 3) || t.total >= 4
|
|
return &models.SanitizedReport{
|
|
Detected: detected,
|
|
Markers: t.total,
|
|
Strong: t.strong,
|
|
Files: len(t.files),
|
|
Evidence: t.evidence,
|
|
}
|
|
}
|