Files
logpile/internal/privacy/customer.go
T
Mikhail ChusavitinandClaude Sonnet 5 475318fc2b refactor: extract parsers and Redfish replay/analysis to external module
logpile no longer carries its own archive/vendor parsers or Redfish
raw-payload replay/analysis engine. Both now live in
git.mchus.pro/reanimator/parser/v13 (already used by the sibling `core`
project), reached through parser.NewService().AnalyzeArchivePayload /
AnalyzeRedfishRawPayloads. logpile keeps live BMC/Redfish collection,
the DB, exporter, server and UI.

- Deleted internal/parser/ (interface, registry, archive extraction,
  all vendor packages), internal/collector/redfish_replay*.go (replay
  engine, confirmed near-identical to the module's redfish/ package),
  internal/ingest/ (pure duplicate of the module's Service),
  internal/models/ (repointed every import to the module's models
  package instead), and internal/rar5 (only used by the deleted
  archive extractor). Removed the now-unused third_party/pciids
  submodule.
- internal/collector/redfish.go: Collect() now calls
  parser.NewService().AnalyzeRedfishRawPayloads(...) at the one place
  it used to call the local ReplayRedfishFromRawPayloads (an existing
  "unified tunnel" between live collection and raw import). A
  deadcode-driven pass then removed ~150 doc-to-model conversion
  helpers in redfish.go that duplicated the module's redfish/ package
  and were already unreachable before this change; kept the small
  generic dedup/scoring helpers the live crawler still calls.
- internal/collector/redfishprofile/ stays whole: Collect() depends on
  it directly for acquisition-time tuning. Its analysis-only methods
  (ApplyAnalysisProfiles/PostAnalyze) are now dead but left in place
  rather than splitting the package's single Profile interface.
- pciids moved with the module (now a public package there); logpile's
  live collector and exporter import it from there instead of keeping
  a local copy.
- go.mod: new require on git.mchus.pro/reanimator/parser/v13 v13.0.0
  (plain module dependency, not a submodule like reanimator/chart,
  since this module is independently versioned and shared with core).
- bible-local docs rewritten for the new boundary; see ADL-080.

See also the companion change in git.mchus.pro/reanimator/parser
(tagged v13.0.0): new top-level exports
(IsSupportedArchiveFilename/SupportedArchiveExtensions/
ParseInDefaultArchiveLocation/PrivacyScanEnabled) and pciids moved to
a public package, both needed to complete this migration.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-08 12:59:39 +03:00

183 lines
4.3 KiB
Go

package privacy
import (
"sort"
"strconv"
"strings"
"git.mchus.pro/reanimator/parser/v13/models"
)
// Multi-label public suffixes handled explicitly; every other final label is
// treated as the suffix. This is a heuristic, not a full PSL - good enough to
// collapse "sn-x.mgmt.corp.example.co.uk" to "example.co.uk".
var multiLabelSuffixes = []string{
"co.uk", "org.uk", "gov.uk", "ac.uk", "me.uk",
"com.cn", "net.cn", "org.cn", "gov.cn", "edu.cn",
"com.au", "net.au", "org.au", "co.jp", "co.kr", "com.br", "com.tr", "com.ua",
}
// Categories whose findings feed the customer guess, and whether a hit in that
// category is strong evidence on its own.
var customerCategories = map[string]bool{
catDomain: false,
catResolv: true,
catADLDAP: true,
catCert: true,
catNSUpdate: true,
catMgmtSubdomain: true,
catDHCP: false,
catEmail: false,
}
type customerTally struct {
domain string
hits int
paths map[string]struct{}
strong bool
evidence []string
evidenceSeen map[string]struct{}
}
func guessCustomers(findings []models.PrivacyFinding) []models.CustomerGuess {
tallies := map[string]*customerTally{}
for _, f := range findings {
strongCat, ok := customerCategories[f.Category]
if !ok {
continue
}
host := f.Match
if f.Category == catEmail {
if at := strings.IndexByte(host, '@'); at >= 0 {
host = host[at+1:]
}
}
reg := registrableDomain(host)
if reg == "" || isAllowlistedDomain(reg) {
continue
}
t := tallies[reg]
if t == nil {
t = &customerTally{domain: reg, paths: map[string]struct{}{}, evidenceSeen: map[string]struct{}{}}
tallies[reg] = t
}
t.hits++
if f.Path != "" {
t.paths[f.Path] = struct{}{}
}
if strongCat {
t.strong = true
}
if len(t.evidence) < 3 && f.Excerpt != "" {
line := evidenceLine(f)
if _, dup := t.evidenceSeen[line]; !dup {
t.evidenceSeen[line] = struct{}{}
t.evidence = append(t.evidence, line)
}
}
}
guesses := make([]models.CustomerGuess, 0, len(tallies))
for _, t := range tallies {
guesses = append(guesses, models.CustomerGuess{
Domain: t.domain,
Confidence: confidence(t),
Hits: t.hits,
Evidence: t.evidence,
})
}
sort.Slice(guesses, func(i, j int) bool {
si, sj := score(tallies[guesses[i].Domain]), score(tallies[guesses[j].Domain])
if si != sj {
return si > sj
}
return guesses[i].Domain < guesses[j].Domain
})
// Drop weak single-hit low-confidence candidates entirely - a "could not
// identify" is more useful than guessing a driver-comment or kernel-source
// domain (nvidia.com, linux.it, ...).
out := make([]models.CustomerGuess, 0, 3)
for _, g := range guesses {
if len(out) >= 3 {
break
}
if g.Hits < 2 && g.Confidence == "low" {
continue
}
out = append(out, g)
}
return out
}
func score(t *customerTally) int {
s := t.hits + 3*len(t.paths)
if t.strong {
s += 10
}
return s
}
func confidence(t *customerTally) string {
switch {
case t.strong && len(t.paths) >= 2:
return "high"
case t.strong || len(t.paths) >= 2:
return "medium"
default:
return "low"
}
}
func evidenceLine(f models.PrivacyFinding) string {
loc := f.Path
if f.Line > 0 {
loc += ":" + strconv.Itoa(f.Line)
}
return loc + ": " + f.Excerpt
}
// registrableDomain returns the registrable ("eTLD+1") portion of a host name,
// or "" if host is an IP, has no dot, or is a bare TLD.
func registrableDomain(host string) string {
host = strings.ToLower(strings.Trim(strings.TrimSpace(host), ".*"))
host = strings.TrimPrefix(host, "*.")
if host == "" || strings.ContainsAny(host, ":/ ") {
return ""
}
if at := strings.IndexByte(host, '@'); at >= 0 {
host = host[at+1:]
}
if !plausibleFQDN(host) {
return ""
}
if strings.Count(host, ".") == 3 {
allDigits := true
for _, r := range host {
if r != '.' && (r < '0' || r > '9') {
allDigits = false
break
}
}
if allDigits {
return ""
}
}
labels := strings.Split(host, ".")
if len(labels) < 2 {
return ""
}
for _, suf := range multiLabelSuffixes {
if strings.HasSuffix(host, "."+suf) {
sufLabels := strings.Count(suf, ".") + 1
if len(labels) > sufLabels {
return strings.Join(labels[len(labels)-sufLabels-1:], ".")
}
return ""
}
}
return strings.Join(labels[len(labels)-2:], ".")
}