Files
logpile/internal/privacy/privacy.go
T
Mikhail ChusavitinandClaude Sonnet 5 475318fc2b refactor: extract parsers and Redfish replay/analysis to external module
logpile no longer carries its own archive/vendor parsers or Redfish
raw-payload replay/analysis engine. Both now live in
git.mchus.pro/reanimator/parser/v13 (already used by the sibling `core`
project), reached through parser.NewService().AnalyzeArchivePayload /
AnalyzeRedfishRawPayloads. logpile keeps live BMC/Redfish collection,
the DB, exporter, server and UI.

- Deleted internal/parser/ (interface, registry, archive extraction,
  all vendor packages), internal/collector/redfish_replay*.go (replay
  engine, confirmed near-identical to the module's redfish/ package),
  internal/ingest/ (pure duplicate of the module's Service),
  internal/models/ (repointed every import to the module's models
  package instead), and internal/rar5 (only used by the deleted
  archive extractor). Removed the now-unused third_party/pciids
  submodule.
- internal/collector/redfish.go: Collect() now calls
  parser.NewService().AnalyzeRedfishRawPayloads(...) at the one place
  it used to call the local ReplayRedfishFromRawPayloads (an existing
  "unified tunnel" between live collection and raw import). A
  deadcode-driven pass then removed ~150 doc-to-model conversion
  helpers in redfish.go that duplicated the module's redfish/ package
  and were already unreachable before this change; kept the small
  generic dedup/scoring helpers the live crawler still calls.
- internal/collector/redfishprofile/ stays whole: Collect() depends on
  it directly for acquisition-time tuning. Its analysis-only methods
  (ApplyAnalysisProfiles/PostAnalyze) are now dead but left in place
  rather than splitting the package's single Profile interface.
- pciids moved with the module (now a public package there); logpile's
  live collector and exporter import it from there instead of keeping
  a local copy.
- go.mod: new require on git.mchus.pro/reanimator/parser/v13 v13.0.0
  (plain module dependency, not a submodule like reanimator/chart,
  since this module is independently versioned and shared with core).
- bible-local docs rewritten for the new boundary; see ADL-080.

See also the companion change in git.mchus.pro/reanimator/parser
(tagged v13.0.0): new top-level exports
(IsSupportedArchiveFilename/SupportedArchiveExtensions/
ParseInDefaultArchiveLocation/PrivacyScanEnabled) and pciids moved to
a public package, both needed to complete this migration.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-08 12:59:39 +03:00

181 lines
3.9 KiB
Go

// Package privacy scans ingested source files for customer-identifying and
// site-identifying data: DNS suffix, AD domain, NTP/DNS/syslog host names,
// timezone, e-mail, public IPs, TLS cert CN/SAN, FRU location fields.
//
// Detection only. Nothing here modifies the source. The rule catalogue is a
// port of the KB grep playbook; see bible-local/docs/privacy-scan.md.
package privacy
import (
"bufio"
"bytes"
"sort"
"strings"
"unicode/utf8"
"git.mchus.pro/reanimator/parser/v13/models"
)
// File is one unit of scanned content (an extracted archive member, or a
// serialized Redfish tree).
type File struct {
Path string
Content []byte
}
const (
maxFindings = 1000
maxExcerptRunes = 200
maxScanLineBytes = 8192
)
// Scan runs every rule over every text file and returns the aggregated report.
// Returns nil when there is nothing to scan (no text files).
func Scan(files []File) *models.PrivacyScan {
scanned := 0
seen := make(map[string]struct{})
var findings []models.PrivacyFinding
emit := func(f models.PrivacyFinding) {
if len(findings) >= maxFindings {
return
}
key := f.Category + "|" + f.Match + "|" + f.Path
if _, ok := seen[key]; ok {
return
}
seen[key] = struct{}{}
findings = append(findings, f)
}
san := newSanTally()
for _, file := range files {
if isAllowlistedFilename(file.Path) {
continue
}
if !looksLikeText(file.Content) {
continue
}
scanned++
scanFileLines(file, emit, san)
}
if scanned == 0 {
return nil
}
sortFindings(findings)
report := &models.PrivacyScan{
FilesScanned: scanned,
Findings: findings,
Customers: guessCustomers(findings),
Summary: summarize(findings),
Sanitized: san.report(),
}
return report
}
func scanFileLines(file File, emit func(models.PrivacyFinding), san *sanTally) {
certFile := isCertFilename(file.Path)
sc := bufio.NewScanner(bytes.NewReader(file.Content))
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
ln := 0
for sc.Scan() {
ln++
line := sc.Text()
if len(line) > maxScanLineBytes {
line = line[:maxScanLineBytes]
}
scanLine(file.Path, line, ln, certFile, emit)
for _, m := range SanitizationMarkers(line) {
san.add(file.Path, ln, line, m)
}
}
}
func looksLikeText(b []byte) bool {
if len(b) == 0 {
return false
}
head := b
if len(head) > 8192 {
head = head[:8192]
}
if bytes.IndexByte(head, 0) >= 0 {
return false
}
// Tolerate a truncated multi-byte rune at the sampled boundary.
if !utf8.Valid(head) && !utf8.Valid(trimIncompleteRune(head)) {
return false
}
return true
}
func trimIncompleteRune(b []byte) []byte {
for i := 0; i < 3 && i < len(b); i++ {
if utf8.RuneStart(b[len(b)-1-i]) {
return b[:len(b)-1-i]
}
}
return b
}
func excerpt(line string) string {
line = strings.Map(func(r rune) rune {
if r == 0 || (r < 0x20 && r != '\t') {
return ' '
}
return r
}, line)
line = strings.Join(strings.Fields(line), " ")
line = strings.TrimSpace(line)
if utf8.RuneCountInString(line) <= maxExcerptRunes {
return line
}
r := []rune(line)
return string(r[:maxExcerptRunes]) + "..."
}
func severityRank(s string) int {
switch s {
case severityHigh:
return 0
case severityMedium:
return 1
default:
return 2
}
}
func sortFindings(f []models.PrivacyFinding) {
sort.SliceStable(f, func(i, j int) bool {
if f[i].Severity != f[j].Severity {
return severityRank(f[i].Severity) < severityRank(f[j].Severity)
}
if f[i].Category != f[j].Category {
return f[i].Category < f[j].Category
}
if f[i].Path != f[j].Path {
return f[i].Path < f[j].Path
}
return f[i].Match < f[j].Match
})
}
func summarize(f []models.PrivacyFinding) models.PrivacySummary {
s := models.PrivacySummary{Total: len(f), ByCategory: map[string]int{}}
for _, x := range f {
switch x.Severity {
case severityHigh:
s.High++
case severityMedium:
s.Medium++
default:
s.Low++
}
s.ByCategory[x.Category]++
}
return s
}