feat(privacy): scan ingested sources for customer-identifying data
Detection-only scan (internal/privacy) attached to every AnalysisResult: a customer-domain guess plus a findings list (category, file, line, match, hint), ported from the KB grep playbook. Runs on archive uploads and the serialized Redfish tree; gated by LOGPILE_PRIVACY_SCAN (default on). Surfaced at GET /api/privacy-scan, in the "Customer data" UI panel, and as privacy_report.json in the raw-export bundle. IP policy keeps RFC1918 and example ranges out of findings; allowlist covers standards-body and vendor infrastructure domains. No customer tokens in the repo. See ADL-066 and bible-local/docs/privacy-scan.md. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
3311bafd8e
commit
4a4910f207
@@ -0,0 +1,175 @@
|
||||
// Package privacy scans ingested source files for customer-identifying and
|
||||
// site-identifying data: DNS suffix, AD domain, NTP/DNS/syslog host names,
|
||||
// timezone, e-mail, public IPs, TLS cert CN/SAN, FRU location fields.
|
||||
//
|
||||
// Detection only. Nothing here modifies the source. The rule catalogue is a
|
||||
// port of the KB grep playbook; see bible-local/docs/privacy-scan.md.
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
)
|
||||
|
||||
// File is one unit of scanned content (an extracted archive member, or a
|
||||
// serialized Redfish tree).
|
||||
type File struct {
|
||||
Path string
|
||||
Content []byte
|
||||
}
|
||||
|
||||
const (
|
||||
maxFindings = 1000
|
||||
maxExcerptRunes = 200
|
||||
maxScanLineBytes = 8192
|
||||
)
|
||||
|
||||
// Scan runs every rule over every text file and returns the aggregated report.
|
||||
// Returns nil when there is nothing to scan (no text files).
|
||||
func Scan(files []File) *models.PrivacyScan {
|
||||
scanned := 0
|
||||
seen := make(map[string]struct{})
|
||||
var findings []models.PrivacyFinding
|
||||
|
||||
emit := func(f models.PrivacyFinding) {
|
||||
if len(findings) >= maxFindings {
|
||||
return
|
||||
}
|
||||
key := f.Category + "|" + f.Match + "|" + f.Path
|
||||
if _, ok := seen[key]; ok {
|
||||
return
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
findings = append(findings, f)
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
if isAllowlistedFilename(file.Path) {
|
||||
continue
|
||||
}
|
||||
if !looksLikeText(file.Content) {
|
||||
continue
|
||||
}
|
||||
scanned++
|
||||
scanFileLines(file, emit)
|
||||
}
|
||||
|
||||
if scanned == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
sortFindings(findings)
|
||||
|
||||
report := &models.PrivacyScan{
|
||||
FilesScanned: scanned,
|
||||
Findings: findings,
|
||||
Customers: guessCustomers(findings),
|
||||
Summary: summarize(findings),
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
func scanFileLines(file File, emit func(models.PrivacyFinding)) {
|
||||
certFile := isCertFilename(file.Path)
|
||||
sc := bufio.NewScanner(bytes.NewReader(file.Content))
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
ln := 0
|
||||
for sc.Scan() {
|
||||
ln++
|
||||
line := sc.Text()
|
||||
if len(line) > maxScanLineBytes {
|
||||
line = line[:maxScanLineBytes]
|
||||
}
|
||||
scanLine(file.Path, line, ln, certFile, emit)
|
||||
}
|
||||
}
|
||||
|
||||
func looksLikeText(b []byte) bool {
|
||||
if len(b) == 0 {
|
||||
return false
|
||||
}
|
||||
head := b
|
||||
if len(head) > 8192 {
|
||||
head = head[:8192]
|
||||
}
|
||||
if bytes.IndexByte(head, 0) >= 0 {
|
||||
return false
|
||||
}
|
||||
// Tolerate a truncated multi-byte rune at the sampled boundary.
|
||||
if !utf8.Valid(head) && !utf8.Valid(trimIncompleteRune(head)) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func trimIncompleteRune(b []byte) []byte {
|
||||
for i := 0; i < 3 && i < len(b); i++ {
|
||||
if utf8.RuneStart(b[len(b)-1-i]) {
|
||||
return b[:len(b)-1-i]
|
||||
}
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func excerpt(line string) string {
|
||||
line = strings.Map(func(r rune) rune {
|
||||
if r == 0 || (r < 0x20 && r != '\t') {
|
||||
return ' '
|
||||
}
|
||||
return r
|
||||
}, line)
|
||||
line = strings.Join(strings.Fields(line), " ")
|
||||
line = strings.TrimSpace(line)
|
||||
if utf8.RuneCountInString(line) <= maxExcerptRunes {
|
||||
return line
|
||||
}
|
||||
r := []rune(line)
|
||||
return string(r[:maxExcerptRunes]) + "..."
|
||||
}
|
||||
|
||||
func severityRank(s string) int {
|
||||
switch s {
|
||||
case severityHigh:
|
||||
return 0
|
||||
case severityMedium:
|
||||
return 1
|
||||
default:
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
func sortFindings(f []models.PrivacyFinding) {
|
||||
sort.SliceStable(f, func(i, j int) bool {
|
||||
if f[i].Severity != f[j].Severity {
|
||||
return severityRank(f[i].Severity) < severityRank(f[j].Severity)
|
||||
}
|
||||
if f[i].Category != f[j].Category {
|
||||
return f[i].Category < f[j].Category
|
||||
}
|
||||
if f[i].Path != f[j].Path {
|
||||
return f[i].Path < f[j].Path
|
||||
}
|
||||
return f[i].Match < f[j].Match
|
||||
})
|
||||
}
|
||||
|
||||
func summarize(f []models.PrivacyFinding) models.PrivacySummary {
|
||||
s := models.PrivacySummary{Total: len(f), ByCategory: map[string]int{}}
|
||||
for _, x := range f {
|
||||
switch x.Severity {
|
||||
case severityHigh:
|
||||
s.High++
|
||||
case severityMedium:
|
||||
s.Medium++
|
||||
default:
|
||||
s.Low++
|
||||
}
|
||||
s.ByCategory[x.Category]++
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in New Issue
Block a user