logpile/internal/privacy/privacy.go

// Package privacy scans ingested source files for customer-identifying and
// site-identifying data: DNS suffix, AD domain, NTP/DNS/syslog host names,
// timezone, e-mail, public IPs, TLS cert CN/SAN, FRU location fields.
//
// Detection only. Nothing here modifies the source. The rule catalogue is a
// port of the KB grep playbook; see bible-local/docs/privacy-scan.md.
package privacy

import (
	"bufio"
	"bytes"
	"sort"
	"strings"
	"unicode/utf8"

	"git.mchus.pro/mchus/logpile/internal/models"
)

// File is one unit of scanned content (an extracted archive member, or a
// serialized Redfish tree).
type File struct {
	Path    string
	Content []byte
}

const (
	maxFindings      = 1000
	maxExcerptRunes  = 200
	maxScanLineBytes = 8192
)

// Scan runs every rule over every text file and returns the aggregated report.
// Returns nil when there is nothing to scan (no text files).
func Scan(files []File) *models.PrivacyScan {
	scanned := 0
	seen := make(map[string]struct{})
	var findings []models.PrivacyFinding

	emit := func(f models.PrivacyFinding) {
		if len(findings) >= maxFindings {
			return
		}
		key := f.Category + "|" + f.Match + "|" + f.Path
		if _, ok := seen[key]; ok {
			return
		}
		seen[key] = struct{}{}
		findings = append(findings, f)
	}

	san := newSanTally()
	for _, file := range files {
		if isAllowlistedFilename(file.Path) {
			continue
		}
		if !looksLikeText(file.Content) {
			continue
		}
		scanned++
		scanFileLines(file, emit, san)
	}

	if scanned == 0 {
		return nil
	}

	sortFindings(findings)

	report := &models.PrivacyScan{
		FilesScanned: scanned,
		Findings:     findings,
		Customers:    guessCustomers(findings),
		Summary:      summarize(findings),
		Sanitized:    san.report(),
	}
	return report
}

func scanFileLines(file File, emit func(models.PrivacyFinding), san *sanTally) {
	certFile := isCertFilename(file.Path)
	sc := bufio.NewScanner(bytes.NewReader(file.Content))
	sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
	ln := 0
	for sc.Scan() {
		ln++
		line := sc.Text()
		if len(line) > maxScanLineBytes {
			line = line[:maxScanLineBytes]
		}
		scanLine(file.Path, line, ln, certFile, emit)
		for _, m := range SanitizationMarkers(line) {
			san.add(file.Path, ln, line, m)
		}
	}
}

func looksLikeText(b []byte) bool {
	if len(b) == 0 {
		return false
	}
	head := b
	if len(head) > 8192 {
		head = head[:8192]
	}
	if bytes.IndexByte(head, 0) >= 0 {
		return false
	}
	// Tolerate a truncated multi-byte rune at the sampled boundary.
	if !utf8.Valid(head) && !utf8.Valid(trimIncompleteRune(head)) {
		return false
	}
	return true
}

func trimIncompleteRune(b []byte) []byte {
	for i := 0; i < 3 && i < len(b); i++ {
		if utf8.RuneStart(b[len(b)-1-i]) {
			return b[:len(b)-1-i]
		}
	}
	return b
}

func excerpt(line string) string {
	line = strings.Map(func(r rune) rune {
		if r == 0 || (r < 0x20 && r != '\t') {
			return ' '
		}
		return r
	}, line)
	line = strings.Join(strings.Fields(line), " ")
	line = strings.TrimSpace(line)
	if utf8.RuneCountInString(line) <= maxExcerptRunes {
		return line
	}
	r := []rune(line)
	return string(r[:maxExcerptRunes]) + "..."
}

func severityRank(s string) int {
	switch s {
	case severityHigh:
		return 0
	case severityMedium:
		return 1
	default:
		return 2
	}
}

func sortFindings(f []models.PrivacyFinding) {
	sort.SliceStable(f, func(i, j int) bool {
		if f[i].Severity != f[j].Severity {
			return severityRank(f[i].Severity) < severityRank(f[j].Severity)
		}
		if f[i].Category != f[j].Category {
			return f[i].Category < f[j].Category
		}
		if f[i].Path != f[j].Path {
			return f[i].Path < f[j].Path
		}
		return f[i].Match < f[j].Match
	})
}

func summarize(f []models.PrivacyFinding) models.PrivacySummary {
	s := models.PrivacySummary{Total: len(f), ByCategory: map[string]int{}}
	for _, x := range f {
		switch x.Severity {
		case severityHigh:
			s.High++
		case severityMedium:
			s.Medium++
		default:
			s.Low++
		}
		s.ByCategory[x.Category]++
	}
	return s
}