logpile/internal/privacy/rules.go

package privacy

import (
	"net"
	"regexp"
	"strings"

	"git.mchus.pro/mchus/logpile/internal/models"
)

const (
	severityHigh   = "high"
	severityMedium = "medium"
	severityLow    = "low"

	catDomain        = "domain"
	catResolv        = "resolv"
	catADLDAP        = "ad_ldap"
	catTimezone      = "timezone"
	catEmail         = "email"
	catPublicIP      = "public_ip"
	catCollector     = "collector"
	catCert          = "cert"
	catFRULocation   = "fru_location"
	catDHCP          = "dhcp"
	catNSUpdate      = "nsupdate"
	catMgmtSubdomain = "mgmt_subdomain"
)

// Span is a matched sensitive substring of one line: line[Start:End]. Detection
// (Scan) and redaction (internal/sanitize) both consume the same spans so they
// can never disagree about what counts as customer data.
type Span struct {
	Start, End int
	Category   string
}

// FindSpans returns every sensitive span in a single line. certFile enables the
// TLS-certificate rules (only meaningful for *.pem / *.csr members).
func FindSpans(line string, certFile bool) []Span {
	ms := findMatches(line, certFile)
	out := make([]Span, 0, len(ms))
	for _, m := range ms {
		out = append(out, Span{Start: m.start, End: m.end, Category: m.category})
	}
	return out
}

// tableRule is a simple line-regexp rule. group is the submatch index used as
// the reported token (0 = whole match).
type tableRule struct {
	category string
	severity string
	re       *regexp.Regexp
	group    int
	hint     string
}

type matchSpan struct {
	start, end int
	category   string
	severity   string
	hint       string
	match      string
}

var (
	// Match any dotted name ending in a 2-24 char alpha label; plausibleFQDN
	// then decides whether that final label is a real TLD.
	reFQDN         = regexp.MustCompile(`(?i)\b((?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,24})\b`)
	mgmtLabels     = map[string]struct{}{"mgmt": {}, "oob": {}, "ipmi": {}, "drac": {}, "idrac": {}, "ilo": {}, "bmc": {}}
	reIPv4         = regexp.MustCompile(`\b((?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3})\b`)
	reEmail        = regexp.MustCompile(`(?i)\b([a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,})\b`)
	reSyslogTarget = regexp.MustCompile(`@((?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}:[0-9]{1,5})`)
	reTZName       = regexp.MustCompile(`\b((?:Africa|America|Antarctica|Asia|Atlantic|Australia|Europe|Indian|Pacific)/[A-Za-z_]+(?:/[A-Za-z_]+)?)\b`)
	reTZAbbr       = regexp.MustCompile(`\b(MSK|MSD|EEST|EET|MDST|CEST|CET|WEST|WET)\b\s+20\d\d`)
	// Kernel ring-buffer line: "[   12.345678] ..." or "[ 12.34][ T1] ...".
	// These carry driver/copyright strings (emails, versions), never customer id.
	reKernelTimestamp = regexp.MustCompile(`^\[\s*\d+\.\d+\]`)
	// Go panic / stack-trace lines: module-cache paths and "file.go:123" frames.
	reGoRuntime = regexp.MustCompile(`/go-mod/|\bgoroutine \d|\.go:\d+`)
	// A dotted quad sitting on a line that is talking about a version or a spec
	// clause, not a host.
	reVersionContext = regexp.MustCompile(`(?i)(version|\bver\.?\b|_ver\b|fw[_ ]?ver|\brevision|vbios|firmware|\bbios\b|microcode|ucode|\bbuild\b|x\.?org|xserver|ieee|\bstd\b|clause|section|\brfc\b)`)
	// syslog selector "local7.info", "mail.err" - a facility.severity pair, not a
	// domain.
	reSyslogSelector = regexp.MustCompile(`(?i)^(?:\*|local[0-7]|auth|authpriv|cron|daemon|ftp|kern|lpr|mail|news|security|syslog|user|uucp)\.(?:\*|emerg|panic|alert|crit|err|error|warn|warning|notice|info|debug|none)$`)
	// A TLD label written Title-case ("OS.It", "Rodolfo.Cn") is prose or mojibake,
	// not a real domain (real ones are lower- or all-upper-case).
	reTitlecaseTail = regexp.MustCompile(`\.[A-Z][a-z]+$`)

	tableRules = []tableRule{
		{catResolv, severityHigh, regexp.MustCompile(`(?i)^\s*(?:domain|search)\s+([a-z0-9][a-z0-9.\-]*\.[a-z0-9\-]+)`), 1,
			"resolv.conf DNS suffix - replace with example.local"},
		{catADLDAP, severityHigh, regexp.MustCompile(`(?i)\b(racdomain|adfilterdc[0-9]|rolegroup[0-9](?:name|domain))\s*=\s*(\S+)`), 2,
			"activedir.conf - AD domain / DC address / role-group name"},
		{catADLDAP, severityHigh, regexp.MustCompile(`(?i)\b(binddn|bindpw)\s*=\s*(\S+)`), 2,
			"ldap.conf - directory service-account bind"},
		{catTimezone, severityMedium, regexp.MustCompile(`(?i)\b(TimeZone|SELTimeUTCOffset)\s*=\s*(\S+)`), 2,
			"timezone reveals region - set Etc/UTC / offset 0"},
		{catCollector, severityMedium, regexp.MustCompile(`(?i)\b(SyslogHostname)\s*=\s*(\S+)`), 2,
			"customer syslog collector"},
		{catCollector, severityMedium, regexp.MustCompile(`(?i)"Destination"\s*:\s*"([^"]+)"`), 1,
			"SNMP trap / event destination"},
		{catFRULocation, severityMedium, regexp.MustCompile(`(?i)\b(Asset Tag|Product Location|Chassis Location|Board Extra)\b\s*[:=]\s*(.+)`), 2,
			"FRU site / inventory field"},
		{catNSUpdate, severityHigh, regexp.MustCompile(`(?i)\bupdate\s+(?:add|delete)\s+(\S+\.\S+)\s+.*\b(?:A|AAAA|PTR|CNAME)\b`), 1,
			"DDNS update - host FQDN + BMC record"},
		{catDHCP, severityMedium, regexp.MustCompile(`(?i)option\s+domain-name\s+"?([^";]+)`), 1,
			"domain-name handed out by customer DHCP"},
	}

	certLineRules = []tableRule{
		{catCert, severityHigh, regexp.MustCompile(`(?i)^\s*(?:Subject|Issuer):\s*(.+)`), 1, "TLS cert subject/issuer"},
		{catCert, severityHigh, regexp.MustCompile(`(?i)\bCN\s*=\s*([^,/]+)`), 1, "TLS cert common name"},
		{catCert, severityHigh, regexp.MustCompile(`(?i)\bDNS:\s*([a-z0-9.\-*]+)`), 1, "TLS cert SAN"},
	}
)

// findMatches is the single matcher shared by Scan and FindSpans. It returns
// every accepted span on the line, before the Scan-only "already covered by a
// more specific category" dedupe.
func findMatches(line string, certFile bool) []matchSpan {
	var out []matchSpan
	specific := map[string]struct{}{}
	noiseLine := reKernelTimestamp.MatchString(line) || reGoRuntime.MatchString(line)

	add := func(cat, sev, hint string, s, e int) {
		if s < 0 || e > len(line) || s >= e {
			return
		}
		s, e = trimSpan(line, s, e)
		if s >= e {
			return
		}
		match := line[s:e]
		if len(match) < 2 || !strings.ContainsFunc(match, isAlnum) || isAllowlistedValue(match) {
			return
		}
		if ip := net.ParseIP(match); ip != nil && !isSensitiveIP(match) {
			return
		}
		if cat == catFRULocation && !usefulFRUValue(match) {
			return
		}
		if cat != catDomain && cat != catEmail && cat != catPublicIP {
			specific[strings.ToLower(match)] = struct{}{}
		}
		out = append(out, matchSpan{start: s, end: e, category: cat, severity: sev, hint: hint, match: match})
	}

	groupSpan := func(m []int, group int) (int, int) {
		if 2*group+1 >= len(m) {
			return -1, -1
		}
		return m[2*group], m[2*group+1]
	}

	for _, r := range tableRules {
		for _, m := range r.re.FindAllStringSubmatchIndex(line, -1) {
			s, e := groupSpan(m, r.group)
			add(r.category, r.severity, r.hint, s, e)
		}
	}

	if certFile {
		for _, r := range certLineRules {
			for _, m := range r.re.FindAllStringSubmatchIndex(line, -1) {
				s, e := groupSpan(m, r.group)
				add(r.category, r.severity, r.hint, s, e)
			}
		}
	}

	if !noiseLine {
		for _, idx := range reFQDN.FindAllStringSubmatchIndex(line, -1) {
			s, e := idx[2], idx[3]
			if !cleanTokenBoundary(line, s, e) {
				continue
			}
			fqdn := line[s:e]
			if isAllowlistedDomain(fqdn) || !plausibleFQDN(fqdn) || reTitlecaseTail.MatchString(fqdn) || reSyslogSelector.MatchString(fqdn) {
				continue
			}
			labels := strings.Split(strings.ToLower(fqdn), ".")
			cat, sev, hint := catDomain, severityLow, "domain / FQDN in the logs"
			if len(labels) >= 3 {
				sev = severityMedium
			}
			for _, lbl := range labels {
				if _, ok := mgmtLabels[lbl]; ok {
					cat, sev, hint = catMgmtSubdomain, severityHigh, "management-network subdomain"
					break
				}
			}
			add(cat, sev, hint, s, e)
		}

		for _, m := range reEmail.FindAllStringSubmatchIndex(line, -1) {
			s, e := m[2], m[3]
			addr := line[s:e]
			host := addr[strings.IndexByte(addr, '@')+1:]
			if isAllowlistedDomain(host) || !looksLikeMailHost(host) {
				continue
			}
			add(catEmail, severityMedium, "e-mail address", s, e)
		}
	}

	for _, m := range reSyslogTarget.FindAllStringSubmatchIndex(line, -1) {
		add(catCollector, severityMedium, "remote syslog target", m[2], m[3])
	}

	commentLine := strings.HasPrefix(strings.TrimSpace(line), "#") || strings.HasPrefix(strings.TrimSpace(line), ";")
	if !noiseLine && !commentLine && !reVersionContext.MatchString(line) {
		for _, loc := range reIPv4.FindAllStringIndex(line, -1) {
			if inDottedNumberRun(line, loc[0], loc[1]) {
				continue
			}
			ip := line[loc[0]:loc[1]]
			if strings.HasSuffix(ip, ".0") || strings.HasPrefix(ip, "0.") || strings.HasPrefix(ip, "1.") {
				continue
			}
			if !isSensitiveIP(ip) {
				continue
			}
			add(catPublicIP, severityMedium, "public IP reveals provider / site", loc[0], loc[1])
		}
	}

	if ll := strings.ToLower(line); strings.Contains(ll, "timezone") || strings.Contains(ll, "zoneinfo") || strings.Contains(ll, "localtime") || strings.Contains(ll, "tz=") {
		for _, m := range reTZName.FindAllStringSubmatchIndex(line, -1) {
			s, e := m[2], m[3]
			if isAllowlistedValue(line[s:e]) {
				continue
			}
			add(catTimezone, severityMedium, "timezone reveals region - set Etc/UTC", s, e)
		}
	}
	for _, m := range reTZAbbr.FindAllStringSubmatchIndex(line, -1) {
		add(catTimezone, severityLow, "localized timestamp reveals region", m[2], m[3])
	}

	// Scan-only dedupe: drop a bare domain/email span when the same token is
	// already covered by a more specific category on this line.
	if len(specific) > 0 {
		filtered := out[:0]
		for _, m := range out {
			if (m.category == catDomain || m.category == catEmail) && matchCoveredBySpecific(m.match, specific) {
				continue
			}
			filtered = append(filtered, m)
		}
		out = filtered
	}
	return out
}

func scanLine(path, line string, ln int, certFile bool, emit func(models.PrivacyFinding)) {
	for _, m := range findMatches(line, certFile) {
		emit(models.PrivacyFinding{
			Category: m.category, Severity: m.severity, Path: path, Line: ln,
			Match: m.match, Excerpt: excerpt(line), Hint: m.hint,
		})
	}
}

func isAlnum(r rune) bool {
	return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9')
}

// trimSpan narrows [s,e) past leading/trailing quoting and separator bytes,
// matching the old strings.Trim(TrimSpace(...), "\"',;:") behaviour.
func trimSpan(line string, s, e int) (int, int) {
	const cut = " \t\"',;:"
	for s < e && strings.IndexByte(cut, line[s]) >= 0 {
		s++
	}
	for e > s && strings.IndexByte(cut, line[e-1]) >= 0 {
		e--
	}
	return s, e
}

// cleanTokenBoundary reports whether line[s:e] stands as its own token - not
// glued to surrounding identifier characters ("auth.backend.gssapi.store-creds",
// "mountall.sh [start]" is fine, "96.00.CF.00" is not) - so it reads as a
// hostname rather than a fragment of code, a path tail, or a version string.
func cleanTokenBoundary(line string, s, e int) bool {
	if s > 0 {
		switch c := line[s-1]; c {
		case ' ', '\t', '"', '\'', '(', '<', '=', ',', '@':
		case '/', ':':
		default:
			return false
		}
	}
	if e < len(line) {
		switch line[e] {
		case ' ', '\t', '"', '\'', ')', '>', ',', ';', ':', '/', '\\', '?', '!':
		default:
			return false
		}
	}
	return true
}

// inDottedNumberRun reports whether line[start:end] (a dotted quad) is embedded
// in a longer version-like run - "18:6.1.4.5", "v1.2.3.4", "1.2.3.4.5",
// "go1.25.0.linux" - rather than a standalone address.
func inDottedNumberRun(line string, start, end int) bool {
	if start > 0 {
		switch c := line[start-1]; {
		case c == '.', c == '-':
			return true
		case c == ':' && start >= 2 && line[start-2] >= '0' && line[start-2] <= '9':
			return true
		case c == 'v' || c == 'V':
			return true
		}
	}
	if end < len(line) && (line[end] == '.' || line[end] == '-') {
		return true
	}
	return false
}

// usefulFRUValue drops FRU asset/location values that carry no site information:
// the field name echoed back, an all-digit manufacturing code, or a string that
// is just the chassis serial (already in the archive name).
func usefulFRUValue(v string) bool {
	lv := strings.ToLower(strings.ReplaceAll(v, " ", ""))
	if strings.Contains(lv, "assettag") || strings.Contains(lv, "serialnumber") {
		return false
	}
	if len(v) < 4 {
		return false
	}
	digits, upperAlnum := 0, 0
	for _, r := range v {
		if r >= '0' && r <= '9' {
			digits++
		}
		if (r >= '0' && r <= '9') || (r >= 'A' && r <= 'Z') {
			upperAlnum++
		}
	}
	if digits == len(v) {
		return false
	}
	if upperAlnum == len(v) && digits >= 4 {
		return false
	}
	return true
}

// looksLikeMailHost rejects the many "local@identifier.token" strings that are
// not e-mail: OData/Redfish JSON annotations (Members@odata.count), SSH
// cipher/kex names, systemd units ("serial-getty@ttyAMA0.service").
func looksLikeMailHost(host string) bool {
	h := strings.ToLower(host)
	if strings.Contains(h, "odata") || strings.Contains(h, "redfish") || strings.Contains(h, "message.") {
		return false
	}
	return plausibleFQDN(h)
}

// matchCoveredBySpecific reports whether a bare FQDN/e-mail match is already
// represented by a more precise finding on the same line.
func matchCoveredBySpecific(match string, specific map[string]struct{}) bool {
	m := strings.ToLower(match)
	if _, ok := specific[m]; ok {
		return true
	}
	if at := strings.IndexByte(m, '@'); at >= 0 {
		if _, ok := specific[m[at+1:]]; ok {
			return true
		}
	}
	return false
}