logpile/internal/sanitize/sanitize.go

// Package sanitize produces a de-identified copy of an uploaded diagnostic file.
//
// It rewrites the customer-identifying spans that internal/privacy detects
// (domains, FQDNs, e-mails, AD config, public IPs, timezone) with same-length
// neutral fillers, in place, without changing the file format: archive
// structure, entry names, modes and embedded timestamps are preserved; only the
// redacted byte ranges - and, for compressed containers, the compression layer
// - differ. Detection only ever reports; this is the matching redactor. See
// bible-local/docs/log-sanitization.md.
package sanitize

import (
	"fmt"
	"sort"
	"strings"
)

// Change is one aggregated group of replacements for the preview UI.
type Change struct {
	Path         string `json:"path"`
	Category     string `json:"category"`
	Count        int    `json:"count"`
	SampleBefore string `json:"sample_before"`
	SampleAfter  string `json:"sample_after"`
}

// Result is the outcome of sanitizing one file.
type Result struct {
	Data          []byte   `json:"-"`
	Changes       []Change `json:"changes"`
	SkippedBinary []string `json:"skipped_binary"`
	TotalReplaced int      `json:"total_replaced"`
}

// Sanitize de-identifies data (named filename so the format can be resolved by
// extension, same set as the parser accepts) and returns the rebuilt file plus
// a summary of what changed.
func Sanitize(filename string, data []byte) (*Result, error) {
	if len(data) == 0 {
		return nil, fmt.Errorf("empty input")
	}
	if len(data) > maxInputBytes {
		return nil, fmt.Errorf("file too large to sanitize in memory: %d bytes (limit %d)", len(data), maxInputBytes)
	}

	out, changes, skipped, err := rewriteBytes(filename, data)
	if err != nil {
		return nil, err
	}

	res := &Result{
		Data:          out,
		TotalReplaced: len(changes),
		Changes:       aggregate(changes),
		SkippedBinary: dedupeStrings(skipped),
	}
	return res, nil
}

func aggregate(raw []change) []Change {
	type key struct{ path, cat string }
	m := map[key]*Change{}
	order := []key{}
	for _, c := range raw {
		k := key{c.path, c.category}
		g := m[k]
		if g == nil {
			g = &Change{Path: c.path, Category: c.category, SampleBefore: c.before, SampleAfter: c.after}
			m[k] = g
			order = append(order, k)
		}
		g.Count++
	}
	out := make([]Change, 0, len(order))
	for _, k := range order {
		out = append(out, *m[k])
	}
	sort.SliceStable(out, func(i, j int) bool {
		if out[i].Category != out[j].Category {
			return out[i].Category < out[j].Category
		}
		return out[i].Path < out[j].Path
	})
	return out
}

func dedupeStrings(in []string) []string {
	if len(in) == 0 {
		return nil
	}
	seen := map[string]struct{}{}
	out := make([]string, 0, len(in))
	for _, s := range in {
		s = strings.TrimSpace(s)
		if s == "" {
			continue
		}
		if _, ok := seen[s]; ok {
			continue
		}
		seen[s] = struct{}{}
		out = append(out, s)
	}
	sort.Strings(out)
	return out
}

// CanSanitize reports whether a file with this name is a format the sanitizer
// can rebuild.
func CanSanitize(filename string) bool {
	switch strings.ToLower(ext(filename)) {
	case ".tar", ".sds", ".gz", ".tgz", ".zip", ".txt", ".log":
		return true
	}
	return false
}

func ext(name string) string {
	if i := strings.LastIndexByte(name, '.'); i >= 0 {
		return name[i:]
	}
	return ""
}