diff --git a/cmd/logpile-cpu-audit/main.go b/cmd/logpile-cpu-audit/main.go new file mode 100644 index 0000000..28f96c1 --- /dev/null +++ b/cmd/logpile-cpu-audit/main.go @@ -0,0 +1,482 @@ +package main + +import ( + "crypto/sha256" + "encoding/json" + "flag" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + "time" + + "git.mchus.pro/mchus/logpile/internal/exporter" + "git.mchus.pro/mchus/logpile/internal/models" + "git.mchus.pro/mchus/logpile/internal/parser" + _ "git.mchus.pro/mchus/logpile/internal/parser/vendors" +) + +const auditSchema = "logpile.cpu-identity-audit.v1" + +type cpuAudit struct { + Socket int `json:"socket"` + Model string `json:"model,omitempty"` + PPIN string `json:"ppin,omitempty"` + SerialNumber string `json:"serial_number,omitempty"` + ExpectedSerialNumber string `json:"expected_serial_number,omitempty"` + PPINPromoted bool `json:"ppin_promoted"` + Finding string `json:"finding"` +} + +type parserAudit struct { + Vendor string `json:"vendor,omitempty"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` +} + +type fileAudit struct { + Schema string `json:"schema"` + SourceFile string `json:"source_file"` + Parser parserAudit `json:"parser,omitempty"` + CPUs []cpuAudit `json:"cpus"` + Error string `json:"error,omitempty"` +} + +type failedAudit struct { + SourceFile string `json:"source_file"` + Error string `json:"error"` +} + +type auditSummary struct { + Schema string `json:"schema"` + InputRoot string `json:"input_root"` + OutputRoot string `json:"output_root"` + FilesFound int `json:"files_found"` + FilesParsed int `json:"files_parsed"` + FilesFailed int `json:"files_failed"` + FilesWithCPU int `json:"files_with_cpu"` + FilesNoCPU int `json:"files_without_cpu"` + CPUsFound int `json:"cpus_found"` + FindingCount map[string]int `json:"finding_count"` + Failures []failedAudit `json:"failures,omitempty"` +} + +type auditJob struct { + index int + sourcePath string + relative string +} + +type auditResult struct { + index int + report fileAudit + err error +} + +type reanimatorResult struct { + index int + sourceFile string + payload *exporter.ReanimatorExport + cpus []cpuAudit + err error + noCPU bool +} + +func main() { + input := flag.String("input", "", "Dump file or directory to scan recursively") + output := flag.String("output", "", "Directory for CPU audit JSON files") + workers := flag.Int("workers", max(1, min(runtime.NumCPU(), 4)), "Number of dumps parsed concurrently") + includePlain := flag.Bool("include-plain", false, "Also scan standalone .txt and .log inputs inside directories") + reanimatorMode := flag.Bool("reanimator", false, "Write flat CPU-only Reanimator ingest JSON files") + prefix := flag.String("prefix", "", "Filename prefix for flat Reanimator output, for example v0") + flag.Parse() + + if strings.TrimSpace(*input) == "" || strings.TrimSpace(*output) == "" { + flag.Usage() + os.Exit(2) + } + if *workers < 1 { + slog.Error("invalid worker count", "workers", *workers) + os.Exit(2) + } + + started := time.Now() + var summary auditSummary + var err error + if *reanimatorMode { + summary, err = runReanimatorExport(*input, *output, *prefix, *workers, *includePlain) + } else { + summary, err = runAudit(*input, *output, *workers, *includePlain) + } + if err != nil { + slog.Error("CPU identity audit failed", "err", err) + os.Exit(1) + } + slog.Info("CPU identity audit completed", + "files", summary.FilesFound, + "parsed", summary.FilesParsed, + "failed", summary.FilesFailed, + "cpus", summary.CPUsFound, + "duration", time.Since(started).Round(time.Millisecond), + "output", summary.OutputRoot, + ) + if summary.FilesFailed > 0 { + os.Exit(2) + } +} + +func runReanimatorExport(inputPath, outputPath, prefix string, workers int, includePlain bool) (auditSummary, error) { + inputRoot, err := filepath.Abs(inputPath) + if err != nil { + return auditSummary{}, fmt.Errorf("resolve input path: %w", err) + } + outputRoot, err := filepath.Abs(outputPath) + if err != nil { + return auditSummary{}, fmt.Errorf("resolve output path: %w", err) + } + info, err := os.Stat(inputRoot) + if err != nil { + return auditSummary{}, fmt.Errorf("inspect input: %w", err) + } + if err := os.MkdirAll(outputRoot, 0o755); err != nil { + return auditSummary{}, fmt.Errorf("create output directory: %w", err) + } + jobs, err := collectAuditJobs(inputRoot, outputRoot, info.IsDir(), includePlain) + if err != nil { + return auditSummary{}, err + } + summary := auditSummary{ + Schema: auditSchema, + InputRoot: inputRoot, + OutputRoot: outputRoot, + FilesFound: len(jobs), + FindingCount: make(map[string]int), + } + + jobCh := make(chan auditJob) + resultCh := make(chan reanimatorResult) + var group sync.WaitGroup + for range workers { + group.Add(1) + go func() { + defer group.Done() + for job := range jobCh { + resultCh <- processReanimatorJob(job) + } + }() + } + go func() { + for _, job := range jobs { + jobCh <- job + } + close(jobCh) + group.Wait() + close(resultCh) + }() + + completed := 0 + for result := range resultCh { + completed++ + if result.err != nil { + summary.FilesFailed++ + summary.Failures = append(summary.Failures, failedAudit{SourceFile: result.sourceFile, Error: result.err.Error()}) + } else { + summary.FilesParsed++ + if result.noCPU { + summary.FilesNoCPU++ + } else { + summary.FilesWithCPU++ + summary.CPUsFound += len(result.payload.Hardware.CPUs) + for _, cpu := range result.cpus { + summary.FindingCount[cpu.Finding]++ + } + name := flatReanimatorName(prefix, result.sourceFile) + path := filepath.Join(outputRoot, name) + if _, statErr := os.Stat(path); statErr == nil { + hash := fmt.Sprintf("%x", sha256.Sum256([]byte(result.sourceFile)))[:8] + name = flatReanimatorName(prefix+"__"+hash, result.sourceFile) + path = filepath.Join(outputRoot, name) + } + if err := writeJSON(path, result.payload); err != nil { + return auditSummary{}, fmt.Errorf("write Reanimator payload %s: %w", path, err) + } + } + } + if completed%25 == 0 || completed == len(jobs) || result.err != nil { + slog.Info("CPU Reanimator export progress", "completed", completed, "total", len(jobs), "source", result.sourceFile) + } + } + return summary, nil +} + +func processReanimatorJob(job auditJob) reanimatorResult { + result := reanimatorResult{index: job.index, sourceFile: filepath.ToSlash(job.relative)} + bmcParser := parser.NewBMCParser() + if err := bmcParser.ParseArchive(job.sourcePath); err != nil { + result.err = err + return result + } + parsed := bmcParser.Result() + if parsed == nil || parsed.Hardware == nil || len(parsed.Hardware.CPUs) == 0 { + result.noCPU = true + return result + } + parsed.Filename = filepath.Base(job.sourcePath) + converted, err := exporter.ConvertToReanimator(parsed) + if err != nil { + result.err = err + return result + } + result.payload = cpuOnlyReanimatorPayload(converted) + for _, cpu := range parsed.Hardware.CPUs { + result.cpus = append(result.cpus, buildCPUAudit(cpu)) + } + return result +} + +func cpuOnlyReanimatorPayload(converted *exporter.ReanimatorExport) *exporter.ReanimatorExport { + converted.Hardware = exporter.ReanimatorHardware{ + Board: converted.Hardware.Board, + CPUs: converted.Hardware.CPUs, + } + return converted +} + +func flatReanimatorName(prefix, sourceFile string) string { + name := filepath.Base(sourceFile) + ".reanimator.json" + prefix = strings.Trim(strings.TrimSpace(prefix), "_-") + if prefix == "" { + return name + } + return prefix + "__" + name +} + +func runAudit(inputPath, outputPath string, workers int, includePlain bool) (auditSummary, error) { + inputRoot, err := filepath.Abs(inputPath) + if err != nil { + return auditSummary{}, fmt.Errorf("resolve input path: %w", err) + } + outputRoot, err := filepath.Abs(outputPath) + if err != nil { + return auditSummary{}, fmt.Errorf("resolve output path: %w", err) + } + info, err := os.Stat(inputRoot) + if err != nil { + return auditSummary{}, fmt.Errorf("inspect input: %w", err) + } + if err := os.MkdirAll(outputRoot, 0o755); err != nil { + return auditSummary{}, fmt.Errorf("create output directory: %w", err) + } + + jobs, err := collectAuditJobs(inputRoot, outputRoot, info.IsDir(), includePlain) + if err != nil { + return auditSummary{}, err + } + summary := auditSummary{ + Schema: auditSchema, + InputRoot: inputRoot, + OutputRoot: outputRoot, + FilesFound: len(jobs), + FindingCount: make(map[string]int), + } + + jobCh := make(chan auditJob) + resultCh := make(chan auditResult) + var group sync.WaitGroup + for range workers { + group.Add(1) + go func() { + defer group.Done() + for job := range jobCh { + resultCh <- processAuditJob(job) + } + }() + } + go func() { + for _, job := range jobs { + jobCh <- job + } + close(jobCh) + group.Wait() + close(resultCh) + }() + + results := make([]auditResult, len(jobs)) + completed := 0 + for result := range resultCh { + results[result.index] = result + completed++ + if completed%25 == 0 || completed == len(jobs) || result.err != nil { + slog.Info("CPU identity audit progress", "completed", completed, "total", len(jobs), "source", result.report.SourceFile) + } + } + + for _, result := range results { + reportPath := filepath.Join(outputRoot, result.report.SourceFile+".cpu-audit.json") + if err := writeJSON(reportPath, result.report); err != nil { + return auditSummary{}, fmt.Errorf("write report %s: %w", reportPath, err) + } + if result.err != nil { + summary.FilesFailed++ + summary.Failures = append(summary.Failures, failedAudit{ + SourceFile: result.report.SourceFile, + Error: result.err.Error(), + }) + continue + } + summary.FilesParsed++ + summary.CPUsFound += len(result.report.CPUs) + if len(result.report.CPUs) == 0 { + summary.FilesNoCPU++ + } else { + summary.FilesWithCPU++ + } + for _, cpu := range result.report.CPUs { + summary.FindingCount[cpu.Finding]++ + } + } + + if err := writeJSON(filepath.Join(outputRoot, "_summary.json"), summary); err != nil { + return auditSummary{}, fmt.Errorf("write summary: %w", err) + } + return summary, nil +} + +func collectAuditJobs(inputRoot, outputRoot string, inputIsDir, includePlain bool) ([]auditJob, error) { + if !inputIsDir { + if !parser.IsSupportedArchiveFilename(inputRoot) { + return nil, fmt.Errorf("unsupported input file: %s", inputRoot) + } + return []auditJob{{sourcePath: inputRoot, relative: filepath.Base(inputRoot)}}, nil + } + + var jobs []auditJob + err := filepath.WalkDir(inputRoot, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return fmt.Errorf("walk %s: %w", path, walkErr) + } + if entry.IsDir() { + if path == outputRoot && path != inputRoot { + return filepath.SkipDir + } + return nil + } + if !isAuditCandidate(path, includePlain) { + return nil + } + relative, err := filepath.Rel(inputRoot, path) + if err != nil { + return fmt.Errorf("resolve relative path for %s: %w", path, err) + } + jobs = append(jobs, auditJob{sourcePath: path, relative: relative}) + return nil + }) + if err != nil { + return nil, fmt.Errorf("scan input directory: %w", err) + } + sort.Slice(jobs, func(i, j int) bool { return jobs[i].relative < jobs[j].relative }) + for i := range jobs { + jobs[i].index = i + } + return jobs, nil +} + +func isAuditCandidate(path string, includePlain bool) bool { + if !parser.IsSupportedArchiveFilename(path) { + return false + } + ext := strings.ToLower(filepath.Ext(path)) + return includePlain || (ext != ".txt" && ext != ".log") +} + +func processAuditJob(job auditJob) auditResult { + report := fileAudit{ + Schema: auditSchema, + SourceFile: filepath.ToSlash(job.relative), + CPUs: []cpuAudit{}, + } + bmcParser := parser.NewBMCParser() + if err := bmcParser.ParseArchive(job.sourcePath); err != nil { + report.Error = err.Error() + return auditResult{index: job.index, report: report, err: err} + } + report.Parser = detectedParserAudit(bmcParser.DetectedVendor()) + result := bmcParser.Result() + if result == nil || result.Hardware == nil { + return auditResult{index: job.index, report: report} + } + for _, cpu := range result.Hardware.CPUs { + report.CPUs = append(report.CPUs, buildCPUAudit(cpu)) + } + sort.SliceStable(report.CPUs, func(i, j int) bool { return report.CPUs[i].Socket < report.CPUs[j].Socket }) + return auditResult{index: job.index, report: report} +} + +func detectedParserAudit(name string) parserAudit { + for _, info := range parser.ListParsersInfo() { + if info.Name == name { + return parserAudit{Vendor: info.Vendor, Name: info.Name, Version: info.Version} + } + } + return parserAudit{Name: name} +} + +func buildCPUAudit(cpu models.CPU) cpuAudit { + ppin := strings.TrimSpace(cpu.PPIN) + serial := strings.TrimSpace(cpu.SerialNumber) + validPPIN := models.ResolveCPUSerialNumber("", ppin) + validSerial := models.ResolveCPUSerialNumber(serial, "") + expected := models.ResolveCPUSerialNumber(serial, ppin) + report := cpuAudit{ + Socket: cpu.Socket, + Model: strings.TrimSpace(cpu.Model), + PPIN: ppin, + SerialNumber: serial, + ExpectedSerialNumber: expected, + PPINPromoted: validPPIN != "" && validSerial == validPPIN, + } + switch { + case validPPIN != "" && validSerial == "": + report.Finding = "missing_serial_with_ppin" + case validPPIN != "" && validSerial == validPPIN: + report.Finding = "ppin_used_as_serial" + case validSerial != "": + report.Finding = "source_serial_present" + default: + report.Finding = "cpu_identity_missing" + } + return report +} + +func writeJSON(path string, value any) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("create parent directory: %w", err) + } + payload, err := json.MarshalIndent(value, "", " ") + if err != nil { + return fmt.Errorf("encode JSON: %w", err) + } + payload = append(payload, '\n') + temporary, err := os.CreateTemp(filepath.Dir(path), ".cpu-audit-*.tmp") + if err != nil { + return fmt.Errorf("create temporary file: %w", err) + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if _, err := temporary.Write(payload); err != nil { + temporary.Close() + return fmt.Errorf("write temporary file: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close temporary file: %w", err) + } + if err := os.Rename(temporaryPath, path); err != nil { + return fmt.Errorf("replace report: %w", err) + } + return nil +} diff --git a/cmd/logpile-cpu-audit/main_test.go b/cmd/logpile-cpu-audit/main_test.go new file mode 100644 index 0000000..ddd96d3 --- /dev/null +++ b/cmd/logpile-cpu-audit/main_test.go @@ -0,0 +1,130 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "git.mchus.pro/mchus/logpile/internal/exporter" + "git.mchus.pro/mchus/logpile/internal/models" +) + +func TestIsAuditCandidate(t *testing.T) { + tests := []struct { + name string + path string + includePlain bool + want bool + }{ + {name: "tar gzip", path: "dump.tar.gz", want: true}, + {name: "zip", path: "dump.zip", want: true}, + {name: "plain log excluded", path: "component.log", want: false}, + {name: "plain log requested", path: "nvidia-bug-report.log", includePlain: true, want: true}, + {name: "JSON report excluded", path: "report.json", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isAuditCandidate(tt.path, tt.includePlain); got != tt.want { + t.Fatalf("isAuditCandidate(%q, %v) = %v, want %v", tt.path, tt.includePlain, got, tt.want) + } + }) + } +} + +func TestCPUOnlyReanimatorPayloadPreservesRequiredFields(t *testing.T) { + payload := &exporter.ReanimatorExport{ + Filename: "dump.tar.gz", + CollectedAt: "2026-07-27T10:34:40Z", + Hardware: exporter.ReanimatorHardware{ + Board: exporter.ReanimatorBoard{SerialNumber: "23E102624"}, + CPUs: []exporter.ReanimatorCPU{{Socket: 0, SerialNumber: "D46E5D6B1D3E40E1"}}, + Firmware: []exporter.ReanimatorFirmware{{DeviceName: "BIOS", Version: "1.0"}}, + }, + } + + got := cpuOnlyReanimatorPayload(payload) + if got.CollectedAt == "" || got.Hardware.Board.SerialNumber == "" || len(got.Hardware.CPUs) != 1 { + t.Fatalf("required ingest fields were lost: %+v", got) + } + if len(got.Hardware.Firmware) != 0 { + t.Fatalf("expected non-CPU hardware to be removed: %+v", got.Hardware.Firmware) + } +} + +func TestFlatReanimatorName(t *testing.T) { + got := flatReanimatorName("v1", "NL/example/dump.tar.gz") + if got != "v1__dump.tar.gz.reanimator.json" { + t.Fatalf("unexpected flat name %q", got) + } +} + +func TestBuildCPUAudit(t *testing.T) { + tests := []struct { + name string + cpu models.CPU + finding string + promoted bool + expectedSer string + }{ + { + name: "PPIN promoted", + cpu: models.CPU{Socket: 0, PPIN: "D46E5D6B1D3E40E1", SerialNumber: "D46E5D6B1D3E40E1"}, + finding: "ppin_used_as_serial", + promoted: true, + expectedSer: "D46E5D6B1D3E40E1", + }, + { + name: "regression detected", + cpu: models.CPU{Socket: 1, PPIN: "D44F8D6B9155EE0E"}, + finding: "missing_serial_with_ppin", + expectedSer: "D44F8D6B9155EE0E", + }, + { + name: "placeholder serial with PPIN", + cpu: models.CPU{Socket: 1, PPIN: "D44F8D6B9155EE0E", SerialNumber: "Unknown"}, + finding: "missing_serial_with_ppin", + expectedSer: "D44F8D6B9155EE0E", + }, + { + name: "native serial", + cpu: models.CPU{Socket: 0, SerialNumber: "CPU-SERIAL"}, + finding: "source_serial_present", + expectedSer: "CPU-SERIAL", + }, + { + name: "placeholder PPIN", + cpu: models.CPU{Socket: 0, PPIN: "N/A"}, + finding: "cpu_identity_missing", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildCPUAudit(tt.cpu) + if got.Finding != tt.finding || got.PPINPromoted != tt.promoted || got.ExpectedSerialNumber != tt.expectedSer { + t.Fatalf("unexpected audit: %+v", got) + } + }) + } +} + +func TestRunAuditWritesFailureReportAndSummary(t *testing.T) { + input := t.TempDir() + output := t.TempDir() + badArchive := filepath.Join(input, "bad.zip") + if err := os.WriteFile(badArchive, []byte("not a zip"), 0o644); err != nil { + t.Fatalf("write fixture: %v", err) + } + + summary, err := runAudit(input, output, 1, false) + if err != nil { + t.Fatalf("runAudit failed: %v", err) + } + if summary.FilesFound != 1 || summary.FilesFailed != 1 || summary.FilesParsed != 0 { + t.Fatalf("unexpected summary: %+v", summary) + } + for _, name := range []string{"bad.zip.cpu-audit.json", "_summary.json"} { + if _, err := os.Stat(filepath.Join(output, name)); err != nil { + t.Fatalf("expected %s: %v", name, err) + } + } +}