refactor: modularize audit and harden build validation

audit/internal/app/app.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
-	"strconv"
 	"strings"
 
 	"bee/audit/internal/collector"
@@ -96,14 +95,38 @@ type installer interface {
 type GPUPresenceResult struct {
 	Nvidia bool
 	AMD    bool
+	// NvidiaInitializing / AMDInitializing report a PCI device of that vendor
+	// when DetectGPUVendor did not report the vendor as operational.
+	NvidiaInitializing bool
+	AMDInitializing    bool
 }
 
+// DetectGPUPresence combines the existing operational vendor detection with a
+// PCI display-class scan. The latter distinguishes absent hardware from a
+// detected PCI function whose runtime is not operational yet.
 func (a *App) DetectGPUPresence() GPUPresenceResult {
 	vendor := a.sat.DetectGPUVendor()
-	return GPUPresenceResult{
+	res := GPUPresenceResult{
 		Nvidia: vendor == "nvidia",
 		AMD:    vendor == "amd",
 	}
+	physNvidia, physAMD := a.sat.PhysicalGPUVendors()
+	res.NvidiaInitializing = physNvidia && !res.Nvidia
+	res.AMDInitializing = physAMD && !res.AMD
+	return res
+}
+
+// RuntimeHealthNow collects a fresh runtime-health snapshot (driver / CUDA
+// readiness, GSP state). Unlike ReadRuntimeHealth it does not read the
+// boot-time JSON; callers that poll for the GPU stack to come up need
+// current data.
+func (a *App) RuntimeHealthNow() (schema.RuntimeHealth, error) {
+	return a.runtime.CollectRuntimeHealth(DefaultExportDir)
+}
+
+// TPMPresent reports whether this host has a TPM the checks can talk to.
+func (a *App) TPMPresent() bool {
+	return a.sat.TPMPresent()
 }
 
 func (a *App) IsLiveMediaInRAM() bool {
@@ -132,13 +155,14 @@ type satRunner interface {
 	RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error)
 	RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error)
 	RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error)
-	RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error)
+	RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, fullMatrix bool, logFunc func(string)) (string, error)
 	RunNvidiaStressPack(ctx context.Context, baseDir string, opts platform.NvidiaStressOptions, logFunc func(string)) (string, error)
 	ListNvidiaGPUStatuses() ([]platform.NvidiaGPUStatus, error)
 	ResetNvidiaGPU(index int) (string, error)
 	RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error)
 	RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error)
 	RunTPMValidationPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
+	TPMPresent() bool
 	RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
 	RunPCIeLinkCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
 	RunNvidiaPCIeBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error)
@@ -151,6 +175,7 @@ type satRunner interface {
 	SetNvidiaGPUPowerLimit(index int, watts float64) (string, error)
 	ResetNvidiaGPUDefaults() (string, error)
 	DetectGPUVendor() string
+	PhysicalGPUVendors() (nvidia bool, amd bool)
 	ListAMDGPUs() ([]platform.AMDGPUInfo, error)
 	RunAMDAcceptancePack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
 	RunAMDMemIntegrityPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
@@ -279,15 +304,6 @@ func (a *App) RunRuntimePreflight(output string) (string, error) {
 	}
 }
 
-func (a *App) RunRuntimePreflightResult() (ActionResult, error) {
-	path, err := a.RunRuntimePreflight("file:" + DefaultRuntimeJSONPath)
-	body := "Runtime preflight completed."
-	if path != "" {
-		body = "Runtime health written to " + path
-	}
-	return ActionResult{Title: "Run self-check", Body: body}, err
-}
-
 func (a *App) RuntimeHealthResult() ActionResult {
 	health, err := ReadRuntimeHealth(DefaultRuntimeJSONPath)
 	if err != nil {
@@ -326,10 +342,6 @@ func (a *App) RunAuditNow(runtimeMode runtimeenv.Mode) (ActionResult, error) {
 	return ActionResult{Title: "Run audit", Body: body}, err
 }
 
-func (a *App) RunAuditToDefaultFile(runtimeMode runtimeenv.Mode) (string, error) {
-	return a.RunAudit(runtimeMode, "file:"+DefaultAuditJSONPath)
-}
-
 func (a *App) HealthSummaryResult() ActionResult {
 	raw, err := os.ReadFile(DefaultAuditJSONPath)
 	if err != nil {
@@ -399,26 +411,6 @@ func (a *App) MainBanner() string {
 	return strings.TrimSpace(strings.Join(lines, "\n"))
 }
 
-func (a *App) FormatToolStatuses(statuses []platform.ToolStatus) string {
-	var body strings.Builder
-	for _, tool := range statuses {
-		status := "MISSING"
-		if tool.OK {
-			status = "OK (" + tool.Path + ")"
-		}
-		fmt.Fprintf(&body, "- %s: %s\n", tool.Name, status)
-	}
-	return strings.TrimSpace(body.String())
-}
-
-func (a *App) ParsePrefix(raw string, fallback int) int {
-	value, err := strconv.Atoi(strings.TrimSpace(raw))
-	if err != nil || value <= 0 {
-		return fallback
-	}
-	return value
-}
-
 // writePSUStatusesToDB records PSU statuses collected during audit into the
 // component-status DB so they are visible in the Hardware Summary card.
 // PSU status is sourced from IPMI (ipmitool fru + sdr) during audit.

audit/internal/app/app_install.go

@@ -62,18 +62,6 @@ func (a *App) ExportLatestAudit(target platform.RemovableTarget) (string, error)
 	return a.exports.ExportFileToTarget(tmpPath, target)
 }
 
-func (a *App) ExportLatestAuditResult(target platform.RemovableTarget) (ActionResult, error) {
-	path, err := a.ExportLatestAudit(target)
-	body := "Audit export failed."
-	if err == nil {
-		body = "Audit exported."
-	}
-	if err == nil && path != "" {
-		body = "Audit exported to " + path
-	}
-	return ActionResult{Title: "Export audit", Body: body}, err
-}
-
 func (a *App) ExportSupportBundle(target platform.RemovableTarget) (string, error) {
 	archive, err := BuildSupportBundle(DefaultExportDir)
 	if err != nil {

audit/internal/app/app_packs.go

@@ -196,11 +196,11 @@ func (a *App) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durati
 	return a.sat.RunNvidiaPulseTestPack(ctx, baseDir, durationSec, gpuIndices, logFunc)
 }
 
-func (a *App) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
+func (a *App) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, fullMatrix bool, logFunc func(string)) (string, error) {
 	if strings.TrimSpace(baseDir) == "" {
 		baseDir = DefaultSATBaseDir
 	}
-	return a.sat.RunNvidiaBandwidthPack(ctx, baseDir, gpuIndices, logFunc)
+	return a.sat.RunNvidiaBandwidthPack(ctx, baseDir, gpuIndices, fullMatrix, logFunc)
 }
 
 func (a *App) RunNvidiaStressPackCtx(ctx context.Context, baseDir string, opts platform.NvidiaStressOptions, logFunc func(string)) (string, error) {
@@ -237,11 +237,6 @@ func (a *App) RunCPUAcceptancePackCtx(ctx context.Context, baseDir string, durat
 	return a.sat.RunCPUAcceptancePack(ctx, baseDir, durationSec, logFunc)
 }
 
-func (a *App) RunCPUAcceptancePackResult(baseDir string, durationSec int) (ActionResult, error) {
-	path, err := a.RunCPUAcceptancePack(baseDir, durationSec, nil)
-	return ActionResult{Title: "CPU SAT", Body: satResultBody(path)}, err
-}
-
 func (a *App) RunStorageAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
 	return a.RunStorageAcceptancePackCtx(context.Background(), baseDir, false, logFunc)
 }
@@ -276,11 +271,6 @@ func (a *App) RunNvidiaConfigCheckPack(baseDir string, logFunc func(string)) (st
 	return a.RunNvidiaConfigCheckPackCtx(context.Background(), baseDir, logFunc)
 }
 
-func (a *App) RunNvidiaConfigCheckPackResult(baseDir string) (ActionResult, error) {
-	path, err := a.RunNvidiaConfigCheckPack(baseDir, nil)
-	return ActionResult{Title: "GPU Config & NVLink Check", Body: satResultBody(path)}, err
-}
-
 func (a *App) RunPCIeLinkCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
 	if strings.TrimSpace(baseDir) == "" {
 		baseDir = DefaultSATBaseDir
@@ -292,11 +282,6 @@ func (a *App) RunPCIeLinkCheckPack(baseDir string, logFunc func(string)) (string
 	return a.RunPCIeLinkCheckPackCtx(context.Background(), baseDir, logFunc)
 }
 
-func (a *App) RunPCIeLinkCheckPackResult(baseDir string) (ActionResult, error) {
-	path, err := a.RunPCIeLinkCheckPack(baseDir, nil)
-	return ActionResult{Title: "PCIe Link Check", Body: satResultBody(path)}, err
-}
-
 func (a *App) RunNvidiaPCIeBandwidthPackCtx(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
 	if strings.TrimSpace(baseDir) == "" {
 		baseDir = DefaultSATBaseDir
@@ -308,11 +293,6 @@ func (a *App) RunNvidiaPCIeBandwidthPack(baseDir string, gpuIndices []int, logFu
 	return a.RunNvidiaPCIeBandwidthPackCtx(context.Background(), baseDir, gpuIndices, logFunc)
 }
 
-func (a *App) RunNvidiaPCIeBandwidthPackResult(baseDir string) (ActionResult, error) {
-	path, err := a.RunNvidiaPCIeBandwidthPack(baseDir, nil, nil)
-	return ActionResult{Title: "NVIDIA GPU PCIe Bandwidth Check", Body: satResultBody(path)}, err
-}
-
 func (a *App) DetectGPUVendor() string {
 	return a.sat.DetectGPUVendor()
 }
@@ -332,11 +312,6 @@ func (a *App) RunAMDAcceptancePackCtx(ctx context.Context, baseDir string, logFu
 	return a.sat.RunAMDAcceptancePack(ctx, baseDir, logFunc)
 }
 
-func (a *App) RunAMDAcceptancePackResult(baseDir string) (ActionResult, error) {
-	path, err := a.RunAMDAcceptancePack(baseDir, nil)
-	return ActionResult{Title: "AMD GPU SAT", Body: satResultBody(path)}, err
-}
-
 func (a *App) RunAMDMemIntegrityPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
 	if strings.TrimSpace(baseDir) == "" {
 		baseDir = DefaultSATBaseDir
@@ -399,69 +374,6 @@ func (a *App) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int
 	return a.sat.RunNCCLTests(ctx, baseDir, gpuIndices, logFunc)
 }
 
-func (a *App) RunNCCLTestsResult(ctx context.Context) (ActionResult, error) {
-	path, err := a.RunNCCLTests(ctx, DefaultSATBaseDir, nil, nil)
-	body := "Results: " + path
-	if err != nil && err != context.Canceled {
-		body += "\nERROR: " + err.Error()
-	}
-	return ActionResult{Title: "NCCL bandwidth test", Body: body}, err
-}
-
-func (a *App) RunFanStressTestResult(ctx context.Context, opts platform.FanStressOptions) (ActionResult, error) {
-	path, err := a.RunFanStressTest(ctx, "", opts)
-	body := formatFanStressResult(path)
-	if err != nil && err != context.Canceled {
-		body += "\nERROR: " + err.Error()
-	}
-	return ActionResult{Title: "GPU Platform Stress Test", Body: body}, err
-}
-
-// formatFanStressResult formats the summary.txt from a fan-stress run, including
-// the per-step pass/fail display and the analysis section (throttling, max temps, fan response).
-func formatFanStressResult(archivePath string) string {
-	if archivePath == "" {
-		return "No output produced."
-	}
-	runDir := strings.TrimSuffix(archivePath, ".tar.gz")
-	raw, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
-	if err != nil {
-		return "Archive written to " + archivePath
-	}
-	content := strings.TrimSpace(string(raw))
-	kv := parseKeyValueSummary(content)
-
-	var b strings.Builder
-	b.WriteString(formatSATDetail(content))
-
-	// Append analysis section.
-	var analysis []string
-	if v, ok := kv["throttling_detected"]; ok {
-		label := "NO"
-		if v == "true" {
-			label = "YES  ← throttling detected during load"
-		}
-		analysis = append(analysis, "Throttling:   "+label)
-	}
-	if v, ok := kv["max_gpu_temp_c"]; ok && v != "0.0" {
-		analysis = append(analysis, "Max GPU temp: "+v+"°C")
-	}
-	if v, ok := kv["max_cpu_temp_c"]; ok && v != "0.0" {
-		analysis = append(analysis, "Max CPU temp: "+v+"°C")
-	}
-	if v, ok := kv["fan_response_sec"]; ok && v != "N/A" && v != "-1.0" {
-		analysis = append(analysis, "Fan response: "+v+"s")
-	}
-
-	if len(analysis) > 0 {
-		b.WriteString("\n\n=== Analysis ===\n")
-		for _, line := range analysis {
-			b.WriteString(line + "\n")
-		}
-	}
-	return strings.TrimSpace(b.String())
-}
-
 // satResultBody reads summary.txt from the SAT run directory (archive path without .tar.gz)
 // and returns a formatted human-readable result. Falls back to a plain message if unreadable.
 func satResultBody(archivePath string) string {

audit/internal/app/app_test.go

@@ -216,7 +216,7 @@ func (f fakeSAT) RunNvidiaPulseTestPack(_ context.Context, baseDir string, durat
 	return f.runNvidiaFn(baseDir)
 }
 
-func (f fakeSAT) RunNvidiaBandwidthPack(_ context.Context, baseDir string, gpuIndices []int, _ func(string)) (string, error) {
+func (f fakeSAT) RunNvidiaBandwidthPack(_ context.Context, baseDir string, gpuIndices []int, _ bool, _ func(string)) (string, error) {
 	if f.runNvidiaBandwidthFn != nil {
 		return f.runNvidiaBandwidthFn(baseDir, gpuIndices)
 	}
@@ -287,6 +287,10 @@ func (f fakeSAT) RunTPMValidationPack(_ context.Context, _ string, _ func(string
 	return "", nil
 }
 
+func (f fakeSAT) TPMPresent() bool { return true }
+
+func (f fakeSAT) PhysicalGPUVendors() (bool, bool) { return false, false }
+
 func (f fakeSAT) RunNvidiaConfigCheckPack(_ context.Context, baseDir string, _ func(string)) (string, error) {
 	return "", nil
 }
@@ -725,6 +729,7 @@ func TestActionResultsUseFallbackBody(t *testing.T) {
 
 func TestExportSupportBundleResultMentionsUnmountedUSB(t *testing.T) {
 	tmp := t.TempDir()
+	t.Setenv("TMPDIR", tmp)
 	oldExportDir := DefaultExportDir
 	DefaultExportDir = tmp
 	t.Cleanup(func() { DefaultExportDir = oldExportDir })
@@ -761,6 +766,7 @@ func TestExportSupportBundleResultMentionsUnmountedUSB(t *testing.T) {
 
 func TestExportSupportBundleResultDoesNotPretendSuccessOnError(t *testing.T) {
 	tmp := t.TempDir()
+	t.Setenv("TMPDIR", tmp)
 	oldExportDir := DefaultExportDir
 	DefaultExportDir = tmp
 	t.Cleanup(func() { DefaultExportDir = oldExportDir })
@@ -939,6 +945,10 @@ func TestApplySATOverlayFiltersIgnoredLegacyDevices(t *testing.T) {
 
 func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) {
 	tmp := t.TempDir()
+	// Isolate os.TempDir() so a concurrent support-bundle test (this or the
+	// webui endpoint test, in another package running in parallel) cannot
+	// collide on the staging tree / archive path.
+	t.Setenv("TMPDIR", tmp)
 	exportDir := filepath.Join(tmp, "export")
 	if err := os.MkdirAll(filepath.Join(exportDir, "bee-sat", "memory-run"), 0755); err != nil {
 		t.Fatal(err)
@@ -1065,6 +1075,7 @@ func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) {
 // too.
 func TestBuildSupportBundleIncludesOrientationDocs(t *testing.T) {
 	tmp := t.TempDir()
+	t.Setenv("TMPDIR", tmp)
 	exportDir := filepath.Join(tmp, "export")
 	if err := os.MkdirAll(exportDir, 0755); err != nil {
 		t.Fatal(err)

audit/internal/app/component_status_db_test.go

@@ -233,7 +233,6 @@ func TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets(t *tes
 	}
 }
 
-
 // TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the
 // hardware-ingest-contract.md rule that status_history is a transition log
 // ("История переходов статусов"), not a per-poll journal. A component

audit/internal/app/support_bundle.go

@@ -455,11 +455,21 @@ func BuildSupportBundle(exportDir string) (string, error) {
 
 	now := time.Now().UTC()
 
-	stageRoot := filepath.Join(os.TempDir(), fmt.Sprintf("bee-support-stage-%s-%s", sanitizeFilename(hostnameOr("unknown")), now.Format("20060102-150405")))
+	// Stage under a private parent dir. Two bundle builds started in the same
+	// wall-clock second (two operators, or an on-demand build racing the
+	// blackbox worker) must not share a staging tree: one's deferred
+	// os.RemoveAll would then wipe the other's half-populated tree and yield a
+	// truncated archive. The leaf name stays meaningful because it becomes the
+	// archive's top-level directory.
+	buildParent, err := os.MkdirTemp(os.TempDir(), "bee-support-build-")
+	if err != nil {
+		return "", err
+	}
+	defer os.RemoveAll(buildParent)
+	stageRoot := filepath.Join(buildParent, fmt.Sprintf("bee-support-stage-%s-%s", sanitizeFilename(hostnameOr("unknown")), now.Format("20060102-150405")))
 	if err := os.MkdirAll(stageRoot, 0755); err != nil {
 		return "", err
 	}
-	defer os.RemoveAll(stageRoot)
 
 	if err := categorizeExportTree(exportDir, stageRoot, true); err != nil {
 		return "", err
@@ -513,10 +523,6 @@ func SupportBundleBaseName(at time.Time) string {
 	return fmt.Sprintf("%s (BEE-SP v%s) %s %s %s", date, ver, model, sn, tod)
 }
 
-func LatestSupportBundlePath() (string, error) {
-	return latestSupportBundlePath(os.TempDir())
-}
-
 func cleanupOldSupportBundles(dir string) error {
 	matches, err := filepath.Glob(filepath.Join(dir, supportBundleGlob))
 	if err != nil {
@@ -538,18 +544,6 @@ func cleanupOldSupportBundles(dir string) error {
 	return nil
 }
 
-func latestSupportBundlePath(dir string) (string, error) {
-	matches, err := filepath.Glob(filepath.Join(dir, supportBundleGlob))
-	if err != nil {
-		return "", err
-	}
-	ordered := orderSupportBundles(supportBundleEntries(matches))
-	if len(ordered) == 0 {
-		return "", os.ErrNotExist
-	}
-	return ordered[0], nil
-}
-
 func supportBundleEntries(matches []string) map[string]time.Time {
 	entries := make(map[string]time.Time, len(matches))
 	for _, match := range matches {
@@ -758,24 +752,6 @@ func buildCommit() string {
 	return strings.TrimSpace(string(raw))
 }
 
-func copyDirContents(srcDir, dstDir string) error {
-	entries, err := os.ReadDir(srcDir)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	for _, entry := range entries {
-		src := filepath.Join(srcDir, entry.Name())
-		dst := filepath.Join(dstDir, entry.Name())
-		if err := copyPath(src, dst); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func copyDirContentsFiltered(srcDir, dstDir string, keep func(rel string, info os.FileInfo) bool) error {
 	entries, err := os.ReadDir(srcDir)
 	if err != nil {

audit/internal/collector/nic_mellanox.go

@@ -30,7 +30,7 @@ var (
 		return string(out), nil
 	}
 
-	netIfacesByBDF = listNetIfacesByBDF
+	netIfacesByBDF     = listNetIfacesByBDF
 	readNetCarrierFile = func(iface string) (string, error) {
 		path := filepath.Join("/sys/class/net", iface, "carrier")
 		raw, err := os.ReadFile(path)

audit/internal/collector/sensors.go

@@ -3,7 +3,6 @@ package collector
 import (
 	"bee/audit/internal/schema"
 	"encoding/json"
-	"log/slog"
 	"os/exec"
 	"sort"
 	"strconv"
@@ -12,25 +11,6 @@ import (
 
 type sensorsDoc map[string]map[string]any
 
-func collectSensors() *schema.HardwareSensors {
-	doc, err := readSensorsJSONDoc()
-	if err != nil {
-		slog.Info("sensors: unavailable, skipping", "err", err)
-		return nil
-	}
-	sensors := buildSensorsFromDoc(doc)
-	if sensors == nil || (len(sensors.Fans) == 0 && len(sensors.Power) == 0 && len(sensors.Temperatures) == 0 && len(sensors.Other) == 0) {
-		return nil
-	}
-	slog.Info("sensors: collected",
-		"fans", len(sensors.Fans),
-		"power", len(sensors.Power),
-		"temperatures", len(sensors.Temperatures),
-		"other", len(sensors.Other),
-	)
-	return sensors
-}
-
 func readSensorsJSONDoc() (sensorsDoc, error) {
 	out, err := exec.Command("sensors", "-j").Output()
 	if err != nil {

audit/internal/collector/storage.go

@@ -280,9 +280,9 @@ type smartctlInfo struct {
 	ScsiVendor    string `json:"scsi_vendor"`
 	ScsiProduct   string `json:"scsi_product"`
 	SerialNumber  string `json:"serial_number"`
-	FirmwareVer  string `json:"firmware_version"`
-	RotationRate int    `json:"rotation_rate"`
-	Temperature  struct {
+	FirmwareVer   string `json:"firmware_version"`
+	RotationRate  int    `json:"rotation_rate"`
+	Temperature   struct {
 		Current int `json:"current"`
 	} `json:"temperature"`
 	SmartStatus struct {

audit/internal/platform/benchmark_analysis.go

@@ -0,0 +1,659 @@
+package platform
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+type benchmarkPlannedPhase struct {
+	PlanLabel   string
+	MetricStage string
+	DurationSec int
+}
+
+func runBenchmarkPlannedCommandWithMetrics(
+	ctx context.Context,
+	verboseLog, name string,
+	cmd []string,
+	env []string,
+	gpuIndices []int,
+	phases []benchmarkPlannedPhase,
+	logFunc func(string),
+) ([]byte, map[string][]GPUMetricRow, map[string][]byte, error) {
+	out, rows, err := runBenchmarkCommandWithMetrics(ctx, verboseLog, name, cmd, env, gpuIndices, logFunc)
+	return out, splitBenchmarkRowsByPlannedPhase(rows, phases), splitBenchmarkLogByPlannedPhase(out), err
+}
+
+func splitBenchmarkRowsByPlannedPhase(rows []GPUMetricRow, phases []benchmarkPlannedPhase) map[string][]GPUMetricRow {
+	out := make(map[string][]GPUMetricRow, len(phases))
+	if len(rows) == 0 || len(phases) == 0 {
+		return out
+	}
+	for _, row := range rows {
+		idx := len(phases) - 1
+		var elapsed float64
+		for i, phase := range phases {
+			durationSec := phase.DurationSec
+			if durationSec <= 0 {
+				durationSec = 1
+			}
+			elapsed += float64(durationSec)
+			if row.ElapsedSec < elapsed {
+				idx = i
+				break
+			}
+		}
+		out[phases[idx].MetricStage] = append(out[phases[idx].MetricStage], row)
+	}
+	return out
+}
+
+func splitBenchmarkLogByPlannedPhase(raw []byte) map[string][]byte {
+	out := make(map[string][]byte)
+	var current string
+	for _, line := range strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") {
+		trimmed := strings.TrimSpace(stripBenchmarkPrefix(line))
+		switch {
+		case strings.HasPrefix(trimmed, "phase_begin="):
+			current = strings.TrimSpace(strings.TrimPrefix(trimmed, "phase_begin="))
+		case strings.HasPrefix(trimmed, "phase_end="):
+			current = ""
+		case current != "":
+			out[current] = append(out[current], []byte(line+"\n")...)
+		}
+	}
+	return out
+}
+
+type benchmarkCoolingSample struct {
+	AvgFanRPM             float64
+	AvgFanDutyCyclePct    float64
+	FanDutyCycleAvailable bool
+	FanDutyCycleEstimated bool
+}
+
+func sampleBenchmarkTelemetry(gpuIndices []int) ([]GPUMetricRow, error) {
+	samples, err := sampleGPUMetrics(gpuIndices)
+	if err != nil {
+		return nil, err
+	}
+	fanSample := sampleBenchmarkCoolingSample()
+	for i := range samples {
+		samples[i].FanAvgRPM = fanSample.AvgFanRPM
+		samples[i].FanDutyCyclePct = fanSample.AvgFanDutyCyclePct
+		samples[i].FanDutyCycleAvailable = fanSample.FanDutyCycleAvailable
+		samples[i].FanDutyCycleEstimated = fanSample.FanDutyCycleEstimated
+	}
+	return samples, nil
+}
+
+func sampleBenchmarkCoolingSample() benchmarkCoolingSample {
+	fans, _ := sampleFanSpeeds()
+	avgRPM, _, _ := fanRPMStats(fans)
+	dutyPct, dutyAvailable, dutyEstimated := sampleFanDutyCyclePctFromFans(fans)
+	return benchmarkCoolingSample{
+		AvgFanRPM:             avgRPM,
+		AvgFanDutyCyclePct:    dutyPct,
+		FanDutyCycleAvailable: dutyAvailable,
+		FanDutyCycleEstimated: dutyEstimated,
+	}
+}
+
+func annotateBenchmarkMetricRows(rows []GPUMetricRow, stage string, offset, durationSec float64) []GPUMetricRow {
+	if len(rows) == 0 {
+		return nil
+	}
+	stageEnd := offset + durationSec
+	if stageEnd <= offset {
+		stageEnd = offset
+		for _, row := range rows {
+			if row.ElapsedSec+offset > stageEnd {
+				stageEnd = row.ElapsedSec + offset
+			}
+		}
+	}
+	out := make([]GPUMetricRow, len(rows))
+	for i, row := range rows {
+		row.Stage = stage
+		row.ElapsedSec += offset
+		row.StageStartSec = offset
+		row.StageEndSec = stageEnd
+		out[i] = row
+	}
+	return out
+}
+
+func appendBenchmarkMetrics(allRows *[]GPUMetricRow, rows []GPUMetricRow, stage string, cursor *float64, durationSec float64) {
+	annotated := annotateBenchmarkMetricRows(rows, stage, *cursor, durationSec)
+	*allRows = append(*allRows, annotated...)
+	*cursor += durationSec
+}
+
+func writeBenchmarkMetricsFiles(runDir string, rows []GPUMetricRow) {
+	if len(rows) == 0 {
+		return
+	}
+	_ = WriteGPUMetricsCSV(filepath.Join(runDir, "gpu-metrics.csv"), rows)
+	_ = WriteGPUMetricsHTML(filepath.Join(runDir, "gpu-metrics.html"), rows)
+}
+
+func appendBenchmarkStageLog(path, source, stage string, raw []byte) {
+	if path == "" || len(raw) == 0 {
+		return
+	}
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	header := fmt.Sprintf("\n========== %s | stage=%s ==========\n", source, stage)
+	_, _ = f.WriteString(header)
+	if len(raw) > 0 {
+		_, _ = f.Write(raw)
+		if raw[len(raw)-1] != '\n' {
+			_, _ = f.WriteString("\n")
+		}
+	}
+}
+
+func parseBenchmarkBurnLog(raw string) benchmarkBurnParseResult {
+	result := benchmarkBurnParseResult{}
+	lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
+	profiles := make(map[string]*benchmarkBurnProfile)
+	for _, line := range lines {
+		line = stripBenchmarkPrefix(strings.TrimSpace(line))
+		if line == "" {
+			continue
+		}
+		switch {
+		case strings.HasPrefix(line, "device="):
+			result.Device = strings.TrimSpace(strings.TrimPrefix(line, "device="))
+		case strings.HasPrefix(line, "compute_capability="):
+			result.ComputeCapability = strings.TrimSpace(strings.TrimPrefix(line, "compute_capability="))
+		case strings.HasPrefix(line, "backend="):
+			result.Backend = strings.TrimSpace(strings.TrimPrefix(line, "backend="))
+			result.Fallback = result.Backend == "driver-ptx"
+		case strings.HasPrefix(line, "duration_s="):
+			result.DurationSec, _ = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "duration_s=")))
+		default:
+			if m := benchmarkReadyPattern.FindStringSubmatch(line); len(m) == 6 {
+				profile := ensureBenchmarkProfile(profiles, m[1])
+				profile.supported = true
+				profile.lanes++
+				profile.m, _ = strconv.ParseUint(m[3], 10, 64)
+				profile.n, _ = strconv.ParseUint(m[4], 10, 64)
+				profile.k, _ = strconv.ParseUint(m[5], 10, 64)
+				continue
+			}
+			if m := benchmarkSkippedPattern.FindStringSubmatch(line); len(m) == 3 {
+				profile := ensureBenchmarkProfile(profiles, m[1])
+				profile.supported = false
+				profile.notes = strings.TrimSpace(m[2])
+				continue
+			}
+			if m := benchmarkIterationsPattern.FindStringSubmatch(line); len(m) == 3 {
+				profile := ensureBenchmarkProfile(profiles, m[1])
+				iters, _ := strconv.ParseUint(m[2], 10, 64)
+				profile.iterations += iters
+			}
+		}
+	}
+
+	keys := make([]string, 0, len(profiles))
+	for key := range profiles {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		profile := profiles[key]
+		precision := BenchmarkPrecisionResult{
+			Name:       profile.name,
+			Category:   profile.category,
+			Supported:  profile.supported,
+			Lanes:      profile.lanes,
+			M:          profile.m,
+			N:          profile.n,
+			K:          profile.k,
+			Iterations: profile.iterations,
+			Notes:      profile.notes,
+		}
+		w := precisionWeight(profile.category)
+		precision.Weight = w
+		if profile.supported && result.DurationSec > 0 && profile.m > 0 && profile.n > 0 && profile.k > 0 && profile.iterations > 0 {
+			precision.TeraOpsPerSec = (2.0 * float64(profile.m) * float64(profile.n) * float64(profile.k) * float64(profile.iterations)) / float64(result.DurationSec) / 1e12
+			precision.WeightedTeraOpsPerSec = precision.TeraOpsPerSec * w
+		}
+		result.Profiles = append(result.Profiles, precision)
+	}
+	return result
+}
+
+func ensureBenchmarkProfile(profiles map[string]*benchmarkBurnProfile, name string) *benchmarkBurnProfile {
+	if profile, ok := profiles[name]; ok {
+		return profile
+	}
+	category := "other"
+	switch {
+	case strings.HasPrefix(name, "fp64"):
+		category = "fp64"
+	case strings.HasPrefix(name, "fp32"):
+		category = "fp32_tf32"
+	case strings.HasPrefix(name, "fp16"):
+		category = "fp16_bf16"
+	case strings.HasPrefix(name, "int8"):
+		category = "int8"
+	case strings.HasPrefix(name, "fp8"):
+		category = "fp8"
+	case strings.HasPrefix(name, "fp4"):
+		category = "fp4"
+	}
+	profile := &benchmarkBurnProfile{name: name, category: category, supported: true}
+	profiles[name] = profile
+	return profile
+}
+
+// precisionWeight returns the fp32-equivalence factor for a precision category.
+// Each factor represents how much "real" numeric work one operation of that
+// type performs relative to fp32 (single precision = 1.0 baseline):
+//
+//	fp64  = 2.0  — double precision, 2× more bits per operand
+//	fp32  = 1.0  — single precision baseline
+//	fp16  = 0.5  — half precision
+//	int8  = 0.25 — quarter precision
+//	fp8   = 0.25 — quarter precision
+//	fp4   = 0.125 — eighth precision
+//
+// Multiplying raw TOPS by the weight gives fp32-equivalent TOPS, enabling
+// cross-precision comparison on the same numeric scale.
+func precisionWeight(category string) float64 {
+	switch category {
+	case "fp64":
+		return 2.0
+	case "fp32_tf32":
+		return 1.0
+	case "fp16_bf16":
+		return 0.5
+	case "int8":
+		return 0.25
+	case "fp8":
+		return 0.25
+	case "fp4":
+		return 0.125
+	default:
+		return 1.0
+	}
+}
+
+func stripBenchmarkPrefix(line string) string {
+	if strings.HasPrefix(line, "[gpu ") {
+		if idx := strings.Index(line, "] "); idx >= 0 {
+			return line[idx+2:]
+		}
+	}
+	return line
+}
+
+func summarizeBenchmarkTelemetry(rows []GPUMetricRow) BenchmarkTelemetrySummary {
+	summary := BenchmarkTelemetrySummary{}
+	if len(rows) == 0 {
+		return summary
+	}
+	temps := make([]float64, 0, len(rows))
+	powers := make([]float64, 0, len(rows))
+	clocks := make([]float64, 0, len(rows))
+	memClocks := make([]float64, 0, len(rows))
+	usages := make([]float64, 0, len(rows))
+	memUsages := make([]float64, 0, len(rows))
+	summary.DurationSec = rows[len(rows)-1].ElapsedSec
+	summary.Samples = len(rows)
+	for _, row := range rows {
+		temps = append(temps, row.TempC)
+		powers = append(powers, row.PowerW)
+		clocks = append(clocks, row.ClockMHz)
+		memClocks = append(memClocks, row.MemClockMHz)
+		usages = append(usages, row.UsagePct)
+		memUsages = append(memUsages, row.MemUsagePct)
+	}
+	summary.AvgTempC = benchmarkMean(temps)
+	summary.P95TempC = benchmarkPercentile(temps, 95)
+	summary.AvgPowerW = benchmarkMean(powers)
+	summary.P95PowerW = benchmarkPercentile(powers, 95)
+	summary.AvgGraphicsClockMHz = benchmarkMean(clocks)
+	summary.P95GraphicsClockMHz = benchmarkPercentile(clocks, 95)
+	summary.AvgMemoryClockMHz = benchmarkMean(memClocks)
+	summary.P95MemoryClockMHz = benchmarkPercentile(memClocks, 95)
+	summary.AvgUsagePct = benchmarkMean(usages)
+	summary.AvgMemUsagePct = benchmarkMean(memUsages)
+	summary.ClockCVPct = benchmarkCV(clocks)
+	summary.PowerCVPct = benchmarkCV(powers)
+	summary.TempCVPct = benchmarkCV(temps)
+	summary.ClockDriftPct = benchmarkClockDrift(clocks)
+	return summary
+}
+
+func summarizeBenchmarkCooling(rows []GPUMetricRow) *BenchmarkCoolingSummary {
+	if len(rows) == 0 {
+		return nil
+	}
+	var rpmValues []float64
+	var dutyValues []float64
+	var dutyEstimated bool
+	for _, row := range rows {
+		if row.FanAvgRPM > 0 {
+			rpmValues = append(rpmValues, row.FanAvgRPM)
+		}
+		if row.FanDutyCycleAvailable {
+			dutyValues = append(dutyValues, row.FanDutyCyclePct)
+			if row.FanDutyCycleEstimated {
+				dutyEstimated = true
+			}
+		}
+	}
+	if len(rpmValues) == 0 && len(dutyValues) == 0 {
+		return nil
+	}
+	summary := &BenchmarkCoolingSummary{
+		Available:             true,
+		AvgFanRPM:             benchmarkMean(rpmValues),
+		FanDutyCycleEstimated: dutyEstimated,
+	}
+	if len(dutyValues) > 0 {
+		summary.FanDutyCycleAvailable = true
+		summary.AvgFanDutyCyclePct = benchmarkMean(dutyValues)
+		summary.P95FanDutyCyclePct = benchmarkPercentile(dutyValues, 95)
+		if summary.FanDutyCycleEstimated {
+			summary.Notes = append(summary.Notes, "fan duty cycle is estimated from the highest fan RPM observed since boot; treat it as an approximation, not a direct PWM reading")
+		}
+	} else {
+		summary.Notes = append(summary.Notes, "fan duty cycle unavailable on this host; RPM-only fan telemetry was collected")
+	}
+	return summary
+}
+
+func benchmarkTelemetryAvailable(summary BenchmarkTelemetrySummary) bool {
+	return summary.Samples > 0 || summary.DurationSec > 0
+}
+
+func benchmarkPrecisionSteadyFallback(phases []BenchmarkPrecisionSteadyPhase) (BenchmarkTelemetrySummary, string, bool) {
+	var (
+		best      BenchmarkTelemetrySummary
+		bestLabel string
+		found     bool
+	)
+	for _, phase := range phases {
+		if !benchmarkTelemetryAvailable(phase.Steady) {
+			continue
+		}
+		if !found ||
+			phase.Steady.DurationSec > best.DurationSec ||
+			(phase.Steady.DurationSec == best.DurationSec && phase.Steady.P95PowerW > best.P95PowerW) {
+			best = phase.Steady
+			bestLabel = phase.Precision
+			found = true
+		}
+	}
+	return best, bestLabel, found
+}
+
+func applyBenchmarkSteadyFallback(gpu *BenchmarkGPUResult) {
+	if gpu == nil || benchmarkTelemetryAvailable(gpu.Steady) {
+		return
+	}
+	if fallback, label, ok := benchmarkPrecisionSteadyFallback(gpu.PrecisionSteady); ok {
+		gpu.Steady = fallback
+		gpu.Notes = append(gpu.Notes,
+			fmt.Sprintf("mixed steady telemetry unavailable; reporting steady-state fallback from %s precision phase", label))
+	}
+}
+
+func scoreBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkScorecard {
+	score := BenchmarkScorecard{}
+
+	// SyntheticScore: sum of fp32-equivalent TOPS from per-precision phases.
+	// Each precision ran alone with full GPU dedicated — peak capability.
+	for _, p := range gpu.PrecisionSteady {
+		if !benchmarkPrecisionEnabled(p.Precision) {
+			continue
+		}
+		score.SyntheticScore += p.WeightedTeraOpsPerSec
+	}
+
+	// MixedScore: sum of fp32-equivalent TOPS from the combined phase.
+	// All precisions compete simultaneously — closer to real inference workloads.
+	for _, p := range gpu.PrecisionResults {
+		if p.Supported && benchmarkPrecisionEnabled(p.Category) {
+			score.MixedScore += p.WeightedTeraOpsPerSec
+		}
+	}
+
+	// MixedEfficiency = MixedScore / SyntheticScore.
+	// Measures how well the GPU sustains throughput under concurrent mixed load.
+	// A healthy GPU scores ~0.8–0.95; severe degradation suggests bandwidth
+	// contention or scheduler inefficiency.
+	if score.SyntheticScore > 0 && score.MixedScore > 0 {
+		score.MixedEfficiency = score.MixedScore / score.SyntheticScore
+	}
+
+	// ComputeScore = SyntheticScore × (1 + MixedEfficiency × 0.3).
+	// SyntheticScore is the primary signal; MixedEfficiency adds up to +30%
+	// bonus for GPUs that handle mixed-precision concurrency well.
+	// Falls back to MixedScore alone when per-precision data is absent.
+	switch {
+	case score.SyntheticScore > 0:
+		score.ComputeScore = score.SyntheticScore * (1 + score.MixedEfficiency*0.3)
+	case score.MixedScore > 0:
+		score.ComputeScore = score.MixedScore
+	}
+	// PowerSustainScore: how stable is GPU power draw during the benchmark?
+	// High variance means the workload is bursting or the power delivery is
+	// unstable. Score = max(0, 100 − PowerCVPct × 3).
+	// At 10% CV → score 70; at 33%+ CV → score 0.
+	// Uses per-precision windows when available (each runs a single kernel,
+	// so CV reflects genuine power regulation, not workload switching).
+	if len(gpu.PrecisionSteady) > 0 {
+		var sum float64
+		var count int
+		for _, p := range gpu.PrecisionSteady {
+			if !benchmarkPrecisionEnabled(p.Precision) {
+				continue
+			}
+			sum += clampScore(100 - p.Steady.PowerCVPct*3)
+			count++
+		}
+		if count > 0 {
+			score.PowerSustainScore = sum / float64(count)
+		}
+	} else if gpu.Steady.PowerCVPct > 0 {
+		score.PowerSustainScore = clampScore(100 - gpu.Steady.PowerCVPct*3)
+	}
+
+	// ThermalSustainScore: how stable is GPU temperature during the benchmark?
+	// High variance means cooling is inconsistent (fan bursts, liquid flow
+	// instability, or frequent transitions in and out of throttle).
+	// Score = max(0, 100 − TempCVPct × 3).
+	if gpu.Steady.TempCVPct > 0 {
+		score.ThermalSustainScore = clampScore(100 - gpu.Steady.TempCVPct*3)
+	} else {
+		// TempCV not recorded — fall back to 100 (no penalty).
+		score.ThermalSustainScore = 100
+	}
+
+	// Throttle breakdown: compute per-type percentages for diagnosis.
+	// Each counter measures microseconds spent in that throttle state during
+	// the steady-state window. Counters can overlap (e.g. thermal + power cap
+	// simultaneously), so they are reported independently, not summed.
+	runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6)
+	score.ThermalThrottlePct = math.Min(100,
+		float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS*100)
+	score.PowerCapThrottlePct = math.Min(100,
+		float64(gpu.Throttle.SWPowerCapUS)/runtimeUS*100)
+	score.SyncBoostThrottlePct = math.Min(100,
+		float64(gpu.Throttle.SyncBoostUS)/runtimeUS*100)
+
+	// StabilityScore: combined throttle signal (thermal + power cap).
+	// Score = max(0, 100 − combined_throttle_pct).
+	// 1% throttle → 99; 10% → 90; any throttle > 0 is penalised.
+	combinedThrottlePct := math.Min(100,
+		float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS+gpu.Throttle.SWPowerCapUS)/runtimeUS*100)
+	score.StabilityScore = clampScore(100 - combinedThrottlePct)
+
+	// TempHeadroomC: distance from p95 temperature to the GPU's hardware
+	// shutdown threshold (sourced from nvidia-smi -q "GPU Shutdown Temp").
+	// Fallback: 90°C when not available.
+	// Assessed independently of throttle — a GPU at 86°C without any throttle
+	// counter still has limited headroom and operates in degraded reliability zone.
+	// Warning zone: headroom < (shutdownTemp - slowdownTemp), i.e. past slowdown onset.
+	// Critical zone: headroom < 10°C from shutdown.
+	if gpu.Steady.P95TempC > 0 {
+		shutdownTemp := gpu.ShutdownTempC
+		if shutdownTemp <= 0 {
+			shutdownTemp = 90
+		}
+		score.TempHeadroomC = shutdownTemp - gpu.Steady.P95TempC
+	}
+	score.ServerQualityScore = serverQualityScore(score)
+	score.CompositeScore = score.ComputeScore
+	if gpu.MultiprocessorCount > 0 && gpu.Steady.AvgGraphicsClockMHz > 0 && score.ComputeScore > 0 {
+		score.TOPSPerSMPerGHz = score.ComputeScore / float64(gpu.MultiprocessorCount) / (gpu.Steady.AvgGraphicsClockMHz / 1000.0)
+	}
+	return score
+}
+
+// compositeBenchmarkScore is kept for compatibility with legacy callers.
+// CompositeScore = ComputeScore (no quality multiplier; throttling already
+// reduces TOPS directly, so no additional penalty is needed).
+func compositeBenchmarkScore(score BenchmarkScorecard) float64 {
+	return score.ComputeScore
+}
+
+// serverQualityScore returns a 0–100 score reflecting server infrastructure
+// quality, independent of GPU model or compute speed.
+//
+//	StabilityScore (throttle time)   0.40 — heaviest: direct evidence GPU can't sustain load
+//	PowerSustainScore (power CV)     0.30 — unstable draw hints at PSU/VRM issues
+//	ThermalSustainScore (temp CV)    0.30 — unstable temp hints at airflow/cooling issues
+func serverQualityScore(score BenchmarkScorecard) float64 {
+	q := 0.40*(score.StabilityScore/100.0) +
+		0.30*(score.PowerSustainScore/100.0) +
+		0.30*(score.ThermalSustainScore/100.0)
+	return clampScore(q * 100)
+}
+
+// detectPowerAnomaly scans per-GPU steady-state metric rows for a sudden
+// power drop — a symptom of bad cable contact, VRM fault, or thermal event
+// on the power delivery path. Returns a non-empty string if an anomaly is found.
+//
+// Algorithm: uses a 5-sample rolling baseline; flags any sample that falls
+// more than 30% below the baseline while the GPU was otherwise loaded
+// (usage > 50%). A sustained throttle (power cap) is not flagged here —
+// that is already captured by PowerCapThrottlePct.
+func detectPowerAnomaly(rows []GPUMetricRow, gpuIndex int) string {
+	const windowSize = 5
+	const dropThresholdPct = 30.0
+	const minUsagePct = 50.0
+
+	// Filter rows for this GPU during steady state only.
+	var steady []GPUMetricRow
+	for _, r := range rows {
+		if r.GPUIndex == gpuIndex && r.Stage != "" && strings.Contains(r.Stage, "steady") {
+			steady = append(steady, r)
+		}
+	}
+	if len(steady) < windowSize+2 {
+		return ""
+	}
+
+	// Compute initial baseline from the first window.
+	var baseSum float64
+	for i := 0; i < windowSize; i++ {
+		baseSum += steady[i].PowerW
+	}
+
+	for i := windowSize; i < len(steady); i++ {
+		baseline := baseSum / float64(windowSize)
+		sample := steady[i]
+		if baseline > 0 && sample.UsagePct >= minUsagePct {
+			dropPct := (baseline - sample.PowerW) / baseline * 100
+			if dropPct >= dropThresholdPct {
+				return fmt.Sprintf("sudden power drop detected at t=%.0fs: %.0f W → %.0f W (%.0f%% below rolling baseline) — possible bad cable contact or VRM fault",
+					sample.ElapsedSec, baseline, sample.PowerW, dropPct)
+			}
+		}
+		// Slide the window baseline.
+		baseSum -= steady[i-windowSize].PowerW
+		baseSum += sample.PowerW
+	}
+	return ""
+}
+
+// detectSlowdownTempExceedance scans steady-state metric rows for a GPU and
+// returns a warning string if any temperature sample exceeded the GPU's
+// SlowdownTempC threshold. Uses fallback 80°C when SlowdownTempC is zero.
+// This is a real-time signal distinct from p95 stats — even a single spike
+// above the slowdown threshold is worth flagging.
+func detectSlowdownTempExceedance(rows []GPUMetricRow, gpuIndex int, slowdownTempC float64) string {
+	if slowdownTempC <= 0 {
+		slowdownTempC = 80
+	}
+	var maxTemp float64
+	var exceedCount int
+	for _, r := range rows {
+		if r.GPUIndex != gpuIndex {
+			continue
+		}
+		if !strings.Contains(r.Stage, "steady") {
+			continue
+		}
+		if r.TempC > maxTemp {
+			maxTemp = r.TempC
+		}
+		if r.TempC >= slowdownTempC {
+			exceedCount++
+		}
+	}
+	if exceedCount == 0 {
+		return ""
+	}
+	return fmt.Sprintf(
+		"temperature exceeded slowdown threshold (%.0f°C) in %d sample(s) during steady state — peak %.1f°C",
+		slowdownTempC, exceedCount, maxTemp)
+}
+
+func detectBenchmarkDegradationReasons(gpu BenchmarkGPUResult, normalizationStatus string) []string {
+	var reasons []string
+	runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6)
+	if float64(gpu.Throttle.SWPowerCapUS)/runtimeUS >= 0.05 {
+		reasons = append(reasons, "power_capped")
+	}
+	if float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS >= 0.01 {
+		reasons = append(reasons, "thermal_limited")
+	}
+	if float64(gpu.Throttle.SyncBoostUS)/runtimeUS >= 0.01 {
+		reasons = append(reasons, "sync_boost_limited")
+	}
+	if gpu.LockedGraphicsClockMHz > 0 && gpu.Steady.AvgGraphicsClockMHz < gpu.LockedGraphicsClockMHz*0.90 {
+		reasons = append(reasons, "low_sm_clock_vs_target")
+	}
+	if gpu.Scores.StabilityScore > 0 && gpu.Scores.StabilityScore < 85 {
+		reasons = append(reasons, "variance_too_high")
+	}
+	if normalizationStatus != "full" {
+		reasons = append(reasons, "normalization_partial")
+	}
+	if gpu.PowerLimitDerated {
+		reasons = append(reasons, "power_limit_derated")
+	}
+	if gpu.ECC.Uncorrected > 0 {
+		reasons = append(reasons, "ecc_uncorrected_errors")
+	}
+	if gpu.ECC.Corrected > 0 {
+		reasons = append(reasons, "ecc_corrected_errors")
+	}
+	return dedupeStrings(reasons)
+}

audit/internal/platform/benchmark_power_autotune.go

@@ -69,16 +69,6 @@ func LoadSystemPowerSourceConfig(exportDir string) (*BenchmarkPowerAutotuneConfi
 	return LoadBenchmarkPowerAutotuneConfig(BenchmarkPowerSourceConfigPath(exportDir))
 }
 
-func ResetBenchmarkPowerAutotuneConfig(path string) error {
-	if strings.TrimSpace(path) == "" {
-		return fmt.Errorf("empty autotune config path")
-	}
-	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	return nil
-}
-
 func normalizeBenchmarkPowerSource(source string) string {
 	switch strings.TrimSpace(strings.ToLower(source)) {
 	case BenchmarkPowerSourceSDRPSUInput:

audit/internal/platform/benchmark_power_calibration.go

@@ -0,0 +1,928 @@
+package platform
+
+import (
+	"bee/audit/internal/collector"
+	"context"
+	"fmt"
+	"math"
+	"os"
+	"os/exec"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func detectDCMIPartialCoverage(sp *BenchmarkServerPower) bool {
+	if sp == nil || !sp.Available {
+		return false
+	}
+	if sp.PSUInputIdleW > 0 && sp.IdleW > 0 {
+		return sp.IdleW/sp.PSUInputIdleW < 0.7
+	}
+	if sp.PSUInputLoadedW > 0 && sp.LoadedW > 0 {
+		return sp.LoadedW/sp.PSUInputLoadedW < 0.7
+	}
+	return false
+}
+
+// detectIPMISaturationFallback is the heuristic used when SDR PSU sensors are
+// absent. It analyses the power ramp: if 2+ of the last 3 per-step incremental
+// DCMI/GPU ratios fall below 25% of the first-step ratio, DCMI has likely
+// plateaued while GPU load continued to grow (saturation proxy).
+// Prefer detectDCMIPartialCoverage when SDR data is available.
+func detectIPMISaturationFallback(steps []NvidiaPowerBenchStep) bool {
+	type pt struct{ incIPMI, incGPU float64 }
+	var pts []pt
+	for i := 1; i < len(steps); i++ {
+		if steps[i].ServerDeltaW <= 0 || steps[i-1].ServerDeltaW <= 0 {
+			continue
+		}
+		incIPMI := steps[i].ServerDeltaW - steps[i-1].ServerDeltaW
+		incGPU := steps[i].TotalObservedPowerW - steps[i-1].TotalObservedPowerW
+		if incGPU <= 0 {
+			continue
+		}
+		pts = append(pts, pt{incIPMI, incGPU})
+	}
+	if len(pts) < 3 {
+		return false
+	}
+	refRatio := pts[0].incIPMI / pts[0].incGPU
+	if refRatio <= 0 {
+		return false
+	}
+	saturated := 0
+	for _, p := range pts[len(pts)-3:] {
+		if p.incIPMI/p.incGPU < refRatio*0.25 {
+			saturated++
+		}
+	}
+	return saturated >= 2
+}
+
+// psuStatusSnapshot samples PSU health sensor states via
+// `ipmitool sdr type "Power Supply"`. Returns a map of sensor name → reading
+// string (e.g. "Presence detected", "Failure detected"). Returns nil when IPMI
+// is unavailable or no Power Supply entity sensors are present.
+func psuStatusSnapshot() map[string]string {
+	out, err := exec.Command("ipmitool", "sdr", "type", "Power Supply").Output()
+	if err != nil || len(out) == 0 {
+		return nil
+	}
+	result := make(map[string]string)
+	for _, line := range strings.Split(string(out), "\n") {
+		parts := strings.Split(line, "|")
+		if len(parts) < 5 {
+			continue
+		}
+		name := strings.TrimSpace(parts[0])
+		reading := strings.TrimSpace(parts[4])
+		if name == "" {
+			continue
+		}
+		result[name] = reading
+	}
+	return result
+}
+
+// diffPSUStatus compares PSU sensor snapshots taken before and after a test.
+// Returns human-readable fault strings for sensors that entered a fault state
+// during the test. Pre-existing faults (present in both snapshots) are excluded
+// so that only new anomalies caused by the test are reported.
+func diffPSUStatus(before, after map[string]string) []string {
+	if len(after) == 0 {
+		return nil
+	}
+	isFault := func(s string) bool {
+		lower := strings.ToLower(s)
+		return strings.Contains(lower, "failure") ||
+			strings.Contains(lower, "fault") ||
+			strings.Contains(lower, "warning") ||
+			strings.Contains(lower, "predictive") ||
+			strings.Contains(lower, "absent") ||
+			strings.Contains(lower, "ac lost")
+	}
+	var issues []string
+	for name, afterReading := range after {
+		if !isFault(afterReading) {
+			continue
+		}
+		if beforeReading, had := before[name]; had && isFault(beforeReading) {
+			continue // pre-existing fault, not caused by this test
+		}
+		if prev, had := before[name]; had {
+			issues = append(issues, fmt.Sprintf("%s: changed from %q to %q during test", name, prev, afterReading))
+		} else {
+			issues = append(issues, fmt.Sprintf("%s: %s (appeared after test start)", name, afterReading))
+		}
+	}
+	sort.Strings(issues)
+	return issues
+}
+
+// sdrPowerSnapshot holds per-source power sums from a single `ipmitool sdr` read.
+type sdrPowerSnapshot struct {
+	PSUInW   float64 // sum of PSU AC input across all slots
+	PSUOutW  float64 // sum of PSU DC output across all slots
+	GPUSlotW float64 // sum of GPU slot/GPU power sensors
+
+	// Per-slot PSU data from collector.PSUSlotsFromSDR — same slot keys as
+	// audit HardwarePowerSupply.Slot (0-based strings).
+	PSUSlots map[string]BenchmarkPSUSlotPower
+
+	SkippedSensors []string // sensors rejected during self-healing
+}
+
+type benchmarkSDRSeriesSummary struct {
+	PSUInW   float64
+	PSUOutW  float64
+	GPUSlotW float64
+	PSUSlots map[string]BenchmarkPSUSlotPower
+	Samples  int
+
+	SkippedSensors []string
+}
+
+// sdrSensor is a name+watts pair used for GPU slot self-healing filtering.
+type sdrSensor struct {
+	name  string
+	watts float64
+}
+
+// filterSensorGroup removes physically implausible readings from a group.
+// Hard bounds: 0 < watts ≤ maxPerSensorW. Within groups of 2+ sensors,
+// values more than 5× the group median are rejected as stuck/fault sensors.
+func filterSensorGroup(sensors []sdrSensor, maxPerSensorW float64) (valid []sdrSensor, skipped []string) {
+	var inBounds []sdrSensor
+	for _, s := range sensors {
+		if s.watts <= 0 || s.watts > maxPerSensorW {
+			skipped = append(skipped, fmt.Sprintf("%s (%.0f W: out of range 0–%.0f W)", s.name, s.watts, maxPerSensorW))
+		} else {
+			inBounds = append(inBounds, s)
+		}
+	}
+	if len(inBounds) < 2 {
+		return inBounds, skipped
+	}
+	vals := make([]float64, len(inBounds))
+	for i, s := range inBounds {
+		vals[i] = s.watts
+	}
+	sort.Float64s(vals)
+	mid := len(vals) / 2
+	var median float64
+	if len(vals)%2 == 0 {
+		median = (vals[mid-1] + vals[mid]) / 2
+	} else {
+		median = vals[mid]
+	}
+	for _, s := range inBounds {
+		if median > 0 && s.watts > median*5 {
+			skipped = append(skipped, fmt.Sprintf("%s (%.0f W: >5× median %.0f W, likely sensor fault)", s.name, s.watts, median))
+		} else {
+			valid = append(valid, s)
+		}
+	}
+	return valid, skipped
+}
+
+// sampleIPMISDRPowerSensors reads power sensors from `ipmitool sdr` in a single
+// invocation and returns self-healed grouped sums.
+//
+// PSU identification delegates to collector.PSUSlotsFromSDR which uses the same
+// slot-detection regexes as the hardware audit (PSU1_POWER_IN, PSU1_PIN, PS1 POut,
+// Power1…). Self-healing: bounds checking + 5× median outlier rejection.
+//
+// GPU slot sensors (GPU_POWER_SLOTx, GPU1 Power, …) are classified separately
+// since the audit collector does not track GPU PCIe slot power.
+func sampleIPMISDRPowerSensors() sdrPowerSnapshot {
+	raw, err := exec.Command("ipmitool", "sdr").Output()
+	if err != nil || len(raw) == 0 {
+		return sdrPowerSnapshot{}
+	}
+	sdrStr := string(raw)
+	var snap sdrPowerSnapshot
+
+	// ── PSU data via audit collector ─────────────────────────────────────────
+	// collector.PSUSlotsFromSDR handles all vendor naming variants and applies
+	// bounds checking inside parseBoundedFloat (0 < w ≤ 6000 W).
+	collectorSlots := collector.PSUSlotsFromSDR(sdrStr)
+
+	// Convert to benchmark type and apply cross-slot median filtering.
+	var psuInSensors, psuOutSensors []sdrSensor
+	for slotKey, sp := range collectorSlots {
+		bsp := BenchmarkPSUSlotPower{Status: sp.Status}
+		if sp.InputW != nil {
+			bsp.InputW = sp.InputW
+			psuInSensors = append(psuInSensors, sdrSensor{name: "PSU-slot-" + slotKey, watts: *sp.InputW})
+		}
+		if sp.OutputW != nil {
+			bsp.OutputW = sp.OutputW
+			psuOutSensors = append(psuOutSensors, sdrSensor{name: "PSU-slot-" + slotKey + "-out", watts: *sp.OutputW})
+		}
+		if snap.PSUSlots == nil {
+			snap.PSUSlots = make(map[string]BenchmarkPSUSlotPower)
+		}
+		snap.PSUSlots[slotKey] = bsp
+	}
+
+	// Apply cross-slot outlier filter and sum.
+	validIn, skIn := filterSensorGroup(psuInSensors, 6000)
+	for _, s := range validIn {
+		snap.PSUInW += s.watts
+	}
+	snap.SkippedSensors = append(snap.SkippedSensors, skIn...)
+
+	validOut, skOut := filterSensorGroup(psuOutSensors, 6000)
+	for _, s := range validOut {
+		snap.PSUOutW += s.watts
+	}
+	snap.SkippedSensors = append(snap.SkippedSensors, skOut...)
+
+	// ── GPU slot sensors ─────────────────────────────────────────────────────
+	// collector does not track GPU PCIe slot power; classify here.
+	// Matches: GPU_POWER_SLOTx (MSI), GPU1 Power (xFusion), GPU_PWR_x (generic).
+	var gpuSensors []sdrSensor
+	for _, line := range strings.Split(sdrStr, "\n") {
+		parts := strings.Split(line, "|")
+		if len(parts) < 2 {
+			continue
+		}
+		name := strings.TrimSpace(parts[0])
+		nameLower := strings.ToLower(name)
+		if !strings.Contains(nameLower, "gpu") {
+			continue
+		}
+		if !strings.Contains(nameLower, "slot") && !strings.Contains(nameLower, "power") &&
+			!strings.Contains(nameLower, "pwr") {
+			continue
+		}
+		var w float64
+		if n, _ := fmt.Sscanf(strings.TrimSpace(parts[1]), "%f Watts", &w); n != 1 {
+			continue
+		}
+		gpuSensors = append(gpuSensors, sdrSensor{name: name, watts: w})
+	}
+	validGPU, skGPU := filterSensorGroup(gpuSensors, 2000)
+	for _, s := range validGPU {
+		snap.GPUSlotW += s.watts
+	}
+	snap.SkippedSensors = append(snap.SkippedSensors, skGPU...)
+
+	return snap
+}
+
+func startIPMISDRSampler(stopCh <-chan struct{}, intervalSec int) <-chan []sdrPowerSnapshot {
+	if intervalSec <= 0 {
+		intervalSec = benchmarkPowerAutotuneSampleInterval
+	}
+	ch := make(chan []sdrPowerSnapshot, 1)
+	go func() {
+		defer close(ch)
+		var samples []sdrPowerSnapshot
+		record := func() {
+			snap := sampleIPMISDRPowerSensors()
+			if snap.PSUInW <= 0 && snap.PSUOutW <= 0 && snap.GPUSlotW <= 0 && len(snap.PSUSlots) == 0 {
+				return
+			}
+			samples = append(samples, snap)
+		}
+		record()
+		ticker := time.NewTicker(time.Duration(intervalSec) * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-stopCh:
+				ch <- samples
+				return
+			case <-ticker.C:
+				record()
+			}
+		}
+	}()
+	return ch
+}
+
+func summarizeSDRPowerSeries(samples []sdrPowerSnapshot) benchmarkSDRSeriesSummary {
+	var summary benchmarkSDRSeriesSummary
+	if len(samples) == 0 {
+		return summary
+	}
+
+	type slotAggregate struct {
+		inputs  []float64
+		outputs []float64
+		status  string
+	}
+
+	slotAgg := make(map[string]*slotAggregate)
+	skippedSet := make(map[string]struct{})
+	var inputTotals []float64
+	var outputTotals []float64
+	var gpuSlotTotals []float64
+
+	for _, sample := range samples {
+		if sample.PSUInW > 0 {
+			inputTotals = append(inputTotals, sample.PSUInW)
+		}
+		if sample.PSUOutW > 0 {
+			outputTotals = append(outputTotals, sample.PSUOutW)
+		}
+		if sample.GPUSlotW > 0 {
+			gpuSlotTotals = append(gpuSlotTotals, sample.GPUSlotW)
+		}
+		for _, skipped := range sample.SkippedSensors {
+			if skipped != "" {
+				skippedSet[skipped] = struct{}{}
+			}
+		}
+		for slot, reading := range sample.PSUSlots {
+			agg := slotAgg[slot]
+			if agg == nil {
+				agg = &slotAggregate{}
+				slotAgg[slot] = agg
+			}
+			if reading.InputW != nil && *reading.InputW > 0 {
+				agg.inputs = append(agg.inputs, *reading.InputW)
+			}
+			if reading.OutputW != nil && *reading.OutputW > 0 {
+				agg.outputs = append(agg.outputs, *reading.OutputW)
+			}
+			switch {
+			case reading.Status == "":
+			case agg.status == "":
+				agg.status = reading.Status
+			case agg.status == "OK" && reading.Status != "OK":
+				agg.status = reading.Status
+			}
+		}
+	}
+
+	summary.PSUInW = benchmarkMean(inputTotals)
+	summary.PSUOutW = benchmarkMean(outputTotals)
+	summary.GPUSlotW = benchmarkMean(gpuSlotTotals)
+	summary.Samples = len(samples)
+
+	if len(slotAgg) > 0 {
+		summary.PSUSlots = make(map[string]BenchmarkPSUSlotPower, len(slotAgg))
+		for slot, agg := range slotAgg {
+			reading := BenchmarkPSUSlotPower{Status: agg.status}
+			if mean := benchmarkMean(agg.inputs); mean > 0 {
+				v := mean
+				reading.InputW = &v
+			}
+			if mean := benchmarkMean(agg.outputs); mean > 0 {
+				v := mean
+				reading.OutputW = &v
+			}
+			summary.PSUSlots[slot] = reading
+		}
+	}
+	if len(skippedSet) > 0 {
+		summary.SkippedSensors = make([]string, 0, len(skippedSet))
+		for skipped := range skippedSet {
+			summary.SkippedSensors = append(summary.SkippedSensors, skipped)
+		}
+		sort.Strings(summary.SkippedSensors)
+	}
+
+	return summary
+}
+
+// queryIPMIServerPowerW reads the current server power draw via ipmitool dcmi.
+// Returns 0 and an error if IPMI is unavailable or the output cannot be parsed.
+func queryIPMIServerPowerW() (float64, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "ipmitool", "dcmi", "power", "reading")
+	out, err := cmd.Output()
+	if err != nil {
+		return 0, fmt.Errorf("ipmitool dcmi power reading: %w", err)
+	}
+	if w := parseDCMIPowerReading(string(out)); w > 0 {
+		return w, nil
+	}
+	return 0, fmt.Errorf("could not parse ipmitool dcmi power reading output")
+}
+
+// sampleIPMIPowerSeries collects IPMI power readings every 2 seconds for
+// durationSec seconds. Returns the mean of all successful samples.
+// Returns 0, false if IPMI is unavailable.
+func sampleIPMIPowerSeries(ctx context.Context, durationSec int) (meanW float64, ok bool) {
+	if durationSec <= 0 {
+		return 0, false
+	}
+	deadline := time.Now().Add(time.Duration(durationSec) * time.Second)
+	var samples []float64
+loop:
+	for {
+		if w, err := queryIPMIServerPowerW(); err == nil {
+			samples = append(samples, w)
+		}
+		if time.Now().After(deadline) {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			break loop
+		case <-time.After(2 * time.Second):
+		}
+	}
+	if len(samples) == 0 {
+		return 0, false
+	}
+	var sum float64
+	for _, w := range samples {
+		sum += w
+	}
+	return sum / float64(len(samples)), true
+}
+
+// characterizeServerPower computes BenchmarkServerPower from idle and loaded
+// samples plus the GPU-reported average power during steady state.
+func characterizeServerPower(idleW, loadedW, gpuReportedSumW float64, source string, available bool) *BenchmarkServerPower {
+	sp := &BenchmarkServerPower{
+		Available:         available,
+		Source:            normalizeBenchmarkPowerSource(source),
+		SampleIntervalSec: benchmarkPowerAutotuneSampleInterval,
+	}
+	if !available {
+		sp.Notes = append(sp.Notes, "IPMI power reading unavailable; server-side power characterization skipped")
+		return sp
+	}
+	sp.IdleW = idleW
+	sp.LoadedW = loadedW
+	sp.DeltaW = loadedW - idleW
+	sp.GPUReportedSumW = gpuReportedSumW
+	if gpuReportedSumW > 0 && sp.DeltaW > 0 {
+		sp.ReportingRatio = sp.DeltaW / gpuReportedSumW
+	}
+	return sp
+}
+
+// readServerModel returns the DMI system product name (e.g. "SuperMicro SYS-421GE-TNRT").
+// Returns empty string if unavailable (non-Linux or missing DMI entry).
+func readServerModel() string {
+	data, err := os.ReadFile("/sys/class/dmi/id/product_name")
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(data))
+}
+
+// filterRowsByGPU returns only the metric rows for a specific GPU index.
+func filterRowsByGPU(rows []GPUMetricRow, gpuIndex int) []GPUMetricRow {
+	var out []GPUMetricRow
+	for _, r := range rows {
+		if r.GPUIndex == gpuIndex {
+			out = append(out, r)
+		}
+	}
+	return out
+}
+
+// parseBenchmarkBurnLogByGPU splits a multi-GPU bee-gpu-burn output by [gpu N] prefix
+// and returns a per-GPU parse result map.
+func parseBenchmarkBurnLogByGPU(raw string) map[int]benchmarkBurnParseResult {
+	gpuLines := make(map[int][]string)
+	for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") {
+		line = strings.TrimSpace(line)
+		if !strings.HasPrefix(line, "[gpu ") {
+			continue
+		}
+		end := strings.Index(line, "] ")
+		if end < 0 {
+			continue
+		}
+		gpuIdx, err := strconv.Atoi(strings.TrimSpace(line[5:end]))
+		if err != nil {
+			continue
+		}
+		gpuLines[gpuIdx] = append(gpuLines[gpuIdx], line[end+2:])
+	}
+	results := make(map[int]benchmarkBurnParseResult, len(gpuLines))
+	for gpuIdx, lines := range gpuLines {
+		// Lines are already stripped of the [gpu N] prefix; parseBenchmarkBurnLog
+		// calls stripBenchmarkPrefix which is a no-op on already-stripped lines.
+		results[gpuIdx] = parseBenchmarkBurnLog(strings.Join(lines, "\n"))
+	}
+	return results
+}
+
+// runNvidiaBenchmarkParallel runs warmup and steady compute on all selected GPUs
+// simultaneously using a single bee-gpu-burn invocation per phase.
+func runNvidiaBenchmarkParallel(
+	ctx context.Context,
+	verboseLog, runDir string,
+	selected []int,
+	infoByIndex map[int]benchmarkGPUInfo,
+	opts NvidiaBenchmarkOptions,
+	spec benchmarkProfileSpec,
+	logFunc func(string),
+	result *NvidiaBenchmarkResult,
+	calibByIndex map[int]benchmarkPowerCalibrationResult,
+	serverIdleW *float64, serverLoadedWSum *float64,
+	serverIdleOK *bool, serverLoadedOK *bool, serverLoadedSamples *int,
+	allMetricRows *[]GPUMetricRow,
+	metricTimelineSec *float64,
+	gpuBurnLog string,
+) {
+	allDevices := joinIndexList(selected)
+
+	// Build per-GPU result stubs.
+	gpuResults := make(map[int]*BenchmarkGPUResult, len(selected))
+	for _, idx := range selected {
+		r := &BenchmarkGPUResult{Index: idx, Status: "FAILED"}
+		if info, ok := infoByIndex[idx]; ok {
+			r.UUID = info.UUID
+			r.Name = info.Name
+			r.BusID = info.BusID
+			r.VBIOS = info.VBIOS
+			r.PowerLimitW = info.PowerLimitW
+			r.MultiprocessorCount = info.MultiprocessorCount
+			r.DefaultPowerLimitW = info.DefaultPowerLimitW
+			r.ShutdownTempC = info.ShutdownTempC
+			r.SlowdownTempC = info.SlowdownTempC
+			r.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz
+			r.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz
+			r.MaxMemoryClockMHz = info.MaxMemoryClockMHz
+		}
+		if calib, ok := calibByIndex[idx]; ok {
+			r.CalibratedPeakPowerW = calib.Summary.P95PowerW
+			r.CalibratedPeakTempC = calib.Summary.P95TempC
+			r.PowerCalibrationTries = calib.Attempts
+			r.PowerLimitDerated = calib.Derated
+			r.Notes = append(r.Notes, calib.Notes...)
+			if calib.CoolingWarning != "" {
+				r.CoolingWarning = calib.CoolingWarning
+			}
+		}
+		if norm := findBenchmarkNormalization(result.Normalization.GPUs, idx); norm != nil {
+			r.LockedGraphicsClockMHz = norm.GPUClockLockMHz
+			r.LockedMemoryClockMHz = norm.MemoryClockLockMHz
+		}
+		gpuResults[idx] = r
+	}
+
+	// Baseline: sample all GPUs together.
+	baselineRows, err := collectBenchmarkSamples(ctx, spec.BaselineSec, selected)
+	if err != nil && err != context.Canceled {
+		for _, idx := range selected {
+			gpuResults[idx].Notes = append(gpuResults[idx].Notes, "baseline sampling failed: "+err.Error())
+		}
+	}
+	for _, idx := range selected {
+		perGPU := filterRowsByGPU(baselineRows, idx)
+		gpuResults[idx].Baseline = summarizeBenchmarkTelemetry(perGPU)
+	}
+	appendBenchmarkMetrics(allMetricRows, baselineRows, "baseline", metricTimelineSec, float64(spec.BaselineSec))
+
+	// Sample server idle power once.
+	if !*serverIdleOK {
+		if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, maxInt(spec.BaselineSec, 10), benchmarkPowerAutotuneSampleInterval); ok {
+			*serverIdleW = w
+			*serverIdleOK = true
+			logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w))
+		}
+	}
+
+	// Warmup: all GPUs simultaneously.
+	warmupCmd := []string{
+		"bee-gpu-burn",
+		"--seconds", strconv.Itoa(spec.WarmupSec),
+		"--size-mb", strconv.Itoa(opts.SizeMB),
+		"--devices", allDevices,
+	}
+	logFunc(fmt.Sprintf("GPUs %s: parallel warmup (%ds)", allDevices, spec.WarmupSec))
+	warmupOut, warmupRows, warmupErr := runBenchmarkCommandWithMetrics(ctx, verboseLog, "gpu-all-warmup.log", warmupCmd, nil, selected, logFunc)
+	appendBenchmarkMetrics(allMetricRows, warmupRows, "warmup", metricTimelineSec, float64(spec.WarmupSec))
+	appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", "warmup", warmupOut)
+	if warmupErr != nil {
+		for _, idx := range selected {
+			gpuResults[idx].Notes = append(gpuResults[idx].Notes, "parallel warmup failed: "+warmupErr.Error())
+		}
+	}
+	warmupParseByGPU := parseBenchmarkBurnLogByGPU(string(warmupOut))
+	supportedPrecisions := append([]string(nil), benchmarkPrecisionPhases...)
+	for _, idx := range selected {
+		if pr, ok := warmupParseByGPU[idx]; ok && pr.ComputeCapability != "" {
+			if gpuResults[idx].ComputeCapability == "" {
+				gpuResults[idx].ComputeCapability = pr.ComputeCapability
+			}
+			if ccPrecisions := benchmarkSupportedPrecisions(pr.ComputeCapability); len(ccPrecisions) < len(supportedPrecisions) {
+				supportedPrecisions = ccPrecisions
+			}
+		}
+	}
+
+	// Run synthetic precision phases and the combined steady phase as one
+	// uninterrupted command so the GPUs stay hot between windows.
+	eccBase := make(map[int]BenchmarkECCCounters, len(selected))
+	for _, idx := range selected {
+		eccBase[idx], _ = queryECCCounters(idx)
+	}
+	planLabels, planPhases, basePhaseSec, mixedPhaseSec := buildBenchmarkSteadyPlan(spec, supportedPrecisions, func(label string) string {
+		if label == "mixed" {
+			return "steady"
+		}
+		return "gpu-all-steady-" + label
+	})
+	planCmd := []string{
+		"bee-gpu-burn",
+		"--seconds", strconv.Itoa(basePhaseSec),
+		"--size-mb", strconv.Itoa(opts.SizeMB),
+		"--devices", allDevices,
+		"--precision-plan", strings.Join(planLabels, ","),
+		"--precision-plan-seconds", benchmarkPlanDurationsCSV(planPhases),
+	}
+	logFunc(fmt.Sprintf("GPUs %s: uninterrupted precision plan (%d precision phases x %ds, mixed %ds)", allDevices, len(supportedPrecisions), basePhaseSec, mixedPhaseSec))
+	serverPowerStopCh := make(chan struct{})
+	serverPowerCh := startSelectedPowerSourceSampler(serverPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
+	_, phaseRowsByStage, phaseLogs, planErr := runBenchmarkPlannedCommandWithMetrics(ctx, verboseLog, "gpu-all-precision-plan.log", planCmd, nil, selected, planPhases, logFunc)
+	close(serverPowerStopCh)
+	if serverPowerSamples := <-serverPowerCh; len(serverPowerSamples) > 0 {
+		*serverLoadedWSum += benchmarkMean(serverPowerSamples)
+		(*serverLoadedSamples)++
+		*serverLoadedOK = true
+		logFunc(fmt.Sprintf("GPUs %s: server loaded power (%s avg): %.0f W", allDevices, opts.ServerPowerSource, benchmarkMean(serverPowerSamples)))
+	}
+	for _, phaseSpec := range planPhases {
+		if rows := phaseRowsByStage[phaseSpec.MetricStage]; len(rows) > 0 {
+			appendBenchmarkMetrics(allMetricRows, rows, phaseSpec.MetricStage, metricTimelineSec, float64(phaseSpec.DurationSec))
+		}
+		appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", phaseSpec.MetricStage, phaseLogs[phaseSpec.PlanLabel])
+	}
+	for _, prec := range supportedPrecisions {
+		phaseLogName := "gpu-all-steady-" + prec
+		phaseRows := phaseRowsByStage[phaseLogName]
+		parseByGPU := parseBenchmarkBurnLogByGPU(string(phaseLogs[prec]))
+		for _, idx := range selected {
+			perGPU := filterRowsByGPU(phaseRows, idx)
+			phase := BenchmarkPrecisionSteadyPhase{
+				Precision: prec,
+				Status:    "OK",
+				Steady:    summarizeBenchmarkTelemetry(perGPU),
+			}
+			if status, note := benchmarkPlannedPhaseStatus(phaseLogs[prec]); status != "OK" {
+				phase.Status = status
+				phase.Notes = note
+				gpuResults[idx].PrecisionFailures = append(gpuResults[idx].PrecisionFailures, prec+":"+status)
+			}
+			if pr, ok := parseByGPU[idx]; ok {
+				for _, p := range pr.Profiles {
+					if p.Supported {
+						phase.TeraOpsPerSec += p.TeraOpsPerSec
+						phase.WeightedTeraOpsPerSec += p.WeightedTeraOpsPerSec
+					}
+				}
+			}
+			gpuResults[idx].PrecisionSteady = append(gpuResults[idx].PrecisionSteady, phase)
+		}
+	}
+
+	// Snapshot throttle counters before steady.
+	beforeThrottle := make(map[int]BenchmarkThrottleCounters, len(selected))
+	for _, idx := range selected {
+		beforeThrottle[idx], _ = queryThrottleCounters(idx)
+	}
+
+	logFunc(fmt.Sprintf("GPUs %s: parallel steady compute (combined, %ds)", allDevices, mixedPhaseSec))
+	afterThrottle := make(map[int]BenchmarkThrottleCounters, len(selected))
+	for _, idx := range selected {
+		afterThrottle[idx], _ = queryThrottleCounters(idx)
+	}
+
+	steadyRows := phaseRowsByStage["steady"]
+	parseResults := parseBenchmarkBurnLogByGPU(string(phaseLogs["mixed"]))
+
+	for _, idx := range selected {
+		perGPU := filterRowsByGPU(steadyRows, idx)
+		gpuResults[idx].Steady = summarizeBenchmarkTelemetry(perGPU)
+		gpuResults[idx].Throttle = diffThrottleCounters(beforeThrottle[idx], afterThrottle[idx])
+		if eccFinal, err := queryECCCounters(idx); err == nil {
+			gpuResults[idx].ECC = diffECCCounters(eccBase[idx], eccFinal)
+		}
+
+		if pr, ok := parseResults[idx]; ok {
+			gpuResults[idx].ComputeCapability = pr.ComputeCapability
+			gpuResults[idx].Backend = pr.Backend
+			gpuResults[idx].PrecisionResults = pr.Profiles
+			if pr.Fallback {
+				gpuResults[idx].Notes = append(gpuResults[idx].Notes, "benchmark used driver PTX fallback; tensor throughput score is not comparable")
+			}
+		}
+		if planErr != nil {
+			gpuResults[idx].Notes = append(gpuResults[idx].Notes, "precision plan failed: "+planErr.Error())
+		}
+	}
+
+	// Cooldown: all GPUs together.
+	if spec.CooldownSec > 0 {
+		cooldownRows, err := collectBenchmarkSamples(ctx, spec.CooldownSec, selected)
+		if err != nil && err != context.Canceled {
+			for _, idx := range selected {
+				gpuResults[idx].Notes = append(gpuResults[idx].Notes, "cooldown sampling failed: "+err.Error())
+			}
+		}
+		for _, idx := range selected {
+			perGPU := filterRowsByGPU(cooldownRows, idx)
+			gpuResults[idx].Cooldown = summarizeBenchmarkTelemetry(perGPU)
+		}
+		appendBenchmarkMetrics(allMetricRows, cooldownRows, "cooldown", metricTimelineSec, float64(spec.CooldownSec))
+	}
+
+	// Score and finalize each GPU.
+	for _, idx := range selected {
+		r := gpuResults[idx]
+		applyBenchmarkSteadyFallback(r)
+		r.Scores = scoreBenchmarkGPUResult(*r)
+		r.DegradationReasons = detectBenchmarkDegradationReasons(*r, result.Normalization.Status)
+		pr := parseResults[idx]
+		switch {
+		case planErr != nil:
+			r.Status = classifySATErrorStatus(phaseLogs["mixed"], planErr)
+		case len(r.PrecisionFailures) > 0:
+			r.Status = "PARTIAL"
+		case pr.Fallback:
+			r.Status = "PARTIAL"
+		default:
+			r.Status = "OK"
+		}
+		result.GPUs = append(result.GPUs, finalizeBenchmarkGPUResult(*r))
+	}
+}
+
+// readBenchmarkHostConfig reads static CPU and memory configuration from
+// /proc/cpuinfo and /proc/meminfo. Returns nil if neither source is readable.
+func readBenchmarkHostConfig() *BenchmarkHostConfig {
+	cfg := &BenchmarkHostConfig{}
+	populated := false
+
+	// Parse /proc/cpuinfo for CPU model, sockets, cores, threads.
+	if data, err := os.ReadFile("/proc/cpuinfo"); err == nil {
+		socketIDs := map[string]struct{}{}
+		coresPerSocket := map[string]int{}
+		var modelName string
+		threads := 0
+		for _, line := range strings.Split(string(data), "\n") {
+			kv := strings.SplitN(line, ":", 2)
+			if len(kv) != 2 {
+				continue
+			}
+			key := strings.TrimSpace(kv[0])
+			val := strings.TrimSpace(kv[1])
+			switch key {
+			case "processor":
+				threads++
+			case "model name":
+				if modelName == "" {
+					modelName = val
+				}
+			case "physical id":
+				socketIDs[val] = struct{}{}
+			case "cpu cores":
+				// Overwrite per-socket core count (last wins per socket, but all
+				// entries for the same socket report the same value).
+				if physLine := ""; physLine == "" {
+					// We accumulate below by treating cpu cores as a per-thread
+					// field; sum by socket requires a two-pass approach. Use the
+					// simpler approximation: totalCores = threads / (threads per core).
+					_ = val
+				}
+			}
+		}
+		// Second pass: per-socket core count.
+		var curSocket string
+		for _, line := range strings.Split(string(data), "\n") {
+			kv := strings.SplitN(line, ":", 2)
+			if len(kv) != 2 {
+				continue
+			}
+			key := strings.TrimSpace(kv[0])
+			val := strings.TrimSpace(kv[1])
+			switch key {
+			case "physical id":
+				curSocket = val
+			case "cpu cores":
+				if curSocket != "" {
+					if _, seen := coresPerSocket[curSocket]; !seen {
+						v, _ := strconv.Atoi(val)
+						coresPerSocket[curSocket] = v
+					}
+				}
+			}
+		}
+		totalCores := 0
+		for _, c := range coresPerSocket {
+			totalCores += c
+		}
+		cfg.CPUModel = modelName
+		cfg.CPUSockets = len(socketIDs)
+		if cfg.CPUSockets == 0 && threads > 0 {
+			cfg.CPUSockets = 1
+		}
+		cfg.CPUCores = totalCores
+		cfg.CPUThreads = threads
+		if modelName != "" || threads > 0 {
+			populated = true
+		}
+	}
+
+	// Parse /proc/meminfo for total physical RAM.
+	if data, err := os.ReadFile("/proc/meminfo"); err == nil {
+		for _, line := range strings.Split(string(data), "\n") {
+			if strings.HasPrefix(line, "MemTotal:") {
+				fields := strings.Fields(line)
+				if len(fields) >= 2 {
+					kb, _ := strconv.ParseUint(fields[1], 10, 64)
+					cfg.MemTotalGiB = float64(kb) / (1024 * 1024)
+					populated = true
+				}
+				break
+			}
+		}
+	}
+
+	if !populated {
+		return nil
+	}
+	return cfg
+}
+
+// startCPULoadSampler starts a goroutine that samples host CPU load every
+// intervalSec seconds until stopCh is closed, then sends the collected
+// samples on the returned channel.
+func startCPULoadSampler(stopCh <-chan struct{}, intervalSec int) <-chan []float64 {
+	ch := make(chan []float64, 1)
+	go func() {
+		var samples []float64
+		ticker := time.NewTicker(time.Duration(intervalSec) * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-stopCh:
+				ch <- samples
+				return
+			case <-ticker.C:
+				if pct := sampleCPULoadPct(); pct > 0 {
+					samples = append(samples, pct)
+				}
+			}
+		}
+	}()
+	return ch
+}
+
+// summarizeCPULoad computes stats over sampled CPU load values and assigns
+// a health status.
+func summarizeCPULoad(samples []float64) *BenchmarkCPULoad {
+	if len(samples) == 0 {
+		return nil
+	}
+	sorted := append([]float64(nil), samples...)
+	sort.Float64s(sorted)
+	var sum float64
+	for _, v := range sorted {
+		sum += v
+	}
+	avg := sum / float64(len(sorted))
+	p95 := sorted[int(float64(len(sorted))*0.95)]
+	max := sorted[len(sorted)-1]
+
+	cl := &BenchmarkCPULoad{
+		AvgPct:  math.Round(avg*10) / 10,
+		MaxPct:  math.Round(max*10) / 10,
+		P95Pct:  math.Round(p95*10) / 10,
+		Samples: len(sorted),
+	}
+
+	// Compute standard deviation to detect instability.
+	var variance float64
+	for _, v := range sorted {
+		d := v - avg
+		variance += d * d
+	}
+	stdDev := math.Sqrt(variance / float64(len(sorted)))
+
+	switch {
+	case avg > 20 || max > 40:
+		cl.Status = "high"
+		cl.Note = fmt.Sprintf("avg %.1f%% max %.1f%% — elevated host CPU load may interfere with GPU benchmark results", avg, max)
+	case stdDev > 12:
+		cl.Status = "unstable"
+		cl.Note = fmt.Sprintf("avg %.1f%% stddev %.1f%% — host CPU load was erratic during the benchmark", avg, stdDev)
+	default:
+		cl.Status = "ok"
+	}
+	return cl
+}
+
+// runBenchmarkPowerCalibration runs the configured power-fit load for the supplied
+// GPU set and actively watches throttle counters. seedLimits, when provided, are treated as
+// the starting point for this calibration pass rather than as immutable fixed
+// limits. This matters during cumulative ramp-up: once an additional GPU is
+// introduced, every already-active GPU must be revalidated under the new
+// thermal state instead of assuming its previous single-step limit is still
+// valid. The selected reduced power limits stay active for the main benchmark
+// and are restored by the caller afterwards.

audit/internal/platform/benchmark_power_calibration_run.go

@@ -0,0 +1,558 @@
+package platform
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func runBenchmarkPowerCalibration(
+	ctx context.Context,
+	verboseLog, runDir string,
+	gpuIndices []int,
+	infoByIndex map[int]benchmarkGPUInfo,
+	logFunc func(string),
+	seedLimits map[int]int,
+	durationSec int,
+) (map[int]benchmarkPowerCalibrationResult, []benchmarkRestoreAction, []GPUMetricRow, benchmarkPowerCalibrationRunSummary) {
+	calibDurationSec := durationSec
+	var runSummary benchmarkPowerCalibrationRunSummary
+	if calibDurationSec <= 0 {
+		calibDurationSec = 120
+	}
+	// calibSearchTolerance is the binary-search convergence threshold in watts.
+	// When hi-lo ≤ this, the highest verified-stable limit (lo) is used.
+	const calibSearchTolerance = 10
+	// dcgmResourceBusyMaxDelaySec caps the exponential back-off when DCGM
+	// returns DCGM_ST_IN_USE (exit 222). The sequence is 1 s, 2 s, 4 s, …
+	// doubling each retry until it would exceed the cap, at which point the
+	// next busy response fails the calibration immediately.
+	const dcgmResourceBusyMaxDelaySec = 300
+	engine := benchmarkPowerEngine()
+	engineLabel := benchmarkPowerEngineLabel(engine)
+
+	if engine == BenchmarkPowerEngineTargetedPower {
+		if _, err := exec.LookPath("dcgmi"); err != nil {
+			logFunc("power calibration: dcgmi not found, skipping (will use default power limit)")
+			return map[int]benchmarkPowerCalibrationResult{}, nil, nil, runSummary
+		}
+	} else {
+		if _, _, err := resolveBenchmarkPowerLoadCommand(calibDurationSec, gpuIndices); err != nil {
+			logFunc("power calibration: dcgmproftester not found, skipping (will use default power limit)")
+			return map[int]benchmarkPowerCalibrationResult{}, nil, nil, runSummary
+		}
+	}
+	if killed := KillTestWorkers(); len(killed) > 0 {
+		for _, p := range killed {
+			logFunc(fmt.Sprintf("power calibration pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
+		}
+	}
+
+	canDerate := os.Geteuid() == 0
+	if !canDerate {
+		logFunc("power calibration: root privileges unavailable, adaptive power-limit derating disabled")
+	}
+
+	type calibrationAttemptResult struct {
+		out  []byte
+		rows []GPUMetricRow
+		err  error
+	}
+
+	// gpuCalibState holds per-GPU binary search state during parallel calibration.
+	type gpuCalibState struct {
+		idx            int
+		info           benchmarkGPUInfo
+		originalLimitW int
+		appliedLimitW  int
+		minLimitW      int
+		lo             int // highest verified-stable limit
+		hi             int // lowest verified-unstable limit (exclusive sentinel above start)
+		loVerified     bool
+		calib          benchmarkPowerCalibrationResult
+		converged      bool
+	}
+
+	results := make(map[int]benchmarkPowerCalibrationResult, len(gpuIndices))
+	var restore []benchmarkRestoreAction
+	var allCalibRows []GPUMetricRow // accumulated telemetry across all attempts
+	var calibCursor float64
+
+	// Initialise per-GPU state.
+	states := make([]*gpuCalibState, 0, len(gpuIndices))
+	for _, idx := range gpuIndices {
+		info := infoByIndex[idx]
+		originalLimitW := int(math.Round(info.PowerLimitW))
+		if originalLimitW <= 0 {
+			originalLimitW = int(math.Round(info.DefaultPowerLimitW))
+		}
+		defaultLimitW := int(math.Round(info.DefaultPowerLimitW))
+		if defaultLimitW <= 0 {
+			defaultLimitW = originalLimitW
+		}
+		appliedLimitW := initialBenchmarkCalibrationLimitW(info)
+		if appliedLimitW <= 0 {
+			appliedLimitW = defaultLimitW
+		}
+		minLimitW := int(math.Round(info.MinPowerLimitW))
+		if minLimitW <= 0 {
+			minLimitW = appliedLimitW
+		}
+		maxLimitW := int(math.Round(info.MaxPowerLimitW))
+		if maxLimitW > 0 && appliedLimitW > maxLimitW {
+			appliedLimitW = maxLimitW
+		}
+		s := &gpuCalibState{
+			idx:            idx,
+			info:           info,
+			originalLimitW: originalLimitW,
+			appliedLimitW:  appliedLimitW,
+			minLimitW:      minLimitW,
+			lo:             minLimitW,
+			hi:             appliedLimitW + 1, // not yet tested, not yet confirmed unstable
+			calib:          benchmarkPowerCalibrationResult{AppliedPowerLimitW: float64(appliedLimitW)},
+		}
+		if minLimitW > 0 && appliedLimitW > 0 && minLimitW >= appliedLimitW {
+			s.appliedLimitW = minLimitW
+			s.hi = minLimitW + 1
+		}
+		if info.MinPowerLimitW <= 0 {
+			s.calib.Notes = append(s.calib.Notes, "minimum power limit was not reported by nvidia-smi; calibration can only validate the current/default power limit")
+		}
+		if seedLimits != nil {
+			if seedW, ok := seedLimits[idx]; ok && seedW > 0 {
+				// A previously validated limit is only a starting point. Re-run
+				// targeted_power under the current multi-GPU thermal load and derate
+				// again if this step shows new throttling.
+				if seedW < s.minLimitW {
+					seedW = s.minLimitW
+				}
+				if maxLimitW > 0 && seedW > maxLimitW {
+					seedW = maxLimitW
+				}
+				if canDerate {
+					_ = setBenchmarkPowerLimit(ctx, verboseLog, idx, seedW)
+				}
+				s.appliedLimitW = seedW
+				s.hi = seedW + 1
+				s.calib.AppliedPowerLimitW = float64(seedW)
+				s.calib.Derated = seedW < s.originalLimitW
+				s.calib.Notes = append(s.calib.Notes,
+					fmt.Sprintf("seed limit: %d W (revalidating under current thermal load)", seedW))
+			}
+		}
+		states = append(states, s)
+		if canDerate && originalLimitW > 0 {
+			idxCopy := idx
+			orig := originalLimitW
+			restore = append(restore, benchmarkRestoreAction{
+				name: fmt.Sprintf("gpu-%d-restore-power-limit", idxCopy),
+				fn: func() {
+					_ = setBenchmarkPowerLimit(context.Background(), verboseLog, idxCopy, orig)
+				},
+			})
+		}
+	}
+
+	// Shared DCGM resource-busy back-off state (single diagnostic session).
+	busyRetries := 0
+	busyDelaySec := 1
+	sharedAttempt := 0
+
+	type sharedAttemptResult struct {
+		out  []byte
+		rows []GPUMetricRow
+		err  error
+	}
+
+calibDone:
+	for {
+		// Collect non-converged GPUs.
+		var active []*gpuCalibState
+		for _, s := range states {
+			if !s.converged {
+				active = append(active, s)
+			}
+		}
+		if len(active) == 0 || ctx.Err() != nil {
+			break
+		}
+
+		sharedAttempt++
+		for _, s := range active {
+			s.calib.Attempts++
+			logFunc(fmt.Sprintf("power calibration: GPU %d %s attempt %d at %d W for %ds", s.idx, engineLabel, s.calib.Attempts, s.appliedLimitW, calibDurationSec))
+		}
+
+		// Snapshot throttle counters for all active GPUs before the run.
+		beforeThrottle := make(map[int]BenchmarkThrottleCounters, len(active))
+		for _, s := range active {
+			beforeThrottle[s.idx], _ = queryThrottleCounters(s.idx)
+		}
+
+		// Run the selected power-fit load for ALL gpuIndices simultaneously so every card
+		// is under load during calibration — this reflects real server thermals.
+		logName := fmt.Sprintf("power-calibration-attempt-%d.log", sharedAttempt)
+		cmd, env, err := resolveBenchmarkPowerLoadCommand(calibDurationSec, gpuIndices)
+		if err != nil {
+			for _, s := range active {
+				s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("failed to resolve %s command: %v", engineLabel, err))
+				s.converged = true
+			}
+			logFunc(fmt.Sprintf("power calibration: failed to resolve %s command: %v", engineLabel, err))
+			break calibDone
+		}
+		attemptCtx, cancelAttempt := context.WithCancel(ctx)
+		doneCh := make(chan sharedAttemptResult, 1)
+		sdrStopCh := make(chan struct{})
+		sdrDoneCh := startIPMISDRSampler(sdrStopCh, benchmarkPowerAutotuneSampleInterval)
+		fanStopCh := make(chan struct{})
+		fanDoneCh := startBenchmarkFanSampler(fanStopCh, benchmarkPowerAutotuneSampleInterval)
+		go func() {
+			out, rows, err := runBenchmarkCommandWithMetrics(attemptCtx, verboseLog, logName, cmd, env, gpuIndices, logFunc)
+			doneCh <- sharedAttemptResult{out: out, rows: rows, err: err}
+		}()
+
+		ticker := time.NewTicker(time.Second)
+		throttleReasons := make(map[int]string, len(active))
+		var ar sharedAttemptResult
+
+	attemptLoop:
+		for {
+			select {
+			case ar = <-doneCh:
+				break attemptLoop
+			case <-ticker.C:
+				// Poll throttle counters for each active GPU independently.
+				for _, s := range active {
+					if throttleReasons[s.idx] != "" {
+						continue // already detected for this GPU
+					}
+					after, err := queryThrottleCounters(s.idx)
+					if err != nil {
+						continue
+					}
+					// Record throttle but do NOT cancel — let the load command finish so
+					// runtime resources release cleanly before the next attempt.
+					if reason := benchmarkCalibrationThrottleReason(beforeThrottle[s.idx], after); reason != "" {
+						throttleReasons[s.idx] = reason
+						logFunc(fmt.Sprintf("power calibration: GPU %d detected %s throttle at %d W, waiting for run to finish", s.idx, reason, s.appliedLimitW))
+					}
+				}
+			case <-ctx.Done():
+				cancelAttempt()
+				ar = <-doneCh
+				break attemptLoop
+			}
+		}
+		ticker.Stop()
+		cancelAttempt()
+		close(sdrStopCh)
+		close(fanStopCh)
+		attemptSDRSummary := summarizeSDRPowerSeries(<-sdrDoneCh)
+		attemptFanSummary := <-fanDoneCh
+		_ = os.WriteFile(filepath.Join(runDir, logName), ar.out, 0644)
+		// Accumulate telemetry rows with attempt stage label.
+		appendBenchmarkMetrics(&allCalibRows, ar.rows, fmt.Sprintf("attempt-%d", sharedAttempt), &calibCursor, float64(calibDurationSec))
+
+		// Resource busy: retry with exponential back-off (shared — one DCGM session).
+		if ar.err != nil && isDCGMResourceBusy(ar.err) {
+			if busyDelaySec > dcgmResourceBusyMaxDelaySec {
+				for _, s := range active {
+					s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("DCGM resource busy after %d retries, giving up", busyRetries))
+					s.converged = true
+				}
+				logFunc(fmt.Sprintf("power calibration: DCGM resource persistently busy after %d retries, stopping", busyRetries))
+				break calibDone
+			}
+			busyRetries++
+			// Undo attempt counter: busy retries don't count as real attempts.
+			for _, s := range active {
+				s.calib.Attempts--
+			}
+			logFunc(fmt.Sprintf("power calibration: DCGM resource busy (attempt %d), retrying in %ds", sharedAttempt, busyDelaySec))
+			select {
+			case <-ctx.Done():
+				break calibDone
+			case <-time.After(time.Duration(busyDelaySec) * time.Second):
+			}
+			next := busyDelaySec * 2
+			if next > dcgmResourceBusyMaxDelaySec {
+				next = dcgmResourceBusyMaxDelaySec + 1
+			}
+			busyDelaySec = next
+			sharedAttempt-- // retry same logical attempt number
+			continue
+		}
+		busyRetries = 0
+		busyDelaySec = 1
+
+		// Per-GPU analysis and binary search update.
+		attemptStable := ar.err == nil
+		for _, s := range active {
+			perGPU := filterRowsByGPU(ar.rows, s.idx)
+			summary := summarizeBenchmarkTelemetry(perGPU)
+			throttle := throttleReasons[s.idx]
+			if throttle != "" || summary.P95PowerW <= 0 {
+				attemptStable = false
+			}
+
+			// Cooling warning: thermal throttle with fans not at maximum.
+			if strings.Contains(throttle, "thermal") && s.calib.CoolingWarning == "" {
+				clocks := make([]float64, 0, len(perGPU))
+				var fanDutyValues []float64
+				fanDutyAvail := false
+				for _, r := range perGPU {
+					if r.ClockMHz > 0 {
+						clocks = append(clocks, r.ClockMHz)
+					}
+					if r.FanDutyCycleAvailable {
+						fanDutyAvail = true
+						fanDutyValues = append(fanDutyValues, r.FanDutyCyclePct)
+					}
+				}
+				dropPct := benchmarkClockDrift(clocks)
+				p95FanDuty := benchmarkPercentile(fanDutyValues, 95)
+				if dropPct >= 20 && fanDutyAvail && p95FanDuty < 98 {
+					s.calib.CoolingWarning = fmt.Sprintf(
+						"thermal throttle (%s) caused a %.0f%% clock drop while fans were at %.0f%% duty cycle — server cooling may not be configured for full GPU load",
+						throttle, dropPct, p95FanDuty,
+					)
+					logFunc(fmt.Sprintf("power calibration: GPU %d cooling warning: %s", s.idx, s.calib.CoolingWarning))
+				}
+			}
+
+			if throttle == "" && ar.err == nil && summary.P95PowerW > 0 {
+				// Stable at current limit — update lo and binary-search upward.
+				s.calib.Summary = summary
+				s.calib.Completed = true
+				s.calib.AppliedPowerLimitW = float64(s.appliedLimitW)
+				logFunc(fmt.Sprintf("power calibration: GPU %d stable at %d W, p95=%.0f W p95_temp=%.1f C (%d samples)", s.idx, s.appliedLimitW, summary.P95PowerW, summary.P95TempC, summary.Samples))
+				s.lo = s.appliedLimitW
+				s.loVerified = true
+				if canDerate && s.hi-s.lo > calibSearchTolerance {
+					next := roundTo5W((s.lo + s.hi) / 2)
+					if next > s.lo && next < s.hi {
+						if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, next); err == nil {
+							s.appliedLimitW = next
+							s.calib.AppliedPowerLimitW = float64(next)
+							s.calib.Completed = false // keep searching
+							s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search: stable at %d W, trying %d W (lo=%d hi=%d)", s.lo, next, s.lo, s.hi))
+							logFunc(fmt.Sprintf("power calibration: GPU %d binary search up: stable at %d W, trying %d W", s.idx, s.lo, next))
+							continue // next GPU in active list
+						}
+					}
+				}
+				s.calib.MetricRows = filterRowsByGPU(ar.rows, s.idx)
+				s.converged = true
+				continue
+			}
+
+			// Failed or throttled — log and binary-search downward.
+			switch {
+			case throttle != "":
+				s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("targeted_power attempt %d: %s throttle at %d W", s.calib.Attempts, throttle, s.appliedLimitW))
+				logFunc(fmt.Sprintf("power calibration: GPU %d throttled (%s) at %d W, reducing power limit", s.idx, throttle, s.appliedLimitW))
+			case ar.err != nil:
+				s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("targeted_power attempt %d failed at %d W: %v", s.calib.Attempts, s.appliedLimitW, ar.err))
+				logFunc(fmt.Sprintf("power calibration: GPU %d %s failed at %d W: %v", s.idx, engineLabel, s.appliedLimitW, ar.err))
+			default:
+				s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("%s attempt %d at %d W: no valid power telemetry", engineLabel, s.calib.Attempts, s.appliedLimitW))
+				logFunc(fmt.Sprintf("power calibration: GPU %d attempt %d at %d W: no valid telemetry", s.idx, s.calib.Attempts, s.appliedLimitW))
+			}
+
+			if !canDerate || s.appliedLimitW <= 0 {
+				s.converged = true
+				continue
+			}
+			s.hi = s.appliedLimitW
+
+			if s.hi-s.lo <= calibSearchTolerance {
+				if !s.loVerified && s.minLimitW > 0 && s.appliedLimitW != s.minLimitW {
+					if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, s.minLimitW); err != nil {
+						s.calib.Notes = append(s.calib.Notes, "failed to set power limit: "+err.Error())
+						logFunc(fmt.Sprintf("power calibration: GPU %d failed to set minimum power limit %d W: %v", s.idx, s.minLimitW, err))
+						s.converged = true
+						continue
+					}
+					s.appliedLimitW = s.minLimitW
+					s.calib.AppliedPowerLimitW = float64(s.minLimitW)
+					s.calib.Derated = s.minLimitW < s.originalLimitW
+					s.info.PowerLimitW = float64(s.minLimitW)
+					infoByIndex[s.idx] = s.info
+					s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search: validating minimum settable limit %d W before concluding failure", s.minLimitW))
+					logFunc(fmt.Sprintf("power calibration: GPU %d binary search: validating minimum settable limit %d W", s.idx, s.minLimitW))
+					continue
+				}
+				if s.loVerified {
+					s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search converged: using %d W (lo=%d hi=%d)", s.lo, s.lo, s.hi))
+					if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, s.lo); err == nil {
+						s.appliedLimitW = s.lo
+						s.calib.AppliedPowerLimitW = float64(s.lo)
+						s.calib.Derated = s.lo < s.originalLimitW
+						// Summary was captured when we last verified stability at s.lo,
+						// so the result is valid — mark as completed even though we
+						// converged from the failure path (tried higher, failed, fell back).
+						s.calib.Completed = true
+					}
+				} else {
+					s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("could not find a stable %s limit down to the minimum settable power limit %d W", engineLabel, s.minLimitW))
+					logFunc(fmt.Sprintf("power calibration: GPU %d no stable limit found down to minimum settable power limit %d W", s.idx, s.minLimitW))
+				}
+				s.calib.MetricRows = filterRowsByGPU(ar.rows, s.idx)
+				s.converged = true
+				continue
+			}
+
+			next := roundTo5W((s.lo + s.hi) / 2)
+			if next <= s.lo {
+				next = s.lo + calibSearchTolerance
+			}
+			if next >= s.hi {
+				next = (s.lo + s.hi) / 2
+			}
+			if next < s.minLimitW {
+				next = s.minLimitW
+			}
+			if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, next); err != nil {
+				s.calib.Notes = append(s.calib.Notes, "failed to set power limit: "+err.Error())
+				logFunc(fmt.Sprintf("power calibration: GPU %d failed to set power limit %d W: %v", s.idx, next, err))
+				s.converged = true
+				continue
+			}
+			s.appliedLimitW = next
+			s.calib.AppliedPowerLimitW = float64(next)
+			s.calib.Derated = next < s.originalLimitW
+			s.info.PowerLimitW = float64(next)
+			infoByIndex[s.idx] = s.info
+			s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search: trying %d W (lo=%d hi=%d)", next, s.lo, s.hi))
+			logFunc(fmt.Sprintf("power calibration: GPU %d binary search: trying %d W (lo=%d hi=%d)", s.idx, next, s.lo, s.hi))
+		}
+		if attemptStable {
+			if attemptSDRSummary.Samples > 0 {
+				runSummary.LoadedSDR = attemptSDRSummary
+			}
+			if attemptFanSummary.FanSamples > 0 {
+				runSummary.AvgFanRPM = attemptFanSummary.AvgFanRPM
+				runSummary.AvgFanDutyCyclePct = attemptFanSummary.AvgFanDutyCyclePct
+				runSummary.FanSamples = attemptFanSummary.FanSamples
+			}
+		}
+	}
+
+	for _, s := range states {
+		if s.calib.Completed || s.calib.Attempts > 0 || len(s.calib.Notes) > 0 {
+			results[s.idx] = s.calib
+		}
+	}
+	writeBenchmarkMetricsFiles(runDir, allCalibRows)
+	return results, restore, allCalibRows, runSummary
+}
+
+// isDCGMResourceBusy returns true when dcgmi exits with DCGM_ST_IN_USE (222),
+// meaning nv-hostengine still holds the diagnostic slot from a prior run.
+func isDCGMResourceBusy(err error) bool {
+	var exitErr *exec.ExitError
+	return errors.As(err, &exitErr) && exitErr.ExitCode() == 222
+}
+
+// roundTo5W rounds w to the nearest 5 W boundary.
+func roundTo5W(w int) int {
+	return ((w + 2) / 5) * 5
+}
+
+func initialBenchmarkCalibrationLimitW(info benchmarkGPUInfo) int {
+	defaultLimitW := int(math.Round(info.DefaultPowerLimitW))
+	currentLimitW := int(math.Round(info.PowerLimitW))
+	maxLimitW := int(math.Round(info.MaxPowerLimitW))
+
+	startW := defaultLimitW
+	if startW <= 0 {
+		startW = currentLimitW
+	}
+	if startW <= 0 {
+		startW = maxLimitW
+	}
+	if maxLimitW > 0 && startW > maxLimitW {
+		startW = maxLimitW
+	}
+	return startW
+}
+
+// meanFanRPM returns the average RPM across a set of fan readings.
+func meanFanRPM(fans []FanReading) float64 {
+	if len(fans) == 0 {
+		return 0
+	}
+	var sum float64
+	for _, f := range fans {
+		sum += f.RPM
+	}
+	return sum / float64(len(fans))
+}
+
+func startBenchmarkFanSampler(stopCh <-chan struct{}, intervalSec int) <-chan benchmarkPowerCalibrationRunSummary {
+	if intervalSec <= 0 {
+		intervalSec = benchmarkPowerAutotuneSampleInterval
+	}
+	ch := make(chan benchmarkPowerCalibrationRunSummary, 1)
+	go func() {
+		defer close(ch)
+		var rpmSamples []float64
+		var dutySamples []float64
+		record := func() {
+			fans, err := sampleFanSpeeds()
+			if err != nil || len(fans) == 0 {
+				return
+			}
+			if rpm := meanFanRPM(fans); rpm > 0 {
+				rpmSamples = append(rpmSamples, rpm)
+			}
+			if duty, ok, _ := sampleFanDutyCyclePctFromFans(fans); ok && duty > 0 {
+				dutySamples = append(dutySamples, duty)
+			}
+		}
+		record()
+		ticker := time.NewTicker(time.Duration(intervalSec) * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-stopCh:
+				ch <- benchmarkPowerCalibrationRunSummary{
+					AvgFanRPM:          benchmarkMean(rpmSamples),
+					AvgFanDutyCyclePct: benchmarkMean(dutySamples),
+					FanSamples:         len(rpmSamples),
+				}
+				return
+			case <-ticker.C:
+				record()
+			}
+		}
+	}()
+	return ch
+}
+
+func powerBenchDurationSec(profile string) int {
+	switch strings.TrimSpace(strings.ToLower(profile)) {
+	case NvidiaBenchmarkProfileStability:
+		return 300
+	case NvidiaBenchmarkProfileOvernight:
+		return 600
+	default:
+		return 120
+	}
+}
+
+func cloneBenchmarkGPUInfoMap(src map[int]benchmarkGPUInfo) map[int]benchmarkGPUInfo {
+	out := make(map[int]benchmarkGPUInfo, len(src))
+	for k, v := range src {
+		out[k] = v
+	}
+	return out
+}

audit/internal/platform/benchmark_power_report.go

@@ -0,0 +1,624 @@
+package platform
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+func renderPowerBenchReport(result NvidiaPowerBenchResult) string {
+	var b strings.Builder
+	b.WriteString("# Bee Bench Power Report\n\n")
+	fmt.Fprintf(&b, "**Benchmark version:** %s  \n", result.BenchmarkVersion)
+	fmt.Fprintf(&b, "**Profile:** %s  \n", result.BenchmarkProfile)
+	fmt.Fprintf(&b, "**Generated:** %s  \n", result.GeneratedAt.Format("2006-01-02 15:04:05 UTC"))
+	fmt.Fprintf(&b, "**Overall status:** %s  \n", result.OverallStatus)
+	fmt.Fprintf(&b, "**Platform max TDP (GPU-reported):** %.0f W  \n", result.PlatformMaxTDPW)
+	if sp := result.ServerPower; sp != nil && sp.Available {
+		sourceLabel := "autotuned source"
+		switch normalizeBenchmarkPowerSource(sp.Source) {
+		case BenchmarkPowerSourceSDRPSUInput:
+			sourceLabel = "autotuned source (SDR PSU AC input)"
+		case BenchmarkPowerSourceDCMI:
+			sourceLabel = "autotuned source (DCMI)"
+		}
+		fmt.Fprintf(&b, "**Server power delta (%s):** %.0f W  \n", sourceLabel, sp.DeltaW)
+		fmt.Fprintf(&b, "**Reporting ratio:** %.2f  \n", sp.ReportingRatio)
+	}
+	b.WriteString("\n")
+	// Server power comparison table.
+	if sp := result.ServerPower; sp != nil {
+		b.WriteString("## Server vs GPU Power Comparison\n\n")
+		selectedSource := normalizeBenchmarkPowerSource(sp.Source)
+		selectedSourceLabel := "Selected source"
+		if selectedSource == BenchmarkPowerSourceSDRPSUInput {
+			selectedSourceLabel = "Selected source (SDR PSU AC input)"
+		} else if selectedSource == BenchmarkPowerSourceDCMI {
+			selectedSourceLabel = "Selected source (DCMI)"
+		}
+		var spRows [][]string
+		spRows = append(spRows, []string{"GPU actual power sum (p95, last step)", fmt.Sprintf("%.0f W", sp.GPUReportedSumW)})
+		if sp.Available {
+			spRows = append(spRows, []string{selectedSourceLabel + " idle power", fmt.Sprintf("%.0f W", sp.IdleW)})
+			spRows = append(spRows, []string{selectedSourceLabel + " loaded power", fmt.Sprintf("%.0f W", sp.LoadedW)})
+			spRows = append(spRows, []string{selectedSourceLabel + " Δ power (loaded − idle)", fmt.Sprintf("%.0f W", sp.DeltaW)})
+		}
+		if selectedSource == BenchmarkPowerSourceSDRPSUInput && sp.PSUInputLoadedW > 0 {
+			spRows = append(spRows, []string{"PSU AC input (idle avg, pre-load phase)", fmt.Sprintf("%.0f W", sp.PSUInputIdleW)})
+			spRows = append(spRows, []string{"PSU AC input (loaded avg, final phase)", fmt.Sprintf("%.0f W", sp.PSUInputLoadedW)})
+			psuDelta := sp.PSUInputLoadedW - sp.PSUInputIdleW
+			spRows = append(spRows, []string{"PSU AC input Δ (loaded − idle)", fmt.Sprintf("%.0f W", psuDelta)})
+		}
+		if sp.Available {
+			ratio := sp.ReportingRatio
+			dcmiPartial := detectDCMIPartialCoverage(sp) ||
+				(sp.PSUInputIdleW == 0 && detectIPMISaturationFallback(result.RampSteps))
+			ratioNote := ""
+			switch {
+			case dcmiPartial:
+				ratioNote = "⚠ IPMI DCMI covers partial PSU set; use SDR ratio below for accuracy assessment"
+			case ratio >= 0.9:
+				ratioNote = "✓ GPU telemetry matches server power"
+			case ratio >= 0.75:
+				ratioNote = "⚠ minor discrepancy — GPU may slightly over-report TDP"
+			default:
+				ratioNote = "✗ significant discrepancy — GPU over-reports TDP vs wall power"
+			}
+			spRows = append(spRows, []string{"Reporting ratio", fmt.Sprintf("%.2f — %s", ratio, ratioNote)})
+			if selectedSource == BenchmarkPowerSourceSDRPSUInput && sp.PSUInputLoadedW > 0 && sp.GPUReportedSumW > 0 {
+				psuDelta := sp.PSUInputLoadedW - sp.PSUInputIdleW
+				sdrRatio := psuDelta / sp.GPUReportedSumW
+				sdrNote := ""
+				switch {
+				case sdrRatio >= 0.9:
+					sdrNote = "✓ GPU telemetry matches wall power"
+				case sdrRatio >= 0.75:
+					sdrNote = "⚠ minor discrepancy"
+				default:
+					sdrNote = "✗ significant discrepancy"
+				}
+				spRows = append(spRows, []string{"PSU AC input reporting ratio", fmt.Sprintf("%.2f — %s", sdrRatio, sdrNote)})
+			}
+		} else {
+			spRows = append(spRows, []string{"IPMI availability", "not available — IPMI not supported or ipmitool not found"})
+		}
+		b.WriteString(fmtMDTable([]string{"Metric", "Value"}, spRows))
+		for _, note := range sp.Notes {
+			fmt.Fprintf(&b, "\n> %s\n", note)
+		}
+		b.WriteString("\n")
+
+		if len(sp.PSUSlotReadingsIdle) > 0 || len(sp.PSUSlotReadingsLoaded) > 0 {
+			b.WriteString("## PSU Load Distribution\n\n")
+
+			slotSet := map[string]struct{}{}
+			for k := range sp.PSUSlotReadingsIdle {
+				slotSet[k] = struct{}{}
+			}
+			for k := range sp.PSUSlotReadingsLoaded {
+				slotSet[k] = struct{}{}
+			}
+			slots := make([]string, 0, len(slotSet))
+			for k := range slotSet {
+				slots = append(slots, k)
+			}
+			sort.Strings(slots)
+
+			fmtW := func(v *float64) string {
+				if v == nil {
+					return "—"
+				}
+				return fmt.Sprintf("%.0f W", *v)
+			}
+
+			var psuDistRows [][]string
+			for _, slot := range slots {
+				idle := sp.PSUSlotReadingsIdle[slot]
+				loaded := sp.PSUSlotReadingsLoaded[slot]
+
+				var deltaStr string
+				if idle.InputW != nil && loaded.InputW != nil {
+					deltaStr = fmt.Sprintf("%+.0f W", *loaded.InputW-*idle.InputW)
+				} else {
+					deltaStr = "—"
+				}
+
+				status := loaded.Status
+				if status == "" {
+					status = idle.Status
+				}
+				if status == "" {
+					status = "—"
+				}
+
+				psuDistRows = append(psuDistRows, []string{
+					slot,
+					fmtW(idle.InputW), fmtW(loaded.InputW),
+					deltaStr, status,
+				})
+			}
+			b.WriteString(fmtMDTable([]string{"Slot", "AC Input (idle avg)", "AC Input (loaded avg)", "Load Δ", "Status"}, psuDistRows))
+			b.WriteString("\n")
+		}
+	}
+
+	if len(result.Findings) > 0 {
+		b.WriteString("## Summary\n\n")
+		for _, finding := range result.Findings {
+			fmt.Fprintf(&b, "- %s\n", finding)
+		}
+		b.WriteString("\n")
+	}
+	// ── Single GPU section ───────────────────────────────────────────────────
+	b.WriteString("## Single GPU\n\n")
+	{
+		var sgRows [][]string
+		for _, gpu := range result.GPUs {
+			clk := "—"
+			mem := "—"
+			temp := "—"
+			pwr := "—"
+			if gpu.Telemetry != nil {
+				clk = fmt.Sprintf("%.0f", gpu.Telemetry.AvgGraphicsClockMHz)
+				mem = fmt.Sprintf("%.0f", gpu.Telemetry.AvgMemoryClockMHz)
+				temp = fmt.Sprintf("%.1f", gpu.Telemetry.AvgTempC)
+				pwr = fmt.Sprintf("%.0f W", gpu.Telemetry.AvgPowerW)
+			}
+			serverDelta := "—"
+			if gpu.ServerDeltaW > 0 {
+				serverDelta = fmt.Sprintf("%.0f W", gpu.ServerDeltaW)
+			}
+			fan := "—"
+			if gpu.AvgFanRPM > 0 {
+				if gpu.AvgFanDutyCyclePct > 0 {
+					fan = fmt.Sprintf("%.0f RPM (%.0f%%)", gpu.AvgFanRPM, gpu.AvgFanDutyCyclePct)
+				} else {
+					fan = fmt.Sprintf("%.0f RPM", gpu.AvgFanRPM)
+				}
+			}
+			sgRows = append(sgRows, []string{
+				fmt.Sprintf("GPU %d", gpu.Index),
+				fmt.Sprintf("%s (%s)", clk, mem),
+				temp,
+				pwr,
+				serverDelta,
+				fan,
+			})
+		}
+		b.WriteString(fmtMDTable([]string{"GPU", "Clock MHz (Mem MHz)", "Avg Temp °C", "Power W", "Server Δ W", "Avg Fan RPM (duty%)"}, sgRows))
+		b.WriteString("\n")
+	}
+	if len(result.RecommendedSlotOrder) > 0 {
+		fmt.Fprintf(&b, "Recommended slot order for best single-card power realization: `%s`\n\n", joinIndexList(result.RecommendedSlotOrder))
+	}
+
+	// ── Ramp Sequence ────────────────────────────────────────────────────────
+	// Rows = run number; Cols = per-GPU power (from step telemetry) + aggregates.
+	if len(result.RampSteps) > 0 {
+		b.WriteString("## Ramp Sequence\n\n")
+
+		// Collect all GPU indices that appear across all steps (ordered by first appearance).
+		allGPUIndices := make([]int, 0, len(result.GPUs))
+		seen := map[int]bool{}
+		for _, step := range result.RampSteps {
+			for _, idx := range step.GPUIndices {
+				if !seen[idx] {
+					seen[idx] = true
+					allGPUIndices = append(allGPUIndices, idx)
+				}
+			}
+		}
+
+		var idleW float64
+		if result.ServerPower != nil {
+			idleW = result.ServerPower.IdleW
+		}
+
+		// Build header: Run | GPU 0 | GPU 1 | ... | GPU total W | Server itself W | Server wall W | Per GPU wall W | Platform eff.
+		headers := []string{"Run"}
+		for _, idx := range allGPUIndices {
+			headers = append(headers, fmt.Sprintf("GPU %d W", idx))
+		}
+		headers = append(headers, "GPU total W", "Server itself W", "Server wall W", "Per GPU wall W", "Platform eff.")
+
+		var rampRows [][]string
+		if idleW > 0 {
+			idleRow := []string{"0 (idle)"}
+			for range allGPUIndices {
+				idleRow = append(idleRow, "—")
+			}
+			// No load: GPU total is negligible, all draw is the server's own baseline.
+			idleRow = append(idleRow, "—", fmt.Sprintf("%.0f", idleW), fmt.Sprintf("%.0f", idleW), "—", "—")
+			rampRows = append(rampRows, idleRow)
+		}
+		for _, step := range result.RampSteps {
+			row := []string{fmt.Sprintf("%d", step.StepIndex)}
+			for _, idx := range allGPUIndices {
+				inStep := false
+				for _, si := range step.GPUIndices {
+					if si == idx {
+						inStep = true
+						break
+					}
+				}
+				if !inStep {
+					row = append(row, "—")
+					continue
+				}
+				gpuPwr := "—"
+				if t, ok := step.PerGPUTelemetry[idx]; ok && t != nil && t.AvgPowerW > 0 {
+					gpuPwr = fmt.Sprintf("%.0f", t.AvgPowerW)
+				}
+				row = append(row, gpuPwr)
+			}
+			// GPU total W = sum of observed GPU power (nvidia-smi)
+			gpuTotal := "—"
+			if step.TotalObservedPowerW > 0 {
+				gpuTotal = fmt.Sprintf("%.0f", step.TotalObservedPowerW)
+			}
+			// Server itself W = server wall power minus GPU total (non-GPU baseline draw)
+			serverItself := "—"
+			if step.ServerLoadedW > 0 && step.TotalObservedPowerW > 0 {
+				serverItself = fmt.Sprintf("%.0f", step.ServerLoadedW-step.TotalObservedPowerW)
+			}
+			// Server wall W
+			serverWall := "—"
+			if step.ServerLoadedW > 0 {
+				serverWall = fmt.Sprintf("%.0f", step.ServerLoadedW)
+			}
+			// Per GPU wall W = ServerDeltaW / len(GPUIndices)
+			perGPUWall := "—"
+			if step.ServerDeltaW > 0 && len(step.GPUIndices) > 0 {
+				perGPUWall = fmt.Sprintf("%.0f", step.ServerDeltaW/float64(len(step.GPUIndices)))
+			}
+			// Platform eff. = (ServerLoadedW − idleW) / TotalObservedPowerW
+			platEff := "—"
+			if step.TotalObservedPowerW > 0 {
+				eff := step.ServerDeltaW / step.TotalObservedPowerW
+				if idleW > 0 && step.ServerLoadedW > 0 {
+					eff = (step.ServerLoadedW - idleW) / step.TotalObservedPowerW
+				}
+				platEff = fmt.Sprintf("%.2f", eff)
+			}
+			row = append(row, gpuTotal, serverItself, serverWall, perGPUWall, platEff)
+			rampRows = append(rampRows, row)
+		}
+		b.WriteString(fmtMDTable(headers, rampRows))
+		b.WriteString("\n")
+	}
+
+	// ── PSU Performance ───────────────────────────────────────────────────────
+	{
+		// Collect all PSU slot keys from any ramp step.
+		psuSlotSet := map[string]struct{}{}
+		for _, step := range result.RampSteps {
+			for k := range step.PSUSlotReadings {
+				psuSlotSet[k] = struct{}{}
+			}
+		}
+		if len(psuSlotSet) > 0 {
+			b.WriteString("## PSU Performance\n\n")
+			psuSlots := make([]string, 0, len(psuSlotSet))
+			for k := range psuSlotSet {
+				psuSlots = append(psuSlots, k)
+			}
+			sort.Strings(psuSlots)
+
+			var idleW float64
+			if result.ServerPower != nil {
+				idleW = result.ServerPower.IdleW
+			}
+
+			psuHeaders := []string{"Run"}
+			for _, slot := range psuSlots {
+				psuHeaders = append(psuHeaders, fmt.Sprintf("PSU %s W", slot))
+			}
+			psuHeaders = append(psuHeaders, "PSU Total W", "Platform eff.", "Avg Fan RPM (duty%)")
+
+			var psuRows [][]string
+			for _, step := range result.RampSteps {
+				row := []string{fmt.Sprintf("%d", step.StepIndex)}
+				var psuTotal float64
+				for _, slot := range psuSlots {
+					sp, ok := step.PSUSlotReadings[slot]
+					if !ok || sp.InputW == nil {
+						row = append(row, "—")
+						continue
+					}
+					row = append(row, fmt.Sprintf("%.0f", *sp.InputW))
+					psuTotal += *sp.InputW
+				}
+				totalStr := "—"
+				if psuTotal > 0 {
+					totalStr = fmt.Sprintf("%.0f", psuTotal)
+				}
+				platEff := "—"
+				if step.TotalObservedPowerW > 0 {
+					eff := step.ServerDeltaW / step.TotalObservedPowerW
+					if idleW > 0 && step.ServerLoadedW > 0 {
+						eff = (step.ServerLoadedW - idleW) / step.TotalObservedPowerW
+					}
+					platEff = fmt.Sprintf("%.2f", eff)
+				}
+				fan := "—"
+				if step.AvgFanRPM > 0 {
+					if step.AvgFanDutyCyclePct > 0 {
+						fan = fmt.Sprintf("%.0f (%.0f%%)", step.AvgFanRPM, step.AvgFanDutyCyclePct)
+					} else {
+						fan = fmt.Sprintf("%.0f", step.AvgFanRPM)
+					}
+				}
+				row = append(row, totalStr, platEff, fan)
+				psuRows = append(psuRows, row)
+			}
+			b.WriteString(fmtMDTable(psuHeaders, psuRows))
+			b.WriteString("\n")
+		}
+	}
+
+	// ── PSU Issues ────────────────────────────────────────────────────────────
+	if len(result.PSUIssues) > 0 {
+		b.WriteString("## PSU Issues\n\n")
+		b.WriteString("The following power supply anomalies were detected during the test:\n\n")
+		for _, issue := range result.PSUIssues {
+			fmt.Fprintf(&b, "- ⛔ %s\n", issue)
+		}
+		b.WriteString("\n")
+	}
+
+	// ── Power Distribution Summary ────────────────────────────────────────────
+	b.WriteString("## Power Distribution Summary\n\n")
+	{
+		var totalDefault, totalStable float64
+		for _, gpu := range result.GPUs {
+			stable := gpu.StablePowerLimitW
+			if stable <= 0 {
+				stable = gpu.AppliedPowerLimitW
+			}
+			totalDefault += gpu.DefaultPowerLimitW
+			totalStable += stable
+		}
+		var pdRows [][]string
+		for _, gpu := range result.GPUs {
+			stable := gpu.StablePowerLimitW
+			if stable <= 0 {
+				stable = gpu.AppliedPowerLimitW
+			}
+			realization := "-"
+			if gpu.DefaultPowerLimitW > 0 && stable > 0 {
+				realization = fmt.Sprintf("%.1f%%", stable/gpu.DefaultPowerLimitW*100)
+			}
+			derated := "-"
+			if gpu.Derated {
+				derated = "⚠ yes"
+			}
+			pdRows = append(pdRows, []string{
+				fmt.Sprintf("GPU %d", gpu.Index),
+				fmt.Sprintf("%.0f W", gpu.AppliedPowerLimitW),
+				fmt.Sprintf("%.0f W", stable),
+				realization,
+				derated,
+			})
+		}
+		platformReal := "-"
+		if totalDefault > 0 && totalStable > 0 {
+			platformReal = fmt.Sprintf("%.1f%%", totalStable/totalDefault*100)
+		}
+		pdRows = append(pdRows, []string{
+			"**Platform**",
+			"—",
+			fmt.Sprintf("**%.0f W**", totalStable),
+			fmt.Sprintf("**%s**", platformReal),
+			"",
+		})
+		b.WriteString(fmtMDTable([]string{"GPU", "Single-card limit", "Stable limit", "Realization", "Derated"}, pdRows))
+		b.WriteString("\n")
+
+		// Balance across GPUs — only meaningful with 2+ GPUs.
+		if len(result.GPUs) > 1 {
+			var minS, maxS, sumS float64
+			var cnt int
+			for _, gpu := range result.GPUs {
+				s := gpu.StablePowerLimitW
+				if s <= 0 {
+					s = gpu.AppliedPowerLimitW
+				}
+				if s <= 0 {
+					continue
+				}
+				sumS += s
+				cnt++
+				if cnt == 1 || s < minS {
+					minS = s
+				}
+				if s > maxS {
+					maxS = s
+				}
+			}
+			if cnt > 0 {
+				avg := sumS / float64(cnt)
+				spread := (maxS - minS) / avg * 100
+				balanceNote := "✓ balanced"
+				switch {
+				case spread > 20:
+					balanceNote = "⚠ significant imbalance — check slot thermals"
+				case spread > 10:
+					balanceNote = "— minor imbalance"
+				}
+				fmt.Fprintf(&b, "**GPU power balance:** avg %.0f W · min %.0f W · max %.0f W · spread %.1f%% — %s\n\n",
+					avg, minS, maxS, spread, balanceNote)
+			}
+		}
+
+		// Ramp scalability table — power efficiency of adding each GPU.
+		if len(result.RampSteps) > 1 {
+			b.WriteString("**Ramp power scalability** (stable TDP per step):\n\n")
+			var firstStable float64
+			if len(result.GPUs) > 0 {
+				firstStable = result.GPUs[0].StablePowerLimitW
+				if firstStable <= 0 {
+					firstStable = result.GPUs[0].AppliedPowerLimitW
+				}
+			}
+			var prevCumulative float64
+			var scalRows [][]string
+			for _, step := range result.RampSteps {
+				var cumulative float64
+				for _, gpuIdx := range step.GPUIndices {
+					for _, g := range result.GPUs {
+						if g.Index != gpuIdx {
+							continue
+						}
+						s := g.StablePowerLimitW
+						if s <= 0 {
+							s = g.AppliedPowerLimitW
+						}
+						cumulative += s
+					}
+				}
+				incremental := cumulative - prevCumulative
+				efficiency := "—"
+				if step.StepIndex > 1 && firstStable > 0 {
+					efficiency = fmt.Sprintf("%.1f%%", incremental/firstStable*100)
+				}
+				scalRows = append(scalRows, []string{
+					fmt.Sprintf("%d", step.StepIndex),
+					joinIndexList(step.GPUIndices),
+					fmt.Sprintf("%.0f W", cumulative),
+					fmt.Sprintf("%.0f W", incremental),
+					efficiency,
+				})
+				prevCumulative = cumulative
+			}
+			b.WriteString(fmtMDTable([]string{"Step", "GPUs", "Cumulative stable TDP", "Incremental", "Efficiency vs GPU 1"}, scalRows))
+			b.WriteString("\n")
+		}
+	}
+
+	// ── Per-GPU sections ──────────────────────────────────────────────────────
+	var lastStep *NvidiaPowerBenchStep
+	if n := len(result.RampSteps); n > 0 {
+		lastStep = &result.RampSteps[n-1]
+	}
+	for _, gpu := range result.GPUs {
+		fmt.Fprintf(&b, "### GPU %d — %s\n\n", gpu.Index, gpu.Name)
+
+		// Transposed comparison table: Single Run vs All GPU Run.
+		singleClk := "—"
+		singleMem := "—"
+		singleTemp := "—"
+		singlePwr := "—"
+		singleWall := "—"
+		singleFan := "—"
+		if gpu.Telemetry != nil {
+			singleClk = fmt.Sprintf("%.0f", gpu.Telemetry.AvgGraphicsClockMHz)
+			singleMem = fmt.Sprintf("%.0f", gpu.Telemetry.AvgMemoryClockMHz)
+			singleTemp = fmt.Sprintf("%.1f", gpu.Telemetry.AvgTempC)
+			singlePwr = fmt.Sprintf("%.0f W", gpu.Telemetry.AvgPowerW)
+		}
+		if gpu.ServerDeltaW > 0 {
+			singleWall = fmt.Sprintf("%.0f W", gpu.ServerDeltaW)
+		}
+		if gpu.AvgFanRPM > 0 {
+			if gpu.AvgFanDutyCyclePct > 0 {
+				singleFan = fmt.Sprintf("%.0f RPM (%.0f%%)", gpu.AvgFanRPM, gpu.AvgFanDutyCyclePct)
+			} else {
+				singleFan = fmt.Sprintf("%.0f RPM", gpu.AvgFanRPM)
+			}
+		}
+
+		allClk := "—"
+		allMem := "—"
+		allTemp := "—"
+		allPwr := "—"
+		allWall := "—"
+		allFan := "—"
+		if lastStep != nil {
+			if t, ok := lastStep.PerGPUTelemetry[gpu.Index]; ok && t != nil {
+				allClk = fmt.Sprintf("%.0f", t.AvgGraphicsClockMHz)
+				allMem = fmt.Sprintf("%.0f", t.AvgMemoryClockMHz)
+				allTemp = fmt.Sprintf("%.1f", t.AvgTempC)
+				allPwr = fmt.Sprintf("%.0f W", t.AvgPowerW)
+			}
+			if lastStep.ServerDeltaW > 0 && len(lastStep.GPUIndices) > 0 {
+				allWall = fmt.Sprintf("%.0f W", lastStep.ServerDeltaW/float64(len(lastStep.GPUIndices)))
+			}
+			if lastStep.AvgFanRPM > 0 {
+				if lastStep.AvgFanDutyCyclePct > 0 {
+					allFan = fmt.Sprintf("%.0f RPM (%.0f%%)", lastStep.AvgFanRPM, lastStep.AvgFanDutyCyclePct)
+				} else {
+					allFan = fmt.Sprintf("%.0f RPM", lastStep.AvgFanRPM)
+				}
+			}
+		}
+
+		tableHeaders := []string{"", "Single Run"}
+		if lastStep != nil {
+			tableHeaders = append(tableHeaders, "All GPU Run")
+		}
+		compRows := [][]string{
+			{"Clock MHz (Mem MHz)", fmt.Sprintf("%s (%s)", singleClk, singleMem)},
+			{"Avg Temp °C", singleTemp},
+			{"Power W", singlePwr},
+			{"Per GPU wall W", singleWall},
+			{"Avg Fan RPM (duty%)", singleFan},
+		}
+		if lastStep != nil {
+			compRows[0] = append(compRows[0], fmt.Sprintf("%s (%s)", allClk, allMem))
+			compRows[1] = append(compRows[1], allTemp)
+			compRows[2] = append(compRows[2], allPwr)
+			compRows[3] = append(compRows[3], allWall)
+			compRows[4] = append(compRows[4], allFan)
+		}
+		b.WriteString(fmtMDTable(tableHeaders, compRows))
+		b.WriteString("\n")
+
+		for _, note := range gpu.Notes {
+			fmt.Fprintf(&b, "- %s\n", note)
+		}
+		if len(gpu.Notes) > 0 {
+			b.WriteString("\n")
+		}
+	}
+	return b.String()
+}
+
+func renderPowerBenchSummary(result NvidiaPowerBenchResult) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "run_at_utc=%s\n", result.GeneratedAt.Format(time.RFC3339))
+	fmt.Fprintf(&b, "benchmark_version=%s\n", result.BenchmarkVersion)
+	fmt.Fprintf(&b, "benchmark_profile=%s\n", result.BenchmarkProfile)
+	fmt.Fprintf(&b, "overall_status=%s\n", result.OverallStatus)
+	fmt.Fprintf(&b, "platform_max_tdp_w=%.0f\n", result.PlatformMaxTDPW)
+	fmt.Fprintf(&b, "gpu_count=%d\n", len(result.GPUs))
+	if len(result.RecommendedSlotOrder) > 0 {
+		fmt.Fprintf(&b, "recommended_slot_order=%s\n", joinIndexList(result.RecommendedSlotOrder))
+	}
+	for _, step := range result.RampSteps {
+		fmt.Fprintf(&b, "ramp_step_%d_gpus=%s\n", step.StepIndex, joinIndexList(step.GPUIndices))
+		fmt.Fprintf(&b, "ramp_step_%d_new_gpu=%d\n", step.StepIndex, step.NewGPUIndex)
+		fmt.Fprintf(&b, "ramp_step_%d_stable_limit_w=%.0f\n", step.StepIndex, step.NewGPUStableLimitW)
+		fmt.Fprintf(&b, "ramp_step_%d_total_power_w=%.0f\n", step.StepIndex, step.TotalObservedPowerW)
+		if step.ServerLoadedW > 0 {
+			fmt.Fprintf(&b, "ramp_step_%d_server_loaded_w=%.0f\n", step.StepIndex, step.ServerLoadedW)
+			fmt.Fprintf(&b, "ramp_step_%d_server_delta_w=%.0f\n", step.StepIndex, step.ServerDeltaW)
+		}
+	}
+	for _, gpu := range result.GPUs {
+		if gpu.StablePowerLimitW > 0 {
+			fmt.Fprintf(&b, "gpu_%d_stable_limit_w=%.0f\n", gpu.Index, gpu.StablePowerLimitW)
+		}
+		if gpu.ServerLoadedW > 0 {
+			fmt.Fprintf(&b, "gpu_%d_server_loaded_w=%.0f\n", gpu.Index, gpu.ServerLoadedW)
+			fmt.Fprintf(&b, "gpu_%d_server_delta_w=%.0f\n", gpu.Index, gpu.ServerDeltaW)
+		}
+	}
+	if sp := result.ServerPower; sp != nil && sp.Available {
+		fmt.Fprintf(&b, "server_idle_w=%.0f\n", sp.IdleW)
+		fmt.Fprintf(&b, "server_loaded_w=%.0f\n", sp.LoadedW)
+		fmt.Fprintf(&b, "server_delta_w=%.0f\n", sp.DeltaW)
+		fmt.Fprintf(&b, "server_reporting_ratio=%.2f\n", sp.ReportingRatio)
+	}
+	return b.String()
+}

audit/internal/platform/benchmark_power_run.go

@@ -0,0 +1,495 @@
+package platform
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+func (s *System) RunNvidiaPowerBench(ctx context.Context, baseDir string, opts NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if logFunc == nil {
+		logFunc = func(string) {}
+	}
+	if strings.TrimSpace(baseDir) == "" {
+		baseDir = "/var/log/bee-bench/power"
+	}
+	opts = normalizeNvidiaBenchmarkOptionsForBenchmark(opts)
+	selected, err := resolveNvidiaGPUSelection(opts.GPUIndices, opts.ExcludeGPUIndices)
+	if err != nil {
+		return "", err
+	}
+	if len(selected) == 0 {
+		return "", fmt.Errorf("no NVIDIA GPUs selected")
+	}
+	ts := time.Now().UTC().Format("20060102-150405")
+	runDir := filepath.Join(baseDir, "power-"+ts)
+	if err := os.MkdirAll(runDir, 0755); err != nil {
+		return "", fmt.Errorf("mkdir %s: %w", runDir, err)
+	}
+	verboseLog := filepath.Join(runDir, "verbose.log")
+	hostname, _ := os.Hostname()
+	result := NvidiaPowerBenchResult{
+		BenchmarkVersion:   benchmarkVersion,
+		GeneratedAt:        time.Now().UTC(),
+		Hostname:           hostname,
+		ServerModel:        readServerModel(),
+		BenchmarkProfile:   opts.Profile,
+		SelectedGPUIndices: append([]int(nil), selected...),
+		OverallStatus:      "OK",
+	}
+	infoByIndex, infoErr := queryBenchmarkGPUInfo(selected)
+	if infoErr != nil {
+		return "", infoErr
+	}
+	// Capture full nvidia-smi -q snapshot at the start of the run.
+	if out, err := runSATCommandCtx(ctx, verboseLog, "00-nvidia-smi-q.log", []string{"nvidia-smi", "-q"}, nil, nil); err == nil {
+		_ = os.WriteFile(filepath.Join(runDir, "00-nvidia-smi-q.log"), out, 0644)
+	}
+	durationSec := powerBenchDurationSec(opts.Profile)
+
+	// Sample server idle power before any GPU load.
+	var serverIdleW float64
+	var serverIdleOK bool
+	idleSDRStopCh := make(chan struct{})
+	idleSDRCh := startIPMISDRSampler(idleSDRStopCh, benchmarkPowerAutotuneSampleInterval)
+	if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, 10, benchmarkPowerAutotuneSampleInterval); ok {
+		serverIdleW = w
+		serverIdleOK = true
+		logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w))
+	}
+	close(idleSDRStopCh)
+	sdrIdle := summarizeSDRPowerSeries(<-idleSDRCh)
+	psuBefore := psuStatusSnapshot()
+
+	// Phase 1: calibrate each GPU individually (sequentially, one at a time) to
+	// establish a true single-card power baseline unaffected by neighbour heat.
+	calibByIndex := make(map[int]benchmarkPowerCalibrationResult, len(selected))
+	singleIPMILoadedW := make(map[int]float64, len(selected))
+	singleRunSummaryByIndex := make(map[int]benchmarkPowerCalibrationRunSummary, len(selected))
+	var allRestoreActions []benchmarkRestoreAction
+	// allPowerRows accumulates telemetry from all phases for the top-level gpu-metrics.csv.
+	var allPowerRows []GPUMetricRow
+	var powerCursor float64
+	for _, idx := range selected {
+		singleDir := filepath.Join(runDir, fmt.Sprintf("single-%02d", idx))
+		_ = os.MkdirAll(singleDir, 0755)
+		singleInfo := cloneBenchmarkGPUInfoMap(infoByIndex)
+		if failed := resetBenchmarkGPUs(ctx, verboseLog, []int{idx}, logFunc); len(failed) > 0 {
+			return "", fmt.Errorf("power benchmark pre-flight: failed to reset GPU %d; benchmark aborted to keep measurements clean", idx)
+		}
+		logFunc(fmt.Sprintf("power calibration: GPU %d single-card baseline", idx))
+		singlePowerStopCh := make(chan struct{})
+		singlePowerCh := startSelectedPowerSourceSampler(singlePowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
+		c, restore, singleRows, singleRun := runBenchmarkPowerCalibration(ctx, verboseLog, singleDir, []int{idx}, singleInfo, logFunc, nil, durationSec)
+		appendBenchmarkMetrics(&allPowerRows, singleRows, fmt.Sprintf("single-gpu-%d", idx), &powerCursor, 0)
+		close(singlePowerStopCh)
+		if samples := <-singlePowerCh; len(samples) > 0 {
+			singleIPMILoadedW[idx] = benchmarkMean(samples)
+			logFunc(fmt.Sprintf("power calibration: GPU %d single-card server power (%s avg): %.0f W", idx, opts.ServerPowerSource, singleIPMILoadedW[idx]))
+		} else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && singleRun.LoadedSDR.PSUInW > 0 {
+			singleIPMILoadedW[idx] = singleRun.LoadedSDR.PSUInW
+			logFunc(fmt.Sprintf("power calibration: GPU %d single-card fallback server power (SDR avg): %.0f W", idx, singleRun.LoadedSDR.PSUInW))
+		}
+		allRestoreActions = append(allRestoreActions, restore...)
+		if r, ok := c[idx]; ok {
+			calibByIndex[idx] = r
+		}
+		singleRunSummaryByIndex[idx] = singleRun
+	}
+	defer func() {
+		for i := len(allRestoreActions) - 1; i >= 0; i-- {
+			allRestoreActions[i].fn()
+		}
+	}()
+	gpus := make([]NvidiaPowerBenchGPU, 0, len(selected))
+	for _, idx := range selected {
+		info := infoByIndex[idx]
+		calib := calibByIndex[idx]
+		status := "OK"
+		if !calib.Completed {
+			status = "FAILED"
+			result.OverallStatus = "PARTIAL"
+		} else if calib.Derated {
+			status = "PARTIAL"
+			if result.OverallStatus == "OK" {
+				result.OverallStatus = "PARTIAL"
+			}
+		}
+		gpu := NvidiaPowerBenchGPU{
+			Index:               idx,
+			Name:                info.Name,
+			BusID:               info.BusID,
+			DefaultPowerLimitW:  info.DefaultPowerLimitW,
+			AppliedPowerLimitW:  calib.AppliedPowerLimitW,
+			MaxObservedPowerW:   calib.Summary.P95PowerW,
+			MaxObservedTempC:    calib.Summary.P95TempC,
+			CalibrationAttempts: calib.Attempts,
+			Derated:             calib.Derated,
+			Status:              status,
+			Notes:               append([]string(nil), calib.Notes...),
+			CoolingWarning:      calib.CoolingWarning,
+		}
+		if w, ok := singleIPMILoadedW[idx]; ok && serverIdleOK && w > 0 {
+			gpu.ServerLoadedW = w
+			gpu.ServerDeltaW = w - serverIdleW
+		}
+		if len(calib.MetricRows) > 0 {
+			t := summarizeBenchmarkTelemetry(calib.MetricRows)
+			gpu.Telemetry = &t
+		}
+		if singleRun := singleRunSummaryByIndex[idx]; singleRun.AvgFanRPM > 0 {
+			gpu.AvgFanRPM = singleRun.AvgFanRPM
+			gpu.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct
+		}
+		gpus = append(gpus, gpu)
+	}
+	sort.Slice(gpus, func(i, j int) bool {
+		if gpus[i].MaxObservedPowerW != gpus[j].MaxObservedPowerW {
+			return gpus[i].MaxObservedPowerW > gpus[j].MaxObservedPowerW
+		}
+		if gpus[i].AppliedPowerLimitW != gpus[j].AppliedPowerLimitW {
+			return gpus[i].AppliedPowerLimitW > gpus[j].AppliedPowerLimitW
+		}
+		if gpus[i].Derated != gpus[j].Derated {
+			return !gpus[i].Derated
+		}
+		return gpus[i].Index < gpus[j].Index
+	})
+	result.GPUs = gpus
+	result.RecommendedSlotOrder = make([]int, 0, len(gpus))
+	for _, gpu := range gpus {
+		result.RecommendedSlotOrder = append(result.RecommendedSlotOrder, gpu.Index)
+	}
+	if len(result.RecommendedSlotOrder) > 0 {
+		result.Findings = append(result.Findings, fmt.Sprintf("Recommended slot order for installation based on single-card %s: %s.", benchmarkPowerEngineLabel(benchmarkPowerEngine()), joinIndexList(result.RecommendedSlotOrder)))
+	}
+	for _, gpu := range gpus {
+		if gpu.Derated {
+			result.Findings = append(result.Findings, fmt.Sprintf("GPU %d required reduced power limit %.0f W to complete %s.", gpu.Index, gpu.AppliedPowerLimitW, benchmarkPowerEngineLabel(benchmarkPowerEngine())))
+		}
+		if gpu.CoolingWarning != "" {
+			result.Findings = append(result.Findings, fmt.Sprintf(
+				"GPU %d: %s. Operator action: rerun the benchmark with fan speed manually fixed at 100%% to confirm actual thermal headroom.",
+				gpu.Index, gpu.CoolingWarning,
+			))
+		}
+	}
+	singleByIndex := make(map[int]NvidiaPowerBenchGPU, len(gpus))
+	for _, gpu := range gpus {
+		singleByIndex[gpu.Index] = gpu
+	}
+
+	// Phase 2: cumulative thermal ramp.
+	// Each step introduces one new GPU into an environment where all previously
+	// calibrated GPUs are already running at their fixed stable limits. The new
+	// GPU's stable TDP is searched via binary search under real
+	// multi-GPU thermal load. Once found, its limit is fixed permanently for all
+	// subsequent steps. This ensures each GPU's limit reflects actual sustained
+	// power in the final full-system thermal state.
+	//
+	// stableLimits accumulates GPU index → fixed stable limit (W) across steps.
+	stableLimits := make(map[int]int, len(result.RecommendedSlotOrder))
+
+	// serverLoadedW tracks the IPMI server power from the final ramp step
+	// (all GPUs simultaneously loaded). Earlier steps' values are stored
+	// per-step in NvidiaPowerBenchStep.ServerLoadedW.
+	var serverLoadedW float64
+	var serverLoadedOK bool
+	// sdrLastStep retains the phase-averaged SDR readings from the last ramp step
+	// while GPUs are loaded. Used in the summary instead of re-sampling after the
+	// test when GPUs have already returned to idle.
+	var sdrLastStep benchmarkSDRSeriesSummary
+
+	// Step 1: reuse single-card calibration result directly.
+	if len(result.RecommendedSlotOrder) > 0 {
+		firstIdx := result.RecommendedSlotOrder[0]
+		firstCalib := calibByIndex[firstIdx]
+		stableLimits[firstIdx] = int(math.Round(firstCalib.AppliedPowerLimitW))
+		ramp := NvidiaPowerBenchStep{
+			StepIndex:           1,
+			GPUIndices:          []int{firstIdx},
+			NewGPUIndex:         firstIdx,
+			NewGPUStableLimitW:  firstCalib.AppliedPowerLimitW,
+			TotalObservedPowerW: firstCalib.Summary.P95PowerW,
+			AvgObservedPowerW:   firstCalib.Summary.P95PowerW,
+			Derated:             firstCalib.Derated,
+			Status:              "OK",
+		}
+		if w, ok := singleIPMILoadedW[firstIdx]; ok && serverIdleOK && w > 0 {
+			ramp.ServerLoadedW = w
+			ramp.ServerDeltaW = w - serverIdleW
+		}
+		if singleRun := singleRunSummaryByIndex[firstIdx]; singleRun.AvgFanRPM > 0 {
+			ramp.AvgFanRPM = singleRun.AvgFanRPM
+			ramp.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct
+		}
+		firstSummary := firstCalib.Summary
+		ramp.PerGPUTelemetry = map[int]*BenchmarkTelemetrySummary{firstIdx: &firstSummary}
+		if !firstCalib.Completed {
+			ramp.Status = "FAILED"
+			ramp.Notes = append(ramp.Notes, fmt.Sprintf("GPU %d did not complete single-card %s", firstIdx, benchmarkPowerEngineLabel(benchmarkPowerEngine())))
+			result.OverallStatus = "PARTIAL"
+		} else if firstCalib.Derated {
+			ramp.Status = "PARTIAL"
+			if result.OverallStatus == "OK" {
+				result.OverallStatus = "PARTIAL"
+			}
+			result.Findings = append(result.Findings, fmt.Sprintf("Ramp step 1 (GPU %d) required derating to %.0f W.", firstIdx, firstCalib.AppliedPowerLimitW))
+		}
+		result.RampSteps = append(result.RampSteps, ramp)
+		logFunc(fmt.Sprintf("power ramp: step 1/%d — reused single-card calibration for GPU %d, stable limit %.0f W",
+			len(result.RecommendedSlotOrder), firstIdx, firstCalib.AppliedPowerLimitW))
+	}
+
+	// Steps 2..N: each step revalidates every already-active GPU under the new
+	// cumulative thermal environment and also calibrates the newly introduced
+	// GPU. Previously found limits are used only as seeds for the search.
+	for stepNum := 1; stepNum < len(result.RecommendedSlotOrder); stepNum++ {
+		step := stepNum + 1
+		subset := append([]int(nil), result.RecommendedSlotOrder[:step]...)
+		newGPUIdx := result.RecommendedSlotOrder[stepNum]
+		stepDir := filepath.Join(runDir, fmt.Sprintf("step-%02d", step))
+		_ = os.MkdirAll(stepDir, 0755)
+
+		// Reuse the latest stable limits as starting points, but re-check every
+		// active GPU in this hotter configuration. For the newly introduced GPU,
+		// seed from its single-card calibration so we do not restart from the
+		// default TDP when a prior derated limit is already known.
+		seedForStep := make(map[int]int, len(subset))
+		for _, idx := range subset {
+			if lim, ok := stableLimits[idx]; ok && lim > 0 {
+				seedForStep[idx] = lim
+				continue
+			}
+			if base, ok := calibByIndex[idx]; ok {
+				lim := int(math.Round(base.AppliedPowerLimitW))
+				if lim > 0 {
+					seedForStep[idx] = lim
+				}
+			}
+		}
+
+		logFunc(fmt.Sprintf("power ramp: step %d/%d — revalidating %d active GPU(s) including new GPU %d",
+			step, len(result.RecommendedSlotOrder), len(subset), newGPUIdx))
+
+		stepInfo := cloneBenchmarkGPUInfoMap(infoByIndex)
+		stepPowerStopCh := make(chan struct{})
+		stepPowerCh := startSelectedPowerSourceSampler(stepPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
+		stepCalib, stepRestore, stepRows, stepRun := runBenchmarkPowerCalibration(ctx, verboseLog, stepDir, subset, stepInfo, logFunc, seedForStep, durationSec)
+		appendBenchmarkMetrics(&allPowerRows, stepRows, fmt.Sprintf("ramp-step-%d", step), &powerCursor, 0)
+		close(stepPowerStopCh)
+		var stepIPMILoadedW float64
+		var stepIPMIOK bool
+		if samples := <-stepPowerCh; len(samples) > 0 {
+			stepIPMILoadedW = benchmarkMean(samples)
+			stepIPMIOK = true
+		}
+		// Accumulate restore actions; they all run in the outer defer.
+		allRestoreActions = append(allRestoreActions, stepRestore...)
+
+		ramp := NvidiaPowerBenchStep{
+			StepIndex:   step,
+			GPUIndices:  subset,
+			NewGPUIndex: newGPUIdx,
+			Status:      "OK",
+		}
+
+		// Total observed power = sum of p95 across all GPUs in this step.
+		for _, idx := range subset {
+			if c, ok := stepCalib[idx]; ok {
+				ramp.TotalObservedPowerW += c.Summary.P95PowerW
+			}
+		}
+		if len(subset) > 0 {
+			ramp.AvgObservedPowerW = ramp.TotalObservedPowerW / float64(len(subset))
+		}
+
+		for _, idx := range subset {
+			c, ok := stepCalib[idx]
+			if !ok || !c.Completed {
+				fallback := 0
+				if lim, ok := stableLimits[idx]; ok && lim > 0 {
+					fallback = lim
+				} else if fb, ok := calibByIndex[idx]; ok {
+					fallback = int(math.Round(fb.AppliedPowerLimitW))
+				}
+				if fallback > 0 {
+					stableLimits[idx] = fallback
+				}
+				ramp.Status = "FAILED"
+				ramp.Notes = append(ramp.Notes,
+					fmt.Sprintf("GPU %d did not complete %s in ramp step %d; keeping previous stable limit %d W", idx, benchmarkPowerEngineLabel(benchmarkPowerEngine()), step, fallback))
+				result.OverallStatus = "PARTIAL"
+				continue
+			}
+
+			prevLimit, hadPrev := stableLimits[idx]
+			newLimit := int(math.Round(c.AppliedPowerLimitW))
+			stableLimits[idx] = newLimit
+			if idx == newGPUIdx {
+				ramp.NewGPUStableLimitW = c.AppliedPowerLimitW
+				ramp.Derated = c.Derated
+			}
+			if c.Derated {
+				ramp.Status = "PARTIAL"
+				if result.OverallStatus == "OK" {
+					result.OverallStatus = "PARTIAL"
+				}
+			}
+			if hadPrev && newLimit < prevLimit {
+				ramp.Notes = append(ramp.Notes,
+					fmt.Sprintf("GPU %d was re-derated from %d W to %d W under combined thermal load.", idx, prevLimit, newLimit))
+			}
+		}
+
+		if c, ok := stepCalib[newGPUIdx]; ok && c.Completed && c.Derated {
+			result.Findings = append(result.Findings, fmt.Sprintf("Ramp step %d (GPU %d) required derating to %.0f W under combined thermal load.", step, newGPUIdx, c.AppliedPowerLimitW))
+		}
+
+		// Per-step PSU slot readings are averaged over the whole load phase rather
+		// than captured as a single end-of-phase snapshot.
+		sdrStep := stepRun.LoadedSDR
+		if len(sdrStep.PSUSlots) > 0 {
+			ramp.PSUSlotReadings = sdrStep.PSUSlots
+		}
+
+		if stepIPMIOK && serverIdleOK && stepIPMILoadedW > 0 {
+			ramp.ServerLoadedW = stepIPMILoadedW
+			ramp.ServerDeltaW = stepIPMILoadedW - serverIdleW
+			logFunc(fmt.Sprintf("power ramp: step %d server loaded power (%s avg): %.0f W", step, opts.ServerPowerSource, stepIPMILoadedW))
+			// The last step has all GPUs loaded — use it as the top-level loaded_w.
+			if step == len(result.RecommendedSlotOrder) {
+				serverLoadedW = stepIPMILoadedW
+				serverLoadedOK = true
+				sdrLastStep = sdrStep
+			}
+		} else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && sdrStep.PSUInW > 0 {
+			ramp.ServerLoadedW = sdrStep.PSUInW
+			ramp.ServerDeltaW = sdrStep.PSUInW - sdrIdle.PSUInW
+			logFunc(fmt.Sprintf("power ramp: step %d fallback server loaded power (SDR avg): %.0f W", step, sdrStep.PSUInW))
+			if step == len(result.RecommendedSlotOrder) {
+				serverLoadedW = sdrStep.PSUInW
+				serverLoadedOK = true
+				sdrLastStep = sdrStep
+			}
+		}
+
+		// Fan values are phase averages over the same load window.
+		if stepRun.AvgFanRPM > 0 {
+			ramp.AvgFanRPM = stepRun.AvgFanRPM
+			ramp.AvgFanDutyCyclePct = stepRun.AvgFanDutyCyclePct
+		}
+
+		// Per-GPU telemetry from this ramp step's calibration.
+		ramp.PerGPUTelemetry = make(map[int]*BenchmarkTelemetrySummary, len(subset))
+		for _, gpuIdx := range subset {
+			if c, ok := stepCalib[gpuIdx]; ok {
+				s := c.Summary
+				ramp.PerGPUTelemetry[gpuIdx] = &s
+			}
+		}
+
+		result.RampSteps = append(result.RampSteps, ramp)
+	}
+
+	// Populate StablePowerLimitW on each GPU entry from the accumulated stable limits.
+	for i := range result.GPUs {
+		if lim, ok := stableLimits[result.GPUs[i].Index]; ok {
+			result.GPUs[i].StablePowerLimitW = float64(lim)
+		}
+		if result.GPUs[i].StablePowerLimitW > 0 && result.GPUs[i].AppliedPowerLimitW > 0 &&
+			result.GPUs[i].StablePowerLimitW < result.GPUs[i].AppliedPowerLimitW {
+			result.GPUs[i].Derated = true
+			result.Findings = append(result.Findings, fmt.Sprintf(
+				"GPU %d required additional derating from %.0f W (single-card) to %.0f W under full-system thermal load.",
+				result.GPUs[i].Index, result.GPUs[i].AppliedPowerLimitW, result.GPUs[i].StablePowerLimitW,
+			))
+		}
+	}
+
+	// PlatformMaxTDPW = sum of all stable limits — the actual sustained power
+	// budget of this server with all GPUs running simultaneously without throttling.
+	for _, lim := range stableLimits {
+		result.PlatformMaxTDPW += float64(lim)
+	}
+
+	// Characterize server power from IPMI idle/loaded samples.
+	// gpuActualSumW = sum of p95 GPU power from the last ramp step — actual
+	// measured consumption, not the stable limit cap. This is the correct
+	// denominator for the reporting ratio: limit caps (PlatformMaxTDPW) inflate
+	// the denominator and make the ratio appear artificially low.
+	var gpuActualSumW float64
+	if n := len(result.RampSteps); n > 0 {
+		gpuActualSumW = result.RampSteps[n-1].TotalObservedPowerW
+	}
+	if gpuActualSumW <= 0 {
+		gpuActualSumW = result.PlatformMaxTDPW
+	}
+	_ = serverIdleOK // used implicitly via characterizeServerPower
+	result.ServerPower = characterizeServerPower(serverIdleW, serverLoadedW, gpuActualSumW, opts.ServerPowerSource, serverIdleOK && serverLoadedOK)
+	// Supplement DCMI with SDR multi-source data via collector's PSU slot patterns.
+	// Per-slot readings enable correlation with audit HardwarePowerSupply entries.
+	if result.ServerPower != nil {
+		// Use the SDR phase average from the last ramp step (GPUs still loaded)
+		// rather than re-sampling here, which would capture post-test idle state.
+		sdrLoaded := sdrLastStep
+		result.ServerPower.PSUInputIdleW = sdrIdle.PSUInW
+		result.ServerPower.PSUInputLoadedW = sdrLoaded.PSUInW
+		result.ServerPower.PSUOutputIdleW = sdrIdle.PSUOutW
+		result.ServerPower.PSUOutputLoadedW = sdrLoaded.PSUOutW
+		result.ServerPower.GPUSlotTotalW = sdrLoaded.GPUSlotW
+		if len(sdrIdle.PSUSlots) > 0 {
+			result.ServerPower.PSUSlotReadingsIdle = sdrIdle.PSUSlots
+		}
+		if len(sdrLoaded.PSUSlots) > 0 {
+			result.ServerPower.PSUSlotReadingsLoaded = sdrLoaded.PSUSlots
+		}
+		if sdrIdle.PSUInW > 0 && result.ServerPower.IdleW > 0 {
+			result.ServerPower.DCMICoverageRatio = result.ServerPower.IdleW / sdrIdle.PSUInW
+		}
+		if len(sdrLoaded.SkippedSensors) > 0 {
+			result.ServerPower.Notes = append(result.ServerPower.Notes,
+				"SDR sensors skipped (self-healed): "+strings.Join(sdrLoaded.SkippedSensors, "; "))
+		}
+		if sdrLoaded.Samples > 0 {
+			result.ServerPower.Notes = append(result.ServerPower.Notes,
+				fmt.Sprintf("Final SDR PSU loaded values are phase averages across %d sample(s) from the last full-load step.", sdrLoaded.Samples))
+		}
+		// Detect DCMI partial coverage: direct SDR comparison first,
+		// ramp heuristic as fallback when SDR PSU sensors are absent.
+		dcmiUnreliable := detectDCMIPartialCoverage(result.ServerPower) ||
+			(sdrIdle.PSUInW == 0 && detectIPMISaturationFallback(result.RampSteps))
+		if dcmiUnreliable {
+			result.ServerPower.Notes = append(result.ServerPower.Notes,
+				fmt.Sprintf("IPMI DCMI covers only a subset of installed PSUs (coverage %.0f%%). "+
+					"Use SDR PSU Δ ratio for GPU accuracy assessment; DCMI ratio is not reliable.",
+					result.ServerPower.DCMICoverageRatio*100))
+		}
+	}
+	result.PSUIssues = diffPSUStatus(psuBefore, psuStatusSnapshot())
+	// Write top-level gpu-metrics.csv/.html aggregating all phases.
+	writeBenchmarkMetricsFiles(runDir, allPowerRows)
+	resultJSON, err := json.MarshalIndent(result, "", "  ")
+	if err != nil {
+		return "", fmt.Errorf("marshal power result: %w", err)
+	}
+	if err := os.WriteFile(filepath.Join(runDir, "result.json"), resultJSON, 0644); err != nil {
+		return "", fmt.Errorf("write result.json: %w", err)
+	}
+	if err := os.WriteFile(filepath.Join(runDir, "report.md"), []byte(renderPowerBenchReport(result)), 0644); err != nil {
+		return "", fmt.Errorf("write report.md: %w", err)
+	}
+	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(renderPowerBenchSummary(result)), 0644); err != nil {
+		return "", fmt.Errorf("write summary.txt: %w", err)
+	}
+	return runDir, nil
+}

audit/internal/platform/benchmark_result_analysis.go

@@ -0,0 +1,537 @@
+package platform
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+func runBenchmarkInterconnect(ctx context.Context, verboseLog, runDir string, gpuIndices []int, spec benchmarkProfileSpec, logFunc func(string)) *BenchmarkInterconnectResult {
+	result := &BenchmarkInterconnectResult{
+		Status:             "UNSUPPORTED",
+		Attempted:          true,
+		SelectedGPUIndices: append([]int(nil), gpuIndices...),
+	}
+	cmd := []string{
+		"all_reduce_perf",
+		"-b", "512M",
+		"-e", "4G",
+		"-f", "2",
+		"-g", strconv.Itoa(len(gpuIndices)),
+		"--iters", strconv.Itoa(maxInt(20, spec.NCCLSec/10)),
+	}
+	env := []string{
+		"CUDA_DEVICE_ORDER=PCI_BUS_ID",
+		"CUDA_VISIBLE_DEVICES=" + joinIndexList(gpuIndices),
+	}
+	logFunc(fmt.Sprintf("NCCL interconnect: gpus=%s", joinIndexList(gpuIndices)))
+	out, err := runSATCommandCtx(ctx, verboseLog, "nccl-all-reduce.log", cmd, env, logFunc)
+	_ = os.WriteFile(filepath.Join(runDir, "nccl-all-reduce.log"), out, 0644)
+	if err != nil {
+		result.Notes = append(result.Notes, strings.TrimSpace(string(out)))
+		return result
+	}
+	avgAlg, maxAlg, avgBus, maxBus := parseNCCLAllReduceOutput(string(out))
+	result.Status = "OK"
+	result.Supported = true
+	result.AvgAlgBWGBps = avgAlg
+	result.MaxAlgBWGBps = maxAlg
+	result.AvgBusBWGBps = avgBus
+	result.MaxBusBWGBps = maxBus
+	return result
+}
+
+func parseNCCLAllReduceOutput(raw string) (avgAlg, maxAlg, avgBus, maxBus float64) {
+	lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
+	var algs []float64
+	var buses []float64
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) < 8 {
+			continue
+		}
+		for i := 0; i+2 < len(fields); i++ {
+			timeVal, err1 := strconv.ParseFloat(fields[i], 64)
+			algVal, err2 := strconv.ParseFloat(fields[i+1], 64)
+			busVal, err3 := strconv.ParseFloat(fields[i+2], 64)
+			if err1 == nil && err2 == nil && err3 == nil && timeVal > 0 {
+				algs = append(algs, algVal)
+				buses = append(buses, busVal)
+				break
+			}
+		}
+	}
+	if len(algs) == 0 {
+		return 0, 0, 0, 0
+	}
+	return benchmarkMean(algs), benchmarkMax(algs), benchmarkMean(buses), benchmarkMax(buses)
+}
+
+func queryThrottleCounters(gpuIndex int) (BenchmarkThrottleCounters, error) {
+	out, err := satExecCommand(
+		"nvidia-smi",
+		"--id="+strconv.Itoa(gpuIndex),
+		"--query-gpu=clocks_event_reasons_counters.sw_power_cap,clocks_event_reasons_counters.sw_thermal_slowdown,clocks_event_reasons_counters.sync_boost,clocks_event_reasons_counters.hw_thermal_slowdown,clocks_event_reasons_counters.hw_power_brake_slowdown",
+		"--format=csv,noheader,nounits",
+	).Output()
+	if err != nil {
+		return BenchmarkThrottleCounters{}, err
+	}
+	fields := strings.Split(strings.TrimSpace(string(out)), ",")
+	if len(fields) < 5 {
+		return BenchmarkThrottleCounters{}, fmt.Errorf("unexpected throttle counter columns: %q", strings.TrimSpace(string(out)))
+	}
+	return BenchmarkThrottleCounters{
+		SWPowerCapUS:           parseBenchmarkUint64(fields[0]),
+		SWThermalSlowdownUS:    parseBenchmarkUint64(fields[1]),
+		SyncBoostUS:            parseBenchmarkUint64(fields[2]),
+		HWThermalSlowdownUS:    parseBenchmarkUint64(fields[3]),
+		HWPowerBrakeSlowdownUS: parseBenchmarkUint64(fields[4]),
+	}, nil
+}
+
+func diffThrottleCounters(before, after BenchmarkThrottleCounters) BenchmarkThrottleCounters {
+	return BenchmarkThrottleCounters{
+		SWPowerCapUS:           saturatingSub(after.SWPowerCapUS, before.SWPowerCapUS),
+		SWThermalSlowdownUS:    saturatingSub(after.SWThermalSlowdownUS, before.SWThermalSlowdownUS),
+		SyncBoostUS:            saturatingSub(after.SyncBoostUS, before.SyncBoostUS),
+		HWThermalSlowdownUS:    saturatingSub(after.HWThermalSlowdownUS, before.HWThermalSlowdownUS),
+		HWPowerBrakeSlowdownUS: saturatingSub(after.HWPowerBrakeSlowdownUS, before.HWPowerBrakeSlowdownUS),
+	}
+}
+
+func queryECCCounters(gpuIndex int) (BenchmarkECCCounters, error) {
+	out, err := satExecCommand(
+		"nvidia-smi",
+		"--id="+strconv.Itoa(gpuIndex),
+		"--query-gpu=ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total",
+		"--format=csv,noheader,nounits",
+	).Output()
+	if err != nil {
+		return BenchmarkECCCounters{}, err
+	}
+	fields := strings.Split(strings.TrimSpace(string(out)), ",")
+	if len(fields) < 2 {
+		return BenchmarkECCCounters{}, fmt.Errorf("unexpected ECC counter columns: %q", strings.TrimSpace(string(out)))
+	}
+	corrected, err1 := strconv.ParseUint(strings.TrimSpace(fields[0]), 10, 64)
+	uncorrected, err2 := strconv.ParseUint(strings.TrimSpace(fields[1]), 10, 64)
+	if err1 != nil || err2 != nil {
+		// ECC may be disabled on this GPU — return zero counters silently.
+		return BenchmarkECCCounters{}, nil
+	}
+	return BenchmarkECCCounters{Corrected: corrected, Uncorrected: uncorrected}, nil
+}
+
+func diffECCCounters(before, after BenchmarkECCCounters) BenchmarkECCCounters {
+	return BenchmarkECCCounters{
+		Corrected:   saturatingSub(after.Corrected, before.Corrected),
+		Uncorrected: saturatingSub(after.Uncorrected, before.Uncorrected),
+	}
+}
+
+func queryActiveComputeApps(gpuIndices []int) ([]string, error) {
+	args := []string{
+		"--query-compute-apps=gpu_uuid,pid,process_name",
+		"--format=csv,noheader,nounits",
+	}
+	if len(gpuIndices) > 0 {
+		args = append([]string{"--id=" + joinIndexList(gpuIndices)}, args...)
+	}
+	out, err := satExecCommand("nvidia-smi", args...).Output()
+	if err != nil {
+		return nil, err
+	}
+	var lines []string
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		lines = append(lines, line)
+	}
+	return lines, nil
+}
+
+func finalizeBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkGPUResult {
+	if gpu.Status == "" {
+		gpu.Status = "OK"
+	}
+	if gpu.Scores.CompositeScore == 0 {
+		gpu.Scores.CompositeScore = gpu.Scores.ComputeScore
+	}
+	return gpu
+}
+
+func buildBenchmarkFindings(result NvidiaBenchmarkResult) []string {
+	var findings []string
+
+	passed := 0
+	for _, gpu := range result.GPUs {
+		if gpu.Status == "OK" {
+			passed++
+		}
+	}
+	total := len(result.GPUs)
+	if total > 0 {
+		if passed == total {
+			findings = append(findings, fmt.Sprintf("All %d GPU(s) passed the benchmark.", total))
+		} else {
+			findings = append(findings, fmt.Sprintf("%d of %d GPU(s) passed the benchmark.", passed, total))
+		}
+	}
+
+	if result.Normalization.Status != "full" {
+		findings = append(findings, "Environment normalization was partial; compare results with caution.")
+	}
+	for _, gpu := range result.GPUs {
+		if gpu.Status == "FAILED" && len(gpu.DegradationReasons) == 0 {
+			findings = append(findings, fmt.Sprintf("GPU %d failed the benchmark (check verbose.log for details).", gpu.Index))
+			continue
+		}
+		if len(gpu.DegradationReasons) == 0 && gpu.Status == "OK" {
+			findings = append(findings, fmt.Sprintf("GPU %d held clocks without observable throttle counters during steady state.", gpu.Index))
+			continue
+		}
+		for _, reason := range gpu.DegradationReasons {
+			switch reason {
+			case "power_capped":
+				findings = append(findings, fmt.Sprintf(
+					"[POWER] GPU %d: power cap throttle %.1f%% of steady state — server is not delivering full TDP to the GPU.",
+					gpu.Index, gpu.Scores.PowerCapThrottlePct))
+			case "thermal_limited":
+				// Hard stop check: thermal throttle while fans are not at maximum.
+				// This means the server does not see GPU thermals — incompatible config.
+				if result.Cooling != nil && result.Cooling.FanDutyCycleAvailable &&
+					result.Cooling.P95FanDutyCyclePct < 95 {
+					findings = append(findings, fmt.Sprintf(
+						"[HARD STOP] GPU %d: thermal throttle (%.1f%% of time) while fans peaked at only %.0f%% duty cycle — server cooling is not responding to GPU heat load. Configuration is likely incompatible.",
+						gpu.Index, gpu.Scores.ThermalThrottlePct, result.Cooling.P95FanDutyCyclePct))
+				} else {
+					findings = append(findings, fmt.Sprintf(
+						"[THERMAL] GPU %d: thermal throttle %.1f%% of steady state.",
+						gpu.Index, gpu.Scores.ThermalThrottlePct))
+				}
+			case "sync_boost_limited":
+				findings = append(findings, fmt.Sprintf(
+					"[SYNC] GPU %d: sync boost throttle %.1f%% of steady state — GPUs are constraining each other's clocks.",
+					gpu.Index, gpu.Scores.SyncBoostThrottlePct))
+			case "low_sm_clock_vs_target":
+				findings = append(findings, fmt.Sprintf("GPU %d average SM clock stayed below the requested lock target.", gpu.Index))
+			case "variance_too_high":
+				findings = append(findings, fmt.Sprintf("GPU %d showed unstable clocks/power over the benchmark window.", gpu.Index))
+			case "normalization_partial":
+				findings = append(findings, fmt.Sprintf("GPU %d ran without full benchmark normalization.", gpu.Index))
+			case "power_limit_derated":
+				findings = append(findings, fmt.Sprintf("[POWER] GPU %d could not sustain full TDP in this server; benchmark ran at reduced limit %.0f W.", gpu.Index, gpu.PowerLimitW))
+			case "ecc_uncorrected_errors":
+				findings = append(findings, fmt.Sprintf(
+					"[HARD STOP] GPU %d: %d uncorrected ECC error(s) detected — possible hardware fault. Do not use in production.",
+					gpu.Index, gpu.ECC.Uncorrected))
+			case "ecc_corrected_errors":
+				findings = append(findings, fmt.Sprintf(
+					"[WARNING] GPU %d: %d corrected ECC error(s) — possible DRAM degradation, monitor closely.",
+					gpu.Index, gpu.ECC.Corrected))
+			}
+		}
+		// Temperature headroom checks — independent of throttle counters.
+		// Shutdown and slowdown thresholds are per-GPU from nvidia-smi -q;
+		// fall back to 90°C / 80°C when unavailable.
+		if gpu.Steady.P95TempC > 0 {
+			shutdownTemp := gpu.ShutdownTempC
+			if shutdownTemp <= 0 {
+				shutdownTemp = 90
+			}
+			slowdownTemp := gpu.SlowdownTempC
+			if slowdownTemp <= 0 {
+				slowdownTemp = 80
+			}
+			headroom := shutdownTemp - gpu.Steady.P95TempC
+			switch {
+			case headroom < 10:
+				findings = append(findings, fmt.Sprintf(
+					"[HARD STOP] GPU %d: p95 temperature %.1f°C — only %.1f°C from shutdown threshold (%.0f°C). Do not operate.",
+					gpu.Index, gpu.Steady.P95TempC, headroom, shutdownTemp))
+			case gpu.Steady.P95TempC >= slowdownTemp:
+				findings = append(findings, fmt.Sprintf(
+					"[THERMAL] GPU %d: p95 temperature %.1f°C exceeds slowdown threshold (%.0f°C) — %.1f°C headroom to shutdown. Operating in degraded reliability zone.",
+					gpu.Index, gpu.Steady.P95TempC, slowdownTemp, headroom))
+			}
+		}
+		if gpu.CoolingWarning != "" {
+			findings = append(findings, fmt.Sprintf(
+				"GPU %d: %s. Operator action: rerun the benchmark with fan speed manually fixed at 100%% to confirm actual thermal headroom.",
+				gpu.Index, gpu.CoolingWarning,
+			))
+		}
+		if len(gpu.PrecisionFailures) > 0 {
+			findings = append(findings, fmt.Sprintf("GPU %d had incomplete precision coverage: %s.", gpu.Index, strings.Join(gpu.PrecisionFailures, ", ")))
+		}
+		if gpu.Backend == "driver-ptx" {
+			findings = append(findings, fmt.Sprintf("GPU %d used driver PTX fallback; tensor score is intentionally degraded.", gpu.Index))
+		}
+		if gpu.DefaultPowerLimitW > 0 && gpu.PowerLimitW > 0 && gpu.PowerLimitW < gpu.DefaultPowerLimitW*0.95 {
+			findings = append(findings, fmt.Sprintf(
+				"GPU %d power limit %.0f W is below default %.0f W (%.0f%%). Performance may be artificially reduced.",
+				gpu.Index, gpu.PowerLimitW, gpu.DefaultPowerLimitW, gpu.PowerLimitW/gpu.DefaultPowerLimitW*100,
+			))
+		}
+		// Flag significant TDP deviation (over or under) from calibration.
+		if gpu.CalibratedPeakPowerW > 0 {
+			ref := gpu.DefaultPowerLimitW
+			if ref <= 0 {
+				ref = gpu.PowerLimitW
+			}
+			if ref > 0 {
+				deviationPct := (gpu.CalibratedPeakPowerW - ref) / ref * 100
+				switch {
+				case deviationPct < -10:
+					findings = append(findings, fmt.Sprintf(
+						"GPU %d reached only %.0f W (%.0f%% of rated %.0f W) under targeted_power. Check power delivery or cooling.",
+						gpu.Index, gpu.CalibratedPeakPowerW, gpu.CalibratedPeakPowerW/ref*100, ref,
+					))
+				case deviationPct > 5:
+					findings = append(findings, fmt.Sprintf(
+						"GPU %d exceeded rated TDP: %.0f W measured vs %.0f W rated (+%.0f%%). Power limit may not be enforced correctly.",
+						gpu.Index, gpu.CalibratedPeakPowerW, ref, deviationPct,
+					))
+				}
+			}
+		}
+	}
+	if result.Interconnect != nil && result.Interconnect.Supported {
+		findings = append(findings, fmt.Sprintf("Multi-GPU all_reduce max bus bandwidth: %.1f GB/s.", result.Interconnect.MaxBusBWGBps))
+	}
+	if cl := result.CPULoad; cl != nil {
+		switch cl.Status {
+		case "high":
+			findings = append(findings, fmt.Sprintf(
+				"Host CPU load was elevated during the benchmark (avg %.1f%%, max %.1f%%). A competing CPU workload may skew GPU results.",
+				cl.AvgPct, cl.MaxPct,
+			))
+		case "unstable":
+			findings = append(findings, fmt.Sprintf(
+				"Host CPU load was erratic during the benchmark (avg %.1f%%, p95 %.1f%%). Results may be less reproducible.",
+				cl.AvgPct, cl.P95Pct,
+			))
+		}
+	}
+	if sp := result.ServerPower; sp != nil && sp.Available && sp.GPUReportedSumW > 0 {
+		dcmiPartial := detectDCMIPartialCoverage(sp)
+		if sp.ReportingRatio < 0.75 && !dcmiPartial {
+			findings = append(findings, fmt.Sprintf(
+				"GPU power reporting may be unreliable: server delta %.0f W vs GPU-reported %.0f W (ratio %.2f). GPU telemetry likely over-reports actual consumption. Composite scores have been penalized accordingly.",
+				sp.DeltaW, sp.GPUReportedSumW, sp.ReportingRatio,
+			))
+		} else if sp.ReportingRatio < 0.75 && dcmiPartial {
+			findings = append(findings, fmt.Sprintf(
+				"IPMI DCMI covers partial PSU set (DCMI/SDR coverage %.0f%%): ratio %.2f reflects DCMI under-reporting, not GPU inaccuracy. GPU telemetry is the reliable power source; use SDR-based ratio for server-side accuracy.",
+				sp.DCMICoverageRatio*100, sp.ReportingRatio,
+			))
+		} else if sp.ReportingRatio > 1.25 {
+			findings = append(findings, fmt.Sprintf(
+				"Server power delta %.0f W exceeds GPU-reported sum %.0f W by %.0f%%. Other components (CPU, NVMe, networking) may be drawing substantial power under GPU load.",
+				sp.DeltaW, sp.GPUReportedSumW, (sp.ReportingRatio-1)*100,
+			))
+		}
+	}
+	return dedupeStrings(findings)
+}
+
+func benchmarkOverallStatus(result NvidiaBenchmarkResult) string {
+	if len(result.GPUs) == 0 {
+		return "FAILED"
+	}
+	hasOK := false
+	hasPartial := result.Normalization.Status != "full"
+	for _, gpu := range result.GPUs {
+		switch gpu.Status {
+		case "OK":
+			hasOK = true
+		case "PARTIAL", "UNSUPPORTED":
+			hasPartial = true
+		}
+	}
+	if !hasOK {
+		return "FAILED"
+	}
+	if hasPartial {
+		return "PARTIAL"
+	}
+	return "OK"
+}
+
+func findBenchmarkNormalization(items []BenchmarkNormalizationGPU, idx int) *BenchmarkNormalizationGPU {
+	for i := range items {
+		if items[i].Index == idx {
+			return &items[i]
+		}
+	}
+	return nil
+}
+
+func classifySATErrorStatus(out []byte, err error) string {
+	status, _ := classifySATResult("benchmark", out, err)
+	if status == "UNSUPPORTED" {
+		return "UNSUPPORTED"
+	}
+	return "FAILED"
+}
+
+func parseBenchmarkFloat(raw string) float64 {
+	raw = strings.TrimSpace(raw)
+	if raw == "" || strings.EqualFold(raw, "n/a") || strings.EqualFold(raw, "[not supported]") {
+		return 0
+	}
+	value, _ := strconv.ParseFloat(raw, 64)
+	return value
+}
+
+func parseBenchmarkUint64(raw string) uint64 {
+	raw = strings.TrimSpace(raw)
+	if raw == "" || strings.EqualFold(raw, "n/a") || strings.EqualFold(raw, "[not supported]") {
+		return 0
+	}
+	value, _ := strconv.ParseUint(raw, 10, 64)
+	return value
+}
+
+func benchmarkMean(values []float64) float64 {
+	if len(values) == 0 {
+		return 0
+	}
+	var sum float64
+	for _, value := range values {
+		sum += value
+	}
+	return sum / float64(len(values))
+}
+
+func benchmarkPercentile(values []float64, p float64) float64 {
+	if len(values) == 0 {
+		return 0
+	}
+	copyValues := append([]float64(nil), values...)
+	sort.Float64s(copyValues)
+	if len(copyValues) == 1 {
+		return copyValues[0]
+	}
+	rank := (p / 100.0) * float64(len(copyValues)-1)
+	lower := int(math.Floor(rank))
+	upper := int(math.Ceil(rank))
+	if lower == upper {
+		return copyValues[lower]
+	}
+	frac := rank - float64(lower)
+	return copyValues[lower] + (copyValues[upper]-copyValues[lower])*frac
+}
+
+func benchmarkCV(values []float64) float64 {
+	if len(values) == 0 {
+		return 0
+	}
+	mean := benchmarkMean(values)
+	if mean == 0 {
+		return 0
+	}
+	var variance float64
+	for _, value := range values {
+		diff := value - mean
+		variance += diff * diff
+	}
+	variance /= float64(len(values))
+	return math.Sqrt(variance) / mean * 100
+}
+
+func benchmarkClockDrift(values []float64) float64 {
+	if len(values) < 4 {
+		return 0
+	}
+	window := len(values) / 4
+	if window < 1 {
+		window = 1
+	}
+	head := benchmarkMean(values[:window])
+	tail := benchmarkMean(values[len(values)-window:])
+	if head <= 0 || tail >= head {
+		return 0
+	}
+	return ((head - tail) / head) * 100
+}
+
+func benchmarkMax(values []float64) float64 {
+	var max float64
+	for i, value := range values {
+		if i == 0 || value > max {
+			max = value
+		}
+	}
+	return max
+}
+
+func clampScore(value float64) float64 {
+	switch {
+	case value < 0:
+		return 0
+	case value > 100:
+		return 100
+	default:
+		return value
+	}
+}
+
+func dedupeStrings(values []string) []string {
+	if len(values) == 0 {
+		return nil
+	}
+	seen := make(map[string]struct{}, len(values))
+	out := make([]string, 0, len(values))
+	for _, value := range values {
+		value = strings.TrimSpace(value)
+		if value == "" {
+			continue
+		}
+		if _, ok := seen[value]; ok {
+			continue
+		}
+		seen[value] = struct{}{}
+		out = append(out, value)
+	}
+	return out
+}
+
+func saturatingSub(after, before uint64) uint64 {
+	if after <= before {
+		return 0
+	}
+	return after - before
+}
+
+func maxInt(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+// detectDCMIPartialCoverage returns true when IPMI DCMI under-reports actual
+// server power by comparing DCMI readings against SDR PSUx_POWER_IN sensor sums.
+//
+// Primary check: DCMI_idle / SDR_PSU_IN_idle — most reliable because GPU load
+// is zero, so both sources measure the same server state. A ratio below 0.7
+// means DCMI misses ≥30% of installed PSUs (e.g. 0.50 = sees 2 of 4 PSUs).
+//
+// Fallback: DCMI_loaded / SDR_PSU_IN_loaded — less precise (GPU load may
+// affect different PSUs differently) but still useful when idle SDR is absent.
+//
+// Returns false when SDR data is unavailable (server has no PSUx_POWER_IN
+// sensors); the heuristic detectIPMISaturationFallback is used in that case.

audit/internal/platform/benchmark_setup.go

@@ -0,0 +1,396 @@
+package platform
+
+import (
+	"context"
+	"encoding/csv"
+	"fmt"
+	"math"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func normalizeNvidiaBenchmarkOptionsForBenchmark(opts NvidiaBenchmarkOptions) NvidiaBenchmarkOptions {
+	switch strings.TrimSpace(strings.ToLower(opts.Profile)) {
+	case NvidiaBenchmarkProfileStability:
+		opts.Profile = NvidiaBenchmarkProfileStability
+	case NvidiaBenchmarkProfileOvernight:
+		opts.Profile = NvidiaBenchmarkProfileOvernight
+	default:
+		opts.Profile = NvidiaBenchmarkProfileStandard
+	}
+	if opts.SizeMB < 0 {
+		opts.SizeMB = 0
+	}
+	opts.ServerPowerSource = normalizeBenchmarkPowerSource(opts.ServerPowerSource)
+	opts.GPUIndices = dedupeSortedIndices(opts.GPUIndices)
+	opts.ExcludeGPUIndices = dedupeSortedIndices(opts.ExcludeGPUIndices)
+	return opts
+}
+
+func resolveBenchmarkProfile(profile string) benchmarkProfileSpec {
+	switch strings.TrimSpace(strings.ToLower(profile)) {
+	case NvidiaBenchmarkProfileStability:
+		return benchmarkProfileSpec{Name: NvidiaBenchmarkProfileStability, BaselineSec: 30, WarmupSec: 120, SteadySec: 3600, NCCLSec: 300, CooldownSec: 0}
+	case NvidiaBenchmarkProfileOvernight:
+		return benchmarkProfileSpec{Name: NvidiaBenchmarkProfileOvernight, BaselineSec: 60, WarmupSec: 180, SteadySec: 27000, NCCLSec: 600, CooldownSec: 0}
+	default:
+		return benchmarkProfileSpec{Name: NvidiaBenchmarkProfileStandard, BaselineSec: 15, WarmupSec: 45, SteadySec: 480, NCCLSec: 180, CooldownSec: 0}
+	}
+}
+
+// benchmarkGPUInfoQuery describes a nvidia-smi --query-gpu field set to try.
+// Fields are tried in order; the first successful query wins. Extended fields
+// (attribute.multiprocessor_count, power.default_limit) are not supported on
+// all driver versions, so we fall back to the base set if the full query fails.
+// The minimal fallback omits clock fields entirely — clocks.max.* returns
+// exit status 2 on some GPU generations (e.g. Blackwell); missing data is
+// then recovered from nvidia-smi -q.
+var benchmarkGPUInfoQueries = []struct {
+	fields   string
+	extended bool // whether this query includes optional extended fields
+	minimal  bool // clock fields omitted; max clocks must be filled separately
+}{
+	{
+		fields:   "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics,attribute.multiprocessor_count,power.default_limit",
+		extended: true,
+	},
+	{
+		fields:   "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics",
+		extended: false,
+	},
+	{
+		fields:  "index,uuid,name,pci.bus_id,vbios_version,power.limit",
+		minimal: true,
+	},
+}
+
+// enrichGPUInfoWithNvidiaSMIQ fills benchmark GPU metadata from nvidia-smi -q
+// for fields that may be missing from --query-gpu on some driver versions.
+func enrichGPUInfoWithNvidiaSMIQ(infoByIndex map[int]benchmarkGPUInfo, nvsmiQ []byte) {
+	if len(infoByIndex) == 0 || len(nvsmiQ) == 0 {
+		return
+	}
+
+	// Build bus_id → index map for matching verbose sections to GPU indices.
+	busToBenchIdx := make(map[string]int, len(infoByIndex))
+	for idx, info := range infoByIndex {
+		if info.BusID != "" {
+			// nvidia-smi -q uses "GPU 00000000:4E:00.0" (8-digit domain),
+			// while --query-gpu returns the same format; normalise to lower.
+			busToBenchIdx[strings.ToLower(strings.TrimSpace(info.BusID))] = idx
+		}
+	}
+
+	// Split the verbose output into per-GPU sections on "^GPU " lines.
+	gpuSectionRe := regexp.MustCompile(`(?m)^GPU\s+([\dA-Fa-f:\.]+)`)
+	maxGfxRe := regexp.MustCompile(`(?i)Max Clocks[\s\S]*?Graphics\s*:\s*(\d+)\s*MHz`)
+	maxMemRe := regexp.MustCompile(`(?i)Max Clocks[\s\S]*?Memory\s*:\s*(\d+)\s*MHz`)
+	defaultPwrRe := regexp.MustCompile(`(?i)Default Power Limit\s*:\s*([0-9.]+)\s*W`)
+	currentPwrRe := regexp.MustCompile(`(?i)Current Power Limit\s*:\s*([0-9.]+)\s*W`)
+	minPwrRe := regexp.MustCompile(`(?i)Min Power Limit\s*:\s*([0-9.]+)\s*W`)
+	maxPwrRe := regexp.MustCompile(`(?i)Max Power Limit\s*:\s*([0-9.]+)\s*W`)
+	smCountRe := regexp.MustCompile(`(?i)Multiprocessor Count\s*:\s*(\d+)`)
+	shutdownTempRe := regexp.MustCompile(`(?i)GPU Shutdown Temp\s*:\s*(\d+)\s*C`)
+	slowdownTempRe := regexp.MustCompile(`(?i)GPU Slowdown Temp\s*:\s*(\d+)\s*C`)
+
+	sectionStarts := gpuSectionRe.FindAllSubmatchIndex(nvsmiQ, -1)
+	for i, loc := range sectionStarts {
+		busID := strings.ToLower(string(nvsmiQ[loc[2]:loc[3]]))
+		benchIdx, ok := busToBenchIdx[busID]
+		if !ok {
+			// Bus IDs from verbose output may have a different domain prefix;
+			// try suffix match on the slot portion (XX:XX.X).
+			for k, v := range busToBenchIdx {
+				if strings.HasSuffix(k, busID) || strings.HasSuffix(busID, k) {
+					benchIdx = v
+					ok = true
+					break
+				}
+			}
+		}
+		if !ok {
+			continue
+		}
+
+		end := len(nvsmiQ)
+		if i+1 < len(sectionStarts) {
+			end = sectionStarts[i+1][0]
+		}
+		section := nvsmiQ[loc[0]:end]
+
+		info := infoByIndex[benchIdx]
+
+		if info.MaxGraphicsClockMHz == 0 {
+			if m := maxGfxRe.FindSubmatch(section); m != nil {
+				if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil {
+					info.MaxGraphicsClockMHz = v
+				}
+			}
+		}
+		if info.MaxMemoryClockMHz == 0 {
+			if m := maxMemRe.FindSubmatch(section); m != nil {
+				if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil {
+					info.MaxMemoryClockMHz = v
+				}
+			}
+		}
+		if info.DefaultPowerLimitW == 0 {
+			if m := defaultPwrRe.FindSubmatch(section); m != nil {
+				if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 {
+					info.DefaultPowerLimitW = v
+				}
+			}
+		}
+		if info.PowerLimitW == 0 {
+			if m := currentPwrRe.FindSubmatch(section); m != nil {
+				if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 {
+					info.PowerLimitW = v
+				}
+			}
+		}
+		if info.MinPowerLimitW == 0 {
+			if m := minPwrRe.FindSubmatch(section); m != nil {
+				if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 {
+					info.MinPowerLimitW = v
+				}
+			}
+		}
+		if info.MaxPowerLimitW == 0 {
+			if m := maxPwrRe.FindSubmatch(section); m != nil {
+				if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 {
+					info.MaxPowerLimitW = v
+				}
+			}
+		}
+		if info.MultiprocessorCount == 0 {
+			if m := smCountRe.FindSubmatch(section); m != nil {
+				if v, err := strconv.Atoi(string(m[1])); err == nil && v > 0 {
+					info.MultiprocessorCount = v
+				}
+			}
+		}
+		if info.ShutdownTempC == 0 {
+			if m := shutdownTempRe.FindSubmatch(section); m != nil {
+				if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 {
+					info.ShutdownTempC = v
+				}
+			}
+		}
+		if info.SlowdownTempC == 0 {
+			if m := slowdownTempRe.FindSubmatch(section); m != nil {
+				if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 {
+					info.SlowdownTempC = v
+				}
+			}
+		}
+		infoByIndex[benchIdx] = info
+	}
+}
+
+func queryBenchmarkGPUInfo(gpuIndices []int) (map[int]benchmarkGPUInfo, error) {
+	var lastErr error
+	for _, q := range benchmarkGPUInfoQueries {
+		args := []string{
+			"--query-gpu=" + q.fields,
+			"--format=csv,noheader,nounits",
+		}
+		if len(gpuIndices) > 0 {
+			args = append([]string{"--id=" + joinIndexList(gpuIndices)}, args...)
+		}
+		out, err := satExecCommand("nvidia-smi", args...).Output()
+		if err != nil {
+			lastErr = fmt.Errorf("nvidia-smi gpu info (%s): %w", q.fields[:min(len(q.fields), 40)], err)
+			continue
+		}
+
+		r := csv.NewReader(strings.NewReader(string(out)))
+		r.TrimLeadingSpace = true
+		r.FieldsPerRecord = -1
+		rows, err := r.ReadAll()
+		if err != nil {
+			lastErr = fmt.Errorf("parse nvidia-smi gpu info: %w", err)
+			continue
+		}
+
+		minFields := 6
+		if !q.minimal {
+			minFields = 9
+		}
+		infoByIndex := make(map[int]benchmarkGPUInfo, len(rows))
+		for _, row := range rows {
+			if len(row) < minFields {
+				continue
+			}
+			idx, err := strconv.Atoi(strings.TrimSpace(row[0]))
+			if err != nil {
+				continue
+			}
+			info := benchmarkGPUInfo{
+				Index:       idx,
+				UUID:        strings.TrimSpace(row[1]),
+				Name:        strings.TrimSpace(row[2]),
+				BusID:       strings.TrimSpace(row[3]),
+				VBIOS:       strings.TrimSpace(row[4]),
+				PowerLimitW: parseBenchmarkFloat(row[5]),
+			}
+			if !q.minimal {
+				info.MaxGraphicsClockMHz = parseBenchmarkFloat(row[6])
+				info.MaxMemoryClockMHz = parseBenchmarkFloat(row[7])
+				if len(row) >= 9 {
+					info.BaseGraphicsClockMHz = parseBenchmarkFloat(row[8])
+				}
+				if q.extended {
+					if len(row) >= 10 {
+						info.MultiprocessorCount = int(parseBenchmarkFloat(row[9]))
+					}
+					if len(row) >= 11 {
+						info.DefaultPowerLimitW = parseBenchmarkFloat(row[10])
+					}
+				}
+			}
+			infoByIndex[idx] = info
+		}
+		return infoByIndex, nil
+	}
+	return nil, lastErr
+}
+
+func applyBenchmarkNormalization(ctx context.Context, verboseLog string, gpuIndices []int, infoByIndex map[int]benchmarkGPUInfo, result *NvidiaBenchmarkResult) []benchmarkRestoreAction {
+	if os.Geteuid() != 0 {
+		result.Normalization.Status = "partial"
+		result.Normalization.Notes = append(result.Normalization.Notes, "benchmark normalization skipped: root privileges are required for persistence mode and clock locks")
+		for _, idx := range gpuIndices {
+			result.Normalization.GPUs = append(result.Normalization.GPUs, BenchmarkNormalizationGPU{
+				Index: idx,
+				Notes: []string{"normalization skipped: root privileges are required"},
+			})
+		}
+		return nil
+	}
+
+	var restore []benchmarkRestoreAction
+	for _, idx := range gpuIndices {
+		rec := BenchmarkNormalizationGPU{Index: idx}
+		if _, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("normalize-gpu-%d-pm", idx), []string{"nvidia-smi", "-i", strconv.Itoa(idx), "-pm", "1"}, nil, nil); err != nil {
+			rec.PersistenceMode = "failed"
+			rec.Notes = append(rec.Notes, "failed to enable persistence mode")
+			result.Normalization.Status = "partial"
+		} else {
+			rec.PersistenceMode = "applied"
+		}
+
+		if info, ok := infoByIndex[idx]; ok && info.MaxGraphicsClockMHz > 0 {
+			target := int(math.Round(info.MaxGraphicsClockMHz))
+			if out, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("normalize-gpu-%d-lgc", idx), []string{"nvidia-smi", "-i", strconv.Itoa(idx), "-lgc", strconv.Itoa(target)}, nil, nil); err != nil {
+				rec.GPUClockLockStatus = "failed"
+				rec.Notes = append(rec.Notes, "graphics clock lock failed: "+strings.TrimSpace(string(out)))
+				result.Normalization.Status = "partial"
+			} else {
+				rec.GPUClockLockStatus = "applied"
+				rec.GPUClockLockMHz = float64(target)
+				idxCopy := idx
+				restore = append(restore, benchmarkRestoreAction{name: fmt.Sprintf("gpu-%d-rgc", idxCopy), fn: func() {
+					_, _ = runSATCommandCtx(context.Background(), verboseLog, fmt.Sprintf("restore-gpu-%d-rgc", idxCopy), []string{"nvidia-smi", "-i", strconv.Itoa(idxCopy), "-rgc"}, nil, nil)
+				}})
+			}
+		} else {
+			rec.GPUClockLockStatus = "skipped"
+			rec.Notes = append(rec.Notes, "graphics clock lock skipped: gpu inventory unavailable or MaxGraphicsClockMHz=0")
+			result.Normalization.Status = "partial"
+		}
+
+		if info, ok := infoByIndex[idx]; ok && info.MaxMemoryClockMHz > 0 {
+			target := int(math.Round(info.MaxMemoryClockMHz))
+			out, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("normalize-gpu-%d-lmc", idx), []string{"nvidia-smi", "-i", strconv.Itoa(idx), "-lmc", strconv.Itoa(target)}, nil, nil)
+			switch {
+			case err == nil:
+				rec.MemoryClockLockStatus = "applied"
+				rec.MemoryClockLockMHz = float64(target)
+				idxCopy := idx
+				restore = append(restore, benchmarkRestoreAction{name: fmt.Sprintf("gpu-%d-rmc", idxCopy), fn: func() {
+					_, _ = runSATCommandCtx(context.Background(), verboseLog, fmt.Sprintf("restore-gpu-%d-rmc", idxCopy), []string{"nvidia-smi", "-i", strconv.Itoa(idxCopy), "-rmc"}, nil, nil)
+				}})
+			case strings.Contains(strings.ToLower(string(out)), "deferred") || strings.Contains(strings.ToLower(string(out)), "not supported"):
+				rec.MemoryClockLockStatus = "unsupported"
+				rec.Notes = append(rec.Notes, "memory clock lock unsupported on this GPU/driver path")
+				result.Normalization.Status = "partial"
+			default:
+				rec.MemoryClockLockStatus = "failed"
+				rec.Notes = append(rec.Notes, "memory clock lock failed: "+strings.TrimSpace(string(out)))
+				result.Normalization.Status = "partial"
+			}
+		}
+
+		result.Normalization.GPUs = append(result.Normalization.GPUs, rec)
+	}
+	return restore
+}
+
+func collectBenchmarkSamples(ctx context.Context, durationSec int, gpuIndices []int) ([]GPUMetricRow, error) {
+	if durationSec <= 0 {
+		return nil, nil
+	}
+	deadline := time.Now().Add(time.Duration(durationSec) * time.Second)
+	var rows []GPUMetricRow
+	start := time.Now()
+	for {
+		if ctx.Err() != nil {
+			return rows, ctx.Err()
+		}
+		samples, err := sampleBenchmarkTelemetry(gpuIndices)
+		if err == nil {
+			elapsed := time.Since(start).Seconds()
+			for i := range samples {
+				samples[i].ElapsedSec = elapsed
+			}
+			rows = append(rows, samples...)
+		}
+		if time.Now().After(deadline) {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			return rows, ctx.Err()
+		case <-time.After(time.Second):
+		}
+	}
+	return rows, nil
+}
+
+func runBenchmarkCommandWithMetrics(ctx context.Context, verboseLog, name string, cmd []string, env []string, gpuIndices []int, logFunc func(string)) ([]byte, []GPUMetricRow, error) {
+	stopCh := make(chan struct{})
+	doneCh := make(chan struct{})
+	var metricRows []GPUMetricRow
+	start := time.Now()
+
+	go func() {
+		defer close(doneCh)
+		ticker := time.NewTicker(time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-stopCh:
+				return
+			case <-ticker.C:
+				samples, err := sampleBenchmarkTelemetry(gpuIndices)
+				if err != nil {
+					continue
+				}
+				elapsed := time.Since(start).Seconds()
+				for i := range samples {
+					samples[i].ElapsedSec = elapsed
+				}
+				metricRows = append(metricRows, samples...)
+			}
+		}
+	}()
+
+	out, err := runSATCommandCtx(ctx, verboseLog, name, cmd, env, logFunc)
+	close(stopCh)
+	<-doneCh
+
+	return out, metricRows, err
+}

audit/internal/platform/nvidia_bandwidth_socket.go

@@ -11,12 +11,8 @@ import (
 // satReadFile is a seam for tests to fake sysfs reads (numa_node files).
 var satReadFile = os.ReadFile
 
-// gpuBandwidthSocketGroups splits gpuIndices into per-socket groups (ordered
-// by ascending NUMA node ID) for RunNvidiaBandwidthPack. A cross-socket
-// peer-to-peer path is a different (and, on platforms without NVLink, far
-// less exercised) fault domain than a same-socket one, so testing each
-// socket's GPUs in isolation before testing all of them together isolates
-// whether a failure is specific to the cross-socket path.
+// gpuBandwidthSocketGroups splits gpuIndices into NUMA-locality groups,
+// ordered by ascending Linux NUMA node ID, for RunNvidiaBandwidthPack.
 //
 // Falls back to a single group containing all of gpuIndices — i.e. no split
 // — whenever the NUMA node can't be resolved for every GPU, or all resolve
@@ -31,22 +27,17 @@ func gpuBandwidthSocketGroups(gpuIndices []int, logFunc func(string)) [][]int {
 	}
 
 	byNode := map[int][]int{}
-	var unresolved []int
 	for _, idx := range gpuIndices {
 		node, ok := nodes[idx]
 		if !ok {
 			if logFunc != nil {
-				logFunc(fmt.Sprintf("nvbandwidth: no NUMA node resolved for GPU %d; will fold it into a resolved socket group instead of dropping the split", idx))
+				logFunc(fmt.Sprintf("nvbandwidth: no NUMA node resolved for GPU %d; running all GPUs as one group", idx))
 			}
-			unresolved = append(unresolved, idx)
-			continue
+			return [][]int{gpuIndices}
 		}
 		byNode[node] = append(byNode[node], idx)
 	}
-	// Fewer than two resolved sockets means there's nothing to split either
-	// way: every GPU's node is unknown, or every resolved GPU shares one
-	// socket. A single unresolved GPU among an otherwise clean multi-socket
-	// system shouldn't cost us the split, so only bail out here.
+	// Fewer than two NUMA nodes means there is nothing meaningful to split.
 	if len(byNode) < 2 {
 		return [][]int{gpuIndices}
 	}
@@ -61,14 +52,6 @@ func gpuBandwidthSocketGroups(gpuIndices []int, logFunc func(string)) [][]int {
 	for _, node := range sortedNodes {
 		groups = append(groups, dedupeSortedIndices(byNode[node]))
 	}
-	if len(unresolved) > 0 {
-		// Fold into the last group rather than running unresolved GPUs in a
-		// group of their own — a lone GPU can't run a GPU-to-GPU bandwidth
-		// test by itself, and the point of the split is to isolate the
-		// sockets we *do* know about, not to also isolate the unknown one.
-		last := len(groups) - 1
-		groups[last] = dedupeSortedIndices(append(groups[last], unresolved...))
-	}
 	return groups
 }
 
@@ -107,10 +90,16 @@ func gpuNUMANodes(gpuIndices []int) (map[int]int, error) {
 	return nodes, nil
 }
 
-// normalizeNvidiaBDF converts nvidia-smi's 8-hex-digit-domain PCI bus ID
-// ("00000000:05:00.0") to the 4-hex-digit-domain form sysfs paths use
-// ("0000:05:00.0").
+// normalizeNvidiaBDF converts nvidia-smi's PCI bus ID to the exact form the
+// sysfs paths under /sys/bus/pci/devices use: an 8-hex-digit domain is
+// narrowed to 4 digits ("00000000:05:00.0" -> "0000:05:00.0"), and the hex
+// is lower-cased ("0000:CB:00.0" -> "0000:cb:00.0"). nvidia-smi upper-cases
+// the bus/device hex; sysfs directory names are always lower-case, so
+// without this a BDF containing a hex letter (e.g. GPUs on bus 4b/cb/cf)
+// would never match a sysfs entry and every numa_node / link-speed read
+// would silently fail.
 func normalizeNvidiaBDF(busID string) string {
+	busID = strings.ToLower(strings.TrimSpace(busID))
 	domain, rest, ok := strings.Cut(busID, ":")
 	if !ok {
 		return busID

audit/internal/platform/nvidia_bandwidth_socket_test.go

@@ -33,10 +33,10 @@ func fakeNUMANodes(t *testing.T, byBDF map[string]string) {
 }
 
 func TestGPUNUMANodesResolvesFromPCIBusID(t *testing.T) {
-	fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:F4:00.0\n")
+	fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:f4:00.0\n")
 	fakeNUMANodes(t, map[string]string{
 		"0000:05:00.0": "0\n",
-		"0000:F4:00.0": "1\n",
+		"0000:f4:00.0": "1\n",
 	})
 
 	nodes, err := gpuNUMANodes([]int{0, 1})
@@ -69,14 +69,14 @@ func TestGPUNUMANodesSkipsUnresolvableNode(t *testing.T) {
 }
 
 func TestGPUBandwidthSocketGroupsSplitsBySocket(t *testing.T) {
-	fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:F4:00.0\n5, 00000000:F5:00.0\n")
+	fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:f4:00.0\n5, 00000000:f5:00.0\n")
 	fakeNUMANodes(t, map[string]string{
 		"0000:05:00.0": "0\n",
 		"0000:06:00.0": "0\n",
 		"0000:76:00.0": "0\n",
 		"0000:77:00.0": "0\n",
-		"0000:F4:00.0": "1\n",
-		"0000:F5:00.0": "1\n",
+		"0000:f4:00.0": "1\n",
+		"0000:f5:00.0": "1\n",
 	})
 
 	groups := gpuBandwidthSocketGroups([]int{0, 1, 2, 3, 4, 5}, nil)
@@ -91,30 +91,20 @@ func TestGPUBandwidthSocketGroupsSplitsBySocket(t *testing.T) {
 	}
 }
 
-func TestGPUBandwidthSocketGroupsFoldsUnresolvedIntoLastGroup(t *testing.T) {
-	// GPU 4's NUMA node fails to resolve (e.g. a flaky sysfs read), but the
-	// other 5 GPUs still clearly span two sockets — the split should survive
-	// and GPU 4 should ride along with the last group rather than being
-	// tested alone or collapsing the whole thing to one pass.
-	fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:F4:00.0\n5, 00000000:F5:00.0\n")
+func TestGPUBandwidthSocketGroupsFallsBackWhenAnyNodeIsUnresolved(t *testing.T) {
+	fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:f4:00.0\n5, 00000000:f5:00.0\n")
 	fakeNUMANodes(t, map[string]string{
 		"0000:05:00.0": "0\n",
 		"0000:06:00.0": "0\n",
 		"0000:76:00.0": "0\n",
 		"0000:77:00.0": "0\n",
 		// GPU 4 (F4:00.0) deliberately missing.
-		"0000:F5:00.0": "1\n",
+		"0000:f5:00.0": "1\n",
 	})
 
 	groups := gpuBandwidthSocketGroups([]int{0, 1, 2, 3, 4, 5}, nil)
-	if len(groups) != 2 {
-		t.Fatalf("groups=%v want 2 groups", groups)
-	}
-	if joinIndexList(groups[0]) != "0,1,2,3" {
-		t.Fatalf("groups[0]=%v want 0,1,2,3", groups[0])
-	}
-	if joinIndexList(groups[1]) != "4,5" {
-		t.Fatalf("groups[1]=%v want 4,5 (unresolved GPU 4 folded into last group)", groups[1])
+	if len(groups) != 1 || joinIndexList(groups[0]) != "0,1,2,3,4,5" {
+		t.Fatalf("groups=%v want single fallback group", groups)
 	}
 }
 
@@ -161,6 +151,8 @@ func TestNormalizeNvidiaBDF(t *testing.T) {
 	cases := map[string]string{
 		"00000000:05:00.0": "0000:05:00.0",
 		"0000:05:00.0":     "0000:05:00.0",
+		"00000000:CB:00.0": "0000:cb:00.0",
+		"0000:4F:00.0":     "0000:4f:00.0",
 		"garbage":          "garbage",
 	}
 	for in, want := range cases {
@@ -171,17 +163,17 @@ func TestNormalizeNvidiaBDF(t *testing.T) {
 }
 
 func TestRunNvidiaBandwidthPackSplitsPerSocketThenAll(t *testing.T) {
-	fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:F4:00.0\n3, 00000000:F5:00.0\n")
+	fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:f4:00.0\n3, 00000000:f5:00.0\n")
 	fakeNUMANodes(t, map[string]string{
 		"0000:05:00.0": "0\n",
 		"0000:06:00.0": "0\n",
-		"0000:F4:00.0": "1\n",
-		"0000:F5:00.0": "1\n",
+		"0000:f4:00.0": "1\n",
+		"0000:f5:00.0": "1\n",
 	})
 
 	dir := t.TempDir()
 	s := &System{}
-	_, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1, 2, 3}, nil)
+	_, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1, 2, 3}, true, nil)
 	if err != nil {
 		t.Fatalf("RunNvidiaBandwidthPack error: %v", err)
 	}
@@ -211,6 +203,34 @@ func TestRunNvidiaBandwidthPackSplitsPerSocketThenAll(t *testing.T) {
 	}
 }
 
+func TestRunNvidiaBandwidthPackValidateNeverSplits(t *testing.T) {
+	// Multi-socket system, but fullMatrix=false (Validate tier): still one
+	// nvbandwidth pass across every GPU, no per-socket split.
+	fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:f4:00.0\n3, 00000000:f5:00.0\n")
+	fakeNUMANodes(t, map[string]string{
+		"0000:05:00.0": "0\n",
+		"0000:06:00.0": "0\n",
+		"0000:f4:00.0": "1\n",
+		"0000:f5:00.0": "1\n",
+	})
+
+	dir := t.TempDir()
+	if _, err := (&System{}).RunNvidiaBandwidthPack(nil, dir, []int{0, 1, 2, 3}, false, nil); err != nil {
+		t.Fatalf("RunNvidiaBandwidthPack error: %v", err)
+	}
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatalf("ReadDir: %v", err)
+	}
+	runDir := filepath.Join(dir, entries[0].Name())
+	if _, err := os.Stat(filepath.Join(runDir, "03-dcgmi-nvbandwidth.log")); err != nil {
+		t.Fatalf("missing single-pass job output: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(runDir, "03-dcgmi-nvbandwidth-socket0.log")); err == nil {
+		t.Fatalf("Validate tier must not split per socket")
+	}
+}
+
 func TestRunNvidiaBandwidthPackSinglePassWhenOneSocket(t *testing.T) {
 	fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n")
 	fakeNUMANodes(t, map[string]string{
@@ -220,7 +240,7 @@ func TestRunNvidiaBandwidthPackSinglePassWhenOneSocket(t *testing.T) {
 
 	dir := t.TempDir()
 	s := &System{}
-	_, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1}, nil)
+	_, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1}, true, nil)
 	if err != nil {
 		t.Fatalf("RunNvidiaBandwidthPack error: %v", err)
 	}

audit/internal/platform/physical_gpu_vendors_test.go

@@ -0,0 +1,53 @@
+package platform
+
+import (
+	"os/exec"
+	"testing"
+)
+
+func TestPhysicalGPUVendors(t *testing.T) {
+	tests := []struct {
+		name       string
+		lspci      string
+		wantNvidia bool
+		wantAMD    bool
+	}{
+		{
+			name: "nvidia 3D controllers",
+			lspci: "4b:00.0 3D controller [0302]: NVIDIA Corporation GH100 [H200 NVL] [10de:233b] (rev a1)\n" +
+				"02:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED [1a03:2000]\n",
+			wantNvidia: true,
+		},
+		{
+			name:    "amd VGA by vendor id",
+			lspci:   "63:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Navi [1002:744c]\n",
+			wantAMD: true,
+		},
+		{
+			name: "amd cpu root complex is not a GPU",
+			lspci: "00:00.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14a4]\n" +
+				"00:01.0 IOMMU [0806]: Advanced Micro Devices, Inc. [AMD] Device [1022:14a1]\n",
+		},
+		{
+			name:  "no GPU",
+			lspci: "02:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED [1a03:2000]\n",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			old := satExecCommand
+			satExecCommand = func(name string, args ...string) *exec.Cmd {
+				if name == "lspci" {
+					return exec.Command("printf", "%s", tt.lspci)
+				}
+				return exec.Command(name, args...)
+			}
+			t.Cleanup(func() { satExecCommand = old })
+
+			nvidia, amd := (&System{}).PhysicalGPUVendors()
+			if nvidia != tt.wantNvidia || amd != tt.wantAMD {
+				t.Fatalf("got nvidia=%v amd=%v, want nvidia=%v amd=%v", nvidia, amd, tt.wantNvidia, tt.wantAMD)
+			}
+		})
+	}
+}

audit/internal/platform/platform_stress.go

@@ -1,9 +1,7 @@
 package platform
 
 import (
-	"archive/tar"
 	"bytes"
-	"compress/gzip"
 	"context"
 	"encoding/csv"
 	"fmt"
@@ -532,44 +530,3 @@ func containsComponent(components []string, name string) bool {
 	}
 	return false
 }
-
-func packPlatformDir(dir, dest string) error {
-	f, err := os.Create(dest)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	gz := gzip.NewWriter(f)
-	defer gz.Close()
-	tw := tar.NewWriter(gz)
-	defer tw.Close()
-
-	entries, err := os.ReadDir(dir)
-	if err != nil {
-		return err
-	}
-	base := filepath.Base(dir)
-	for _, e := range entries {
-		if e.IsDir() {
-			continue
-		}
-		fpath := filepath.Join(dir, e.Name())
-		data, err := os.ReadFile(fpath)
-		if err != nil {
-			continue
-		}
-		hdr := &tar.Header{
-			Name:    filepath.Join(base, e.Name()),
-			Size:    int64(len(data)),
-			Mode:    0644,
-			ModTime: time.Now(),
-		}
-		if err := tw.WriteHeader(hdr); err != nil {
-			return err
-		}
-		if _, err := tw.Write(data); err != nil {
-			return err
-		}
-	}
-	return nil
-}

audit/internal/platform/sat.go

@@ -1,10 +1,8 @@
 package platform
 
 import (
-	"archive/tar"
 	"bufio"
 	"bytes"
-	"compress/gzip"
 	"context"
 	"errors"
 	"fmt"
@@ -194,774 +192,6 @@ func streamExecOutput(cmd *exec.Cmd, logFunc func(string), livePath string) ([]b
 }
 
 // NvidiaGPU holds basic GPU info from nvidia-smi.
-type NvidiaGPU struct {
-	Index    int    `json:"index"`
-	Name     string `json:"name"`
-	MemoryMB int    `json:"memory_mb"`
-}
-
-type NvidiaGPUStatus struct {
-	Index        int    `json:"index"`
-	Name         string `json:"name"`
-	BDF          string `json:"bdf,omitempty"`
-	Serial       string `json:"serial,omitempty"`
-	Status       string `json:"status"`
-	RawLine      string `json:"raw_line,omitempty"`
-	NeedsReset   bool   `json:"needs_reset"`
-	ParseFailure bool   `json:"parse_failure,omitempty"`
-}
-
-type nvidiaGPUHealth struct {
-	Index        int
-	Name         string
-	NeedsReset   bool
-	RawLine      string
-	ParseFailure bool
-}
-
-type nvidiaGPUStatusFile struct {
-	Index      int
-	Name       string
-	RunStatus  string
-	Reason     string
-	Health     string
-	HealthRaw  string
-	Observed   bool
-	Selected   bool
-	FailingJob string
-}
-
-// AMDGPUInfo holds basic info about an AMD GPU from rocm-smi.
-type AMDGPUInfo struct {
-	Index int    `json:"index"`
-	Name  string `json:"name"`
-}
-
-// DetectGPUVendor returns "nvidia" if /dev/nvidia0 exists, "amd" if /dev/kfd exists, or "" otherwise.
-func (s *System) DetectGPUVendor() string {
-	if _, err := os.Stat("/dev/nvidia0"); err == nil {
-		return "nvidia"
-	}
-	if _, err := os.Stat("/dev/kfd"); err == nil {
-		return "amd"
-	}
-	if raw, err := exec.Command("lspci", "-nn").Output(); err == nil {
-		// Only match AMD GPU device classes [0300]=VGA, [0302]=3D controller, [0380]=Display.
-		// AMD CPUs also appear in lspci as "Advanced Micro Devices" (Root Complex, IOMMU, etc.)
-		// so matching vendor alone causes false positives on AMD CPU servers without GPUs.
-		for _, line := range strings.Split(strings.ToLower(string(raw)), "\n") {
-			if !strings.Contains(line, "advanced micro devices") && !strings.Contains(line, "amd/ati") {
-				continue
-			}
-			if strings.Contains(line, "[0300]") || strings.Contains(line, "[0302]") || strings.Contains(line, "[0380]") {
-				return "amd"
-			}
-		}
-	}
-	return ""
-}
-
-// ListAMDGPUs returns AMD GPUs visible to rocm-smi.
-func (s *System) ListAMDGPUs() ([]AMDGPUInfo, error) {
-	out, err := runROCmSMI("--showproductname", "--csv")
-	if err != nil {
-		return nil, fmt.Errorf("rocm-smi: %w", err)
-	}
-	var gpus []AMDGPUInfo
-	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" || strings.HasPrefix(strings.ToLower(line), "device") {
-			continue
-		}
-		parts := strings.SplitN(line, ",", 2)
-		name := ""
-		if len(parts) >= 2 {
-			name = strings.TrimSpace(parts[1])
-		}
-		idx := len(gpus)
-		gpus = append(gpus, AMDGPUInfo{Index: idx, Name: name})
-	}
-	return gpus, nil
-}
-
-// RunAMDAcceptancePack runs an AMD GPU diagnostic pack using rocm-smi.
-func (s *System) RunAMDAcceptancePack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-amd", []satJob{
-		{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
-		{name: "02-rocm-smi-showallinfo.log", cmd: []string{"rocm-smi", "--showallinfo"}},
-		{name: "03-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
-		{name: "04-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
-	}, logFunc)
-}
-
-// RunAMDMemIntegrityPack runs the official RVS MEM module as a validate-style memory integrity test.
-func (s *System) RunAMDMemIntegrityPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
-	if err := ensureAMDRuntimeReady(); err != nil {
-		return "", err
-	}
-	cfgFile := "/tmp/bee-amd-mem.conf"
-	cfg := `actions:
-- name: mem_integrity
-  device: all
-  module: mem
-  parallel: true
-  duration: 60000
-  copy_matrix: false
-  target_stress: 90
-  matrix_size: 8640
-`
-	_ = os.WriteFile(cfgFile, []byte(cfg), 0644)
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-mem", []satJob{
-		{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
-		{name: "02-rvs-mem.log", cmd: []string{"rvs", "-c", cfgFile}},
-		{name: "03-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}},
-	}, logFunc)
-}
-
-// RunAMDMemBandwidthPack runs AMD's memory/interconnect bandwidth-oriented tools.
-func (s *System) RunAMDMemBandwidthPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
-	if err := ensureAMDRuntimeReady(); err != nil {
-		return "", err
-	}
-	cfgFile := "/tmp/bee-amd-babel.conf"
-	cfg := `actions:
-- name: babel_mem_bw
-  device: all
-  module: babel
-  parallel: true
-  copy_matrix: true
-  target_stress: 90
-  matrix_size: 134217728
-`
-	_ = os.WriteFile(cfgFile, []byte(cfg), 0644)
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-bandwidth", []satJob{
-		{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
-		{name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}},
-		{name: "03-rvs-babel.log", cmd: []string{"rvs", "-c", cfgFile}},
-		{name: "04-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}},
-	}, logFunc)
-}
-
-// RunAMDStressPack runs an AMD GPU burn-in pack.
-// Missing tools are reported as UNSUPPORTED, consistent with the existing SAT pattern.
-func (s *System) RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
-	seconds := durationSec
-	if seconds <= 0 {
-		seconds = envInt("BEE_AMD_STRESS_SECONDS", 300)
-	}
-	if err := ensureAMDRuntimeReady(); err != nil {
-		return "", err
-	}
-	// Enable copy_matrix so the same GST run drives VRAM traffic in addition to compute.
-	rvsCfg := amdStressRVSConfig(seconds)
-	cfgFile := "/tmp/bee-amd-gst.conf"
-	_ = os.WriteFile(cfgFile, []byte(rvsCfg), 0644)
-
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-stress", amdStressJobs(seconds, cfgFile), logFunc)
-}
-
-func amdStressRVSConfig(seconds int) string {
-	return fmt.Sprintf(`actions:
-- name: gst_stress
-  device: all
-  module: gst
-  parallel: true
-  duration: %d
-  copy_matrix: false
-  target_stress: 90
-  matrix_size_a: 8640
-  matrix_size_b: 8640
-  matrix_size_c: 8640
-`, seconds*1000)
-}
-
-func amdStressJobs(seconds int, cfgFile string) []satJob {
-	return []satJob{
-		{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
-		{name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}},
-		{name: fmt.Sprintf("03-rvs-gst-%ds.log", seconds), cmd: []string{"rvs", "-c", cfgFile}},
-		{name: fmt.Sprintf("04-rocm-smi-after.log"), cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--csv"}},
-	}
-}
-
-// ListNvidiaGPUs returns GPUs visible to nvidia-smi.
-func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) {
-	out, err := exec.Command("nvidia-smi",
-		"--query-gpu=index,name,memory.total",
-		"--format=csv,noheader,nounits").Output()
-	if err != nil {
-		return nil, fmt.Errorf("nvidia-smi: %w", err)
-	}
-	var gpus []NvidiaGPU
-	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" {
-			continue
-		}
-		parts := strings.SplitN(line, ", ", 3)
-		if len(parts) != 3 {
-			continue
-		}
-		idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
-		if err != nil {
-			continue
-		}
-		memMB, _ := strconv.Atoi(strings.TrimSpace(parts[2]))
-		gpus = append(gpus, NvidiaGPU{
-			Index:    idx,
-			Name:     strings.TrimSpace(parts[1]),
-			MemoryMB: memMB,
-		})
-	}
-	sort.Slice(gpus, func(i, j int) bool {
-		return gpus[i].Index < gpus[j].Index
-	})
-	return gpus, nil
-}
-
-func (s *System) ListNvidiaGPUStatuses() ([]NvidiaGPUStatus, error) {
-	out, err := satExecCommand(
-		"nvidia-smi",
-		"--query-gpu=index,name,pci.bus_id,serial,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total",
-		"--format=csv,noheader,nounits",
-	).Output()
-	if err != nil {
-		return nil, fmt.Errorf("nvidia-smi: %w", err)
-	}
-	var gpus []NvidiaGPUStatus
-	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" {
-			continue
-		}
-		parts := strings.Split(line, ",")
-		if len(parts) < 4 {
-			gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true})
-			continue
-		}
-		idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
-		if err != nil {
-			gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true})
-			continue
-		}
-		upper := strings.ToUpper(line)
-		needsReset := strings.Contains(upper, "GPU REQUIRES RESET")
-		status := "OK"
-		if needsReset {
-			status = "RESET_REQUIRED"
-		}
-		gpus = append(gpus, NvidiaGPUStatus{
-			Index:      idx,
-			Name:       strings.TrimSpace(parts[1]),
-			BDF:        normalizeNvidiaBusID(strings.TrimSpace(parts[2])),
-			Serial:     strings.TrimSpace(parts[3]),
-			Status:     status,
-			RawLine:    line,
-			NeedsReset: needsReset,
-		})
-	}
-	sort.Slice(gpus, func(i, j int) bool { return gpus[i].Index < gpus[j].Index })
-	return gpus, nil
-}
-
-func normalizeNvidiaBusID(v string) string {
-	v = strings.TrimSpace(strings.ToLower(v))
-	parts := strings.Split(v, ":")
-	if len(parts) == 3 && len(parts[0]) > 4 {
-		parts[0] = parts[0][len(parts[0])-4:]
-		return strings.Join(parts, ":")
-	}
-	return v
-}
-
-func (s *System) ResetNvidiaGPU(index int) (string, error) {
-	return resetNvidiaGPU(index)
-}
-
-// RunNCCLTests runs nccl-tests all_reduce_perf across the selected NVIDIA GPUs.
-// Measures collective communication bandwidth over NVLink/PCIe.
-func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	gpuCount := len(selected)
-	if gpuCount < 1 {
-		gpuCount = 1
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", withNvidiaPersistenceMode(
-		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-all-reduce-perf.log", cmd: []string{
-			"all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2",
-			"-g", strconv.Itoa(gpuCount), "--iters", "20",
-		}, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true},
-	), logFunc)
-}
-
-func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	var (
-		profCmd []string
-		profEnv []string
-	)
-	if len(selected) > 1 {
-		// For multiple GPUs, always spawn one dcgmproftester process per GPU via
-		// bee-dcgmproftester-staggered (stagger=0 means all start simultaneously).
-		// A single dcgmproftester process without -i only loads GPU 0 regardless
-		// of CUDA_VISIBLE_DEVICES.
-		stagger := staggerSec
-		if stagger < 0 {
-			stagger = 0
-		}
-		profCmd = []string{
-			"bee-dcgmproftester-staggered",
-			"--seconds", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec)),
-			"--stagger-seconds", strconv.Itoa(stagger),
-			"--devices", joinIndexList(selected),
-		}
-	} else {
-		profCmd, err = resolveDCGMProfTesterCommand("--no-dcgm-validation", "-t", "1004", "-d", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec)))
-		if err != nil {
-			return "", err
-		}
-		profEnv = nvidiaVisibleDevicesEnv(selected)
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-compute", withNvidiaPersistenceMode(
-		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-version.log", cmd: []string{"dcgmi", "-v"}},
-		satJob{
-			name:        "03-dcgmproftester.log",
-			cmd:         profCmd,
-			env:         profEnv,
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		},
-		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
-	), logFunc)
-}
-
-func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
-	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
-	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
-		for _, p := range killed {
-			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
-		}
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode(
-		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
-		satJob{
-			name:        "03-dcgmi-targeted-power.log",
-			cmd:         nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		},
-		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
-	), logFunc)
-}
-
-func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
-	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
-	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
-		for _, p := range killed {
-			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
-		}
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode(
-		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
-		satJob{
-			name:        "03-dcgmi-pulse-test.log",
-			cmd:         nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		},
-		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
-	), logFunc)
-}
-
-func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
-	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
-	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
-		for _, p := range killed {
-			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
-		}
-	}
-	jobs := []satJob{
-		{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
-	}
-
-	// On a system with GPUs on more than one CPU socket, run each socket's
-	// GPUs through nvbandwidth in isolation before the all-GPU pass. Without
-	// NVLink, cross-socket peer-to-peer traffic is a distinct fault domain
-	// from same-socket traffic; if the single-socket passes log clean and
-	// only the all-GPU pass doesn't complete, that isolates the cross-socket
-	// path as the trigger instead of leaving it conflated with a general
-	// GPU/PCIe fault. Systems with one socket (or no resolvable NUMA
-	// affinity) get a single group back and keep the original one-pass shape.
-	step := 3
-	socketGroups := gpuBandwidthSocketGroups(selected, logFunc)
-	if len(socketGroups) <= 1 {
-		jobs = append(jobs, satJob{
-			name:        fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step),
-			cmd:         nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		})
-		step++
-	} else {
-		for i, group := range socketGroups {
-			jobs = append(jobs, satJob{
-				name:        fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i),
-				cmd:         nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group),
-				collectGPU:  true,
-				gpuIndices:  group,
-				syncBracket: true,
-			})
-			step++
-		}
-		jobs = append(jobs, satJob{
-			name:        fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step),
-			cmd:         nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		})
-		step++
-	}
-	jobs = append(jobs, satJob{
-		name: fmt.Sprintf("%02d-nvidia-smi-after.log", step),
-		cmd:  []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"},
-	})
-
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(jobs...), logFunc)
-}
-
-func (s *System) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
-	return runAcceptancePackCtx(context.Background(), baseDir, "gpu-nvidia", nvidiaSATJobs(), logFunc)
-}
-
-// RunNvidiaAcceptancePackWithOptions runs the NVIDIA diagnostics via DCGM.
-// diagLevel: 1=quick, 2=medium, 3=targeted stress, 4=extended stress.
-// gpuIndices: specific GPU indices to test (empty = all GPUs).
-// ctx cancellation kills the running job.
-func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int, logFunc func(string)) (string, error) {
-	resolvedGPUIndices, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia", nvidiaDCGMJobs(diagLevel, resolvedGPUIndices), logFunc)
-}
-
-func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
-	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
-	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
-		for _, p := range killed {
-			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
-		}
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode(
-		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
-		satJob{
-			name:        "03-dcgmi-targeted-stress.log",
-			cmd:         nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		},
-		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
-	), logFunc)
-}
-
-func resolveDCGMGPUIndices(gpuIndices []int) ([]int, error) {
-	if len(gpuIndices) > 0 {
-		return dedupeSortedIndices(gpuIndices), nil
-	}
-	all, err := listNvidiaGPUIndices()
-	if err != nil {
-		return nil, err
-	}
-	if len(all) == 0 {
-		return nil, fmt.Errorf("nvidia-smi found no NVIDIA GPUs")
-	}
-	return all, nil
-}
-
-func memoryStressSizeArg() string {
-	if mb := envInt("BEE_VM_STRESS_SIZE_MB", 0); mb > 0 {
-		return fmt.Sprintf("%dM", mb)
-	}
-	availBytes := satFreeMemBytes()
-	if availBytes <= 0 {
-		return "80%"
-	}
-	availMB := availBytes / (1024 * 1024)
-	targetMB := (availMB * 2) / 3
-	if targetMB >= 256 {
-		targetMB = (targetMB / 256) * 256
-	}
-	if targetMB <= 0 {
-		return "80%"
-	}
-	return fmt.Sprintf("%dM", targetMB)
-}
-
-func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error) {
-	if sizeMB <= 0 {
-		sizeMB = 256
-	}
-	if passes <= 0 {
-		passes = 1
-	}
-	// Keep Validate Memory bounded to a quick diagnostic window. The timeout is
-	// intentionally conservative enough for healthy systems while avoiding the
-	// prior 30-80 minute hangs caused by memtester spinning on a bad subtest.
-	timeoutSec := sizeMB*passes*20/100 + 60
-	if timeoutSec < 180 {
-		timeoutSec = 180
-	}
-	if timeoutSec > 900 {
-		timeoutSec = 900
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{
-		{name: "01-free-before.log", cmd: []string{"free", "-h"}},
-		{name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}, syncBracket: true},
-		{name: "03-free-after.log", cmd: []string{"free", "-h"}},
-	}, logFunc)
-}
-
-func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
-	seconds := durationSec
-	if seconds <= 0 {
-		seconds = envInt("BEE_VM_STRESS_SECONDS", 300)
-	}
-	// Base the default on current MemAvailable and keep headroom for the OS and
-	// concurrent stressors so mixed burn runs do not trip the OOM killer.
-	sizeArg := memoryStressSizeArg()
-	return runAcceptancePackCtx(ctx, baseDir, "memory-stress", []satJob{
-		{name: "01-free-before.log", cmd: []string{"free", "-h"}},
-		{name: "02-stress-ng-vm.log", cmd: []string{
-			"stress-ng", "--vm", "1",
-			"--vm-bytes", sizeArg,
-			"--vm-method", "all",
-			"--timeout", fmt.Sprintf("%d", seconds),
-			"--metrics-brief",
-		}, syncBracket: true},
-		{name: "03-free-after.log", cmd: []string{"free", "-h"}},
-	}, logFunc)
-}
-
-func (s *System) RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
-	seconds := durationSec
-	if seconds <= 0 {
-		seconds = envInt("BEE_SAT_STRESS_SECONDS", 300)
-	}
-	cmd := []string{"stressapptest", "-s", fmt.Sprintf("%d", seconds), "-W", "--cc_test"}
-	if mb := envInt("BEE_SAT_STRESS_MB", 0); mb > 0 {
-		cmd = append(cmd, "-M", fmt.Sprintf("%d", mb))
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "sat-stress", []satJob{
-		{name: "01-free-before.log", cmd: []string{"free", "-h"}},
-		{name: "02-stressapptest.log", cmd: cmd},
-		{name: "03-free-after.log", cmd: []string{"free", "-h"}},
-	}, logFunc)
-}
-
-// cpuThermalThrottleSysDir is the sysfs root the throttle-check scripts glob
-// under. Overridden in tests so they can point at a fake directory tree
-// instead of the real /sys.
-var cpuThermalThrottleSysDir = "/sys/devices/system/cpu"
-
-// cpuThrottleSumScript is the shell fragment both before/after scripts use to
-// sum the kernel's cumulative-since-boot thermal throttle counters across
-// every CPU.
-func cpuThrottleSumScript() string {
-	return fmt.Sprintf(`
-sum=0
-for f in %[1]s/cpu*/thermal_throttle/core_throttle_count %[1]s/cpu*/thermal_throttle/package_throttle_count; do
-  [ -f "$f" ] || continue
-  v=$(cat "$f" 2>/dev/null)
-  case "$v" in ''|*[!0-9]*) continue ;; esac
-  sum=$((sum + v))
-done
-`, cpuThermalThrottleSysDir)
-}
-
-// cpuThrottleBeforeScript snapshots the throttle counter sum into a file in
-// {{run_dir}} so cpuThrottleCheckScript can later diff before/after despite
-// each satJob running as an independent process.
-func cpuThrottleBeforeScript() string {
-	return cpuThrottleSumScript() + `echo "$sum" | tee {{run_dir}}/.cpu-throttle-before` + "\n"
-}
-
-// cpuThrottleCheckScript compares the after-run throttle counter sum against
-// the snapshot cpuThrottleBeforeScript took, and fails (non-zero exit) if it
-// increased — i.e. the CPU actually hit thermal throttling during this
-// specific run, not just at some earlier point this boot. classifySATResult
-// maps a failed job here to SAT status FAILED, which ApplySATResultToDB
-// records as component status "Warning" for cpu:all — without this, the
-// "cpu" SAT pack only checks stress-ng's exit code, which is 0 whether or
-// not the CPU throttled while running it.
-func cpuThrottleCheckScript() string {
-	return `before=$(cat {{run_dir}}/.cpu-throttle-before 2>/dev/null)
-case "$before" in ''|*[!0-9]*) before=0 ;; esac
-` + cpuThrottleSumScript() + `after=$sum
-echo "throttle_count_before=$before"
-echo "throttle_count_after=$after"
-if [ "$after" -gt "$before" ]; then
-  echo "THROTTLE DETECTED: CPU package/core hit thermal throttling during this stress-ng run ($before -> $after)"
-  exit 1
-fi
-echo "no new thermal throttling detected during this run"
-`
-}
-
-func cpuSATJobs(durationSec int) []satJob {
-	return []satJob{
-		{name: "01-lscpu.log", cmd: []string{"lscpu"}},
-		{name: "02-sensors-before.log", cmd: []string{"sensors"}},
-		{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
-		{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
-		{name: "04-sensors-after.log", cmd: []string{"sensors"}},
-		{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
-	}
-}
-
-func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
-	if durationSec <= 0 {
-		durationSec = 60
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "cpu", cpuSATJobs(durationSec), logFunc)
-}
-
-func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) {
-	if baseDir == "" {
-		baseDir = "/var/log/bee-sat"
-	}
-	ts := time.Now().UTC().Format("20060102-150405")
-	runDir := filepath.Join(baseDir, "storage-"+ts)
-	if err := os.MkdirAll(runDir, 0755); err != nil {
-		return "", err
-	}
-	verboseLog := filepath.Join(runDir, "verbose.log")
-
-	devices, err := listStorageDevices()
-	if err != nil {
-		return "", err
-	}
-	sort.Strings(devices)
-
-	var summary strings.Builder
-	stats := satStats{}
-	fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
-	if len(devices) == 0 {
-		fmt.Fprintln(&summary, "devices=0")
-		stats.Unsupported++
-	} else {
-		fmt.Fprintf(&summary, "devices=%d\n", len(devices))
-	}
-
-	for index, devPath := range devices {
-		if ctx.Err() != nil {
-			break
-		}
-		prefix := fmt.Sprintf("%02d-%s", index+1, filepath.Base(devPath))
-		commands := storageSATCommands(devPath, extended)
-		deviceOutputs := make(map[string][]byte, len(commands))
-		for cmdIndex, job := range commands {
-			if ctx.Err() != nil {
-				break
-			}
-			name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
-			livePath := filepath.Join(runDir, name)
-			runSyncBracketHook(job, "before", logFunc)
-			out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc, livePath)
-			deviceOutputs[job.name] = out
-			if writeErr := os.WriteFile(livePath, out, 0644); writeErr != nil {
-				return "", writeErr
-			}
-			if satJobBoundaryHook != nil {
-				satJobBoundaryHook(name)
-			}
-			// smartctl -t short only launches the self-test on the drive firmware and
-			// returns immediately ("Testing has begun"); unlike `nvme device-self-test
-			// --wait`, smartctl has no blocking mode, so we must poll the drive
-			// ourselves until the self-test actually finishes. Hold the "after" sync
-			// until that poll completes — the self-test itself, not just its launch,
-			// is the load worth having durable evidence of.
-			deferSyncBracketAfter := job.name == "smartctl-self-test-short" && err == nil
-			if !deferSyncBracketAfter {
-				runSyncBracketHook(job, "after", logFunc)
-			}
-			status, rc := classifySATResult(job.name, out, err)
-			// A zero smartctl exit status only proves the command ran.  If the
-			// drive did not return its overall-health verdict, it must not turn
-			// the storage SAT green.
-			if job.name == "smartctl-health" && status == "OK" && !hasSMARTOverallHealth(out) {
-				status = "UNSUPPORTED"
-			}
-			stats.Add(status)
-			key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_")
-			fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
-			fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
-
-			if deferSyncBracketAfter {
-				statusName := "smartctl-self-test-status"
-				statusOut := waitForSmartctlSelfTest(ctx, verboseLog, devPath, logFunc)
-				deviceOutputs[statusName] = statusOut
-				statusFile := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+2, statusName)
-				if writeErr := os.WriteFile(filepath.Join(runDir, statusFile), statusOut, 0644); writeErr != nil {
-					return "", writeErr
-				}
-				runSyncBracketHook(job, "after", logFunc)
-				sStatus, sRC := classifySATResult(statusName, statusOut, nil)
-				stats.Add(sStatus)
-				sKey := filepath.Base(devPath) + "_" + strings.ReplaceAll(statusName, "-", "_")
-				fmt.Fprintf(&summary, "%s_rc=%d\n", sKey, sRC)
-				fmt.Fprintf(&summary, "%s_status=%s\n", sKey, sStatus)
-			}
-		}
-		reportText := GenerateDiskReportText(index+1, devPath, deviceOutputs, time.Now().UTC())
-		reportName := "disk-" + prefix + "-report.txt"
-		_ = os.WriteFile(filepath.Join(runDir, reportName), []byte(reportText), 0644)
-	}
-
-	writeSATStats(&summary, stats)
-	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
-		return "", err
-	}
-	return runDir, nil
-}
-
 type satJob struct {
 	name       string
 	cmd        []string
@@ -1574,41 +804,6 @@ func hasSMARTOverallHealth(out []byte) bool {
 	return len(m) > 1 && strings.TrimSpace(m[1]) != ""
 }
 
-func runSATCommand(verboseLog, name string, cmd []string, logFunc func(string)) ([]byte, error) {
-	start := time.Now().UTC()
-	resolvedCmd, err := resolveSATCommand(cmd)
-	appendSATVerboseLog(verboseLog,
-		fmt.Sprintf("[%s] start %s", start.Format(time.RFC3339), name),
-		"cmd: "+strings.Join(resolvedCmd, " "),
-	)
-	if logFunc != nil {
-		logFunc(fmt.Sprintf("=== %s ===", name))
-	}
-	if err != nil {
-		appendSATVerboseLog(verboseLog,
-			fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
-			"rc: 1",
-			fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()),
-			"",
-		)
-		return []byte(err.Error() + "\n"), err
-	}
-
-	out, err := streamExecOutput(satExecCommand(resolvedCmd[0], resolvedCmd[1:]...), logFunc, "")
-
-	rc := 0
-	if err != nil {
-		rc = 1
-	}
-	appendSATVerboseLog(verboseLog,
-		fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
-		fmt.Sprintf("rc: %d", rc),
-		fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()),
-		"",
-	)
-	return out, err
-}
-
 func runROCmSMI(args ...string) ([]byte, error) {
 	cmd, err := resolveROCmSMICommand(args...)
 	if err != nil {
@@ -1802,46 +997,3 @@ func envInt(name string, fallback int) int {
 	}
 	return value
 }
-
-func createTarGz(dst, srcDir string) error {
-	file, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	gz := gzip.NewWriter(file)
-	defer gz.Close()
-
-	tw := tar.NewWriter(gz)
-	defer tw.Close()
-
-	base := filepath.Dir(srcDir)
-	return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if info.IsDir() {
-			return nil
-		}
-		header, err := tar.FileInfoHeader(info, "")
-		if err != nil {
-			return err
-		}
-		rel, err := filepath.Rel(base, path)
-		if err != nil {
-			return err
-		}
-		header.Name = rel
-		if err := tw.WriteHeader(header); err != nil {
-			return err
-		}
-		file, err := os.Open(path)
-		if err != nil {
-			return err
-		}
-		defer file.Close()
-		_, err = io.Copy(tw, file)
-		return err
-	})
-}

audit/internal/platform/sat_gpu_packs.go

@@ -0,0 +1,568 @@
+package platform
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+type NvidiaGPU struct {
+	Index    int    `json:"index"`
+	Name     string `json:"name"`
+	MemoryMB int    `json:"memory_mb"`
+}
+
+type NvidiaGPUStatus struct {
+	Index        int    `json:"index"`
+	Name         string `json:"name"`
+	BDF          string `json:"bdf,omitempty"`
+	Serial       string `json:"serial,omitempty"`
+	Status       string `json:"status"`
+	RawLine      string `json:"raw_line,omitempty"`
+	NeedsReset   bool   `json:"needs_reset"`
+	ParseFailure bool   `json:"parse_failure,omitempty"`
+}
+
+type nvidiaGPUHealth struct {
+	Index        int
+	Name         string
+	NeedsReset   bool
+	RawLine      string
+	ParseFailure bool
+}
+
+type nvidiaGPUStatusFile struct {
+	Index      int
+	Name       string
+	RunStatus  string
+	Reason     string
+	Health     string
+	HealthRaw  string
+	Observed   bool
+	Selected   bool
+	FailingJob string
+}
+
+// AMDGPUInfo holds basic info about an AMD GPU from rocm-smi.
+type AMDGPUInfo struct {
+	Index int    `json:"index"`
+	Name  string `json:"name"`
+}
+
+// DetectGPUVendor returns "nvidia" if /dev/nvidia0 exists, "amd" if /dev/kfd exists, or "" otherwise.
+func (s *System) DetectGPUVendor() string {
+	if _, err := os.Stat("/dev/nvidia0"); err == nil {
+		return "nvidia"
+	}
+	if _, err := os.Stat("/dev/kfd"); err == nil {
+		return "amd"
+	}
+	if raw, err := exec.Command("lspci", "-nn").Output(); err == nil {
+		// Only match AMD GPU device classes [0300]=VGA, [0302]=3D controller, [0380]=Display.
+		// AMD CPUs also appear in lspci as "Advanced Micro Devices" (Root Complex, IOMMU, etc.)
+		// so matching vendor alone causes false positives on AMD CPU servers without GPUs.
+		for _, line := range strings.Split(strings.ToLower(string(raw)), "\n") {
+			if !strings.Contains(line, "advanced micro devices") && !strings.Contains(line, "amd/ati") {
+				continue
+			}
+			if strings.Contains(line, "[0300]") || strings.Contains(line, "[0302]") || strings.Contains(line, "[0380]") {
+				return "amd"
+			}
+		}
+	}
+	return ""
+}
+
+// PhysicalGPUVendors reports which supported vendors have a display-class PCI
+// function, regardless of driver state. It is used to distinguish absent
+// hardware from a PCI function whose runtime is not operational yet.
+func (s *System) PhysicalGPUVendors() (nvidia bool, amd bool) {
+	raw, err := satExecCommand("lspci", "-nn").Output()
+	if err != nil {
+		return false, false
+	}
+	for _, line := range strings.Split(strings.ToLower(string(raw)), "\n") {
+		// [0300]=VGA, [0302]=3D controller, [0380]=Display controller.
+		if !strings.Contains(line, "[0300]") && !strings.Contains(line, "[0302]") && !strings.Contains(line, "[0380]") {
+			continue
+		}
+		switch {
+		case strings.Contains(line, "[10de:"):
+			nvidia = true
+		case strings.Contains(line, "[1002:"), strings.Contains(line, "advanced micro devices"), strings.Contains(line, "amd/ati"):
+			amd = true
+		}
+	}
+	return nvidia, amd
+}
+
+// ListAMDGPUs returns AMD GPUs visible to rocm-smi.
+func (s *System) ListAMDGPUs() ([]AMDGPUInfo, error) {
+	out, err := runROCmSMI("--showproductname", "--csv")
+	if err != nil {
+		return nil, fmt.Errorf("rocm-smi: %w", err)
+	}
+	var gpus []AMDGPUInfo
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(strings.ToLower(line), "device") {
+			continue
+		}
+		parts := strings.SplitN(line, ",", 2)
+		name := ""
+		if len(parts) >= 2 {
+			name = strings.TrimSpace(parts[1])
+		}
+		idx := len(gpus)
+		gpus = append(gpus, AMDGPUInfo{Index: idx, Name: name})
+	}
+	return gpus, nil
+}
+
+// RunAMDAcceptancePack runs an AMD GPU diagnostic pack using rocm-smi.
+func (s *System) RunAMDAcceptancePack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
+	return runAcceptancePackCtx(ctx, baseDir, "gpu-amd", []satJob{
+		{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
+		{name: "02-rocm-smi-showallinfo.log", cmd: []string{"rocm-smi", "--showallinfo"}},
+		{name: "03-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
+		{name: "04-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
+	}, logFunc)
+}
+
+// RunAMDMemIntegrityPack runs the official RVS MEM module as a validate-style memory integrity test.
+func (s *System) RunAMDMemIntegrityPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
+	if err := ensureAMDRuntimeReady(); err != nil {
+		return "", err
+	}
+	cfgFile := "/tmp/bee-amd-mem.conf"
+	cfg := `actions:
+- name: mem_integrity
+  device: all
+  module: mem
+  parallel: true
+  duration: 60000
+  copy_matrix: false
+  target_stress: 90
+  matrix_size: 8640
+`
+	_ = os.WriteFile(cfgFile, []byte(cfg), 0644)
+	return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-mem", []satJob{
+		{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
+		{name: "02-rvs-mem.log", cmd: []string{"rvs", "-c", cfgFile}},
+		{name: "03-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}},
+	}, logFunc)
+}
+
+// RunAMDMemBandwidthPack runs AMD's memory/interconnect bandwidth-oriented tools.
+func (s *System) RunAMDMemBandwidthPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
+	if err := ensureAMDRuntimeReady(); err != nil {
+		return "", err
+	}
+	cfgFile := "/tmp/bee-amd-babel.conf"
+	cfg := `actions:
+- name: babel_mem_bw
+  device: all
+  module: babel
+  parallel: true
+  copy_matrix: true
+  target_stress: 90
+  matrix_size: 134217728
+`
+	_ = os.WriteFile(cfgFile, []byte(cfg), 0644)
+	return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-bandwidth", []satJob{
+		{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
+		{name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}},
+		{name: "03-rvs-babel.log", cmd: []string{"rvs", "-c", cfgFile}},
+		{name: "04-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}},
+	}, logFunc)
+}
+
+// RunAMDStressPack runs an AMD GPU burn-in pack.
+// Missing tools are reported as UNSUPPORTED, consistent with the existing SAT pattern.
+func (s *System) RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
+	seconds := durationSec
+	if seconds <= 0 {
+		seconds = envInt("BEE_AMD_STRESS_SECONDS", 300)
+	}
+	if err := ensureAMDRuntimeReady(); err != nil {
+		return "", err
+	}
+	// Enable copy_matrix so the same GST run drives VRAM traffic in addition to compute.
+	rvsCfg := amdStressRVSConfig(seconds)
+	cfgFile := "/tmp/bee-amd-gst.conf"
+	_ = os.WriteFile(cfgFile, []byte(rvsCfg), 0644)
+
+	return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-stress", amdStressJobs(seconds, cfgFile), logFunc)
+}
+
+func amdStressRVSConfig(seconds int) string {
+	return fmt.Sprintf(`actions:
+- name: gst_stress
+  device: all
+  module: gst
+  parallel: true
+  duration: %d
+  copy_matrix: false
+  target_stress: 90
+  matrix_size_a: 8640
+  matrix_size_b: 8640
+  matrix_size_c: 8640
+`, seconds*1000)
+}
+
+func amdStressJobs(seconds int, cfgFile string) []satJob {
+	return []satJob{
+		{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
+		{name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}},
+		{name: fmt.Sprintf("03-rvs-gst-%ds.log", seconds), cmd: []string{"rvs", "-c", cfgFile}},
+		{name: fmt.Sprintf("04-rocm-smi-after.log"), cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--csv"}},
+	}
+}
+
+// ListNvidiaGPUs returns GPUs visible to nvidia-smi.
+func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) {
+	out, err := exec.Command("nvidia-smi",
+		"--query-gpu=index,name,memory.total",
+		"--format=csv,noheader,nounits").Output()
+	if err != nil {
+		return nil, fmt.Errorf("nvidia-smi: %w", err)
+	}
+	var gpus []NvidiaGPU
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		parts := strings.SplitN(line, ", ", 3)
+		if len(parts) != 3 {
+			continue
+		}
+		idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
+		if err != nil {
+			continue
+		}
+		memMB, _ := strconv.Atoi(strings.TrimSpace(parts[2]))
+		gpus = append(gpus, NvidiaGPU{
+			Index:    idx,
+			Name:     strings.TrimSpace(parts[1]),
+			MemoryMB: memMB,
+		})
+	}
+	sort.Slice(gpus, func(i, j int) bool {
+		return gpus[i].Index < gpus[j].Index
+	})
+	return gpus, nil
+}
+
+func (s *System) ListNvidiaGPUStatuses() ([]NvidiaGPUStatus, error) {
+	out, err := satExecCommand(
+		"nvidia-smi",
+		"--query-gpu=index,name,pci.bus_id,serial,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total",
+		"--format=csv,noheader,nounits",
+	).Output()
+	if err != nil {
+		return nil, fmt.Errorf("nvidia-smi: %w", err)
+	}
+	var gpus []NvidiaGPUStatus
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		parts := strings.Split(line, ",")
+		if len(parts) < 4 {
+			gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true})
+			continue
+		}
+		idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
+		if err != nil {
+			gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true})
+			continue
+		}
+		upper := strings.ToUpper(line)
+		needsReset := strings.Contains(upper, "GPU REQUIRES RESET")
+		status := "OK"
+		if needsReset {
+			status = "RESET_REQUIRED"
+		}
+		gpus = append(gpus, NvidiaGPUStatus{
+			Index:      idx,
+			Name:       strings.TrimSpace(parts[1]),
+			BDF:        normalizeNvidiaBusID(strings.TrimSpace(parts[2])),
+			Serial:     strings.TrimSpace(parts[3]),
+			Status:     status,
+			RawLine:    line,
+			NeedsReset: needsReset,
+		})
+	}
+	sort.Slice(gpus, func(i, j int) bool { return gpus[i].Index < gpus[j].Index })
+	return gpus, nil
+}
+
+func normalizeNvidiaBusID(v string) string {
+	v = strings.TrimSpace(strings.ToLower(v))
+	parts := strings.Split(v, ":")
+	if len(parts) == 3 && len(parts[0]) > 4 {
+		parts[0] = parts[0][len(parts[0])-4:]
+		return strings.Join(parts, ":")
+	}
+	return v
+}
+
+func (s *System) ResetNvidiaGPU(index int) (string, error) {
+	return resetNvidiaGPU(index)
+}
+
+// RunNCCLTests runs nccl-tests all_reduce_perf across the selected NVIDIA GPUs.
+// Measures collective communication bandwidth over NVLink/PCIe.
+func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
+	selected, err := resolveDCGMGPUIndices(gpuIndices)
+	if err != nil {
+		return "", err
+	}
+	gpuCount := len(selected)
+	if gpuCount < 1 {
+		gpuCount = 1
+	}
+	return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", withNvidiaPersistenceMode(
+		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
+		satJob{name: "02-all-reduce-perf.log", cmd: []string{
+			"all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2",
+			"-g", strconv.Itoa(gpuCount), "--iters", "20",
+		}, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true},
+	), logFunc)
+}
+
+func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error) {
+	selected, err := resolveDCGMGPUIndices(gpuIndices)
+	if err != nil {
+		return "", err
+	}
+	var (
+		profCmd []string
+		profEnv []string
+	)
+	if len(selected) > 1 {
+		// For multiple GPUs, always spawn one dcgmproftester process per GPU via
+		// bee-dcgmproftester-staggered (stagger=0 means all start simultaneously).
+		// A single dcgmproftester process without -i only loads GPU 0 regardless
+		// of CUDA_VISIBLE_DEVICES.
+		stagger := staggerSec
+		if stagger < 0 {
+			stagger = 0
+		}
+		profCmd = []string{
+			"bee-dcgmproftester-staggered",
+			"--seconds", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec)),
+			"--stagger-seconds", strconv.Itoa(stagger),
+			"--devices", joinIndexList(selected),
+		}
+	} else {
+		profCmd, err = resolveDCGMProfTesterCommand("--no-dcgm-validation", "-t", "1004", "-d", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec)))
+		if err != nil {
+			return "", err
+		}
+		profEnv = nvidiaVisibleDevicesEnv(selected)
+	}
+	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-compute", withNvidiaPersistenceMode(
+		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
+		satJob{name: "02-dcgmi-version.log", cmd: []string{"dcgmi", "-v"}},
+		satJob{
+			name:        "03-dcgmproftester.log",
+			cmd:         profCmd,
+			env:         profEnv,
+			collectGPU:  true,
+			gpuIndices:  selected,
+			syncBracket: true,
+		},
+		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
+	), logFunc)
+}
+
+func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
+	selected, err := resolveDCGMGPUIndices(gpuIndices)
+	if err != nil {
+		return "", err
+	}
+	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
+	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
+	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
+		for _, p := range killed {
+			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
+		}
+	}
+	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode(
+		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
+		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
+		satJob{
+			name:        "03-dcgmi-targeted-power.log",
+			cmd:         nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
+			collectGPU:  true,
+			gpuIndices:  selected,
+			syncBracket: true,
+		},
+		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
+	), logFunc)
+}
+
+func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
+	selected, err := resolveDCGMGPUIndices(gpuIndices)
+	if err != nil {
+		return "", err
+	}
+	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
+	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
+	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
+		for _, p := range killed {
+			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
+		}
+	}
+	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode(
+		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
+		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
+		satJob{
+			name:        "03-dcgmi-pulse-test.log",
+			cmd:         nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
+			collectGPU:  true,
+			gpuIndices:  selected,
+			syncBracket: true,
+		},
+		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
+	), logFunc)
+}
+
+// RunNvidiaBandwidthPack runs `dcgmi diag -r nvbandwidth`. The only thing
+// fullMatrix changes is which GPU sets each invocation gets via `-i`:
+//
+//   - fullMatrix=false (Validate): a single pass across every selected GPU.
+//   - fullMatrix=true (deep/Stress): on a system whose GPUs span more than
+//     one CPU socket, one pass per socket group and then one pass across all
+//     of them, isolating the cross-socket peer-to-peer path as its own
+//     fault domain (see bible-local/decisions/2026-07-27-nvbandwidth-per-socket-split.md).
+//     Single-socket systems collapse back to one pass.
+func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, fullMatrix bool, logFunc func(string)) (string, error) {
+	selected, err := resolveDCGMGPUIndices(gpuIndices)
+	if err != nil {
+		return "", err
+	}
+	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
+	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
+	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
+		for _, p := range killed {
+			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
+		}
+	}
+	jobs := []satJob{
+		{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
+		{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
+	}
+
+	// On a system with GPUs on more than one CPU socket, run each socket's
+	// GPUs through nvbandwidth in isolation before the all-GPU pass. Without
+	// NVLink, cross-socket peer-to-peer traffic is a distinct fault domain
+	// from same-socket traffic; if the single-socket passes log clean and
+	// only the all-GPU pass doesn't complete, that isolates the cross-socket
+	// path as the trigger instead of leaving it conflated with a general
+	// GPU/PCIe fault. Systems with one socket (or no resolvable NUMA
+	// affinity) get a single group back and keep the original one-pass shape.
+	step := 3
+	socketGroups := [][]int{selected}
+	if fullMatrix {
+		socketGroups = gpuBandwidthSocketGroups(selected, logFunc)
+	}
+	if len(socketGroups) <= 1 {
+		jobs = append(jobs, satJob{
+			name:        fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step),
+			cmd:         nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
+			collectGPU:  true,
+			gpuIndices:  selected,
+			syncBracket: true,
+		})
+		step++
+	} else {
+		for i, group := range socketGroups {
+			jobs = append(jobs, satJob{
+				name:        fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i),
+				cmd:         nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group),
+				collectGPU:  true,
+				gpuIndices:  group,
+				syncBracket: true,
+			})
+			step++
+		}
+		jobs = append(jobs, satJob{
+			name:        fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step),
+			cmd:         nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
+			collectGPU:  true,
+			gpuIndices:  selected,
+			syncBracket: true,
+		})
+		step++
+	}
+	jobs = append(jobs, satJob{
+		name: fmt.Sprintf("%02d-nvidia-smi-after.log", step),
+		cmd:  []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"},
+	})
+
+	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(jobs...), logFunc)
+}
+
+func (s *System) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
+	return runAcceptancePackCtx(context.Background(), baseDir, "gpu-nvidia", nvidiaSATJobs(), logFunc)
+}
+
+// RunNvidiaAcceptancePackWithOptions runs the NVIDIA diagnostics via DCGM.
+// diagLevel: 1=quick, 2=medium, 3=targeted stress, 4=extended stress.
+// gpuIndices: specific GPU indices to test (empty = all GPUs).
+// ctx cancellation kills the running job.
+func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int, logFunc func(string)) (string, error) {
+	resolvedGPUIndices, err := resolveDCGMGPUIndices(gpuIndices)
+	if err != nil {
+		return "", err
+	}
+	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia", nvidiaDCGMJobs(diagLevel, resolvedGPUIndices), logFunc)
+}
+
+func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
+	selected, err := resolveDCGMGPUIndices(gpuIndices)
+	if err != nil {
+		return "", err
+	}
+	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
+	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
+	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
+		for _, p := range killed {
+			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
+		}
+	}
+	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode(
+		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
+		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
+		satJob{
+			name:        "03-dcgmi-targeted-stress.log",
+			cmd:         nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
+			collectGPU:  true,
+			gpuIndices:  selected,
+			syncBracket: true,
+		},
+		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
+	), logFunc)
+}
+
+func resolveDCGMGPUIndices(gpuIndices []int) ([]int, error) {
+	if len(gpuIndices) > 0 {
+		return dedupeSortedIndices(gpuIndices), nil
+	}
+	all, err := listNvidiaGPUIndices()
+	if err != nil {
+		return nil, err
+	}
+	if len(all) == 0 {
+		return nil, fmt.Errorf("nvidia-smi found no NVIDIA GPUs")
+	}
+	return all, nil
+}

audit/internal/platform/sat_host_packs.go

@@ -0,0 +1,257 @@
+package platform
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+func memoryStressSizeArg() string {
+	if mb := envInt("BEE_VM_STRESS_SIZE_MB", 0); mb > 0 {
+		return fmt.Sprintf("%dM", mb)
+	}
+	availBytes := satFreeMemBytes()
+	if availBytes <= 0 {
+		return "80%"
+	}
+	availMB := availBytes / (1024 * 1024)
+	targetMB := (availMB * 2) / 3
+	if targetMB >= 256 {
+		targetMB = (targetMB / 256) * 256
+	}
+	if targetMB <= 0 {
+		return "80%"
+	}
+	return fmt.Sprintf("%dM", targetMB)
+}
+
+func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error) {
+	if sizeMB <= 0 {
+		sizeMB = 256
+	}
+	if passes <= 0 {
+		passes = 1
+	}
+	// Keep Validate Memory bounded to a quick diagnostic window. The timeout is
+	// intentionally conservative enough for healthy systems while avoiding the
+	// prior 30-80 minute hangs caused by memtester spinning on a bad subtest.
+	timeoutSec := sizeMB*passes*20/100 + 60
+	if timeoutSec < 180 {
+		timeoutSec = 180
+	}
+	if timeoutSec > 900 {
+		timeoutSec = 900
+	}
+	return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{
+		{name: "01-free-before.log", cmd: []string{"free", "-h"}},
+		{name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}, syncBracket: true},
+		{name: "03-free-after.log", cmd: []string{"free", "-h"}},
+	}, logFunc)
+}
+
+func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
+	seconds := durationSec
+	if seconds <= 0 {
+		seconds = envInt("BEE_VM_STRESS_SECONDS", 300)
+	}
+	// Base the default on current MemAvailable and keep headroom for the OS and
+	// concurrent stressors so mixed burn runs do not trip the OOM killer.
+	sizeArg := memoryStressSizeArg()
+	return runAcceptancePackCtx(ctx, baseDir, "memory-stress", []satJob{
+		{name: "01-free-before.log", cmd: []string{"free", "-h"}},
+		{name: "02-stress-ng-vm.log", cmd: []string{
+			"stress-ng", "--vm", "1",
+			"--vm-bytes", sizeArg,
+			"--vm-method", "all",
+			"--timeout", fmt.Sprintf("%d", seconds),
+			"--metrics-brief",
+		}, syncBracket: true},
+		{name: "03-free-after.log", cmd: []string{"free", "-h"}},
+	}, logFunc)
+}
+
+func (s *System) RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
+	seconds := durationSec
+	if seconds <= 0 {
+		seconds = envInt("BEE_SAT_STRESS_SECONDS", 300)
+	}
+	cmd := []string{"stressapptest", "-s", fmt.Sprintf("%d", seconds), "-W", "--cc_test"}
+	if mb := envInt("BEE_SAT_STRESS_MB", 0); mb > 0 {
+		cmd = append(cmd, "-M", fmt.Sprintf("%d", mb))
+	}
+	return runAcceptancePackCtx(ctx, baseDir, "sat-stress", []satJob{
+		{name: "01-free-before.log", cmd: []string{"free", "-h"}},
+		{name: "02-stressapptest.log", cmd: cmd},
+		{name: "03-free-after.log", cmd: []string{"free", "-h"}},
+	}, logFunc)
+}
+
+// cpuThermalThrottleSysDir is the sysfs root the throttle-check scripts glob
+// under. Overridden in tests so they can point at a fake directory tree
+// instead of the real /sys.
+var cpuThermalThrottleSysDir = "/sys/devices/system/cpu"
+
+// cpuThrottleSumScript is the shell fragment both before/after scripts use to
+// sum the kernel's cumulative-since-boot thermal throttle counters across
+// every CPU.
+func cpuThrottleSumScript() string {
+	return fmt.Sprintf(`
+sum=0
+for f in %[1]s/cpu*/thermal_throttle/core_throttle_count %[1]s/cpu*/thermal_throttle/package_throttle_count; do
+  [ -f "$f" ] || continue
+  v=$(cat "$f" 2>/dev/null)
+  case "$v" in ''|*[!0-9]*) continue ;; esac
+  sum=$((sum + v))
+done
+`, cpuThermalThrottleSysDir)
+}
+
+// cpuThrottleBeforeScript snapshots the throttle counter sum into a file in
+// {{run_dir}} so cpuThrottleCheckScript can later diff before/after despite
+// each satJob running as an independent process.
+func cpuThrottleBeforeScript() string {
+	return cpuThrottleSumScript() + `echo "$sum" | tee {{run_dir}}/.cpu-throttle-before` + "\n"
+}
+
+// cpuThrottleCheckScript compares the after-run throttle counter sum against
+// the snapshot cpuThrottleBeforeScript took, and fails (non-zero exit) if it
+// increased — i.e. the CPU actually hit thermal throttling during this
+// specific run, not just at some earlier point this boot. classifySATResult
+// maps a failed job here to SAT status FAILED, which ApplySATResultToDB
+// records as component status "Warning" for cpu:all — without this, the
+// "cpu" SAT pack only checks stress-ng's exit code, which is 0 whether or
+// not the CPU throttled while running it.
+func cpuThrottleCheckScript() string {
+	return `before=$(cat {{run_dir}}/.cpu-throttle-before 2>/dev/null)
+case "$before" in ''|*[!0-9]*) before=0 ;; esac
+` + cpuThrottleSumScript() + `after=$sum
+echo "throttle_count_before=$before"
+echo "throttle_count_after=$after"
+if [ "$after" -gt "$before" ]; then
+  echo "THROTTLE DETECTED: CPU package/core hit thermal throttling during this stress-ng run ($before -> $after)"
+  exit 1
+fi
+echo "no new thermal throttling detected during this run"
+`
+}
+
+func cpuSATJobs(durationSec int) []satJob {
+	return []satJob{
+		{name: "01-lscpu.log", cmd: []string{"lscpu"}},
+		{name: "02-sensors-before.log", cmd: []string{"sensors"}},
+		{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
+		{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
+		{name: "04-sensors-after.log", cmd: []string{"sensors"}},
+		{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
+	}
+}
+
+func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
+	if durationSec <= 0 {
+		durationSec = 60
+	}
+	return runAcceptancePackCtx(ctx, baseDir, "cpu", cpuSATJobs(durationSec), logFunc)
+}
+
+func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) {
+	if baseDir == "" {
+		baseDir = "/var/log/bee-sat"
+	}
+	ts := time.Now().UTC().Format("20060102-150405")
+	runDir := filepath.Join(baseDir, "storage-"+ts)
+	if err := os.MkdirAll(runDir, 0755); err != nil {
+		return "", err
+	}
+	verboseLog := filepath.Join(runDir, "verbose.log")
+
+	devices, err := listStorageDevices()
+	if err != nil {
+		return "", err
+	}
+	sort.Strings(devices)
+
+	var summary strings.Builder
+	stats := satStats{}
+	fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
+	if len(devices) == 0 {
+		fmt.Fprintln(&summary, "devices=0")
+		stats.Unsupported++
+	} else {
+		fmt.Fprintf(&summary, "devices=%d\n", len(devices))
+	}
+
+	for index, devPath := range devices {
+		if ctx.Err() != nil {
+			break
+		}
+		prefix := fmt.Sprintf("%02d-%s", index+1, filepath.Base(devPath))
+		commands := storageSATCommands(devPath, extended)
+		deviceOutputs := make(map[string][]byte, len(commands))
+		for cmdIndex, job := range commands {
+			if ctx.Err() != nil {
+				break
+			}
+			name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
+			livePath := filepath.Join(runDir, name)
+			runSyncBracketHook(job, "before", logFunc)
+			out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc, livePath)
+			deviceOutputs[job.name] = out
+			if writeErr := os.WriteFile(livePath, out, 0644); writeErr != nil {
+				return "", writeErr
+			}
+			if satJobBoundaryHook != nil {
+				satJobBoundaryHook(name)
+			}
+			// smartctl -t short only launches the self-test on the drive firmware and
+			// returns immediately ("Testing has begun"); unlike `nvme device-self-test
+			// --wait`, smartctl has no blocking mode, so we must poll the drive
+			// ourselves until the self-test actually finishes. Hold the "after" sync
+			// until that poll completes — the self-test itself, not just its launch,
+			// is the load worth having durable evidence of.
+			deferSyncBracketAfter := job.name == "smartctl-self-test-short" && err == nil
+			if !deferSyncBracketAfter {
+				runSyncBracketHook(job, "after", logFunc)
+			}
+			status, rc := classifySATResult(job.name, out, err)
+			// A zero smartctl exit status only proves the command ran.  If the
+			// drive did not return its overall-health verdict, it must not turn
+			// the storage SAT green.
+			if job.name == "smartctl-health" && status == "OK" && !hasSMARTOverallHealth(out) {
+				status = "UNSUPPORTED"
+			}
+			stats.Add(status)
+			key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_")
+			fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
+			fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
+
+			if deferSyncBracketAfter {
+				statusName := "smartctl-self-test-status"
+				statusOut := waitForSmartctlSelfTest(ctx, verboseLog, devPath, logFunc)
+				deviceOutputs[statusName] = statusOut
+				statusFile := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+2, statusName)
+				if writeErr := os.WriteFile(filepath.Join(runDir, statusFile), statusOut, 0644); writeErr != nil {
+					return "", writeErr
+				}
+				runSyncBracketHook(job, "after", logFunc)
+				sStatus, sRC := classifySATResult(statusName, statusOut, nil)
+				stats.Add(sStatus)
+				sKey := filepath.Base(devPath) + "_" + strings.ReplaceAll(statusName, "-", "_")
+				fmt.Fprintf(&summary, "%s_rc=%d\n", sKey, sRC)
+				fmt.Fprintf(&summary, "%s_status=%s\n", sKey, sStatus)
+			}
+		}
+		reportText := GenerateDiskReportText(index+1, devPath, deviceOutputs, time.Now().UTC())
+		reportName := "disk-" + prefix + "-report.txt"
+		_ = os.WriteFile(filepath.Join(runDir, reportName), []byte(reportText), 0644)
+	}
+
+	writeSATStats(&summary, stats)
+	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
+		return "", err
+	}
+	return runDir, nil
+}

audit/internal/platform/sat_test.go

@@ -145,8 +145,6 @@ func TestNvidiaDCGMJobsEnablePersistenceModeBeforeDiag(t *testing.T) {
 }
 
 func TestBuildNvidiaStressJobUsesSelectedLoaderAndDevices(t *testing.T) {
-	t.Parallel()
-
 	oldExecCommand := satExecCommand
 	satExecCommand = func(name string, args ...string) *exec.Cmd {
 		if name == "nvidia-smi" {
@@ -179,8 +177,6 @@ func TestBuildNvidiaStressJobUsesSelectedLoaderAndDevices(t *testing.T) {
 }
 
 func TestBuildNvidiaStressJobUsesNCCLLoader(t *testing.T) {
-	t.Parallel()
-
 	oldExecCommand := satExecCommand
 	satExecCommand = func(name string, args ...string) *exec.Cmd {
 		if name == "nvidia-smi" {
@@ -213,8 +209,6 @@ func TestBuildNvidiaStressJobUsesNCCLLoader(t *testing.T) {
 }
 
 func TestResolveDCGMGPUIndicesUsesDetectedGPUsWhenUnset(t *testing.T) {
-	t.Parallel()
-
 	oldExecCommand := satExecCommand
 	satExecCommand = func(name string, args ...string) *exec.Cmd {
 		if name == "nvidia-smi" {

audit/internal/platform/tpm.go

@@ -1,14 +1,82 @@
 package platform
 
-import "context"
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// tpmDeviceGlob is a seam for tests. It reports the sysfs TPM device nodes
+// the kernel has registered; an empty result means the platform exposes no
+// TPM at all (no discrete chip, or firmware/BIOS has it disabled).
+var tpmDeviceGlob = func() []string {
+	matches, _ := filepath.Glob("/sys/class/tpm/tpm*")
+	return matches
+}
+
+var tpmReadFile = os.ReadFile
+
+// TPMPresent reports whether sysfs identifies a registered device as TPM 2.x.
+// The validation pack uses tpm2-tools, so a TPM 1.2 device is not sufficient.
+func (s *System) TPMPresent() bool {
+	for _, device := range tpmDeviceGlob() {
+		version, err := tpmReadFile(filepath.Join(device, "tpm_version_major"))
+		if err == nil && strings.TrimSpace(string(version)) == "2" {
+			return true
+		}
+	}
+	return false
+}
 
 // RunTPMValidationPack verifies TPM 2.0 communication using read-only
 // commands. It deliberately excludes SelfTest, provisioning, NV writes, PCR
 // changes, key creation, and ownership operations.
+//
+// When the platform exposes no TPM device at all, the pack does not run the
+// tpm2_* tools: without a TCTI device they only ever emit a wall of
+// "Failed to open ... /dev/tpmrm0" errors that read as a hard failure when
+// the real situation is "this machine has no TPM". Instead it writes an
+// UNSUPPORTED summary and returns, the same way the storage pack handles a
+// host with no drives.
 func (s *System) RunTPMValidationPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
+	if !s.TPMPresent() {
+		return writeTPMUnsupportedRun(baseDir, logFunc)
+	}
 	return runAcceptancePackCtx(ctx, baseDir, "tpm", tpmValidationJobs(), logFunc)
 }
 
+func writeTPMUnsupportedRun(baseDir string, logFunc func(string)) (string, error) {
+	if strings.TrimSpace(baseDir) == "" {
+		baseDir = "/var/log/bee-sat"
+	}
+	now := time.Now().UTC()
+	runDir := filepath.Join(baseDir, "tpm-"+now.Format("20060102-150405"))
+	if err := os.MkdirAll(runDir, 0755); err != nil {
+		return "", err
+	}
+	if logFunc != nil {
+		logFunc("no TPM 2.x device reported by sysfs; skipping read-only TPM checks")
+	}
+
+	var summary strings.Builder
+	fmt.Fprintf(&summary, "run_at_utc=%s\n", now.Format(time.RFC3339))
+	summary.WriteString("tpm_present=false\n")
+	summary.WriteString("skip_reason=no TPM 2.x device reported by sysfs; tpm2-tools are not applicable\n")
+	summary.WriteString("tpm_check_status=UNSUPPORTED\n")
+	summary.WriteString("overall_status=UNSUPPORTED\n")
+	summary.WriteString("job_ok=0\n")
+	summary.WriteString("job_failed=0\n")
+	summary.WriteString("job_unsupported=1\n")
+	summary.WriteString("job_informational_failed=0\n")
+	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
+		return "", err
+	}
+	return runDir, nil
+}
+
 func tpmValidationJobs() []satJob {
 	return []satJob{
 		{name: "01-properties-fixed.log", cmd: []string{"tpm2_getcap", "properties-fixed"}},

audit/internal/platform/tpm_test.go

@@ -1,11 +1,66 @@
 package platform
 
 import (
+	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"testing"
 )
 
+func TestRunTPMValidationPackSkipsWhenNoTPMDevice(t *testing.T) {
+	old := tpmDeviceGlob
+	tpmDeviceGlob = func() []string { return nil }
+	t.Cleanup(func() { tpmDeviceGlob = old })
+
+	dir := t.TempDir()
+	runDir, err := (&System{}).RunTPMValidationPack(nil, dir, nil)
+	if err != nil {
+		t.Fatalf("RunTPMValidationPack: %v", err)
+	}
+	for _, name := range []string{"01-properties-fixed.log", "02-pcr-banks.log", "03-pcr-values.log", "04-test-result.log"} {
+		if _, err := os.Stat(filepath.Join(runDir, name)); err == nil {
+			t.Fatalf("tpm2 job %q ran despite no TPM device", name)
+		}
+	}
+	summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
+	if err != nil {
+		t.Fatalf("read summary: %v", err)
+	}
+	if !strings.Contains(string(summary), "overall_status=UNSUPPORTED") ||
+		!strings.Contains(string(summary), "tpm_present=false") {
+		t.Fatalf("summary missing UNSUPPORTED/tpm_present markers:\n%s", summary)
+	}
+}
+
+func TestTPMPresentRequiresVersion2(t *testing.T) {
+	oldGlob, oldRead := tpmDeviceGlob, tpmReadFile
+	t.Cleanup(func() {
+		tpmDeviceGlob = oldGlob
+		tpmReadFile = oldRead
+	})
+
+	tests := []struct {
+		name    string
+		version string
+		readErr error
+		want    bool
+	}{
+		{name: "TPM 2", version: "2\n", want: true},
+		{name: "TPM 1.2", version: "1\n", want: false},
+		{name: "missing version attribute", readErr: os.ErrNotExist, want: false},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			tpmDeviceGlob = func() []string { return []string{"/sys/class/tpm/tpm0"} }
+			tpmReadFile = func(string) ([]byte, error) { return []byte(test.version), test.readErr }
+			if got := (&System{}).TPMPresent(); got != test.want {
+				t.Fatalf("TPMPresent()=%v want %v", got, test.want)
+			}
+		})
+	}
+}
+
 func TestTPMValidationJobsAreReadOnly(t *testing.T) {
 	t.Parallel()
 

audit/internal/webui/api_gpu_tools.go

@@ -0,0 +1,509 @@
+package webui
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"bee/audit/internal/platform"
+)
+
+func (h *handler) handleAPIGNVIDIAGPUs(w http.ResponseWriter, _ *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	gpus, err := h.opts.App.ListNvidiaGPUs()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if gpus == nil {
+		gpus = []platform.NvidiaGPU{}
+	}
+	writeJSON(w, gpus)
+}
+
+func (h *handler) handleAPIGNVIDIAGPUStatuses(w http.ResponseWriter, _ *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	gpus, err := apiListNvidiaGPUStatuses(h.opts.App)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if gpus == nil {
+		gpus = []platform.NvidiaGPUStatus{}
+	}
+	writeJSON(w, gpus)
+}
+
+func (h *handler) handleAPIGNVIDIAReset(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	var req struct {
+		Index int `json:"index"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid request body")
+		return
+	}
+	result, err := h.opts.App.ResetNvidiaGPU(req.Index)
+	status := "ok"
+	if err != nil {
+		status = "error"
+	}
+	writeJSON(w, map[string]string{"status": status, "output": result.Body})
+}
+
+// ── GPU settings (ECC / power limit) ──────────────────────────────────────────
+
+func (h *handler) handleAPIGNVIDIAGPUSettings(w http.ResponseWriter, _ *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	settings, err := h.opts.App.ListNvidiaGPUSettings()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if settings == nil {
+		settings = []platform.NvidiaGPUSetting{}
+	}
+	writeJSON(w, settings)
+}
+
+func (h *handler) handleAPIGNVIDIASetECC(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	var req struct {
+		Index   int  `json:"index"`
+		Enabled bool `json:"enabled"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid request body")
+		return
+	}
+	result, err := h.opts.App.SetNvidiaGPUECC(req.Index, req.Enabled)
+	status := "ok"
+	if err != nil {
+		status = "error"
+	}
+	writeJSON(w, map[string]string{"status": status, "output": result.Body})
+}
+
+func (h *handler) handleAPIGNVIDIASetMIG(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	var req struct {
+		Index   int  `json:"index"`
+		Enabled bool `json:"enabled"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid request body")
+		return
+	}
+	result, err := h.opts.App.SetNvidiaGPUMIG(req.Index, req.Enabled)
+	status := "ok"
+	if err != nil {
+		status = "error"
+	}
+	writeJSON(w, map[string]string{"status": status, "output": result.Body})
+}
+
+func (h *handler) handleAPIGNVIDIASetCCMode(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	var req struct {
+		Index   int  `json:"index"`
+		Enabled bool `json:"enabled"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid request body")
+		return
+	}
+	result, err := h.opts.App.SetNvidiaGPUCCMode(req.Index, req.Enabled)
+	status := "ok"
+	if err != nil {
+		status = "error"
+	}
+	writeJSON(w, map[string]string{"status": status, "output": result.Body})
+}
+
+func (h *handler) handleAPIGNVIDIASetPowerLimit(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	var req struct {
+		Index int     `json:"index"`
+		Watts float64 `json:"watts"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid request body")
+		return
+	}
+	if req.Watts <= 0 {
+		writeError(w, http.StatusBadRequest, "watts must be > 0")
+		return
+	}
+	result, err := h.opts.App.SetNvidiaGPUPowerLimit(req.Index, req.Watts)
+	status := "ok"
+	if err != nil {
+		status = "error"
+	}
+	writeJSON(w, map[string]string{"status": status, "output": result.Body})
+}
+
+func (h *handler) handleAPIGNVIDIAResetDefaults(w http.ResponseWriter, _ *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	result, err := h.opts.App.ResetNvidiaGPUDefaults()
+	status := "ok"
+	if err != nil {
+		status = "error"
+	}
+	writeJSON(w, map[string]string{"status": status, "output": result.Body})
+}
+
+func (h *handler) handleAPIGPUPresence(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	gp := h.opts.App.DetectGPUPresence()
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]bool{
+		"nvidia":              gp.Nvidia,
+		"amd":                 gp.AMD,
+		"nvidia_initializing": gp.NvidiaInitializing,
+		"amd_initializing":    gp.AMDInitializing,
+	})
+}
+
+// ── GPU tools ─────────────────────────────────────────────────────────────────
+
+func (h *handler) handleAPIGPUTools(w http.ResponseWriter, _ *http.Request) {
+	type toolEntry struct {
+		ID        string `json:"id"`
+		Available bool   `json:"available"`
+		Vendor    string `json:"vendor"` // "nvidia" | "amd"
+	}
+	// Single source of truth for GPU presence: see app.DetectGPUPresence.
+	var nvidiaUp, amdUp bool
+	if h.opts.App != nil {
+		gp := h.opts.App.DetectGPUPresence()
+		nvidiaUp, amdUp = gp.Nvidia, gp.AMD
+	} else {
+		_, nvidiaErr := os.Stat("/dev/nvidia0")
+		_, amdErr := os.Stat("/dev/kfd")
+		nvidiaUp, amdUp = nvidiaErr == nil, amdErr == nil
+	}
+	_, dcgmErr := exec.LookPath("dcgmi")
+	_, ncclStressErr := exec.LookPath("bee-nccl-gpu-stress")
+	_, johnErr := exec.LookPath("bee-john-gpu-stress")
+	_, beeBurnErr := exec.LookPath("bee-gpu-burn")
+	_, nvBandwidthErr := exec.LookPath("nvbandwidth")
+	profErr := lookPathAny("dcgmproftester", "dcgmproftester13", "dcgmproftester12", "dcgmproftester11")
+	writeJSON(w, []toolEntry{
+		{ID: "nvidia-compute", Available: nvidiaUp && profErr == nil, Vendor: "nvidia"},
+		{ID: "nvidia-targeted-power", Available: nvidiaUp && dcgmErr == nil, Vendor: "nvidia"},
+		{ID: "nvidia-pulse", Available: nvidiaUp && dcgmErr == nil, Vendor: "nvidia"},
+		{ID: "nvidia-interconnect", Available: nvidiaUp && ncclStressErr == nil, Vendor: "nvidia"},
+		{ID: "nvidia-bandwidth", Available: nvidiaUp && dcgmErr == nil && nvBandwidthErr == nil, Vendor: "nvidia"},
+		{ID: "bee-gpu-burn", Available: nvidiaUp && beeBurnErr == nil, Vendor: "nvidia"},
+		{ID: "john", Available: nvidiaUp && johnErr == nil, Vendor: "nvidia"},
+		{ID: "rvs", Available: amdUp, Vendor: "amd"},
+	})
+}
+
+func lookPathAny(names ...string) error {
+	for _, name := range names {
+		if _, err := exec.LookPath(name); err == nil {
+			return nil
+		}
+	}
+	return exec.ErrNotFound
+}
+
+// ── System ────────────────────────────────────────────────────────────────────
+
+func (h *handler) handleAPIRAMStatus(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	status := h.currentRAMStatus()
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(status)
+}
+
+type ramStatusResponse struct {
+	platform.LiveMediaRAMState
+	InstallTaskActive bool   `json:"install_task_active,omitempty"`
+	CopyTaskActive    bool   `json:"copy_task_active,omitempty"`
+	CanStartTask      bool   `json:"can_start_task,omitempty"`
+	BlockedReason     string `json:"blocked_reason,omitempty"`
+}
+
+func (h *handler) currentRAMStatus() ramStatusResponse {
+	state := h.opts.App.LiveMediaRAMState()
+	resp := ramStatusResponse{LiveMediaRAMState: state}
+	if globalQueue.hasActiveTarget("install") {
+		resp.InstallTaskActive = true
+		resp.BlockedReason = "install to disk is already running"
+		return resp
+	}
+	if globalQueue.hasActiveTarget("install-to-ram") {
+		resp.CopyTaskActive = true
+		resp.BlockedReason = "install to RAM task is already pending or running"
+		return resp
+	}
+	if state.InRAM {
+		resp.BlockedReason = "system is already running from RAM"
+		return resp
+	}
+	resp.CanStartTask = state.CanStartCopy
+	if !resp.CanStartTask && resp.BlockedReason == "" {
+		resp.BlockedReason = state.Message
+	}
+	return resp
+}
+
+func (h *handler) handleAPIInstallToRAM(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	status := h.currentRAMStatus()
+	if !status.CanStartTask {
+		msg := strings.TrimSpace(status.BlockedReason)
+		if msg == "" {
+			msg = "install to RAM is not available"
+		}
+		writeError(w, http.StatusConflict, msg)
+		return
+	}
+	t := &Task{
+		ID:        newJobID("install-to-ram"),
+		Name:      "Install to RAM",
+		Target:    "install-to-ram",
+		Priority:  defaultTaskPriority("install-to-ram", taskParams{}),
+		Status:    TaskPending,
+		CreatedAt: time.Now(),
+	}
+	globalQueue.enqueue(t)
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]string{"task_id": t.ID})
+}
+
+func (h *handler) handleAPISystemReboot(w http.ResponseWriter, r *http.Request) {
+	if err := exec.Command("systemctl", "reboot").Start(); err != nil {
+		writeError(w, http.StatusInternalServerError, "reboot failed: "+err.Error())
+		return
+	}
+	writeJSON(w, map[string]string{"status": "rebooting"})
+}
+
+func (h *handler) handleAPISystemShutdown(w http.ResponseWriter, r *http.Request) {
+	if err := exec.Command("systemctl", "poweroff").Start(); err != nil {
+		writeError(w, http.StatusInternalServerError, "shutdown failed: "+err.Error())
+		return
+	}
+	writeJSON(w, map[string]string{"status": "shutting down"})
+}
+
+// timezoneNameRE matches IANA timezone identifiers like "Europe/Moscow" or "UTC".
+var timezoneNameRE = regexp.MustCompile(`^[A-Za-z0-9_+\-]+(/[A-Za-z0-9_+\-]+)*$`)
+
+func validTimezoneName(tz string) bool {
+	if tz == "" || !timezoneNameRE.MatchString(tz) {
+		return false
+	}
+	_, err := os.Stat(filepath.Join("/usr/share/zoneinfo", tz))
+	return err == nil
+}
+
+// handleAPISystemTimeSync sets the host's timezone and wall-clock time from
+// values supplied by the client's browser (used when the appliance has no
+// network/NTP access to keep its own clock in sync).
+func (h *handler) handleAPISystemTimeSync(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		Timezone string `json:"timezone"`
+		EpochMS  int64  `json:"epoch_ms"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid request body")
+		return
+	}
+	if req.EpochMS <= 0 {
+		writeError(w, http.StatusBadRequest, "epoch_ms required")
+		return
+	}
+
+	var out strings.Builder
+
+	if req.Timezone != "" {
+		if !validTimezoneName(req.Timezone) {
+			writeError(w, http.StatusBadRequest, "invalid timezone")
+			return
+		}
+		if b, err := exec.Command("timedatectl", "set-timezone", req.Timezone).CombinedOutput(); err != nil {
+			writeError(w, http.StatusInternalServerError, "set-timezone failed: "+strings.TrimSpace(string(b)))
+			return
+		}
+		fmt.Fprintf(&out, "timezone set to %s\n", req.Timezone)
+	}
+
+	// Manual time only sticks if NTP sync is off.
+	_ = exec.Command("timedatectl", "set-ntp", "false").Run()
+
+	sec := req.EpochMS / 1000
+	if b, err := exec.Command("date", "-u", "-s", "@"+strconv.FormatInt(sec, 10)).CombinedOutput(); err != nil {
+		writeError(w, http.StatusInternalServerError, "set-time failed: "+strings.TrimSpace(string(b)))
+		return
+	}
+	out.WriteString("system clock synced\n")
+
+	writeJSON(w, map[string]string{"status": "ok", "output": out.String()})
+}
+
+// ── Tools ─────────────────────────────────────────────────────────────────────
+
+var standardTools = []string{
+	"dmidecode", "smartctl", "nvme", "lspci", "ipmitool",
+	"tpm2_getcap", "tpm2_pcrread", "tpm2_gettestresult",
+	"nvidia-smi", "dcgmi", "nv-hostengine", "memtester", "stress-ng", "nvtop",
+	"mstflint", "saa",
+}
+
+func (h *handler) handleAPIToolsCheck(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	statuses := h.opts.App.CheckTools(standardTools)
+	writeJSON(w, statuses)
+}
+
+// ── Preflight ─────────────────────────────────────────────────────────────────
+
+func (h *handler) handleAPIPreflight(w http.ResponseWriter, r *http.Request) {
+	data, err := loadSnapshot(filepath.Join(h.opts.ExportDir, "runtime-health.json"))
+	if err != nil {
+		writeError(w, http.StatusNotFound, "runtime health not found")
+		return
+	}
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Cache-Control", "no-store")
+	_, _ = w.Write(data)
+}
+
+// ── Install ───────────────────────────────────────────────────────────────────
+
+func (h *handler) handleAPIInstallDisks(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	disks, err := h.opts.App.ListInstallDisks()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	type diskJSON struct {
+		Device       string   `json:"device"`
+		Model        string   `json:"model"`
+		Size         string   `json:"size"`
+		SizeBytes    int64    `json:"size_bytes"`
+		MountedParts []string `json:"mounted_parts"`
+		Warnings     []string `json:"warnings"`
+	}
+	result := make([]diskJSON, 0, len(disks))
+	for _, d := range disks {
+		result = append(result, diskJSON{
+			Device:       d.Device,
+			Model:        d.Model,
+			Size:         d.Size,
+			SizeBytes:    d.SizeBytes,
+			MountedParts: d.MountedParts,
+			Warnings:     platform.DiskWarnings(d),
+		})
+	}
+	writeJSON(w, result)
+}
+
+func (h *handler) handleAPIInstallRun(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	var req struct {
+		Device string `json:"device"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Device == "" {
+		writeError(w, http.StatusBadRequest, "device is required")
+		return
+	}
+
+	// Whitelist: only allow devices that ListInstallDisks() returns.
+	disks, err := h.opts.App.ListInstallDisks()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	allowed := false
+	for _, d := range disks {
+		if d.Device == req.Device {
+			allowed = true
+			break
+		}
+	}
+	if !allowed {
+		writeError(w, http.StatusBadRequest, "device not in install candidate list")
+		return
+	}
+	if globalQueue.hasActiveTarget("install-to-ram") {
+		writeError(w, http.StatusConflict, "install to RAM task is already pending or running")
+		return
+	}
+	if globalQueue.hasActiveTarget("install") {
+		writeError(w, http.StatusConflict, "install task is already pending or running")
+		return
+	}
+	t := &Task{
+		ID:        newJobID("install"),
+		Name:      "Install to Disk",
+		Target:    "install",
+		Priority:  defaultTaskPriority("install", taskParams{}),
+		Status:    TaskPending,
+		CreatedAt: time.Now(),
+		params: taskParams{
+			Device: req.Device,
+		},
+	}
+	globalQueue.enqueue(t)
+	writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID})
+}
+
+// ── Metrics SSE ───────────────────────────────────────────────────────────────

audit/internal/webui/api_metrics_network.go

@@ -0,0 +1,360 @@
+package webui
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"bee/audit/internal/app"
+	"bee/audit/internal/platform"
+)
+
+func (h *handler) handleAPIMetricsLatest(w http.ResponseWriter, r *http.Request) {
+	sample, ok := h.latestMetric()
+	if !ok {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte("{}"))
+		return
+	}
+	b, err := json.Marshal(sample)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_, _ = w.Write(b)
+}
+
+func (h *handler) handleAPIMetricsStream(w http.ResponseWriter, r *http.Request) {
+	if !sseStart(w) {
+		return
+	}
+	ticker := time.NewTicker(1 * time.Second)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-r.Context().Done():
+			return
+		case <-ticker.C:
+			sample, ok := h.latestMetric()
+			if !ok {
+				continue
+			}
+			b, err := json.Marshal(sample)
+			if err != nil {
+				continue
+			}
+			if !sseWrite(w, "metrics", string(b)) {
+				return
+			}
+		}
+	}
+}
+
+// feedRings pushes one sample into all in-memory ring buffers.
+func (h *handler) feedRings(sample platform.LiveMetricSample) {
+	for _, t := range sample.Temps {
+		switch t.Group {
+		case "cpu":
+			h.pushNamedMetricRing(&h.cpuTempRings, t.Name, t.Celsius)
+		case "ambient":
+			h.pushNamedMetricRing(&h.ambientTempRings, t.Name, t.Celsius)
+		}
+	}
+	h.ringPower.push(sample.PowerW)
+	h.ringCPULoad.push(sample.CPULoadPct)
+	h.ringMemLoad.push(sample.MemLoadPct)
+
+	h.ringsMu.Lock()
+	h.pushFanRings(sample.Fans)
+	for _, gpu := range sample.GPUs {
+		idx := gpu.GPUIndex
+		for len(h.gpuRings) <= idx {
+			h.gpuRings = append(h.gpuRings, &gpuRings{
+				Temp:    newMetricsRing(120),
+				Util:    newMetricsRing(120),
+				MemUtil: newMetricsRing(120),
+				Power:   newMetricsRing(120),
+			})
+		}
+		h.gpuRings[idx].Temp.push(gpu.TempC)
+		h.gpuRings[idx].Util.push(gpu.UsagePct)
+		h.gpuRings[idx].MemUtil.push(gpu.MemUsagePct)
+		h.gpuRings[idx].Power.push(gpu.PowerW)
+	}
+	h.ringsMu.Unlock()
+}
+
+func (h *handler) pushFanRings(fans []platform.FanReading) {
+	if len(fans) == 0 && len(h.ringFans) == 0 {
+		return
+	}
+	fanValues := make(map[string]float64, len(fans))
+	for _, fan := range fans {
+		if fan.Name == "" {
+			continue
+		}
+		fanValues[fan.Name] = fan.RPM
+		found := false
+		for i, name := range h.fanNames {
+			if name == fan.Name {
+				found = true
+				if i >= len(h.ringFans) {
+					h.ringFans = append(h.ringFans, newMetricsRing(120))
+				}
+				break
+			}
+		}
+		if !found {
+			h.fanNames = append(h.fanNames, fan.Name)
+			h.ringFans = append(h.ringFans, newMetricsRing(120))
+		}
+	}
+	for i, ring := range h.ringFans {
+		if ring == nil {
+			continue
+		}
+		name := ""
+		if i < len(h.fanNames) {
+			name = h.fanNames[i]
+		}
+		if rpm, ok := fanValues[name]; ok {
+			ring.push(rpm)
+			continue
+		}
+		if last, ok := ring.latest(); ok {
+			ring.push(last)
+			continue
+		}
+		ring.push(0)
+	}
+}
+
+func (h *handler) pushNamedMetricRing(dst *[]*namedMetricsRing, name string, value float64) {
+	if name == "" {
+		return
+	}
+	for _, item := range *dst {
+		if item != nil && item.Name == name && item.Ring != nil {
+			item.Ring.push(value)
+			return
+		}
+	}
+	*dst = append(*dst, &namedMetricsRing{
+		Name: name,
+		Ring: newMetricsRing(120),
+	})
+	(*dst)[len(*dst)-1].Ring.push(value)
+}
+
+// ── Network toggle ────────────────────────────────────────────────────────────
+
+const netRollbackTimeout = 60 * time.Second
+
+func (h *handler) handleAPINetworkToggle(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	var req struct {
+		Iface string `json:"iface"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Iface == "" {
+		writeError(w, http.StatusBadRequest, "iface is required")
+		return
+	}
+
+	wasUp, err := h.opts.App.GetInterfaceState(req.Iface)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	if _, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) {
+		err := h.opts.App.SetInterfaceState(req.Iface, !wasUp)
+		return app.ActionResult{}, err
+	}); err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	newState := "up"
+	if wasUp {
+		newState = "down"
+	}
+	writeJSON(w, map[string]any{
+		"iface":       req.Iface,
+		"new_state":   newState,
+		"rollback_in": int(netRollbackTimeout.Seconds()),
+	})
+}
+
+func (h *handler) applyPendingNetworkChange(apply func() (app.ActionResult, error)) (app.ActionResult, error) {
+	if h.opts.App == nil {
+		return app.ActionResult{}, fmt.Errorf("app not configured")
+	}
+
+	if err := h.rollbackPendingNetworkChange(); err != nil && err.Error() != "no pending network change" {
+		return app.ActionResult{}, err
+	}
+
+	snapshot, err := h.opts.App.CaptureNetworkSnapshot()
+	if err != nil {
+		return app.ActionResult{}, err
+	}
+
+	result, err := apply()
+	if err != nil {
+		return result, err
+	}
+
+	pnc := &pendingNetChange{
+		snapshot: snapshot,
+		deadline: time.Now().Add(netRollbackTimeout),
+	}
+	pnc.timer = time.AfterFunc(netRollbackTimeout, func() {
+		_ = h.opts.App.RestoreNetworkSnapshot(snapshot)
+		h.pendingNetMu.Lock()
+		if h.pendingNet == pnc {
+			h.pendingNet = nil
+		}
+		h.pendingNetMu.Unlock()
+	})
+
+	h.pendingNetMu.Lock()
+	h.pendingNet = pnc
+	h.pendingNetMu.Unlock()
+
+	return result, nil
+}
+
+func (h *handler) hasPendingNetworkChange() bool {
+	h.pendingNetMu.Lock()
+	defer h.pendingNetMu.Unlock()
+	return h.pendingNet != nil
+}
+
+func (h *handler) pendingNetworkRollbackIn() int {
+	h.pendingNetMu.Lock()
+	defer h.pendingNetMu.Unlock()
+	if h.pendingNet == nil {
+		return 0
+	}
+	remaining := int(time.Until(h.pendingNet.deadline).Seconds())
+	if remaining < 1 {
+		return 1
+	}
+	return remaining
+}
+
+func (h *handler) handleAPINetworkConfirm(w http.ResponseWriter, _ *http.Request) {
+	h.pendingNetMu.Lock()
+	pnc := h.pendingNet
+	h.pendingNet = nil
+	h.pendingNetMu.Unlock()
+	if pnc != nil {
+		pnc.mu.Lock()
+		pnc.timer.Stop()
+		pnc.mu.Unlock()
+	}
+	writeJSON(w, map[string]string{"status": "confirmed"})
+}
+
+func (h *handler) handleAPINetworkRollback(w http.ResponseWriter, _ *http.Request) {
+	if err := h.rollbackPendingNetworkChange(); err != nil {
+		if err.Error() == "no pending network change" {
+			writeError(w, http.StatusConflict, err.Error())
+			return
+		}
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, map[string]string{"status": "rolled back"})
+}
+
+func (h *handler) handleAPIBenchmarkResults(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	fmt.Fprint(w, renderBenchmarkResultsCard(h.opts.ExportDir))
+}
+
+// ── Hardware summary / component detail ──────────────────────────────────────
+
+// handleAPIHardwareSummary returns the hardware summary card HTML fragment for
+// htmx polling (hx-get="/api/hardware-summary" hx-swap="outerHTML").
+func (h *handler) handleAPIHardwareSummary(w http.ResponseWriter, _ *http.Request) {
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	w.Header().Set("Cache-Control", "no-store")
+	fmt.Fprint(w, renderHardwareSummaryCard(h.opts))
+}
+
+// handleAPIComponentDetail returns an HTML fragment describing the current and
+// historical status for one component type (cpu, memory, storage, gpu, psu).
+func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Request) {
+	compType := r.PathValue("type")
+	var exact, prefixes []string
+	var title string
+	switch compType {
+	case "cpu":
+		title = "CPU"
+		exact = []string{"cpu:all"}
+	case "memory":
+		title = "Memory"
+		exact = []string{"memory:all"}
+		prefixes = []string{"memory:"}
+	case "storage":
+		title = "Storage"
+		exact = []string{"storage:all"}
+		prefixes = []string{"storage:"}
+	case "gpu":
+		title = "GPU"
+		prefixes = []string{"pcie:gpu:"}
+	case "nic":
+		title = "NIC"
+		prefixes = []string{"pcie:nic:"}
+	case "psu":
+		title = "PSU"
+		prefixes = []string{"psu:"}
+	case "raid":
+		title = "RAID"
+		prefixes = []string{"pcie:raid:"}
+	default:
+		http.NotFound(w, r)
+		return
+	}
+
+	var records []app.ComponentStatusRecord
+	if h.opts.App != nil && h.opts.App.StatusDB != nil {
+		all := h.opts.App.StatusDB.All()
+		records = matchedRecords(all, exact, prefixes)
+	}
+
+	fromInventory := false
+	if len(records) == 0 {
+		if fallback := inventoryFallbackRecords(compType, h.opts); len(fallback) > 0 {
+			records = fallback
+			fromInventory = true
+		}
+	}
+
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	w.Header().Set("Cache-Control", "no-store")
+	fmt.Fprint(w, renderComponentDetail(title, records, fromInventory))
+}
+
+func (h *handler) rollbackPendingNetworkChange() error {
+	h.pendingNetMu.Lock()
+	pnc := h.pendingNet
+	h.pendingNet = nil
+	h.pendingNetMu.Unlock()
+	if pnc == nil {
+		return fmt.Errorf("no pending network change")
+	}
+	pnc.mu.Lock()
+	pnc.timer.Stop()
+	pnc.mu.Unlock()
+	if h.opts.App != nil {
+		return h.opts.App.RestoreNetworkSnapshot(pnc.snapshot)
+	}
+	return nil
+}

audit/internal/webui/api_runs.go

@@ -0,0 +1,430 @@
+package webui
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"bee/audit/internal/platform"
+)
+
+func (h *handler) handleAPIAuditRun(w http.ResponseWriter, _ *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	t := &Task{
+		ID:        newJobID("audit"),
+		Name:      "Audit",
+		Target:    "audit",
+		Priority:  defaultTaskPriority("audit", taskParams{}),
+		Status:    TaskPending,
+		CreatedAt: time.Now(),
+	}
+	globalQueue.enqueue(t)
+	writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID})
+}
+
+func (h *handler) handleAPIAuditStream(w http.ResponseWriter, r *http.Request) {
+	id := r.URL.Query().Get("job_id")
+	if id == "" {
+		id = r.URL.Query().Get("task_id")
+	}
+	// Try task queue first, then legacy job manager
+	if j, ok := globalQueue.findJob(id); ok {
+		streamJob(w, r, j)
+		return
+	}
+	if j, ok := globalJobs.get(id); ok {
+		streamJob(w, r, j)
+		return
+	}
+	http.Error(w, "job not found", http.StatusNotFound)
+}
+
+// ── SAT ───────────────────────────────────────────────────────────────────────
+
+func (h *handler) handleAPISATRun(target string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if h.opts.App == nil {
+			writeError(w, http.StatusServiceUnavailable, "app not configured")
+			return
+		}
+
+		var body struct {
+			Duration           int      `json:"duration"`
+			StressMode         bool     `json:"stress_mode"`
+			GPUIndices         []int    `json:"gpu_indices"`
+			ExcludeGPUIndices  []int    `json:"exclude_gpu_indices"`
+			StaggerGPUStart    bool     `json:"stagger_gpu_start"`
+			ParallelGPUs       bool     `json:"parallel_gpus"`
+			Loader             string   `json:"loader"`
+			Profile            string   `json:"profile"`
+			DisplayName        string   `json:"display_name"`
+			PlatformComponents []string `json:"platform_components"`
+		}
+		if r.Body != nil {
+			if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
+				writeError(w, http.StatusBadRequest, "invalid request body")
+				return
+			}
+		}
+
+		params := taskParams{
+			Duration:           body.Duration,
+			StressMode:         body.StressMode,
+			GPUIndices:         body.GPUIndices,
+			ExcludeGPUIndices:  body.ExcludeGPUIndices,
+			StaggerGPUStart:    body.StaggerGPUStart,
+			ParallelGPUs:       body.ParallelGPUs,
+			Loader:             body.Loader,
+			BurnProfile:        body.Profile,
+			DisplayName:        body.DisplayName,
+			PlatformComponents: body.PlatformComponents,
+		}
+		tasks, err := h.enqueueSATTarget(target, params)
+		if err != nil {
+			writeError(w, http.StatusBadRequest, err.Error())
+			return
+		}
+		writeTaskRunResponse(w, tasks)
+	}
+}
+
+// enqueueSATTarget builds the task set for one SAT target (splitting
+// homogeneous multi-GPU NVIDIA targets as needed) and enqueues it. Shared by
+// the single-target /api/sat/<target>/run endpoints and /api/sat/run-all.
+func (h *handler) enqueueSATTarget(target string, params taskParams) ([]*Task, error) {
+	name := taskDisplayName(target, params.BurnProfile, params.Loader)
+	if strings.TrimSpace(params.DisplayName) != "" {
+		name = params.DisplayName
+	}
+	tasks, err := buildNvidiaTaskSet(target, defaultTaskPriority(target, params), time.Now(), params, name, h.opts.App, "sat-"+target)
+	if err != nil {
+		return nil, err
+	}
+	for _, t := range tasks {
+		globalQueue.enqueue(t)
+	}
+	return tasks, nil
+}
+
+// ── Scenario ─────────────────────────────────────────────────────────────────
+
+func (h *handler) handleAPIScenarioList(w http.ResponseWriter, _ *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	files, err := h.opts.App.ListAvailableScenarios()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	type scenarioFile struct {
+		Name        string `json:"name"`
+		Description string `json:"description"`
+		Device      string `json:"device"`
+	}
+	out := make([]scenarioFile, 0, len(files))
+	for _, f := range files {
+		out = append(out, scenarioFile{Name: f.Name, Description: f.Description, Device: f.Device})
+	}
+	writeJSON(w, out)
+}
+
+func (h *handler) handleAPIScenarioRun(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	var body struct {
+		Name string `json:"name"`
+	}
+	if r.Body != nil {
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
+			writeError(w, http.StatusBadRequest, "invalid request body")
+			return
+		}
+	}
+	name := strings.TrimSpace(body.Name)
+	if name == "" {
+		writeError(w, http.StatusBadRequest, "scenario name is required")
+		return
+	}
+	t := &Task{
+		ID:        newJobID("scenario"),
+		Name:      "Scenario: " + name,
+		Target:    "scenario",
+		Priority:  defaultTaskPriority("scenario", taskParams{}),
+		Status:    TaskPending,
+		CreatedAt: time.Now(),
+	}
+	t.params.ScenarioName = name
+	globalQueue.enqueue(t)
+	writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID})
+}
+
+func (h *handler) handleAPIBenchmarkNvidiaRunKind(target string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if h.opts.App == nil {
+			writeError(w, http.StatusServiceUnavailable, "app not configured")
+			return
+		}
+
+		var body struct {
+			Profile           string `json:"profile"`
+			SizeMB            int    `json:"size_mb"`
+			GPUIndices        []int  `json:"gpu_indices"`
+			ExcludeGPUIndices []int  `json:"exclude_gpu_indices"`
+			RunNCCL           *bool  `json:"run_nccl"`
+			ParallelGPUs      *bool  `json:"parallel_gpus"`
+			RampUp            *bool  `json:"ramp_up"`
+			DisplayName       string `json:"display_name"`
+		}
+		if r.Body != nil {
+			if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
+				writeError(w, http.StatusBadRequest, "invalid request body")
+				return
+			}
+		}
+
+		runNCCL := true
+		if body.RunNCCL != nil {
+			runNCCL = *body.RunNCCL
+		}
+		parallelGPUs := false
+		if body.ParallelGPUs != nil {
+			parallelGPUs = *body.ParallelGPUs
+		}
+		rampUp := false
+		if body.RampUp != nil {
+			rampUp = *body.RampUp
+		}
+		// Build a descriptive base name that includes profile and mode so the task
+		// list is self-explanatory without opening individual task detail pages.
+		profile := strings.TrimSpace(body.Profile)
+		if profile == "" {
+			profile = "standard"
+		}
+		name := taskDisplayName(target, "", "")
+		if strings.TrimSpace(body.DisplayName) != "" {
+			name = body.DisplayName
+		}
+		// Append profile tag.
+		name = fmt.Sprintf("%s · %s", name, profile)
+
+		if target == "nvidia-bench-power" && parallelGPUs {
+			writeError(w, http.StatusBadRequest, "power / thermal fit benchmark uses sequential or ramp-up modes only")
+			return
+		}
+
+		if rampUp && len(body.GPUIndices) > 1 {
+			// Ramp-up mode: RunNvidiaPowerBench internally ramps from 1 to N GPUs
+			// in Phase 2 (one additional GPU per step). A single task with all
+			// selected GPUs is sufficient — spawning N tasks with growing subsets
+			// would repeat all earlier steps redundantly.
+			gpus, err := apiListNvidiaGPUs(h.opts.App)
+			if err != nil {
+				writeError(w, http.StatusBadRequest, err.Error())
+				return
+			}
+			resolved, err := expandSelectedGPUIndices(gpus, body.GPUIndices, body.ExcludeGPUIndices)
+			if err != nil {
+				writeError(w, http.StatusBadRequest, err.Error())
+				return
+			}
+			if len(resolved) < 2 {
+				// Fall through to normal single-task path.
+				rampUp = false
+			} else {
+				now := time.Now()
+				rampRunID := fmt.Sprintf("ramp-%s", now.UTC().Format("20060102-150405"))
+				taskName := fmt.Sprintf("%s · ramp 1–%d · GPU %s", name, len(resolved), formatGPUIndexList(resolved))
+				t := &Task{
+					ID:        newJobID("bee-bench-nvidia"),
+					Name:      taskName,
+					Target:    target,
+					Priority:  defaultTaskPriority(target, taskParams{}),
+					Status:    TaskPending,
+					CreatedAt: now,
+					params: taskParams{
+						GPUIndices:       append([]int(nil), resolved...),
+						SizeMB:           body.SizeMB,
+						BenchmarkProfile: body.Profile,
+						RunNCCL:          runNCCL,
+						ParallelGPUs:     true,
+						RampTotal:        len(resolved),
+						RampRunID:        rampRunID,
+						DisplayName:      taskName,
+					},
+				}
+				globalQueue.enqueue(t)
+				writeTaskRunResponse(w, []*Task{t})
+				return
+			}
+		}
+
+		// For non-ramp tasks append mode tag.
+		if parallelGPUs {
+			name = fmt.Sprintf("%s · parallel", name)
+		} else {
+			name = fmt.Sprintf("%s · sequential", name)
+		}
+
+		params := taskParams{
+			GPUIndices:        body.GPUIndices,
+			ExcludeGPUIndices: body.ExcludeGPUIndices,
+			SizeMB:            body.SizeMB,
+			BenchmarkProfile:  body.Profile,
+			RunNCCL:           runNCCL,
+			ParallelGPUs:      parallelGPUs,
+			DisplayName:       body.DisplayName,
+		}
+		tasks, err := buildNvidiaTaskSet(target, defaultTaskPriority(target, params), time.Now(), params, name, h.opts.App, "bee-bench-nvidia")
+		if err != nil {
+			writeError(w, http.StatusBadRequest, err.Error())
+			return
+		}
+		for _, t := range tasks {
+			globalQueue.enqueue(t)
+		}
+		writeTaskRunResponse(w, tasks)
+	}
+}
+
+func (h *handler) handleAPIBenchmarkAutotuneRun() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if h.opts.App == nil {
+			writeError(w, http.StatusServiceUnavailable, "app not configured")
+			return
+		}
+		var body struct {
+			Profile       string `json:"profile"`
+			BenchmarkKind string `json:"benchmark_kind"`
+			SizeMB        int    `json:"size_mb"`
+		}
+		if r.Body != nil {
+			if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
+				writeError(w, http.StatusBadRequest, "invalid request body")
+				return
+			}
+		}
+		profile := strings.TrimSpace(body.Profile)
+		if profile == "" {
+			profile = "standard"
+		}
+		benchmarkKind := strings.TrimSpace(body.BenchmarkKind)
+		if benchmarkKind == "" {
+			benchmarkKind = "power-fit"
+		}
+		now := time.Now()
+		taskName := fmt.Sprintf("NVIDIA Benchmark Autotune · %s · %s", profile, benchmarkKind)
+		t := &Task{
+			ID:        newJobID("bee-bench-autotune"),
+			Name:      taskName,
+			Target:    "nvidia-bench-autotune",
+			Priority:  defaultTaskPriority("nvidia-bench-autotune", taskParams{}),
+			Status:    TaskPending,
+			CreatedAt: now,
+			params: taskParams{
+				BenchmarkProfile: profile,
+				BenchmarkKind:    benchmarkKind,
+				SizeMB:           body.SizeMB,
+				DisplayName:      taskName,
+			},
+		}
+		globalQueue.enqueue(t)
+		writeTaskRunResponse(w, []*Task{t})
+	}
+}
+
+func (h *handler) handleAPIBenchmarkAutotuneStatus(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	cfg, err := h.opts.App.LoadBenchmarkPowerAutotune()
+	if err != nil {
+		if os.IsNotExist(err) {
+			w.WriteHeader(http.StatusOK)
+			writeJSON(w, map[string]any{
+				"configured": false,
+				"decision":   platform.ResolveSystemPowerDecision(h.opts.ExportDir),
+			})
+			return
+		}
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+	writeJSON(w, map[string]any{
+		"configured": true,
+		"config":     cfg,
+		"decision":   platform.ResolveSystemPowerDecision(h.opts.ExportDir),
+	})
+}
+
+func (h *handler) handleAPIBenchmarkNvidiaRun(w http.ResponseWriter, r *http.Request) {
+	h.handleAPIBenchmarkNvidiaRunKind("nvidia-bench-perf").ServeHTTP(w, r)
+}
+
+func (h *handler) handleAPISATStream(w http.ResponseWriter, r *http.Request) {
+	id := r.URL.Query().Get("job_id")
+	if id == "" {
+		id = r.URL.Query().Get("task_id")
+	}
+	if j, ok := globalQueue.findJob(id); ok {
+		streamJob(w, r, j)
+		return
+	}
+	if j, ok := globalJobs.get(id); ok {
+		streamJob(w, r, j)
+		return
+	}
+	http.Error(w, "job not found", http.StatusNotFound)
+}
+
+func (h *handler) handleAPISATAbort(w http.ResponseWriter, r *http.Request) {
+	id := r.URL.Query().Get("job_id")
+	if id == "" {
+		id = r.URL.Query().Get("task_id")
+	}
+	if t, ok := globalQueue.findByID(id); ok {
+		globalQueue.mu.Lock()
+		switch t.Status {
+		case TaskPending:
+			t.Status = TaskCancelled
+			now := time.Now()
+			t.DoneAt = &now
+		case TaskRunning:
+			if t.job == nil || !t.job.abort() {
+				globalQueue.mu.Unlock()
+				writeJSON(w, map[string]string{"status": "not_running"})
+				return
+			}
+			globalQueue.mu.Unlock()
+			writeJSON(w, map[string]string{"status": "aborting"})
+			return
+		}
+		globalQueue.mu.Unlock()
+		writeJSON(w, map[string]string{"status": "aborted"})
+		return
+	}
+	if j, ok := globalJobs.get(id); ok {
+		if j.abort() {
+			writeJSON(w, map[string]string{"status": "aborted"})
+		} else {
+			writeJSON(w, map[string]string{"status": "not_running"})
+		}
+		return
+	}
+	http.Error(w, "job not found", http.StatusNotFound)
+}
+
+// ── Services ──────────────────────────────────────────────────────────────────

audit/internal/webui/api_sat_runall.go

@@ -0,0 +1,212 @@
+package webui
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"sort"
+	"time"
+
+	"bee/audit/internal/app"
+	"bee/audit/internal/platform"
+	"bee/audit/internal/schema"
+)
+
+// gpuReadyWait bounds how long /api/sat/run-all waits for the GPU driver
+// stack to come up before it plans the GPU tests. The per-GPU GSP firmware
+// boot on a multi-GPU box lags the device nodes by tens of seconds.
+// Overridable from tests.
+var (
+	gpuReadyWait         = 75 * time.Second
+	gpuReadyPollInterval = 3 * time.Second
+	apiRuntimeHealthNow  = func(a *app.App) (schema.RuntimeHealth, error) {
+		return a.RuntimeHealthNow()
+	}
+)
+
+type satRunAllRequest struct {
+	StressMode bool `json:"stress_mode"`
+	// AMDTargets is the operator's AMD check selection (intent). It is still
+	// gated on an AMD GPU actually being present.
+	AMDTargets []string `json:"amd_targets"`
+	// NvidiaGPUIndices optionally narrows the NVIDIA tests to a subset; empty
+	// means "every GPU the backend enumerates".
+	NvidiaGPUIndices []int `json:"nvidia_gpu_indices"`
+}
+
+type satRunAllResponse struct {
+	TaskIDs   []string `json:"task_ids"`
+	TaskCount int      `json:"task_count"`
+	// Notes carries anything the backend decided to skip or override, so the
+	// page can show it without reasoning about hardware itself.
+	Notes []string `json:"notes,omitempty"`
+}
+
+// handleAPISATRunAll plans and enqueues the full validate/check task set
+// server-side. Hardware presence and readiness are decided here, never in the
+// browser: the page sends only operator intent (stress toggle, AMD checkbox
+// selection, an optional GPU subset).
+func (h *handler) handleAPISATRunAll(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	var req satRunAllRequest
+	if r.Body != nil {
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil && !errors.Is(err, io.EOF) {
+			writeError(w, http.StatusBadRequest, "invalid request body")
+			return
+		}
+	}
+	specs, notes := h.planSATRunAll(r.Context(), req)
+
+	var ids []string
+	for _, spec := range specs {
+		tasks, err := h.enqueueSATTarget(spec.target, spec.params)
+		if err != nil {
+			writeError(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+		for _, t := range tasks {
+			if t != nil {
+				ids = append(ids, t.ID)
+			}
+		}
+	}
+	slog.Info("sat run-all planned", "tasks", len(ids), "stress", req.StressMode, "notes", len(notes))
+	writeJSON(w, satRunAllResponse{TaskIDs: ids, TaskCount: len(ids), Notes: notes})
+}
+
+type satRunAllSpec struct {
+	target string
+	params taskParams
+}
+
+func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]satRunAllSpec, []string) {
+	var specs []satRunAllSpec
+	var notes []string
+	skip := func(msg string) {
+		notes = append(notes, msg)
+		slog.Warn("sat run-all: check skipped", "reason", msg)
+	}
+
+	cpuDur := 60
+	if req.StressMode {
+		cpuDur = 1800
+	}
+	specs = append(specs,
+		satRunAllSpec{target: "cpu", params: taskParams{Duration: cpuDur, StressMode: req.StressMode}},
+		satRunAllSpec{target: "memory", params: taskParams{StressMode: req.StressMode}},
+		satRunAllSpec{target: "storage", params: taskParams{StressMode: req.StressMode}},
+		satRunAllSpec{target: "pcie-link", params: taskParams{}},
+	)
+
+	if h.opts.App.TPMPresent() {
+		specs = append(specs, satRunAllSpec{target: "tpm", params: taskParams{}})
+	} else {
+		skip("TPM: no TPM device on this host; check skipped")
+	}
+
+	gp := h.opts.App.DetectGPUPresence()
+
+	if gp.Nvidia || gp.NvidiaInitializing {
+		// nvidia-config only collects inventory and NVLink state; safe to run
+		// even while the compute stack is still coming up.
+		specs = append(specs, satRunAllSpec{target: "nvidia-config", params: taskParams{}})
+
+		health, gpus, ready := h.waitForNvidiaReady(ctx)
+		switch {
+		case health.NvidiaGSPMode == "gsp-stuck":
+			skip("NVIDIA: GSP firmware init is stuck; reboot with GSP=off. GPU compute/interconnect/bandwidth tests skipped")
+		case !ready:
+			skip("NVIDIA: nvidia-smi did not enumerate a GPU after " + gpuReadyWait.String() +
+				". GPU compute/interconnect/bandwidth tests skipped; see the GPU Config check")
+		default:
+			indices := make([]int, 0, len(gpus))
+			for _, g := range gpus {
+				indices = append(indices, g.Index)
+			}
+			if len(req.NvidiaGPUIndices) > 0 {
+				indices = intersectSortedInts(indices, req.NvidiaGPUIndices)
+			}
+			if len(indices) == 0 {
+				skip("NVIDIA: driver ready but no GPU to test (enumeration empty, or the requested subset matched nothing); GPU tests skipped")
+				break
+			}
+			if !health.CUDAReady {
+				notes = append(notes, "NVIDIA: CUDA runtime not confirmed ready; GPU tests queued anyway")
+			}
+			gpuTargets := []string{"nvidia", "nvidia-interconnect", "nvidia-bandwidth", "nvidia-pcie-bandwidth"}
+			if req.StressMode {
+				// Stress tier adds the targeted dcgmi diag load tests.
+				gpuTargets = append(gpuTargets, "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse")
+			}
+			for _, target := range gpuTargets {
+				specs = append(specs, satRunAllSpec{
+					target: target,
+					params: taskParams{GPUIndices: append([]int(nil), indices...), StressMode: req.StressMode},
+				})
+			}
+		}
+	}
+
+	if gp.AMD {
+		for _, target := range req.AMDTargets {
+			switch target {
+			case "amd", "amd-mem", "amd-bandwidth":
+				specs = append(specs, satRunAllSpec{target: target, params: taskParams{StressMode: req.StressMode}})
+			}
+		}
+	}
+
+	return specs, notes
+}
+
+// waitForNvidiaReady waits until the same nvidia-smi query used by
+// ListNvidiaGPUs returns at least one GPU. A loaded kernel module alone is
+// not evidence that NVIDIA user-space commands can address a GPU yet.
+func (h *handler) waitForNvidiaReady(ctx context.Context) (schema.RuntimeHealth, []platform.NvidiaGPU, bool) {
+	deadline := time.Now().Add(gpuReadyWait)
+	var last schema.RuntimeHealth
+	for {
+		health, err := apiRuntimeHealthNow(h.opts.App)
+		if err == nil {
+			last = health
+			if health.NvidiaGSPMode == "gsp-stuck" {
+				return health, nil, false
+			}
+		}
+		gpus, listErr := apiListNvidiaGPUs(h.opts.App)
+		if listErr == nil && len(gpus) > 0 {
+			return last, gpus, true
+		}
+		if time.Now().After(deadline) || ctx.Err() != nil {
+			return last, nil, false
+		}
+		select {
+		case <-ctx.Done():
+			return last, nil, false
+		case <-time.After(gpuReadyPollInterval):
+		}
+	}
+}
+
+// intersectSortedInts returns the ascending-sorted values present in both a
+// and b.
+func intersectSortedInts(a, b []int) []int {
+	set := make(map[int]struct{}, len(b))
+	for _, v := range b {
+		set[v] = struct{}{}
+	}
+	out := make([]int, 0, len(a))
+	for _, v := range a {
+		if _, ok := set[v]; ok {
+			out = append(out, v)
+		}
+	}
+	sort.Ints(out)
+	return out
+}

audit/internal/webui/api_sat_runall_test.go

@@ -0,0 +1,90 @@
+package webui
+
+import (
+	"context"
+	"reflect"
+	"testing"
+	"time"
+
+	"bee/audit/internal/app"
+	"bee/audit/internal/platform"
+	"bee/audit/internal/schema"
+)
+
+func TestIntersectSortedInts(t *testing.T) {
+	got := intersectSortedInts([]int{0, 1, 2, 3, 4, 5, 6, 7}, []int{5, 1, 9})
+	if want := []int{1, 5}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("intersectSortedInts=%v want %v", got, want)
+	}
+	if got := intersectSortedInts([]int{0, 1}, []int{9}); len(got) != 0 {
+		t.Fatalf("want empty, got %v", got)
+	}
+}
+
+func TestWaitForNvidiaReadyDoesNotTreatLoadedDriverAsEnumeration(t *testing.T) {
+	oldWait, oldPoll := gpuReadyWait, gpuReadyPollInterval
+	oldHealth, oldList := apiRuntimeHealthNow, apiListNvidiaGPUs
+	gpuReadyWait, gpuReadyPollInterval = 5*time.Millisecond, time.Millisecond
+	apiRuntimeHealthNow = func(*app.App) (schema.RuntimeHealth, error) {
+		return schema.RuntimeHealth{DriverReady: true}, nil
+	}
+	apiListNvidiaGPUs = func(*app.App) ([]platform.NvidiaGPU, error) { return nil, nil }
+	t.Cleanup(func() {
+		gpuReadyWait, gpuReadyPollInterval = oldWait, oldPoll
+		apiRuntimeHealthNow, apiListNvidiaGPUs = oldHealth, oldList
+	})
+
+	_, gpus, ready := (&handler{opts: HandlerOptions{App: app.New(&platform.System{})}}).waitForNvidiaReady(context.Background())
+	if ready || len(gpus) != 0 {
+		t.Fatalf("ready=%v gpus=%v; loaded module without enumerated GPUs must not be ready", ready, gpus)
+	}
+}
+
+func TestWaitForNvidiaReadyReturnsFreshEnumeration(t *testing.T) {
+	oldWait, oldPoll := gpuReadyWait, gpuReadyPollInterval
+	oldHealth, oldList := apiRuntimeHealthNow, apiListNvidiaGPUs
+	gpuReadyWait, gpuReadyPollInterval = 20*time.Millisecond, time.Millisecond
+	apiRuntimeHealthNow = func(*app.App) (schema.RuntimeHealth, error) {
+		return schema.RuntimeHealth{DriverReady: true, CUDAReady: true}, nil
+	}
+	calls := 0
+	apiListNvidiaGPUs = func(*app.App) ([]platform.NvidiaGPU, error) {
+		calls++
+		if calls < 2 {
+			return nil, nil
+		}
+		return []platform.NvidiaGPU{{Index: 3}}, nil
+	}
+	t.Cleanup(func() {
+		gpuReadyWait, gpuReadyPollInterval = oldWait, oldPoll
+		apiRuntimeHealthNow, apiListNvidiaGPUs = oldHealth, oldList
+	})
+
+	health, gpus, ready := (&handler{opts: HandlerOptions{App: app.New(&platform.System{})}}).waitForNvidiaReady(context.Background())
+	if !ready || !health.CUDAReady || len(gpus) != 1 || gpus[0].Index != 3 {
+		t.Fatalf("ready=%v health=%+v gpus=%v", ready, health, gpus)
+	}
+}
+
+// On a host with no GPU and no TPM the plan is the base checks plus a note
+// that TPM was skipped, and no GPU tasks are invented.
+func TestPlanSATRunAllNoAcceleratorNoTPM(t *testing.T) {
+	oldWait := gpuReadyWait
+	gpuReadyWait = 10 * time.Millisecond
+	t.Cleanup(func() { gpuReadyWait = oldWait })
+
+	h := &handler{opts: HandlerOptions{App: app.New(&platform.System{})}}
+	specs, notes := h.planSATRunAll(context.Background(), satRunAllRequest{})
+
+	var targets []string
+	for _, s := range specs {
+		targets = append(targets, s.target)
+	}
+	want := []string{"cpu", "memory", "storage", "pcie-link"}
+	if !reflect.DeepEqual(targets, want) {
+		t.Fatalf("targets=%v want %v", targets, want)
+	}
+	if len(notes) == 0 {
+		t.Fatalf("expected a note about TPM being skipped")
+	}
+}

audit/internal/webui/api_services_system.go

@@ -0,0 +1,260 @@
+package webui
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"bee/audit/internal/app"
+	"bee/audit/internal/platform"
+)
+
+func (h *handler) handleAPIServicesList(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	names, err := h.opts.App.ListBeeServices()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	type serviceInfo struct {
+		Name  string `json:"name"`
+		State string `json:"state"`
+		Body  string `json:"body"`
+	}
+	result := make([]serviceInfo, 0, len(names))
+	for _, name := range names {
+		state := h.opts.App.ServiceState(name)
+		body, _ := h.opts.App.ServiceStatus(name)
+		result = append(result, serviceInfo{Name: name, State: state, Body: body})
+	}
+	writeJSON(w, result)
+}
+
+func (h *handler) handleAPIServicesAction(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	var req struct {
+		Name   string `json:"name"`
+		Action string `json:"action"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid request body")
+		return
+	}
+	var action platform.ServiceAction
+	switch req.Action {
+	case "start":
+		action = platform.ServiceStart
+	case "stop":
+		action = platform.ServiceStop
+	case "restart":
+		action = platform.ServiceRestart
+	default:
+		writeError(w, http.StatusBadRequest, "action must be start|stop|restart")
+		return
+	}
+	result, err := h.opts.App.ServiceActionResult(req.Name, action)
+	status := "ok"
+	if err != nil {
+		status = "error"
+	}
+	// Always return 200 with output so the frontend can display the actual
+	// systemctl error message instead of a generic "exit status 1".
+	writeJSON(w, map[string]string{"status": status, "output": result.Body})
+}
+
+// ── Network ───────────────────────────────────────────────────────────────────
+
+func (h *handler) handleAPINetworkStatus(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	ifaces, err := h.opts.App.ListInterfaces()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, map[string]any{
+		"interfaces":     ifaces,
+		"default_route":  h.opts.App.DefaultRoute(),
+		"pending_change": h.hasPendingNetworkChange(),
+		"rollback_in":    h.pendingNetworkRollbackIn(),
+	})
+}
+
+func (h *handler) handleAPINetworkDHCP(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	var req struct {
+		Interface string `json:"interface"`
+	}
+	_ = json.NewDecoder(r.Body).Decode(&req)
+
+	result, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) {
+		if req.Interface == "" || req.Interface == "all" {
+			return h.opts.App.DHCPAllResult()
+		}
+		return h.opts.App.DHCPOneResult(req.Interface)
+	})
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, map[string]any{
+		"status":      "ok",
+		"output":      result.Body,
+		"rollback_in": int(netRollbackTimeout.Seconds()),
+	})
+}
+
+func (h *handler) handleAPINetworkStatic(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	var req struct {
+		Interface string   `json:"interface"`
+		Address   string   `json:"address"`
+		Prefix    string   `json:"prefix"`
+		Gateway   string   `json:"gateway"`
+		DNS       []string `json:"dns"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid request body")
+		return
+	}
+	cfg := platform.StaticIPv4Config{
+		Interface: req.Interface,
+		Address:   req.Address,
+		Prefix:    req.Prefix,
+		Gateway:   req.Gateway,
+		DNS:       req.DNS,
+	}
+	result, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) {
+		return h.opts.App.SetStaticIPv4Result(cfg)
+	})
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, map[string]any{
+		"status":      "ok",
+		"output":      result.Body,
+		"rollback_in": int(netRollbackTimeout.Seconds()),
+	})
+}
+
+// ── Export ────────────────────────────────────────────────────────────────────
+
+func (h *handler) handleAPIExportList(w http.ResponseWriter, r *http.Request) {
+	entries, err := listExportFiles(h.opts.ExportDir)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, entries)
+}
+
+func (h *handler) handleAPIExportUSBTargets(w http.ResponseWriter, _ *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	targets, err := h.opts.App.ListRemovableTargets()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if targets == nil {
+		targets = []platform.RemovableTarget{}
+	}
+	writeJSON(w, targets)
+}
+
+func (h *handler) handleAPIBlackboxStatus(w http.ResponseWriter, _ *http.Request) {
+	state, err := app.ReadBlackboxState(filepath.Join(h.opts.ExportDir, "blackbox-state.json"))
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			writeJSON(w, app.BlackboxState{Status: "disabled", Targets: []app.BlackboxTargetStatus{}})
+			return
+		}
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if state.Targets == nil {
+		state.Targets = []app.BlackboxTargetStatus{}
+	}
+	writeJSON(w, state)
+}
+
+func (h *handler) handleAPIBlackboxEnable(w http.ResponseWriter, r *http.Request) {
+	if h.opts.App == nil {
+		writeError(w, http.StatusServiceUnavailable, "app not configured")
+		return
+	}
+	var target platform.RemovableTarget
+	if err := json.NewDecoder(r.Body).Decode(&target); err != nil || strings.TrimSpace(target.Device) == "" {
+		writeError(w, http.StatusBadRequest, "device is required")
+		return
+	}
+	targets, err := h.opts.App.ListRemovableTargets()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	allowed := false
+	for _, candidate := range targets {
+		if candidate.Device == target.Device {
+			target = candidate
+			allowed = true
+			break
+		}
+	}
+	if !allowed {
+		writeError(w, http.StatusBadRequest, "device not in removable target list")
+		return
+	}
+	marker, err := app.EnableBlackboxTarget(target)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, map[string]any{
+		"status":        "ok",
+		"message":       "Black-box marker written.",
+		"enrollment_id": marker.EnrollmentID,
+	})
+}
+
+func (h *handler) handleAPIBlackboxDisable(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		Device       string `json:"device"`
+		EnrollmentID string `json:"enrollment_id"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid request body")
+		return
+	}
+	if err := app.DisableBlackboxTarget(req.Device, req.EnrollmentID); err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			writeError(w, http.StatusNotFound, "black-box target not found")
+			return
+		}
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, map[string]string{"status": "ok", "message": "Black-box marker removed."})
+}
+
+// ── GPU presence ──────────────────────────────────────────────────────────────

audit/internal/webui/huawei_elabel.go

@@ -33,19 +33,19 @@ type huaweiFieldDef struct {
 }
 
 var huaweiElabelDefs = []huaweiFieldDef{
-	{"Device Name",            "DeviceName",            0x00, 0x06, 0x01, ""},
-	{"Device Serial Number",   "DeviceSerialNumber",    0x00, 0x06, 0x03, ""},
-	{"Product Name",           "ProductName",           0x00, 0x03, 0x01, ""},
-	{"Product Serial Number",  "ProductSerialNumber",   0x00, 0x03, 0x04, ""},
-	{"Product Asset Tag",      "ProductAssetTag",       0x00, 0x03, 0x05, ""},
-	{"Product Manufacturer",   "ProductManufacturer",   0x00, 0x03, 0x00, ""},
+	{"Device Name", "DeviceName", 0x00, 0x06, 0x01, ""},
+	{"Device Serial Number", "DeviceSerialNumber", 0x00, 0x06, 0x03, ""},
+	{"Product Name", "ProductName", 0x00, 0x03, 0x01, ""},
+	{"Product Serial Number", "ProductSerialNumber", 0x00, 0x03, 0x04, ""},
+	{"Product Asset Tag", "ProductAssetTag", 0x00, 0x03, 0x05, ""},
+	{"Product Manufacturer", "ProductManufacturer", 0x00, 0x03, 0x00, ""},
 	{"Mainboard Manufacturer", "MainboardManufacturer", 0x00, 0x02, 0x01, ""},
-	{"Board Product Name",     "BoardProductName",      0x00, 0x02, 0x02, ""},
-	{"Chassis Part Number",    "ChassisPartnumber",     0x00, 0x01, 0x01, ""},
-	{"Chassis Type",           "ChassisType",           0x00, 0x01, 0x00, "chassis-type"},
-	{"IO Chassis Serial",      "IOChassisSerialNumber", 0x01, 0x03, 0x04, ""},
-	{"IO Chassis Asset Tag",   "IOChassisAssetTag",     0x01, 0x03, 0x05, ""},
-	{"GUID",                   "GUID",                  0x00, 0x00, 0x00, "guid"},
+	{"Board Product Name", "BoardProductName", 0x00, 0x02, 0x02, ""},
+	{"Chassis Part Number", "ChassisPartnumber", 0x00, 0x01, 0x01, ""},
+	{"Chassis Type", "ChassisType", 0x00, 0x01, 0x00, "chassis-type"},
+	{"IO Chassis Serial", "IOChassisSerialNumber", 0x01, 0x03, 0x04, ""},
+	{"IO Chassis Asset Tag", "IOChassisAssetTag", 0x01, 0x03, 0x05, ""},
+	{"GUID", "GUID", 0x00, 0x00, 0x00, "guid"},
 }
 
 // huaweiGetRaw reads a string elabel field via OEM IPMI raw command.

audit/internal/webui/layout.go

@@ -100,7 +100,7 @@ tbody tr:hover td{background:rgba(0,0,0,.03)}
 func layoutNav(active string, buildLabel string) string {
 	type navItem struct {
 		id, label, href string
-		sep              bool
+		sep             bool
 	}
 	items := []navItem{
 		{id: "dashboard", label: "Dashboard", href: "/"},

audit/internal/webui/metrics_charts_http.go

@@ -0,0 +1,665 @@
+package webui
+
+import (
+	"fmt"
+	"html"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+
+	"bee/audit/internal/platform"
+)
+
+func (h *handler) handleMetricsChartSVG(w http.ResponseWriter, r *http.Request) {
+	path := strings.TrimPrefix(r.URL.Path, "/api/metrics/chart/")
+	path = strings.TrimSuffix(path, ".svg")
+
+	if h.metricsDB == nil {
+		http.Error(w, "metrics database not available", http.StatusServiceUnavailable)
+		return
+	}
+	samples, err := h.metricsDB.LoadAll()
+	if err != nil || len(samples) == 0 {
+		http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable)
+		return
+	}
+	timeline := metricsTimelineSegments(samples, time.Now())
+	if idx, sub, ok := parseGPUChartPath(path); ok && sub == "overview" {
+		var overviewOk bool
+		var buf []byte
+		buf, overviewOk, err = renderGPUOverviewChartSVG(idx, samples, timeline)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		if !overviewOk {
+			http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable)
+			return
+		}
+		w.Header().Set("Content-Type", "image/svg+xml")
+		w.Header().Set("Cache-Control", "no-store")
+		_, _ = w.Write(buf)
+		return
+	}
+	datasets, names, labels, title, yMin, yMax, stacked, ok := chartDataFromSamples(path, samples)
+	if !ok {
+		http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable)
+		return
+	}
+
+	var buf []byte
+	if stacked {
+		buf, err = renderStackedMetricChartSVG(
+			title,
+			labels,
+			sampleTimes(samples),
+			datasets,
+			names,
+			yMax,
+			chartCanvasHeightForPath(path, len(names)),
+			timeline,
+		)
+	} else {
+		buf, err = renderMetricChartSVG(
+			title,
+			labels,
+			sampleTimes(samples),
+			datasets,
+			names,
+			yMin,
+			yMax,
+			chartCanvasHeightForPath(path, len(names)),
+			timeline,
+		)
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "image/svg+xml")
+	w.Header().Set("Cache-Control", "no-store")
+	_, _ = w.Write(buf)
+}
+
+func chartDataFromSamples(path string, samples []platform.LiveMetricSample) (datasets [][]float64, names []string, labels []string, title string, yMin, yMax *float64, stacked bool, ok bool) {
+	labels = sampleTimeLabels(samples)
+
+	switch {
+	case path == "server-load":
+		title = "CPU / Memory Load"
+		cpu := make([]float64, len(samples))
+		mem := make([]float64, len(samples))
+		for i, s := range samples {
+			cpu[i] = s.CPULoadPct
+			mem[i] = s.MemLoadPct
+		}
+		datasets = [][]float64{cpu, mem}
+		names = []string{"CPU Load %", "Mem Load %"}
+		yMin = floatPtr(0)
+		yMax = floatPtr(100)
+
+	case path == "server-temp", path == "server-temp-cpu":
+		title = "CPU Temperature"
+		datasets, names = namedTempDatasets(samples, "cpu")
+		yMin = floatPtr(0)
+		yMax = autoMax120(datasets...)
+
+	case path == "server-temp-gpu":
+		title = "GPU Temperature"
+		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.TempC })
+		yMin = floatPtr(0)
+		yMax = autoMax120(datasets...)
+
+	case path == "server-temp-ambient":
+		title = "Ambient / Other Sensors"
+		datasets, names = namedTempDatasets(samples, "ambient")
+		yMin = floatPtr(0)
+		yMax = autoMax120(datasets...)
+
+	case path == "server-power":
+		title = "System Power"
+		power := make([]float64, len(samples))
+		label := "Power W"
+		for i, s := range samples {
+			power[i] = s.PowerW
+			if strings.TrimSpace(s.PowerSource) != "" {
+				label = fmt.Sprintf("Power W · %s", s.PowerSource)
+				if strings.TrimSpace(s.PowerMode) != "" {
+					label += fmt.Sprintf(" (%s)", s.PowerMode)
+				}
+			}
+		}
+		power = normalizePowerSeries(power)
+		datasets = [][]float64{power}
+		names = []string{label}
+		yMin = floatPtr(0)
+		yMax = autoMax120(power)
+
+	case path == "server-fans":
+		title = "Fan RPM"
+		datasets, names = namedFanDatasets(samples)
+		yMin, yMax = autoBounds120(datasets...)
+
+	case path == "gpu-all-load":
+		title = "GPU Compute Load"
+		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.UsagePct })
+		yMin = floatPtr(0)
+		yMax = floatPtr(100)
+
+	case path == "gpu-all-memload":
+		title = "GPU Memory Load"
+		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.MemUsagePct })
+		yMin = floatPtr(0)
+		yMax = floatPtr(100)
+
+	case path == "gpu-all-power":
+		title = "GPU Power"
+		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.PowerW })
+		yMin, yMax = autoBounds120(datasets...)
+
+	case path == "gpu-all-temp":
+		title = "GPU Temperature"
+		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.TempC })
+		yMin = floatPtr(0)
+		yMax = autoMax120(datasets...)
+
+	case path == "gpu-all-clock":
+		title = "GPU Core Clock"
+		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.ClockMHz })
+		yMin, yMax = autoBounds120(datasets...)
+
+	case path == "gpu-all-memclock":
+		title = "GPU Memory Clock"
+		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.MemClockMHz })
+		yMin, yMax = autoBounds120(datasets...)
+
+	case strings.HasPrefix(path, "gpu/"):
+		idx, sub, ok := parseGPUChartPath(path)
+		if !ok {
+			return nil, nil, nil, "", nil, nil, false, false
+		}
+		switch sub {
+		case "load":
+			title = gpuDisplayLabel(idx) + " Load"
+			util := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.UsagePct })
+			mem := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.MemUsagePct })
+			if util == nil && mem == nil {
+				return nil, nil, nil, "", nil, nil, false, false
+			}
+			datasets = [][]float64{coalesceDataset(util, len(samples)), coalesceDataset(mem, len(samples))}
+			names = []string{"Load %", "Mem %"}
+			yMin = floatPtr(0)
+			yMax = floatPtr(100)
+		case "temp":
+			title = gpuDisplayLabel(idx) + " Temperature"
+			temp := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.TempC })
+			if temp == nil {
+				return nil, nil, nil, "", nil, nil, false, false
+			}
+			datasets = [][]float64{temp}
+			names = []string{"Temp °C"}
+			yMin = floatPtr(0)
+			yMax = autoMax120(temp)
+		case "clock":
+			title = gpuDisplayLabel(idx) + " Core Clock"
+			clock := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.ClockMHz })
+			if clock == nil {
+				return nil, nil, nil, "", nil, nil, false, false
+			}
+			datasets = [][]float64{clock}
+			names = []string{"Core Clock MHz"}
+			yMin, yMax = autoBounds120(clock)
+		case "memclock":
+			title = gpuDisplayLabel(idx) + " Memory Clock"
+			clock := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.MemClockMHz })
+			if clock == nil {
+				return nil, nil, nil, "", nil, nil, false, false
+			}
+			datasets = [][]float64{clock}
+			names = []string{"Memory Clock MHz"}
+			yMin, yMax = autoBounds120(clock)
+		default:
+			title = gpuDisplayLabel(idx) + " Power"
+			power := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.PowerW })
+			if power == nil {
+				return nil, nil, nil, "", nil, nil, false, false
+			}
+			datasets = [][]float64{power}
+			names = []string{"Power W"}
+			yMin, yMax = autoBounds120(power)
+		}
+
+	default:
+		return nil, nil, nil, "", nil, nil, false, false
+	}
+
+	return datasets, names, labels, title, yMin, yMax, stacked, len(datasets) > 0
+}
+
+func parseGPUChartPath(path string) (idx int, sub string, ok bool) {
+	if !strings.HasPrefix(path, "gpu/") {
+		return 0, "", false
+	}
+	rest := strings.TrimPrefix(path, "gpu/")
+	if rest == "" {
+		return 0, "", false
+	}
+	sub = ""
+	if i := strings.LastIndex(rest, "-"); i > 0 {
+		sub = rest[i+1:]
+		rest = rest[:i]
+	}
+	n, err := fmt.Sscanf(rest, "%d", &idx)
+	if err != nil || n != 1 {
+		return 0, "", false
+	}
+	return idx, sub, true
+}
+
+func sampleTimeLabels(samples []platform.LiveMetricSample) []string {
+	labels := make([]string, len(samples))
+	if len(samples) == 0 {
+		return labels
+	}
+	times := make([]time.Time, len(samples))
+	for i, s := range samples {
+		times[i] = s.Timestamp
+	}
+	sameDay := timestampsSameLocalDay(times)
+	for i, s := range samples {
+		labels[i] = formatTimelineLabel(s.Timestamp.Local(), sameDay)
+	}
+	return labels
+}
+
+func namedTempDatasets(samples []platform.LiveMetricSample, group string) ([][]float64, []string) {
+	seen := map[string]bool{}
+	var names []string
+	for _, s := range samples {
+		for _, t := range s.Temps {
+			if t.Group == group && !seen[t.Name] {
+				seen[t.Name] = true
+				names = append(names, t.Name)
+			}
+		}
+	}
+	sort.Strings(names)
+	datasets := make([][]float64, 0, len(names))
+	for _, name := range names {
+		ds := make([]float64, len(samples))
+		for i, s := range samples {
+			for _, t := range s.Temps {
+				if t.Group == group && t.Name == name {
+					ds[i] = t.Celsius
+					break
+				}
+			}
+		}
+		datasets = append(datasets, ds)
+	}
+	return datasets, names
+}
+
+func namedFanDatasets(samples []platform.LiveMetricSample) ([][]float64, []string) {
+	seen := map[string]bool{}
+	var names []string
+	for _, s := range samples {
+		for _, f := range s.Fans {
+			if !seen[f.Name] {
+				seen[f.Name] = true
+				names = append(names, f.Name)
+			}
+		}
+	}
+	sort.Strings(names)
+	datasets := make([][]float64, 0, len(names))
+	for _, name := range names {
+		ds := make([]float64, len(samples))
+		for i, s := range samples {
+			for _, f := range s.Fans {
+				if f.Name == name {
+					ds[i] = f.RPM
+					break
+				}
+			}
+		}
+		datasets = append(datasets, normalizeFanSeries(ds))
+	}
+	return datasets, names
+}
+
+func gpuDatasets(samples []platform.LiveMetricSample, pick func(platform.GPUMetricRow) float64) ([][]float64, []string) {
+	seen := map[int]bool{}
+	var indices []int
+	for _, s := range samples {
+		for _, g := range s.GPUs {
+			if !seen[g.GPUIndex] {
+				seen[g.GPUIndex] = true
+				indices = append(indices, g.GPUIndex)
+			}
+		}
+	}
+	sort.Ints(indices)
+	datasets := make([][]float64, 0, len(indices))
+	names := make([]string, 0, len(indices))
+	for _, idx := range indices {
+		ds := gpuDatasetByIndex(samples, idx, pick)
+		if ds == nil {
+			continue
+		}
+		datasets = append(datasets, ds)
+		names = append(names, gpuDisplayLabel(idx))
+	}
+	return datasets, names
+}
+
+func gpuDatasetByIndex(samples []platform.LiveMetricSample, idx int, pick func(platform.GPUMetricRow) float64) []float64 {
+	found := false
+	ds := make([]float64, len(samples))
+	for i, s := range samples {
+		for _, g := range s.GPUs {
+			if g.GPUIndex == idx {
+				ds[i] = pick(g)
+				found = true
+				break
+			}
+		}
+	}
+	if !found {
+		return nil
+	}
+	return ds
+}
+
+func coalesceDataset(ds []float64, n int) []float64 {
+	if ds != nil {
+		return ds
+	}
+	return make([]float64, n)
+}
+
+func normalizePowerSeries(ds []float64) []float64 {
+	if len(ds) == 0 {
+		return nil
+	}
+	out := make([]float64, len(ds))
+	copy(out, ds)
+	last := 0.0
+	haveLast := false
+	for i, v := range out {
+		if v > 0 {
+			last = v
+			haveLast = true
+			continue
+		}
+		if haveLast {
+			out[i] = last
+		}
+	}
+	return out
+}
+
+// psuSlotsFromSamples returns the sorted list of PSU slot numbers seen across samples.
+func psuSlotsFromSamples(samples []platform.LiveMetricSample) []int {
+	seen := map[int]struct{}{}
+	for _, s := range samples {
+		for _, p := range s.PSUs {
+			seen[p.Slot] = struct{}{}
+		}
+	}
+	slots := make([]int, 0, len(seen))
+	for s := range seen {
+		slots = append(slots, s)
+	}
+	sort.Ints(slots)
+	return slots
+}
+
+// psuStackedTotal returns the point-by-point sum of all PSU datasets (for scale calculation).
+func psuStackedTotal(datasets [][]float64) []float64 {
+	if len(datasets) == 0 {
+		return nil
+	}
+	n := len(datasets[0])
+	total := make([]float64, n)
+	for _, ds := range datasets {
+		for i, v := range ds {
+			total[i] += v
+		}
+	}
+	return total
+}
+
+func normalizeFanSeries(ds []float64) []float64 {
+	if len(ds) == 0 {
+		return nil
+	}
+	out := make([]float64, len(ds))
+	var lastPositive float64
+	for i, v := range ds {
+		if v > 0 {
+			lastPositive = v
+			out[i] = v
+			continue
+		}
+		if lastPositive > 0 {
+			out[i] = lastPositive
+			continue
+		}
+		out[i] = 0
+	}
+	return out
+}
+
+// floatPtr returns a pointer to a float64 value.
+func floatPtr(v float64) *float64 { return &v }
+
+// autoMax120 returns 0→max+20% Y-axis max across all datasets.
+func autoMax120(datasets ...[]float64) *float64 {
+	max := 0.0
+	for _, ds := range datasets {
+		for _, v := range ds {
+			if v > max {
+				max = v
+			}
+		}
+	}
+	if max == 0 {
+		return nil // let library auto-scale
+	}
+	v := max * 1.2
+	return &v
+}
+
+func autoBounds120(datasets ...[]float64) (*float64, *float64) {
+	min := 0.0
+	max := 0.0
+	first := true
+	for _, ds := range datasets {
+		for _, v := range ds {
+			if first {
+				min, max = v, v
+				first = false
+				continue
+			}
+			if v < min {
+				min = v
+			}
+			if v > max {
+				max = v
+			}
+		}
+	}
+	if first {
+		return nil, nil
+	}
+	if max <= 0 {
+		return floatPtr(0), nil
+	}
+	span := max - min
+	if span <= 0 {
+		span = max * 0.1
+		if span <= 0 {
+			span = 1
+		}
+	}
+	pad := span * 0.2
+	low := min - pad
+	if low < 0 {
+		low = 0
+	}
+	high := max + pad
+	return floatPtr(low), floatPtr(high)
+}
+
+func gpuChartLabelIndices(total, target int) []int {
+	if total <= 0 {
+		return nil
+	}
+	if total == 1 {
+		return []int{0}
+	}
+	step := total / target
+	if step < 1 {
+		step = 1
+	}
+	var indices []int
+	for i := 0; i < total; i += step {
+		indices = append(indices, i)
+	}
+	if indices[len(indices)-1] != total-1 {
+		indices = append(indices, total-1)
+	}
+	return indices
+}
+
+func chartCanvasHeightForPath(path string, seriesCount int) int {
+	height := chartCanvasHeight(seriesCount)
+	if isGPUChartPath(path) {
+		return height * 2
+	}
+	return height
+}
+
+func isGPUChartPath(path string) bool {
+	return strings.HasPrefix(path, "gpu-all-") || strings.HasPrefix(path, "gpu/")
+}
+
+func chartLegendVisible(seriesCount int) bool {
+	return seriesCount <= 8
+}
+
+func chartCanvasHeight(seriesCount int) int {
+	if chartLegendVisible(seriesCount) {
+		return 360
+	}
+	return 288
+}
+
+// globalStats returns min, average, and max across all values in all datasets.
+func globalStats(datasets [][]float64) (mn, avg, mx float64) {
+	var sum float64
+	var count int
+	first := true
+	for _, ds := range datasets {
+		for _, v := range ds {
+			if first {
+				mn, mx = v, v
+				first = false
+			}
+			if v < mn {
+				mn = v
+			}
+			if v > mx {
+				mx = v
+			}
+			sum += v
+			count++
+		}
+	}
+	if count > 0 {
+		avg = sum / float64(count)
+	}
+	return mn, avg, mx
+}
+
+func sanitizeChartText(s string) string {
+	if s == "" {
+		return ""
+	}
+	return html.EscapeString(strings.Map(func(r rune) rune {
+		if r < 0x20 && r != '\t' && r != '\n' && r != '\r' {
+			return -1
+		}
+		return r
+	}, s))
+}
+
+func snapshotFanRings(rings []*metricsRing, fanNames []string) ([][]float64, []string, []string) {
+	var datasets [][]float64
+	var names []string
+	var labels []string
+	for i, ring := range rings {
+		if ring == nil {
+			continue
+		}
+		vals, l := ring.snapshot()
+		datasets = append(datasets, normalizeFanSeries(vals))
+		name := "Fan"
+		if i < len(fanNames) {
+			name = fanNames[i]
+		}
+		names = append(names, name+" RPM")
+		if len(labels) == 0 {
+			labels = l
+		}
+	}
+	return datasets, names, labels
+}
+
+func chartLegendNumber(v float64) string {
+	neg := v < 0
+	if v < 0 {
+		v = -v
+	}
+	var out string
+	switch {
+	case v >= 10000:
+		out = fmt.Sprintf("%dk", int((v+500)/1000))
+	case v >= 1000:
+		s := fmt.Sprintf("%.2f", v/1000)
+		s = strings.TrimRight(strings.TrimRight(s, "0"), ".")
+		out = strings.ReplaceAll(s, ".", ",") + "k"
+	default:
+		out = fmt.Sprintf("%.0f", v)
+	}
+	if neg {
+		return "-" + out
+	}
+	return out
+}
+
+func chartYAxisNumber(v float64) string {
+	neg := v < 0
+	if neg {
+		v = -v
+	}
+	var out string
+	switch {
+	case v >= 10000:
+		out = fmt.Sprintf("%dк", int((v+500)/1000))
+	case v >= 1000:
+		// Use one decimal place so ticks like 1400, 1600, 1800 read as
+		// "1,4к", "1,6к", "1,8к" instead of the ambiguous "1к"/"2к".
+		s := fmt.Sprintf("%.1f", v/1000)
+		s = strings.TrimRight(strings.TrimRight(s, "0"), ".")
+		out = strings.ReplaceAll(s, ".", ",") + "к"
+	default:
+		out = fmt.Sprintf("%.0f", v)
+	}
+	if neg {
+		return "-" + out
+	}
+	return out
+}

audit/internal/webui/metricsdb.go

@@ -455,7 +455,3 @@ func (m *MetricsDB) ExportCSV(w io.Writer) error {
 	cw.Flush()
 	return cw.Error()
 }
-
-func nullFloat(v float64) sql.NullFloat64 {
-	return sql.NullFloat64{Float64: v, Valid: true}
-}

audit/internal/webui/page_benchmark.go

@@ -613,5 +613,5 @@ func renderPowerBenchmarkResultsCard(exportDir string) string {
 }
 
 // renderSpeed and renderEndurance are legacy wrappers; canonical page is 5. Benchmark at /benchmark.
-func renderSpeed(opts HandlerOptions) string    { return renderBenchmark(opts) }
+func renderSpeed(opts HandlerOptions) string     { return renderBenchmark(opts) }
 func renderEndurance(opts HandlerOptions) string { return renderBenchmark(opts) }

audit/internal/webui/page_topo.go

@@ -11,7 +11,6 @@ import (
 	"strconv"
 	"strings"
 
-	"bee/audit/internal/app"
 	"bee/audit/internal/schema"
 )
 
@@ -201,6 +200,7 @@ func parseDIMMBankLocatorNodes(raw string) map[string]int {
 //  2. A node number from the Bank Locator via parseDIMMBankLocatorNodes,
 //     e.g. Locator "DIMM000(A)" whose Bank Locator is
 //     "_Node1_Channel0_Dimm0" -> 1.
+//
 // ok=false means neither pattern matched, so this DIMM can't be confidently
 // attached to a CPU column and falls back to the unattached Memory row.
 func dimmRawNode(mem schema.HardwareMemory, bankNodeByLocator map[string]int) (int, bool) {
@@ -582,772 +582,3 @@ type topoEdge struct {
 	x1, y1, x2, y2 int
 	color          string
 }
-
-func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string {
-	// A GPU hardware fault that needs a physical reboot (Xid 79 "fallen off
-	// the bus", Xid 154 "Node Reboot Required" — see gpuNeedsPhysicalReboot)
-	// isn't visible in dev.Status: the collector only sets that from PCIe
-	// link-speed checks, not from SAT/kmsg results. Without this, a GPU that
-	// dropped off the bus mid-test still renders green here even though the
-	// Hardware Summary card is showing a critical banner for it.
-	gpuHardwareFault := false
-	if db, err := app.OpenComponentStatusDB(filepath.Join(exportDir, "component-status.json")); err == nil {
-		if _, needsReboot := gpuNeedsPhysicalReboot(matchedRecords(db.All(), nil, []string{"pcie:gpu:"})); needsReboot {
-			gpuHardwareFault = true
-		}
-	}
-
-	socketIdx := buildSocketIndex(hw.CPUs)
-	numCols := len(hw.CPUs)
-	if numCols == 0 {
-		numCols = 1
-	}
-	unknownCol := numCols // extra trailing column for unmatched devices
-
-	// Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node
-	// types) into columns by NUMA node, falling back to the "unknown" bucket.
-	type placedDevice struct {
-		dev  schema.HardwarePCIeDevice
-		kind string // "gpu", "nic", "raid"
-		col  int
-		bdf  string
-	}
-	var placed []placedDevice
-	for _, dev := range hw.PCIeDevices {
-		kind := pcieDeviceKind(dev)
-		if kind == "" {
-			continue
-		}
-		col := unknownCol
-		if dev.NUMANode != nil {
-			if ci, ok := socketIdx[*dev.NUMANode]; ok {
-				col = ci
-			}
-		}
-		bdf := ""
-		if dev.Slot != nil {
-			bdf = normalizeTopoBDF(*dev.Slot)
-		} else if dev.BDF != nil {
-			bdf = normalizeTopoBDF(*dev.BDF)
-		}
-		placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf})
-	}
-	hasUnknownCol := false
-	for _, p := range placed {
-		if p.col == unknownCol {
-			hasUnknownCol = true
-			break
-		}
-	}
-	totalCols := numCols
-	if hasUnknownCol {
-		totalCols++
-	}
-
-	// GPU index<->BDF map + pairwise NVLink adjacency, read from the
-	// persisted techdump captured during the last audit cycle, best-effort:
-	// if the dump is missing (older audit, no NVIDIA GPUs), this is simply
-	// skipped. Used only to detect the cross-NUMA-bonded-pair anomaly below;
-	// the pairwise links themselves are drawn in the separate NVLink
-	// Topology card, since grouping same-kind/same-column devices into one
-	// stacked card here leaves no single per-GPU anchor point to draw a
-	// pairwise connector to or from.
-	bdfToIndex, _ := readNVIDIAIndexByBDF(exportDir)
-	var pairs []gpuPairLink
-	if topoMatrix, err := readGPUTopologyMatrix(exportDir); err == nil {
-		pairs = parseGPUPairAdjacency(topoMatrix)
-	}
-	gpuNUMAByIndex := map[int]*int{}
-	gpuBDFByIndex := map[int]string{}
-	for _, p := range placed {
-		if p.kind != "gpu" || p.bdf == "" {
-			continue
-		}
-		if idx, ok := bdfToIndex[p.bdf]; ok {
-			gpuNUMAByIndex[idx] = p.dev.NUMANode
-			gpuBDFByIndex[idx] = p.bdf
-		}
-	}
-	// A bonded pair spanning two different NUMA nodes is treated as an
-	// anomaly (not a neutral fact) per project decision: a bonded pair is
-	// expected to sit on one NUMA node, so a cross-NUMA bond escalates both
-	// GPUs' effective severity to at least Warning, regardless of their own
-	// reported SAT status.
-	crossNUMAWarnBDF := map[string]bool{}
-	for _, pair := range pairs {
-		numaA, okA := gpuNUMAByIndex[pair.GPUA]
-		numaB, okB := gpuNUMAByIndex[pair.GPUB]
-		if !okA || !okB || numaA == nil || numaB == nil || *numaA == *numaB {
-			continue
-		}
-		crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUA]] = true
-		crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUB]] = true
-	}
-
-	kindOrder := []string{"gpu", "nic", "raid"}
-	kindLabel := map[string]string{"gpu": "GPU", "nic": "NIC", "raid": "RAID"}
-
-	// Attach memory DIMMs to their CPU column too, the same way GPU/NIC/RAID
-	// PCIe devices are attached via NUMANode — memory has no NUMANode field
-	// in the schema, so this reads the DIMM's own Locator/Bank Locator
-	// strings instead (see dimmRawNode). DIMMs that can't be confidently
-	// attached fall back to the unattached "Memory" row below the diagram,
-	// same as before this existed.
-	memBankNodes := map[string]int{}
-	if raw, err := readTopoTechDump(exportDir, "dmidecode-type17.txt"); err == nil {
-		memBankNodes = parseDIMMBankLocatorNodes(raw)
-	}
-	memCol := make([]int, len(hw.Memory))
-	memMatched := make([]bool, len(hw.Memory))
-	var memRawNodes []int
-	for i, m := range hw.Memory {
-		if node, ok := dimmRawNode(m, memBankNodes); ok {
-			memCol[i] = node
-			memMatched[i] = true
-			memRawNodes = append(memRawNodes, node)
-		}
-	}
-	memColIdx := buildMemoryColumnIndex(memRawNodes)
-	for i := range hw.Memory {
-		if !memMatched[i] {
-			continue
-		}
-		col := memColIdx[memCol[i]]
-		if col >= numCols {
-			memMatched[i] = false
-			continue
-		}
-		memCol[i] = col
-	}
-
-	var boxes []topoBox
-	var pcieEdges []topoEdge
-
-	for col := 0; col < totalCols; col++ {
-		colX := (col+1)*24 + col*topoColWidth
-		if col < len(hw.CPUs) {
-			cpu := hw.CPUs[col]
-			model := ""
-			if cpu.Model != nil {
-				model = *cpu.Model
-			}
-			socket := col
-			if cpu.Socket != nil {
-				socket = *cpu.Socket
-			}
-			var tally topoStatusTally
-			tally.add(classifyTopoSeverity(cpu.Status))
-			fill, stroke, text := topoSeverityColors(tally.worst())
-			boxes = append(boxes, topoBox{
-				x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight,
-				topoCardInfo: topoCardInfo{
-					label: fmt.Sprintf("CPU %d", socket), sublabel: model, count: 1,
-					statusLine: tally.line(),
-					fillVar:    fill, strokeVar: stroke, textVar: text,
-					detailType: "cpu",
-				},
-			})
-		}
-
-		y := topoTopMargin + topoBoxHeight + topoDeviceGap*2
-
-		// Memory goes first in the chain, directly under the CPU box: DIMMs
-		// are wired straight to the socket's memory controller, not reached
-		// over PCIe like the GPU/NIC/RAID chain below it.
-		var memGroup []schema.HardwareMemory
-		for i, m := range hw.Memory {
-			if memMatched[i] && memCol[i] == col {
-				memGroup = append(memGroup, m)
-			}
-		}
-		if len(memGroup) > 0 {
-			var tally topoStatusTally
-			sizeGB := 0
-			for _, m := range memGroup {
-				tally.add(classifyTopoSeverity(m.Status))
-				if m.SizeMB != nil {
-					sizeGB += *m.SizeMB / 1024
-				}
-			}
-			fill, stroke, text := topoSeverityColors(tally.worst())
-			sublabel := ""
-			if sizeGB > 0 {
-				sublabel = fmt.Sprintf("%d GB total", sizeGB)
-			}
-			stackLayers := topoStackLayers(len(memGroup))
-			boxes = append(boxes, topoBox{
-				x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
-				topoCardInfo: topoCardInfo{
-					label: "Memory", sublabel: sublabel, count: len(memGroup),
-					statusLine: tally.line(),
-					fillVar:    fill, strokeVar: stroke, textVar: text,
-					detailType: "memory",
-				},
-			})
-			if col < len(hw.CPUs) {
-				pcieEdges = append(pcieEdges, topoEdge{
-					x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
-					x2: colX + topoBoxWidth/2, y2: y,
-					color: "var(--ok-fg)",
-				})
-			}
-			y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
-		}
-
-		for _, kind := range kindOrder {
-			var group []placedDevice
-			for _, p := range placed {
-				if p.col == col && p.kind == kind {
-					group = append(group, p)
-				}
-			}
-			if len(group) == 0 {
-				continue
-			}
-
-			var tally topoStatusTally
-			model := ""
-			edgeColor := "var(--ok-fg)"
-			for i, p := range group {
-				sev := classifyTopoSeverity(p.dev.Status)
-				if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 {
-					sev = 2
-				}
-				if kind == "gpu" && gpuHardwareFault && sev < 3 {
-					sev = 3
-				}
-				tally.add(sev)
-				if i == 0 && p.dev.Model != nil {
-					model = *p.dev.Model
-				}
-				if topoEdgeColorVar(p.dev) == "var(--warn-fg)" {
-					edgeColor = "var(--warn-fg)"
-				}
-			}
-			fill, stroke, text := topoSeverityColors(tally.worst())
-			stackLayers := topoStackLayers(len(group))
-			boxes = append(boxes, topoBox{
-				x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
-				topoCardInfo: topoCardInfo{
-					label: kindLabel[kind], sublabel: model, count: len(group),
-					statusLine: tally.line(),
-					fillVar:    fill, strokeVar: stroke, textVar: text,
-					detailType: kind,
-				},
-			})
-
-			if col < len(hw.CPUs) {
-				pcieEdges = append(pcieEdges, topoEdge{
-					x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
-					x2: colX + topoBoxWidth/2, y2: y,
-					color: edgeColor,
-				})
-			}
-
-			y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
-		}
-	}
-
-	maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2
-	for _, box := range boxes {
-		bottom := box.y + box.h + topoStackLayers(box.count)*topoStackStep
-		if bottom > maxDeviceY {
-			maxDeviceY = bottom
-		}
-	}
-
-	svgHeight := maxDeviceY + 24
-	svgWidth := totalCols*topoColWidth + 48
-
-	var b strings.Builder
-	// Wrapped in its own horizontally-scrolling container (matching the
-	// overflow-x:auto convention used for wide tables elsewhere in webui)
-	// rather than max-width:100% — squashing a node/edge diagram to fit a
-	// narrow viewport makes labels and badges illegible, whereas scrolling
-	// keeps the diagram readable at its natural size on any screen width.
-	b.WriteString(`<div style="overflow-x:auto">`)
-	fmt.Fprintf(&b, `<svg width="%d" height="%d" viewBox="0 0 %d %d">`+"\n", svgWidth, svgHeight, svgWidth, svgHeight)
-	for _, e := range pcieEdges {
-		fmt.Fprintf(&b, `<line x1="%d" y1="%d" x2="%d" y2="%d" style="stroke:%s;stroke-width:2"/>`+"\n", e.x1, e.y1, e.x2, e.y2, e.color)
-	}
-	for _, box := range boxes {
-		writeTopoBoxSVG(&b, box)
-	}
-	b.WriteString(`</svg></div>`)
-
-	// Firmware (BMC/BIOS/...) and PSUs have no PCIe/CPU affinity to anchor
-	// them to a column, and there can be an arbitrary number of any of them
-	// — so unlike the diagram above, they're plain flex-wrap HTML below the
-	// SVG rather than absolutely-positioned SVG boxes. A fixed-size SVG
-	// canvas has no way to wrap overflow onto a new row, which is exactly
-	// what caused these to pile up and overlap once a board had more
-	// PSUs/firmware records than fit in one fixed-width row.
-	//
-	// Memory DIMMs that were matched to a CPU column above already got a
-	// box in the SVG diagram; only DIMMs that couldn't be attached to a
-	// column (see memMatched above) fall back to this row.
-	var unmatchedMem []schema.HardwareMemory
-	for i, m := range hw.Memory {
-		if !memMatched[i] {
-			unmatchedMem = append(unmatchedMem, m)
-		}
-	}
-	if len(unmatchedMem) > 0 {
-		var tally topoStatusTally
-		for _, m := range unmatchedMem {
-			tally.add(classifyTopoSeverity(m.Status))
-		}
-		fill, stroke, text := topoSeverityColors(tally.worst())
-		sizeGB := 0
-		for _, m := range unmatchedMem {
-			if m.SizeMB != nil {
-				sizeGB += *m.SizeMB / 1024
-			}
-		}
-		sublabel := ""
-		if sizeGB > 0 {
-			sublabel = fmt.Sprintf("%d GB total", sizeGB)
-		}
-		b.WriteString(renderTopoFlexRow("Memory", []topoCardInfo{{
-			label: "Memory", sublabel: sublabel, count: len(unmatchedMem),
-			statusLine: tally.line(),
-			fillVar:    fill, strokeVar: stroke, textVar: text,
-			detailType: "memory",
-		}}))
-	}
-
-	var firmwareItems []topoCardInfo
-	for _, rec := range hw.Firmware {
-		// Firmware records carry no per-item status in the schema (they are
-		// identity, not health, facts), so each stays a neutral, uncolored
-		// card rather than forcing a fake "Unknown" status line.
-		fillVar, strokeVar, textVar := topoSeverityColors(0)
-		firmwareItems = append(firmwareItems, topoCardInfo{
-			label: rec.DeviceName, sublabel: "fw " + rec.Version, count: 1,
-			fillVar: fillVar, strokeVar: strokeVar, textVar: textVar,
-		})
-	}
-	b.WriteString(renderTopoFlexRow("Firmware", firmwareItems))
-
-	if len(hw.PowerSupplies) > 0 {
-		var tally topoStatusTally
-		watt := 0
-		for _, psu := range hw.PowerSupplies {
-			tally.add(classifyTopoSeverity(psu.Status))
-			if psu.WattageW != nil {
-				watt = *psu.WattageW
-			}
-		}
-		fill, stroke, text := topoSeverityColors(tally.worst())
-		sublabel := ""
-		if watt > 0 {
-			sublabel = fmt.Sprintf("%dW each", watt)
-		}
-		b.WriteString(renderTopoFlexRow("Power Supplies", []topoCardInfo{{
-			label: "Power Supplies", sublabel: sublabel, count: len(hw.PowerSupplies),
-			statusLine: tally.line(),
-			fillVar:    fill, strokeVar: stroke, textVar: text,
-			detailType: "psu",
-		}}))
-	}
-
-	return topoCard("Topology", b.String())
-}
-
-// renderTopoFlexRow renders a labeled, wrapping row of component cards.
-// Returns "" if items is empty (e.g. no PSU data in this audit).
-func renderTopoFlexRow(title string, items []topoCardInfo) string {
-	if len(items) == 0 {
-		return ""
-	}
-	var b strings.Builder
-	fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
-		html.EscapeString(title))
-	b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:10px">`)
-	for _, item := range items {
-		onclick := ""
-		cursor := "default"
-		if item.detailType != "" {
-			onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, item.detailType)
-			cursor = "pointer"
-		}
-		stackLayers := topoStackLayers(item.count)
-		// Extra right/bottom padding on the wrapper reserves room for the
-		// backing layers of the stack effect so they aren't clipped by the
-		// flex container.
-		fmt.Fprintf(&b, `<div style="position:relative;padding-right:%dpx;padding-bottom:%dpx">`,
-			stackLayers*topoStackStep, stackLayers*topoStackStep)
-		for i := stackLayers; i >= 1; i-- {
-			off := i * topoStackStep
-			fmt.Fprintf(&b, `<div style="position:absolute;top:%dpx;left:%dpx;right:0;bottom:0;border-radius:6px;background:%s;border:1px solid %s;opacity:.55"></div>`,
-				off, off, item.fillVar, item.strokeVar)
-		}
-		fmt.Fprintf(&b, `<div%s style="position:relative;cursor:%s;min-width:160px;padding:10px 12px;border-radius:6px;background:%s;border:1px solid %s;color:%s">`,
-			onclick, cursor, item.fillVar, item.strokeVar, item.textVar)
-		label := item.label
-		if item.count > 1 {
-			label = fmt.Sprintf("%s ×%d", item.label, item.count)
-		}
-		fmt.Fprintf(&b, `<div style="font-size:13px;font-weight:700">%s</div>`, html.EscapeString(label))
-		if item.sublabel != "" {
-			fmt.Fprintf(&b, `<div style="font-size:11px;opacity:.85">%s</div>`, html.EscapeString(item.sublabel))
-		}
-		if item.statusLine != "" {
-			fmt.Fprintf(&b, `<div style="font-size:11px;font-weight:600;margin-top:4px">%s</div>`, html.EscapeString(item.statusLine))
-		}
-		b.WriteString(`</div></div>`)
-	}
-	b.WriteString(`</div>`)
-	return b.String()
-}
-
-func writeTopoBoxSVG(b *strings.Builder, box topoBox) {
-	onclick := ""
-	cursor := "default"
-	if box.detailType != "" {
-		onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType)
-		cursor = "pointer"
-	}
-	fmt.Fprintf(b, `<g%s style="cursor:%s">`, onclick, cursor)
-
-	// Stack-of-cards effect: faint offset rects behind the front card when
-	// this box represents more than one physical component (e.g. 4 GPUs in
-	// one NUMA column), so a group reads as "a deck of N" rather than a
-	// single item. Peeks toward the bottom-right, into space already
-	// reserved between this box and the next one in the column.
-	for i := topoStackLayers(box.count); i >= 1; i-- {
-		off := i * topoStackStep
-		fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s;opacity:.55"/>`+"\n",
-			box.x+off, box.y+off, box.w, box.h, box.fillVar, box.strokeVar)
-	}
-
-	fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s"/>`+"\n",
-		box.x, box.y, box.w, box.h, box.fillVar, box.strokeVar)
-
-	label := box.label
-	if box.count > 1 {
-		label = fmt.Sprintf("%s ×%d", box.label, box.count)
-	}
-	fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:13px;font-weight:700">%s</text>`+"\n",
-		box.x+10, box.y+20, box.textVar, html.EscapeString(label))
-	if box.sublabel != "" {
-		fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:11px;opacity:.85">%s</text>`+"\n",
-			box.x+10, box.y+36, box.textVar, html.EscapeString(truncateTopoLabel(box.sublabel, 26)))
-	}
-	if box.statusLine != "" {
-		fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:10px;font-weight:600">%s</text>`+"\n",
-			box.x+10, box.y+box.h-10, box.textVar, html.EscapeString(box.statusLine))
-	}
-	b.WriteString(`</g>` + "\n")
-}
-
-func truncateTopoLabel(s string, max int) string {
-	if len(s) <= max {
-		return s
-	}
-	if max <= 1 {
-		return s[:max]
-	}
-	return s[:max-1] + "…"
-}
-
-// ---------------------------------------------------------------------------
-// Separate NVLink topology card (read from techdump, not written to any
-// ingest contract)
-// ---------------------------------------------------------------------------
-
-type topoNVLinkPort struct {
-	Index          int
-	Active         bool
-	SpeedGBs       *float64
-	ReplayErrors   int64
-	RecoveryErrors int64
-	CRCErrors      int64
-}
-
-var (
-	topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
-	topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
-	topoNVLinkInactiveRe  = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
-	topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
-)
-
-func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) {
-	raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-status.txt")
-	if err != nil {
-		return nil, err
-	}
-	return parseTopoNVLinkStatus(raw), nil
-}
-
-func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort {
-	result := map[int][]topoNVLinkPort{}
-	currentGPU := -1
-	for _, line := range strings.Split(raw, "\n") {
-		trimmed := strings.TrimSpace(line)
-		if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
-			currentGPU, _ = strconv.Atoi(m[1])
-			continue
-		}
-		if currentGPU < 0 {
-			continue
-		}
-		if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
-			idx, _ := strconv.Atoi(m[1])
-			result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false})
-			continue
-		}
-		if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
-			idx, _ := strconv.Atoi(m[1])
-			port := topoNVLinkPort{Index: idx, Active: true}
-			if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
-				port.SpeedGBs = &speed
-			}
-			result[currentGPU] = append(result[currentGPU], port)
-		}
-	}
-	return result
-}
-
-func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) {
-	raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-errors.txt")
-	if err != nil {
-		return nil, err
-	}
-	return parseTopoNVLinkErrors(raw), nil
-}
-
-// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc].
-func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 {
-	result := map[int]map[int][3]int64{}
-	currentGPU := -1
-	for _, line := range strings.Split(raw, "\n") {
-		trimmed := strings.TrimSpace(line)
-		if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
-			currentGPU, _ = strconv.Atoi(m[1])
-			continue
-		}
-		if currentGPU < 0 {
-			continue
-		}
-		m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed)
-		if m == nil {
-			continue
-		}
-		linkIdx, _ := strconv.Atoi(m[1])
-		count, _ := strconv.ParseInt(m[3], 10, 64)
-		if result[currentGPU] == nil {
-			result[currentGPU] = map[int][3]int64{}
-		}
-		c := result[currentGPU][linkIdx]
-		switch m[2] {
-		case "Replay":
-			c[0] = count
-		case "Recovery":
-			c[1] = count
-		case "CRC":
-			c[2] = count
-		}
-		result[currentGPU][linkIdx] = c
-	}
-	return result
-}
-
-// renderTopoNVLinkCard renders the separate NVLink topology card. Returns ""
-// if there are fewer than 2 NVIDIA GPUs, or the nvidia-smi nvlink techdump
-// wasn't captured (older audit, or nvidia-smi unavailable on that run).
-func renderTopoNVLinkCard(hw schema.HardwareSnapshot, exportDir string) string {
-	gpuCount := 0
-	for _, dev := range hw.PCIeDevices {
-		if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) {
-			gpuCount++
-		}
-	}
-	if gpuCount < 2 {
-		return ""
-	}
-
-	status, err := readTopoNVLinkStatus(exportDir)
-	if err != nil || len(status) == 0 {
-		return topoCard("NVLink Topology", `<span class="badge badge-unknown">nvidia-smi nvlink data unavailable</span>`)
-	}
-	errors, _ := readTopoNVLinkErrors(exportDir)
-
-	topoMatrix, _ := readGPUTopologyMatrix(exportDir)
-	pairs := parseGPUPairAdjacency(topoMatrix)
-
-	var bodyB strings.Builder
-	if gpuCount <= 4 && len(pairs) > 0 {
-		// Small GPU count: per-pair box+line with per-link detail.
-		for _, pair := range pairs {
-			activeCount, total, hasError := 0, 0, false
-			for _, port := range status[pair.GPUA] {
-				total++
-				if port.Active {
-					activeCount++
-				}
-			}
-			for _, counters := range errors[pair.GPUA] {
-				if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
-					hasError = true
-				}
-			}
-			color := "var(--ok-fg)"
-			switch {
-			case hasError:
-				color = "var(--crit-fg)"
-			case total > 0 && activeCount < total:
-				color = "var(--warn-fg)"
-			}
-			fmt.Fprintf(&bodyB, `<div style="display:flex;align-items:center;gap:12px;margin-bottom:10px">`+
-				`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
-				`<div style="flex:1;height:2px;background:%s"></div>`+
-				`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
-				`<div style="font-size:12px;color:var(--muted)">%d/%d links active%s</div>`+
-				`</div>`,
-				pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError))
-		}
-	} else if len(pairs) > 0 {
-		// Larger GPU counts (NVSwitch fabric): aggregate pair table instead of
-		// an unreadable all-to-all graph.
-		bodyB.WriteString(`<table><thead><tr><th>GPU A</th><th>GPU B</th><th>NVLinks</th></tr></thead><tbody>`)
-		for _, pair := range pairs {
-			fmt.Fprintf(&bodyB, `<tr><td>GPU %d</td><td>GPU %d</td><td>%d</td></tr>`, pair.GPUA, pair.GPUB, pair.NVLinks)
-		}
-		bodyB.WriteString(`</tbody></table>`)
-	} else {
-		bodyB.WriteString(`<span class="badge badge-unknown">No NVLink-bonded GPU pairs found</span>`)
-	}
-
-	return topoCard("NVLink Topology", bodyB.String())
-}
-
-func errNoteSuffix(hasError bool) string {
-	if hasError {
-		return " — errors detected"
-	}
-	return ""
-}
-
-// ---------------------------------------------------------------------------
-// Inventory fallback for the component-detail modal
-//
-// handleAPIComponentDetail normally sources records from app.ComponentStatusDB,
-// which only gains entries once something has actually written a status
-// observation (SAT run, watchdog tick, ...). On a freshly booted host that
-// hasn't run SAT yet, StatusDB can be entirely empty for a component type even
-// though the /topo card for it already shows "N OK" — that card reads
-// schema.HardwareComponentStatus.Status straight from the audit snapshot.
-// inventoryFallbackRecords bridges that gap by building synthetic records
-// from the same snapshot/classifiers the topology card uses, so the two
-// views never disagree about how many devices exist or their status.
-// ---------------------------------------------------------------------------
-
-// topoSeverityStatus renders classifyTopoSeverity's rank back into the status
-// string vocabulary renderComponentDetail/chipLetterClass expect ("OK",
-// "Warning", "Critical", "Unknown") — kept in lockstep with classifyTopoSeverity
-// so a device the topo card counts as "OK" is never shown here as "Unknown".
-func topoSeverityStatus(status *string) string {
-	switch classifyTopoSeverity(status) {
-	case 3:
-		return "Critical"
-	case 2:
-		return "Warning"
-	case 1:
-		return "OK"
-	default:
-		return "Unknown"
-	}
-}
-
-// pcieDeviceKind classifies a PCIe device the same way renderTopoMainDiagram
-// does, returning "" for devices that aren't GPU/NIC/RAID.
-func pcieDeviceKind(dev schema.HardwarePCIeDevice) string {
-	switch {
-	case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass):
-		return "gpu"
-	case isNICDeviceClassDev(dev):
-		return "nic"
-	case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass):
-		return "raid"
-	default:
-		return ""
-	}
-}
-
-// pcieDeviceKey builds a stable, human-readable component key for a PCIe
-// device: "<kind>:<bdf>" when a slot/BDF is known, else "<kind>:<index>".
-func pcieDeviceKey(kind string, index int, dev schema.HardwarePCIeDevice) string {
-	bdf := ""
-	if dev.Slot != nil {
-		bdf = normalizeTopoBDF(*dev.Slot)
-	} else if dev.BDF != nil {
-		bdf = normalizeTopoBDF(*dev.BDF)
-	}
-	if bdf != "" {
-		return kind + ":" + bdf
-	}
-	return fmt.Sprintf("%s:%d", kind, index)
-}
-
-// inventoryFallbackRecords builds ComponentStatusRecord entries straight from
-// the audit inventory (bee-audit.json) for the given component type, used
-// when ComponentStatusDB has no matching records yet. Records carry only
-// ComponentKey/Status — no LastCheckedAt/History — so renderComponentDetail
-// renders them without a "checked at" timestamp or sparkline.
-func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.ComponentStatusRecord {
-	data, err := loadSnapshot(opts.AuditPath)
-	if err != nil {
-		return nil
-	}
-	var ingest schema.HardwareIngestRequest
-	if err := json.Unmarshal(data, &ingest); err != nil {
-		return nil
-	}
-	hw := ingest.Hardware
-
-	var records []app.ComponentStatusRecord
-	switch compType {
-	case "cpu":
-		for i, cpu := range hw.CPUs {
-			key := fmt.Sprintf("cpu:%d", i)
-			if cpu.Socket != nil {
-				key = fmt.Sprintf("cpu:socket%d", *cpu.Socket)
-			}
-			records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(cpu.Status)})
-		}
-	case "memory":
-		for i, m := range hw.Memory {
-			key := fmt.Sprintf("memory:%d", i)
-			if m.Slot != nil && strings.TrimSpace(*m.Slot) != "" {
-				key = "memory:" + strings.TrimSpace(*m.Slot)
-			}
-			records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(m.Status)})
-		}
-	case "storage":
-		for i, s := range hw.Storage {
-			key := fmt.Sprintf("storage:%d", i)
-			if s.Slot != nil && strings.TrimSpace(*s.Slot) != "" {
-				key = "storage:" + strings.TrimSpace(*s.Slot)
-			}
-			records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(s.Status)})
-		}
-	case "psu":
-		for i, p := range hw.PowerSupplies {
-			key := fmt.Sprintf("psu:%d", i)
-			if p.Slot != nil && strings.TrimSpace(*p.Slot) != "" {
-				key = "psu:" + strings.TrimSpace(*p.Slot)
-			}
-			records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)})
-		}
-	case "gpu", "nic", "raid":
-		for i, dev := range hw.PCIeDevices {
-			if pcieDeviceKind(dev) != compType {
-				continue
-			}
-			key := pcieDeviceKey(compType, i, dev)
-			records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(dev.Status)})
-		}
-	}
-	return records
-}

audit/internal/webui/page_topo_diagram.go

@@ -0,0 +1,783 @@
+package webui
+
+import (
+	"encoding/json"
+	"fmt"
+	"html"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"bee/audit/internal/app"
+	"bee/audit/internal/schema"
+)
+
+func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string {
+	// A GPU hardware fault that needs a physical reboot (Xid 79 "fallen off
+	// the bus", Xid 154 "Node Reboot Required" — see gpuNeedsPhysicalReboot)
+	// isn't visible in dev.Status: the collector only sets that from PCIe
+	// link-speed checks, not from SAT/kmsg results. Without this, a GPU that
+	// dropped off the bus mid-test still renders green here even though the
+	// Hardware Summary card is showing a critical banner for it.
+	gpuHardwareFault := false
+	if db, err := app.OpenComponentStatusDB(filepath.Join(exportDir, "component-status.json")); err == nil {
+		if _, needsReboot := gpuNeedsPhysicalReboot(matchedRecords(db.All(), nil, []string{"pcie:gpu:"})); needsReboot {
+			gpuHardwareFault = true
+		}
+	}
+
+	socketIdx := buildSocketIndex(hw.CPUs)
+	numCols := len(hw.CPUs)
+	if numCols == 0 {
+		numCols = 1
+	}
+	unknownCol := numCols // extra trailing column for unmatched devices
+
+	// Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node
+	// types) into columns by NUMA node, falling back to the "unknown" bucket.
+	type placedDevice struct {
+		dev  schema.HardwarePCIeDevice
+		kind string // "gpu", "nic", "raid"
+		col  int
+		bdf  string
+	}
+	var placed []placedDevice
+	for _, dev := range hw.PCIeDevices {
+		kind := pcieDeviceKind(dev)
+		if kind == "" {
+			continue
+		}
+		col := unknownCol
+		if dev.NUMANode != nil {
+			if ci, ok := socketIdx[*dev.NUMANode]; ok {
+				col = ci
+			}
+		}
+		bdf := ""
+		if dev.Slot != nil {
+			bdf = normalizeTopoBDF(*dev.Slot)
+		} else if dev.BDF != nil {
+			bdf = normalizeTopoBDF(*dev.BDF)
+		}
+		placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf})
+	}
+	hasUnknownCol := false
+	for _, p := range placed {
+		if p.col == unknownCol {
+			hasUnknownCol = true
+			break
+		}
+	}
+	totalCols := numCols
+	if hasUnknownCol {
+		totalCols++
+	}
+
+	// GPU index<->BDF map + pairwise NVLink adjacency, read from the
+	// persisted techdump captured during the last audit cycle, best-effort:
+	// if the dump is missing (older audit, no NVIDIA GPUs), this is simply
+	// skipped. Used only to detect the cross-NUMA-bonded-pair anomaly below;
+	// the pairwise links themselves are drawn in the separate NVLink
+	// Topology card, since grouping same-kind/same-column devices into one
+	// stacked card here leaves no single per-GPU anchor point to draw a
+	// pairwise connector to or from.
+	bdfToIndex, _ := readNVIDIAIndexByBDF(exportDir)
+	var pairs []gpuPairLink
+	if topoMatrix, err := readGPUTopologyMatrix(exportDir); err == nil {
+		pairs = parseGPUPairAdjacency(topoMatrix)
+	}
+	gpuNUMAByIndex := map[int]*int{}
+	gpuBDFByIndex := map[int]string{}
+	for _, p := range placed {
+		if p.kind != "gpu" || p.bdf == "" {
+			continue
+		}
+		if idx, ok := bdfToIndex[p.bdf]; ok {
+			gpuNUMAByIndex[idx] = p.dev.NUMANode
+			gpuBDFByIndex[idx] = p.bdf
+		}
+	}
+	// A bonded pair spanning two different NUMA nodes is treated as an
+	// anomaly (not a neutral fact) per project decision: a bonded pair is
+	// expected to sit on one NUMA node, so a cross-NUMA bond escalates both
+	// GPUs' effective severity to at least Warning, regardless of their own
+	// reported SAT status.
+	crossNUMAWarnBDF := map[string]bool{}
+	for _, pair := range pairs {
+		numaA, okA := gpuNUMAByIndex[pair.GPUA]
+		numaB, okB := gpuNUMAByIndex[pair.GPUB]
+		if !okA || !okB || numaA == nil || numaB == nil || *numaA == *numaB {
+			continue
+		}
+		crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUA]] = true
+		crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUB]] = true
+	}
+
+	kindOrder := []string{"gpu", "nic", "raid"}
+	kindLabel := map[string]string{"gpu": "GPU", "nic": "NIC", "raid": "RAID"}
+
+	// Attach memory DIMMs to their CPU column too, the same way GPU/NIC/RAID
+	// PCIe devices are attached via NUMANode — memory has no NUMANode field
+	// in the schema, so this reads the DIMM's own Locator/Bank Locator
+	// strings instead (see dimmRawNode). DIMMs that can't be confidently
+	// attached fall back to the unattached "Memory" row below the diagram,
+	// same as before this existed.
+	memBankNodes := map[string]int{}
+	if raw, err := readTopoTechDump(exportDir, "dmidecode-type17.txt"); err == nil {
+		memBankNodes = parseDIMMBankLocatorNodes(raw)
+	}
+	memCol := make([]int, len(hw.Memory))
+	memMatched := make([]bool, len(hw.Memory))
+	var memRawNodes []int
+	for i, m := range hw.Memory {
+		if node, ok := dimmRawNode(m, memBankNodes); ok {
+			memCol[i] = node
+			memMatched[i] = true
+			memRawNodes = append(memRawNodes, node)
+		}
+	}
+	memColIdx := buildMemoryColumnIndex(memRawNodes)
+	for i := range hw.Memory {
+		if !memMatched[i] {
+			continue
+		}
+		col := memColIdx[memCol[i]]
+		if col >= numCols {
+			memMatched[i] = false
+			continue
+		}
+		memCol[i] = col
+	}
+
+	var boxes []topoBox
+	var pcieEdges []topoEdge
+
+	for col := 0; col < totalCols; col++ {
+		colX := (col+1)*24 + col*topoColWidth
+		if col < len(hw.CPUs) {
+			cpu := hw.CPUs[col]
+			model := ""
+			if cpu.Model != nil {
+				model = *cpu.Model
+			}
+			socket := col
+			if cpu.Socket != nil {
+				socket = *cpu.Socket
+			}
+			var tally topoStatusTally
+			tally.add(classifyTopoSeverity(cpu.Status))
+			fill, stroke, text := topoSeverityColors(tally.worst())
+			boxes = append(boxes, topoBox{
+				x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight,
+				topoCardInfo: topoCardInfo{
+					label: fmt.Sprintf("CPU %d", socket), sublabel: model, count: 1,
+					statusLine: tally.line(),
+					fillVar:    fill, strokeVar: stroke, textVar: text,
+					detailType: "cpu",
+				},
+			})
+		}
+
+		y := topoTopMargin + topoBoxHeight + topoDeviceGap*2
+
+		// Memory goes first in the chain, directly under the CPU box: DIMMs
+		// are wired straight to the socket's memory controller, not reached
+		// over PCIe like the GPU/NIC/RAID chain below it.
+		var memGroup []schema.HardwareMemory
+		for i, m := range hw.Memory {
+			if memMatched[i] && memCol[i] == col {
+				memGroup = append(memGroup, m)
+			}
+		}
+		if len(memGroup) > 0 {
+			var tally topoStatusTally
+			sizeGB := 0
+			for _, m := range memGroup {
+				tally.add(classifyTopoSeverity(m.Status))
+				if m.SizeMB != nil {
+					sizeGB += *m.SizeMB / 1024
+				}
+			}
+			fill, stroke, text := topoSeverityColors(tally.worst())
+			sublabel := ""
+			if sizeGB > 0 {
+				sublabel = fmt.Sprintf("%d GB total", sizeGB)
+			}
+			stackLayers := topoStackLayers(len(memGroup))
+			boxes = append(boxes, topoBox{
+				x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
+				topoCardInfo: topoCardInfo{
+					label: "Memory", sublabel: sublabel, count: len(memGroup),
+					statusLine: tally.line(),
+					fillVar:    fill, strokeVar: stroke, textVar: text,
+					detailType: "memory",
+				},
+			})
+			if col < len(hw.CPUs) {
+				pcieEdges = append(pcieEdges, topoEdge{
+					x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
+					x2: colX + topoBoxWidth/2, y2: y,
+					color: "var(--ok-fg)",
+				})
+			}
+			y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
+		}
+
+		for _, kind := range kindOrder {
+			var group []placedDevice
+			for _, p := range placed {
+				if p.col == col && p.kind == kind {
+					group = append(group, p)
+				}
+			}
+			if len(group) == 0 {
+				continue
+			}
+
+			var tally topoStatusTally
+			model := ""
+			edgeColor := "var(--ok-fg)"
+			for i, p := range group {
+				sev := classifyTopoSeverity(p.dev.Status)
+				if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 {
+					sev = 2
+				}
+				if kind == "gpu" && gpuHardwareFault && sev < 3 {
+					sev = 3
+				}
+				tally.add(sev)
+				if i == 0 && p.dev.Model != nil {
+					model = *p.dev.Model
+				}
+				if topoEdgeColorVar(p.dev) == "var(--warn-fg)" {
+					edgeColor = "var(--warn-fg)"
+				}
+			}
+			fill, stroke, text := topoSeverityColors(tally.worst())
+			stackLayers := topoStackLayers(len(group))
+			boxes = append(boxes, topoBox{
+				x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
+				topoCardInfo: topoCardInfo{
+					label: kindLabel[kind], sublabel: model, count: len(group),
+					statusLine: tally.line(),
+					fillVar:    fill, strokeVar: stroke, textVar: text,
+					detailType: kind,
+				},
+			})
+
+			if col < len(hw.CPUs) {
+				pcieEdges = append(pcieEdges, topoEdge{
+					x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
+					x2: colX + topoBoxWidth/2, y2: y,
+					color: edgeColor,
+				})
+			}
+
+			y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
+		}
+	}
+
+	maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2
+	for _, box := range boxes {
+		bottom := box.y + box.h + topoStackLayers(box.count)*topoStackStep
+		if bottom > maxDeviceY {
+			maxDeviceY = bottom
+		}
+	}
+
+	svgHeight := maxDeviceY + 24
+	svgWidth := totalCols*topoColWidth + 48
+
+	var b strings.Builder
+	// Wrapped in its own horizontally-scrolling container (matching the
+	// overflow-x:auto convention used for wide tables elsewhere in webui)
+	// rather than max-width:100% — squashing a node/edge diagram to fit a
+	// narrow viewport makes labels and badges illegible, whereas scrolling
+	// keeps the diagram readable at its natural size on any screen width.
+	b.WriteString(`<div style="overflow-x:auto">`)
+	fmt.Fprintf(&b, `<svg width="%d" height="%d" viewBox="0 0 %d %d">`+"\n", svgWidth, svgHeight, svgWidth, svgHeight)
+	for _, e := range pcieEdges {
+		fmt.Fprintf(&b, `<line x1="%d" y1="%d" x2="%d" y2="%d" style="stroke:%s;stroke-width:2"/>`+"\n", e.x1, e.y1, e.x2, e.y2, e.color)
+	}
+	for _, box := range boxes {
+		writeTopoBoxSVG(&b, box)
+	}
+	b.WriteString(`</svg></div>`)
+
+	// Firmware (BMC/BIOS/...) and PSUs have no PCIe/CPU affinity to anchor
+	// them to a column, and there can be an arbitrary number of any of them
+	// — so unlike the diagram above, they're plain flex-wrap HTML below the
+	// SVG rather than absolutely-positioned SVG boxes. A fixed-size SVG
+	// canvas has no way to wrap overflow onto a new row, which is exactly
+	// what caused these to pile up and overlap once a board had more
+	// PSUs/firmware records than fit in one fixed-width row.
+	//
+	// Memory DIMMs that were matched to a CPU column above already got a
+	// box in the SVG diagram; only DIMMs that couldn't be attached to a
+	// column (see memMatched above) fall back to this row.
+	var unmatchedMem []schema.HardwareMemory
+	for i, m := range hw.Memory {
+		if !memMatched[i] {
+			unmatchedMem = append(unmatchedMem, m)
+		}
+	}
+	if len(unmatchedMem) > 0 {
+		var tally topoStatusTally
+		for _, m := range unmatchedMem {
+			tally.add(classifyTopoSeverity(m.Status))
+		}
+		fill, stroke, text := topoSeverityColors(tally.worst())
+		sizeGB := 0
+		for _, m := range unmatchedMem {
+			if m.SizeMB != nil {
+				sizeGB += *m.SizeMB / 1024
+			}
+		}
+		sublabel := ""
+		if sizeGB > 0 {
+			sublabel = fmt.Sprintf("%d GB total", sizeGB)
+		}
+		b.WriteString(renderTopoFlexRow("Memory", []topoCardInfo{{
+			label: "Memory", sublabel: sublabel, count: len(unmatchedMem),
+			statusLine: tally.line(),
+			fillVar:    fill, strokeVar: stroke, textVar: text,
+			detailType: "memory",
+		}}))
+	}
+
+	var firmwareItems []topoCardInfo
+	for _, rec := range hw.Firmware {
+		// Firmware records carry no per-item status in the schema (they are
+		// identity, not health, facts), so each stays a neutral, uncolored
+		// card rather than forcing a fake "Unknown" status line.
+		fillVar, strokeVar, textVar := topoSeverityColors(0)
+		firmwareItems = append(firmwareItems, topoCardInfo{
+			label: rec.DeviceName, sublabel: "fw " + rec.Version, count: 1,
+			fillVar: fillVar, strokeVar: strokeVar, textVar: textVar,
+		})
+	}
+	b.WriteString(renderTopoFlexRow("Firmware", firmwareItems))
+
+	if len(hw.PowerSupplies) > 0 {
+		var tally topoStatusTally
+		watt := 0
+		for _, psu := range hw.PowerSupplies {
+			tally.add(classifyTopoSeverity(psu.Status))
+			if psu.WattageW != nil {
+				watt = *psu.WattageW
+			}
+		}
+		fill, stroke, text := topoSeverityColors(tally.worst())
+		sublabel := ""
+		if watt > 0 {
+			sublabel = fmt.Sprintf("%dW each", watt)
+		}
+		b.WriteString(renderTopoFlexRow("Power Supplies", []topoCardInfo{{
+			label: "Power Supplies", sublabel: sublabel, count: len(hw.PowerSupplies),
+			statusLine: tally.line(),
+			fillVar:    fill, strokeVar: stroke, textVar: text,
+			detailType: "psu",
+		}}))
+	}
+
+	return topoCard("Topology", b.String())
+}
+
+// renderTopoFlexRow renders a labeled, wrapping row of component cards.
+// Returns "" if items is empty (e.g. no PSU data in this audit).
+func renderTopoFlexRow(title string, items []topoCardInfo) string {
+	if len(items) == 0 {
+		return ""
+	}
+	var b strings.Builder
+	fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
+		html.EscapeString(title))
+	b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:10px">`)
+	for _, item := range items {
+		onclick := ""
+		cursor := "default"
+		if item.detailType != "" {
+			onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, item.detailType)
+			cursor = "pointer"
+		}
+		stackLayers := topoStackLayers(item.count)
+		// Extra right/bottom padding on the wrapper reserves room for the
+		// backing layers of the stack effect so they aren't clipped by the
+		// flex container.
+		fmt.Fprintf(&b, `<div style="position:relative;padding-right:%dpx;padding-bottom:%dpx">`,
+			stackLayers*topoStackStep, stackLayers*topoStackStep)
+		for i := stackLayers; i >= 1; i-- {
+			off := i * topoStackStep
+			fmt.Fprintf(&b, `<div style="position:absolute;top:%dpx;left:%dpx;right:0;bottom:0;border-radius:6px;background:%s;border:1px solid %s;opacity:.55"></div>`,
+				off, off, item.fillVar, item.strokeVar)
+		}
+		fmt.Fprintf(&b, `<div%s style="position:relative;cursor:%s;min-width:160px;padding:10px 12px;border-radius:6px;background:%s;border:1px solid %s;color:%s">`,
+			onclick, cursor, item.fillVar, item.strokeVar, item.textVar)
+		label := item.label
+		if item.count > 1 {
+			label = fmt.Sprintf("%s ×%d", item.label, item.count)
+		}
+		fmt.Fprintf(&b, `<div style="font-size:13px;font-weight:700">%s</div>`, html.EscapeString(label))
+		if item.sublabel != "" {
+			fmt.Fprintf(&b, `<div style="font-size:11px;opacity:.85">%s</div>`, html.EscapeString(item.sublabel))
+		}
+		if item.statusLine != "" {
+			fmt.Fprintf(&b, `<div style="font-size:11px;font-weight:600;margin-top:4px">%s</div>`, html.EscapeString(item.statusLine))
+		}
+		b.WriteString(`</div></div>`)
+	}
+	b.WriteString(`</div>`)
+	return b.String()
+}
+
+func writeTopoBoxSVG(b *strings.Builder, box topoBox) {
+	onclick := ""
+	cursor := "default"
+	if box.detailType != "" {
+		onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType)
+		cursor = "pointer"
+	}
+	fmt.Fprintf(b, `<g%s style="cursor:%s">`, onclick, cursor)
+
+	// Stack-of-cards effect: faint offset rects behind the front card when
+	// this box represents more than one physical component (e.g. 4 GPUs in
+	// one NUMA column), so a group reads as "a deck of N" rather than a
+	// single item. Peeks toward the bottom-right, into space already
+	// reserved between this box and the next one in the column.
+	for i := topoStackLayers(box.count); i >= 1; i-- {
+		off := i * topoStackStep
+		fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s;opacity:.55"/>`+"\n",
+			box.x+off, box.y+off, box.w, box.h, box.fillVar, box.strokeVar)
+	}
+
+	fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s"/>`+"\n",
+		box.x, box.y, box.w, box.h, box.fillVar, box.strokeVar)
+
+	label := box.label
+	if box.count > 1 {
+		label = fmt.Sprintf("%s ×%d", box.label, box.count)
+	}
+	fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:13px;font-weight:700">%s</text>`+"\n",
+		box.x+10, box.y+20, box.textVar, html.EscapeString(label))
+	if box.sublabel != "" {
+		fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:11px;opacity:.85">%s</text>`+"\n",
+			box.x+10, box.y+36, box.textVar, html.EscapeString(truncateTopoLabel(box.sublabel, 26)))
+	}
+	if box.statusLine != "" {
+		fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:10px;font-weight:600">%s</text>`+"\n",
+			box.x+10, box.y+box.h-10, box.textVar, html.EscapeString(box.statusLine))
+	}
+	b.WriteString(`</g>` + "\n")
+}
+
+func truncateTopoLabel(s string, max int) string {
+	if len(s) <= max {
+		return s
+	}
+	if max <= 1 {
+		return s[:max]
+	}
+	return s[:max-1] + "…"
+}
+
+// ---------------------------------------------------------------------------
+// Separate NVLink topology card (read from techdump, not written to any
+// ingest contract)
+// ---------------------------------------------------------------------------
+
+type topoNVLinkPort struct {
+	Index          int
+	Active         bool
+	SpeedGBs       *float64
+	ReplayErrors   int64
+	RecoveryErrors int64
+	CRCErrors      int64
+}
+
+var (
+	topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
+	topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
+	topoNVLinkInactiveRe  = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
+	topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
+)
+
+func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) {
+	raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-status.txt")
+	if err != nil {
+		return nil, err
+	}
+	return parseTopoNVLinkStatus(raw), nil
+}
+
+func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort {
+	result := map[int][]topoNVLinkPort{}
+	currentGPU := -1
+	for _, line := range strings.Split(raw, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
+			currentGPU, _ = strconv.Atoi(m[1])
+			continue
+		}
+		if currentGPU < 0 {
+			continue
+		}
+		if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
+			idx, _ := strconv.Atoi(m[1])
+			result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false})
+			continue
+		}
+		if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
+			idx, _ := strconv.Atoi(m[1])
+			port := topoNVLinkPort{Index: idx, Active: true}
+			if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
+				port.SpeedGBs = &speed
+			}
+			result[currentGPU] = append(result[currentGPU], port)
+		}
+	}
+	return result
+}
+
+func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) {
+	raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-errors.txt")
+	if err != nil {
+		return nil, err
+	}
+	return parseTopoNVLinkErrors(raw), nil
+}
+
+// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc].
+func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 {
+	result := map[int]map[int][3]int64{}
+	currentGPU := -1
+	for _, line := range strings.Split(raw, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
+			currentGPU, _ = strconv.Atoi(m[1])
+			continue
+		}
+		if currentGPU < 0 {
+			continue
+		}
+		m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed)
+		if m == nil {
+			continue
+		}
+		linkIdx, _ := strconv.Atoi(m[1])
+		count, _ := strconv.ParseInt(m[3], 10, 64)
+		if result[currentGPU] == nil {
+			result[currentGPU] = map[int][3]int64{}
+		}
+		c := result[currentGPU][linkIdx]
+		switch m[2] {
+		case "Replay":
+			c[0] = count
+		case "Recovery":
+			c[1] = count
+		case "CRC":
+			c[2] = count
+		}
+		result[currentGPU][linkIdx] = c
+	}
+	return result
+}
+
+// renderTopoNVLinkCard renders the separate NVLink topology card. Returns ""
+// if there are fewer than 2 NVIDIA GPUs, or the nvidia-smi nvlink techdump
+// wasn't captured (older audit, or nvidia-smi unavailable on that run).
+func renderTopoNVLinkCard(hw schema.HardwareSnapshot, exportDir string) string {
+	gpuCount := 0
+	for _, dev := range hw.PCIeDevices {
+		if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) {
+			gpuCount++
+		}
+	}
+	if gpuCount < 2 {
+		return ""
+	}
+
+	status, err := readTopoNVLinkStatus(exportDir)
+	if err != nil || len(status) == 0 {
+		return topoCard("NVLink Topology", `<span class="badge badge-unknown">nvidia-smi nvlink data unavailable</span>`)
+	}
+	errors, _ := readTopoNVLinkErrors(exportDir)
+
+	topoMatrix, _ := readGPUTopologyMatrix(exportDir)
+	pairs := parseGPUPairAdjacency(topoMatrix)
+
+	var bodyB strings.Builder
+	if gpuCount <= 4 && len(pairs) > 0 {
+		// Small GPU count: per-pair box+line with per-link detail.
+		for _, pair := range pairs {
+			activeCount, total, hasError := 0, 0, false
+			for _, port := range status[pair.GPUA] {
+				total++
+				if port.Active {
+					activeCount++
+				}
+			}
+			for _, counters := range errors[pair.GPUA] {
+				if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
+					hasError = true
+				}
+			}
+			color := "var(--ok-fg)"
+			switch {
+			case hasError:
+				color = "var(--crit-fg)"
+			case total > 0 && activeCount < total:
+				color = "var(--warn-fg)"
+			}
+			fmt.Fprintf(&bodyB, `<div style="display:flex;align-items:center;gap:12px;margin-bottom:10px">`+
+				`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
+				`<div style="flex:1;height:2px;background:%s"></div>`+
+				`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
+				`<div style="font-size:12px;color:var(--muted)">%d/%d links active%s</div>`+
+				`</div>`,
+				pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError))
+		}
+	} else if len(pairs) > 0 {
+		// Larger GPU counts (NVSwitch fabric): aggregate pair table instead of
+		// an unreadable all-to-all graph.
+		bodyB.WriteString(`<table><thead><tr><th>GPU A</th><th>GPU B</th><th>NVLinks</th></tr></thead><tbody>`)
+		for _, pair := range pairs {
+			fmt.Fprintf(&bodyB, `<tr><td>GPU %d</td><td>GPU %d</td><td>%d</td></tr>`, pair.GPUA, pair.GPUB, pair.NVLinks)
+		}
+		bodyB.WriteString(`</tbody></table>`)
+	} else {
+		bodyB.WriteString(`<span class="badge badge-unknown">No NVLink-bonded GPU pairs found</span>`)
+	}
+
+	return topoCard("NVLink Topology", bodyB.String())
+}
+
+func errNoteSuffix(hasError bool) string {
+	if hasError {
+		return " — errors detected"
+	}
+	return ""
+}
+
+// ---------------------------------------------------------------------------
+// Inventory fallback for the component-detail modal
+//
+// handleAPIComponentDetail normally sources records from app.ComponentStatusDB,
+// which only gains entries once something has actually written a status
+// observation (SAT run, watchdog tick, ...). On a freshly booted host that
+// hasn't run SAT yet, StatusDB can be entirely empty for a component type even
+// though the /topo card for it already shows "N OK" — that card reads
+// schema.HardwareComponentStatus.Status straight from the audit snapshot.
+// inventoryFallbackRecords bridges that gap by building synthetic records
+// from the same snapshot/classifiers the topology card uses, so the two
+// views never disagree about how many devices exist or their status.
+// ---------------------------------------------------------------------------
+
+// topoSeverityStatus renders classifyTopoSeverity's rank back into the status
+// string vocabulary renderComponentDetail/chipLetterClass expect ("OK",
+// "Warning", "Critical", "Unknown") — kept in lockstep with classifyTopoSeverity
+// so a device the topo card counts as "OK" is never shown here as "Unknown".
+func topoSeverityStatus(status *string) string {
+	switch classifyTopoSeverity(status) {
+	case 3:
+		return "Critical"
+	case 2:
+		return "Warning"
+	case 1:
+		return "OK"
+	default:
+		return "Unknown"
+	}
+}
+
+// pcieDeviceKind classifies a PCIe device the same way renderTopoMainDiagram
+// does, returning "" for devices that aren't GPU/NIC/RAID.
+func pcieDeviceKind(dev schema.HardwarePCIeDevice) string {
+	switch {
+	case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass):
+		return "gpu"
+	case isNICDeviceClassDev(dev):
+		return "nic"
+	case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass):
+		return "raid"
+	default:
+		return ""
+	}
+}
+
+// pcieDeviceKey builds a stable, human-readable component key for a PCIe
+// device: "<kind>:<bdf>" when a slot/BDF is known, else "<kind>:<index>".
+func pcieDeviceKey(kind string, index int, dev schema.HardwarePCIeDevice) string {
+	bdf := ""
+	if dev.Slot != nil {
+		bdf = normalizeTopoBDF(*dev.Slot)
+	} else if dev.BDF != nil {
+		bdf = normalizeTopoBDF(*dev.BDF)
+	}
+	if bdf != "" {
+		return kind + ":" + bdf
+	}
+	return fmt.Sprintf("%s:%d", kind, index)
+}
+
+// inventoryFallbackRecords builds ComponentStatusRecord entries straight from
+// the audit inventory (bee-audit.json) for the given component type, used
+// when ComponentStatusDB has no matching records yet. Records carry only
+// ComponentKey/Status — no LastCheckedAt/History — so renderComponentDetail
+// renders them without a "checked at" timestamp or sparkline.
+func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.ComponentStatusRecord {
+	data, err := loadSnapshot(opts.AuditPath)
+	if err != nil {
+		return nil
+	}
+	var ingest schema.HardwareIngestRequest
+	if err := json.Unmarshal(data, &ingest); err != nil {
+		return nil
+	}
+	hw := ingest.Hardware
+
+	var records []app.ComponentStatusRecord
+	switch compType {
+	case "cpu":
+		for i, cpu := range hw.CPUs {
+			key := fmt.Sprintf("cpu:%d", i)
+			if cpu.Socket != nil {
+				key = fmt.Sprintf("cpu:socket%d", *cpu.Socket)
+			}
+			records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(cpu.Status)})
+		}
+	case "memory":
+		for i, m := range hw.Memory {
+			key := fmt.Sprintf("memory:%d", i)
+			if m.Slot != nil && strings.TrimSpace(*m.Slot) != "" {
+				key = "memory:" + strings.TrimSpace(*m.Slot)
+			}
+			records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(m.Status)})
+		}
+	case "storage":
+		for i, s := range hw.Storage {
+			key := fmt.Sprintf("storage:%d", i)
+			if s.Slot != nil && strings.TrimSpace(*s.Slot) != "" {
+				key = "storage:" + strings.TrimSpace(*s.Slot)
+			}
+			records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(s.Status)})
+		}
+	case "psu":
+		for i, p := range hw.PowerSupplies {
+			key := fmt.Sprintf("psu:%d", i)
+			if p.Slot != nil && strings.TrimSpace(*p.Slot) != "" {
+				key = "psu:" + strings.TrimSpace(*p.Slot)
+			}
+			records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)})
+		}
+	case "gpu", "nic", "raid":
+		for i, dev := range hw.PCIeDevices {
+			if pcieDeviceKind(dev) != compType {
+				continue
+			}
+			key := pcieDeviceKey(compType, i, dev)
+			records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(dev.Status)})
+		}
+	}
+	return records
+}

audit/internal/webui/page_validate.go

@@ -286,11 +286,6 @@ function satLoadGPUs() {
     satUpdateGPUSelectionNote();
   });
 }
-function satGPUDisplayName(gpu) {
-  const idx = (gpu && Number.isFinite(Number(gpu.index))) ? Number(gpu.index) : 0;
-  const name = gpu && gpu.name ? gpu.name : ('GPU ' + idx);
-  return 'GPU ' + idx + ' — ' + name;
-}
 function satRequestBody(target, overrides) {
   const body = {};
   const labels = satLabels();
@@ -358,34 +353,9 @@ function runSATWithOverrides(target, overrides) {
   return enqueueSATTarget(target, overrides)
     .then(d => streamSATTask(d.task_id, title, false));
 }
-const nvidiaPerGPUTargets = [];
-const nvidiaAllGPUTargets = ['nvidia', 'nvidia-targeted-stress', 'nvidia-targeted-power', 'nvidia-pulse', 'nvidia-interconnect', 'nvidia-bandwidth'];
 function satAllGPUIndicesForMulti() {
   return Promise.resolve(satSelectedGPUIndices());
 }
-function expandSATTarget(target) {
-  if (nvidiaAllGPUTargets.indexOf(target) >= 0) {
-    return satAllGPUIndicesForMulti().then(function(indices) {
-      if (!indices.length) return Promise.reject(new Error('No NVIDIA GPUs available.'));
-      return [{target: target, overrides: {gpu_indices: indices, display_name: satLabels()[target] || target}}];
-    });
-  }
-  if (nvidiaPerGPUTargets.indexOf(target) < 0) {
-    return Promise.resolve([{target: target}]);
-  }
-  const selected = satSelectedGPUIndices();
-  if (!selected.length) {
-    return Promise.reject(new Error('Select at least one NVIDIA GPU.'));
-  }
-  return loadSatNvidiaGPUs().then(gpus => gpus.filter(gpu => selected.indexOf(Number(gpu.index)) >= 0).map(gpu => ({
-    target: target,
-    overrides: {
-      gpu_indices: [Number(gpu.index)],
-      display_name: (satLabels()[target] || ('Validate ' + target)) + ' (' + satGPUDisplayName(gpu) + ')'
-    },
-    label: satGPUDisplayName(gpu),
-  })));
-}
 function runNvidiaFabricValidate(target) {
   satAllGPUIndicesForMulti().then(function(indices) {
     if (!indices.length) { alert('No NVIDIA GPUs available.'); return; }
@@ -419,52 +389,40 @@ function runAMDValidateSet() {
   };
   return runNext(0);
 }
+// runAllSAT hands the whole decision to the backend: which hardware is
+// present and ready, and therefore which tasks to enqueue, is decided by
+// POST /api/sat/run-all. The page only sends operator intent.
 function runAllSAT() {
-  const cycles = 1;
   const status = document.getElementById('sat-all-status');
-  status.textContent = 'Enqueuing...';
-  const stressOnlyTargets = ['nvidia-targeted-stress', 'nvidia-targeted-power', 'nvidia-pulse'];
-  const baseTargets = ['nvidia','nvidia-targeted-stress','nvidia-targeted-power','nvidia-pulse','nvidia-interconnect','nvidia-bandwidth','memory','storage','tpm','cpu'].concat(selectedAMDValidateTargets());
-  const activeTargets = baseTargets.filter(target => {
-    if (stressOnlyTargets.indexOf(target) >= 0 && !satStressMode()) return false;
-    const btn = document.getElementById('sat-btn-' + target);
-    return !(btn && btn.disabled);
-  });
-  Promise.all(activeTargets.map(expandSATTarget)).then(groups => {
-    const expanded = [];
-    for (let cycle = 0; cycle < cycles; cycle++) {
-      groups.forEach(group => group.forEach(item => expanded.push(item)));
-    }
-    const total = expanded.length;
-    let enqueued = 0;
-    if (!total) {
-      status.textContent = 'No tasks selected.';
-      return;
-    }
-    const runNext = (idx) => {
-      if (idx >= expanded.length) { status.textContent = 'Completed ' + total + ' task(s).'; return Promise.resolve(); }
-      const item = expanded[idx];
-      status.textContent = 'Running ' + (idx + 1) + '/' + total + '...';
-      return enqueueSATTarget(item.target, item.overrides)
-        .then(() => {
-          enqueued++;
-          return runNext(idx + 1);
-        });
-    };
-    return runNext(0);
-  }).catch(err => {
-    status.textContent = 'Error: ' + err.message;
-  });
+  status.textContent = 'Planning on server...';
+  const body = {
+    stress_mode: satStressMode(),
+    amd_targets: selectedAMDValidateTargets(),
+  };
+  const gpuSubset = satSelectedGPUIndices();
+  if (gpuSubset.length) body.nvidia_gpu_indices = gpuSubset;
+  fetch('/api/sat/run-all', {
+    method: 'POST',
+    headers: {'Content-Type': 'application/json'},
+    body: JSON.stringify(body),
+  }).then(r => r.json().then(d => { if (!r.ok) throw new Error(d.error || ('HTTP ' + r.status)); return d; }))
+    .then(d => {
+      let msg = 'Enqueued ' + (d.task_count || 0) + ' task(s).';
+      if (d.notes && d.notes.length) msg += ' ' + d.notes.join('; ');
+      status.textContent = msg;
+    })
+    .catch(err => { status.textContent = 'Error: ' + err.message; });
 }
 </script>
 <script>
 fetch('/api/gpu/presence').then(r=>r.json()).then(gp => {
-    if (!gp.nvidia) disableSATCard('nvidia', 'No NVIDIA GPU detected');
-    if (!gp.nvidia) disableSATCard('nvidia-targeted-stress', 'No NVIDIA GPU detected');
-    if (!gp.nvidia) disableSATCard('nvidia-targeted-power', 'No NVIDIA GPU detected');
-    if (!gp.nvidia) disableSATCard('nvidia-pulse', 'No NVIDIA GPU detected');
-    if (!gp.nvidia) disableSATCard('nvidia-interconnect', 'No NVIDIA GPU detected');
-    if (!gp.nvidia) disableSATCard('nvidia-bandwidth', 'No NVIDIA GPU detected');
+    if (!gp.nvidia) {
+      const why = gp.nvidia_initializing
+        ? 'NVIDIA GPU present, driver still initializing; Run All will wait for it'
+        : 'No NVIDIA GPU detected';
+      ['nvidia','nvidia-targeted-stress','nvidia-targeted-power','nvidia-pulse','nvidia-interconnect','nvidia-bandwidth']
+        .forEach(t => disableSATCard(t, why));
+    }
     if (!gp.amd) disableSATCard('amd', 'No AMD GPU detected');
     if (!gp.amd) disableSATAMDOptions('No AMD GPU detected');
 });
@@ -915,31 +873,27 @@ function runAMDValidateSet() {
   };
   return runNext(0);
 }
+// runAllCheckSAT delegates hardware detection and task planning to the
+// backend (POST /api/sat/run-all). The browser no longer decides whether a
+// GPU is present: a stale or empty GPU list can no longer silently drop the
+// GPU checks.
 function runAllCheckSAT() {
   const status = document.getElementById('sat-all-status');
-  status.textContent = 'Enqueuing...';
-  const nvidiaIndices = satSelectedGPUIndices();
-  const nvidiaAllTargets = ['nvidia', 'nvidia-interconnect', 'nvidia-bandwidth', 'nvidia-pcie-bandwidth'];
-  const baseTargets = ['cpu', 'memory', 'storage', 'tpm', 'nvidia-config', 'pcie-link'];
-  const amdTargets = selectedAMDValidateTargets();
-  const expanded = [];
-  baseTargets.forEach(t => expanded.push({target: t}));
-  if (nvidiaIndices.length) {
-    nvidiaAllTargets.forEach(t => {
-      const btn = document.getElementById('sat-btn-' + t);
-      if (!(btn && btn.disabled)) expanded.push({target: t, overrides: {gpu_indices: nvidiaIndices, display_name: satLabels()[t] || t}});
-    });
-  }
-  amdTargets.forEach(t => expanded.push({target: t}));
-  if (!expanded.length) { status.textContent = 'No tasks selected.'; return; }
-  const total = expanded.length;
-  const runNext = idx => {
-    if (idx >= expanded.length) { status.textContent = 'Completed ' + total + ' task(s).'; return Promise.resolve(); }
-    const item = expanded[idx];
-    status.textContent = 'Running ' + (idx + 1) + '/' + total + '...';
-    return enqueueSATTarget(item.target, item.overrides).then(() => runNext(idx + 1));
-  };
-  runNext(0).catch(err => { status.textContent = 'Error: ' + err.message; });
+  status.textContent = 'Planning on server...';
+  const body = {stress_mode: false, amd_targets: selectedAMDValidateTargets()};
+  const gpuSubset = satSelectedGPUIndices();
+  if (gpuSubset.length) body.nvidia_gpu_indices = gpuSubset;
+  fetch('/api/sat/run-all', {
+    method: 'POST',
+    headers: {'Content-Type': 'application/json'},
+    body: JSON.stringify(body),
+  }).then(r => r.json().then(d => { if (!r.ok) throw new Error(d.error || ('HTTP ' + r.status)); return d; }))
+    .then(d => {
+      let msg = 'Enqueued ' + (d.task_count || 0) + ' task(s).';
+      if (d.notes && d.notes.length) msg += ' ' + d.notes.join('; ');
+      status.textContent = msg;
+    })
+    .catch(err => { status.textContent = 'Error: ' + err.message; });
 }
 function disableSATCard(id, reason) {
   const btn = document.getElementById('sat-btn-' + id);
@@ -959,7 +913,12 @@ function disableSATCard(id, reason) {
   }
 }
 fetch('/api/gpu/presence').then(r => r.json()).then(gp => {
-  if (!gp.nvidia) ['nvidia','nvidia-interconnect','nvidia-bandwidth'].forEach(t => disableSATCard(t, 'No NVIDIA GPU detected'));
+  if (!gp.nvidia) {
+    const why = gp.nvidia_initializing
+      ? 'NVIDIA GPU present, driver still initializing; Run All will wait for it'
+      : 'No NVIDIA GPU detected';
+    ['nvidia','nvidia-interconnect','nvidia-bandwidth'].forEach(t => disableSATCard(t, why));
+  }
   if (!gp.amd) {
     disableSATCard('amd', 'No AMD GPU detected');
     ['sat-amd-target','sat-amd-mem-target','sat-amd-bandwidth-target'].forEach(id => {

audit/internal/webui/page_validate_test.go

@@ -37,7 +37,6 @@ func TestRenderCheckIncludesReadOnlyTPMValidation(t *testing.T) {
 		`tpm2_getcap properties-fixed`,
 		`tpm2_pcrread`,
 		`tpm2_gettestresult`,
-		`'storage', 'tpm', 'nvidia-config'`,
 	} {
 		if !strings.Contains(page, want) {
 			t.Fatalf("check page does not contain %q", want)

audit/internal/webui/pages.go

@@ -5,9 +5,6 @@ import (
 	"fmt"
 	"html"
 	"path/filepath"
-	"regexp"
-	"sort"
-	"strconv"
 	"strings"
 
 	"bee/audit/internal/app"
@@ -645,684 +642,3 @@ function auditModalRun() {
 }
 </script>`
 }
-
-func renderHealthCard(opts HandlerOptions) string {
-	data, err := loadSnapshot(filepath.Join(opts.ExportDir, "runtime-health.json"))
-	if err != nil {
-		return `<div class="card"><div class="card-head">Runtime Health</div><div class="card-body"><span class="badge badge-unknown">No data</span></div></div>`
-	}
-	var health schema.RuntimeHealth
-	if err := json.Unmarshal(data, &health); err != nil {
-		return `<div class="card"><div class="card-head">Runtime Health</div><div class="card-body"><span class="badge badge-err">Parse error</span></div></div>`
-	}
-	status := strings.TrimSpace(health.Status)
-	if status == "" {
-		status = "UNKNOWN"
-	}
-	badge := "badge-ok"
-	if status == "PARTIAL" {
-		badge = "badge-warn"
-	} else if status == "FAIL" || status == "FAILED" {
-		badge = "badge-err"
-	}
-	var b strings.Builder
-	b.WriteString(`<div class="card"><div class="card-head">Runtime Health</div><div class="card-body">`)
-	b.WriteString(fmt.Sprintf(`<div style="margin-bottom:10px"><span class="badge %s">%s</span></div>`, badge, html.EscapeString(status)))
-	if checkedAt := strings.TrimSpace(health.CheckedAt); checkedAt != "" {
-		b.WriteString(`<div style="font-size:12px;color:var(--muted);margin-bottom:12px">Checked at: ` + html.EscapeString(checkedAt) + `</div>`)
-	}
-	rows := []runtimeHealthRow{
-		buildRuntimeExportRow(health),
-		buildRuntimeNetworkRow(health),
-		buildRuntimeDriverRow(health),
-		buildRuntimeAccelerationRow(health),
-		buildRuntimeToolsRow(health),
-		buildRuntimeServicesRow(health),
-		buildRuntimeUSBExportRow(health),
-		buildRuntimeToRAMRow(health),
-	}
-	b.WriteString(`<table><thead><tr><th>Check</th><th>Status</th><th>Source</th><th>Issue</th></tr></thead><tbody>`)
-	for _, row := range rows {
-		b.WriteString(`<tr><td>` + html.EscapeString(row.Title) + `</td><td>` + runtimeStatusBadge(row.Status) + `</td><td>` + html.EscapeString(row.Source) + `</td><td>` + rowIssueHTML(row.Issue) + `</td></tr>`)
-	}
-	b.WriteString(`</tbody></table>`)
-	b.WriteString(`</div></div>`)
-	return b.String()
-}
-
-type runtimeHealthRow struct {
-	Title  string
-	Status string
-	Source string
-	Issue  string
-}
-
-func buildRuntimeExportRow(health schema.RuntimeHealth) runtimeHealthRow {
-	issue := runtimeIssueDescriptions(health.Issues, "export_dir_unavailable")
-	status := "UNKNOWN"
-	switch {
-	case issue != "":
-		status = "FAILED"
-	case strings.TrimSpace(health.ExportDir) != "":
-		status = "OK"
-	}
-	source := "os.MkdirAll"
-	if dir := strings.TrimSpace(health.ExportDir); dir != "" {
-		source += " " + dir
-	}
-	return runtimeHealthRow{Title: "Export Directory", Status: status, Source: source, Issue: issue}
-}
-
-func buildRuntimeNetworkRow(health schema.RuntimeHealth) runtimeHealthRow {
-	status := strings.TrimSpace(health.NetworkStatus)
-	if status == "" {
-		status = "UNKNOWN"
-	}
-	issue := runtimeIssueDescriptions(health.Issues, "dhcp_failed")
-	return runtimeHealthRow{Title: "Network", Status: status, Source: "ListInterfaces / DHCP", Issue: issue}
-}
-
-func buildRuntimeDriverRow(health schema.RuntimeHealth) runtimeHealthRow {
-	issue := runtimeIssueDescriptions(health.Issues, "nvidia_kernel_module_missing", "nvidia_modeset_failed", "amdgpu_kernel_module_missing")
-	status := "UNKNOWN"
-	switch {
-	case health.DriverReady && issue == "":
-		status = "OK"
-	case health.DriverReady:
-		status = "PARTIAL"
-	case issue != "":
-		status = "FAILED"
-	}
-	return runtimeHealthRow{Title: "NVIDIA/AMD Driver", Status: status, Source: "lsmod / vendor probe", Issue: issue}
-}
-
-func buildRuntimeAccelerationRow(health schema.RuntimeHealth) runtimeHealthRow {
-	issue := runtimeIssueDescriptions(health.Issues, "cuda_runtime_not_ready", "rocm_smi_unavailable")
-	status := "UNKNOWN"
-	switch {
-	case health.CUDAReady && issue == "":
-		status = "OK"
-	case health.CUDAReady:
-		status = "PARTIAL"
-	case issue != "":
-		status = "FAILED"
-	}
-	return runtimeHealthRow{Title: "CUDA / ROCm", Status: status, Source: "bee-gpu-burn / rocm-smi", Issue: issue}
-}
-
-func buildRuntimeToolsRow(health schema.RuntimeHealth) runtimeHealthRow {
-	if len(health.Tools) == 0 {
-		return runtimeHealthRow{Title: "Required Utilities", Status: "UNKNOWN", Source: "CheckTools", Issue: "No tool status data."}
-	}
-	missing := make([]string, 0)
-	for _, tool := range health.Tools {
-		if !tool.OK {
-			missing = append(missing, tool.Name)
-		}
-	}
-	status := "OK"
-	issue := ""
-	if len(missing) > 0 {
-		status = "PARTIAL"
-		issue = "Missing: " + strings.Join(missing, ", ")
-	}
-	return runtimeHealthRow{Title: "Required Utilities", Status: status, Source: "CheckTools", Issue: issue}
-}
-
-func buildRuntimeServicesRow(health schema.RuntimeHealth) runtimeHealthRow {
-	if len(health.Services) == 0 {
-		return runtimeHealthRow{Title: "Bee Services", Status: "UNKNOWN", Source: "systemctl is-active", Issue: "No service status data."}
-	}
-	nonActive := make([]string, 0)
-	for _, svc := range health.Services {
-		state := strings.TrimSpace(strings.ToLower(svc.Status))
-		// "inactive" is OK for oneshot services that have completed successfully
-		// (bee-sshsetup, bee-preflight, bee-audit, bee-network, etc.).
-		// Only "failed" is a genuine problem.
-		switch state {
-		case "active", "activating", "deactivating", "reloading", "inactive":
-			// OK — service is running, transitioning normally, or completed successfully
-		default:
-			nonActive = append(nonActive, svc.Name+"="+svc.Status)
-		}
-	}
-	status := "OK"
-	issue := ""
-	if len(nonActive) > 0 {
-		status = "PARTIAL"
-		issue = strings.Join(nonActive, ", ")
-	}
-	return runtimeHealthRow{Title: "Bee Services", Status: status, Source: "ServiceState", Issue: issue}
-}
-
-func buildRuntimeUSBExportRow(health schema.RuntimeHealth) runtimeHealthRow {
-	path := strings.TrimSpace(health.USBExportPath)
-	if path != "" {
-		return runtimeHealthRow{
-			Title:  "USB Export Drive",
-			Status: "OK",
-			Source: "/proc/mounts + lsblk",
-			Issue:  path,
-		}
-	}
-	return runtimeHealthRow{
-		Title:  "USB Export Drive",
-		Status: "WARNING",
-		Source: "/proc/mounts + lsblk",
-		Issue:  "No writable USB drive mounted. Plug in a USB drive to enable log export.",
-	}
-}
-
-func buildRuntimeToRAMRow(health schema.RuntimeHealth) runtimeHealthRow {
-	switch strings.ToLower(strings.TrimSpace(health.ToRAMStatus)) {
-	case "ok":
-		return runtimeHealthRow{
-			Title:  "LiveCD in RAM",
-			Status: "OK",
-			Source: "live-boot / /proc/mounts",
-			Issue:  "",
-		}
-	case "partial":
-		return runtimeHealthRow{
-			Title:  "LiveCD in RAM",
-			Status: "WARNING",
-			Source: "live-boot / /proc/mounts / /dev/shm/bee-live",
-			Issue:  "Partial or staged RAM copy detected. System is not fully running from RAM; Copy to RAM can be retried.",
-		}
-	case "failed":
-		return runtimeHealthRow{
-			Title:  "LiveCD in RAM",
-			Status: "FAILED",
-			Source: "live-boot / /proc/mounts",
-			Issue:  "toram boot parameter set but ISO is not mounted from RAM. Copy may have failed.",
-		}
-	default:
-		// toram not active — ISO still on original boot media (USB/CD)
-		return runtimeHealthRow{
-			Title:  "LiveCD in RAM",
-			Status: "WARNING",
-			Source: "live-boot / /proc/mounts",
-			Issue:  "ISO not copied to RAM. Use \u201cCopy to RAM\u201d to free the boot drive and improve performance.",
-		}
-	}
-}
-
-func buildHardwareComponentRows(exportDir string) []runtimeHealthRow {
-	path := filepath.Join(exportDir, "component-status.json")
-	db, err := app.OpenComponentStatusDB(path)
-	if err != nil {
-		return []runtimeHealthRow{
-			{Title: "CPU Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."},
-			{Title: "Memory Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."},
-			{Title: "Storage Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."},
-			{Title: "GPU Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."},
-			{Title: "PSU Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "No PSU component checks recorded."},
-		}
-	}
-	records := db.All()
-	return []runtimeHealthRow{
-		aggregateComponentStatus("CPU", records, []string{"cpu:all"}, nil),
-		aggregateComponentStatus("Memory", records, []string{"memory:all"}, []string{"memory:"}),
-		aggregateComponentStatus("Storage", records, []string{"storage:all"}, []string{"storage:"}),
-		aggregateComponentStatus("GPU", records, nil, []string{"pcie:gpu:"}),
-		aggregateComponentStatus("PSU", records, nil, []string{"psu:"}),
-	}
-}
-
-// matchedRecords returns all ComponentStatusRecord entries whose key matches
-// any exact key or any of the given prefixes. Used for per-device chip rendering.
-func firstNonEmpty(vals ...string) string {
-	for _, v := range vals {
-		if v != "" {
-			return v
-		}
-	}
-	return ""
-}
-
-func matchedRecords(records []app.ComponentStatusRecord, exact []string, prefixes []string) []app.ComponentStatusRecord {
-	var matched []app.ComponentStatusRecord
-	for _, rec := range records {
-		key := strings.TrimSpace(rec.ComponentKey)
-		if key == "" {
-			continue
-		}
-		if containsExactKey(key, exact) || hasAnyPrefix(key, prefixes) {
-			matched = append(matched, rec)
-		}
-	}
-	return matched
-}
-
-func aggregateComponentStatus(title string, records []app.ComponentStatusRecord, exact []string, prefixes []string) runtimeHealthRow {
-	matched := make([]app.ComponentStatusRecord, 0)
-	for _, rec := range records {
-		key := strings.TrimSpace(rec.ComponentKey)
-		if key == "" {
-			continue
-		}
-		if containsExactKey(key, exact) || hasAnyPrefix(key, prefixes) {
-			matched = append(matched, rec)
-		}
-	}
-	if len(matched) == 0 {
-		return runtimeHealthRow{Title: title, Status: "UNKNOWN", Source: "component-status.json", Issue: "No component status data."}
-	}
-
-	maxSev := -1
-	for _, rec := range matched {
-		if sev := runtimeComponentSeverity(rec.Status); sev > maxSev {
-			maxSev = sev
-		}
-	}
-	status := "UNKNOWN"
-	switch maxSev {
-	case 3:
-		status = "CRITICAL"
-	case 2:
-		status = "WARNING"
-	case 1:
-		status = "OK"
-	}
-
-	sources := make([]string, 0)
-	sourceSeen := map[string]struct{}{}
-	issues := make([]string, 0)
-	issueSeen := map[string]struct{}{}
-	for _, rec := range matched {
-		if runtimeComponentSeverity(rec.Status) != maxSev {
-			continue
-		}
-		source := latestComponentSource(rec)
-		if source == "" {
-			source = "component-status.json"
-		}
-		if _, ok := sourceSeen[source]; !ok {
-			sourceSeen[source] = struct{}{}
-			sources = append(sources, source)
-		}
-		issue := strings.TrimSpace(rec.ErrorSummary)
-		if issue == "" {
-			issue = latestComponentDetail(rec)
-		}
-		if issue == "" {
-			continue
-		}
-		if _, ok := issueSeen[issue]; ok {
-			continue
-		}
-		issueSeen[issue] = struct{}{}
-		issues = append(issues, issue)
-	}
-	if len(sources) == 0 {
-		sources = append(sources, "component-status.json")
-	}
-	issue := strings.Join(issues, "; ")
-	if issue == "" {
-		issue = "—"
-	}
-	return runtimeHealthRow{
-		Title:  title,
-		Status: status,
-		Source: strings.Join(sources, ", "),
-		Issue:  issue,
-	}
-}
-
-func containsExactKey(key string, exact []string) bool {
-	for _, candidate := range exact {
-		if key == candidate {
-			return true
-		}
-	}
-	return false
-}
-
-func hasAnyPrefix(key string, prefixes []string) bool {
-	for _, prefix := range prefixes {
-		if strings.HasPrefix(key, prefix) {
-			return true
-		}
-	}
-	return false
-}
-
-func runtimeComponentSeverity(status string) int {
-	switch strings.TrimSpace(strings.ToLower(status)) {
-	case "critical":
-		return 3
-	case "warning":
-		return 2
-	case "ok":
-		return 1
-	default:
-		return 0
-	}
-}
-
-func latestComponentSource(rec app.ComponentStatusRecord) string {
-	if len(rec.History) == 0 {
-		return ""
-	}
-	return strings.TrimSpace(rec.History[len(rec.History)-1].Source)
-}
-
-func latestComponentDetail(rec app.ComponentStatusRecord) string {
-	if len(rec.History) == 0 {
-		return ""
-	}
-	return strings.TrimSpace(rec.History[len(rec.History)-1].Detail)
-}
-
-func runtimeIssueDescriptions(issues []schema.RuntimeIssue, codes ...string) string {
-	if len(issues) == 0 || len(codes) == 0 {
-		return ""
-	}
-	allowed := make(map[string]struct{}, len(codes))
-	for _, code := range codes {
-		allowed[code] = struct{}{}
-	}
-	messages := make([]string, 0)
-	for _, issue := range issues {
-		if _, ok := allowed[issue.Code]; !ok {
-			continue
-		}
-		desc := strings.TrimSpace(issue.Description)
-		if desc == "" {
-			desc = issue.Code
-		}
-		messages = append(messages, desc)
-	}
-	return strings.Join(messages, "; ")
-}
-
-// gpuNeedsPhysicalReboot reports whether any GPU component record carries a
-// hardware-fault reason that a driver reset can't clear (e.g. Xid 79 "GPU
-// has fallen off the bus", Xid 154 "Node Reboot Required" — see
-// collector.xidHardwareFaultMessages). Those errors mean every subsequent
-// GPU SAT job will keep failing until the node is physically power-cycled,
-// so this drives a dashboard banner that says so up front instead of making
-// an operator burn another test cycle to rediscover it.
-func gpuNeedsPhysicalReboot(records []app.ComponentStatusRecord) (reason string, needsReboot bool) {
-	for _, rec := range records {
-		if !strings.EqualFold(strings.TrimSpace(rec.Status), "Critical") {
-			continue
-		}
-		if strings.Contains(strings.ToLower(rec.ErrorSummary), "reboot") {
-			return rec.ErrorSummary, true
-		}
-	}
-	return "", false
-}
-
-// chipLetterClass maps a component status to a single display letter and CSS class.
-func chipLetterClass(status string) (letter, cls string) {
-	switch strings.ToUpper(strings.TrimSpace(status)) {
-	case "OK":
-		return "O", "chip-ok"
-	case "WARNING", "WARN", "PARTIAL":
-		return "W", "chip-warn"
-	case "CRITICAL", "FAIL", "FAILED", "ERROR":
-		return "F", "chip-fail"
-	default:
-		return "?", "chip-unknown"
-	}
-}
-
-// renderComponentChips renders one 20×20 chip per ComponentStatusRecord.
-// Hover tooltip shows component key, status, error summary and last check time.
-// Falls back to a single unknown chip when no records are available.
-func renderComponentChips(matched []app.ComponentStatusRecord) string {
-	if len(matched) == 0 {
-		return `<span class="chips"><span class="chip chip-unknown" title="No data">?</span></span>`
-	}
-	sort.Slice(matched, func(i, j int) bool {
-		return matched[i].ComponentKey < matched[j].ComponentKey
-	})
-	var b strings.Builder
-	b.WriteString(`<span class="chips">`)
-	for _, rec := range matched {
-		letter, cls := chipLetterClass(rec.Status)
-		var tooltip strings.Builder
-		tooltip.WriteString(rec.ComponentKey)
-		tooltip.WriteString(": ")
-		tooltip.WriteString(firstNonEmpty(rec.Status, "UNKNOWN"))
-		if rec.ErrorSummary != "" {
-			tooltip.WriteString(" — ")
-			tooltip.WriteString(rec.ErrorSummary)
-		}
-		if !rec.LastCheckedAt.IsZero() {
-			fmt.Fprintf(&tooltip, " (checked %s)", rec.LastCheckedAt.Format("15:04:05"))
-		}
-		fmt.Fprintf(&b, `<span class="chip %s" title="%s">%s</span>`,
-			cls, html.EscapeString(tooltip.String()), letter)
-	}
-	b.WriteString(`</span>`)
-	return b.String()
-}
-
-func runtimeStatusBadge(status string) string {
-	status = strings.ToUpper(strings.TrimSpace(status))
-	badge := "badge-unknown"
-	switch status {
-	case "OK":
-		badge = "badge-ok"
-	case "PARTIAL", "WARNING", "WARN":
-		badge = "badge-warn"
-	case "FAIL", "FAILED", "CRITICAL":
-		badge = "badge-err"
-	}
-	return `<span class="badge ` + badge + `">` + html.EscapeString(status) + `</span>`
-}
-
-func rowIssueHTML(issue string) string {
-	issue = strings.TrimSpace(issue)
-	if issue == "" {
-		return `<span style="color:var(--muted)">—</span>`
-	}
-	return html.EscapeString(issue)
-}
-
-var aerStatusRe = regexp.MustCompile(`aer_status:\s*0x([0-9a-fA-F]{1,8})`)
-
-// decodeAERStatus parses an AER status hex value from a kernel error detail string
-// and returns a human-readable list of set bit names with correctable/uncorrectable label,
-// or "" if no AER status is found.
-func decodeAERStatus(detail string) string {
-	m := aerStatusRe.FindStringSubmatch(detail)
-	if m == nil {
-		return ""
-	}
-	v64, err := strconv.ParseUint(m[1], 16, 32)
-	if err != nil {
-		return ""
-	}
-	val := uint32(v64)
-
-	type bitDef struct {
-		bit  uint32
-		name string
-	}
-	corrBits := []bitDef{
-		{0, "Receiver Error"}, {6, "Replay Timer Timeout"}, {7, "Advisory Non-Fatal"},
-		{8, "Corrected Internal Error"}, {9, "Header Log Overflow"},
-		{13, "Replay Num Rollover"}, {14, "Bad DLLP"}, {15, "Bad TLP"},
-	}
-	uncorrBits := []bitDef{
-		{4, "Data Link Protocol Error"}, {5, "Surprise Down Error"},
-		{12, "Poisoned TLP Received"}, {13, "Flow Control Protocol Error"},
-		{14, "Completion Timeout"}, {15, "Completer Abort"}, {16, "Unexpected Completion"},
-		{17, "Receiver Overflow"}, {18, "Malformed TLP"}, {19, "ECRC Error"},
-		{20, "Unsupported Request Error"}, {21, "ACS Violation"}, {22, "Uncorrectable Internal Error"},
-	}
-	var corrNames, uncorrNames []string
-	for _, b := range corrBits {
-		if val&(1<<b.bit) != 0 {
-			corrNames = append(corrNames, b.name)
-		}
-	}
-	for _, b := range uncorrBits {
-		if val&(1<<b.bit) != 0 {
-			uncorrNames = append(uncorrNames, b.name)
-		}
-	}
-	if len(corrNames) >= len(uncorrNames) && len(corrNames) > 0 {
-		return strings.Join(corrNames, ", ") + " (correctable)"
-	}
-	if len(uncorrNames) > 0 {
-		return strings.Join(uncorrNames, ", ") + " (uncorrectable)"
-	}
-	return fmt.Sprintf("unknown bits: 0x%08x", val)
-}
-
-// renderSparkline returns a small inline SVG showing non-OK events over time.
-// Events are positioned proportionally along the time axis; if all share the same
-// timestamp they are spaced evenly. Width is always 100px.
-func renderSparkline(history []app.ComponentStatusEntry) string {
-	const (
-		svgW = 100
-		svgH = 20
-		barW = 3
-		barH = 14
-	)
-	var events []app.ComponentStatusEntry
-	for _, e := range history {
-		if e.Status != "OK" {
-			events = append(events, e)
-		}
-	}
-	if len(events) == 0 {
-		return ""
-	}
-	n := len(events)
-	barColor := func(status string) string {
-		if status == "Critical" {
-			return "#c0392b"
-		}
-		return "#d97706"
-	}
-	yTop := (svgH - barH) / 2
-
-	var bars strings.Builder
-	if n == 1 {
-		x := (svgW - barW) / 2
-		fmt.Fprintf(&bars, `<rect x="%d" y="%d" width="%d" height="%d" fill="%s" rx="1"/>`,
-			x, yTop, barW, barH, barColor(events[0].Status))
-	} else {
-		minT := events[0].At
-		maxT := events[n-1].At
-		dur := maxT.Sub(minT).Seconds()
-		for i, e := range events {
-			var x int
-			if dur <= 0 {
-				step := svgW / n
-				x = i*step + (step-barW)/2
-			} else {
-				frac := e.At.Sub(minT).Seconds() / dur
-				x = int(frac * float64(svgW-barW))
-			}
-			fmt.Fprintf(&bars, `<rect x="%d" y="%d" width="%d" height="%d" fill="%s" rx="1"/>`,
-				x, yTop, barW, barH, barColor(e.Status))
-		}
-	}
-	return fmt.Sprintf(
-		`<svg width="%d" height="%d" style="display:inline-block;vertical-align:middle;margin-left:6px;flex-shrink:0" xmlns="http://www.w3.org/2000/svg">`+
-			`<rect x="0" y="0" width="%d" height="%d" fill="var(--surface-alt,#ebebeb)" rx="3"/>%s</svg>`,
-		svgW, svgH, svgW, svgH, bars.String())
-}
-
-// renderComponentDetail renders a modal content fragment for one component type.
-// Called by handleAPIComponentDetail and displayed inside #component-detail-dialog.
-// fromInventory marks that records were synthesized from the audit inventory
-// snapshot (no ComponentStatusDB history yet) rather than real SAT/watchdog
-// observations — see inventoryFallbackRecords.
-func renderComponentDetail(title string, records []app.ComponentStatusRecord, fromInventory bool) string {
-	var b strings.Builder
-	fmt.Fprintf(&b, `<div style="padding:20px 24px 0">`)
-	fmt.Fprintf(&b, `<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:16px">`)
-	fmt.Fprintf(&b, `<span style="font-size:16px;font-weight:700">%s — Status Detail</span>`, html.EscapeString(title))
-	b.WriteString(`<button class="btn btn-sm btn-secondary" onclick="document.getElementById('component-detail-dialog').close()">Close</button>`)
-	b.WriteString(`</div>`)
-
-	if len(records) == 0 {
-		b.WriteString(`<p style="color:var(--muted)">No status data recorded yet for this component type.</p>`)
-		b.WriteString(`</div>`)
-		return b.String()
-	}
-
-	if fromInventory {
-		b.WriteString(`<p style="color:var(--muted);font-size:12px;margin-top:-8px;margin-bottom:16px">No SAT-test history yet — showing latest inventory snapshot.</p>`)
-	}
-
-	sort.Slice(records, func(i, j int) bool {
-		return records[i].ComponentKey < records[j].ComponentKey
-	})
-
-	for _, rec := range records {
-		letter, cls := chipLetterClass(rec.Status)
-
-		// Count non-OK events across the full history for the badge + sparkline.
-		warnCount := 0
-		for _, e := range rec.History {
-			if e.Status != "OK" {
-				warnCount++
-			}
-		}
-
-		fmt.Fprintf(&b, `<div style="margin-bottom:20px">`)
-		fmt.Fprintf(&b, `<div style="display:flex;align-items:center;gap:8px;margin-bottom:8px;flex-wrap:wrap">`)
-		fmt.Fprintf(&b, `<span class="chip %s">%s</span>`, cls, letter)
-		fmt.Fprintf(&b, `<span style="font-weight:700;font-size:13px">%s</span>`, html.EscapeString(rec.ComponentKey))
-		if !rec.LastCheckedAt.IsZero() {
-			fmt.Fprintf(&b, `<span style="color:var(--muted);font-size:12px">checked %s</span>`, rec.LastCheckedAt.Format("2006-01-02 15:04:05"))
-		}
-		if warnCount > 0 {
-			noun := "events"
-			if warnCount == 1 {
-				noun = "event"
-			}
-			fmt.Fprintf(&b,
-				`<span style="font-size:11px;background:var(--warn-bg,#fffbeb);color:var(--warn-fg,#92400e);border:1px solid var(--warn-border,#fde68a);border-radius:10px;padding:1px 7px;white-space:nowrap">%d %s</span>`,
-				warnCount, noun)
-			b.WriteString(renderSparkline(rec.History))
-		}
-		b.WriteString(`</div>`)
-
-		if rec.ErrorSummary != "" {
-			fmt.Fprintf(&b, `<div style="font-size:12px;margin-bottom:4px;color:var(--muted)">%s</div>`, html.EscapeString(rec.ErrorSummary))
-			if decoded := decodeAERStatus(rec.ErrorSummary); decoded != "" {
-				fmt.Fprintf(&b,
-					`<div style="font-size:12px;margin-bottom:8px;color:var(--muted)"><span style="background:var(--surface-alt,#f5f5f5);border-radius:4px;padding:1px 6px;font-family:monospace">AER: %s</span></div>`,
-					html.EscapeString(decoded))
-			}
-		}
-
-		// History table — newest first, cap at 20 entries.
-		history := rec.History
-		if len(history) > 20 {
-			history = history[len(history)-20:]
-		}
-		b.WriteString(`<table style="width:100%;font-size:12px;border-collapse:collapse">`)
-		b.WriteString(`<tr style="color:var(--muted)"><th style="text-align:left;padding:2px 10px 2px 0;white-space:nowrap">Time</th><th style="text-align:left;padding:2px 10px 2px 0">Status</th><th style="text-align:left;padding:2px 10px 2px 0">Source</th><th style="text-align:left;padding:2px 0">Detail</th></tr>`)
-		for i := len(history) - 1; i >= 0; i-- {
-			e := history[i]
-			eLetter, eCls := chipLetterClass(e.Status)
-			detail := e.Detail
-			if detail == "" {
-				detail = "—"
-			}
-			fmt.Fprintf(&b,
-				`<tr><td style="padding:3px 10px 3px 0;white-space:nowrap;color:var(--muted)">%s</td><td style="padding:3px 10px 3px 0"><span class="chip %s" style="font-size:10px;width:16px;height:16px">%s</span></td><td style="padding:3px 10px 3px 0;white-space:nowrap">%s</td><td style="padding:3px 0;color:var(--muted)">%s</td></tr>`,
-				html.EscapeString(e.At.Format("2006-01-02 15:04:05")),
-				eCls, eLetter,
-				html.EscapeString(e.Source),
-				html.EscapeString(detail),
-			)
-		}
-		b.WriteString(`</table>`)
-		b.WriteString(`</div>`)
-	}
-
-	b.WriteString(`</div>`)
-	return b.String()
-}

audit/internal/webui/pages_health.go

@@ -0,0 +1,572 @@
+package webui
+
+import (
+	"encoding/json"
+	"fmt"
+	"html"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"bee/audit/internal/app"
+	"bee/audit/internal/schema"
+)
+
+func renderHealthCard(opts HandlerOptions) string {
+	data, err := loadSnapshot(filepath.Join(opts.ExportDir, "runtime-health.json"))
+	if err != nil {
+		return `<div class="card"><div class="card-head">Runtime Health</div><div class="card-body"><span class="badge badge-unknown">No data</span></div></div>`
+	}
+	var health schema.RuntimeHealth
+	if err := json.Unmarshal(data, &health); err != nil {
+		return `<div class="card"><div class="card-head">Runtime Health</div><div class="card-body"><span class="badge badge-err">Parse error</span></div></div>`
+	}
+	status := strings.TrimSpace(health.Status)
+	if status == "" {
+		status = "UNKNOWN"
+	}
+	badge := "badge-ok"
+	if status == "PARTIAL" {
+		badge = "badge-warn"
+	} else if status == "FAIL" || status == "FAILED" {
+		badge = "badge-err"
+	}
+	var b strings.Builder
+	b.WriteString(`<div class="card"><div class="card-head">Runtime Health</div><div class="card-body">`)
+	b.WriteString(fmt.Sprintf(`<div style="margin-bottom:10px"><span class="badge %s">%s</span></div>`, badge, html.EscapeString(status)))
+	if checkedAt := strings.TrimSpace(health.CheckedAt); checkedAt != "" {
+		b.WriteString(`<div style="font-size:12px;color:var(--muted);margin-bottom:12px">Checked at: ` + html.EscapeString(checkedAt) + `</div>`)
+	}
+	rows := []runtimeHealthRow{
+		buildRuntimeExportRow(health),
+		buildRuntimeNetworkRow(health),
+		buildRuntimeDriverRow(health),
+		buildRuntimeAccelerationRow(health),
+		buildRuntimeToolsRow(health),
+		buildRuntimeServicesRow(health),
+		buildRuntimeUSBExportRow(health),
+		buildRuntimeToRAMRow(health),
+	}
+	b.WriteString(`<table><thead><tr><th>Check</th><th>Status</th><th>Source</th><th>Issue</th></tr></thead><tbody>`)
+	for _, row := range rows {
+		b.WriteString(`<tr><td>` + html.EscapeString(row.Title) + `</td><td>` + runtimeStatusBadge(row.Status) + `</td><td>` + html.EscapeString(row.Source) + `</td><td>` + rowIssueHTML(row.Issue) + `</td></tr>`)
+	}
+	b.WriteString(`</tbody></table>`)
+	b.WriteString(`</div></div>`)
+	return b.String()
+}
+
+type runtimeHealthRow struct {
+	Title  string
+	Status string
+	Source string
+	Issue  string
+}
+
+func buildRuntimeExportRow(health schema.RuntimeHealth) runtimeHealthRow {
+	issue := runtimeIssueDescriptions(health.Issues, "export_dir_unavailable")
+	status := "UNKNOWN"
+	switch {
+	case issue != "":
+		status = "FAILED"
+	case strings.TrimSpace(health.ExportDir) != "":
+		status = "OK"
+	}
+	source := "os.MkdirAll"
+	if dir := strings.TrimSpace(health.ExportDir); dir != "" {
+		source += " " + dir
+	}
+	return runtimeHealthRow{Title: "Export Directory", Status: status, Source: source, Issue: issue}
+}
+
+func buildRuntimeNetworkRow(health schema.RuntimeHealth) runtimeHealthRow {
+	status := strings.TrimSpace(health.NetworkStatus)
+	if status == "" {
+		status = "UNKNOWN"
+	}
+	issue := runtimeIssueDescriptions(health.Issues, "dhcp_failed")
+	return runtimeHealthRow{Title: "Network", Status: status, Source: "ListInterfaces / DHCP", Issue: issue}
+}
+
+func buildRuntimeDriverRow(health schema.RuntimeHealth) runtimeHealthRow {
+	issue := runtimeIssueDescriptions(health.Issues, "nvidia_kernel_module_missing", "nvidia_modeset_failed", "amdgpu_kernel_module_missing")
+	status := "UNKNOWN"
+	switch {
+	case health.DriverReady && issue == "":
+		status = "OK"
+	case health.DriverReady:
+		status = "PARTIAL"
+	case issue != "":
+		status = "FAILED"
+	}
+	return runtimeHealthRow{Title: "NVIDIA/AMD Driver", Status: status, Source: "lsmod / vendor probe", Issue: issue}
+}
+
+func buildRuntimeAccelerationRow(health schema.RuntimeHealth) runtimeHealthRow {
+	issue := runtimeIssueDescriptions(health.Issues, "cuda_runtime_not_ready", "rocm_smi_unavailable")
+	status := "UNKNOWN"
+	switch {
+	case health.CUDAReady && issue == "":
+		status = "OK"
+	case health.CUDAReady:
+		status = "PARTIAL"
+	case issue != "":
+		status = "FAILED"
+	}
+	return runtimeHealthRow{Title: "CUDA / ROCm", Status: status, Source: "bee-gpu-burn / rocm-smi", Issue: issue}
+}
+
+func buildRuntimeToolsRow(health schema.RuntimeHealth) runtimeHealthRow {
+	if len(health.Tools) == 0 {
+		return runtimeHealthRow{Title: "Required Utilities", Status: "UNKNOWN", Source: "CheckTools", Issue: "No tool status data."}
+	}
+	missing := make([]string, 0)
+	for _, tool := range health.Tools {
+		if !tool.OK {
+			missing = append(missing, tool.Name)
+		}
+	}
+	status := "OK"
+	issue := ""
+	if len(missing) > 0 {
+		status = "PARTIAL"
+		issue = "Missing: " + strings.Join(missing, ", ")
+	}
+	return runtimeHealthRow{Title: "Required Utilities", Status: status, Source: "CheckTools", Issue: issue}
+}
+
+func buildRuntimeServicesRow(health schema.RuntimeHealth) runtimeHealthRow {
+	if len(health.Services) == 0 {
+		return runtimeHealthRow{Title: "Bee Services", Status: "UNKNOWN", Source: "systemctl is-active", Issue: "No service status data."}
+	}
+	nonActive := make([]string, 0)
+	for _, svc := range health.Services {
+		state := strings.TrimSpace(strings.ToLower(svc.Status))
+		// "inactive" is OK for oneshot services that have completed successfully
+		// (bee-sshsetup, bee-preflight, bee-audit, bee-network, etc.).
+		// Only "failed" is a genuine problem.
+		switch state {
+		case "active", "activating", "deactivating", "reloading", "inactive":
+			// OK — service is running, transitioning normally, or completed successfully
+		default:
+			nonActive = append(nonActive, svc.Name+"="+svc.Status)
+		}
+	}
+	status := "OK"
+	issue := ""
+	if len(nonActive) > 0 {
+		status = "PARTIAL"
+		issue = strings.Join(nonActive, ", ")
+	}
+	return runtimeHealthRow{Title: "Bee Services", Status: status, Source: "ServiceState", Issue: issue}
+}
+
+func buildRuntimeUSBExportRow(health schema.RuntimeHealth) runtimeHealthRow {
+	path := strings.TrimSpace(health.USBExportPath)
+	if path != "" {
+		return runtimeHealthRow{
+			Title:  "USB Export Drive",
+			Status: "OK",
+			Source: "/proc/mounts + lsblk",
+			Issue:  path,
+		}
+	}
+	return runtimeHealthRow{
+		Title:  "USB Export Drive",
+		Status: "WARNING",
+		Source: "/proc/mounts + lsblk",
+		Issue:  "No writable USB drive mounted. Plug in a USB drive to enable log export.",
+	}
+}
+
+func buildRuntimeToRAMRow(health schema.RuntimeHealth) runtimeHealthRow {
+	switch strings.ToLower(strings.TrimSpace(health.ToRAMStatus)) {
+	case "ok":
+		return runtimeHealthRow{
+			Title:  "LiveCD in RAM",
+			Status: "OK",
+			Source: "live-boot / /proc/mounts",
+			Issue:  "",
+		}
+	case "partial":
+		return runtimeHealthRow{
+			Title:  "LiveCD in RAM",
+			Status: "WARNING",
+			Source: "live-boot / /proc/mounts / /dev/shm/bee-live",
+			Issue:  "Partial or staged RAM copy detected. System is not fully running from RAM; Copy to RAM can be retried.",
+		}
+	case "failed":
+		return runtimeHealthRow{
+			Title:  "LiveCD in RAM",
+			Status: "FAILED",
+			Source: "live-boot / /proc/mounts",
+			Issue:  "toram boot parameter set but ISO is not mounted from RAM. Copy may have failed.",
+		}
+	default:
+		// toram not active — ISO still on original boot media (USB/CD)
+		return runtimeHealthRow{
+			Title:  "LiveCD in RAM",
+			Status: "WARNING",
+			Source: "live-boot / /proc/mounts",
+			Issue:  "ISO not copied to RAM. Use \u201cCopy to RAM\u201d to free the boot drive and improve performance.",
+		}
+	}
+}
+
+// matchedRecords returns all ComponentStatusRecord entries whose key matches
+// any exact key or any of the given prefixes. Used for per-device chip rendering.
+func firstNonEmpty(vals ...string) string {
+	for _, v := range vals {
+		if v != "" {
+			return v
+		}
+	}
+	return ""
+}
+
+func matchedRecords(records []app.ComponentStatusRecord, exact []string, prefixes []string) []app.ComponentStatusRecord {
+	var matched []app.ComponentStatusRecord
+	for _, rec := range records {
+		key := strings.TrimSpace(rec.ComponentKey)
+		if key == "" {
+			continue
+		}
+		if containsExactKey(key, exact) || hasAnyPrefix(key, prefixes) {
+			matched = append(matched, rec)
+		}
+	}
+	return matched
+}
+
+func containsExactKey(key string, exact []string) bool {
+	for _, candidate := range exact {
+		if key == candidate {
+			return true
+		}
+	}
+	return false
+}
+
+func hasAnyPrefix(key string, prefixes []string) bool {
+	for _, prefix := range prefixes {
+		if strings.HasPrefix(key, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+func runtimeIssueDescriptions(issues []schema.RuntimeIssue, codes ...string) string {
+	if len(issues) == 0 || len(codes) == 0 {
+		return ""
+	}
+	allowed := make(map[string]struct{}, len(codes))
+	for _, code := range codes {
+		allowed[code] = struct{}{}
+	}
+	messages := make([]string, 0)
+	for _, issue := range issues {
+		if _, ok := allowed[issue.Code]; !ok {
+			continue
+		}
+		desc := strings.TrimSpace(issue.Description)
+		if desc == "" {
+			desc = issue.Code
+		}
+		messages = append(messages, desc)
+	}
+	return strings.Join(messages, "; ")
+}
+
+// gpuNeedsPhysicalReboot reports whether any GPU component record carries a
+// hardware-fault reason that a driver reset can't clear (e.g. Xid 79 "GPU
+// has fallen off the bus", Xid 154 "Node Reboot Required" — see
+// collector.xidHardwareFaultMessages). Those errors mean every subsequent
+// GPU SAT job will keep failing until the node is physically power-cycled,
+// so this drives a dashboard banner that says so up front instead of making
+// an operator burn another test cycle to rediscover it.
+func gpuNeedsPhysicalReboot(records []app.ComponentStatusRecord) (reason string, needsReboot bool) {
+	for _, rec := range records {
+		if !strings.EqualFold(strings.TrimSpace(rec.Status), "Critical") {
+			continue
+		}
+		if strings.Contains(strings.ToLower(rec.ErrorSummary), "reboot") {
+			return rec.ErrorSummary, true
+		}
+	}
+	return "", false
+}
+
+// chipLetterClass maps a component status to a single display letter and CSS class.
+func chipLetterClass(status string) (letter, cls string) {
+	switch strings.ToUpper(strings.TrimSpace(status)) {
+	case "OK":
+		return "O", "chip-ok"
+	case "WARNING", "WARN", "PARTIAL":
+		return "W", "chip-warn"
+	case "CRITICAL", "FAIL", "FAILED", "ERROR":
+		return "F", "chip-fail"
+	default:
+		return "?", "chip-unknown"
+	}
+}
+
+// renderComponentChips renders one 20×20 chip per ComponentStatusRecord.
+// Hover tooltip shows component key, status, error summary and last check time.
+// Falls back to a single unknown chip when no records are available.
+func renderComponentChips(matched []app.ComponentStatusRecord) string {
+	if len(matched) == 0 {
+		return `<span class="chips"><span class="chip chip-unknown" title="No data">?</span></span>`
+	}
+	sort.Slice(matched, func(i, j int) bool {
+		return matched[i].ComponentKey < matched[j].ComponentKey
+	})
+	var b strings.Builder
+	b.WriteString(`<span class="chips">`)
+	for _, rec := range matched {
+		letter, cls := chipLetterClass(rec.Status)
+		var tooltip strings.Builder
+		tooltip.WriteString(rec.ComponentKey)
+		tooltip.WriteString(": ")
+		tooltip.WriteString(firstNonEmpty(rec.Status, "UNKNOWN"))
+		if rec.ErrorSummary != "" {
+			tooltip.WriteString(" — ")
+			tooltip.WriteString(rec.ErrorSummary)
+		}
+		if !rec.LastCheckedAt.IsZero() {
+			fmt.Fprintf(&tooltip, " (checked %s)", rec.LastCheckedAt.Format("15:04:05"))
+		}
+		fmt.Fprintf(&b, `<span class="chip %s" title="%s">%s</span>`,
+			cls, html.EscapeString(tooltip.String()), letter)
+	}
+	b.WriteString(`</span>`)
+	return b.String()
+}
+
+func runtimeStatusBadge(status string) string {
+	status = strings.ToUpper(strings.TrimSpace(status))
+	badge := "badge-unknown"
+	switch status {
+	case "OK":
+		badge = "badge-ok"
+	case "PARTIAL", "WARNING", "WARN":
+		badge = "badge-warn"
+	case "FAIL", "FAILED", "CRITICAL":
+		badge = "badge-err"
+	}
+	return `<span class="badge ` + badge + `">` + html.EscapeString(status) + `</span>`
+}
+
+func rowIssueHTML(issue string) string {
+	issue = strings.TrimSpace(issue)
+	if issue == "" {
+		return `<span style="color:var(--muted)">—</span>`
+	}
+	return html.EscapeString(issue)
+}
+
+var aerStatusRe = regexp.MustCompile(`aer_status:\s*0x([0-9a-fA-F]{1,8})`)
+
+// decodeAERStatus parses an AER status hex value from a kernel error detail string
+// and returns a human-readable list of set bit names with correctable/uncorrectable label,
+// or "" if no AER status is found.
+func decodeAERStatus(detail string) string {
+	m := aerStatusRe.FindStringSubmatch(detail)
+	if m == nil {
+		return ""
+	}
+	v64, err := strconv.ParseUint(m[1], 16, 32)
+	if err != nil {
+		return ""
+	}
+	val := uint32(v64)
+
+	type bitDef struct {
+		bit  uint32
+		name string
+	}
+	corrBits := []bitDef{
+		{0, "Receiver Error"}, {6, "Replay Timer Timeout"}, {7, "Advisory Non-Fatal"},
+		{8, "Corrected Internal Error"}, {9, "Header Log Overflow"},
+		{13, "Replay Num Rollover"}, {14, "Bad DLLP"}, {15, "Bad TLP"},
+	}
+	uncorrBits := []bitDef{
+		{4, "Data Link Protocol Error"}, {5, "Surprise Down Error"},
+		{12, "Poisoned TLP Received"}, {13, "Flow Control Protocol Error"},
+		{14, "Completion Timeout"}, {15, "Completer Abort"}, {16, "Unexpected Completion"},
+		{17, "Receiver Overflow"}, {18, "Malformed TLP"}, {19, "ECRC Error"},
+		{20, "Unsupported Request Error"}, {21, "ACS Violation"}, {22, "Uncorrectable Internal Error"},
+	}
+	var corrNames, uncorrNames []string
+	for _, b := range corrBits {
+		if val&(1<<b.bit) != 0 {
+			corrNames = append(corrNames, b.name)
+		}
+	}
+	for _, b := range uncorrBits {
+		if val&(1<<b.bit) != 0 {
+			uncorrNames = append(uncorrNames, b.name)
+		}
+	}
+	if len(corrNames) >= len(uncorrNames) && len(corrNames) > 0 {
+		return strings.Join(corrNames, ", ") + " (correctable)"
+	}
+	if len(uncorrNames) > 0 {
+		return strings.Join(uncorrNames, ", ") + " (uncorrectable)"
+	}
+	return fmt.Sprintf("unknown bits: 0x%08x", val)
+}
+
+// renderSparkline returns a small inline SVG showing non-OK events over time.
+// Events are positioned proportionally along the time axis; if all share the same
+// timestamp they are spaced evenly. Width is always 100px.
+func renderSparkline(history []app.ComponentStatusEntry) string {
+	const (
+		svgW = 100
+		svgH = 20
+		barW = 3
+		barH = 14
+	)
+	var events []app.ComponentStatusEntry
+	for _, e := range history {
+		if e.Status != "OK" {
+			events = append(events, e)
+		}
+	}
+	if len(events) == 0 {
+		return ""
+	}
+	n := len(events)
+	barColor := func(status string) string {
+		if status == "Critical" {
+			return "#c0392b"
+		}
+		return "#d97706"
+	}
+	yTop := (svgH - barH) / 2
+
+	var bars strings.Builder
+	if n == 1 {
+		x := (svgW - barW) / 2
+		fmt.Fprintf(&bars, `<rect x="%d" y="%d" width="%d" height="%d" fill="%s" rx="1"/>`,
+			x, yTop, barW, barH, barColor(events[0].Status))
+	} else {
+		minT := events[0].At
+		maxT := events[n-1].At
+		dur := maxT.Sub(minT).Seconds()
+		for i, e := range events {
+			var x int
+			if dur <= 0 {
+				step := svgW / n
+				x = i*step + (step-barW)/2
+			} else {
+				frac := e.At.Sub(minT).Seconds() / dur
+				x = int(frac * float64(svgW-barW))
+			}
+			fmt.Fprintf(&bars, `<rect x="%d" y="%d" width="%d" height="%d" fill="%s" rx="1"/>`,
+				x, yTop, barW, barH, barColor(e.Status))
+		}
+	}
+	return fmt.Sprintf(
+		`<svg width="%d" height="%d" style="display:inline-block;vertical-align:middle;margin-left:6px;flex-shrink:0" xmlns="http://www.w3.org/2000/svg">`+
+			`<rect x="0" y="0" width="%d" height="%d" fill="var(--surface-alt,#ebebeb)" rx="3"/>%s</svg>`,
+		svgW, svgH, svgW, svgH, bars.String())
+}
+
+// renderComponentDetail renders a modal content fragment for one component type.
+// Called by handleAPIComponentDetail and displayed inside #component-detail-dialog.
+// fromInventory marks that records were synthesized from the audit inventory
+// snapshot (no ComponentStatusDB history yet) rather than real SAT/watchdog
+// observations — see inventoryFallbackRecords.
+func renderComponentDetail(title string, records []app.ComponentStatusRecord, fromInventory bool) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, `<div style="padding:20px 24px 0">`)
+	fmt.Fprintf(&b, `<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:16px">`)
+	fmt.Fprintf(&b, `<span style="font-size:16px;font-weight:700">%s — Status Detail</span>`, html.EscapeString(title))
+	b.WriteString(`<button class="btn btn-sm btn-secondary" onclick="document.getElementById('component-detail-dialog').close()">Close</button>`)
+	b.WriteString(`</div>`)
+
+	if len(records) == 0 {
+		b.WriteString(`<p style="color:var(--muted)">No status data recorded yet for this component type.</p>`)
+		b.WriteString(`</div>`)
+		return b.String()
+	}
+
+	if fromInventory {
+		b.WriteString(`<p style="color:var(--muted);font-size:12px;margin-top:-8px;margin-bottom:16px">No SAT-test history yet — showing latest inventory snapshot.</p>`)
+	}
+
+	sort.Slice(records, func(i, j int) bool {
+		return records[i].ComponentKey < records[j].ComponentKey
+	})
+
+	for _, rec := range records {
+		letter, cls := chipLetterClass(rec.Status)
+
+		// Count non-OK events across the full history for the badge + sparkline.
+		warnCount := 0
+		for _, e := range rec.History {
+			if e.Status != "OK" {
+				warnCount++
+			}
+		}
+
+		fmt.Fprintf(&b, `<div style="margin-bottom:20px">`)
+		fmt.Fprintf(&b, `<div style="display:flex;align-items:center;gap:8px;margin-bottom:8px;flex-wrap:wrap">`)
+		fmt.Fprintf(&b, `<span class="chip %s">%s</span>`, cls, letter)
+		fmt.Fprintf(&b, `<span style="font-weight:700;font-size:13px">%s</span>`, html.EscapeString(rec.ComponentKey))
+		if !rec.LastCheckedAt.IsZero() {
+			fmt.Fprintf(&b, `<span style="color:var(--muted);font-size:12px">checked %s</span>`, rec.LastCheckedAt.Format("2006-01-02 15:04:05"))
+		}
+		if warnCount > 0 {
+			noun := "events"
+			if warnCount == 1 {
+				noun = "event"
+			}
+			fmt.Fprintf(&b,
+				`<span style="font-size:11px;background:var(--warn-bg,#fffbeb);color:var(--warn-fg,#92400e);border:1px solid var(--warn-border,#fde68a);border-radius:10px;padding:1px 7px;white-space:nowrap">%d %s</span>`,
+				warnCount, noun)
+			b.WriteString(renderSparkline(rec.History))
+		}
+		b.WriteString(`</div>`)
+
+		if rec.ErrorSummary != "" {
+			fmt.Fprintf(&b, `<div style="font-size:12px;margin-bottom:4px;color:var(--muted)">%s</div>`, html.EscapeString(rec.ErrorSummary))
+			if decoded := decodeAERStatus(rec.ErrorSummary); decoded != "" {
+				fmt.Fprintf(&b,
+					`<div style="font-size:12px;margin-bottom:8px;color:var(--muted)"><span style="background:var(--surface-alt,#f5f5f5);border-radius:4px;padding:1px 6px;font-family:monospace">AER: %s</span></div>`,
+					html.EscapeString(decoded))
+			}
+		}
+
+		// History table — newest first, cap at 20 entries.
+		history := rec.History
+		if len(history) > 20 {
+			history = history[len(history)-20:]
+		}
+		b.WriteString(`<table style="width:100%;font-size:12px;border-collapse:collapse">`)
+		b.WriteString(`<tr style="color:var(--muted)"><th style="text-align:left;padding:2px 10px 2px 0;white-space:nowrap">Time</th><th style="text-align:left;padding:2px 10px 2px 0">Status</th><th style="text-align:left;padding:2px 10px 2px 0">Source</th><th style="text-align:left;padding:2px 0">Detail</th></tr>`)
+		for i := len(history) - 1; i >= 0; i-- {
+			e := history[i]
+			eLetter, eCls := chipLetterClass(e.Status)
+			detail := e.Detail
+			if detail == "" {
+				detail = "—"
+			}
+			fmt.Fprintf(&b,
+				`<tr><td style="padding:3px 10px 3px 0;white-space:nowrap;color:var(--muted)">%s</td><td style="padding:3px 10px 3px 0"><span class="chip %s" style="font-size:10px;width:16px;height:16px">%s</span></td><td style="padding:3px 10px 3px 0;white-space:nowrap">%s</td><td style="padding:3px 0;color:var(--muted)">%s</td></tr>`,
+				html.EscapeString(e.At.Format("2006-01-02 15:04:05")),
+				eCls, eLetter,
+				html.EscapeString(e.Source),
+				html.EscapeString(detail),
+			)
+		}
+		b.WriteString(`</table>`)
+		b.WriteString(`</div>`)
+	}
+
+	b.WriteString(`</div>`)
+	return b.String()
+}

audit/internal/webui/raid_actions.go

@@ -0,0 +1,707 @@
+package webui
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func (h *handler) handleAPIRAIDStatus(w http.ResponseWriter, r *http.Request) {
+	resp := raidStatusResp{Controllers: []raidControllerInfo{}}
+
+	lsi2 := detectStorcli2Controllers()
+	if lsi := detectLSIControllers(); len(lsi) > 0 {
+		// storcli64 can enumerate a Tri-Mode controller (SAS3808-iMR/9500
+		// series) at a basic level but its drive-listing JSON parser finds
+		// no "Drive Information" for these — a zero-drives entry that
+		// storcli2 (run above) already covers correctly. Only drop it when
+		// storcli2 actually found something, so a genuinely drive-populated
+		// classic controller elsewhere in a mixed setup is never hidden.
+		for _, c := range lsi {
+			if len(c.AllDrives) == 0 && len(lsi2) > 0 {
+				continue
+			}
+			resp.Controllers = append(resp.Controllers, c)
+		}
+	}
+	if len(lsi2) > 0 {
+		resp.Controllers = append(resp.Controllers, lsi2...)
+	}
+	if vroc := detectVROCController(); vroc != nil {
+		resp.Controllers = append(resp.Controllers, *vroc)
+	}
+
+	writeJSON(w, resp)
+}
+
+func (h *handler) handleAPIRAIDForeignAction(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		ControllerID string `json:"controller_id"`
+		Action       string `json:"action"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON")
+		return
+	}
+	if req.Action != "import" && req.Action != "clear" {
+		writeError(w, http.StatusBadRequest, "action must be 'import' or 'clear'")
+		return
+	}
+	ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID)
+	if !ok {
+		writeError(w, http.StatusBadRequest, "invalid controller_id")
+		return
+	}
+
+	target := "raid-foreign-clear"
+	name := fmt.Sprintf("RAID Foreign Clear (ctrl %d)", ctrlIdx)
+	if req.Action == "import" {
+		target = "raid-foreign-import"
+		name = fmt.Sprintf("RAID Foreign Import (ctrl %d)", ctrlIdx)
+	}
+
+	t := &Task{
+		ID:        newJobID(target),
+		Name:      name,
+		Target:    target,
+		Priority:  defaultTaskPriority(target, taskParams{}),
+		Status:    TaskPending,
+		CreatedAt: time.Now(),
+		params:    taskParams{RAIDController: ctrlIdx},
+	}
+	globalQueue.enqueue(t)
+	writeJSON(w, map[string]string{"task_id": t.ID})
+}
+
+func (h *handler) handleAPIRAIDCreateMirror(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		ControllerID string   `json:"controller_id"`
+		Devices      []string `json:"devices"`
+		ArrayName    string   `json:"array_name"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON")
+		return
+	}
+	if len(req.Devices) < 2 {
+		writeError(w, http.StatusBadRequest, "at least 2 devices required")
+		return
+	}
+
+	var target, name string
+	var params taskParams
+
+	switch {
+	case strings.HasPrefix(req.ControllerID, "lsi-"):
+		ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID)
+		if !ok {
+			writeError(w, http.StatusBadRequest, "invalid controller_id")
+			return
+		}
+		target = "raid-lsi-create-mirror"
+		name = fmt.Sprintf("Create RAID 1 Mirror (LSI ctrl %d)", ctrlIdx)
+		params = taskParams{RAIDController: ctrlIdx, RAIDDevices: req.Devices}
+
+	case req.ControllerID == "vroc-0":
+		arrayName := strings.TrimSpace(req.ArrayName)
+		if arrayName == "" {
+			arrayName = "bee-mirror0"
+		}
+		target = "raid-vroc-create-mirror"
+		name = fmt.Sprintf("Create VROC RAID 1 (%s)", arrayName)
+		params = taskParams{RAIDDevices: req.Devices, RAIDArrayName: arrayName}
+
+	default:
+		writeError(w, http.StatusBadRequest, "unknown controller_id")
+		return
+	}
+
+	t := &Task{
+		ID:        newJobID(target),
+		Name:      name,
+		Target:    target,
+		Priority:  defaultTaskPriority(target, taskParams{}),
+		Status:    TaskPending,
+		CreatedAt: time.Now(),
+		params:    params,
+	}
+	globalQueue.enqueue(t)
+	writeJSON(w, map[string]string{"task_id": t.ID})
+}
+
+func (h *handler) handleAPIRAIDPrepareDrive(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		ControllerID string `json:"controller_id"`
+		Slot         string `json:"slot"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON")
+		return
+	}
+	ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID)
+	if !ok {
+		writeError(w, http.StatusBadRequest, "invalid controller_id")
+		return
+	}
+	if _, _, ok := parseRAIDSlot(req.Slot); !ok {
+		writeError(w, http.StatusBadRequest, "invalid slot")
+		return
+	}
+
+	t := &Task{
+		ID:        newJobID("raid-lsi-prepare-drive"),
+		Name:      fmt.Sprintf("Prepare drive %s (LSI ctrl %d)", req.Slot, ctrlIdx),
+		Target:    "raid-lsi-prepare-drive",
+		Priority:  defaultTaskPriority("raid-lsi-prepare-drive", taskParams{}),
+		Status:    TaskPending,
+		CreatedAt: time.Now(),
+		params:    taskParams{RAIDController: ctrlIdx, RAIDSlot: req.Slot},
+	}
+	globalQueue.enqueue(t)
+	writeJSON(w, map[string]string{"task_id": t.ID})
+}
+
+func parseLSIControllerIndex(id string) (int, bool) {
+	if !strings.HasPrefix(id, "lsi-") {
+		return 0, false
+	}
+	n, err := strconv.Atoi(strings.TrimPrefix(id, "lsi-"))
+	if err != nil || n < 0 {
+		return 0, false
+	}
+	return n, true
+}
+
+// --- Task runner functions ---
+
+func runRAIDForeignClearTask(ctx context.Context, j *jobState, ctrl int) error {
+	j.append(fmt.Sprintf("Clearing foreign configuration on controller %d...", ctrl))
+	cmd := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d/fall", ctrl), "del", "noprompt")
+	return streamCmdJob(j, cmd)
+}
+
+func runRAIDForeignImportTask(ctx context.Context, j *jobState, ctrl int) error {
+	j.append(fmt.Sprintf("Importing foreign configuration on controller %d...", ctrl))
+	cmd := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d/fall", ctrl), "import", "noprompt")
+	return streamCmdJob(j, cmd)
+}
+
+// raidPrepareAction says what (if anything) must be done to a drive in the
+// given storcli state before it can join a new VD. Derived from the Broadcom
+// StorCLI drive-state matrix: only UGood drives are accepted by "add vd";
+// JBOD/UBad convert with "set good force"; hotspares must be released first;
+// Frgn/Onln hold configuration data and must not be silently destroyed.
+type raidPrepareAction int
+
+const (
+	raidPrepNone     raidPrepareAction = iota // UGood or unknown — try add vd as-is
+	raidPrepSetGood                           // JBOD, UBad — "set good force"
+	raidPrepHotspare                          // GHS, DHS — "delete hotsparedrive", then set good
+	raidPrepBlockedFrgn
+	raidPrepBlockedOnln
+)
+
+func classifyRAIDPrepareAction(state string) raidPrepareAction {
+	switch strings.TrimSpace(state) {
+	case "JBOD", "UBad":
+		return raidPrepSetGood
+	case "GHS", "DHS":
+		return raidPrepHotspare
+	case "Frgn":
+		return raidPrepBlockedFrgn
+	case "Onln", "Offln":
+		return raidPrepBlockedOnln
+	default: // "UGood", "" (state unknown — let add vd decide)
+		return raidPrepNone
+	}
+}
+
+// raidLSIDriveStates returns EID:Slt -> State for one controller, or nil if
+// storcli/parsing fails (callers then fall back to unconditional prepare).
+func raidLSIDriveStates(ctx context.Context, ctrl int) map[string]string {
+	out, err := exec.CommandContext(ctx, "storcli64",
+		fmt.Sprintf("/c%d/eall/sall", ctrl), "show", "all", "J").Output()
+	if err != nil {
+		return nil
+	}
+	var doc struct {
+		Controllers []struct {
+			ResponseData map[string]json.RawMessage `json:"Response Data"`
+		} `json:"Controllers"`
+	}
+	if err := json.Unmarshal(out, &doc); err != nil || len(doc.Controllers) == 0 {
+		return nil
+	}
+	states := map[string]string{}
+	for _, c := range doc.Controllers {
+		for _, d := range parseStorcliResponseDataDrives(c.ResponseData) {
+			states[strings.TrimSpace(d.EIDSlt)] = strings.TrimSpace(d.State)
+		}
+	}
+	return states
+}
+
+func runRAIDLSICreateMirrorTask(ctx context.Context, j *jobState, ctrl int, drives []string) error {
+	driveList := strings.Join(drives, ",")
+	states := raidLSIDriveStates(ctx, ctrl)
+
+	// Non-UGood drives cannot be added to a VD directly — storcli fails with
+	// "resources already in use" (exit 11) or similar. Fix what is safely
+	// fixable (JBOD/UBad/hotspare), refuse what holds data (Frgn/Onln).
+	for _, drive := range drives {
+		eid, slt, ok := parseRAIDSlot(drive)
+		if !ok {
+			return fmt.Errorf("invalid drive slot %q", drive)
+		}
+		state := states[drive]
+		slotPath := fmt.Sprintf("/c%d/e%d/s%d", ctrl, eid, slt)
+
+		switch classifyRAIDPrepareAction(state) {
+		case raidPrepBlockedFrgn:
+			return fmt.Errorf("drive %s carries a foreign configuration; run the RAID Foreign Clear (or Import) task first, then retry", drive)
+		case raidPrepBlockedOnln:
+			return fmt.Errorf("drive %s is part of an existing virtual drive (state %s); delete that VD first", drive, state)
+		case raidPrepHotspare:
+			j.append(fmt.Sprintf("Drive %s is a hotspare (%s); releasing it...", drive, state))
+			rel := exec.CommandContext(ctx, "storcli64", slotPath, "delete", "hotsparedrive")
+			if err := streamCmdJob(j, rel); err != nil {
+				return fmt.Errorf("release hotspare %s: %w", drive, err)
+			}
+		case raidPrepSetGood:
+			j.append(fmt.Sprintf("Drive %s is %s; converting to Unconfigured Good (set good force)...", drive, state))
+			prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force")
+			if err := streamCmdJob(j, prep); err != nil {
+				return fmt.Errorf("set good on %s: %w", drive, err)
+			}
+		case raidPrepNone:
+			if state == "" {
+				// Drive state unknown (storcli query failed) — attempt the
+				// conversion anyway; harmless on an already-UGood drive with
+				// force, and add vd below is the real verdict.
+				j.append(fmt.Sprintf("Preparing drive %s (set good, force)...", drive))
+				prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force")
+				if err := streamCmdJob(j, prep); err != nil {
+					j.append(fmt.Sprintf("note: set good on %s: %v (continuing)", drive, err))
+				}
+			}
+		}
+	}
+
+	j.append(fmt.Sprintf("Creating RAID 1 on controller %d with drives: %s", ctrl, driveList))
+	cmd := exec.CommandContext(ctx, "storcli64",
+		fmt.Sprintf("/c%d", ctrl),
+		"add", "vd", "type=raid1",
+		fmt.Sprintf("drives=%s", driveList),
+		"pdperarray=2",
+	)
+	if err := streamCmdJob(j, cmd); err != nil {
+		// A blocked add vd is often preserved cache from a dead VD
+		// ("controller has data in cache for offline or missing virtual
+		// drives"). Surface it so the log is actionable.
+		j.append("add vd failed; checking for preserved cache...")
+		pc := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d", ctrl), "show", "preservedcache")
+		_ = streamCmdJob(j, pc)
+		j.append(fmt.Sprintf("hint: if preserved cache is listed above, clear it with: storcli64 /c%d/vall delete preservedcache (invalidates cached data of dead VDs), then retry", ctrl))
+		return err
+	}
+	return nil
+}
+
+// parseRAIDSlot splits a storcli "EID:Slt" identifier (e.g. "252:0") into
+// enclosure and slot numbers.
+func parseRAIDSlot(slot string) (eid int, slt int, ok bool) {
+	parts := strings.SplitN(strings.TrimSpace(slot), ":", 2)
+	if len(parts) != 2 {
+		return 0, 0, false
+	}
+	eid, err1 := strconv.Atoi(strings.TrimSpace(parts[0]))
+	slt, err2 := strconv.Atoi(strings.TrimSpace(parts[1]))
+	if err1 != nil || err2 != nil {
+		return 0, 0, false
+	}
+	return eid, slt, true
+}
+
+func runRAIDPrepareDriveTask(ctx context.Context, j *jobState, ctrl int, slot string) error {
+	eid, slt, ok := parseRAIDSlot(slot)
+	if !ok {
+		return fmt.Errorf("invalid slot %q", slot)
+	}
+	j.append(fmt.Sprintf("Preparing drive %s on controller %d (set good, force)...", slot, ctrl))
+	cmd := exec.CommandContext(ctx, "storcli64",
+		fmt.Sprintf("/c%d/e%d/s%d", ctrl, eid, slt),
+		"set", "good", "force",
+	)
+	return streamCmdJob(j, cmd)
+}
+
+func runRAIDVROCCreateMirrorTask(ctx context.Context, j *jobState, devices []string, arrayName string) error {
+	if arrayName == "" {
+		arrayName = "bee-mirror0"
+	}
+	devPath := "/dev/md/" + arrayName
+	args := []string{
+		"--create", devPath,
+		"--level=1",
+		fmt.Sprintf("--raid-devices=%d", len(devices)),
+		"--run",
+	}
+	args = append(args, devices...)
+	j.append(fmt.Sprintf("Creating VROC RAID 1 array %s with: %s", devPath, strings.Join(devices, " ")))
+	cmd := exec.CommandContext(ctx, "mdadm", args...)
+	return streamCmdJob(j, cmd)
+}
+
+// raidParseHumanSizeGB parses storcli size strings like "1.818 TB", "745.211 GB".
+func raidParseHumanSizeGB(s string) float64 {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return 0
+	}
+	upper := strings.ToUpper(s)
+	var mul float64
+	var numStr string
+	switch {
+	case strings.Contains(upper, " TB"):
+		mul = 1024
+		numStr = strings.TrimSpace(strings.SplitN(upper, " T", 2)[0])
+	case strings.Contains(upper, " GB"):
+		mul = 1
+		numStr = strings.TrimSpace(strings.SplitN(upper, " G", 2)[0])
+	case strings.Contains(upper, " MB"):
+		mul = 1.0 / 1024
+		numStr = strings.TrimSpace(strings.SplitN(upper, " M", 2)[0])
+	default:
+		return 0
+	}
+	v, err := strconv.ParseFloat(numStr, 64)
+	if err != nil {
+		return 0
+	}
+	return v * mul
+}
+
+// --- UI card ---
+
+func renderRAIDMgmtCard() string {
+	return `<div class="card"><div class="card-head card-head-actions">RAID Controller Management<div class="card-head-buttons"><button class="btn btn-sm btn-secondary" onclick="raidLoad()">&#8635; Refresh</button></div></div><div class="card-body">
+<div id="raid-status" style="font-size:13px;color:var(--muted);margin-bottom:8px">Loading...</div>
+<div id="raid-content"></div>
+<div id="raid-out-wrap" style="display:none;margin-top:14px">
+  <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:4px">
+    <span id="raid-out-label" style="font-size:12px;font-weight:600;color:var(--muted)">Output</span>
+    <span id="raid-out-status" style="font-size:12px"></span>
+  </div>
+  <div id="raid-terminal" class="terminal" style="max-height:260px;width:100%;box-sizing:border-box"></div>
+</div>
+</div></div>
+<script>
+(function(){
+function escHtml(s) {
+  return String(s||'').replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;');
+}
+
+var _raidControllers = [];
+
+function raidLoad() {
+  var status = document.getElementById('raid-status');
+  var content = document.getElementById('raid-content');
+  status.textContent = 'Detecting RAID controllers...';
+  status.style.color = 'var(--muted)';
+  content.innerHTML = '';
+  fetch('/api/tools/raid/status', {cache:'no-store'})
+    .then(function(r) {
+      if (!r.ok) return r.json().then(function(e) { throw new Error(e.error || r.statusText); });
+      return r.json();
+    })
+    .then(function(data) {
+      _raidControllers = data.controllers || [];
+      if (_raidControllers.length === 0) {
+        status.textContent = 'No RAID controllers detected.';
+        return;
+      }
+      status.textContent = _raidControllers.length + ' controller(s) detected.';
+      content.innerHTML = _raidControllers.map(function(c, i) {
+        return raidRenderController(c, i);
+      }).join('<hr style="margin:16px 0;border:none;border-top:1px solid var(--border)">');
+    })
+    .catch(function(e) {
+      status.textContent = 'Error: ' + e.message;
+      status.style.color = 'var(--crit-fg)';
+    });
+}
+
+function raidRenderController(c, idx) {
+  var html = '';
+  var typeLabel = c.type === 'lsi' ? 'LSI / Broadcom' : 'Intel VROC';
+  html += '<div style="font-weight:600;font-size:13px;margin-bottom:10px">' + typeLabel + ' &mdash; ' + escHtml(c.model) + '</div>';
+
+  if (c.type === 'lsi') {
+    var foreign = c.foreign_drives || [];
+    if (foreign.length > 0) {
+      html += '<div style="background:var(--warn-bg,rgba(240,192,0,0.1));border:1px solid var(--warn-border,#c8a800);border-radius:4px;padding:10px 12px;margin-bottom:12px">';
+      html += '<div style="font-weight:600;font-size:13px;margin-bottom:6px">&#9888;&#xFE0E; Foreign Configuration Detected (' + foreign.length + ' drive(s))</div>';
+      html += '<table style="margin-bottom:10px"><tr><th>Slot</th><th>Model</th><th>Size</th><th>State</th></tr>';
+      foreign.forEach(function(d) {
+        html += '<tr>'
+          + '<td style="font-family:monospace">' + escHtml(d.slot) + '</td>'
+          + '<td>' + escHtml(d.model||'—') + '</td>'
+          + '<td>' + (d.size_gb > 0 ? Math.round(d.size_gb) + ' GB' : '—') + '</td>'
+          + '<td><span class="badge badge-warn">' + escHtml(d.state) + '</span></td>'
+          + '</tr>';
+      });
+      html += '</table>';
+      html += '<div style="display:flex;gap:8px;flex-wrap:wrap">';
+      html += '<button class="btn btn-sm btn-primary" onclick="raidForeignAction(\'' + escHtml(c.id) + '\',\'import\',this)">Import Foreign Config</button>';
+      html += '<button class="btn btn-sm btn-secondary" style="color:var(--crit-fg)" onclick="raidForeignAction(\'' + escHtml(c.id) + '\',\'clear\',this)">Clear Foreign Config</button>';
+      html += '</div></div>';
+    }
+
+    html += raidRenderAllDrives(c, idx);
+    html += raidRenderMirrorSection(c, idx, 'lsi');
+  }
+
+  if (c.type === 'vroc') {
+    var arrays = c.arrays || [];
+    if (arrays.length > 0) {
+      html += '<div style="font-size:12px;font-weight:600;color:var(--muted);margin-bottom:6px;text-transform:uppercase;letter-spacing:.04em">Active Arrays</div>';
+      html += '<table style="margin-bottom:14px"><tr><th>Name</th><th>Level</th><th>Members</th><th>Status</th></tr>';
+      arrays.forEach(function(a) {
+        var badge = a.degraded
+          ? '<span class="badge badge-err">Degraded</span>'
+          : '<span class="badge badge-ok">OK</span>';
+        html += '<tr>'
+          + '<td style="font-family:monospace">' + escHtml(a.name) + '</td>'
+          + '<td>' + escHtml(a.level||'—') + '</td>'
+          + '<td style="font-family:monospace;font-size:12px">' + (a.members||[]).map(escHtml).join(', ') + '</td>'
+          + '<td>' + badge + '</td>'
+          + '</tr>';
+      });
+      html += '</table>';
+    }
+
+    html += raidRenderAllDrives(c, idx);
+    html += raidRenderMirrorSection(c, idx, 'vroc');
+  }
+
+  return html;
+}
+
+var RAID_READY_STATES = {'UGood': true, 'JBOD': true, 'available': true};
+var RAID_NO_PREPARE_STATES = {'UGood': true, 'JBOD': true, 'Frgn': true, 'Onln': true, 'Msng': true};
+
+function raidRenderAllDrives(c, idx) {
+  var drives = c.all_drives || [];
+  var isLSI = c.type === 'lsi';
+  if (drives.length === 0) {
+    return '<p style="font-size:13px;color:var(--muted);margin-bottom:12px">No drives detected on this controller.</p>';
+  }
+  var html = '<div style="font-size:12px;font-weight:600;color:var(--muted);margin-bottom:6px;text-transform:uppercase;letter-spacing:.04em">All Drives on This Controller</div>';
+  html += '<table style="margin-bottom:14px"><tr><th>' + (isLSI ? 'Slot' : 'Device') + '</th><th>Model</th><th>Size</th><th>State</th>' + (isLSI ? '<th></th>' : '') + '</tr>';
+  drives.forEach(function(d) {
+    var ready = !!RAID_READY_STATES[d.state];
+    var badgeClass = ready ? 'badge-ok' : 'badge-warn';
+    var actionCell = '';
+    if (isLSI && !RAID_NO_PREPARE_STATES[d.state]) {
+      actionCell = '<td><button class="btn btn-sm btn-secondary" onclick="raidPrepareDrive(\'' + escHtml(c.id) + '\',\'' + escHtml(d.slot) + '\',this)">Prepare</button></td>';
+    } else if (isLSI) {
+      actionCell = '<td></td>';
+    }
+    html += '<tr>'
+      + '<td style="font-family:monospace">' + escHtml(isLSI ? d.slot : d.device) + '</td>'
+      + '<td>' + escHtml(d.model||'—') + (d.serial ? ' [' + escHtml(d.serial) + ']' : '') + '</td>'
+      + '<td>' + (d.size_gb > 0 ? Math.round(d.size_gb) + ' GB' : '—') + '</td>'
+      + '<td><span class="badge ' + badgeClass + '">' + escHtml(d.state||'—') + '</span></td>'
+      + actionCell
+      + '</tr>';
+  });
+  html += '</table>';
+  return html;
+}
+
+function raidPrepareDrive(ctrlID, slot, btn) {
+  if (!confirm('Prepare drive ' + slot + ' on ' + ctrlID + ' for array creation?\n\nThis forces the drive into Unconfigured Good state. If it currently belongs to a virtual drive or holds data, that data will become inaccessible.')) {
+    return;
+  }
+  var original = btn ? btn.textContent : '';
+  if (btn) { btn.disabled = true; btn.textContent = 'Preparing...'; }
+  raidShowOutput('Prepare drive ' + slot, '', '');
+  fetch('/api/tools/raid/prepare-drive', {
+    method: 'POST',
+    headers: {'Content-Type': 'application/json'},
+    body: JSON.stringify({controller_id: ctrlID, slot: slot})
+  })
+    .then(function(r) { return r.json(); })
+    .then(function(d) {
+      if (d.error) throw new Error(d.error);
+      raidStreamTask(d.task_id, 'Prepare drive ' + slot, function() {
+        if (btn) { btn.disabled = false; btn.textContent = original; }
+        raidLoad();
+      });
+    })
+    .catch(function(e) {
+      raidShowOutput('Error', 'failed', e.message);
+      if (btn) { btn.disabled = false; btn.textContent = original; }
+    });
+}
+
+function raidRenderMirrorSection(c, idx, kind) {
+  var free = c.free_drives || [];
+  var html = '<div style="font-size:12px;font-weight:600;color:var(--muted);margin-bottom:6px;text-transform:uppercase;letter-spacing:.04em">Create RAID 1 Mirror</div>';
+
+  if (free.length < 2) {
+    html += '<p style="font-size:13px;color:var(--muted)">No unconfigured drives available (need at least 2).</p>';
+    return html;
+  }
+
+  html += '<p style="font-size:13px;color:var(--muted);margin-bottom:8px">Select exactly 2 drives:</p>';
+  html += '<div>';
+  free.forEach(function(d) {
+    var val = kind === 'lsi' ? d.slot : d.device;
+    var label = kind === 'lsi'
+      ? escHtml(d.slot) + (d.model ? ' &mdash; ' + escHtml(d.model) : '') + (d.size_gb > 0 ? ' (' + Math.round(d.size_gb) + ' GB)' : '')
+      : escHtml(d.device) + (d.model ? ' &mdash; ' + escHtml(d.model) : '') + (d.serial ? ' [' + escHtml(d.serial) + ']' : '');
+    html += '<label style="display:block;margin-bottom:4px;font-size:13px;cursor:pointer">'
+      + '<input type="checkbox" class="raid-mirror-check-' + idx + '" value="' + escHtml(val) + '"> '
+      + label + '</label>';
+  });
+  html += '</div>';
+
+  if (kind === 'vroc') {
+    html += '<div style="margin-top:10px;display:flex;align-items:center;gap:8px;flex-wrap:wrap">'
+      + '<label style="font-size:13px">Array name:&nbsp;<input type="text" id="vroc-arrayname-' + idx + '" value="bee-mirror0" style="font-family:monospace;padding:2px 6px;width:140px"></label>';
+  } else {
+    html += '<div style="margin-top:10px;display:flex;gap:8px">';
+  }
+
+  html += '<button class="btn btn-sm btn-primary raid-mirror-btn-' + idx + '" onclick="raidCreateMirror(\'' + escHtml(c.id) + '\',' + idx + ',\'' + kind + '\',this)">Create Mirror</button>';
+  html += '</div>';
+
+  return html;
+}
+
+function raidForeignAction(ctrlID, action, btn) {
+  if (action === 'clear' && !confirm('Clear foreign configuration on ' + ctrlID + '?\n\nThis will DELETE the foreign RAID metadata. Data on those drives may become inaccessible.')) {
+    return;
+  }
+  var original = btn ? btn.textContent : '';
+  if (btn) { btn.disabled = true; btn.textContent = action === 'import' ? 'Importing...' : 'Clearing...'; }
+  raidShowOutput('RAID foreign ' + action, '', '');
+  fetch('/api/tools/raid/foreign', {
+    method: 'POST',
+    headers: {'Content-Type': 'application/json'},
+    body: JSON.stringify({controller_id: ctrlID, action: action})
+  })
+    .then(function(r) { return r.json(); })
+    .then(function(d) {
+      if (d.error) throw new Error(d.error);
+      var actionLabel = action === 'import' ? 'Import foreign config' : 'Clear foreign config';
+      raidStreamTask(d.task_id, actionLabel, function() {
+        if (btn) { btn.disabled = false; btn.textContent = original; }
+        raidLoad();
+      });
+    })
+    .catch(function(e) {
+      raidShowOutput('Error', 'failed', e.message);
+      if (btn) { btn.disabled = false; btn.textContent = original; }
+    });
+}
+
+function raidCreateMirror(ctrlID, idx, kind, btn) {
+  var checks = document.querySelectorAll('.raid-mirror-check-' + idx + ':checked');
+  if (checks.length !== 2) {
+    alert('Select exactly 2 drives.');
+    return;
+  }
+  var devices = Array.from(checks).map(function(c) { return c.value; });
+  var arrayName = '';
+  if (kind === 'vroc') {
+    var nameEl = document.getElementById('vroc-arrayname-' + idx);
+    arrayName = nameEl ? nameEl.value.trim() : 'bee-mirror0';
+    if (!arrayName) arrayName = 'bee-mirror0';
+  }
+  var original = btn ? btn.textContent : '';
+  if (btn) { btn.disabled = true; btn.textContent = 'Creating...'; }
+  raidShowOutput('Create RAID 1', '', '');
+  fetch('/api/tools/raid/create-mirror', {
+    method: 'POST',
+    headers: {'Content-Type': 'application/json'},
+    body: JSON.stringify({controller_id: ctrlID, devices: devices, array_name: arrayName})
+  })
+    .then(function(r) { return r.json(); })
+    .then(function(d) {
+      if (d.error) throw new Error(d.error);
+      raidStreamTask(d.task_id, 'Create RAID 1 mirror', function() {
+        if (btn) { btn.disabled = false; btn.textContent = original; }
+        raidLoad();
+      });
+    })
+    .catch(function(e) {
+      raidShowOutput('Error', 'failed', e.message);
+      if (btn) { btn.disabled = false; btn.textContent = original; }
+    });
+}
+
+function raidShowOutput(label, status, text) {
+  var wrap = document.getElementById('raid-out-wrap');
+  var labelEl = document.getElementById('raid-out-label');
+  var statusEl = document.getElementById('raid-out-status');
+  var term = document.getElementById('raid-terminal');
+  wrap.style.display = 'block';
+  labelEl.textContent = label;
+  if (status === 'ok') {
+    statusEl.textContent = '✓ done';
+    statusEl.style.color = 'var(--ok-fg)';
+  } else if (status === 'failed') {
+    statusEl.textContent = '✗ failed';
+    statusEl.style.color = 'var(--crit-fg)';
+  } else {
+    statusEl.textContent = status;
+    statusEl.style.color = 'var(--muted)';
+  }
+  if (text !== undefined) {
+    term.textContent = text;
+    term.scrollTop = term.scrollHeight;
+  }
+}
+
+function raidStreamTask(taskID, taskName, onDone) {
+  var term = document.getElementById('raid-terminal');
+  term.textContent = '';
+  raidShowOutput(taskName || 'Running…', 'running…', undefined);
+  var es = new EventSource('/api/tasks/' + taskID + '/stream');
+  es.onmessage = function(e) {
+    term.textContent += e.data + '\n';
+    term.scrollTop = term.scrollHeight;
+  };
+  es.addEventListener('done', function(e) {
+    es.close();
+    if (!e.data) {
+      raidShowOutput(taskName, 'ok', undefined);
+    } else {
+      raidShowOutput(taskName, 'failed', undefined);
+      term.textContent += '\nFailed: ' + e.data;
+      term.scrollTop = term.scrollHeight;
+    }
+    if (onDone) onDone();
+  });
+  es.onerror = function() {
+    es.close();
+    raidShowOutput(taskName, 'failed', undefined);
+    if (onDone) onDone();
+  };
+}
+
+window.raidLoad = raidLoad;
+window.raidForeignAction = raidForeignAction;
+window.raidCreateMirror = raidCreateMirror;
+window.raidPrepareDrive = raidPrepareDrive;
+raidLoad();
+})();
+</script>`
+}

audit/internal/webui/raid_mgmt.go

@@ -1,16 +1,12 @@
 package webui
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
-	"net/http"
 	"os"
 	"os/exec"
 	"regexp"
-	"strconv"
 	"strings"
-	"time"
 )
 
 // --- Response types ---
@@ -395,698 +391,3 @@ func detectVROCController() *raidControllerInfo {
 }
 
 // --- API handlers ---
-
-func (h *handler) handleAPIRAIDStatus(w http.ResponseWriter, r *http.Request) {
-	resp := raidStatusResp{Controllers: []raidControllerInfo{}}
-
-	lsi2 := detectStorcli2Controllers()
-	if lsi := detectLSIControllers(); len(lsi) > 0 {
-		// storcli64 can enumerate a Tri-Mode controller (SAS3808-iMR/9500
-		// series) at a basic level but its drive-listing JSON parser finds
-		// no "Drive Information" for these — a zero-drives entry that
-		// storcli2 (run above) already covers correctly. Only drop it when
-		// storcli2 actually found something, so a genuinely drive-populated
-		// classic controller elsewhere in a mixed setup is never hidden.
-		for _, c := range lsi {
-			if len(c.AllDrives) == 0 && len(lsi2) > 0 {
-				continue
-			}
-			resp.Controllers = append(resp.Controllers, c)
-		}
-	}
-	if len(lsi2) > 0 {
-		resp.Controllers = append(resp.Controllers, lsi2...)
-	}
-	if vroc := detectVROCController(); vroc != nil {
-		resp.Controllers = append(resp.Controllers, *vroc)
-	}
-
-	writeJSON(w, resp)
-}
-
-func (h *handler) handleAPIRAIDForeignAction(w http.ResponseWriter, r *http.Request) {
-	var req struct {
-		ControllerID string `json:"controller_id"`
-		Action       string `json:"action"`
-	}
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON")
-		return
-	}
-	if req.Action != "import" && req.Action != "clear" {
-		writeError(w, http.StatusBadRequest, "action must be 'import' or 'clear'")
-		return
-	}
-	ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID)
-	if !ok {
-		writeError(w, http.StatusBadRequest, "invalid controller_id")
-		return
-	}
-
-	target := "raid-foreign-clear"
-	name := fmt.Sprintf("RAID Foreign Clear (ctrl %d)", ctrlIdx)
-	if req.Action == "import" {
-		target = "raid-foreign-import"
-		name = fmt.Sprintf("RAID Foreign Import (ctrl %d)", ctrlIdx)
-	}
-
-	t := &Task{
-		ID:        newJobID(target),
-		Name:      name,
-		Target:    target,
-		Priority:  defaultTaskPriority(target, taskParams{}),
-		Status:    TaskPending,
-		CreatedAt: time.Now(),
-		params:    taskParams{RAIDController: ctrlIdx},
-	}
-	globalQueue.enqueue(t)
-	writeJSON(w, map[string]string{"task_id": t.ID})
-}
-
-func (h *handler) handleAPIRAIDCreateMirror(w http.ResponseWriter, r *http.Request) {
-	var req struct {
-		ControllerID string   `json:"controller_id"`
-		Devices      []string `json:"devices"`
-		ArrayName    string   `json:"array_name"`
-	}
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON")
-		return
-	}
-	if len(req.Devices) < 2 {
-		writeError(w, http.StatusBadRequest, "at least 2 devices required")
-		return
-	}
-
-	var target, name string
-	var params taskParams
-
-	switch {
-	case strings.HasPrefix(req.ControllerID, "lsi-"):
-		ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID)
-		if !ok {
-			writeError(w, http.StatusBadRequest, "invalid controller_id")
-			return
-		}
-		target = "raid-lsi-create-mirror"
-		name = fmt.Sprintf("Create RAID 1 Mirror (LSI ctrl %d)", ctrlIdx)
-		params = taskParams{RAIDController: ctrlIdx, RAIDDevices: req.Devices}
-
-	case req.ControllerID == "vroc-0":
-		arrayName := strings.TrimSpace(req.ArrayName)
-		if arrayName == "" {
-			arrayName = "bee-mirror0"
-		}
-		target = "raid-vroc-create-mirror"
-		name = fmt.Sprintf("Create VROC RAID 1 (%s)", arrayName)
-		params = taskParams{RAIDDevices: req.Devices, RAIDArrayName: arrayName}
-
-	default:
-		writeError(w, http.StatusBadRequest, "unknown controller_id")
-		return
-	}
-
-	t := &Task{
-		ID:        newJobID(target),
-		Name:      name,
-		Target:    target,
-		Priority:  defaultTaskPriority(target, taskParams{}),
-		Status:    TaskPending,
-		CreatedAt: time.Now(),
-		params:    params,
-	}
-	globalQueue.enqueue(t)
-	writeJSON(w, map[string]string{"task_id": t.ID})
-}
-
-func (h *handler) handleAPIRAIDPrepareDrive(w http.ResponseWriter, r *http.Request) {
-	var req struct {
-		ControllerID string `json:"controller_id"`
-		Slot         string `json:"slot"`
-	}
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON")
-		return
-	}
-	ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID)
-	if !ok {
-		writeError(w, http.StatusBadRequest, "invalid controller_id")
-		return
-	}
-	if _, _, ok := parseRAIDSlot(req.Slot); !ok {
-		writeError(w, http.StatusBadRequest, "invalid slot")
-		return
-	}
-
-	t := &Task{
-		ID:        newJobID("raid-lsi-prepare-drive"),
-		Name:      fmt.Sprintf("Prepare drive %s (LSI ctrl %d)", req.Slot, ctrlIdx),
-		Target:    "raid-lsi-prepare-drive",
-		Priority:  defaultTaskPriority("raid-lsi-prepare-drive", taskParams{}),
-		Status:    TaskPending,
-		CreatedAt: time.Now(),
-		params:    taskParams{RAIDController: ctrlIdx, RAIDSlot: req.Slot},
-	}
-	globalQueue.enqueue(t)
-	writeJSON(w, map[string]string{"task_id": t.ID})
-}
-
-func parseLSIControllerIndex(id string) (int, bool) {
-	if !strings.HasPrefix(id, "lsi-") {
-		return 0, false
-	}
-	n, err := strconv.Atoi(strings.TrimPrefix(id, "lsi-"))
-	if err != nil || n < 0 {
-		return 0, false
-	}
-	return n, true
-}
-
-// --- Task runner functions ---
-
-func runRAIDForeignClearTask(ctx context.Context, j *jobState, ctrl int) error {
-	j.append(fmt.Sprintf("Clearing foreign configuration on controller %d...", ctrl))
-	cmd := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d/fall", ctrl), "del", "noprompt")
-	return streamCmdJob(j, cmd)
-}
-
-func runRAIDForeignImportTask(ctx context.Context, j *jobState, ctrl int) error {
-	j.append(fmt.Sprintf("Importing foreign configuration on controller %d...", ctrl))
-	cmd := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d/fall", ctrl), "import", "noprompt")
-	return streamCmdJob(j, cmd)
-}
-
-// raidPrepareAction says what (if anything) must be done to a drive in the
-// given storcli state before it can join a new VD. Derived from the Broadcom
-// StorCLI drive-state matrix: only UGood drives are accepted by "add vd";
-// JBOD/UBad convert with "set good force"; hotspares must be released first;
-// Frgn/Onln hold configuration data and must not be silently destroyed.
-type raidPrepareAction int
-
-const (
-	raidPrepNone      raidPrepareAction = iota // UGood or unknown — try add vd as-is
-	raidPrepSetGood                            // JBOD, UBad — "set good force"
-	raidPrepHotspare                           // GHS, DHS — "delete hotsparedrive", then set good
-	raidPrepBlockedFrgn
-	raidPrepBlockedOnln
-)
-
-func classifyRAIDPrepareAction(state string) raidPrepareAction {
-	switch strings.TrimSpace(state) {
-	case "JBOD", "UBad":
-		return raidPrepSetGood
-	case "GHS", "DHS":
-		return raidPrepHotspare
-	case "Frgn":
-		return raidPrepBlockedFrgn
-	case "Onln", "Offln":
-		return raidPrepBlockedOnln
-	default: // "UGood", "" (state unknown — let add vd decide)
-		return raidPrepNone
-	}
-}
-
-// raidLSIDriveStates returns EID:Slt -> State for one controller, or nil if
-// storcli/parsing fails (callers then fall back to unconditional prepare).
-func raidLSIDriveStates(ctx context.Context, ctrl int) map[string]string {
-	out, err := exec.CommandContext(ctx, "storcli64",
-		fmt.Sprintf("/c%d/eall/sall", ctrl), "show", "all", "J").Output()
-	if err != nil {
-		return nil
-	}
-	var doc struct {
-		Controllers []struct {
-			ResponseData map[string]json.RawMessage `json:"Response Data"`
-		} `json:"Controllers"`
-	}
-	if err := json.Unmarshal(out, &doc); err != nil || len(doc.Controllers) == 0 {
-		return nil
-	}
-	states := map[string]string{}
-	for _, c := range doc.Controllers {
-		for _, d := range parseStorcliResponseDataDrives(c.ResponseData) {
-			states[strings.TrimSpace(d.EIDSlt)] = strings.TrimSpace(d.State)
-		}
-	}
-	return states
-}
-
-func runRAIDLSICreateMirrorTask(ctx context.Context, j *jobState, ctrl int, drives []string) error {
-	driveList := strings.Join(drives, ",")
-	states := raidLSIDriveStates(ctx, ctrl)
-
-	// Non-UGood drives cannot be added to a VD directly — storcli fails with
-	// "resources already in use" (exit 11) or similar. Fix what is safely
-	// fixable (JBOD/UBad/hotspare), refuse what holds data (Frgn/Onln).
-	for _, drive := range drives {
-		eid, slt, ok := parseRAIDSlot(drive)
-		if !ok {
-			return fmt.Errorf("invalid drive slot %q", drive)
-		}
-		state := states[drive]
-		slotPath := fmt.Sprintf("/c%d/e%d/s%d", ctrl, eid, slt)
-
-		switch classifyRAIDPrepareAction(state) {
-		case raidPrepBlockedFrgn:
-			return fmt.Errorf("drive %s carries a foreign configuration; run the RAID Foreign Clear (or Import) task first, then retry", drive)
-		case raidPrepBlockedOnln:
-			return fmt.Errorf("drive %s is part of an existing virtual drive (state %s); delete that VD first", drive, state)
-		case raidPrepHotspare:
-			j.append(fmt.Sprintf("Drive %s is a hotspare (%s); releasing it...", drive, state))
-			rel := exec.CommandContext(ctx, "storcli64", slotPath, "delete", "hotsparedrive")
-			if err := streamCmdJob(j, rel); err != nil {
-				return fmt.Errorf("release hotspare %s: %w", drive, err)
-			}
-		case raidPrepSetGood:
-			j.append(fmt.Sprintf("Drive %s is %s; converting to Unconfigured Good (set good force)...", drive, state))
-			prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force")
-			if err := streamCmdJob(j, prep); err != nil {
-				return fmt.Errorf("set good on %s: %w", drive, err)
-			}
-		case raidPrepNone:
-			if state == "" {
-				// Drive state unknown (storcli query failed) — attempt the
-				// conversion anyway; harmless on an already-UGood drive with
-				// force, and add vd below is the real verdict.
-				j.append(fmt.Sprintf("Preparing drive %s (set good, force)...", drive))
-				prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force")
-				if err := streamCmdJob(j, prep); err != nil {
-					j.append(fmt.Sprintf("note: set good on %s: %v (continuing)", drive, err))
-				}
-			}
-		}
-	}
-
-	j.append(fmt.Sprintf("Creating RAID 1 on controller %d with drives: %s", ctrl, driveList))
-	cmd := exec.CommandContext(ctx, "storcli64",
-		fmt.Sprintf("/c%d", ctrl),
-		"add", "vd", "type=raid1",
-		fmt.Sprintf("drives=%s", driveList),
-		"pdperarray=2",
-	)
-	if err := streamCmdJob(j, cmd); err != nil {
-		// A blocked add vd is often preserved cache from a dead VD
-		// ("controller has data in cache for offline or missing virtual
-		// drives"). Surface it so the log is actionable.
-		j.append("add vd failed; checking for preserved cache...")
-		pc := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d", ctrl), "show", "preservedcache")
-		_ = streamCmdJob(j, pc)
-		j.append(fmt.Sprintf("hint: if preserved cache is listed above, clear it with: storcli64 /c%d/vall delete preservedcache (invalidates cached data of dead VDs), then retry", ctrl))
-		return err
-	}
-	return nil
-}
-
-// parseRAIDSlot splits a storcli "EID:Slt" identifier (e.g. "252:0") into
-// enclosure and slot numbers.
-func parseRAIDSlot(slot string) (eid int, slt int, ok bool) {
-	parts := strings.SplitN(strings.TrimSpace(slot), ":", 2)
-	if len(parts) != 2 {
-		return 0, 0, false
-	}
-	eid, err1 := strconv.Atoi(strings.TrimSpace(parts[0]))
-	slt, err2 := strconv.Atoi(strings.TrimSpace(parts[1]))
-	if err1 != nil || err2 != nil {
-		return 0, 0, false
-	}
-	return eid, slt, true
-}
-
-func runRAIDPrepareDriveTask(ctx context.Context, j *jobState, ctrl int, slot string) error {
-	eid, slt, ok := parseRAIDSlot(slot)
-	if !ok {
-		return fmt.Errorf("invalid slot %q", slot)
-	}
-	j.append(fmt.Sprintf("Preparing drive %s on controller %d (set good, force)...", slot, ctrl))
-	cmd := exec.CommandContext(ctx, "storcli64",
-		fmt.Sprintf("/c%d/e%d/s%d", ctrl, eid, slt),
-		"set", "good", "force",
-	)
-	return streamCmdJob(j, cmd)
-}
-
-func runRAIDVROCCreateMirrorTask(ctx context.Context, j *jobState, devices []string, arrayName string) error {
-	if arrayName == "" {
-		arrayName = "bee-mirror0"
-	}
-	devPath := "/dev/md/" + arrayName
-	args := []string{
-		"--create", devPath,
-		"--level=1",
-		fmt.Sprintf("--raid-devices=%d", len(devices)),
-		"--run",
-	}
-	args = append(args, devices...)
-	j.append(fmt.Sprintf("Creating VROC RAID 1 array %s with: %s", devPath, strings.Join(devices, " ")))
-	cmd := exec.CommandContext(ctx, "mdadm", args...)
-	return streamCmdJob(j, cmd)
-}
-
-// raidParseHumanSizeGB parses storcli size strings like "1.818 TB", "745.211 GB".
-func raidParseHumanSizeGB(s string) float64 {
-	s = strings.TrimSpace(s)
-	if s == "" {
-		return 0
-	}
-	upper := strings.ToUpper(s)
-	var mul float64
-	var numStr string
-	switch {
-	case strings.Contains(upper, " TB"):
-		mul = 1024
-		numStr = strings.TrimSpace(strings.SplitN(upper, " T", 2)[0])
-	case strings.Contains(upper, " GB"):
-		mul = 1
-		numStr = strings.TrimSpace(strings.SplitN(upper, " G", 2)[0])
-	case strings.Contains(upper, " MB"):
-		mul = 1.0 / 1024
-		numStr = strings.TrimSpace(strings.SplitN(upper, " M", 2)[0])
-	default:
-		return 0
-	}
-	v, err := strconv.ParseFloat(numStr, 64)
-	if err != nil {
-		return 0
-	}
-	return v * mul
-}
-
-// --- UI card ---
-
-func renderRAIDMgmtCard() string {
-	return `<div class="card"><div class="card-head card-head-actions">RAID Controller Management<div class="card-head-buttons"><button class="btn btn-sm btn-secondary" onclick="raidLoad()">&#8635; Refresh</button></div></div><div class="card-body">
-<div id="raid-status" style="font-size:13px;color:var(--muted);margin-bottom:8px">Loading...</div>
-<div id="raid-content"></div>
-<div id="raid-out-wrap" style="display:none;margin-top:14px">
-  <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:4px">
-    <span id="raid-out-label" style="font-size:12px;font-weight:600;color:var(--muted)">Output</span>
-    <span id="raid-out-status" style="font-size:12px"></span>
-  </div>
-  <div id="raid-terminal" class="terminal" style="max-height:260px;width:100%;box-sizing:border-box"></div>
-</div>
-</div></div>
-<script>
-(function(){
-function escHtml(s) {
-  return String(s||'').replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;');
-}
-
-var _raidControllers = [];
-
-function raidLoad() {
-  var status = document.getElementById('raid-status');
-  var content = document.getElementById('raid-content');
-  status.textContent = 'Detecting RAID controllers...';
-  status.style.color = 'var(--muted)';
-  content.innerHTML = '';
-  fetch('/api/tools/raid/status', {cache:'no-store'})
-    .then(function(r) {
-      if (!r.ok) return r.json().then(function(e) { throw new Error(e.error || r.statusText); });
-      return r.json();
-    })
-    .then(function(data) {
-      _raidControllers = data.controllers || [];
-      if (_raidControllers.length === 0) {
-        status.textContent = 'No RAID controllers detected.';
-        return;
-      }
-      status.textContent = _raidControllers.length + ' controller(s) detected.';
-      content.innerHTML = _raidControllers.map(function(c, i) {
-        return raidRenderController(c, i);
-      }).join('<hr style="margin:16px 0;border:none;border-top:1px solid var(--border)">');
-    })
-    .catch(function(e) {
-      status.textContent = 'Error: ' + e.message;
-      status.style.color = 'var(--crit-fg)';
-    });
-}
-
-function raidRenderController(c, idx) {
-  var html = '';
-  var typeLabel = c.type === 'lsi' ? 'LSI / Broadcom' : 'Intel VROC';
-  html += '<div style="font-weight:600;font-size:13px;margin-bottom:10px">' + typeLabel + ' &mdash; ' + escHtml(c.model) + '</div>';
-
-  if (c.type === 'lsi') {
-    var foreign = c.foreign_drives || [];
-    if (foreign.length > 0) {
-      html += '<div style="background:var(--warn-bg,rgba(240,192,0,0.1));border:1px solid var(--warn-border,#c8a800);border-radius:4px;padding:10px 12px;margin-bottom:12px">';
-      html += '<div style="font-weight:600;font-size:13px;margin-bottom:6px">&#9888;&#xFE0E; Foreign Configuration Detected (' + foreign.length + ' drive(s))</div>';
-      html += '<table style="margin-bottom:10px"><tr><th>Slot</th><th>Model</th><th>Size</th><th>State</th></tr>';
-      foreign.forEach(function(d) {
-        html += '<tr>'
-          + '<td style="font-family:monospace">' + escHtml(d.slot) + '</td>'
-          + '<td>' + escHtml(d.model||'—') + '</td>'
-          + '<td>' + (d.size_gb > 0 ? Math.round(d.size_gb) + ' GB' : '—') + '</td>'
-          + '<td><span class="badge badge-warn">' + escHtml(d.state) + '</span></td>'
-          + '</tr>';
-      });
-      html += '</table>';
-      html += '<div style="display:flex;gap:8px;flex-wrap:wrap">';
-      html += '<button class="btn btn-sm btn-primary" onclick="raidForeignAction(\'' + escHtml(c.id) + '\',\'import\',this)">Import Foreign Config</button>';
-      html += '<button class="btn btn-sm btn-secondary" style="color:var(--crit-fg)" onclick="raidForeignAction(\'' + escHtml(c.id) + '\',\'clear\',this)">Clear Foreign Config</button>';
-      html += '</div></div>';
-    }
-
-    html += raidRenderAllDrives(c, idx);
-    html += raidRenderMirrorSection(c, idx, 'lsi');
-  }
-
-  if (c.type === 'vroc') {
-    var arrays = c.arrays || [];
-    if (arrays.length > 0) {
-      html += '<div style="font-size:12px;font-weight:600;color:var(--muted);margin-bottom:6px;text-transform:uppercase;letter-spacing:.04em">Active Arrays</div>';
-      html += '<table style="margin-bottom:14px"><tr><th>Name</th><th>Level</th><th>Members</th><th>Status</th></tr>';
-      arrays.forEach(function(a) {
-        var badge = a.degraded
-          ? '<span class="badge badge-err">Degraded</span>'
-          : '<span class="badge badge-ok">OK</span>';
-        html += '<tr>'
-          + '<td style="font-family:monospace">' + escHtml(a.name) + '</td>'
-          + '<td>' + escHtml(a.level||'—') + '</td>'
-          + '<td style="font-family:monospace;font-size:12px">' + (a.members||[]).map(escHtml).join(', ') + '</td>'
-          + '<td>' + badge + '</td>'
-          + '</tr>';
-      });
-      html += '</table>';
-    }
-
-    html += raidRenderAllDrives(c, idx);
-    html += raidRenderMirrorSection(c, idx, 'vroc');
-  }
-
-  return html;
-}
-
-var RAID_READY_STATES = {'UGood': true, 'JBOD': true, 'available': true};
-var RAID_NO_PREPARE_STATES = {'UGood': true, 'JBOD': true, 'Frgn': true, 'Onln': true, 'Msng': true};
-
-function raidRenderAllDrives(c, idx) {
-  var drives = c.all_drives || [];
-  var isLSI = c.type === 'lsi';
-  if (drives.length === 0) {
-    return '<p style="font-size:13px;color:var(--muted);margin-bottom:12px">No drives detected on this controller.</p>';
-  }
-  var html = '<div style="font-size:12px;font-weight:600;color:var(--muted);margin-bottom:6px;text-transform:uppercase;letter-spacing:.04em">All Drives on This Controller</div>';
-  html += '<table style="margin-bottom:14px"><tr><th>' + (isLSI ? 'Slot' : 'Device') + '</th><th>Model</th><th>Size</th><th>State</th>' + (isLSI ? '<th></th>' : '') + '</tr>';
-  drives.forEach(function(d) {
-    var ready = !!RAID_READY_STATES[d.state];
-    var badgeClass = ready ? 'badge-ok' : 'badge-warn';
-    var actionCell = '';
-    if (isLSI && !RAID_NO_PREPARE_STATES[d.state]) {
-      actionCell = '<td><button class="btn btn-sm btn-secondary" onclick="raidPrepareDrive(\'' + escHtml(c.id) + '\',\'' + escHtml(d.slot) + '\',this)">Prepare</button></td>';
-    } else if (isLSI) {
-      actionCell = '<td></td>';
-    }
-    html += '<tr>'
-      + '<td style="font-family:monospace">' + escHtml(isLSI ? d.slot : d.device) + '</td>'
-      + '<td>' + escHtml(d.model||'—') + (d.serial ? ' [' + escHtml(d.serial) + ']' : '') + '</td>'
-      + '<td>' + (d.size_gb > 0 ? Math.round(d.size_gb) + ' GB' : '—') + '</td>'
-      + '<td><span class="badge ' + badgeClass + '">' + escHtml(d.state||'—') + '</span></td>'
-      + actionCell
-      + '</tr>';
-  });
-  html += '</table>';
-  return html;
-}
-
-function raidPrepareDrive(ctrlID, slot, btn) {
-  if (!confirm('Prepare drive ' + slot + ' on ' + ctrlID + ' for array creation?\n\nThis forces the drive into Unconfigured Good state. If it currently belongs to a virtual drive or holds data, that data will become inaccessible.')) {
-    return;
-  }
-  var original = btn ? btn.textContent : '';
-  if (btn) { btn.disabled = true; btn.textContent = 'Preparing...'; }
-  raidShowOutput('Prepare drive ' + slot, '', '');
-  fetch('/api/tools/raid/prepare-drive', {
-    method: 'POST',
-    headers: {'Content-Type': 'application/json'},
-    body: JSON.stringify({controller_id: ctrlID, slot: slot})
-  })
-    .then(function(r) { return r.json(); })
-    .then(function(d) {
-      if (d.error) throw new Error(d.error);
-      raidStreamTask(d.task_id, 'Prepare drive ' + slot, function() {
-        if (btn) { btn.disabled = false; btn.textContent = original; }
-        raidLoad();
-      });
-    })
-    .catch(function(e) {
-      raidShowOutput('Error', 'failed', e.message);
-      if (btn) { btn.disabled = false; btn.textContent = original; }
-    });
-}
-
-function raidRenderMirrorSection(c, idx, kind) {
-  var free = c.free_drives || [];
-  var html = '<div style="font-size:12px;font-weight:600;color:var(--muted);margin-bottom:6px;text-transform:uppercase;letter-spacing:.04em">Create RAID 1 Mirror</div>';
-
-  if (free.length < 2) {
-    html += '<p style="font-size:13px;color:var(--muted)">No unconfigured drives available (need at least 2).</p>';
-    return html;
-  }
-
-  html += '<p style="font-size:13px;color:var(--muted);margin-bottom:8px">Select exactly 2 drives:</p>';
-  html += '<div>';
-  free.forEach(function(d) {
-    var val = kind === 'lsi' ? d.slot : d.device;
-    var label = kind === 'lsi'
-      ? escHtml(d.slot) + (d.model ? ' &mdash; ' + escHtml(d.model) : '') + (d.size_gb > 0 ? ' (' + Math.round(d.size_gb) + ' GB)' : '')
-      : escHtml(d.device) + (d.model ? ' &mdash; ' + escHtml(d.model) : '') + (d.serial ? ' [' + escHtml(d.serial) + ']' : '');
-    html += '<label style="display:block;margin-bottom:4px;font-size:13px;cursor:pointer">'
-      + '<input type="checkbox" class="raid-mirror-check-' + idx + '" value="' + escHtml(val) + '"> '
-      + label + '</label>';
-  });
-  html += '</div>';
-
-  if (kind === 'vroc') {
-    html += '<div style="margin-top:10px;display:flex;align-items:center;gap:8px;flex-wrap:wrap">'
-      + '<label style="font-size:13px">Array name:&nbsp;<input type="text" id="vroc-arrayname-' + idx + '" value="bee-mirror0" style="font-family:monospace;padding:2px 6px;width:140px"></label>';
-  } else {
-    html += '<div style="margin-top:10px;display:flex;gap:8px">';
-  }
-
-  html += '<button class="btn btn-sm btn-primary raid-mirror-btn-' + idx + '" onclick="raidCreateMirror(\'' + escHtml(c.id) + '\',' + idx + ',\'' + kind + '\',this)">Create Mirror</button>';
-  html += '</div>';
-
-  return html;
-}
-
-function raidForeignAction(ctrlID, action, btn) {
-  if (action === 'clear' && !confirm('Clear foreign configuration on ' + ctrlID + '?\n\nThis will DELETE the foreign RAID metadata. Data on those drives may become inaccessible.')) {
-    return;
-  }
-  var original = btn ? btn.textContent : '';
-  if (btn) { btn.disabled = true; btn.textContent = action === 'import' ? 'Importing...' : 'Clearing...'; }
-  raidShowOutput('RAID foreign ' + action, '', '');
-  fetch('/api/tools/raid/foreign', {
-    method: 'POST',
-    headers: {'Content-Type': 'application/json'},
-    body: JSON.stringify({controller_id: ctrlID, action: action})
-  })
-    .then(function(r) { return r.json(); })
-    .then(function(d) {
-      if (d.error) throw new Error(d.error);
-      var actionLabel = action === 'import' ? 'Import foreign config' : 'Clear foreign config';
-      raidStreamTask(d.task_id, actionLabel, function() {
-        if (btn) { btn.disabled = false; btn.textContent = original; }
-        raidLoad();
-      });
-    })
-    .catch(function(e) {
-      raidShowOutput('Error', 'failed', e.message);
-      if (btn) { btn.disabled = false; btn.textContent = original; }
-    });
-}
-
-function raidCreateMirror(ctrlID, idx, kind, btn) {
-  var checks = document.querySelectorAll('.raid-mirror-check-' + idx + ':checked');
-  if (checks.length !== 2) {
-    alert('Select exactly 2 drives.');
-    return;
-  }
-  var devices = Array.from(checks).map(function(c) { return c.value; });
-  var arrayName = '';
-  if (kind === 'vroc') {
-    var nameEl = document.getElementById('vroc-arrayname-' + idx);
-    arrayName = nameEl ? nameEl.value.trim() : 'bee-mirror0';
-    if (!arrayName) arrayName = 'bee-mirror0';
-  }
-  var original = btn ? btn.textContent : '';
-  if (btn) { btn.disabled = true; btn.textContent = 'Creating...'; }
-  raidShowOutput('Create RAID 1', '', '');
-  fetch('/api/tools/raid/create-mirror', {
-    method: 'POST',
-    headers: {'Content-Type': 'application/json'},
-    body: JSON.stringify({controller_id: ctrlID, devices: devices, array_name: arrayName})
-  })
-    .then(function(r) { return r.json(); })
-    .then(function(d) {
-      if (d.error) throw new Error(d.error);
-      raidStreamTask(d.task_id, 'Create RAID 1 mirror', function() {
-        if (btn) { btn.disabled = false; btn.textContent = original; }
-        raidLoad();
-      });
-    })
-    .catch(function(e) {
-      raidShowOutput('Error', 'failed', e.message);
-      if (btn) { btn.disabled = false; btn.textContent = original; }
-    });
-}
-
-function raidShowOutput(label, status, text) {
-  var wrap = document.getElementById('raid-out-wrap');
-  var labelEl = document.getElementById('raid-out-label');
-  var statusEl = document.getElementById('raid-out-status');
-  var term = document.getElementById('raid-terminal');
-  wrap.style.display = 'block';
-  labelEl.textContent = label;
-  if (status === 'ok') {
-    statusEl.textContent = '✓ done';
-    statusEl.style.color = 'var(--ok-fg)';
-  } else if (status === 'failed') {
-    statusEl.textContent = '✗ failed';
-    statusEl.style.color = 'var(--crit-fg)';
-  } else {
-    statusEl.textContent = status;
-    statusEl.style.color = 'var(--muted)';
-  }
-  if (text !== undefined) {
-    term.textContent = text;
-    term.scrollTop = term.scrollHeight;
-  }
-}
-
-function raidStreamTask(taskID, taskName, onDone) {
-  var term = document.getElementById('raid-terminal');
-  term.textContent = '';
-  raidShowOutput(taskName || 'Running…', 'running…', undefined);
-  var es = new EventSource('/api/tasks/' + taskID + '/stream');
-  es.onmessage = function(e) {
-    term.textContent += e.data + '\n';
-    term.scrollTop = term.scrollHeight;
-  };
-  es.addEventListener('done', function(e) {
-    es.close();
-    if (!e.data) {
-      raidShowOutput(taskName, 'ok', undefined);
-    } else {
-      raidShowOutput(taskName, 'failed', undefined);
-      term.textContent += '\nFailed: ' + e.data;
-      term.scrollTop = term.scrollHeight;
-    }
-    if (onDone) onDone();
-  });
-  es.onerror = function() {
-    es.close();
-    raidShowOutput(taskName, 'failed', undefined);
-    if (onDone) onDone();
-  };
-}
-
-window.raidLoad = raidLoad;
-window.raidForeignAction = raidForeignAction;
-window.raidCreateMirror = raidCreateMirror;
-window.raidPrepareDrive = raidPrepareDrive;
-raidLoad();
-})();
-</script>`
-}

audit/internal/webui/saa_dmi.go

@@ -33,7 +33,6 @@ var (
 	dmiVersionRE = regexp.MustCompile(`(?i)^version\s*=`)
 )
 
-
 // parseDMIFile parses the DMI.txt produced by "saa GetDmiInfo".
 // Real format (from SAA User Guide 4.8.1):
 //
@@ -211,4 +210,3 @@ func runSAADMIWriteTask(ctx context.Context, j *jobState, exportDir string, p ta
 	j.append("Done. Reboot the server for changes to take effect.")
 	return nil
 }
-

audit/internal/webui/server.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"html"
 	"io"
 	"log/slog"
 	"mime"
@@ -14,7 +13,6 @@ import (
 	"os"
 	"path/filepath"
 	"runtime/debug"
-	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -276,6 +274,7 @@ func NewHandler(opts HandlerOptions) http.Handler {
 	mux.HandleFunc("POST /api/sat/memory-stress/run", h.handleAPISATRun("memory-stress"))
 	mux.HandleFunc("POST /api/sat/sat-stress/run", h.handleAPISATRun("sat-stress"))
 	mux.HandleFunc("POST /api/sat/platform-stress/run", h.handleAPISATRun("platform-stress"))
+	mux.HandleFunc("POST /api/sat/run-all", h.handleAPISATRunAll)
 	mux.HandleFunc("GET /api/sat/stream", h.handleAPISATStream)
 	mux.HandleFunc("POST /api/sat/abort", h.handleAPISATAbort)
 	mux.HandleFunc("POST /api/bee-bench/nvidia/perf/run", h.handleAPIBenchmarkNvidiaRunKind("nvidia-bench-perf"))
@@ -619,677 +618,6 @@ func (h *handler) handleViewer(w http.ResponseWriter, r *http.Request) {
 	_, _ = w.Write(body)
 }
 
-func (h *handler) handleMetricsChartSVG(w http.ResponseWriter, r *http.Request) {
-	path := strings.TrimPrefix(r.URL.Path, "/api/metrics/chart/")
-	path = strings.TrimSuffix(path, ".svg")
-
-	if h.metricsDB == nil {
-		http.Error(w, "metrics database not available", http.StatusServiceUnavailable)
-		return
-	}
-	samples, err := h.metricsDB.LoadAll()
-	if err != nil || len(samples) == 0 {
-		http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable)
-		return
-	}
-	timeline := metricsTimelineSegments(samples, time.Now())
-	if idx, sub, ok := parseGPUChartPath(path); ok && sub == "overview" {
-		var overviewOk bool
-		var buf []byte
-		buf, overviewOk, err = renderGPUOverviewChartSVG(idx, samples, timeline)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-		if !overviewOk {
-			http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable)
-			return
-		}
-		w.Header().Set("Content-Type", "image/svg+xml")
-		w.Header().Set("Cache-Control", "no-store")
-		_, _ = w.Write(buf)
-		return
-	}
-	datasets, names, labels, title, yMin, yMax, stacked, ok := chartDataFromSamples(path, samples)
-	if !ok {
-		http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable)
-		return
-	}
-
-	var buf []byte
-	if stacked {
-		buf, err = renderStackedMetricChartSVG(
-			title,
-			labels,
-			sampleTimes(samples),
-			datasets,
-			names,
-			yMax,
-			chartCanvasHeightForPath(path, len(names)),
-			timeline,
-		)
-	} else {
-		buf, err = renderMetricChartSVG(
-			title,
-			labels,
-			sampleTimes(samples),
-			datasets,
-			names,
-			yMin,
-			yMax,
-			chartCanvasHeightForPath(path, len(names)),
-			timeline,
-		)
-	}
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	w.Header().Set("Content-Type", "image/svg+xml")
-	w.Header().Set("Cache-Control", "no-store")
-	_, _ = w.Write(buf)
-}
-
-func chartDataFromSamples(path string, samples []platform.LiveMetricSample) (datasets [][]float64, names []string, labels []string, title string, yMin, yMax *float64, stacked bool, ok bool) {
-	labels = sampleTimeLabels(samples)
-
-	switch {
-	case path == "server-load":
-		title = "CPU / Memory Load"
-		cpu := make([]float64, len(samples))
-		mem := make([]float64, len(samples))
-		for i, s := range samples {
-			cpu[i] = s.CPULoadPct
-			mem[i] = s.MemLoadPct
-		}
-		datasets = [][]float64{cpu, mem}
-		names = []string{"CPU Load %", "Mem Load %"}
-		yMin = floatPtr(0)
-		yMax = floatPtr(100)
-
-	case path == "server-temp", path == "server-temp-cpu":
-		title = "CPU Temperature"
-		datasets, names = namedTempDatasets(samples, "cpu")
-		yMin = floatPtr(0)
-		yMax = autoMax120(datasets...)
-
-	case path == "server-temp-gpu":
-		title = "GPU Temperature"
-		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.TempC })
-		yMin = floatPtr(0)
-		yMax = autoMax120(datasets...)
-
-	case path == "server-temp-ambient":
-		title = "Ambient / Other Sensors"
-		datasets, names = namedTempDatasets(samples, "ambient")
-		yMin = floatPtr(0)
-		yMax = autoMax120(datasets...)
-
-	case path == "server-power":
-		title = "System Power"
-		power := make([]float64, len(samples))
-		label := "Power W"
-		for i, s := range samples {
-			power[i] = s.PowerW
-			if strings.TrimSpace(s.PowerSource) != "" {
-				label = fmt.Sprintf("Power W · %s", s.PowerSource)
-				if strings.TrimSpace(s.PowerMode) != "" {
-					label += fmt.Sprintf(" (%s)", s.PowerMode)
-				}
-			}
-		}
-		power = normalizePowerSeries(power)
-		datasets = [][]float64{power}
-		names = []string{label}
-		yMin = floatPtr(0)
-		yMax = autoMax120(power)
-
-	case path == "server-fans":
-		title = "Fan RPM"
-		datasets, names = namedFanDatasets(samples)
-		yMin, yMax = autoBounds120(datasets...)
-
-	case path == "gpu-all-load":
-		title = "GPU Compute Load"
-		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.UsagePct })
-		yMin = floatPtr(0)
-		yMax = floatPtr(100)
-
-	case path == "gpu-all-memload":
-		title = "GPU Memory Load"
-		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.MemUsagePct })
-		yMin = floatPtr(0)
-		yMax = floatPtr(100)
-
-	case path == "gpu-all-power":
-		title = "GPU Power"
-		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.PowerW })
-		yMin, yMax = autoBounds120(datasets...)
-
-	case path == "gpu-all-temp":
-		title = "GPU Temperature"
-		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.TempC })
-		yMin = floatPtr(0)
-		yMax = autoMax120(datasets...)
-
-	case path == "gpu-all-clock":
-		title = "GPU Core Clock"
-		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.ClockMHz })
-		yMin, yMax = autoBounds120(datasets...)
-
-	case path == "gpu-all-memclock":
-		title = "GPU Memory Clock"
-		datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.MemClockMHz })
-		yMin, yMax = autoBounds120(datasets...)
-
-	case strings.HasPrefix(path, "gpu/"):
-		idx, sub, ok := parseGPUChartPath(path)
-		if !ok {
-			return nil, nil, nil, "", nil, nil, false, false
-		}
-		switch sub {
-		case "load":
-			title = gpuDisplayLabel(idx) + " Load"
-			util := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.UsagePct })
-			mem := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.MemUsagePct })
-			if util == nil && mem == nil {
-				return nil, nil, nil, "", nil, nil, false, false
-			}
-			datasets = [][]float64{coalesceDataset(util, len(samples)), coalesceDataset(mem, len(samples))}
-			names = []string{"Load %", "Mem %"}
-			yMin = floatPtr(0)
-			yMax = floatPtr(100)
-		case "temp":
-			title = gpuDisplayLabel(idx) + " Temperature"
-			temp := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.TempC })
-			if temp == nil {
-				return nil, nil, nil, "", nil, nil, false, false
-			}
-			datasets = [][]float64{temp}
-			names = []string{"Temp °C"}
-			yMin = floatPtr(0)
-			yMax = autoMax120(temp)
-		case "clock":
-			title = gpuDisplayLabel(idx) + " Core Clock"
-			clock := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.ClockMHz })
-			if clock == nil {
-				return nil, nil, nil, "", nil, nil, false, false
-			}
-			datasets = [][]float64{clock}
-			names = []string{"Core Clock MHz"}
-			yMin, yMax = autoBounds120(clock)
-		case "memclock":
-			title = gpuDisplayLabel(idx) + " Memory Clock"
-			clock := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.MemClockMHz })
-			if clock == nil {
-				return nil, nil, nil, "", nil, nil, false, false
-			}
-			datasets = [][]float64{clock}
-			names = []string{"Memory Clock MHz"}
-			yMin, yMax = autoBounds120(clock)
-		default:
-			title = gpuDisplayLabel(idx) + " Power"
-			power := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.PowerW })
-			if power == nil {
-				return nil, nil, nil, "", nil, nil, false, false
-			}
-			datasets = [][]float64{power}
-			names = []string{"Power W"}
-			yMin, yMax = autoBounds120(power)
-		}
-
-	default:
-		return nil, nil, nil, "", nil, nil, false, false
-	}
-
-	return datasets, names, labels, title, yMin, yMax, stacked, len(datasets) > 0
-}
-
-func parseGPUChartPath(path string) (idx int, sub string, ok bool) {
-	if !strings.HasPrefix(path, "gpu/") {
-		return 0, "", false
-	}
-	rest := strings.TrimPrefix(path, "gpu/")
-	if rest == "" {
-		return 0, "", false
-	}
-	sub = ""
-	if i := strings.LastIndex(rest, "-"); i > 0 {
-		sub = rest[i+1:]
-		rest = rest[:i]
-	}
-	n, err := fmt.Sscanf(rest, "%d", &idx)
-	if err != nil || n != 1 {
-		return 0, "", false
-	}
-	return idx, sub, true
-}
-
-func sampleTimeLabels(samples []platform.LiveMetricSample) []string {
-	labels := make([]string, len(samples))
-	if len(samples) == 0 {
-		return labels
-	}
-	times := make([]time.Time, len(samples))
-	for i, s := range samples {
-		times[i] = s.Timestamp
-	}
-	sameDay := timestampsSameLocalDay(times)
-	for i, s := range samples {
-		labels[i] = formatTimelineLabel(s.Timestamp.Local(), sameDay)
-	}
-	return labels
-}
-
-func namedTempDatasets(samples []platform.LiveMetricSample, group string) ([][]float64, []string) {
-	seen := map[string]bool{}
-	var names []string
-	for _, s := range samples {
-		for _, t := range s.Temps {
-			if t.Group == group && !seen[t.Name] {
-				seen[t.Name] = true
-				names = append(names, t.Name)
-			}
-		}
-	}
-	sort.Strings(names)
-	datasets := make([][]float64, 0, len(names))
-	for _, name := range names {
-		ds := make([]float64, len(samples))
-		for i, s := range samples {
-			for _, t := range s.Temps {
-				if t.Group == group && t.Name == name {
-					ds[i] = t.Celsius
-					break
-				}
-			}
-		}
-		datasets = append(datasets, ds)
-	}
-	return datasets, names
-}
-
-func namedFanDatasets(samples []platform.LiveMetricSample) ([][]float64, []string) {
-	seen := map[string]bool{}
-	var names []string
-	for _, s := range samples {
-		for _, f := range s.Fans {
-			if !seen[f.Name] {
-				seen[f.Name] = true
-				names = append(names, f.Name)
-			}
-		}
-	}
-	sort.Strings(names)
-	datasets := make([][]float64, 0, len(names))
-	for _, name := range names {
-		ds := make([]float64, len(samples))
-		for i, s := range samples {
-			for _, f := range s.Fans {
-				if f.Name == name {
-					ds[i] = f.RPM
-					break
-				}
-			}
-		}
-		datasets = append(datasets, normalizeFanSeries(ds))
-	}
-	return datasets, names
-}
-
-func gpuDatasets(samples []platform.LiveMetricSample, pick func(platform.GPUMetricRow) float64) ([][]float64, []string) {
-	seen := map[int]bool{}
-	var indices []int
-	for _, s := range samples {
-		for _, g := range s.GPUs {
-			if !seen[g.GPUIndex] {
-				seen[g.GPUIndex] = true
-				indices = append(indices, g.GPUIndex)
-			}
-		}
-	}
-	sort.Ints(indices)
-	datasets := make([][]float64, 0, len(indices))
-	names := make([]string, 0, len(indices))
-	for _, idx := range indices {
-		ds := gpuDatasetByIndex(samples, idx, pick)
-		if ds == nil {
-			continue
-		}
-		datasets = append(datasets, ds)
-		names = append(names, gpuDisplayLabel(idx))
-	}
-	return datasets, names
-}
-
-func gpuDatasetByIndex(samples []platform.LiveMetricSample, idx int, pick func(platform.GPUMetricRow) float64) []float64 {
-	found := false
-	ds := make([]float64, len(samples))
-	for i, s := range samples {
-		for _, g := range s.GPUs {
-			if g.GPUIndex == idx {
-				ds[i] = pick(g)
-				found = true
-				break
-			}
-		}
-	}
-	if !found {
-		return nil
-	}
-	return ds
-}
-
-func coalesceDataset(ds []float64, n int) []float64 {
-	if ds != nil {
-		return ds
-	}
-	return make([]float64, n)
-}
-
-func normalizePowerSeries(ds []float64) []float64 {
-	if len(ds) == 0 {
-		return nil
-	}
-	out := make([]float64, len(ds))
-	copy(out, ds)
-	last := 0.0
-	haveLast := false
-	for i, v := range out {
-		if v > 0 {
-			last = v
-			haveLast = true
-			continue
-		}
-		if haveLast {
-			out[i] = last
-		}
-	}
-	return out
-}
-
-// psuSlotsFromSamples returns the sorted list of PSU slot numbers seen across samples.
-func psuSlotsFromSamples(samples []platform.LiveMetricSample) []int {
-	seen := map[int]struct{}{}
-	for _, s := range samples {
-		for _, p := range s.PSUs {
-			seen[p.Slot] = struct{}{}
-		}
-	}
-	slots := make([]int, 0, len(seen))
-	for s := range seen {
-		slots = append(slots, s)
-	}
-	sort.Ints(slots)
-	return slots
-}
-
-// psuStackedTotal returns the point-by-point sum of all PSU datasets (for scale calculation).
-func psuStackedTotal(datasets [][]float64) []float64 {
-	if len(datasets) == 0 {
-		return nil
-	}
-	n := len(datasets[0])
-	total := make([]float64, n)
-	for _, ds := range datasets {
-		for i, v := range ds {
-			total[i] += v
-		}
-	}
-	return total
-}
-
-func normalizeFanSeries(ds []float64) []float64 {
-	if len(ds) == 0 {
-		return nil
-	}
-	out := make([]float64, len(ds))
-	var lastPositive float64
-	for i, v := range ds {
-		if v > 0 {
-			lastPositive = v
-			out[i] = v
-			continue
-		}
-		if lastPositive > 0 {
-			out[i] = lastPositive
-			continue
-		}
-		out[i] = 0
-	}
-	return out
-}
-
-// floatPtr returns a pointer to a float64 value.
-func floatPtr(v float64) *float64 { return &v }
-
-// autoMax120 returns 0→max+20% Y-axis max across all datasets.
-func autoMax120(datasets ...[]float64) *float64 {
-	max := 0.0
-	for _, ds := range datasets {
-		for _, v := range ds {
-			if v > max {
-				max = v
-			}
-		}
-	}
-	if max == 0 {
-		return nil // let library auto-scale
-	}
-	v := max * 1.2
-	return &v
-}
-
-func autoBounds120(datasets ...[]float64) (*float64, *float64) {
-	min := 0.0
-	max := 0.0
-	first := true
-	for _, ds := range datasets {
-		for _, v := range ds {
-			if first {
-				min, max = v, v
-				first = false
-				continue
-			}
-			if v < min {
-				min = v
-			}
-			if v > max {
-				max = v
-			}
-		}
-	}
-	if first {
-		return nil, nil
-	}
-	if max <= 0 {
-		return floatPtr(0), nil
-	}
-	span := max - min
-	if span <= 0 {
-		span = max * 0.1
-		if span <= 0 {
-			span = 1
-		}
-	}
-	pad := span * 0.2
-	low := min - pad
-	if low < 0 {
-		low = 0
-	}
-	high := max + pad
-	return floatPtr(low), floatPtr(high)
-}
-
-func gpuChartLabelIndices(total, target int) []int {
-	if total <= 0 {
-		return nil
-	}
-	if total == 1 {
-		return []int{0}
-	}
-	step := total / target
-	if step < 1 {
-		step = 1
-	}
-	var indices []int
-	for i := 0; i < total; i += step {
-		indices = append(indices, i)
-	}
-	if indices[len(indices)-1] != total-1 {
-		indices = append(indices, total-1)
-	}
-	return indices
-}
-
-func chartCanvasHeightForPath(path string, seriesCount int) int {
-	height := chartCanvasHeight(seriesCount)
-	if isGPUChartPath(path) {
-		return height * 2
-	}
-	return height
-}
-
-func isGPUChartPath(path string) bool {
-	return strings.HasPrefix(path, "gpu-all-") || strings.HasPrefix(path, "gpu/")
-}
-
-func chartLegendVisible(seriesCount int) bool {
-	return seriesCount <= 8
-}
-
-func chartCanvasHeight(seriesCount int) int {
-	if chartLegendVisible(seriesCount) {
-		return 360
-	}
-	return 288
-}
-
-// globalStats returns min, average, and max across all values in all datasets.
-func globalStats(datasets [][]float64) (mn, avg, mx float64) {
-	var sum float64
-	var count int
-	first := true
-	for _, ds := range datasets {
-		for _, v := range ds {
-			if first {
-				mn, mx = v, v
-				first = false
-			}
-			if v < mn {
-				mn = v
-			}
-			if v > mx {
-				mx = v
-			}
-			sum += v
-			count++
-		}
-	}
-	if count > 0 {
-		avg = sum / float64(count)
-	}
-	return mn, avg, mx
-}
-
-func sanitizeChartText(s string) string {
-	if s == "" {
-		return ""
-	}
-	return html.EscapeString(strings.Map(func(r rune) rune {
-		if r < 0x20 && r != '\t' && r != '\n' && r != '\r' {
-			return -1
-		}
-		return r
-	}, s))
-}
-
-func snapshotNamedRings(rings []*namedMetricsRing) ([][]float64, []string, []string) {
-	var datasets [][]float64
-	var names []string
-	var labels []string
-	for _, item := range rings {
-		if item == nil || item.Ring == nil {
-			continue
-		}
-		vals, l := item.Ring.snapshot()
-		datasets = append(datasets, vals)
-		names = append(names, item.Name)
-		if len(labels) == 0 {
-			labels = l
-		}
-	}
-	return datasets, names, labels
-}
-
-func snapshotFanRings(rings []*metricsRing, fanNames []string) ([][]float64, []string, []string) {
-	var datasets [][]float64
-	var names []string
-	var labels []string
-	for i, ring := range rings {
-		if ring == nil {
-			continue
-		}
-		vals, l := ring.snapshot()
-		datasets = append(datasets, normalizeFanSeries(vals))
-		name := "Fan"
-		if i < len(fanNames) {
-			name = fanNames[i]
-		}
-		names = append(names, name+" RPM")
-		if len(labels) == 0 {
-			labels = l
-		}
-	}
-	return datasets, names, labels
-}
-
-func chartLegendNumber(v float64) string {
-	neg := v < 0
-	if v < 0 {
-		v = -v
-	}
-	var out string
-	switch {
-	case v >= 10000:
-		out = fmt.Sprintf("%dk", int((v+500)/1000))
-	case v >= 1000:
-		s := fmt.Sprintf("%.2f", v/1000)
-		s = strings.TrimRight(strings.TrimRight(s, "0"), ".")
-		out = strings.ReplaceAll(s, ".", ",") + "k"
-	default:
-		out = fmt.Sprintf("%.0f", v)
-	}
-	if neg {
-		return "-" + out
-	}
-	return out
-}
-
-func chartYAxisNumber(v float64) string {
-	neg := v < 0
-	if neg {
-		v = -v
-	}
-	var out string
-	switch {
-	case v >= 10000:
-		out = fmt.Sprintf("%dк", int((v+500)/1000))
-	case v >= 1000:
-		// Use one decimal place so ticks like 1400, 1600, 1800 read as
-		// "1,4к", "1,6к", "1,8к" instead of the ambiguous "1к"/"2к".
-		s := fmt.Sprintf("%.1f", v/1000)
-		s = strings.TrimRight(strings.TrimRight(s, "0"), ".")
-		out = strings.ReplaceAll(s, ".", ",") + "к"
-	default:
-		out = fmt.Sprintf("%.0f", v)
-	}
-	if neg {
-		return "-" + out
-	}
-	return out
-}
-
 func (h *handler) handleAPIMetricsExportCSV(w http.ResponseWriter, r *http.Request) {
 	if h.metricsDB == nil {
 		http.Error(w, "metrics database not available", http.StatusServiceUnavailable)

audit/internal/webui/server_test.go

@@ -1145,6 +1145,10 @@ func TestMissingAuditJSONReturnsNotFound(t *testing.T) {
 
 func TestSupportBundleEndpointReturnsArchive(t *testing.T) {
 	dir := t.TempDir()
+	// Isolate os.TempDir(): BuildSupportBundle stages and writes its archive
+	// there, and a concurrent support-bundle test in another package would
+	// otherwise race on the same paths.
+	t.Setenv("TMPDIR", dir)
 	exportDir := filepath.Join(dir, "export")
 	if err := os.MkdirAll(exportDir, 0755); err != nil {
 		t.Fatal(err)

audit/internal/webui/task_page.go

@@ -2,7 +2,6 @@ package webui
 
 import (
 	"encoding/json"
-	"fmt"
 	"html"
 	"net/http"
 	"os"
@@ -225,13 +224,6 @@ func loadTaskReportFragment(task Task) string {
 	return string(data)
 }
 
-func taskArtifactDownloadLink(task Task, absPath string) string {
-	if strings.TrimSpace(absPath) == "" {
-		return ""
-	}
-	return fmt.Sprintf(`/export/file?path=%s`, absPath)
-}
-
 func (h *handler) taskSamplesForRequest(r *http.Request) (Task, []platform.LiveMetricSample, time.Time, time.Time, bool) {
 	id := r.PathValue("id")
 	taskPtr, ok := globalQueue.findByID(id)

audit/internal/webui/task_runner.go

@@ -224,7 +224,10 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
 			err = fmt.Errorf("app not configured")
 			break
 		}
-		archive, err = a.RunNvidiaBandwidthPack(ctx, "", t.params.GPUIndices, j.append)
+		// Validate: one nvbandwidth pass over all selected GPUs. Stress
+		// (deep): per-socket passes then an all-GPU pass, so a cross-socket
+		// P2P fault is isolated from a same-socket one.
+		archive, err = a.RunNvidiaBandwidthPack(ctx, "", t.params.GPUIndices, t.params.StressMode, j.append)
 	case "nvidia-interconnect":
 		if a == nil {
 			err = fmt.Errorf("app not configured")

audit/internal/webui/tasks.go

@@ -2,11 +2,9 @@ package webui
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"log/slog"
-	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -892,402 +890,3 @@ func splitNL(s string) []string {
 }
 
 // ── HTTP handlers ─────────────────────────────────────────────────────────────
-
-func (h *handler) handleAPITasksList(w http.ResponseWriter, _ *http.Request) {
-	tasks := globalQueue.snapshot()
-	writeJSON(w, tasks)
-}
-
-func (h *handler) handleAPITasksCancel(w http.ResponseWriter, r *http.Request) {
-	id := r.PathValue("id")
-	t, ok := globalQueue.findByID(id)
-	if !ok {
-		writeError(w, http.StatusNotFound, "task not found")
-		return
-	}
-	globalQueue.mu.Lock()
-	defer globalQueue.mu.Unlock()
-	switch t.Status {
-	case TaskPending:
-		t.Status = TaskCancelled
-		now := time.Now()
-		t.DoneAt = &now
-		globalQueue.persistLocked()
-		taskSerialEvent(t, "finished with status="+t.Status)
-		writeJSON(w, map[string]string{"status": "cancelled"})
-	case TaskRunning:
-		if t.job == nil || !t.job.abort() {
-			writeError(w, http.StatusConflict, "task is not cancellable")
-			return
-		}
-		writeJSON(w, map[string]string{"status": "aborting"})
-	default:
-		writeError(w, http.StatusConflict, "task is not running or pending")
-	}
-}
-
-func (h *handler) handleAPITasksPriority(w http.ResponseWriter, r *http.Request) {
-	id := r.PathValue("id")
-	t, ok := globalQueue.findByID(id)
-	if !ok {
-		writeError(w, http.StatusNotFound, "task not found")
-		return
-	}
-	var req struct {
-		Delta int `json:"delta"`
-	}
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid body")
-		return
-	}
-	globalQueue.mu.Lock()
-	defer globalQueue.mu.Unlock()
-	if t.Status != TaskPending {
-		writeError(w, http.StatusConflict, "only pending tasks can be reprioritised")
-		return
-	}
-	t.Priority += req.Delta
-	globalQueue.persistLocked()
-	writeJSON(w, map[string]int{"priority": t.Priority})
-}
-
-func (h *handler) handleAPITasksCancelAll(w http.ResponseWriter, _ *http.Request) {
-	globalQueue.mu.Lock()
-	now := time.Now()
-	n := 0
-	for _, t := range globalQueue.tasks {
-		switch t.Status {
-		case TaskPending:
-			t.Status = TaskCancelled
-			t.DoneAt = &now
-			taskSerialEvent(t, "finished with status="+t.Status)
-			n++
-		case TaskRunning:
-			if t.job != nil {
-				t.job.abort()
-			}
-			n++
-		}
-	}
-	globalQueue.persistLocked()
-	globalQueue.mu.Unlock()
-	writeJSON(w, map[string]int{"cancelled": n})
-}
-
-func (h *handler) handleAPITasksKillWorkers(w http.ResponseWriter, _ *http.Request) {
-	// Cancel all queued/running tasks in the queue first.
-	globalQueue.mu.Lock()
-	now := time.Now()
-	cancelled := 0
-	for _, t := range globalQueue.tasks {
-		switch t.Status {
-		case TaskPending:
-			t.Status = TaskCancelled
-			t.DoneAt = &now
-			taskSerialEvent(t, "finished with status="+t.Status)
-			cancelled++
-		case TaskRunning:
-			if t.job != nil {
-				t.job.abort()
-			}
-			if taskMayLeaveOrphanWorkers(t.Target) {
-				platform.KillTestWorkers()
-			}
-			t.Status = TaskCancelled
-			t.DoneAt = &now
-			taskSerialEvent(t, "finished with status="+t.Status)
-			cancelled++
-		}
-	}
-	globalQueue.persistLocked()
-	globalQueue.mu.Unlock()
-
-	// Kill orphaned test worker processes at the OS level.
-	killed := platform.KillTestWorkers()
-	writeJSON(w, map[string]any{
-		"cancelled": cancelled,
-		"killed":    len(killed),
-		"processes": killed,
-	})
-}
-
-func (h *handler) handleAPITasksStream(w http.ResponseWriter, r *http.Request) {
-	id := r.PathValue("id")
-	src, ok := globalQueue.taskStreamSource(id)
-	if !ok {
-		http.Error(w, "task not found", http.StatusNotFound)
-		return
-	}
-	if src.job != nil {
-		streamJob(w, r, src.job)
-		return
-	}
-	if src.status == TaskDone || src.status == TaskFailed || src.status == TaskCancelled {
-		j := newTaskJobState(src.logPath)
-		j.finish(src.errMsg)
-		streamJob(w, r, j)
-		return
-	}
-	if !sseStart(w) {
-		return
-	}
-	sseWrite(w, "", "Task is queued. Waiting for worker...")
-	ticker := time.NewTicker(200 * time.Millisecond)
-	defer ticker.Stop()
-	for {
-		select {
-		case <-ticker.C:
-			src, ok = globalQueue.taskStreamSource(id)
-			if !ok {
-				sseWrite(w, "done", "task not found")
-				return
-			}
-			if src.job != nil {
-				streamSubscribedJob(w, r, src.job)
-				return
-			}
-			if src.status == TaskDone || src.status == TaskFailed || src.status == TaskCancelled {
-				j := newTaskJobState(src.logPath)
-				j.finish(src.errMsg)
-				streamSubscribedJob(w, r, j)
-				return
-			}
-		case <-r.Context().Done():
-			return
-		}
-	}
-}
-
-func (q *taskQueue) assignTaskLogPathLocked(t *Task) {
-	if q.logsDir == "" || t.ID == "" {
-		return
-	}
-	q.ensureTaskArtifactPathsLocked(t)
-}
-
-func (q *taskQueue) loadLocked() {
-	if q.statePath == "" {
-		return
-	}
-	data, err := os.ReadFile(q.statePath)
-	if err != nil || len(data) == 0 {
-		return
-	}
-	var persisted []persistedTask
-	if err := json.Unmarshal(data, &persisted); err != nil {
-		return
-	}
-	for _, pt := range persisted {
-		t := &Task{
-			ID:             pt.ID,
-			Name:           pt.Name,
-			Target:         pt.Target,
-			Priority:       pt.Priority,
-			Status:         pt.Status,
-			CreatedAt:      pt.CreatedAt,
-			StartedAt:      pt.StartedAt,
-			DoneAt:         pt.DoneAt,
-			ErrMsg:         pt.ErrMsg,
-			LogPath:        pt.LogPath,
-			ArtifactsDir:   pt.ArtifactsDir,
-			ReportJSONPath: pt.ReportJSONPath,
-			ReportHTMLPath: pt.ReportHTMLPath,
-			params:         pt.Params,
-		}
-		q.assignTaskLogPathLocked(t)
-		if t.Status == TaskRunning {
-			state, ok := readTaskRunnerState(t)
-			switch {
-			case ok && state.Status == TaskRunning && processAlive(state.PID):
-				t.runnerPID = state.PID
-				t.job = newTaskJobState(t.LogPath)
-			case ok && state.Status != TaskRunning:
-				t.runnerPID = state.PID
-				t.Status = state.Status
-				t.ErrMsg = state.Error
-				now := state.UpdatedAt
-				if now.IsZero() {
-					now = time.Now()
-				}
-				t.DoneAt = &now
-			default:
-				if taskMayLeaveOrphanWorkers(t.Target) {
-					_ = platform.KillTestWorkers()
-				}
-				now := time.Now()
-				t.Status = TaskFailed
-				t.DoneAt = &now
-				t.ErrMsg = "interrupted by bee-web restart"
-			}
-		} else if t.Status == TaskPending {
-			t.StartedAt = nil
-			t.DoneAt = nil
-			t.ErrMsg = ""
-		}
-		q.tasks = append(q.tasks, t)
-	}
-	q.prune()
-	q.persistLocked()
-}
-
-func (q *taskQueue) persistLocked() {
-	if q.statePath == "" {
-		return
-	}
-	state := make([]persistedTask, 0, len(q.tasks))
-	for _, t := range q.tasks {
-		state = append(state, persistedTask{
-			ID:             t.ID,
-			Name:           t.Name,
-			Target:         t.Target,
-			Priority:       t.Priority,
-			Status:         t.Status,
-			CreatedAt:      t.CreatedAt,
-			StartedAt:      t.StartedAt,
-			DoneAt:         t.DoneAt,
-			ErrMsg:         t.ErrMsg,
-			LogPath:        t.LogPath,
-			ArtifactsDir:   t.ArtifactsDir,
-			ReportJSONPath: t.ReportJSONPath,
-			ReportHTMLPath: t.ReportHTMLPath,
-			Params:         t.params,
-		})
-	}
-	data, err := json.MarshalIndent(state, "", "  ")
-	if err != nil {
-		return
-	}
-	tmp := q.statePath + ".tmp"
-	if err := os.WriteFile(tmp, data, 0644); err != nil {
-		return
-	}
-	_ = os.Rename(tmp, q.statePath)
-}
-
-func taskElapsedSec(t *Task, now time.Time) int {
-	if t == nil || t.StartedAt == nil || t.StartedAt.IsZero() {
-		return 0
-	}
-	start := *t.StartedAt
-	if !t.CreatedAt.IsZero() && start.Before(t.CreatedAt) {
-		start = t.CreatedAt
-	}
-	end := now
-	if t.DoneAt != nil && !t.DoneAt.IsZero() {
-		end = *t.DoneAt
-	}
-	if end.Before(start) {
-		return 0
-	}
-	return int(end.Sub(start).Round(time.Second) / time.Second)
-}
-
-func taskFolderStatus(status string) string {
-	status = strings.TrimSpace(strings.ToLower(status))
-	switch status {
-	case TaskRunning, TaskDone, TaskFailed, TaskCancelled:
-		return status
-	default:
-		return TaskPending
-	}
-}
-
-func sanitizeTaskFolderPart(s string) string {
-	s = strings.TrimSpace(strings.ToLower(s))
-	if s == "" {
-		return "task"
-	}
-	var b strings.Builder
-	lastDash := false
-	for _, r := range s {
-		isAlnum := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9')
-		if isAlnum {
-			b.WriteRune(r)
-			lastDash = false
-			continue
-		}
-		if !lastDash {
-			b.WriteByte('-')
-			lastDash = true
-		}
-	}
-	out := strings.Trim(b.String(), "-")
-	if out == "" {
-		return "task"
-	}
-	return out
-}
-
-func taskArtifactsDir(root string, t *Task, status string) string {
-	if strings.TrimSpace(root) == "" || t == nil {
-		return ""
-	}
-	prefix := taskFolderNumberPrefix(t.ID)
-	return filepath.Join(root, fmt.Sprintf("%s_%s_%s", prefix, sanitizeTaskFolderPart(t.Name), taskFolderStatus(status)))
-}
-
-func taskFolderNumberPrefix(taskID string) string {
-	taskID = strings.TrimSpace(taskID)
-	if strings.HasPrefix(taskID, "TASK-") && len(taskID) >= len("TASK-000") {
-		num := strings.TrimSpace(strings.TrimPrefix(taskID, "TASK-"))
-		if len(num) == 3 {
-			allDigits := true
-			for _, r := range num {
-				if r < '0' || r > '9' {
-					allDigits = false
-					break
-				}
-			}
-			if allDigits {
-				return num
-			}
-		}
-	}
-	fallback := sanitizeTaskFolderPart(taskID)
-	if fallback == "" {
-		return "000"
-	}
-	return fallback
-}
-
-func ensureTaskReportPaths(t *Task) {
-	if t == nil || strings.TrimSpace(t.ArtifactsDir) == "" {
-		return
-	}
-	if t.LogPath == "" || filepath.Base(t.LogPath) == "task.log" {
-		t.LogPath = filepath.Join(t.ArtifactsDir, "task.log")
-	}
-	t.ReportJSONPath = filepath.Join(t.ArtifactsDir, "report.json")
-	t.ReportHTMLPath = filepath.Join(t.ArtifactsDir, "report.html")
-}
-
-func (q *taskQueue) ensureTaskArtifactPathsLocked(t *Task) {
-	if t == nil || strings.TrimSpace(q.logsDir) == "" || strings.TrimSpace(t.ID) == "" {
-		return
-	}
-	if strings.TrimSpace(t.ArtifactsDir) == "" {
-		t.ArtifactsDir = taskArtifactsDir(q.logsDir, t, t.Status)
-	}
-	if t.ArtifactsDir != "" {
-		_ = os.MkdirAll(t.ArtifactsDir, 0755)
-	}
-	ensureTaskReportPaths(t)
-}
-
-func (q *taskQueue) finalizeTaskArtifactPathsLocked(t *Task) {
-	if t == nil || strings.TrimSpace(q.logsDir) == "" || strings.TrimSpace(t.ID) == "" {
-		return
-	}
-	q.ensureTaskArtifactPathsLocked(t)
-	dstDir := taskArtifactsDir(q.logsDir, t, t.Status)
-	if dstDir == "" {
-		return
-	}
-	if t.ArtifactsDir != "" && t.ArtifactsDir != dstDir {
-		if _, err := os.Stat(dstDir); err != nil {
-			_ = os.Rename(t.ArtifactsDir, dstDir)
-		}
-		t.ArtifactsDir = dstDir
-	}
-	ensureTaskReportPaths(t)
-}

audit/internal/webui/tasks_http_persistence.go

@@ -0,0 +1,412 @@
+package webui
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"bee/audit/internal/platform"
+)
+
+func (h *handler) handleAPITasksList(w http.ResponseWriter, _ *http.Request) {
+	tasks := globalQueue.snapshot()
+	writeJSON(w, tasks)
+}
+
+func (h *handler) handleAPITasksCancel(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	t, ok := globalQueue.findByID(id)
+	if !ok {
+		writeError(w, http.StatusNotFound, "task not found")
+		return
+	}
+	globalQueue.mu.Lock()
+	defer globalQueue.mu.Unlock()
+	switch t.Status {
+	case TaskPending:
+		t.Status = TaskCancelled
+		now := time.Now()
+		t.DoneAt = &now
+		globalQueue.persistLocked()
+		taskSerialEvent(t, "finished with status="+t.Status)
+		writeJSON(w, map[string]string{"status": "cancelled"})
+	case TaskRunning:
+		if t.job == nil || !t.job.abort() {
+			writeError(w, http.StatusConflict, "task is not cancellable")
+			return
+		}
+		writeJSON(w, map[string]string{"status": "aborting"})
+	default:
+		writeError(w, http.StatusConflict, "task is not running or pending")
+	}
+}
+
+func (h *handler) handleAPITasksPriority(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	t, ok := globalQueue.findByID(id)
+	if !ok {
+		writeError(w, http.StatusNotFound, "task not found")
+		return
+	}
+	var req struct {
+		Delta int `json:"delta"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid body")
+		return
+	}
+	globalQueue.mu.Lock()
+	defer globalQueue.mu.Unlock()
+	if t.Status != TaskPending {
+		writeError(w, http.StatusConflict, "only pending tasks can be reprioritised")
+		return
+	}
+	t.Priority += req.Delta
+	globalQueue.persistLocked()
+	writeJSON(w, map[string]int{"priority": t.Priority})
+}
+
+func (h *handler) handleAPITasksCancelAll(w http.ResponseWriter, _ *http.Request) {
+	globalQueue.mu.Lock()
+	now := time.Now()
+	n := 0
+	for _, t := range globalQueue.tasks {
+		switch t.Status {
+		case TaskPending:
+			t.Status = TaskCancelled
+			t.DoneAt = &now
+			taskSerialEvent(t, "finished with status="+t.Status)
+			n++
+		case TaskRunning:
+			if t.job != nil {
+				t.job.abort()
+			}
+			n++
+		}
+	}
+	globalQueue.persistLocked()
+	globalQueue.mu.Unlock()
+	writeJSON(w, map[string]int{"cancelled": n})
+}
+
+func (h *handler) handleAPITasksKillWorkers(w http.ResponseWriter, _ *http.Request) {
+	// Cancel all queued/running tasks in the queue first.
+	globalQueue.mu.Lock()
+	now := time.Now()
+	cancelled := 0
+	for _, t := range globalQueue.tasks {
+		switch t.Status {
+		case TaskPending:
+			t.Status = TaskCancelled
+			t.DoneAt = &now
+			taskSerialEvent(t, "finished with status="+t.Status)
+			cancelled++
+		case TaskRunning:
+			if t.job != nil {
+				t.job.abort()
+			}
+			if taskMayLeaveOrphanWorkers(t.Target) {
+				platform.KillTestWorkers()
+			}
+			t.Status = TaskCancelled
+			t.DoneAt = &now
+			taskSerialEvent(t, "finished with status="+t.Status)
+			cancelled++
+		}
+	}
+	globalQueue.persistLocked()
+	globalQueue.mu.Unlock()
+
+	// Kill orphaned test worker processes at the OS level.
+	killed := platform.KillTestWorkers()
+	writeJSON(w, map[string]any{
+		"cancelled": cancelled,
+		"killed":    len(killed),
+		"processes": killed,
+	})
+}
+
+func (h *handler) handleAPITasksStream(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	src, ok := globalQueue.taskStreamSource(id)
+	if !ok {
+		http.Error(w, "task not found", http.StatusNotFound)
+		return
+	}
+	if src.job != nil {
+		streamJob(w, r, src.job)
+		return
+	}
+	if src.status == TaskDone || src.status == TaskFailed || src.status == TaskCancelled {
+		j := newTaskJobState(src.logPath)
+		j.finish(src.errMsg)
+		streamJob(w, r, j)
+		return
+	}
+	if !sseStart(w) {
+		return
+	}
+	sseWrite(w, "", "Task is queued. Waiting for worker...")
+	ticker := time.NewTicker(200 * time.Millisecond)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			src, ok = globalQueue.taskStreamSource(id)
+			if !ok {
+				sseWrite(w, "done", "task not found")
+				return
+			}
+			if src.job != nil {
+				streamSubscribedJob(w, r, src.job)
+				return
+			}
+			if src.status == TaskDone || src.status == TaskFailed || src.status == TaskCancelled {
+				j := newTaskJobState(src.logPath)
+				j.finish(src.errMsg)
+				streamSubscribedJob(w, r, j)
+				return
+			}
+		case <-r.Context().Done():
+			return
+		}
+	}
+}
+
+func (q *taskQueue) assignTaskLogPathLocked(t *Task) {
+	if q.logsDir == "" || t.ID == "" {
+		return
+	}
+	q.ensureTaskArtifactPathsLocked(t)
+}
+
+func (q *taskQueue) loadLocked() {
+	if q.statePath == "" {
+		return
+	}
+	data, err := os.ReadFile(q.statePath)
+	if err != nil || len(data) == 0 {
+		return
+	}
+	var persisted []persistedTask
+	if err := json.Unmarshal(data, &persisted); err != nil {
+		return
+	}
+	for _, pt := range persisted {
+		t := &Task{
+			ID:             pt.ID,
+			Name:           pt.Name,
+			Target:         pt.Target,
+			Priority:       pt.Priority,
+			Status:         pt.Status,
+			CreatedAt:      pt.CreatedAt,
+			StartedAt:      pt.StartedAt,
+			DoneAt:         pt.DoneAt,
+			ErrMsg:         pt.ErrMsg,
+			LogPath:        pt.LogPath,
+			ArtifactsDir:   pt.ArtifactsDir,
+			ReportJSONPath: pt.ReportJSONPath,
+			ReportHTMLPath: pt.ReportHTMLPath,
+			params:         pt.Params,
+		}
+		q.assignTaskLogPathLocked(t)
+		if t.Status == TaskRunning {
+			state, ok := readTaskRunnerState(t)
+			switch {
+			case ok && state.Status == TaskRunning && processAlive(state.PID):
+				t.runnerPID = state.PID
+				t.job = newTaskJobState(t.LogPath)
+			case ok && state.Status != TaskRunning:
+				t.runnerPID = state.PID
+				t.Status = state.Status
+				t.ErrMsg = state.Error
+				now := state.UpdatedAt
+				if now.IsZero() {
+					now = time.Now()
+				}
+				t.DoneAt = &now
+			default:
+				if taskMayLeaveOrphanWorkers(t.Target) {
+					_ = platform.KillTestWorkers()
+				}
+				now := time.Now()
+				t.Status = TaskFailed
+				t.DoneAt = &now
+				t.ErrMsg = "interrupted by bee-web restart"
+			}
+		} else if t.Status == TaskPending {
+			t.StartedAt = nil
+			t.DoneAt = nil
+			t.ErrMsg = ""
+		}
+		q.tasks = append(q.tasks, t)
+	}
+	q.prune()
+	q.persistLocked()
+}
+
+func (q *taskQueue) persistLocked() {
+	if q.statePath == "" {
+		return
+	}
+	state := make([]persistedTask, 0, len(q.tasks))
+	for _, t := range q.tasks {
+		state = append(state, persistedTask{
+			ID:             t.ID,
+			Name:           t.Name,
+			Target:         t.Target,
+			Priority:       t.Priority,
+			Status:         t.Status,
+			CreatedAt:      t.CreatedAt,
+			StartedAt:      t.StartedAt,
+			DoneAt:         t.DoneAt,
+			ErrMsg:         t.ErrMsg,
+			LogPath:        t.LogPath,
+			ArtifactsDir:   t.ArtifactsDir,
+			ReportJSONPath: t.ReportJSONPath,
+			ReportHTMLPath: t.ReportHTMLPath,
+			Params:         t.params,
+		})
+	}
+	data, err := json.MarshalIndent(state, "", "  ")
+	if err != nil {
+		return
+	}
+	tmp := q.statePath + ".tmp"
+	if err := os.WriteFile(tmp, data, 0644); err != nil {
+		return
+	}
+	_ = os.Rename(tmp, q.statePath)
+}
+
+func taskElapsedSec(t *Task, now time.Time) int {
+	if t == nil || t.StartedAt == nil || t.StartedAt.IsZero() {
+		return 0
+	}
+	start := *t.StartedAt
+	if !t.CreatedAt.IsZero() && start.Before(t.CreatedAt) {
+		start = t.CreatedAt
+	}
+	end := now
+	if t.DoneAt != nil && !t.DoneAt.IsZero() {
+		end = *t.DoneAt
+	}
+	if end.Before(start) {
+		return 0
+	}
+	return int(end.Sub(start).Round(time.Second) / time.Second)
+}
+
+func taskFolderStatus(status string) string {
+	status = strings.TrimSpace(strings.ToLower(status))
+	switch status {
+	case TaskRunning, TaskDone, TaskFailed, TaskCancelled:
+		return status
+	default:
+		return TaskPending
+	}
+}
+
+func sanitizeTaskFolderPart(s string) string {
+	s = strings.TrimSpace(strings.ToLower(s))
+	if s == "" {
+		return "task"
+	}
+	var b strings.Builder
+	lastDash := false
+	for _, r := range s {
+		isAlnum := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9')
+		if isAlnum {
+			b.WriteRune(r)
+			lastDash = false
+			continue
+		}
+		if !lastDash {
+			b.WriteByte('-')
+			lastDash = true
+		}
+	}
+	out := strings.Trim(b.String(), "-")
+	if out == "" {
+		return "task"
+	}
+	return out
+}
+
+func taskArtifactsDir(root string, t *Task, status string) string {
+	if strings.TrimSpace(root) == "" || t == nil {
+		return ""
+	}
+	prefix := taskFolderNumberPrefix(t.ID)
+	return filepath.Join(root, fmt.Sprintf("%s_%s_%s", prefix, sanitizeTaskFolderPart(t.Name), taskFolderStatus(status)))
+}
+
+func taskFolderNumberPrefix(taskID string) string {
+	taskID = strings.TrimSpace(taskID)
+	if strings.HasPrefix(taskID, "TASK-") && len(taskID) >= len("TASK-000") {
+		num := strings.TrimSpace(strings.TrimPrefix(taskID, "TASK-"))
+		if len(num) == 3 {
+			allDigits := true
+			for _, r := range num {
+				if r < '0' || r > '9' {
+					allDigits = false
+					break
+				}
+			}
+			if allDigits {
+				return num
+			}
+		}
+	}
+	fallback := sanitizeTaskFolderPart(taskID)
+	if fallback == "" {
+		return "000"
+	}
+	return fallback
+}
+
+func ensureTaskReportPaths(t *Task) {
+	if t == nil || strings.TrimSpace(t.ArtifactsDir) == "" {
+		return
+	}
+	if t.LogPath == "" || filepath.Base(t.LogPath) == "task.log" {
+		t.LogPath = filepath.Join(t.ArtifactsDir, "task.log")
+	}
+	t.ReportJSONPath = filepath.Join(t.ArtifactsDir, "report.json")
+	t.ReportHTMLPath = filepath.Join(t.ArtifactsDir, "report.html")
+}
+
+func (q *taskQueue) ensureTaskArtifactPathsLocked(t *Task) {
+	if t == nil || strings.TrimSpace(q.logsDir) == "" || strings.TrimSpace(t.ID) == "" {
+		return
+	}
+	if strings.TrimSpace(t.ArtifactsDir) == "" {
+		t.ArtifactsDir = taskArtifactsDir(q.logsDir, t, t.Status)
+	}
+	if t.ArtifactsDir != "" {
+		_ = os.MkdirAll(t.ArtifactsDir, 0755)
+	}
+	ensureTaskReportPaths(t)
+}
+
+func (q *taskQueue) finalizeTaskArtifactPathsLocked(t *Task) {
+	if t == nil || strings.TrimSpace(q.logsDir) == "" || strings.TrimSpace(t.ID) == "" {
+		return
+	}
+	q.ensureTaskArtifactPathsLocked(t)
+	dstDir := taskArtifactsDir(q.logsDir, t, t.Status)
+	if dstDir == "" {
+		return
+	}
+	if t.ArtifactsDir != "" && t.ArtifactsDir != dstDir {
+		if _, err := os.Stat(dstDir); err != nil {
+			_ = os.Rename(t.ArtifactsDir, dstDir)
+		}
+		t.ArtifactsDir = dstDir
+	}
+	ensureTaskReportPaths(t)
+}

audit/internal/webui/tasks_test.go

@@ -16,6 +16,19 @@ import (
 	"bee/audit/internal/platform"
 )
 
+type flushNotifyRecorder struct {
+	*httptest.ResponseRecorder
+	flushed chan struct{}
+}
+
+func (r *flushNotifyRecorder) Flush() {
+	r.ResponseRecorder.Flush()
+	select {
+	case r.flushed <- struct{}{}:
+	default:
+	}
+}
+
 func TestTaskQueuePersistsAndRecoversPendingTasks(t *testing.T) {
 	dir := t.TempDir()
 	q := &taskQueue{
@@ -275,7 +288,10 @@ func TestHandleAPITasksStreamPendingTaskStartsSSEImmediately(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	req := httptest.NewRequest(http.MethodGet, "/api/tasks/pending-1/stream", nil).WithContext(ctx)
 	req.SetPathValue("id", "pending-1")
-	rec := httptest.NewRecorder()
+	rec := &flushNotifyRecorder{
+		ResponseRecorder: httptest.NewRecorder(),
+		flushed:          make(chan struct{}, 1),
+	}
 
 	done := make(chan struct{})
 	go func() {
@@ -284,17 +300,18 @@ func TestHandleAPITasksStreamPendingTaskStartsSSEImmediately(t *testing.T) {
 		close(done)
 	}()
 
-	deadline := time.Now().Add(2 * time.Second)
-	for time.Now().Before(deadline) {
-		if strings.Contains(rec.Body.String(), "Task is queued. Waiting for worker...") {
-			cancel()
-			<-done
-			if rec.Code != http.StatusOK {
-				t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
-			}
-			return
+	select {
+	case <-rec.flushed:
+		cancel()
+		<-done
+		if rec.Code != http.StatusOK {
+			t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
 		}
-		time.Sleep(20 * time.Millisecond)
+		if !strings.Contains(rec.Body.String(), "Task is queued. Waiting for worker...") {
+			t.Fatalf("missing queued status, body=%q", rec.Body.String())
+		}
+		return
+	case <-time.After(2 * time.Second):
 	}
 	cancel()
 	<-done