refactor: modularize audit and harden build validation

audit/internal/app/app.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
-	"strconv"
 	"strings"
 
 	"bee/audit/internal/collector"
@@ -96,14 +95,38 @@ type installer interface {
 type GPUPresenceResult struct {
 	Nvidia bool
 	AMD    bool
+	// NvidiaInitializing / AMDInitializing report a PCI device of that vendor
+	// when DetectGPUVendor did not report the vendor as operational.
+	NvidiaInitializing bool
+	AMDInitializing    bool
 }
 
+// DetectGPUPresence combines the existing operational vendor detection with a
+// PCI display-class scan. The latter distinguishes absent hardware from a
+// detected PCI function whose runtime is not operational yet.
 func (a *App) DetectGPUPresence() GPUPresenceResult {
 	vendor := a.sat.DetectGPUVendor()
-	return GPUPresenceResult{
+	res := GPUPresenceResult{
 		Nvidia: vendor == "nvidia",
 		AMD:    vendor == "amd",
 	}
+	physNvidia, physAMD := a.sat.PhysicalGPUVendors()
+	res.NvidiaInitializing = physNvidia && !res.Nvidia
+	res.AMDInitializing = physAMD && !res.AMD
+	return res
+}
+
+// RuntimeHealthNow collects a fresh runtime-health snapshot (driver / CUDA
+// readiness, GSP state). Unlike ReadRuntimeHealth it does not read the
+// boot-time JSON; callers that poll for the GPU stack to come up need
+// current data.
+func (a *App) RuntimeHealthNow() (schema.RuntimeHealth, error) {
+	return a.runtime.CollectRuntimeHealth(DefaultExportDir)
+}
+
+// TPMPresent reports whether this host has a TPM the checks can talk to.
+func (a *App) TPMPresent() bool {
+	return a.sat.TPMPresent()
 }
 
 func (a *App) IsLiveMediaInRAM() bool {
@@ -132,13 +155,14 @@ type satRunner interface {
 	RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error)
 	RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error)
 	RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error)
-	RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error)
+	RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, fullMatrix bool, logFunc func(string)) (string, error)
 	RunNvidiaStressPack(ctx context.Context, baseDir string, opts platform.NvidiaStressOptions, logFunc func(string)) (string, error)
 	ListNvidiaGPUStatuses() ([]platform.NvidiaGPUStatus, error)
 	ResetNvidiaGPU(index int) (string, error)
 	RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error)
 	RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error)
 	RunTPMValidationPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
+	TPMPresent() bool
 	RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
 	RunPCIeLinkCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
 	RunNvidiaPCIeBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error)
@@ -151,6 +175,7 @@ type satRunner interface {
 	SetNvidiaGPUPowerLimit(index int, watts float64) (string, error)
 	ResetNvidiaGPUDefaults() (string, error)
 	DetectGPUVendor() string
+	PhysicalGPUVendors() (nvidia bool, amd bool)
 	ListAMDGPUs() ([]platform.AMDGPUInfo, error)
 	RunAMDAcceptancePack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
 	RunAMDMemIntegrityPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
@@ -279,15 +304,6 @@ func (a *App) RunRuntimePreflight(output string) (string, error) {
 	}
 }
 
-func (a *App) RunRuntimePreflightResult() (ActionResult, error) {
-	path, err := a.RunRuntimePreflight("file:" + DefaultRuntimeJSONPath)
-	body := "Runtime preflight completed."
-	if path != "" {
-		body = "Runtime health written to " + path
-	}
-	return ActionResult{Title: "Run self-check", Body: body}, err
-}
-
 func (a *App) RuntimeHealthResult() ActionResult {
 	health, err := ReadRuntimeHealth(DefaultRuntimeJSONPath)
 	if err != nil {
@@ -326,10 +342,6 @@ func (a *App) RunAuditNow(runtimeMode runtimeenv.Mode) (ActionResult, error) {
 	return ActionResult{Title: "Run audit", Body: body}, err
 }
 
-func (a *App) RunAuditToDefaultFile(runtimeMode runtimeenv.Mode) (string, error) {
-	return a.RunAudit(runtimeMode, "file:"+DefaultAuditJSONPath)
-}
-
 func (a *App) HealthSummaryResult() ActionResult {
 	raw, err := os.ReadFile(DefaultAuditJSONPath)
 	if err != nil {
@@ -399,26 +411,6 @@ func (a *App) MainBanner() string {
 	return strings.TrimSpace(strings.Join(lines, "\n"))
 }
 
-func (a *App) FormatToolStatuses(statuses []platform.ToolStatus) string {
-	var body strings.Builder
-	for _, tool := range statuses {
-		status := "MISSING"
-		if tool.OK {
-			status = "OK (" + tool.Path + ")"
-		}
-		fmt.Fprintf(&body, "- %s: %s\n", tool.Name, status)
-	}
-	return strings.TrimSpace(body.String())
-}
-
-func (a *App) ParsePrefix(raw string, fallback int) int {
-	value, err := strconv.Atoi(strings.TrimSpace(raw))
-	if err != nil || value <= 0 {
-		return fallback
-	}
-	return value
-}
-
 // writePSUStatusesToDB records PSU statuses collected during audit into the
 // component-status DB so they are visible in the Hardware Summary card.
 // PSU status is sourced from IPMI (ipmitool fru + sdr) during audit.

audit/internal/app/app_install.go

@@ -62,18 +62,6 @@ func (a *App) ExportLatestAudit(target platform.RemovableTarget) (string, error)
 	return a.exports.ExportFileToTarget(tmpPath, target)
 }
 
-func (a *App) ExportLatestAuditResult(target platform.RemovableTarget) (ActionResult, error) {
-	path, err := a.ExportLatestAudit(target)
-	body := "Audit export failed."
-	if err == nil {
-		body = "Audit exported."
-	}
-	if err == nil && path != "" {
-		body = "Audit exported to " + path
-	}
-	return ActionResult{Title: "Export audit", Body: body}, err
-}
-
 func (a *App) ExportSupportBundle(target platform.RemovableTarget) (string, error) {
 	archive, err := BuildSupportBundle(DefaultExportDir)
 	if err != nil {

audit/internal/app/app_packs.go

@@ -196,11 +196,11 @@ func (a *App) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durati
 	return a.sat.RunNvidiaPulseTestPack(ctx, baseDir, durationSec, gpuIndices, logFunc)
 }
 
-func (a *App) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
+func (a *App) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, fullMatrix bool, logFunc func(string)) (string, error) {
 	if strings.TrimSpace(baseDir) == "" {
 		baseDir = DefaultSATBaseDir
 	}
-	return a.sat.RunNvidiaBandwidthPack(ctx, baseDir, gpuIndices, logFunc)
+	return a.sat.RunNvidiaBandwidthPack(ctx, baseDir, gpuIndices, fullMatrix, logFunc)
 }
 
 func (a *App) RunNvidiaStressPackCtx(ctx context.Context, baseDir string, opts platform.NvidiaStressOptions, logFunc func(string)) (string, error) {
@@ -237,11 +237,6 @@ func (a *App) RunCPUAcceptancePackCtx(ctx context.Context, baseDir string, durat
 	return a.sat.RunCPUAcceptancePack(ctx, baseDir, durationSec, logFunc)
 }
 
-func (a *App) RunCPUAcceptancePackResult(baseDir string, durationSec int) (ActionResult, error) {
-	path, err := a.RunCPUAcceptancePack(baseDir, durationSec, nil)
-	return ActionResult{Title: "CPU SAT", Body: satResultBody(path)}, err
-}
-
 func (a *App) RunStorageAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
 	return a.RunStorageAcceptancePackCtx(context.Background(), baseDir, false, logFunc)
 }
@@ -276,11 +271,6 @@ func (a *App) RunNvidiaConfigCheckPack(baseDir string, logFunc func(string)) (st
 	return a.RunNvidiaConfigCheckPackCtx(context.Background(), baseDir, logFunc)
 }
 
-func (a *App) RunNvidiaConfigCheckPackResult(baseDir string) (ActionResult, error) {
-	path, err := a.RunNvidiaConfigCheckPack(baseDir, nil)
-	return ActionResult{Title: "GPU Config & NVLink Check", Body: satResultBody(path)}, err
-}
-
 func (a *App) RunPCIeLinkCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
 	if strings.TrimSpace(baseDir) == "" {
 		baseDir = DefaultSATBaseDir
@@ -292,11 +282,6 @@ func (a *App) RunPCIeLinkCheckPack(baseDir string, logFunc func(string)) (string
 	return a.RunPCIeLinkCheckPackCtx(context.Background(), baseDir, logFunc)
 }
 
-func (a *App) RunPCIeLinkCheckPackResult(baseDir string) (ActionResult, error) {
-	path, err := a.RunPCIeLinkCheckPack(baseDir, nil)
-	return ActionResult{Title: "PCIe Link Check", Body: satResultBody(path)}, err
-}
-
 func (a *App) RunNvidiaPCIeBandwidthPackCtx(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
 	if strings.TrimSpace(baseDir) == "" {
 		baseDir = DefaultSATBaseDir
@@ -308,11 +293,6 @@ func (a *App) RunNvidiaPCIeBandwidthPack(baseDir string, gpuIndices []int, logFu
 	return a.RunNvidiaPCIeBandwidthPackCtx(context.Background(), baseDir, gpuIndices, logFunc)
 }
 
-func (a *App) RunNvidiaPCIeBandwidthPackResult(baseDir string) (ActionResult, error) {
-	path, err := a.RunNvidiaPCIeBandwidthPack(baseDir, nil, nil)
-	return ActionResult{Title: "NVIDIA GPU PCIe Bandwidth Check", Body: satResultBody(path)}, err
-}
-
 func (a *App) DetectGPUVendor() string {
 	return a.sat.DetectGPUVendor()
 }
@@ -332,11 +312,6 @@ func (a *App) RunAMDAcceptancePackCtx(ctx context.Context, baseDir string, logFu
 	return a.sat.RunAMDAcceptancePack(ctx, baseDir, logFunc)
 }
 
-func (a *App) RunAMDAcceptancePackResult(baseDir string) (ActionResult, error) {
-	path, err := a.RunAMDAcceptancePack(baseDir, nil)
-	return ActionResult{Title: "AMD GPU SAT", Body: satResultBody(path)}, err
-}
-
 func (a *App) RunAMDMemIntegrityPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
 	if strings.TrimSpace(baseDir) == "" {
 		baseDir = DefaultSATBaseDir
@@ -399,69 +374,6 @@ func (a *App) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int
 	return a.sat.RunNCCLTests(ctx, baseDir, gpuIndices, logFunc)
 }
 
-func (a *App) RunNCCLTestsResult(ctx context.Context) (ActionResult, error) {
-	path, err := a.RunNCCLTests(ctx, DefaultSATBaseDir, nil, nil)
-	body := "Results: " + path
-	if err != nil && err != context.Canceled {
-		body += "\nERROR: " + err.Error()
-	}
-	return ActionResult{Title: "NCCL bandwidth test", Body: body}, err
-}
-
-func (a *App) RunFanStressTestResult(ctx context.Context, opts platform.FanStressOptions) (ActionResult, error) {
-	path, err := a.RunFanStressTest(ctx, "", opts)
-	body := formatFanStressResult(path)
-	if err != nil && err != context.Canceled {
-		body += "\nERROR: " + err.Error()
-	}
-	return ActionResult{Title: "GPU Platform Stress Test", Body: body}, err
-}
-
-// formatFanStressResult formats the summary.txt from a fan-stress run, including
-// the per-step pass/fail display and the analysis section (throttling, max temps, fan response).
-func formatFanStressResult(archivePath string) string {
-	if archivePath == "" {
-		return "No output produced."
-	}
-	runDir := strings.TrimSuffix(archivePath, ".tar.gz")
-	raw, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
-	if err != nil {
-		return "Archive written to " + archivePath
-	}
-	content := strings.TrimSpace(string(raw))
-	kv := parseKeyValueSummary(content)
-
-	var b strings.Builder
-	b.WriteString(formatSATDetail(content))
-
-	// Append analysis section.
-	var analysis []string
-	if v, ok := kv["throttling_detected"]; ok {
-		label := "NO"
-		if v == "true" {
-			label = "YES  ← throttling detected during load"
-		}
-		analysis = append(analysis, "Throttling:   "+label)
-	}
-	if v, ok := kv["max_gpu_temp_c"]; ok && v != "0.0" {
-		analysis = append(analysis, "Max GPU temp: "+v+"°C")
-	}
-	if v, ok := kv["max_cpu_temp_c"]; ok && v != "0.0" {
-		analysis = append(analysis, "Max CPU temp: "+v+"°C")
-	}
-	if v, ok := kv["fan_response_sec"]; ok && v != "N/A" && v != "-1.0" {
-		analysis = append(analysis, "Fan response: "+v+"s")
-	}
-
-	if len(analysis) > 0 {
-		b.WriteString("\n\n=== Analysis ===\n")
-		for _, line := range analysis {
-			b.WriteString(line + "\n")
-		}
-	}
-	return strings.TrimSpace(b.String())
-}
-
 // satResultBody reads summary.txt from the SAT run directory (archive path without .tar.gz)
 // and returns a formatted human-readable result. Falls back to a plain message if unreadable.
 func satResultBody(archivePath string) string {

audit/internal/app/app_test.go

@@ -216,7 +216,7 @@ func (f fakeSAT) RunNvidiaPulseTestPack(_ context.Context, baseDir string, durat
 	return f.runNvidiaFn(baseDir)
 }
 
-func (f fakeSAT) RunNvidiaBandwidthPack(_ context.Context, baseDir string, gpuIndices []int, _ func(string)) (string, error) {
+func (f fakeSAT) RunNvidiaBandwidthPack(_ context.Context, baseDir string, gpuIndices []int, _ bool, _ func(string)) (string, error) {
 	if f.runNvidiaBandwidthFn != nil {
 		return f.runNvidiaBandwidthFn(baseDir, gpuIndices)
 	}
@@ -287,6 +287,10 @@ func (f fakeSAT) RunTPMValidationPack(_ context.Context, _ string, _ func(string
 	return "", nil
 }
 
+func (f fakeSAT) TPMPresent() bool { return true }
+
+func (f fakeSAT) PhysicalGPUVendors() (bool, bool) { return false, false }
+
 func (f fakeSAT) RunNvidiaConfigCheckPack(_ context.Context, baseDir string, _ func(string)) (string, error) {
 	return "", nil
 }
@@ -725,6 +729,7 @@ func TestActionResultsUseFallbackBody(t *testing.T) {
 
 func TestExportSupportBundleResultMentionsUnmountedUSB(t *testing.T) {
 	tmp := t.TempDir()
+	t.Setenv("TMPDIR", tmp)
 	oldExportDir := DefaultExportDir
 	DefaultExportDir = tmp
 	t.Cleanup(func() { DefaultExportDir = oldExportDir })
@@ -761,6 +766,7 @@ func TestExportSupportBundleResultMentionsUnmountedUSB(t *testing.T) {
 
 func TestExportSupportBundleResultDoesNotPretendSuccessOnError(t *testing.T) {
 	tmp := t.TempDir()
+	t.Setenv("TMPDIR", tmp)
 	oldExportDir := DefaultExportDir
 	DefaultExportDir = tmp
 	t.Cleanup(func() { DefaultExportDir = oldExportDir })
@@ -939,6 +945,10 @@ func TestApplySATOverlayFiltersIgnoredLegacyDevices(t *testing.T) {
 
 func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) {
 	tmp := t.TempDir()
+	// Isolate os.TempDir() so a concurrent support-bundle test (this or the
+	// webui endpoint test, in another package running in parallel) cannot
+	// collide on the staging tree / archive path.
+	t.Setenv("TMPDIR", tmp)
 	exportDir := filepath.Join(tmp, "export")
 	if err := os.MkdirAll(filepath.Join(exportDir, "bee-sat", "memory-run"), 0755); err != nil {
 		t.Fatal(err)
@@ -1065,6 +1075,7 @@ func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) {
 // too.
 func TestBuildSupportBundleIncludesOrientationDocs(t *testing.T) {
 	tmp := t.TempDir()
+	t.Setenv("TMPDIR", tmp)
 	exportDir := filepath.Join(tmp, "export")
 	if err := os.MkdirAll(exportDir, 0755); err != nil {
 		t.Fatal(err)

audit/internal/app/component_status_db_test.go

@@ -233,7 +233,6 @@ func TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets(t *tes
 	}
 }
 
-
 // TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the
 // hardware-ingest-contract.md rule that status_history is a transition log
 // ("История переходов статусов"), not a per-poll journal. A component

audit/internal/app/support_bundle.go

@@ -455,11 +455,21 @@ func BuildSupportBundle(exportDir string) (string, error) {
 
 	now := time.Now().UTC()
 
-	stageRoot := filepath.Join(os.TempDir(), fmt.Sprintf("bee-support-stage-%s-%s", sanitizeFilename(hostnameOr("unknown")), now.Format("20060102-150405")))
+	// Stage under a private parent dir. Two bundle builds started in the same
+	// wall-clock second (two operators, or an on-demand build racing the
+	// blackbox worker) must not share a staging tree: one's deferred
+	// os.RemoveAll would then wipe the other's half-populated tree and yield a
+	// truncated archive. The leaf name stays meaningful because it becomes the
+	// archive's top-level directory.
+	buildParent, err := os.MkdirTemp(os.TempDir(), "bee-support-build-")
+	if err != nil {
+		return "", err
+	}
+	defer os.RemoveAll(buildParent)
+	stageRoot := filepath.Join(buildParent, fmt.Sprintf("bee-support-stage-%s-%s", sanitizeFilename(hostnameOr("unknown")), now.Format("20060102-150405")))
 	if err := os.MkdirAll(stageRoot, 0755); err != nil {
 		return "", err
 	}
-	defer os.RemoveAll(stageRoot)
 
 	if err := categorizeExportTree(exportDir, stageRoot, true); err != nil {
 		return "", err
@@ -513,10 +523,6 @@ func SupportBundleBaseName(at time.Time) string {
 	return fmt.Sprintf("%s (BEE-SP v%s) %s %s %s", date, ver, model, sn, tod)
 }
 
-func LatestSupportBundlePath() (string, error) {
-	return latestSupportBundlePath(os.TempDir())
-}
-
 func cleanupOldSupportBundles(dir string) error {
 	matches, err := filepath.Glob(filepath.Join(dir, supportBundleGlob))
 	if err != nil {
@@ -538,18 +544,6 @@ func cleanupOldSupportBundles(dir string) error {
 	return nil
 }
 
-func latestSupportBundlePath(dir string) (string, error) {
-	matches, err := filepath.Glob(filepath.Join(dir, supportBundleGlob))
-	if err != nil {
-		return "", err
-	}
-	ordered := orderSupportBundles(supportBundleEntries(matches))
-	if len(ordered) == 0 {
-		return "", os.ErrNotExist
-	}
-	return ordered[0], nil
-}
-
 func supportBundleEntries(matches []string) map[string]time.Time {
 	entries := make(map[string]time.Time, len(matches))
 	for _, match := range matches {
@@ -758,24 +752,6 @@ func buildCommit() string {
 	return strings.TrimSpace(string(raw))
 }
 
-func copyDirContents(srcDir, dstDir string) error {
-	entries, err := os.ReadDir(srcDir)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	for _, entry := range entries {
-		src := filepath.Join(srcDir, entry.Name())
-		dst := filepath.Join(dstDir, entry.Name())
-		if err := copyPath(src, dst); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func copyDirContentsFiltered(srcDir, dstDir string, keep func(rel string, info os.FileInfo) bool) error {
 	entries, err := os.ReadDir(srcDir)
 	if err != nil {