From ac4bc0b2b70901cfa6ba4aa473d6df0793f57553 Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Mon, 31 Aug 2026 21:22:16 +0300 Subject: [PATCH] refactor: modularize audit and harden build validation --- audit/internal/app/app.go | 64 +- audit/internal/app/app_install.go | 12 - audit/internal/app/app_packs.go | 92 +- audit/internal/app/app_test.go | 13 +- .../internal/app/component_status_db_test.go | 1 - audit/internal/app/support_bundle.go | 48 +- audit/internal/collector/nic_mellanox.go | 2 +- audit/internal/collector/sensors.go | 20 - audit/internal/collector/storage.go | 6 +- audit/internal/platform/benchmark.go | 4130 ----------------- audit/internal/platform/benchmark_analysis.go | 659 +++ .../platform/benchmark_power_autotune.go | 10 - .../platform/benchmark_power_calibration.go | 928 ++++ .../benchmark_power_calibration_run.go | 558 +++ .../platform/benchmark_power_report.go | 624 +++ .../internal/platform/benchmark_power_run.go | 495 ++ .../platform/benchmark_result_analysis.go | 537 +++ audit/internal/platform/benchmark_setup.go | 396 ++ .../platform/nvidia_bandwidth_socket.go | 39 +- .../platform/nvidia_bandwidth_socket_test.go | 70 +- .../platform/physical_gpu_vendors_test.go | 53 + audit/internal/platform/platform_stress.go | 43 - audit/internal/platform/sat.go | 848 ---- audit/internal/platform/sat_gpu_packs.go | 568 +++ audit/internal/platform/sat_host_packs.go | 257 + audit/internal/platform/sat_test.go | 6 - audit/internal/platform/tpm.go | 70 +- audit/internal/platform/tpm_test.go | 55 + audit/internal/webui/api.go | 1525 ------ audit/internal/webui/api_gpu_tools.go | 509 ++ audit/internal/webui/api_metrics_network.go | 360 ++ audit/internal/webui/api_runs.go | 430 ++ audit/internal/webui/api_sat_runall.go | 212 + audit/internal/webui/api_sat_runall_test.go | 90 + audit/internal/webui/api_services_system.go | 260 ++ audit/internal/webui/huawei_elabel.go | 24 +- audit/internal/webui/layout.go | 2 +- audit/internal/webui/metrics_charts_http.go | 665 +++ audit/internal/webui/metricsdb.go | 4 - audit/internal/webui/page_benchmark.go | 2 +- audit/internal/webui/page_topo.go | 771 +-- audit/internal/webui/page_topo_diagram.go | 783 ++++ audit/internal/webui/page_validate.go | 147 +- audit/internal/webui/page_validate_test.go | 1 - audit/internal/webui/pages.go | 684 --- audit/internal/webui/pages_health.go | 572 +++ audit/internal/webui/raid_actions.go | 707 +++ audit/internal/webui/raid_mgmt.go | 699 --- audit/internal/webui/saa_dmi.go | 2 - audit/internal/webui/server.go | 674 +-- audit/internal/webui/server_test.go | 4 + audit/internal/webui/task_page.go | 8 - audit/internal/webui/task_runner.go | 5 +- audit/internal/webui/tasks.go | 401 -- .../internal/webui/tasks_http_persistence.go | 412 ++ audit/internal/webui/tasks_test.go | 39 +- bible-local/architecture/api-surface.md | 1 + bible-local/architecture/runtime-flows.md | 87 +- .../2026-04-01-memtest-build-strategy.md | 7 +- ...pci-realloc-out-of-default-boot-entries.md | 9 +- .../2026-08-31-backend-driven-sat-planning.md | 67 + .../2026-08-31-bee-nvidia-restart-deadlock.md | 66 + ...nvbandwidth-validate-single-deep-matrix.md | 47 + bible-local/decisions/README.md | 4 + .../docs/customer-gpu-test-methodology.md | 81 +- bible-local/docs/iso-build-rules.md | 23 +- iso/builder/bee-gpu-stress-cublaslt.inc | 944 ++++ iso/builder/bee-gpu-stress-cuda.inc | 356 ++ iso/builder/bee-gpu-stress-main.inc | 202 + iso/builder/bee-gpu-stress.c | 1495 +----- iso/builder/build.sh | 1339 +----- .../hooks/normal/9100-memtest.hook.binary | 76 +- iso/builder/lib/bootloader.sh | 146 + iso/builder/lib/build-log.sh | 137 + iso/builder/lib/fast-path.sh | 206 + iso/builder/lib/iso-validation.sh | 636 +++ iso/builder/test-build-libs.sh | 98 + iso/overlay/usr/local/bin/bee-nvidia-load | 105 +- 78 files changed, 13598 insertions(+), 13130 deletions(-) create mode 100644 audit/internal/platform/benchmark_analysis.go create mode 100644 audit/internal/platform/benchmark_power_calibration.go create mode 100644 audit/internal/platform/benchmark_power_calibration_run.go create mode 100644 audit/internal/platform/benchmark_power_report.go create mode 100644 audit/internal/platform/benchmark_power_run.go create mode 100644 audit/internal/platform/benchmark_result_analysis.go create mode 100644 audit/internal/platform/benchmark_setup.go create mode 100644 audit/internal/platform/physical_gpu_vendors_test.go create mode 100644 audit/internal/platform/sat_gpu_packs.go create mode 100644 audit/internal/platform/sat_host_packs.go create mode 100644 audit/internal/webui/api_gpu_tools.go create mode 100644 audit/internal/webui/api_metrics_network.go create mode 100644 audit/internal/webui/api_runs.go create mode 100644 audit/internal/webui/api_sat_runall.go create mode 100644 audit/internal/webui/api_sat_runall_test.go create mode 100644 audit/internal/webui/api_services_system.go create mode 100644 audit/internal/webui/metrics_charts_http.go create mode 100644 audit/internal/webui/page_topo_diagram.go create mode 100644 audit/internal/webui/pages_health.go create mode 100644 audit/internal/webui/raid_actions.go create mode 100644 audit/internal/webui/tasks_http_persistence.go create mode 100644 bible-local/decisions/2026-08-31-backend-driven-sat-planning.md create mode 100644 bible-local/decisions/2026-08-31-bee-nvidia-restart-deadlock.md create mode 100644 bible-local/decisions/2026-08-31-nvbandwidth-validate-single-deep-matrix.md create mode 100644 iso/builder/bee-gpu-stress-cublaslt.inc create mode 100644 iso/builder/bee-gpu-stress-cuda.inc create mode 100644 iso/builder/bee-gpu-stress-main.inc create mode 100755 iso/builder/lib/bootloader.sh create mode 100755 iso/builder/lib/build-log.sh create mode 100755 iso/builder/lib/fast-path.sh create mode 100755 iso/builder/lib/iso-validation.sh create mode 100755 iso/builder/test-build-libs.sh diff --git a/audit/internal/app/app.go b/audit/internal/app/app.go index c681bc3..4b982ea 100644 --- a/audit/internal/app/app.go +++ b/audit/internal/app/app.go @@ -6,7 +6,6 @@ import ( "fmt" "log/slog" "os" - "strconv" "strings" "bee/audit/internal/collector" @@ -96,14 +95,38 @@ type installer interface { type GPUPresenceResult struct { Nvidia bool AMD bool + // NvidiaInitializing / AMDInitializing report a PCI device of that vendor + // when DetectGPUVendor did not report the vendor as operational. + NvidiaInitializing bool + AMDInitializing bool } +// DetectGPUPresence combines the existing operational vendor detection with a +// PCI display-class scan. The latter distinguishes absent hardware from a +// detected PCI function whose runtime is not operational yet. func (a *App) DetectGPUPresence() GPUPresenceResult { vendor := a.sat.DetectGPUVendor() - return GPUPresenceResult{ + res := GPUPresenceResult{ Nvidia: vendor == "nvidia", AMD: vendor == "amd", } + physNvidia, physAMD := a.sat.PhysicalGPUVendors() + res.NvidiaInitializing = physNvidia && !res.Nvidia + res.AMDInitializing = physAMD && !res.AMD + return res +} + +// RuntimeHealthNow collects a fresh runtime-health snapshot (driver / CUDA +// readiness, GSP state). Unlike ReadRuntimeHealth it does not read the +// boot-time JSON; callers that poll for the GPU stack to come up need +// current data. +func (a *App) RuntimeHealthNow() (schema.RuntimeHealth, error) { + return a.runtime.CollectRuntimeHealth(DefaultExportDir) +} + +// TPMPresent reports whether this host has a TPM the checks can talk to. +func (a *App) TPMPresent() bool { + return a.sat.TPMPresent() } func (a *App) IsLiveMediaInRAM() bool { @@ -132,13 +155,14 @@ type satRunner interface { RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) - RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) + RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, fullMatrix bool, logFunc func(string)) (string, error) RunNvidiaStressPack(ctx context.Context, baseDir string, opts platform.NvidiaStressOptions, logFunc func(string)) (string, error) ListNvidiaGPUStatuses() ([]platform.NvidiaGPUStatus, error) ResetNvidiaGPU(index int) (string, error) RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) RunTPMValidationPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) + TPMPresent() bool RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) RunPCIeLinkCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) RunNvidiaPCIeBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) @@ -151,6 +175,7 @@ type satRunner interface { SetNvidiaGPUPowerLimit(index int, watts float64) (string, error) ResetNvidiaGPUDefaults() (string, error) DetectGPUVendor() string + PhysicalGPUVendors() (nvidia bool, amd bool) ListAMDGPUs() ([]platform.AMDGPUInfo, error) RunAMDAcceptancePack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) RunAMDMemIntegrityPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) @@ -279,15 +304,6 @@ func (a *App) RunRuntimePreflight(output string) (string, error) { } } -func (a *App) RunRuntimePreflightResult() (ActionResult, error) { - path, err := a.RunRuntimePreflight("file:" + DefaultRuntimeJSONPath) - body := "Runtime preflight completed." - if path != "" { - body = "Runtime health written to " + path - } - return ActionResult{Title: "Run self-check", Body: body}, err -} - func (a *App) RuntimeHealthResult() ActionResult { health, err := ReadRuntimeHealth(DefaultRuntimeJSONPath) if err != nil { @@ -326,10 +342,6 @@ func (a *App) RunAuditNow(runtimeMode runtimeenv.Mode) (ActionResult, error) { return ActionResult{Title: "Run audit", Body: body}, err } -func (a *App) RunAuditToDefaultFile(runtimeMode runtimeenv.Mode) (string, error) { - return a.RunAudit(runtimeMode, "file:"+DefaultAuditJSONPath) -} - func (a *App) HealthSummaryResult() ActionResult { raw, err := os.ReadFile(DefaultAuditJSONPath) if err != nil { @@ -399,26 +411,6 @@ func (a *App) MainBanner() string { return strings.TrimSpace(strings.Join(lines, "\n")) } -func (a *App) FormatToolStatuses(statuses []platform.ToolStatus) string { - var body strings.Builder - for _, tool := range statuses { - status := "MISSING" - if tool.OK { - status = "OK (" + tool.Path + ")" - } - fmt.Fprintf(&body, "- %s: %s\n", tool.Name, status) - } - return strings.TrimSpace(body.String()) -} - -func (a *App) ParsePrefix(raw string, fallback int) int { - value, err := strconv.Atoi(strings.TrimSpace(raw)) - if err != nil || value <= 0 { - return fallback - } - return value -} - // writePSUStatusesToDB records PSU statuses collected during audit into the // component-status DB so they are visible in the Hardware Summary card. // PSU status is sourced from IPMI (ipmitool fru + sdr) during audit. diff --git a/audit/internal/app/app_install.go b/audit/internal/app/app_install.go index f728ffe..c13c2b4 100644 --- a/audit/internal/app/app_install.go +++ b/audit/internal/app/app_install.go @@ -62,18 +62,6 @@ func (a *App) ExportLatestAudit(target platform.RemovableTarget) (string, error) return a.exports.ExportFileToTarget(tmpPath, target) } -func (a *App) ExportLatestAuditResult(target platform.RemovableTarget) (ActionResult, error) { - path, err := a.ExportLatestAudit(target) - body := "Audit export failed." - if err == nil { - body = "Audit exported." - } - if err == nil && path != "" { - body = "Audit exported to " + path - } - return ActionResult{Title: "Export audit", Body: body}, err -} - func (a *App) ExportSupportBundle(target platform.RemovableTarget) (string, error) { archive, err := BuildSupportBundle(DefaultExportDir) if err != nil { diff --git a/audit/internal/app/app_packs.go b/audit/internal/app/app_packs.go index 2977fc2..310fd88 100644 --- a/audit/internal/app/app_packs.go +++ b/audit/internal/app/app_packs.go @@ -196,11 +196,11 @@ func (a *App) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durati return a.sat.RunNvidiaPulseTestPack(ctx, baseDir, durationSec, gpuIndices, logFunc) } -func (a *App) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) { +func (a *App) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, fullMatrix bool, logFunc func(string)) (string, error) { if strings.TrimSpace(baseDir) == "" { baseDir = DefaultSATBaseDir } - return a.sat.RunNvidiaBandwidthPack(ctx, baseDir, gpuIndices, logFunc) + return a.sat.RunNvidiaBandwidthPack(ctx, baseDir, gpuIndices, fullMatrix, logFunc) } func (a *App) RunNvidiaStressPackCtx(ctx context.Context, baseDir string, opts platform.NvidiaStressOptions, logFunc func(string)) (string, error) { @@ -237,11 +237,6 @@ func (a *App) RunCPUAcceptancePackCtx(ctx context.Context, baseDir string, durat return a.sat.RunCPUAcceptancePack(ctx, baseDir, durationSec, logFunc) } -func (a *App) RunCPUAcceptancePackResult(baseDir string, durationSec int) (ActionResult, error) { - path, err := a.RunCPUAcceptancePack(baseDir, durationSec, nil) - return ActionResult{Title: "CPU SAT", Body: satResultBody(path)}, err -} - func (a *App) RunStorageAcceptancePack(baseDir string, logFunc func(string)) (string, error) { return a.RunStorageAcceptancePackCtx(context.Background(), baseDir, false, logFunc) } @@ -276,11 +271,6 @@ func (a *App) RunNvidiaConfigCheckPack(baseDir string, logFunc func(string)) (st return a.RunNvidiaConfigCheckPackCtx(context.Background(), baseDir, logFunc) } -func (a *App) RunNvidiaConfigCheckPackResult(baseDir string) (ActionResult, error) { - path, err := a.RunNvidiaConfigCheckPack(baseDir, nil) - return ActionResult{Title: "GPU Config & NVLink Check", Body: satResultBody(path)}, err -} - func (a *App) RunPCIeLinkCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { if strings.TrimSpace(baseDir) == "" { baseDir = DefaultSATBaseDir @@ -292,11 +282,6 @@ func (a *App) RunPCIeLinkCheckPack(baseDir string, logFunc func(string)) (string return a.RunPCIeLinkCheckPackCtx(context.Background(), baseDir, logFunc) } -func (a *App) RunPCIeLinkCheckPackResult(baseDir string) (ActionResult, error) { - path, err := a.RunPCIeLinkCheckPack(baseDir, nil) - return ActionResult{Title: "PCIe Link Check", Body: satResultBody(path)}, err -} - func (a *App) RunNvidiaPCIeBandwidthPackCtx(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) { if strings.TrimSpace(baseDir) == "" { baseDir = DefaultSATBaseDir @@ -308,11 +293,6 @@ func (a *App) RunNvidiaPCIeBandwidthPack(baseDir string, gpuIndices []int, logFu return a.RunNvidiaPCIeBandwidthPackCtx(context.Background(), baseDir, gpuIndices, logFunc) } -func (a *App) RunNvidiaPCIeBandwidthPackResult(baseDir string) (ActionResult, error) { - path, err := a.RunNvidiaPCIeBandwidthPack(baseDir, nil, nil) - return ActionResult{Title: "NVIDIA GPU PCIe Bandwidth Check", Body: satResultBody(path)}, err -} - func (a *App) DetectGPUVendor() string { return a.sat.DetectGPUVendor() } @@ -332,11 +312,6 @@ func (a *App) RunAMDAcceptancePackCtx(ctx context.Context, baseDir string, logFu return a.sat.RunAMDAcceptancePack(ctx, baseDir, logFunc) } -func (a *App) RunAMDAcceptancePackResult(baseDir string) (ActionResult, error) { - path, err := a.RunAMDAcceptancePack(baseDir, nil) - return ActionResult{Title: "AMD GPU SAT", Body: satResultBody(path)}, err -} - func (a *App) RunAMDMemIntegrityPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { if strings.TrimSpace(baseDir) == "" { baseDir = DefaultSATBaseDir @@ -399,69 +374,6 @@ func (a *App) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int return a.sat.RunNCCLTests(ctx, baseDir, gpuIndices, logFunc) } -func (a *App) RunNCCLTestsResult(ctx context.Context) (ActionResult, error) { - path, err := a.RunNCCLTests(ctx, DefaultSATBaseDir, nil, nil) - body := "Results: " + path - if err != nil && err != context.Canceled { - body += "\nERROR: " + err.Error() - } - return ActionResult{Title: "NCCL bandwidth test", Body: body}, err -} - -func (a *App) RunFanStressTestResult(ctx context.Context, opts platform.FanStressOptions) (ActionResult, error) { - path, err := a.RunFanStressTest(ctx, "", opts) - body := formatFanStressResult(path) - if err != nil && err != context.Canceled { - body += "\nERROR: " + err.Error() - } - return ActionResult{Title: "GPU Platform Stress Test", Body: body}, err -} - -// formatFanStressResult formats the summary.txt from a fan-stress run, including -// the per-step pass/fail display and the analysis section (throttling, max temps, fan response). -func formatFanStressResult(archivePath string) string { - if archivePath == "" { - return "No output produced." - } - runDir := strings.TrimSuffix(archivePath, ".tar.gz") - raw, err := os.ReadFile(filepath.Join(runDir, "summary.txt")) - if err != nil { - return "Archive written to " + archivePath - } - content := strings.TrimSpace(string(raw)) - kv := parseKeyValueSummary(content) - - var b strings.Builder - b.WriteString(formatSATDetail(content)) - - // Append analysis section. - var analysis []string - if v, ok := kv["throttling_detected"]; ok { - label := "NO" - if v == "true" { - label = "YES ← throttling detected during load" - } - analysis = append(analysis, "Throttling: "+label) - } - if v, ok := kv["max_gpu_temp_c"]; ok && v != "0.0" { - analysis = append(analysis, "Max GPU temp: "+v+"°C") - } - if v, ok := kv["max_cpu_temp_c"]; ok && v != "0.0" { - analysis = append(analysis, "Max CPU temp: "+v+"°C") - } - if v, ok := kv["fan_response_sec"]; ok && v != "N/A" && v != "-1.0" { - analysis = append(analysis, "Fan response: "+v+"s") - } - - if len(analysis) > 0 { - b.WriteString("\n\n=== Analysis ===\n") - for _, line := range analysis { - b.WriteString(line + "\n") - } - } - return strings.TrimSpace(b.String()) -} - // satResultBody reads summary.txt from the SAT run directory (archive path without .tar.gz) // and returns a formatted human-readable result. Falls back to a plain message if unreadable. func satResultBody(archivePath string) string { diff --git a/audit/internal/app/app_test.go b/audit/internal/app/app_test.go index 04bab78..c5ccbbf 100644 --- a/audit/internal/app/app_test.go +++ b/audit/internal/app/app_test.go @@ -216,7 +216,7 @@ func (f fakeSAT) RunNvidiaPulseTestPack(_ context.Context, baseDir string, durat return f.runNvidiaFn(baseDir) } -func (f fakeSAT) RunNvidiaBandwidthPack(_ context.Context, baseDir string, gpuIndices []int, _ func(string)) (string, error) { +func (f fakeSAT) RunNvidiaBandwidthPack(_ context.Context, baseDir string, gpuIndices []int, _ bool, _ func(string)) (string, error) { if f.runNvidiaBandwidthFn != nil { return f.runNvidiaBandwidthFn(baseDir, gpuIndices) } @@ -287,6 +287,10 @@ func (f fakeSAT) RunTPMValidationPack(_ context.Context, _ string, _ func(string return "", nil } +func (f fakeSAT) TPMPresent() bool { return true } + +func (f fakeSAT) PhysicalGPUVendors() (bool, bool) { return false, false } + func (f fakeSAT) RunNvidiaConfigCheckPack(_ context.Context, baseDir string, _ func(string)) (string, error) { return "", nil } @@ -725,6 +729,7 @@ func TestActionResultsUseFallbackBody(t *testing.T) { func TestExportSupportBundleResultMentionsUnmountedUSB(t *testing.T) { tmp := t.TempDir() + t.Setenv("TMPDIR", tmp) oldExportDir := DefaultExportDir DefaultExportDir = tmp t.Cleanup(func() { DefaultExportDir = oldExportDir }) @@ -761,6 +766,7 @@ func TestExportSupportBundleResultMentionsUnmountedUSB(t *testing.T) { func TestExportSupportBundleResultDoesNotPretendSuccessOnError(t *testing.T) { tmp := t.TempDir() + t.Setenv("TMPDIR", tmp) oldExportDir := DefaultExportDir DefaultExportDir = tmp t.Cleanup(func() { DefaultExportDir = oldExportDir }) @@ -939,6 +945,10 @@ func TestApplySATOverlayFiltersIgnoredLegacyDevices(t *testing.T) { func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) { tmp := t.TempDir() + // Isolate os.TempDir() so a concurrent support-bundle test (this or the + // webui endpoint test, in another package running in parallel) cannot + // collide on the staging tree / archive path. + t.Setenv("TMPDIR", tmp) exportDir := filepath.Join(tmp, "export") if err := os.MkdirAll(filepath.Join(exportDir, "bee-sat", "memory-run"), 0755); err != nil { t.Fatal(err) @@ -1065,6 +1075,7 @@ func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) { // too. func TestBuildSupportBundleIncludesOrientationDocs(t *testing.T) { tmp := t.TempDir() + t.Setenv("TMPDIR", tmp) exportDir := filepath.Join(tmp, "export") if err := os.MkdirAll(exportDir, 0755); err != nil { t.Fatal(err) diff --git a/audit/internal/app/component_status_db_test.go b/audit/internal/app/component_status_db_test.go index c440d88..143a053 100644 --- a/audit/internal/app/component_status_db_test.go +++ b/audit/internal/app/component_status_db_test.go @@ -233,7 +233,6 @@ func TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets(t *tes } } - // TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the // hardware-ingest-contract.md rule that status_history is a transition log // ("История переходов статусов"), not a per-poll journal. A component diff --git a/audit/internal/app/support_bundle.go b/audit/internal/app/support_bundle.go index 7154dc3..f802993 100644 --- a/audit/internal/app/support_bundle.go +++ b/audit/internal/app/support_bundle.go @@ -455,11 +455,21 @@ func BuildSupportBundle(exportDir string) (string, error) { now := time.Now().UTC() - stageRoot := filepath.Join(os.TempDir(), fmt.Sprintf("bee-support-stage-%s-%s", sanitizeFilename(hostnameOr("unknown")), now.Format("20060102-150405"))) + // Stage under a private parent dir. Two bundle builds started in the same + // wall-clock second (two operators, or an on-demand build racing the + // blackbox worker) must not share a staging tree: one's deferred + // os.RemoveAll would then wipe the other's half-populated tree and yield a + // truncated archive. The leaf name stays meaningful because it becomes the + // archive's top-level directory. + buildParent, err := os.MkdirTemp(os.TempDir(), "bee-support-build-") + if err != nil { + return "", err + } + defer os.RemoveAll(buildParent) + stageRoot := filepath.Join(buildParent, fmt.Sprintf("bee-support-stage-%s-%s", sanitizeFilename(hostnameOr("unknown")), now.Format("20060102-150405"))) if err := os.MkdirAll(stageRoot, 0755); err != nil { return "", err } - defer os.RemoveAll(stageRoot) if err := categorizeExportTree(exportDir, stageRoot, true); err != nil { return "", err @@ -513,10 +523,6 @@ func SupportBundleBaseName(at time.Time) string { return fmt.Sprintf("%s (BEE-SP v%s) %s %s %s", date, ver, model, sn, tod) } -func LatestSupportBundlePath() (string, error) { - return latestSupportBundlePath(os.TempDir()) -} - func cleanupOldSupportBundles(dir string) error { matches, err := filepath.Glob(filepath.Join(dir, supportBundleGlob)) if err != nil { @@ -538,18 +544,6 @@ func cleanupOldSupportBundles(dir string) error { return nil } -func latestSupportBundlePath(dir string) (string, error) { - matches, err := filepath.Glob(filepath.Join(dir, supportBundleGlob)) - if err != nil { - return "", err - } - ordered := orderSupportBundles(supportBundleEntries(matches)) - if len(ordered) == 0 { - return "", os.ErrNotExist - } - return ordered[0], nil -} - func supportBundleEntries(matches []string) map[string]time.Time { entries := make(map[string]time.Time, len(matches)) for _, match := range matches { @@ -758,24 +752,6 @@ func buildCommit() string { return strings.TrimSpace(string(raw)) } -func copyDirContents(srcDir, dstDir string) error { - entries, err := os.ReadDir(srcDir) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - for _, entry := range entries { - src := filepath.Join(srcDir, entry.Name()) - dst := filepath.Join(dstDir, entry.Name()) - if err := copyPath(src, dst); err != nil { - return err - } - } - return nil -} - func copyDirContentsFiltered(srcDir, dstDir string, keep func(rel string, info os.FileInfo) bool) error { entries, err := os.ReadDir(srcDir) if err != nil { diff --git a/audit/internal/collector/nic_mellanox.go b/audit/internal/collector/nic_mellanox.go index 20215ab..b9aac9e 100644 --- a/audit/internal/collector/nic_mellanox.go +++ b/audit/internal/collector/nic_mellanox.go @@ -30,7 +30,7 @@ var ( return string(out), nil } - netIfacesByBDF = listNetIfacesByBDF + netIfacesByBDF = listNetIfacesByBDF readNetCarrierFile = func(iface string) (string, error) { path := filepath.Join("/sys/class/net", iface, "carrier") raw, err := os.ReadFile(path) diff --git a/audit/internal/collector/sensors.go b/audit/internal/collector/sensors.go index 0a448b2..b12ca88 100644 --- a/audit/internal/collector/sensors.go +++ b/audit/internal/collector/sensors.go @@ -3,7 +3,6 @@ package collector import ( "bee/audit/internal/schema" "encoding/json" - "log/slog" "os/exec" "sort" "strconv" @@ -12,25 +11,6 @@ import ( type sensorsDoc map[string]map[string]any -func collectSensors() *schema.HardwareSensors { - doc, err := readSensorsJSONDoc() - if err != nil { - slog.Info("sensors: unavailable, skipping", "err", err) - return nil - } - sensors := buildSensorsFromDoc(doc) - if sensors == nil || (len(sensors.Fans) == 0 && len(sensors.Power) == 0 && len(sensors.Temperatures) == 0 && len(sensors.Other) == 0) { - return nil - } - slog.Info("sensors: collected", - "fans", len(sensors.Fans), - "power", len(sensors.Power), - "temperatures", len(sensors.Temperatures), - "other", len(sensors.Other), - ) - return sensors -} - func readSensorsJSONDoc() (sensorsDoc, error) { out, err := exec.Command("sensors", "-j").Output() if err != nil { diff --git a/audit/internal/collector/storage.go b/audit/internal/collector/storage.go index 2f69983..033e5bc 100644 --- a/audit/internal/collector/storage.go +++ b/audit/internal/collector/storage.go @@ -280,9 +280,9 @@ type smartctlInfo struct { ScsiVendor string `json:"scsi_vendor"` ScsiProduct string `json:"scsi_product"` SerialNumber string `json:"serial_number"` - FirmwareVer string `json:"firmware_version"` - RotationRate int `json:"rotation_rate"` - Temperature struct { + FirmwareVer string `json:"firmware_version"` + RotationRate int `json:"rotation_rate"` + Temperature struct { Current int `json:"current"` } `json:"temperature"` SmartStatus struct { diff --git a/audit/internal/platform/benchmark.go b/audit/internal/platform/benchmark.go index a630707..01be3cd 100644 --- a/audit/internal/platform/benchmark.go +++ b/audit/internal/platform/benchmark.go @@ -1,18 +1,12 @@ package platform import ( - "bee/audit/internal/collector" "context" - "encoding/csv" "encoding/json" - "errors" "fmt" - "math" "os" - "os/exec" "path/filepath" "regexp" - "sort" "strconv" "strings" "time" @@ -773,4127 +767,3 @@ func (s *System) RunNvidiaBenchmark(ctx context.Context, baseDir string, opts Nv return runDir, nil } - -func normalizeNvidiaBenchmarkOptionsForBenchmark(opts NvidiaBenchmarkOptions) NvidiaBenchmarkOptions { - switch strings.TrimSpace(strings.ToLower(opts.Profile)) { - case NvidiaBenchmarkProfileStability: - opts.Profile = NvidiaBenchmarkProfileStability - case NvidiaBenchmarkProfileOvernight: - opts.Profile = NvidiaBenchmarkProfileOvernight - default: - opts.Profile = NvidiaBenchmarkProfileStandard - } - if opts.SizeMB < 0 { - opts.SizeMB = 0 - } - opts.ServerPowerSource = normalizeBenchmarkPowerSource(opts.ServerPowerSource) - opts.GPUIndices = dedupeSortedIndices(opts.GPUIndices) - opts.ExcludeGPUIndices = dedupeSortedIndices(opts.ExcludeGPUIndices) - return opts -} - -func resolveBenchmarkProfile(profile string) benchmarkProfileSpec { - switch strings.TrimSpace(strings.ToLower(profile)) { - case NvidiaBenchmarkProfileStability: - return benchmarkProfileSpec{Name: NvidiaBenchmarkProfileStability, BaselineSec: 30, WarmupSec: 120, SteadySec: 3600, NCCLSec: 300, CooldownSec: 0} - case NvidiaBenchmarkProfileOvernight: - return benchmarkProfileSpec{Name: NvidiaBenchmarkProfileOvernight, BaselineSec: 60, WarmupSec: 180, SteadySec: 27000, NCCLSec: 600, CooldownSec: 0} - default: - return benchmarkProfileSpec{Name: NvidiaBenchmarkProfileStandard, BaselineSec: 15, WarmupSec: 45, SteadySec: 480, NCCLSec: 180, CooldownSec: 0} - } -} - -// benchmarkGPUInfoQuery describes a nvidia-smi --query-gpu field set to try. -// Fields are tried in order; the first successful query wins. Extended fields -// (attribute.multiprocessor_count, power.default_limit) are not supported on -// all driver versions, so we fall back to the base set if the full query fails. -// The minimal fallback omits clock fields entirely — clocks.max.* returns -// exit status 2 on some GPU generations (e.g. Blackwell); missing data is -// then recovered from nvidia-smi -q. -var benchmarkGPUInfoQueries = []struct { - fields string - extended bool // whether this query includes optional extended fields - minimal bool // clock fields omitted; max clocks must be filled separately -}{ - { - fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics,attribute.multiprocessor_count,power.default_limit", - extended: true, - }, - { - fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics", - extended: false, - }, - { - fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit", - minimal: true, - }, -} - -// enrichGPUInfoWithNvidiaSMIQ fills benchmark GPU metadata from nvidia-smi -q -// for fields that may be missing from --query-gpu on some driver versions. -func enrichGPUInfoWithNvidiaSMIQ(infoByIndex map[int]benchmarkGPUInfo, nvsmiQ []byte) { - if len(infoByIndex) == 0 || len(nvsmiQ) == 0 { - return - } - - // Build bus_id → index map for matching verbose sections to GPU indices. - busToBenchIdx := make(map[string]int, len(infoByIndex)) - for idx, info := range infoByIndex { - if info.BusID != "" { - // nvidia-smi -q uses "GPU 00000000:4E:00.0" (8-digit domain), - // while --query-gpu returns the same format; normalise to lower. - busToBenchIdx[strings.ToLower(strings.TrimSpace(info.BusID))] = idx - } - } - - // Split the verbose output into per-GPU sections on "^GPU " lines. - gpuSectionRe := regexp.MustCompile(`(?m)^GPU\s+([\dA-Fa-f:\.]+)`) - maxGfxRe := regexp.MustCompile(`(?i)Max Clocks[\s\S]*?Graphics\s*:\s*(\d+)\s*MHz`) - maxMemRe := regexp.MustCompile(`(?i)Max Clocks[\s\S]*?Memory\s*:\s*(\d+)\s*MHz`) - defaultPwrRe := regexp.MustCompile(`(?i)Default Power Limit\s*:\s*([0-9.]+)\s*W`) - currentPwrRe := regexp.MustCompile(`(?i)Current Power Limit\s*:\s*([0-9.]+)\s*W`) - minPwrRe := regexp.MustCompile(`(?i)Min Power Limit\s*:\s*([0-9.]+)\s*W`) - maxPwrRe := regexp.MustCompile(`(?i)Max Power Limit\s*:\s*([0-9.]+)\s*W`) - smCountRe := regexp.MustCompile(`(?i)Multiprocessor Count\s*:\s*(\d+)`) - shutdownTempRe := regexp.MustCompile(`(?i)GPU Shutdown Temp\s*:\s*(\d+)\s*C`) - slowdownTempRe := regexp.MustCompile(`(?i)GPU Slowdown Temp\s*:\s*(\d+)\s*C`) - - sectionStarts := gpuSectionRe.FindAllSubmatchIndex(nvsmiQ, -1) - for i, loc := range sectionStarts { - busID := strings.ToLower(string(nvsmiQ[loc[2]:loc[3]])) - benchIdx, ok := busToBenchIdx[busID] - if !ok { - // Bus IDs from verbose output may have a different domain prefix; - // try suffix match on the slot portion (XX:XX.X). - for k, v := range busToBenchIdx { - if strings.HasSuffix(k, busID) || strings.HasSuffix(busID, k) { - benchIdx = v - ok = true - break - } - } - } - if !ok { - continue - } - - end := len(nvsmiQ) - if i+1 < len(sectionStarts) { - end = sectionStarts[i+1][0] - } - section := nvsmiQ[loc[0]:end] - - info := infoByIndex[benchIdx] - - if info.MaxGraphicsClockMHz == 0 { - if m := maxGfxRe.FindSubmatch(section); m != nil { - if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil { - info.MaxGraphicsClockMHz = v - } - } - } - if info.MaxMemoryClockMHz == 0 { - if m := maxMemRe.FindSubmatch(section); m != nil { - if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil { - info.MaxMemoryClockMHz = v - } - } - } - if info.DefaultPowerLimitW == 0 { - if m := defaultPwrRe.FindSubmatch(section); m != nil { - if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 { - info.DefaultPowerLimitW = v - } - } - } - if info.PowerLimitW == 0 { - if m := currentPwrRe.FindSubmatch(section); m != nil { - if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 { - info.PowerLimitW = v - } - } - } - if info.MinPowerLimitW == 0 { - if m := minPwrRe.FindSubmatch(section); m != nil { - if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 { - info.MinPowerLimitW = v - } - } - } - if info.MaxPowerLimitW == 0 { - if m := maxPwrRe.FindSubmatch(section); m != nil { - if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 { - info.MaxPowerLimitW = v - } - } - } - if info.MultiprocessorCount == 0 { - if m := smCountRe.FindSubmatch(section); m != nil { - if v, err := strconv.Atoi(string(m[1])); err == nil && v > 0 { - info.MultiprocessorCount = v - } - } - } - if info.ShutdownTempC == 0 { - if m := shutdownTempRe.FindSubmatch(section); m != nil { - if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 { - info.ShutdownTempC = v - } - } - } - if info.SlowdownTempC == 0 { - if m := slowdownTempRe.FindSubmatch(section); m != nil { - if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 { - info.SlowdownTempC = v - } - } - } - infoByIndex[benchIdx] = info - } -} - -func queryBenchmarkGPUInfo(gpuIndices []int) (map[int]benchmarkGPUInfo, error) { - var lastErr error - for _, q := range benchmarkGPUInfoQueries { - args := []string{ - "--query-gpu=" + q.fields, - "--format=csv,noheader,nounits", - } - if len(gpuIndices) > 0 { - args = append([]string{"--id=" + joinIndexList(gpuIndices)}, args...) - } - out, err := satExecCommand("nvidia-smi", args...).Output() - if err != nil { - lastErr = fmt.Errorf("nvidia-smi gpu info (%s): %w", q.fields[:min(len(q.fields), 40)], err) - continue - } - - r := csv.NewReader(strings.NewReader(string(out))) - r.TrimLeadingSpace = true - r.FieldsPerRecord = -1 - rows, err := r.ReadAll() - if err != nil { - lastErr = fmt.Errorf("parse nvidia-smi gpu info: %w", err) - continue - } - - minFields := 6 - if !q.minimal { - minFields = 9 - } - infoByIndex := make(map[int]benchmarkGPUInfo, len(rows)) - for _, row := range rows { - if len(row) < minFields { - continue - } - idx, err := strconv.Atoi(strings.TrimSpace(row[0])) - if err != nil { - continue - } - info := benchmarkGPUInfo{ - Index: idx, - UUID: strings.TrimSpace(row[1]), - Name: strings.TrimSpace(row[2]), - BusID: strings.TrimSpace(row[3]), - VBIOS: strings.TrimSpace(row[4]), - PowerLimitW: parseBenchmarkFloat(row[5]), - } - if !q.minimal { - info.MaxGraphicsClockMHz = parseBenchmarkFloat(row[6]) - info.MaxMemoryClockMHz = parseBenchmarkFloat(row[7]) - if len(row) >= 9 { - info.BaseGraphicsClockMHz = parseBenchmarkFloat(row[8]) - } - if q.extended { - if len(row) >= 10 { - info.MultiprocessorCount = int(parseBenchmarkFloat(row[9])) - } - if len(row) >= 11 { - info.DefaultPowerLimitW = parseBenchmarkFloat(row[10]) - } - } - } - infoByIndex[idx] = info - } - return infoByIndex, nil - } - return nil, lastErr -} - -func applyBenchmarkNormalization(ctx context.Context, verboseLog string, gpuIndices []int, infoByIndex map[int]benchmarkGPUInfo, result *NvidiaBenchmarkResult) []benchmarkRestoreAction { - if os.Geteuid() != 0 { - result.Normalization.Status = "partial" - result.Normalization.Notes = append(result.Normalization.Notes, "benchmark normalization skipped: root privileges are required for persistence mode and clock locks") - for _, idx := range gpuIndices { - result.Normalization.GPUs = append(result.Normalization.GPUs, BenchmarkNormalizationGPU{ - Index: idx, - Notes: []string{"normalization skipped: root privileges are required"}, - }) - } - return nil - } - - var restore []benchmarkRestoreAction - for _, idx := range gpuIndices { - rec := BenchmarkNormalizationGPU{Index: idx} - if _, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("normalize-gpu-%d-pm", idx), []string{"nvidia-smi", "-i", strconv.Itoa(idx), "-pm", "1"}, nil, nil); err != nil { - rec.PersistenceMode = "failed" - rec.Notes = append(rec.Notes, "failed to enable persistence mode") - result.Normalization.Status = "partial" - } else { - rec.PersistenceMode = "applied" - } - - if info, ok := infoByIndex[idx]; ok && info.MaxGraphicsClockMHz > 0 { - target := int(math.Round(info.MaxGraphicsClockMHz)) - if out, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("normalize-gpu-%d-lgc", idx), []string{"nvidia-smi", "-i", strconv.Itoa(idx), "-lgc", strconv.Itoa(target)}, nil, nil); err != nil { - rec.GPUClockLockStatus = "failed" - rec.Notes = append(rec.Notes, "graphics clock lock failed: "+strings.TrimSpace(string(out))) - result.Normalization.Status = "partial" - } else { - rec.GPUClockLockStatus = "applied" - rec.GPUClockLockMHz = float64(target) - idxCopy := idx - restore = append(restore, benchmarkRestoreAction{name: fmt.Sprintf("gpu-%d-rgc", idxCopy), fn: func() { - _, _ = runSATCommandCtx(context.Background(), verboseLog, fmt.Sprintf("restore-gpu-%d-rgc", idxCopy), []string{"nvidia-smi", "-i", strconv.Itoa(idxCopy), "-rgc"}, nil, nil) - }}) - } - } else { - rec.GPUClockLockStatus = "skipped" - rec.Notes = append(rec.Notes, "graphics clock lock skipped: gpu inventory unavailable or MaxGraphicsClockMHz=0") - result.Normalization.Status = "partial" - } - - if info, ok := infoByIndex[idx]; ok && info.MaxMemoryClockMHz > 0 { - target := int(math.Round(info.MaxMemoryClockMHz)) - out, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("normalize-gpu-%d-lmc", idx), []string{"nvidia-smi", "-i", strconv.Itoa(idx), "-lmc", strconv.Itoa(target)}, nil, nil) - switch { - case err == nil: - rec.MemoryClockLockStatus = "applied" - rec.MemoryClockLockMHz = float64(target) - idxCopy := idx - restore = append(restore, benchmarkRestoreAction{name: fmt.Sprintf("gpu-%d-rmc", idxCopy), fn: func() { - _, _ = runSATCommandCtx(context.Background(), verboseLog, fmt.Sprintf("restore-gpu-%d-rmc", idxCopy), []string{"nvidia-smi", "-i", strconv.Itoa(idxCopy), "-rmc"}, nil, nil) - }}) - case strings.Contains(strings.ToLower(string(out)), "deferred") || strings.Contains(strings.ToLower(string(out)), "not supported"): - rec.MemoryClockLockStatus = "unsupported" - rec.Notes = append(rec.Notes, "memory clock lock unsupported on this GPU/driver path") - result.Normalization.Status = "partial" - default: - rec.MemoryClockLockStatus = "failed" - rec.Notes = append(rec.Notes, "memory clock lock failed: "+strings.TrimSpace(string(out))) - result.Normalization.Status = "partial" - } - } - - result.Normalization.GPUs = append(result.Normalization.GPUs, rec) - } - return restore -} - -func collectBenchmarkSamples(ctx context.Context, durationSec int, gpuIndices []int) ([]GPUMetricRow, error) { - if durationSec <= 0 { - return nil, nil - } - deadline := time.Now().Add(time.Duration(durationSec) * time.Second) - var rows []GPUMetricRow - start := time.Now() - for { - if ctx.Err() != nil { - return rows, ctx.Err() - } - samples, err := sampleBenchmarkTelemetry(gpuIndices) - if err == nil { - elapsed := time.Since(start).Seconds() - for i := range samples { - samples[i].ElapsedSec = elapsed - } - rows = append(rows, samples...) - } - if time.Now().After(deadline) { - break - } - select { - case <-ctx.Done(): - return rows, ctx.Err() - case <-time.After(time.Second): - } - } - return rows, nil -} - -func runBenchmarkCommandWithMetrics(ctx context.Context, verboseLog, name string, cmd []string, env []string, gpuIndices []int, logFunc func(string)) ([]byte, []GPUMetricRow, error) { - stopCh := make(chan struct{}) - doneCh := make(chan struct{}) - var metricRows []GPUMetricRow - start := time.Now() - - go func() { - defer close(doneCh) - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - for { - select { - case <-stopCh: - return - case <-ticker.C: - samples, err := sampleBenchmarkTelemetry(gpuIndices) - if err != nil { - continue - } - elapsed := time.Since(start).Seconds() - for i := range samples { - samples[i].ElapsedSec = elapsed - } - metricRows = append(metricRows, samples...) - } - } - }() - - out, err := runSATCommandCtx(ctx, verboseLog, name, cmd, env, logFunc) - close(stopCh) - <-doneCh - - return out, metricRows, err -} - -type benchmarkPlannedPhase struct { - PlanLabel string - MetricStage string - DurationSec int -} - -func runBenchmarkPlannedCommandWithMetrics( - ctx context.Context, - verboseLog, name string, - cmd []string, - env []string, - gpuIndices []int, - phases []benchmarkPlannedPhase, - logFunc func(string), -) ([]byte, map[string][]GPUMetricRow, map[string][]byte, error) { - out, rows, err := runBenchmarkCommandWithMetrics(ctx, verboseLog, name, cmd, env, gpuIndices, logFunc) - return out, splitBenchmarkRowsByPlannedPhase(rows, phases), splitBenchmarkLogByPlannedPhase(out), err -} - -func splitBenchmarkRowsByPlannedPhase(rows []GPUMetricRow, phases []benchmarkPlannedPhase) map[string][]GPUMetricRow { - out := make(map[string][]GPUMetricRow, len(phases)) - if len(rows) == 0 || len(phases) == 0 { - return out - } - for _, row := range rows { - idx := len(phases) - 1 - var elapsed float64 - for i, phase := range phases { - durationSec := phase.DurationSec - if durationSec <= 0 { - durationSec = 1 - } - elapsed += float64(durationSec) - if row.ElapsedSec < elapsed { - idx = i - break - } - } - out[phases[idx].MetricStage] = append(out[phases[idx].MetricStage], row) - } - return out -} - -func splitBenchmarkLogByPlannedPhase(raw []byte) map[string][]byte { - out := make(map[string][]byte) - var current string - for _, line := range strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") { - trimmed := strings.TrimSpace(stripBenchmarkPrefix(line)) - switch { - case strings.HasPrefix(trimmed, "phase_begin="): - current = strings.TrimSpace(strings.TrimPrefix(trimmed, "phase_begin=")) - case strings.HasPrefix(trimmed, "phase_end="): - current = "" - case current != "": - out[current] = append(out[current], []byte(line+"\n")...) - } - } - return out -} - -type benchmarkCoolingSample struct { - AvgFanRPM float64 - AvgFanDutyCyclePct float64 - FanDutyCycleAvailable bool - FanDutyCycleEstimated bool -} - -func sampleBenchmarkTelemetry(gpuIndices []int) ([]GPUMetricRow, error) { - samples, err := sampleGPUMetrics(gpuIndices) - if err != nil { - return nil, err - } - fanSample := sampleBenchmarkCoolingSample() - for i := range samples { - samples[i].FanAvgRPM = fanSample.AvgFanRPM - samples[i].FanDutyCyclePct = fanSample.AvgFanDutyCyclePct - samples[i].FanDutyCycleAvailable = fanSample.FanDutyCycleAvailable - samples[i].FanDutyCycleEstimated = fanSample.FanDutyCycleEstimated - } - return samples, nil -} - -func sampleBenchmarkCoolingSample() benchmarkCoolingSample { - fans, _ := sampleFanSpeeds() - avgRPM, _, _ := fanRPMStats(fans) - dutyPct, dutyAvailable, dutyEstimated := sampleFanDutyCyclePctFromFans(fans) - return benchmarkCoolingSample{ - AvgFanRPM: avgRPM, - AvgFanDutyCyclePct: dutyPct, - FanDutyCycleAvailable: dutyAvailable, - FanDutyCycleEstimated: dutyEstimated, - } -} - -func annotateBenchmarkMetricRows(rows []GPUMetricRow, stage string, offset, durationSec float64) []GPUMetricRow { - if len(rows) == 0 { - return nil - } - stageEnd := offset + durationSec - if stageEnd <= offset { - stageEnd = offset - for _, row := range rows { - if row.ElapsedSec+offset > stageEnd { - stageEnd = row.ElapsedSec + offset - } - } - } - out := make([]GPUMetricRow, len(rows)) - for i, row := range rows { - row.Stage = stage - row.ElapsedSec += offset - row.StageStartSec = offset - row.StageEndSec = stageEnd - out[i] = row - } - return out -} - -func appendBenchmarkMetrics(allRows *[]GPUMetricRow, rows []GPUMetricRow, stage string, cursor *float64, durationSec float64) { - annotated := annotateBenchmarkMetricRows(rows, stage, *cursor, durationSec) - *allRows = append(*allRows, annotated...) - *cursor += durationSec -} - -func writeBenchmarkMetricsFiles(runDir string, rows []GPUMetricRow) { - if len(rows) == 0 { - return - } - _ = WriteGPUMetricsCSV(filepath.Join(runDir, "gpu-metrics.csv"), rows) - _ = WriteGPUMetricsHTML(filepath.Join(runDir, "gpu-metrics.html"), rows) -} - -func appendBenchmarkStageLog(path, source, stage string, raw []byte) { - if path == "" || len(raw) == 0 { - return - } - f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) - if err != nil { - return - } - defer f.Close() - header := fmt.Sprintf("\n========== %s | stage=%s ==========\n", source, stage) - _, _ = f.WriteString(header) - if len(raw) > 0 { - _, _ = f.Write(raw) - if raw[len(raw)-1] != '\n' { - _, _ = f.WriteString("\n") - } - } -} - -func parseBenchmarkBurnLog(raw string) benchmarkBurnParseResult { - result := benchmarkBurnParseResult{} - lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") - profiles := make(map[string]*benchmarkBurnProfile) - for _, line := range lines { - line = stripBenchmarkPrefix(strings.TrimSpace(line)) - if line == "" { - continue - } - switch { - case strings.HasPrefix(line, "device="): - result.Device = strings.TrimSpace(strings.TrimPrefix(line, "device=")) - case strings.HasPrefix(line, "compute_capability="): - result.ComputeCapability = strings.TrimSpace(strings.TrimPrefix(line, "compute_capability=")) - case strings.HasPrefix(line, "backend="): - result.Backend = strings.TrimSpace(strings.TrimPrefix(line, "backend=")) - result.Fallback = result.Backend == "driver-ptx" - case strings.HasPrefix(line, "duration_s="): - result.DurationSec, _ = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "duration_s="))) - default: - if m := benchmarkReadyPattern.FindStringSubmatch(line); len(m) == 6 { - profile := ensureBenchmarkProfile(profiles, m[1]) - profile.supported = true - profile.lanes++ - profile.m, _ = strconv.ParseUint(m[3], 10, 64) - profile.n, _ = strconv.ParseUint(m[4], 10, 64) - profile.k, _ = strconv.ParseUint(m[5], 10, 64) - continue - } - if m := benchmarkSkippedPattern.FindStringSubmatch(line); len(m) == 3 { - profile := ensureBenchmarkProfile(profiles, m[1]) - profile.supported = false - profile.notes = strings.TrimSpace(m[2]) - continue - } - if m := benchmarkIterationsPattern.FindStringSubmatch(line); len(m) == 3 { - profile := ensureBenchmarkProfile(profiles, m[1]) - iters, _ := strconv.ParseUint(m[2], 10, 64) - profile.iterations += iters - } - } - } - - keys := make([]string, 0, len(profiles)) - for key := range profiles { - keys = append(keys, key) - } - sort.Strings(keys) - for _, key := range keys { - profile := profiles[key] - precision := BenchmarkPrecisionResult{ - Name: profile.name, - Category: profile.category, - Supported: profile.supported, - Lanes: profile.lanes, - M: profile.m, - N: profile.n, - K: profile.k, - Iterations: profile.iterations, - Notes: profile.notes, - } - w := precisionWeight(profile.category) - precision.Weight = w - if profile.supported && result.DurationSec > 0 && profile.m > 0 && profile.n > 0 && profile.k > 0 && profile.iterations > 0 { - precision.TeraOpsPerSec = (2.0 * float64(profile.m) * float64(profile.n) * float64(profile.k) * float64(profile.iterations)) / float64(result.DurationSec) / 1e12 - precision.WeightedTeraOpsPerSec = precision.TeraOpsPerSec * w - } - result.Profiles = append(result.Profiles, precision) - } - return result -} - -func ensureBenchmarkProfile(profiles map[string]*benchmarkBurnProfile, name string) *benchmarkBurnProfile { - if profile, ok := profiles[name]; ok { - return profile - } - category := "other" - switch { - case strings.HasPrefix(name, "fp64"): - category = "fp64" - case strings.HasPrefix(name, "fp32"): - category = "fp32_tf32" - case strings.HasPrefix(name, "fp16"): - category = "fp16_bf16" - case strings.HasPrefix(name, "int8"): - category = "int8" - case strings.HasPrefix(name, "fp8"): - category = "fp8" - case strings.HasPrefix(name, "fp4"): - category = "fp4" - } - profile := &benchmarkBurnProfile{name: name, category: category, supported: true} - profiles[name] = profile - return profile -} - -// precisionWeight returns the fp32-equivalence factor for a precision category. -// Each factor represents how much "real" numeric work one operation of that -// type performs relative to fp32 (single precision = 1.0 baseline): -// -// fp64 = 2.0 — double precision, 2× more bits per operand -// fp32 = 1.0 — single precision baseline -// fp16 = 0.5 — half precision -// int8 = 0.25 — quarter precision -// fp8 = 0.25 — quarter precision -// fp4 = 0.125 — eighth precision -// -// Multiplying raw TOPS by the weight gives fp32-equivalent TOPS, enabling -// cross-precision comparison on the same numeric scale. -func precisionWeight(category string) float64 { - switch category { - case "fp64": - return 2.0 - case "fp32_tf32": - return 1.0 - case "fp16_bf16": - return 0.5 - case "int8": - return 0.25 - case "fp8": - return 0.25 - case "fp4": - return 0.125 - default: - return 1.0 - } -} - -func stripBenchmarkPrefix(line string) string { - if strings.HasPrefix(line, "[gpu ") { - if idx := strings.Index(line, "] "); idx >= 0 { - return line[idx+2:] - } - } - return line -} - -func summarizeBenchmarkTelemetry(rows []GPUMetricRow) BenchmarkTelemetrySummary { - summary := BenchmarkTelemetrySummary{} - if len(rows) == 0 { - return summary - } - temps := make([]float64, 0, len(rows)) - powers := make([]float64, 0, len(rows)) - clocks := make([]float64, 0, len(rows)) - memClocks := make([]float64, 0, len(rows)) - usages := make([]float64, 0, len(rows)) - memUsages := make([]float64, 0, len(rows)) - summary.DurationSec = rows[len(rows)-1].ElapsedSec - summary.Samples = len(rows) - for _, row := range rows { - temps = append(temps, row.TempC) - powers = append(powers, row.PowerW) - clocks = append(clocks, row.ClockMHz) - memClocks = append(memClocks, row.MemClockMHz) - usages = append(usages, row.UsagePct) - memUsages = append(memUsages, row.MemUsagePct) - } - summary.AvgTempC = benchmarkMean(temps) - summary.P95TempC = benchmarkPercentile(temps, 95) - summary.AvgPowerW = benchmarkMean(powers) - summary.P95PowerW = benchmarkPercentile(powers, 95) - summary.AvgGraphicsClockMHz = benchmarkMean(clocks) - summary.P95GraphicsClockMHz = benchmarkPercentile(clocks, 95) - summary.AvgMemoryClockMHz = benchmarkMean(memClocks) - summary.P95MemoryClockMHz = benchmarkPercentile(memClocks, 95) - summary.AvgUsagePct = benchmarkMean(usages) - summary.AvgMemUsagePct = benchmarkMean(memUsages) - summary.ClockCVPct = benchmarkCV(clocks) - summary.PowerCVPct = benchmarkCV(powers) - summary.TempCVPct = benchmarkCV(temps) - summary.ClockDriftPct = benchmarkClockDrift(clocks) - return summary -} - -func summarizeBenchmarkCooling(rows []GPUMetricRow) *BenchmarkCoolingSummary { - if len(rows) == 0 { - return nil - } - var rpmValues []float64 - var dutyValues []float64 - var dutyEstimated bool - for _, row := range rows { - if row.FanAvgRPM > 0 { - rpmValues = append(rpmValues, row.FanAvgRPM) - } - if row.FanDutyCycleAvailable { - dutyValues = append(dutyValues, row.FanDutyCyclePct) - if row.FanDutyCycleEstimated { - dutyEstimated = true - } - } - } - if len(rpmValues) == 0 && len(dutyValues) == 0 { - return nil - } - summary := &BenchmarkCoolingSummary{ - Available: true, - AvgFanRPM: benchmarkMean(rpmValues), - FanDutyCycleEstimated: dutyEstimated, - } - if len(dutyValues) > 0 { - summary.FanDutyCycleAvailable = true - summary.AvgFanDutyCyclePct = benchmarkMean(dutyValues) - summary.P95FanDutyCyclePct = benchmarkPercentile(dutyValues, 95) - if summary.FanDutyCycleEstimated { - summary.Notes = append(summary.Notes, "fan duty cycle is estimated from the highest fan RPM observed since boot; treat it as an approximation, not a direct PWM reading") - } - } else { - summary.Notes = append(summary.Notes, "fan duty cycle unavailable on this host; RPM-only fan telemetry was collected") - } - return summary -} - -func benchmarkTelemetryAvailable(summary BenchmarkTelemetrySummary) bool { - return summary.Samples > 0 || summary.DurationSec > 0 -} - -func benchmarkPrecisionSteadyFallback(phases []BenchmarkPrecisionSteadyPhase) (BenchmarkTelemetrySummary, string, bool) { - var ( - best BenchmarkTelemetrySummary - bestLabel string - found bool - ) - for _, phase := range phases { - if !benchmarkTelemetryAvailable(phase.Steady) { - continue - } - if !found || - phase.Steady.DurationSec > best.DurationSec || - (phase.Steady.DurationSec == best.DurationSec && phase.Steady.P95PowerW > best.P95PowerW) { - best = phase.Steady - bestLabel = phase.Precision - found = true - } - } - return best, bestLabel, found -} - -func applyBenchmarkSteadyFallback(gpu *BenchmarkGPUResult) { - if gpu == nil || benchmarkTelemetryAvailable(gpu.Steady) { - return - } - if fallback, label, ok := benchmarkPrecisionSteadyFallback(gpu.PrecisionSteady); ok { - gpu.Steady = fallback - gpu.Notes = append(gpu.Notes, - fmt.Sprintf("mixed steady telemetry unavailable; reporting steady-state fallback from %s precision phase", label)) - } -} - -func scoreBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkScorecard { - score := BenchmarkScorecard{} - - // SyntheticScore: sum of fp32-equivalent TOPS from per-precision phases. - // Each precision ran alone with full GPU dedicated — peak capability. - for _, p := range gpu.PrecisionSteady { - if !benchmarkPrecisionEnabled(p.Precision) { - continue - } - score.SyntheticScore += p.WeightedTeraOpsPerSec - } - - // MixedScore: sum of fp32-equivalent TOPS from the combined phase. - // All precisions compete simultaneously — closer to real inference workloads. - for _, p := range gpu.PrecisionResults { - if p.Supported && benchmarkPrecisionEnabled(p.Category) { - score.MixedScore += p.WeightedTeraOpsPerSec - } - } - - // MixedEfficiency = MixedScore / SyntheticScore. - // Measures how well the GPU sustains throughput under concurrent mixed load. - // A healthy GPU scores ~0.8–0.95; severe degradation suggests bandwidth - // contention or scheduler inefficiency. - if score.SyntheticScore > 0 && score.MixedScore > 0 { - score.MixedEfficiency = score.MixedScore / score.SyntheticScore - } - - // ComputeScore = SyntheticScore × (1 + MixedEfficiency × 0.3). - // SyntheticScore is the primary signal; MixedEfficiency adds up to +30% - // bonus for GPUs that handle mixed-precision concurrency well. - // Falls back to MixedScore alone when per-precision data is absent. - switch { - case score.SyntheticScore > 0: - score.ComputeScore = score.SyntheticScore * (1 + score.MixedEfficiency*0.3) - case score.MixedScore > 0: - score.ComputeScore = score.MixedScore - } - // PowerSustainScore: how stable is GPU power draw during the benchmark? - // High variance means the workload is bursting or the power delivery is - // unstable. Score = max(0, 100 − PowerCVPct × 3). - // At 10% CV → score 70; at 33%+ CV → score 0. - // Uses per-precision windows when available (each runs a single kernel, - // so CV reflects genuine power regulation, not workload switching). - if len(gpu.PrecisionSteady) > 0 { - var sum float64 - var count int - for _, p := range gpu.PrecisionSteady { - if !benchmarkPrecisionEnabled(p.Precision) { - continue - } - sum += clampScore(100 - p.Steady.PowerCVPct*3) - count++ - } - if count > 0 { - score.PowerSustainScore = sum / float64(count) - } - } else if gpu.Steady.PowerCVPct > 0 { - score.PowerSustainScore = clampScore(100 - gpu.Steady.PowerCVPct*3) - } - - // ThermalSustainScore: how stable is GPU temperature during the benchmark? - // High variance means cooling is inconsistent (fan bursts, liquid flow - // instability, or frequent transitions in and out of throttle). - // Score = max(0, 100 − TempCVPct × 3). - if gpu.Steady.TempCVPct > 0 { - score.ThermalSustainScore = clampScore(100 - gpu.Steady.TempCVPct*3) - } else { - // TempCV not recorded — fall back to 100 (no penalty). - score.ThermalSustainScore = 100 - } - - // Throttle breakdown: compute per-type percentages for diagnosis. - // Each counter measures microseconds spent in that throttle state during - // the steady-state window. Counters can overlap (e.g. thermal + power cap - // simultaneously), so they are reported independently, not summed. - runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6) - score.ThermalThrottlePct = math.Min(100, - float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS*100) - score.PowerCapThrottlePct = math.Min(100, - float64(gpu.Throttle.SWPowerCapUS)/runtimeUS*100) - score.SyncBoostThrottlePct = math.Min(100, - float64(gpu.Throttle.SyncBoostUS)/runtimeUS*100) - - // StabilityScore: combined throttle signal (thermal + power cap). - // Score = max(0, 100 − combined_throttle_pct). - // 1% throttle → 99; 10% → 90; any throttle > 0 is penalised. - combinedThrottlePct := math.Min(100, - float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS+gpu.Throttle.SWPowerCapUS)/runtimeUS*100) - score.StabilityScore = clampScore(100 - combinedThrottlePct) - - // TempHeadroomC: distance from p95 temperature to the GPU's hardware - // shutdown threshold (sourced from nvidia-smi -q "GPU Shutdown Temp"). - // Fallback: 90°C when not available. - // Assessed independently of throttle — a GPU at 86°C without any throttle - // counter still has limited headroom and operates in degraded reliability zone. - // Warning zone: headroom < (shutdownTemp - slowdownTemp), i.e. past slowdown onset. - // Critical zone: headroom < 10°C from shutdown. - if gpu.Steady.P95TempC > 0 { - shutdownTemp := gpu.ShutdownTempC - if shutdownTemp <= 0 { - shutdownTemp = 90 - } - score.TempHeadroomC = shutdownTemp - gpu.Steady.P95TempC - } - score.ServerQualityScore = serverQualityScore(score) - score.CompositeScore = score.ComputeScore - if gpu.MultiprocessorCount > 0 && gpu.Steady.AvgGraphicsClockMHz > 0 && score.ComputeScore > 0 { - score.TOPSPerSMPerGHz = score.ComputeScore / float64(gpu.MultiprocessorCount) / (gpu.Steady.AvgGraphicsClockMHz / 1000.0) - } - return score -} - -// compositeBenchmarkScore is kept for compatibility with legacy callers. -// CompositeScore = ComputeScore (no quality multiplier; throttling already -// reduces TOPS directly, so no additional penalty is needed). -func compositeBenchmarkScore(score BenchmarkScorecard) float64 { - return score.ComputeScore -} - -// serverQualityScore returns a 0–100 score reflecting server infrastructure -// quality, independent of GPU model or compute speed. -// -// StabilityScore (throttle time) 0.40 — heaviest: direct evidence GPU can't sustain load -// PowerSustainScore (power CV) 0.30 — unstable draw hints at PSU/VRM issues -// ThermalSustainScore (temp CV) 0.30 — unstable temp hints at airflow/cooling issues -func serverQualityScore(score BenchmarkScorecard) float64 { - q := 0.40*(score.StabilityScore/100.0) + - 0.30*(score.PowerSustainScore/100.0) + - 0.30*(score.ThermalSustainScore/100.0) - return clampScore(q * 100) -} - -// detectPowerAnomaly scans per-GPU steady-state metric rows for a sudden -// power drop — a symptom of bad cable contact, VRM fault, or thermal event -// on the power delivery path. Returns a non-empty string if an anomaly is found. -// -// Algorithm: uses a 5-sample rolling baseline; flags any sample that falls -// more than 30% below the baseline while the GPU was otherwise loaded -// (usage > 50%). A sustained throttle (power cap) is not flagged here — -// that is already captured by PowerCapThrottlePct. -func detectPowerAnomaly(rows []GPUMetricRow, gpuIndex int) string { - const windowSize = 5 - const dropThresholdPct = 30.0 - const minUsagePct = 50.0 - - // Filter rows for this GPU during steady state only. - var steady []GPUMetricRow - for _, r := range rows { - if r.GPUIndex == gpuIndex && r.Stage != "" && strings.Contains(r.Stage, "steady") { - steady = append(steady, r) - } - } - if len(steady) < windowSize+2 { - return "" - } - - // Compute initial baseline from the first window. - var baseSum float64 - for i := 0; i < windowSize; i++ { - baseSum += steady[i].PowerW - } - - for i := windowSize; i < len(steady); i++ { - baseline := baseSum / float64(windowSize) - sample := steady[i] - if baseline > 0 && sample.UsagePct >= minUsagePct { - dropPct := (baseline - sample.PowerW) / baseline * 100 - if dropPct >= dropThresholdPct { - return fmt.Sprintf("sudden power drop detected at t=%.0fs: %.0f W → %.0f W (%.0f%% below rolling baseline) — possible bad cable contact or VRM fault", - sample.ElapsedSec, baseline, sample.PowerW, dropPct) - } - } - // Slide the window baseline. - baseSum -= steady[i-windowSize].PowerW - baseSum += sample.PowerW - } - return "" -} - -// detectSlowdownTempExceedance scans steady-state metric rows for a GPU and -// returns a warning string if any temperature sample exceeded the GPU's -// SlowdownTempC threshold. Uses fallback 80°C when SlowdownTempC is zero. -// This is a real-time signal distinct from p95 stats — even a single spike -// above the slowdown threshold is worth flagging. -func detectSlowdownTempExceedance(rows []GPUMetricRow, gpuIndex int, slowdownTempC float64) string { - if slowdownTempC <= 0 { - slowdownTempC = 80 - } - var maxTemp float64 - var exceedCount int - for _, r := range rows { - if r.GPUIndex != gpuIndex { - continue - } - if !strings.Contains(r.Stage, "steady") { - continue - } - if r.TempC > maxTemp { - maxTemp = r.TempC - } - if r.TempC >= slowdownTempC { - exceedCount++ - } - } - if exceedCount == 0 { - return "" - } - return fmt.Sprintf( - "temperature exceeded slowdown threshold (%.0f°C) in %d sample(s) during steady state — peak %.1f°C", - slowdownTempC, exceedCount, maxTemp) -} - -func detectBenchmarkDegradationReasons(gpu BenchmarkGPUResult, normalizationStatus string) []string { - var reasons []string - runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6) - if float64(gpu.Throttle.SWPowerCapUS)/runtimeUS >= 0.05 { - reasons = append(reasons, "power_capped") - } - if float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS >= 0.01 { - reasons = append(reasons, "thermal_limited") - } - if float64(gpu.Throttle.SyncBoostUS)/runtimeUS >= 0.01 { - reasons = append(reasons, "sync_boost_limited") - } - if gpu.LockedGraphicsClockMHz > 0 && gpu.Steady.AvgGraphicsClockMHz < gpu.LockedGraphicsClockMHz*0.90 { - reasons = append(reasons, "low_sm_clock_vs_target") - } - if gpu.Scores.StabilityScore > 0 && gpu.Scores.StabilityScore < 85 { - reasons = append(reasons, "variance_too_high") - } - if normalizationStatus != "full" { - reasons = append(reasons, "normalization_partial") - } - if gpu.PowerLimitDerated { - reasons = append(reasons, "power_limit_derated") - } - if gpu.ECC.Uncorrected > 0 { - reasons = append(reasons, "ecc_uncorrected_errors") - } - if gpu.ECC.Corrected > 0 { - reasons = append(reasons, "ecc_corrected_errors") - } - return dedupeStrings(reasons) -} - -func runBenchmarkInterconnect(ctx context.Context, verboseLog, runDir string, gpuIndices []int, spec benchmarkProfileSpec, logFunc func(string)) *BenchmarkInterconnectResult { - result := &BenchmarkInterconnectResult{ - Status: "UNSUPPORTED", - Attempted: true, - SelectedGPUIndices: append([]int(nil), gpuIndices...), - } - cmd := []string{ - "all_reduce_perf", - "-b", "512M", - "-e", "4G", - "-f", "2", - "-g", strconv.Itoa(len(gpuIndices)), - "--iters", strconv.Itoa(maxInt(20, spec.NCCLSec/10)), - } - env := []string{ - "CUDA_DEVICE_ORDER=PCI_BUS_ID", - "CUDA_VISIBLE_DEVICES=" + joinIndexList(gpuIndices), - } - logFunc(fmt.Sprintf("NCCL interconnect: gpus=%s", joinIndexList(gpuIndices))) - out, err := runSATCommandCtx(ctx, verboseLog, "nccl-all-reduce.log", cmd, env, logFunc) - _ = os.WriteFile(filepath.Join(runDir, "nccl-all-reduce.log"), out, 0644) - if err != nil { - result.Notes = append(result.Notes, strings.TrimSpace(string(out))) - return result - } - avgAlg, maxAlg, avgBus, maxBus := parseNCCLAllReduceOutput(string(out)) - result.Status = "OK" - result.Supported = true - result.AvgAlgBWGBps = avgAlg - result.MaxAlgBWGBps = maxAlg - result.AvgBusBWGBps = avgBus - result.MaxBusBWGBps = maxBus - return result -} - -func parseNCCLAllReduceOutput(raw string) (avgAlg, maxAlg, avgBus, maxBus float64) { - lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") - var algs []float64 - var buses []float64 - for _, line := range lines { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - fields := strings.Fields(line) - if len(fields) < 8 { - continue - } - for i := 0; i+2 < len(fields); i++ { - timeVal, err1 := strconv.ParseFloat(fields[i], 64) - algVal, err2 := strconv.ParseFloat(fields[i+1], 64) - busVal, err3 := strconv.ParseFloat(fields[i+2], 64) - if err1 == nil && err2 == nil && err3 == nil && timeVal > 0 { - algs = append(algs, algVal) - buses = append(buses, busVal) - break - } - } - } - if len(algs) == 0 { - return 0, 0, 0, 0 - } - return benchmarkMean(algs), benchmarkMax(algs), benchmarkMean(buses), benchmarkMax(buses) -} - -func queryThrottleCounters(gpuIndex int) (BenchmarkThrottleCounters, error) { - out, err := satExecCommand( - "nvidia-smi", - "--id="+strconv.Itoa(gpuIndex), - "--query-gpu=clocks_event_reasons_counters.sw_power_cap,clocks_event_reasons_counters.sw_thermal_slowdown,clocks_event_reasons_counters.sync_boost,clocks_event_reasons_counters.hw_thermal_slowdown,clocks_event_reasons_counters.hw_power_brake_slowdown", - "--format=csv,noheader,nounits", - ).Output() - if err != nil { - return BenchmarkThrottleCounters{}, err - } - fields := strings.Split(strings.TrimSpace(string(out)), ",") - if len(fields) < 5 { - return BenchmarkThrottleCounters{}, fmt.Errorf("unexpected throttle counter columns: %q", strings.TrimSpace(string(out))) - } - return BenchmarkThrottleCounters{ - SWPowerCapUS: parseBenchmarkUint64(fields[0]), - SWThermalSlowdownUS: parseBenchmarkUint64(fields[1]), - SyncBoostUS: parseBenchmarkUint64(fields[2]), - HWThermalSlowdownUS: parseBenchmarkUint64(fields[3]), - HWPowerBrakeSlowdownUS: parseBenchmarkUint64(fields[4]), - }, nil -} - -func diffThrottleCounters(before, after BenchmarkThrottleCounters) BenchmarkThrottleCounters { - return BenchmarkThrottleCounters{ - SWPowerCapUS: saturatingSub(after.SWPowerCapUS, before.SWPowerCapUS), - SWThermalSlowdownUS: saturatingSub(after.SWThermalSlowdownUS, before.SWThermalSlowdownUS), - SyncBoostUS: saturatingSub(after.SyncBoostUS, before.SyncBoostUS), - HWThermalSlowdownUS: saturatingSub(after.HWThermalSlowdownUS, before.HWThermalSlowdownUS), - HWPowerBrakeSlowdownUS: saturatingSub(after.HWPowerBrakeSlowdownUS, before.HWPowerBrakeSlowdownUS), - } -} - -func queryECCCounters(gpuIndex int) (BenchmarkECCCounters, error) { - out, err := satExecCommand( - "nvidia-smi", - "--id="+strconv.Itoa(gpuIndex), - "--query-gpu=ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total", - "--format=csv,noheader,nounits", - ).Output() - if err != nil { - return BenchmarkECCCounters{}, err - } - fields := strings.Split(strings.TrimSpace(string(out)), ",") - if len(fields) < 2 { - return BenchmarkECCCounters{}, fmt.Errorf("unexpected ECC counter columns: %q", strings.TrimSpace(string(out))) - } - corrected, err1 := strconv.ParseUint(strings.TrimSpace(fields[0]), 10, 64) - uncorrected, err2 := strconv.ParseUint(strings.TrimSpace(fields[1]), 10, 64) - if err1 != nil || err2 != nil { - // ECC may be disabled on this GPU — return zero counters silently. - return BenchmarkECCCounters{}, nil - } - return BenchmarkECCCounters{Corrected: corrected, Uncorrected: uncorrected}, nil -} - -func diffECCCounters(before, after BenchmarkECCCounters) BenchmarkECCCounters { - return BenchmarkECCCounters{ - Corrected: saturatingSub(after.Corrected, before.Corrected), - Uncorrected: saturatingSub(after.Uncorrected, before.Uncorrected), - } -} - -func queryActiveComputeApps(gpuIndices []int) ([]string, error) { - args := []string{ - "--query-compute-apps=gpu_uuid,pid,process_name", - "--format=csv,noheader,nounits", - } - if len(gpuIndices) > 0 { - args = append([]string{"--id=" + joinIndexList(gpuIndices)}, args...) - } - out, err := satExecCommand("nvidia-smi", args...).Output() - if err != nil { - return nil, err - } - var lines []string - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - line = strings.TrimSpace(line) - if line == "" { - continue - } - lines = append(lines, line) - } - return lines, nil -} - -func finalizeBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkGPUResult { - if gpu.Status == "" { - gpu.Status = "OK" - } - if gpu.Scores.CompositeScore == 0 { - gpu.Scores.CompositeScore = gpu.Scores.ComputeScore - } - return gpu -} - -func buildBenchmarkFindings(result NvidiaBenchmarkResult) []string { - var findings []string - - passed := 0 - for _, gpu := range result.GPUs { - if gpu.Status == "OK" { - passed++ - } - } - total := len(result.GPUs) - if total > 0 { - if passed == total { - findings = append(findings, fmt.Sprintf("All %d GPU(s) passed the benchmark.", total)) - } else { - findings = append(findings, fmt.Sprintf("%d of %d GPU(s) passed the benchmark.", passed, total)) - } - } - - if result.Normalization.Status != "full" { - findings = append(findings, "Environment normalization was partial; compare results with caution.") - } - for _, gpu := range result.GPUs { - if gpu.Status == "FAILED" && len(gpu.DegradationReasons) == 0 { - findings = append(findings, fmt.Sprintf("GPU %d failed the benchmark (check verbose.log for details).", gpu.Index)) - continue - } - if len(gpu.DegradationReasons) == 0 && gpu.Status == "OK" { - findings = append(findings, fmt.Sprintf("GPU %d held clocks without observable throttle counters during steady state.", gpu.Index)) - continue - } - for _, reason := range gpu.DegradationReasons { - switch reason { - case "power_capped": - findings = append(findings, fmt.Sprintf( - "[POWER] GPU %d: power cap throttle %.1f%% of steady state — server is not delivering full TDP to the GPU.", - gpu.Index, gpu.Scores.PowerCapThrottlePct)) - case "thermal_limited": - // Hard stop check: thermal throttle while fans are not at maximum. - // This means the server does not see GPU thermals — incompatible config. - if result.Cooling != nil && result.Cooling.FanDutyCycleAvailable && - result.Cooling.P95FanDutyCyclePct < 95 { - findings = append(findings, fmt.Sprintf( - "[HARD STOP] GPU %d: thermal throttle (%.1f%% of time) while fans peaked at only %.0f%% duty cycle — server cooling is not responding to GPU heat load. Configuration is likely incompatible.", - gpu.Index, gpu.Scores.ThermalThrottlePct, result.Cooling.P95FanDutyCyclePct)) - } else { - findings = append(findings, fmt.Sprintf( - "[THERMAL] GPU %d: thermal throttle %.1f%% of steady state.", - gpu.Index, gpu.Scores.ThermalThrottlePct)) - } - case "sync_boost_limited": - findings = append(findings, fmt.Sprintf( - "[SYNC] GPU %d: sync boost throttle %.1f%% of steady state — GPUs are constraining each other's clocks.", - gpu.Index, gpu.Scores.SyncBoostThrottlePct)) - case "low_sm_clock_vs_target": - findings = append(findings, fmt.Sprintf("GPU %d average SM clock stayed below the requested lock target.", gpu.Index)) - case "variance_too_high": - findings = append(findings, fmt.Sprintf("GPU %d showed unstable clocks/power over the benchmark window.", gpu.Index)) - case "normalization_partial": - findings = append(findings, fmt.Sprintf("GPU %d ran without full benchmark normalization.", gpu.Index)) - case "power_limit_derated": - findings = append(findings, fmt.Sprintf("[POWER] GPU %d could not sustain full TDP in this server; benchmark ran at reduced limit %.0f W.", gpu.Index, gpu.PowerLimitW)) - case "ecc_uncorrected_errors": - findings = append(findings, fmt.Sprintf( - "[HARD STOP] GPU %d: %d uncorrected ECC error(s) detected — possible hardware fault. Do not use in production.", - gpu.Index, gpu.ECC.Uncorrected)) - case "ecc_corrected_errors": - findings = append(findings, fmt.Sprintf( - "[WARNING] GPU %d: %d corrected ECC error(s) — possible DRAM degradation, monitor closely.", - gpu.Index, gpu.ECC.Corrected)) - } - } - // Temperature headroom checks — independent of throttle counters. - // Shutdown and slowdown thresholds are per-GPU from nvidia-smi -q; - // fall back to 90°C / 80°C when unavailable. - if gpu.Steady.P95TempC > 0 { - shutdownTemp := gpu.ShutdownTempC - if shutdownTemp <= 0 { - shutdownTemp = 90 - } - slowdownTemp := gpu.SlowdownTempC - if slowdownTemp <= 0 { - slowdownTemp = 80 - } - headroom := shutdownTemp - gpu.Steady.P95TempC - switch { - case headroom < 10: - findings = append(findings, fmt.Sprintf( - "[HARD STOP] GPU %d: p95 temperature %.1f°C — only %.1f°C from shutdown threshold (%.0f°C). Do not operate.", - gpu.Index, gpu.Steady.P95TempC, headroom, shutdownTemp)) - case gpu.Steady.P95TempC >= slowdownTemp: - findings = append(findings, fmt.Sprintf( - "[THERMAL] GPU %d: p95 temperature %.1f°C exceeds slowdown threshold (%.0f°C) — %.1f°C headroom to shutdown. Operating in degraded reliability zone.", - gpu.Index, gpu.Steady.P95TempC, slowdownTemp, headroom)) - } - } - if gpu.CoolingWarning != "" { - findings = append(findings, fmt.Sprintf( - "GPU %d: %s. Operator action: rerun the benchmark with fan speed manually fixed at 100%% to confirm actual thermal headroom.", - gpu.Index, gpu.CoolingWarning, - )) - } - if len(gpu.PrecisionFailures) > 0 { - findings = append(findings, fmt.Sprintf("GPU %d had incomplete precision coverage: %s.", gpu.Index, strings.Join(gpu.PrecisionFailures, ", "))) - } - if gpu.Backend == "driver-ptx" { - findings = append(findings, fmt.Sprintf("GPU %d used driver PTX fallback; tensor score is intentionally degraded.", gpu.Index)) - } - if gpu.DefaultPowerLimitW > 0 && gpu.PowerLimitW > 0 && gpu.PowerLimitW < gpu.DefaultPowerLimitW*0.95 { - findings = append(findings, fmt.Sprintf( - "GPU %d power limit %.0f W is below default %.0f W (%.0f%%). Performance may be artificially reduced.", - gpu.Index, gpu.PowerLimitW, gpu.DefaultPowerLimitW, gpu.PowerLimitW/gpu.DefaultPowerLimitW*100, - )) - } - // Flag significant TDP deviation (over or under) from calibration. - if gpu.CalibratedPeakPowerW > 0 { - ref := gpu.DefaultPowerLimitW - if ref <= 0 { - ref = gpu.PowerLimitW - } - if ref > 0 { - deviationPct := (gpu.CalibratedPeakPowerW - ref) / ref * 100 - switch { - case deviationPct < -10: - findings = append(findings, fmt.Sprintf( - "GPU %d reached only %.0f W (%.0f%% of rated %.0f W) under targeted_power. Check power delivery or cooling.", - gpu.Index, gpu.CalibratedPeakPowerW, gpu.CalibratedPeakPowerW/ref*100, ref, - )) - case deviationPct > 5: - findings = append(findings, fmt.Sprintf( - "GPU %d exceeded rated TDP: %.0f W measured vs %.0f W rated (+%.0f%%). Power limit may not be enforced correctly.", - gpu.Index, gpu.CalibratedPeakPowerW, ref, deviationPct, - )) - } - } - } - } - if result.Interconnect != nil && result.Interconnect.Supported { - findings = append(findings, fmt.Sprintf("Multi-GPU all_reduce max bus bandwidth: %.1f GB/s.", result.Interconnect.MaxBusBWGBps)) - } - if cl := result.CPULoad; cl != nil { - switch cl.Status { - case "high": - findings = append(findings, fmt.Sprintf( - "Host CPU load was elevated during the benchmark (avg %.1f%%, max %.1f%%). A competing CPU workload may skew GPU results.", - cl.AvgPct, cl.MaxPct, - )) - case "unstable": - findings = append(findings, fmt.Sprintf( - "Host CPU load was erratic during the benchmark (avg %.1f%%, p95 %.1f%%). Results may be less reproducible.", - cl.AvgPct, cl.P95Pct, - )) - } - } - if sp := result.ServerPower; sp != nil && sp.Available && sp.GPUReportedSumW > 0 { - dcmiPartial := detectDCMIPartialCoverage(sp) - if sp.ReportingRatio < 0.75 && !dcmiPartial { - findings = append(findings, fmt.Sprintf( - "GPU power reporting may be unreliable: server delta %.0f W vs GPU-reported %.0f W (ratio %.2f). GPU telemetry likely over-reports actual consumption. Composite scores have been penalized accordingly.", - sp.DeltaW, sp.GPUReportedSumW, sp.ReportingRatio, - )) - } else if sp.ReportingRatio < 0.75 && dcmiPartial { - findings = append(findings, fmt.Sprintf( - "IPMI DCMI covers partial PSU set (DCMI/SDR coverage %.0f%%): ratio %.2f reflects DCMI under-reporting, not GPU inaccuracy. GPU telemetry is the reliable power source; use SDR-based ratio for server-side accuracy.", - sp.DCMICoverageRatio*100, sp.ReportingRatio, - )) - } else if sp.ReportingRatio > 1.25 { - findings = append(findings, fmt.Sprintf( - "Server power delta %.0f W exceeds GPU-reported sum %.0f W by %.0f%%. Other components (CPU, NVMe, networking) may be drawing substantial power under GPU load.", - sp.DeltaW, sp.GPUReportedSumW, (sp.ReportingRatio-1)*100, - )) - } - } - return dedupeStrings(findings) -} - -func benchmarkOverallStatus(result NvidiaBenchmarkResult) string { - if len(result.GPUs) == 0 { - return "FAILED" - } - hasOK := false - hasPartial := result.Normalization.Status != "full" - for _, gpu := range result.GPUs { - switch gpu.Status { - case "OK": - hasOK = true - case "PARTIAL", "UNSUPPORTED": - hasPartial = true - } - } - if !hasOK { - return "FAILED" - } - if hasPartial { - return "PARTIAL" - } - return "OK" -} - -func findBenchmarkNormalization(items []BenchmarkNormalizationGPU, idx int) *BenchmarkNormalizationGPU { - for i := range items { - if items[i].Index == idx { - return &items[i] - } - } - return nil -} - -func classifySATErrorStatus(out []byte, err error) string { - status, _ := classifySATResult("benchmark", out, err) - if status == "UNSUPPORTED" { - return "UNSUPPORTED" - } - return "FAILED" -} - -func parseBenchmarkFloat(raw string) float64 { - raw = strings.TrimSpace(raw) - if raw == "" || strings.EqualFold(raw, "n/a") || strings.EqualFold(raw, "[not supported]") { - return 0 - } - value, _ := strconv.ParseFloat(raw, 64) - return value -} - -func parseBenchmarkUint64(raw string) uint64 { - raw = strings.TrimSpace(raw) - if raw == "" || strings.EqualFold(raw, "n/a") || strings.EqualFold(raw, "[not supported]") { - return 0 - } - value, _ := strconv.ParseUint(raw, 10, 64) - return value -} - -func benchmarkMean(values []float64) float64 { - if len(values) == 0 { - return 0 - } - var sum float64 - for _, value := range values { - sum += value - } - return sum / float64(len(values)) -} - -func benchmarkPercentile(values []float64, p float64) float64 { - if len(values) == 0 { - return 0 - } - copyValues := append([]float64(nil), values...) - sort.Float64s(copyValues) - if len(copyValues) == 1 { - return copyValues[0] - } - rank := (p / 100.0) * float64(len(copyValues)-1) - lower := int(math.Floor(rank)) - upper := int(math.Ceil(rank)) - if lower == upper { - return copyValues[lower] - } - frac := rank - float64(lower) - return copyValues[lower] + (copyValues[upper]-copyValues[lower])*frac -} - -func benchmarkCV(values []float64) float64 { - if len(values) == 0 { - return 0 - } - mean := benchmarkMean(values) - if mean == 0 { - return 0 - } - var variance float64 - for _, value := range values { - diff := value - mean - variance += diff * diff - } - variance /= float64(len(values)) - return math.Sqrt(variance) / mean * 100 -} - -func benchmarkClockDrift(values []float64) float64 { - if len(values) < 4 { - return 0 - } - window := len(values) / 4 - if window < 1 { - window = 1 - } - head := benchmarkMean(values[:window]) - tail := benchmarkMean(values[len(values)-window:]) - if head <= 0 || tail >= head { - return 0 - } - return ((head - tail) / head) * 100 -} - -func benchmarkMax(values []float64) float64 { - var max float64 - for i, value := range values { - if i == 0 || value > max { - max = value - } - } - return max -} - -func clampScore(value float64) float64 { - switch { - case value < 0: - return 0 - case value > 100: - return 100 - default: - return value - } -} - -func dedupeStrings(values []string) []string { - if len(values) == 0 { - return nil - } - seen := make(map[string]struct{}, len(values)) - out := make([]string, 0, len(values)) - for _, value := range values { - value = strings.TrimSpace(value) - if value == "" { - continue - } - if _, ok := seen[value]; ok { - continue - } - seen[value] = struct{}{} - out = append(out, value) - } - return out -} - -func saturatingSub(after, before uint64) uint64 { - if after <= before { - return 0 - } - return after - before -} - -func maxInt(a, b int) int { - if a > b { - return a - } - return b -} - -// detectDCMIPartialCoverage returns true when IPMI DCMI under-reports actual -// server power by comparing DCMI readings against SDR PSUx_POWER_IN sensor sums. -// -// Primary check: DCMI_idle / SDR_PSU_IN_idle — most reliable because GPU load -// is zero, so both sources measure the same server state. A ratio below 0.7 -// means DCMI misses ≥30% of installed PSUs (e.g. 0.50 = sees 2 of 4 PSUs). -// -// Fallback: DCMI_loaded / SDR_PSU_IN_loaded — less precise (GPU load may -// affect different PSUs differently) but still useful when idle SDR is absent. -// -// Returns false when SDR data is unavailable (server has no PSUx_POWER_IN -// sensors); the heuristic detectIPMISaturationFallback is used in that case. -func detectDCMIPartialCoverage(sp *BenchmarkServerPower) bool { - if sp == nil || !sp.Available { - return false - } - if sp.PSUInputIdleW > 0 && sp.IdleW > 0 { - return sp.IdleW/sp.PSUInputIdleW < 0.7 - } - if sp.PSUInputLoadedW > 0 && sp.LoadedW > 0 { - return sp.LoadedW/sp.PSUInputLoadedW < 0.7 - } - return false -} - -// detectIPMISaturationFallback is the heuristic used when SDR PSU sensors are -// absent. It analyses the power ramp: if 2+ of the last 3 per-step incremental -// DCMI/GPU ratios fall below 25% of the first-step ratio, DCMI has likely -// plateaued while GPU load continued to grow (saturation proxy). -// Prefer detectDCMIPartialCoverage when SDR data is available. -func detectIPMISaturationFallback(steps []NvidiaPowerBenchStep) bool { - type pt struct{ incIPMI, incGPU float64 } - var pts []pt - for i := 1; i < len(steps); i++ { - if steps[i].ServerDeltaW <= 0 || steps[i-1].ServerDeltaW <= 0 { - continue - } - incIPMI := steps[i].ServerDeltaW - steps[i-1].ServerDeltaW - incGPU := steps[i].TotalObservedPowerW - steps[i-1].TotalObservedPowerW - if incGPU <= 0 { - continue - } - pts = append(pts, pt{incIPMI, incGPU}) - } - if len(pts) < 3 { - return false - } - refRatio := pts[0].incIPMI / pts[0].incGPU - if refRatio <= 0 { - return false - } - saturated := 0 - for _, p := range pts[len(pts)-3:] { - if p.incIPMI/p.incGPU < refRatio*0.25 { - saturated++ - } - } - return saturated >= 2 -} - -// psuStatusSnapshot samples PSU health sensor states via -// `ipmitool sdr type "Power Supply"`. Returns a map of sensor name → reading -// string (e.g. "Presence detected", "Failure detected"). Returns nil when IPMI -// is unavailable or no Power Supply entity sensors are present. -func psuStatusSnapshot() map[string]string { - out, err := exec.Command("ipmitool", "sdr", "type", "Power Supply").Output() - if err != nil || len(out) == 0 { - return nil - } - result := make(map[string]string) - for _, line := range strings.Split(string(out), "\n") { - parts := strings.Split(line, "|") - if len(parts) < 5 { - continue - } - name := strings.TrimSpace(parts[0]) - reading := strings.TrimSpace(parts[4]) - if name == "" { - continue - } - result[name] = reading - } - return result -} - -// diffPSUStatus compares PSU sensor snapshots taken before and after a test. -// Returns human-readable fault strings for sensors that entered a fault state -// during the test. Pre-existing faults (present in both snapshots) are excluded -// so that only new anomalies caused by the test are reported. -func diffPSUStatus(before, after map[string]string) []string { - if len(after) == 0 { - return nil - } - isFault := func(s string) bool { - lower := strings.ToLower(s) - return strings.Contains(lower, "failure") || - strings.Contains(lower, "fault") || - strings.Contains(lower, "warning") || - strings.Contains(lower, "predictive") || - strings.Contains(lower, "absent") || - strings.Contains(lower, "ac lost") - } - var issues []string - for name, afterReading := range after { - if !isFault(afterReading) { - continue - } - if beforeReading, had := before[name]; had && isFault(beforeReading) { - continue // pre-existing fault, not caused by this test - } - if prev, had := before[name]; had { - issues = append(issues, fmt.Sprintf("%s: changed from %q to %q during test", name, prev, afterReading)) - } else { - issues = append(issues, fmt.Sprintf("%s: %s (appeared after test start)", name, afterReading)) - } - } - sort.Strings(issues) - return issues -} - -// sdrPowerSnapshot holds per-source power sums from a single `ipmitool sdr` read. -type sdrPowerSnapshot struct { - PSUInW float64 // sum of PSU AC input across all slots - PSUOutW float64 // sum of PSU DC output across all slots - GPUSlotW float64 // sum of GPU slot/GPU power sensors - - // Per-slot PSU data from collector.PSUSlotsFromSDR — same slot keys as - // audit HardwarePowerSupply.Slot (0-based strings). - PSUSlots map[string]BenchmarkPSUSlotPower - - SkippedSensors []string // sensors rejected during self-healing -} - -type benchmarkSDRSeriesSummary struct { - PSUInW float64 - PSUOutW float64 - GPUSlotW float64 - PSUSlots map[string]BenchmarkPSUSlotPower - Samples int - - SkippedSensors []string -} - -// sdrSensor is a name+watts pair used for GPU slot self-healing filtering. -type sdrSensor struct { - name string - watts float64 -} - -// filterSensorGroup removes physically implausible readings from a group. -// Hard bounds: 0 < watts ≤ maxPerSensorW. Within groups of 2+ sensors, -// values more than 5× the group median are rejected as stuck/fault sensors. -func filterSensorGroup(sensors []sdrSensor, maxPerSensorW float64) (valid []sdrSensor, skipped []string) { - var inBounds []sdrSensor - for _, s := range sensors { - if s.watts <= 0 || s.watts > maxPerSensorW { - skipped = append(skipped, fmt.Sprintf("%s (%.0f W: out of range 0–%.0f W)", s.name, s.watts, maxPerSensorW)) - } else { - inBounds = append(inBounds, s) - } - } - if len(inBounds) < 2 { - return inBounds, skipped - } - vals := make([]float64, len(inBounds)) - for i, s := range inBounds { - vals[i] = s.watts - } - sort.Float64s(vals) - mid := len(vals) / 2 - var median float64 - if len(vals)%2 == 0 { - median = (vals[mid-1] + vals[mid]) / 2 - } else { - median = vals[mid] - } - for _, s := range inBounds { - if median > 0 && s.watts > median*5 { - skipped = append(skipped, fmt.Sprintf("%s (%.0f W: >5× median %.0f W, likely sensor fault)", s.name, s.watts, median)) - } else { - valid = append(valid, s) - } - } - return valid, skipped -} - -// sampleIPMISDRPowerSensors reads power sensors from `ipmitool sdr` in a single -// invocation and returns self-healed grouped sums. -// -// PSU identification delegates to collector.PSUSlotsFromSDR which uses the same -// slot-detection regexes as the hardware audit (PSU1_POWER_IN, PSU1_PIN, PS1 POut, -// Power1…). Self-healing: bounds checking + 5× median outlier rejection. -// -// GPU slot sensors (GPU_POWER_SLOTx, GPU1 Power, …) are classified separately -// since the audit collector does not track GPU PCIe slot power. -func sampleIPMISDRPowerSensors() sdrPowerSnapshot { - raw, err := exec.Command("ipmitool", "sdr").Output() - if err != nil || len(raw) == 0 { - return sdrPowerSnapshot{} - } - sdrStr := string(raw) - var snap sdrPowerSnapshot - - // ── PSU data via audit collector ───────────────────────────────────────── - // collector.PSUSlotsFromSDR handles all vendor naming variants and applies - // bounds checking inside parseBoundedFloat (0 < w ≤ 6000 W). - collectorSlots := collector.PSUSlotsFromSDR(sdrStr) - - // Convert to benchmark type and apply cross-slot median filtering. - var psuInSensors, psuOutSensors []sdrSensor - for slotKey, sp := range collectorSlots { - bsp := BenchmarkPSUSlotPower{Status: sp.Status} - if sp.InputW != nil { - bsp.InputW = sp.InputW - psuInSensors = append(psuInSensors, sdrSensor{name: "PSU-slot-" + slotKey, watts: *sp.InputW}) - } - if sp.OutputW != nil { - bsp.OutputW = sp.OutputW - psuOutSensors = append(psuOutSensors, sdrSensor{name: "PSU-slot-" + slotKey + "-out", watts: *sp.OutputW}) - } - if snap.PSUSlots == nil { - snap.PSUSlots = make(map[string]BenchmarkPSUSlotPower) - } - snap.PSUSlots[slotKey] = bsp - } - - // Apply cross-slot outlier filter and sum. - validIn, skIn := filterSensorGroup(psuInSensors, 6000) - for _, s := range validIn { - snap.PSUInW += s.watts - } - snap.SkippedSensors = append(snap.SkippedSensors, skIn...) - - validOut, skOut := filterSensorGroup(psuOutSensors, 6000) - for _, s := range validOut { - snap.PSUOutW += s.watts - } - snap.SkippedSensors = append(snap.SkippedSensors, skOut...) - - // ── GPU slot sensors ───────────────────────────────────────────────────── - // collector does not track GPU PCIe slot power; classify here. - // Matches: GPU_POWER_SLOTx (MSI), GPU1 Power (xFusion), GPU_PWR_x (generic). - var gpuSensors []sdrSensor - for _, line := range strings.Split(sdrStr, "\n") { - parts := strings.Split(line, "|") - if len(parts) < 2 { - continue - } - name := strings.TrimSpace(parts[0]) - nameLower := strings.ToLower(name) - if !strings.Contains(nameLower, "gpu") { - continue - } - if !strings.Contains(nameLower, "slot") && !strings.Contains(nameLower, "power") && - !strings.Contains(nameLower, "pwr") { - continue - } - var w float64 - if n, _ := fmt.Sscanf(strings.TrimSpace(parts[1]), "%f Watts", &w); n != 1 { - continue - } - gpuSensors = append(gpuSensors, sdrSensor{name: name, watts: w}) - } - validGPU, skGPU := filterSensorGroup(gpuSensors, 2000) - for _, s := range validGPU { - snap.GPUSlotW += s.watts - } - snap.SkippedSensors = append(snap.SkippedSensors, skGPU...) - - return snap -} - -func startIPMISDRSampler(stopCh <-chan struct{}, intervalSec int) <-chan []sdrPowerSnapshot { - if intervalSec <= 0 { - intervalSec = benchmarkPowerAutotuneSampleInterval - } - ch := make(chan []sdrPowerSnapshot, 1) - go func() { - defer close(ch) - var samples []sdrPowerSnapshot - record := func() { - snap := sampleIPMISDRPowerSensors() - if snap.PSUInW <= 0 && snap.PSUOutW <= 0 && snap.GPUSlotW <= 0 && len(snap.PSUSlots) == 0 { - return - } - samples = append(samples, snap) - } - record() - ticker := time.NewTicker(time.Duration(intervalSec) * time.Second) - defer ticker.Stop() - for { - select { - case <-stopCh: - ch <- samples - return - case <-ticker.C: - record() - } - } - }() - return ch -} - -func summarizeSDRPowerSeries(samples []sdrPowerSnapshot) benchmarkSDRSeriesSummary { - var summary benchmarkSDRSeriesSummary - if len(samples) == 0 { - return summary - } - - type slotAggregate struct { - inputs []float64 - outputs []float64 - status string - } - - slotAgg := make(map[string]*slotAggregate) - skippedSet := make(map[string]struct{}) - var inputTotals []float64 - var outputTotals []float64 - var gpuSlotTotals []float64 - - for _, sample := range samples { - if sample.PSUInW > 0 { - inputTotals = append(inputTotals, sample.PSUInW) - } - if sample.PSUOutW > 0 { - outputTotals = append(outputTotals, sample.PSUOutW) - } - if sample.GPUSlotW > 0 { - gpuSlotTotals = append(gpuSlotTotals, sample.GPUSlotW) - } - for _, skipped := range sample.SkippedSensors { - if skipped != "" { - skippedSet[skipped] = struct{}{} - } - } - for slot, reading := range sample.PSUSlots { - agg := slotAgg[slot] - if agg == nil { - agg = &slotAggregate{} - slotAgg[slot] = agg - } - if reading.InputW != nil && *reading.InputW > 0 { - agg.inputs = append(agg.inputs, *reading.InputW) - } - if reading.OutputW != nil && *reading.OutputW > 0 { - agg.outputs = append(agg.outputs, *reading.OutputW) - } - switch { - case reading.Status == "": - case agg.status == "": - agg.status = reading.Status - case agg.status == "OK" && reading.Status != "OK": - agg.status = reading.Status - } - } - } - - summary.PSUInW = benchmarkMean(inputTotals) - summary.PSUOutW = benchmarkMean(outputTotals) - summary.GPUSlotW = benchmarkMean(gpuSlotTotals) - summary.Samples = len(samples) - - if len(slotAgg) > 0 { - summary.PSUSlots = make(map[string]BenchmarkPSUSlotPower, len(slotAgg)) - for slot, agg := range slotAgg { - reading := BenchmarkPSUSlotPower{Status: agg.status} - if mean := benchmarkMean(agg.inputs); mean > 0 { - v := mean - reading.InputW = &v - } - if mean := benchmarkMean(agg.outputs); mean > 0 { - v := mean - reading.OutputW = &v - } - summary.PSUSlots[slot] = reading - } - } - if len(skippedSet) > 0 { - summary.SkippedSensors = make([]string, 0, len(skippedSet)) - for skipped := range skippedSet { - summary.SkippedSensors = append(summary.SkippedSensors, skipped) - } - sort.Strings(summary.SkippedSensors) - } - - return summary -} - -func collectIPMISDRPowerSeries(ctx context.Context, durationSec, intervalSec int) benchmarkSDRSeriesSummary { - if durationSec <= 0 { - return benchmarkSDRSeriesSummary{} - } - stopCh := make(chan struct{}) - doneCh := startIPMISDRSampler(stopCh, intervalSec) - select { - case <-ctx.Done(): - case <-time.After(time.Duration(durationSec) * time.Second): - } - close(stopCh) - return summarizeSDRPowerSeries(<-doneCh) -} - -// queryIPMIServerPowerW reads the current server power draw via ipmitool dcmi. -// Returns 0 and an error if IPMI is unavailable or the output cannot be parsed. -func queryIPMIServerPowerW() (float64, error) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cmd := exec.CommandContext(ctx, "ipmitool", "dcmi", "power", "reading") - out, err := cmd.Output() - if err != nil { - return 0, fmt.Errorf("ipmitool dcmi power reading: %w", err) - } - if w := parseDCMIPowerReading(string(out)); w > 0 { - return w, nil - } - return 0, fmt.Errorf("could not parse ipmitool dcmi power reading output") -} - -// sampleIPMIPowerSeries collects IPMI power readings every 2 seconds for -// durationSec seconds. Returns the mean of all successful samples. -// Returns 0, false if IPMI is unavailable. -func sampleIPMIPowerSeries(ctx context.Context, durationSec int) (meanW float64, ok bool) { - if durationSec <= 0 { - return 0, false - } - deadline := time.Now().Add(time.Duration(durationSec) * time.Second) - var samples []float64 -loop: - for { - if w, err := queryIPMIServerPowerW(); err == nil { - samples = append(samples, w) - } - if time.Now().After(deadline) { - break - } - select { - case <-ctx.Done(): - break loop - case <-time.After(2 * time.Second): - } - } - if len(samples) == 0 { - return 0, false - } - var sum float64 - for _, w := range samples { - sum += w - } - return sum / float64(len(samples)), true -} - -// characterizeServerPower computes BenchmarkServerPower from idle and loaded -// samples plus the GPU-reported average power during steady state. -func characterizeServerPower(idleW, loadedW, gpuReportedSumW float64, source string, available bool) *BenchmarkServerPower { - sp := &BenchmarkServerPower{ - Available: available, - Source: normalizeBenchmarkPowerSource(source), - SampleIntervalSec: benchmarkPowerAutotuneSampleInterval, - } - if !available { - sp.Notes = append(sp.Notes, "IPMI power reading unavailable; server-side power characterization skipped") - return sp - } - sp.IdleW = idleW - sp.LoadedW = loadedW - sp.DeltaW = loadedW - idleW - sp.GPUReportedSumW = gpuReportedSumW - if gpuReportedSumW > 0 && sp.DeltaW > 0 { - sp.ReportingRatio = sp.DeltaW / gpuReportedSumW - } - return sp -} - -// readServerModel returns the DMI system product name (e.g. "SuperMicro SYS-421GE-TNRT"). -// Returns empty string if unavailable (non-Linux or missing DMI entry). -func readServerModel() string { - data, err := os.ReadFile("/sys/class/dmi/id/product_name") - if err != nil { - return "" - } - return strings.TrimSpace(string(data)) -} - -// filterRowsByGPU returns only the metric rows for a specific GPU index. -func filterRowsByGPU(rows []GPUMetricRow, gpuIndex int) []GPUMetricRow { - var out []GPUMetricRow - for _, r := range rows { - if r.GPUIndex == gpuIndex { - out = append(out, r) - } - } - return out -} - -// parseBenchmarkBurnLogByGPU splits a multi-GPU bee-gpu-burn output by [gpu N] prefix -// and returns a per-GPU parse result map. -func parseBenchmarkBurnLogByGPU(raw string) map[int]benchmarkBurnParseResult { - gpuLines := make(map[int][]string) - for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") { - line = strings.TrimSpace(line) - if !strings.HasPrefix(line, "[gpu ") { - continue - } - end := strings.Index(line, "] ") - if end < 0 { - continue - } - gpuIdx, err := strconv.Atoi(strings.TrimSpace(line[5:end])) - if err != nil { - continue - } - gpuLines[gpuIdx] = append(gpuLines[gpuIdx], line[end+2:]) - } - results := make(map[int]benchmarkBurnParseResult, len(gpuLines)) - for gpuIdx, lines := range gpuLines { - // Lines are already stripped of the [gpu N] prefix; parseBenchmarkBurnLog - // calls stripBenchmarkPrefix which is a no-op on already-stripped lines. - results[gpuIdx] = parseBenchmarkBurnLog(strings.Join(lines, "\n")) - } - return results -} - -// runNvidiaBenchmarkParallel runs warmup and steady compute on all selected GPUs -// simultaneously using a single bee-gpu-burn invocation per phase. -func runNvidiaBenchmarkParallel( - ctx context.Context, - verboseLog, runDir string, - selected []int, - infoByIndex map[int]benchmarkGPUInfo, - opts NvidiaBenchmarkOptions, - spec benchmarkProfileSpec, - logFunc func(string), - result *NvidiaBenchmarkResult, - calibByIndex map[int]benchmarkPowerCalibrationResult, - serverIdleW *float64, serverLoadedWSum *float64, - serverIdleOK *bool, serverLoadedOK *bool, serverLoadedSamples *int, - allMetricRows *[]GPUMetricRow, - metricTimelineSec *float64, - gpuBurnLog string, -) { - allDevices := joinIndexList(selected) - - // Build per-GPU result stubs. - gpuResults := make(map[int]*BenchmarkGPUResult, len(selected)) - for _, idx := range selected { - r := &BenchmarkGPUResult{Index: idx, Status: "FAILED"} - if info, ok := infoByIndex[idx]; ok { - r.UUID = info.UUID - r.Name = info.Name - r.BusID = info.BusID - r.VBIOS = info.VBIOS - r.PowerLimitW = info.PowerLimitW - r.MultiprocessorCount = info.MultiprocessorCount - r.DefaultPowerLimitW = info.DefaultPowerLimitW - r.ShutdownTempC = info.ShutdownTempC - r.SlowdownTempC = info.SlowdownTempC - r.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz - r.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz - r.MaxMemoryClockMHz = info.MaxMemoryClockMHz - } - if calib, ok := calibByIndex[idx]; ok { - r.CalibratedPeakPowerW = calib.Summary.P95PowerW - r.CalibratedPeakTempC = calib.Summary.P95TempC - r.PowerCalibrationTries = calib.Attempts - r.PowerLimitDerated = calib.Derated - r.Notes = append(r.Notes, calib.Notes...) - if calib.CoolingWarning != "" { - r.CoolingWarning = calib.CoolingWarning - } - } - if norm := findBenchmarkNormalization(result.Normalization.GPUs, idx); norm != nil { - r.LockedGraphicsClockMHz = norm.GPUClockLockMHz - r.LockedMemoryClockMHz = norm.MemoryClockLockMHz - } - gpuResults[idx] = r - } - - // Baseline: sample all GPUs together. - baselineRows, err := collectBenchmarkSamples(ctx, spec.BaselineSec, selected) - if err != nil && err != context.Canceled { - for _, idx := range selected { - gpuResults[idx].Notes = append(gpuResults[idx].Notes, "baseline sampling failed: "+err.Error()) - } - } - for _, idx := range selected { - perGPU := filterRowsByGPU(baselineRows, idx) - gpuResults[idx].Baseline = summarizeBenchmarkTelemetry(perGPU) - } - appendBenchmarkMetrics(allMetricRows, baselineRows, "baseline", metricTimelineSec, float64(spec.BaselineSec)) - - // Sample server idle power once. - if !*serverIdleOK { - if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, maxInt(spec.BaselineSec, 10), benchmarkPowerAutotuneSampleInterval); ok { - *serverIdleW = w - *serverIdleOK = true - logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w)) - } - } - - // Warmup: all GPUs simultaneously. - warmupCmd := []string{ - "bee-gpu-burn", - "--seconds", strconv.Itoa(spec.WarmupSec), - "--size-mb", strconv.Itoa(opts.SizeMB), - "--devices", allDevices, - } - logFunc(fmt.Sprintf("GPUs %s: parallel warmup (%ds)", allDevices, spec.WarmupSec)) - warmupOut, warmupRows, warmupErr := runBenchmarkCommandWithMetrics(ctx, verboseLog, "gpu-all-warmup.log", warmupCmd, nil, selected, logFunc) - appendBenchmarkMetrics(allMetricRows, warmupRows, "warmup", metricTimelineSec, float64(spec.WarmupSec)) - appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", "warmup", warmupOut) - if warmupErr != nil { - for _, idx := range selected { - gpuResults[idx].Notes = append(gpuResults[idx].Notes, "parallel warmup failed: "+warmupErr.Error()) - } - } - warmupParseByGPU := parseBenchmarkBurnLogByGPU(string(warmupOut)) - supportedPrecisions := append([]string(nil), benchmarkPrecisionPhases...) - for _, idx := range selected { - if pr, ok := warmupParseByGPU[idx]; ok && pr.ComputeCapability != "" { - if gpuResults[idx].ComputeCapability == "" { - gpuResults[idx].ComputeCapability = pr.ComputeCapability - } - if ccPrecisions := benchmarkSupportedPrecisions(pr.ComputeCapability); len(ccPrecisions) < len(supportedPrecisions) { - supportedPrecisions = ccPrecisions - } - } - } - - // Run synthetic precision phases and the combined steady phase as one - // uninterrupted command so the GPUs stay hot between windows. - eccBase := make(map[int]BenchmarkECCCounters, len(selected)) - for _, idx := range selected { - eccBase[idx], _ = queryECCCounters(idx) - } - planLabels, planPhases, basePhaseSec, mixedPhaseSec := buildBenchmarkSteadyPlan(spec, supportedPrecisions, func(label string) string { - if label == "mixed" { - return "steady" - } - return "gpu-all-steady-" + label - }) - planCmd := []string{ - "bee-gpu-burn", - "--seconds", strconv.Itoa(basePhaseSec), - "--size-mb", strconv.Itoa(opts.SizeMB), - "--devices", allDevices, - "--precision-plan", strings.Join(planLabels, ","), - "--precision-plan-seconds", benchmarkPlanDurationsCSV(planPhases), - } - logFunc(fmt.Sprintf("GPUs %s: uninterrupted precision plan (%d precision phases x %ds, mixed %ds)", allDevices, len(supportedPrecisions), basePhaseSec, mixedPhaseSec)) - serverPowerStopCh := make(chan struct{}) - serverPowerCh := startSelectedPowerSourceSampler(serverPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval) - _, phaseRowsByStage, phaseLogs, planErr := runBenchmarkPlannedCommandWithMetrics(ctx, verboseLog, "gpu-all-precision-plan.log", planCmd, nil, selected, planPhases, logFunc) - close(serverPowerStopCh) - if serverPowerSamples := <-serverPowerCh; len(serverPowerSamples) > 0 { - *serverLoadedWSum += benchmarkMean(serverPowerSamples) - (*serverLoadedSamples)++ - *serverLoadedOK = true - logFunc(fmt.Sprintf("GPUs %s: server loaded power (%s avg): %.0f W", allDevices, opts.ServerPowerSource, benchmarkMean(serverPowerSamples))) - } - for _, phaseSpec := range planPhases { - if rows := phaseRowsByStage[phaseSpec.MetricStage]; len(rows) > 0 { - appendBenchmarkMetrics(allMetricRows, rows, phaseSpec.MetricStage, metricTimelineSec, float64(phaseSpec.DurationSec)) - } - appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", phaseSpec.MetricStage, phaseLogs[phaseSpec.PlanLabel]) - } - for _, prec := range supportedPrecisions { - phaseLogName := "gpu-all-steady-" + prec - phaseRows := phaseRowsByStage[phaseLogName] - parseByGPU := parseBenchmarkBurnLogByGPU(string(phaseLogs[prec])) - for _, idx := range selected { - perGPU := filterRowsByGPU(phaseRows, idx) - phase := BenchmarkPrecisionSteadyPhase{ - Precision: prec, - Status: "OK", - Steady: summarizeBenchmarkTelemetry(perGPU), - } - if status, note := benchmarkPlannedPhaseStatus(phaseLogs[prec]); status != "OK" { - phase.Status = status - phase.Notes = note - gpuResults[idx].PrecisionFailures = append(gpuResults[idx].PrecisionFailures, prec+":"+status) - } - if pr, ok := parseByGPU[idx]; ok { - for _, p := range pr.Profiles { - if p.Supported { - phase.TeraOpsPerSec += p.TeraOpsPerSec - phase.WeightedTeraOpsPerSec += p.WeightedTeraOpsPerSec - } - } - } - gpuResults[idx].PrecisionSteady = append(gpuResults[idx].PrecisionSteady, phase) - } - } - - // Snapshot throttle counters before steady. - beforeThrottle := make(map[int]BenchmarkThrottleCounters, len(selected)) - for _, idx := range selected { - beforeThrottle[idx], _ = queryThrottleCounters(idx) - } - - logFunc(fmt.Sprintf("GPUs %s: parallel steady compute (combined, %ds)", allDevices, mixedPhaseSec)) - afterThrottle := make(map[int]BenchmarkThrottleCounters, len(selected)) - for _, idx := range selected { - afterThrottle[idx], _ = queryThrottleCounters(idx) - } - - steadyRows := phaseRowsByStage["steady"] - parseResults := parseBenchmarkBurnLogByGPU(string(phaseLogs["mixed"])) - - for _, idx := range selected { - perGPU := filterRowsByGPU(steadyRows, idx) - gpuResults[idx].Steady = summarizeBenchmarkTelemetry(perGPU) - gpuResults[idx].Throttle = diffThrottleCounters(beforeThrottle[idx], afterThrottle[idx]) - if eccFinal, err := queryECCCounters(idx); err == nil { - gpuResults[idx].ECC = diffECCCounters(eccBase[idx], eccFinal) - } - - if pr, ok := parseResults[idx]; ok { - gpuResults[idx].ComputeCapability = pr.ComputeCapability - gpuResults[idx].Backend = pr.Backend - gpuResults[idx].PrecisionResults = pr.Profiles - if pr.Fallback { - gpuResults[idx].Notes = append(gpuResults[idx].Notes, "benchmark used driver PTX fallback; tensor throughput score is not comparable") - } - } - if planErr != nil { - gpuResults[idx].Notes = append(gpuResults[idx].Notes, "precision plan failed: "+planErr.Error()) - } - } - - // Cooldown: all GPUs together. - if spec.CooldownSec > 0 { - cooldownRows, err := collectBenchmarkSamples(ctx, spec.CooldownSec, selected) - if err != nil && err != context.Canceled { - for _, idx := range selected { - gpuResults[idx].Notes = append(gpuResults[idx].Notes, "cooldown sampling failed: "+err.Error()) - } - } - for _, idx := range selected { - perGPU := filterRowsByGPU(cooldownRows, idx) - gpuResults[idx].Cooldown = summarizeBenchmarkTelemetry(perGPU) - } - appendBenchmarkMetrics(allMetricRows, cooldownRows, "cooldown", metricTimelineSec, float64(spec.CooldownSec)) - } - - // Score and finalize each GPU. - for _, idx := range selected { - r := gpuResults[idx] - applyBenchmarkSteadyFallback(r) - r.Scores = scoreBenchmarkGPUResult(*r) - r.DegradationReasons = detectBenchmarkDegradationReasons(*r, result.Normalization.Status) - pr := parseResults[idx] - switch { - case planErr != nil: - r.Status = classifySATErrorStatus(phaseLogs["mixed"], planErr) - case len(r.PrecisionFailures) > 0: - r.Status = "PARTIAL" - case pr.Fallback: - r.Status = "PARTIAL" - default: - r.Status = "OK" - } - result.GPUs = append(result.GPUs, finalizeBenchmarkGPUResult(*r)) - } -} - -// readBenchmarkHostConfig reads static CPU and memory configuration from -// /proc/cpuinfo and /proc/meminfo. Returns nil if neither source is readable. -func readBenchmarkHostConfig() *BenchmarkHostConfig { - cfg := &BenchmarkHostConfig{} - populated := false - - // Parse /proc/cpuinfo for CPU model, sockets, cores, threads. - if data, err := os.ReadFile("/proc/cpuinfo"); err == nil { - socketIDs := map[string]struct{}{} - coresPerSocket := map[string]int{} - var modelName string - threads := 0 - for _, line := range strings.Split(string(data), "\n") { - kv := strings.SplitN(line, ":", 2) - if len(kv) != 2 { - continue - } - key := strings.TrimSpace(kv[0]) - val := strings.TrimSpace(kv[1]) - switch key { - case "processor": - threads++ - case "model name": - if modelName == "" { - modelName = val - } - case "physical id": - socketIDs[val] = struct{}{} - case "cpu cores": - // Overwrite per-socket core count (last wins per socket, but all - // entries for the same socket report the same value). - if physLine := ""; physLine == "" { - // We accumulate below by treating cpu cores as a per-thread - // field; sum by socket requires a two-pass approach. Use the - // simpler approximation: totalCores = threads / (threads per core). - _ = val - } - } - } - // Second pass: per-socket core count. - var curSocket string - for _, line := range strings.Split(string(data), "\n") { - kv := strings.SplitN(line, ":", 2) - if len(kv) != 2 { - continue - } - key := strings.TrimSpace(kv[0]) - val := strings.TrimSpace(kv[1]) - switch key { - case "physical id": - curSocket = val - case "cpu cores": - if curSocket != "" { - if _, seen := coresPerSocket[curSocket]; !seen { - v, _ := strconv.Atoi(val) - coresPerSocket[curSocket] = v - } - } - } - } - totalCores := 0 - for _, c := range coresPerSocket { - totalCores += c - } - cfg.CPUModel = modelName - cfg.CPUSockets = len(socketIDs) - if cfg.CPUSockets == 0 && threads > 0 { - cfg.CPUSockets = 1 - } - cfg.CPUCores = totalCores - cfg.CPUThreads = threads - if modelName != "" || threads > 0 { - populated = true - } - } - - // Parse /proc/meminfo for total physical RAM. - if data, err := os.ReadFile("/proc/meminfo"); err == nil { - for _, line := range strings.Split(string(data), "\n") { - if strings.HasPrefix(line, "MemTotal:") { - fields := strings.Fields(line) - if len(fields) >= 2 { - kb, _ := strconv.ParseUint(fields[1], 10, 64) - cfg.MemTotalGiB = float64(kb) / (1024 * 1024) - populated = true - } - break - } - } - } - - if !populated { - return nil - } - return cfg -} - -// startCPULoadSampler starts a goroutine that samples host CPU load every -// intervalSec seconds until stopCh is closed, then sends the collected -// samples on the returned channel. -func startCPULoadSampler(stopCh <-chan struct{}, intervalSec int) <-chan []float64 { - ch := make(chan []float64, 1) - go func() { - var samples []float64 - ticker := time.NewTicker(time.Duration(intervalSec) * time.Second) - defer ticker.Stop() - for { - select { - case <-stopCh: - ch <- samples - return - case <-ticker.C: - if pct := sampleCPULoadPct(); pct > 0 { - samples = append(samples, pct) - } - } - } - }() - return ch -} - -// summarizeCPULoad computes stats over sampled CPU load values and assigns -// a health status. -func summarizeCPULoad(samples []float64) *BenchmarkCPULoad { - if len(samples) == 0 { - return nil - } - sorted := append([]float64(nil), samples...) - sort.Float64s(sorted) - var sum float64 - for _, v := range sorted { - sum += v - } - avg := sum / float64(len(sorted)) - p95 := sorted[int(float64(len(sorted))*0.95)] - max := sorted[len(sorted)-1] - - cl := &BenchmarkCPULoad{ - AvgPct: math.Round(avg*10) / 10, - MaxPct: math.Round(max*10) / 10, - P95Pct: math.Round(p95*10) / 10, - Samples: len(sorted), - } - - // Compute standard deviation to detect instability. - var variance float64 - for _, v := range sorted { - d := v - avg - variance += d * d - } - stdDev := math.Sqrt(variance / float64(len(sorted))) - - switch { - case avg > 20 || max > 40: - cl.Status = "high" - cl.Note = fmt.Sprintf("avg %.1f%% max %.1f%% — elevated host CPU load may interfere with GPU benchmark results", avg, max) - case stdDev > 12: - cl.Status = "unstable" - cl.Note = fmt.Sprintf("avg %.1f%% stddev %.1f%% — host CPU load was erratic during the benchmark", avg, stdDev) - default: - cl.Status = "ok" - } - return cl -} - -// runBenchmarkPowerCalibration runs the configured power-fit load for the supplied -// GPU set and actively watches throttle counters. seedLimits, when provided, are treated as -// the starting point for this calibration pass rather than as immutable fixed -// limits. This matters during cumulative ramp-up: once an additional GPU is -// introduced, every already-active GPU must be revalidated under the new -// thermal state instead of assuming its previous single-step limit is still -// valid. The selected reduced power limits stay active for the main benchmark -// and are restored by the caller afterwards. -func runBenchmarkPowerCalibration( - ctx context.Context, - verboseLog, runDir string, - gpuIndices []int, - infoByIndex map[int]benchmarkGPUInfo, - logFunc func(string), - seedLimits map[int]int, - durationSec int, -) (map[int]benchmarkPowerCalibrationResult, []benchmarkRestoreAction, []GPUMetricRow, benchmarkPowerCalibrationRunSummary) { - calibDurationSec := durationSec - var runSummary benchmarkPowerCalibrationRunSummary - if calibDurationSec <= 0 { - calibDurationSec = 120 - } - // calibSearchTolerance is the binary-search convergence threshold in watts. - // When hi-lo ≤ this, the highest verified-stable limit (lo) is used. - const calibSearchTolerance = 10 - // dcgmResourceBusyMaxDelaySec caps the exponential back-off when DCGM - // returns DCGM_ST_IN_USE (exit 222). The sequence is 1 s, 2 s, 4 s, … - // doubling each retry until it would exceed the cap, at which point the - // next busy response fails the calibration immediately. - const dcgmResourceBusyMaxDelaySec = 300 - engine := benchmarkPowerEngine() - engineLabel := benchmarkPowerEngineLabel(engine) - - if engine == BenchmarkPowerEngineTargetedPower { - if _, err := exec.LookPath("dcgmi"); err != nil { - logFunc("power calibration: dcgmi not found, skipping (will use default power limit)") - return map[int]benchmarkPowerCalibrationResult{}, nil, nil, runSummary - } - } else { - if _, _, err := resolveBenchmarkPowerLoadCommand(calibDurationSec, gpuIndices); err != nil { - logFunc("power calibration: dcgmproftester not found, skipping (will use default power limit)") - return map[int]benchmarkPowerCalibrationResult{}, nil, nil, runSummary - } - } - if killed := KillTestWorkers(); len(killed) > 0 { - for _, p := range killed { - logFunc(fmt.Sprintf("power calibration pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) - } - } - - canDerate := os.Geteuid() == 0 - if !canDerate { - logFunc("power calibration: root privileges unavailable, adaptive power-limit derating disabled") - } - - type calibrationAttemptResult struct { - out []byte - rows []GPUMetricRow - err error - } - - // gpuCalibState holds per-GPU binary search state during parallel calibration. - type gpuCalibState struct { - idx int - info benchmarkGPUInfo - originalLimitW int - appliedLimitW int - minLimitW int - lo int // highest verified-stable limit - hi int // lowest verified-unstable limit (exclusive sentinel above start) - loVerified bool - calib benchmarkPowerCalibrationResult - converged bool - } - - results := make(map[int]benchmarkPowerCalibrationResult, len(gpuIndices)) - var restore []benchmarkRestoreAction - var allCalibRows []GPUMetricRow // accumulated telemetry across all attempts - var calibCursor float64 - - // Initialise per-GPU state. - states := make([]*gpuCalibState, 0, len(gpuIndices)) - for _, idx := range gpuIndices { - info := infoByIndex[idx] - originalLimitW := int(math.Round(info.PowerLimitW)) - if originalLimitW <= 0 { - originalLimitW = int(math.Round(info.DefaultPowerLimitW)) - } - defaultLimitW := int(math.Round(info.DefaultPowerLimitW)) - if defaultLimitW <= 0 { - defaultLimitW = originalLimitW - } - appliedLimitW := initialBenchmarkCalibrationLimitW(info) - if appliedLimitW <= 0 { - appliedLimitW = defaultLimitW - } - minLimitW := int(math.Round(info.MinPowerLimitW)) - if minLimitW <= 0 { - minLimitW = appliedLimitW - } - maxLimitW := int(math.Round(info.MaxPowerLimitW)) - if maxLimitW > 0 && appliedLimitW > maxLimitW { - appliedLimitW = maxLimitW - } - s := &gpuCalibState{ - idx: idx, - info: info, - originalLimitW: originalLimitW, - appliedLimitW: appliedLimitW, - minLimitW: minLimitW, - lo: minLimitW, - hi: appliedLimitW + 1, // not yet tested, not yet confirmed unstable - calib: benchmarkPowerCalibrationResult{AppliedPowerLimitW: float64(appliedLimitW)}, - } - if minLimitW > 0 && appliedLimitW > 0 && minLimitW >= appliedLimitW { - s.appliedLimitW = minLimitW - s.hi = minLimitW + 1 - } - if info.MinPowerLimitW <= 0 { - s.calib.Notes = append(s.calib.Notes, "minimum power limit was not reported by nvidia-smi; calibration can only validate the current/default power limit") - } - if seedLimits != nil { - if seedW, ok := seedLimits[idx]; ok && seedW > 0 { - // A previously validated limit is only a starting point. Re-run - // targeted_power under the current multi-GPU thermal load and derate - // again if this step shows new throttling. - if seedW < s.minLimitW { - seedW = s.minLimitW - } - if maxLimitW > 0 && seedW > maxLimitW { - seedW = maxLimitW - } - if canDerate { - _ = setBenchmarkPowerLimit(ctx, verboseLog, idx, seedW) - } - s.appliedLimitW = seedW - s.hi = seedW + 1 - s.calib.AppliedPowerLimitW = float64(seedW) - s.calib.Derated = seedW < s.originalLimitW - s.calib.Notes = append(s.calib.Notes, - fmt.Sprintf("seed limit: %d W (revalidating under current thermal load)", seedW)) - } - } - states = append(states, s) - if canDerate && originalLimitW > 0 { - idxCopy := idx - orig := originalLimitW - restore = append(restore, benchmarkRestoreAction{ - name: fmt.Sprintf("gpu-%d-restore-power-limit", idxCopy), - fn: func() { - _ = setBenchmarkPowerLimit(context.Background(), verboseLog, idxCopy, orig) - }, - }) - } - } - - // Shared DCGM resource-busy back-off state (single diagnostic session). - busyRetries := 0 - busyDelaySec := 1 - sharedAttempt := 0 - - type sharedAttemptResult struct { - out []byte - rows []GPUMetricRow - err error - } - -calibDone: - for { - // Collect non-converged GPUs. - var active []*gpuCalibState - for _, s := range states { - if !s.converged { - active = append(active, s) - } - } - if len(active) == 0 || ctx.Err() != nil { - break - } - - sharedAttempt++ - for _, s := range active { - s.calib.Attempts++ - logFunc(fmt.Sprintf("power calibration: GPU %d %s attempt %d at %d W for %ds", s.idx, engineLabel, s.calib.Attempts, s.appliedLimitW, calibDurationSec)) - } - - // Snapshot throttle counters for all active GPUs before the run. - beforeThrottle := make(map[int]BenchmarkThrottleCounters, len(active)) - for _, s := range active { - beforeThrottle[s.idx], _ = queryThrottleCounters(s.idx) - } - - // Run the selected power-fit load for ALL gpuIndices simultaneously so every card - // is under load during calibration — this reflects real server thermals. - logName := fmt.Sprintf("power-calibration-attempt-%d.log", sharedAttempt) - cmd, env, err := resolveBenchmarkPowerLoadCommand(calibDurationSec, gpuIndices) - if err != nil { - for _, s := range active { - s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("failed to resolve %s command: %v", engineLabel, err)) - s.converged = true - } - logFunc(fmt.Sprintf("power calibration: failed to resolve %s command: %v", engineLabel, err)) - break calibDone - } - attemptCtx, cancelAttempt := context.WithCancel(ctx) - doneCh := make(chan sharedAttemptResult, 1) - sdrStopCh := make(chan struct{}) - sdrDoneCh := startIPMISDRSampler(sdrStopCh, benchmarkPowerAutotuneSampleInterval) - fanStopCh := make(chan struct{}) - fanDoneCh := startBenchmarkFanSampler(fanStopCh, benchmarkPowerAutotuneSampleInterval) - go func() { - out, rows, err := runBenchmarkCommandWithMetrics(attemptCtx, verboseLog, logName, cmd, env, gpuIndices, logFunc) - doneCh <- sharedAttemptResult{out: out, rows: rows, err: err} - }() - - ticker := time.NewTicker(time.Second) - throttleReasons := make(map[int]string, len(active)) - var ar sharedAttemptResult - - attemptLoop: - for { - select { - case ar = <-doneCh: - break attemptLoop - case <-ticker.C: - // Poll throttle counters for each active GPU independently. - for _, s := range active { - if throttleReasons[s.idx] != "" { - continue // already detected for this GPU - } - after, err := queryThrottleCounters(s.idx) - if err != nil { - continue - } - // Record throttle but do NOT cancel — let the load command finish so - // runtime resources release cleanly before the next attempt. - if reason := benchmarkCalibrationThrottleReason(beforeThrottle[s.idx], after); reason != "" { - throttleReasons[s.idx] = reason - logFunc(fmt.Sprintf("power calibration: GPU %d detected %s throttle at %d W, waiting for run to finish", s.idx, reason, s.appliedLimitW)) - } - } - case <-ctx.Done(): - cancelAttempt() - ar = <-doneCh - break attemptLoop - } - } - ticker.Stop() - cancelAttempt() - close(sdrStopCh) - close(fanStopCh) - attemptSDRSummary := summarizeSDRPowerSeries(<-sdrDoneCh) - attemptFanSummary := <-fanDoneCh - _ = os.WriteFile(filepath.Join(runDir, logName), ar.out, 0644) - // Accumulate telemetry rows with attempt stage label. - appendBenchmarkMetrics(&allCalibRows, ar.rows, fmt.Sprintf("attempt-%d", sharedAttempt), &calibCursor, float64(calibDurationSec)) - - // Resource busy: retry with exponential back-off (shared — one DCGM session). - if ar.err != nil && isDCGMResourceBusy(ar.err) { - if busyDelaySec > dcgmResourceBusyMaxDelaySec { - for _, s := range active { - s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("DCGM resource busy after %d retries, giving up", busyRetries)) - s.converged = true - } - logFunc(fmt.Sprintf("power calibration: DCGM resource persistently busy after %d retries, stopping", busyRetries)) - break calibDone - } - busyRetries++ - // Undo attempt counter: busy retries don't count as real attempts. - for _, s := range active { - s.calib.Attempts-- - } - logFunc(fmt.Sprintf("power calibration: DCGM resource busy (attempt %d), retrying in %ds", sharedAttempt, busyDelaySec)) - select { - case <-ctx.Done(): - break calibDone - case <-time.After(time.Duration(busyDelaySec) * time.Second): - } - next := busyDelaySec * 2 - if next > dcgmResourceBusyMaxDelaySec { - next = dcgmResourceBusyMaxDelaySec + 1 - } - busyDelaySec = next - sharedAttempt-- // retry same logical attempt number - continue - } - busyRetries = 0 - busyDelaySec = 1 - - // Per-GPU analysis and binary search update. - attemptStable := ar.err == nil - for _, s := range active { - perGPU := filterRowsByGPU(ar.rows, s.idx) - summary := summarizeBenchmarkTelemetry(perGPU) - throttle := throttleReasons[s.idx] - if throttle != "" || summary.P95PowerW <= 0 { - attemptStable = false - } - - // Cooling warning: thermal throttle with fans not at maximum. - if strings.Contains(throttle, "thermal") && s.calib.CoolingWarning == "" { - clocks := make([]float64, 0, len(perGPU)) - var fanDutyValues []float64 - fanDutyAvail := false - for _, r := range perGPU { - if r.ClockMHz > 0 { - clocks = append(clocks, r.ClockMHz) - } - if r.FanDutyCycleAvailable { - fanDutyAvail = true - fanDutyValues = append(fanDutyValues, r.FanDutyCyclePct) - } - } - dropPct := benchmarkClockDrift(clocks) - p95FanDuty := benchmarkPercentile(fanDutyValues, 95) - if dropPct >= 20 && fanDutyAvail && p95FanDuty < 98 { - s.calib.CoolingWarning = fmt.Sprintf( - "thermal throttle (%s) caused a %.0f%% clock drop while fans were at %.0f%% duty cycle — server cooling may not be configured for full GPU load", - throttle, dropPct, p95FanDuty, - ) - logFunc(fmt.Sprintf("power calibration: GPU %d cooling warning: %s", s.idx, s.calib.CoolingWarning)) - } - } - - if throttle == "" && ar.err == nil && summary.P95PowerW > 0 { - // Stable at current limit — update lo and binary-search upward. - s.calib.Summary = summary - s.calib.Completed = true - s.calib.AppliedPowerLimitW = float64(s.appliedLimitW) - logFunc(fmt.Sprintf("power calibration: GPU %d stable at %d W, p95=%.0f W p95_temp=%.1f C (%d samples)", s.idx, s.appliedLimitW, summary.P95PowerW, summary.P95TempC, summary.Samples)) - s.lo = s.appliedLimitW - s.loVerified = true - if canDerate && s.hi-s.lo > calibSearchTolerance { - next := roundTo5W((s.lo + s.hi) / 2) - if next > s.lo && next < s.hi { - if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, next); err == nil { - s.appliedLimitW = next - s.calib.AppliedPowerLimitW = float64(next) - s.calib.Completed = false // keep searching - s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search: stable at %d W, trying %d W (lo=%d hi=%d)", s.lo, next, s.lo, s.hi)) - logFunc(fmt.Sprintf("power calibration: GPU %d binary search up: stable at %d W, trying %d W", s.idx, s.lo, next)) - continue // next GPU in active list - } - } - } - s.calib.MetricRows = filterRowsByGPU(ar.rows, s.idx) - s.converged = true - continue - } - - // Failed or throttled — log and binary-search downward. - switch { - case throttle != "": - s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("targeted_power attempt %d: %s throttle at %d W", s.calib.Attempts, throttle, s.appliedLimitW)) - logFunc(fmt.Sprintf("power calibration: GPU %d throttled (%s) at %d W, reducing power limit", s.idx, throttle, s.appliedLimitW)) - case ar.err != nil: - s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("targeted_power attempt %d failed at %d W: %v", s.calib.Attempts, s.appliedLimitW, ar.err)) - logFunc(fmt.Sprintf("power calibration: GPU %d %s failed at %d W: %v", s.idx, engineLabel, s.appliedLimitW, ar.err)) - default: - s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("%s attempt %d at %d W: no valid power telemetry", engineLabel, s.calib.Attempts, s.appliedLimitW)) - logFunc(fmt.Sprintf("power calibration: GPU %d attempt %d at %d W: no valid telemetry", s.idx, s.calib.Attempts, s.appliedLimitW)) - } - - if !canDerate || s.appliedLimitW <= 0 { - s.converged = true - continue - } - s.hi = s.appliedLimitW - - if s.hi-s.lo <= calibSearchTolerance { - if !s.loVerified && s.minLimitW > 0 && s.appliedLimitW != s.minLimitW { - if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, s.minLimitW); err != nil { - s.calib.Notes = append(s.calib.Notes, "failed to set power limit: "+err.Error()) - logFunc(fmt.Sprintf("power calibration: GPU %d failed to set minimum power limit %d W: %v", s.idx, s.minLimitW, err)) - s.converged = true - continue - } - s.appliedLimitW = s.minLimitW - s.calib.AppliedPowerLimitW = float64(s.minLimitW) - s.calib.Derated = s.minLimitW < s.originalLimitW - s.info.PowerLimitW = float64(s.minLimitW) - infoByIndex[s.idx] = s.info - s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search: validating minimum settable limit %d W before concluding failure", s.minLimitW)) - logFunc(fmt.Sprintf("power calibration: GPU %d binary search: validating minimum settable limit %d W", s.idx, s.minLimitW)) - continue - } - if s.loVerified { - s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search converged: using %d W (lo=%d hi=%d)", s.lo, s.lo, s.hi)) - if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, s.lo); err == nil { - s.appliedLimitW = s.lo - s.calib.AppliedPowerLimitW = float64(s.lo) - s.calib.Derated = s.lo < s.originalLimitW - // Summary was captured when we last verified stability at s.lo, - // so the result is valid — mark as completed even though we - // converged from the failure path (tried higher, failed, fell back). - s.calib.Completed = true - } - } else { - s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("could not find a stable %s limit down to the minimum settable power limit %d W", engineLabel, s.minLimitW)) - logFunc(fmt.Sprintf("power calibration: GPU %d no stable limit found down to minimum settable power limit %d W", s.idx, s.minLimitW)) - } - s.calib.MetricRows = filterRowsByGPU(ar.rows, s.idx) - s.converged = true - continue - } - - next := roundTo5W((s.lo + s.hi) / 2) - if next <= s.lo { - next = s.lo + calibSearchTolerance - } - if next >= s.hi { - next = (s.lo + s.hi) / 2 - } - if next < s.minLimitW { - next = s.minLimitW - } - if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, next); err != nil { - s.calib.Notes = append(s.calib.Notes, "failed to set power limit: "+err.Error()) - logFunc(fmt.Sprintf("power calibration: GPU %d failed to set power limit %d W: %v", s.idx, next, err)) - s.converged = true - continue - } - s.appliedLimitW = next - s.calib.AppliedPowerLimitW = float64(next) - s.calib.Derated = next < s.originalLimitW - s.info.PowerLimitW = float64(next) - infoByIndex[s.idx] = s.info - s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search: trying %d W (lo=%d hi=%d)", next, s.lo, s.hi)) - logFunc(fmt.Sprintf("power calibration: GPU %d binary search: trying %d W (lo=%d hi=%d)", s.idx, next, s.lo, s.hi)) - } - if attemptStable { - if attemptSDRSummary.Samples > 0 { - runSummary.LoadedSDR = attemptSDRSummary - } - if attemptFanSummary.FanSamples > 0 { - runSummary.AvgFanRPM = attemptFanSummary.AvgFanRPM - runSummary.AvgFanDutyCyclePct = attemptFanSummary.AvgFanDutyCyclePct - runSummary.FanSamples = attemptFanSummary.FanSamples - } - } - } - - for _, s := range states { - if s.calib.Completed || s.calib.Attempts > 0 || len(s.calib.Notes) > 0 { - results[s.idx] = s.calib - } - } - writeBenchmarkMetricsFiles(runDir, allCalibRows) - return results, restore, allCalibRows, runSummary -} - -// isDCGMResourceBusy returns true when dcgmi exits with DCGM_ST_IN_USE (222), -// meaning nv-hostengine still holds the diagnostic slot from a prior run. -func isDCGMResourceBusy(err error) bool { - var exitErr *exec.ExitError - return errors.As(err, &exitErr) && exitErr.ExitCode() == 222 -} - -// roundTo5W rounds w to the nearest 5 W boundary. -func roundTo5W(w int) int { - return ((w + 2) / 5) * 5 -} - -func initialBenchmarkCalibrationLimitW(info benchmarkGPUInfo) int { - defaultLimitW := int(math.Round(info.DefaultPowerLimitW)) - currentLimitW := int(math.Round(info.PowerLimitW)) - maxLimitW := int(math.Round(info.MaxPowerLimitW)) - - startW := defaultLimitW - if startW <= 0 { - startW = currentLimitW - } - if startW <= 0 { - startW = maxLimitW - } - if maxLimitW > 0 && startW > maxLimitW { - startW = maxLimitW - } - return startW -} - -// meanFanRPM returns the average RPM across a set of fan readings. -func meanFanRPM(fans []FanReading) float64 { - if len(fans) == 0 { - return 0 - } - var sum float64 - for _, f := range fans { - sum += f.RPM - } - return sum / float64(len(fans)) -} - -func startBenchmarkFanSampler(stopCh <-chan struct{}, intervalSec int) <-chan benchmarkPowerCalibrationRunSummary { - if intervalSec <= 0 { - intervalSec = benchmarkPowerAutotuneSampleInterval - } - ch := make(chan benchmarkPowerCalibrationRunSummary, 1) - go func() { - defer close(ch) - var rpmSamples []float64 - var dutySamples []float64 - record := func() { - fans, err := sampleFanSpeeds() - if err != nil || len(fans) == 0 { - return - } - if rpm := meanFanRPM(fans); rpm > 0 { - rpmSamples = append(rpmSamples, rpm) - } - if duty, ok, _ := sampleFanDutyCyclePctFromFans(fans); ok && duty > 0 { - dutySamples = append(dutySamples, duty) - } - } - record() - ticker := time.NewTicker(time.Duration(intervalSec) * time.Second) - defer ticker.Stop() - for { - select { - case <-stopCh: - ch <- benchmarkPowerCalibrationRunSummary{ - AvgFanRPM: benchmarkMean(rpmSamples), - AvgFanDutyCyclePct: benchmarkMean(dutySamples), - FanSamples: len(rpmSamples), - } - return - case <-ticker.C: - record() - } - } - }() - return ch -} - -func powerBenchDurationSec(profile string) int { - switch strings.TrimSpace(strings.ToLower(profile)) { - case NvidiaBenchmarkProfileStability: - return 300 - case NvidiaBenchmarkProfileOvernight: - return 600 - default: - return 120 - } -} - -func cloneBenchmarkGPUInfoMap(src map[int]benchmarkGPUInfo) map[int]benchmarkGPUInfo { - out := make(map[int]benchmarkGPUInfo, len(src)) - for k, v := range src { - out[k] = v - } - return out -} - -func renderPowerBenchReport(result NvidiaPowerBenchResult) string { - var b strings.Builder - b.WriteString("# Bee Bench Power Report\n\n") - fmt.Fprintf(&b, "**Benchmark version:** %s \n", result.BenchmarkVersion) - fmt.Fprintf(&b, "**Profile:** %s \n", result.BenchmarkProfile) - fmt.Fprintf(&b, "**Generated:** %s \n", result.GeneratedAt.Format("2006-01-02 15:04:05 UTC")) - fmt.Fprintf(&b, "**Overall status:** %s \n", result.OverallStatus) - fmt.Fprintf(&b, "**Platform max TDP (GPU-reported):** %.0f W \n", result.PlatformMaxTDPW) - if sp := result.ServerPower; sp != nil && sp.Available { - sourceLabel := "autotuned source" - switch normalizeBenchmarkPowerSource(sp.Source) { - case BenchmarkPowerSourceSDRPSUInput: - sourceLabel = "autotuned source (SDR PSU AC input)" - case BenchmarkPowerSourceDCMI: - sourceLabel = "autotuned source (DCMI)" - } - fmt.Fprintf(&b, "**Server power delta (%s):** %.0f W \n", sourceLabel, sp.DeltaW) - fmt.Fprintf(&b, "**Reporting ratio:** %.2f \n", sp.ReportingRatio) - } - b.WriteString("\n") - // Server power comparison table. - if sp := result.ServerPower; sp != nil { - b.WriteString("## Server vs GPU Power Comparison\n\n") - selectedSource := normalizeBenchmarkPowerSource(sp.Source) - selectedSourceLabel := "Selected source" - if selectedSource == BenchmarkPowerSourceSDRPSUInput { - selectedSourceLabel = "Selected source (SDR PSU AC input)" - } else if selectedSource == BenchmarkPowerSourceDCMI { - selectedSourceLabel = "Selected source (DCMI)" - } - var spRows [][]string - spRows = append(spRows, []string{"GPU actual power sum (p95, last step)", fmt.Sprintf("%.0f W", sp.GPUReportedSumW)}) - if sp.Available { - spRows = append(spRows, []string{selectedSourceLabel + " idle power", fmt.Sprintf("%.0f W", sp.IdleW)}) - spRows = append(spRows, []string{selectedSourceLabel + " loaded power", fmt.Sprintf("%.0f W", sp.LoadedW)}) - spRows = append(spRows, []string{selectedSourceLabel + " Δ power (loaded − idle)", fmt.Sprintf("%.0f W", sp.DeltaW)}) - } - if selectedSource == BenchmarkPowerSourceSDRPSUInput && sp.PSUInputLoadedW > 0 { - spRows = append(spRows, []string{"PSU AC input (idle avg, pre-load phase)", fmt.Sprintf("%.0f W", sp.PSUInputIdleW)}) - spRows = append(spRows, []string{"PSU AC input (loaded avg, final phase)", fmt.Sprintf("%.0f W", sp.PSUInputLoadedW)}) - psuDelta := sp.PSUInputLoadedW - sp.PSUInputIdleW - spRows = append(spRows, []string{"PSU AC input Δ (loaded − idle)", fmt.Sprintf("%.0f W", psuDelta)}) - } - if sp.Available { - ratio := sp.ReportingRatio - dcmiPartial := detectDCMIPartialCoverage(sp) || - (sp.PSUInputIdleW == 0 && detectIPMISaturationFallback(result.RampSteps)) - ratioNote := "" - switch { - case dcmiPartial: - ratioNote = "⚠ IPMI DCMI covers partial PSU set; use SDR ratio below for accuracy assessment" - case ratio >= 0.9: - ratioNote = "✓ GPU telemetry matches server power" - case ratio >= 0.75: - ratioNote = "⚠ minor discrepancy — GPU may slightly over-report TDP" - default: - ratioNote = "✗ significant discrepancy — GPU over-reports TDP vs wall power" - } - spRows = append(spRows, []string{"Reporting ratio", fmt.Sprintf("%.2f — %s", ratio, ratioNote)}) - if selectedSource == BenchmarkPowerSourceSDRPSUInput && sp.PSUInputLoadedW > 0 && sp.GPUReportedSumW > 0 { - psuDelta := sp.PSUInputLoadedW - sp.PSUInputIdleW - sdrRatio := psuDelta / sp.GPUReportedSumW - sdrNote := "" - switch { - case sdrRatio >= 0.9: - sdrNote = "✓ GPU telemetry matches wall power" - case sdrRatio >= 0.75: - sdrNote = "⚠ minor discrepancy" - default: - sdrNote = "✗ significant discrepancy" - } - spRows = append(spRows, []string{"PSU AC input reporting ratio", fmt.Sprintf("%.2f — %s", sdrRatio, sdrNote)}) - } - } else { - spRows = append(spRows, []string{"IPMI availability", "not available — IPMI not supported or ipmitool not found"}) - } - b.WriteString(fmtMDTable([]string{"Metric", "Value"}, spRows)) - for _, note := range sp.Notes { - fmt.Fprintf(&b, "\n> %s\n", note) - } - b.WriteString("\n") - - if len(sp.PSUSlotReadingsIdle) > 0 || len(sp.PSUSlotReadingsLoaded) > 0 { - b.WriteString("## PSU Load Distribution\n\n") - - slotSet := map[string]struct{}{} - for k := range sp.PSUSlotReadingsIdle { - slotSet[k] = struct{}{} - } - for k := range sp.PSUSlotReadingsLoaded { - slotSet[k] = struct{}{} - } - slots := make([]string, 0, len(slotSet)) - for k := range slotSet { - slots = append(slots, k) - } - sort.Strings(slots) - - fmtW := func(v *float64) string { - if v == nil { - return "—" - } - return fmt.Sprintf("%.0f W", *v) - } - - var psuDistRows [][]string - for _, slot := range slots { - idle := sp.PSUSlotReadingsIdle[slot] - loaded := sp.PSUSlotReadingsLoaded[slot] - - var deltaStr string - if idle.InputW != nil && loaded.InputW != nil { - deltaStr = fmt.Sprintf("%+.0f W", *loaded.InputW-*idle.InputW) - } else { - deltaStr = "—" - } - - status := loaded.Status - if status == "" { - status = idle.Status - } - if status == "" { - status = "—" - } - - psuDistRows = append(psuDistRows, []string{ - slot, - fmtW(idle.InputW), fmtW(loaded.InputW), - deltaStr, status, - }) - } - b.WriteString(fmtMDTable([]string{"Slot", "AC Input (idle avg)", "AC Input (loaded avg)", "Load Δ", "Status"}, psuDistRows)) - b.WriteString("\n") - } - } - - if len(result.Findings) > 0 { - b.WriteString("## Summary\n\n") - for _, finding := range result.Findings { - fmt.Fprintf(&b, "- %s\n", finding) - } - b.WriteString("\n") - } - // ── Single GPU section ─────────────────────────────────────────────────── - b.WriteString("## Single GPU\n\n") - { - var sgRows [][]string - for _, gpu := range result.GPUs { - clk := "—" - mem := "—" - temp := "—" - pwr := "—" - if gpu.Telemetry != nil { - clk = fmt.Sprintf("%.0f", gpu.Telemetry.AvgGraphicsClockMHz) - mem = fmt.Sprintf("%.0f", gpu.Telemetry.AvgMemoryClockMHz) - temp = fmt.Sprintf("%.1f", gpu.Telemetry.AvgTempC) - pwr = fmt.Sprintf("%.0f W", gpu.Telemetry.AvgPowerW) - } - serverDelta := "—" - if gpu.ServerDeltaW > 0 { - serverDelta = fmt.Sprintf("%.0f W", gpu.ServerDeltaW) - } - fan := "—" - if gpu.AvgFanRPM > 0 { - if gpu.AvgFanDutyCyclePct > 0 { - fan = fmt.Sprintf("%.0f RPM (%.0f%%)", gpu.AvgFanRPM, gpu.AvgFanDutyCyclePct) - } else { - fan = fmt.Sprintf("%.0f RPM", gpu.AvgFanRPM) - } - } - sgRows = append(sgRows, []string{ - fmt.Sprintf("GPU %d", gpu.Index), - fmt.Sprintf("%s (%s)", clk, mem), - temp, - pwr, - serverDelta, - fan, - }) - } - b.WriteString(fmtMDTable([]string{"GPU", "Clock MHz (Mem MHz)", "Avg Temp °C", "Power W", "Server Δ W", "Avg Fan RPM (duty%)"}, sgRows)) - b.WriteString("\n") - } - if len(result.RecommendedSlotOrder) > 0 { - fmt.Fprintf(&b, "Recommended slot order for best single-card power realization: `%s`\n\n", joinIndexList(result.RecommendedSlotOrder)) - } - - // ── Ramp Sequence ──────────────────────────────────────────────────────── - // Rows = run number; Cols = per-GPU power (from step telemetry) + aggregates. - if len(result.RampSteps) > 0 { - b.WriteString("## Ramp Sequence\n\n") - - // Collect all GPU indices that appear across all steps (ordered by first appearance). - allGPUIndices := make([]int, 0, len(result.GPUs)) - seen := map[int]bool{} - for _, step := range result.RampSteps { - for _, idx := range step.GPUIndices { - if !seen[idx] { - seen[idx] = true - allGPUIndices = append(allGPUIndices, idx) - } - } - } - - var idleW float64 - if result.ServerPower != nil { - idleW = result.ServerPower.IdleW - } - - // Build header: Run | GPU 0 | GPU 1 | ... | GPU total W | Server itself W | Server wall W | Per GPU wall W | Platform eff. - headers := []string{"Run"} - for _, idx := range allGPUIndices { - headers = append(headers, fmt.Sprintf("GPU %d W", idx)) - } - headers = append(headers, "GPU total W", "Server itself W", "Server wall W", "Per GPU wall W", "Platform eff.") - - var rampRows [][]string - if idleW > 0 { - idleRow := []string{"0 (idle)"} - for range allGPUIndices { - idleRow = append(idleRow, "—") - } - // No load: GPU total is negligible, all draw is the server's own baseline. - idleRow = append(idleRow, "—", fmt.Sprintf("%.0f", idleW), fmt.Sprintf("%.0f", idleW), "—", "—") - rampRows = append(rampRows, idleRow) - } - for _, step := range result.RampSteps { - row := []string{fmt.Sprintf("%d", step.StepIndex)} - for _, idx := range allGPUIndices { - inStep := false - for _, si := range step.GPUIndices { - if si == idx { - inStep = true - break - } - } - if !inStep { - row = append(row, "—") - continue - } - gpuPwr := "—" - if t, ok := step.PerGPUTelemetry[idx]; ok && t != nil && t.AvgPowerW > 0 { - gpuPwr = fmt.Sprintf("%.0f", t.AvgPowerW) - } - row = append(row, gpuPwr) - } - // GPU total W = sum of observed GPU power (nvidia-smi) - gpuTotal := "—" - if step.TotalObservedPowerW > 0 { - gpuTotal = fmt.Sprintf("%.0f", step.TotalObservedPowerW) - } - // Server itself W = server wall power minus GPU total (non-GPU baseline draw) - serverItself := "—" - if step.ServerLoadedW > 0 && step.TotalObservedPowerW > 0 { - serverItself = fmt.Sprintf("%.0f", step.ServerLoadedW-step.TotalObservedPowerW) - } - // Server wall W - serverWall := "—" - if step.ServerLoadedW > 0 { - serverWall = fmt.Sprintf("%.0f", step.ServerLoadedW) - } - // Per GPU wall W = ServerDeltaW / len(GPUIndices) - perGPUWall := "—" - if step.ServerDeltaW > 0 && len(step.GPUIndices) > 0 { - perGPUWall = fmt.Sprintf("%.0f", step.ServerDeltaW/float64(len(step.GPUIndices))) - } - // Platform eff. = (ServerLoadedW − idleW) / TotalObservedPowerW - platEff := "—" - if step.TotalObservedPowerW > 0 { - eff := step.ServerDeltaW / step.TotalObservedPowerW - if idleW > 0 && step.ServerLoadedW > 0 { - eff = (step.ServerLoadedW - idleW) / step.TotalObservedPowerW - } - platEff = fmt.Sprintf("%.2f", eff) - } - row = append(row, gpuTotal, serverItself, serverWall, perGPUWall, platEff) - rampRows = append(rampRows, row) - } - b.WriteString(fmtMDTable(headers, rampRows)) - b.WriteString("\n") - } - - // ── PSU Performance ─────────────────────────────────────────────────────── - { - // Collect all PSU slot keys from any ramp step. - psuSlotSet := map[string]struct{}{} - for _, step := range result.RampSteps { - for k := range step.PSUSlotReadings { - psuSlotSet[k] = struct{}{} - } - } - if len(psuSlotSet) > 0 { - b.WriteString("## PSU Performance\n\n") - psuSlots := make([]string, 0, len(psuSlotSet)) - for k := range psuSlotSet { - psuSlots = append(psuSlots, k) - } - sort.Strings(psuSlots) - - var idleW float64 - if result.ServerPower != nil { - idleW = result.ServerPower.IdleW - } - - psuHeaders := []string{"Run"} - for _, slot := range psuSlots { - psuHeaders = append(psuHeaders, fmt.Sprintf("PSU %s W", slot)) - } - psuHeaders = append(psuHeaders, "PSU Total W", "Platform eff.", "Avg Fan RPM (duty%)") - - var psuRows [][]string - for _, step := range result.RampSteps { - row := []string{fmt.Sprintf("%d", step.StepIndex)} - var psuTotal float64 - for _, slot := range psuSlots { - sp, ok := step.PSUSlotReadings[slot] - if !ok || sp.InputW == nil { - row = append(row, "—") - continue - } - row = append(row, fmt.Sprintf("%.0f", *sp.InputW)) - psuTotal += *sp.InputW - } - totalStr := "—" - if psuTotal > 0 { - totalStr = fmt.Sprintf("%.0f", psuTotal) - } - platEff := "—" - if step.TotalObservedPowerW > 0 { - eff := step.ServerDeltaW / step.TotalObservedPowerW - if idleW > 0 && step.ServerLoadedW > 0 { - eff = (step.ServerLoadedW - idleW) / step.TotalObservedPowerW - } - platEff = fmt.Sprintf("%.2f", eff) - } - fan := "—" - if step.AvgFanRPM > 0 { - if step.AvgFanDutyCyclePct > 0 { - fan = fmt.Sprintf("%.0f (%.0f%%)", step.AvgFanRPM, step.AvgFanDutyCyclePct) - } else { - fan = fmt.Sprintf("%.0f", step.AvgFanRPM) - } - } - row = append(row, totalStr, platEff, fan) - psuRows = append(psuRows, row) - } - b.WriteString(fmtMDTable(psuHeaders, psuRows)) - b.WriteString("\n") - } - } - - // ── PSU Issues ──────────────────────────────────────────────────────────── - if len(result.PSUIssues) > 0 { - b.WriteString("## PSU Issues\n\n") - b.WriteString("The following power supply anomalies were detected during the test:\n\n") - for _, issue := range result.PSUIssues { - fmt.Fprintf(&b, "- ⛔ %s\n", issue) - } - b.WriteString("\n") - } - - // ── Power Distribution Summary ──────────────────────────────────────────── - b.WriteString("## Power Distribution Summary\n\n") - { - var totalDefault, totalStable float64 - for _, gpu := range result.GPUs { - stable := gpu.StablePowerLimitW - if stable <= 0 { - stable = gpu.AppliedPowerLimitW - } - totalDefault += gpu.DefaultPowerLimitW - totalStable += stable - } - var pdRows [][]string - for _, gpu := range result.GPUs { - stable := gpu.StablePowerLimitW - if stable <= 0 { - stable = gpu.AppliedPowerLimitW - } - realization := "-" - if gpu.DefaultPowerLimitW > 0 && stable > 0 { - realization = fmt.Sprintf("%.1f%%", stable/gpu.DefaultPowerLimitW*100) - } - derated := "-" - if gpu.Derated { - derated = "⚠ yes" - } - pdRows = append(pdRows, []string{ - fmt.Sprintf("GPU %d", gpu.Index), - fmt.Sprintf("%.0f W", gpu.AppliedPowerLimitW), - fmt.Sprintf("%.0f W", stable), - realization, - derated, - }) - } - platformReal := "-" - if totalDefault > 0 && totalStable > 0 { - platformReal = fmt.Sprintf("%.1f%%", totalStable/totalDefault*100) - } - pdRows = append(pdRows, []string{ - "**Platform**", - "—", - fmt.Sprintf("**%.0f W**", totalStable), - fmt.Sprintf("**%s**", platformReal), - "", - }) - b.WriteString(fmtMDTable([]string{"GPU", "Single-card limit", "Stable limit", "Realization", "Derated"}, pdRows)) - b.WriteString("\n") - - // Balance across GPUs — only meaningful with 2+ GPUs. - if len(result.GPUs) > 1 { - var minS, maxS, sumS float64 - var cnt int - for _, gpu := range result.GPUs { - s := gpu.StablePowerLimitW - if s <= 0 { - s = gpu.AppliedPowerLimitW - } - if s <= 0 { - continue - } - sumS += s - cnt++ - if cnt == 1 || s < minS { - minS = s - } - if s > maxS { - maxS = s - } - } - if cnt > 0 { - avg := sumS / float64(cnt) - spread := (maxS - minS) / avg * 100 - balanceNote := "✓ balanced" - switch { - case spread > 20: - balanceNote = "⚠ significant imbalance — check slot thermals" - case spread > 10: - balanceNote = "— minor imbalance" - } - fmt.Fprintf(&b, "**GPU power balance:** avg %.0f W · min %.0f W · max %.0f W · spread %.1f%% — %s\n\n", - avg, minS, maxS, spread, balanceNote) - } - } - - // Ramp scalability table — power efficiency of adding each GPU. - if len(result.RampSteps) > 1 { - b.WriteString("**Ramp power scalability** (stable TDP per step):\n\n") - var firstStable float64 - if len(result.GPUs) > 0 { - firstStable = result.GPUs[0].StablePowerLimitW - if firstStable <= 0 { - firstStable = result.GPUs[0].AppliedPowerLimitW - } - } - var prevCumulative float64 - var scalRows [][]string - for _, step := range result.RampSteps { - var cumulative float64 - for _, gpuIdx := range step.GPUIndices { - for _, g := range result.GPUs { - if g.Index != gpuIdx { - continue - } - s := g.StablePowerLimitW - if s <= 0 { - s = g.AppliedPowerLimitW - } - cumulative += s - } - } - incremental := cumulative - prevCumulative - efficiency := "—" - if step.StepIndex > 1 && firstStable > 0 { - efficiency = fmt.Sprintf("%.1f%%", incremental/firstStable*100) - } - scalRows = append(scalRows, []string{ - fmt.Sprintf("%d", step.StepIndex), - joinIndexList(step.GPUIndices), - fmt.Sprintf("%.0f W", cumulative), - fmt.Sprintf("%.0f W", incremental), - efficiency, - }) - prevCumulative = cumulative - } - b.WriteString(fmtMDTable([]string{"Step", "GPUs", "Cumulative stable TDP", "Incremental", "Efficiency vs GPU 1"}, scalRows)) - b.WriteString("\n") - } - } - - // ── Per-GPU sections ────────────────────────────────────────────────────── - var lastStep *NvidiaPowerBenchStep - if n := len(result.RampSteps); n > 0 { - lastStep = &result.RampSteps[n-1] - } - for _, gpu := range result.GPUs { - fmt.Fprintf(&b, "### GPU %d — %s\n\n", gpu.Index, gpu.Name) - - // Transposed comparison table: Single Run vs All GPU Run. - singleClk := "—" - singleMem := "—" - singleTemp := "—" - singlePwr := "—" - singleWall := "—" - singleFan := "—" - if gpu.Telemetry != nil { - singleClk = fmt.Sprintf("%.0f", gpu.Telemetry.AvgGraphicsClockMHz) - singleMem = fmt.Sprintf("%.0f", gpu.Telemetry.AvgMemoryClockMHz) - singleTemp = fmt.Sprintf("%.1f", gpu.Telemetry.AvgTempC) - singlePwr = fmt.Sprintf("%.0f W", gpu.Telemetry.AvgPowerW) - } - if gpu.ServerDeltaW > 0 { - singleWall = fmt.Sprintf("%.0f W", gpu.ServerDeltaW) - } - if gpu.AvgFanRPM > 0 { - if gpu.AvgFanDutyCyclePct > 0 { - singleFan = fmt.Sprintf("%.0f RPM (%.0f%%)", gpu.AvgFanRPM, gpu.AvgFanDutyCyclePct) - } else { - singleFan = fmt.Sprintf("%.0f RPM", gpu.AvgFanRPM) - } - } - - allClk := "—" - allMem := "—" - allTemp := "—" - allPwr := "—" - allWall := "—" - allFan := "—" - if lastStep != nil { - if t, ok := lastStep.PerGPUTelemetry[gpu.Index]; ok && t != nil { - allClk = fmt.Sprintf("%.0f", t.AvgGraphicsClockMHz) - allMem = fmt.Sprintf("%.0f", t.AvgMemoryClockMHz) - allTemp = fmt.Sprintf("%.1f", t.AvgTempC) - allPwr = fmt.Sprintf("%.0f W", t.AvgPowerW) - } - if lastStep.ServerDeltaW > 0 && len(lastStep.GPUIndices) > 0 { - allWall = fmt.Sprintf("%.0f W", lastStep.ServerDeltaW/float64(len(lastStep.GPUIndices))) - } - if lastStep.AvgFanRPM > 0 { - if lastStep.AvgFanDutyCyclePct > 0 { - allFan = fmt.Sprintf("%.0f RPM (%.0f%%)", lastStep.AvgFanRPM, lastStep.AvgFanDutyCyclePct) - } else { - allFan = fmt.Sprintf("%.0f RPM", lastStep.AvgFanRPM) - } - } - } - - tableHeaders := []string{"", "Single Run"} - if lastStep != nil { - tableHeaders = append(tableHeaders, "All GPU Run") - } - compRows := [][]string{ - {"Clock MHz (Mem MHz)", fmt.Sprintf("%s (%s)", singleClk, singleMem)}, - {"Avg Temp °C", singleTemp}, - {"Power W", singlePwr}, - {"Per GPU wall W", singleWall}, - {"Avg Fan RPM (duty%)", singleFan}, - } - if lastStep != nil { - compRows[0] = append(compRows[0], fmt.Sprintf("%s (%s)", allClk, allMem)) - compRows[1] = append(compRows[1], allTemp) - compRows[2] = append(compRows[2], allPwr) - compRows[3] = append(compRows[3], allWall) - compRows[4] = append(compRows[4], allFan) - } - b.WriteString(fmtMDTable(tableHeaders, compRows)) - b.WriteString("\n") - - for _, note := range gpu.Notes { - fmt.Fprintf(&b, "- %s\n", note) - } - if len(gpu.Notes) > 0 { - b.WriteString("\n") - } - } - return b.String() -} - -func renderPowerBenchSummary(result NvidiaPowerBenchResult) string { - var b strings.Builder - fmt.Fprintf(&b, "run_at_utc=%s\n", result.GeneratedAt.Format(time.RFC3339)) - fmt.Fprintf(&b, "benchmark_version=%s\n", result.BenchmarkVersion) - fmt.Fprintf(&b, "benchmark_profile=%s\n", result.BenchmarkProfile) - fmt.Fprintf(&b, "overall_status=%s\n", result.OverallStatus) - fmt.Fprintf(&b, "platform_max_tdp_w=%.0f\n", result.PlatformMaxTDPW) - fmt.Fprintf(&b, "gpu_count=%d\n", len(result.GPUs)) - if len(result.RecommendedSlotOrder) > 0 { - fmt.Fprintf(&b, "recommended_slot_order=%s\n", joinIndexList(result.RecommendedSlotOrder)) - } - for _, step := range result.RampSteps { - fmt.Fprintf(&b, "ramp_step_%d_gpus=%s\n", step.StepIndex, joinIndexList(step.GPUIndices)) - fmt.Fprintf(&b, "ramp_step_%d_new_gpu=%d\n", step.StepIndex, step.NewGPUIndex) - fmt.Fprintf(&b, "ramp_step_%d_stable_limit_w=%.0f\n", step.StepIndex, step.NewGPUStableLimitW) - fmt.Fprintf(&b, "ramp_step_%d_total_power_w=%.0f\n", step.StepIndex, step.TotalObservedPowerW) - if step.ServerLoadedW > 0 { - fmt.Fprintf(&b, "ramp_step_%d_server_loaded_w=%.0f\n", step.StepIndex, step.ServerLoadedW) - fmt.Fprintf(&b, "ramp_step_%d_server_delta_w=%.0f\n", step.StepIndex, step.ServerDeltaW) - } - } - for _, gpu := range result.GPUs { - if gpu.StablePowerLimitW > 0 { - fmt.Fprintf(&b, "gpu_%d_stable_limit_w=%.0f\n", gpu.Index, gpu.StablePowerLimitW) - } - if gpu.ServerLoadedW > 0 { - fmt.Fprintf(&b, "gpu_%d_server_loaded_w=%.0f\n", gpu.Index, gpu.ServerLoadedW) - fmt.Fprintf(&b, "gpu_%d_server_delta_w=%.0f\n", gpu.Index, gpu.ServerDeltaW) - } - } - if sp := result.ServerPower; sp != nil && sp.Available { - fmt.Fprintf(&b, "server_idle_w=%.0f\n", sp.IdleW) - fmt.Fprintf(&b, "server_loaded_w=%.0f\n", sp.LoadedW) - fmt.Fprintf(&b, "server_delta_w=%.0f\n", sp.DeltaW) - fmt.Fprintf(&b, "server_reporting_ratio=%.2f\n", sp.ReportingRatio) - } - return b.String() -} - -func (s *System) RunNvidiaPowerBench(ctx context.Context, baseDir string, opts NvidiaBenchmarkOptions, logFunc func(string)) (string, error) { - if ctx == nil { - ctx = context.Background() - } - if logFunc == nil { - logFunc = func(string) {} - } - if strings.TrimSpace(baseDir) == "" { - baseDir = "/var/log/bee-bench/power" - } - opts = normalizeNvidiaBenchmarkOptionsForBenchmark(opts) - selected, err := resolveNvidiaGPUSelection(opts.GPUIndices, opts.ExcludeGPUIndices) - if err != nil { - return "", err - } - if len(selected) == 0 { - return "", fmt.Errorf("no NVIDIA GPUs selected") - } - ts := time.Now().UTC().Format("20060102-150405") - runDir := filepath.Join(baseDir, "power-"+ts) - if err := os.MkdirAll(runDir, 0755); err != nil { - return "", fmt.Errorf("mkdir %s: %w", runDir, err) - } - verboseLog := filepath.Join(runDir, "verbose.log") - hostname, _ := os.Hostname() - result := NvidiaPowerBenchResult{ - BenchmarkVersion: benchmarkVersion, - GeneratedAt: time.Now().UTC(), - Hostname: hostname, - ServerModel: readServerModel(), - BenchmarkProfile: opts.Profile, - SelectedGPUIndices: append([]int(nil), selected...), - OverallStatus: "OK", - } - infoByIndex, infoErr := queryBenchmarkGPUInfo(selected) - if infoErr != nil { - return "", infoErr - } - // Capture full nvidia-smi -q snapshot at the start of the run. - if out, err := runSATCommandCtx(ctx, verboseLog, "00-nvidia-smi-q.log", []string{"nvidia-smi", "-q"}, nil, nil); err == nil { - _ = os.WriteFile(filepath.Join(runDir, "00-nvidia-smi-q.log"), out, 0644) - } - durationSec := powerBenchDurationSec(opts.Profile) - - // Sample server idle power before any GPU load. - var serverIdleW float64 - var serverIdleOK bool - idleSDRStopCh := make(chan struct{}) - idleSDRCh := startIPMISDRSampler(idleSDRStopCh, benchmarkPowerAutotuneSampleInterval) - if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, 10, benchmarkPowerAutotuneSampleInterval); ok { - serverIdleW = w - serverIdleOK = true - logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w)) - } - close(idleSDRStopCh) - sdrIdle := summarizeSDRPowerSeries(<-idleSDRCh) - psuBefore := psuStatusSnapshot() - - // Phase 1: calibrate each GPU individually (sequentially, one at a time) to - // establish a true single-card power baseline unaffected by neighbour heat. - calibByIndex := make(map[int]benchmarkPowerCalibrationResult, len(selected)) - singleIPMILoadedW := make(map[int]float64, len(selected)) - singleRunSummaryByIndex := make(map[int]benchmarkPowerCalibrationRunSummary, len(selected)) - var allRestoreActions []benchmarkRestoreAction - // allPowerRows accumulates telemetry from all phases for the top-level gpu-metrics.csv. - var allPowerRows []GPUMetricRow - var powerCursor float64 - for _, idx := range selected { - singleDir := filepath.Join(runDir, fmt.Sprintf("single-%02d", idx)) - _ = os.MkdirAll(singleDir, 0755) - singleInfo := cloneBenchmarkGPUInfoMap(infoByIndex) - if failed := resetBenchmarkGPUs(ctx, verboseLog, []int{idx}, logFunc); len(failed) > 0 { - return "", fmt.Errorf("power benchmark pre-flight: failed to reset GPU %d; benchmark aborted to keep measurements clean", idx) - } - logFunc(fmt.Sprintf("power calibration: GPU %d single-card baseline", idx)) - singlePowerStopCh := make(chan struct{}) - singlePowerCh := startSelectedPowerSourceSampler(singlePowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval) - c, restore, singleRows, singleRun := runBenchmarkPowerCalibration(ctx, verboseLog, singleDir, []int{idx}, singleInfo, logFunc, nil, durationSec) - appendBenchmarkMetrics(&allPowerRows, singleRows, fmt.Sprintf("single-gpu-%d", idx), &powerCursor, 0) - close(singlePowerStopCh) - if samples := <-singlePowerCh; len(samples) > 0 { - singleIPMILoadedW[idx] = benchmarkMean(samples) - logFunc(fmt.Sprintf("power calibration: GPU %d single-card server power (%s avg): %.0f W", idx, opts.ServerPowerSource, singleIPMILoadedW[idx])) - } else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && singleRun.LoadedSDR.PSUInW > 0 { - singleIPMILoadedW[idx] = singleRun.LoadedSDR.PSUInW - logFunc(fmt.Sprintf("power calibration: GPU %d single-card fallback server power (SDR avg): %.0f W", idx, singleRun.LoadedSDR.PSUInW)) - } - allRestoreActions = append(allRestoreActions, restore...) - if r, ok := c[idx]; ok { - calibByIndex[idx] = r - } - singleRunSummaryByIndex[idx] = singleRun - } - defer func() { - for i := len(allRestoreActions) - 1; i >= 0; i-- { - allRestoreActions[i].fn() - } - }() - gpus := make([]NvidiaPowerBenchGPU, 0, len(selected)) - for _, idx := range selected { - info := infoByIndex[idx] - calib := calibByIndex[idx] - status := "OK" - if !calib.Completed { - status = "FAILED" - result.OverallStatus = "PARTIAL" - } else if calib.Derated { - status = "PARTIAL" - if result.OverallStatus == "OK" { - result.OverallStatus = "PARTIAL" - } - } - gpu := NvidiaPowerBenchGPU{ - Index: idx, - Name: info.Name, - BusID: info.BusID, - DefaultPowerLimitW: info.DefaultPowerLimitW, - AppliedPowerLimitW: calib.AppliedPowerLimitW, - MaxObservedPowerW: calib.Summary.P95PowerW, - MaxObservedTempC: calib.Summary.P95TempC, - CalibrationAttempts: calib.Attempts, - Derated: calib.Derated, - Status: status, - Notes: append([]string(nil), calib.Notes...), - CoolingWarning: calib.CoolingWarning, - } - if w, ok := singleIPMILoadedW[idx]; ok && serverIdleOK && w > 0 { - gpu.ServerLoadedW = w - gpu.ServerDeltaW = w - serverIdleW - } - if len(calib.MetricRows) > 0 { - t := summarizeBenchmarkTelemetry(calib.MetricRows) - gpu.Telemetry = &t - } - if singleRun := singleRunSummaryByIndex[idx]; singleRun.AvgFanRPM > 0 { - gpu.AvgFanRPM = singleRun.AvgFanRPM - gpu.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct - } - gpus = append(gpus, gpu) - } - sort.Slice(gpus, func(i, j int) bool { - if gpus[i].MaxObservedPowerW != gpus[j].MaxObservedPowerW { - return gpus[i].MaxObservedPowerW > gpus[j].MaxObservedPowerW - } - if gpus[i].AppliedPowerLimitW != gpus[j].AppliedPowerLimitW { - return gpus[i].AppliedPowerLimitW > gpus[j].AppliedPowerLimitW - } - if gpus[i].Derated != gpus[j].Derated { - return !gpus[i].Derated - } - return gpus[i].Index < gpus[j].Index - }) - result.GPUs = gpus - result.RecommendedSlotOrder = make([]int, 0, len(gpus)) - for _, gpu := range gpus { - result.RecommendedSlotOrder = append(result.RecommendedSlotOrder, gpu.Index) - } - if len(result.RecommendedSlotOrder) > 0 { - result.Findings = append(result.Findings, fmt.Sprintf("Recommended slot order for installation based on single-card %s: %s.", benchmarkPowerEngineLabel(benchmarkPowerEngine()), joinIndexList(result.RecommendedSlotOrder))) - } - for _, gpu := range gpus { - if gpu.Derated { - result.Findings = append(result.Findings, fmt.Sprintf("GPU %d required reduced power limit %.0f W to complete %s.", gpu.Index, gpu.AppliedPowerLimitW, benchmarkPowerEngineLabel(benchmarkPowerEngine()))) - } - if gpu.CoolingWarning != "" { - result.Findings = append(result.Findings, fmt.Sprintf( - "GPU %d: %s. Operator action: rerun the benchmark with fan speed manually fixed at 100%% to confirm actual thermal headroom.", - gpu.Index, gpu.CoolingWarning, - )) - } - } - singleByIndex := make(map[int]NvidiaPowerBenchGPU, len(gpus)) - for _, gpu := range gpus { - singleByIndex[gpu.Index] = gpu - } - - // Phase 2: cumulative thermal ramp. - // Each step introduces one new GPU into an environment where all previously - // calibrated GPUs are already running at their fixed stable limits. The new - // GPU's stable TDP is searched via binary search under real - // multi-GPU thermal load. Once found, its limit is fixed permanently for all - // subsequent steps. This ensures each GPU's limit reflects actual sustained - // power in the final full-system thermal state. - // - // stableLimits accumulates GPU index → fixed stable limit (W) across steps. - stableLimits := make(map[int]int, len(result.RecommendedSlotOrder)) - - // serverLoadedW tracks the IPMI server power from the final ramp step - // (all GPUs simultaneously loaded). Earlier steps' values are stored - // per-step in NvidiaPowerBenchStep.ServerLoadedW. - var serverLoadedW float64 - var serverLoadedOK bool - // sdrLastStep retains the phase-averaged SDR readings from the last ramp step - // while GPUs are loaded. Used in the summary instead of re-sampling after the - // test when GPUs have already returned to idle. - var sdrLastStep benchmarkSDRSeriesSummary - - // Step 1: reuse single-card calibration result directly. - if len(result.RecommendedSlotOrder) > 0 { - firstIdx := result.RecommendedSlotOrder[0] - firstCalib := calibByIndex[firstIdx] - stableLimits[firstIdx] = int(math.Round(firstCalib.AppliedPowerLimitW)) - ramp := NvidiaPowerBenchStep{ - StepIndex: 1, - GPUIndices: []int{firstIdx}, - NewGPUIndex: firstIdx, - NewGPUStableLimitW: firstCalib.AppliedPowerLimitW, - TotalObservedPowerW: firstCalib.Summary.P95PowerW, - AvgObservedPowerW: firstCalib.Summary.P95PowerW, - Derated: firstCalib.Derated, - Status: "OK", - } - if w, ok := singleIPMILoadedW[firstIdx]; ok && serverIdleOK && w > 0 { - ramp.ServerLoadedW = w - ramp.ServerDeltaW = w - serverIdleW - } - if singleRun := singleRunSummaryByIndex[firstIdx]; singleRun.AvgFanRPM > 0 { - ramp.AvgFanRPM = singleRun.AvgFanRPM - ramp.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct - } - firstSummary := firstCalib.Summary - ramp.PerGPUTelemetry = map[int]*BenchmarkTelemetrySummary{firstIdx: &firstSummary} - if !firstCalib.Completed { - ramp.Status = "FAILED" - ramp.Notes = append(ramp.Notes, fmt.Sprintf("GPU %d did not complete single-card %s", firstIdx, benchmarkPowerEngineLabel(benchmarkPowerEngine()))) - result.OverallStatus = "PARTIAL" - } else if firstCalib.Derated { - ramp.Status = "PARTIAL" - if result.OverallStatus == "OK" { - result.OverallStatus = "PARTIAL" - } - result.Findings = append(result.Findings, fmt.Sprintf("Ramp step 1 (GPU %d) required derating to %.0f W.", firstIdx, firstCalib.AppliedPowerLimitW)) - } - result.RampSteps = append(result.RampSteps, ramp) - logFunc(fmt.Sprintf("power ramp: step 1/%d — reused single-card calibration for GPU %d, stable limit %.0f W", - len(result.RecommendedSlotOrder), firstIdx, firstCalib.AppliedPowerLimitW)) - } - - // Steps 2..N: each step revalidates every already-active GPU under the new - // cumulative thermal environment and also calibrates the newly introduced - // GPU. Previously found limits are used only as seeds for the search. - for stepNum := 1; stepNum < len(result.RecommendedSlotOrder); stepNum++ { - step := stepNum + 1 - subset := append([]int(nil), result.RecommendedSlotOrder[:step]...) - newGPUIdx := result.RecommendedSlotOrder[stepNum] - stepDir := filepath.Join(runDir, fmt.Sprintf("step-%02d", step)) - _ = os.MkdirAll(stepDir, 0755) - - // Reuse the latest stable limits as starting points, but re-check every - // active GPU in this hotter configuration. For the newly introduced GPU, - // seed from its single-card calibration so we do not restart from the - // default TDP when a prior derated limit is already known. - seedForStep := make(map[int]int, len(subset)) - for _, idx := range subset { - if lim, ok := stableLimits[idx]; ok && lim > 0 { - seedForStep[idx] = lim - continue - } - if base, ok := calibByIndex[idx]; ok { - lim := int(math.Round(base.AppliedPowerLimitW)) - if lim > 0 { - seedForStep[idx] = lim - } - } - } - - logFunc(fmt.Sprintf("power ramp: step %d/%d — revalidating %d active GPU(s) including new GPU %d", - step, len(result.RecommendedSlotOrder), len(subset), newGPUIdx)) - - stepInfo := cloneBenchmarkGPUInfoMap(infoByIndex) - stepPowerStopCh := make(chan struct{}) - stepPowerCh := startSelectedPowerSourceSampler(stepPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval) - stepCalib, stepRestore, stepRows, stepRun := runBenchmarkPowerCalibration(ctx, verboseLog, stepDir, subset, stepInfo, logFunc, seedForStep, durationSec) - appendBenchmarkMetrics(&allPowerRows, stepRows, fmt.Sprintf("ramp-step-%d", step), &powerCursor, 0) - close(stepPowerStopCh) - var stepIPMILoadedW float64 - var stepIPMIOK bool - if samples := <-stepPowerCh; len(samples) > 0 { - stepIPMILoadedW = benchmarkMean(samples) - stepIPMIOK = true - } - // Accumulate restore actions; they all run in the outer defer. - allRestoreActions = append(allRestoreActions, stepRestore...) - - ramp := NvidiaPowerBenchStep{ - StepIndex: step, - GPUIndices: subset, - NewGPUIndex: newGPUIdx, - Status: "OK", - } - - // Total observed power = sum of p95 across all GPUs in this step. - for _, idx := range subset { - if c, ok := stepCalib[idx]; ok { - ramp.TotalObservedPowerW += c.Summary.P95PowerW - } - } - if len(subset) > 0 { - ramp.AvgObservedPowerW = ramp.TotalObservedPowerW / float64(len(subset)) - } - - for _, idx := range subset { - c, ok := stepCalib[idx] - if !ok || !c.Completed { - fallback := 0 - if lim, ok := stableLimits[idx]; ok && lim > 0 { - fallback = lim - } else if fb, ok := calibByIndex[idx]; ok { - fallback = int(math.Round(fb.AppliedPowerLimitW)) - } - if fallback > 0 { - stableLimits[idx] = fallback - } - ramp.Status = "FAILED" - ramp.Notes = append(ramp.Notes, - fmt.Sprintf("GPU %d did not complete %s in ramp step %d; keeping previous stable limit %d W", idx, benchmarkPowerEngineLabel(benchmarkPowerEngine()), step, fallback)) - result.OverallStatus = "PARTIAL" - continue - } - - prevLimit, hadPrev := stableLimits[idx] - newLimit := int(math.Round(c.AppliedPowerLimitW)) - stableLimits[idx] = newLimit - if idx == newGPUIdx { - ramp.NewGPUStableLimitW = c.AppliedPowerLimitW - ramp.Derated = c.Derated - } - if c.Derated { - ramp.Status = "PARTIAL" - if result.OverallStatus == "OK" { - result.OverallStatus = "PARTIAL" - } - } - if hadPrev && newLimit < prevLimit { - ramp.Notes = append(ramp.Notes, - fmt.Sprintf("GPU %d was re-derated from %d W to %d W under combined thermal load.", idx, prevLimit, newLimit)) - } - } - - if c, ok := stepCalib[newGPUIdx]; ok && c.Completed && c.Derated { - result.Findings = append(result.Findings, fmt.Sprintf("Ramp step %d (GPU %d) required derating to %.0f W under combined thermal load.", step, newGPUIdx, c.AppliedPowerLimitW)) - } - - // Per-step PSU slot readings are averaged over the whole load phase rather - // than captured as a single end-of-phase snapshot. - sdrStep := stepRun.LoadedSDR - if len(sdrStep.PSUSlots) > 0 { - ramp.PSUSlotReadings = sdrStep.PSUSlots - } - - if stepIPMIOK && serverIdleOK && stepIPMILoadedW > 0 { - ramp.ServerLoadedW = stepIPMILoadedW - ramp.ServerDeltaW = stepIPMILoadedW - serverIdleW - logFunc(fmt.Sprintf("power ramp: step %d server loaded power (%s avg): %.0f W", step, opts.ServerPowerSource, stepIPMILoadedW)) - // The last step has all GPUs loaded — use it as the top-level loaded_w. - if step == len(result.RecommendedSlotOrder) { - serverLoadedW = stepIPMILoadedW - serverLoadedOK = true - sdrLastStep = sdrStep - } - } else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && sdrStep.PSUInW > 0 { - ramp.ServerLoadedW = sdrStep.PSUInW - ramp.ServerDeltaW = sdrStep.PSUInW - sdrIdle.PSUInW - logFunc(fmt.Sprintf("power ramp: step %d fallback server loaded power (SDR avg): %.0f W", step, sdrStep.PSUInW)) - if step == len(result.RecommendedSlotOrder) { - serverLoadedW = sdrStep.PSUInW - serverLoadedOK = true - sdrLastStep = sdrStep - } - } - - // Fan values are phase averages over the same load window. - if stepRun.AvgFanRPM > 0 { - ramp.AvgFanRPM = stepRun.AvgFanRPM - ramp.AvgFanDutyCyclePct = stepRun.AvgFanDutyCyclePct - } - - // Per-GPU telemetry from this ramp step's calibration. - ramp.PerGPUTelemetry = make(map[int]*BenchmarkTelemetrySummary, len(subset)) - for _, gpuIdx := range subset { - if c, ok := stepCalib[gpuIdx]; ok { - s := c.Summary - ramp.PerGPUTelemetry[gpuIdx] = &s - } - } - - result.RampSteps = append(result.RampSteps, ramp) - } - - // Populate StablePowerLimitW on each GPU entry from the accumulated stable limits. - for i := range result.GPUs { - if lim, ok := stableLimits[result.GPUs[i].Index]; ok { - result.GPUs[i].StablePowerLimitW = float64(lim) - } - if result.GPUs[i].StablePowerLimitW > 0 && result.GPUs[i].AppliedPowerLimitW > 0 && - result.GPUs[i].StablePowerLimitW < result.GPUs[i].AppliedPowerLimitW { - result.GPUs[i].Derated = true - result.Findings = append(result.Findings, fmt.Sprintf( - "GPU %d required additional derating from %.0f W (single-card) to %.0f W under full-system thermal load.", - result.GPUs[i].Index, result.GPUs[i].AppliedPowerLimitW, result.GPUs[i].StablePowerLimitW, - )) - } - } - - // PlatformMaxTDPW = sum of all stable limits — the actual sustained power - // budget of this server with all GPUs running simultaneously without throttling. - for _, lim := range stableLimits { - result.PlatformMaxTDPW += float64(lim) - } - - // Characterize server power from IPMI idle/loaded samples. - // gpuActualSumW = sum of p95 GPU power from the last ramp step — actual - // measured consumption, not the stable limit cap. This is the correct - // denominator for the reporting ratio: limit caps (PlatformMaxTDPW) inflate - // the denominator and make the ratio appear artificially low. - var gpuActualSumW float64 - if n := len(result.RampSteps); n > 0 { - gpuActualSumW = result.RampSteps[n-1].TotalObservedPowerW - } - if gpuActualSumW <= 0 { - gpuActualSumW = result.PlatformMaxTDPW - } - _ = serverIdleOK // used implicitly via characterizeServerPower - result.ServerPower = characterizeServerPower(serverIdleW, serverLoadedW, gpuActualSumW, opts.ServerPowerSource, serverIdleOK && serverLoadedOK) - // Supplement DCMI with SDR multi-source data via collector's PSU slot patterns. - // Per-slot readings enable correlation with audit HardwarePowerSupply entries. - if result.ServerPower != nil { - // Use the SDR phase average from the last ramp step (GPUs still loaded) - // rather than re-sampling here, which would capture post-test idle state. - sdrLoaded := sdrLastStep - result.ServerPower.PSUInputIdleW = sdrIdle.PSUInW - result.ServerPower.PSUInputLoadedW = sdrLoaded.PSUInW - result.ServerPower.PSUOutputIdleW = sdrIdle.PSUOutW - result.ServerPower.PSUOutputLoadedW = sdrLoaded.PSUOutW - result.ServerPower.GPUSlotTotalW = sdrLoaded.GPUSlotW - if len(sdrIdle.PSUSlots) > 0 { - result.ServerPower.PSUSlotReadingsIdle = sdrIdle.PSUSlots - } - if len(sdrLoaded.PSUSlots) > 0 { - result.ServerPower.PSUSlotReadingsLoaded = sdrLoaded.PSUSlots - } - if sdrIdle.PSUInW > 0 && result.ServerPower.IdleW > 0 { - result.ServerPower.DCMICoverageRatio = result.ServerPower.IdleW / sdrIdle.PSUInW - } - if len(sdrLoaded.SkippedSensors) > 0 { - result.ServerPower.Notes = append(result.ServerPower.Notes, - "SDR sensors skipped (self-healed): "+strings.Join(sdrLoaded.SkippedSensors, "; ")) - } - if sdrLoaded.Samples > 0 { - result.ServerPower.Notes = append(result.ServerPower.Notes, - fmt.Sprintf("Final SDR PSU loaded values are phase averages across %d sample(s) from the last full-load step.", sdrLoaded.Samples)) - } - // Detect DCMI partial coverage: direct SDR comparison first, - // ramp heuristic as fallback when SDR PSU sensors are absent. - dcmiUnreliable := detectDCMIPartialCoverage(result.ServerPower) || - (sdrIdle.PSUInW == 0 && detectIPMISaturationFallback(result.RampSteps)) - if dcmiUnreliable { - result.ServerPower.Notes = append(result.ServerPower.Notes, - fmt.Sprintf("IPMI DCMI covers only a subset of installed PSUs (coverage %.0f%%). "+ - "Use SDR PSU Δ ratio for GPU accuracy assessment; DCMI ratio is not reliable.", - result.ServerPower.DCMICoverageRatio*100)) - } - } - result.PSUIssues = diffPSUStatus(psuBefore, psuStatusSnapshot()) - // Write top-level gpu-metrics.csv/.html aggregating all phases. - writeBenchmarkMetricsFiles(runDir, allPowerRows) - resultJSON, err := json.MarshalIndent(result, "", " ") - if err != nil { - return "", fmt.Errorf("marshal power result: %w", err) - } - if err := os.WriteFile(filepath.Join(runDir, "result.json"), resultJSON, 0644); err != nil { - return "", fmt.Errorf("write result.json: %w", err) - } - if err := os.WriteFile(filepath.Join(runDir, "report.md"), []byte(renderPowerBenchReport(result)), 0644); err != nil { - return "", fmt.Errorf("write report.md: %w", err) - } - if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(renderPowerBenchSummary(result)), 0644); err != nil { - return "", fmt.Errorf("write summary.txt: %w", err) - } - return runDir, nil -} diff --git a/audit/internal/platform/benchmark_analysis.go b/audit/internal/platform/benchmark_analysis.go new file mode 100644 index 0000000..d1d14e0 --- /dev/null +++ b/audit/internal/platform/benchmark_analysis.go @@ -0,0 +1,659 @@ +package platform + +import ( + "context" + "fmt" + "math" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +type benchmarkPlannedPhase struct { + PlanLabel string + MetricStage string + DurationSec int +} + +func runBenchmarkPlannedCommandWithMetrics( + ctx context.Context, + verboseLog, name string, + cmd []string, + env []string, + gpuIndices []int, + phases []benchmarkPlannedPhase, + logFunc func(string), +) ([]byte, map[string][]GPUMetricRow, map[string][]byte, error) { + out, rows, err := runBenchmarkCommandWithMetrics(ctx, verboseLog, name, cmd, env, gpuIndices, logFunc) + return out, splitBenchmarkRowsByPlannedPhase(rows, phases), splitBenchmarkLogByPlannedPhase(out), err +} + +func splitBenchmarkRowsByPlannedPhase(rows []GPUMetricRow, phases []benchmarkPlannedPhase) map[string][]GPUMetricRow { + out := make(map[string][]GPUMetricRow, len(phases)) + if len(rows) == 0 || len(phases) == 0 { + return out + } + for _, row := range rows { + idx := len(phases) - 1 + var elapsed float64 + for i, phase := range phases { + durationSec := phase.DurationSec + if durationSec <= 0 { + durationSec = 1 + } + elapsed += float64(durationSec) + if row.ElapsedSec < elapsed { + idx = i + break + } + } + out[phases[idx].MetricStage] = append(out[phases[idx].MetricStage], row) + } + return out +} + +func splitBenchmarkLogByPlannedPhase(raw []byte) map[string][]byte { + out := make(map[string][]byte) + var current string + for _, line := range strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") { + trimmed := strings.TrimSpace(stripBenchmarkPrefix(line)) + switch { + case strings.HasPrefix(trimmed, "phase_begin="): + current = strings.TrimSpace(strings.TrimPrefix(trimmed, "phase_begin=")) + case strings.HasPrefix(trimmed, "phase_end="): + current = "" + case current != "": + out[current] = append(out[current], []byte(line+"\n")...) + } + } + return out +} + +type benchmarkCoolingSample struct { + AvgFanRPM float64 + AvgFanDutyCyclePct float64 + FanDutyCycleAvailable bool + FanDutyCycleEstimated bool +} + +func sampleBenchmarkTelemetry(gpuIndices []int) ([]GPUMetricRow, error) { + samples, err := sampleGPUMetrics(gpuIndices) + if err != nil { + return nil, err + } + fanSample := sampleBenchmarkCoolingSample() + for i := range samples { + samples[i].FanAvgRPM = fanSample.AvgFanRPM + samples[i].FanDutyCyclePct = fanSample.AvgFanDutyCyclePct + samples[i].FanDutyCycleAvailable = fanSample.FanDutyCycleAvailable + samples[i].FanDutyCycleEstimated = fanSample.FanDutyCycleEstimated + } + return samples, nil +} + +func sampleBenchmarkCoolingSample() benchmarkCoolingSample { + fans, _ := sampleFanSpeeds() + avgRPM, _, _ := fanRPMStats(fans) + dutyPct, dutyAvailable, dutyEstimated := sampleFanDutyCyclePctFromFans(fans) + return benchmarkCoolingSample{ + AvgFanRPM: avgRPM, + AvgFanDutyCyclePct: dutyPct, + FanDutyCycleAvailable: dutyAvailable, + FanDutyCycleEstimated: dutyEstimated, + } +} + +func annotateBenchmarkMetricRows(rows []GPUMetricRow, stage string, offset, durationSec float64) []GPUMetricRow { + if len(rows) == 0 { + return nil + } + stageEnd := offset + durationSec + if stageEnd <= offset { + stageEnd = offset + for _, row := range rows { + if row.ElapsedSec+offset > stageEnd { + stageEnd = row.ElapsedSec + offset + } + } + } + out := make([]GPUMetricRow, len(rows)) + for i, row := range rows { + row.Stage = stage + row.ElapsedSec += offset + row.StageStartSec = offset + row.StageEndSec = stageEnd + out[i] = row + } + return out +} + +func appendBenchmarkMetrics(allRows *[]GPUMetricRow, rows []GPUMetricRow, stage string, cursor *float64, durationSec float64) { + annotated := annotateBenchmarkMetricRows(rows, stage, *cursor, durationSec) + *allRows = append(*allRows, annotated...) + *cursor += durationSec +} + +func writeBenchmarkMetricsFiles(runDir string, rows []GPUMetricRow) { + if len(rows) == 0 { + return + } + _ = WriteGPUMetricsCSV(filepath.Join(runDir, "gpu-metrics.csv"), rows) + _ = WriteGPUMetricsHTML(filepath.Join(runDir, "gpu-metrics.html"), rows) +} + +func appendBenchmarkStageLog(path, source, stage string, raw []byte) { + if path == "" || len(raw) == 0 { + return + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return + } + defer f.Close() + header := fmt.Sprintf("\n========== %s | stage=%s ==========\n", source, stage) + _, _ = f.WriteString(header) + if len(raw) > 0 { + _, _ = f.Write(raw) + if raw[len(raw)-1] != '\n' { + _, _ = f.WriteString("\n") + } + } +} + +func parseBenchmarkBurnLog(raw string) benchmarkBurnParseResult { + result := benchmarkBurnParseResult{} + lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") + profiles := make(map[string]*benchmarkBurnProfile) + for _, line := range lines { + line = stripBenchmarkPrefix(strings.TrimSpace(line)) + if line == "" { + continue + } + switch { + case strings.HasPrefix(line, "device="): + result.Device = strings.TrimSpace(strings.TrimPrefix(line, "device=")) + case strings.HasPrefix(line, "compute_capability="): + result.ComputeCapability = strings.TrimSpace(strings.TrimPrefix(line, "compute_capability=")) + case strings.HasPrefix(line, "backend="): + result.Backend = strings.TrimSpace(strings.TrimPrefix(line, "backend=")) + result.Fallback = result.Backend == "driver-ptx" + case strings.HasPrefix(line, "duration_s="): + result.DurationSec, _ = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "duration_s="))) + default: + if m := benchmarkReadyPattern.FindStringSubmatch(line); len(m) == 6 { + profile := ensureBenchmarkProfile(profiles, m[1]) + profile.supported = true + profile.lanes++ + profile.m, _ = strconv.ParseUint(m[3], 10, 64) + profile.n, _ = strconv.ParseUint(m[4], 10, 64) + profile.k, _ = strconv.ParseUint(m[5], 10, 64) + continue + } + if m := benchmarkSkippedPattern.FindStringSubmatch(line); len(m) == 3 { + profile := ensureBenchmarkProfile(profiles, m[1]) + profile.supported = false + profile.notes = strings.TrimSpace(m[2]) + continue + } + if m := benchmarkIterationsPattern.FindStringSubmatch(line); len(m) == 3 { + profile := ensureBenchmarkProfile(profiles, m[1]) + iters, _ := strconv.ParseUint(m[2], 10, 64) + profile.iterations += iters + } + } + } + + keys := make([]string, 0, len(profiles)) + for key := range profiles { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + profile := profiles[key] + precision := BenchmarkPrecisionResult{ + Name: profile.name, + Category: profile.category, + Supported: profile.supported, + Lanes: profile.lanes, + M: profile.m, + N: profile.n, + K: profile.k, + Iterations: profile.iterations, + Notes: profile.notes, + } + w := precisionWeight(profile.category) + precision.Weight = w + if profile.supported && result.DurationSec > 0 && profile.m > 0 && profile.n > 0 && profile.k > 0 && profile.iterations > 0 { + precision.TeraOpsPerSec = (2.0 * float64(profile.m) * float64(profile.n) * float64(profile.k) * float64(profile.iterations)) / float64(result.DurationSec) / 1e12 + precision.WeightedTeraOpsPerSec = precision.TeraOpsPerSec * w + } + result.Profiles = append(result.Profiles, precision) + } + return result +} + +func ensureBenchmarkProfile(profiles map[string]*benchmarkBurnProfile, name string) *benchmarkBurnProfile { + if profile, ok := profiles[name]; ok { + return profile + } + category := "other" + switch { + case strings.HasPrefix(name, "fp64"): + category = "fp64" + case strings.HasPrefix(name, "fp32"): + category = "fp32_tf32" + case strings.HasPrefix(name, "fp16"): + category = "fp16_bf16" + case strings.HasPrefix(name, "int8"): + category = "int8" + case strings.HasPrefix(name, "fp8"): + category = "fp8" + case strings.HasPrefix(name, "fp4"): + category = "fp4" + } + profile := &benchmarkBurnProfile{name: name, category: category, supported: true} + profiles[name] = profile + return profile +} + +// precisionWeight returns the fp32-equivalence factor for a precision category. +// Each factor represents how much "real" numeric work one operation of that +// type performs relative to fp32 (single precision = 1.0 baseline): +// +// fp64 = 2.0 — double precision, 2× more bits per operand +// fp32 = 1.0 — single precision baseline +// fp16 = 0.5 — half precision +// int8 = 0.25 — quarter precision +// fp8 = 0.25 — quarter precision +// fp4 = 0.125 — eighth precision +// +// Multiplying raw TOPS by the weight gives fp32-equivalent TOPS, enabling +// cross-precision comparison on the same numeric scale. +func precisionWeight(category string) float64 { + switch category { + case "fp64": + return 2.0 + case "fp32_tf32": + return 1.0 + case "fp16_bf16": + return 0.5 + case "int8": + return 0.25 + case "fp8": + return 0.25 + case "fp4": + return 0.125 + default: + return 1.0 + } +} + +func stripBenchmarkPrefix(line string) string { + if strings.HasPrefix(line, "[gpu ") { + if idx := strings.Index(line, "] "); idx >= 0 { + return line[idx+2:] + } + } + return line +} + +func summarizeBenchmarkTelemetry(rows []GPUMetricRow) BenchmarkTelemetrySummary { + summary := BenchmarkTelemetrySummary{} + if len(rows) == 0 { + return summary + } + temps := make([]float64, 0, len(rows)) + powers := make([]float64, 0, len(rows)) + clocks := make([]float64, 0, len(rows)) + memClocks := make([]float64, 0, len(rows)) + usages := make([]float64, 0, len(rows)) + memUsages := make([]float64, 0, len(rows)) + summary.DurationSec = rows[len(rows)-1].ElapsedSec + summary.Samples = len(rows) + for _, row := range rows { + temps = append(temps, row.TempC) + powers = append(powers, row.PowerW) + clocks = append(clocks, row.ClockMHz) + memClocks = append(memClocks, row.MemClockMHz) + usages = append(usages, row.UsagePct) + memUsages = append(memUsages, row.MemUsagePct) + } + summary.AvgTempC = benchmarkMean(temps) + summary.P95TempC = benchmarkPercentile(temps, 95) + summary.AvgPowerW = benchmarkMean(powers) + summary.P95PowerW = benchmarkPercentile(powers, 95) + summary.AvgGraphicsClockMHz = benchmarkMean(clocks) + summary.P95GraphicsClockMHz = benchmarkPercentile(clocks, 95) + summary.AvgMemoryClockMHz = benchmarkMean(memClocks) + summary.P95MemoryClockMHz = benchmarkPercentile(memClocks, 95) + summary.AvgUsagePct = benchmarkMean(usages) + summary.AvgMemUsagePct = benchmarkMean(memUsages) + summary.ClockCVPct = benchmarkCV(clocks) + summary.PowerCVPct = benchmarkCV(powers) + summary.TempCVPct = benchmarkCV(temps) + summary.ClockDriftPct = benchmarkClockDrift(clocks) + return summary +} + +func summarizeBenchmarkCooling(rows []GPUMetricRow) *BenchmarkCoolingSummary { + if len(rows) == 0 { + return nil + } + var rpmValues []float64 + var dutyValues []float64 + var dutyEstimated bool + for _, row := range rows { + if row.FanAvgRPM > 0 { + rpmValues = append(rpmValues, row.FanAvgRPM) + } + if row.FanDutyCycleAvailable { + dutyValues = append(dutyValues, row.FanDutyCyclePct) + if row.FanDutyCycleEstimated { + dutyEstimated = true + } + } + } + if len(rpmValues) == 0 && len(dutyValues) == 0 { + return nil + } + summary := &BenchmarkCoolingSummary{ + Available: true, + AvgFanRPM: benchmarkMean(rpmValues), + FanDutyCycleEstimated: dutyEstimated, + } + if len(dutyValues) > 0 { + summary.FanDutyCycleAvailable = true + summary.AvgFanDutyCyclePct = benchmarkMean(dutyValues) + summary.P95FanDutyCyclePct = benchmarkPercentile(dutyValues, 95) + if summary.FanDutyCycleEstimated { + summary.Notes = append(summary.Notes, "fan duty cycle is estimated from the highest fan RPM observed since boot; treat it as an approximation, not a direct PWM reading") + } + } else { + summary.Notes = append(summary.Notes, "fan duty cycle unavailable on this host; RPM-only fan telemetry was collected") + } + return summary +} + +func benchmarkTelemetryAvailable(summary BenchmarkTelemetrySummary) bool { + return summary.Samples > 0 || summary.DurationSec > 0 +} + +func benchmarkPrecisionSteadyFallback(phases []BenchmarkPrecisionSteadyPhase) (BenchmarkTelemetrySummary, string, bool) { + var ( + best BenchmarkTelemetrySummary + bestLabel string + found bool + ) + for _, phase := range phases { + if !benchmarkTelemetryAvailable(phase.Steady) { + continue + } + if !found || + phase.Steady.DurationSec > best.DurationSec || + (phase.Steady.DurationSec == best.DurationSec && phase.Steady.P95PowerW > best.P95PowerW) { + best = phase.Steady + bestLabel = phase.Precision + found = true + } + } + return best, bestLabel, found +} + +func applyBenchmarkSteadyFallback(gpu *BenchmarkGPUResult) { + if gpu == nil || benchmarkTelemetryAvailable(gpu.Steady) { + return + } + if fallback, label, ok := benchmarkPrecisionSteadyFallback(gpu.PrecisionSteady); ok { + gpu.Steady = fallback + gpu.Notes = append(gpu.Notes, + fmt.Sprintf("mixed steady telemetry unavailable; reporting steady-state fallback from %s precision phase", label)) + } +} + +func scoreBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkScorecard { + score := BenchmarkScorecard{} + + // SyntheticScore: sum of fp32-equivalent TOPS from per-precision phases. + // Each precision ran alone with full GPU dedicated — peak capability. + for _, p := range gpu.PrecisionSteady { + if !benchmarkPrecisionEnabled(p.Precision) { + continue + } + score.SyntheticScore += p.WeightedTeraOpsPerSec + } + + // MixedScore: sum of fp32-equivalent TOPS from the combined phase. + // All precisions compete simultaneously — closer to real inference workloads. + for _, p := range gpu.PrecisionResults { + if p.Supported && benchmarkPrecisionEnabled(p.Category) { + score.MixedScore += p.WeightedTeraOpsPerSec + } + } + + // MixedEfficiency = MixedScore / SyntheticScore. + // Measures how well the GPU sustains throughput under concurrent mixed load. + // A healthy GPU scores ~0.8–0.95; severe degradation suggests bandwidth + // contention or scheduler inefficiency. + if score.SyntheticScore > 0 && score.MixedScore > 0 { + score.MixedEfficiency = score.MixedScore / score.SyntheticScore + } + + // ComputeScore = SyntheticScore × (1 + MixedEfficiency × 0.3). + // SyntheticScore is the primary signal; MixedEfficiency adds up to +30% + // bonus for GPUs that handle mixed-precision concurrency well. + // Falls back to MixedScore alone when per-precision data is absent. + switch { + case score.SyntheticScore > 0: + score.ComputeScore = score.SyntheticScore * (1 + score.MixedEfficiency*0.3) + case score.MixedScore > 0: + score.ComputeScore = score.MixedScore + } + // PowerSustainScore: how stable is GPU power draw during the benchmark? + // High variance means the workload is bursting or the power delivery is + // unstable. Score = max(0, 100 − PowerCVPct × 3). + // At 10% CV → score 70; at 33%+ CV → score 0. + // Uses per-precision windows when available (each runs a single kernel, + // so CV reflects genuine power regulation, not workload switching). + if len(gpu.PrecisionSteady) > 0 { + var sum float64 + var count int + for _, p := range gpu.PrecisionSteady { + if !benchmarkPrecisionEnabled(p.Precision) { + continue + } + sum += clampScore(100 - p.Steady.PowerCVPct*3) + count++ + } + if count > 0 { + score.PowerSustainScore = sum / float64(count) + } + } else if gpu.Steady.PowerCVPct > 0 { + score.PowerSustainScore = clampScore(100 - gpu.Steady.PowerCVPct*3) + } + + // ThermalSustainScore: how stable is GPU temperature during the benchmark? + // High variance means cooling is inconsistent (fan bursts, liquid flow + // instability, or frequent transitions in and out of throttle). + // Score = max(0, 100 − TempCVPct × 3). + if gpu.Steady.TempCVPct > 0 { + score.ThermalSustainScore = clampScore(100 - gpu.Steady.TempCVPct*3) + } else { + // TempCV not recorded — fall back to 100 (no penalty). + score.ThermalSustainScore = 100 + } + + // Throttle breakdown: compute per-type percentages for diagnosis. + // Each counter measures microseconds spent in that throttle state during + // the steady-state window. Counters can overlap (e.g. thermal + power cap + // simultaneously), so they are reported independently, not summed. + runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6) + score.ThermalThrottlePct = math.Min(100, + float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS*100) + score.PowerCapThrottlePct = math.Min(100, + float64(gpu.Throttle.SWPowerCapUS)/runtimeUS*100) + score.SyncBoostThrottlePct = math.Min(100, + float64(gpu.Throttle.SyncBoostUS)/runtimeUS*100) + + // StabilityScore: combined throttle signal (thermal + power cap). + // Score = max(0, 100 − combined_throttle_pct). + // 1% throttle → 99; 10% → 90; any throttle > 0 is penalised. + combinedThrottlePct := math.Min(100, + float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS+gpu.Throttle.SWPowerCapUS)/runtimeUS*100) + score.StabilityScore = clampScore(100 - combinedThrottlePct) + + // TempHeadroomC: distance from p95 temperature to the GPU's hardware + // shutdown threshold (sourced from nvidia-smi -q "GPU Shutdown Temp"). + // Fallback: 90°C when not available. + // Assessed independently of throttle — a GPU at 86°C without any throttle + // counter still has limited headroom and operates in degraded reliability zone. + // Warning zone: headroom < (shutdownTemp - slowdownTemp), i.e. past slowdown onset. + // Critical zone: headroom < 10°C from shutdown. + if gpu.Steady.P95TempC > 0 { + shutdownTemp := gpu.ShutdownTempC + if shutdownTemp <= 0 { + shutdownTemp = 90 + } + score.TempHeadroomC = shutdownTemp - gpu.Steady.P95TempC + } + score.ServerQualityScore = serverQualityScore(score) + score.CompositeScore = score.ComputeScore + if gpu.MultiprocessorCount > 0 && gpu.Steady.AvgGraphicsClockMHz > 0 && score.ComputeScore > 0 { + score.TOPSPerSMPerGHz = score.ComputeScore / float64(gpu.MultiprocessorCount) / (gpu.Steady.AvgGraphicsClockMHz / 1000.0) + } + return score +} + +// compositeBenchmarkScore is kept for compatibility with legacy callers. +// CompositeScore = ComputeScore (no quality multiplier; throttling already +// reduces TOPS directly, so no additional penalty is needed). +func compositeBenchmarkScore(score BenchmarkScorecard) float64 { + return score.ComputeScore +} + +// serverQualityScore returns a 0–100 score reflecting server infrastructure +// quality, independent of GPU model or compute speed. +// +// StabilityScore (throttle time) 0.40 — heaviest: direct evidence GPU can't sustain load +// PowerSustainScore (power CV) 0.30 — unstable draw hints at PSU/VRM issues +// ThermalSustainScore (temp CV) 0.30 — unstable temp hints at airflow/cooling issues +func serverQualityScore(score BenchmarkScorecard) float64 { + q := 0.40*(score.StabilityScore/100.0) + + 0.30*(score.PowerSustainScore/100.0) + + 0.30*(score.ThermalSustainScore/100.0) + return clampScore(q * 100) +} + +// detectPowerAnomaly scans per-GPU steady-state metric rows for a sudden +// power drop — a symptom of bad cable contact, VRM fault, or thermal event +// on the power delivery path. Returns a non-empty string if an anomaly is found. +// +// Algorithm: uses a 5-sample rolling baseline; flags any sample that falls +// more than 30% below the baseline while the GPU was otherwise loaded +// (usage > 50%). A sustained throttle (power cap) is not flagged here — +// that is already captured by PowerCapThrottlePct. +func detectPowerAnomaly(rows []GPUMetricRow, gpuIndex int) string { + const windowSize = 5 + const dropThresholdPct = 30.0 + const minUsagePct = 50.0 + + // Filter rows for this GPU during steady state only. + var steady []GPUMetricRow + for _, r := range rows { + if r.GPUIndex == gpuIndex && r.Stage != "" && strings.Contains(r.Stage, "steady") { + steady = append(steady, r) + } + } + if len(steady) < windowSize+2 { + return "" + } + + // Compute initial baseline from the first window. + var baseSum float64 + for i := 0; i < windowSize; i++ { + baseSum += steady[i].PowerW + } + + for i := windowSize; i < len(steady); i++ { + baseline := baseSum / float64(windowSize) + sample := steady[i] + if baseline > 0 && sample.UsagePct >= minUsagePct { + dropPct := (baseline - sample.PowerW) / baseline * 100 + if dropPct >= dropThresholdPct { + return fmt.Sprintf("sudden power drop detected at t=%.0fs: %.0f W → %.0f W (%.0f%% below rolling baseline) — possible bad cable contact or VRM fault", + sample.ElapsedSec, baseline, sample.PowerW, dropPct) + } + } + // Slide the window baseline. + baseSum -= steady[i-windowSize].PowerW + baseSum += sample.PowerW + } + return "" +} + +// detectSlowdownTempExceedance scans steady-state metric rows for a GPU and +// returns a warning string if any temperature sample exceeded the GPU's +// SlowdownTempC threshold. Uses fallback 80°C when SlowdownTempC is zero. +// This is a real-time signal distinct from p95 stats — even a single spike +// above the slowdown threshold is worth flagging. +func detectSlowdownTempExceedance(rows []GPUMetricRow, gpuIndex int, slowdownTempC float64) string { + if slowdownTempC <= 0 { + slowdownTempC = 80 + } + var maxTemp float64 + var exceedCount int + for _, r := range rows { + if r.GPUIndex != gpuIndex { + continue + } + if !strings.Contains(r.Stage, "steady") { + continue + } + if r.TempC > maxTemp { + maxTemp = r.TempC + } + if r.TempC >= slowdownTempC { + exceedCount++ + } + } + if exceedCount == 0 { + return "" + } + return fmt.Sprintf( + "temperature exceeded slowdown threshold (%.0f°C) in %d sample(s) during steady state — peak %.1f°C", + slowdownTempC, exceedCount, maxTemp) +} + +func detectBenchmarkDegradationReasons(gpu BenchmarkGPUResult, normalizationStatus string) []string { + var reasons []string + runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6) + if float64(gpu.Throttle.SWPowerCapUS)/runtimeUS >= 0.05 { + reasons = append(reasons, "power_capped") + } + if float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS >= 0.01 { + reasons = append(reasons, "thermal_limited") + } + if float64(gpu.Throttle.SyncBoostUS)/runtimeUS >= 0.01 { + reasons = append(reasons, "sync_boost_limited") + } + if gpu.LockedGraphicsClockMHz > 0 && gpu.Steady.AvgGraphicsClockMHz < gpu.LockedGraphicsClockMHz*0.90 { + reasons = append(reasons, "low_sm_clock_vs_target") + } + if gpu.Scores.StabilityScore > 0 && gpu.Scores.StabilityScore < 85 { + reasons = append(reasons, "variance_too_high") + } + if normalizationStatus != "full" { + reasons = append(reasons, "normalization_partial") + } + if gpu.PowerLimitDerated { + reasons = append(reasons, "power_limit_derated") + } + if gpu.ECC.Uncorrected > 0 { + reasons = append(reasons, "ecc_uncorrected_errors") + } + if gpu.ECC.Corrected > 0 { + reasons = append(reasons, "ecc_corrected_errors") + } + return dedupeStrings(reasons) +} diff --git a/audit/internal/platform/benchmark_power_autotune.go b/audit/internal/platform/benchmark_power_autotune.go index 7ab7776..247aacd 100644 --- a/audit/internal/platform/benchmark_power_autotune.go +++ b/audit/internal/platform/benchmark_power_autotune.go @@ -69,16 +69,6 @@ func LoadSystemPowerSourceConfig(exportDir string) (*BenchmarkPowerAutotuneConfi return LoadBenchmarkPowerAutotuneConfig(BenchmarkPowerSourceConfigPath(exportDir)) } -func ResetBenchmarkPowerAutotuneConfig(path string) error { - if strings.TrimSpace(path) == "" { - return fmt.Errorf("empty autotune config path") - } - if err := os.Remove(path); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - func normalizeBenchmarkPowerSource(source string) string { switch strings.TrimSpace(strings.ToLower(source)) { case BenchmarkPowerSourceSDRPSUInput: diff --git a/audit/internal/platform/benchmark_power_calibration.go b/audit/internal/platform/benchmark_power_calibration.go new file mode 100644 index 0000000..8a72501 --- /dev/null +++ b/audit/internal/platform/benchmark_power_calibration.go @@ -0,0 +1,928 @@ +package platform + +import ( + "bee/audit/internal/collector" + "context" + "fmt" + "math" + "os" + "os/exec" + "sort" + "strconv" + "strings" + "time" +) + +func detectDCMIPartialCoverage(sp *BenchmarkServerPower) bool { + if sp == nil || !sp.Available { + return false + } + if sp.PSUInputIdleW > 0 && sp.IdleW > 0 { + return sp.IdleW/sp.PSUInputIdleW < 0.7 + } + if sp.PSUInputLoadedW > 0 && sp.LoadedW > 0 { + return sp.LoadedW/sp.PSUInputLoadedW < 0.7 + } + return false +} + +// detectIPMISaturationFallback is the heuristic used when SDR PSU sensors are +// absent. It analyses the power ramp: if 2+ of the last 3 per-step incremental +// DCMI/GPU ratios fall below 25% of the first-step ratio, DCMI has likely +// plateaued while GPU load continued to grow (saturation proxy). +// Prefer detectDCMIPartialCoverage when SDR data is available. +func detectIPMISaturationFallback(steps []NvidiaPowerBenchStep) bool { + type pt struct{ incIPMI, incGPU float64 } + var pts []pt + for i := 1; i < len(steps); i++ { + if steps[i].ServerDeltaW <= 0 || steps[i-1].ServerDeltaW <= 0 { + continue + } + incIPMI := steps[i].ServerDeltaW - steps[i-1].ServerDeltaW + incGPU := steps[i].TotalObservedPowerW - steps[i-1].TotalObservedPowerW + if incGPU <= 0 { + continue + } + pts = append(pts, pt{incIPMI, incGPU}) + } + if len(pts) < 3 { + return false + } + refRatio := pts[0].incIPMI / pts[0].incGPU + if refRatio <= 0 { + return false + } + saturated := 0 + for _, p := range pts[len(pts)-3:] { + if p.incIPMI/p.incGPU < refRatio*0.25 { + saturated++ + } + } + return saturated >= 2 +} + +// psuStatusSnapshot samples PSU health sensor states via +// `ipmitool sdr type "Power Supply"`. Returns a map of sensor name → reading +// string (e.g. "Presence detected", "Failure detected"). Returns nil when IPMI +// is unavailable or no Power Supply entity sensors are present. +func psuStatusSnapshot() map[string]string { + out, err := exec.Command("ipmitool", "sdr", "type", "Power Supply").Output() + if err != nil || len(out) == 0 { + return nil + } + result := make(map[string]string) + for _, line := range strings.Split(string(out), "\n") { + parts := strings.Split(line, "|") + if len(parts) < 5 { + continue + } + name := strings.TrimSpace(parts[0]) + reading := strings.TrimSpace(parts[4]) + if name == "" { + continue + } + result[name] = reading + } + return result +} + +// diffPSUStatus compares PSU sensor snapshots taken before and after a test. +// Returns human-readable fault strings for sensors that entered a fault state +// during the test. Pre-existing faults (present in both snapshots) are excluded +// so that only new anomalies caused by the test are reported. +func diffPSUStatus(before, after map[string]string) []string { + if len(after) == 0 { + return nil + } + isFault := func(s string) bool { + lower := strings.ToLower(s) + return strings.Contains(lower, "failure") || + strings.Contains(lower, "fault") || + strings.Contains(lower, "warning") || + strings.Contains(lower, "predictive") || + strings.Contains(lower, "absent") || + strings.Contains(lower, "ac lost") + } + var issues []string + for name, afterReading := range after { + if !isFault(afterReading) { + continue + } + if beforeReading, had := before[name]; had && isFault(beforeReading) { + continue // pre-existing fault, not caused by this test + } + if prev, had := before[name]; had { + issues = append(issues, fmt.Sprintf("%s: changed from %q to %q during test", name, prev, afterReading)) + } else { + issues = append(issues, fmt.Sprintf("%s: %s (appeared after test start)", name, afterReading)) + } + } + sort.Strings(issues) + return issues +} + +// sdrPowerSnapshot holds per-source power sums from a single `ipmitool sdr` read. +type sdrPowerSnapshot struct { + PSUInW float64 // sum of PSU AC input across all slots + PSUOutW float64 // sum of PSU DC output across all slots + GPUSlotW float64 // sum of GPU slot/GPU power sensors + + // Per-slot PSU data from collector.PSUSlotsFromSDR — same slot keys as + // audit HardwarePowerSupply.Slot (0-based strings). + PSUSlots map[string]BenchmarkPSUSlotPower + + SkippedSensors []string // sensors rejected during self-healing +} + +type benchmarkSDRSeriesSummary struct { + PSUInW float64 + PSUOutW float64 + GPUSlotW float64 + PSUSlots map[string]BenchmarkPSUSlotPower + Samples int + + SkippedSensors []string +} + +// sdrSensor is a name+watts pair used for GPU slot self-healing filtering. +type sdrSensor struct { + name string + watts float64 +} + +// filterSensorGroup removes physically implausible readings from a group. +// Hard bounds: 0 < watts ≤ maxPerSensorW. Within groups of 2+ sensors, +// values more than 5× the group median are rejected as stuck/fault sensors. +func filterSensorGroup(sensors []sdrSensor, maxPerSensorW float64) (valid []sdrSensor, skipped []string) { + var inBounds []sdrSensor + for _, s := range sensors { + if s.watts <= 0 || s.watts > maxPerSensorW { + skipped = append(skipped, fmt.Sprintf("%s (%.0f W: out of range 0–%.0f W)", s.name, s.watts, maxPerSensorW)) + } else { + inBounds = append(inBounds, s) + } + } + if len(inBounds) < 2 { + return inBounds, skipped + } + vals := make([]float64, len(inBounds)) + for i, s := range inBounds { + vals[i] = s.watts + } + sort.Float64s(vals) + mid := len(vals) / 2 + var median float64 + if len(vals)%2 == 0 { + median = (vals[mid-1] + vals[mid]) / 2 + } else { + median = vals[mid] + } + for _, s := range inBounds { + if median > 0 && s.watts > median*5 { + skipped = append(skipped, fmt.Sprintf("%s (%.0f W: >5× median %.0f W, likely sensor fault)", s.name, s.watts, median)) + } else { + valid = append(valid, s) + } + } + return valid, skipped +} + +// sampleIPMISDRPowerSensors reads power sensors from `ipmitool sdr` in a single +// invocation and returns self-healed grouped sums. +// +// PSU identification delegates to collector.PSUSlotsFromSDR which uses the same +// slot-detection regexes as the hardware audit (PSU1_POWER_IN, PSU1_PIN, PS1 POut, +// Power1…). Self-healing: bounds checking + 5× median outlier rejection. +// +// GPU slot sensors (GPU_POWER_SLOTx, GPU1 Power, …) are classified separately +// since the audit collector does not track GPU PCIe slot power. +func sampleIPMISDRPowerSensors() sdrPowerSnapshot { + raw, err := exec.Command("ipmitool", "sdr").Output() + if err != nil || len(raw) == 0 { + return sdrPowerSnapshot{} + } + sdrStr := string(raw) + var snap sdrPowerSnapshot + + // ── PSU data via audit collector ───────────────────────────────────────── + // collector.PSUSlotsFromSDR handles all vendor naming variants and applies + // bounds checking inside parseBoundedFloat (0 < w ≤ 6000 W). + collectorSlots := collector.PSUSlotsFromSDR(sdrStr) + + // Convert to benchmark type and apply cross-slot median filtering. + var psuInSensors, psuOutSensors []sdrSensor + for slotKey, sp := range collectorSlots { + bsp := BenchmarkPSUSlotPower{Status: sp.Status} + if sp.InputW != nil { + bsp.InputW = sp.InputW + psuInSensors = append(psuInSensors, sdrSensor{name: "PSU-slot-" + slotKey, watts: *sp.InputW}) + } + if sp.OutputW != nil { + bsp.OutputW = sp.OutputW + psuOutSensors = append(psuOutSensors, sdrSensor{name: "PSU-slot-" + slotKey + "-out", watts: *sp.OutputW}) + } + if snap.PSUSlots == nil { + snap.PSUSlots = make(map[string]BenchmarkPSUSlotPower) + } + snap.PSUSlots[slotKey] = bsp + } + + // Apply cross-slot outlier filter and sum. + validIn, skIn := filterSensorGroup(psuInSensors, 6000) + for _, s := range validIn { + snap.PSUInW += s.watts + } + snap.SkippedSensors = append(snap.SkippedSensors, skIn...) + + validOut, skOut := filterSensorGroup(psuOutSensors, 6000) + for _, s := range validOut { + snap.PSUOutW += s.watts + } + snap.SkippedSensors = append(snap.SkippedSensors, skOut...) + + // ── GPU slot sensors ───────────────────────────────────────────────────── + // collector does not track GPU PCIe slot power; classify here. + // Matches: GPU_POWER_SLOTx (MSI), GPU1 Power (xFusion), GPU_PWR_x (generic). + var gpuSensors []sdrSensor + for _, line := range strings.Split(sdrStr, "\n") { + parts := strings.Split(line, "|") + if len(parts) < 2 { + continue + } + name := strings.TrimSpace(parts[0]) + nameLower := strings.ToLower(name) + if !strings.Contains(nameLower, "gpu") { + continue + } + if !strings.Contains(nameLower, "slot") && !strings.Contains(nameLower, "power") && + !strings.Contains(nameLower, "pwr") { + continue + } + var w float64 + if n, _ := fmt.Sscanf(strings.TrimSpace(parts[1]), "%f Watts", &w); n != 1 { + continue + } + gpuSensors = append(gpuSensors, sdrSensor{name: name, watts: w}) + } + validGPU, skGPU := filterSensorGroup(gpuSensors, 2000) + for _, s := range validGPU { + snap.GPUSlotW += s.watts + } + snap.SkippedSensors = append(snap.SkippedSensors, skGPU...) + + return snap +} + +func startIPMISDRSampler(stopCh <-chan struct{}, intervalSec int) <-chan []sdrPowerSnapshot { + if intervalSec <= 0 { + intervalSec = benchmarkPowerAutotuneSampleInterval + } + ch := make(chan []sdrPowerSnapshot, 1) + go func() { + defer close(ch) + var samples []sdrPowerSnapshot + record := func() { + snap := sampleIPMISDRPowerSensors() + if snap.PSUInW <= 0 && snap.PSUOutW <= 0 && snap.GPUSlotW <= 0 && len(snap.PSUSlots) == 0 { + return + } + samples = append(samples, snap) + } + record() + ticker := time.NewTicker(time.Duration(intervalSec) * time.Second) + defer ticker.Stop() + for { + select { + case <-stopCh: + ch <- samples + return + case <-ticker.C: + record() + } + } + }() + return ch +} + +func summarizeSDRPowerSeries(samples []sdrPowerSnapshot) benchmarkSDRSeriesSummary { + var summary benchmarkSDRSeriesSummary + if len(samples) == 0 { + return summary + } + + type slotAggregate struct { + inputs []float64 + outputs []float64 + status string + } + + slotAgg := make(map[string]*slotAggregate) + skippedSet := make(map[string]struct{}) + var inputTotals []float64 + var outputTotals []float64 + var gpuSlotTotals []float64 + + for _, sample := range samples { + if sample.PSUInW > 0 { + inputTotals = append(inputTotals, sample.PSUInW) + } + if sample.PSUOutW > 0 { + outputTotals = append(outputTotals, sample.PSUOutW) + } + if sample.GPUSlotW > 0 { + gpuSlotTotals = append(gpuSlotTotals, sample.GPUSlotW) + } + for _, skipped := range sample.SkippedSensors { + if skipped != "" { + skippedSet[skipped] = struct{}{} + } + } + for slot, reading := range sample.PSUSlots { + agg := slotAgg[slot] + if agg == nil { + agg = &slotAggregate{} + slotAgg[slot] = agg + } + if reading.InputW != nil && *reading.InputW > 0 { + agg.inputs = append(agg.inputs, *reading.InputW) + } + if reading.OutputW != nil && *reading.OutputW > 0 { + agg.outputs = append(agg.outputs, *reading.OutputW) + } + switch { + case reading.Status == "": + case agg.status == "": + agg.status = reading.Status + case agg.status == "OK" && reading.Status != "OK": + agg.status = reading.Status + } + } + } + + summary.PSUInW = benchmarkMean(inputTotals) + summary.PSUOutW = benchmarkMean(outputTotals) + summary.GPUSlotW = benchmarkMean(gpuSlotTotals) + summary.Samples = len(samples) + + if len(slotAgg) > 0 { + summary.PSUSlots = make(map[string]BenchmarkPSUSlotPower, len(slotAgg)) + for slot, agg := range slotAgg { + reading := BenchmarkPSUSlotPower{Status: agg.status} + if mean := benchmarkMean(agg.inputs); mean > 0 { + v := mean + reading.InputW = &v + } + if mean := benchmarkMean(agg.outputs); mean > 0 { + v := mean + reading.OutputW = &v + } + summary.PSUSlots[slot] = reading + } + } + if len(skippedSet) > 0 { + summary.SkippedSensors = make([]string, 0, len(skippedSet)) + for skipped := range skippedSet { + summary.SkippedSensors = append(summary.SkippedSensors, skipped) + } + sort.Strings(summary.SkippedSensors) + } + + return summary +} + +// queryIPMIServerPowerW reads the current server power draw via ipmitool dcmi. +// Returns 0 and an error if IPMI is unavailable or the output cannot be parsed. +func queryIPMIServerPowerW() (float64, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "ipmitool", "dcmi", "power", "reading") + out, err := cmd.Output() + if err != nil { + return 0, fmt.Errorf("ipmitool dcmi power reading: %w", err) + } + if w := parseDCMIPowerReading(string(out)); w > 0 { + return w, nil + } + return 0, fmt.Errorf("could not parse ipmitool dcmi power reading output") +} + +// sampleIPMIPowerSeries collects IPMI power readings every 2 seconds for +// durationSec seconds. Returns the mean of all successful samples. +// Returns 0, false if IPMI is unavailable. +func sampleIPMIPowerSeries(ctx context.Context, durationSec int) (meanW float64, ok bool) { + if durationSec <= 0 { + return 0, false + } + deadline := time.Now().Add(time.Duration(durationSec) * time.Second) + var samples []float64 +loop: + for { + if w, err := queryIPMIServerPowerW(); err == nil { + samples = append(samples, w) + } + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + break loop + case <-time.After(2 * time.Second): + } + } + if len(samples) == 0 { + return 0, false + } + var sum float64 + for _, w := range samples { + sum += w + } + return sum / float64(len(samples)), true +} + +// characterizeServerPower computes BenchmarkServerPower from idle and loaded +// samples plus the GPU-reported average power during steady state. +func characterizeServerPower(idleW, loadedW, gpuReportedSumW float64, source string, available bool) *BenchmarkServerPower { + sp := &BenchmarkServerPower{ + Available: available, + Source: normalizeBenchmarkPowerSource(source), + SampleIntervalSec: benchmarkPowerAutotuneSampleInterval, + } + if !available { + sp.Notes = append(sp.Notes, "IPMI power reading unavailable; server-side power characterization skipped") + return sp + } + sp.IdleW = idleW + sp.LoadedW = loadedW + sp.DeltaW = loadedW - idleW + sp.GPUReportedSumW = gpuReportedSumW + if gpuReportedSumW > 0 && sp.DeltaW > 0 { + sp.ReportingRatio = sp.DeltaW / gpuReportedSumW + } + return sp +} + +// readServerModel returns the DMI system product name (e.g. "SuperMicro SYS-421GE-TNRT"). +// Returns empty string if unavailable (non-Linux or missing DMI entry). +func readServerModel() string { + data, err := os.ReadFile("/sys/class/dmi/id/product_name") + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + +// filterRowsByGPU returns only the metric rows for a specific GPU index. +func filterRowsByGPU(rows []GPUMetricRow, gpuIndex int) []GPUMetricRow { + var out []GPUMetricRow + for _, r := range rows { + if r.GPUIndex == gpuIndex { + out = append(out, r) + } + } + return out +} + +// parseBenchmarkBurnLogByGPU splits a multi-GPU bee-gpu-burn output by [gpu N] prefix +// and returns a per-GPU parse result map. +func parseBenchmarkBurnLogByGPU(raw string) map[int]benchmarkBurnParseResult { + gpuLines := make(map[int][]string) + for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "[gpu ") { + continue + } + end := strings.Index(line, "] ") + if end < 0 { + continue + } + gpuIdx, err := strconv.Atoi(strings.TrimSpace(line[5:end])) + if err != nil { + continue + } + gpuLines[gpuIdx] = append(gpuLines[gpuIdx], line[end+2:]) + } + results := make(map[int]benchmarkBurnParseResult, len(gpuLines)) + for gpuIdx, lines := range gpuLines { + // Lines are already stripped of the [gpu N] prefix; parseBenchmarkBurnLog + // calls stripBenchmarkPrefix which is a no-op on already-stripped lines. + results[gpuIdx] = parseBenchmarkBurnLog(strings.Join(lines, "\n")) + } + return results +} + +// runNvidiaBenchmarkParallel runs warmup and steady compute on all selected GPUs +// simultaneously using a single bee-gpu-burn invocation per phase. +func runNvidiaBenchmarkParallel( + ctx context.Context, + verboseLog, runDir string, + selected []int, + infoByIndex map[int]benchmarkGPUInfo, + opts NvidiaBenchmarkOptions, + spec benchmarkProfileSpec, + logFunc func(string), + result *NvidiaBenchmarkResult, + calibByIndex map[int]benchmarkPowerCalibrationResult, + serverIdleW *float64, serverLoadedWSum *float64, + serverIdleOK *bool, serverLoadedOK *bool, serverLoadedSamples *int, + allMetricRows *[]GPUMetricRow, + metricTimelineSec *float64, + gpuBurnLog string, +) { + allDevices := joinIndexList(selected) + + // Build per-GPU result stubs. + gpuResults := make(map[int]*BenchmarkGPUResult, len(selected)) + for _, idx := range selected { + r := &BenchmarkGPUResult{Index: idx, Status: "FAILED"} + if info, ok := infoByIndex[idx]; ok { + r.UUID = info.UUID + r.Name = info.Name + r.BusID = info.BusID + r.VBIOS = info.VBIOS + r.PowerLimitW = info.PowerLimitW + r.MultiprocessorCount = info.MultiprocessorCount + r.DefaultPowerLimitW = info.DefaultPowerLimitW + r.ShutdownTempC = info.ShutdownTempC + r.SlowdownTempC = info.SlowdownTempC + r.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz + r.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz + r.MaxMemoryClockMHz = info.MaxMemoryClockMHz + } + if calib, ok := calibByIndex[idx]; ok { + r.CalibratedPeakPowerW = calib.Summary.P95PowerW + r.CalibratedPeakTempC = calib.Summary.P95TempC + r.PowerCalibrationTries = calib.Attempts + r.PowerLimitDerated = calib.Derated + r.Notes = append(r.Notes, calib.Notes...) + if calib.CoolingWarning != "" { + r.CoolingWarning = calib.CoolingWarning + } + } + if norm := findBenchmarkNormalization(result.Normalization.GPUs, idx); norm != nil { + r.LockedGraphicsClockMHz = norm.GPUClockLockMHz + r.LockedMemoryClockMHz = norm.MemoryClockLockMHz + } + gpuResults[idx] = r + } + + // Baseline: sample all GPUs together. + baselineRows, err := collectBenchmarkSamples(ctx, spec.BaselineSec, selected) + if err != nil && err != context.Canceled { + for _, idx := range selected { + gpuResults[idx].Notes = append(gpuResults[idx].Notes, "baseline sampling failed: "+err.Error()) + } + } + for _, idx := range selected { + perGPU := filterRowsByGPU(baselineRows, idx) + gpuResults[idx].Baseline = summarizeBenchmarkTelemetry(perGPU) + } + appendBenchmarkMetrics(allMetricRows, baselineRows, "baseline", metricTimelineSec, float64(spec.BaselineSec)) + + // Sample server idle power once. + if !*serverIdleOK { + if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, maxInt(spec.BaselineSec, 10), benchmarkPowerAutotuneSampleInterval); ok { + *serverIdleW = w + *serverIdleOK = true + logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w)) + } + } + + // Warmup: all GPUs simultaneously. + warmupCmd := []string{ + "bee-gpu-burn", + "--seconds", strconv.Itoa(spec.WarmupSec), + "--size-mb", strconv.Itoa(opts.SizeMB), + "--devices", allDevices, + } + logFunc(fmt.Sprintf("GPUs %s: parallel warmup (%ds)", allDevices, spec.WarmupSec)) + warmupOut, warmupRows, warmupErr := runBenchmarkCommandWithMetrics(ctx, verboseLog, "gpu-all-warmup.log", warmupCmd, nil, selected, logFunc) + appendBenchmarkMetrics(allMetricRows, warmupRows, "warmup", metricTimelineSec, float64(spec.WarmupSec)) + appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", "warmup", warmupOut) + if warmupErr != nil { + for _, idx := range selected { + gpuResults[idx].Notes = append(gpuResults[idx].Notes, "parallel warmup failed: "+warmupErr.Error()) + } + } + warmupParseByGPU := parseBenchmarkBurnLogByGPU(string(warmupOut)) + supportedPrecisions := append([]string(nil), benchmarkPrecisionPhases...) + for _, idx := range selected { + if pr, ok := warmupParseByGPU[idx]; ok && pr.ComputeCapability != "" { + if gpuResults[idx].ComputeCapability == "" { + gpuResults[idx].ComputeCapability = pr.ComputeCapability + } + if ccPrecisions := benchmarkSupportedPrecisions(pr.ComputeCapability); len(ccPrecisions) < len(supportedPrecisions) { + supportedPrecisions = ccPrecisions + } + } + } + + // Run synthetic precision phases and the combined steady phase as one + // uninterrupted command so the GPUs stay hot between windows. + eccBase := make(map[int]BenchmarkECCCounters, len(selected)) + for _, idx := range selected { + eccBase[idx], _ = queryECCCounters(idx) + } + planLabels, planPhases, basePhaseSec, mixedPhaseSec := buildBenchmarkSteadyPlan(spec, supportedPrecisions, func(label string) string { + if label == "mixed" { + return "steady" + } + return "gpu-all-steady-" + label + }) + planCmd := []string{ + "bee-gpu-burn", + "--seconds", strconv.Itoa(basePhaseSec), + "--size-mb", strconv.Itoa(opts.SizeMB), + "--devices", allDevices, + "--precision-plan", strings.Join(planLabels, ","), + "--precision-plan-seconds", benchmarkPlanDurationsCSV(planPhases), + } + logFunc(fmt.Sprintf("GPUs %s: uninterrupted precision plan (%d precision phases x %ds, mixed %ds)", allDevices, len(supportedPrecisions), basePhaseSec, mixedPhaseSec)) + serverPowerStopCh := make(chan struct{}) + serverPowerCh := startSelectedPowerSourceSampler(serverPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval) + _, phaseRowsByStage, phaseLogs, planErr := runBenchmarkPlannedCommandWithMetrics(ctx, verboseLog, "gpu-all-precision-plan.log", planCmd, nil, selected, planPhases, logFunc) + close(serverPowerStopCh) + if serverPowerSamples := <-serverPowerCh; len(serverPowerSamples) > 0 { + *serverLoadedWSum += benchmarkMean(serverPowerSamples) + (*serverLoadedSamples)++ + *serverLoadedOK = true + logFunc(fmt.Sprintf("GPUs %s: server loaded power (%s avg): %.0f W", allDevices, opts.ServerPowerSource, benchmarkMean(serverPowerSamples))) + } + for _, phaseSpec := range planPhases { + if rows := phaseRowsByStage[phaseSpec.MetricStage]; len(rows) > 0 { + appendBenchmarkMetrics(allMetricRows, rows, phaseSpec.MetricStage, metricTimelineSec, float64(phaseSpec.DurationSec)) + } + appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", phaseSpec.MetricStage, phaseLogs[phaseSpec.PlanLabel]) + } + for _, prec := range supportedPrecisions { + phaseLogName := "gpu-all-steady-" + prec + phaseRows := phaseRowsByStage[phaseLogName] + parseByGPU := parseBenchmarkBurnLogByGPU(string(phaseLogs[prec])) + for _, idx := range selected { + perGPU := filterRowsByGPU(phaseRows, idx) + phase := BenchmarkPrecisionSteadyPhase{ + Precision: prec, + Status: "OK", + Steady: summarizeBenchmarkTelemetry(perGPU), + } + if status, note := benchmarkPlannedPhaseStatus(phaseLogs[prec]); status != "OK" { + phase.Status = status + phase.Notes = note + gpuResults[idx].PrecisionFailures = append(gpuResults[idx].PrecisionFailures, prec+":"+status) + } + if pr, ok := parseByGPU[idx]; ok { + for _, p := range pr.Profiles { + if p.Supported { + phase.TeraOpsPerSec += p.TeraOpsPerSec + phase.WeightedTeraOpsPerSec += p.WeightedTeraOpsPerSec + } + } + } + gpuResults[idx].PrecisionSteady = append(gpuResults[idx].PrecisionSteady, phase) + } + } + + // Snapshot throttle counters before steady. + beforeThrottle := make(map[int]BenchmarkThrottleCounters, len(selected)) + for _, idx := range selected { + beforeThrottle[idx], _ = queryThrottleCounters(idx) + } + + logFunc(fmt.Sprintf("GPUs %s: parallel steady compute (combined, %ds)", allDevices, mixedPhaseSec)) + afterThrottle := make(map[int]BenchmarkThrottleCounters, len(selected)) + for _, idx := range selected { + afterThrottle[idx], _ = queryThrottleCounters(idx) + } + + steadyRows := phaseRowsByStage["steady"] + parseResults := parseBenchmarkBurnLogByGPU(string(phaseLogs["mixed"])) + + for _, idx := range selected { + perGPU := filterRowsByGPU(steadyRows, idx) + gpuResults[idx].Steady = summarizeBenchmarkTelemetry(perGPU) + gpuResults[idx].Throttle = diffThrottleCounters(beforeThrottle[idx], afterThrottle[idx]) + if eccFinal, err := queryECCCounters(idx); err == nil { + gpuResults[idx].ECC = diffECCCounters(eccBase[idx], eccFinal) + } + + if pr, ok := parseResults[idx]; ok { + gpuResults[idx].ComputeCapability = pr.ComputeCapability + gpuResults[idx].Backend = pr.Backend + gpuResults[idx].PrecisionResults = pr.Profiles + if pr.Fallback { + gpuResults[idx].Notes = append(gpuResults[idx].Notes, "benchmark used driver PTX fallback; tensor throughput score is not comparable") + } + } + if planErr != nil { + gpuResults[idx].Notes = append(gpuResults[idx].Notes, "precision plan failed: "+planErr.Error()) + } + } + + // Cooldown: all GPUs together. + if spec.CooldownSec > 0 { + cooldownRows, err := collectBenchmarkSamples(ctx, spec.CooldownSec, selected) + if err != nil && err != context.Canceled { + for _, idx := range selected { + gpuResults[idx].Notes = append(gpuResults[idx].Notes, "cooldown sampling failed: "+err.Error()) + } + } + for _, idx := range selected { + perGPU := filterRowsByGPU(cooldownRows, idx) + gpuResults[idx].Cooldown = summarizeBenchmarkTelemetry(perGPU) + } + appendBenchmarkMetrics(allMetricRows, cooldownRows, "cooldown", metricTimelineSec, float64(spec.CooldownSec)) + } + + // Score and finalize each GPU. + for _, idx := range selected { + r := gpuResults[idx] + applyBenchmarkSteadyFallback(r) + r.Scores = scoreBenchmarkGPUResult(*r) + r.DegradationReasons = detectBenchmarkDegradationReasons(*r, result.Normalization.Status) + pr := parseResults[idx] + switch { + case planErr != nil: + r.Status = classifySATErrorStatus(phaseLogs["mixed"], planErr) + case len(r.PrecisionFailures) > 0: + r.Status = "PARTIAL" + case pr.Fallback: + r.Status = "PARTIAL" + default: + r.Status = "OK" + } + result.GPUs = append(result.GPUs, finalizeBenchmarkGPUResult(*r)) + } +} + +// readBenchmarkHostConfig reads static CPU and memory configuration from +// /proc/cpuinfo and /proc/meminfo. Returns nil if neither source is readable. +func readBenchmarkHostConfig() *BenchmarkHostConfig { + cfg := &BenchmarkHostConfig{} + populated := false + + // Parse /proc/cpuinfo for CPU model, sockets, cores, threads. + if data, err := os.ReadFile("/proc/cpuinfo"); err == nil { + socketIDs := map[string]struct{}{} + coresPerSocket := map[string]int{} + var modelName string + threads := 0 + for _, line := range strings.Split(string(data), "\n") { + kv := strings.SplitN(line, ":", 2) + if len(kv) != 2 { + continue + } + key := strings.TrimSpace(kv[0]) + val := strings.TrimSpace(kv[1]) + switch key { + case "processor": + threads++ + case "model name": + if modelName == "" { + modelName = val + } + case "physical id": + socketIDs[val] = struct{}{} + case "cpu cores": + // Overwrite per-socket core count (last wins per socket, but all + // entries for the same socket report the same value). + if physLine := ""; physLine == "" { + // We accumulate below by treating cpu cores as a per-thread + // field; sum by socket requires a two-pass approach. Use the + // simpler approximation: totalCores = threads / (threads per core). + _ = val + } + } + } + // Second pass: per-socket core count. + var curSocket string + for _, line := range strings.Split(string(data), "\n") { + kv := strings.SplitN(line, ":", 2) + if len(kv) != 2 { + continue + } + key := strings.TrimSpace(kv[0]) + val := strings.TrimSpace(kv[1]) + switch key { + case "physical id": + curSocket = val + case "cpu cores": + if curSocket != "" { + if _, seen := coresPerSocket[curSocket]; !seen { + v, _ := strconv.Atoi(val) + coresPerSocket[curSocket] = v + } + } + } + } + totalCores := 0 + for _, c := range coresPerSocket { + totalCores += c + } + cfg.CPUModel = modelName + cfg.CPUSockets = len(socketIDs) + if cfg.CPUSockets == 0 && threads > 0 { + cfg.CPUSockets = 1 + } + cfg.CPUCores = totalCores + cfg.CPUThreads = threads + if modelName != "" || threads > 0 { + populated = true + } + } + + // Parse /proc/meminfo for total physical RAM. + if data, err := os.ReadFile("/proc/meminfo"); err == nil { + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "MemTotal:") { + fields := strings.Fields(line) + if len(fields) >= 2 { + kb, _ := strconv.ParseUint(fields[1], 10, 64) + cfg.MemTotalGiB = float64(kb) / (1024 * 1024) + populated = true + } + break + } + } + } + + if !populated { + return nil + } + return cfg +} + +// startCPULoadSampler starts a goroutine that samples host CPU load every +// intervalSec seconds until stopCh is closed, then sends the collected +// samples on the returned channel. +func startCPULoadSampler(stopCh <-chan struct{}, intervalSec int) <-chan []float64 { + ch := make(chan []float64, 1) + go func() { + var samples []float64 + ticker := time.NewTicker(time.Duration(intervalSec) * time.Second) + defer ticker.Stop() + for { + select { + case <-stopCh: + ch <- samples + return + case <-ticker.C: + if pct := sampleCPULoadPct(); pct > 0 { + samples = append(samples, pct) + } + } + } + }() + return ch +} + +// summarizeCPULoad computes stats over sampled CPU load values and assigns +// a health status. +func summarizeCPULoad(samples []float64) *BenchmarkCPULoad { + if len(samples) == 0 { + return nil + } + sorted := append([]float64(nil), samples...) + sort.Float64s(sorted) + var sum float64 + for _, v := range sorted { + sum += v + } + avg := sum / float64(len(sorted)) + p95 := sorted[int(float64(len(sorted))*0.95)] + max := sorted[len(sorted)-1] + + cl := &BenchmarkCPULoad{ + AvgPct: math.Round(avg*10) / 10, + MaxPct: math.Round(max*10) / 10, + P95Pct: math.Round(p95*10) / 10, + Samples: len(sorted), + } + + // Compute standard deviation to detect instability. + var variance float64 + for _, v := range sorted { + d := v - avg + variance += d * d + } + stdDev := math.Sqrt(variance / float64(len(sorted))) + + switch { + case avg > 20 || max > 40: + cl.Status = "high" + cl.Note = fmt.Sprintf("avg %.1f%% max %.1f%% — elevated host CPU load may interfere with GPU benchmark results", avg, max) + case stdDev > 12: + cl.Status = "unstable" + cl.Note = fmt.Sprintf("avg %.1f%% stddev %.1f%% — host CPU load was erratic during the benchmark", avg, stdDev) + default: + cl.Status = "ok" + } + return cl +} + +// runBenchmarkPowerCalibration runs the configured power-fit load for the supplied +// GPU set and actively watches throttle counters. seedLimits, when provided, are treated as +// the starting point for this calibration pass rather than as immutable fixed +// limits. This matters during cumulative ramp-up: once an additional GPU is +// introduced, every already-active GPU must be revalidated under the new +// thermal state instead of assuming its previous single-step limit is still +// valid. The selected reduced power limits stay active for the main benchmark +// and are restored by the caller afterwards. diff --git a/audit/internal/platform/benchmark_power_calibration_run.go b/audit/internal/platform/benchmark_power_calibration_run.go new file mode 100644 index 0000000..450daa1 --- /dev/null +++ b/audit/internal/platform/benchmark_power_calibration_run.go @@ -0,0 +1,558 @@ +package platform + +import ( + "context" + "errors" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +func runBenchmarkPowerCalibration( + ctx context.Context, + verboseLog, runDir string, + gpuIndices []int, + infoByIndex map[int]benchmarkGPUInfo, + logFunc func(string), + seedLimits map[int]int, + durationSec int, +) (map[int]benchmarkPowerCalibrationResult, []benchmarkRestoreAction, []GPUMetricRow, benchmarkPowerCalibrationRunSummary) { + calibDurationSec := durationSec + var runSummary benchmarkPowerCalibrationRunSummary + if calibDurationSec <= 0 { + calibDurationSec = 120 + } + // calibSearchTolerance is the binary-search convergence threshold in watts. + // When hi-lo ≤ this, the highest verified-stable limit (lo) is used. + const calibSearchTolerance = 10 + // dcgmResourceBusyMaxDelaySec caps the exponential back-off when DCGM + // returns DCGM_ST_IN_USE (exit 222). The sequence is 1 s, 2 s, 4 s, … + // doubling each retry until it would exceed the cap, at which point the + // next busy response fails the calibration immediately. + const dcgmResourceBusyMaxDelaySec = 300 + engine := benchmarkPowerEngine() + engineLabel := benchmarkPowerEngineLabel(engine) + + if engine == BenchmarkPowerEngineTargetedPower { + if _, err := exec.LookPath("dcgmi"); err != nil { + logFunc("power calibration: dcgmi not found, skipping (will use default power limit)") + return map[int]benchmarkPowerCalibrationResult{}, nil, nil, runSummary + } + } else { + if _, _, err := resolveBenchmarkPowerLoadCommand(calibDurationSec, gpuIndices); err != nil { + logFunc("power calibration: dcgmproftester not found, skipping (will use default power limit)") + return map[int]benchmarkPowerCalibrationResult{}, nil, nil, runSummary + } + } + if killed := KillTestWorkers(); len(killed) > 0 { + for _, p := range killed { + logFunc(fmt.Sprintf("power calibration pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) + } + } + + canDerate := os.Geteuid() == 0 + if !canDerate { + logFunc("power calibration: root privileges unavailable, adaptive power-limit derating disabled") + } + + type calibrationAttemptResult struct { + out []byte + rows []GPUMetricRow + err error + } + + // gpuCalibState holds per-GPU binary search state during parallel calibration. + type gpuCalibState struct { + idx int + info benchmarkGPUInfo + originalLimitW int + appliedLimitW int + minLimitW int + lo int // highest verified-stable limit + hi int // lowest verified-unstable limit (exclusive sentinel above start) + loVerified bool + calib benchmarkPowerCalibrationResult + converged bool + } + + results := make(map[int]benchmarkPowerCalibrationResult, len(gpuIndices)) + var restore []benchmarkRestoreAction + var allCalibRows []GPUMetricRow // accumulated telemetry across all attempts + var calibCursor float64 + + // Initialise per-GPU state. + states := make([]*gpuCalibState, 0, len(gpuIndices)) + for _, idx := range gpuIndices { + info := infoByIndex[idx] + originalLimitW := int(math.Round(info.PowerLimitW)) + if originalLimitW <= 0 { + originalLimitW = int(math.Round(info.DefaultPowerLimitW)) + } + defaultLimitW := int(math.Round(info.DefaultPowerLimitW)) + if defaultLimitW <= 0 { + defaultLimitW = originalLimitW + } + appliedLimitW := initialBenchmarkCalibrationLimitW(info) + if appliedLimitW <= 0 { + appliedLimitW = defaultLimitW + } + minLimitW := int(math.Round(info.MinPowerLimitW)) + if minLimitW <= 0 { + minLimitW = appliedLimitW + } + maxLimitW := int(math.Round(info.MaxPowerLimitW)) + if maxLimitW > 0 && appliedLimitW > maxLimitW { + appliedLimitW = maxLimitW + } + s := &gpuCalibState{ + idx: idx, + info: info, + originalLimitW: originalLimitW, + appliedLimitW: appliedLimitW, + minLimitW: minLimitW, + lo: minLimitW, + hi: appliedLimitW + 1, // not yet tested, not yet confirmed unstable + calib: benchmarkPowerCalibrationResult{AppliedPowerLimitW: float64(appliedLimitW)}, + } + if minLimitW > 0 && appliedLimitW > 0 && minLimitW >= appliedLimitW { + s.appliedLimitW = minLimitW + s.hi = minLimitW + 1 + } + if info.MinPowerLimitW <= 0 { + s.calib.Notes = append(s.calib.Notes, "minimum power limit was not reported by nvidia-smi; calibration can only validate the current/default power limit") + } + if seedLimits != nil { + if seedW, ok := seedLimits[idx]; ok && seedW > 0 { + // A previously validated limit is only a starting point. Re-run + // targeted_power under the current multi-GPU thermal load and derate + // again if this step shows new throttling. + if seedW < s.minLimitW { + seedW = s.minLimitW + } + if maxLimitW > 0 && seedW > maxLimitW { + seedW = maxLimitW + } + if canDerate { + _ = setBenchmarkPowerLimit(ctx, verboseLog, idx, seedW) + } + s.appliedLimitW = seedW + s.hi = seedW + 1 + s.calib.AppliedPowerLimitW = float64(seedW) + s.calib.Derated = seedW < s.originalLimitW + s.calib.Notes = append(s.calib.Notes, + fmt.Sprintf("seed limit: %d W (revalidating under current thermal load)", seedW)) + } + } + states = append(states, s) + if canDerate && originalLimitW > 0 { + idxCopy := idx + orig := originalLimitW + restore = append(restore, benchmarkRestoreAction{ + name: fmt.Sprintf("gpu-%d-restore-power-limit", idxCopy), + fn: func() { + _ = setBenchmarkPowerLimit(context.Background(), verboseLog, idxCopy, orig) + }, + }) + } + } + + // Shared DCGM resource-busy back-off state (single diagnostic session). + busyRetries := 0 + busyDelaySec := 1 + sharedAttempt := 0 + + type sharedAttemptResult struct { + out []byte + rows []GPUMetricRow + err error + } + +calibDone: + for { + // Collect non-converged GPUs. + var active []*gpuCalibState + for _, s := range states { + if !s.converged { + active = append(active, s) + } + } + if len(active) == 0 || ctx.Err() != nil { + break + } + + sharedAttempt++ + for _, s := range active { + s.calib.Attempts++ + logFunc(fmt.Sprintf("power calibration: GPU %d %s attempt %d at %d W for %ds", s.idx, engineLabel, s.calib.Attempts, s.appliedLimitW, calibDurationSec)) + } + + // Snapshot throttle counters for all active GPUs before the run. + beforeThrottle := make(map[int]BenchmarkThrottleCounters, len(active)) + for _, s := range active { + beforeThrottle[s.idx], _ = queryThrottleCounters(s.idx) + } + + // Run the selected power-fit load for ALL gpuIndices simultaneously so every card + // is under load during calibration — this reflects real server thermals. + logName := fmt.Sprintf("power-calibration-attempt-%d.log", sharedAttempt) + cmd, env, err := resolveBenchmarkPowerLoadCommand(calibDurationSec, gpuIndices) + if err != nil { + for _, s := range active { + s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("failed to resolve %s command: %v", engineLabel, err)) + s.converged = true + } + logFunc(fmt.Sprintf("power calibration: failed to resolve %s command: %v", engineLabel, err)) + break calibDone + } + attemptCtx, cancelAttempt := context.WithCancel(ctx) + doneCh := make(chan sharedAttemptResult, 1) + sdrStopCh := make(chan struct{}) + sdrDoneCh := startIPMISDRSampler(sdrStopCh, benchmarkPowerAutotuneSampleInterval) + fanStopCh := make(chan struct{}) + fanDoneCh := startBenchmarkFanSampler(fanStopCh, benchmarkPowerAutotuneSampleInterval) + go func() { + out, rows, err := runBenchmarkCommandWithMetrics(attemptCtx, verboseLog, logName, cmd, env, gpuIndices, logFunc) + doneCh <- sharedAttemptResult{out: out, rows: rows, err: err} + }() + + ticker := time.NewTicker(time.Second) + throttleReasons := make(map[int]string, len(active)) + var ar sharedAttemptResult + + attemptLoop: + for { + select { + case ar = <-doneCh: + break attemptLoop + case <-ticker.C: + // Poll throttle counters for each active GPU independently. + for _, s := range active { + if throttleReasons[s.idx] != "" { + continue // already detected for this GPU + } + after, err := queryThrottleCounters(s.idx) + if err != nil { + continue + } + // Record throttle but do NOT cancel — let the load command finish so + // runtime resources release cleanly before the next attempt. + if reason := benchmarkCalibrationThrottleReason(beforeThrottle[s.idx], after); reason != "" { + throttleReasons[s.idx] = reason + logFunc(fmt.Sprintf("power calibration: GPU %d detected %s throttle at %d W, waiting for run to finish", s.idx, reason, s.appliedLimitW)) + } + } + case <-ctx.Done(): + cancelAttempt() + ar = <-doneCh + break attemptLoop + } + } + ticker.Stop() + cancelAttempt() + close(sdrStopCh) + close(fanStopCh) + attemptSDRSummary := summarizeSDRPowerSeries(<-sdrDoneCh) + attemptFanSummary := <-fanDoneCh + _ = os.WriteFile(filepath.Join(runDir, logName), ar.out, 0644) + // Accumulate telemetry rows with attempt stage label. + appendBenchmarkMetrics(&allCalibRows, ar.rows, fmt.Sprintf("attempt-%d", sharedAttempt), &calibCursor, float64(calibDurationSec)) + + // Resource busy: retry with exponential back-off (shared — one DCGM session). + if ar.err != nil && isDCGMResourceBusy(ar.err) { + if busyDelaySec > dcgmResourceBusyMaxDelaySec { + for _, s := range active { + s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("DCGM resource busy after %d retries, giving up", busyRetries)) + s.converged = true + } + logFunc(fmt.Sprintf("power calibration: DCGM resource persistently busy after %d retries, stopping", busyRetries)) + break calibDone + } + busyRetries++ + // Undo attempt counter: busy retries don't count as real attempts. + for _, s := range active { + s.calib.Attempts-- + } + logFunc(fmt.Sprintf("power calibration: DCGM resource busy (attempt %d), retrying in %ds", sharedAttempt, busyDelaySec)) + select { + case <-ctx.Done(): + break calibDone + case <-time.After(time.Duration(busyDelaySec) * time.Second): + } + next := busyDelaySec * 2 + if next > dcgmResourceBusyMaxDelaySec { + next = dcgmResourceBusyMaxDelaySec + 1 + } + busyDelaySec = next + sharedAttempt-- // retry same logical attempt number + continue + } + busyRetries = 0 + busyDelaySec = 1 + + // Per-GPU analysis and binary search update. + attemptStable := ar.err == nil + for _, s := range active { + perGPU := filterRowsByGPU(ar.rows, s.idx) + summary := summarizeBenchmarkTelemetry(perGPU) + throttle := throttleReasons[s.idx] + if throttle != "" || summary.P95PowerW <= 0 { + attemptStable = false + } + + // Cooling warning: thermal throttle with fans not at maximum. + if strings.Contains(throttle, "thermal") && s.calib.CoolingWarning == "" { + clocks := make([]float64, 0, len(perGPU)) + var fanDutyValues []float64 + fanDutyAvail := false + for _, r := range perGPU { + if r.ClockMHz > 0 { + clocks = append(clocks, r.ClockMHz) + } + if r.FanDutyCycleAvailable { + fanDutyAvail = true + fanDutyValues = append(fanDutyValues, r.FanDutyCyclePct) + } + } + dropPct := benchmarkClockDrift(clocks) + p95FanDuty := benchmarkPercentile(fanDutyValues, 95) + if dropPct >= 20 && fanDutyAvail && p95FanDuty < 98 { + s.calib.CoolingWarning = fmt.Sprintf( + "thermal throttle (%s) caused a %.0f%% clock drop while fans were at %.0f%% duty cycle — server cooling may not be configured for full GPU load", + throttle, dropPct, p95FanDuty, + ) + logFunc(fmt.Sprintf("power calibration: GPU %d cooling warning: %s", s.idx, s.calib.CoolingWarning)) + } + } + + if throttle == "" && ar.err == nil && summary.P95PowerW > 0 { + // Stable at current limit — update lo and binary-search upward. + s.calib.Summary = summary + s.calib.Completed = true + s.calib.AppliedPowerLimitW = float64(s.appliedLimitW) + logFunc(fmt.Sprintf("power calibration: GPU %d stable at %d W, p95=%.0f W p95_temp=%.1f C (%d samples)", s.idx, s.appliedLimitW, summary.P95PowerW, summary.P95TempC, summary.Samples)) + s.lo = s.appliedLimitW + s.loVerified = true + if canDerate && s.hi-s.lo > calibSearchTolerance { + next := roundTo5W((s.lo + s.hi) / 2) + if next > s.lo && next < s.hi { + if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, next); err == nil { + s.appliedLimitW = next + s.calib.AppliedPowerLimitW = float64(next) + s.calib.Completed = false // keep searching + s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search: stable at %d W, trying %d W (lo=%d hi=%d)", s.lo, next, s.lo, s.hi)) + logFunc(fmt.Sprintf("power calibration: GPU %d binary search up: stable at %d W, trying %d W", s.idx, s.lo, next)) + continue // next GPU in active list + } + } + } + s.calib.MetricRows = filterRowsByGPU(ar.rows, s.idx) + s.converged = true + continue + } + + // Failed or throttled — log and binary-search downward. + switch { + case throttle != "": + s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("targeted_power attempt %d: %s throttle at %d W", s.calib.Attempts, throttle, s.appliedLimitW)) + logFunc(fmt.Sprintf("power calibration: GPU %d throttled (%s) at %d W, reducing power limit", s.idx, throttle, s.appliedLimitW)) + case ar.err != nil: + s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("targeted_power attempt %d failed at %d W: %v", s.calib.Attempts, s.appliedLimitW, ar.err)) + logFunc(fmt.Sprintf("power calibration: GPU %d %s failed at %d W: %v", s.idx, engineLabel, s.appliedLimitW, ar.err)) + default: + s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("%s attempt %d at %d W: no valid power telemetry", engineLabel, s.calib.Attempts, s.appliedLimitW)) + logFunc(fmt.Sprintf("power calibration: GPU %d attempt %d at %d W: no valid telemetry", s.idx, s.calib.Attempts, s.appliedLimitW)) + } + + if !canDerate || s.appliedLimitW <= 0 { + s.converged = true + continue + } + s.hi = s.appliedLimitW + + if s.hi-s.lo <= calibSearchTolerance { + if !s.loVerified && s.minLimitW > 0 && s.appliedLimitW != s.minLimitW { + if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, s.minLimitW); err != nil { + s.calib.Notes = append(s.calib.Notes, "failed to set power limit: "+err.Error()) + logFunc(fmt.Sprintf("power calibration: GPU %d failed to set minimum power limit %d W: %v", s.idx, s.minLimitW, err)) + s.converged = true + continue + } + s.appliedLimitW = s.minLimitW + s.calib.AppliedPowerLimitW = float64(s.minLimitW) + s.calib.Derated = s.minLimitW < s.originalLimitW + s.info.PowerLimitW = float64(s.minLimitW) + infoByIndex[s.idx] = s.info + s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search: validating minimum settable limit %d W before concluding failure", s.minLimitW)) + logFunc(fmt.Sprintf("power calibration: GPU %d binary search: validating minimum settable limit %d W", s.idx, s.minLimitW)) + continue + } + if s.loVerified { + s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search converged: using %d W (lo=%d hi=%d)", s.lo, s.lo, s.hi)) + if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, s.lo); err == nil { + s.appliedLimitW = s.lo + s.calib.AppliedPowerLimitW = float64(s.lo) + s.calib.Derated = s.lo < s.originalLimitW + // Summary was captured when we last verified stability at s.lo, + // so the result is valid — mark as completed even though we + // converged from the failure path (tried higher, failed, fell back). + s.calib.Completed = true + } + } else { + s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("could not find a stable %s limit down to the minimum settable power limit %d W", engineLabel, s.minLimitW)) + logFunc(fmt.Sprintf("power calibration: GPU %d no stable limit found down to minimum settable power limit %d W", s.idx, s.minLimitW)) + } + s.calib.MetricRows = filterRowsByGPU(ar.rows, s.idx) + s.converged = true + continue + } + + next := roundTo5W((s.lo + s.hi) / 2) + if next <= s.lo { + next = s.lo + calibSearchTolerance + } + if next >= s.hi { + next = (s.lo + s.hi) / 2 + } + if next < s.minLimitW { + next = s.minLimitW + } + if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, next); err != nil { + s.calib.Notes = append(s.calib.Notes, "failed to set power limit: "+err.Error()) + logFunc(fmt.Sprintf("power calibration: GPU %d failed to set power limit %d W: %v", s.idx, next, err)) + s.converged = true + continue + } + s.appliedLimitW = next + s.calib.AppliedPowerLimitW = float64(next) + s.calib.Derated = next < s.originalLimitW + s.info.PowerLimitW = float64(next) + infoByIndex[s.idx] = s.info + s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search: trying %d W (lo=%d hi=%d)", next, s.lo, s.hi)) + logFunc(fmt.Sprintf("power calibration: GPU %d binary search: trying %d W (lo=%d hi=%d)", s.idx, next, s.lo, s.hi)) + } + if attemptStable { + if attemptSDRSummary.Samples > 0 { + runSummary.LoadedSDR = attemptSDRSummary + } + if attemptFanSummary.FanSamples > 0 { + runSummary.AvgFanRPM = attemptFanSummary.AvgFanRPM + runSummary.AvgFanDutyCyclePct = attemptFanSummary.AvgFanDutyCyclePct + runSummary.FanSamples = attemptFanSummary.FanSamples + } + } + } + + for _, s := range states { + if s.calib.Completed || s.calib.Attempts > 0 || len(s.calib.Notes) > 0 { + results[s.idx] = s.calib + } + } + writeBenchmarkMetricsFiles(runDir, allCalibRows) + return results, restore, allCalibRows, runSummary +} + +// isDCGMResourceBusy returns true when dcgmi exits with DCGM_ST_IN_USE (222), +// meaning nv-hostengine still holds the diagnostic slot from a prior run. +func isDCGMResourceBusy(err error) bool { + var exitErr *exec.ExitError + return errors.As(err, &exitErr) && exitErr.ExitCode() == 222 +} + +// roundTo5W rounds w to the nearest 5 W boundary. +func roundTo5W(w int) int { + return ((w + 2) / 5) * 5 +} + +func initialBenchmarkCalibrationLimitW(info benchmarkGPUInfo) int { + defaultLimitW := int(math.Round(info.DefaultPowerLimitW)) + currentLimitW := int(math.Round(info.PowerLimitW)) + maxLimitW := int(math.Round(info.MaxPowerLimitW)) + + startW := defaultLimitW + if startW <= 0 { + startW = currentLimitW + } + if startW <= 0 { + startW = maxLimitW + } + if maxLimitW > 0 && startW > maxLimitW { + startW = maxLimitW + } + return startW +} + +// meanFanRPM returns the average RPM across a set of fan readings. +func meanFanRPM(fans []FanReading) float64 { + if len(fans) == 0 { + return 0 + } + var sum float64 + for _, f := range fans { + sum += f.RPM + } + return sum / float64(len(fans)) +} + +func startBenchmarkFanSampler(stopCh <-chan struct{}, intervalSec int) <-chan benchmarkPowerCalibrationRunSummary { + if intervalSec <= 0 { + intervalSec = benchmarkPowerAutotuneSampleInterval + } + ch := make(chan benchmarkPowerCalibrationRunSummary, 1) + go func() { + defer close(ch) + var rpmSamples []float64 + var dutySamples []float64 + record := func() { + fans, err := sampleFanSpeeds() + if err != nil || len(fans) == 0 { + return + } + if rpm := meanFanRPM(fans); rpm > 0 { + rpmSamples = append(rpmSamples, rpm) + } + if duty, ok, _ := sampleFanDutyCyclePctFromFans(fans); ok && duty > 0 { + dutySamples = append(dutySamples, duty) + } + } + record() + ticker := time.NewTicker(time.Duration(intervalSec) * time.Second) + defer ticker.Stop() + for { + select { + case <-stopCh: + ch <- benchmarkPowerCalibrationRunSummary{ + AvgFanRPM: benchmarkMean(rpmSamples), + AvgFanDutyCyclePct: benchmarkMean(dutySamples), + FanSamples: len(rpmSamples), + } + return + case <-ticker.C: + record() + } + } + }() + return ch +} + +func powerBenchDurationSec(profile string) int { + switch strings.TrimSpace(strings.ToLower(profile)) { + case NvidiaBenchmarkProfileStability: + return 300 + case NvidiaBenchmarkProfileOvernight: + return 600 + default: + return 120 + } +} + +func cloneBenchmarkGPUInfoMap(src map[int]benchmarkGPUInfo) map[int]benchmarkGPUInfo { + out := make(map[int]benchmarkGPUInfo, len(src)) + for k, v := range src { + out[k] = v + } + return out +} diff --git a/audit/internal/platform/benchmark_power_report.go b/audit/internal/platform/benchmark_power_report.go new file mode 100644 index 0000000..30a173a --- /dev/null +++ b/audit/internal/platform/benchmark_power_report.go @@ -0,0 +1,624 @@ +package platform + +import ( + "fmt" + "sort" + "strings" + "time" +) + +func renderPowerBenchReport(result NvidiaPowerBenchResult) string { + var b strings.Builder + b.WriteString("# Bee Bench Power Report\n\n") + fmt.Fprintf(&b, "**Benchmark version:** %s \n", result.BenchmarkVersion) + fmt.Fprintf(&b, "**Profile:** %s \n", result.BenchmarkProfile) + fmt.Fprintf(&b, "**Generated:** %s \n", result.GeneratedAt.Format("2006-01-02 15:04:05 UTC")) + fmt.Fprintf(&b, "**Overall status:** %s \n", result.OverallStatus) + fmt.Fprintf(&b, "**Platform max TDP (GPU-reported):** %.0f W \n", result.PlatformMaxTDPW) + if sp := result.ServerPower; sp != nil && sp.Available { + sourceLabel := "autotuned source" + switch normalizeBenchmarkPowerSource(sp.Source) { + case BenchmarkPowerSourceSDRPSUInput: + sourceLabel = "autotuned source (SDR PSU AC input)" + case BenchmarkPowerSourceDCMI: + sourceLabel = "autotuned source (DCMI)" + } + fmt.Fprintf(&b, "**Server power delta (%s):** %.0f W \n", sourceLabel, sp.DeltaW) + fmt.Fprintf(&b, "**Reporting ratio:** %.2f \n", sp.ReportingRatio) + } + b.WriteString("\n") + // Server power comparison table. + if sp := result.ServerPower; sp != nil { + b.WriteString("## Server vs GPU Power Comparison\n\n") + selectedSource := normalizeBenchmarkPowerSource(sp.Source) + selectedSourceLabel := "Selected source" + if selectedSource == BenchmarkPowerSourceSDRPSUInput { + selectedSourceLabel = "Selected source (SDR PSU AC input)" + } else if selectedSource == BenchmarkPowerSourceDCMI { + selectedSourceLabel = "Selected source (DCMI)" + } + var spRows [][]string + spRows = append(spRows, []string{"GPU actual power sum (p95, last step)", fmt.Sprintf("%.0f W", sp.GPUReportedSumW)}) + if sp.Available { + spRows = append(spRows, []string{selectedSourceLabel + " idle power", fmt.Sprintf("%.0f W", sp.IdleW)}) + spRows = append(spRows, []string{selectedSourceLabel + " loaded power", fmt.Sprintf("%.0f W", sp.LoadedW)}) + spRows = append(spRows, []string{selectedSourceLabel + " Δ power (loaded − idle)", fmt.Sprintf("%.0f W", sp.DeltaW)}) + } + if selectedSource == BenchmarkPowerSourceSDRPSUInput && sp.PSUInputLoadedW > 0 { + spRows = append(spRows, []string{"PSU AC input (idle avg, pre-load phase)", fmt.Sprintf("%.0f W", sp.PSUInputIdleW)}) + spRows = append(spRows, []string{"PSU AC input (loaded avg, final phase)", fmt.Sprintf("%.0f W", sp.PSUInputLoadedW)}) + psuDelta := sp.PSUInputLoadedW - sp.PSUInputIdleW + spRows = append(spRows, []string{"PSU AC input Δ (loaded − idle)", fmt.Sprintf("%.0f W", psuDelta)}) + } + if sp.Available { + ratio := sp.ReportingRatio + dcmiPartial := detectDCMIPartialCoverage(sp) || + (sp.PSUInputIdleW == 0 && detectIPMISaturationFallback(result.RampSteps)) + ratioNote := "" + switch { + case dcmiPartial: + ratioNote = "⚠ IPMI DCMI covers partial PSU set; use SDR ratio below for accuracy assessment" + case ratio >= 0.9: + ratioNote = "✓ GPU telemetry matches server power" + case ratio >= 0.75: + ratioNote = "⚠ minor discrepancy — GPU may slightly over-report TDP" + default: + ratioNote = "✗ significant discrepancy — GPU over-reports TDP vs wall power" + } + spRows = append(spRows, []string{"Reporting ratio", fmt.Sprintf("%.2f — %s", ratio, ratioNote)}) + if selectedSource == BenchmarkPowerSourceSDRPSUInput && sp.PSUInputLoadedW > 0 && sp.GPUReportedSumW > 0 { + psuDelta := sp.PSUInputLoadedW - sp.PSUInputIdleW + sdrRatio := psuDelta / sp.GPUReportedSumW + sdrNote := "" + switch { + case sdrRatio >= 0.9: + sdrNote = "✓ GPU telemetry matches wall power" + case sdrRatio >= 0.75: + sdrNote = "⚠ minor discrepancy" + default: + sdrNote = "✗ significant discrepancy" + } + spRows = append(spRows, []string{"PSU AC input reporting ratio", fmt.Sprintf("%.2f — %s", sdrRatio, sdrNote)}) + } + } else { + spRows = append(spRows, []string{"IPMI availability", "not available — IPMI not supported or ipmitool not found"}) + } + b.WriteString(fmtMDTable([]string{"Metric", "Value"}, spRows)) + for _, note := range sp.Notes { + fmt.Fprintf(&b, "\n> %s\n", note) + } + b.WriteString("\n") + + if len(sp.PSUSlotReadingsIdle) > 0 || len(sp.PSUSlotReadingsLoaded) > 0 { + b.WriteString("## PSU Load Distribution\n\n") + + slotSet := map[string]struct{}{} + for k := range sp.PSUSlotReadingsIdle { + slotSet[k] = struct{}{} + } + for k := range sp.PSUSlotReadingsLoaded { + slotSet[k] = struct{}{} + } + slots := make([]string, 0, len(slotSet)) + for k := range slotSet { + slots = append(slots, k) + } + sort.Strings(slots) + + fmtW := func(v *float64) string { + if v == nil { + return "—" + } + return fmt.Sprintf("%.0f W", *v) + } + + var psuDistRows [][]string + for _, slot := range slots { + idle := sp.PSUSlotReadingsIdle[slot] + loaded := sp.PSUSlotReadingsLoaded[slot] + + var deltaStr string + if idle.InputW != nil && loaded.InputW != nil { + deltaStr = fmt.Sprintf("%+.0f W", *loaded.InputW-*idle.InputW) + } else { + deltaStr = "—" + } + + status := loaded.Status + if status == "" { + status = idle.Status + } + if status == "" { + status = "—" + } + + psuDistRows = append(psuDistRows, []string{ + slot, + fmtW(idle.InputW), fmtW(loaded.InputW), + deltaStr, status, + }) + } + b.WriteString(fmtMDTable([]string{"Slot", "AC Input (idle avg)", "AC Input (loaded avg)", "Load Δ", "Status"}, psuDistRows)) + b.WriteString("\n") + } + } + + if len(result.Findings) > 0 { + b.WriteString("## Summary\n\n") + for _, finding := range result.Findings { + fmt.Fprintf(&b, "- %s\n", finding) + } + b.WriteString("\n") + } + // ── Single GPU section ─────────────────────────────────────────────────── + b.WriteString("## Single GPU\n\n") + { + var sgRows [][]string + for _, gpu := range result.GPUs { + clk := "—" + mem := "—" + temp := "—" + pwr := "—" + if gpu.Telemetry != nil { + clk = fmt.Sprintf("%.0f", gpu.Telemetry.AvgGraphicsClockMHz) + mem = fmt.Sprintf("%.0f", gpu.Telemetry.AvgMemoryClockMHz) + temp = fmt.Sprintf("%.1f", gpu.Telemetry.AvgTempC) + pwr = fmt.Sprintf("%.0f W", gpu.Telemetry.AvgPowerW) + } + serverDelta := "—" + if gpu.ServerDeltaW > 0 { + serverDelta = fmt.Sprintf("%.0f W", gpu.ServerDeltaW) + } + fan := "—" + if gpu.AvgFanRPM > 0 { + if gpu.AvgFanDutyCyclePct > 0 { + fan = fmt.Sprintf("%.0f RPM (%.0f%%)", gpu.AvgFanRPM, gpu.AvgFanDutyCyclePct) + } else { + fan = fmt.Sprintf("%.0f RPM", gpu.AvgFanRPM) + } + } + sgRows = append(sgRows, []string{ + fmt.Sprintf("GPU %d", gpu.Index), + fmt.Sprintf("%s (%s)", clk, mem), + temp, + pwr, + serverDelta, + fan, + }) + } + b.WriteString(fmtMDTable([]string{"GPU", "Clock MHz (Mem MHz)", "Avg Temp °C", "Power W", "Server Δ W", "Avg Fan RPM (duty%)"}, sgRows)) + b.WriteString("\n") + } + if len(result.RecommendedSlotOrder) > 0 { + fmt.Fprintf(&b, "Recommended slot order for best single-card power realization: `%s`\n\n", joinIndexList(result.RecommendedSlotOrder)) + } + + // ── Ramp Sequence ──────────────────────────────────────────────────────── + // Rows = run number; Cols = per-GPU power (from step telemetry) + aggregates. + if len(result.RampSteps) > 0 { + b.WriteString("## Ramp Sequence\n\n") + + // Collect all GPU indices that appear across all steps (ordered by first appearance). + allGPUIndices := make([]int, 0, len(result.GPUs)) + seen := map[int]bool{} + for _, step := range result.RampSteps { + for _, idx := range step.GPUIndices { + if !seen[idx] { + seen[idx] = true + allGPUIndices = append(allGPUIndices, idx) + } + } + } + + var idleW float64 + if result.ServerPower != nil { + idleW = result.ServerPower.IdleW + } + + // Build header: Run | GPU 0 | GPU 1 | ... | GPU total W | Server itself W | Server wall W | Per GPU wall W | Platform eff. + headers := []string{"Run"} + for _, idx := range allGPUIndices { + headers = append(headers, fmt.Sprintf("GPU %d W", idx)) + } + headers = append(headers, "GPU total W", "Server itself W", "Server wall W", "Per GPU wall W", "Platform eff.") + + var rampRows [][]string + if idleW > 0 { + idleRow := []string{"0 (idle)"} + for range allGPUIndices { + idleRow = append(idleRow, "—") + } + // No load: GPU total is negligible, all draw is the server's own baseline. + idleRow = append(idleRow, "—", fmt.Sprintf("%.0f", idleW), fmt.Sprintf("%.0f", idleW), "—", "—") + rampRows = append(rampRows, idleRow) + } + for _, step := range result.RampSteps { + row := []string{fmt.Sprintf("%d", step.StepIndex)} + for _, idx := range allGPUIndices { + inStep := false + for _, si := range step.GPUIndices { + if si == idx { + inStep = true + break + } + } + if !inStep { + row = append(row, "—") + continue + } + gpuPwr := "—" + if t, ok := step.PerGPUTelemetry[idx]; ok && t != nil && t.AvgPowerW > 0 { + gpuPwr = fmt.Sprintf("%.0f", t.AvgPowerW) + } + row = append(row, gpuPwr) + } + // GPU total W = sum of observed GPU power (nvidia-smi) + gpuTotal := "—" + if step.TotalObservedPowerW > 0 { + gpuTotal = fmt.Sprintf("%.0f", step.TotalObservedPowerW) + } + // Server itself W = server wall power minus GPU total (non-GPU baseline draw) + serverItself := "—" + if step.ServerLoadedW > 0 && step.TotalObservedPowerW > 0 { + serverItself = fmt.Sprintf("%.0f", step.ServerLoadedW-step.TotalObservedPowerW) + } + // Server wall W + serverWall := "—" + if step.ServerLoadedW > 0 { + serverWall = fmt.Sprintf("%.0f", step.ServerLoadedW) + } + // Per GPU wall W = ServerDeltaW / len(GPUIndices) + perGPUWall := "—" + if step.ServerDeltaW > 0 && len(step.GPUIndices) > 0 { + perGPUWall = fmt.Sprintf("%.0f", step.ServerDeltaW/float64(len(step.GPUIndices))) + } + // Platform eff. = (ServerLoadedW − idleW) / TotalObservedPowerW + platEff := "—" + if step.TotalObservedPowerW > 0 { + eff := step.ServerDeltaW / step.TotalObservedPowerW + if idleW > 0 && step.ServerLoadedW > 0 { + eff = (step.ServerLoadedW - idleW) / step.TotalObservedPowerW + } + platEff = fmt.Sprintf("%.2f", eff) + } + row = append(row, gpuTotal, serverItself, serverWall, perGPUWall, platEff) + rampRows = append(rampRows, row) + } + b.WriteString(fmtMDTable(headers, rampRows)) + b.WriteString("\n") + } + + // ── PSU Performance ─────────────────────────────────────────────────────── + { + // Collect all PSU slot keys from any ramp step. + psuSlotSet := map[string]struct{}{} + for _, step := range result.RampSteps { + for k := range step.PSUSlotReadings { + psuSlotSet[k] = struct{}{} + } + } + if len(psuSlotSet) > 0 { + b.WriteString("## PSU Performance\n\n") + psuSlots := make([]string, 0, len(psuSlotSet)) + for k := range psuSlotSet { + psuSlots = append(psuSlots, k) + } + sort.Strings(psuSlots) + + var idleW float64 + if result.ServerPower != nil { + idleW = result.ServerPower.IdleW + } + + psuHeaders := []string{"Run"} + for _, slot := range psuSlots { + psuHeaders = append(psuHeaders, fmt.Sprintf("PSU %s W", slot)) + } + psuHeaders = append(psuHeaders, "PSU Total W", "Platform eff.", "Avg Fan RPM (duty%)") + + var psuRows [][]string + for _, step := range result.RampSteps { + row := []string{fmt.Sprintf("%d", step.StepIndex)} + var psuTotal float64 + for _, slot := range psuSlots { + sp, ok := step.PSUSlotReadings[slot] + if !ok || sp.InputW == nil { + row = append(row, "—") + continue + } + row = append(row, fmt.Sprintf("%.0f", *sp.InputW)) + psuTotal += *sp.InputW + } + totalStr := "—" + if psuTotal > 0 { + totalStr = fmt.Sprintf("%.0f", psuTotal) + } + platEff := "—" + if step.TotalObservedPowerW > 0 { + eff := step.ServerDeltaW / step.TotalObservedPowerW + if idleW > 0 && step.ServerLoadedW > 0 { + eff = (step.ServerLoadedW - idleW) / step.TotalObservedPowerW + } + platEff = fmt.Sprintf("%.2f", eff) + } + fan := "—" + if step.AvgFanRPM > 0 { + if step.AvgFanDutyCyclePct > 0 { + fan = fmt.Sprintf("%.0f (%.0f%%)", step.AvgFanRPM, step.AvgFanDutyCyclePct) + } else { + fan = fmt.Sprintf("%.0f", step.AvgFanRPM) + } + } + row = append(row, totalStr, platEff, fan) + psuRows = append(psuRows, row) + } + b.WriteString(fmtMDTable(psuHeaders, psuRows)) + b.WriteString("\n") + } + } + + // ── PSU Issues ──────────────────────────────────────────────────────────── + if len(result.PSUIssues) > 0 { + b.WriteString("## PSU Issues\n\n") + b.WriteString("The following power supply anomalies were detected during the test:\n\n") + for _, issue := range result.PSUIssues { + fmt.Fprintf(&b, "- ⛔ %s\n", issue) + } + b.WriteString("\n") + } + + // ── Power Distribution Summary ──────────────────────────────────────────── + b.WriteString("## Power Distribution Summary\n\n") + { + var totalDefault, totalStable float64 + for _, gpu := range result.GPUs { + stable := gpu.StablePowerLimitW + if stable <= 0 { + stable = gpu.AppliedPowerLimitW + } + totalDefault += gpu.DefaultPowerLimitW + totalStable += stable + } + var pdRows [][]string + for _, gpu := range result.GPUs { + stable := gpu.StablePowerLimitW + if stable <= 0 { + stable = gpu.AppliedPowerLimitW + } + realization := "-" + if gpu.DefaultPowerLimitW > 0 && stable > 0 { + realization = fmt.Sprintf("%.1f%%", stable/gpu.DefaultPowerLimitW*100) + } + derated := "-" + if gpu.Derated { + derated = "⚠ yes" + } + pdRows = append(pdRows, []string{ + fmt.Sprintf("GPU %d", gpu.Index), + fmt.Sprintf("%.0f W", gpu.AppliedPowerLimitW), + fmt.Sprintf("%.0f W", stable), + realization, + derated, + }) + } + platformReal := "-" + if totalDefault > 0 && totalStable > 0 { + platformReal = fmt.Sprintf("%.1f%%", totalStable/totalDefault*100) + } + pdRows = append(pdRows, []string{ + "**Platform**", + "—", + fmt.Sprintf("**%.0f W**", totalStable), + fmt.Sprintf("**%s**", platformReal), + "", + }) + b.WriteString(fmtMDTable([]string{"GPU", "Single-card limit", "Stable limit", "Realization", "Derated"}, pdRows)) + b.WriteString("\n") + + // Balance across GPUs — only meaningful with 2+ GPUs. + if len(result.GPUs) > 1 { + var minS, maxS, sumS float64 + var cnt int + for _, gpu := range result.GPUs { + s := gpu.StablePowerLimitW + if s <= 0 { + s = gpu.AppliedPowerLimitW + } + if s <= 0 { + continue + } + sumS += s + cnt++ + if cnt == 1 || s < minS { + minS = s + } + if s > maxS { + maxS = s + } + } + if cnt > 0 { + avg := sumS / float64(cnt) + spread := (maxS - minS) / avg * 100 + balanceNote := "✓ balanced" + switch { + case spread > 20: + balanceNote = "⚠ significant imbalance — check slot thermals" + case spread > 10: + balanceNote = "— minor imbalance" + } + fmt.Fprintf(&b, "**GPU power balance:** avg %.0f W · min %.0f W · max %.0f W · spread %.1f%% — %s\n\n", + avg, minS, maxS, spread, balanceNote) + } + } + + // Ramp scalability table — power efficiency of adding each GPU. + if len(result.RampSteps) > 1 { + b.WriteString("**Ramp power scalability** (stable TDP per step):\n\n") + var firstStable float64 + if len(result.GPUs) > 0 { + firstStable = result.GPUs[0].StablePowerLimitW + if firstStable <= 0 { + firstStable = result.GPUs[0].AppliedPowerLimitW + } + } + var prevCumulative float64 + var scalRows [][]string + for _, step := range result.RampSteps { + var cumulative float64 + for _, gpuIdx := range step.GPUIndices { + for _, g := range result.GPUs { + if g.Index != gpuIdx { + continue + } + s := g.StablePowerLimitW + if s <= 0 { + s = g.AppliedPowerLimitW + } + cumulative += s + } + } + incremental := cumulative - prevCumulative + efficiency := "—" + if step.StepIndex > 1 && firstStable > 0 { + efficiency = fmt.Sprintf("%.1f%%", incremental/firstStable*100) + } + scalRows = append(scalRows, []string{ + fmt.Sprintf("%d", step.StepIndex), + joinIndexList(step.GPUIndices), + fmt.Sprintf("%.0f W", cumulative), + fmt.Sprintf("%.0f W", incremental), + efficiency, + }) + prevCumulative = cumulative + } + b.WriteString(fmtMDTable([]string{"Step", "GPUs", "Cumulative stable TDP", "Incremental", "Efficiency vs GPU 1"}, scalRows)) + b.WriteString("\n") + } + } + + // ── Per-GPU sections ────────────────────────────────────────────────────── + var lastStep *NvidiaPowerBenchStep + if n := len(result.RampSteps); n > 0 { + lastStep = &result.RampSteps[n-1] + } + for _, gpu := range result.GPUs { + fmt.Fprintf(&b, "### GPU %d — %s\n\n", gpu.Index, gpu.Name) + + // Transposed comparison table: Single Run vs All GPU Run. + singleClk := "—" + singleMem := "—" + singleTemp := "—" + singlePwr := "—" + singleWall := "—" + singleFan := "—" + if gpu.Telemetry != nil { + singleClk = fmt.Sprintf("%.0f", gpu.Telemetry.AvgGraphicsClockMHz) + singleMem = fmt.Sprintf("%.0f", gpu.Telemetry.AvgMemoryClockMHz) + singleTemp = fmt.Sprintf("%.1f", gpu.Telemetry.AvgTempC) + singlePwr = fmt.Sprintf("%.0f W", gpu.Telemetry.AvgPowerW) + } + if gpu.ServerDeltaW > 0 { + singleWall = fmt.Sprintf("%.0f W", gpu.ServerDeltaW) + } + if gpu.AvgFanRPM > 0 { + if gpu.AvgFanDutyCyclePct > 0 { + singleFan = fmt.Sprintf("%.0f RPM (%.0f%%)", gpu.AvgFanRPM, gpu.AvgFanDutyCyclePct) + } else { + singleFan = fmt.Sprintf("%.0f RPM", gpu.AvgFanRPM) + } + } + + allClk := "—" + allMem := "—" + allTemp := "—" + allPwr := "—" + allWall := "—" + allFan := "—" + if lastStep != nil { + if t, ok := lastStep.PerGPUTelemetry[gpu.Index]; ok && t != nil { + allClk = fmt.Sprintf("%.0f", t.AvgGraphicsClockMHz) + allMem = fmt.Sprintf("%.0f", t.AvgMemoryClockMHz) + allTemp = fmt.Sprintf("%.1f", t.AvgTempC) + allPwr = fmt.Sprintf("%.0f W", t.AvgPowerW) + } + if lastStep.ServerDeltaW > 0 && len(lastStep.GPUIndices) > 0 { + allWall = fmt.Sprintf("%.0f W", lastStep.ServerDeltaW/float64(len(lastStep.GPUIndices))) + } + if lastStep.AvgFanRPM > 0 { + if lastStep.AvgFanDutyCyclePct > 0 { + allFan = fmt.Sprintf("%.0f RPM (%.0f%%)", lastStep.AvgFanRPM, lastStep.AvgFanDutyCyclePct) + } else { + allFan = fmt.Sprintf("%.0f RPM", lastStep.AvgFanRPM) + } + } + } + + tableHeaders := []string{"", "Single Run"} + if lastStep != nil { + tableHeaders = append(tableHeaders, "All GPU Run") + } + compRows := [][]string{ + {"Clock MHz (Mem MHz)", fmt.Sprintf("%s (%s)", singleClk, singleMem)}, + {"Avg Temp °C", singleTemp}, + {"Power W", singlePwr}, + {"Per GPU wall W", singleWall}, + {"Avg Fan RPM (duty%)", singleFan}, + } + if lastStep != nil { + compRows[0] = append(compRows[0], fmt.Sprintf("%s (%s)", allClk, allMem)) + compRows[1] = append(compRows[1], allTemp) + compRows[2] = append(compRows[2], allPwr) + compRows[3] = append(compRows[3], allWall) + compRows[4] = append(compRows[4], allFan) + } + b.WriteString(fmtMDTable(tableHeaders, compRows)) + b.WriteString("\n") + + for _, note := range gpu.Notes { + fmt.Fprintf(&b, "- %s\n", note) + } + if len(gpu.Notes) > 0 { + b.WriteString("\n") + } + } + return b.String() +} + +func renderPowerBenchSummary(result NvidiaPowerBenchResult) string { + var b strings.Builder + fmt.Fprintf(&b, "run_at_utc=%s\n", result.GeneratedAt.Format(time.RFC3339)) + fmt.Fprintf(&b, "benchmark_version=%s\n", result.BenchmarkVersion) + fmt.Fprintf(&b, "benchmark_profile=%s\n", result.BenchmarkProfile) + fmt.Fprintf(&b, "overall_status=%s\n", result.OverallStatus) + fmt.Fprintf(&b, "platform_max_tdp_w=%.0f\n", result.PlatformMaxTDPW) + fmt.Fprintf(&b, "gpu_count=%d\n", len(result.GPUs)) + if len(result.RecommendedSlotOrder) > 0 { + fmt.Fprintf(&b, "recommended_slot_order=%s\n", joinIndexList(result.RecommendedSlotOrder)) + } + for _, step := range result.RampSteps { + fmt.Fprintf(&b, "ramp_step_%d_gpus=%s\n", step.StepIndex, joinIndexList(step.GPUIndices)) + fmt.Fprintf(&b, "ramp_step_%d_new_gpu=%d\n", step.StepIndex, step.NewGPUIndex) + fmt.Fprintf(&b, "ramp_step_%d_stable_limit_w=%.0f\n", step.StepIndex, step.NewGPUStableLimitW) + fmt.Fprintf(&b, "ramp_step_%d_total_power_w=%.0f\n", step.StepIndex, step.TotalObservedPowerW) + if step.ServerLoadedW > 0 { + fmt.Fprintf(&b, "ramp_step_%d_server_loaded_w=%.0f\n", step.StepIndex, step.ServerLoadedW) + fmt.Fprintf(&b, "ramp_step_%d_server_delta_w=%.0f\n", step.StepIndex, step.ServerDeltaW) + } + } + for _, gpu := range result.GPUs { + if gpu.StablePowerLimitW > 0 { + fmt.Fprintf(&b, "gpu_%d_stable_limit_w=%.0f\n", gpu.Index, gpu.StablePowerLimitW) + } + if gpu.ServerLoadedW > 0 { + fmt.Fprintf(&b, "gpu_%d_server_loaded_w=%.0f\n", gpu.Index, gpu.ServerLoadedW) + fmt.Fprintf(&b, "gpu_%d_server_delta_w=%.0f\n", gpu.Index, gpu.ServerDeltaW) + } + } + if sp := result.ServerPower; sp != nil && sp.Available { + fmt.Fprintf(&b, "server_idle_w=%.0f\n", sp.IdleW) + fmt.Fprintf(&b, "server_loaded_w=%.0f\n", sp.LoadedW) + fmt.Fprintf(&b, "server_delta_w=%.0f\n", sp.DeltaW) + fmt.Fprintf(&b, "server_reporting_ratio=%.2f\n", sp.ReportingRatio) + } + return b.String() +} diff --git a/audit/internal/platform/benchmark_power_run.go b/audit/internal/platform/benchmark_power_run.go new file mode 100644 index 0000000..78d0c2d --- /dev/null +++ b/audit/internal/platform/benchmark_power_run.go @@ -0,0 +1,495 @@ +package platform + +import ( + "context" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +func (s *System) RunNvidiaPowerBench(ctx context.Context, baseDir string, opts NvidiaBenchmarkOptions, logFunc func(string)) (string, error) { + if ctx == nil { + ctx = context.Background() + } + if logFunc == nil { + logFunc = func(string) {} + } + if strings.TrimSpace(baseDir) == "" { + baseDir = "/var/log/bee-bench/power" + } + opts = normalizeNvidiaBenchmarkOptionsForBenchmark(opts) + selected, err := resolveNvidiaGPUSelection(opts.GPUIndices, opts.ExcludeGPUIndices) + if err != nil { + return "", err + } + if len(selected) == 0 { + return "", fmt.Errorf("no NVIDIA GPUs selected") + } + ts := time.Now().UTC().Format("20060102-150405") + runDir := filepath.Join(baseDir, "power-"+ts) + if err := os.MkdirAll(runDir, 0755); err != nil { + return "", fmt.Errorf("mkdir %s: %w", runDir, err) + } + verboseLog := filepath.Join(runDir, "verbose.log") + hostname, _ := os.Hostname() + result := NvidiaPowerBenchResult{ + BenchmarkVersion: benchmarkVersion, + GeneratedAt: time.Now().UTC(), + Hostname: hostname, + ServerModel: readServerModel(), + BenchmarkProfile: opts.Profile, + SelectedGPUIndices: append([]int(nil), selected...), + OverallStatus: "OK", + } + infoByIndex, infoErr := queryBenchmarkGPUInfo(selected) + if infoErr != nil { + return "", infoErr + } + // Capture full nvidia-smi -q snapshot at the start of the run. + if out, err := runSATCommandCtx(ctx, verboseLog, "00-nvidia-smi-q.log", []string{"nvidia-smi", "-q"}, nil, nil); err == nil { + _ = os.WriteFile(filepath.Join(runDir, "00-nvidia-smi-q.log"), out, 0644) + } + durationSec := powerBenchDurationSec(opts.Profile) + + // Sample server idle power before any GPU load. + var serverIdleW float64 + var serverIdleOK bool + idleSDRStopCh := make(chan struct{}) + idleSDRCh := startIPMISDRSampler(idleSDRStopCh, benchmarkPowerAutotuneSampleInterval) + if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, 10, benchmarkPowerAutotuneSampleInterval); ok { + serverIdleW = w + serverIdleOK = true + logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w)) + } + close(idleSDRStopCh) + sdrIdle := summarizeSDRPowerSeries(<-idleSDRCh) + psuBefore := psuStatusSnapshot() + + // Phase 1: calibrate each GPU individually (sequentially, one at a time) to + // establish a true single-card power baseline unaffected by neighbour heat. + calibByIndex := make(map[int]benchmarkPowerCalibrationResult, len(selected)) + singleIPMILoadedW := make(map[int]float64, len(selected)) + singleRunSummaryByIndex := make(map[int]benchmarkPowerCalibrationRunSummary, len(selected)) + var allRestoreActions []benchmarkRestoreAction + // allPowerRows accumulates telemetry from all phases for the top-level gpu-metrics.csv. + var allPowerRows []GPUMetricRow + var powerCursor float64 + for _, idx := range selected { + singleDir := filepath.Join(runDir, fmt.Sprintf("single-%02d", idx)) + _ = os.MkdirAll(singleDir, 0755) + singleInfo := cloneBenchmarkGPUInfoMap(infoByIndex) + if failed := resetBenchmarkGPUs(ctx, verboseLog, []int{idx}, logFunc); len(failed) > 0 { + return "", fmt.Errorf("power benchmark pre-flight: failed to reset GPU %d; benchmark aborted to keep measurements clean", idx) + } + logFunc(fmt.Sprintf("power calibration: GPU %d single-card baseline", idx)) + singlePowerStopCh := make(chan struct{}) + singlePowerCh := startSelectedPowerSourceSampler(singlePowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval) + c, restore, singleRows, singleRun := runBenchmarkPowerCalibration(ctx, verboseLog, singleDir, []int{idx}, singleInfo, logFunc, nil, durationSec) + appendBenchmarkMetrics(&allPowerRows, singleRows, fmt.Sprintf("single-gpu-%d", idx), &powerCursor, 0) + close(singlePowerStopCh) + if samples := <-singlePowerCh; len(samples) > 0 { + singleIPMILoadedW[idx] = benchmarkMean(samples) + logFunc(fmt.Sprintf("power calibration: GPU %d single-card server power (%s avg): %.0f W", idx, opts.ServerPowerSource, singleIPMILoadedW[idx])) + } else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && singleRun.LoadedSDR.PSUInW > 0 { + singleIPMILoadedW[idx] = singleRun.LoadedSDR.PSUInW + logFunc(fmt.Sprintf("power calibration: GPU %d single-card fallback server power (SDR avg): %.0f W", idx, singleRun.LoadedSDR.PSUInW)) + } + allRestoreActions = append(allRestoreActions, restore...) + if r, ok := c[idx]; ok { + calibByIndex[idx] = r + } + singleRunSummaryByIndex[idx] = singleRun + } + defer func() { + for i := len(allRestoreActions) - 1; i >= 0; i-- { + allRestoreActions[i].fn() + } + }() + gpus := make([]NvidiaPowerBenchGPU, 0, len(selected)) + for _, idx := range selected { + info := infoByIndex[idx] + calib := calibByIndex[idx] + status := "OK" + if !calib.Completed { + status = "FAILED" + result.OverallStatus = "PARTIAL" + } else if calib.Derated { + status = "PARTIAL" + if result.OverallStatus == "OK" { + result.OverallStatus = "PARTIAL" + } + } + gpu := NvidiaPowerBenchGPU{ + Index: idx, + Name: info.Name, + BusID: info.BusID, + DefaultPowerLimitW: info.DefaultPowerLimitW, + AppliedPowerLimitW: calib.AppliedPowerLimitW, + MaxObservedPowerW: calib.Summary.P95PowerW, + MaxObservedTempC: calib.Summary.P95TempC, + CalibrationAttempts: calib.Attempts, + Derated: calib.Derated, + Status: status, + Notes: append([]string(nil), calib.Notes...), + CoolingWarning: calib.CoolingWarning, + } + if w, ok := singleIPMILoadedW[idx]; ok && serverIdleOK && w > 0 { + gpu.ServerLoadedW = w + gpu.ServerDeltaW = w - serverIdleW + } + if len(calib.MetricRows) > 0 { + t := summarizeBenchmarkTelemetry(calib.MetricRows) + gpu.Telemetry = &t + } + if singleRun := singleRunSummaryByIndex[idx]; singleRun.AvgFanRPM > 0 { + gpu.AvgFanRPM = singleRun.AvgFanRPM + gpu.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct + } + gpus = append(gpus, gpu) + } + sort.Slice(gpus, func(i, j int) bool { + if gpus[i].MaxObservedPowerW != gpus[j].MaxObservedPowerW { + return gpus[i].MaxObservedPowerW > gpus[j].MaxObservedPowerW + } + if gpus[i].AppliedPowerLimitW != gpus[j].AppliedPowerLimitW { + return gpus[i].AppliedPowerLimitW > gpus[j].AppliedPowerLimitW + } + if gpus[i].Derated != gpus[j].Derated { + return !gpus[i].Derated + } + return gpus[i].Index < gpus[j].Index + }) + result.GPUs = gpus + result.RecommendedSlotOrder = make([]int, 0, len(gpus)) + for _, gpu := range gpus { + result.RecommendedSlotOrder = append(result.RecommendedSlotOrder, gpu.Index) + } + if len(result.RecommendedSlotOrder) > 0 { + result.Findings = append(result.Findings, fmt.Sprintf("Recommended slot order for installation based on single-card %s: %s.", benchmarkPowerEngineLabel(benchmarkPowerEngine()), joinIndexList(result.RecommendedSlotOrder))) + } + for _, gpu := range gpus { + if gpu.Derated { + result.Findings = append(result.Findings, fmt.Sprintf("GPU %d required reduced power limit %.0f W to complete %s.", gpu.Index, gpu.AppliedPowerLimitW, benchmarkPowerEngineLabel(benchmarkPowerEngine()))) + } + if gpu.CoolingWarning != "" { + result.Findings = append(result.Findings, fmt.Sprintf( + "GPU %d: %s. Operator action: rerun the benchmark with fan speed manually fixed at 100%% to confirm actual thermal headroom.", + gpu.Index, gpu.CoolingWarning, + )) + } + } + singleByIndex := make(map[int]NvidiaPowerBenchGPU, len(gpus)) + for _, gpu := range gpus { + singleByIndex[gpu.Index] = gpu + } + + // Phase 2: cumulative thermal ramp. + // Each step introduces one new GPU into an environment where all previously + // calibrated GPUs are already running at their fixed stable limits. The new + // GPU's stable TDP is searched via binary search under real + // multi-GPU thermal load. Once found, its limit is fixed permanently for all + // subsequent steps. This ensures each GPU's limit reflects actual sustained + // power in the final full-system thermal state. + // + // stableLimits accumulates GPU index → fixed stable limit (W) across steps. + stableLimits := make(map[int]int, len(result.RecommendedSlotOrder)) + + // serverLoadedW tracks the IPMI server power from the final ramp step + // (all GPUs simultaneously loaded). Earlier steps' values are stored + // per-step in NvidiaPowerBenchStep.ServerLoadedW. + var serverLoadedW float64 + var serverLoadedOK bool + // sdrLastStep retains the phase-averaged SDR readings from the last ramp step + // while GPUs are loaded. Used in the summary instead of re-sampling after the + // test when GPUs have already returned to idle. + var sdrLastStep benchmarkSDRSeriesSummary + + // Step 1: reuse single-card calibration result directly. + if len(result.RecommendedSlotOrder) > 0 { + firstIdx := result.RecommendedSlotOrder[0] + firstCalib := calibByIndex[firstIdx] + stableLimits[firstIdx] = int(math.Round(firstCalib.AppliedPowerLimitW)) + ramp := NvidiaPowerBenchStep{ + StepIndex: 1, + GPUIndices: []int{firstIdx}, + NewGPUIndex: firstIdx, + NewGPUStableLimitW: firstCalib.AppliedPowerLimitW, + TotalObservedPowerW: firstCalib.Summary.P95PowerW, + AvgObservedPowerW: firstCalib.Summary.P95PowerW, + Derated: firstCalib.Derated, + Status: "OK", + } + if w, ok := singleIPMILoadedW[firstIdx]; ok && serverIdleOK && w > 0 { + ramp.ServerLoadedW = w + ramp.ServerDeltaW = w - serverIdleW + } + if singleRun := singleRunSummaryByIndex[firstIdx]; singleRun.AvgFanRPM > 0 { + ramp.AvgFanRPM = singleRun.AvgFanRPM + ramp.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct + } + firstSummary := firstCalib.Summary + ramp.PerGPUTelemetry = map[int]*BenchmarkTelemetrySummary{firstIdx: &firstSummary} + if !firstCalib.Completed { + ramp.Status = "FAILED" + ramp.Notes = append(ramp.Notes, fmt.Sprintf("GPU %d did not complete single-card %s", firstIdx, benchmarkPowerEngineLabel(benchmarkPowerEngine()))) + result.OverallStatus = "PARTIAL" + } else if firstCalib.Derated { + ramp.Status = "PARTIAL" + if result.OverallStatus == "OK" { + result.OverallStatus = "PARTIAL" + } + result.Findings = append(result.Findings, fmt.Sprintf("Ramp step 1 (GPU %d) required derating to %.0f W.", firstIdx, firstCalib.AppliedPowerLimitW)) + } + result.RampSteps = append(result.RampSteps, ramp) + logFunc(fmt.Sprintf("power ramp: step 1/%d — reused single-card calibration for GPU %d, stable limit %.0f W", + len(result.RecommendedSlotOrder), firstIdx, firstCalib.AppliedPowerLimitW)) + } + + // Steps 2..N: each step revalidates every already-active GPU under the new + // cumulative thermal environment and also calibrates the newly introduced + // GPU. Previously found limits are used only as seeds for the search. + for stepNum := 1; stepNum < len(result.RecommendedSlotOrder); stepNum++ { + step := stepNum + 1 + subset := append([]int(nil), result.RecommendedSlotOrder[:step]...) + newGPUIdx := result.RecommendedSlotOrder[stepNum] + stepDir := filepath.Join(runDir, fmt.Sprintf("step-%02d", step)) + _ = os.MkdirAll(stepDir, 0755) + + // Reuse the latest stable limits as starting points, but re-check every + // active GPU in this hotter configuration. For the newly introduced GPU, + // seed from its single-card calibration so we do not restart from the + // default TDP when a prior derated limit is already known. + seedForStep := make(map[int]int, len(subset)) + for _, idx := range subset { + if lim, ok := stableLimits[idx]; ok && lim > 0 { + seedForStep[idx] = lim + continue + } + if base, ok := calibByIndex[idx]; ok { + lim := int(math.Round(base.AppliedPowerLimitW)) + if lim > 0 { + seedForStep[idx] = lim + } + } + } + + logFunc(fmt.Sprintf("power ramp: step %d/%d — revalidating %d active GPU(s) including new GPU %d", + step, len(result.RecommendedSlotOrder), len(subset), newGPUIdx)) + + stepInfo := cloneBenchmarkGPUInfoMap(infoByIndex) + stepPowerStopCh := make(chan struct{}) + stepPowerCh := startSelectedPowerSourceSampler(stepPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval) + stepCalib, stepRestore, stepRows, stepRun := runBenchmarkPowerCalibration(ctx, verboseLog, stepDir, subset, stepInfo, logFunc, seedForStep, durationSec) + appendBenchmarkMetrics(&allPowerRows, stepRows, fmt.Sprintf("ramp-step-%d", step), &powerCursor, 0) + close(stepPowerStopCh) + var stepIPMILoadedW float64 + var stepIPMIOK bool + if samples := <-stepPowerCh; len(samples) > 0 { + stepIPMILoadedW = benchmarkMean(samples) + stepIPMIOK = true + } + // Accumulate restore actions; they all run in the outer defer. + allRestoreActions = append(allRestoreActions, stepRestore...) + + ramp := NvidiaPowerBenchStep{ + StepIndex: step, + GPUIndices: subset, + NewGPUIndex: newGPUIdx, + Status: "OK", + } + + // Total observed power = sum of p95 across all GPUs in this step. + for _, idx := range subset { + if c, ok := stepCalib[idx]; ok { + ramp.TotalObservedPowerW += c.Summary.P95PowerW + } + } + if len(subset) > 0 { + ramp.AvgObservedPowerW = ramp.TotalObservedPowerW / float64(len(subset)) + } + + for _, idx := range subset { + c, ok := stepCalib[idx] + if !ok || !c.Completed { + fallback := 0 + if lim, ok := stableLimits[idx]; ok && lim > 0 { + fallback = lim + } else if fb, ok := calibByIndex[idx]; ok { + fallback = int(math.Round(fb.AppliedPowerLimitW)) + } + if fallback > 0 { + stableLimits[idx] = fallback + } + ramp.Status = "FAILED" + ramp.Notes = append(ramp.Notes, + fmt.Sprintf("GPU %d did not complete %s in ramp step %d; keeping previous stable limit %d W", idx, benchmarkPowerEngineLabel(benchmarkPowerEngine()), step, fallback)) + result.OverallStatus = "PARTIAL" + continue + } + + prevLimit, hadPrev := stableLimits[idx] + newLimit := int(math.Round(c.AppliedPowerLimitW)) + stableLimits[idx] = newLimit + if idx == newGPUIdx { + ramp.NewGPUStableLimitW = c.AppliedPowerLimitW + ramp.Derated = c.Derated + } + if c.Derated { + ramp.Status = "PARTIAL" + if result.OverallStatus == "OK" { + result.OverallStatus = "PARTIAL" + } + } + if hadPrev && newLimit < prevLimit { + ramp.Notes = append(ramp.Notes, + fmt.Sprintf("GPU %d was re-derated from %d W to %d W under combined thermal load.", idx, prevLimit, newLimit)) + } + } + + if c, ok := stepCalib[newGPUIdx]; ok && c.Completed && c.Derated { + result.Findings = append(result.Findings, fmt.Sprintf("Ramp step %d (GPU %d) required derating to %.0f W under combined thermal load.", step, newGPUIdx, c.AppliedPowerLimitW)) + } + + // Per-step PSU slot readings are averaged over the whole load phase rather + // than captured as a single end-of-phase snapshot. + sdrStep := stepRun.LoadedSDR + if len(sdrStep.PSUSlots) > 0 { + ramp.PSUSlotReadings = sdrStep.PSUSlots + } + + if stepIPMIOK && serverIdleOK && stepIPMILoadedW > 0 { + ramp.ServerLoadedW = stepIPMILoadedW + ramp.ServerDeltaW = stepIPMILoadedW - serverIdleW + logFunc(fmt.Sprintf("power ramp: step %d server loaded power (%s avg): %.0f W", step, opts.ServerPowerSource, stepIPMILoadedW)) + // The last step has all GPUs loaded — use it as the top-level loaded_w. + if step == len(result.RecommendedSlotOrder) { + serverLoadedW = stepIPMILoadedW + serverLoadedOK = true + sdrLastStep = sdrStep + } + } else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && sdrStep.PSUInW > 0 { + ramp.ServerLoadedW = sdrStep.PSUInW + ramp.ServerDeltaW = sdrStep.PSUInW - sdrIdle.PSUInW + logFunc(fmt.Sprintf("power ramp: step %d fallback server loaded power (SDR avg): %.0f W", step, sdrStep.PSUInW)) + if step == len(result.RecommendedSlotOrder) { + serverLoadedW = sdrStep.PSUInW + serverLoadedOK = true + sdrLastStep = sdrStep + } + } + + // Fan values are phase averages over the same load window. + if stepRun.AvgFanRPM > 0 { + ramp.AvgFanRPM = stepRun.AvgFanRPM + ramp.AvgFanDutyCyclePct = stepRun.AvgFanDutyCyclePct + } + + // Per-GPU telemetry from this ramp step's calibration. + ramp.PerGPUTelemetry = make(map[int]*BenchmarkTelemetrySummary, len(subset)) + for _, gpuIdx := range subset { + if c, ok := stepCalib[gpuIdx]; ok { + s := c.Summary + ramp.PerGPUTelemetry[gpuIdx] = &s + } + } + + result.RampSteps = append(result.RampSteps, ramp) + } + + // Populate StablePowerLimitW on each GPU entry from the accumulated stable limits. + for i := range result.GPUs { + if lim, ok := stableLimits[result.GPUs[i].Index]; ok { + result.GPUs[i].StablePowerLimitW = float64(lim) + } + if result.GPUs[i].StablePowerLimitW > 0 && result.GPUs[i].AppliedPowerLimitW > 0 && + result.GPUs[i].StablePowerLimitW < result.GPUs[i].AppliedPowerLimitW { + result.GPUs[i].Derated = true + result.Findings = append(result.Findings, fmt.Sprintf( + "GPU %d required additional derating from %.0f W (single-card) to %.0f W under full-system thermal load.", + result.GPUs[i].Index, result.GPUs[i].AppliedPowerLimitW, result.GPUs[i].StablePowerLimitW, + )) + } + } + + // PlatformMaxTDPW = sum of all stable limits — the actual sustained power + // budget of this server with all GPUs running simultaneously without throttling. + for _, lim := range stableLimits { + result.PlatformMaxTDPW += float64(lim) + } + + // Characterize server power from IPMI idle/loaded samples. + // gpuActualSumW = sum of p95 GPU power from the last ramp step — actual + // measured consumption, not the stable limit cap. This is the correct + // denominator for the reporting ratio: limit caps (PlatformMaxTDPW) inflate + // the denominator and make the ratio appear artificially low. + var gpuActualSumW float64 + if n := len(result.RampSteps); n > 0 { + gpuActualSumW = result.RampSteps[n-1].TotalObservedPowerW + } + if gpuActualSumW <= 0 { + gpuActualSumW = result.PlatformMaxTDPW + } + _ = serverIdleOK // used implicitly via characterizeServerPower + result.ServerPower = characterizeServerPower(serverIdleW, serverLoadedW, gpuActualSumW, opts.ServerPowerSource, serverIdleOK && serverLoadedOK) + // Supplement DCMI with SDR multi-source data via collector's PSU slot patterns. + // Per-slot readings enable correlation with audit HardwarePowerSupply entries. + if result.ServerPower != nil { + // Use the SDR phase average from the last ramp step (GPUs still loaded) + // rather than re-sampling here, which would capture post-test idle state. + sdrLoaded := sdrLastStep + result.ServerPower.PSUInputIdleW = sdrIdle.PSUInW + result.ServerPower.PSUInputLoadedW = sdrLoaded.PSUInW + result.ServerPower.PSUOutputIdleW = sdrIdle.PSUOutW + result.ServerPower.PSUOutputLoadedW = sdrLoaded.PSUOutW + result.ServerPower.GPUSlotTotalW = sdrLoaded.GPUSlotW + if len(sdrIdle.PSUSlots) > 0 { + result.ServerPower.PSUSlotReadingsIdle = sdrIdle.PSUSlots + } + if len(sdrLoaded.PSUSlots) > 0 { + result.ServerPower.PSUSlotReadingsLoaded = sdrLoaded.PSUSlots + } + if sdrIdle.PSUInW > 0 && result.ServerPower.IdleW > 0 { + result.ServerPower.DCMICoverageRatio = result.ServerPower.IdleW / sdrIdle.PSUInW + } + if len(sdrLoaded.SkippedSensors) > 0 { + result.ServerPower.Notes = append(result.ServerPower.Notes, + "SDR sensors skipped (self-healed): "+strings.Join(sdrLoaded.SkippedSensors, "; ")) + } + if sdrLoaded.Samples > 0 { + result.ServerPower.Notes = append(result.ServerPower.Notes, + fmt.Sprintf("Final SDR PSU loaded values are phase averages across %d sample(s) from the last full-load step.", sdrLoaded.Samples)) + } + // Detect DCMI partial coverage: direct SDR comparison first, + // ramp heuristic as fallback when SDR PSU sensors are absent. + dcmiUnreliable := detectDCMIPartialCoverage(result.ServerPower) || + (sdrIdle.PSUInW == 0 && detectIPMISaturationFallback(result.RampSteps)) + if dcmiUnreliable { + result.ServerPower.Notes = append(result.ServerPower.Notes, + fmt.Sprintf("IPMI DCMI covers only a subset of installed PSUs (coverage %.0f%%). "+ + "Use SDR PSU Δ ratio for GPU accuracy assessment; DCMI ratio is not reliable.", + result.ServerPower.DCMICoverageRatio*100)) + } + } + result.PSUIssues = diffPSUStatus(psuBefore, psuStatusSnapshot()) + // Write top-level gpu-metrics.csv/.html aggregating all phases. + writeBenchmarkMetricsFiles(runDir, allPowerRows) + resultJSON, err := json.MarshalIndent(result, "", " ") + if err != nil { + return "", fmt.Errorf("marshal power result: %w", err) + } + if err := os.WriteFile(filepath.Join(runDir, "result.json"), resultJSON, 0644); err != nil { + return "", fmt.Errorf("write result.json: %w", err) + } + if err := os.WriteFile(filepath.Join(runDir, "report.md"), []byte(renderPowerBenchReport(result)), 0644); err != nil { + return "", fmt.Errorf("write report.md: %w", err) + } + if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(renderPowerBenchSummary(result)), 0644); err != nil { + return "", fmt.Errorf("write summary.txt: %w", err) + } + return runDir, nil +} diff --git a/audit/internal/platform/benchmark_result_analysis.go b/audit/internal/platform/benchmark_result_analysis.go new file mode 100644 index 0000000..cb60638 --- /dev/null +++ b/audit/internal/platform/benchmark_result_analysis.go @@ -0,0 +1,537 @@ +package platform + +import ( + "context" + "fmt" + "math" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +func runBenchmarkInterconnect(ctx context.Context, verboseLog, runDir string, gpuIndices []int, spec benchmarkProfileSpec, logFunc func(string)) *BenchmarkInterconnectResult { + result := &BenchmarkInterconnectResult{ + Status: "UNSUPPORTED", + Attempted: true, + SelectedGPUIndices: append([]int(nil), gpuIndices...), + } + cmd := []string{ + "all_reduce_perf", + "-b", "512M", + "-e", "4G", + "-f", "2", + "-g", strconv.Itoa(len(gpuIndices)), + "--iters", strconv.Itoa(maxInt(20, spec.NCCLSec/10)), + } + env := []string{ + "CUDA_DEVICE_ORDER=PCI_BUS_ID", + "CUDA_VISIBLE_DEVICES=" + joinIndexList(gpuIndices), + } + logFunc(fmt.Sprintf("NCCL interconnect: gpus=%s", joinIndexList(gpuIndices))) + out, err := runSATCommandCtx(ctx, verboseLog, "nccl-all-reduce.log", cmd, env, logFunc) + _ = os.WriteFile(filepath.Join(runDir, "nccl-all-reduce.log"), out, 0644) + if err != nil { + result.Notes = append(result.Notes, strings.TrimSpace(string(out))) + return result + } + avgAlg, maxAlg, avgBus, maxBus := parseNCCLAllReduceOutput(string(out)) + result.Status = "OK" + result.Supported = true + result.AvgAlgBWGBps = avgAlg + result.MaxAlgBWGBps = maxAlg + result.AvgBusBWGBps = avgBus + result.MaxBusBWGBps = maxBus + return result +} + +func parseNCCLAllReduceOutput(raw string) (avgAlg, maxAlg, avgBus, maxBus float64) { + lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") + var algs []float64 + var buses []float64 + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + if len(fields) < 8 { + continue + } + for i := 0; i+2 < len(fields); i++ { + timeVal, err1 := strconv.ParseFloat(fields[i], 64) + algVal, err2 := strconv.ParseFloat(fields[i+1], 64) + busVal, err3 := strconv.ParseFloat(fields[i+2], 64) + if err1 == nil && err2 == nil && err3 == nil && timeVal > 0 { + algs = append(algs, algVal) + buses = append(buses, busVal) + break + } + } + } + if len(algs) == 0 { + return 0, 0, 0, 0 + } + return benchmarkMean(algs), benchmarkMax(algs), benchmarkMean(buses), benchmarkMax(buses) +} + +func queryThrottleCounters(gpuIndex int) (BenchmarkThrottleCounters, error) { + out, err := satExecCommand( + "nvidia-smi", + "--id="+strconv.Itoa(gpuIndex), + "--query-gpu=clocks_event_reasons_counters.sw_power_cap,clocks_event_reasons_counters.sw_thermal_slowdown,clocks_event_reasons_counters.sync_boost,clocks_event_reasons_counters.hw_thermal_slowdown,clocks_event_reasons_counters.hw_power_brake_slowdown", + "--format=csv,noheader,nounits", + ).Output() + if err != nil { + return BenchmarkThrottleCounters{}, err + } + fields := strings.Split(strings.TrimSpace(string(out)), ",") + if len(fields) < 5 { + return BenchmarkThrottleCounters{}, fmt.Errorf("unexpected throttle counter columns: %q", strings.TrimSpace(string(out))) + } + return BenchmarkThrottleCounters{ + SWPowerCapUS: parseBenchmarkUint64(fields[0]), + SWThermalSlowdownUS: parseBenchmarkUint64(fields[1]), + SyncBoostUS: parseBenchmarkUint64(fields[2]), + HWThermalSlowdownUS: parseBenchmarkUint64(fields[3]), + HWPowerBrakeSlowdownUS: parseBenchmarkUint64(fields[4]), + }, nil +} + +func diffThrottleCounters(before, after BenchmarkThrottleCounters) BenchmarkThrottleCounters { + return BenchmarkThrottleCounters{ + SWPowerCapUS: saturatingSub(after.SWPowerCapUS, before.SWPowerCapUS), + SWThermalSlowdownUS: saturatingSub(after.SWThermalSlowdownUS, before.SWThermalSlowdownUS), + SyncBoostUS: saturatingSub(after.SyncBoostUS, before.SyncBoostUS), + HWThermalSlowdownUS: saturatingSub(after.HWThermalSlowdownUS, before.HWThermalSlowdownUS), + HWPowerBrakeSlowdownUS: saturatingSub(after.HWPowerBrakeSlowdownUS, before.HWPowerBrakeSlowdownUS), + } +} + +func queryECCCounters(gpuIndex int) (BenchmarkECCCounters, error) { + out, err := satExecCommand( + "nvidia-smi", + "--id="+strconv.Itoa(gpuIndex), + "--query-gpu=ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total", + "--format=csv,noheader,nounits", + ).Output() + if err != nil { + return BenchmarkECCCounters{}, err + } + fields := strings.Split(strings.TrimSpace(string(out)), ",") + if len(fields) < 2 { + return BenchmarkECCCounters{}, fmt.Errorf("unexpected ECC counter columns: %q", strings.TrimSpace(string(out))) + } + corrected, err1 := strconv.ParseUint(strings.TrimSpace(fields[0]), 10, 64) + uncorrected, err2 := strconv.ParseUint(strings.TrimSpace(fields[1]), 10, 64) + if err1 != nil || err2 != nil { + // ECC may be disabled on this GPU — return zero counters silently. + return BenchmarkECCCounters{}, nil + } + return BenchmarkECCCounters{Corrected: corrected, Uncorrected: uncorrected}, nil +} + +func diffECCCounters(before, after BenchmarkECCCounters) BenchmarkECCCounters { + return BenchmarkECCCounters{ + Corrected: saturatingSub(after.Corrected, before.Corrected), + Uncorrected: saturatingSub(after.Uncorrected, before.Uncorrected), + } +} + +func queryActiveComputeApps(gpuIndices []int) ([]string, error) { + args := []string{ + "--query-compute-apps=gpu_uuid,pid,process_name", + "--format=csv,noheader,nounits", + } + if len(gpuIndices) > 0 { + args = append([]string{"--id=" + joinIndexList(gpuIndices)}, args...) + } + out, err := satExecCommand("nvidia-smi", args...).Output() + if err != nil { + return nil, err + } + var lines []string + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + lines = append(lines, line) + } + return lines, nil +} + +func finalizeBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkGPUResult { + if gpu.Status == "" { + gpu.Status = "OK" + } + if gpu.Scores.CompositeScore == 0 { + gpu.Scores.CompositeScore = gpu.Scores.ComputeScore + } + return gpu +} + +func buildBenchmarkFindings(result NvidiaBenchmarkResult) []string { + var findings []string + + passed := 0 + for _, gpu := range result.GPUs { + if gpu.Status == "OK" { + passed++ + } + } + total := len(result.GPUs) + if total > 0 { + if passed == total { + findings = append(findings, fmt.Sprintf("All %d GPU(s) passed the benchmark.", total)) + } else { + findings = append(findings, fmt.Sprintf("%d of %d GPU(s) passed the benchmark.", passed, total)) + } + } + + if result.Normalization.Status != "full" { + findings = append(findings, "Environment normalization was partial; compare results with caution.") + } + for _, gpu := range result.GPUs { + if gpu.Status == "FAILED" && len(gpu.DegradationReasons) == 0 { + findings = append(findings, fmt.Sprintf("GPU %d failed the benchmark (check verbose.log for details).", gpu.Index)) + continue + } + if len(gpu.DegradationReasons) == 0 && gpu.Status == "OK" { + findings = append(findings, fmt.Sprintf("GPU %d held clocks without observable throttle counters during steady state.", gpu.Index)) + continue + } + for _, reason := range gpu.DegradationReasons { + switch reason { + case "power_capped": + findings = append(findings, fmt.Sprintf( + "[POWER] GPU %d: power cap throttle %.1f%% of steady state — server is not delivering full TDP to the GPU.", + gpu.Index, gpu.Scores.PowerCapThrottlePct)) + case "thermal_limited": + // Hard stop check: thermal throttle while fans are not at maximum. + // This means the server does not see GPU thermals — incompatible config. + if result.Cooling != nil && result.Cooling.FanDutyCycleAvailable && + result.Cooling.P95FanDutyCyclePct < 95 { + findings = append(findings, fmt.Sprintf( + "[HARD STOP] GPU %d: thermal throttle (%.1f%% of time) while fans peaked at only %.0f%% duty cycle — server cooling is not responding to GPU heat load. Configuration is likely incompatible.", + gpu.Index, gpu.Scores.ThermalThrottlePct, result.Cooling.P95FanDutyCyclePct)) + } else { + findings = append(findings, fmt.Sprintf( + "[THERMAL] GPU %d: thermal throttle %.1f%% of steady state.", + gpu.Index, gpu.Scores.ThermalThrottlePct)) + } + case "sync_boost_limited": + findings = append(findings, fmt.Sprintf( + "[SYNC] GPU %d: sync boost throttle %.1f%% of steady state — GPUs are constraining each other's clocks.", + gpu.Index, gpu.Scores.SyncBoostThrottlePct)) + case "low_sm_clock_vs_target": + findings = append(findings, fmt.Sprintf("GPU %d average SM clock stayed below the requested lock target.", gpu.Index)) + case "variance_too_high": + findings = append(findings, fmt.Sprintf("GPU %d showed unstable clocks/power over the benchmark window.", gpu.Index)) + case "normalization_partial": + findings = append(findings, fmt.Sprintf("GPU %d ran without full benchmark normalization.", gpu.Index)) + case "power_limit_derated": + findings = append(findings, fmt.Sprintf("[POWER] GPU %d could not sustain full TDP in this server; benchmark ran at reduced limit %.0f W.", gpu.Index, gpu.PowerLimitW)) + case "ecc_uncorrected_errors": + findings = append(findings, fmt.Sprintf( + "[HARD STOP] GPU %d: %d uncorrected ECC error(s) detected — possible hardware fault. Do not use in production.", + gpu.Index, gpu.ECC.Uncorrected)) + case "ecc_corrected_errors": + findings = append(findings, fmt.Sprintf( + "[WARNING] GPU %d: %d corrected ECC error(s) — possible DRAM degradation, monitor closely.", + gpu.Index, gpu.ECC.Corrected)) + } + } + // Temperature headroom checks — independent of throttle counters. + // Shutdown and slowdown thresholds are per-GPU from nvidia-smi -q; + // fall back to 90°C / 80°C when unavailable. + if gpu.Steady.P95TempC > 0 { + shutdownTemp := gpu.ShutdownTempC + if shutdownTemp <= 0 { + shutdownTemp = 90 + } + slowdownTemp := gpu.SlowdownTempC + if slowdownTemp <= 0 { + slowdownTemp = 80 + } + headroom := shutdownTemp - gpu.Steady.P95TempC + switch { + case headroom < 10: + findings = append(findings, fmt.Sprintf( + "[HARD STOP] GPU %d: p95 temperature %.1f°C — only %.1f°C from shutdown threshold (%.0f°C). Do not operate.", + gpu.Index, gpu.Steady.P95TempC, headroom, shutdownTemp)) + case gpu.Steady.P95TempC >= slowdownTemp: + findings = append(findings, fmt.Sprintf( + "[THERMAL] GPU %d: p95 temperature %.1f°C exceeds slowdown threshold (%.0f°C) — %.1f°C headroom to shutdown. Operating in degraded reliability zone.", + gpu.Index, gpu.Steady.P95TempC, slowdownTemp, headroom)) + } + } + if gpu.CoolingWarning != "" { + findings = append(findings, fmt.Sprintf( + "GPU %d: %s. Operator action: rerun the benchmark with fan speed manually fixed at 100%% to confirm actual thermal headroom.", + gpu.Index, gpu.CoolingWarning, + )) + } + if len(gpu.PrecisionFailures) > 0 { + findings = append(findings, fmt.Sprintf("GPU %d had incomplete precision coverage: %s.", gpu.Index, strings.Join(gpu.PrecisionFailures, ", "))) + } + if gpu.Backend == "driver-ptx" { + findings = append(findings, fmt.Sprintf("GPU %d used driver PTX fallback; tensor score is intentionally degraded.", gpu.Index)) + } + if gpu.DefaultPowerLimitW > 0 && gpu.PowerLimitW > 0 && gpu.PowerLimitW < gpu.DefaultPowerLimitW*0.95 { + findings = append(findings, fmt.Sprintf( + "GPU %d power limit %.0f W is below default %.0f W (%.0f%%). Performance may be artificially reduced.", + gpu.Index, gpu.PowerLimitW, gpu.DefaultPowerLimitW, gpu.PowerLimitW/gpu.DefaultPowerLimitW*100, + )) + } + // Flag significant TDP deviation (over or under) from calibration. + if gpu.CalibratedPeakPowerW > 0 { + ref := gpu.DefaultPowerLimitW + if ref <= 0 { + ref = gpu.PowerLimitW + } + if ref > 0 { + deviationPct := (gpu.CalibratedPeakPowerW - ref) / ref * 100 + switch { + case deviationPct < -10: + findings = append(findings, fmt.Sprintf( + "GPU %d reached only %.0f W (%.0f%% of rated %.0f W) under targeted_power. Check power delivery or cooling.", + gpu.Index, gpu.CalibratedPeakPowerW, gpu.CalibratedPeakPowerW/ref*100, ref, + )) + case deviationPct > 5: + findings = append(findings, fmt.Sprintf( + "GPU %d exceeded rated TDP: %.0f W measured vs %.0f W rated (+%.0f%%). Power limit may not be enforced correctly.", + gpu.Index, gpu.CalibratedPeakPowerW, ref, deviationPct, + )) + } + } + } + } + if result.Interconnect != nil && result.Interconnect.Supported { + findings = append(findings, fmt.Sprintf("Multi-GPU all_reduce max bus bandwidth: %.1f GB/s.", result.Interconnect.MaxBusBWGBps)) + } + if cl := result.CPULoad; cl != nil { + switch cl.Status { + case "high": + findings = append(findings, fmt.Sprintf( + "Host CPU load was elevated during the benchmark (avg %.1f%%, max %.1f%%). A competing CPU workload may skew GPU results.", + cl.AvgPct, cl.MaxPct, + )) + case "unstable": + findings = append(findings, fmt.Sprintf( + "Host CPU load was erratic during the benchmark (avg %.1f%%, p95 %.1f%%). Results may be less reproducible.", + cl.AvgPct, cl.P95Pct, + )) + } + } + if sp := result.ServerPower; sp != nil && sp.Available && sp.GPUReportedSumW > 0 { + dcmiPartial := detectDCMIPartialCoverage(sp) + if sp.ReportingRatio < 0.75 && !dcmiPartial { + findings = append(findings, fmt.Sprintf( + "GPU power reporting may be unreliable: server delta %.0f W vs GPU-reported %.0f W (ratio %.2f). GPU telemetry likely over-reports actual consumption. Composite scores have been penalized accordingly.", + sp.DeltaW, sp.GPUReportedSumW, sp.ReportingRatio, + )) + } else if sp.ReportingRatio < 0.75 && dcmiPartial { + findings = append(findings, fmt.Sprintf( + "IPMI DCMI covers partial PSU set (DCMI/SDR coverage %.0f%%): ratio %.2f reflects DCMI under-reporting, not GPU inaccuracy. GPU telemetry is the reliable power source; use SDR-based ratio for server-side accuracy.", + sp.DCMICoverageRatio*100, sp.ReportingRatio, + )) + } else if sp.ReportingRatio > 1.25 { + findings = append(findings, fmt.Sprintf( + "Server power delta %.0f W exceeds GPU-reported sum %.0f W by %.0f%%. Other components (CPU, NVMe, networking) may be drawing substantial power under GPU load.", + sp.DeltaW, sp.GPUReportedSumW, (sp.ReportingRatio-1)*100, + )) + } + } + return dedupeStrings(findings) +} + +func benchmarkOverallStatus(result NvidiaBenchmarkResult) string { + if len(result.GPUs) == 0 { + return "FAILED" + } + hasOK := false + hasPartial := result.Normalization.Status != "full" + for _, gpu := range result.GPUs { + switch gpu.Status { + case "OK": + hasOK = true + case "PARTIAL", "UNSUPPORTED": + hasPartial = true + } + } + if !hasOK { + return "FAILED" + } + if hasPartial { + return "PARTIAL" + } + return "OK" +} + +func findBenchmarkNormalization(items []BenchmarkNormalizationGPU, idx int) *BenchmarkNormalizationGPU { + for i := range items { + if items[i].Index == idx { + return &items[i] + } + } + return nil +} + +func classifySATErrorStatus(out []byte, err error) string { + status, _ := classifySATResult("benchmark", out, err) + if status == "UNSUPPORTED" { + return "UNSUPPORTED" + } + return "FAILED" +} + +func parseBenchmarkFloat(raw string) float64 { + raw = strings.TrimSpace(raw) + if raw == "" || strings.EqualFold(raw, "n/a") || strings.EqualFold(raw, "[not supported]") { + return 0 + } + value, _ := strconv.ParseFloat(raw, 64) + return value +} + +func parseBenchmarkUint64(raw string) uint64 { + raw = strings.TrimSpace(raw) + if raw == "" || strings.EqualFold(raw, "n/a") || strings.EqualFold(raw, "[not supported]") { + return 0 + } + value, _ := strconv.ParseUint(raw, 10, 64) + return value +} + +func benchmarkMean(values []float64) float64 { + if len(values) == 0 { + return 0 + } + var sum float64 + for _, value := range values { + sum += value + } + return sum / float64(len(values)) +} + +func benchmarkPercentile(values []float64, p float64) float64 { + if len(values) == 0 { + return 0 + } + copyValues := append([]float64(nil), values...) + sort.Float64s(copyValues) + if len(copyValues) == 1 { + return copyValues[0] + } + rank := (p / 100.0) * float64(len(copyValues)-1) + lower := int(math.Floor(rank)) + upper := int(math.Ceil(rank)) + if lower == upper { + return copyValues[lower] + } + frac := rank - float64(lower) + return copyValues[lower] + (copyValues[upper]-copyValues[lower])*frac +} + +func benchmarkCV(values []float64) float64 { + if len(values) == 0 { + return 0 + } + mean := benchmarkMean(values) + if mean == 0 { + return 0 + } + var variance float64 + for _, value := range values { + diff := value - mean + variance += diff * diff + } + variance /= float64(len(values)) + return math.Sqrt(variance) / mean * 100 +} + +func benchmarkClockDrift(values []float64) float64 { + if len(values) < 4 { + return 0 + } + window := len(values) / 4 + if window < 1 { + window = 1 + } + head := benchmarkMean(values[:window]) + tail := benchmarkMean(values[len(values)-window:]) + if head <= 0 || tail >= head { + return 0 + } + return ((head - tail) / head) * 100 +} + +func benchmarkMax(values []float64) float64 { + var max float64 + for i, value := range values { + if i == 0 || value > max { + max = value + } + } + return max +} + +func clampScore(value float64) float64 { + switch { + case value < 0: + return 0 + case value > 100: + return 100 + default: + return value + } +} + +func dedupeStrings(values []string) []string { + if len(values) == 0 { + return nil + } + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + return out +} + +func saturatingSub(after, before uint64) uint64 { + if after <= before { + return 0 + } + return after - before +} + +func maxInt(a, b int) int { + if a > b { + return a + } + return b +} + +// detectDCMIPartialCoverage returns true when IPMI DCMI under-reports actual +// server power by comparing DCMI readings against SDR PSUx_POWER_IN sensor sums. +// +// Primary check: DCMI_idle / SDR_PSU_IN_idle — most reliable because GPU load +// is zero, so both sources measure the same server state. A ratio below 0.7 +// means DCMI misses ≥30% of installed PSUs (e.g. 0.50 = sees 2 of 4 PSUs). +// +// Fallback: DCMI_loaded / SDR_PSU_IN_loaded — less precise (GPU load may +// affect different PSUs differently) but still useful when idle SDR is absent. +// +// Returns false when SDR data is unavailable (server has no PSUx_POWER_IN +// sensors); the heuristic detectIPMISaturationFallback is used in that case. diff --git a/audit/internal/platform/benchmark_setup.go b/audit/internal/platform/benchmark_setup.go new file mode 100644 index 0000000..f6f26de --- /dev/null +++ b/audit/internal/platform/benchmark_setup.go @@ -0,0 +1,396 @@ +package platform + +import ( + "context" + "encoding/csv" + "fmt" + "math" + "os" + "regexp" + "strconv" + "strings" + "time" +) + +func normalizeNvidiaBenchmarkOptionsForBenchmark(opts NvidiaBenchmarkOptions) NvidiaBenchmarkOptions { + switch strings.TrimSpace(strings.ToLower(opts.Profile)) { + case NvidiaBenchmarkProfileStability: + opts.Profile = NvidiaBenchmarkProfileStability + case NvidiaBenchmarkProfileOvernight: + opts.Profile = NvidiaBenchmarkProfileOvernight + default: + opts.Profile = NvidiaBenchmarkProfileStandard + } + if opts.SizeMB < 0 { + opts.SizeMB = 0 + } + opts.ServerPowerSource = normalizeBenchmarkPowerSource(opts.ServerPowerSource) + opts.GPUIndices = dedupeSortedIndices(opts.GPUIndices) + opts.ExcludeGPUIndices = dedupeSortedIndices(opts.ExcludeGPUIndices) + return opts +} + +func resolveBenchmarkProfile(profile string) benchmarkProfileSpec { + switch strings.TrimSpace(strings.ToLower(profile)) { + case NvidiaBenchmarkProfileStability: + return benchmarkProfileSpec{Name: NvidiaBenchmarkProfileStability, BaselineSec: 30, WarmupSec: 120, SteadySec: 3600, NCCLSec: 300, CooldownSec: 0} + case NvidiaBenchmarkProfileOvernight: + return benchmarkProfileSpec{Name: NvidiaBenchmarkProfileOvernight, BaselineSec: 60, WarmupSec: 180, SteadySec: 27000, NCCLSec: 600, CooldownSec: 0} + default: + return benchmarkProfileSpec{Name: NvidiaBenchmarkProfileStandard, BaselineSec: 15, WarmupSec: 45, SteadySec: 480, NCCLSec: 180, CooldownSec: 0} + } +} + +// benchmarkGPUInfoQuery describes a nvidia-smi --query-gpu field set to try. +// Fields are tried in order; the first successful query wins. Extended fields +// (attribute.multiprocessor_count, power.default_limit) are not supported on +// all driver versions, so we fall back to the base set if the full query fails. +// The minimal fallback omits clock fields entirely — clocks.max.* returns +// exit status 2 on some GPU generations (e.g. Blackwell); missing data is +// then recovered from nvidia-smi -q. +var benchmarkGPUInfoQueries = []struct { + fields string + extended bool // whether this query includes optional extended fields + minimal bool // clock fields omitted; max clocks must be filled separately +}{ + { + fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics,attribute.multiprocessor_count,power.default_limit", + extended: true, + }, + { + fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics", + extended: false, + }, + { + fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit", + minimal: true, + }, +} + +// enrichGPUInfoWithNvidiaSMIQ fills benchmark GPU metadata from nvidia-smi -q +// for fields that may be missing from --query-gpu on some driver versions. +func enrichGPUInfoWithNvidiaSMIQ(infoByIndex map[int]benchmarkGPUInfo, nvsmiQ []byte) { + if len(infoByIndex) == 0 || len(nvsmiQ) == 0 { + return + } + + // Build bus_id → index map for matching verbose sections to GPU indices. + busToBenchIdx := make(map[string]int, len(infoByIndex)) + for idx, info := range infoByIndex { + if info.BusID != "" { + // nvidia-smi -q uses "GPU 00000000:4E:00.0" (8-digit domain), + // while --query-gpu returns the same format; normalise to lower. + busToBenchIdx[strings.ToLower(strings.TrimSpace(info.BusID))] = idx + } + } + + // Split the verbose output into per-GPU sections on "^GPU " lines. + gpuSectionRe := regexp.MustCompile(`(?m)^GPU\s+([\dA-Fa-f:\.]+)`) + maxGfxRe := regexp.MustCompile(`(?i)Max Clocks[\s\S]*?Graphics\s*:\s*(\d+)\s*MHz`) + maxMemRe := regexp.MustCompile(`(?i)Max Clocks[\s\S]*?Memory\s*:\s*(\d+)\s*MHz`) + defaultPwrRe := regexp.MustCompile(`(?i)Default Power Limit\s*:\s*([0-9.]+)\s*W`) + currentPwrRe := regexp.MustCompile(`(?i)Current Power Limit\s*:\s*([0-9.]+)\s*W`) + minPwrRe := regexp.MustCompile(`(?i)Min Power Limit\s*:\s*([0-9.]+)\s*W`) + maxPwrRe := regexp.MustCompile(`(?i)Max Power Limit\s*:\s*([0-9.]+)\s*W`) + smCountRe := regexp.MustCompile(`(?i)Multiprocessor Count\s*:\s*(\d+)`) + shutdownTempRe := regexp.MustCompile(`(?i)GPU Shutdown Temp\s*:\s*(\d+)\s*C`) + slowdownTempRe := regexp.MustCompile(`(?i)GPU Slowdown Temp\s*:\s*(\d+)\s*C`) + + sectionStarts := gpuSectionRe.FindAllSubmatchIndex(nvsmiQ, -1) + for i, loc := range sectionStarts { + busID := strings.ToLower(string(nvsmiQ[loc[2]:loc[3]])) + benchIdx, ok := busToBenchIdx[busID] + if !ok { + // Bus IDs from verbose output may have a different domain prefix; + // try suffix match on the slot portion (XX:XX.X). + for k, v := range busToBenchIdx { + if strings.HasSuffix(k, busID) || strings.HasSuffix(busID, k) { + benchIdx = v + ok = true + break + } + } + } + if !ok { + continue + } + + end := len(nvsmiQ) + if i+1 < len(sectionStarts) { + end = sectionStarts[i+1][0] + } + section := nvsmiQ[loc[0]:end] + + info := infoByIndex[benchIdx] + + if info.MaxGraphicsClockMHz == 0 { + if m := maxGfxRe.FindSubmatch(section); m != nil { + if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil { + info.MaxGraphicsClockMHz = v + } + } + } + if info.MaxMemoryClockMHz == 0 { + if m := maxMemRe.FindSubmatch(section); m != nil { + if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil { + info.MaxMemoryClockMHz = v + } + } + } + if info.DefaultPowerLimitW == 0 { + if m := defaultPwrRe.FindSubmatch(section); m != nil { + if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 { + info.DefaultPowerLimitW = v + } + } + } + if info.PowerLimitW == 0 { + if m := currentPwrRe.FindSubmatch(section); m != nil { + if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 { + info.PowerLimitW = v + } + } + } + if info.MinPowerLimitW == 0 { + if m := minPwrRe.FindSubmatch(section); m != nil { + if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 { + info.MinPowerLimitW = v + } + } + } + if info.MaxPowerLimitW == 0 { + if m := maxPwrRe.FindSubmatch(section); m != nil { + if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 { + info.MaxPowerLimitW = v + } + } + } + if info.MultiprocessorCount == 0 { + if m := smCountRe.FindSubmatch(section); m != nil { + if v, err := strconv.Atoi(string(m[1])); err == nil && v > 0 { + info.MultiprocessorCount = v + } + } + } + if info.ShutdownTempC == 0 { + if m := shutdownTempRe.FindSubmatch(section); m != nil { + if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 { + info.ShutdownTempC = v + } + } + } + if info.SlowdownTempC == 0 { + if m := slowdownTempRe.FindSubmatch(section); m != nil { + if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 { + info.SlowdownTempC = v + } + } + } + infoByIndex[benchIdx] = info + } +} + +func queryBenchmarkGPUInfo(gpuIndices []int) (map[int]benchmarkGPUInfo, error) { + var lastErr error + for _, q := range benchmarkGPUInfoQueries { + args := []string{ + "--query-gpu=" + q.fields, + "--format=csv,noheader,nounits", + } + if len(gpuIndices) > 0 { + args = append([]string{"--id=" + joinIndexList(gpuIndices)}, args...) + } + out, err := satExecCommand("nvidia-smi", args...).Output() + if err != nil { + lastErr = fmt.Errorf("nvidia-smi gpu info (%s): %w", q.fields[:min(len(q.fields), 40)], err) + continue + } + + r := csv.NewReader(strings.NewReader(string(out))) + r.TrimLeadingSpace = true + r.FieldsPerRecord = -1 + rows, err := r.ReadAll() + if err != nil { + lastErr = fmt.Errorf("parse nvidia-smi gpu info: %w", err) + continue + } + + minFields := 6 + if !q.minimal { + minFields = 9 + } + infoByIndex := make(map[int]benchmarkGPUInfo, len(rows)) + for _, row := range rows { + if len(row) < minFields { + continue + } + idx, err := strconv.Atoi(strings.TrimSpace(row[0])) + if err != nil { + continue + } + info := benchmarkGPUInfo{ + Index: idx, + UUID: strings.TrimSpace(row[1]), + Name: strings.TrimSpace(row[2]), + BusID: strings.TrimSpace(row[3]), + VBIOS: strings.TrimSpace(row[4]), + PowerLimitW: parseBenchmarkFloat(row[5]), + } + if !q.minimal { + info.MaxGraphicsClockMHz = parseBenchmarkFloat(row[6]) + info.MaxMemoryClockMHz = parseBenchmarkFloat(row[7]) + if len(row) >= 9 { + info.BaseGraphicsClockMHz = parseBenchmarkFloat(row[8]) + } + if q.extended { + if len(row) >= 10 { + info.MultiprocessorCount = int(parseBenchmarkFloat(row[9])) + } + if len(row) >= 11 { + info.DefaultPowerLimitW = parseBenchmarkFloat(row[10]) + } + } + } + infoByIndex[idx] = info + } + return infoByIndex, nil + } + return nil, lastErr +} + +func applyBenchmarkNormalization(ctx context.Context, verboseLog string, gpuIndices []int, infoByIndex map[int]benchmarkGPUInfo, result *NvidiaBenchmarkResult) []benchmarkRestoreAction { + if os.Geteuid() != 0 { + result.Normalization.Status = "partial" + result.Normalization.Notes = append(result.Normalization.Notes, "benchmark normalization skipped: root privileges are required for persistence mode and clock locks") + for _, idx := range gpuIndices { + result.Normalization.GPUs = append(result.Normalization.GPUs, BenchmarkNormalizationGPU{ + Index: idx, + Notes: []string{"normalization skipped: root privileges are required"}, + }) + } + return nil + } + + var restore []benchmarkRestoreAction + for _, idx := range gpuIndices { + rec := BenchmarkNormalizationGPU{Index: idx} + if _, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("normalize-gpu-%d-pm", idx), []string{"nvidia-smi", "-i", strconv.Itoa(idx), "-pm", "1"}, nil, nil); err != nil { + rec.PersistenceMode = "failed" + rec.Notes = append(rec.Notes, "failed to enable persistence mode") + result.Normalization.Status = "partial" + } else { + rec.PersistenceMode = "applied" + } + + if info, ok := infoByIndex[idx]; ok && info.MaxGraphicsClockMHz > 0 { + target := int(math.Round(info.MaxGraphicsClockMHz)) + if out, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("normalize-gpu-%d-lgc", idx), []string{"nvidia-smi", "-i", strconv.Itoa(idx), "-lgc", strconv.Itoa(target)}, nil, nil); err != nil { + rec.GPUClockLockStatus = "failed" + rec.Notes = append(rec.Notes, "graphics clock lock failed: "+strings.TrimSpace(string(out))) + result.Normalization.Status = "partial" + } else { + rec.GPUClockLockStatus = "applied" + rec.GPUClockLockMHz = float64(target) + idxCopy := idx + restore = append(restore, benchmarkRestoreAction{name: fmt.Sprintf("gpu-%d-rgc", idxCopy), fn: func() { + _, _ = runSATCommandCtx(context.Background(), verboseLog, fmt.Sprintf("restore-gpu-%d-rgc", idxCopy), []string{"nvidia-smi", "-i", strconv.Itoa(idxCopy), "-rgc"}, nil, nil) + }}) + } + } else { + rec.GPUClockLockStatus = "skipped" + rec.Notes = append(rec.Notes, "graphics clock lock skipped: gpu inventory unavailable or MaxGraphicsClockMHz=0") + result.Normalization.Status = "partial" + } + + if info, ok := infoByIndex[idx]; ok && info.MaxMemoryClockMHz > 0 { + target := int(math.Round(info.MaxMemoryClockMHz)) + out, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("normalize-gpu-%d-lmc", idx), []string{"nvidia-smi", "-i", strconv.Itoa(idx), "-lmc", strconv.Itoa(target)}, nil, nil) + switch { + case err == nil: + rec.MemoryClockLockStatus = "applied" + rec.MemoryClockLockMHz = float64(target) + idxCopy := idx + restore = append(restore, benchmarkRestoreAction{name: fmt.Sprintf("gpu-%d-rmc", idxCopy), fn: func() { + _, _ = runSATCommandCtx(context.Background(), verboseLog, fmt.Sprintf("restore-gpu-%d-rmc", idxCopy), []string{"nvidia-smi", "-i", strconv.Itoa(idxCopy), "-rmc"}, nil, nil) + }}) + case strings.Contains(strings.ToLower(string(out)), "deferred") || strings.Contains(strings.ToLower(string(out)), "not supported"): + rec.MemoryClockLockStatus = "unsupported" + rec.Notes = append(rec.Notes, "memory clock lock unsupported on this GPU/driver path") + result.Normalization.Status = "partial" + default: + rec.MemoryClockLockStatus = "failed" + rec.Notes = append(rec.Notes, "memory clock lock failed: "+strings.TrimSpace(string(out))) + result.Normalization.Status = "partial" + } + } + + result.Normalization.GPUs = append(result.Normalization.GPUs, rec) + } + return restore +} + +func collectBenchmarkSamples(ctx context.Context, durationSec int, gpuIndices []int) ([]GPUMetricRow, error) { + if durationSec <= 0 { + return nil, nil + } + deadline := time.Now().Add(time.Duration(durationSec) * time.Second) + var rows []GPUMetricRow + start := time.Now() + for { + if ctx.Err() != nil { + return rows, ctx.Err() + } + samples, err := sampleBenchmarkTelemetry(gpuIndices) + if err == nil { + elapsed := time.Since(start).Seconds() + for i := range samples { + samples[i].ElapsedSec = elapsed + } + rows = append(rows, samples...) + } + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return rows, ctx.Err() + case <-time.After(time.Second): + } + } + return rows, nil +} + +func runBenchmarkCommandWithMetrics(ctx context.Context, verboseLog, name string, cmd []string, env []string, gpuIndices []int, logFunc func(string)) ([]byte, []GPUMetricRow, error) { + stopCh := make(chan struct{}) + doneCh := make(chan struct{}) + var metricRows []GPUMetricRow + start := time.Now() + + go func() { + defer close(doneCh) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-stopCh: + return + case <-ticker.C: + samples, err := sampleBenchmarkTelemetry(gpuIndices) + if err != nil { + continue + } + elapsed := time.Since(start).Seconds() + for i := range samples { + samples[i].ElapsedSec = elapsed + } + metricRows = append(metricRows, samples...) + } + } + }() + + out, err := runSATCommandCtx(ctx, verboseLog, name, cmd, env, logFunc) + close(stopCh) + <-doneCh + + return out, metricRows, err +} diff --git a/audit/internal/platform/nvidia_bandwidth_socket.go b/audit/internal/platform/nvidia_bandwidth_socket.go index 74b46ae..07b2f89 100644 --- a/audit/internal/platform/nvidia_bandwidth_socket.go +++ b/audit/internal/platform/nvidia_bandwidth_socket.go @@ -11,12 +11,8 @@ import ( // satReadFile is a seam for tests to fake sysfs reads (numa_node files). var satReadFile = os.ReadFile -// gpuBandwidthSocketGroups splits gpuIndices into per-socket groups (ordered -// by ascending NUMA node ID) for RunNvidiaBandwidthPack. A cross-socket -// peer-to-peer path is a different (and, on platforms without NVLink, far -// less exercised) fault domain than a same-socket one, so testing each -// socket's GPUs in isolation before testing all of them together isolates -// whether a failure is specific to the cross-socket path. +// gpuBandwidthSocketGroups splits gpuIndices into NUMA-locality groups, +// ordered by ascending Linux NUMA node ID, for RunNvidiaBandwidthPack. // // Falls back to a single group containing all of gpuIndices — i.e. no split // — whenever the NUMA node can't be resolved for every GPU, or all resolve @@ -31,22 +27,17 @@ func gpuBandwidthSocketGroups(gpuIndices []int, logFunc func(string)) [][]int { } byNode := map[int][]int{} - var unresolved []int for _, idx := range gpuIndices { node, ok := nodes[idx] if !ok { if logFunc != nil { - logFunc(fmt.Sprintf("nvbandwidth: no NUMA node resolved for GPU %d; will fold it into a resolved socket group instead of dropping the split", idx)) + logFunc(fmt.Sprintf("nvbandwidth: no NUMA node resolved for GPU %d; running all GPUs as one group", idx)) } - unresolved = append(unresolved, idx) - continue + return [][]int{gpuIndices} } byNode[node] = append(byNode[node], idx) } - // Fewer than two resolved sockets means there's nothing to split either - // way: every GPU's node is unknown, or every resolved GPU shares one - // socket. A single unresolved GPU among an otherwise clean multi-socket - // system shouldn't cost us the split, so only bail out here. + // Fewer than two NUMA nodes means there is nothing meaningful to split. if len(byNode) < 2 { return [][]int{gpuIndices} } @@ -61,14 +52,6 @@ func gpuBandwidthSocketGroups(gpuIndices []int, logFunc func(string)) [][]int { for _, node := range sortedNodes { groups = append(groups, dedupeSortedIndices(byNode[node])) } - if len(unresolved) > 0 { - // Fold into the last group rather than running unresolved GPUs in a - // group of their own — a lone GPU can't run a GPU-to-GPU bandwidth - // test by itself, and the point of the split is to isolate the - // sockets we *do* know about, not to also isolate the unknown one. - last := len(groups) - 1 - groups[last] = dedupeSortedIndices(append(groups[last], unresolved...)) - } return groups } @@ -107,10 +90,16 @@ func gpuNUMANodes(gpuIndices []int) (map[int]int, error) { return nodes, nil } -// normalizeNvidiaBDF converts nvidia-smi's 8-hex-digit-domain PCI bus ID -// ("00000000:05:00.0") to the 4-hex-digit-domain form sysfs paths use -// ("0000:05:00.0"). +// normalizeNvidiaBDF converts nvidia-smi's PCI bus ID to the exact form the +// sysfs paths under /sys/bus/pci/devices use: an 8-hex-digit domain is +// narrowed to 4 digits ("00000000:05:00.0" -> "0000:05:00.0"), and the hex +// is lower-cased ("0000:CB:00.0" -> "0000:cb:00.0"). nvidia-smi upper-cases +// the bus/device hex; sysfs directory names are always lower-case, so +// without this a BDF containing a hex letter (e.g. GPUs on bus 4b/cb/cf) +// would never match a sysfs entry and every numa_node / link-speed read +// would silently fail. func normalizeNvidiaBDF(busID string) string { + busID = strings.ToLower(strings.TrimSpace(busID)) domain, rest, ok := strings.Cut(busID, ":") if !ok { return busID diff --git a/audit/internal/platform/nvidia_bandwidth_socket_test.go b/audit/internal/platform/nvidia_bandwidth_socket_test.go index 8ae8b32..d46529d 100644 --- a/audit/internal/platform/nvidia_bandwidth_socket_test.go +++ b/audit/internal/platform/nvidia_bandwidth_socket_test.go @@ -33,10 +33,10 @@ func fakeNUMANodes(t *testing.T, byBDF map[string]string) { } func TestGPUNUMANodesResolvesFromPCIBusID(t *testing.T) { - fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:F4:00.0\n") + fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:f4:00.0\n") fakeNUMANodes(t, map[string]string{ "0000:05:00.0": "0\n", - "0000:F4:00.0": "1\n", + "0000:f4:00.0": "1\n", }) nodes, err := gpuNUMANodes([]int{0, 1}) @@ -69,14 +69,14 @@ func TestGPUNUMANodesSkipsUnresolvableNode(t *testing.T) { } func TestGPUBandwidthSocketGroupsSplitsBySocket(t *testing.T) { - fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:F4:00.0\n5, 00000000:F5:00.0\n") + fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:f4:00.0\n5, 00000000:f5:00.0\n") fakeNUMANodes(t, map[string]string{ "0000:05:00.0": "0\n", "0000:06:00.0": "0\n", "0000:76:00.0": "0\n", "0000:77:00.0": "0\n", - "0000:F4:00.0": "1\n", - "0000:F5:00.0": "1\n", + "0000:f4:00.0": "1\n", + "0000:f5:00.0": "1\n", }) groups := gpuBandwidthSocketGroups([]int{0, 1, 2, 3, 4, 5}, nil) @@ -91,30 +91,20 @@ func TestGPUBandwidthSocketGroupsSplitsBySocket(t *testing.T) { } } -func TestGPUBandwidthSocketGroupsFoldsUnresolvedIntoLastGroup(t *testing.T) { - // GPU 4's NUMA node fails to resolve (e.g. a flaky sysfs read), but the - // other 5 GPUs still clearly span two sockets — the split should survive - // and GPU 4 should ride along with the last group rather than being - // tested alone or collapsing the whole thing to one pass. - fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:F4:00.0\n5, 00000000:F5:00.0\n") +func TestGPUBandwidthSocketGroupsFallsBackWhenAnyNodeIsUnresolved(t *testing.T) { + fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:f4:00.0\n5, 00000000:f5:00.0\n") fakeNUMANodes(t, map[string]string{ "0000:05:00.0": "0\n", "0000:06:00.0": "0\n", "0000:76:00.0": "0\n", "0000:77:00.0": "0\n", // GPU 4 (F4:00.0) deliberately missing. - "0000:F5:00.0": "1\n", + "0000:f5:00.0": "1\n", }) groups := gpuBandwidthSocketGroups([]int{0, 1, 2, 3, 4, 5}, nil) - if len(groups) != 2 { - t.Fatalf("groups=%v want 2 groups", groups) - } - if joinIndexList(groups[0]) != "0,1,2,3" { - t.Fatalf("groups[0]=%v want 0,1,2,3", groups[0]) - } - if joinIndexList(groups[1]) != "4,5" { - t.Fatalf("groups[1]=%v want 4,5 (unresolved GPU 4 folded into last group)", groups[1]) + if len(groups) != 1 || joinIndexList(groups[0]) != "0,1,2,3,4,5" { + t.Fatalf("groups=%v want single fallback group", groups) } } @@ -161,6 +151,8 @@ func TestNormalizeNvidiaBDF(t *testing.T) { cases := map[string]string{ "00000000:05:00.0": "0000:05:00.0", "0000:05:00.0": "0000:05:00.0", + "00000000:CB:00.0": "0000:cb:00.0", + "0000:4F:00.0": "0000:4f:00.0", "garbage": "garbage", } for in, want := range cases { @@ -171,17 +163,17 @@ func TestNormalizeNvidiaBDF(t *testing.T) { } func TestRunNvidiaBandwidthPackSplitsPerSocketThenAll(t *testing.T) { - fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:F4:00.0\n3, 00000000:F5:00.0\n") + fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:f4:00.0\n3, 00000000:f5:00.0\n") fakeNUMANodes(t, map[string]string{ "0000:05:00.0": "0\n", "0000:06:00.0": "0\n", - "0000:F4:00.0": "1\n", - "0000:F5:00.0": "1\n", + "0000:f4:00.0": "1\n", + "0000:f5:00.0": "1\n", }) dir := t.TempDir() s := &System{} - _, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1, 2, 3}, nil) + _, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1, 2, 3}, true, nil) if err != nil { t.Fatalf("RunNvidiaBandwidthPack error: %v", err) } @@ -211,6 +203,34 @@ func TestRunNvidiaBandwidthPackSplitsPerSocketThenAll(t *testing.T) { } } +func TestRunNvidiaBandwidthPackValidateNeverSplits(t *testing.T) { + // Multi-socket system, but fullMatrix=false (Validate tier): still one + // nvbandwidth pass across every GPU, no per-socket split. + fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:f4:00.0\n3, 00000000:f5:00.0\n") + fakeNUMANodes(t, map[string]string{ + "0000:05:00.0": "0\n", + "0000:06:00.0": "0\n", + "0000:f4:00.0": "1\n", + "0000:f5:00.0": "1\n", + }) + + dir := t.TempDir() + if _, err := (&System{}).RunNvidiaBandwidthPack(nil, dir, []int{0, 1, 2, 3}, false, nil); err != nil { + t.Fatalf("RunNvidiaBandwidthPack error: %v", err) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + runDir := filepath.Join(dir, entries[0].Name()) + if _, err := os.Stat(filepath.Join(runDir, "03-dcgmi-nvbandwidth.log")); err != nil { + t.Fatalf("missing single-pass job output: %v", err) + } + if _, err := os.Stat(filepath.Join(runDir, "03-dcgmi-nvbandwidth-socket0.log")); err == nil { + t.Fatalf("Validate tier must not split per socket") + } +} + func TestRunNvidiaBandwidthPackSinglePassWhenOneSocket(t *testing.T) { fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n") fakeNUMANodes(t, map[string]string{ @@ -220,7 +240,7 @@ func TestRunNvidiaBandwidthPackSinglePassWhenOneSocket(t *testing.T) { dir := t.TempDir() s := &System{} - _, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1}, nil) + _, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1}, true, nil) if err != nil { t.Fatalf("RunNvidiaBandwidthPack error: %v", err) } diff --git a/audit/internal/platform/physical_gpu_vendors_test.go b/audit/internal/platform/physical_gpu_vendors_test.go new file mode 100644 index 0000000..8808670 --- /dev/null +++ b/audit/internal/platform/physical_gpu_vendors_test.go @@ -0,0 +1,53 @@ +package platform + +import ( + "os/exec" + "testing" +) + +func TestPhysicalGPUVendors(t *testing.T) { + tests := []struct { + name string + lspci string + wantNvidia bool + wantAMD bool + }{ + { + name: "nvidia 3D controllers", + lspci: "4b:00.0 3D controller [0302]: NVIDIA Corporation GH100 [H200 NVL] [10de:233b] (rev a1)\n" + + "02:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED [1a03:2000]\n", + wantNvidia: true, + }, + { + name: "amd VGA by vendor id", + lspci: "63:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Navi [1002:744c]\n", + wantAMD: true, + }, + { + name: "amd cpu root complex is not a GPU", + lspci: "00:00.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14a4]\n" + + "00:01.0 IOMMU [0806]: Advanced Micro Devices, Inc. [AMD] Device [1022:14a1]\n", + }, + { + name: "no GPU", + lspci: "02:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED [1a03:2000]\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + old := satExecCommand + satExecCommand = func(name string, args ...string) *exec.Cmd { + if name == "lspci" { + return exec.Command("printf", "%s", tt.lspci) + } + return exec.Command(name, args...) + } + t.Cleanup(func() { satExecCommand = old }) + + nvidia, amd := (&System{}).PhysicalGPUVendors() + if nvidia != tt.wantNvidia || amd != tt.wantAMD { + t.Fatalf("got nvidia=%v amd=%v, want nvidia=%v amd=%v", nvidia, amd, tt.wantNvidia, tt.wantAMD) + } + }) + } +} diff --git a/audit/internal/platform/platform_stress.go b/audit/internal/platform/platform_stress.go index c56ab97..726b10e 100644 --- a/audit/internal/platform/platform_stress.go +++ b/audit/internal/platform/platform_stress.go @@ -1,9 +1,7 @@ package platform import ( - "archive/tar" "bytes" - "compress/gzip" "context" "encoding/csv" "fmt" @@ -532,44 +530,3 @@ func containsComponent(components []string, name string) bool { } return false } - -func packPlatformDir(dir, dest string) error { - f, err := os.Create(dest) - if err != nil { - return err - } - defer f.Close() - gz := gzip.NewWriter(f) - defer gz.Close() - tw := tar.NewWriter(gz) - defer tw.Close() - - entries, err := os.ReadDir(dir) - if err != nil { - return err - } - base := filepath.Base(dir) - for _, e := range entries { - if e.IsDir() { - continue - } - fpath := filepath.Join(dir, e.Name()) - data, err := os.ReadFile(fpath) - if err != nil { - continue - } - hdr := &tar.Header{ - Name: filepath.Join(base, e.Name()), - Size: int64(len(data)), - Mode: 0644, - ModTime: time.Now(), - } - if err := tw.WriteHeader(hdr); err != nil { - return err - } - if _, err := tw.Write(data); err != nil { - return err - } - } - return nil -} diff --git a/audit/internal/platform/sat.go b/audit/internal/platform/sat.go index 9dde5f3..da07c3a 100644 --- a/audit/internal/platform/sat.go +++ b/audit/internal/platform/sat.go @@ -1,10 +1,8 @@ package platform import ( - "archive/tar" "bufio" "bytes" - "compress/gzip" "context" "errors" "fmt" @@ -194,774 +192,6 @@ func streamExecOutput(cmd *exec.Cmd, logFunc func(string), livePath string) ([]b } // NvidiaGPU holds basic GPU info from nvidia-smi. -type NvidiaGPU struct { - Index int `json:"index"` - Name string `json:"name"` - MemoryMB int `json:"memory_mb"` -} - -type NvidiaGPUStatus struct { - Index int `json:"index"` - Name string `json:"name"` - BDF string `json:"bdf,omitempty"` - Serial string `json:"serial,omitempty"` - Status string `json:"status"` - RawLine string `json:"raw_line,omitempty"` - NeedsReset bool `json:"needs_reset"` - ParseFailure bool `json:"parse_failure,omitempty"` -} - -type nvidiaGPUHealth struct { - Index int - Name string - NeedsReset bool - RawLine string - ParseFailure bool -} - -type nvidiaGPUStatusFile struct { - Index int - Name string - RunStatus string - Reason string - Health string - HealthRaw string - Observed bool - Selected bool - FailingJob string -} - -// AMDGPUInfo holds basic info about an AMD GPU from rocm-smi. -type AMDGPUInfo struct { - Index int `json:"index"` - Name string `json:"name"` -} - -// DetectGPUVendor returns "nvidia" if /dev/nvidia0 exists, "amd" if /dev/kfd exists, or "" otherwise. -func (s *System) DetectGPUVendor() string { - if _, err := os.Stat("/dev/nvidia0"); err == nil { - return "nvidia" - } - if _, err := os.Stat("/dev/kfd"); err == nil { - return "amd" - } - if raw, err := exec.Command("lspci", "-nn").Output(); err == nil { - // Only match AMD GPU device classes [0300]=VGA, [0302]=3D controller, [0380]=Display. - // AMD CPUs also appear in lspci as "Advanced Micro Devices" (Root Complex, IOMMU, etc.) - // so matching vendor alone causes false positives on AMD CPU servers without GPUs. - for _, line := range strings.Split(strings.ToLower(string(raw)), "\n") { - if !strings.Contains(line, "advanced micro devices") && !strings.Contains(line, "amd/ati") { - continue - } - if strings.Contains(line, "[0300]") || strings.Contains(line, "[0302]") || strings.Contains(line, "[0380]") { - return "amd" - } - } - } - return "" -} - -// ListAMDGPUs returns AMD GPUs visible to rocm-smi. -func (s *System) ListAMDGPUs() ([]AMDGPUInfo, error) { - out, err := runROCmSMI("--showproductname", "--csv") - if err != nil { - return nil, fmt.Errorf("rocm-smi: %w", err) - } - var gpus []AMDGPUInfo - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(strings.ToLower(line), "device") { - continue - } - parts := strings.SplitN(line, ",", 2) - name := "" - if len(parts) >= 2 { - name = strings.TrimSpace(parts[1]) - } - idx := len(gpus) - gpus = append(gpus, AMDGPUInfo{Index: idx, Name: name}) - } - return gpus, nil -} - -// RunAMDAcceptancePack runs an AMD GPU diagnostic pack using rocm-smi. -func (s *System) RunAMDAcceptancePack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { - return runAcceptancePackCtx(ctx, baseDir, "gpu-amd", []satJob{ - {name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}}, - {name: "02-rocm-smi-showallinfo.log", cmd: []string{"rocm-smi", "--showallinfo"}}, - {name: "03-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}}, - {name: "04-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}}, - }, logFunc) -} - -// RunAMDMemIntegrityPack runs the official RVS MEM module as a validate-style memory integrity test. -func (s *System) RunAMDMemIntegrityPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { - if err := ensureAMDRuntimeReady(); err != nil { - return "", err - } - cfgFile := "/tmp/bee-amd-mem.conf" - cfg := `actions: -- name: mem_integrity - device: all - module: mem - parallel: true - duration: 60000 - copy_matrix: false - target_stress: 90 - matrix_size: 8640 -` - _ = os.WriteFile(cfgFile, []byte(cfg), 0644) - return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-mem", []satJob{ - {name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}}, - {name: "02-rvs-mem.log", cmd: []string{"rvs", "-c", cfgFile}}, - {name: "03-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}}, - }, logFunc) -} - -// RunAMDMemBandwidthPack runs AMD's memory/interconnect bandwidth-oriented tools. -func (s *System) RunAMDMemBandwidthPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { - if err := ensureAMDRuntimeReady(); err != nil { - return "", err - } - cfgFile := "/tmp/bee-amd-babel.conf" - cfg := `actions: -- name: babel_mem_bw - device: all - module: babel - parallel: true - copy_matrix: true - target_stress: 90 - matrix_size: 134217728 -` - _ = os.WriteFile(cfgFile, []byte(cfg), 0644) - return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-bandwidth", []satJob{ - {name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}}, - {name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}}, - {name: "03-rvs-babel.log", cmd: []string{"rvs", "-c", cfgFile}}, - {name: "04-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}}, - }, logFunc) -} - -// RunAMDStressPack runs an AMD GPU burn-in pack. -// Missing tools are reported as UNSUPPORTED, consistent with the existing SAT pattern. -func (s *System) RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { - seconds := durationSec - if seconds <= 0 { - seconds = envInt("BEE_AMD_STRESS_SECONDS", 300) - } - if err := ensureAMDRuntimeReady(); err != nil { - return "", err - } - // Enable copy_matrix so the same GST run drives VRAM traffic in addition to compute. - rvsCfg := amdStressRVSConfig(seconds) - cfgFile := "/tmp/bee-amd-gst.conf" - _ = os.WriteFile(cfgFile, []byte(rvsCfg), 0644) - - return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-stress", amdStressJobs(seconds, cfgFile), logFunc) -} - -func amdStressRVSConfig(seconds int) string { - return fmt.Sprintf(`actions: -- name: gst_stress - device: all - module: gst - parallel: true - duration: %d - copy_matrix: false - target_stress: 90 - matrix_size_a: 8640 - matrix_size_b: 8640 - matrix_size_c: 8640 -`, seconds*1000) -} - -func amdStressJobs(seconds int, cfgFile string) []satJob { - return []satJob{ - {name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}}, - {name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}}, - {name: fmt.Sprintf("03-rvs-gst-%ds.log", seconds), cmd: []string{"rvs", "-c", cfgFile}}, - {name: fmt.Sprintf("04-rocm-smi-after.log"), cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--csv"}}, - } -} - -// ListNvidiaGPUs returns GPUs visible to nvidia-smi. -func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) { - out, err := exec.Command("nvidia-smi", - "--query-gpu=index,name,memory.total", - "--format=csv,noheader,nounits").Output() - if err != nil { - return nil, fmt.Errorf("nvidia-smi: %w", err) - } - var gpus []NvidiaGPU - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - line = strings.TrimSpace(line) - if line == "" { - continue - } - parts := strings.SplitN(line, ", ", 3) - if len(parts) != 3 { - continue - } - idx, err := strconv.Atoi(strings.TrimSpace(parts[0])) - if err != nil { - continue - } - memMB, _ := strconv.Atoi(strings.TrimSpace(parts[2])) - gpus = append(gpus, NvidiaGPU{ - Index: idx, - Name: strings.TrimSpace(parts[1]), - MemoryMB: memMB, - }) - } - sort.Slice(gpus, func(i, j int) bool { - return gpus[i].Index < gpus[j].Index - }) - return gpus, nil -} - -func (s *System) ListNvidiaGPUStatuses() ([]NvidiaGPUStatus, error) { - out, err := satExecCommand( - "nvidia-smi", - "--query-gpu=index,name,pci.bus_id,serial,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", - "--format=csv,noheader,nounits", - ).Output() - if err != nil { - return nil, fmt.Errorf("nvidia-smi: %w", err) - } - var gpus []NvidiaGPUStatus - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - line = strings.TrimSpace(line) - if line == "" { - continue - } - parts := strings.Split(line, ",") - if len(parts) < 4 { - gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true}) - continue - } - idx, err := strconv.Atoi(strings.TrimSpace(parts[0])) - if err != nil { - gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true}) - continue - } - upper := strings.ToUpper(line) - needsReset := strings.Contains(upper, "GPU REQUIRES RESET") - status := "OK" - if needsReset { - status = "RESET_REQUIRED" - } - gpus = append(gpus, NvidiaGPUStatus{ - Index: idx, - Name: strings.TrimSpace(parts[1]), - BDF: normalizeNvidiaBusID(strings.TrimSpace(parts[2])), - Serial: strings.TrimSpace(parts[3]), - Status: status, - RawLine: line, - NeedsReset: needsReset, - }) - } - sort.Slice(gpus, func(i, j int) bool { return gpus[i].Index < gpus[j].Index }) - return gpus, nil -} - -func normalizeNvidiaBusID(v string) string { - v = strings.TrimSpace(strings.ToLower(v)) - parts := strings.Split(v, ":") - if len(parts) == 3 && len(parts[0]) > 4 { - parts[0] = parts[0][len(parts[0])-4:] - return strings.Join(parts, ":") - } - return v -} - -func (s *System) ResetNvidiaGPU(index int) (string, error) { - return resetNvidiaGPU(index) -} - -// RunNCCLTests runs nccl-tests all_reduce_perf across the selected NVIDIA GPUs. -// Measures collective communication bandwidth over NVLink/PCIe. -func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) { - selected, err := resolveDCGMGPUIndices(gpuIndices) - if err != nil { - return "", err - } - gpuCount := len(selected) - if gpuCount < 1 { - gpuCount = 1 - } - return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", withNvidiaPersistenceMode( - satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, - satJob{name: "02-all-reduce-perf.log", cmd: []string{ - "all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2", - "-g", strconv.Itoa(gpuCount), "--iters", "20", - }, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true}, - ), logFunc) -} - -func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error) { - selected, err := resolveDCGMGPUIndices(gpuIndices) - if err != nil { - return "", err - } - var ( - profCmd []string - profEnv []string - ) - if len(selected) > 1 { - // For multiple GPUs, always spawn one dcgmproftester process per GPU via - // bee-dcgmproftester-staggered (stagger=0 means all start simultaneously). - // A single dcgmproftester process without -i only loads GPU 0 regardless - // of CUDA_VISIBLE_DEVICES. - stagger := staggerSec - if stagger < 0 { - stagger = 0 - } - profCmd = []string{ - "bee-dcgmproftester-staggered", - "--seconds", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec)), - "--stagger-seconds", strconv.Itoa(stagger), - "--devices", joinIndexList(selected), - } - } else { - profCmd, err = resolveDCGMProfTesterCommand("--no-dcgm-validation", "-t", "1004", "-d", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec))) - if err != nil { - return "", err - } - profEnv = nvidiaVisibleDevicesEnv(selected) - } - return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-compute", withNvidiaPersistenceMode( - satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, - satJob{name: "02-dcgmi-version.log", cmd: []string{"dcgmi", "-v"}}, - satJob{ - name: "03-dcgmproftester.log", - cmd: profCmd, - env: profEnv, - collectGPU: true, - gpuIndices: selected, - syncBracket: true, - }, - satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, - ), logFunc) -} - -func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) { - selected, err := resolveDCGMGPUIndices(gpuIndices) - if err != nil { - return "", err - } - // Kill any lingering nvvs/dcgmi processes from a previous interrupted run - // before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34). - if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil { - for _, p := range killed { - logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) - } - } - return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode( - satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, - satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, - satJob{ - name: "03-dcgmi-targeted-power.log", - cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected), - collectGPU: true, - gpuIndices: selected, - syncBracket: true, - }, - satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, - ), logFunc) -} - -func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) { - selected, err := resolveDCGMGPUIndices(gpuIndices) - if err != nil { - return "", err - } - // Kill any lingering nvvs/dcgmi processes from a previous interrupted run - // before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34). - if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil { - for _, p := range killed { - logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) - } - } - return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode( - satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, - satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, - satJob{ - name: "03-dcgmi-pulse-test.log", - cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected), - collectGPU: true, - gpuIndices: selected, - syncBracket: true, - }, - satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, - ), logFunc) -} - -func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) { - selected, err := resolveDCGMGPUIndices(gpuIndices) - if err != nil { - return "", err - } - // Kill any lingering nvvs/dcgmi processes from a previous interrupted run - // before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34). - if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil { - for _, p := range killed { - logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) - } - } - jobs := []satJob{ - {name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, - {name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, - } - - // On a system with GPUs on more than one CPU socket, run each socket's - // GPUs through nvbandwidth in isolation before the all-GPU pass. Without - // NVLink, cross-socket peer-to-peer traffic is a distinct fault domain - // from same-socket traffic; if the single-socket passes log clean and - // only the all-GPU pass doesn't complete, that isolates the cross-socket - // path as the trigger instead of leaving it conflated with a general - // GPU/PCIe fault. Systems with one socket (or no resolvable NUMA - // affinity) get a single group back and keep the original one-pass shape. - step := 3 - socketGroups := gpuBandwidthSocketGroups(selected, logFunc) - if len(socketGroups) <= 1 { - jobs = append(jobs, satJob{ - name: fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step), - cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected), - collectGPU: true, - gpuIndices: selected, - syncBracket: true, - }) - step++ - } else { - for i, group := range socketGroups { - jobs = append(jobs, satJob{ - name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i), - cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group), - collectGPU: true, - gpuIndices: group, - syncBracket: true, - }) - step++ - } - jobs = append(jobs, satJob{ - name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step), - cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected), - collectGPU: true, - gpuIndices: selected, - syncBracket: true, - }) - step++ - } - jobs = append(jobs, satJob{ - name: fmt.Sprintf("%02d-nvidia-smi-after.log", step), - cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}, - }) - - return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(jobs...), logFunc) -} - -func (s *System) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) { - return runAcceptancePackCtx(context.Background(), baseDir, "gpu-nvidia", nvidiaSATJobs(), logFunc) -} - -// RunNvidiaAcceptancePackWithOptions runs the NVIDIA diagnostics via DCGM. -// diagLevel: 1=quick, 2=medium, 3=targeted stress, 4=extended stress. -// gpuIndices: specific GPU indices to test (empty = all GPUs). -// ctx cancellation kills the running job. -func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int, logFunc func(string)) (string, error) { - resolvedGPUIndices, err := resolveDCGMGPUIndices(gpuIndices) - if err != nil { - return "", err - } - return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia", nvidiaDCGMJobs(diagLevel, resolvedGPUIndices), logFunc) -} - -func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) { - selected, err := resolveDCGMGPUIndices(gpuIndices) - if err != nil { - return "", err - } - // Kill any lingering nvvs/dcgmi processes from a previous interrupted run - // before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34). - if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil { - for _, p := range killed { - logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) - } - } - return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode( - satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, - satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, - satJob{ - name: "03-dcgmi-targeted-stress.log", - cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected), - collectGPU: true, - gpuIndices: selected, - syncBracket: true, - }, - satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, - ), logFunc) -} - -func resolveDCGMGPUIndices(gpuIndices []int) ([]int, error) { - if len(gpuIndices) > 0 { - return dedupeSortedIndices(gpuIndices), nil - } - all, err := listNvidiaGPUIndices() - if err != nil { - return nil, err - } - if len(all) == 0 { - return nil, fmt.Errorf("nvidia-smi found no NVIDIA GPUs") - } - return all, nil -} - -func memoryStressSizeArg() string { - if mb := envInt("BEE_VM_STRESS_SIZE_MB", 0); mb > 0 { - return fmt.Sprintf("%dM", mb) - } - availBytes := satFreeMemBytes() - if availBytes <= 0 { - return "80%" - } - availMB := availBytes / (1024 * 1024) - targetMB := (availMB * 2) / 3 - if targetMB >= 256 { - targetMB = (targetMB / 256) * 256 - } - if targetMB <= 0 { - return "80%" - } - return fmt.Sprintf("%dM", targetMB) -} - -func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error) { - if sizeMB <= 0 { - sizeMB = 256 - } - if passes <= 0 { - passes = 1 - } - // Keep Validate Memory bounded to a quick diagnostic window. The timeout is - // intentionally conservative enough for healthy systems while avoiding the - // prior 30-80 minute hangs caused by memtester spinning on a bad subtest. - timeoutSec := sizeMB*passes*20/100 + 60 - if timeoutSec < 180 { - timeoutSec = 180 - } - if timeoutSec > 900 { - timeoutSec = 900 - } - return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{ - {name: "01-free-before.log", cmd: []string{"free", "-h"}}, - {name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}, syncBracket: true}, - {name: "03-free-after.log", cmd: []string{"free", "-h"}}, - }, logFunc) -} - -func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { - seconds := durationSec - if seconds <= 0 { - seconds = envInt("BEE_VM_STRESS_SECONDS", 300) - } - // Base the default on current MemAvailable and keep headroom for the OS and - // concurrent stressors so mixed burn runs do not trip the OOM killer. - sizeArg := memoryStressSizeArg() - return runAcceptancePackCtx(ctx, baseDir, "memory-stress", []satJob{ - {name: "01-free-before.log", cmd: []string{"free", "-h"}}, - {name: "02-stress-ng-vm.log", cmd: []string{ - "stress-ng", "--vm", "1", - "--vm-bytes", sizeArg, - "--vm-method", "all", - "--timeout", fmt.Sprintf("%d", seconds), - "--metrics-brief", - }, syncBracket: true}, - {name: "03-free-after.log", cmd: []string{"free", "-h"}}, - }, logFunc) -} - -func (s *System) RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { - seconds := durationSec - if seconds <= 0 { - seconds = envInt("BEE_SAT_STRESS_SECONDS", 300) - } - cmd := []string{"stressapptest", "-s", fmt.Sprintf("%d", seconds), "-W", "--cc_test"} - if mb := envInt("BEE_SAT_STRESS_MB", 0); mb > 0 { - cmd = append(cmd, "-M", fmt.Sprintf("%d", mb)) - } - return runAcceptancePackCtx(ctx, baseDir, "sat-stress", []satJob{ - {name: "01-free-before.log", cmd: []string{"free", "-h"}}, - {name: "02-stressapptest.log", cmd: cmd}, - {name: "03-free-after.log", cmd: []string{"free", "-h"}}, - }, logFunc) -} - -// cpuThermalThrottleSysDir is the sysfs root the throttle-check scripts glob -// under. Overridden in tests so they can point at a fake directory tree -// instead of the real /sys. -var cpuThermalThrottleSysDir = "/sys/devices/system/cpu" - -// cpuThrottleSumScript is the shell fragment both before/after scripts use to -// sum the kernel's cumulative-since-boot thermal throttle counters across -// every CPU. -func cpuThrottleSumScript() string { - return fmt.Sprintf(` -sum=0 -for f in %[1]s/cpu*/thermal_throttle/core_throttle_count %[1]s/cpu*/thermal_throttle/package_throttle_count; do - [ -f "$f" ] || continue - v=$(cat "$f" 2>/dev/null) - case "$v" in ''|*[!0-9]*) continue ;; esac - sum=$((sum + v)) -done -`, cpuThermalThrottleSysDir) -} - -// cpuThrottleBeforeScript snapshots the throttle counter sum into a file in -// {{run_dir}} so cpuThrottleCheckScript can later diff before/after despite -// each satJob running as an independent process. -func cpuThrottleBeforeScript() string { - return cpuThrottleSumScript() + `echo "$sum" | tee {{run_dir}}/.cpu-throttle-before` + "\n" -} - -// cpuThrottleCheckScript compares the after-run throttle counter sum against -// the snapshot cpuThrottleBeforeScript took, and fails (non-zero exit) if it -// increased — i.e. the CPU actually hit thermal throttling during this -// specific run, not just at some earlier point this boot. classifySATResult -// maps a failed job here to SAT status FAILED, which ApplySATResultToDB -// records as component status "Warning" for cpu:all — without this, the -// "cpu" SAT pack only checks stress-ng's exit code, which is 0 whether or -// not the CPU throttled while running it. -func cpuThrottleCheckScript() string { - return `before=$(cat {{run_dir}}/.cpu-throttle-before 2>/dev/null) -case "$before" in ''|*[!0-9]*) before=0 ;; esac -` + cpuThrottleSumScript() + `after=$sum -echo "throttle_count_before=$before" -echo "throttle_count_after=$after" -if [ "$after" -gt "$before" ]; then - echo "THROTTLE DETECTED: CPU package/core hit thermal throttling during this stress-ng run ($before -> $after)" - exit 1 -fi -echo "no new thermal throttling detected during this run" -` -} - -func cpuSATJobs(durationSec int) []satJob { - return []satJob{ - {name: "01-lscpu.log", cmd: []string{"lscpu"}}, - {name: "02-sensors-before.log", cmd: []string{"sensors"}}, - {name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true}, - {name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true}, - {name: "04-sensors-after.log", cmd: []string{"sensors"}}, - {name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}}, - } -} - -func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { - if durationSec <= 0 { - durationSec = 60 - } - return runAcceptancePackCtx(ctx, baseDir, "cpu", cpuSATJobs(durationSec), logFunc) -} - -func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) { - if baseDir == "" { - baseDir = "/var/log/bee-sat" - } - ts := time.Now().UTC().Format("20060102-150405") - runDir := filepath.Join(baseDir, "storage-"+ts) - if err := os.MkdirAll(runDir, 0755); err != nil { - return "", err - } - verboseLog := filepath.Join(runDir, "verbose.log") - - devices, err := listStorageDevices() - if err != nil { - return "", err - } - sort.Strings(devices) - - var summary strings.Builder - stats := satStats{} - fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339)) - if len(devices) == 0 { - fmt.Fprintln(&summary, "devices=0") - stats.Unsupported++ - } else { - fmt.Fprintf(&summary, "devices=%d\n", len(devices)) - } - - for index, devPath := range devices { - if ctx.Err() != nil { - break - } - prefix := fmt.Sprintf("%02d-%s", index+1, filepath.Base(devPath)) - commands := storageSATCommands(devPath, extended) - deviceOutputs := make(map[string][]byte, len(commands)) - for cmdIndex, job := range commands { - if ctx.Err() != nil { - break - } - name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name) - livePath := filepath.Join(runDir, name) - runSyncBracketHook(job, "before", logFunc) - out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc, livePath) - deviceOutputs[job.name] = out - if writeErr := os.WriteFile(livePath, out, 0644); writeErr != nil { - return "", writeErr - } - if satJobBoundaryHook != nil { - satJobBoundaryHook(name) - } - // smartctl -t short only launches the self-test on the drive firmware and - // returns immediately ("Testing has begun"); unlike `nvme device-self-test - // --wait`, smartctl has no blocking mode, so we must poll the drive - // ourselves until the self-test actually finishes. Hold the "after" sync - // until that poll completes — the self-test itself, not just its launch, - // is the load worth having durable evidence of. - deferSyncBracketAfter := job.name == "smartctl-self-test-short" && err == nil - if !deferSyncBracketAfter { - runSyncBracketHook(job, "after", logFunc) - } - status, rc := classifySATResult(job.name, out, err) - // A zero smartctl exit status only proves the command ran. If the - // drive did not return its overall-health verdict, it must not turn - // the storage SAT green. - if job.name == "smartctl-health" && status == "OK" && !hasSMARTOverallHealth(out) { - status = "UNSUPPORTED" - } - stats.Add(status) - key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_") - fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc) - fmt.Fprintf(&summary, "%s_status=%s\n", key, status) - - if deferSyncBracketAfter { - statusName := "smartctl-self-test-status" - statusOut := waitForSmartctlSelfTest(ctx, verboseLog, devPath, logFunc) - deviceOutputs[statusName] = statusOut - statusFile := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+2, statusName) - if writeErr := os.WriteFile(filepath.Join(runDir, statusFile), statusOut, 0644); writeErr != nil { - return "", writeErr - } - runSyncBracketHook(job, "after", logFunc) - sStatus, sRC := classifySATResult(statusName, statusOut, nil) - stats.Add(sStatus) - sKey := filepath.Base(devPath) + "_" + strings.ReplaceAll(statusName, "-", "_") - fmt.Fprintf(&summary, "%s_rc=%d\n", sKey, sRC) - fmt.Fprintf(&summary, "%s_status=%s\n", sKey, sStatus) - } - } - reportText := GenerateDiskReportText(index+1, devPath, deviceOutputs, time.Now().UTC()) - reportName := "disk-" + prefix + "-report.txt" - _ = os.WriteFile(filepath.Join(runDir, reportName), []byte(reportText), 0644) - } - - writeSATStats(&summary, stats) - if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil { - return "", err - } - return runDir, nil -} - type satJob struct { name string cmd []string @@ -1574,41 +804,6 @@ func hasSMARTOverallHealth(out []byte) bool { return len(m) > 1 && strings.TrimSpace(m[1]) != "" } -func runSATCommand(verboseLog, name string, cmd []string, logFunc func(string)) ([]byte, error) { - start := time.Now().UTC() - resolvedCmd, err := resolveSATCommand(cmd) - appendSATVerboseLog(verboseLog, - fmt.Sprintf("[%s] start %s", start.Format(time.RFC3339), name), - "cmd: "+strings.Join(resolvedCmd, " "), - ) - if logFunc != nil { - logFunc(fmt.Sprintf("=== %s ===", name)) - } - if err != nil { - appendSATVerboseLog(verboseLog, - fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name), - "rc: 1", - fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()), - "", - ) - return []byte(err.Error() + "\n"), err - } - - out, err := streamExecOutput(satExecCommand(resolvedCmd[0], resolvedCmd[1:]...), logFunc, "") - - rc := 0 - if err != nil { - rc = 1 - } - appendSATVerboseLog(verboseLog, - fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name), - fmt.Sprintf("rc: %d", rc), - fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()), - "", - ) - return out, err -} - func runROCmSMI(args ...string) ([]byte, error) { cmd, err := resolveROCmSMICommand(args...) if err != nil { @@ -1802,46 +997,3 @@ func envInt(name string, fallback int) int { } return value } - -func createTarGz(dst, srcDir string) error { - file, err := os.Create(dst) - if err != nil { - return err - } - defer file.Close() - - gz := gzip.NewWriter(file) - defer gz.Close() - - tw := tar.NewWriter(gz) - defer tw.Close() - - base := filepath.Dir(srcDir) - return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - header, err := tar.FileInfoHeader(info, "") - if err != nil { - return err - } - rel, err := filepath.Rel(base, path) - if err != nil { - return err - } - header.Name = rel - if err := tw.WriteHeader(header); err != nil { - return err - } - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - _, err = io.Copy(tw, file) - return err - }) -} diff --git a/audit/internal/platform/sat_gpu_packs.go b/audit/internal/platform/sat_gpu_packs.go new file mode 100644 index 0000000..7f016da --- /dev/null +++ b/audit/internal/platform/sat_gpu_packs.go @@ -0,0 +1,568 @@ +package platform + +import ( + "context" + "fmt" + "os" + "os/exec" + "sort" + "strconv" + "strings" +) + +type NvidiaGPU struct { + Index int `json:"index"` + Name string `json:"name"` + MemoryMB int `json:"memory_mb"` +} + +type NvidiaGPUStatus struct { + Index int `json:"index"` + Name string `json:"name"` + BDF string `json:"bdf,omitempty"` + Serial string `json:"serial,omitempty"` + Status string `json:"status"` + RawLine string `json:"raw_line,omitempty"` + NeedsReset bool `json:"needs_reset"` + ParseFailure bool `json:"parse_failure,omitempty"` +} + +type nvidiaGPUHealth struct { + Index int + Name string + NeedsReset bool + RawLine string + ParseFailure bool +} + +type nvidiaGPUStatusFile struct { + Index int + Name string + RunStatus string + Reason string + Health string + HealthRaw string + Observed bool + Selected bool + FailingJob string +} + +// AMDGPUInfo holds basic info about an AMD GPU from rocm-smi. +type AMDGPUInfo struct { + Index int `json:"index"` + Name string `json:"name"` +} + +// DetectGPUVendor returns "nvidia" if /dev/nvidia0 exists, "amd" if /dev/kfd exists, or "" otherwise. +func (s *System) DetectGPUVendor() string { + if _, err := os.Stat("/dev/nvidia0"); err == nil { + return "nvidia" + } + if _, err := os.Stat("/dev/kfd"); err == nil { + return "amd" + } + if raw, err := exec.Command("lspci", "-nn").Output(); err == nil { + // Only match AMD GPU device classes [0300]=VGA, [0302]=3D controller, [0380]=Display. + // AMD CPUs also appear in lspci as "Advanced Micro Devices" (Root Complex, IOMMU, etc.) + // so matching vendor alone causes false positives on AMD CPU servers without GPUs. + for _, line := range strings.Split(strings.ToLower(string(raw)), "\n") { + if !strings.Contains(line, "advanced micro devices") && !strings.Contains(line, "amd/ati") { + continue + } + if strings.Contains(line, "[0300]") || strings.Contains(line, "[0302]") || strings.Contains(line, "[0380]") { + return "amd" + } + } + } + return "" +} + +// PhysicalGPUVendors reports which supported vendors have a display-class PCI +// function, regardless of driver state. It is used to distinguish absent +// hardware from a PCI function whose runtime is not operational yet. +func (s *System) PhysicalGPUVendors() (nvidia bool, amd bool) { + raw, err := satExecCommand("lspci", "-nn").Output() + if err != nil { + return false, false + } + for _, line := range strings.Split(strings.ToLower(string(raw)), "\n") { + // [0300]=VGA, [0302]=3D controller, [0380]=Display controller. + if !strings.Contains(line, "[0300]") && !strings.Contains(line, "[0302]") && !strings.Contains(line, "[0380]") { + continue + } + switch { + case strings.Contains(line, "[10de:"): + nvidia = true + case strings.Contains(line, "[1002:"), strings.Contains(line, "advanced micro devices"), strings.Contains(line, "amd/ati"): + amd = true + } + } + return nvidia, amd +} + +// ListAMDGPUs returns AMD GPUs visible to rocm-smi. +func (s *System) ListAMDGPUs() ([]AMDGPUInfo, error) { + out, err := runROCmSMI("--showproductname", "--csv") + if err != nil { + return nil, fmt.Errorf("rocm-smi: %w", err) + } + var gpus []AMDGPUInfo + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(strings.ToLower(line), "device") { + continue + } + parts := strings.SplitN(line, ",", 2) + name := "" + if len(parts) >= 2 { + name = strings.TrimSpace(parts[1]) + } + idx := len(gpus) + gpus = append(gpus, AMDGPUInfo{Index: idx, Name: name}) + } + return gpus, nil +} + +// RunAMDAcceptancePack runs an AMD GPU diagnostic pack using rocm-smi. +func (s *System) RunAMDAcceptancePack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { + return runAcceptancePackCtx(ctx, baseDir, "gpu-amd", []satJob{ + {name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}}, + {name: "02-rocm-smi-showallinfo.log", cmd: []string{"rocm-smi", "--showallinfo"}}, + {name: "03-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}}, + {name: "04-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}}, + }, logFunc) +} + +// RunAMDMemIntegrityPack runs the official RVS MEM module as a validate-style memory integrity test. +func (s *System) RunAMDMemIntegrityPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { + if err := ensureAMDRuntimeReady(); err != nil { + return "", err + } + cfgFile := "/tmp/bee-amd-mem.conf" + cfg := `actions: +- name: mem_integrity + device: all + module: mem + parallel: true + duration: 60000 + copy_matrix: false + target_stress: 90 + matrix_size: 8640 +` + _ = os.WriteFile(cfgFile, []byte(cfg), 0644) + return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-mem", []satJob{ + {name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}}, + {name: "02-rvs-mem.log", cmd: []string{"rvs", "-c", cfgFile}}, + {name: "03-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}}, + }, logFunc) +} + +// RunAMDMemBandwidthPack runs AMD's memory/interconnect bandwidth-oriented tools. +func (s *System) RunAMDMemBandwidthPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { + if err := ensureAMDRuntimeReady(); err != nil { + return "", err + } + cfgFile := "/tmp/bee-amd-babel.conf" + cfg := `actions: +- name: babel_mem_bw + device: all + module: babel + parallel: true + copy_matrix: true + target_stress: 90 + matrix_size: 134217728 +` + _ = os.WriteFile(cfgFile, []byte(cfg), 0644) + return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-bandwidth", []satJob{ + {name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}}, + {name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}}, + {name: "03-rvs-babel.log", cmd: []string{"rvs", "-c", cfgFile}}, + {name: "04-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}}, + }, logFunc) +} + +// RunAMDStressPack runs an AMD GPU burn-in pack. +// Missing tools are reported as UNSUPPORTED, consistent with the existing SAT pattern. +func (s *System) RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { + seconds := durationSec + if seconds <= 0 { + seconds = envInt("BEE_AMD_STRESS_SECONDS", 300) + } + if err := ensureAMDRuntimeReady(); err != nil { + return "", err + } + // Enable copy_matrix so the same GST run drives VRAM traffic in addition to compute. + rvsCfg := amdStressRVSConfig(seconds) + cfgFile := "/tmp/bee-amd-gst.conf" + _ = os.WriteFile(cfgFile, []byte(rvsCfg), 0644) + + return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-stress", amdStressJobs(seconds, cfgFile), logFunc) +} + +func amdStressRVSConfig(seconds int) string { + return fmt.Sprintf(`actions: +- name: gst_stress + device: all + module: gst + parallel: true + duration: %d + copy_matrix: false + target_stress: 90 + matrix_size_a: 8640 + matrix_size_b: 8640 + matrix_size_c: 8640 +`, seconds*1000) +} + +func amdStressJobs(seconds int, cfgFile string) []satJob { + return []satJob{ + {name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}}, + {name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}}, + {name: fmt.Sprintf("03-rvs-gst-%ds.log", seconds), cmd: []string{"rvs", "-c", cfgFile}}, + {name: fmt.Sprintf("04-rocm-smi-after.log"), cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--csv"}}, + } +} + +// ListNvidiaGPUs returns GPUs visible to nvidia-smi. +func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) { + out, err := exec.Command("nvidia-smi", + "--query-gpu=index,name,memory.total", + "--format=csv,noheader,nounits").Output() + if err != nil { + return nil, fmt.Errorf("nvidia-smi: %w", err) + } + var gpus []NvidiaGPU + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.SplitN(line, ", ", 3) + if len(parts) != 3 { + continue + } + idx, err := strconv.Atoi(strings.TrimSpace(parts[0])) + if err != nil { + continue + } + memMB, _ := strconv.Atoi(strings.TrimSpace(parts[2])) + gpus = append(gpus, NvidiaGPU{ + Index: idx, + Name: strings.TrimSpace(parts[1]), + MemoryMB: memMB, + }) + } + sort.Slice(gpus, func(i, j int) bool { + return gpus[i].Index < gpus[j].Index + }) + return gpus, nil +} + +func (s *System) ListNvidiaGPUStatuses() ([]NvidiaGPUStatus, error) { + out, err := satExecCommand( + "nvidia-smi", + "--query-gpu=index,name,pci.bus_id,serial,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", + "--format=csv,noheader,nounits", + ).Output() + if err != nil { + return nil, fmt.Errorf("nvidia-smi: %w", err) + } + var gpus []NvidiaGPUStatus + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.Split(line, ",") + if len(parts) < 4 { + gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true}) + continue + } + idx, err := strconv.Atoi(strings.TrimSpace(parts[0])) + if err != nil { + gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true}) + continue + } + upper := strings.ToUpper(line) + needsReset := strings.Contains(upper, "GPU REQUIRES RESET") + status := "OK" + if needsReset { + status = "RESET_REQUIRED" + } + gpus = append(gpus, NvidiaGPUStatus{ + Index: idx, + Name: strings.TrimSpace(parts[1]), + BDF: normalizeNvidiaBusID(strings.TrimSpace(parts[2])), + Serial: strings.TrimSpace(parts[3]), + Status: status, + RawLine: line, + NeedsReset: needsReset, + }) + } + sort.Slice(gpus, func(i, j int) bool { return gpus[i].Index < gpus[j].Index }) + return gpus, nil +} + +func normalizeNvidiaBusID(v string) string { + v = strings.TrimSpace(strings.ToLower(v)) + parts := strings.Split(v, ":") + if len(parts) == 3 && len(parts[0]) > 4 { + parts[0] = parts[0][len(parts[0])-4:] + return strings.Join(parts, ":") + } + return v +} + +func (s *System) ResetNvidiaGPU(index int) (string, error) { + return resetNvidiaGPU(index) +} + +// RunNCCLTests runs nccl-tests all_reduce_perf across the selected NVIDIA GPUs. +// Measures collective communication bandwidth over NVLink/PCIe. +func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) { + selected, err := resolveDCGMGPUIndices(gpuIndices) + if err != nil { + return "", err + } + gpuCount := len(selected) + if gpuCount < 1 { + gpuCount = 1 + } + return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", withNvidiaPersistenceMode( + satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, + satJob{name: "02-all-reduce-perf.log", cmd: []string{ + "all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2", + "-g", strconv.Itoa(gpuCount), "--iters", "20", + }, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true}, + ), logFunc) +} + +func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error) { + selected, err := resolveDCGMGPUIndices(gpuIndices) + if err != nil { + return "", err + } + var ( + profCmd []string + profEnv []string + ) + if len(selected) > 1 { + // For multiple GPUs, always spawn one dcgmproftester process per GPU via + // bee-dcgmproftester-staggered (stagger=0 means all start simultaneously). + // A single dcgmproftester process without -i only loads GPU 0 regardless + // of CUDA_VISIBLE_DEVICES. + stagger := staggerSec + if stagger < 0 { + stagger = 0 + } + profCmd = []string{ + "bee-dcgmproftester-staggered", + "--seconds", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec)), + "--stagger-seconds", strconv.Itoa(stagger), + "--devices", joinIndexList(selected), + } + } else { + profCmd, err = resolveDCGMProfTesterCommand("--no-dcgm-validation", "-t", "1004", "-d", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec))) + if err != nil { + return "", err + } + profEnv = nvidiaVisibleDevicesEnv(selected) + } + return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-compute", withNvidiaPersistenceMode( + satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, + satJob{name: "02-dcgmi-version.log", cmd: []string{"dcgmi", "-v"}}, + satJob{ + name: "03-dcgmproftester.log", + cmd: profCmd, + env: profEnv, + collectGPU: true, + gpuIndices: selected, + syncBracket: true, + }, + satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, + ), logFunc) +} + +func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) { + selected, err := resolveDCGMGPUIndices(gpuIndices) + if err != nil { + return "", err + } + // Kill any lingering nvvs/dcgmi processes from a previous interrupted run + // before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34). + if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil { + for _, p := range killed { + logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) + } + } + return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode( + satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, + satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, + satJob{ + name: "03-dcgmi-targeted-power.log", + cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected), + collectGPU: true, + gpuIndices: selected, + syncBracket: true, + }, + satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, + ), logFunc) +} + +func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) { + selected, err := resolveDCGMGPUIndices(gpuIndices) + if err != nil { + return "", err + } + // Kill any lingering nvvs/dcgmi processes from a previous interrupted run + // before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34). + if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil { + for _, p := range killed { + logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) + } + } + return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode( + satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, + satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, + satJob{ + name: "03-dcgmi-pulse-test.log", + cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected), + collectGPU: true, + gpuIndices: selected, + syncBracket: true, + }, + satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, + ), logFunc) +} + +// RunNvidiaBandwidthPack runs `dcgmi diag -r nvbandwidth`. The only thing +// fullMatrix changes is which GPU sets each invocation gets via `-i`: +// +// - fullMatrix=false (Validate): a single pass across every selected GPU. +// - fullMatrix=true (deep/Stress): on a system whose GPUs span more than +// one CPU socket, one pass per socket group and then one pass across all +// of them, isolating the cross-socket peer-to-peer path as its own +// fault domain (see bible-local/decisions/2026-07-27-nvbandwidth-per-socket-split.md). +// Single-socket systems collapse back to one pass. +func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, fullMatrix bool, logFunc func(string)) (string, error) { + selected, err := resolveDCGMGPUIndices(gpuIndices) + if err != nil { + return "", err + } + // Kill any lingering nvvs/dcgmi processes from a previous interrupted run + // before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34). + if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil { + for _, p := range killed { + logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) + } + } + jobs := []satJob{ + {name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, + {name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, + } + + // On a system with GPUs on more than one CPU socket, run each socket's + // GPUs through nvbandwidth in isolation before the all-GPU pass. Without + // NVLink, cross-socket peer-to-peer traffic is a distinct fault domain + // from same-socket traffic; if the single-socket passes log clean and + // only the all-GPU pass doesn't complete, that isolates the cross-socket + // path as the trigger instead of leaving it conflated with a general + // GPU/PCIe fault. Systems with one socket (or no resolvable NUMA + // affinity) get a single group back and keep the original one-pass shape. + step := 3 + socketGroups := [][]int{selected} + if fullMatrix { + socketGroups = gpuBandwidthSocketGroups(selected, logFunc) + } + if len(socketGroups) <= 1 { + jobs = append(jobs, satJob{ + name: fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step), + cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected), + collectGPU: true, + gpuIndices: selected, + syncBracket: true, + }) + step++ + } else { + for i, group := range socketGroups { + jobs = append(jobs, satJob{ + name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i), + cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group), + collectGPU: true, + gpuIndices: group, + syncBracket: true, + }) + step++ + } + jobs = append(jobs, satJob{ + name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step), + cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected), + collectGPU: true, + gpuIndices: selected, + syncBracket: true, + }) + step++ + } + jobs = append(jobs, satJob{ + name: fmt.Sprintf("%02d-nvidia-smi-after.log", step), + cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}, + }) + + return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(jobs...), logFunc) +} + +func (s *System) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) { + return runAcceptancePackCtx(context.Background(), baseDir, "gpu-nvidia", nvidiaSATJobs(), logFunc) +} + +// RunNvidiaAcceptancePackWithOptions runs the NVIDIA diagnostics via DCGM. +// diagLevel: 1=quick, 2=medium, 3=targeted stress, 4=extended stress. +// gpuIndices: specific GPU indices to test (empty = all GPUs). +// ctx cancellation kills the running job. +func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int, logFunc func(string)) (string, error) { + resolvedGPUIndices, err := resolveDCGMGPUIndices(gpuIndices) + if err != nil { + return "", err + } + return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia", nvidiaDCGMJobs(diagLevel, resolvedGPUIndices), logFunc) +} + +func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) { + selected, err := resolveDCGMGPUIndices(gpuIndices) + if err != nil { + return "", err + } + // Kill any lingering nvvs/dcgmi processes from a previous interrupted run + // before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34). + if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil { + for _, p := range killed { + logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) + } + } + return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode( + satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, + satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, + satJob{ + name: "03-dcgmi-targeted-stress.log", + cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected), + collectGPU: true, + gpuIndices: selected, + syncBracket: true, + }, + satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, + ), logFunc) +} + +func resolveDCGMGPUIndices(gpuIndices []int) ([]int, error) { + if len(gpuIndices) > 0 { + return dedupeSortedIndices(gpuIndices), nil + } + all, err := listNvidiaGPUIndices() + if err != nil { + return nil, err + } + if len(all) == 0 { + return nil, fmt.Errorf("nvidia-smi found no NVIDIA GPUs") + } + return all, nil +} diff --git a/audit/internal/platform/sat_host_packs.go b/audit/internal/platform/sat_host_packs.go new file mode 100644 index 0000000..7da077b --- /dev/null +++ b/audit/internal/platform/sat_host_packs.go @@ -0,0 +1,257 @@ +package platform + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +func memoryStressSizeArg() string { + if mb := envInt("BEE_VM_STRESS_SIZE_MB", 0); mb > 0 { + return fmt.Sprintf("%dM", mb) + } + availBytes := satFreeMemBytes() + if availBytes <= 0 { + return "80%" + } + availMB := availBytes / (1024 * 1024) + targetMB := (availMB * 2) / 3 + if targetMB >= 256 { + targetMB = (targetMB / 256) * 256 + } + if targetMB <= 0 { + return "80%" + } + return fmt.Sprintf("%dM", targetMB) +} + +func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error) { + if sizeMB <= 0 { + sizeMB = 256 + } + if passes <= 0 { + passes = 1 + } + // Keep Validate Memory bounded to a quick diagnostic window. The timeout is + // intentionally conservative enough for healthy systems while avoiding the + // prior 30-80 minute hangs caused by memtester spinning on a bad subtest. + timeoutSec := sizeMB*passes*20/100 + 60 + if timeoutSec < 180 { + timeoutSec = 180 + } + if timeoutSec > 900 { + timeoutSec = 900 + } + return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{ + {name: "01-free-before.log", cmd: []string{"free", "-h"}}, + {name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}, syncBracket: true}, + {name: "03-free-after.log", cmd: []string{"free", "-h"}}, + }, logFunc) +} + +func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { + seconds := durationSec + if seconds <= 0 { + seconds = envInt("BEE_VM_STRESS_SECONDS", 300) + } + // Base the default on current MemAvailable and keep headroom for the OS and + // concurrent stressors so mixed burn runs do not trip the OOM killer. + sizeArg := memoryStressSizeArg() + return runAcceptancePackCtx(ctx, baseDir, "memory-stress", []satJob{ + {name: "01-free-before.log", cmd: []string{"free", "-h"}}, + {name: "02-stress-ng-vm.log", cmd: []string{ + "stress-ng", "--vm", "1", + "--vm-bytes", sizeArg, + "--vm-method", "all", + "--timeout", fmt.Sprintf("%d", seconds), + "--metrics-brief", + }, syncBracket: true}, + {name: "03-free-after.log", cmd: []string{"free", "-h"}}, + }, logFunc) +} + +func (s *System) RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { + seconds := durationSec + if seconds <= 0 { + seconds = envInt("BEE_SAT_STRESS_SECONDS", 300) + } + cmd := []string{"stressapptest", "-s", fmt.Sprintf("%d", seconds), "-W", "--cc_test"} + if mb := envInt("BEE_SAT_STRESS_MB", 0); mb > 0 { + cmd = append(cmd, "-M", fmt.Sprintf("%d", mb)) + } + return runAcceptancePackCtx(ctx, baseDir, "sat-stress", []satJob{ + {name: "01-free-before.log", cmd: []string{"free", "-h"}}, + {name: "02-stressapptest.log", cmd: cmd}, + {name: "03-free-after.log", cmd: []string{"free", "-h"}}, + }, logFunc) +} + +// cpuThermalThrottleSysDir is the sysfs root the throttle-check scripts glob +// under. Overridden in tests so they can point at a fake directory tree +// instead of the real /sys. +var cpuThermalThrottleSysDir = "/sys/devices/system/cpu" + +// cpuThrottleSumScript is the shell fragment both before/after scripts use to +// sum the kernel's cumulative-since-boot thermal throttle counters across +// every CPU. +func cpuThrottleSumScript() string { + return fmt.Sprintf(` +sum=0 +for f in %[1]s/cpu*/thermal_throttle/core_throttle_count %[1]s/cpu*/thermal_throttle/package_throttle_count; do + [ -f "$f" ] || continue + v=$(cat "$f" 2>/dev/null) + case "$v" in ''|*[!0-9]*) continue ;; esac + sum=$((sum + v)) +done +`, cpuThermalThrottleSysDir) +} + +// cpuThrottleBeforeScript snapshots the throttle counter sum into a file in +// {{run_dir}} so cpuThrottleCheckScript can later diff before/after despite +// each satJob running as an independent process. +func cpuThrottleBeforeScript() string { + return cpuThrottleSumScript() + `echo "$sum" | tee {{run_dir}}/.cpu-throttle-before` + "\n" +} + +// cpuThrottleCheckScript compares the after-run throttle counter sum against +// the snapshot cpuThrottleBeforeScript took, and fails (non-zero exit) if it +// increased — i.e. the CPU actually hit thermal throttling during this +// specific run, not just at some earlier point this boot. classifySATResult +// maps a failed job here to SAT status FAILED, which ApplySATResultToDB +// records as component status "Warning" for cpu:all — without this, the +// "cpu" SAT pack only checks stress-ng's exit code, which is 0 whether or +// not the CPU throttled while running it. +func cpuThrottleCheckScript() string { + return `before=$(cat {{run_dir}}/.cpu-throttle-before 2>/dev/null) +case "$before" in ''|*[!0-9]*) before=0 ;; esac +` + cpuThrottleSumScript() + `after=$sum +echo "throttle_count_before=$before" +echo "throttle_count_after=$after" +if [ "$after" -gt "$before" ]; then + echo "THROTTLE DETECTED: CPU package/core hit thermal throttling during this stress-ng run ($before -> $after)" + exit 1 +fi +echo "no new thermal throttling detected during this run" +` +} + +func cpuSATJobs(durationSec int) []satJob { + return []satJob{ + {name: "01-lscpu.log", cmd: []string{"lscpu"}}, + {name: "02-sensors-before.log", cmd: []string{"sensors"}}, + {name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true}, + {name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true}, + {name: "04-sensors-after.log", cmd: []string{"sensors"}}, + {name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}}, + } +} + +func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { + if durationSec <= 0 { + durationSec = 60 + } + return runAcceptancePackCtx(ctx, baseDir, "cpu", cpuSATJobs(durationSec), logFunc) +} + +func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) { + if baseDir == "" { + baseDir = "/var/log/bee-sat" + } + ts := time.Now().UTC().Format("20060102-150405") + runDir := filepath.Join(baseDir, "storage-"+ts) + if err := os.MkdirAll(runDir, 0755); err != nil { + return "", err + } + verboseLog := filepath.Join(runDir, "verbose.log") + + devices, err := listStorageDevices() + if err != nil { + return "", err + } + sort.Strings(devices) + + var summary strings.Builder + stats := satStats{} + fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339)) + if len(devices) == 0 { + fmt.Fprintln(&summary, "devices=0") + stats.Unsupported++ + } else { + fmt.Fprintf(&summary, "devices=%d\n", len(devices)) + } + + for index, devPath := range devices { + if ctx.Err() != nil { + break + } + prefix := fmt.Sprintf("%02d-%s", index+1, filepath.Base(devPath)) + commands := storageSATCommands(devPath, extended) + deviceOutputs := make(map[string][]byte, len(commands)) + for cmdIndex, job := range commands { + if ctx.Err() != nil { + break + } + name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name) + livePath := filepath.Join(runDir, name) + runSyncBracketHook(job, "before", logFunc) + out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc, livePath) + deviceOutputs[job.name] = out + if writeErr := os.WriteFile(livePath, out, 0644); writeErr != nil { + return "", writeErr + } + if satJobBoundaryHook != nil { + satJobBoundaryHook(name) + } + // smartctl -t short only launches the self-test on the drive firmware and + // returns immediately ("Testing has begun"); unlike `nvme device-self-test + // --wait`, smartctl has no blocking mode, so we must poll the drive + // ourselves until the self-test actually finishes. Hold the "after" sync + // until that poll completes — the self-test itself, not just its launch, + // is the load worth having durable evidence of. + deferSyncBracketAfter := job.name == "smartctl-self-test-short" && err == nil + if !deferSyncBracketAfter { + runSyncBracketHook(job, "after", logFunc) + } + status, rc := classifySATResult(job.name, out, err) + // A zero smartctl exit status only proves the command ran. If the + // drive did not return its overall-health verdict, it must not turn + // the storage SAT green. + if job.name == "smartctl-health" && status == "OK" && !hasSMARTOverallHealth(out) { + status = "UNSUPPORTED" + } + stats.Add(status) + key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_") + fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc) + fmt.Fprintf(&summary, "%s_status=%s\n", key, status) + + if deferSyncBracketAfter { + statusName := "smartctl-self-test-status" + statusOut := waitForSmartctlSelfTest(ctx, verboseLog, devPath, logFunc) + deviceOutputs[statusName] = statusOut + statusFile := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+2, statusName) + if writeErr := os.WriteFile(filepath.Join(runDir, statusFile), statusOut, 0644); writeErr != nil { + return "", writeErr + } + runSyncBracketHook(job, "after", logFunc) + sStatus, sRC := classifySATResult(statusName, statusOut, nil) + stats.Add(sStatus) + sKey := filepath.Base(devPath) + "_" + strings.ReplaceAll(statusName, "-", "_") + fmt.Fprintf(&summary, "%s_rc=%d\n", sKey, sRC) + fmt.Fprintf(&summary, "%s_status=%s\n", sKey, sStatus) + } + } + reportText := GenerateDiskReportText(index+1, devPath, deviceOutputs, time.Now().UTC()) + reportName := "disk-" + prefix + "-report.txt" + _ = os.WriteFile(filepath.Join(runDir, reportName), []byte(reportText), 0644) + } + + writeSATStats(&summary, stats) + if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil { + return "", err + } + return runDir, nil +} diff --git a/audit/internal/platform/sat_test.go b/audit/internal/platform/sat_test.go index 58a112a..df689c3 100644 --- a/audit/internal/platform/sat_test.go +++ b/audit/internal/platform/sat_test.go @@ -145,8 +145,6 @@ func TestNvidiaDCGMJobsEnablePersistenceModeBeforeDiag(t *testing.T) { } func TestBuildNvidiaStressJobUsesSelectedLoaderAndDevices(t *testing.T) { - t.Parallel() - oldExecCommand := satExecCommand satExecCommand = func(name string, args ...string) *exec.Cmd { if name == "nvidia-smi" { @@ -179,8 +177,6 @@ func TestBuildNvidiaStressJobUsesSelectedLoaderAndDevices(t *testing.T) { } func TestBuildNvidiaStressJobUsesNCCLLoader(t *testing.T) { - t.Parallel() - oldExecCommand := satExecCommand satExecCommand = func(name string, args ...string) *exec.Cmd { if name == "nvidia-smi" { @@ -213,8 +209,6 @@ func TestBuildNvidiaStressJobUsesNCCLLoader(t *testing.T) { } func TestResolveDCGMGPUIndicesUsesDetectedGPUsWhenUnset(t *testing.T) { - t.Parallel() - oldExecCommand := satExecCommand satExecCommand = func(name string, args ...string) *exec.Cmd { if name == "nvidia-smi" { diff --git a/audit/internal/platform/tpm.go b/audit/internal/platform/tpm.go index 038b0a7..2ba94e6 100644 --- a/audit/internal/platform/tpm.go +++ b/audit/internal/platform/tpm.go @@ -1,14 +1,82 @@ package platform -import "context" +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// tpmDeviceGlob is a seam for tests. It reports the sysfs TPM device nodes +// the kernel has registered; an empty result means the platform exposes no +// TPM at all (no discrete chip, or firmware/BIOS has it disabled). +var tpmDeviceGlob = func() []string { + matches, _ := filepath.Glob("/sys/class/tpm/tpm*") + return matches +} + +var tpmReadFile = os.ReadFile + +// TPMPresent reports whether sysfs identifies a registered device as TPM 2.x. +// The validation pack uses tpm2-tools, so a TPM 1.2 device is not sufficient. +func (s *System) TPMPresent() bool { + for _, device := range tpmDeviceGlob() { + version, err := tpmReadFile(filepath.Join(device, "tpm_version_major")) + if err == nil && strings.TrimSpace(string(version)) == "2" { + return true + } + } + return false +} // RunTPMValidationPack verifies TPM 2.0 communication using read-only // commands. It deliberately excludes SelfTest, provisioning, NV writes, PCR // changes, key creation, and ownership operations. +// +// When the platform exposes no TPM device at all, the pack does not run the +// tpm2_* tools: without a TCTI device they only ever emit a wall of +// "Failed to open ... /dev/tpmrm0" errors that read as a hard failure when +// the real situation is "this machine has no TPM". Instead it writes an +// UNSUPPORTED summary and returns, the same way the storage pack handles a +// host with no drives. func (s *System) RunTPMValidationPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { + if !s.TPMPresent() { + return writeTPMUnsupportedRun(baseDir, logFunc) + } return runAcceptancePackCtx(ctx, baseDir, "tpm", tpmValidationJobs(), logFunc) } +func writeTPMUnsupportedRun(baseDir string, logFunc func(string)) (string, error) { + if strings.TrimSpace(baseDir) == "" { + baseDir = "/var/log/bee-sat" + } + now := time.Now().UTC() + runDir := filepath.Join(baseDir, "tpm-"+now.Format("20060102-150405")) + if err := os.MkdirAll(runDir, 0755); err != nil { + return "", err + } + if logFunc != nil { + logFunc("no TPM 2.x device reported by sysfs; skipping read-only TPM checks") + } + + var summary strings.Builder + fmt.Fprintf(&summary, "run_at_utc=%s\n", now.Format(time.RFC3339)) + summary.WriteString("tpm_present=false\n") + summary.WriteString("skip_reason=no TPM 2.x device reported by sysfs; tpm2-tools are not applicable\n") + summary.WriteString("tpm_check_status=UNSUPPORTED\n") + summary.WriteString("overall_status=UNSUPPORTED\n") + summary.WriteString("job_ok=0\n") + summary.WriteString("job_failed=0\n") + summary.WriteString("job_unsupported=1\n") + summary.WriteString("job_informational_failed=0\n") + if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil { + return "", err + } + return runDir, nil +} + func tpmValidationJobs() []satJob { return []satJob{ {name: "01-properties-fixed.log", cmd: []string{"tpm2_getcap", "properties-fixed"}}, diff --git a/audit/internal/platform/tpm_test.go b/audit/internal/platform/tpm_test.go index 4834a5d..bc935b3 100644 --- a/audit/internal/platform/tpm_test.go +++ b/audit/internal/platform/tpm_test.go @@ -1,11 +1,66 @@ package platform import ( + "os" + "path/filepath" "reflect" "strings" "testing" ) +func TestRunTPMValidationPackSkipsWhenNoTPMDevice(t *testing.T) { + old := tpmDeviceGlob + tpmDeviceGlob = func() []string { return nil } + t.Cleanup(func() { tpmDeviceGlob = old }) + + dir := t.TempDir() + runDir, err := (&System{}).RunTPMValidationPack(nil, dir, nil) + if err != nil { + t.Fatalf("RunTPMValidationPack: %v", err) + } + for _, name := range []string{"01-properties-fixed.log", "02-pcr-banks.log", "03-pcr-values.log", "04-test-result.log"} { + if _, err := os.Stat(filepath.Join(runDir, name)); err == nil { + t.Fatalf("tpm2 job %q ran despite no TPM device", name) + } + } + summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt")) + if err != nil { + t.Fatalf("read summary: %v", err) + } + if !strings.Contains(string(summary), "overall_status=UNSUPPORTED") || + !strings.Contains(string(summary), "tpm_present=false") { + t.Fatalf("summary missing UNSUPPORTED/tpm_present markers:\n%s", summary) + } +} + +func TestTPMPresentRequiresVersion2(t *testing.T) { + oldGlob, oldRead := tpmDeviceGlob, tpmReadFile + t.Cleanup(func() { + tpmDeviceGlob = oldGlob + tpmReadFile = oldRead + }) + + tests := []struct { + name string + version string + readErr error + want bool + }{ + {name: "TPM 2", version: "2\n", want: true}, + {name: "TPM 1.2", version: "1\n", want: false}, + {name: "missing version attribute", readErr: os.ErrNotExist, want: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tpmDeviceGlob = func() []string { return []string{"/sys/class/tpm/tpm0"} } + tpmReadFile = func(string) ([]byte, error) { return []byte(test.version), test.readErr } + if got := (&System{}).TPMPresent(); got != test.want { + t.Fatalf("TPMPresent()=%v want %v", got, test.want) + } + }) + } +} + func TestTPMValidationJobsAreReadOnly(t *testing.T) { t.Parallel() diff --git a/audit/internal/webui/api.go b/audit/internal/webui/api.go index 0bf6942..b25e54b 100644 --- a/audit/internal/webui/api.go +++ b/audit/internal/webui/api.go @@ -2,14 +2,11 @@ package webui import ( "bufio" - "encoding/json" "errors" "fmt" "io" "net/http" - "os" "os/exec" - "path/filepath" "regexp" "sort" "strconv" @@ -484,1525 +481,3 @@ func streamCmdJob(j *jobState, cmd *exec.Cmd) error { } // ── Audit ───────────────────────────────────────────────────────────────────── - -func (h *handler) handleAPIAuditRun(w http.ResponseWriter, _ *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - t := &Task{ - ID: newJobID("audit"), - Name: "Audit", - Target: "audit", - Priority: defaultTaskPriority("audit", taskParams{}), - Status: TaskPending, - CreatedAt: time.Now(), - } - globalQueue.enqueue(t) - writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID}) -} - -func (h *handler) handleAPIAuditStream(w http.ResponseWriter, r *http.Request) { - id := r.URL.Query().Get("job_id") - if id == "" { - id = r.URL.Query().Get("task_id") - } - // Try task queue first, then legacy job manager - if j, ok := globalQueue.findJob(id); ok { - streamJob(w, r, j) - return - } - if j, ok := globalJobs.get(id); ok { - streamJob(w, r, j) - return - } - http.Error(w, "job not found", http.StatusNotFound) -} - -// ── SAT ─────────────────────────────────────────────────────────────────────── - -func (h *handler) handleAPISATRun(target string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - - var body struct { - Duration int `json:"duration"` - StressMode bool `json:"stress_mode"` - GPUIndices []int `json:"gpu_indices"` - ExcludeGPUIndices []int `json:"exclude_gpu_indices"` - StaggerGPUStart bool `json:"stagger_gpu_start"` - ParallelGPUs bool `json:"parallel_gpus"` - Loader string `json:"loader"` - Profile string `json:"profile"` - DisplayName string `json:"display_name"` - PlatformComponents []string `json:"platform_components"` - } - if r.Body != nil { - if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - } - - name := taskDisplayName(target, body.Profile, body.Loader) - if strings.TrimSpace(body.DisplayName) != "" { - name = body.DisplayName - } - params := taskParams{ - Duration: body.Duration, - StressMode: body.StressMode, - GPUIndices: body.GPUIndices, - ExcludeGPUIndices: body.ExcludeGPUIndices, - StaggerGPUStart: body.StaggerGPUStart, - ParallelGPUs: body.ParallelGPUs, - Loader: body.Loader, - BurnProfile: body.Profile, - DisplayName: body.DisplayName, - PlatformComponents: body.PlatformComponents, - } - tasks, err := buildNvidiaTaskSet(target, defaultTaskPriority(target, params), time.Now(), params, name, h.opts.App, "sat-"+target) - if err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return - } - for _, t := range tasks { - globalQueue.enqueue(t) - } - writeTaskRunResponse(w, tasks) - } -} - -// ── Scenario ───────────────────────────────────────────────────────────────── - -func (h *handler) handleAPIScenarioList(w http.ResponseWriter, _ *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - files, err := h.opts.App.ListAvailableScenarios() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - type scenarioFile struct { - Name string `json:"name"` - Description string `json:"description"` - Device string `json:"device"` - } - out := make([]scenarioFile, 0, len(files)) - for _, f := range files { - out = append(out, scenarioFile{Name: f.Name, Description: f.Description, Device: f.Device}) - } - writeJSON(w, out) -} - -func (h *handler) handleAPIScenarioRun(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var body struct { - Name string `json:"name"` - } - if r.Body != nil { - if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - } - name := strings.TrimSpace(body.Name) - if name == "" { - writeError(w, http.StatusBadRequest, "scenario name is required") - return - } - t := &Task{ - ID: newJobID("scenario"), - Name: "Scenario: " + name, - Target: "scenario", - Priority: defaultTaskPriority("scenario", taskParams{}), - Status: TaskPending, - CreatedAt: time.Now(), - } - t.params.ScenarioName = name - globalQueue.enqueue(t) - writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID}) -} - -func (h *handler) handleAPIBenchmarkNvidiaRunKind(target string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - - var body struct { - Profile string `json:"profile"` - SizeMB int `json:"size_mb"` - GPUIndices []int `json:"gpu_indices"` - ExcludeGPUIndices []int `json:"exclude_gpu_indices"` - RunNCCL *bool `json:"run_nccl"` - ParallelGPUs *bool `json:"parallel_gpus"` - RampUp *bool `json:"ramp_up"` - DisplayName string `json:"display_name"` - } - if r.Body != nil { - if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - } - - runNCCL := true - if body.RunNCCL != nil { - runNCCL = *body.RunNCCL - } - parallelGPUs := false - if body.ParallelGPUs != nil { - parallelGPUs = *body.ParallelGPUs - } - rampUp := false - if body.RampUp != nil { - rampUp = *body.RampUp - } - // Build a descriptive base name that includes profile and mode so the task - // list is self-explanatory without opening individual task detail pages. - profile := strings.TrimSpace(body.Profile) - if profile == "" { - profile = "standard" - } - name := taskDisplayName(target, "", "") - if strings.TrimSpace(body.DisplayName) != "" { - name = body.DisplayName - } - // Append profile tag. - name = fmt.Sprintf("%s · %s", name, profile) - - if target == "nvidia-bench-power" && parallelGPUs { - writeError(w, http.StatusBadRequest, "power / thermal fit benchmark uses sequential or ramp-up modes only") - return - } - - if rampUp && len(body.GPUIndices) > 1 { - // Ramp-up mode: RunNvidiaPowerBench internally ramps from 1 to N GPUs - // in Phase 2 (one additional GPU per step). A single task with all - // selected GPUs is sufficient — spawning N tasks with growing subsets - // would repeat all earlier steps redundantly. - gpus, err := apiListNvidiaGPUs(h.opts.App) - if err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return - } - resolved, err := expandSelectedGPUIndices(gpus, body.GPUIndices, body.ExcludeGPUIndices) - if err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return - } - if len(resolved) < 2 { - // Fall through to normal single-task path. - rampUp = false - } else { - now := time.Now() - rampRunID := fmt.Sprintf("ramp-%s", now.UTC().Format("20060102-150405")) - taskName := fmt.Sprintf("%s · ramp 1–%d · GPU %s", name, len(resolved), formatGPUIndexList(resolved)) - t := &Task{ - ID: newJobID("bee-bench-nvidia"), - Name: taskName, - Target: target, - Priority: defaultTaskPriority(target, taskParams{}), - Status: TaskPending, - CreatedAt: now, - params: taskParams{ - GPUIndices: append([]int(nil), resolved...), - SizeMB: body.SizeMB, - BenchmarkProfile: body.Profile, - RunNCCL: runNCCL, - ParallelGPUs: true, - RampTotal: len(resolved), - RampRunID: rampRunID, - DisplayName: taskName, - }, - } - globalQueue.enqueue(t) - writeTaskRunResponse(w, []*Task{t}) - return - } - } - - // For non-ramp tasks append mode tag. - if parallelGPUs { - name = fmt.Sprintf("%s · parallel", name) - } else { - name = fmt.Sprintf("%s · sequential", name) - } - - params := taskParams{ - GPUIndices: body.GPUIndices, - ExcludeGPUIndices: body.ExcludeGPUIndices, - SizeMB: body.SizeMB, - BenchmarkProfile: body.Profile, - RunNCCL: runNCCL, - ParallelGPUs: parallelGPUs, - DisplayName: body.DisplayName, - } - tasks, err := buildNvidiaTaskSet(target, defaultTaskPriority(target, params), time.Now(), params, name, h.opts.App, "bee-bench-nvidia") - if err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return - } - for _, t := range tasks { - globalQueue.enqueue(t) - } - writeTaskRunResponse(w, tasks) - } -} - -func (h *handler) handleAPIBenchmarkAutotuneRun() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var body struct { - Profile string `json:"profile"` - BenchmarkKind string `json:"benchmark_kind"` - SizeMB int `json:"size_mb"` - } - if r.Body != nil { - if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - } - profile := strings.TrimSpace(body.Profile) - if profile == "" { - profile = "standard" - } - benchmarkKind := strings.TrimSpace(body.BenchmarkKind) - if benchmarkKind == "" { - benchmarkKind = "power-fit" - } - now := time.Now() - taskName := fmt.Sprintf("NVIDIA Benchmark Autotune · %s · %s", profile, benchmarkKind) - t := &Task{ - ID: newJobID("bee-bench-autotune"), - Name: taskName, - Target: "nvidia-bench-autotune", - Priority: defaultTaskPriority("nvidia-bench-autotune", taskParams{}), - Status: TaskPending, - CreatedAt: now, - params: taskParams{ - BenchmarkProfile: profile, - BenchmarkKind: benchmarkKind, - SizeMB: body.SizeMB, - DisplayName: taskName, - }, - } - globalQueue.enqueue(t) - writeTaskRunResponse(w, []*Task{t}) - } -} - -func (h *handler) handleAPIBenchmarkAutotuneStatus(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - cfg, err := h.opts.App.LoadBenchmarkPowerAutotune() - if err != nil { - if os.IsNotExist(err) { - w.WriteHeader(http.StatusOK) - writeJSON(w, map[string]any{ - "configured": false, - "decision": platform.ResolveSystemPowerDecision(h.opts.ExportDir), - }) - return - } - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - w.WriteHeader(http.StatusOK) - writeJSON(w, map[string]any{ - "configured": true, - "config": cfg, - "decision": platform.ResolveSystemPowerDecision(h.opts.ExportDir), - }) -} - -func (h *handler) handleAPIBenchmarkNvidiaRun(w http.ResponseWriter, r *http.Request) { - h.handleAPIBenchmarkNvidiaRunKind("nvidia-bench-perf").ServeHTTP(w, r) -} - -func (h *handler) handleAPISATStream(w http.ResponseWriter, r *http.Request) { - id := r.URL.Query().Get("job_id") - if id == "" { - id = r.URL.Query().Get("task_id") - } - if j, ok := globalQueue.findJob(id); ok { - streamJob(w, r, j) - return - } - if j, ok := globalJobs.get(id); ok { - streamJob(w, r, j) - return - } - http.Error(w, "job not found", http.StatusNotFound) -} - -func (h *handler) handleAPISATAbort(w http.ResponseWriter, r *http.Request) { - id := r.URL.Query().Get("job_id") - if id == "" { - id = r.URL.Query().Get("task_id") - } - if t, ok := globalQueue.findByID(id); ok { - globalQueue.mu.Lock() - switch t.Status { - case TaskPending: - t.Status = TaskCancelled - now := time.Now() - t.DoneAt = &now - case TaskRunning: - if t.job == nil || !t.job.abort() { - globalQueue.mu.Unlock() - writeJSON(w, map[string]string{"status": "not_running"}) - return - } - globalQueue.mu.Unlock() - writeJSON(w, map[string]string{"status": "aborting"}) - return - } - globalQueue.mu.Unlock() - writeJSON(w, map[string]string{"status": "aborted"}) - return - } - if j, ok := globalJobs.get(id); ok { - if j.abort() { - writeJSON(w, map[string]string{"status": "aborted"}) - } else { - writeJSON(w, map[string]string{"status": "not_running"}) - } - return - } - http.Error(w, "job not found", http.StatusNotFound) -} - -// ── Services ────────────────────────────────────────────────────────────────── - -func (h *handler) handleAPIServicesList(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - names, err := h.opts.App.ListBeeServices() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - type serviceInfo struct { - Name string `json:"name"` - State string `json:"state"` - Body string `json:"body"` - } - result := make([]serviceInfo, 0, len(names)) - for _, name := range names { - state := h.opts.App.ServiceState(name) - body, _ := h.opts.App.ServiceStatus(name) - result = append(result, serviceInfo{Name: name, State: state, Body: body}) - } - writeJSON(w, result) -} - -func (h *handler) handleAPIServicesAction(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var req struct { - Name string `json:"name"` - Action string `json:"action"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - var action platform.ServiceAction - switch req.Action { - case "start": - action = platform.ServiceStart - case "stop": - action = platform.ServiceStop - case "restart": - action = platform.ServiceRestart - default: - writeError(w, http.StatusBadRequest, "action must be start|stop|restart") - return - } - result, err := h.opts.App.ServiceActionResult(req.Name, action) - status := "ok" - if err != nil { - status = "error" - } - // Always return 200 with output so the frontend can display the actual - // systemctl error message instead of a generic "exit status 1". - writeJSON(w, map[string]string{"status": status, "output": result.Body}) -} - -// ── Network ─────────────────────────────────────────────────────────────────── - -func (h *handler) handleAPINetworkStatus(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - ifaces, err := h.opts.App.ListInterfaces() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, map[string]any{ - "interfaces": ifaces, - "default_route": h.opts.App.DefaultRoute(), - "pending_change": h.hasPendingNetworkChange(), - "rollback_in": h.pendingNetworkRollbackIn(), - }) -} - -func (h *handler) handleAPINetworkDHCP(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var req struct { - Interface string `json:"interface"` - } - _ = json.NewDecoder(r.Body).Decode(&req) - - result, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) { - if req.Interface == "" || req.Interface == "all" { - return h.opts.App.DHCPAllResult() - } - return h.opts.App.DHCPOneResult(req.Interface) - }) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, map[string]any{ - "status": "ok", - "output": result.Body, - "rollback_in": int(netRollbackTimeout.Seconds()), - }) -} - -func (h *handler) handleAPINetworkStatic(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var req struct { - Interface string `json:"interface"` - Address string `json:"address"` - Prefix string `json:"prefix"` - Gateway string `json:"gateway"` - DNS []string `json:"dns"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - cfg := platform.StaticIPv4Config{ - Interface: req.Interface, - Address: req.Address, - Prefix: req.Prefix, - Gateway: req.Gateway, - DNS: req.DNS, - } - result, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) { - return h.opts.App.SetStaticIPv4Result(cfg) - }) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, map[string]any{ - "status": "ok", - "output": result.Body, - "rollback_in": int(netRollbackTimeout.Seconds()), - }) -} - -// ── Export ──────────────────────────────────────────────────────────────────── - -func (h *handler) handleAPIExportList(w http.ResponseWriter, r *http.Request) { - entries, err := listExportFiles(h.opts.ExportDir) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, entries) -} - -func (h *handler) handleAPIExportUSBTargets(w http.ResponseWriter, _ *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - targets, err := h.opts.App.ListRemovableTargets() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - if targets == nil { - targets = []platform.RemovableTarget{} - } - writeJSON(w, targets) -} - -func (h *handler) handleAPIExportUSBAudit(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var target platform.RemovableTarget - if err := json.NewDecoder(r.Body).Decode(&target); err != nil || target.Device == "" { - writeError(w, http.StatusBadRequest, "device is required") - return - } - result, err := h.opts.App.ExportLatestAuditResult(target) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, map[string]string{"status": "ok", "message": result.Body}) -} - -func (h *handler) handleAPIExportUSBBundle(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var target platform.RemovableTarget - if err := json.NewDecoder(r.Body).Decode(&target); err != nil || target.Device == "" { - writeError(w, http.StatusBadRequest, "device is required") - return - } - result, err := h.opts.App.ExportSupportBundleResult(target) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, map[string]string{"status": "ok", "message": result.Body}) -} - -func (h *handler) handleAPIBlackboxStatus(w http.ResponseWriter, _ *http.Request) { - state, err := app.ReadBlackboxState(filepath.Join(h.opts.ExportDir, "blackbox-state.json")) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - writeJSON(w, app.BlackboxState{Status: "disabled", Targets: []app.BlackboxTargetStatus{}}) - return - } - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - if state.Targets == nil { - state.Targets = []app.BlackboxTargetStatus{} - } - writeJSON(w, state) -} - -func (h *handler) handleAPIBlackboxEnable(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var target platform.RemovableTarget - if err := json.NewDecoder(r.Body).Decode(&target); err != nil || strings.TrimSpace(target.Device) == "" { - writeError(w, http.StatusBadRequest, "device is required") - return - } - targets, err := h.opts.App.ListRemovableTargets() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - allowed := false - for _, candidate := range targets { - if candidate.Device == target.Device { - target = candidate - allowed = true - break - } - } - if !allowed { - writeError(w, http.StatusBadRequest, "device not in removable target list") - return - } - marker, err := app.EnableBlackboxTarget(target) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, map[string]any{ - "status": "ok", - "message": "Black-box marker written.", - "enrollment_id": marker.EnrollmentID, - }) -} - -func (h *handler) handleAPIBlackboxDisable(w http.ResponseWriter, r *http.Request) { - var req struct { - Device string `json:"device"` - EnrollmentID string `json:"enrollment_id"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - if err := app.DisableBlackboxTarget(req.Device, req.EnrollmentID); err != nil { - if errors.Is(err, os.ErrNotExist) { - writeError(w, http.StatusNotFound, "black-box target not found") - return - } - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, map[string]string{"status": "ok", "message": "Black-box marker removed."}) -} - -// ── GPU presence ────────────────────────────────────────────────────────────── - -func (h *handler) handleAPIGNVIDIAGPUs(w http.ResponseWriter, _ *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - gpus, err := h.opts.App.ListNvidiaGPUs() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - if gpus == nil { - gpus = []platform.NvidiaGPU{} - } - writeJSON(w, gpus) -} - -func (h *handler) handleAPIGNVIDIAGPUStatuses(w http.ResponseWriter, _ *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - gpus, err := apiListNvidiaGPUStatuses(h.opts.App) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - if gpus == nil { - gpus = []platform.NvidiaGPUStatus{} - } - writeJSON(w, gpus) -} - -func (h *handler) handleAPIGNVIDIAReset(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var req struct { - Index int `json:"index"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - result, err := h.opts.App.ResetNvidiaGPU(req.Index) - status := "ok" - if err != nil { - status = "error" - } - writeJSON(w, map[string]string{"status": status, "output": result.Body}) -} - -// ── GPU settings (ECC / power limit) ────────────────────────────────────────── - -func (h *handler) handleAPIGNVIDIAGPUSettings(w http.ResponseWriter, _ *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - settings, err := h.opts.App.ListNvidiaGPUSettings() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - if settings == nil { - settings = []platform.NvidiaGPUSetting{} - } - writeJSON(w, settings) -} - -func (h *handler) handleAPIGNVIDIASetECC(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var req struct { - Index int `json:"index"` - Enabled bool `json:"enabled"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - result, err := h.opts.App.SetNvidiaGPUECC(req.Index, req.Enabled) - status := "ok" - if err != nil { - status = "error" - } - writeJSON(w, map[string]string{"status": status, "output": result.Body}) -} - -func (h *handler) handleAPIGNVIDIASetMIG(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var req struct { - Index int `json:"index"` - Enabled bool `json:"enabled"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - result, err := h.opts.App.SetNvidiaGPUMIG(req.Index, req.Enabled) - status := "ok" - if err != nil { - status = "error" - } - writeJSON(w, map[string]string{"status": status, "output": result.Body}) -} - -func (h *handler) handleAPIGNVIDIASetCCMode(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var req struct { - Index int `json:"index"` - Enabled bool `json:"enabled"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - result, err := h.opts.App.SetNvidiaGPUCCMode(req.Index, req.Enabled) - status := "ok" - if err != nil { - status = "error" - } - writeJSON(w, map[string]string{"status": status, "output": result.Body}) -} - -func (h *handler) handleAPIGNVIDIASetPowerLimit(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var req struct { - Index int `json:"index"` - Watts float64 `json:"watts"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - if req.Watts <= 0 { - writeError(w, http.StatusBadRequest, "watts must be > 0") - return - } - result, err := h.opts.App.SetNvidiaGPUPowerLimit(req.Index, req.Watts) - status := "ok" - if err != nil { - status = "error" - } - writeJSON(w, map[string]string{"status": status, "output": result.Body}) -} - -func (h *handler) handleAPIGNVIDIAResetDefaults(w http.ResponseWriter, _ *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - result, err := h.opts.App.ResetNvidiaGPUDefaults() - status := "ok" - if err != nil { - status = "error" - } - writeJSON(w, map[string]string{"status": status, "output": result.Body}) -} - -func (h *handler) handleAPIGPUPresence(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - gp := h.opts.App.DetectGPUPresence() - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]bool{ - "nvidia": gp.Nvidia, - "amd": gp.AMD, - }) -} - -// ── GPU tools ───────────────────────────────────────────────────────────────── - -func (h *handler) handleAPIGPUTools(w http.ResponseWriter, _ *http.Request) { - type toolEntry struct { - ID string `json:"id"` - Available bool `json:"available"` - Vendor string `json:"vendor"` // "nvidia" | "amd" - } - _, nvidiaErr := os.Stat("/dev/nvidia0") - _, amdErr := os.Stat("/dev/kfd") - nvidiaUp := nvidiaErr == nil - amdUp := amdErr == nil - _, dcgmErr := exec.LookPath("dcgmi") - _, ncclStressErr := exec.LookPath("bee-nccl-gpu-stress") - _, johnErr := exec.LookPath("bee-john-gpu-stress") - _, beeBurnErr := exec.LookPath("bee-gpu-burn") - _, nvBandwidthErr := exec.LookPath("nvbandwidth") - profErr := lookPathAny("dcgmproftester", "dcgmproftester13", "dcgmproftester12", "dcgmproftester11") - writeJSON(w, []toolEntry{ - {ID: "nvidia-compute", Available: nvidiaUp && profErr == nil, Vendor: "nvidia"}, - {ID: "nvidia-targeted-power", Available: nvidiaUp && dcgmErr == nil, Vendor: "nvidia"}, - {ID: "nvidia-pulse", Available: nvidiaUp && dcgmErr == nil, Vendor: "nvidia"}, - {ID: "nvidia-interconnect", Available: nvidiaUp && ncclStressErr == nil, Vendor: "nvidia"}, - {ID: "nvidia-bandwidth", Available: nvidiaUp && dcgmErr == nil && nvBandwidthErr == nil, Vendor: "nvidia"}, - {ID: "bee-gpu-burn", Available: nvidiaUp && beeBurnErr == nil, Vendor: "nvidia"}, - {ID: "john", Available: nvidiaUp && johnErr == nil, Vendor: "nvidia"}, - {ID: "rvs", Available: amdUp, Vendor: "amd"}, - }) -} - -func lookPathAny(names ...string) error { - for _, name := range names { - if _, err := exec.LookPath(name); err == nil { - return nil - } - } - return exec.ErrNotFound -} - -// ── System ──────────────────────────────────────────────────────────────────── - -func (h *handler) handleAPIRAMStatus(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - status := h.currentRAMStatus() - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(status) -} - -type ramStatusResponse struct { - platform.LiveMediaRAMState - InstallTaskActive bool `json:"install_task_active,omitempty"` - CopyTaskActive bool `json:"copy_task_active,omitempty"` - CanStartTask bool `json:"can_start_task,omitempty"` - BlockedReason string `json:"blocked_reason,omitempty"` -} - -func (h *handler) currentRAMStatus() ramStatusResponse { - state := h.opts.App.LiveMediaRAMState() - resp := ramStatusResponse{LiveMediaRAMState: state} - if globalQueue.hasActiveTarget("install") { - resp.InstallTaskActive = true - resp.BlockedReason = "install to disk is already running" - return resp - } - if globalQueue.hasActiveTarget("install-to-ram") { - resp.CopyTaskActive = true - resp.BlockedReason = "install to RAM task is already pending or running" - return resp - } - if state.InRAM { - resp.BlockedReason = "system is already running from RAM" - return resp - } - resp.CanStartTask = state.CanStartCopy - if !resp.CanStartTask && resp.BlockedReason == "" { - resp.BlockedReason = state.Message - } - return resp -} - -func (h *handler) handleAPIInstallToRAM(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - status := h.currentRAMStatus() - if !status.CanStartTask { - msg := strings.TrimSpace(status.BlockedReason) - if msg == "" { - msg = "install to RAM is not available" - } - writeError(w, http.StatusConflict, msg) - return - } - t := &Task{ - ID: newJobID("install-to-ram"), - Name: "Install to RAM", - Target: "install-to-ram", - Priority: defaultTaskPriority("install-to-ram", taskParams{}), - Status: TaskPending, - CreatedAt: time.Now(), - } - globalQueue.enqueue(t) - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]string{"task_id": t.ID}) -} - -func (h *handler) handleAPISystemReboot(w http.ResponseWriter, r *http.Request) { - if err := exec.Command("systemctl", "reboot").Start(); err != nil { - writeError(w, http.StatusInternalServerError, "reboot failed: "+err.Error()) - return - } - writeJSON(w, map[string]string{"status": "rebooting"}) -} - -func (h *handler) handleAPISystemShutdown(w http.ResponseWriter, r *http.Request) { - if err := exec.Command("systemctl", "poweroff").Start(); err != nil { - writeError(w, http.StatusInternalServerError, "shutdown failed: "+err.Error()) - return - } - writeJSON(w, map[string]string{"status": "shutting down"}) -} - -// timezoneNameRE matches IANA timezone identifiers like "Europe/Moscow" or "UTC". -var timezoneNameRE = regexp.MustCompile(`^[A-Za-z0-9_+\-]+(/[A-Za-z0-9_+\-]+)*$`) - -func validTimezoneName(tz string) bool { - if tz == "" || !timezoneNameRE.MatchString(tz) { - return false - } - _, err := os.Stat(filepath.Join("/usr/share/zoneinfo", tz)) - return err == nil -} - -// handleAPISystemTimeSync sets the host's timezone and wall-clock time from -// values supplied by the client's browser (used when the appliance has no -// network/NTP access to keep its own clock in sync). -func (h *handler) handleAPISystemTimeSync(w http.ResponseWriter, r *http.Request) { - var req struct { - Timezone string `json:"timezone"` - EpochMS int64 `json:"epoch_ms"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - if req.EpochMS <= 0 { - writeError(w, http.StatusBadRequest, "epoch_ms required") - return - } - - var out strings.Builder - - if req.Timezone != "" { - if !validTimezoneName(req.Timezone) { - writeError(w, http.StatusBadRequest, "invalid timezone") - return - } - if b, err := exec.Command("timedatectl", "set-timezone", req.Timezone).CombinedOutput(); err != nil { - writeError(w, http.StatusInternalServerError, "set-timezone failed: "+strings.TrimSpace(string(b))) - return - } - fmt.Fprintf(&out, "timezone set to %s\n", req.Timezone) - } - - // Manual time only sticks if NTP sync is off. - _ = exec.Command("timedatectl", "set-ntp", "false").Run() - - sec := req.EpochMS / 1000 - if b, err := exec.Command("date", "-u", "-s", "@"+strconv.FormatInt(sec, 10)).CombinedOutput(); err != nil { - writeError(w, http.StatusInternalServerError, "set-time failed: "+strings.TrimSpace(string(b))) - return - } - out.WriteString("system clock synced\n") - - writeJSON(w, map[string]string{"status": "ok", "output": out.String()}) -} - -// ── Tools ───────────────────────────────────────────────────────────────────── - -var standardTools = []string{ - "dmidecode", "smartctl", "nvme", "lspci", "ipmitool", - "tpm2_getcap", "tpm2_pcrread", "tpm2_gettestresult", - "nvidia-smi", "dcgmi", "nv-hostengine", "memtester", "stress-ng", "nvtop", - "mstflint", "saa", -} - -func (h *handler) handleAPIToolsCheck(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - statuses := h.opts.App.CheckTools(standardTools) - writeJSON(w, statuses) -} - -// ── Preflight ───────────────────────────────────────────────────────────────── - -func (h *handler) handleAPIPreflight(w http.ResponseWriter, r *http.Request) { - data, err := loadSnapshot(filepath.Join(h.opts.ExportDir, "runtime-health.json")) - if err != nil { - writeError(w, http.StatusNotFound, "runtime health not found") - return - } - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.Header().Set("Cache-Control", "no-store") - _, _ = w.Write(data) -} - -// ── Install ─────────────────────────────────────────────────────────────────── - -func (h *handler) handleAPIInstallDisks(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - disks, err := h.opts.App.ListInstallDisks() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - type diskJSON struct { - Device string `json:"device"` - Model string `json:"model"` - Size string `json:"size"` - SizeBytes int64 `json:"size_bytes"` - MountedParts []string `json:"mounted_parts"` - Warnings []string `json:"warnings"` - } - result := make([]diskJSON, 0, len(disks)) - for _, d := range disks { - result = append(result, diskJSON{ - Device: d.Device, - Model: d.Model, - Size: d.Size, - SizeBytes: d.SizeBytes, - MountedParts: d.MountedParts, - Warnings: platform.DiskWarnings(d), - }) - } - writeJSON(w, result) -} - -func (h *handler) handleAPIInstallRun(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var req struct { - Device string `json:"device"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Device == "" { - writeError(w, http.StatusBadRequest, "device is required") - return - } - - // Whitelist: only allow devices that ListInstallDisks() returns. - disks, err := h.opts.App.ListInstallDisks() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - allowed := false - for _, d := range disks { - if d.Device == req.Device { - allowed = true - break - } - } - if !allowed { - writeError(w, http.StatusBadRequest, "device not in install candidate list") - return - } - if globalQueue.hasActiveTarget("install-to-ram") { - writeError(w, http.StatusConflict, "install to RAM task is already pending or running") - return - } - if globalQueue.hasActiveTarget("install") { - writeError(w, http.StatusConflict, "install task is already pending or running") - return - } - t := &Task{ - ID: newJobID("install"), - Name: "Install to Disk", - Target: "install", - Priority: defaultTaskPriority("install", taskParams{}), - Status: TaskPending, - CreatedAt: time.Now(), - params: taskParams{ - Device: req.Device, - }, - } - globalQueue.enqueue(t) - writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID}) -} - -// ── Metrics SSE ─────────────────────────────────────────────────────────────── - -func (h *handler) handleAPIMetricsLatest(w http.ResponseWriter, r *http.Request) { - sample, ok := h.latestMetric() - if !ok { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte("{}")) - return - } - b, err := json.Marshal(sample) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write(b) -} - -func (h *handler) handleAPIMetricsStream(w http.ResponseWriter, r *http.Request) { - if !sseStart(w) { - return - } - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - for { - select { - case <-r.Context().Done(): - return - case <-ticker.C: - sample, ok := h.latestMetric() - if !ok { - continue - } - b, err := json.Marshal(sample) - if err != nil { - continue - } - if !sseWrite(w, "metrics", string(b)) { - return - } - } - } -} - -// feedRings pushes one sample into all in-memory ring buffers. -func (h *handler) feedRings(sample platform.LiveMetricSample) { - for _, t := range sample.Temps { - switch t.Group { - case "cpu": - h.pushNamedMetricRing(&h.cpuTempRings, t.Name, t.Celsius) - case "ambient": - h.pushNamedMetricRing(&h.ambientTempRings, t.Name, t.Celsius) - } - } - h.ringPower.push(sample.PowerW) - h.ringCPULoad.push(sample.CPULoadPct) - h.ringMemLoad.push(sample.MemLoadPct) - - h.ringsMu.Lock() - h.pushFanRings(sample.Fans) - for _, gpu := range sample.GPUs { - idx := gpu.GPUIndex - for len(h.gpuRings) <= idx { - h.gpuRings = append(h.gpuRings, &gpuRings{ - Temp: newMetricsRing(120), - Util: newMetricsRing(120), - MemUtil: newMetricsRing(120), - Power: newMetricsRing(120), - }) - } - h.gpuRings[idx].Temp.push(gpu.TempC) - h.gpuRings[idx].Util.push(gpu.UsagePct) - h.gpuRings[idx].MemUtil.push(gpu.MemUsagePct) - h.gpuRings[idx].Power.push(gpu.PowerW) - } - h.ringsMu.Unlock() -} - -func (h *handler) pushFanRings(fans []platform.FanReading) { - if len(fans) == 0 && len(h.ringFans) == 0 { - return - } - fanValues := make(map[string]float64, len(fans)) - for _, fan := range fans { - if fan.Name == "" { - continue - } - fanValues[fan.Name] = fan.RPM - found := false - for i, name := range h.fanNames { - if name == fan.Name { - found = true - if i >= len(h.ringFans) { - h.ringFans = append(h.ringFans, newMetricsRing(120)) - } - break - } - } - if !found { - h.fanNames = append(h.fanNames, fan.Name) - h.ringFans = append(h.ringFans, newMetricsRing(120)) - } - } - for i, ring := range h.ringFans { - if ring == nil { - continue - } - name := "" - if i < len(h.fanNames) { - name = h.fanNames[i] - } - if rpm, ok := fanValues[name]; ok { - ring.push(rpm) - continue - } - if last, ok := ring.latest(); ok { - ring.push(last) - continue - } - ring.push(0) - } -} - -func (h *handler) pushNamedMetricRing(dst *[]*namedMetricsRing, name string, value float64) { - if name == "" { - return - } - for _, item := range *dst { - if item != nil && item.Name == name && item.Ring != nil { - item.Ring.push(value) - return - } - } - *dst = append(*dst, &namedMetricsRing{ - Name: name, - Ring: newMetricsRing(120), - }) - (*dst)[len(*dst)-1].Ring.push(value) -} - -// ── Network toggle ──────────────────────────────────────────────────────────── - -const netRollbackTimeout = 60 * time.Second - -func (h *handler) handleAPINetworkToggle(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var req struct { - Iface string `json:"iface"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Iface == "" { - writeError(w, http.StatusBadRequest, "iface is required") - return - } - - wasUp, err := h.opts.App.GetInterfaceState(req.Iface) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - if _, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) { - err := h.opts.App.SetInterfaceState(req.Iface, !wasUp) - return app.ActionResult{}, err - }); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - newState := "up" - if wasUp { - newState = "down" - } - writeJSON(w, map[string]any{ - "iface": req.Iface, - "new_state": newState, - "rollback_in": int(netRollbackTimeout.Seconds()), - }) -} - -func (h *handler) applyPendingNetworkChange(apply func() (app.ActionResult, error)) (app.ActionResult, error) { - if h.opts.App == nil { - return app.ActionResult{}, fmt.Errorf("app not configured") - } - - if err := h.rollbackPendingNetworkChange(); err != nil && err.Error() != "no pending network change" { - return app.ActionResult{}, err - } - - snapshot, err := h.opts.App.CaptureNetworkSnapshot() - if err != nil { - return app.ActionResult{}, err - } - - result, err := apply() - if err != nil { - return result, err - } - - pnc := &pendingNetChange{ - snapshot: snapshot, - deadline: time.Now().Add(netRollbackTimeout), - } - pnc.timer = time.AfterFunc(netRollbackTimeout, func() { - _ = h.opts.App.RestoreNetworkSnapshot(snapshot) - h.pendingNetMu.Lock() - if h.pendingNet == pnc { - h.pendingNet = nil - } - h.pendingNetMu.Unlock() - }) - - h.pendingNetMu.Lock() - h.pendingNet = pnc - h.pendingNetMu.Unlock() - - return result, nil -} - -func (h *handler) hasPendingNetworkChange() bool { - h.pendingNetMu.Lock() - defer h.pendingNetMu.Unlock() - return h.pendingNet != nil -} - -func (h *handler) pendingNetworkRollbackIn() int { - h.pendingNetMu.Lock() - defer h.pendingNetMu.Unlock() - if h.pendingNet == nil { - return 0 - } - remaining := int(time.Until(h.pendingNet.deadline).Seconds()) - if remaining < 1 { - return 1 - } - return remaining -} - -func (h *handler) handleAPINetworkConfirm(w http.ResponseWriter, _ *http.Request) { - h.pendingNetMu.Lock() - pnc := h.pendingNet - h.pendingNet = nil - h.pendingNetMu.Unlock() - if pnc != nil { - pnc.mu.Lock() - pnc.timer.Stop() - pnc.mu.Unlock() - } - writeJSON(w, map[string]string{"status": "confirmed"}) -} - -func (h *handler) handleAPINetworkRollback(w http.ResponseWriter, _ *http.Request) { - if err := h.rollbackPendingNetworkChange(); err != nil { - if err.Error() == "no pending network change" { - writeError(w, http.StatusConflict, err.Error()) - return - } - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, map[string]string{"status": "rolled back"}) -} - -func (h *handler) handleAPIBenchmarkResults(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/html; charset=utf-8") - fmt.Fprint(w, renderBenchmarkResultsCard(h.opts.ExportDir)) -} - -// ── Hardware summary / component detail ────────────────────────────────────── - -// handleAPIHardwareSummary returns the hardware summary card HTML fragment for -// htmx polling (hx-get="/api/hardware-summary" hx-swap="outerHTML"). -func (h *handler) handleAPIHardwareSummary(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "text/html; charset=utf-8") - w.Header().Set("Cache-Control", "no-store") - fmt.Fprint(w, renderHardwareSummaryCard(h.opts)) -} - -// handleAPIComponentDetail returns an HTML fragment describing the current and -// historical status for one component type (cpu, memory, storage, gpu, psu). -func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Request) { - compType := r.PathValue("type") - var exact, prefixes []string - var title string - switch compType { - case "cpu": - title = "CPU" - exact = []string{"cpu:all"} - case "memory": - title = "Memory" - exact = []string{"memory:all"} - prefixes = []string{"memory:"} - case "storage": - title = "Storage" - exact = []string{"storage:all"} - prefixes = []string{"storage:"} - case "gpu": - title = "GPU" - prefixes = []string{"pcie:gpu:"} - case "nic": - title = "NIC" - prefixes = []string{"pcie:nic:"} - case "psu": - title = "PSU" - prefixes = []string{"psu:"} - case "raid": - title = "RAID" - prefixes = []string{"pcie:raid:"} - default: - http.NotFound(w, r) - return - } - - var records []app.ComponentStatusRecord - if h.opts.App != nil && h.opts.App.StatusDB != nil { - all := h.opts.App.StatusDB.All() - records = matchedRecords(all, exact, prefixes) - } - - fromInventory := false - if len(records) == 0 { - if fallback := inventoryFallbackRecords(compType, h.opts); len(fallback) > 0 { - records = fallback - fromInventory = true - } - } - - w.Header().Set("Content-Type", "text/html; charset=utf-8") - w.Header().Set("Cache-Control", "no-store") - fmt.Fprint(w, renderComponentDetail(title, records, fromInventory)) -} - -func (h *handler) rollbackPendingNetworkChange() error { - h.pendingNetMu.Lock() - pnc := h.pendingNet - h.pendingNet = nil - h.pendingNetMu.Unlock() - if pnc == nil { - return fmt.Errorf("no pending network change") - } - pnc.mu.Lock() - pnc.timer.Stop() - pnc.mu.Unlock() - if h.opts.App != nil { - return h.opts.App.RestoreNetworkSnapshot(pnc.snapshot) - } - return nil -} diff --git a/audit/internal/webui/api_gpu_tools.go b/audit/internal/webui/api_gpu_tools.go new file mode 100644 index 0000000..208b6d8 --- /dev/null +++ b/audit/internal/webui/api_gpu_tools.go @@ -0,0 +1,509 @@ +package webui + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "bee/audit/internal/platform" +) + +func (h *handler) handleAPIGNVIDIAGPUs(w http.ResponseWriter, _ *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + gpus, err := h.opts.App.ListNvidiaGPUs() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if gpus == nil { + gpus = []platform.NvidiaGPU{} + } + writeJSON(w, gpus) +} + +func (h *handler) handleAPIGNVIDIAGPUStatuses(w http.ResponseWriter, _ *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + gpus, err := apiListNvidiaGPUStatuses(h.opts.App) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if gpus == nil { + gpus = []platform.NvidiaGPUStatus{} + } + writeJSON(w, gpus) +} + +func (h *handler) handleAPIGNVIDIAReset(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var req struct { + Index int `json:"index"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + result, err := h.opts.App.ResetNvidiaGPU(req.Index) + status := "ok" + if err != nil { + status = "error" + } + writeJSON(w, map[string]string{"status": status, "output": result.Body}) +} + +// ── GPU settings (ECC / power limit) ────────────────────────────────────────── + +func (h *handler) handleAPIGNVIDIAGPUSettings(w http.ResponseWriter, _ *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + settings, err := h.opts.App.ListNvidiaGPUSettings() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if settings == nil { + settings = []platform.NvidiaGPUSetting{} + } + writeJSON(w, settings) +} + +func (h *handler) handleAPIGNVIDIASetECC(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var req struct { + Index int `json:"index"` + Enabled bool `json:"enabled"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + result, err := h.opts.App.SetNvidiaGPUECC(req.Index, req.Enabled) + status := "ok" + if err != nil { + status = "error" + } + writeJSON(w, map[string]string{"status": status, "output": result.Body}) +} + +func (h *handler) handleAPIGNVIDIASetMIG(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var req struct { + Index int `json:"index"` + Enabled bool `json:"enabled"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + result, err := h.opts.App.SetNvidiaGPUMIG(req.Index, req.Enabled) + status := "ok" + if err != nil { + status = "error" + } + writeJSON(w, map[string]string{"status": status, "output": result.Body}) +} + +func (h *handler) handleAPIGNVIDIASetCCMode(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var req struct { + Index int `json:"index"` + Enabled bool `json:"enabled"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + result, err := h.opts.App.SetNvidiaGPUCCMode(req.Index, req.Enabled) + status := "ok" + if err != nil { + status = "error" + } + writeJSON(w, map[string]string{"status": status, "output": result.Body}) +} + +func (h *handler) handleAPIGNVIDIASetPowerLimit(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var req struct { + Index int `json:"index"` + Watts float64 `json:"watts"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + if req.Watts <= 0 { + writeError(w, http.StatusBadRequest, "watts must be > 0") + return + } + result, err := h.opts.App.SetNvidiaGPUPowerLimit(req.Index, req.Watts) + status := "ok" + if err != nil { + status = "error" + } + writeJSON(w, map[string]string{"status": status, "output": result.Body}) +} + +func (h *handler) handleAPIGNVIDIAResetDefaults(w http.ResponseWriter, _ *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + result, err := h.opts.App.ResetNvidiaGPUDefaults() + status := "ok" + if err != nil { + status = "error" + } + writeJSON(w, map[string]string{"status": status, "output": result.Body}) +} + +func (h *handler) handleAPIGPUPresence(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + gp := h.opts.App.DetectGPUPresence() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]bool{ + "nvidia": gp.Nvidia, + "amd": gp.AMD, + "nvidia_initializing": gp.NvidiaInitializing, + "amd_initializing": gp.AMDInitializing, + }) +} + +// ── GPU tools ───────────────────────────────────────────────────────────────── + +func (h *handler) handleAPIGPUTools(w http.ResponseWriter, _ *http.Request) { + type toolEntry struct { + ID string `json:"id"` + Available bool `json:"available"` + Vendor string `json:"vendor"` // "nvidia" | "amd" + } + // Single source of truth for GPU presence: see app.DetectGPUPresence. + var nvidiaUp, amdUp bool + if h.opts.App != nil { + gp := h.opts.App.DetectGPUPresence() + nvidiaUp, amdUp = gp.Nvidia, gp.AMD + } else { + _, nvidiaErr := os.Stat("/dev/nvidia0") + _, amdErr := os.Stat("/dev/kfd") + nvidiaUp, amdUp = nvidiaErr == nil, amdErr == nil + } + _, dcgmErr := exec.LookPath("dcgmi") + _, ncclStressErr := exec.LookPath("bee-nccl-gpu-stress") + _, johnErr := exec.LookPath("bee-john-gpu-stress") + _, beeBurnErr := exec.LookPath("bee-gpu-burn") + _, nvBandwidthErr := exec.LookPath("nvbandwidth") + profErr := lookPathAny("dcgmproftester", "dcgmproftester13", "dcgmproftester12", "dcgmproftester11") + writeJSON(w, []toolEntry{ + {ID: "nvidia-compute", Available: nvidiaUp && profErr == nil, Vendor: "nvidia"}, + {ID: "nvidia-targeted-power", Available: nvidiaUp && dcgmErr == nil, Vendor: "nvidia"}, + {ID: "nvidia-pulse", Available: nvidiaUp && dcgmErr == nil, Vendor: "nvidia"}, + {ID: "nvidia-interconnect", Available: nvidiaUp && ncclStressErr == nil, Vendor: "nvidia"}, + {ID: "nvidia-bandwidth", Available: nvidiaUp && dcgmErr == nil && nvBandwidthErr == nil, Vendor: "nvidia"}, + {ID: "bee-gpu-burn", Available: nvidiaUp && beeBurnErr == nil, Vendor: "nvidia"}, + {ID: "john", Available: nvidiaUp && johnErr == nil, Vendor: "nvidia"}, + {ID: "rvs", Available: amdUp, Vendor: "amd"}, + }) +} + +func lookPathAny(names ...string) error { + for _, name := range names { + if _, err := exec.LookPath(name); err == nil { + return nil + } + } + return exec.ErrNotFound +} + +// ── System ──────────────────────────────────────────────────────────────────── + +func (h *handler) handleAPIRAMStatus(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + status := h.currentRAMStatus() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(status) +} + +type ramStatusResponse struct { + platform.LiveMediaRAMState + InstallTaskActive bool `json:"install_task_active,omitempty"` + CopyTaskActive bool `json:"copy_task_active,omitempty"` + CanStartTask bool `json:"can_start_task,omitempty"` + BlockedReason string `json:"blocked_reason,omitempty"` +} + +func (h *handler) currentRAMStatus() ramStatusResponse { + state := h.opts.App.LiveMediaRAMState() + resp := ramStatusResponse{LiveMediaRAMState: state} + if globalQueue.hasActiveTarget("install") { + resp.InstallTaskActive = true + resp.BlockedReason = "install to disk is already running" + return resp + } + if globalQueue.hasActiveTarget("install-to-ram") { + resp.CopyTaskActive = true + resp.BlockedReason = "install to RAM task is already pending or running" + return resp + } + if state.InRAM { + resp.BlockedReason = "system is already running from RAM" + return resp + } + resp.CanStartTask = state.CanStartCopy + if !resp.CanStartTask && resp.BlockedReason == "" { + resp.BlockedReason = state.Message + } + return resp +} + +func (h *handler) handleAPIInstallToRAM(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + status := h.currentRAMStatus() + if !status.CanStartTask { + msg := strings.TrimSpace(status.BlockedReason) + if msg == "" { + msg = "install to RAM is not available" + } + writeError(w, http.StatusConflict, msg) + return + } + t := &Task{ + ID: newJobID("install-to-ram"), + Name: "Install to RAM", + Target: "install-to-ram", + Priority: defaultTaskPriority("install-to-ram", taskParams{}), + Status: TaskPending, + CreatedAt: time.Now(), + } + globalQueue.enqueue(t) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{"task_id": t.ID}) +} + +func (h *handler) handleAPISystemReboot(w http.ResponseWriter, r *http.Request) { + if err := exec.Command("systemctl", "reboot").Start(); err != nil { + writeError(w, http.StatusInternalServerError, "reboot failed: "+err.Error()) + return + } + writeJSON(w, map[string]string{"status": "rebooting"}) +} + +func (h *handler) handleAPISystemShutdown(w http.ResponseWriter, r *http.Request) { + if err := exec.Command("systemctl", "poweroff").Start(); err != nil { + writeError(w, http.StatusInternalServerError, "shutdown failed: "+err.Error()) + return + } + writeJSON(w, map[string]string{"status": "shutting down"}) +} + +// timezoneNameRE matches IANA timezone identifiers like "Europe/Moscow" or "UTC". +var timezoneNameRE = regexp.MustCompile(`^[A-Za-z0-9_+\-]+(/[A-Za-z0-9_+\-]+)*$`) + +func validTimezoneName(tz string) bool { + if tz == "" || !timezoneNameRE.MatchString(tz) { + return false + } + _, err := os.Stat(filepath.Join("/usr/share/zoneinfo", tz)) + return err == nil +} + +// handleAPISystemTimeSync sets the host's timezone and wall-clock time from +// values supplied by the client's browser (used when the appliance has no +// network/NTP access to keep its own clock in sync). +func (h *handler) handleAPISystemTimeSync(w http.ResponseWriter, r *http.Request) { + var req struct { + Timezone string `json:"timezone"` + EpochMS int64 `json:"epoch_ms"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + if req.EpochMS <= 0 { + writeError(w, http.StatusBadRequest, "epoch_ms required") + return + } + + var out strings.Builder + + if req.Timezone != "" { + if !validTimezoneName(req.Timezone) { + writeError(w, http.StatusBadRequest, "invalid timezone") + return + } + if b, err := exec.Command("timedatectl", "set-timezone", req.Timezone).CombinedOutput(); err != nil { + writeError(w, http.StatusInternalServerError, "set-timezone failed: "+strings.TrimSpace(string(b))) + return + } + fmt.Fprintf(&out, "timezone set to %s\n", req.Timezone) + } + + // Manual time only sticks if NTP sync is off. + _ = exec.Command("timedatectl", "set-ntp", "false").Run() + + sec := req.EpochMS / 1000 + if b, err := exec.Command("date", "-u", "-s", "@"+strconv.FormatInt(sec, 10)).CombinedOutput(); err != nil { + writeError(w, http.StatusInternalServerError, "set-time failed: "+strings.TrimSpace(string(b))) + return + } + out.WriteString("system clock synced\n") + + writeJSON(w, map[string]string{"status": "ok", "output": out.String()}) +} + +// ── Tools ───────────────────────────────────────────────────────────────────── + +var standardTools = []string{ + "dmidecode", "smartctl", "nvme", "lspci", "ipmitool", + "tpm2_getcap", "tpm2_pcrread", "tpm2_gettestresult", + "nvidia-smi", "dcgmi", "nv-hostengine", "memtester", "stress-ng", "nvtop", + "mstflint", "saa", +} + +func (h *handler) handleAPIToolsCheck(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + statuses := h.opts.App.CheckTools(standardTools) + writeJSON(w, statuses) +} + +// ── Preflight ───────────────────────────────────────────────────────────────── + +func (h *handler) handleAPIPreflight(w http.ResponseWriter, r *http.Request) { + data, err := loadSnapshot(filepath.Join(h.opts.ExportDir, "runtime-health.json")) + if err != nil { + writeError(w, http.StatusNotFound, "runtime health not found") + return + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write(data) +} + +// ── Install ─────────────────────────────────────────────────────────────────── + +func (h *handler) handleAPIInstallDisks(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + disks, err := h.opts.App.ListInstallDisks() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + type diskJSON struct { + Device string `json:"device"` + Model string `json:"model"` + Size string `json:"size"` + SizeBytes int64 `json:"size_bytes"` + MountedParts []string `json:"mounted_parts"` + Warnings []string `json:"warnings"` + } + result := make([]diskJSON, 0, len(disks)) + for _, d := range disks { + result = append(result, diskJSON{ + Device: d.Device, + Model: d.Model, + Size: d.Size, + SizeBytes: d.SizeBytes, + MountedParts: d.MountedParts, + Warnings: platform.DiskWarnings(d), + }) + } + writeJSON(w, result) +} + +func (h *handler) handleAPIInstallRun(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var req struct { + Device string `json:"device"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Device == "" { + writeError(w, http.StatusBadRequest, "device is required") + return + } + + // Whitelist: only allow devices that ListInstallDisks() returns. + disks, err := h.opts.App.ListInstallDisks() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + allowed := false + for _, d := range disks { + if d.Device == req.Device { + allowed = true + break + } + } + if !allowed { + writeError(w, http.StatusBadRequest, "device not in install candidate list") + return + } + if globalQueue.hasActiveTarget("install-to-ram") { + writeError(w, http.StatusConflict, "install to RAM task is already pending or running") + return + } + if globalQueue.hasActiveTarget("install") { + writeError(w, http.StatusConflict, "install task is already pending or running") + return + } + t := &Task{ + ID: newJobID("install"), + Name: "Install to Disk", + Target: "install", + Priority: defaultTaskPriority("install", taskParams{}), + Status: TaskPending, + CreatedAt: time.Now(), + params: taskParams{ + Device: req.Device, + }, + } + globalQueue.enqueue(t) + writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID}) +} + +// ── Metrics SSE ─────────────────────────────────────────────────────────────── diff --git a/audit/internal/webui/api_metrics_network.go b/audit/internal/webui/api_metrics_network.go new file mode 100644 index 0000000..3ae78cc --- /dev/null +++ b/audit/internal/webui/api_metrics_network.go @@ -0,0 +1,360 @@ +package webui + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "bee/audit/internal/app" + "bee/audit/internal/platform" +) + +func (h *handler) handleAPIMetricsLatest(w http.ResponseWriter, r *http.Request) { + sample, ok := h.latestMetric() + if !ok { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte("{}")) + return + } + b, err := json.Marshal(sample) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(b) +} + +func (h *handler) handleAPIMetricsStream(w http.ResponseWriter, r *http.Request) { + if !sseStart(w) { + return + } + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + for { + select { + case <-r.Context().Done(): + return + case <-ticker.C: + sample, ok := h.latestMetric() + if !ok { + continue + } + b, err := json.Marshal(sample) + if err != nil { + continue + } + if !sseWrite(w, "metrics", string(b)) { + return + } + } + } +} + +// feedRings pushes one sample into all in-memory ring buffers. +func (h *handler) feedRings(sample platform.LiveMetricSample) { + for _, t := range sample.Temps { + switch t.Group { + case "cpu": + h.pushNamedMetricRing(&h.cpuTempRings, t.Name, t.Celsius) + case "ambient": + h.pushNamedMetricRing(&h.ambientTempRings, t.Name, t.Celsius) + } + } + h.ringPower.push(sample.PowerW) + h.ringCPULoad.push(sample.CPULoadPct) + h.ringMemLoad.push(sample.MemLoadPct) + + h.ringsMu.Lock() + h.pushFanRings(sample.Fans) + for _, gpu := range sample.GPUs { + idx := gpu.GPUIndex + for len(h.gpuRings) <= idx { + h.gpuRings = append(h.gpuRings, &gpuRings{ + Temp: newMetricsRing(120), + Util: newMetricsRing(120), + MemUtil: newMetricsRing(120), + Power: newMetricsRing(120), + }) + } + h.gpuRings[idx].Temp.push(gpu.TempC) + h.gpuRings[idx].Util.push(gpu.UsagePct) + h.gpuRings[idx].MemUtil.push(gpu.MemUsagePct) + h.gpuRings[idx].Power.push(gpu.PowerW) + } + h.ringsMu.Unlock() +} + +func (h *handler) pushFanRings(fans []platform.FanReading) { + if len(fans) == 0 && len(h.ringFans) == 0 { + return + } + fanValues := make(map[string]float64, len(fans)) + for _, fan := range fans { + if fan.Name == "" { + continue + } + fanValues[fan.Name] = fan.RPM + found := false + for i, name := range h.fanNames { + if name == fan.Name { + found = true + if i >= len(h.ringFans) { + h.ringFans = append(h.ringFans, newMetricsRing(120)) + } + break + } + } + if !found { + h.fanNames = append(h.fanNames, fan.Name) + h.ringFans = append(h.ringFans, newMetricsRing(120)) + } + } + for i, ring := range h.ringFans { + if ring == nil { + continue + } + name := "" + if i < len(h.fanNames) { + name = h.fanNames[i] + } + if rpm, ok := fanValues[name]; ok { + ring.push(rpm) + continue + } + if last, ok := ring.latest(); ok { + ring.push(last) + continue + } + ring.push(0) + } +} + +func (h *handler) pushNamedMetricRing(dst *[]*namedMetricsRing, name string, value float64) { + if name == "" { + return + } + for _, item := range *dst { + if item != nil && item.Name == name && item.Ring != nil { + item.Ring.push(value) + return + } + } + *dst = append(*dst, &namedMetricsRing{ + Name: name, + Ring: newMetricsRing(120), + }) + (*dst)[len(*dst)-1].Ring.push(value) +} + +// ── Network toggle ──────────────────────────────────────────────────────────── + +const netRollbackTimeout = 60 * time.Second + +func (h *handler) handleAPINetworkToggle(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var req struct { + Iface string `json:"iface"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Iface == "" { + writeError(w, http.StatusBadRequest, "iface is required") + return + } + + wasUp, err := h.opts.App.GetInterfaceState(req.Iface) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + + if _, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) { + err := h.opts.App.SetInterfaceState(req.Iface, !wasUp) + return app.ActionResult{}, err + }); err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + + newState := "up" + if wasUp { + newState = "down" + } + writeJSON(w, map[string]any{ + "iface": req.Iface, + "new_state": newState, + "rollback_in": int(netRollbackTimeout.Seconds()), + }) +} + +func (h *handler) applyPendingNetworkChange(apply func() (app.ActionResult, error)) (app.ActionResult, error) { + if h.opts.App == nil { + return app.ActionResult{}, fmt.Errorf("app not configured") + } + + if err := h.rollbackPendingNetworkChange(); err != nil && err.Error() != "no pending network change" { + return app.ActionResult{}, err + } + + snapshot, err := h.opts.App.CaptureNetworkSnapshot() + if err != nil { + return app.ActionResult{}, err + } + + result, err := apply() + if err != nil { + return result, err + } + + pnc := &pendingNetChange{ + snapshot: snapshot, + deadline: time.Now().Add(netRollbackTimeout), + } + pnc.timer = time.AfterFunc(netRollbackTimeout, func() { + _ = h.opts.App.RestoreNetworkSnapshot(snapshot) + h.pendingNetMu.Lock() + if h.pendingNet == pnc { + h.pendingNet = nil + } + h.pendingNetMu.Unlock() + }) + + h.pendingNetMu.Lock() + h.pendingNet = pnc + h.pendingNetMu.Unlock() + + return result, nil +} + +func (h *handler) hasPendingNetworkChange() bool { + h.pendingNetMu.Lock() + defer h.pendingNetMu.Unlock() + return h.pendingNet != nil +} + +func (h *handler) pendingNetworkRollbackIn() int { + h.pendingNetMu.Lock() + defer h.pendingNetMu.Unlock() + if h.pendingNet == nil { + return 0 + } + remaining := int(time.Until(h.pendingNet.deadline).Seconds()) + if remaining < 1 { + return 1 + } + return remaining +} + +func (h *handler) handleAPINetworkConfirm(w http.ResponseWriter, _ *http.Request) { + h.pendingNetMu.Lock() + pnc := h.pendingNet + h.pendingNet = nil + h.pendingNetMu.Unlock() + if pnc != nil { + pnc.mu.Lock() + pnc.timer.Stop() + pnc.mu.Unlock() + } + writeJSON(w, map[string]string{"status": "confirmed"}) +} + +func (h *handler) handleAPINetworkRollback(w http.ResponseWriter, _ *http.Request) { + if err := h.rollbackPendingNetworkChange(); err != nil { + if err.Error() == "no pending network change" { + writeError(w, http.StatusConflict, err.Error()) + return + } + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, map[string]string{"status": "rolled back"}) +} + +func (h *handler) handleAPIBenchmarkResults(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + fmt.Fprint(w, renderBenchmarkResultsCard(h.opts.ExportDir)) +} + +// ── Hardware summary / component detail ────────────────────────────────────── + +// handleAPIHardwareSummary returns the hardware summary card HTML fragment for +// htmx polling (hx-get="/api/hardware-summary" hx-swap="outerHTML"). +func (h *handler) handleAPIHardwareSummary(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + fmt.Fprint(w, renderHardwareSummaryCard(h.opts)) +} + +// handleAPIComponentDetail returns an HTML fragment describing the current and +// historical status for one component type (cpu, memory, storage, gpu, psu). +func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Request) { + compType := r.PathValue("type") + var exact, prefixes []string + var title string + switch compType { + case "cpu": + title = "CPU" + exact = []string{"cpu:all"} + case "memory": + title = "Memory" + exact = []string{"memory:all"} + prefixes = []string{"memory:"} + case "storage": + title = "Storage" + exact = []string{"storage:all"} + prefixes = []string{"storage:"} + case "gpu": + title = "GPU" + prefixes = []string{"pcie:gpu:"} + case "nic": + title = "NIC" + prefixes = []string{"pcie:nic:"} + case "psu": + title = "PSU" + prefixes = []string{"psu:"} + case "raid": + title = "RAID" + prefixes = []string{"pcie:raid:"} + default: + http.NotFound(w, r) + return + } + + var records []app.ComponentStatusRecord + if h.opts.App != nil && h.opts.App.StatusDB != nil { + all := h.opts.App.StatusDB.All() + records = matchedRecords(all, exact, prefixes) + } + + fromInventory := false + if len(records) == 0 { + if fallback := inventoryFallbackRecords(compType, h.opts); len(fallback) > 0 { + records = fallback + fromInventory = true + } + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + fmt.Fprint(w, renderComponentDetail(title, records, fromInventory)) +} + +func (h *handler) rollbackPendingNetworkChange() error { + h.pendingNetMu.Lock() + pnc := h.pendingNet + h.pendingNet = nil + h.pendingNetMu.Unlock() + if pnc == nil { + return fmt.Errorf("no pending network change") + } + pnc.mu.Lock() + pnc.timer.Stop() + pnc.mu.Unlock() + if h.opts.App != nil { + return h.opts.App.RestoreNetworkSnapshot(pnc.snapshot) + } + return nil +} diff --git a/audit/internal/webui/api_runs.go b/audit/internal/webui/api_runs.go new file mode 100644 index 0000000..bbb9df9 --- /dev/null +++ b/audit/internal/webui/api_runs.go @@ -0,0 +1,430 @@ +package webui + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" + + "bee/audit/internal/platform" +) + +func (h *handler) handleAPIAuditRun(w http.ResponseWriter, _ *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + t := &Task{ + ID: newJobID("audit"), + Name: "Audit", + Target: "audit", + Priority: defaultTaskPriority("audit", taskParams{}), + Status: TaskPending, + CreatedAt: time.Now(), + } + globalQueue.enqueue(t) + writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID}) +} + +func (h *handler) handleAPIAuditStream(w http.ResponseWriter, r *http.Request) { + id := r.URL.Query().Get("job_id") + if id == "" { + id = r.URL.Query().Get("task_id") + } + // Try task queue first, then legacy job manager + if j, ok := globalQueue.findJob(id); ok { + streamJob(w, r, j) + return + } + if j, ok := globalJobs.get(id); ok { + streamJob(w, r, j) + return + } + http.Error(w, "job not found", http.StatusNotFound) +} + +// ── SAT ─────────────────────────────────────────────────────────────────────── + +func (h *handler) handleAPISATRun(target string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + + var body struct { + Duration int `json:"duration"` + StressMode bool `json:"stress_mode"` + GPUIndices []int `json:"gpu_indices"` + ExcludeGPUIndices []int `json:"exclude_gpu_indices"` + StaggerGPUStart bool `json:"stagger_gpu_start"` + ParallelGPUs bool `json:"parallel_gpus"` + Loader string `json:"loader"` + Profile string `json:"profile"` + DisplayName string `json:"display_name"` + PlatformComponents []string `json:"platform_components"` + } + if r.Body != nil { + if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + } + + params := taskParams{ + Duration: body.Duration, + StressMode: body.StressMode, + GPUIndices: body.GPUIndices, + ExcludeGPUIndices: body.ExcludeGPUIndices, + StaggerGPUStart: body.StaggerGPUStart, + ParallelGPUs: body.ParallelGPUs, + Loader: body.Loader, + BurnProfile: body.Profile, + DisplayName: body.DisplayName, + PlatformComponents: body.PlatformComponents, + } + tasks, err := h.enqueueSATTarget(target, params) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeTaskRunResponse(w, tasks) + } +} + +// enqueueSATTarget builds the task set for one SAT target (splitting +// homogeneous multi-GPU NVIDIA targets as needed) and enqueues it. Shared by +// the single-target /api/sat//run endpoints and /api/sat/run-all. +func (h *handler) enqueueSATTarget(target string, params taskParams) ([]*Task, error) { + name := taskDisplayName(target, params.BurnProfile, params.Loader) + if strings.TrimSpace(params.DisplayName) != "" { + name = params.DisplayName + } + tasks, err := buildNvidiaTaskSet(target, defaultTaskPriority(target, params), time.Now(), params, name, h.opts.App, "sat-"+target) + if err != nil { + return nil, err + } + for _, t := range tasks { + globalQueue.enqueue(t) + } + return tasks, nil +} + +// ── Scenario ───────────────────────────────────────────────────────────────── + +func (h *handler) handleAPIScenarioList(w http.ResponseWriter, _ *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + files, err := h.opts.App.ListAvailableScenarios() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + type scenarioFile struct { + Name string `json:"name"` + Description string `json:"description"` + Device string `json:"device"` + } + out := make([]scenarioFile, 0, len(files)) + for _, f := range files { + out = append(out, scenarioFile{Name: f.Name, Description: f.Description, Device: f.Device}) + } + writeJSON(w, out) +} + +func (h *handler) handleAPIScenarioRun(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var body struct { + Name string `json:"name"` + } + if r.Body != nil { + if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + } + name := strings.TrimSpace(body.Name) + if name == "" { + writeError(w, http.StatusBadRequest, "scenario name is required") + return + } + t := &Task{ + ID: newJobID("scenario"), + Name: "Scenario: " + name, + Target: "scenario", + Priority: defaultTaskPriority("scenario", taskParams{}), + Status: TaskPending, + CreatedAt: time.Now(), + } + t.params.ScenarioName = name + globalQueue.enqueue(t) + writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID}) +} + +func (h *handler) handleAPIBenchmarkNvidiaRunKind(target string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + + var body struct { + Profile string `json:"profile"` + SizeMB int `json:"size_mb"` + GPUIndices []int `json:"gpu_indices"` + ExcludeGPUIndices []int `json:"exclude_gpu_indices"` + RunNCCL *bool `json:"run_nccl"` + ParallelGPUs *bool `json:"parallel_gpus"` + RampUp *bool `json:"ramp_up"` + DisplayName string `json:"display_name"` + } + if r.Body != nil { + if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + } + + runNCCL := true + if body.RunNCCL != nil { + runNCCL = *body.RunNCCL + } + parallelGPUs := false + if body.ParallelGPUs != nil { + parallelGPUs = *body.ParallelGPUs + } + rampUp := false + if body.RampUp != nil { + rampUp = *body.RampUp + } + // Build a descriptive base name that includes profile and mode so the task + // list is self-explanatory without opening individual task detail pages. + profile := strings.TrimSpace(body.Profile) + if profile == "" { + profile = "standard" + } + name := taskDisplayName(target, "", "") + if strings.TrimSpace(body.DisplayName) != "" { + name = body.DisplayName + } + // Append profile tag. + name = fmt.Sprintf("%s · %s", name, profile) + + if target == "nvidia-bench-power" && parallelGPUs { + writeError(w, http.StatusBadRequest, "power / thermal fit benchmark uses sequential or ramp-up modes only") + return + } + + if rampUp && len(body.GPUIndices) > 1 { + // Ramp-up mode: RunNvidiaPowerBench internally ramps from 1 to N GPUs + // in Phase 2 (one additional GPU per step). A single task with all + // selected GPUs is sufficient — spawning N tasks with growing subsets + // would repeat all earlier steps redundantly. + gpus, err := apiListNvidiaGPUs(h.opts.App) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + resolved, err := expandSelectedGPUIndices(gpus, body.GPUIndices, body.ExcludeGPUIndices) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + if len(resolved) < 2 { + // Fall through to normal single-task path. + rampUp = false + } else { + now := time.Now() + rampRunID := fmt.Sprintf("ramp-%s", now.UTC().Format("20060102-150405")) + taskName := fmt.Sprintf("%s · ramp 1–%d · GPU %s", name, len(resolved), formatGPUIndexList(resolved)) + t := &Task{ + ID: newJobID("bee-bench-nvidia"), + Name: taskName, + Target: target, + Priority: defaultTaskPriority(target, taskParams{}), + Status: TaskPending, + CreatedAt: now, + params: taskParams{ + GPUIndices: append([]int(nil), resolved...), + SizeMB: body.SizeMB, + BenchmarkProfile: body.Profile, + RunNCCL: runNCCL, + ParallelGPUs: true, + RampTotal: len(resolved), + RampRunID: rampRunID, + DisplayName: taskName, + }, + } + globalQueue.enqueue(t) + writeTaskRunResponse(w, []*Task{t}) + return + } + } + + // For non-ramp tasks append mode tag. + if parallelGPUs { + name = fmt.Sprintf("%s · parallel", name) + } else { + name = fmt.Sprintf("%s · sequential", name) + } + + params := taskParams{ + GPUIndices: body.GPUIndices, + ExcludeGPUIndices: body.ExcludeGPUIndices, + SizeMB: body.SizeMB, + BenchmarkProfile: body.Profile, + RunNCCL: runNCCL, + ParallelGPUs: parallelGPUs, + DisplayName: body.DisplayName, + } + tasks, err := buildNvidiaTaskSet(target, defaultTaskPriority(target, params), time.Now(), params, name, h.opts.App, "bee-bench-nvidia") + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + for _, t := range tasks { + globalQueue.enqueue(t) + } + writeTaskRunResponse(w, tasks) + } +} + +func (h *handler) handleAPIBenchmarkAutotuneRun() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var body struct { + Profile string `json:"profile"` + BenchmarkKind string `json:"benchmark_kind"` + SizeMB int `json:"size_mb"` + } + if r.Body != nil { + if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + } + profile := strings.TrimSpace(body.Profile) + if profile == "" { + profile = "standard" + } + benchmarkKind := strings.TrimSpace(body.BenchmarkKind) + if benchmarkKind == "" { + benchmarkKind = "power-fit" + } + now := time.Now() + taskName := fmt.Sprintf("NVIDIA Benchmark Autotune · %s · %s", profile, benchmarkKind) + t := &Task{ + ID: newJobID("bee-bench-autotune"), + Name: taskName, + Target: "nvidia-bench-autotune", + Priority: defaultTaskPriority("nvidia-bench-autotune", taskParams{}), + Status: TaskPending, + CreatedAt: now, + params: taskParams{ + BenchmarkProfile: profile, + BenchmarkKind: benchmarkKind, + SizeMB: body.SizeMB, + DisplayName: taskName, + }, + } + globalQueue.enqueue(t) + writeTaskRunResponse(w, []*Task{t}) + } +} + +func (h *handler) handleAPIBenchmarkAutotuneStatus(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + cfg, err := h.opts.App.LoadBenchmarkPowerAutotune() + if err != nil { + if os.IsNotExist(err) { + w.WriteHeader(http.StatusOK) + writeJSON(w, map[string]any{ + "configured": false, + "decision": platform.ResolveSystemPowerDecision(h.opts.ExportDir), + }) + return + } + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + w.WriteHeader(http.StatusOK) + writeJSON(w, map[string]any{ + "configured": true, + "config": cfg, + "decision": platform.ResolveSystemPowerDecision(h.opts.ExportDir), + }) +} + +func (h *handler) handleAPIBenchmarkNvidiaRun(w http.ResponseWriter, r *http.Request) { + h.handleAPIBenchmarkNvidiaRunKind("nvidia-bench-perf").ServeHTTP(w, r) +} + +func (h *handler) handleAPISATStream(w http.ResponseWriter, r *http.Request) { + id := r.URL.Query().Get("job_id") + if id == "" { + id = r.URL.Query().Get("task_id") + } + if j, ok := globalQueue.findJob(id); ok { + streamJob(w, r, j) + return + } + if j, ok := globalJobs.get(id); ok { + streamJob(w, r, j) + return + } + http.Error(w, "job not found", http.StatusNotFound) +} + +func (h *handler) handleAPISATAbort(w http.ResponseWriter, r *http.Request) { + id := r.URL.Query().Get("job_id") + if id == "" { + id = r.URL.Query().Get("task_id") + } + if t, ok := globalQueue.findByID(id); ok { + globalQueue.mu.Lock() + switch t.Status { + case TaskPending: + t.Status = TaskCancelled + now := time.Now() + t.DoneAt = &now + case TaskRunning: + if t.job == nil || !t.job.abort() { + globalQueue.mu.Unlock() + writeJSON(w, map[string]string{"status": "not_running"}) + return + } + globalQueue.mu.Unlock() + writeJSON(w, map[string]string{"status": "aborting"}) + return + } + globalQueue.mu.Unlock() + writeJSON(w, map[string]string{"status": "aborted"}) + return + } + if j, ok := globalJobs.get(id); ok { + if j.abort() { + writeJSON(w, map[string]string{"status": "aborted"}) + } else { + writeJSON(w, map[string]string{"status": "not_running"}) + } + return + } + http.Error(w, "job not found", http.StatusNotFound) +} + +// ── Services ────────────────────────────────────────────────────────────────── diff --git a/audit/internal/webui/api_sat_runall.go b/audit/internal/webui/api_sat_runall.go new file mode 100644 index 0000000..53d4757 --- /dev/null +++ b/audit/internal/webui/api_sat_runall.go @@ -0,0 +1,212 @@ +package webui + +import ( + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net/http" + "sort" + "time" + + "bee/audit/internal/app" + "bee/audit/internal/platform" + "bee/audit/internal/schema" +) + +// gpuReadyWait bounds how long /api/sat/run-all waits for the GPU driver +// stack to come up before it plans the GPU tests. The per-GPU GSP firmware +// boot on a multi-GPU box lags the device nodes by tens of seconds. +// Overridable from tests. +var ( + gpuReadyWait = 75 * time.Second + gpuReadyPollInterval = 3 * time.Second + apiRuntimeHealthNow = func(a *app.App) (schema.RuntimeHealth, error) { + return a.RuntimeHealthNow() + } +) + +type satRunAllRequest struct { + StressMode bool `json:"stress_mode"` + // AMDTargets is the operator's AMD check selection (intent). It is still + // gated on an AMD GPU actually being present. + AMDTargets []string `json:"amd_targets"` + // NvidiaGPUIndices optionally narrows the NVIDIA tests to a subset; empty + // means "every GPU the backend enumerates". + NvidiaGPUIndices []int `json:"nvidia_gpu_indices"` +} + +type satRunAllResponse struct { + TaskIDs []string `json:"task_ids"` + TaskCount int `json:"task_count"` + // Notes carries anything the backend decided to skip or override, so the + // page can show it without reasoning about hardware itself. + Notes []string `json:"notes,omitempty"` +} + +// handleAPISATRunAll plans and enqueues the full validate/check task set +// server-side. Hardware presence and readiness are decided here, never in the +// browser: the page sends only operator intent (stress toggle, AMD checkbox +// selection, an optional GPU subset). +func (h *handler) handleAPISATRunAll(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var req satRunAllRequest + if r.Body != nil { + if err := json.NewDecoder(r.Body).Decode(&req); err != nil && !errors.Is(err, io.EOF) { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + } + specs, notes := h.planSATRunAll(r.Context(), req) + + var ids []string + for _, spec := range specs { + tasks, err := h.enqueueSATTarget(spec.target, spec.params) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + for _, t := range tasks { + if t != nil { + ids = append(ids, t.ID) + } + } + } + slog.Info("sat run-all planned", "tasks", len(ids), "stress", req.StressMode, "notes", len(notes)) + writeJSON(w, satRunAllResponse{TaskIDs: ids, TaskCount: len(ids), Notes: notes}) +} + +type satRunAllSpec struct { + target string + params taskParams +} + +func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]satRunAllSpec, []string) { + var specs []satRunAllSpec + var notes []string + skip := func(msg string) { + notes = append(notes, msg) + slog.Warn("sat run-all: check skipped", "reason", msg) + } + + cpuDur := 60 + if req.StressMode { + cpuDur = 1800 + } + specs = append(specs, + satRunAllSpec{target: "cpu", params: taskParams{Duration: cpuDur, StressMode: req.StressMode}}, + satRunAllSpec{target: "memory", params: taskParams{StressMode: req.StressMode}}, + satRunAllSpec{target: "storage", params: taskParams{StressMode: req.StressMode}}, + satRunAllSpec{target: "pcie-link", params: taskParams{}}, + ) + + if h.opts.App.TPMPresent() { + specs = append(specs, satRunAllSpec{target: "tpm", params: taskParams{}}) + } else { + skip("TPM: no TPM device on this host; check skipped") + } + + gp := h.opts.App.DetectGPUPresence() + + if gp.Nvidia || gp.NvidiaInitializing { + // nvidia-config only collects inventory and NVLink state; safe to run + // even while the compute stack is still coming up. + specs = append(specs, satRunAllSpec{target: "nvidia-config", params: taskParams{}}) + + health, gpus, ready := h.waitForNvidiaReady(ctx) + switch { + case health.NvidiaGSPMode == "gsp-stuck": + skip("NVIDIA: GSP firmware init is stuck; reboot with GSP=off. GPU compute/interconnect/bandwidth tests skipped") + case !ready: + skip("NVIDIA: nvidia-smi did not enumerate a GPU after " + gpuReadyWait.String() + + ". GPU compute/interconnect/bandwidth tests skipped; see the GPU Config check") + default: + indices := make([]int, 0, len(gpus)) + for _, g := range gpus { + indices = append(indices, g.Index) + } + if len(req.NvidiaGPUIndices) > 0 { + indices = intersectSortedInts(indices, req.NvidiaGPUIndices) + } + if len(indices) == 0 { + skip("NVIDIA: driver ready but no GPU to test (enumeration empty, or the requested subset matched nothing); GPU tests skipped") + break + } + if !health.CUDAReady { + notes = append(notes, "NVIDIA: CUDA runtime not confirmed ready; GPU tests queued anyway") + } + gpuTargets := []string{"nvidia", "nvidia-interconnect", "nvidia-bandwidth", "nvidia-pcie-bandwidth"} + if req.StressMode { + // Stress tier adds the targeted dcgmi diag load tests. + gpuTargets = append(gpuTargets, "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse") + } + for _, target := range gpuTargets { + specs = append(specs, satRunAllSpec{ + target: target, + params: taskParams{GPUIndices: append([]int(nil), indices...), StressMode: req.StressMode}, + }) + } + } + } + + if gp.AMD { + for _, target := range req.AMDTargets { + switch target { + case "amd", "amd-mem", "amd-bandwidth": + specs = append(specs, satRunAllSpec{target: target, params: taskParams{StressMode: req.StressMode}}) + } + } + } + + return specs, notes +} + +// waitForNvidiaReady waits until the same nvidia-smi query used by +// ListNvidiaGPUs returns at least one GPU. A loaded kernel module alone is +// not evidence that NVIDIA user-space commands can address a GPU yet. +func (h *handler) waitForNvidiaReady(ctx context.Context) (schema.RuntimeHealth, []platform.NvidiaGPU, bool) { + deadline := time.Now().Add(gpuReadyWait) + var last schema.RuntimeHealth + for { + health, err := apiRuntimeHealthNow(h.opts.App) + if err == nil { + last = health + if health.NvidiaGSPMode == "gsp-stuck" { + return health, nil, false + } + } + gpus, listErr := apiListNvidiaGPUs(h.opts.App) + if listErr == nil && len(gpus) > 0 { + return last, gpus, true + } + if time.Now().After(deadline) || ctx.Err() != nil { + return last, nil, false + } + select { + case <-ctx.Done(): + return last, nil, false + case <-time.After(gpuReadyPollInterval): + } + } +} + +// intersectSortedInts returns the ascending-sorted values present in both a +// and b. +func intersectSortedInts(a, b []int) []int { + set := make(map[int]struct{}, len(b)) + for _, v := range b { + set[v] = struct{}{} + } + out := make([]int, 0, len(a)) + for _, v := range a { + if _, ok := set[v]; ok { + out = append(out, v) + } + } + sort.Ints(out) + return out +} diff --git a/audit/internal/webui/api_sat_runall_test.go b/audit/internal/webui/api_sat_runall_test.go new file mode 100644 index 0000000..39bf16b --- /dev/null +++ b/audit/internal/webui/api_sat_runall_test.go @@ -0,0 +1,90 @@ +package webui + +import ( + "context" + "reflect" + "testing" + "time" + + "bee/audit/internal/app" + "bee/audit/internal/platform" + "bee/audit/internal/schema" +) + +func TestIntersectSortedInts(t *testing.T) { + got := intersectSortedInts([]int{0, 1, 2, 3, 4, 5, 6, 7}, []int{5, 1, 9}) + if want := []int{1, 5}; !reflect.DeepEqual(got, want) { + t.Fatalf("intersectSortedInts=%v want %v", got, want) + } + if got := intersectSortedInts([]int{0, 1}, []int{9}); len(got) != 0 { + t.Fatalf("want empty, got %v", got) + } +} + +func TestWaitForNvidiaReadyDoesNotTreatLoadedDriverAsEnumeration(t *testing.T) { + oldWait, oldPoll := gpuReadyWait, gpuReadyPollInterval + oldHealth, oldList := apiRuntimeHealthNow, apiListNvidiaGPUs + gpuReadyWait, gpuReadyPollInterval = 5*time.Millisecond, time.Millisecond + apiRuntimeHealthNow = func(*app.App) (schema.RuntimeHealth, error) { + return schema.RuntimeHealth{DriverReady: true}, nil + } + apiListNvidiaGPUs = func(*app.App) ([]platform.NvidiaGPU, error) { return nil, nil } + t.Cleanup(func() { + gpuReadyWait, gpuReadyPollInterval = oldWait, oldPoll + apiRuntimeHealthNow, apiListNvidiaGPUs = oldHealth, oldList + }) + + _, gpus, ready := (&handler{opts: HandlerOptions{App: app.New(&platform.System{})}}).waitForNvidiaReady(context.Background()) + if ready || len(gpus) != 0 { + t.Fatalf("ready=%v gpus=%v; loaded module without enumerated GPUs must not be ready", ready, gpus) + } +} + +func TestWaitForNvidiaReadyReturnsFreshEnumeration(t *testing.T) { + oldWait, oldPoll := gpuReadyWait, gpuReadyPollInterval + oldHealth, oldList := apiRuntimeHealthNow, apiListNvidiaGPUs + gpuReadyWait, gpuReadyPollInterval = 20*time.Millisecond, time.Millisecond + apiRuntimeHealthNow = func(*app.App) (schema.RuntimeHealth, error) { + return schema.RuntimeHealth{DriverReady: true, CUDAReady: true}, nil + } + calls := 0 + apiListNvidiaGPUs = func(*app.App) ([]platform.NvidiaGPU, error) { + calls++ + if calls < 2 { + return nil, nil + } + return []platform.NvidiaGPU{{Index: 3}}, nil + } + t.Cleanup(func() { + gpuReadyWait, gpuReadyPollInterval = oldWait, oldPoll + apiRuntimeHealthNow, apiListNvidiaGPUs = oldHealth, oldList + }) + + health, gpus, ready := (&handler{opts: HandlerOptions{App: app.New(&platform.System{})}}).waitForNvidiaReady(context.Background()) + if !ready || !health.CUDAReady || len(gpus) != 1 || gpus[0].Index != 3 { + t.Fatalf("ready=%v health=%+v gpus=%v", ready, health, gpus) + } +} + +// On a host with no GPU and no TPM the plan is the base checks plus a note +// that TPM was skipped, and no GPU tasks are invented. +func TestPlanSATRunAllNoAcceleratorNoTPM(t *testing.T) { + oldWait := gpuReadyWait + gpuReadyWait = 10 * time.Millisecond + t.Cleanup(func() { gpuReadyWait = oldWait }) + + h := &handler{opts: HandlerOptions{App: app.New(&platform.System{})}} + specs, notes := h.planSATRunAll(context.Background(), satRunAllRequest{}) + + var targets []string + for _, s := range specs { + targets = append(targets, s.target) + } + want := []string{"cpu", "memory", "storage", "pcie-link"} + if !reflect.DeepEqual(targets, want) { + t.Fatalf("targets=%v want %v", targets, want) + } + if len(notes) == 0 { + t.Fatalf("expected a note about TPM being skipped") + } +} diff --git a/audit/internal/webui/api_services_system.go b/audit/internal/webui/api_services_system.go new file mode 100644 index 0000000..8344adb --- /dev/null +++ b/audit/internal/webui/api_services_system.go @@ -0,0 +1,260 @@ +package webui + +import ( + "encoding/json" + "errors" + "net/http" + "os" + "path/filepath" + "strings" + + "bee/audit/internal/app" + "bee/audit/internal/platform" +) + +func (h *handler) handleAPIServicesList(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + names, err := h.opts.App.ListBeeServices() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + type serviceInfo struct { + Name string `json:"name"` + State string `json:"state"` + Body string `json:"body"` + } + result := make([]serviceInfo, 0, len(names)) + for _, name := range names { + state := h.opts.App.ServiceState(name) + body, _ := h.opts.App.ServiceStatus(name) + result = append(result, serviceInfo{Name: name, State: state, Body: body}) + } + writeJSON(w, result) +} + +func (h *handler) handleAPIServicesAction(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var req struct { + Name string `json:"name"` + Action string `json:"action"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + var action platform.ServiceAction + switch req.Action { + case "start": + action = platform.ServiceStart + case "stop": + action = platform.ServiceStop + case "restart": + action = platform.ServiceRestart + default: + writeError(w, http.StatusBadRequest, "action must be start|stop|restart") + return + } + result, err := h.opts.App.ServiceActionResult(req.Name, action) + status := "ok" + if err != nil { + status = "error" + } + // Always return 200 with output so the frontend can display the actual + // systemctl error message instead of a generic "exit status 1". + writeJSON(w, map[string]string{"status": status, "output": result.Body}) +} + +// ── Network ─────────────────────────────────────────────────────────────────── + +func (h *handler) handleAPINetworkStatus(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + ifaces, err := h.opts.App.ListInterfaces() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, map[string]any{ + "interfaces": ifaces, + "default_route": h.opts.App.DefaultRoute(), + "pending_change": h.hasPendingNetworkChange(), + "rollback_in": h.pendingNetworkRollbackIn(), + }) +} + +func (h *handler) handleAPINetworkDHCP(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var req struct { + Interface string `json:"interface"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + + result, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) { + if req.Interface == "" || req.Interface == "all" { + return h.opts.App.DHCPAllResult() + } + return h.opts.App.DHCPOneResult(req.Interface) + }) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, map[string]any{ + "status": "ok", + "output": result.Body, + "rollback_in": int(netRollbackTimeout.Seconds()), + }) +} + +func (h *handler) handleAPINetworkStatic(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var req struct { + Interface string `json:"interface"` + Address string `json:"address"` + Prefix string `json:"prefix"` + Gateway string `json:"gateway"` + DNS []string `json:"dns"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + cfg := platform.StaticIPv4Config{ + Interface: req.Interface, + Address: req.Address, + Prefix: req.Prefix, + Gateway: req.Gateway, + DNS: req.DNS, + } + result, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) { + return h.opts.App.SetStaticIPv4Result(cfg) + }) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, map[string]any{ + "status": "ok", + "output": result.Body, + "rollback_in": int(netRollbackTimeout.Seconds()), + }) +} + +// ── Export ──────────────────────────────────────────────────────────────────── + +func (h *handler) handleAPIExportList(w http.ResponseWriter, r *http.Request) { + entries, err := listExportFiles(h.opts.ExportDir) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, entries) +} + +func (h *handler) handleAPIExportUSBTargets(w http.ResponseWriter, _ *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + targets, err := h.opts.App.ListRemovableTargets() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if targets == nil { + targets = []platform.RemovableTarget{} + } + writeJSON(w, targets) +} + +func (h *handler) handleAPIBlackboxStatus(w http.ResponseWriter, _ *http.Request) { + state, err := app.ReadBlackboxState(filepath.Join(h.opts.ExportDir, "blackbox-state.json")) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + writeJSON(w, app.BlackboxState{Status: "disabled", Targets: []app.BlackboxTargetStatus{}}) + return + } + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if state.Targets == nil { + state.Targets = []app.BlackboxTargetStatus{} + } + writeJSON(w, state) +} + +func (h *handler) handleAPIBlackboxEnable(w http.ResponseWriter, r *http.Request) { + if h.opts.App == nil { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + var target platform.RemovableTarget + if err := json.NewDecoder(r.Body).Decode(&target); err != nil || strings.TrimSpace(target.Device) == "" { + writeError(w, http.StatusBadRequest, "device is required") + return + } + targets, err := h.opts.App.ListRemovableTargets() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + allowed := false + for _, candidate := range targets { + if candidate.Device == target.Device { + target = candidate + allowed = true + break + } + } + if !allowed { + writeError(w, http.StatusBadRequest, "device not in removable target list") + return + } + marker, err := app.EnableBlackboxTarget(target) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, map[string]any{ + "status": "ok", + "message": "Black-box marker written.", + "enrollment_id": marker.EnrollmentID, + }) +} + +func (h *handler) handleAPIBlackboxDisable(w http.ResponseWriter, r *http.Request) { + var req struct { + Device string `json:"device"` + EnrollmentID string `json:"enrollment_id"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + if err := app.DisableBlackboxTarget(req.Device, req.EnrollmentID); err != nil { + if errors.Is(err, os.ErrNotExist) { + writeError(w, http.StatusNotFound, "black-box target not found") + return + } + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, map[string]string{"status": "ok", "message": "Black-box marker removed."}) +} + +// ── GPU presence ────────────────────────────────────────────────────────────── diff --git a/audit/internal/webui/huawei_elabel.go b/audit/internal/webui/huawei_elabel.go index a1e1519..0286556 100644 --- a/audit/internal/webui/huawei_elabel.go +++ b/audit/internal/webui/huawei_elabel.go @@ -33,19 +33,19 @@ type huaweiFieldDef struct { } var huaweiElabelDefs = []huaweiFieldDef{ - {"Device Name", "DeviceName", 0x00, 0x06, 0x01, ""}, - {"Device Serial Number", "DeviceSerialNumber", 0x00, 0x06, 0x03, ""}, - {"Product Name", "ProductName", 0x00, 0x03, 0x01, ""}, - {"Product Serial Number", "ProductSerialNumber", 0x00, 0x03, 0x04, ""}, - {"Product Asset Tag", "ProductAssetTag", 0x00, 0x03, 0x05, ""}, - {"Product Manufacturer", "ProductManufacturer", 0x00, 0x03, 0x00, ""}, + {"Device Name", "DeviceName", 0x00, 0x06, 0x01, ""}, + {"Device Serial Number", "DeviceSerialNumber", 0x00, 0x06, 0x03, ""}, + {"Product Name", "ProductName", 0x00, 0x03, 0x01, ""}, + {"Product Serial Number", "ProductSerialNumber", 0x00, 0x03, 0x04, ""}, + {"Product Asset Tag", "ProductAssetTag", 0x00, 0x03, 0x05, ""}, + {"Product Manufacturer", "ProductManufacturer", 0x00, 0x03, 0x00, ""}, {"Mainboard Manufacturer", "MainboardManufacturer", 0x00, 0x02, 0x01, ""}, - {"Board Product Name", "BoardProductName", 0x00, 0x02, 0x02, ""}, - {"Chassis Part Number", "ChassisPartnumber", 0x00, 0x01, 0x01, ""}, - {"Chassis Type", "ChassisType", 0x00, 0x01, 0x00, "chassis-type"}, - {"IO Chassis Serial", "IOChassisSerialNumber", 0x01, 0x03, 0x04, ""}, - {"IO Chassis Asset Tag", "IOChassisAssetTag", 0x01, 0x03, 0x05, ""}, - {"GUID", "GUID", 0x00, 0x00, 0x00, "guid"}, + {"Board Product Name", "BoardProductName", 0x00, 0x02, 0x02, ""}, + {"Chassis Part Number", "ChassisPartnumber", 0x00, 0x01, 0x01, ""}, + {"Chassis Type", "ChassisType", 0x00, 0x01, 0x00, "chassis-type"}, + {"IO Chassis Serial", "IOChassisSerialNumber", 0x01, 0x03, 0x04, ""}, + {"IO Chassis Asset Tag", "IOChassisAssetTag", 0x01, 0x03, 0x05, ""}, + {"GUID", "GUID", 0x00, 0x00, 0x00, "guid"}, } // huaweiGetRaw reads a string elabel field via OEM IPMI raw command. diff --git a/audit/internal/webui/layout.go b/audit/internal/webui/layout.go index 04eb932..a7ec1ce 100644 --- a/audit/internal/webui/layout.go +++ b/audit/internal/webui/layout.go @@ -100,7 +100,7 @@ tbody tr:hover td{background:rgba(0,0,0,.03)} func layoutNav(active string, buildLabel string) string { type navItem struct { id, label, href string - sep bool + sep bool } items := []navItem{ {id: "dashboard", label: "Dashboard", href: "/"}, diff --git a/audit/internal/webui/metrics_charts_http.go b/audit/internal/webui/metrics_charts_http.go new file mode 100644 index 0000000..7ffdd5d --- /dev/null +++ b/audit/internal/webui/metrics_charts_http.go @@ -0,0 +1,665 @@ +package webui + +import ( + "fmt" + "html" + "net/http" + "sort" + "strings" + "time" + + "bee/audit/internal/platform" +) + +func (h *handler) handleMetricsChartSVG(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/api/metrics/chart/") + path = strings.TrimSuffix(path, ".svg") + + if h.metricsDB == nil { + http.Error(w, "metrics database not available", http.StatusServiceUnavailable) + return + } + samples, err := h.metricsDB.LoadAll() + if err != nil || len(samples) == 0 { + http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable) + return + } + timeline := metricsTimelineSegments(samples, time.Now()) + if idx, sub, ok := parseGPUChartPath(path); ok && sub == "overview" { + var overviewOk bool + var buf []byte + buf, overviewOk, err = renderGPUOverviewChartSVG(idx, samples, timeline) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if !overviewOk { + http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", "image/svg+xml") + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write(buf) + return + } + datasets, names, labels, title, yMin, yMax, stacked, ok := chartDataFromSamples(path, samples) + if !ok { + http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable) + return + } + + var buf []byte + if stacked { + buf, err = renderStackedMetricChartSVG( + title, + labels, + sampleTimes(samples), + datasets, + names, + yMax, + chartCanvasHeightForPath(path, len(names)), + timeline, + ) + } else { + buf, err = renderMetricChartSVG( + title, + labels, + sampleTimes(samples), + datasets, + names, + yMin, + yMax, + chartCanvasHeightForPath(path, len(names)), + timeline, + ) + } + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "image/svg+xml") + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write(buf) +} + +func chartDataFromSamples(path string, samples []platform.LiveMetricSample) (datasets [][]float64, names []string, labels []string, title string, yMin, yMax *float64, stacked bool, ok bool) { + labels = sampleTimeLabels(samples) + + switch { + case path == "server-load": + title = "CPU / Memory Load" + cpu := make([]float64, len(samples)) + mem := make([]float64, len(samples)) + for i, s := range samples { + cpu[i] = s.CPULoadPct + mem[i] = s.MemLoadPct + } + datasets = [][]float64{cpu, mem} + names = []string{"CPU Load %", "Mem Load %"} + yMin = floatPtr(0) + yMax = floatPtr(100) + + case path == "server-temp", path == "server-temp-cpu": + title = "CPU Temperature" + datasets, names = namedTempDatasets(samples, "cpu") + yMin = floatPtr(0) + yMax = autoMax120(datasets...) + + case path == "server-temp-gpu": + title = "GPU Temperature" + datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.TempC }) + yMin = floatPtr(0) + yMax = autoMax120(datasets...) + + case path == "server-temp-ambient": + title = "Ambient / Other Sensors" + datasets, names = namedTempDatasets(samples, "ambient") + yMin = floatPtr(0) + yMax = autoMax120(datasets...) + + case path == "server-power": + title = "System Power" + power := make([]float64, len(samples)) + label := "Power W" + for i, s := range samples { + power[i] = s.PowerW + if strings.TrimSpace(s.PowerSource) != "" { + label = fmt.Sprintf("Power W · %s", s.PowerSource) + if strings.TrimSpace(s.PowerMode) != "" { + label += fmt.Sprintf(" (%s)", s.PowerMode) + } + } + } + power = normalizePowerSeries(power) + datasets = [][]float64{power} + names = []string{label} + yMin = floatPtr(0) + yMax = autoMax120(power) + + case path == "server-fans": + title = "Fan RPM" + datasets, names = namedFanDatasets(samples) + yMin, yMax = autoBounds120(datasets...) + + case path == "gpu-all-load": + title = "GPU Compute Load" + datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.UsagePct }) + yMin = floatPtr(0) + yMax = floatPtr(100) + + case path == "gpu-all-memload": + title = "GPU Memory Load" + datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.MemUsagePct }) + yMin = floatPtr(0) + yMax = floatPtr(100) + + case path == "gpu-all-power": + title = "GPU Power" + datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.PowerW }) + yMin, yMax = autoBounds120(datasets...) + + case path == "gpu-all-temp": + title = "GPU Temperature" + datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.TempC }) + yMin = floatPtr(0) + yMax = autoMax120(datasets...) + + case path == "gpu-all-clock": + title = "GPU Core Clock" + datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.ClockMHz }) + yMin, yMax = autoBounds120(datasets...) + + case path == "gpu-all-memclock": + title = "GPU Memory Clock" + datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.MemClockMHz }) + yMin, yMax = autoBounds120(datasets...) + + case strings.HasPrefix(path, "gpu/"): + idx, sub, ok := parseGPUChartPath(path) + if !ok { + return nil, nil, nil, "", nil, nil, false, false + } + switch sub { + case "load": + title = gpuDisplayLabel(idx) + " Load" + util := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.UsagePct }) + mem := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.MemUsagePct }) + if util == nil && mem == nil { + return nil, nil, nil, "", nil, nil, false, false + } + datasets = [][]float64{coalesceDataset(util, len(samples)), coalesceDataset(mem, len(samples))} + names = []string{"Load %", "Mem %"} + yMin = floatPtr(0) + yMax = floatPtr(100) + case "temp": + title = gpuDisplayLabel(idx) + " Temperature" + temp := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.TempC }) + if temp == nil { + return nil, nil, nil, "", nil, nil, false, false + } + datasets = [][]float64{temp} + names = []string{"Temp °C"} + yMin = floatPtr(0) + yMax = autoMax120(temp) + case "clock": + title = gpuDisplayLabel(idx) + " Core Clock" + clock := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.ClockMHz }) + if clock == nil { + return nil, nil, nil, "", nil, nil, false, false + } + datasets = [][]float64{clock} + names = []string{"Core Clock MHz"} + yMin, yMax = autoBounds120(clock) + case "memclock": + title = gpuDisplayLabel(idx) + " Memory Clock" + clock := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.MemClockMHz }) + if clock == nil { + return nil, nil, nil, "", nil, nil, false, false + } + datasets = [][]float64{clock} + names = []string{"Memory Clock MHz"} + yMin, yMax = autoBounds120(clock) + default: + title = gpuDisplayLabel(idx) + " Power" + power := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.PowerW }) + if power == nil { + return nil, nil, nil, "", nil, nil, false, false + } + datasets = [][]float64{power} + names = []string{"Power W"} + yMin, yMax = autoBounds120(power) + } + + default: + return nil, nil, nil, "", nil, nil, false, false + } + + return datasets, names, labels, title, yMin, yMax, stacked, len(datasets) > 0 +} + +func parseGPUChartPath(path string) (idx int, sub string, ok bool) { + if !strings.HasPrefix(path, "gpu/") { + return 0, "", false + } + rest := strings.TrimPrefix(path, "gpu/") + if rest == "" { + return 0, "", false + } + sub = "" + if i := strings.LastIndex(rest, "-"); i > 0 { + sub = rest[i+1:] + rest = rest[:i] + } + n, err := fmt.Sscanf(rest, "%d", &idx) + if err != nil || n != 1 { + return 0, "", false + } + return idx, sub, true +} + +func sampleTimeLabels(samples []platform.LiveMetricSample) []string { + labels := make([]string, len(samples)) + if len(samples) == 0 { + return labels + } + times := make([]time.Time, len(samples)) + for i, s := range samples { + times[i] = s.Timestamp + } + sameDay := timestampsSameLocalDay(times) + for i, s := range samples { + labels[i] = formatTimelineLabel(s.Timestamp.Local(), sameDay) + } + return labels +} + +func namedTempDatasets(samples []platform.LiveMetricSample, group string) ([][]float64, []string) { + seen := map[string]bool{} + var names []string + for _, s := range samples { + for _, t := range s.Temps { + if t.Group == group && !seen[t.Name] { + seen[t.Name] = true + names = append(names, t.Name) + } + } + } + sort.Strings(names) + datasets := make([][]float64, 0, len(names)) + for _, name := range names { + ds := make([]float64, len(samples)) + for i, s := range samples { + for _, t := range s.Temps { + if t.Group == group && t.Name == name { + ds[i] = t.Celsius + break + } + } + } + datasets = append(datasets, ds) + } + return datasets, names +} + +func namedFanDatasets(samples []platform.LiveMetricSample) ([][]float64, []string) { + seen := map[string]bool{} + var names []string + for _, s := range samples { + for _, f := range s.Fans { + if !seen[f.Name] { + seen[f.Name] = true + names = append(names, f.Name) + } + } + } + sort.Strings(names) + datasets := make([][]float64, 0, len(names)) + for _, name := range names { + ds := make([]float64, len(samples)) + for i, s := range samples { + for _, f := range s.Fans { + if f.Name == name { + ds[i] = f.RPM + break + } + } + } + datasets = append(datasets, normalizeFanSeries(ds)) + } + return datasets, names +} + +func gpuDatasets(samples []platform.LiveMetricSample, pick func(platform.GPUMetricRow) float64) ([][]float64, []string) { + seen := map[int]bool{} + var indices []int + for _, s := range samples { + for _, g := range s.GPUs { + if !seen[g.GPUIndex] { + seen[g.GPUIndex] = true + indices = append(indices, g.GPUIndex) + } + } + } + sort.Ints(indices) + datasets := make([][]float64, 0, len(indices)) + names := make([]string, 0, len(indices)) + for _, idx := range indices { + ds := gpuDatasetByIndex(samples, idx, pick) + if ds == nil { + continue + } + datasets = append(datasets, ds) + names = append(names, gpuDisplayLabel(idx)) + } + return datasets, names +} + +func gpuDatasetByIndex(samples []platform.LiveMetricSample, idx int, pick func(platform.GPUMetricRow) float64) []float64 { + found := false + ds := make([]float64, len(samples)) + for i, s := range samples { + for _, g := range s.GPUs { + if g.GPUIndex == idx { + ds[i] = pick(g) + found = true + break + } + } + } + if !found { + return nil + } + return ds +} + +func coalesceDataset(ds []float64, n int) []float64 { + if ds != nil { + return ds + } + return make([]float64, n) +} + +func normalizePowerSeries(ds []float64) []float64 { + if len(ds) == 0 { + return nil + } + out := make([]float64, len(ds)) + copy(out, ds) + last := 0.0 + haveLast := false + for i, v := range out { + if v > 0 { + last = v + haveLast = true + continue + } + if haveLast { + out[i] = last + } + } + return out +} + +// psuSlotsFromSamples returns the sorted list of PSU slot numbers seen across samples. +func psuSlotsFromSamples(samples []platform.LiveMetricSample) []int { + seen := map[int]struct{}{} + for _, s := range samples { + for _, p := range s.PSUs { + seen[p.Slot] = struct{}{} + } + } + slots := make([]int, 0, len(seen)) + for s := range seen { + slots = append(slots, s) + } + sort.Ints(slots) + return slots +} + +// psuStackedTotal returns the point-by-point sum of all PSU datasets (for scale calculation). +func psuStackedTotal(datasets [][]float64) []float64 { + if len(datasets) == 0 { + return nil + } + n := len(datasets[0]) + total := make([]float64, n) + for _, ds := range datasets { + for i, v := range ds { + total[i] += v + } + } + return total +} + +func normalizeFanSeries(ds []float64) []float64 { + if len(ds) == 0 { + return nil + } + out := make([]float64, len(ds)) + var lastPositive float64 + for i, v := range ds { + if v > 0 { + lastPositive = v + out[i] = v + continue + } + if lastPositive > 0 { + out[i] = lastPositive + continue + } + out[i] = 0 + } + return out +} + +// floatPtr returns a pointer to a float64 value. +func floatPtr(v float64) *float64 { return &v } + +// autoMax120 returns 0→max+20% Y-axis max across all datasets. +func autoMax120(datasets ...[]float64) *float64 { + max := 0.0 + for _, ds := range datasets { + for _, v := range ds { + if v > max { + max = v + } + } + } + if max == 0 { + return nil // let library auto-scale + } + v := max * 1.2 + return &v +} + +func autoBounds120(datasets ...[]float64) (*float64, *float64) { + min := 0.0 + max := 0.0 + first := true + for _, ds := range datasets { + for _, v := range ds { + if first { + min, max = v, v + first = false + continue + } + if v < min { + min = v + } + if v > max { + max = v + } + } + } + if first { + return nil, nil + } + if max <= 0 { + return floatPtr(0), nil + } + span := max - min + if span <= 0 { + span = max * 0.1 + if span <= 0 { + span = 1 + } + } + pad := span * 0.2 + low := min - pad + if low < 0 { + low = 0 + } + high := max + pad + return floatPtr(low), floatPtr(high) +} + +func gpuChartLabelIndices(total, target int) []int { + if total <= 0 { + return nil + } + if total == 1 { + return []int{0} + } + step := total / target + if step < 1 { + step = 1 + } + var indices []int + for i := 0; i < total; i += step { + indices = append(indices, i) + } + if indices[len(indices)-1] != total-1 { + indices = append(indices, total-1) + } + return indices +} + +func chartCanvasHeightForPath(path string, seriesCount int) int { + height := chartCanvasHeight(seriesCount) + if isGPUChartPath(path) { + return height * 2 + } + return height +} + +func isGPUChartPath(path string) bool { + return strings.HasPrefix(path, "gpu-all-") || strings.HasPrefix(path, "gpu/") +} + +func chartLegendVisible(seriesCount int) bool { + return seriesCount <= 8 +} + +func chartCanvasHeight(seriesCount int) int { + if chartLegendVisible(seriesCount) { + return 360 + } + return 288 +} + +// globalStats returns min, average, and max across all values in all datasets. +func globalStats(datasets [][]float64) (mn, avg, mx float64) { + var sum float64 + var count int + first := true + for _, ds := range datasets { + for _, v := range ds { + if first { + mn, mx = v, v + first = false + } + if v < mn { + mn = v + } + if v > mx { + mx = v + } + sum += v + count++ + } + } + if count > 0 { + avg = sum / float64(count) + } + return mn, avg, mx +} + +func sanitizeChartText(s string) string { + if s == "" { + return "" + } + return html.EscapeString(strings.Map(func(r rune) rune { + if r < 0x20 && r != '\t' && r != '\n' && r != '\r' { + return -1 + } + return r + }, s)) +} + +func snapshotFanRings(rings []*metricsRing, fanNames []string) ([][]float64, []string, []string) { + var datasets [][]float64 + var names []string + var labels []string + for i, ring := range rings { + if ring == nil { + continue + } + vals, l := ring.snapshot() + datasets = append(datasets, normalizeFanSeries(vals)) + name := "Fan" + if i < len(fanNames) { + name = fanNames[i] + } + names = append(names, name+" RPM") + if len(labels) == 0 { + labels = l + } + } + return datasets, names, labels +} + +func chartLegendNumber(v float64) string { + neg := v < 0 + if v < 0 { + v = -v + } + var out string + switch { + case v >= 10000: + out = fmt.Sprintf("%dk", int((v+500)/1000)) + case v >= 1000: + s := fmt.Sprintf("%.2f", v/1000) + s = strings.TrimRight(strings.TrimRight(s, "0"), ".") + out = strings.ReplaceAll(s, ".", ",") + "k" + default: + out = fmt.Sprintf("%.0f", v) + } + if neg { + return "-" + out + } + return out +} + +func chartYAxisNumber(v float64) string { + neg := v < 0 + if neg { + v = -v + } + var out string + switch { + case v >= 10000: + out = fmt.Sprintf("%dк", int((v+500)/1000)) + case v >= 1000: + // Use one decimal place so ticks like 1400, 1600, 1800 read as + // "1,4к", "1,6к", "1,8к" instead of the ambiguous "1к"/"2к". + s := fmt.Sprintf("%.1f", v/1000) + s = strings.TrimRight(strings.TrimRight(s, "0"), ".") + out = strings.ReplaceAll(s, ".", ",") + "к" + default: + out = fmt.Sprintf("%.0f", v) + } + if neg { + return "-" + out + } + return out +} diff --git a/audit/internal/webui/metricsdb.go b/audit/internal/webui/metricsdb.go index 1919c8b..ea9ac22 100644 --- a/audit/internal/webui/metricsdb.go +++ b/audit/internal/webui/metricsdb.go @@ -455,7 +455,3 @@ func (m *MetricsDB) ExportCSV(w io.Writer) error { cw.Flush() return cw.Error() } - -func nullFloat(v float64) sql.NullFloat64 { - return sql.NullFloat64{Float64: v, Valid: true} -} diff --git a/audit/internal/webui/page_benchmark.go b/audit/internal/webui/page_benchmark.go index aac0457..676dc1a 100644 --- a/audit/internal/webui/page_benchmark.go +++ b/audit/internal/webui/page_benchmark.go @@ -613,5 +613,5 @@ func renderPowerBenchmarkResultsCard(exportDir string) string { } // renderSpeed and renderEndurance are legacy wrappers; canonical page is 5. Benchmark at /benchmark. -func renderSpeed(opts HandlerOptions) string { return renderBenchmark(opts) } +func renderSpeed(opts HandlerOptions) string { return renderBenchmark(opts) } func renderEndurance(opts HandlerOptions) string { return renderBenchmark(opts) } diff --git a/audit/internal/webui/page_topo.go b/audit/internal/webui/page_topo.go index 333b4be..19f27e3 100644 --- a/audit/internal/webui/page_topo.go +++ b/audit/internal/webui/page_topo.go @@ -11,7 +11,6 @@ import ( "strconv" "strings" - "bee/audit/internal/app" "bee/audit/internal/schema" ) @@ -201,6 +200,7 @@ func parseDIMMBankLocatorNodes(raw string) map[string]int { // 2. A node number from the Bank Locator via parseDIMMBankLocatorNodes, // e.g. Locator "DIMM000(A)" whose Bank Locator is // "_Node1_Channel0_Dimm0" -> 1. +// // ok=false means neither pattern matched, so this DIMM can't be confidently // attached to a CPU column and falls back to the unattached Memory row. func dimmRawNode(mem schema.HardwareMemory, bankNodeByLocator map[string]int) (int, bool) { @@ -582,772 +582,3 @@ type topoEdge struct { x1, y1, x2, y2 int color string } - -func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string { - // A GPU hardware fault that needs a physical reboot (Xid 79 "fallen off - // the bus", Xid 154 "Node Reboot Required" — see gpuNeedsPhysicalReboot) - // isn't visible in dev.Status: the collector only sets that from PCIe - // link-speed checks, not from SAT/kmsg results. Without this, a GPU that - // dropped off the bus mid-test still renders green here even though the - // Hardware Summary card is showing a critical banner for it. - gpuHardwareFault := false - if db, err := app.OpenComponentStatusDB(filepath.Join(exportDir, "component-status.json")); err == nil { - if _, needsReboot := gpuNeedsPhysicalReboot(matchedRecords(db.All(), nil, []string{"pcie:gpu:"})); needsReboot { - gpuHardwareFault = true - } - } - - socketIdx := buildSocketIndex(hw.CPUs) - numCols := len(hw.CPUs) - if numCols == 0 { - numCols = 1 - } - unknownCol := numCols // extra trailing column for unmatched devices - - // Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node - // types) into columns by NUMA node, falling back to the "unknown" bucket. - type placedDevice struct { - dev schema.HardwarePCIeDevice - kind string // "gpu", "nic", "raid" - col int - bdf string - } - var placed []placedDevice - for _, dev := range hw.PCIeDevices { - kind := pcieDeviceKind(dev) - if kind == "" { - continue - } - col := unknownCol - if dev.NUMANode != nil { - if ci, ok := socketIdx[*dev.NUMANode]; ok { - col = ci - } - } - bdf := "" - if dev.Slot != nil { - bdf = normalizeTopoBDF(*dev.Slot) - } else if dev.BDF != nil { - bdf = normalizeTopoBDF(*dev.BDF) - } - placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf}) - } - hasUnknownCol := false - for _, p := range placed { - if p.col == unknownCol { - hasUnknownCol = true - break - } - } - totalCols := numCols - if hasUnknownCol { - totalCols++ - } - - // GPU index<->BDF map + pairwise NVLink adjacency, read from the - // persisted techdump captured during the last audit cycle, best-effort: - // if the dump is missing (older audit, no NVIDIA GPUs), this is simply - // skipped. Used only to detect the cross-NUMA-bonded-pair anomaly below; - // the pairwise links themselves are drawn in the separate NVLink - // Topology card, since grouping same-kind/same-column devices into one - // stacked card here leaves no single per-GPU anchor point to draw a - // pairwise connector to or from. - bdfToIndex, _ := readNVIDIAIndexByBDF(exportDir) - var pairs []gpuPairLink - if topoMatrix, err := readGPUTopologyMatrix(exportDir); err == nil { - pairs = parseGPUPairAdjacency(topoMatrix) - } - gpuNUMAByIndex := map[int]*int{} - gpuBDFByIndex := map[int]string{} - for _, p := range placed { - if p.kind != "gpu" || p.bdf == "" { - continue - } - if idx, ok := bdfToIndex[p.bdf]; ok { - gpuNUMAByIndex[idx] = p.dev.NUMANode - gpuBDFByIndex[idx] = p.bdf - } - } - // A bonded pair spanning two different NUMA nodes is treated as an - // anomaly (not a neutral fact) per project decision: a bonded pair is - // expected to sit on one NUMA node, so a cross-NUMA bond escalates both - // GPUs' effective severity to at least Warning, regardless of their own - // reported SAT status. - crossNUMAWarnBDF := map[string]bool{} - for _, pair := range pairs { - numaA, okA := gpuNUMAByIndex[pair.GPUA] - numaB, okB := gpuNUMAByIndex[pair.GPUB] - if !okA || !okB || numaA == nil || numaB == nil || *numaA == *numaB { - continue - } - crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUA]] = true - crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUB]] = true - } - - kindOrder := []string{"gpu", "nic", "raid"} - kindLabel := map[string]string{"gpu": "GPU", "nic": "NIC", "raid": "RAID"} - - // Attach memory DIMMs to their CPU column too, the same way GPU/NIC/RAID - // PCIe devices are attached via NUMANode — memory has no NUMANode field - // in the schema, so this reads the DIMM's own Locator/Bank Locator - // strings instead (see dimmRawNode). DIMMs that can't be confidently - // attached fall back to the unattached "Memory" row below the diagram, - // same as before this existed. - memBankNodes := map[string]int{} - if raw, err := readTopoTechDump(exportDir, "dmidecode-type17.txt"); err == nil { - memBankNodes = parseDIMMBankLocatorNodes(raw) - } - memCol := make([]int, len(hw.Memory)) - memMatched := make([]bool, len(hw.Memory)) - var memRawNodes []int - for i, m := range hw.Memory { - if node, ok := dimmRawNode(m, memBankNodes); ok { - memCol[i] = node - memMatched[i] = true - memRawNodes = append(memRawNodes, node) - } - } - memColIdx := buildMemoryColumnIndex(memRawNodes) - for i := range hw.Memory { - if !memMatched[i] { - continue - } - col := memColIdx[memCol[i]] - if col >= numCols { - memMatched[i] = false - continue - } - memCol[i] = col - } - - var boxes []topoBox - var pcieEdges []topoEdge - - for col := 0; col < totalCols; col++ { - colX := (col+1)*24 + col*topoColWidth - if col < len(hw.CPUs) { - cpu := hw.CPUs[col] - model := "" - if cpu.Model != nil { - model = *cpu.Model - } - socket := col - if cpu.Socket != nil { - socket = *cpu.Socket - } - var tally topoStatusTally - tally.add(classifyTopoSeverity(cpu.Status)) - fill, stroke, text := topoSeverityColors(tally.worst()) - boxes = append(boxes, topoBox{ - x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight, - topoCardInfo: topoCardInfo{ - label: fmt.Sprintf("CPU %d", socket), sublabel: model, count: 1, - statusLine: tally.line(), - fillVar: fill, strokeVar: stroke, textVar: text, - detailType: "cpu", - }, - }) - } - - y := topoTopMargin + topoBoxHeight + topoDeviceGap*2 - - // Memory goes first in the chain, directly under the CPU box: DIMMs - // are wired straight to the socket's memory controller, not reached - // over PCIe like the GPU/NIC/RAID chain below it. - var memGroup []schema.HardwareMemory - for i, m := range hw.Memory { - if memMatched[i] && memCol[i] == col { - memGroup = append(memGroup, m) - } - } - if len(memGroup) > 0 { - var tally topoStatusTally - sizeGB := 0 - for _, m := range memGroup { - tally.add(classifyTopoSeverity(m.Status)) - if m.SizeMB != nil { - sizeGB += *m.SizeMB / 1024 - } - } - fill, stroke, text := topoSeverityColors(tally.worst()) - sublabel := "" - if sizeGB > 0 { - sublabel = fmt.Sprintf("%d GB total", sizeGB) - } - stackLayers := topoStackLayers(len(memGroup)) - boxes = append(boxes, topoBox{ - x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight, - topoCardInfo: topoCardInfo{ - label: "Memory", sublabel: sublabel, count: len(memGroup), - statusLine: tally.line(), - fillVar: fill, strokeVar: stroke, textVar: text, - detailType: "memory", - }, - }) - if col < len(hw.CPUs) { - pcieEdges = append(pcieEdges, topoEdge{ - x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight, - x2: colX + topoBoxWidth/2, y2: y, - color: "var(--ok-fg)", - }) - } - y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep - } - - for _, kind := range kindOrder { - var group []placedDevice - for _, p := range placed { - if p.col == col && p.kind == kind { - group = append(group, p) - } - } - if len(group) == 0 { - continue - } - - var tally topoStatusTally - model := "" - edgeColor := "var(--ok-fg)" - for i, p := range group { - sev := classifyTopoSeverity(p.dev.Status) - if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 { - sev = 2 - } - if kind == "gpu" && gpuHardwareFault && sev < 3 { - sev = 3 - } - tally.add(sev) - if i == 0 && p.dev.Model != nil { - model = *p.dev.Model - } - if topoEdgeColorVar(p.dev) == "var(--warn-fg)" { - edgeColor = "var(--warn-fg)" - } - } - fill, stroke, text := topoSeverityColors(tally.worst()) - stackLayers := topoStackLayers(len(group)) - boxes = append(boxes, topoBox{ - x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight, - topoCardInfo: topoCardInfo{ - label: kindLabel[kind], sublabel: model, count: len(group), - statusLine: tally.line(), - fillVar: fill, strokeVar: stroke, textVar: text, - detailType: kind, - }, - }) - - if col < len(hw.CPUs) { - pcieEdges = append(pcieEdges, topoEdge{ - x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight, - x2: colX + topoBoxWidth/2, y2: y, - color: edgeColor, - }) - } - - y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep - } - } - - maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2 - for _, box := range boxes { - bottom := box.y + box.h + topoStackLayers(box.count)*topoStackStep - if bottom > maxDeviceY { - maxDeviceY = bottom - } - } - - svgHeight := maxDeviceY + 24 - svgWidth := totalCols*topoColWidth + 48 - - var b strings.Builder - // Wrapped in its own horizontally-scrolling container (matching the - // overflow-x:auto convention used for wide tables elsewhere in webui) - // rather than max-width:100% — squashing a node/edge diagram to fit a - // narrow viewport makes labels and badges illegible, whereas scrolling - // keeps the diagram readable at its natural size on any screen width. - b.WriteString(`
`) - fmt.Fprintf(&b, ``+"\n", svgWidth, svgHeight, svgWidth, svgHeight) - for _, e := range pcieEdges { - fmt.Fprintf(&b, ``+"\n", e.x1, e.y1, e.x2, e.y2, e.color) - } - for _, box := range boxes { - writeTopoBoxSVG(&b, box) - } - b.WriteString(`
`) - - // Firmware (BMC/BIOS/...) and PSUs have no PCIe/CPU affinity to anchor - // them to a column, and there can be an arbitrary number of any of them - // — so unlike the diagram above, they're plain flex-wrap HTML below the - // SVG rather than absolutely-positioned SVG boxes. A fixed-size SVG - // canvas has no way to wrap overflow onto a new row, which is exactly - // what caused these to pile up and overlap once a board had more - // PSUs/firmware records than fit in one fixed-width row. - // - // Memory DIMMs that were matched to a CPU column above already got a - // box in the SVG diagram; only DIMMs that couldn't be attached to a - // column (see memMatched above) fall back to this row. - var unmatchedMem []schema.HardwareMemory - for i, m := range hw.Memory { - if !memMatched[i] { - unmatchedMem = append(unmatchedMem, m) - } - } - if len(unmatchedMem) > 0 { - var tally topoStatusTally - for _, m := range unmatchedMem { - tally.add(classifyTopoSeverity(m.Status)) - } - fill, stroke, text := topoSeverityColors(tally.worst()) - sizeGB := 0 - for _, m := range unmatchedMem { - if m.SizeMB != nil { - sizeGB += *m.SizeMB / 1024 - } - } - sublabel := "" - if sizeGB > 0 { - sublabel = fmt.Sprintf("%d GB total", sizeGB) - } - b.WriteString(renderTopoFlexRow("Memory", []topoCardInfo{{ - label: "Memory", sublabel: sublabel, count: len(unmatchedMem), - statusLine: tally.line(), - fillVar: fill, strokeVar: stroke, textVar: text, - detailType: "memory", - }})) - } - - var firmwareItems []topoCardInfo - for _, rec := range hw.Firmware { - // Firmware records carry no per-item status in the schema (they are - // identity, not health, facts), so each stays a neutral, uncolored - // card rather than forcing a fake "Unknown" status line. - fillVar, strokeVar, textVar := topoSeverityColors(0) - firmwareItems = append(firmwareItems, topoCardInfo{ - label: rec.DeviceName, sublabel: "fw " + rec.Version, count: 1, - fillVar: fillVar, strokeVar: strokeVar, textVar: textVar, - }) - } - b.WriteString(renderTopoFlexRow("Firmware", firmwareItems)) - - if len(hw.PowerSupplies) > 0 { - var tally topoStatusTally - watt := 0 - for _, psu := range hw.PowerSupplies { - tally.add(classifyTopoSeverity(psu.Status)) - if psu.WattageW != nil { - watt = *psu.WattageW - } - } - fill, stroke, text := topoSeverityColors(tally.worst()) - sublabel := "" - if watt > 0 { - sublabel = fmt.Sprintf("%dW each", watt) - } - b.WriteString(renderTopoFlexRow("Power Supplies", []topoCardInfo{{ - label: "Power Supplies", sublabel: sublabel, count: len(hw.PowerSupplies), - statusLine: tally.line(), - fillVar: fill, strokeVar: stroke, textVar: text, - detailType: "psu", - }})) - } - - return topoCard("Topology", b.String()) -} - -// renderTopoFlexRow renders a labeled, wrapping row of component cards. -// Returns "" if items is empty (e.g. no PSU data in this audit). -func renderTopoFlexRow(title string, items []topoCardInfo) string { - if len(items) == 0 { - return "" - } - var b strings.Builder - fmt.Fprintf(&b, `
%s
`, - html.EscapeString(title)) - b.WriteString(`
`) - for _, item := range items { - onclick := "" - cursor := "default" - if item.detailType != "" { - onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, item.detailType) - cursor = "pointer" - } - stackLayers := topoStackLayers(item.count) - // Extra right/bottom padding on the wrapper reserves room for the - // backing layers of the stack effect so they aren't clipped by the - // flex container. - fmt.Fprintf(&b, `
`, - stackLayers*topoStackStep, stackLayers*topoStackStep) - for i := stackLayers; i >= 1; i-- { - off := i * topoStackStep - fmt.Fprintf(&b, `
`, - off, off, item.fillVar, item.strokeVar) - } - fmt.Fprintf(&b, ``, - onclick, cursor, item.fillVar, item.strokeVar, item.textVar) - label := item.label - if item.count > 1 { - label = fmt.Sprintf("%s ×%d", item.label, item.count) - } - fmt.Fprintf(&b, `
%s
`, html.EscapeString(label)) - if item.sublabel != "" { - fmt.Fprintf(&b, `
%s
`, html.EscapeString(item.sublabel)) - } - if item.statusLine != "" { - fmt.Fprintf(&b, `
%s
`, html.EscapeString(item.statusLine)) - } - b.WriteString(`
`) - } - b.WriteString(``) - return b.String() -} - -func writeTopoBoxSVG(b *strings.Builder, box topoBox) { - onclick := "" - cursor := "default" - if box.detailType != "" { - onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType) - cursor = "pointer" - } - fmt.Fprintf(b, ``, onclick, cursor) - - // Stack-of-cards effect: faint offset rects behind the front card when - // this box represents more than one physical component (e.g. 4 GPUs in - // one NUMA column), so a group reads as "a deck of N" rather than a - // single item. Peeks toward the bottom-right, into space already - // reserved between this box and the next one in the column. - for i := topoStackLayers(box.count); i >= 1; i-- { - off := i * topoStackStep - fmt.Fprintf(b, ``+"\n", - box.x+off, box.y+off, box.w, box.h, box.fillVar, box.strokeVar) - } - - fmt.Fprintf(b, ``+"\n", - box.x, box.y, box.w, box.h, box.fillVar, box.strokeVar) - - label := box.label - if box.count > 1 { - label = fmt.Sprintf("%s ×%d", box.label, box.count) - } - fmt.Fprintf(b, `%s`+"\n", - box.x+10, box.y+20, box.textVar, html.EscapeString(label)) - if box.sublabel != "" { - fmt.Fprintf(b, `%s`+"\n", - box.x+10, box.y+36, box.textVar, html.EscapeString(truncateTopoLabel(box.sublabel, 26))) - } - if box.statusLine != "" { - fmt.Fprintf(b, `%s`+"\n", - box.x+10, box.y+box.h-10, box.textVar, html.EscapeString(box.statusLine)) - } - b.WriteString(`` + "\n") -} - -func truncateTopoLabel(s string, max int) string { - if len(s) <= max { - return s - } - if max <= 1 { - return s[:max] - } - return s[:max-1] + "…" -} - -// --------------------------------------------------------------------------- -// Separate NVLink topology card (read from techdump, not written to any -// ingest contract) -// --------------------------------------------------------------------------- - -type topoNVLinkPort struct { - Index int - Active bool - SpeedGBs *float64 - ReplayErrors int64 - RecoveryErrors int64 - CRCErrors int64 -} - -var ( - topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`) - topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`) - topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*`) - topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`) -) - -func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) { - raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-status.txt") - if err != nil { - return nil, err - } - return parseTopoNVLinkStatus(raw), nil -} - -func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort { - result := map[int][]topoNVLinkPort{} - currentGPU := -1 - for _, line := range strings.Split(raw, "\n") { - trimmed := strings.TrimSpace(line) - if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { - currentGPU, _ = strconv.Atoi(m[1]) - continue - } - if currentGPU < 0 { - continue - } - if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil { - idx, _ := strconv.Atoi(m[1]) - result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false}) - continue - } - if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil { - idx, _ := strconv.Atoi(m[1]) - port := topoNVLinkPort{Index: idx, Active: true} - if speed, err := strconv.ParseFloat(m[2], 64); err == nil { - port.SpeedGBs = &speed - } - result[currentGPU] = append(result[currentGPU], port) - } - } - return result -} - -func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) { - raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-errors.txt") - if err != nil { - return nil, err - } - return parseTopoNVLinkErrors(raw), nil -} - -// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc]. -func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 { - result := map[int]map[int][3]int64{} - currentGPU := -1 - for _, line := range strings.Split(raw, "\n") { - trimmed := strings.TrimSpace(line) - if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { - currentGPU, _ = strconv.Atoi(m[1]) - continue - } - if currentGPU < 0 { - continue - } - m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed) - if m == nil { - continue - } - linkIdx, _ := strconv.Atoi(m[1]) - count, _ := strconv.ParseInt(m[3], 10, 64) - if result[currentGPU] == nil { - result[currentGPU] = map[int][3]int64{} - } - c := result[currentGPU][linkIdx] - switch m[2] { - case "Replay": - c[0] = count - case "Recovery": - c[1] = count - case "CRC": - c[2] = count - } - result[currentGPU][linkIdx] = c - } - return result -} - -// renderTopoNVLinkCard renders the separate NVLink topology card. Returns "" -// if there are fewer than 2 NVIDIA GPUs, or the nvidia-smi nvlink techdump -// wasn't captured (older audit, or nvidia-smi unavailable on that run). -func renderTopoNVLinkCard(hw schema.HardwareSnapshot, exportDir string) string { - gpuCount := 0 - for _, dev := range hw.PCIeDevices { - if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) { - gpuCount++ - } - } - if gpuCount < 2 { - return "" - } - - status, err := readTopoNVLinkStatus(exportDir) - if err != nil || len(status) == 0 { - return topoCard("NVLink Topology", `nvidia-smi nvlink data unavailable`) - } - errors, _ := readTopoNVLinkErrors(exportDir) - - topoMatrix, _ := readGPUTopologyMatrix(exportDir) - pairs := parseGPUPairAdjacency(topoMatrix) - - var bodyB strings.Builder - if gpuCount <= 4 && len(pairs) > 0 { - // Small GPU count: per-pair box+line with per-link detail. - for _, pair := range pairs { - activeCount, total, hasError := 0, 0, false - for _, port := range status[pair.GPUA] { - total++ - if port.Active { - activeCount++ - } - } - for _, counters := range errors[pair.GPUA] { - if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 { - hasError = true - } - } - color := "var(--ok-fg)" - switch { - case hasError: - color = "var(--crit-fg)" - case total > 0 && activeCount < total: - color = "var(--warn-fg)" - } - fmt.Fprintf(&bodyB, `
`+ - `
GPU %d
`+ - `
`+ - `
GPU %d
`+ - `
%d/%d links active%s
`+ - `
`, - pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError)) - } - } else if len(pairs) > 0 { - // Larger GPU counts (NVSwitch fabric): aggregate pair table instead of - // an unreadable all-to-all graph. - bodyB.WriteString(``) - for _, pair := range pairs { - fmt.Fprintf(&bodyB, ``, pair.GPUA, pair.GPUB, pair.NVLinks) - } - bodyB.WriteString(`
GPU AGPU BNVLinks
GPU %dGPU %d%d
`) - } else { - bodyB.WriteString(`No NVLink-bonded GPU pairs found`) - } - - return topoCard("NVLink Topology", bodyB.String()) -} - -func errNoteSuffix(hasError bool) string { - if hasError { - return " — errors detected" - } - return "" -} - -// --------------------------------------------------------------------------- -// Inventory fallback for the component-detail modal -// -// handleAPIComponentDetail normally sources records from app.ComponentStatusDB, -// which only gains entries once something has actually written a status -// observation (SAT run, watchdog tick, ...). On a freshly booted host that -// hasn't run SAT yet, StatusDB can be entirely empty for a component type even -// though the /topo card for it already shows "N OK" — that card reads -// schema.HardwareComponentStatus.Status straight from the audit snapshot. -// inventoryFallbackRecords bridges that gap by building synthetic records -// from the same snapshot/classifiers the topology card uses, so the two -// views never disagree about how many devices exist or their status. -// --------------------------------------------------------------------------- - -// topoSeverityStatus renders classifyTopoSeverity's rank back into the status -// string vocabulary renderComponentDetail/chipLetterClass expect ("OK", -// "Warning", "Critical", "Unknown") — kept in lockstep with classifyTopoSeverity -// so a device the topo card counts as "OK" is never shown here as "Unknown". -func topoSeverityStatus(status *string) string { - switch classifyTopoSeverity(status) { - case 3: - return "Critical" - case 2: - return "Warning" - case 1: - return "OK" - default: - return "Unknown" - } -} - -// pcieDeviceKind classifies a PCIe device the same way renderTopoMainDiagram -// does, returning "" for devices that aren't GPU/NIC/RAID. -func pcieDeviceKind(dev schema.HardwarePCIeDevice) string { - switch { - case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass): - return "gpu" - case isNICDeviceClassDev(dev): - return "nic" - case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass): - return "raid" - default: - return "" - } -} - -// pcieDeviceKey builds a stable, human-readable component key for a PCIe -// device: ":" when a slot/BDF is known, else ":". -func pcieDeviceKey(kind string, index int, dev schema.HardwarePCIeDevice) string { - bdf := "" - if dev.Slot != nil { - bdf = normalizeTopoBDF(*dev.Slot) - } else if dev.BDF != nil { - bdf = normalizeTopoBDF(*dev.BDF) - } - if bdf != "" { - return kind + ":" + bdf - } - return fmt.Sprintf("%s:%d", kind, index) -} - -// inventoryFallbackRecords builds ComponentStatusRecord entries straight from -// the audit inventory (bee-audit.json) for the given component type, used -// when ComponentStatusDB has no matching records yet. Records carry only -// ComponentKey/Status — no LastCheckedAt/History — so renderComponentDetail -// renders them without a "checked at" timestamp or sparkline. -func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.ComponentStatusRecord { - data, err := loadSnapshot(opts.AuditPath) - if err != nil { - return nil - } - var ingest schema.HardwareIngestRequest - if err := json.Unmarshal(data, &ingest); err != nil { - return nil - } - hw := ingest.Hardware - - var records []app.ComponentStatusRecord - switch compType { - case "cpu": - for i, cpu := range hw.CPUs { - key := fmt.Sprintf("cpu:%d", i) - if cpu.Socket != nil { - key = fmt.Sprintf("cpu:socket%d", *cpu.Socket) - } - records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(cpu.Status)}) - } - case "memory": - for i, m := range hw.Memory { - key := fmt.Sprintf("memory:%d", i) - if m.Slot != nil && strings.TrimSpace(*m.Slot) != "" { - key = "memory:" + strings.TrimSpace(*m.Slot) - } - records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(m.Status)}) - } - case "storage": - for i, s := range hw.Storage { - key := fmt.Sprintf("storage:%d", i) - if s.Slot != nil && strings.TrimSpace(*s.Slot) != "" { - key = "storage:" + strings.TrimSpace(*s.Slot) - } - records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(s.Status)}) - } - case "psu": - for i, p := range hw.PowerSupplies { - key := fmt.Sprintf("psu:%d", i) - if p.Slot != nil && strings.TrimSpace(*p.Slot) != "" { - key = "psu:" + strings.TrimSpace(*p.Slot) - } - records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)}) - } - case "gpu", "nic", "raid": - for i, dev := range hw.PCIeDevices { - if pcieDeviceKind(dev) != compType { - continue - } - key := pcieDeviceKey(compType, i, dev) - records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(dev.Status)}) - } - } - return records -} diff --git a/audit/internal/webui/page_topo_diagram.go b/audit/internal/webui/page_topo_diagram.go new file mode 100644 index 0000000..09ca865 --- /dev/null +++ b/audit/internal/webui/page_topo_diagram.go @@ -0,0 +1,783 @@ +package webui + +import ( + "encoding/json" + "fmt" + "html" + "path/filepath" + "regexp" + "strconv" + "strings" + + "bee/audit/internal/app" + "bee/audit/internal/schema" +) + +func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string { + // A GPU hardware fault that needs a physical reboot (Xid 79 "fallen off + // the bus", Xid 154 "Node Reboot Required" — see gpuNeedsPhysicalReboot) + // isn't visible in dev.Status: the collector only sets that from PCIe + // link-speed checks, not from SAT/kmsg results. Without this, a GPU that + // dropped off the bus mid-test still renders green here even though the + // Hardware Summary card is showing a critical banner for it. + gpuHardwareFault := false + if db, err := app.OpenComponentStatusDB(filepath.Join(exportDir, "component-status.json")); err == nil { + if _, needsReboot := gpuNeedsPhysicalReboot(matchedRecords(db.All(), nil, []string{"pcie:gpu:"})); needsReboot { + gpuHardwareFault = true + } + } + + socketIdx := buildSocketIndex(hw.CPUs) + numCols := len(hw.CPUs) + if numCols == 0 { + numCols = 1 + } + unknownCol := numCols // extra trailing column for unmatched devices + + // Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node + // types) into columns by NUMA node, falling back to the "unknown" bucket. + type placedDevice struct { + dev schema.HardwarePCIeDevice + kind string // "gpu", "nic", "raid" + col int + bdf string + } + var placed []placedDevice + for _, dev := range hw.PCIeDevices { + kind := pcieDeviceKind(dev) + if kind == "" { + continue + } + col := unknownCol + if dev.NUMANode != nil { + if ci, ok := socketIdx[*dev.NUMANode]; ok { + col = ci + } + } + bdf := "" + if dev.Slot != nil { + bdf = normalizeTopoBDF(*dev.Slot) + } else if dev.BDF != nil { + bdf = normalizeTopoBDF(*dev.BDF) + } + placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf}) + } + hasUnknownCol := false + for _, p := range placed { + if p.col == unknownCol { + hasUnknownCol = true + break + } + } + totalCols := numCols + if hasUnknownCol { + totalCols++ + } + + // GPU index<->BDF map + pairwise NVLink adjacency, read from the + // persisted techdump captured during the last audit cycle, best-effort: + // if the dump is missing (older audit, no NVIDIA GPUs), this is simply + // skipped. Used only to detect the cross-NUMA-bonded-pair anomaly below; + // the pairwise links themselves are drawn in the separate NVLink + // Topology card, since grouping same-kind/same-column devices into one + // stacked card here leaves no single per-GPU anchor point to draw a + // pairwise connector to or from. + bdfToIndex, _ := readNVIDIAIndexByBDF(exportDir) + var pairs []gpuPairLink + if topoMatrix, err := readGPUTopologyMatrix(exportDir); err == nil { + pairs = parseGPUPairAdjacency(topoMatrix) + } + gpuNUMAByIndex := map[int]*int{} + gpuBDFByIndex := map[int]string{} + for _, p := range placed { + if p.kind != "gpu" || p.bdf == "" { + continue + } + if idx, ok := bdfToIndex[p.bdf]; ok { + gpuNUMAByIndex[idx] = p.dev.NUMANode + gpuBDFByIndex[idx] = p.bdf + } + } + // A bonded pair spanning two different NUMA nodes is treated as an + // anomaly (not a neutral fact) per project decision: a bonded pair is + // expected to sit on one NUMA node, so a cross-NUMA bond escalates both + // GPUs' effective severity to at least Warning, regardless of their own + // reported SAT status. + crossNUMAWarnBDF := map[string]bool{} + for _, pair := range pairs { + numaA, okA := gpuNUMAByIndex[pair.GPUA] + numaB, okB := gpuNUMAByIndex[pair.GPUB] + if !okA || !okB || numaA == nil || numaB == nil || *numaA == *numaB { + continue + } + crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUA]] = true + crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUB]] = true + } + + kindOrder := []string{"gpu", "nic", "raid"} + kindLabel := map[string]string{"gpu": "GPU", "nic": "NIC", "raid": "RAID"} + + // Attach memory DIMMs to their CPU column too, the same way GPU/NIC/RAID + // PCIe devices are attached via NUMANode — memory has no NUMANode field + // in the schema, so this reads the DIMM's own Locator/Bank Locator + // strings instead (see dimmRawNode). DIMMs that can't be confidently + // attached fall back to the unattached "Memory" row below the diagram, + // same as before this existed. + memBankNodes := map[string]int{} + if raw, err := readTopoTechDump(exportDir, "dmidecode-type17.txt"); err == nil { + memBankNodes = parseDIMMBankLocatorNodes(raw) + } + memCol := make([]int, len(hw.Memory)) + memMatched := make([]bool, len(hw.Memory)) + var memRawNodes []int + for i, m := range hw.Memory { + if node, ok := dimmRawNode(m, memBankNodes); ok { + memCol[i] = node + memMatched[i] = true + memRawNodes = append(memRawNodes, node) + } + } + memColIdx := buildMemoryColumnIndex(memRawNodes) + for i := range hw.Memory { + if !memMatched[i] { + continue + } + col := memColIdx[memCol[i]] + if col >= numCols { + memMatched[i] = false + continue + } + memCol[i] = col + } + + var boxes []topoBox + var pcieEdges []topoEdge + + for col := 0; col < totalCols; col++ { + colX := (col+1)*24 + col*topoColWidth + if col < len(hw.CPUs) { + cpu := hw.CPUs[col] + model := "" + if cpu.Model != nil { + model = *cpu.Model + } + socket := col + if cpu.Socket != nil { + socket = *cpu.Socket + } + var tally topoStatusTally + tally.add(classifyTopoSeverity(cpu.Status)) + fill, stroke, text := topoSeverityColors(tally.worst()) + boxes = append(boxes, topoBox{ + x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight, + topoCardInfo: topoCardInfo{ + label: fmt.Sprintf("CPU %d", socket), sublabel: model, count: 1, + statusLine: tally.line(), + fillVar: fill, strokeVar: stroke, textVar: text, + detailType: "cpu", + }, + }) + } + + y := topoTopMargin + topoBoxHeight + topoDeviceGap*2 + + // Memory goes first in the chain, directly under the CPU box: DIMMs + // are wired straight to the socket's memory controller, not reached + // over PCIe like the GPU/NIC/RAID chain below it. + var memGroup []schema.HardwareMemory + for i, m := range hw.Memory { + if memMatched[i] && memCol[i] == col { + memGroup = append(memGroup, m) + } + } + if len(memGroup) > 0 { + var tally topoStatusTally + sizeGB := 0 + for _, m := range memGroup { + tally.add(classifyTopoSeverity(m.Status)) + if m.SizeMB != nil { + sizeGB += *m.SizeMB / 1024 + } + } + fill, stroke, text := topoSeverityColors(tally.worst()) + sublabel := "" + if sizeGB > 0 { + sublabel = fmt.Sprintf("%d GB total", sizeGB) + } + stackLayers := topoStackLayers(len(memGroup)) + boxes = append(boxes, topoBox{ + x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight, + topoCardInfo: topoCardInfo{ + label: "Memory", sublabel: sublabel, count: len(memGroup), + statusLine: tally.line(), + fillVar: fill, strokeVar: stroke, textVar: text, + detailType: "memory", + }, + }) + if col < len(hw.CPUs) { + pcieEdges = append(pcieEdges, topoEdge{ + x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight, + x2: colX + topoBoxWidth/2, y2: y, + color: "var(--ok-fg)", + }) + } + y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep + } + + for _, kind := range kindOrder { + var group []placedDevice + for _, p := range placed { + if p.col == col && p.kind == kind { + group = append(group, p) + } + } + if len(group) == 0 { + continue + } + + var tally topoStatusTally + model := "" + edgeColor := "var(--ok-fg)" + for i, p := range group { + sev := classifyTopoSeverity(p.dev.Status) + if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 { + sev = 2 + } + if kind == "gpu" && gpuHardwareFault && sev < 3 { + sev = 3 + } + tally.add(sev) + if i == 0 && p.dev.Model != nil { + model = *p.dev.Model + } + if topoEdgeColorVar(p.dev) == "var(--warn-fg)" { + edgeColor = "var(--warn-fg)" + } + } + fill, stroke, text := topoSeverityColors(tally.worst()) + stackLayers := topoStackLayers(len(group)) + boxes = append(boxes, topoBox{ + x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight, + topoCardInfo: topoCardInfo{ + label: kindLabel[kind], sublabel: model, count: len(group), + statusLine: tally.line(), + fillVar: fill, strokeVar: stroke, textVar: text, + detailType: kind, + }, + }) + + if col < len(hw.CPUs) { + pcieEdges = append(pcieEdges, topoEdge{ + x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight, + x2: colX + topoBoxWidth/2, y2: y, + color: edgeColor, + }) + } + + y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep + } + } + + maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2 + for _, box := range boxes { + bottom := box.y + box.h + topoStackLayers(box.count)*topoStackStep + if bottom > maxDeviceY { + maxDeviceY = bottom + } + } + + svgHeight := maxDeviceY + 24 + svgWidth := totalCols*topoColWidth + 48 + + var b strings.Builder + // Wrapped in its own horizontally-scrolling container (matching the + // overflow-x:auto convention used for wide tables elsewhere in webui) + // rather than max-width:100% — squashing a node/edge diagram to fit a + // narrow viewport makes labels and badges illegible, whereas scrolling + // keeps the diagram readable at its natural size on any screen width. + b.WriteString(`
`) + fmt.Fprintf(&b, ``+"\n", svgWidth, svgHeight, svgWidth, svgHeight) + for _, e := range pcieEdges { + fmt.Fprintf(&b, ``+"\n", e.x1, e.y1, e.x2, e.y2, e.color) + } + for _, box := range boxes { + writeTopoBoxSVG(&b, box) + } + b.WriteString(`
`) + + // Firmware (BMC/BIOS/...) and PSUs have no PCIe/CPU affinity to anchor + // them to a column, and there can be an arbitrary number of any of them + // — so unlike the diagram above, they're plain flex-wrap HTML below the + // SVG rather than absolutely-positioned SVG boxes. A fixed-size SVG + // canvas has no way to wrap overflow onto a new row, which is exactly + // what caused these to pile up and overlap once a board had more + // PSUs/firmware records than fit in one fixed-width row. + // + // Memory DIMMs that were matched to a CPU column above already got a + // box in the SVG diagram; only DIMMs that couldn't be attached to a + // column (see memMatched above) fall back to this row. + var unmatchedMem []schema.HardwareMemory + for i, m := range hw.Memory { + if !memMatched[i] { + unmatchedMem = append(unmatchedMem, m) + } + } + if len(unmatchedMem) > 0 { + var tally topoStatusTally + for _, m := range unmatchedMem { + tally.add(classifyTopoSeverity(m.Status)) + } + fill, stroke, text := topoSeverityColors(tally.worst()) + sizeGB := 0 + for _, m := range unmatchedMem { + if m.SizeMB != nil { + sizeGB += *m.SizeMB / 1024 + } + } + sublabel := "" + if sizeGB > 0 { + sublabel = fmt.Sprintf("%d GB total", sizeGB) + } + b.WriteString(renderTopoFlexRow("Memory", []topoCardInfo{{ + label: "Memory", sublabel: sublabel, count: len(unmatchedMem), + statusLine: tally.line(), + fillVar: fill, strokeVar: stroke, textVar: text, + detailType: "memory", + }})) + } + + var firmwareItems []topoCardInfo + for _, rec := range hw.Firmware { + // Firmware records carry no per-item status in the schema (they are + // identity, not health, facts), so each stays a neutral, uncolored + // card rather than forcing a fake "Unknown" status line. + fillVar, strokeVar, textVar := topoSeverityColors(0) + firmwareItems = append(firmwareItems, topoCardInfo{ + label: rec.DeviceName, sublabel: "fw " + rec.Version, count: 1, + fillVar: fillVar, strokeVar: strokeVar, textVar: textVar, + }) + } + b.WriteString(renderTopoFlexRow("Firmware", firmwareItems)) + + if len(hw.PowerSupplies) > 0 { + var tally topoStatusTally + watt := 0 + for _, psu := range hw.PowerSupplies { + tally.add(classifyTopoSeverity(psu.Status)) + if psu.WattageW != nil { + watt = *psu.WattageW + } + } + fill, stroke, text := topoSeverityColors(tally.worst()) + sublabel := "" + if watt > 0 { + sublabel = fmt.Sprintf("%dW each", watt) + } + b.WriteString(renderTopoFlexRow("Power Supplies", []topoCardInfo{{ + label: "Power Supplies", sublabel: sublabel, count: len(hw.PowerSupplies), + statusLine: tally.line(), + fillVar: fill, strokeVar: stroke, textVar: text, + detailType: "psu", + }})) + } + + return topoCard("Topology", b.String()) +} + +// renderTopoFlexRow renders a labeled, wrapping row of component cards. +// Returns "" if items is empty (e.g. no PSU data in this audit). +func renderTopoFlexRow(title string, items []topoCardInfo) string { + if len(items) == 0 { + return "" + } + var b strings.Builder + fmt.Fprintf(&b, `
%s
`, + html.EscapeString(title)) + b.WriteString(`
`) + for _, item := range items { + onclick := "" + cursor := "default" + if item.detailType != "" { + onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, item.detailType) + cursor = "pointer" + } + stackLayers := topoStackLayers(item.count) + // Extra right/bottom padding on the wrapper reserves room for the + // backing layers of the stack effect so they aren't clipped by the + // flex container. + fmt.Fprintf(&b, `
`, + stackLayers*topoStackStep, stackLayers*topoStackStep) + for i := stackLayers; i >= 1; i-- { + off := i * topoStackStep + fmt.Fprintf(&b, `
`, + off, off, item.fillVar, item.strokeVar) + } + fmt.Fprintf(&b, ``, + onclick, cursor, item.fillVar, item.strokeVar, item.textVar) + label := item.label + if item.count > 1 { + label = fmt.Sprintf("%s ×%d", item.label, item.count) + } + fmt.Fprintf(&b, `
%s
`, html.EscapeString(label)) + if item.sublabel != "" { + fmt.Fprintf(&b, `
%s
`, html.EscapeString(item.sublabel)) + } + if item.statusLine != "" { + fmt.Fprintf(&b, `
%s
`, html.EscapeString(item.statusLine)) + } + b.WriteString(`
`) + } + b.WriteString(``) + return b.String() +} + +func writeTopoBoxSVG(b *strings.Builder, box topoBox) { + onclick := "" + cursor := "default" + if box.detailType != "" { + onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType) + cursor = "pointer" + } + fmt.Fprintf(b, ``, onclick, cursor) + + // Stack-of-cards effect: faint offset rects behind the front card when + // this box represents more than one physical component (e.g. 4 GPUs in + // one NUMA column), so a group reads as "a deck of N" rather than a + // single item. Peeks toward the bottom-right, into space already + // reserved between this box and the next one in the column. + for i := topoStackLayers(box.count); i >= 1; i-- { + off := i * topoStackStep + fmt.Fprintf(b, ``+"\n", + box.x+off, box.y+off, box.w, box.h, box.fillVar, box.strokeVar) + } + + fmt.Fprintf(b, ``+"\n", + box.x, box.y, box.w, box.h, box.fillVar, box.strokeVar) + + label := box.label + if box.count > 1 { + label = fmt.Sprintf("%s ×%d", box.label, box.count) + } + fmt.Fprintf(b, `%s`+"\n", + box.x+10, box.y+20, box.textVar, html.EscapeString(label)) + if box.sublabel != "" { + fmt.Fprintf(b, `%s`+"\n", + box.x+10, box.y+36, box.textVar, html.EscapeString(truncateTopoLabel(box.sublabel, 26))) + } + if box.statusLine != "" { + fmt.Fprintf(b, `%s`+"\n", + box.x+10, box.y+box.h-10, box.textVar, html.EscapeString(box.statusLine)) + } + b.WriteString(`` + "\n") +} + +func truncateTopoLabel(s string, max int) string { + if len(s) <= max { + return s + } + if max <= 1 { + return s[:max] + } + return s[:max-1] + "…" +} + +// --------------------------------------------------------------------------- +// Separate NVLink topology card (read from techdump, not written to any +// ingest contract) +// --------------------------------------------------------------------------- + +type topoNVLinkPort struct { + Index int + Active bool + SpeedGBs *float64 + ReplayErrors int64 + RecoveryErrors int64 + CRCErrors int64 +} + +var ( + topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`) + topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`) + topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*`) + topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`) +) + +func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) { + raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-status.txt") + if err != nil { + return nil, err + } + return parseTopoNVLinkStatus(raw), nil +} + +func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort { + result := map[int][]topoNVLinkPort{} + currentGPU := -1 + for _, line := range strings.Split(raw, "\n") { + trimmed := strings.TrimSpace(line) + if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { + currentGPU, _ = strconv.Atoi(m[1]) + continue + } + if currentGPU < 0 { + continue + } + if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil { + idx, _ := strconv.Atoi(m[1]) + result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false}) + continue + } + if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil { + idx, _ := strconv.Atoi(m[1]) + port := topoNVLinkPort{Index: idx, Active: true} + if speed, err := strconv.ParseFloat(m[2], 64); err == nil { + port.SpeedGBs = &speed + } + result[currentGPU] = append(result[currentGPU], port) + } + } + return result +} + +func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) { + raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-errors.txt") + if err != nil { + return nil, err + } + return parseTopoNVLinkErrors(raw), nil +} + +// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc]. +func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 { + result := map[int]map[int][3]int64{} + currentGPU := -1 + for _, line := range strings.Split(raw, "\n") { + trimmed := strings.TrimSpace(line) + if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { + currentGPU, _ = strconv.Atoi(m[1]) + continue + } + if currentGPU < 0 { + continue + } + m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed) + if m == nil { + continue + } + linkIdx, _ := strconv.Atoi(m[1]) + count, _ := strconv.ParseInt(m[3], 10, 64) + if result[currentGPU] == nil { + result[currentGPU] = map[int][3]int64{} + } + c := result[currentGPU][linkIdx] + switch m[2] { + case "Replay": + c[0] = count + case "Recovery": + c[1] = count + case "CRC": + c[2] = count + } + result[currentGPU][linkIdx] = c + } + return result +} + +// renderTopoNVLinkCard renders the separate NVLink topology card. Returns "" +// if there are fewer than 2 NVIDIA GPUs, or the nvidia-smi nvlink techdump +// wasn't captured (older audit, or nvidia-smi unavailable on that run). +func renderTopoNVLinkCard(hw schema.HardwareSnapshot, exportDir string) string { + gpuCount := 0 + for _, dev := range hw.PCIeDevices { + if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) { + gpuCount++ + } + } + if gpuCount < 2 { + return "" + } + + status, err := readTopoNVLinkStatus(exportDir) + if err != nil || len(status) == 0 { + return topoCard("NVLink Topology", `nvidia-smi nvlink data unavailable`) + } + errors, _ := readTopoNVLinkErrors(exportDir) + + topoMatrix, _ := readGPUTopologyMatrix(exportDir) + pairs := parseGPUPairAdjacency(topoMatrix) + + var bodyB strings.Builder + if gpuCount <= 4 && len(pairs) > 0 { + // Small GPU count: per-pair box+line with per-link detail. + for _, pair := range pairs { + activeCount, total, hasError := 0, 0, false + for _, port := range status[pair.GPUA] { + total++ + if port.Active { + activeCount++ + } + } + for _, counters := range errors[pair.GPUA] { + if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 { + hasError = true + } + } + color := "var(--ok-fg)" + switch { + case hasError: + color = "var(--crit-fg)" + case total > 0 && activeCount < total: + color = "var(--warn-fg)" + } + fmt.Fprintf(&bodyB, `
`+ + `
GPU %d
`+ + `
`+ + `
GPU %d
`+ + `
%d/%d links active%s
`+ + `
`, + pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError)) + } + } else if len(pairs) > 0 { + // Larger GPU counts (NVSwitch fabric): aggregate pair table instead of + // an unreadable all-to-all graph. + bodyB.WriteString(``) + for _, pair := range pairs { + fmt.Fprintf(&bodyB, ``, pair.GPUA, pair.GPUB, pair.NVLinks) + } + bodyB.WriteString(`
GPU AGPU BNVLinks
GPU %dGPU %d%d
`) + } else { + bodyB.WriteString(`No NVLink-bonded GPU pairs found`) + } + + return topoCard("NVLink Topology", bodyB.String()) +} + +func errNoteSuffix(hasError bool) string { + if hasError { + return " — errors detected" + } + return "" +} + +// --------------------------------------------------------------------------- +// Inventory fallback for the component-detail modal +// +// handleAPIComponentDetail normally sources records from app.ComponentStatusDB, +// which only gains entries once something has actually written a status +// observation (SAT run, watchdog tick, ...). On a freshly booted host that +// hasn't run SAT yet, StatusDB can be entirely empty for a component type even +// though the /topo card for it already shows "N OK" — that card reads +// schema.HardwareComponentStatus.Status straight from the audit snapshot. +// inventoryFallbackRecords bridges that gap by building synthetic records +// from the same snapshot/classifiers the topology card uses, so the two +// views never disagree about how many devices exist or their status. +// --------------------------------------------------------------------------- + +// topoSeverityStatus renders classifyTopoSeverity's rank back into the status +// string vocabulary renderComponentDetail/chipLetterClass expect ("OK", +// "Warning", "Critical", "Unknown") — kept in lockstep with classifyTopoSeverity +// so a device the topo card counts as "OK" is never shown here as "Unknown". +func topoSeverityStatus(status *string) string { + switch classifyTopoSeverity(status) { + case 3: + return "Critical" + case 2: + return "Warning" + case 1: + return "OK" + default: + return "Unknown" + } +} + +// pcieDeviceKind classifies a PCIe device the same way renderTopoMainDiagram +// does, returning "" for devices that aren't GPU/NIC/RAID. +func pcieDeviceKind(dev schema.HardwarePCIeDevice) string { + switch { + case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass): + return "gpu" + case isNICDeviceClassDev(dev): + return "nic" + case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass): + return "raid" + default: + return "" + } +} + +// pcieDeviceKey builds a stable, human-readable component key for a PCIe +// device: ":" when a slot/BDF is known, else ":". +func pcieDeviceKey(kind string, index int, dev schema.HardwarePCIeDevice) string { + bdf := "" + if dev.Slot != nil { + bdf = normalizeTopoBDF(*dev.Slot) + } else if dev.BDF != nil { + bdf = normalizeTopoBDF(*dev.BDF) + } + if bdf != "" { + return kind + ":" + bdf + } + return fmt.Sprintf("%s:%d", kind, index) +} + +// inventoryFallbackRecords builds ComponentStatusRecord entries straight from +// the audit inventory (bee-audit.json) for the given component type, used +// when ComponentStatusDB has no matching records yet. Records carry only +// ComponentKey/Status — no LastCheckedAt/History — so renderComponentDetail +// renders them without a "checked at" timestamp or sparkline. +func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.ComponentStatusRecord { + data, err := loadSnapshot(opts.AuditPath) + if err != nil { + return nil + } + var ingest schema.HardwareIngestRequest + if err := json.Unmarshal(data, &ingest); err != nil { + return nil + } + hw := ingest.Hardware + + var records []app.ComponentStatusRecord + switch compType { + case "cpu": + for i, cpu := range hw.CPUs { + key := fmt.Sprintf("cpu:%d", i) + if cpu.Socket != nil { + key = fmt.Sprintf("cpu:socket%d", *cpu.Socket) + } + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(cpu.Status)}) + } + case "memory": + for i, m := range hw.Memory { + key := fmt.Sprintf("memory:%d", i) + if m.Slot != nil && strings.TrimSpace(*m.Slot) != "" { + key = "memory:" + strings.TrimSpace(*m.Slot) + } + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(m.Status)}) + } + case "storage": + for i, s := range hw.Storage { + key := fmt.Sprintf("storage:%d", i) + if s.Slot != nil && strings.TrimSpace(*s.Slot) != "" { + key = "storage:" + strings.TrimSpace(*s.Slot) + } + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(s.Status)}) + } + case "psu": + for i, p := range hw.PowerSupplies { + key := fmt.Sprintf("psu:%d", i) + if p.Slot != nil && strings.TrimSpace(*p.Slot) != "" { + key = "psu:" + strings.TrimSpace(*p.Slot) + } + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)}) + } + case "gpu", "nic", "raid": + for i, dev := range hw.PCIeDevices { + if pcieDeviceKind(dev) != compType { + continue + } + key := pcieDeviceKey(compType, i, dev) + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(dev.Status)}) + } + } + return records +} diff --git a/audit/internal/webui/page_validate.go b/audit/internal/webui/page_validate.go index 0cdda47..a37640c 100644 --- a/audit/internal/webui/page_validate.go +++ b/audit/internal/webui/page_validate.go @@ -286,11 +286,6 @@ function satLoadGPUs() { satUpdateGPUSelectionNote(); }); } -function satGPUDisplayName(gpu) { - const idx = (gpu && Number.isFinite(Number(gpu.index))) ? Number(gpu.index) : 0; - const name = gpu && gpu.name ? gpu.name : ('GPU ' + idx); - return 'GPU ' + idx + ' — ' + name; -} function satRequestBody(target, overrides) { const body = {}; const labels = satLabels(); @@ -358,34 +353,9 @@ function runSATWithOverrides(target, overrides) { return enqueueSATTarget(target, overrides) .then(d => streamSATTask(d.task_id, title, false)); } -const nvidiaPerGPUTargets = []; -const nvidiaAllGPUTargets = ['nvidia', 'nvidia-targeted-stress', 'nvidia-targeted-power', 'nvidia-pulse', 'nvidia-interconnect', 'nvidia-bandwidth']; function satAllGPUIndicesForMulti() { return Promise.resolve(satSelectedGPUIndices()); } -function expandSATTarget(target) { - if (nvidiaAllGPUTargets.indexOf(target) >= 0) { - return satAllGPUIndicesForMulti().then(function(indices) { - if (!indices.length) return Promise.reject(new Error('No NVIDIA GPUs available.')); - return [{target: target, overrides: {gpu_indices: indices, display_name: satLabels()[target] || target}}]; - }); - } - if (nvidiaPerGPUTargets.indexOf(target) < 0) { - return Promise.resolve([{target: target}]); - } - const selected = satSelectedGPUIndices(); - if (!selected.length) { - return Promise.reject(new Error('Select at least one NVIDIA GPU.')); - } - return loadSatNvidiaGPUs().then(gpus => gpus.filter(gpu => selected.indexOf(Number(gpu.index)) >= 0).map(gpu => ({ - target: target, - overrides: { - gpu_indices: [Number(gpu.index)], - display_name: (satLabels()[target] || ('Validate ' + target)) + ' (' + satGPUDisplayName(gpu) + ')' - }, - label: satGPUDisplayName(gpu), - }))); -} function runNvidiaFabricValidate(target) { satAllGPUIndicesForMulti().then(function(indices) { if (!indices.length) { alert('No NVIDIA GPUs available.'); return; } @@ -419,52 +389,40 @@ function runAMDValidateSet() { }; return runNext(0); } +// runAllSAT hands the whole decision to the backend: which hardware is +// present and ready, and therefore which tasks to enqueue, is decided by +// POST /api/sat/run-all. The page only sends operator intent. function runAllSAT() { - const cycles = 1; const status = document.getElementById('sat-all-status'); - status.textContent = 'Enqueuing...'; - const stressOnlyTargets = ['nvidia-targeted-stress', 'nvidia-targeted-power', 'nvidia-pulse']; - const baseTargets = ['nvidia','nvidia-targeted-stress','nvidia-targeted-power','nvidia-pulse','nvidia-interconnect','nvidia-bandwidth','memory','storage','tpm','cpu'].concat(selectedAMDValidateTargets()); - const activeTargets = baseTargets.filter(target => { - if (stressOnlyTargets.indexOf(target) >= 0 && !satStressMode()) return false; - const btn = document.getElementById('sat-btn-' + target); - return !(btn && btn.disabled); - }); - Promise.all(activeTargets.map(expandSATTarget)).then(groups => { - const expanded = []; - for (let cycle = 0; cycle < cycles; cycle++) { - groups.forEach(group => group.forEach(item => expanded.push(item))); - } - const total = expanded.length; - let enqueued = 0; - if (!total) { - status.textContent = 'No tasks selected.'; - return; - } - const runNext = (idx) => { - if (idx >= expanded.length) { status.textContent = 'Completed ' + total + ' task(s).'; return Promise.resolve(); } - const item = expanded[idx]; - status.textContent = 'Running ' + (idx + 1) + '/' + total + '...'; - return enqueueSATTarget(item.target, item.overrides) - .then(() => { - enqueued++; - return runNext(idx + 1); - }); - }; - return runNext(0); - }).catch(err => { - status.textContent = 'Error: ' + err.message; - }); + status.textContent = 'Planning on server...'; + const body = { + stress_mode: satStressMode(), + amd_targets: selectedAMDValidateTargets(), + }; + const gpuSubset = satSelectedGPUIndices(); + if (gpuSubset.length) body.nvidia_gpu_indices = gpuSubset; + fetch('/api/sat/run-all', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify(body), + }).then(r => r.json().then(d => { if (!r.ok) throw new Error(d.error || ('HTTP ' + r.status)); return d; })) + .then(d => { + let msg = 'Enqueued ' + (d.task_count || 0) + ' task(s).'; + if (d.notes && d.notes.length) msg += ' ' + d.notes.join('; '); + status.textContent = msg; + }) + .catch(err => { status.textContent = 'Error: ' + err.message; }); } ` } - -func renderHealthCard(opts HandlerOptions) string { - data, err := loadSnapshot(filepath.Join(opts.ExportDir, "runtime-health.json")) - if err != nil { - return `
Runtime Health
No data
` - } - var health schema.RuntimeHealth - if err := json.Unmarshal(data, &health); err != nil { - return `
Runtime Health
Parse error
` - } - status := strings.TrimSpace(health.Status) - if status == "" { - status = "UNKNOWN" - } - badge := "badge-ok" - if status == "PARTIAL" { - badge = "badge-warn" - } else if status == "FAIL" || status == "FAILED" { - badge = "badge-err" - } - var b strings.Builder - b.WriteString(`
Runtime Health
`) - b.WriteString(fmt.Sprintf(`
%s
`, badge, html.EscapeString(status))) - if checkedAt := strings.TrimSpace(health.CheckedAt); checkedAt != "" { - b.WriteString(`
Checked at: ` + html.EscapeString(checkedAt) + `
`) - } - rows := []runtimeHealthRow{ - buildRuntimeExportRow(health), - buildRuntimeNetworkRow(health), - buildRuntimeDriverRow(health), - buildRuntimeAccelerationRow(health), - buildRuntimeToolsRow(health), - buildRuntimeServicesRow(health), - buildRuntimeUSBExportRow(health), - buildRuntimeToRAMRow(health), - } - b.WriteString(``) - for _, row := range rows { - b.WriteString(``) - } - b.WriteString(`
CheckStatusSourceIssue
` + html.EscapeString(row.Title) + `` + runtimeStatusBadge(row.Status) + `` + html.EscapeString(row.Source) + `` + rowIssueHTML(row.Issue) + `
`) - b.WriteString(`
`) - return b.String() -} - -type runtimeHealthRow struct { - Title string - Status string - Source string - Issue string -} - -func buildRuntimeExportRow(health schema.RuntimeHealth) runtimeHealthRow { - issue := runtimeIssueDescriptions(health.Issues, "export_dir_unavailable") - status := "UNKNOWN" - switch { - case issue != "": - status = "FAILED" - case strings.TrimSpace(health.ExportDir) != "": - status = "OK" - } - source := "os.MkdirAll" - if dir := strings.TrimSpace(health.ExportDir); dir != "" { - source += " " + dir - } - return runtimeHealthRow{Title: "Export Directory", Status: status, Source: source, Issue: issue} -} - -func buildRuntimeNetworkRow(health schema.RuntimeHealth) runtimeHealthRow { - status := strings.TrimSpace(health.NetworkStatus) - if status == "" { - status = "UNKNOWN" - } - issue := runtimeIssueDescriptions(health.Issues, "dhcp_failed") - return runtimeHealthRow{Title: "Network", Status: status, Source: "ListInterfaces / DHCP", Issue: issue} -} - -func buildRuntimeDriverRow(health schema.RuntimeHealth) runtimeHealthRow { - issue := runtimeIssueDescriptions(health.Issues, "nvidia_kernel_module_missing", "nvidia_modeset_failed", "amdgpu_kernel_module_missing") - status := "UNKNOWN" - switch { - case health.DriverReady && issue == "": - status = "OK" - case health.DriverReady: - status = "PARTIAL" - case issue != "": - status = "FAILED" - } - return runtimeHealthRow{Title: "NVIDIA/AMD Driver", Status: status, Source: "lsmod / vendor probe", Issue: issue} -} - -func buildRuntimeAccelerationRow(health schema.RuntimeHealth) runtimeHealthRow { - issue := runtimeIssueDescriptions(health.Issues, "cuda_runtime_not_ready", "rocm_smi_unavailable") - status := "UNKNOWN" - switch { - case health.CUDAReady && issue == "": - status = "OK" - case health.CUDAReady: - status = "PARTIAL" - case issue != "": - status = "FAILED" - } - return runtimeHealthRow{Title: "CUDA / ROCm", Status: status, Source: "bee-gpu-burn / rocm-smi", Issue: issue} -} - -func buildRuntimeToolsRow(health schema.RuntimeHealth) runtimeHealthRow { - if len(health.Tools) == 0 { - return runtimeHealthRow{Title: "Required Utilities", Status: "UNKNOWN", Source: "CheckTools", Issue: "No tool status data."} - } - missing := make([]string, 0) - for _, tool := range health.Tools { - if !tool.OK { - missing = append(missing, tool.Name) - } - } - status := "OK" - issue := "" - if len(missing) > 0 { - status = "PARTIAL" - issue = "Missing: " + strings.Join(missing, ", ") - } - return runtimeHealthRow{Title: "Required Utilities", Status: status, Source: "CheckTools", Issue: issue} -} - -func buildRuntimeServicesRow(health schema.RuntimeHealth) runtimeHealthRow { - if len(health.Services) == 0 { - return runtimeHealthRow{Title: "Bee Services", Status: "UNKNOWN", Source: "systemctl is-active", Issue: "No service status data."} - } - nonActive := make([]string, 0) - for _, svc := range health.Services { - state := strings.TrimSpace(strings.ToLower(svc.Status)) - // "inactive" is OK for oneshot services that have completed successfully - // (bee-sshsetup, bee-preflight, bee-audit, bee-network, etc.). - // Only "failed" is a genuine problem. - switch state { - case "active", "activating", "deactivating", "reloading", "inactive": - // OK — service is running, transitioning normally, or completed successfully - default: - nonActive = append(nonActive, svc.Name+"="+svc.Status) - } - } - status := "OK" - issue := "" - if len(nonActive) > 0 { - status = "PARTIAL" - issue = strings.Join(nonActive, ", ") - } - return runtimeHealthRow{Title: "Bee Services", Status: status, Source: "ServiceState", Issue: issue} -} - -func buildRuntimeUSBExportRow(health schema.RuntimeHealth) runtimeHealthRow { - path := strings.TrimSpace(health.USBExportPath) - if path != "" { - return runtimeHealthRow{ - Title: "USB Export Drive", - Status: "OK", - Source: "/proc/mounts + lsblk", - Issue: path, - } - } - return runtimeHealthRow{ - Title: "USB Export Drive", - Status: "WARNING", - Source: "/proc/mounts + lsblk", - Issue: "No writable USB drive mounted. Plug in a USB drive to enable log export.", - } -} - -func buildRuntimeToRAMRow(health schema.RuntimeHealth) runtimeHealthRow { - switch strings.ToLower(strings.TrimSpace(health.ToRAMStatus)) { - case "ok": - return runtimeHealthRow{ - Title: "LiveCD in RAM", - Status: "OK", - Source: "live-boot / /proc/mounts", - Issue: "", - } - case "partial": - return runtimeHealthRow{ - Title: "LiveCD in RAM", - Status: "WARNING", - Source: "live-boot / /proc/mounts / /dev/shm/bee-live", - Issue: "Partial or staged RAM copy detected. System is not fully running from RAM; Copy to RAM can be retried.", - } - case "failed": - return runtimeHealthRow{ - Title: "LiveCD in RAM", - Status: "FAILED", - Source: "live-boot / /proc/mounts", - Issue: "toram boot parameter set but ISO is not mounted from RAM. Copy may have failed.", - } - default: - // toram not active — ISO still on original boot media (USB/CD) - return runtimeHealthRow{ - Title: "LiveCD in RAM", - Status: "WARNING", - Source: "live-boot / /proc/mounts", - Issue: "ISO not copied to RAM. Use \u201cCopy to RAM\u201d to free the boot drive and improve performance.", - } - } -} - -func buildHardwareComponentRows(exportDir string) []runtimeHealthRow { - path := filepath.Join(exportDir, "component-status.json") - db, err := app.OpenComponentStatusDB(path) - if err != nil { - return []runtimeHealthRow{ - {Title: "CPU Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."}, - {Title: "Memory Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."}, - {Title: "Storage Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."}, - {Title: "GPU Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."}, - {Title: "PSU Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "No PSU component checks recorded."}, - } - } - records := db.All() - return []runtimeHealthRow{ - aggregateComponentStatus("CPU", records, []string{"cpu:all"}, nil), - aggregateComponentStatus("Memory", records, []string{"memory:all"}, []string{"memory:"}), - aggregateComponentStatus("Storage", records, []string{"storage:all"}, []string{"storage:"}), - aggregateComponentStatus("GPU", records, nil, []string{"pcie:gpu:"}), - aggregateComponentStatus("PSU", records, nil, []string{"psu:"}), - } -} - -// matchedRecords returns all ComponentStatusRecord entries whose key matches -// any exact key or any of the given prefixes. Used for per-device chip rendering. -func firstNonEmpty(vals ...string) string { - for _, v := range vals { - if v != "" { - return v - } - } - return "" -} - -func matchedRecords(records []app.ComponentStatusRecord, exact []string, prefixes []string) []app.ComponentStatusRecord { - var matched []app.ComponentStatusRecord - for _, rec := range records { - key := strings.TrimSpace(rec.ComponentKey) - if key == "" { - continue - } - if containsExactKey(key, exact) || hasAnyPrefix(key, prefixes) { - matched = append(matched, rec) - } - } - return matched -} - -func aggregateComponentStatus(title string, records []app.ComponentStatusRecord, exact []string, prefixes []string) runtimeHealthRow { - matched := make([]app.ComponentStatusRecord, 0) - for _, rec := range records { - key := strings.TrimSpace(rec.ComponentKey) - if key == "" { - continue - } - if containsExactKey(key, exact) || hasAnyPrefix(key, prefixes) { - matched = append(matched, rec) - } - } - if len(matched) == 0 { - return runtimeHealthRow{Title: title, Status: "UNKNOWN", Source: "component-status.json", Issue: "No component status data."} - } - - maxSev := -1 - for _, rec := range matched { - if sev := runtimeComponentSeverity(rec.Status); sev > maxSev { - maxSev = sev - } - } - status := "UNKNOWN" - switch maxSev { - case 3: - status = "CRITICAL" - case 2: - status = "WARNING" - case 1: - status = "OK" - } - - sources := make([]string, 0) - sourceSeen := map[string]struct{}{} - issues := make([]string, 0) - issueSeen := map[string]struct{}{} - for _, rec := range matched { - if runtimeComponentSeverity(rec.Status) != maxSev { - continue - } - source := latestComponentSource(rec) - if source == "" { - source = "component-status.json" - } - if _, ok := sourceSeen[source]; !ok { - sourceSeen[source] = struct{}{} - sources = append(sources, source) - } - issue := strings.TrimSpace(rec.ErrorSummary) - if issue == "" { - issue = latestComponentDetail(rec) - } - if issue == "" { - continue - } - if _, ok := issueSeen[issue]; ok { - continue - } - issueSeen[issue] = struct{}{} - issues = append(issues, issue) - } - if len(sources) == 0 { - sources = append(sources, "component-status.json") - } - issue := strings.Join(issues, "; ") - if issue == "" { - issue = "—" - } - return runtimeHealthRow{ - Title: title, - Status: status, - Source: strings.Join(sources, ", "), - Issue: issue, - } -} - -func containsExactKey(key string, exact []string) bool { - for _, candidate := range exact { - if key == candidate { - return true - } - } - return false -} - -func hasAnyPrefix(key string, prefixes []string) bool { - for _, prefix := range prefixes { - if strings.HasPrefix(key, prefix) { - return true - } - } - return false -} - -func runtimeComponentSeverity(status string) int { - switch strings.TrimSpace(strings.ToLower(status)) { - case "critical": - return 3 - case "warning": - return 2 - case "ok": - return 1 - default: - return 0 - } -} - -func latestComponentSource(rec app.ComponentStatusRecord) string { - if len(rec.History) == 0 { - return "" - } - return strings.TrimSpace(rec.History[len(rec.History)-1].Source) -} - -func latestComponentDetail(rec app.ComponentStatusRecord) string { - if len(rec.History) == 0 { - return "" - } - return strings.TrimSpace(rec.History[len(rec.History)-1].Detail) -} - -func runtimeIssueDescriptions(issues []schema.RuntimeIssue, codes ...string) string { - if len(issues) == 0 || len(codes) == 0 { - return "" - } - allowed := make(map[string]struct{}, len(codes)) - for _, code := range codes { - allowed[code] = struct{}{} - } - messages := make([]string, 0) - for _, issue := range issues { - if _, ok := allowed[issue.Code]; !ok { - continue - } - desc := strings.TrimSpace(issue.Description) - if desc == "" { - desc = issue.Code - } - messages = append(messages, desc) - } - return strings.Join(messages, "; ") -} - -// gpuNeedsPhysicalReboot reports whether any GPU component record carries a -// hardware-fault reason that a driver reset can't clear (e.g. Xid 79 "GPU -// has fallen off the bus", Xid 154 "Node Reboot Required" — see -// collector.xidHardwareFaultMessages). Those errors mean every subsequent -// GPU SAT job will keep failing until the node is physically power-cycled, -// so this drives a dashboard banner that says so up front instead of making -// an operator burn another test cycle to rediscover it. -func gpuNeedsPhysicalReboot(records []app.ComponentStatusRecord) (reason string, needsReboot bool) { - for _, rec := range records { - if !strings.EqualFold(strings.TrimSpace(rec.Status), "Critical") { - continue - } - if strings.Contains(strings.ToLower(rec.ErrorSummary), "reboot") { - return rec.ErrorSummary, true - } - } - return "", false -} - -// chipLetterClass maps a component status to a single display letter and CSS class. -func chipLetterClass(status string) (letter, cls string) { - switch strings.ToUpper(strings.TrimSpace(status)) { - case "OK": - return "O", "chip-ok" - case "WARNING", "WARN", "PARTIAL": - return "W", "chip-warn" - case "CRITICAL", "FAIL", "FAILED", "ERROR": - return "F", "chip-fail" - default: - return "?", "chip-unknown" - } -} - -// renderComponentChips renders one 20×20 chip per ComponentStatusRecord. -// Hover tooltip shows component key, status, error summary and last check time. -// Falls back to a single unknown chip when no records are available. -func renderComponentChips(matched []app.ComponentStatusRecord) string { - if len(matched) == 0 { - return `?` - } - sort.Slice(matched, func(i, j int) bool { - return matched[i].ComponentKey < matched[j].ComponentKey - }) - var b strings.Builder - b.WriteString(``) - for _, rec := range matched { - letter, cls := chipLetterClass(rec.Status) - var tooltip strings.Builder - tooltip.WriteString(rec.ComponentKey) - tooltip.WriteString(": ") - tooltip.WriteString(firstNonEmpty(rec.Status, "UNKNOWN")) - if rec.ErrorSummary != "" { - tooltip.WriteString(" — ") - tooltip.WriteString(rec.ErrorSummary) - } - if !rec.LastCheckedAt.IsZero() { - fmt.Fprintf(&tooltip, " (checked %s)", rec.LastCheckedAt.Format("15:04:05")) - } - fmt.Fprintf(&b, `%s`, - cls, html.EscapeString(tooltip.String()), letter) - } - b.WriteString(``) - return b.String() -} - -func runtimeStatusBadge(status string) string { - status = strings.ToUpper(strings.TrimSpace(status)) - badge := "badge-unknown" - switch status { - case "OK": - badge = "badge-ok" - case "PARTIAL", "WARNING", "WARN": - badge = "badge-warn" - case "FAIL", "FAILED", "CRITICAL": - badge = "badge-err" - } - return `` + html.EscapeString(status) + `` -} - -func rowIssueHTML(issue string) string { - issue = strings.TrimSpace(issue) - if issue == "" { - return `—` - } - return html.EscapeString(issue) -} - -var aerStatusRe = regexp.MustCompile(`aer_status:\s*0x([0-9a-fA-F]{1,8})`) - -// decodeAERStatus parses an AER status hex value from a kernel error detail string -// and returns a human-readable list of set bit names with correctable/uncorrectable label, -// or "" if no AER status is found. -func decodeAERStatus(detail string) string { - m := aerStatusRe.FindStringSubmatch(detail) - if m == nil { - return "" - } - v64, err := strconv.ParseUint(m[1], 16, 32) - if err != nil { - return "" - } - val := uint32(v64) - - type bitDef struct { - bit uint32 - name string - } - corrBits := []bitDef{ - {0, "Receiver Error"}, {6, "Replay Timer Timeout"}, {7, "Advisory Non-Fatal"}, - {8, "Corrected Internal Error"}, {9, "Header Log Overflow"}, - {13, "Replay Num Rollover"}, {14, "Bad DLLP"}, {15, "Bad TLP"}, - } - uncorrBits := []bitDef{ - {4, "Data Link Protocol Error"}, {5, "Surprise Down Error"}, - {12, "Poisoned TLP Received"}, {13, "Flow Control Protocol Error"}, - {14, "Completion Timeout"}, {15, "Completer Abort"}, {16, "Unexpected Completion"}, - {17, "Receiver Overflow"}, {18, "Malformed TLP"}, {19, "ECRC Error"}, - {20, "Unsupported Request Error"}, {21, "ACS Violation"}, {22, "Uncorrectable Internal Error"}, - } - var corrNames, uncorrNames []string - for _, b := range corrBits { - if val&(1<= len(uncorrNames) && len(corrNames) > 0 { - return strings.Join(corrNames, ", ") + " (correctable)" - } - if len(uncorrNames) > 0 { - return strings.Join(uncorrNames, ", ") + " (uncorrectable)" - } - return fmt.Sprintf("unknown bits: 0x%08x", val) -} - -// renderSparkline returns a small inline SVG showing non-OK events over time. -// Events are positioned proportionally along the time axis; if all share the same -// timestamp they are spaced evenly. Width is always 100px. -func renderSparkline(history []app.ComponentStatusEntry) string { - const ( - svgW = 100 - svgH = 20 - barW = 3 - barH = 14 - ) - var events []app.ComponentStatusEntry - for _, e := range history { - if e.Status != "OK" { - events = append(events, e) - } - } - if len(events) == 0 { - return "" - } - n := len(events) - barColor := func(status string) string { - if status == "Critical" { - return "#c0392b" - } - return "#d97706" - } - yTop := (svgH - barH) / 2 - - var bars strings.Builder - if n == 1 { - x := (svgW - barW) / 2 - fmt.Fprintf(&bars, ``, - x, yTop, barW, barH, barColor(events[0].Status)) - } else { - minT := events[0].At - maxT := events[n-1].At - dur := maxT.Sub(minT).Seconds() - for i, e := range events { - var x int - if dur <= 0 { - step := svgW / n - x = i*step + (step-barW)/2 - } else { - frac := e.At.Sub(minT).Seconds() / dur - x = int(frac * float64(svgW-barW)) - } - fmt.Fprintf(&bars, ``, - x, yTop, barW, barH, barColor(e.Status)) - } - } - return fmt.Sprintf( - ``+ - `%s`, - svgW, svgH, svgW, svgH, bars.String()) -} - -// renderComponentDetail renders a modal content fragment for one component type. -// Called by handleAPIComponentDetail and displayed inside #component-detail-dialog. -// fromInventory marks that records were synthesized from the audit inventory -// snapshot (no ComponentStatusDB history yet) rather than real SAT/watchdog -// observations — see inventoryFallbackRecords. -func renderComponentDetail(title string, records []app.ComponentStatusRecord, fromInventory bool) string { - var b strings.Builder - fmt.Fprintf(&b, `
`) - fmt.Fprintf(&b, `
`) - fmt.Fprintf(&b, `%s — Status Detail`, html.EscapeString(title)) - b.WriteString(``) - b.WriteString(`
`) - - if len(records) == 0 { - b.WriteString(`

No status data recorded yet for this component type.

`) - b.WriteString(`
`) - return b.String() - } - - if fromInventory { - b.WriteString(`

No SAT-test history yet — showing latest inventory snapshot.

`) - } - - sort.Slice(records, func(i, j int) bool { - return records[i].ComponentKey < records[j].ComponentKey - }) - - for _, rec := range records { - letter, cls := chipLetterClass(rec.Status) - - // Count non-OK events across the full history for the badge + sparkline. - warnCount := 0 - for _, e := range rec.History { - if e.Status != "OK" { - warnCount++ - } - } - - fmt.Fprintf(&b, `
`) - fmt.Fprintf(&b, `
`) - fmt.Fprintf(&b, `%s`, cls, letter) - fmt.Fprintf(&b, `%s`, html.EscapeString(rec.ComponentKey)) - if !rec.LastCheckedAt.IsZero() { - fmt.Fprintf(&b, `checked %s`, rec.LastCheckedAt.Format("2006-01-02 15:04:05")) - } - if warnCount > 0 { - noun := "events" - if warnCount == 1 { - noun = "event" - } - fmt.Fprintf(&b, - `%d %s`, - warnCount, noun) - b.WriteString(renderSparkline(rec.History)) - } - b.WriteString(`
`) - - if rec.ErrorSummary != "" { - fmt.Fprintf(&b, `
%s
`, html.EscapeString(rec.ErrorSummary)) - if decoded := decodeAERStatus(rec.ErrorSummary); decoded != "" { - fmt.Fprintf(&b, - `
AER: %s
`, - html.EscapeString(decoded)) - } - } - - // History table — newest first, cap at 20 entries. - history := rec.History - if len(history) > 20 { - history = history[len(history)-20:] - } - b.WriteString(``) - b.WriteString(``) - for i := len(history) - 1; i >= 0; i-- { - e := history[i] - eLetter, eCls := chipLetterClass(e.Status) - detail := e.Detail - if detail == "" { - detail = "—" - } - fmt.Fprintf(&b, - ``, - html.EscapeString(e.At.Format("2006-01-02 15:04:05")), - eCls, eLetter, - html.EscapeString(e.Source), - html.EscapeString(detail), - ) - } - b.WriteString(`
TimeStatusSourceDetail
%s%s%s%s
`) - b.WriteString(`
`) - } - - b.WriteString(``) - return b.String() -} diff --git a/audit/internal/webui/pages_health.go b/audit/internal/webui/pages_health.go new file mode 100644 index 0000000..d5ca292 --- /dev/null +++ b/audit/internal/webui/pages_health.go @@ -0,0 +1,572 @@ +package webui + +import ( + "encoding/json" + "fmt" + "html" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "bee/audit/internal/app" + "bee/audit/internal/schema" +) + +func renderHealthCard(opts HandlerOptions) string { + data, err := loadSnapshot(filepath.Join(opts.ExportDir, "runtime-health.json")) + if err != nil { + return `
Runtime Health
No data
` + } + var health schema.RuntimeHealth + if err := json.Unmarshal(data, &health); err != nil { + return `
Runtime Health
Parse error
` + } + status := strings.TrimSpace(health.Status) + if status == "" { + status = "UNKNOWN" + } + badge := "badge-ok" + if status == "PARTIAL" { + badge = "badge-warn" + } else if status == "FAIL" || status == "FAILED" { + badge = "badge-err" + } + var b strings.Builder + b.WriteString(`
Runtime Health
`) + b.WriteString(fmt.Sprintf(`
%s
`, badge, html.EscapeString(status))) + if checkedAt := strings.TrimSpace(health.CheckedAt); checkedAt != "" { + b.WriteString(`
Checked at: ` + html.EscapeString(checkedAt) + `
`) + } + rows := []runtimeHealthRow{ + buildRuntimeExportRow(health), + buildRuntimeNetworkRow(health), + buildRuntimeDriverRow(health), + buildRuntimeAccelerationRow(health), + buildRuntimeToolsRow(health), + buildRuntimeServicesRow(health), + buildRuntimeUSBExportRow(health), + buildRuntimeToRAMRow(health), + } + b.WriteString(``) + for _, row := range rows { + b.WriteString(``) + } + b.WriteString(`
CheckStatusSourceIssue
` + html.EscapeString(row.Title) + `` + runtimeStatusBadge(row.Status) + `` + html.EscapeString(row.Source) + `` + rowIssueHTML(row.Issue) + `
`) + b.WriteString(`
`) + return b.String() +} + +type runtimeHealthRow struct { + Title string + Status string + Source string + Issue string +} + +func buildRuntimeExportRow(health schema.RuntimeHealth) runtimeHealthRow { + issue := runtimeIssueDescriptions(health.Issues, "export_dir_unavailable") + status := "UNKNOWN" + switch { + case issue != "": + status = "FAILED" + case strings.TrimSpace(health.ExportDir) != "": + status = "OK" + } + source := "os.MkdirAll" + if dir := strings.TrimSpace(health.ExportDir); dir != "" { + source += " " + dir + } + return runtimeHealthRow{Title: "Export Directory", Status: status, Source: source, Issue: issue} +} + +func buildRuntimeNetworkRow(health schema.RuntimeHealth) runtimeHealthRow { + status := strings.TrimSpace(health.NetworkStatus) + if status == "" { + status = "UNKNOWN" + } + issue := runtimeIssueDescriptions(health.Issues, "dhcp_failed") + return runtimeHealthRow{Title: "Network", Status: status, Source: "ListInterfaces / DHCP", Issue: issue} +} + +func buildRuntimeDriverRow(health schema.RuntimeHealth) runtimeHealthRow { + issue := runtimeIssueDescriptions(health.Issues, "nvidia_kernel_module_missing", "nvidia_modeset_failed", "amdgpu_kernel_module_missing") + status := "UNKNOWN" + switch { + case health.DriverReady && issue == "": + status = "OK" + case health.DriverReady: + status = "PARTIAL" + case issue != "": + status = "FAILED" + } + return runtimeHealthRow{Title: "NVIDIA/AMD Driver", Status: status, Source: "lsmod / vendor probe", Issue: issue} +} + +func buildRuntimeAccelerationRow(health schema.RuntimeHealth) runtimeHealthRow { + issue := runtimeIssueDescriptions(health.Issues, "cuda_runtime_not_ready", "rocm_smi_unavailable") + status := "UNKNOWN" + switch { + case health.CUDAReady && issue == "": + status = "OK" + case health.CUDAReady: + status = "PARTIAL" + case issue != "": + status = "FAILED" + } + return runtimeHealthRow{Title: "CUDA / ROCm", Status: status, Source: "bee-gpu-burn / rocm-smi", Issue: issue} +} + +func buildRuntimeToolsRow(health schema.RuntimeHealth) runtimeHealthRow { + if len(health.Tools) == 0 { + return runtimeHealthRow{Title: "Required Utilities", Status: "UNKNOWN", Source: "CheckTools", Issue: "No tool status data."} + } + missing := make([]string, 0) + for _, tool := range health.Tools { + if !tool.OK { + missing = append(missing, tool.Name) + } + } + status := "OK" + issue := "" + if len(missing) > 0 { + status = "PARTIAL" + issue = "Missing: " + strings.Join(missing, ", ") + } + return runtimeHealthRow{Title: "Required Utilities", Status: status, Source: "CheckTools", Issue: issue} +} + +func buildRuntimeServicesRow(health schema.RuntimeHealth) runtimeHealthRow { + if len(health.Services) == 0 { + return runtimeHealthRow{Title: "Bee Services", Status: "UNKNOWN", Source: "systemctl is-active", Issue: "No service status data."} + } + nonActive := make([]string, 0) + for _, svc := range health.Services { + state := strings.TrimSpace(strings.ToLower(svc.Status)) + // "inactive" is OK for oneshot services that have completed successfully + // (bee-sshsetup, bee-preflight, bee-audit, bee-network, etc.). + // Only "failed" is a genuine problem. + switch state { + case "active", "activating", "deactivating", "reloading", "inactive": + // OK — service is running, transitioning normally, or completed successfully + default: + nonActive = append(nonActive, svc.Name+"="+svc.Status) + } + } + status := "OK" + issue := "" + if len(nonActive) > 0 { + status = "PARTIAL" + issue = strings.Join(nonActive, ", ") + } + return runtimeHealthRow{Title: "Bee Services", Status: status, Source: "ServiceState", Issue: issue} +} + +func buildRuntimeUSBExportRow(health schema.RuntimeHealth) runtimeHealthRow { + path := strings.TrimSpace(health.USBExportPath) + if path != "" { + return runtimeHealthRow{ + Title: "USB Export Drive", + Status: "OK", + Source: "/proc/mounts + lsblk", + Issue: path, + } + } + return runtimeHealthRow{ + Title: "USB Export Drive", + Status: "WARNING", + Source: "/proc/mounts + lsblk", + Issue: "No writable USB drive mounted. Plug in a USB drive to enable log export.", + } +} + +func buildRuntimeToRAMRow(health schema.RuntimeHealth) runtimeHealthRow { + switch strings.ToLower(strings.TrimSpace(health.ToRAMStatus)) { + case "ok": + return runtimeHealthRow{ + Title: "LiveCD in RAM", + Status: "OK", + Source: "live-boot / /proc/mounts", + Issue: "", + } + case "partial": + return runtimeHealthRow{ + Title: "LiveCD in RAM", + Status: "WARNING", + Source: "live-boot / /proc/mounts / /dev/shm/bee-live", + Issue: "Partial or staged RAM copy detected. System is not fully running from RAM; Copy to RAM can be retried.", + } + case "failed": + return runtimeHealthRow{ + Title: "LiveCD in RAM", + Status: "FAILED", + Source: "live-boot / /proc/mounts", + Issue: "toram boot parameter set but ISO is not mounted from RAM. Copy may have failed.", + } + default: + // toram not active — ISO still on original boot media (USB/CD) + return runtimeHealthRow{ + Title: "LiveCD in RAM", + Status: "WARNING", + Source: "live-boot / /proc/mounts", + Issue: "ISO not copied to RAM. Use \u201cCopy to RAM\u201d to free the boot drive and improve performance.", + } + } +} + +// matchedRecords returns all ComponentStatusRecord entries whose key matches +// any exact key or any of the given prefixes. Used for per-device chip rendering. +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "" +} + +func matchedRecords(records []app.ComponentStatusRecord, exact []string, prefixes []string) []app.ComponentStatusRecord { + var matched []app.ComponentStatusRecord + for _, rec := range records { + key := strings.TrimSpace(rec.ComponentKey) + if key == "" { + continue + } + if containsExactKey(key, exact) || hasAnyPrefix(key, prefixes) { + matched = append(matched, rec) + } + } + return matched +} + +func containsExactKey(key string, exact []string) bool { + for _, candidate := range exact { + if key == candidate { + return true + } + } + return false +} + +func hasAnyPrefix(key string, prefixes []string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(key, prefix) { + return true + } + } + return false +} + +func runtimeIssueDescriptions(issues []schema.RuntimeIssue, codes ...string) string { + if len(issues) == 0 || len(codes) == 0 { + return "" + } + allowed := make(map[string]struct{}, len(codes)) + for _, code := range codes { + allowed[code] = struct{}{} + } + messages := make([]string, 0) + for _, issue := range issues { + if _, ok := allowed[issue.Code]; !ok { + continue + } + desc := strings.TrimSpace(issue.Description) + if desc == "" { + desc = issue.Code + } + messages = append(messages, desc) + } + return strings.Join(messages, "; ") +} + +// gpuNeedsPhysicalReboot reports whether any GPU component record carries a +// hardware-fault reason that a driver reset can't clear (e.g. Xid 79 "GPU +// has fallen off the bus", Xid 154 "Node Reboot Required" — see +// collector.xidHardwareFaultMessages). Those errors mean every subsequent +// GPU SAT job will keep failing until the node is physically power-cycled, +// so this drives a dashboard banner that says so up front instead of making +// an operator burn another test cycle to rediscover it. +func gpuNeedsPhysicalReboot(records []app.ComponentStatusRecord) (reason string, needsReboot bool) { + for _, rec := range records { + if !strings.EqualFold(strings.TrimSpace(rec.Status), "Critical") { + continue + } + if strings.Contains(strings.ToLower(rec.ErrorSummary), "reboot") { + return rec.ErrorSummary, true + } + } + return "", false +} + +// chipLetterClass maps a component status to a single display letter and CSS class. +func chipLetterClass(status string) (letter, cls string) { + switch strings.ToUpper(strings.TrimSpace(status)) { + case "OK": + return "O", "chip-ok" + case "WARNING", "WARN", "PARTIAL": + return "W", "chip-warn" + case "CRITICAL", "FAIL", "FAILED", "ERROR": + return "F", "chip-fail" + default: + return "?", "chip-unknown" + } +} + +// renderComponentChips renders one 20×20 chip per ComponentStatusRecord. +// Hover tooltip shows component key, status, error summary and last check time. +// Falls back to a single unknown chip when no records are available. +func renderComponentChips(matched []app.ComponentStatusRecord) string { + if len(matched) == 0 { + return `?` + } + sort.Slice(matched, func(i, j int) bool { + return matched[i].ComponentKey < matched[j].ComponentKey + }) + var b strings.Builder + b.WriteString(``) + for _, rec := range matched { + letter, cls := chipLetterClass(rec.Status) + var tooltip strings.Builder + tooltip.WriteString(rec.ComponentKey) + tooltip.WriteString(": ") + tooltip.WriteString(firstNonEmpty(rec.Status, "UNKNOWN")) + if rec.ErrorSummary != "" { + tooltip.WriteString(" — ") + tooltip.WriteString(rec.ErrorSummary) + } + if !rec.LastCheckedAt.IsZero() { + fmt.Fprintf(&tooltip, " (checked %s)", rec.LastCheckedAt.Format("15:04:05")) + } + fmt.Fprintf(&b, `%s`, + cls, html.EscapeString(tooltip.String()), letter) + } + b.WriteString(``) + return b.String() +} + +func runtimeStatusBadge(status string) string { + status = strings.ToUpper(strings.TrimSpace(status)) + badge := "badge-unknown" + switch status { + case "OK": + badge = "badge-ok" + case "PARTIAL", "WARNING", "WARN": + badge = "badge-warn" + case "FAIL", "FAILED", "CRITICAL": + badge = "badge-err" + } + return `` + html.EscapeString(status) + `` +} + +func rowIssueHTML(issue string) string { + issue = strings.TrimSpace(issue) + if issue == "" { + return `—` + } + return html.EscapeString(issue) +} + +var aerStatusRe = regexp.MustCompile(`aer_status:\s*0x([0-9a-fA-F]{1,8})`) + +// decodeAERStatus parses an AER status hex value from a kernel error detail string +// and returns a human-readable list of set bit names with correctable/uncorrectable label, +// or "" if no AER status is found. +func decodeAERStatus(detail string) string { + m := aerStatusRe.FindStringSubmatch(detail) + if m == nil { + return "" + } + v64, err := strconv.ParseUint(m[1], 16, 32) + if err != nil { + return "" + } + val := uint32(v64) + + type bitDef struct { + bit uint32 + name string + } + corrBits := []bitDef{ + {0, "Receiver Error"}, {6, "Replay Timer Timeout"}, {7, "Advisory Non-Fatal"}, + {8, "Corrected Internal Error"}, {9, "Header Log Overflow"}, + {13, "Replay Num Rollover"}, {14, "Bad DLLP"}, {15, "Bad TLP"}, + } + uncorrBits := []bitDef{ + {4, "Data Link Protocol Error"}, {5, "Surprise Down Error"}, + {12, "Poisoned TLP Received"}, {13, "Flow Control Protocol Error"}, + {14, "Completion Timeout"}, {15, "Completer Abort"}, {16, "Unexpected Completion"}, + {17, "Receiver Overflow"}, {18, "Malformed TLP"}, {19, "ECRC Error"}, + {20, "Unsupported Request Error"}, {21, "ACS Violation"}, {22, "Uncorrectable Internal Error"}, + } + var corrNames, uncorrNames []string + for _, b := range corrBits { + if val&(1<= len(uncorrNames) && len(corrNames) > 0 { + return strings.Join(corrNames, ", ") + " (correctable)" + } + if len(uncorrNames) > 0 { + return strings.Join(uncorrNames, ", ") + " (uncorrectable)" + } + return fmt.Sprintf("unknown bits: 0x%08x", val) +} + +// renderSparkline returns a small inline SVG showing non-OK events over time. +// Events are positioned proportionally along the time axis; if all share the same +// timestamp they are spaced evenly. Width is always 100px. +func renderSparkline(history []app.ComponentStatusEntry) string { + const ( + svgW = 100 + svgH = 20 + barW = 3 + barH = 14 + ) + var events []app.ComponentStatusEntry + for _, e := range history { + if e.Status != "OK" { + events = append(events, e) + } + } + if len(events) == 0 { + return "" + } + n := len(events) + barColor := func(status string) string { + if status == "Critical" { + return "#c0392b" + } + return "#d97706" + } + yTop := (svgH - barH) / 2 + + var bars strings.Builder + if n == 1 { + x := (svgW - barW) / 2 + fmt.Fprintf(&bars, ``, + x, yTop, barW, barH, barColor(events[0].Status)) + } else { + minT := events[0].At + maxT := events[n-1].At + dur := maxT.Sub(minT).Seconds() + for i, e := range events { + var x int + if dur <= 0 { + step := svgW / n + x = i*step + (step-barW)/2 + } else { + frac := e.At.Sub(minT).Seconds() / dur + x = int(frac * float64(svgW-barW)) + } + fmt.Fprintf(&bars, ``, + x, yTop, barW, barH, barColor(e.Status)) + } + } + return fmt.Sprintf( + ``+ + `%s`, + svgW, svgH, svgW, svgH, bars.String()) +} + +// renderComponentDetail renders a modal content fragment for one component type. +// Called by handleAPIComponentDetail and displayed inside #component-detail-dialog. +// fromInventory marks that records were synthesized from the audit inventory +// snapshot (no ComponentStatusDB history yet) rather than real SAT/watchdog +// observations — see inventoryFallbackRecords. +func renderComponentDetail(title string, records []app.ComponentStatusRecord, fromInventory bool) string { + var b strings.Builder + fmt.Fprintf(&b, `
`) + fmt.Fprintf(&b, `
`) + fmt.Fprintf(&b, `%s — Status Detail`, html.EscapeString(title)) + b.WriteString(``) + b.WriteString(`
`) + + if len(records) == 0 { + b.WriteString(`

No status data recorded yet for this component type.

`) + b.WriteString(`
`) + return b.String() + } + + if fromInventory { + b.WriteString(`

No SAT-test history yet — showing latest inventory snapshot.

`) + } + + sort.Slice(records, func(i, j int) bool { + return records[i].ComponentKey < records[j].ComponentKey + }) + + for _, rec := range records { + letter, cls := chipLetterClass(rec.Status) + + // Count non-OK events across the full history for the badge + sparkline. + warnCount := 0 + for _, e := range rec.History { + if e.Status != "OK" { + warnCount++ + } + } + + fmt.Fprintf(&b, `
`) + fmt.Fprintf(&b, `
`) + fmt.Fprintf(&b, `%s`, cls, letter) + fmt.Fprintf(&b, `%s`, html.EscapeString(rec.ComponentKey)) + if !rec.LastCheckedAt.IsZero() { + fmt.Fprintf(&b, `checked %s`, rec.LastCheckedAt.Format("2006-01-02 15:04:05")) + } + if warnCount > 0 { + noun := "events" + if warnCount == 1 { + noun = "event" + } + fmt.Fprintf(&b, + `%d %s`, + warnCount, noun) + b.WriteString(renderSparkline(rec.History)) + } + b.WriteString(`
`) + + if rec.ErrorSummary != "" { + fmt.Fprintf(&b, `
%s
`, html.EscapeString(rec.ErrorSummary)) + if decoded := decodeAERStatus(rec.ErrorSummary); decoded != "" { + fmt.Fprintf(&b, + `
AER: %s
`, + html.EscapeString(decoded)) + } + } + + // History table — newest first, cap at 20 entries. + history := rec.History + if len(history) > 20 { + history = history[len(history)-20:] + } + b.WriteString(``) + b.WriteString(``) + for i := len(history) - 1; i >= 0; i-- { + e := history[i] + eLetter, eCls := chipLetterClass(e.Status) + detail := e.Detail + if detail == "" { + detail = "—" + } + fmt.Fprintf(&b, + ``, + html.EscapeString(e.At.Format("2006-01-02 15:04:05")), + eCls, eLetter, + html.EscapeString(e.Source), + html.EscapeString(detail), + ) + } + b.WriteString(`
TimeStatusSourceDetail
%s%s%s%s
`) + b.WriteString(`
`) + } + + b.WriteString(``) + return b.String() +} diff --git a/audit/internal/webui/raid_actions.go b/audit/internal/webui/raid_actions.go new file mode 100644 index 0000000..efc4bd2 --- /dev/null +++ b/audit/internal/webui/raid_actions.go @@ -0,0 +1,707 @@ +package webui + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os/exec" + "strconv" + "strings" + "time" +) + +func (h *handler) handleAPIRAIDStatus(w http.ResponseWriter, r *http.Request) { + resp := raidStatusResp{Controllers: []raidControllerInfo{}} + + lsi2 := detectStorcli2Controllers() + if lsi := detectLSIControllers(); len(lsi) > 0 { + // storcli64 can enumerate a Tri-Mode controller (SAS3808-iMR/9500 + // series) at a basic level but its drive-listing JSON parser finds + // no "Drive Information" for these — a zero-drives entry that + // storcli2 (run above) already covers correctly. Only drop it when + // storcli2 actually found something, so a genuinely drive-populated + // classic controller elsewhere in a mixed setup is never hidden. + for _, c := range lsi { + if len(c.AllDrives) == 0 && len(lsi2) > 0 { + continue + } + resp.Controllers = append(resp.Controllers, c) + } + } + if len(lsi2) > 0 { + resp.Controllers = append(resp.Controllers, lsi2...) + } + if vroc := detectVROCController(); vroc != nil { + resp.Controllers = append(resp.Controllers, *vroc) + } + + writeJSON(w, resp) +} + +func (h *handler) handleAPIRAIDForeignAction(w http.ResponseWriter, r *http.Request) { + var req struct { + ControllerID string `json:"controller_id"` + Action string `json:"action"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid JSON") + return + } + if req.Action != "import" && req.Action != "clear" { + writeError(w, http.StatusBadRequest, "action must be 'import' or 'clear'") + return + } + ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID) + if !ok { + writeError(w, http.StatusBadRequest, "invalid controller_id") + return + } + + target := "raid-foreign-clear" + name := fmt.Sprintf("RAID Foreign Clear (ctrl %d)", ctrlIdx) + if req.Action == "import" { + target = "raid-foreign-import" + name = fmt.Sprintf("RAID Foreign Import (ctrl %d)", ctrlIdx) + } + + t := &Task{ + ID: newJobID(target), + Name: name, + Target: target, + Priority: defaultTaskPriority(target, taskParams{}), + Status: TaskPending, + CreatedAt: time.Now(), + params: taskParams{RAIDController: ctrlIdx}, + } + globalQueue.enqueue(t) + writeJSON(w, map[string]string{"task_id": t.ID}) +} + +func (h *handler) handleAPIRAIDCreateMirror(w http.ResponseWriter, r *http.Request) { + var req struct { + ControllerID string `json:"controller_id"` + Devices []string `json:"devices"` + ArrayName string `json:"array_name"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid JSON") + return + } + if len(req.Devices) < 2 { + writeError(w, http.StatusBadRequest, "at least 2 devices required") + return + } + + var target, name string + var params taskParams + + switch { + case strings.HasPrefix(req.ControllerID, "lsi-"): + ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID) + if !ok { + writeError(w, http.StatusBadRequest, "invalid controller_id") + return + } + target = "raid-lsi-create-mirror" + name = fmt.Sprintf("Create RAID 1 Mirror (LSI ctrl %d)", ctrlIdx) + params = taskParams{RAIDController: ctrlIdx, RAIDDevices: req.Devices} + + case req.ControllerID == "vroc-0": + arrayName := strings.TrimSpace(req.ArrayName) + if arrayName == "" { + arrayName = "bee-mirror0" + } + target = "raid-vroc-create-mirror" + name = fmt.Sprintf("Create VROC RAID 1 (%s)", arrayName) + params = taskParams{RAIDDevices: req.Devices, RAIDArrayName: arrayName} + + default: + writeError(w, http.StatusBadRequest, "unknown controller_id") + return + } + + t := &Task{ + ID: newJobID(target), + Name: name, + Target: target, + Priority: defaultTaskPriority(target, taskParams{}), + Status: TaskPending, + CreatedAt: time.Now(), + params: params, + } + globalQueue.enqueue(t) + writeJSON(w, map[string]string{"task_id": t.ID}) +} + +func (h *handler) handleAPIRAIDPrepareDrive(w http.ResponseWriter, r *http.Request) { + var req struct { + ControllerID string `json:"controller_id"` + Slot string `json:"slot"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid JSON") + return + } + ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID) + if !ok { + writeError(w, http.StatusBadRequest, "invalid controller_id") + return + } + if _, _, ok := parseRAIDSlot(req.Slot); !ok { + writeError(w, http.StatusBadRequest, "invalid slot") + return + } + + t := &Task{ + ID: newJobID("raid-lsi-prepare-drive"), + Name: fmt.Sprintf("Prepare drive %s (LSI ctrl %d)", req.Slot, ctrlIdx), + Target: "raid-lsi-prepare-drive", + Priority: defaultTaskPriority("raid-lsi-prepare-drive", taskParams{}), + Status: TaskPending, + CreatedAt: time.Now(), + params: taskParams{RAIDController: ctrlIdx, RAIDSlot: req.Slot}, + } + globalQueue.enqueue(t) + writeJSON(w, map[string]string{"task_id": t.ID}) +} + +func parseLSIControllerIndex(id string) (int, bool) { + if !strings.HasPrefix(id, "lsi-") { + return 0, false + } + n, err := strconv.Atoi(strings.TrimPrefix(id, "lsi-")) + if err != nil || n < 0 { + return 0, false + } + return n, true +} + +// --- Task runner functions --- + +func runRAIDForeignClearTask(ctx context.Context, j *jobState, ctrl int) error { + j.append(fmt.Sprintf("Clearing foreign configuration on controller %d...", ctrl)) + cmd := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d/fall", ctrl), "del", "noprompt") + return streamCmdJob(j, cmd) +} + +func runRAIDForeignImportTask(ctx context.Context, j *jobState, ctrl int) error { + j.append(fmt.Sprintf("Importing foreign configuration on controller %d...", ctrl)) + cmd := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d/fall", ctrl), "import", "noprompt") + return streamCmdJob(j, cmd) +} + +// raidPrepareAction says what (if anything) must be done to a drive in the +// given storcli state before it can join a new VD. Derived from the Broadcom +// StorCLI drive-state matrix: only UGood drives are accepted by "add vd"; +// JBOD/UBad convert with "set good force"; hotspares must be released first; +// Frgn/Onln hold configuration data and must not be silently destroyed. +type raidPrepareAction int + +const ( + raidPrepNone raidPrepareAction = iota // UGood or unknown — try add vd as-is + raidPrepSetGood // JBOD, UBad — "set good force" + raidPrepHotspare // GHS, DHS — "delete hotsparedrive", then set good + raidPrepBlockedFrgn + raidPrepBlockedOnln +) + +func classifyRAIDPrepareAction(state string) raidPrepareAction { + switch strings.TrimSpace(state) { + case "JBOD", "UBad": + return raidPrepSetGood + case "GHS", "DHS": + return raidPrepHotspare + case "Frgn": + return raidPrepBlockedFrgn + case "Onln", "Offln": + return raidPrepBlockedOnln + default: // "UGood", "" (state unknown — let add vd decide) + return raidPrepNone + } +} + +// raidLSIDriveStates returns EID:Slt -> State for one controller, or nil if +// storcli/parsing fails (callers then fall back to unconditional prepare). +func raidLSIDriveStates(ctx context.Context, ctrl int) map[string]string { + out, err := exec.CommandContext(ctx, "storcli64", + fmt.Sprintf("/c%d/eall/sall", ctrl), "show", "all", "J").Output() + if err != nil { + return nil + } + var doc struct { + Controllers []struct { + ResponseData map[string]json.RawMessage `json:"Response Data"` + } `json:"Controllers"` + } + if err := json.Unmarshal(out, &doc); err != nil || len(doc.Controllers) == 0 { + return nil + } + states := map[string]string{} + for _, c := range doc.Controllers { + for _, d := range parseStorcliResponseDataDrives(c.ResponseData) { + states[strings.TrimSpace(d.EIDSlt)] = strings.TrimSpace(d.State) + } + } + return states +} + +func runRAIDLSICreateMirrorTask(ctx context.Context, j *jobState, ctrl int, drives []string) error { + driveList := strings.Join(drives, ",") + states := raidLSIDriveStates(ctx, ctrl) + + // Non-UGood drives cannot be added to a VD directly — storcli fails with + // "resources already in use" (exit 11) or similar. Fix what is safely + // fixable (JBOD/UBad/hotspare), refuse what holds data (Frgn/Onln). + for _, drive := range drives { + eid, slt, ok := parseRAIDSlot(drive) + if !ok { + return fmt.Errorf("invalid drive slot %q", drive) + } + state := states[drive] + slotPath := fmt.Sprintf("/c%d/e%d/s%d", ctrl, eid, slt) + + switch classifyRAIDPrepareAction(state) { + case raidPrepBlockedFrgn: + return fmt.Errorf("drive %s carries a foreign configuration; run the RAID Foreign Clear (or Import) task first, then retry", drive) + case raidPrepBlockedOnln: + return fmt.Errorf("drive %s is part of an existing virtual drive (state %s); delete that VD first", drive, state) + case raidPrepHotspare: + j.append(fmt.Sprintf("Drive %s is a hotspare (%s); releasing it...", drive, state)) + rel := exec.CommandContext(ctx, "storcli64", slotPath, "delete", "hotsparedrive") + if err := streamCmdJob(j, rel); err != nil { + return fmt.Errorf("release hotspare %s: %w", drive, err) + } + case raidPrepSetGood: + j.append(fmt.Sprintf("Drive %s is %s; converting to Unconfigured Good (set good force)...", drive, state)) + prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force") + if err := streamCmdJob(j, prep); err != nil { + return fmt.Errorf("set good on %s: %w", drive, err) + } + case raidPrepNone: + if state == "" { + // Drive state unknown (storcli query failed) — attempt the + // conversion anyway; harmless on an already-UGood drive with + // force, and add vd below is the real verdict. + j.append(fmt.Sprintf("Preparing drive %s (set good, force)...", drive)) + prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force") + if err := streamCmdJob(j, prep); err != nil { + j.append(fmt.Sprintf("note: set good on %s: %v (continuing)", drive, err)) + } + } + } + } + + j.append(fmt.Sprintf("Creating RAID 1 on controller %d with drives: %s", ctrl, driveList)) + cmd := exec.CommandContext(ctx, "storcli64", + fmt.Sprintf("/c%d", ctrl), + "add", "vd", "type=raid1", + fmt.Sprintf("drives=%s", driveList), + "pdperarray=2", + ) + if err := streamCmdJob(j, cmd); err != nil { + // A blocked add vd is often preserved cache from a dead VD + // ("controller has data in cache for offline or missing virtual + // drives"). Surface it so the log is actionable. + j.append("add vd failed; checking for preserved cache...") + pc := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d", ctrl), "show", "preservedcache") + _ = streamCmdJob(j, pc) + j.append(fmt.Sprintf("hint: if preserved cache is listed above, clear it with: storcli64 /c%d/vall delete preservedcache (invalidates cached data of dead VDs), then retry", ctrl)) + return err + } + return nil +} + +// parseRAIDSlot splits a storcli "EID:Slt" identifier (e.g. "252:0") into +// enclosure and slot numbers. +func parseRAIDSlot(slot string) (eid int, slt int, ok bool) { + parts := strings.SplitN(strings.TrimSpace(slot), ":", 2) + if len(parts) != 2 { + return 0, 0, false + } + eid, err1 := strconv.Atoi(strings.TrimSpace(parts[0])) + slt, err2 := strconv.Atoi(strings.TrimSpace(parts[1])) + if err1 != nil || err2 != nil { + return 0, 0, false + } + return eid, slt, true +} + +func runRAIDPrepareDriveTask(ctx context.Context, j *jobState, ctrl int, slot string) error { + eid, slt, ok := parseRAIDSlot(slot) + if !ok { + return fmt.Errorf("invalid slot %q", slot) + } + j.append(fmt.Sprintf("Preparing drive %s on controller %d (set good, force)...", slot, ctrl)) + cmd := exec.CommandContext(ctx, "storcli64", + fmt.Sprintf("/c%d/e%d/s%d", ctrl, eid, slt), + "set", "good", "force", + ) + return streamCmdJob(j, cmd) +} + +func runRAIDVROCCreateMirrorTask(ctx context.Context, j *jobState, devices []string, arrayName string) error { + if arrayName == "" { + arrayName = "bee-mirror0" + } + devPath := "/dev/md/" + arrayName + args := []string{ + "--create", devPath, + "--level=1", + fmt.Sprintf("--raid-devices=%d", len(devices)), + "--run", + } + args = append(args, devices...) + j.append(fmt.Sprintf("Creating VROC RAID 1 array %s with: %s", devPath, strings.Join(devices, " "))) + cmd := exec.CommandContext(ctx, "mdadm", args...) + return streamCmdJob(j, cmd) +} + +// raidParseHumanSizeGB parses storcli size strings like "1.818 TB", "745.211 GB". +func raidParseHumanSizeGB(s string) float64 { + s = strings.TrimSpace(s) + if s == "" { + return 0 + } + upper := strings.ToUpper(s) + var mul float64 + var numStr string + switch { + case strings.Contains(upper, " TB"): + mul = 1024 + numStr = strings.TrimSpace(strings.SplitN(upper, " T", 2)[0]) + case strings.Contains(upper, " GB"): + mul = 1 + numStr = strings.TrimSpace(strings.SplitN(upper, " G", 2)[0]) + case strings.Contains(upper, " MB"): + mul = 1.0 / 1024 + numStr = strings.TrimSpace(strings.SplitN(upper, " M", 2)[0]) + default: + return 0 + } + v, err := strconv.ParseFloat(numStr, 64) + if err != nil { + return 0 + } + return v * mul +} + +// --- UI card --- + +func renderRAIDMgmtCard() string { + return `
RAID Controller Management
+
Loading...
+
+ +
+` +} diff --git a/audit/internal/webui/raid_mgmt.go b/audit/internal/webui/raid_mgmt.go index 125234d..3cb7469 100644 --- a/audit/internal/webui/raid_mgmt.go +++ b/audit/internal/webui/raid_mgmt.go @@ -1,16 +1,12 @@ package webui import ( - "context" "encoding/json" "fmt" - "net/http" "os" "os/exec" "regexp" - "strconv" "strings" - "time" ) // --- Response types --- @@ -395,698 +391,3 @@ func detectVROCController() *raidControllerInfo { } // --- API handlers --- - -func (h *handler) handleAPIRAIDStatus(w http.ResponseWriter, r *http.Request) { - resp := raidStatusResp{Controllers: []raidControllerInfo{}} - - lsi2 := detectStorcli2Controllers() - if lsi := detectLSIControllers(); len(lsi) > 0 { - // storcli64 can enumerate a Tri-Mode controller (SAS3808-iMR/9500 - // series) at a basic level but its drive-listing JSON parser finds - // no "Drive Information" for these — a zero-drives entry that - // storcli2 (run above) already covers correctly. Only drop it when - // storcli2 actually found something, so a genuinely drive-populated - // classic controller elsewhere in a mixed setup is never hidden. - for _, c := range lsi { - if len(c.AllDrives) == 0 && len(lsi2) > 0 { - continue - } - resp.Controllers = append(resp.Controllers, c) - } - } - if len(lsi2) > 0 { - resp.Controllers = append(resp.Controllers, lsi2...) - } - if vroc := detectVROCController(); vroc != nil { - resp.Controllers = append(resp.Controllers, *vroc) - } - - writeJSON(w, resp) -} - -func (h *handler) handleAPIRAIDForeignAction(w http.ResponseWriter, r *http.Request) { - var req struct { - ControllerID string `json:"controller_id"` - Action string `json:"action"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid JSON") - return - } - if req.Action != "import" && req.Action != "clear" { - writeError(w, http.StatusBadRequest, "action must be 'import' or 'clear'") - return - } - ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID) - if !ok { - writeError(w, http.StatusBadRequest, "invalid controller_id") - return - } - - target := "raid-foreign-clear" - name := fmt.Sprintf("RAID Foreign Clear (ctrl %d)", ctrlIdx) - if req.Action == "import" { - target = "raid-foreign-import" - name = fmt.Sprintf("RAID Foreign Import (ctrl %d)", ctrlIdx) - } - - t := &Task{ - ID: newJobID(target), - Name: name, - Target: target, - Priority: defaultTaskPriority(target, taskParams{}), - Status: TaskPending, - CreatedAt: time.Now(), - params: taskParams{RAIDController: ctrlIdx}, - } - globalQueue.enqueue(t) - writeJSON(w, map[string]string{"task_id": t.ID}) -} - -func (h *handler) handleAPIRAIDCreateMirror(w http.ResponseWriter, r *http.Request) { - var req struct { - ControllerID string `json:"controller_id"` - Devices []string `json:"devices"` - ArrayName string `json:"array_name"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid JSON") - return - } - if len(req.Devices) < 2 { - writeError(w, http.StatusBadRequest, "at least 2 devices required") - return - } - - var target, name string - var params taskParams - - switch { - case strings.HasPrefix(req.ControllerID, "lsi-"): - ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID) - if !ok { - writeError(w, http.StatusBadRequest, "invalid controller_id") - return - } - target = "raid-lsi-create-mirror" - name = fmt.Sprintf("Create RAID 1 Mirror (LSI ctrl %d)", ctrlIdx) - params = taskParams{RAIDController: ctrlIdx, RAIDDevices: req.Devices} - - case req.ControllerID == "vroc-0": - arrayName := strings.TrimSpace(req.ArrayName) - if arrayName == "" { - arrayName = "bee-mirror0" - } - target = "raid-vroc-create-mirror" - name = fmt.Sprintf("Create VROC RAID 1 (%s)", arrayName) - params = taskParams{RAIDDevices: req.Devices, RAIDArrayName: arrayName} - - default: - writeError(w, http.StatusBadRequest, "unknown controller_id") - return - } - - t := &Task{ - ID: newJobID(target), - Name: name, - Target: target, - Priority: defaultTaskPriority(target, taskParams{}), - Status: TaskPending, - CreatedAt: time.Now(), - params: params, - } - globalQueue.enqueue(t) - writeJSON(w, map[string]string{"task_id": t.ID}) -} - -func (h *handler) handleAPIRAIDPrepareDrive(w http.ResponseWriter, r *http.Request) { - var req struct { - ControllerID string `json:"controller_id"` - Slot string `json:"slot"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid JSON") - return - } - ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID) - if !ok { - writeError(w, http.StatusBadRequest, "invalid controller_id") - return - } - if _, _, ok := parseRAIDSlot(req.Slot); !ok { - writeError(w, http.StatusBadRequest, "invalid slot") - return - } - - t := &Task{ - ID: newJobID("raid-lsi-prepare-drive"), - Name: fmt.Sprintf("Prepare drive %s (LSI ctrl %d)", req.Slot, ctrlIdx), - Target: "raid-lsi-prepare-drive", - Priority: defaultTaskPriority("raid-lsi-prepare-drive", taskParams{}), - Status: TaskPending, - CreatedAt: time.Now(), - params: taskParams{RAIDController: ctrlIdx, RAIDSlot: req.Slot}, - } - globalQueue.enqueue(t) - writeJSON(w, map[string]string{"task_id": t.ID}) -} - -func parseLSIControllerIndex(id string) (int, bool) { - if !strings.HasPrefix(id, "lsi-") { - return 0, false - } - n, err := strconv.Atoi(strings.TrimPrefix(id, "lsi-")) - if err != nil || n < 0 { - return 0, false - } - return n, true -} - -// --- Task runner functions --- - -func runRAIDForeignClearTask(ctx context.Context, j *jobState, ctrl int) error { - j.append(fmt.Sprintf("Clearing foreign configuration on controller %d...", ctrl)) - cmd := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d/fall", ctrl), "del", "noprompt") - return streamCmdJob(j, cmd) -} - -func runRAIDForeignImportTask(ctx context.Context, j *jobState, ctrl int) error { - j.append(fmt.Sprintf("Importing foreign configuration on controller %d...", ctrl)) - cmd := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d/fall", ctrl), "import", "noprompt") - return streamCmdJob(j, cmd) -} - -// raidPrepareAction says what (if anything) must be done to a drive in the -// given storcli state before it can join a new VD. Derived from the Broadcom -// StorCLI drive-state matrix: only UGood drives are accepted by "add vd"; -// JBOD/UBad convert with "set good force"; hotspares must be released first; -// Frgn/Onln hold configuration data and must not be silently destroyed. -type raidPrepareAction int - -const ( - raidPrepNone raidPrepareAction = iota // UGood or unknown — try add vd as-is - raidPrepSetGood // JBOD, UBad — "set good force" - raidPrepHotspare // GHS, DHS — "delete hotsparedrive", then set good - raidPrepBlockedFrgn - raidPrepBlockedOnln -) - -func classifyRAIDPrepareAction(state string) raidPrepareAction { - switch strings.TrimSpace(state) { - case "JBOD", "UBad": - return raidPrepSetGood - case "GHS", "DHS": - return raidPrepHotspare - case "Frgn": - return raidPrepBlockedFrgn - case "Onln", "Offln": - return raidPrepBlockedOnln - default: // "UGood", "" (state unknown — let add vd decide) - return raidPrepNone - } -} - -// raidLSIDriveStates returns EID:Slt -> State for one controller, or nil if -// storcli/parsing fails (callers then fall back to unconditional prepare). -func raidLSIDriveStates(ctx context.Context, ctrl int) map[string]string { - out, err := exec.CommandContext(ctx, "storcli64", - fmt.Sprintf("/c%d/eall/sall", ctrl), "show", "all", "J").Output() - if err != nil { - return nil - } - var doc struct { - Controllers []struct { - ResponseData map[string]json.RawMessage `json:"Response Data"` - } `json:"Controllers"` - } - if err := json.Unmarshal(out, &doc); err != nil || len(doc.Controllers) == 0 { - return nil - } - states := map[string]string{} - for _, c := range doc.Controllers { - for _, d := range parseStorcliResponseDataDrives(c.ResponseData) { - states[strings.TrimSpace(d.EIDSlt)] = strings.TrimSpace(d.State) - } - } - return states -} - -func runRAIDLSICreateMirrorTask(ctx context.Context, j *jobState, ctrl int, drives []string) error { - driveList := strings.Join(drives, ",") - states := raidLSIDriveStates(ctx, ctrl) - - // Non-UGood drives cannot be added to a VD directly — storcli fails with - // "resources already in use" (exit 11) or similar. Fix what is safely - // fixable (JBOD/UBad/hotspare), refuse what holds data (Frgn/Onln). - for _, drive := range drives { - eid, slt, ok := parseRAIDSlot(drive) - if !ok { - return fmt.Errorf("invalid drive slot %q", drive) - } - state := states[drive] - slotPath := fmt.Sprintf("/c%d/e%d/s%d", ctrl, eid, slt) - - switch classifyRAIDPrepareAction(state) { - case raidPrepBlockedFrgn: - return fmt.Errorf("drive %s carries a foreign configuration; run the RAID Foreign Clear (or Import) task first, then retry", drive) - case raidPrepBlockedOnln: - return fmt.Errorf("drive %s is part of an existing virtual drive (state %s); delete that VD first", drive, state) - case raidPrepHotspare: - j.append(fmt.Sprintf("Drive %s is a hotspare (%s); releasing it...", drive, state)) - rel := exec.CommandContext(ctx, "storcli64", slotPath, "delete", "hotsparedrive") - if err := streamCmdJob(j, rel); err != nil { - return fmt.Errorf("release hotspare %s: %w", drive, err) - } - case raidPrepSetGood: - j.append(fmt.Sprintf("Drive %s is %s; converting to Unconfigured Good (set good force)...", drive, state)) - prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force") - if err := streamCmdJob(j, prep); err != nil { - return fmt.Errorf("set good on %s: %w", drive, err) - } - case raidPrepNone: - if state == "" { - // Drive state unknown (storcli query failed) — attempt the - // conversion anyway; harmless on an already-UGood drive with - // force, and add vd below is the real verdict. - j.append(fmt.Sprintf("Preparing drive %s (set good, force)...", drive)) - prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force") - if err := streamCmdJob(j, prep); err != nil { - j.append(fmt.Sprintf("note: set good on %s: %v (continuing)", drive, err)) - } - } - } - } - - j.append(fmt.Sprintf("Creating RAID 1 on controller %d with drives: %s", ctrl, driveList)) - cmd := exec.CommandContext(ctx, "storcli64", - fmt.Sprintf("/c%d", ctrl), - "add", "vd", "type=raid1", - fmt.Sprintf("drives=%s", driveList), - "pdperarray=2", - ) - if err := streamCmdJob(j, cmd); err != nil { - // A blocked add vd is often preserved cache from a dead VD - // ("controller has data in cache for offline or missing virtual - // drives"). Surface it so the log is actionable. - j.append("add vd failed; checking for preserved cache...") - pc := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d", ctrl), "show", "preservedcache") - _ = streamCmdJob(j, pc) - j.append(fmt.Sprintf("hint: if preserved cache is listed above, clear it with: storcli64 /c%d/vall delete preservedcache (invalidates cached data of dead VDs), then retry", ctrl)) - return err - } - return nil -} - -// parseRAIDSlot splits a storcli "EID:Slt" identifier (e.g. "252:0") into -// enclosure and slot numbers. -func parseRAIDSlot(slot string) (eid int, slt int, ok bool) { - parts := strings.SplitN(strings.TrimSpace(slot), ":", 2) - if len(parts) != 2 { - return 0, 0, false - } - eid, err1 := strconv.Atoi(strings.TrimSpace(parts[0])) - slt, err2 := strconv.Atoi(strings.TrimSpace(parts[1])) - if err1 != nil || err2 != nil { - return 0, 0, false - } - return eid, slt, true -} - -func runRAIDPrepareDriveTask(ctx context.Context, j *jobState, ctrl int, slot string) error { - eid, slt, ok := parseRAIDSlot(slot) - if !ok { - return fmt.Errorf("invalid slot %q", slot) - } - j.append(fmt.Sprintf("Preparing drive %s on controller %d (set good, force)...", slot, ctrl)) - cmd := exec.CommandContext(ctx, "storcli64", - fmt.Sprintf("/c%d/e%d/s%d", ctrl, eid, slt), - "set", "good", "force", - ) - return streamCmdJob(j, cmd) -} - -func runRAIDVROCCreateMirrorTask(ctx context.Context, j *jobState, devices []string, arrayName string) error { - if arrayName == "" { - arrayName = "bee-mirror0" - } - devPath := "/dev/md/" + arrayName - args := []string{ - "--create", devPath, - "--level=1", - fmt.Sprintf("--raid-devices=%d", len(devices)), - "--run", - } - args = append(args, devices...) - j.append(fmt.Sprintf("Creating VROC RAID 1 array %s with: %s", devPath, strings.Join(devices, " "))) - cmd := exec.CommandContext(ctx, "mdadm", args...) - return streamCmdJob(j, cmd) -} - -// raidParseHumanSizeGB parses storcli size strings like "1.818 TB", "745.211 GB". -func raidParseHumanSizeGB(s string) float64 { - s = strings.TrimSpace(s) - if s == "" { - return 0 - } - upper := strings.ToUpper(s) - var mul float64 - var numStr string - switch { - case strings.Contains(upper, " TB"): - mul = 1024 - numStr = strings.TrimSpace(strings.SplitN(upper, " T", 2)[0]) - case strings.Contains(upper, " GB"): - mul = 1 - numStr = strings.TrimSpace(strings.SplitN(upper, " G", 2)[0]) - case strings.Contains(upper, " MB"): - mul = 1.0 / 1024 - numStr = strings.TrimSpace(strings.SplitN(upper, " M", 2)[0]) - default: - return 0 - } - v, err := strconv.ParseFloat(numStr, 64) - if err != nil { - return 0 - } - return v * mul -} - -// --- UI card --- - -func renderRAIDMgmtCard() string { - return `
RAID Controller Management
-
Loading...
-
- -
-` -} diff --git a/audit/internal/webui/saa_dmi.go b/audit/internal/webui/saa_dmi.go index 1342ec1..fc9a62a 100644 --- a/audit/internal/webui/saa_dmi.go +++ b/audit/internal/webui/saa_dmi.go @@ -33,7 +33,6 @@ var ( dmiVersionRE = regexp.MustCompile(`(?i)^version\s*=`) ) - // parseDMIFile parses the DMI.txt produced by "saa GetDmiInfo". // Real format (from SAA User Guide 4.8.1): // @@ -211,4 +210,3 @@ func runSAADMIWriteTask(ctx context.Context, j *jobState, exportDir string, p ta j.append("Done. Reboot the server for changes to take effect.") return nil } - diff --git a/audit/internal/webui/server.go b/audit/internal/webui/server.go index 6674967..87acb3d 100644 --- a/audit/internal/webui/server.go +++ b/audit/internal/webui/server.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "html" "io" "log/slog" "mime" @@ -14,7 +13,6 @@ import ( "os" "path/filepath" "runtime/debug" - "sort" "strings" "sync" "time" @@ -276,6 +274,7 @@ func NewHandler(opts HandlerOptions) http.Handler { mux.HandleFunc("POST /api/sat/memory-stress/run", h.handleAPISATRun("memory-stress")) mux.HandleFunc("POST /api/sat/sat-stress/run", h.handleAPISATRun("sat-stress")) mux.HandleFunc("POST /api/sat/platform-stress/run", h.handleAPISATRun("platform-stress")) + mux.HandleFunc("POST /api/sat/run-all", h.handleAPISATRunAll) mux.HandleFunc("GET /api/sat/stream", h.handleAPISATStream) mux.HandleFunc("POST /api/sat/abort", h.handleAPISATAbort) mux.HandleFunc("POST /api/bee-bench/nvidia/perf/run", h.handleAPIBenchmarkNvidiaRunKind("nvidia-bench-perf")) @@ -619,677 +618,6 @@ func (h *handler) handleViewer(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(body) } -func (h *handler) handleMetricsChartSVG(w http.ResponseWriter, r *http.Request) { - path := strings.TrimPrefix(r.URL.Path, "/api/metrics/chart/") - path = strings.TrimSuffix(path, ".svg") - - if h.metricsDB == nil { - http.Error(w, "metrics database not available", http.StatusServiceUnavailable) - return - } - samples, err := h.metricsDB.LoadAll() - if err != nil || len(samples) == 0 { - http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable) - return - } - timeline := metricsTimelineSegments(samples, time.Now()) - if idx, sub, ok := parseGPUChartPath(path); ok && sub == "overview" { - var overviewOk bool - var buf []byte - buf, overviewOk, err = renderGPUOverviewChartSVG(idx, samples, timeline) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if !overviewOk { - http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable) - return - } - w.Header().Set("Content-Type", "image/svg+xml") - w.Header().Set("Cache-Control", "no-store") - _, _ = w.Write(buf) - return - } - datasets, names, labels, title, yMin, yMax, stacked, ok := chartDataFromSamples(path, samples) - if !ok { - http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable) - return - } - - var buf []byte - if stacked { - buf, err = renderStackedMetricChartSVG( - title, - labels, - sampleTimes(samples), - datasets, - names, - yMax, - chartCanvasHeightForPath(path, len(names)), - timeline, - ) - } else { - buf, err = renderMetricChartSVG( - title, - labels, - sampleTimes(samples), - datasets, - names, - yMin, - yMax, - chartCanvasHeightForPath(path, len(names)), - timeline, - ) - } - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "image/svg+xml") - w.Header().Set("Cache-Control", "no-store") - _, _ = w.Write(buf) -} - -func chartDataFromSamples(path string, samples []platform.LiveMetricSample) (datasets [][]float64, names []string, labels []string, title string, yMin, yMax *float64, stacked bool, ok bool) { - labels = sampleTimeLabels(samples) - - switch { - case path == "server-load": - title = "CPU / Memory Load" - cpu := make([]float64, len(samples)) - mem := make([]float64, len(samples)) - for i, s := range samples { - cpu[i] = s.CPULoadPct - mem[i] = s.MemLoadPct - } - datasets = [][]float64{cpu, mem} - names = []string{"CPU Load %", "Mem Load %"} - yMin = floatPtr(0) - yMax = floatPtr(100) - - case path == "server-temp", path == "server-temp-cpu": - title = "CPU Temperature" - datasets, names = namedTempDatasets(samples, "cpu") - yMin = floatPtr(0) - yMax = autoMax120(datasets...) - - case path == "server-temp-gpu": - title = "GPU Temperature" - datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.TempC }) - yMin = floatPtr(0) - yMax = autoMax120(datasets...) - - case path == "server-temp-ambient": - title = "Ambient / Other Sensors" - datasets, names = namedTempDatasets(samples, "ambient") - yMin = floatPtr(0) - yMax = autoMax120(datasets...) - - case path == "server-power": - title = "System Power" - power := make([]float64, len(samples)) - label := "Power W" - for i, s := range samples { - power[i] = s.PowerW - if strings.TrimSpace(s.PowerSource) != "" { - label = fmt.Sprintf("Power W · %s", s.PowerSource) - if strings.TrimSpace(s.PowerMode) != "" { - label += fmt.Sprintf(" (%s)", s.PowerMode) - } - } - } - power = normalizePowerSeries(power) - datasets = [][]float64{power} - names = []string{label} - yMin = floatPtr(0) - yMax = autoMax120(power) - - case path == "server-fans": - title = "Fan RPM" - datasets, names = namedFanDatasets(samples) - yMin, yMax = autoBounds120(datasets...) - - case path == "gpu-all-load": - title = "GPU Compute Load" - datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.UsagePct }) - yMin = floatPtr(0) - yMax = floatPtr(100) - - case path == "gpu-all-memload": - title = "GPU Memory Load" - datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.MemUsagePct }) - yMin = floatPtr(0) - yMax = floatPtr(100) - - case path == "gpu-all-power": - title = "GPU Power" - datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.PowerW }) - yMin, yMax = autoBounds120(datasets...) - - case path == "gpu-all-temp": - title = "GPU Temperature" - datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.TempC }) - yMin = floatPtr(0) - yMax = autoMax120(datasets...) - - case path == "gpu-all-clock": - title = "GPU Core Clock" - datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.ClockMHz }) - yMin, yMax = autoBounds120(datasets...) - - case path == "gpu-all-memclock": - title = "GPU Memory Clock" - datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.MemClockMHz }) - yMin, yMax = autoBounds120(datasets...) - - case strings.HasPrefix(path, "gpu/"): - idx, sub, ok := parseGPUChartPath(path) - if !ok { - return nil, nil, nil, "", nil, nil, false, false - } - switch sub { - case "load": - title = gpuDisplayLabel(idx) + " Load" - util := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.UsagePct }) - mem := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.MemUsagePct }) - if util == nil && mem == nil { - return nil, nil, nil, "", nil, nil, false, false - } - datasets = [][]float64{coalesceDataset(util, len(samples)), coalesceDataset(mem, len(samples))} - names = []string{"Load %", "Mem %"} - yMin = floatPtr(0) - yMax = floatPtr(100) - case "temp": - title = gpuDisplayLabel(idx) + " Temperature" - temp := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.TempC }) - if temp == nil { - return nil, nil, nil, "", nil, nil, false, false - } - datasets = [][]float64{temp} - names = []string{"Temp °C"} - yMin = floatPtr(0) - yMax = autoMax120(temp) - case "clock": - title = gpuDisplayLabel(idx) + " Core Clock" - clock := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.ClockMHz }) - if clock == nil { - return nil, nil, nil, "", nil, nil, false, false - } - datasets = [][]float64{clock} - names = []string{"Core Clock MHz"} - yMin, yMax = autoBounds120(clock) - case "memclock": - title = gpuDisplayLabel(idx) + " Memory Clock" - clock := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.MemClockMHz }) - if clock == nil { - return nil, nil, nil, "", nil, nil, false, false - } - datasets = [][]float64{clock} - names = []string{"Memory Clock MHz"} - yMin, yMax = autoBounds120(clock) - default: - title = gpuDisplayLabel(idx) + " Power" - power := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.PowerW }) - if power == nil { - return nil, nil, nil, "", nil, nil, false, false - } - datasets = [][]float64{power} - names = []string{"Power W"} - yMin, yMax = autoBounds120(power) - } - - default: - return nil, nil, nil, "", nil, nil, false, false - } - - return datasets, names, labels, title, yMin, yMax, stacked, len(datasets) > 0 -} - -func parseGPUChartPath(path string) (idx int, sub string, ok bool) { - if !strings.HasPrefix(path, "gpu/") { - return 0, "", false - } - rest := strings.TrimPrefix(path, "gpu/") - if rest == "" { - return 0, "", false - } - sub = "" - if i := strings.LastIndex(rest, "-"); i > 0 { - sub = rest[i+1:] - rest = rest[:i] - } - n, err := fmt.Sscanf(rest, "%d", &idx) - if err != nil || n != 1 { - return 0, "", false - } - return idx, sub, true -} - -func sampleTimeLabels(samples []platform.LiveMetricSample) []string { - labels := make([]string, len(samples)) - if len(samples) == 0 { - return labels - } - times := make([]time.Time, len(samples)) - for i, s := range samples { - times[i] = s.Timestamp - } - sameDay := timestampsSameLocalDay(times) - for i, s := range samples { - labels[i] = formatTimelineLabel(s.Timestamp.Local(), sameDay) - } - return labels -} - -func namedTempDatasets(samples []platform.LiveMetricSample, group string) ([][]float64, []string) { - seen := map[string]bool{} - var names []string - for _, s := range samples { - for _, t := range s.Temps { - if t.Group == group && !seen[t.Name] { - seen[t.Name] = true - names = append(names, t.Name) - } - } - } - sort.Strings(names) - datasets := make([][]float64, 0, len(names)) - for _, name := range names { - ds := make([]float64, len(samples)) - for i, s := range samples { - for _, t := range s.Temps { - if t.Group == group && t.Name == name { - ds[i] = t.Celsius - break - } - } - } - datasets = append(datasets, ds) - } - return datasets, names -} - -func namedFanDatasets(samples []platform.LiveMetricSample) ([][]float64, []string) { - seen := map[string]bool{} - var names []string - for _, s := range samples { - for _, f := range s.Fans { - if !seen[f.Name] { - seen[f.Name] = true - names = append(names, f.Name) - } - } - } - sort.Strings(names) - datasets := make([][]float64, 0, len(names)) - for _, name := range names { - ds := make([]float64, len(samples)) - for i, s := range samples { - for _, f := range s.Fans { - if f.Name == name { - ds[i] = f.RPM - break - } - } - } - datasets = append(datasets, normalizeFanSeries(ds)) - } - return datasets, names -} - -func gpuDatasets(samples []platform.LiveMetricSample, pick func(platform.GPUMetricRow) float64) ([][]float64, []string) { - seen := map[int]bool{} - var indices []int - for _, s := range samples { - for _, g := range s.GPUs { - if !seen[g.GPUIndex] { - seen[g.GPUIndex] = true - indices = append(indices, g.GPUIndex) - } - } - } - sort.Ints(indices) - datasets := make([][]float64, 0, len(indices)) - names := make([]string, 0, len(indices)) - for _, idx := range indices { - ds := gpuDatasetByIndex(samples, idx, pick) - if ds == nil { - continue - } - datasets = append(datasets, ds) - names = append(names, gpuDisplayLabel(idx)) - } - return datasets, names -} - -func gpuDatasetByIndex(samples []platform.LiveMetricSample, idx int, pick func(platform.GPUMetricRow) float64) []float64 { - found := false - ds := make([]float64, len(samples)) - for i, s := range samples { - for _, g := range s.GPUs { - if g.GPUIndex == idx { - ds[i] = pick(g) - found = true - break - } - } - } - if !found { - return nil - } - return ds -} - -func coalesceDataset(ds []float64, n int) []float64 { - if ds != nil { - return ds - } - return make([]float64, n) -} - -func normalizePowerSeries(ds []float64) []float64 { - if len(ds) == 0 { - return nil - } - out := make([]float64, len(ds)) - copy(out, ds) - last := 0.0 - haveLast := false - for i, v := range out { - if v > 0 { - last = v - haveLast = true - continue - } - if haveLast { - out[i] = last - } - } - return out -} - -// psuSlotsFromSamples returns the sorted list of PSU slot numbers seen across samples. -func psuSlotsFromSamples(samples []platform.LiveMetricSample) []int { - seen := map[int]struct{}{} - for _, s := range samples { - for _, p := range s.PSUs { - seen[p.Slot] = struct{}{} - } - } - slots := make([]int, 0, len(seen)) - for s := range seen { - slots = append(slots, s) - } - sort.Ints(slots) - return slots -} - -// psuStackedTotal returns the point-by-point sum of all PSU datasets (for scale calculation). -func psuStackedTotal(datasets [][]float64) []float64 { - if len(datasets) == 0 { - return nil - } - n := len(datasets[0]) - total := make([]float64, n) - for _, ds := range datasets { - for i, v := range ds { - total[i] += v - } - } - return total -} - -func normalizeFanSeries(ds []float64) []float64 { - if len(ds) == 0 { - return nil - } - out := make([]float64, len(ds)) - var lastPositive float64 - for i, v := range ds { - if v > 0 { - lastPositive = v - out[i] = v - continue - } - if lastPositive > 0 { - out[i] = lastPositive - continue - } - out[i] = 0 - } - return out -} - -// floatPtr returns a pointer to a float64 value. -func floatPtr(v float64) *float64 { return &v } - -// autoMax120 returns 0→max+20% Y-axis max across all datasets. -func autoMax120(datasets ...[]float64) *float64 { - max := 0.0 - for _, ds := range datasets { - for _, v := range ds { - if v > max { - max = v - } - } - } - if max == 0 { - return nil // let library auto-scale - } - v := max * 1.2 - return &v -} - -func autoBounds120(datasets ...[]float64) (*float64, *float64) { - min := 0.0 - max := 0.0 - first := true - for _, ds := range datasets { - for _, v := range ds { - if first { - min, max = v, v - first = false - continue - } - if v < min { - min = v - } - if v > max { - max = v - } - } - } - if first { - return nil, nil - } - if max <= 0 { - return floatPtr(0), nil - } - span := max - min - if span <= 0 { - span = max * 0.1 - if span <= 0 { - span = 1 - } - } - pad := span * 0.2 - low := min - pad - if low < 0 { - low = 0 - } - high := max + pad - return floatPtr(low), floatPtr(high) -} - -func gpuChartLabelIndices(total, target int) []int { - if total <= 0 { - return nil - } - if total == 1 { - return []int{0} - } - step := total / target - if step < 1 { - step = 1 - } - var indices []int - for i := 0; i < total; i += step { - indices = append(indices, i) - } - if indices[len(indices)-1] != total-1 { - indices = append(indices, total-1) - } - return indices -} - -func chartCanvasHeightForPath(path string, seriesCount int) int { - height := chartCanvasHeight(seriesCount) - if isGPUChartPath(path) { - return height * 2 - } - return height -} - -func isGPUChartPath(path string) bool { - return strings.HasPrefix(path, "gpu-all-") || strings.HasPrefix(path, "gpu/") -} - -func chartLegendVisible(seriesCount int) bool { - return seriesCount <= 8 -} - -func chartCanvasHeight(seriesCount int) int { - if chartLegendVisible(seriesCount) { - return 360 - } - return 288 -} - -// globalStats returns min, average, and max across all values in all datasets. -func globalStats(datasets [][]float64) (mn, avg, mx float64) { - var sum float64 - var count int - first := true - for _, ds := range datasets { - for _, v := range ds { - if first { - mn, mx = v, v - first = false - } - if v < mn { - mn = v - } - if v > mx { - mx = v - } - sum += v - count++ - } - } - if count > 0 { - avg = sum / float64(count) - } - return mn, avg, mx -} - -func sanitizeChartText(s string) string { - if s == "" { - return "" - } - return html.EscapeString(strings.Map(func(r rune) rune { - if r < 0x20 && r != '\t' && r != '\n' && r != '\r' { - return -1 - } - return r - }, s)) -} - -func snapshotNamedRings(rings []*namedMetricsRing) ([][]float64, []string, []string) { - var datasets [][]float64 - var names []string - var labels []string - for _, item := range rings { - if item == nil || item.Ring == nil { - continue - } - vals, l := item.Ring.snapshot() - datasets = append(datasets, vals) - names = append(names, item.Name) - if len(labels) == 0 { - labels = l - } - } - return datasets, names, labels -} - -func snapshotFanRings(rings []*metricsRing, fanNames []string) ([][]float64, []string, []string) { - var datasets [][]float64 - var names []string - var labels []string - for i, ring := range rings { - if ring == nil { - continue - } - vals, l := ring.snapshot() - datasets = append(datasets, normalizeFanSeries(vals)) - name := "Fan" - if i < len(fanNames) { - name = fanNames[i] - } - names = append(names, name+" RPM") - if len(labels) == 0 { - labels = l - } - } - return datasets, names, labels -} - -func chartLegendNumber(v float64) string { - neg := v < 0 - if v < 0 { - v = -v - } - var out string - switch { - case v >= 10000: - out = fmt.Sprintf("%dk", int((v+500)/1000)) - case v >= 1000: - s := fmt.Sprintf("%.2f", v/1000) - s = strings.TrimRight(strings.TrimRight(s, "0"), ".") - out = strings.ReplaceAll(s, ".", ",") + "k" - default: - out = fmt.Sprintf("%.0f", v) - } - if neg { - return "-" + out - } - return out -} - -func chartYAxisNumber(v float64) string { - neg := v < 0 - if neg { - v = -v - } - var out string - switch { - case v >= 10000: - out = fmt.Sprintf("%dк", int((v+500)/1000)) - case v >= 1000: - // Use one decimal place so ticks like 1400, 1600, 1800 read as - // "1,4к", "1,6к", "1,8к" instead of the ambiguous "1к"/"2к". - s := fmt.Sprintf("%.1f", v/1000) - s = strings.TrimRight(strings.TrimRight(s, "0"), ".") - out = strings.ReplaceAll(s, ".", ",") + "к" - default: - out = fmt.Sprintf("%.0f", v) - } - if neg { - return "-" + out - } - return out -} - func (h *handler) handleAPIMetricsExportCSV(w http.ResponseWriter, r *http.Request) { if h.metricsDB == nil { http.Error(w, "metrics database not available", http.StatusServiceUnavailable) diff --git a/audit/internal/webui/server_test.go b/audit/internal/webui/server_test.go index e418250..685db15 100644 --- a/audit/internal/webui/server_test.go +++ b/audit/internal/webui/server_test.go @@ -1145,6 +1145,10 @@ func TestMissingAuditJSONReturnsNotFound(t *testing.T) { func TestSupportBundleEndpointReturnsArchive(t *testing.T) { dir := t.TempDir() + // Isolate os.TempDir(): BuildSupportBundle stages and writes its archive + // there, and a concurrent support-bundle test in another package would + // otherwise race on the same paths. + t.Setenv("TMPDIR", dir) exportDir := filepath.Join(dir, "export") if err := os.MkdirAll(exportDir, 0755); err != nil { t.Fatal(err) diff --git a/audit/internal/webui/task_page.go b/audit/internal/webui/task_page.go index 949bd55..5725d89 100644 --- a/audit/internal/webui/task_page.go +++ b/audit/internal/webui/task_page.go @@ -2,7 +2,6 @@ package webui import ( "encoding/json" - "fmt" "html" "net/http" "os" @@ -225,13 +224,6 @@ func loadTaskReportFragment(task Task) string { return string(data) } -func taskArtifactDownloadLink(task Task, absPath string) string { - if strings.TrimSpace(absPath) == "" { - return "" - } - return fmt.Sprintf(`/export/file?path=%s`, absPath) -} - func (h *handler) taskSamplesForRequest(r *http.Request) (Task, []platform.LiveMetricSample, time.Time, time.Time, bool) { id := r.PathValue("id") taskPtr, ok := globalQueue.findByID(id) diff --git a/audit/internal/webui/task_runner.go b/audit/internal/webui/task_runner.go index 2c11e12..c46ab9a 100644 --- a/audit/internal/webui/task_runner.go +++ b/audit/internal/webui/task_runner.go @@ -224,7 +224,10 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont err = fmt.Errorf("app not configured") break } - archive, err = a.RunNvidiaBandwidthPack(ctx, "", t.params.GPUIndices, j.append) + // Validate: one nvbandwidth pass over all selected GPUs. Stress + // (deep): per-socket passes then an all-GPU pass, so a cross-socket + // P2P fault is isolated from a same-socket one. + archive, err = a.RunNvidiaBandwidthPack(ctx, "", t.params.GPUIndices, t.params.StressMode, j.append) case "nvidia-interconnect": if a == nil { err = fmt.Errorf("app not configured") diff --git a/audit/internal/webui/tasks.go b/audit/internal/webui/tasks.go index f90b112..429cea1 100644 --- a/audit/internal/webui/tasks.go +++ b/audit/internal/webui/tasks.go @@ -2,11 +2,9 @@ package webui import ( "context" - "encoding/json" "fmt" "io" "log/slog" - "net/http" "os" "os/exec" "path/filepath" @@ -892,402 +890,3 @@ func splitNL(s string) []string { } // ── HTTP handlers ───────────────────────────────────────────────────────────── - -func (h *handler) handleAPITasksList(w http.ResponseWriter, _ *http.Request) { - tasks := globalQueue.snapshot() - writeJSON(w, tasks) -} - -func (h *handler) handleAPITasksCancel(w http.ResponseWriter, r *http.Request) { - id := r.PathValue("id") - t, ok := globalQueue.findByID(id) - if !ok { - writeError(w, http.StatusNotFound, "task not found") - return - } - globalQueue.mu.Lock() - defer globalQueue.mu.Unlock() - switch t.Status { - case TaskPending: - t.Status = TaskCancelled - now := time.Now() - t.DoneAt = &now - globalQueue.persistLocked() - taskSerialEvent(t, "finished with status="+t.Status) - writeJSON(w, map[string]string{"status": "cancelled"}) - case TaskRunning: - if t.job == nil || !t.job.abort() { - writeError(w, http.StatusConflict, "task is not cancellable") - return - } - writeJSON(w, map[string]string{"status": "aborting"}) - default: - writeError(w, http.StatusConflict, "task is not running or pending") - } -} - -func (h *handler) handleAPITasksPriority(w http.ResponseWriter, r *http.Request) { - id := r.PathValue("id") - t, ok := globalQueue.findByID(id) - if !ok { - writeError(w, http.StatusNotFound, "task not found") - return - } - var req struct { - Delta int `json:"delta"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid body") - return - } - globalQueue.mu.Lock() - defer globalQueue.mu.Unlock() - if t.Status != TaskPending { - writeError(w, http.StatusConflict, "only pending tasks can be reprioritised") - return - } - t.Priority += req.Delta - globalQueue.persistLocked() - writeJSON(w, map[string]int{"priority": t.Priority}) -} - -func (h *handler) handleAPITasksCancelAll(w http.ResponseWriter, _ *http.Request) { - globalQueue.mu.Lock() - now := time.Now() - n := 0 - for _, t := range globalQueue.tasks { - switch t.Status { - case TaskPending: - t.Status = TaskCancelled - t.DoneAt = &now - taskSerialEvent(t, "finished with status="+t.Status) - n++ - case TaskRunning: - if t.job != nil { - t.job.abort() - } - n++ - } - } - globalQueue.persistLocked() - globalQueue.mu.Unlock() - writeJSON(w, map[string]int{"cancelled": n}) -} - -func (h *handler) handleAPITasksKillWorkers(w http.ResponseWriter, _ *http.Request) { - // Cancel all queued/running tasks in the queue first. - globalQueue.mu.Lock() - now := time.Now() - cancelled := 0 - for _, t := range globalQueue.tasks { - switch t.Status { - case TaskPending: - t.Status = TaskCancelled - t.DoneAt = &now - taskSerialEvent(t, "finished with status="+t.Status) - cancelled++ - case TaskRunning: - if t.job != nil { - t.job.abort() - } - if taskMayLeaveOrphanWorkers(t.Target) { - platform.KillTestWorkers() - } - t.Status = TaskCancelled - t.DoneAt = &now - taskSerialEvent(t, "finished with status="+t.Status) - cancelled++ - } - } - globalQueue.persistLocked() - globalQueue.mu.Unlock() - - // Kill orphaned test worker processes at the OS level. - killed := platform.KillTestWorkers() - writeJSON(w, map[string]any{ - "cancelled": cancelled, - "killed": len(killed), - "processes": killed, - }) -} - -func (h *handler) handleAPITasksStream(w http.ResponseWriter, r *http.Request) { - id := r.PathValue("id") - src, ok := globalQueue.taskStreamSource(id) - if !ok { - http.Error(w, "task not found", http.StatusNotFound) - return - } - if src.job != nil { - streamJob(w, r, src.job) - return - } - if src.status == TaskDone || src.status == TaskFailed || src.status == TaskCancelled { - j := newTaskJobState(src.logPath) - j.finish(src.errMsg) - streamJob(w, r, j) - return - } - if !sseStart(w) { - return - } - sseWrite(w, "", "Task is queued. Waiting for worker...") - ticker := time.NewTicker(200 * time.Millisecond) - defer ticker.Stop() - for { - select { - case <-ticker.C: - src, ok = globalQueue.taskStreamSource(id) - if !ok { - sseWrite(w, "done", "task not found") - return - } - if src.job != nil { - streamSubscribedJob(w, r, src.job) - return - } - if src.status == TaskDone || src.status == TaskFailed || src.status == TaskCancelled { - j := newTaskJobState(src.logPath) - j.finish(src.errMsg) - streamSubscribedJob(w, r, j) - return - } - case <-r.Context().Done(): - return - } - } -} - -func (q *taskQueue) assignTaskLogPathLocked(t *Task) { - if q.logsDir == "" || t.ID == "" { - return - } - q.ensureTaskArtifactPathsLocked(t) -} - -func (q *taskQueue) loadLocked() { - if q.statePath == "" { - return - } - data, err := os.ReadFile(q.statePath) - if err != nil || len(data) == 0 { - return - } - var persisted []persistedTask - if err := json.Unmarshal(data, &persisted); err != nil { - return - } - for _, pt := range persisted { - t := &Task{ - ID: pt.ID, - Name: pt.Name, - Target: pt.Target, - Priority: pt.Priority, - Status: pt.Status, - CreatedAt: pt.CreatedAt, - StartedAt: pt.StartedAt, - DoneAt: pt.DoneAt, - ErrMsg: pt.ErrMsg, - LogPath: pt.LogPath, - ArtifactsDir: pt.ArtifactsDir, - ReportJSONPath: pt.ReportJSONPath, - ReportHTMLPath: pt.ReportHTMLPath, - params: pt.Params, - } - q.assignTaskLogPathLocked(t) - if t.Status == TaskRunning { - state, ok := readTaskRunnerState(t) - switch { - case ok && state.Status == TaskRunning && processAlive(state.PID): - t.runnerPID = state.PID - t.job = newTaskJobState(t.LogPath) - case ok && state.Status != TaskRunning: - t.runnerPID = state.PID - t.Status = state.Status - t.ErrMsg = state.Error - now := state.UpdatedAt - if now.IsZero() { - now = time.Now() - } - t.DoneAt = &now - default: - if taskMayLeaveOrphanWorkers(t.Target) { - _ = platform.KillTestWorkers() - } - now := time.Now() - t.Status = TaskFailed - t.DoneAt = &now - t.ErrMsg = "interrupted by bee-web restart" - } - } else if t.Status == TaskPending { - t.StartedAt = nil - t.DoneAt = nil - t.ErrMsg = "" - } - q.tasks = append(q.tasks, t) - } - q.prune() - q.persistLocked() -} - -func (q *taskQueue) persistLocked() { - if q.statePath == "" { - return - } - state := make([]persistedTask, 0, len(q.tasks)) - for _, t := range q.tasks { - state = append(state, persistedTask{ - ID: t.ID, - Name: t.Name, - Target: t.Target, - Priority: t.Priority, - Status: t.Status, - CreatedAt: t.CreatedAt, - StartedAt: t.StartedAt, - DoneAt: t.DoneAt, - ErrMsg: t.ErrMsg, - LogPath: t.LogPath, - ArtifactsDir: t.ArtifactsDir, - ReportJSONPath: t.ReportJSONPath, - ReportHTMLPath: t.ReportHTMLPath, - Params: t.params, - }) - } - data, err := json.MarshalIndent(state, "", " ") - if err != nil { - return - } - tmp := q.statePath + ".tmp" - if err := os.WriteFile(tmp, data, 0644); err != nil { - return - } - _ = os.Rename(tmp, q.statePath) -} - -func taskElapsedSec(t *Task, now time.Time) int { - if t == nil || t.StartedAt == nil || t.StartedAt.IsZero() { - return 0 - } - start := *t.StartedAt - if !t.CreatedAt.IsZero() && start.Before(t.CreatedAt) { - start = t.CreatedAt - } - end := now - if t.DoneAt != nil && !t.DoneAt.IsZero() { - end = *t.DoneAt - } - if end.Before(start) { - return 0 - } - return int(end.Sub(start).Round(time.Second) / time.Second) -} - -func taskFolderStatus(status string) string { - status = strings.TrimSpace(strings.ToLower(status)) - switch status { - case TaskRunning, TaskDone, TaskFailed, TaskCancelled: - return status - default: - return TaskPending - } -} - -func sanitizeTaskFolderPart(s string) string { - s = strings.TrimSpace(strings.ToLower(s)) - if s == "" { - return "task" - } - var b strings.Builder - lastDash := false - for _, r := range s { - isAlnum := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') - if isAlnum { - b.WriteRune(r) - lastDash = false - continue - } - if !lastDash { - b.WriteByte('-') - lastDash = true - } - } - out := strings.Trim(b.String(), "-") - if out == "" { - return "task" - } - return out -} - -func taskArtifactsDir(root string, t *Task, status string) string { - if strings.TrimSpace(root) == "" || t == nil { - return "" - } - prefix := taskFolderNumberPrefix(t.ID) - return filepath.Join(root, fmt.Sprintf("%s_%s_%s", prefix, sanitizeTaskFolderPart(t.Name), taskFolderStatus(status))) -} - -func taskFolderNumberPrefix(taskID string) string { - taskID = strings.TrimSpace(taskID) - if strings.HasPrefix(taskID, "TASK-") && len(taskID) >= len("TASK-000") { - num := strings.TrimSpace(strings.TrimPrefix(taskID, "TASK-")) - if len(num) == 3 { - allDigits := true - for _, r := range num { - if r < '0' || r > '9' { - allDigits = false - break - } - } - if allDigits { - return num - } - } - } - fallback := sanitizeTaskFolderPart(taskID) - if fallback == "" { - return "000" - } - return fallback -} - -func ensureTaskReportPaths(t *Task) { - if t == nil || strings.TrimSpace(t.ArtifactsDir) == "" { - return - } - if t.LogPath == "" || filepath.Base(t.LogPath) == "task.log" { - t.LogPath = filepath.Join(t.ArtifactsDir, "task.log") - } - t.ReportJSONPath = filepath.Join(t.ArtifactsDir, "report.json") - t.ReportHTMLPath = filepath.Join(t.ArtifactsDir, "report.html") -} - -func (q *taskQueue) ensureTaskArtifactPathsLocked(t *Task) { - if t == nil || strings.TrimSpace(q.logsDir) == "" || strings.TrimSpace(t.ID) == "" { - return - } - if strings.TrimSpace(t.ArtifactsDir) == "" { - t.ArtifactsDir = taskArtifactsDir(q.logsDir, t, t.Status) - } - if t.ArtifactsDir != "" { - _ = os.MkdirAll(t.ArtifactsDir, 0755) - } - ensureTaskReportPaths(t) -} - -func (q *taskQueue) finalizeTaskArtifactPathsLocked(t *Task) { - if t == nil || strings.TrimSpace(q.logsDir) == "" || strings.TrimSpace(t.ID) == "" { - return - } - q.ensureTaskArtifactPathsLocked(t) - dstDir := taskArtifactsDir(q.logsDir, t, t.Status) - if dstDir == "" { - return - } - if t.ArtifactsDir != "" && t.ArtifactsDir != dstDir { - if _, err := os.Stat(dstDir); err != nil { - _ = os.Rename(t.ArtifactsDir, dstDir) - } - t.ArtifactsDir = dstDir - } - ensureTaskReportPaths(t) -} diff --git a/audit/internal/webui/tasks_http_persistence.go b/audit/internal/webui/tasks_http_persistence.go new file mode 100644 index 0000000..df1742f --- /dev/null +++ b/audit/internal/webui/tasks_http_persistence.go @@ -0,0 +1,412 @@ +package webui + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "bee/audit/internal/platform" +) + +func (h *handler) handleAPITasksList(w http.ResponseWriter, _ *http.Request) { + tasks := globalQueue.snapshot() + writeJSON(w, tasks) +} + +func (h *handler) handleAPITasksCancel(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + t, ok := globalQueue.findByID(id) + if !ok { + writeError(w, http.StatusNotFound, "task not found") + return + } + globalQueue.mu.Lock() + defer globalQueue.mu.Unlock() + switch t.Status { + case TaskPending: + t.Status = TaskCancelled + now := time.Now() + t.DoneAt = &now + globalQueue.persistLocked() + taskSerialEvent(t, "finished with status="+t.Status) + writeJSON(w, map[string]string{"status": "cancelled"}) + case TaskRunning: + if t.job == nil || !t.job.abort() { + writeError(w, http.StatusConflict, "task is not cancellable") + return + } + writeJSON(w, map[string]string{"status": "aborting"}) + default: + writeError(w, http.StatusConflict, "task is not running or pending") + } +} + +func (h *handler) handleAPITasksPriority(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + t, ok := globalQueue.findByID(id) + if !ok { + writeError(w, http.StatusNotFound, "task not found") + return + } + var req struct { + Delta int `json:"delta"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid body") + return + } + globalQueue.mu.Lock() + defer globalQueue.mu.Unlock() + if t.Status != TaskPending { + writeError(w, http.StatusConflict, "only pending tasks can be reprioritised") + return + } + t.Priority += req.Delta + globalQueue.persistLocked() + writeJSON(w, map[string]int{"priority": t.Priority}) +} + +func (h *handler) handleAPITasksCancelAll(w http.ResponseWriter, _ *http.Request) { + globalQueue.mu.Lock() + now := time.Now() + n := 0 + for _, t := range globalQueue.tasks { + switch t.Status { + case TaskPending: + t.Status = TaskCancelled + t.DoneAt = &now + taskSerialEvent(t, "finished with status="+t.Status) + n++ + case TaskRunning: + if t.job != nil { + t.job.abort() + } + n++ + } + } + globalQueue.persistLocked() + globalQueue.mu.Unlock() + writeJSON(w, map[string]int{"cancelled": n}) +} + +func (h *handler) handleAPITasksKillWorkers(w http.ResponseWriter, _ *http.Request) { + // Cancel all queued/running tasks in the queue first. + globalQueue.mu.Lock() + now := time.Now() + cancelled := 0 + for _, t := range globalQueue.tasks { + switch t.Status { + case TaskPending: + t.Status = TaskCancelled + t.DoneAt = &now + taskSerialEvent(t, "finished with status="+t.Status) + cancelled++ + case TaskRunning: + if t.job != nil { + t.job.abort() + } + if taskMayLeaveOrphanWorkers(t.Target) { + platform.KillTestWorkers() + } + t.Status = TaskCancelled + t.DoneAt = &now + taskSerialEvent(t, "finished with status="+t.Status) + cancelled++ + } + } + globalQueue.persistLocked() + globalQueue.mu.Unlock() + + // Kill orphaned test worker processes at the OS level. + killed := platform.KillTestWorkers() + writeJSON(w, map[string]any{ + "cancelled": cancelled, + "killed": len(killed), + "processes": killed, + }) +} + +func (h *handler) handleAPITasksStream(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + src, ok := globalQueue.taskStreamSource(id) + if !ok { + http.Error(w, "task not found", http.StatusNotFound) + return + } + if src.job != nil { + streamJob(w, r, src.job) + return + } + if src.status == TaskDone || src.status == TaskFailed || src.status == TaskCancelled { + j := newTaskJobState(src.logPath) + j.finish(src.errMsg) + streamJob(w, r, j) + return + } + if !sseStart(w) { + return + } + sseWrite(w, "", "Task is queued. Waiting for worker...") + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ticker.C: + src, ok = globalQueue.taskStreamSource(id) + if !ok { + sseWrite(w, "done", "task not found") + return + } + if src.job != nil { + streamSubscribedJob(w, r, src.job) + return + } + if src.status == TaskDone || src.status == TaskFailed || src.status == TaskCancelled { + j := newTaskJobState(src.logPath) + j.finish(src.errMsg) + streamSubscribedJob(w, r, j) + return + } + case <-r.Context().Done(): + return + } + } +} + +func (q *taskQueue) assignTaskLogPathLocked(t *Task) { + if q.logsDir == "" || t.ID == "" { + return + } + q.ensureTaskArtifactPathsLocked(t) +} + +func (q *taskQueue) loadLocked() { + if q.statePath == "" { + return + } + data, err := os.ReadFile(q.statePath) + if err != nil || len(data) == 0 { + return + } + var persisted []persistedTask + if err := json.Unmarshal(data, &persisted); err != nil { + return + } + for _, pt := range persisted { + t := &Task{ + ID: pt.ID, + Name: pt.Name, + Target: pt.Target, + Priority: pt.Priority, + Status: pt.Status, + CreatedAt: pt.CreatedAt, + StartedAt: pt.StartedAt, + DoneAt: pt.DoneAt, + ErrMsg: pt.ErrMsg, + LogPath: pt.LogPath, + ArtifactsDir: pt.ArtifactsDir, + ReportJSONPath: pt.ReportJSONPath, + ReportHTMLPath: pt.ReportHTMLPath, + params: pt.Params, + } + q.assignTaskLogPathLocked(t) + if t.Status == TaskRunning { + state, ok := readTaskRunnerState(t) + switch { + case ok && state.Status == TaskRunning && processAlive(state.PID): + t.runnerPID = state.PID + t.job = newTaskJobState(t.LogPath) + case ok && state.Status != TaskRunning: + t.runnerPID = state.PID + t.Status = state.Status + t.ErrMsg = state.Error + now := state.UpdatedAt + if now.IsZero() { + now = time.Now() + } + t.DoneAt = &now + default: + if taskMayLeaveOrphanWorkers(t.Target) { + _ = platform.KillTestWorkers() + } + now := time.Now() + t.Status = TaskFailed + t.DoneAt = &now + t.ErrMsg = "interrupted by bee-web restart" + } + } else if t.Status == TaskPending { + t.StartedAt = nil + t.DoneAt = nil + t.ErrMsg = "" + } + q.tasks = append(q.tasks, t) + } + q.prune() + q.persistLocked() +} + +func (q *taskQueue) persistLocked() { + if q.statePath == "" { + return + } + state := make([]persistedTask, 0, len(q.tasks)) + for _, t := range q.tasks { + state = append(state, persistedTask{ + ID: t.ID, + Name: t.Name, + Target: t.Target, + Priority: t.Priority, + Status: t.Status, + CreatedAt: t.CreatedAt, + StartedAt: t.StartedAt, + DoneAt: t.DoneAt, + ErrMsg: t.ErrMsg, + LogPath: t.LogPath, + ArtifactsDir: t.ArtifactsDir, + ReportJSONPath: t.ReportJSONPath, + ReportHTMLPath: t.ReportHTMLPath, + Params: t.params, + }) + } + data, err := json.MarshalIndent(state, "", " ") + if err != nil { + return + } + tmp := q.statePath + ".tmp" + if err := os.WriteFile(tmp, data, 0644); err != nil { + return + } + _ = os.Rename(tmp, q.statePath) +} + +func taskElapsedSec(t *Task, now time.Time) int { + if t == nil || t.StartedAt == nil || t.StartedAt.IsZero() { + return 0 + } + start := *t.StartedAt + if !t.CreatedAt.IsZero() && start.Before(t.CreatedAt) { + start = t.CreatedAt + } + end := now + if t.DoneAt != nil && !t.DoneAt.IsZero() { + end = *t.DoneAt + } + if end.Before(start) { + return 0 + } + return int(end.Sub(start).Round(time.Second) / time.Second) +} + +func taskFolderStatus(status string) string { + status = strings.TrimSpace(strings.ToLower(status)) + switch status { + case TaskRunning, TaskDone, TaskFailed, TaskCancelled: + return status + default: + return TaskPending + } +} + +func sanitizeTaskFolderPart(s string) string { + s = strings.TrimSpace(strings.ToLower(s)) + if s == "" { + return "task" + } + var b strings.Builder + lastDash := false + for _, r := range s { + isAlnum := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') + if isAlnum { + b.WriteRune(r) + lastDash = false + continue + } + if !lastDash { + b.WriteByte('-') + lastDash = true + } + } + out := strings.Trim(b.String(), "-") + if out == "" { + return "task" + } + return out +} + +func taskArtifactsDir(root string, t *Task, status string) string { + if strings.TrimSpace(root) == "" || t == nil { + return "" + } + prefix := taskFolderNumberPrefix(t.ID) + return filepath.Join(root, fmt.Sprintf("%s_%s_%s", prefix, sanitizeTaskFolderPart(t.Name), taskFolderStatus(status))) +} + +func taskFolderNumberPrefix(taskID string) string { + taskID = strings.TrimSpace(taskID) + if strings.HasPrefix(taskID, "TASK-") && len(taskID) >= len("TASK-000") { + num := strings.TrimSpace(strings.TrimPrefix(taskID, "TASK-")) + if len(num) == 3 { + allDigits := true + for _, r := range num { + if r < '0' || r > '9' { + allDigits = false + break + } + } + if allDigits { + return num + } + } + } + fallback := sanitizeTaskFolderPart(taskID) + if fallback == "" { + return "000" + } + return fallback +} + +func ensureTaskReportPaths(t *Task) { + if t == nil || strings.TrimSpace(t.ArtifactsDir) == "" { + return + } + if t.LogPath == "" || filepath.Base(t.LogPath) == "task.log" { + t.LogPath = filepath.Join(t.ArtifactsDir, "task.log") + } + t.ReportJSONPath = filepath.Join(t.ArtifactsDir, "report.json") + t.ReportHTMLPath = filepath.Join(t.ArtifactsDir, "report.html") +} + +func (q *taskQueue) ensureTaskArtifactPathsLocked(t *Task) { + if t == nil || strings.TrimSpace(q.logsDir) == "" || strings.TrimSpace(t.ID) == "" { + return + } + if strings.TrimSpace(t.ArtifactsDir) == "" { + t.ArtifactsDir = taskArtifactsDir(q.logsDir, t, t.Status) + } + if t.ArtifactsDir != "" { + _ = os.MkdirAll(t.ArtifactsDir, 0755) + } + ensureTaskReportPaths(t) +} + +func (q *taskQueue) finalizeTaskArtifactPathsLocked(t *Task) { + if t == nil || strings.TrimSpace(q.logsDir) == "" || strings.TrimSpace(t.ID) == "" { + return + } + q.ensureTaskArtifactPathsLocked(t) + dstDir := taskArtifactsDir(q.logsDir, t, t.Status) + if dstDir == "" { + return + } + if t.ArtifactsDir != "" && t.ArtifactsDir != dstDir { + if _, err := os.Stat(dstDir); err != nil { + _ = os.Rename(t.ArtifactsDir, dstDir) + } + t.ArtifactsDir = dstDir + } + ensureTaskReportPaths(t) +} diff --git a/audit/internal/webui/tasks_test.go b/audit/internal/webui/tasks_test.go index 8251b26..aa1f292 100644 --- a/audit/internal/webui/tasks_test.go +++ b/audit/internal/webui/tasks_test.go @@ -16,6 +16,19 @@ import ( "bee/audit/internal/platform" ) +type flushNotifyRecorder struct { + *httptest.ResponseRecorder + flushed chan struct{} +} + +func (r *flushNotifyRecorder) Flush() { + r.ResponseRecorder.Flush() + select { + case r.flushed <- struct{}{}: + default: + } +} + func TestTaskQueuePersistsAndRecoversPendingTasks(t *testing.T) { dir := t.TempDir() q := &taskQueue{ @@ -275,7 +288,10 @@ func TestHandleAPITasksStreamPendingTaskStartsSSEImmediately(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) req := httptest.NewRequest(http.MethodGet, "/api/tasks/pending-1/stream", nil).WithContext(ctx) req.SetPathValue("id", "pending-1") - rec := httptest.NewRecorder() + rec := &flushNotifyRecorder{ + ResponseRecorder: httptest.NewRecorder(), + flushed: make(chan struct{}, 1), + } done := make(chan struct{}) go func() { @@ -284,17 +300,18 @@ func TestHandleAPITasksStreamPendingTaskStartsSSEImmediately(t *testing.T) { close(done) }() - deadline := time.Now().Add(2 * time.Second) - for time.Now().Before(deadline) { - if strings.Contains(rec.Body.String(), "Task is queued. Waiting for worker...") { - cancel() - <-done - if rec.Code != http.StatusOK { - t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String()) - } - return + select { + case <-rec.flushed: + cancel() + <-done + if rec.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String()) } - time.Sleep(20 * time.Millisecond) + if !strings.Contains(rec.Body.String(), "Task is queued. Waiting for worker...") { + t.Fatalf("missing queued status, body=%q", rec.Body.String()) + } + return + case <-time.After(2 * time.Second): } cancel() <-done diff --git a/bible-local/architecture/api-surface.md b/bible-local/architecture/api-surface.md index 75626f2..74c8083 100644 --- a/bible-local/architecture/api-surface.md +++ b/bible-local/architecture/api-surface.md @@ -53,6 +53,7 @@ All SAT run endpoints enqueue an async task. Response: `{"task_id": "..."}`. | POST | `/api/sat/memory-stress/run` | Memory stress | | POST | `/api/sat/sat-stress/run` | Combined storage+memory stress | | POST | `/api/sat/platform-stress/run` | Fan + thermal stress | +| POST | `/api/sat/run-all` | Plan + enqueue the whole validate/check set server-side. Body: `{stress_mode, amd_targets[], nvidia_gpu_indices[]}` (operator intent only). Response: `{task_ids[], task_count, notes[]}`. Hardware presence/readiness and which tasks to run are decided by `handler.planSATRunAll`, not the page. | | GET | `/api/sat/stream` | SSE: live SAT log stream | | POST | `/api/sat/abort` | Abort the running SAT task | diff --git a/bible-local/architecture/runtime-flows.md b/bible-local/architecture/runtime-flows.md index 1775c72..e2385db 100644 --- a/bible-local/architecture/runtime-flows.md +++ b/bible-local/architecture/runtime-flows.md @@ -32,6 +32,12 @@ local-fs.target - `bee-network.service` uses `dhclient -nw` (background) — network bring-up is best effort and non-blocking. - `bee-nvidia.service` loads modules via `insmod` with absolute paths — NOT `modprobe`. Reason: the modules are shipped in the ISO overlay under `/usr/local/lib/nvidia/`, not in the host module tree. +- `bee-nvidia-load` refreshes `nvidia-fabricmanager.service` / `nvidia-dcgm.service` + with `systemctl --no-block try-restart` only. DO NOT make it a blocking + `systemctl {start,restart}`: `bee-nvidia.service` is `Type=oneshot` and + `Before=` both units, so a synchronous call deadlocks against that ordering + and previously reached a 60-second wrapper timeout for each unit. See + `decisions/2026-08-31-bee-nvidia-restart-deadlock.md`. - `bee-audit.service` does not wait for `network-online.target`; audit is local and must run even if DHCP is broken. - `bee-audit.service` logs audit failures but does not turn partial collector problems into a boot blocker. - `bee-web.service` binds `0.0.0.0:80` and always renders the current `/var/log/bee-audit.json` contents. @@ -67,7 +73,7 @@ Rules: ``` build-in-container.sh [--authorized-keys /path/to/keys] - 1. compile `bee` binary (skip if .go files older than binary) + 1. compile `bee` binary (always; version/git state is part of the artifact) 2. create a temporary overlay staging dir under `dist/` 3. inject authorized_keys into staged `root/.ssh/` (or set password fallback marker) 4. copy `bee` binary → staged `/usr/local/bin/bee` @@ -93,7 +99,12 @@ build-in-container.sh [--authorized-keys /path/to/keys] 13. patch staged `motd` with build metadata 14. copy `iso/builder/` into a temporary live-build workdir under `dist/` 15. sync staged overlay into workdir `config/includes.chroot/` - 16. run `lb config && lb build` inside the privileged builder container + 16. choose the build path from persisted content/ABI/overlay state: + a. full: run `lb clean --all && lb config && lb build` + b. fast: unpack the last squashfs, sync the staged overlay, repack it, + then rebuild checksums, bootloader assets, ISO, and zsync + 17. validate the final ISO boot menus, volume label, memtest, GRUB assets, + and variant runtime before publishing it ``` Build host notes: @@ -109,7 +120,26 @@ Build host notes: - `bee-gpu-burn` worker must be built against cached CUDA userspace headers from `build-cublas.sh`, not against random host-installed CUDA headers. - The live ISO must ship `libcublas`, `libcublasLt`, and `libcudart` together with `libcuda` so tensor-core stress works without internet or package installs at boot. - The source overlay in `iso/overlay/` is treated as immutable source. Build-time files are injected only into the staged overlay. +- Fast-path state lives outside the rsync-managed live-build workdir and is + accepted only when the heavy-input content hash and resolved kernel ABI + match the last successful full build. A failed full build never leaves a + valid completion marker. The workdir's `binary/` tree is preserved because + it is the source artifact for squashfs reuse. +- Bootloader menu text has two canonical sources only: + `config/bootloaders/grub-efi/grub.cfg` and + `config/bootloaders/isolinux/live.cfg.in`. `lib/bootloader.sh` renders those + templates after both full and fast builds; hooks do not append duplicate + menu entries. +- Build orchestration stays in `build.sh`; ISO validation, bootloader + rendering, fast-path/memtest recovery, and logging helpers live under + `iso/builder/lib/`. Run `iso/builder/test-build-libs.sh` after changing + those helpers or the canonical boot parameters. - ISO filename, squashfs filename, ISO volume label, and the live system's hostname all derive from the same `easy-bee--v` scheme (`ISO_BASENAME`/`SQUASHFS_FILENAME`/`BEE_ISO_VOLUME`/`BEE_HOSTNAME` in `build.sh`) instead of the live-build default (`debian`). Keep new naming derived from `PROJECT_VERSION_EFFECTIVE`/`BUILD_VARIANT` in sync with this set rather than hardcoding a new scheme. +- Every live boot entry carries `udev.children_max=1`, + `intel_iommu=on`, `iommu.passthrough=0`, and + `efi=disable_early_pci_dma`. Only the single failsafe entry additionally + carries `pci=realloc iommu.strict=1`; `iommu=pt` is forbidden. The final-ISO + validator enforces this for both GRUB and isolinux. - The live-build workdir under `dist/` is disposable; source files under `iso/builder/` stay clean. - Container build requires `--privileged` because `live-build` uses mounts/chroots/loop devices during ISO assembly. - On macOS / Docker Desktop, the builder still must run as `linux/amd64` so the shipped ISO binaries remain `amd64`. @@ -170,11 +200,11 @@ Acceptance flows: - Runtime overrides: - `BEE_MEMTESTER_SIZE_MB` - `BEE_MEMTESTER_PASSES` -- NVIDIA Bandwidth SAT (`RunNvidiaBandwidthPack`, `dcgmi diag -r nvbandwidth`) on a - multi-socket system runs per CPU socket first, then all selected GPUs together +- NVIDIA Bandwidth SAT (`RunNvidiaBandwidthPack`, `dcgmi diag -r nvbandwidth`) in + Stress mode runs per resolved PCI NUMA node first, then all selected GPUs together (`03-dcgmi-nvbandwidth-socket0.log`, `...-socket1.log`, `...-all.log`) -- - see `decisions/2026-07-27-nvbandwidth-per-socket-split.md`. Single-socket - systems (or systems where a GPU's NUMA node can't be resolved) keep the + see `decisions/2026-07-27-nvbandwidth-per-socket-split.md`. Single-node + systems (or systems where any GPU's NUMA node cannot be resolved) keep the original single `NN-dcgmi-nvbandwidth.log` shape. ## SAT job output durability @@ -228,18 +258,51 @@ bee-blackbox.service (separate process from bee-web/bee-audit) - DO NOT assume a local write under the live ISO's export directory is durable on its own (RAM-backed overlay) -- blackbox's mirror to removable media is the only real persistence boundary across a hard reset. +- `BuildSupportBundle` stages into a private `os.MkdirTemp` parent, not a + shared `os.TempDir()/bee-support-stage--` path. DO NOT go back to + a time-derived staging path: two builds in the same wall-clock second (two + operators, or an on-demand build racing the blackbox worker) then share one + tree and one's deferred `os.RemoveAll` truncates the other's archive. ## NVIDIA SAT Web UI flow ``` -Web UI: Acceptance Tests page → Run Test button - 1. POST /api/sat/nvidia/run → returns job_id - 2. GET /api/sat/stream?job_id=... (SSE) — streams stdout/stderr lines live - 3. After completion — archive written to /appdata/bee/export/bee-sat/ - summary.txt contains overall_status (OK / FAILED) and per-job status values +Web UI: Acceptance Tests page -> Run Test button + 1. POST /api/sat/nvidia/run -> returns job_id + 2. GET /api/sat/stream?job_id=... (SSE): streams stdout/stderr lines live + 3. After completion: archive written to /appdata/bee/export/bee-sat/ + summary.txt contains overall_status (OK / FAILED / UNSUPPORTED) and per-job status +``` + +## Run All (validate / check) flow + +``` +Web UI: "Run All" button -> POST /api/sat/run-all + body: operator intent only { stress_mode, amd_targets[], nvidia_gpu_indices[] } + server (handler.planSATRunAll): + 1. always: cpu, memory, storage, pcie-link + 2. tpm - only if App.TPMPresent() finds tpm_version_major=2 + 3. nvidia-config - if DetectGPUPresence().Nvidia || NvidiaInitializing + 4. wait for NVIDIA enumeration: repeat fresh ListNvidiaGPUs queries until + at least one GPU is returned, NvidiaGSPMode=="gsp-stuck", or 75s + 5. nvidia / nvidia-interconnect / nvidia-bandwidth / nvidia-pcie-bandwidth + (+ targeted-stress/power/pulse when stress_mode) - only once ready, + -i = App.ListNvidiaGPUs() indices (intersected with the requested subset) + 6. amd / amd-mem / amd-bandwidth - if DetectGPUPresence().AMD and selected + response: { task_ids[], task_count, notes[] } (notes = what was skipped and why) ``` **Critical invariants:** -- `bee-gpu-burn` / `bee-john-gpu-stress` use `exec.CommandContext` — killed on job context cancel. +- Hardware presence, readiness, and which tasks to run are decided server-side. + DO NOT move this back into page JS (`satSelectedGPUIndices().length` gating): + a browser-cached empty GPU list then silently drops every GPU test. See + `decisions/2026-08-31-backend-driven-sat-planning.md`. +- `DetectGPUPresence` is the shared detection source (existing operational + vendor detection plus an lspci display-class fallback). + `/api/gpu/presence`, `/api/gpu/tools` and the planner all use it. +- `bee-gpu-burn` / `bee-john-gpu-stress` use `exec.CommandContext`: killed on job context cancel. - Metric goroutine uses stopCh/doneCh pattern; main goroutine waits `<-doneCh` before reading rows (no mutex needed). - SVG chart is fully offline: no JS, no external CSS, pure inline SVG. +- `RunNvidiaBandwidthPack` runs one all-GPU `nvbandwidth` pass in Validate; the + per-NUMA-node matrix is Stress-tier only (`fullMatrix` arg). See + `decisions/2026-08-31-nvbandwidth-validate-single-deep-matrix.md`. diff --git a/bible-local/decisions/2026-04-01-memtest-build-strategy.md b/bible-local/decisions/2026-04-01-memtest-build-strategy.md index 297c1ca..27c3639 100644 --- a/bible-local/decisions/2026-04-01-memtest-build-strategy.md +++ b/bible-local/decisions/2026-04-01-memtest-build-strategy.md @@ -175,8 +175,8 @@ those files may not exist yet. Instead: - Tries to copy `memtest86+x64.bin` / `memtest86+x64.efi` from `chroot/boot/` first. - Falls back to extracting from the cached `.deb` (via `dpkg-deb -x`) if `chroot/boot/` is empty. -- Appends GRUB and isolinux menu entries only if the respective cfg files already exist at hook time. - If they do not exist, the hook warns and continues (does not fail). +- It does not edit bootloader menus. The complete menus are rendered later + from the two canonical project templates. Controlled by `BEE_REQUIRE_MEMTEST=1` env var to turn warnings into hard errors when needed. @@ -186,7 +186,8 @@ After `lb build` completes, `build.sh` checks whether the fully materialized `bi contains all required memtest artifacts. If not: - Copies/extracts memtest binaries into `binary/boot/`. -- Patches `binary/boot/grub/grub.cfg` and `binary/isolinux/live.cfg` directly. +- Calls `enforce_live_build_bootloader_assets`, which renders the complete + GRUB and isolinux configs from the canonical project templates. - Reruns the late binary stages (`binary_checksums`, `binary_iso`, `binary_zsync`) to rebuild the ISO with the patched tree. diff --git a/bible-local/decisions/2026-07-28-pci-realloc-out-of-default-boot-entries.md b/bible-local/decisions/2026-07-28-pci-realloc-out-of-default-boot-entries.md index 34a76c4..b34d6c2 100644 --- a/bible-local/decisions/2026-07-28-pci-realloc-out-of-default-boot-entries.md +++ b/bible-local/decisions/2026-07-28-pci-realloc-out-of-default-boot-entries.md @@ -59,10 +59,11 @@ individual failure mode. This keeps the total entry count in entries plus wipe/memtest/firmware-settings), rather than letting it grow linearly with every workaround discovered. -The isolinux (BIOS/legacy boot) menu in `iso/builder/config/bootloaders/isolinux/live.cfg.in` -already had no `pci=realloc` entry at all (pre-existing asymmetry with -grub-efi, not introduced by this change) -- not addressed here since no -incident has been observed via legacy boot. +As of v13.0, GRUB and isolinux both expose exactly one troubleshooting entry +with `pci=realloc`. It remains absent from every normal, toram, no-GUI, and +wipe entry. The same v13.0 amendment replaces `iommu=pt` in that entry with +translated strict IOMMU mode; this changes DMA diagnostics, not the scope of +the PCI reallocation workaround. ## Consequences diff --git a/bible-local/decisions/2026-08-31-backend-driven-sat-planning.md b/bible-local/decisions/2026-08-31-backend-driven-sat-planning.md new file mode 100644 index 0000000..1a2c7a7 --- /dev/null +++ b/bible-local/decisions/2026-08-31-backend-driven-sat-planning.md @@ -0,0 +1,67 @@ +# "Run All" SAT planning happens on the backend, not in the browser + +**Date:** 2026-08-31 +**Status:** active + +## Symptom + +Support bundle `210619KUGGXGS2000017` (8x H200 NVL, identical to `...008`): +the operator hit "Run All" and only 6 tasks were queued - cpu, memory, +storage, tpm, nvidia-config, pcie-link. No `nvidia`, `nvidia-interconnect`, +`nvidia-bandwidth`, `nvidia-pcie-bandwidth`. The GPUs were physically present +and `nvidia-config` (which enumerates them) passed. + +## Root cause + +Task planning lived in page JavaScript: + +- `runAllCheckSAT()` / `runAllSAT()` added the NVIDIA targets only + `if (satSelectedGPUIndices().length)`. +- That list came from `/api/gpu/nvidia` (`nvidia-smi --query-gpu=...`), fetched + **once per page load and cached** in `satNvidiaGPUsPromise`. +- If the page first queried while the driver was still enumerating GPUs (the + two 60-second service timeouts described in + [2026-08-31-bee-nvidia-restart-deadlock.md](2026-08-31-bee-nvidia-restart-deadlock.md), + plus per-GPU GSP firmware boot), it got an empty list and cached it for the + whole session. "Run All" then silently dropped every GPU test - no banner, + no error. `/api/gpu/presence` (a separate `os.Stat("/dev/nvidia0")` check) + meanwhile said "GPU present", so the UI even contradicted itself. + +## Decision + +- **New endpoint `POST /api/sat/run-all`.** The browser sends only operator + *intent*: `stress_mode`, `amd_targets` (checkbox selection), and + an optional `nvidia_gpu_indices` subset. The server decides what hardware + is present/ready and what to enqueue (`handler.planSATRunAll`). + `runAllSAT()` / `runAllCheckSAT()` are now thin `fetch` wrappers and render + the `notes[]` the server returns ("TPM: no device - skipped", etc.). +- **GPU planning waits on a readiness gate, not a device probe.** + `planSATRunAll` calls `waitForNvidiaReady`, which repeats the fresh + `ListNvidiaGPUs` query until `nvidia-smi` enumerates at least one GPU, the + runtime snapshot reports `NvidiaGSPMode == "gsp-stuck"`, or the 75-second + deadline expires. A loaded kernel module is not treated as proof that + user-space tools can address a GPU. `CUDAReady == false` after enumeration + is a note, not a blocker. +- **One presence source with a PCI fallback.** `app.DetectGPUPresence`: + - primary - the existing operational vendor result (`DetectGPUVendor`); + - fallback - `System.PhysicalGPUVendors()` (lspci VGA/3D/Display class + + vendor id): a GPU on the bus not reported by operational detection sets + `NvidiaInitializing` / `AMDInitializing`, an explicit state distinct from + "absent". For NVIDIA, the page can show that enumeration is still pending + instead of reporting no hardware. + `/api/gpu/tools`, `/api/gpu/presence` and the run-all planner all go through + it. +- **TPM gate.** `tpm` is planned only when `app.TPMPresent()` finds a sysfs TPM + whose stable `tpm_version_major` attribute is exactly `2`; otherwise a note explains the skip. This + matches the pack-level guard already in `RunTPMValidationPack`. + +## Consequences + +- "Run All" can no longer skip hardware because a browser-cached probe was + early/empty. A genuinely absent or dead GPU still gets skipped - but with a + `notes[]` entry, and `nvidia-config` still runs to capture diagnostics. +- The per-card `disableSATCard('...','No NVIDIA GPU detected')` hints in the + page still use `/api/gpu/presence` - cosmetic only now, and self-heal on + reload. +- Automated / headless callers get correct planning for free by POSTing the + same endpoint instead of replicating the JS logic. diff --git a/bible-local/decisions/2026-08-31-bee-nvidia-restart-deadlock.md b/bible-local/decisions/2026-08-31-bee-nvidia-restart-deadlock.md new file mode 100644 index 0000000..e8a636c --- /dev/null +++ b/bible-local/decisions/2026-08-31-bee-nvidia-restart-deadlock.md @@ -0,0 +1,66 @@ +# bee-nvidia.service: never call blocking `systemctl restart` on units ordered After= itself + +**Date:** 2026-08-31 +**Status:** active + +## Symptom + +Every affected NVIDIA boot reaches two 60-second wrapper timeouts in +`bee-nvidia.service`. `bee-nvidia.log`: + +``` +[bee-nvidia] restarting nvidia-fabricmanager.service (timeout 60s) +[bee-nvidia] WARN: systemctl restart nvidia-fabricmanager.service timed out after 60s +[bee-nvidia] restarting nvidia-dcgm.service (timeout 60s) +[bee-nvidia] WARN: systemctl restart nvidia-dcgm.service timed out after 60s +[bee-nvidia] done +``` + +Seen on both `210619KUGGXGS2000008` and `...017` (8x H200 NVL). That 120 s +window overlaps exactly with when an operator opens the web UI and clicks +"Run All" - during it `nvidia-smi` may not yet enumerate all GPUs, which is +how bundle `...017` ended up running the check set with **no GPU tests** +(see [2026-08-31-backend-driven-sat-planning.md](2026-08-31-backend-driven-sat-planning.md)). + +## Root cause + +`bee-nvidia.service` is `Type=oneshot` and `Before=nvidia-fabricmanager.service +nvidia-dcgm.service`. Its `ExecStart` (`bee-nvidia-load`) then ran, synchronously: + +``` +timeout 60 systemctl restart nvidia-fabricmanager.service +timeout 60 systemctl restart nvidia-dcgm.service +``` + +A oneshot unit is not "active" until `ExecStart` returns. Both target units +are ordered `After=bee-nvidia.service`, so systemd queues them behind +bee-nvidia and will not run them while `bee-nvidia-load` is still executing. +`bee-nvidia-load` blocks on `systemctl restart` waiting for exactly that job +to complete -> deadlock -> broken only when `timeout 60` fires. Twice. + +`nvidia-smi -q` inside `bee-check-nvswitch` (the fabricmanager ExecCondition) +is not part of this ordering cycle; the deadlock is structural. + +## Decision + +`bee-nvidia-load` no longer blocks on those units. It calls +`systemctl --no-block try-restart` for each: + +- `--no-block` returns immediately; systemd runs the job after bee-nvidia + exits, via the existing `Before=` ordering. +- `try-restart` only acts if the unit is already running (the "stale instance + from a reload / re-run" case the old code worried about). If it is inactive, + this command does nothing; an enabled unit already queued by the normal boot + transaction starts after `bee-nvidia.service` via the declared ordering. + +The `SYSTEMCTL_TIMEOUT` / `timeout_systemctl` wrapper and the fallback +`systemctl start` / `systemctl status` branches are gone. `--no-block` means +systemctl does not wait for the queued unit job to finish. + +## Consequences + +- NVIDIA boot no longer waits for those two 60-second wrapper deadlines. +- DO NOT reintroduce a synchronous `systemctl {start,restart}` of any unit + that is `After=bee-nvidia.service` from inside `bee-nvidia-load`. If a unit + genuinely must be up before the script returns, invert the ordering + instead. diff --git a/bible-local/decisions/2026-08-31-nvbandwidth-validate-single-deep-matrix.md b/bible-local/decisions/2026-08-31-nvbandwidth-validate-single-deep-matrix.md new file mode 100644 index 0000000..9111b51 --- /dev/null +++ b/bible-local/decisions/2026-08-31-nvbandwidth-validate-single-deep-matrix.md @@ -0,0 +1,47 @@ +# nvbandwidth: single all-GPU pass in Validate, per-NUMA-node matrix only in the deep tier + +**Date:** 2026-08-31 +**Status:** active +**Amends:** [2026-07-27-nvbandwidth-per-socket-split.md](2026-07-27-nvbandwidth-per-socket-split.md) + +## Context + +`2026-07-27` made `RunNvidiaBandwidthPack` split `dcgmi diag -r nvbandwidth` +using Linux PCI `numa_node` locality, followed by an all-GPU pass. A NUMA node +is not assumed to be identical to a physical CPU socket. + +Two problems showed up on an 8x H200 NVL / dual-socket EPYC box +(`210619KUGGXGS2000008`): + +1. `SATEstimatedNvidiaBandwidthSec` assigns 2700 seconds to one all-GPU pass. + Three such invocations are assigned 8100 seconds, which is outside the + intended **Validate** duration. +2. The split silently never engaged anyway: `normalizeNvidiaBDF` returned + nvidia-smi's upper-case PCI BDF (`0000:CB:00.0`) while `/sys/bus/pci/devices` + entries are lower-case, so `readPCINumaNode` failed for every GPU on a bus + with a hex letter and `gpuBandwidthSocketGroups` fell back to one group. + +## Decision + +- `normalizeNvidiaBDF` now lower-cases (and trims) the BDF, so sysfs + `numa_node` / link-speed reads actually resolve. +- `RunNvidiaBandwidthPack` takes a `fullMatrix bool`. The **only** thing it + changes is which GPU set each `nvbandwidth` invocation gets via `-i` - the + command itself is untouched (no extra flags, no testcase filtering). + - **Validate** (`stress_mode=false`) -> `fullMatrix=false`: one pass, + `-i `. No NUMA-locality split. + - **Stress / deep** (`stress_mode=true`) -> `fullMatrix=true`: the + `2026-07-27` behaviour - one pass per resolved NUMA-node group, then one + all-GPU pass. If any selected GPU has no resolved NUMA node, the code does + not guess a group and falls back to the single all-GPU pass. +- `task_runner` passes `t.params.StressMode` through. + +## Consequences + +- Validate schedules exactly one nvbandwidth invocation regardless of NUMA-node count. +- The NUMA-locality isolation from `2026-07-27` is preserved, just moved to + the tier where a 2-3x runtime is acceptable. DO NOT re-add the split to the + Validate path. +- `SATEstimatedNvidiaBandwidthSec` still reflects a single pass; the deep + tier's multi-pass duration is not modelled - revisit once real + multi-node deep-run logs exist (same open item as `2026-07-27`). diff --git a/bible-local/decisions/README.md b/bible-local/decisions/README.md index 9d1b1d9..ab599a4 100644 --- a/bible-local/decisions/README.md +++ b/bible-local/decisions/README.md @@ -11,3 +11,7 @@ One file per decision, named `YYYY-MM-DD-short-topic.md`. | 2026-07-27 | Split the NVIDIA Bandwidth SAT into per-socket passes before the all-GPU pass | active | | 2026-07-27 | Stream SAT job output live to disk and kick blackbox sync on job completion | active | | 2026-07-28 | Move pci=realloc out of the default/toram/no-GUI GRUB entries | active | +| 2026-08-24 | PCIe Gen1-at-idle GPU warning: load-bearing link check, not idle sysfs | active | +| 2026-08-31 | nvbandwidth: single all-GPU pass in Validate, per-socket matrix only in deep tier | active | +| 2026-08-31 | bee-nvidia.service: never blocking `systemctl restart` on units ordered After= itself | active | +| 2026-08-31 | "Run All" SAT planning happens on the backend, not the browser | active | diff --git a/bible-local/docs/customer-gpu-test-methodology.md b/bible-local/docs/customer-gpu-test-methodology.md index 0c1f160..0ce4081 100644 --- a/bible-local/docs/customer-gpu-test-methodology.md +++ b/bible-local/docs/customer-gpu-test-methodology.md @@ -1,54 +1,67 @@ # GPU PCIe Test Methodology +Which of the checks below run is decided by the backend (`POST +/api/sat/run-all` -> `handler.planSATRunAll`), not the web page: it enumerates +the hardware, waits for the NVIDIA driver to finish enumerating GPUs, and +enqueues only the applicable checks. The TPM check is planned only when sysfs +reports a TPM 2.x device (`tpm_version_major=2`). See +`bible-local/decisions/2026-08-31-backend-driven-sat-planning.md`. + ## Validate - CPU check - - `lscpu` - - `sensors` - - `stress-ng` + - `lscpu` + - `sensors` + - `stress-ng` - Memory check - - `free` - - `timeout memtester` - - `free` + - `free` + - `timeout memtester` + - `free` - NVMe storage check - - `nvme id-ctrl` - - `nvme smart-log` - - `nvme device-self-test` + - `nvme id-ctrl` + - `nvme smart-log` + - `nvme device-self-test` - SATA/SAS storage check - - `smartctl -H -A` - - `smartctl -t short` + - `smartctl -H -A` + - `smartctl -t short` - Basic NVIDIA GPU check - - `nvidia-smi -pm 1` - - `nvidia-smi -q` - - `dmidecode -t baseboard` - - `dmidecode -t system` - - `dcgmi diag -r 2` + - `nvidia-smi -pm 1` + - `nvidia-smi -q` + - `dmidecode -t baseboard` + - `dmidecode -t system` + - `dcgmi diag -r 2` - Inter-GPU communication check - - `all_reduce_perf` + - `all_reduce_perf` - GPU bandwidth check - - `dcgmi diag -r nvbandwidth` + - `dcgmi diag -r nvbandwidth` - **one pass across all GPUs** (`-i `). + No NUMA-locality split in Validate. ## Validate -> Stress - Extended NVIDIA GPU check - - `nvidia-smi -pm 1` - - `nvidia-smi -q` - - `dmidecode -t baseboard` - - `dmidecode -t system` - - `dcgmi diag -r 3` + - `nvidia-smi -pm 1` + - `nvidia-smi -q` + - `dmidecode -t baseboard` + - `dmidecode -t system` + - `dcgmi diag -r 3` - NVIDIA targeted stress - - `nvidia-smi -pm 1` - - `nvidia-smi -q` - - `dcgmi diag -r targeted_stress` + - `nvidia-smi -pm 1` + - `nvidia-smi -q` + - `dcgmi diag -r targeted_stress` - NVIDIA targeted power - - `nvidia-smi -pm 1` - - `nvidia-smi -q` - - `dcgmi diag -r targeted_power` + - `nvidia-smi -pm 1` + - `nvidia-smi -q` + - `dcgmi diag -r targeted_power` - NVIDIA pulse test - - `nvidia-smi -pm 1` - - `nvidia-smi -q` - - `dcgmi diag -r pulse_test` + - `nvidia-smi -pm 1` + - `nvidia-smi -q` + - `dcgmi diag -r pulse_test` - Inter-GPU communication check - - `all_reduce_perf` + - `all_reduce_perf` - GPU bandwidth check - - `dcgmi diag -r nvbandwidth` + - `dcgmi diag -r nvbandwidth` - **per-NUMA-node matrix**: one pass per + completely resolved Linux PCI NUMA group, then one all-GPU pass. If any + selected GPU has no resolved `numa_node`, no group is guessed and the test + falls back to one all-GPU pass. See + `bible-local/decisions/2026-07-27-nvbandwidth-per-socket-split.md` and + `bible-local/decisions/2026-08-31-nvbandwidth-validate-single-deep-matrix.md`. diff --git a/bible-local/docs/iso-build-rules.md b/bible-local/docs/iso-build-rules.md index a24c047..e1d9544 100644 --- a/bible-local/docs/iso-build-rules.md +++ b/bible-local/docs/iso-build-rules.md @@ -17,15 +17,18 @@ This applies to: ## Bootloader sync rule -The ISO has two independent bootloader configs that must be kept in sync manually: +The ISO has two canonical bootloader templates whose live entries must remain +semantically equivalent: | File | Used by | |------|---------| | `config/bootloaders/grub-efi/grub.cfg` | UEFI (all modern servers) | | `config/bootloaders/isolinux/live.cfg.in` | CSM / legacy BIOS (syslinux) | -live-build does NOT derive one from the other. Any new boot entry, kernel parameter -change, or new mode added to one file must be manually mirrored in the other. +live-build does not derive one from the other. `lib/bootloader.sh` renders both +templates into the final `binary/` tree after live-build has created it, and the +ISO validator checks every live entry. Any menu or kernel-parameter change must +still be made in both templates. **Canonical entry list** (both files must have all of these): @@ -33,18 +36,20 @@ change, or new mode added to one file must be manually mirrored in the other. |-------|-----------| | normal (default) | `nomodeset bee.nvidia.mode=normal` + full param set | | load to RAM | `toram nomodeset bee.nvidia.mode=normal` + full param set | -| GSP=off | `nomodeset bee.nvidia.mode=gsp-off` + full param set | -| KMS | no `nomodeset`, `bee.nvidia.mode=normal` + full param set | -| KMS + GSP=off | no `nomodeset`, `bee.nvidia.mode=gsp-off` + full param set | -| fail-safe | `nomodeset bee.nvidia.mode=gsp-off noapic noapm nodma nomce nolapic nosmp` | +| no GUI | `nomodeset bee.gui=off bee.nvidia.mode=normal` + full param set | +| fail-safe | normal parameters plus `pci=realloc iommu.strict=1` | +| wipe | `toram nomodeset bee.gui=off bee.wipe=all` + reduced performance params | **Full standard param set** (append after `@APPEND_LIVE@` / `nomodeset` flags): ``` net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1 -nowatchdog nosoftlockup +nowatchdog nosoftlockup udev.children_max=1 intel_iommu=on +iommu.passthrough=0 efi=disable_early_pci_dma ``` -(fail-safe is the exception — it deliberately uses minimal params.) +The fail-safe entry alone adds `pci=realloc iommu.strict=1`. `pci=realloc` must +not be copied into normal entries. Memtest and firmware-setup entries are not +Linux live entries and therefore do not carry these parameters. **Historical note:** `grub-pc/` was mistakenly used instead of `grub-efi/` until v8.25. live-build reads `config/bootloaders/grub-efi/` for UEFI because the build is diff --git a/iso/builder/bee-gpu-stress-cublaslt.inc b/iso/builder/bee-gpu-stress-cublaslt.inc new file mode 100644 index 0000000..1665c76 --- /dev/null +++ b/iso/builder/bee-gpu-stress-cublaslt.inc @@ -0,0 +1,944 @@ +#if HAVE_CUBLASLT_HEADERS +typedef cublasStatus_t (*cublasLtCreate_fn)(cublasLtHandle_t *); +typedef cublasStatus_t (*cublasLtDestroy_fn)(cublasLtHandle_t); +typedef cublasStatus_t (*cublasLtMatmulDescCreate_fn)(cublasLtMatmulDesc_t *, + cublasComputeType_t, + cudaDataType_t); +typedef cublasStatus_t (*cublasLtMatmulDescDestroy_fn)(cublasLtMatmulDesc_t); +typedef cublasStatus_t (*cublasLtMatmulDescSetAttribute_fn)(cublasLtMatmulDesc_t, + cublasLtMatmulDescAttributes_t, + const void *, + size_t); +typedef cublasStatus_t (*cublasLtMatrixLayoutCreate_fn)(cublasLtMatrixLayout_t *, + cudaDataType_t, + uint64_t, + uint64_t, + int64_t); +typedef cublasStatus_t (*cublasLtMatrixLayoutDestroy_fn)(cublasLtMatrixLayout_t); +typedef cublasStatus_t (*cublasLtMatmulPreferenceCreate_fn)(cublasLtMatmulPreference_t *); +typedef cublasStatus_t (*cublasLtMatmulPreferenceDestroy_fn)(cublasLtMatmulPreference_t); +typedef cublasStatus_t (*cublasLtMatmulPreferenceSetAttribute_fn)(cublasLtMatmulPreference_t, + cublasLtMatmulPreferenceAttributes_t, + const void *, + size_t); +typedef cublasStatus_t (*cublasLtMatmulAlgoGetHeuristic_fn)( + cublasLtHandle_t, + cublasLtMatmulDesc_t, + cublasLtMatrixLayout_t, + cublasLtMatrixLayout_t, + cublasLtMatrixLayout_t, + cublasLtMatrixLayout_t, + cublasLtMatmulPreference_t, + int, + cublasLtMatmulHeuristicResult_t *, + int *); +typedef cublasStatus_t (*cublasLtMatmul_fn)(cublasLtHandle_t, + cublasLtMatmulDesc_t, + const void *, + const void *, + cublasLtMatrixLayout_t, + const void *, + cublasLtMatrixLayout_t, + const void *, + const void *, + cublasLtMatrixLayout_t, + void *, + cublasLtMatrixLayout_t, + const cublasLtMatmulAlgo_t *, + void *, + size_t, + cudaStream_t); + +struct cublaslt_api { + void *lib; + cublasLtCreate_fn cublasLtCreate; + cublasLtDestroy_fn cublasLtDestroy; + cublasLtMatmulDescCreate_fn cublasLtMatmulDescCreate; + cublasLtMatmulDescDestroy_fn cublasLtMatmulDescDestroy; + cublasLtMatmulDescSetAttribute_fn cublasLtMatmulDescSetAttribute; + cublasLtMatrixLayoutCreate_fn cublasLtMatrixLayoutCreate; + cublasLtMatrixLayoutDestroy_fn cublasLtMatrixLayoutDestroy; + cublasLtMatmulPreferenceCreate_fn cublasLtMatmulPreferenceCreate; + cublasLtMatmulPreferenceDestroy_fn cublasLtMatmulPreferenceDestroy; + cublasLtMatmulPreferenceSetAttribute_fn cublasLtMatmulPreferenceSetAttribute; + cublasLtMatmulAlgoGetHeuristic_fn cublasLtMatmulAlgoGetHeuristic; + cublasLtMatmul_fn cublasLtMatmul; +}; + +struct profile_desc { + const char *name; + const char *block_label; + int min_cc; + int enabled; + int needs_scalar_scale; + int needs_block_scale; + int min_multiple; + cudaDataType_t a_type; + cudaDataType_t b_type; + cudaDataType_t c_type; + cudaDataType_t d_type; + cublasComputeType_t compute_type; +}; + +struct prepared_profile { + struct profile_desc desc; + CUstream stream; + cublasLtMatmulDesc_t op_desc; + cublasLtMatrixLayout_t a_layout; + cublasLtMatrixLayout_t b_layout; + cublasLtMatrixLayout_t c_layout; + cublasLtMatrixLayout_t d_layout; + cublasLtMatmulPreference_t preference; + cublasLtMatmulHeuristicResult_t heuristic; + CUdeviceptr a_dev; + CUdeviceptr b_dev; + CUdeviceptr c_dev; + CUdeviceptr d_dev; + CUdeviceptr a_scale_dev; + CUdeviceptr b_scale_dev; + CUdeviceptr workspace_dev; + size_t workspace_size; + uint64_t m; + uint64_t n; + uint64_t k; + unsigned long iterations; + int ready; +}; + +static const struct profile_desc k_profiles[] = { + { + "fp64", + "fp64", + 80, + 1, + 0, + 0, + 8, + CUDA_R_64F, + CUDA_R_64F, + CUDA_R_64F, + CUDA_R_64F, + CUBLAS_COMPUTE_64F, + }, + { + "fp32_tf32", + "fp32", + 80, + 1, + 0, + 0, + 128, + CUDA_R_32F, + CUDA_R_32F, + CUDA_R_32F, + CUDA_R_32F, + CUBLAS_COMPUTE_32F_FAST_TF32, + }, + { + "fp16_tensor", + "fp16", + 80, + 1, + 0, + 0, + 128, + CUDA_R_16F, + CUDA_R_16F, + CUDA_R_16F, + CUDA_R_16F, + CUBLAS_COMPUTE_32F_FAST_16F, + }, + { + "int8_tensor", + "int8", + 75, + 1, + 0, + 0, + 128, + CUDA_R_8I, + CUDA_R_8I, + CUDA_R_32I, + CUDA_R_32I, + CUBLAS_COMPUTE_32I, + }, + { + "fp8_e4m3", + "fp8", + 89, + 1, + 1, + 0, + 128, + CUDA_R_8F_E4M3, + CUDA_R_8F_E4M3, + CUDA_R_16BF, + CUDA_R_16BF, + CUBLAS_COMPUTE_32F, + }, + { + "fp8_e5m2", + "fp8", + 89, + 1, + 1, + 0, + 128, + CUDA_R_8F_E5M2, + CUDA_R_8F_E5M2, + CUDA_R_16BF, + CUDA_R_16BF, + CUBLAS_COMPUTE_32F, + }, +#if defined(CUDA_R_4F_E2M1) && defined(CUBLASLT_MATMUL_MATRIX_SCALE_VEC16_UE4M3) + { + "fp4_e2m1", + "fp4", + 100, + 1, + 0, + 1, + 128, + CUDA_R_4F_E2M1, + CUDA_R_4F_E2M1, + CUDA_R_16BF, + CUDA_R_16BF, + CUBLAS_COMPUTE_32F, + }, +#endif +}; + +#define PROFILE_COUNT ((int)(sizeof(k_profiles) / sizeof(k_profiles[0]))) + +static int profile_allowed_for_run(const struct profile_desc *desc, int cc, const char *precision_filter) { + if (!(desc->enabled && cc >= desc->min_cc)) { + return 0; + } + if (precision_filter != NULL) { + return strcmp(desc->block_label, precision_filter) == 0; + } + /* Mixed/all phases intentionally exclude fp64/fp4 for now: both paths are + * unstable on the current benchmark fleet and can abort the whole mixed + * pass after earlier phases already collected useful telemetry. */ + return strcmp(desc->block_label, "fp64") != 0 && strcmp(desc->block_label, "fp4") != 0; +} + +static int load_cublaslt(struct cublaslt_api *api) { + memset(api, 0, sizeof(*api)); + api->lib = dlopen("libcublasLt.so.13", RTLD_NOW | RTLD_LOCAL); + if (!api->lib) { + api->lib = dlopen("libcublasLt.so", RTLD_NOW | RTLD_LOCAL); + } + if (!api->lib) { + return 0; + } + return + load_symbol(api->lib, "cublasLtCreate", (void **)&api->cublasLtCreate) && + load_symbol(api->lib, "cublasLtDestroy", (void **)&api->cublasLtDestroy) && + load_symbol(api->lib, "cublasLtMatmulDescCreate", (void **)&api->cublasLtMatmulDescCreate) && + load_symbol(api->lib, "cublasLtMatmulDescDestroy", (void **)&api->cublasLtMatmulDescDestroy) && + load_symbol(api->lib, + "cublasLtMatmulDescSetAttribute", + (void **)&api->cublasLtMatmulDescSetAttribute) && + load_symbol(api->lib, "cublasLtMatrixLayoutCreate", (void **)&api->cublasLtMatrixLayoutCreate) && + load_symbol(api->lib, "cublasLtMatrixLayoutDestroy", (void **)&api->cublasLtMatrixLayoutDestroy) && + load_symbol(api->lib, + "cublasLtMatmulPreferenceCreate", + (void **)&api->cublasLtMatmulPreferenceCreate) && + load_symbol(api->lib, + "cublasLtMatmulPreferenceDestroy", + (void **)&api->cublasLtMatmulPreferenceDestroy) && + load_symbol(api->lib, + "cublasLtMatmulPreferenceSetAttribute", + (void **)&api->cublasLtMatmulPreferenceSetAttribute) && + load_symbol(api->lib, + "cublasLtMatmulAlgoGetHeuristic", + (void **)&api->cublasLtMatmulAlgoGetHeuristic) && + load_symbol(api->lib, "cublasLtMatmul", (void **)&api->cublasLtMatmul); +} + +static const char *cublas_status_text(cublasStatus_t status) { + switch (status) { + case CUBLAS_STATUS_SUCCESS: + return "CUBLAS_STATUS_SUCCESS"; + case CUBLAS_STATUS_NOT_INITIALIZED: + return "CUBLAS_STATUS_NOT_INITIALIZED"; + case CUBLAS_STATUS_ALLOC_FAILED: + return "CUBLAS_STATUS_ALLOC_FAILED"; + case CUBLAS_STATUS_INVALID_VALUE: + return "CUBLAS_STATUS_INVALID_VALUE"; + case CUBLAS_STATUS_ARCH_MISMATCH: + return "CUBLAS_STATUS_ARCH_MISMATCH"; + case CUBLAS_STATUS_MAPPING_ERROR: + return "CUBLAS_STATUS_MAPPING_ERROR"; + case CUBLAS_STATUS_EXECUTION_FAILED: + return "CUBLAS_STATUS_EXECUTION_FAILED"; + case CUBLAS_STATUS_INTERNAL_ERROR: + return "CUBLAS_STATUS_INTERNAL_ERROR"; + case CUBLAS_STATUS_NOT_SUPPORTED: + return "CUBLAS_STATUS_NOT_SUPPORTED"; + default: + return "CUBLAS_STATUS_UNKNOWN"; + } +} + +static int check_cublas(const char *step, cublasStatus_t status) { + if (status == CUBLAS_STATUS_SUCCESS) { + return 1; + } + fprintf(stderr, "%s failed: %s (%d)\n", step, cublas_status_text(status), (int)status); + return 0; +} + +static size_t bytes_for_elements(cudaDataType_t type, uint64_t elements) { + switch (type) { + case CUDA_R_32F: + case CUDA_R_32I: + return (size_t)(elements * 4u); + case CUDA_R_16F: + case CUDA_R_16BF: + return (size_t)(elements * 2u); + case CUDA_R_8I: + case CUDA_R_8F_E4M3: + case CUDA_R_8F_E5M2: + return (size_t)(elements); +#if defined(CUDA_R_4F_E2M1) + case CUDA_R_4F_E2M1: + return (size_t)((elements + 1u) / 2u); +#endif + default: + return (size_t)(elements * 4u); + } +} + +static cudaDataType_t matmul_scale_type(const struct profile_desc *desc) { + if (desc->compute_type == CUBLAS_COMPUTE_32I) { + return CUDA_R_32I; + } + if (desc->compute_type == CUBLAS_COMPUTE_64F) { + return CUDA_R_64F; + } + return CUDA_R_32F; +} + +static size_t fp4_scale_bytes(uint64_t rows, uint64_t cols) { + uint64_t row_tiles = (rows + 127u) / 128u; + uint64_t col_tiles = (cols + 63u) / 64u; + return (size_t)(row_tiles * col_tiles * 128u); +} + +static uint64_t choose_square_dim(size_t budget_bytes, size_t bytes_per_cell, int multiple) { + double approx = sqrt((double)budget_bytes / (double)bytes_per_cell); + uint64_t dim = (uint64_t)approx; + if (dim < (uint64_t)multiple) { + dim = (uint64_t)multiple; + } + dim = (uint64_t)round_down_size((size_t)dim, (size_t)multiple); + if (dim < (uint64_t)multiple) { + dim = (uint64_t)multiple; + } + if (dim > 65536u) { + dim = 65536u; + } + return dim; +} + +static int device_upload(struct cuda_api *cuda, CUdeviceptr dev, const void *src, size_t bytes) { + return check_rc(cuda, "cuMemcpyHtoD", cuda->cuMemcpyHtoD(dev, src, bytes)); +} + +static int alloc_filled(struct cuda_api *cuda, CUdeviceptr *ptr, size_t bytes, unsigned char pattern) { + if (!check_rc(cuda, "cuMemAlloc", cuda->cuMemAlloc(ptr, bytes))) { + return 0; + } + if (!check_rc(cuda, "cuMemsetD8", cuda->cuMemsetD8(*ptr, pattern, bytes))) { + cuda->cuMemFree(*ptr); + *ptr = 0; + return 0; + } + return 1; +} + +static size_t profile_scale_bytes(const struct profile_desc *desc, uint64_t m, uint64_t n, uint64_t k) { + size_t bytes = 0; + if (desc->needs_scalar_scale) { + bytes += 2u * sizeof(float); + } +#if defined(CUBLASLT_MATMUL_MATRIX_SCALE_VEC16_UE4M3) + if (desc->needs_block_scale) { + bytes += fp4_scale_bytes(k, m); + bytes += fp4_scale_bytes(k, n); + } +#else + (void)m; + (void)n; + (void)k; +#endif + return bytes; +} + +static void destroy_profile(struct cublaslt_api *cublas, struct cuda_api *cuda, struct prepared_profile *profile) { + if (profile->workspace_dev) { + cuda->cuMemFree(profile->workspace_dev); + } + if (profile->a_scale_dev) { + cuda->cuMemFree(profile->a_scale_dev); + } + if (profile->b_scale_dev) { + cuda->cuMemFree(profile->b_scale_dev); + } + if (profile->d_dev) { + cuda->cuMemFree(profile->d_dev); + } + if (profile->c_dev) { + cuda->cuMemFree(profile->c_dev); + } + if (profile->b_dev) { + cuda->cuMemFree(profile->b_dev); + } + if (profile->a_dev) { + cuda->cuMemFree(profile->a_dev); + } + if (profile->preference) { + cublas->cublasLtMatmulPreferenceDestroy(profile->preference); + } + if (profile->d_layout) { + cublas->cublasLtMatrixLayoutDestroy(profile->d_layout); + } + if (profile->c_layout) { + cublas->cublasLtMatrixLayoutDestroy(profile->c_layout); + } + if (profile->b_layout) { + cublas->cublasLtMatrixLayoutDestroy(profile->b_layout); + } + if (profile->a_layout) { + cublas->cublasLtMatrixLayoutDestroy(profile->a_layout); + } + if (profile->op_desc) { + cublas->cublasLtMatmulDescDestroy(profile->op_desc); + } + memset(profile, 0, sizeof(*profile)); +} + +static int prepare_profile(struct cublaslt_api *cublas, + cublasLtHandle_t handle, + struct cuda_api *cuda, + const struct profile_desc *desc, + CUstream stream, + size_t profile_budget_bytes, + struct prepared_profile *out) { + size_t bytes_per_cell = 0; + size_t attempt_budget = profile_budget_bytes; + + bytes_per_cell += bytes_for_elements(desc->a_type, 1); + bytes_per_cell += bytes_for_elements(desc->b_type, 1); + bytes_per_cell += bytes_for_elements(desc->c_type, 1); + bytes_per_cell += bytes_for_elements(desc->d_type, 1); + if (bytes_per_cell == 0) { + return 0; + } + + while (attempt_budget >= MIN_PROFILE_BUDGET_BYTES) { + memset(out, 0, sizeof(*out)); + out->desc = *desc; + out->stream = stream; + + uint64_t dim = choose_square_dim(attempt_budget, bytes_per_cell, desc->min_multiple); + out->m = dim; + out->n = dim; + out->k = dim; + + size_t desired_workspace = attempt_budget / 8u; + if (desired_workspace > 32u * 1024u * 1024u) { + desired_workspace = 32u * 1024u * 1024u; + } + desired_workspace = round_down_size(desired_workspace, 256u); + + size_t a_bytes = 0; + size_t b_bytes = 0; + size_t c_bytes = 0; + size_t d_bytes = 0; + size_t scale_bytes = 0; + while (1) { + a_bytes = bytes_for_elements(desc->a_type, out->k * out->m); + b_bytes = bytes_for_elements(desc->b_type, out->k * out->n); + c_bytes = bytes_for_elements(desc->c_type, out->m * out->n); + d_bytes = bytes_for_elements(desc->d_type, out->m * out->n); + scale_bytes = profile_scale_bytes(desc, out->m, out->n, out->k); + + size_t matrix_bytes = a_bytes + b_bytes + c_bytes + d_bytes + scale_bytes; + if (matrix_bytes <= attempt_budget) { + size_t remaining = attempt_budget - matrix_bytes; + out->workspace_size = desired_workspace; + if (out->workspace_size > remaining) { + out->workspace_size = round_down_size(remaining, 256u); + } + break; + } + + if (out->m <= (uint64_t)desc->min_multiple) { + break; + } + out->m -= (uint64_t)desc->min_multiple; + out->n = out->m; + out->k = out->m; + } + if (out->m < (uint64_t)desc->min_multiple) { + attempt_budget /= 2u; + continue; + } + + if (!alloc_filled(cuda, &out->a_dev, a_bytes, 0x11) || + !alloc_filled(cuda, &out->b_dev, b_bytes, 0x11) || + !alloc_filled(cuda, &out->c_dev, c_bytes, 0x00) || + !alloc_filled(cuda, &out->d_dev, d_bytes, 0x00)) { + destroy_profile(cublas, cuda, out); + return 0; + } + + cudaDataType_t scale_type = matmul_scale_type(desc); + if (!check_cublas("cublasLtMatmulDescCreate", + cublas->cublasLtMatmulDescCreate(&out->op_desc, desc->compute_type, scale_type))) { + destroy_profile(cublas, cuda, out); + return 0; + } + + cublasOperation_t transa = CUBLAS_OP_T; + cublasOperation_t transb = CUBLAS_OP_N; + if (!check_cublas("set TRANSA", + cublas->cublasLtMatmulDescSetAttribute(out->op_desc, + CUBLASLT_MATMUL_DESC_TRANSA, + &transa, + sizeof(transa))) || + !check_cublas("set TRANSB", + cublas->cublasLtMatmulDescSetAttribute(out->op_desc, + CUBLASLT_MATMUL_DESC_TRANSB, + &transb, + sizeof(transb)))) { + destroy_profile(cublas, cuda, out); + return 0; + } + + if (desc->needs_scalar_scale) { + float one = 1.0f; + if (!alloc_filled(cuda, &out->a_scale_dev, sizeof(one), 0x00) || + !alloc_filled(cuda, &out->b_scale_dev, sizeof(one), 0x00)) { + destroy_profile(cublas, cuda, out); + return 0; + } + if (!device_upload(cuda, out->a_scale_dev, &one, sizeof(one)) || + !device_upload(cuda, out->b_scale_dev, &one, sizeof(one))) { + destroy_profile(cublas, cuda, out); + return 0; + } + void *a_scale_ptr = (void *)(uintptr_t)out->a_scale_dev; + void *b_scale_ptr = (void *)(uintptr_t)out->b_scale_dev; + if (!check_cublas("set A scale ptr", + cublas->cublasLtMatmulDescSetAttribute(out->op_desc, + CUBLASLT_MATMUL_DESC_A_SCALE_POINTER, + &a_scale_ptr, + sizeof(a_scale_ptr))) || + !check_cublas("set B scale ptr", + cublas->cublasLtMatmulDescSetAttribute(out->op_desc, + CUBLASLT_MATMUL_DESC_B_SCALE_POINTER, + &b_scale_ptr, + sizeof(b_scale_ptr)))) { + destroy_profile(cublas, cuda, out); + return 0; + } + } + +#if defined(CUBLASLT_MATMUL_MATRIX_SCALE_VEC16_UE4M3) + if (desc->needs_block_scale) { + size_t a_scale_bytes = fp4_scale_bytes(out->k, out->m); + size_t b_scale_bytes = fp4_scale_bytes(out->k, out->n); + if (!alloc_filled(cuda, &out->a_scale_dev, a_scale_bytes, 0x11) || + !alloc_filled(cuda, &out->b_scale_dev, b_scale_bytes, 0x11)) { + destroy_profile(cublas, cuda, out); + return 0; + } + cublasLtMatmulMatrixScale_t scale_mode = CUBLASLT_MATMUL_MATRIX_SCALE_VEC16_UE4M3; + void *a_scale_ptr = (void *)(uintptr_t)out->a_scale_dev; + void *b_scale_ptr = (void *)(uintptr_t)out->b_scale_dev; + if (!check_cublas("set A scale mode", + cublas->cublasLtMatmulDescSetAttribute(out->op_desc, + CUBLASLT_MATMUL_DESC_A_SCALE_MODE, + &scale_mode, + sizeof(scale_mode))) || + !check_cublas("set B scale mode", + cublas->cublasLtMatmulDescSetAttribute(out->op_desc, + CUBLASLT_MATMUL_DESC_B_SCALE_MODE, + &scale_mode, + sizeof(scale_mode))) || + !check_cublas("set A block scale ptr", + cublas->cublasLtMatmulDescSetAttribute(out->op_desc, + CUBLASLT_MATMUL_DESC_A_SCALE_POINTER, + &a_scale_ptr, + sizeof(a_scale_ptr))) || + !check_cublas("set B block scale ptr", + cublas->cublasLtMatmulDescSetAttribute(out->op_desc, + CUBLASLT_MATMUL_DESC_B_SCALE_POINTER, + &b_scale_ptr, + sizeof(b_scale_ptr)))) { + destroy_profile(cublas, cuda, out); + return 0; + } + } +#endif + + if (!check_cublas("create A layout", + cublas->cublasLtMatrixLayoutCreate(&out->a_layout, desc->a_type, out->k, out->m, out->k)) || + !check_cublas("create B layout", + cublas->cublasLtMatrixLayoutCreate(&out->b_layout, desc->b_type, out->k, out->n, out->k)) || + !check_cublas("create C layout", + cublas->cublasLtMatrixLayoutCreate(&out->c_layout, desc->c_type, out->m, out->n, out->m)) || + !check_cublas("create D layout", + cublas->cublasLtMatrixLayoutCreate(&out->d_layout, desc->d_type, out->m, out->n, out->m))) { + destroy_profile(cublas, cuda, out); + return 0; + } + + if (!check_cublas("create preference", cublas->cublasLtMatmulPreferenceCreate(&out->preference))) { + destroy_profile(cublas, cuda, out); + return 0; + } + + if (out->workspace_size > 0) { + if (!alloc_filled(cuda, &out->workspace_dev, out->workspace_size, 0x00)) { + destroy_profile(cublas, cuda, out); + return 0; + } + } + + if (!check_cublas("set workspace", + cublas->cublasLtMatmulPreferenceSetAttribute( + out->preference, + CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, + &out->workspace_size, + sizeof(out->workspace_size)))) { + destroy_profile(cublas, cuda, out); + return 0; + } + + int found = 0; + if (check_cublas("heuristic", + cublas->cublasLtMatmulAlgoGetHeuristic(handle, + out->op_desc, + out->a_layout, + out->b_layout, + out->c_layout, + out->d_layout, + out->preference, + 1, + &out->heuristic, + &found)) && + found > 0) { + out->ready = 1; + return 1; + } + + destroy_profile(cublas, cuda, out); + attempt_budget = round_down_size(attempt_budget * 3u / 4u, 256u); + if (attempt_budget < MIN_PROFILE_BUDGET_BYTES) { + break; + } + } + + return 0; +} + +static int run_cublas_profile(cublasLtHandle_t handle, + struct cublaslt_api *cublas, + struct prepared_profile *profile) { + int32_t alpha_i32 = 1; + int32_t beta_i32 = 0; + double alpha_f64 = 1.0; + double beta_f64 = 0.0; + float alpha = 1.0f; + float beta = 0.0f; + const void *alpha_ptr = α + const void *beta_ptr = β + if (profile->desc.compute_type == CUBLAS_COMPUTE_32I) { + alpha_ptr = &alpha_i32; + beta_ptr = &beta_i32; + } else if (profile->desc.compute_type == CUBLAS_COMPUTE_64F) { + alpha_ptr = &alpha_f64; + beta_ptr = &beta_f64; + } + return check_cublas(profile->desc.name, + cublas->cublasLtMatmul(handle, + profile->op_desc, + alpha_ptr, + (const void *)(uintptr_t)profile->a_dev, + profile->a_layout, + (const void *)(uintptr_t)profile->b_dev, + profile->b_layout, + beta_ptr, + (const void *)(uintptr_t)profile->c_dev, + profile->c_layout, + (void *)(uintptr_t)profile->d_dev, + profile->d_layout, + &profile->heuristic.algo, + (void *)(uintptr_t)profile->workspace_dev, + profile->workspace_size, + profile->stream)); +} + +static int run_cublaslt_stress(struct cuda_api *cuda, + CUdevice dev, + const char *device_name, + int cc_major, + int cc_minor, + int seconds, + int size_mb, + const char *precision_filter, + struct stress_report *report) { + struct cublaslt_api cublas; + struct prepared_profile prepared[MAX_STRESS_STREAMS * PROFILE_COUNT]; + cublasLtHandle_t handle = NULL; + CUcontext ctx = NULL; + CUstream streams[MAX_STRESS_STREAMS] = {0}; + uint16_t sample[256]; + int cc = cc_major * 10 + cc_minor; + int planned = 0; + int active = 0; + int mp_count = 0; + int stream_count = 1; + int profile_count = PROFILE_COUNT; + int prepared_count = 0; + size_t requested_budget = 0; + size_t total_budget = 0; + size_t per_profile_budget = 0; + int budget_profiles = 0; + + memset(report, 0, sizeof(*report)); + snprintf(report->backend, sizeof(report->backend), "cublasLt"); + snprintf(report->device, sizeof(report->device), "%s", device_name); + report->cc_major = cc_major; + report->cc_minor = cc_minor; + report->buffer_mb = size_mb; + + if (!load_cublaslt(&cublas)) { + snprintf(report->details, sizeof(report->details), "cublasLt=unavailable\n"); + return 0; + } + if (!check_rc(cuda, "cuCtxCreate", cuda->cuCtxCreate(&ctx, 0, dev))) { + return 0; + } + if (!check_cublas("cublasLtCreate", cublas.cublasLtCreate(&handle))) { + cuda->cuCtxDestroy(ctx); + return 0; + } + + /* Count profiles matching the filter (for deciding what to run). */ + for (size_t i = 0; i < sizeof(k_profiles) / sizeof(k_profiles[0]); i++) { + if (profile_allowed_for_run(&k_profiles[i], cc, precision_filter)) { + planned++; + } + } + if (planned <= 0) { + snprintf(report->details, sizeof(report->details), "cublasLt_profiles=unsupported\n"); + cublas.cublasLtDestroy(handle); + cuda->cuCtxDestroy(ctx); + return 0; + } + + /* Count all profiles active on this GPU regardless of filter. + * Mixed phases still divide budget across the full precision set, while + * single-precision benchmark phases dedicate budget only to active + * profiles matching precision_filter. */ + int planned_total = 0; + for (size_t i = 0; i < sizeof(k_profiles) / sizeof(k_profiles[0]); i++) { + if (profile_allowed_for_run(&k_profiles[i], cc, precision_filter)) { + planned_total++; + } + } + if (planned_total < planned) { + planned_total = planned; + } + budget_profiles = planned_total; + if (precision_filter != NULL) { + budget_profiles = planned; + } + if (budget_profiles <= 0) { + budget_profiles = planned_total; + } + + requested_budget = (size_t)size_mb * 1024u * 1024u; + if (requested_budget < (size_t)budget_profiles * MIN_PROFILE_BUDGET_BYTES) { + requested_budget = (size_t)budget_profiles * MIN_PROFILE_BUDGET_BYTES; + } + total_budget = clamp_budget_to_free_memory(cuda, requested_budget); + if (total_budget < (size_t)budget_profiles * MIN_PROFILE_BUDGET_BYTES) { + total_budget = (size_t)budget_profiles * MIN_PROFILE_BUDGET_BYTES; + } + if (query_multiprocessor_count(cuda, dev, &mp_count) && + cuda->cuStreamCreate && + cuda->cuStreamDestroy) { + stream_count = choose_stream_count(mp_count, budget_profiles, total_budget, 1); + } + if (precision_filter != NULL && stream_count > MAX_SINGLE_PRECISION_STREAMS) { + stream_count = MAX_SINGLE_PRECISION_STREAMS; + } + if (stream_count > 1) { + int created = 0; + for (; created < stream_count; created++) { + if (!check_rc(cuda, "cuStreamCreate", cuda->cuStreamCreate(&streams[created], 0))) { + destroy_streams(cuda, streams, created); + stream_count = 1; + break; + } + } + } + report->stream_count = stream_count; + per_profile_budget = total_budget / ((size_t)budget_profiles * (size_t)stream_count); + if (per_profile_budget < MIN_PROFILE_BUDGET_BYTES) { + per_profile_budget = MIN_PROFILE_BUDGET_BYTES; + } + if (precision_filter != NULL) { + per_profile_budget = clamp_single_precision_profile_budget(per_profile_budget); + } + report->buffer_mb = (int)(total_budget / (1024u * 1024u)); + append_detail(report->details, + sizeof(report->details), + "requested_mb=%d actual_mb=%d streams=%d mp_count=%d budget_profiles=%d per_worker_mb=%zu\n", + size_mb, + report->buffer_mb, + report->stream_count, + mp_count, + budget_profiles, + per_profile_budget / (1024u * 1024u)); + + for (int i = 0; i < profile_count; i++) { + const struct profile_desc *desc = &k_profiles[i]; + if (!(desc->enabled && cc >= desc->min_cc)) { + append_detail(report->details, + sizeof(report->details), + "%s=SKIPPED cc<%d\n", + desc->name, + desc->min_cc); + continue; + } + if (!profile_allowed_for_run(desc, cc, precision_filter)) { + append_detail(report->details, + sizeof(report->details), + "%s=SKIPPED benchmark_disabled\n", + desc->name); + continue; + } + for (int lane = 0; lane < stream_count; lane++) { + CUstream stream = streams[lane]; + if (prepared_count >= (int)(sizeof(prepared) / sizeof(prepared[0]))) { + break; + } + if (prepare_profile(&cublas, handle, cuda, desc, stream, per_profile_budget, &prepared[prepared_count])) { + active++; + append_detail(report->details, + sizeof(report->details), + "%s[%d]=READY dim=%llux%llux%llu block=%s stream=%d\n", + desc->name, + lane, + (unsigned long long)prepared[prepared_count].m, + (unsigned long long)prepared[prepared_count].n, + (unsigned long long)prepared[prepared_count].k, + desc->block_label, + lane); + prepared_count++; + } else { + append_detail(report->details, + sizeof(report->details), + "%s[%d]=SKIPPED unsupported\n", + desc->name, + lane); + } + } + } + + if (active <= 0) { + cublas.cublasLtDestroy(handle); + destroy_streams(cuda, streams, stream_count); + cuda->cuCtxDestroy(ctx); + return 0; + } + + /* Keep the GPU queue continuously full by submitting kernels without + * synchronizing after every wave. A sync barrier after each small batch + * creates CPU-to-GPU ping-pong gaps that prevent full TDP utilisation, + * especially when individual kernels are short. Instead we sync at most + * once per second (for error detection) and once at the very end. */ + double deadline = now_seconds() + (double)seconds; + double next_sync = now_seconds() + 1.0; + while (now_seconds() < deadline) { + int launched = 0; + for (int i = 0; i < prepared_count; i++) { + if (!prepared[i].ready) { + continue; + } + if (!run_cublas_profile(handle, &cublas, &prepared[i])) { + append_detail(report->details, + sizeof(report->details), + "%s=FAILED runtime\n", + prepared[i].desc.name); + for (int j = 0; j < prepared_count; j++) { + destroy_profile(&cublas, cuda, &prepared[j]); + } + cublas.cublasLtDestroy(handle); + destroy_streams(cuda, streams, stream_count); + cuda->cuCtxDestroy(ctx); + return 0; + } + prepared[i].iterations++; + report->iterations++; + launched++; + } + if (launched <= 0) { + break; + } + double now = now_seconds(); + if (now >= next_sync || now >= deadline) { + if (!check_rc(cuda, "cuCtxSynchronize", cuda->cuCtxSynchronize())) { + for (int i = 0; i < prepared_count; i++) { + destroy_profile(&cublas, cuda, &prepared[i]); + } + cublas.cublasLtDestroy(handle); + destroy_streams(cuda, streams, stream_count); + cuda->cuCtxDestroy(ctx); + return 0; + } + next_sync = now + 1.0; + } + } + /* Final drain: ensure all queued work finishes before we read results. */ + cuda->cuCtxSynchronize(); + + for (int i = 0; i < prepared_count; i++) { + if (!prepared[i].ready) { + continue; + } + append_detail(report->details, + sizeof(report->details), + "%s_iterations=%lu\n", + prepared[i].desc.name, + prepared[i].iterations); + } + + for (int i = 0; i < prepared_count; i++) { + if (prepared[i].ready) { + if (check_rc(cuda, "cuMemcpyDtoH", cuda->cuMemcpyDtoH(sample, prepared[i].d_dev, sizeof(sample)))) { + for (size_t j = 0; j < sizeof(sample) / sizeof(sample[0]); j++) { + report->checksum += sample[j]; + } + } + break; + } + } + + for (int i = 0; i < prepared_count; i++) { + destroy_profile(&cublas, cuda, &prepared[i]); + } + cublas.cublasLtDestroy(handle); + destroy_streams(cuda, streams, stream_count); + cuda->cuCtxDestroy(ctx); + return 1; +} +#endif diff --git a/iso/builder/bee-gpu-stress-cuda.inc b/iso/builder/bee-gpu-stress-cuda.inc new file mode 100644 index 0000000..7db2be6 --- /dev/null +++ b/iso/builder/bee-gpu-stress-cuda.inc @@ -0,0 +1,356 @@ +static int load_symbol(void *lib, const char *name, void **out) { + *out = dlsym(lib, name); + return *out != NULL; +} + +static int load_cuda(struct cuda_api *api) { + memset(api, 0, sizeof(*api)); + api->lib = dlopen("libcuda.so.1", RTLD_NOW | RTLD_LOCAL); + if (!api->lib) { + return 0; + } + if (!( + load_symbol(api->lib, "cuInit", (void **)&api->cuInit) && + load_symbol(api->lib, "cuDeviceGetCount", (void **)&api->cuDeviceGetCount) && + load_symbol(api->lib, "cuDeviceGet", (void **)&api->cuDeviceGet) && + load_symbol(api->lib, "cuDeviceGetName", (void **)&api->cuDeviceGetName) && + load_symbol(api->lib, "cuDeviceGetAttribute", (void **)&api->cuDeviceGetAttribute) && + load_symbol(api->lib, "cuCtxCreate_v2", (void **)&api->cuCtxCreate) && + load_symbol(api->lib, "cuCtxDestroy_v2", (void **)&api->cuCtxDestroy) && + load_symbol(api->lib, "cuCtxSynchronize", (void **)&api->cuCtxSynchronize) && + load_symbol(api->lib, "cuMemAlloc_v2", (void **)&api->cuMemAlloc) && + load_symbol(api->lib, "cuMemFree_v2", (void **)&api->cuMemFree) && + load_symbol(api->lib, "cuMemsetD8_v2", (void **)&api->cuMemsetD8) && + load_symbol(api->lib, "cuMemcpyHtoD_v2", (void **)&api->cuMemcpyHtoD) && + load_symbol(api->lib, "cuMemcpyDtoH_v2", (void **)&api->cuMemcpyDtoH) && + load_symbol(api->lib, "cuModuleLoadDataEx", (void **)&api->cuModuleLoadDataEx) && + load_symbol(api->lib, "cuModuleGetFunction", (void **)&api->cuModuleGetFunction) && + load_symbol(api->lib, "cuLaunchKernel", (void **)&api->cuLaunchKernel))) { + dlclose(api->lib); + memset(api, 0, sizeof(*api)); + return 0; + } + load_symbol(api->lib, "cuMemGetInfo_v2", (void **)&api->cuMemGetInfo); + load_symbol(api->lib, "cuStreamCreate", (void **)&api->cuStreamCreate); + if (!load_symbol(api->lib, "cuStreamDestroy_v2", (void **)&api->cuStreamDestroy)) { + load_symbol(api->lib, "cuStreamDestroy", (void **)&api->cuStreamDestroy); + } + return 1; +} + +static const char *cu_error_name(struct cuda_api *api, CUresult rc) { + const char *value = NULL; + if (api->cuGetErrorName && api->cuGetErrorName(rc, &value) == CU_SUCCESS && value) { + return value; + } + return "CUDA_ERROR"; +} + +static const char *cu_error_string(struct cuda_api *api, CUresult rc) { + const char *value = NULL; + if (api->cuGetErrorString && api->cuGetErrorString(rc, &value) == CU_SUCCESS && value) { + return value; + } + return "unknown"; +} + +static int check_rc(struct cuda_api *api, const char *step, CUresult rc) { + if (rc == CU_SUCCESS) { + return 1; + } + fprintf(stderr, "%s failed: %s (%s)\n", step, cu_error_name(api, rc), cu_error_string(api, rc)); + return 0; +} + +static double now_seconds(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + ((double)ts.tv_nsec / 1000000000.0); +} + +static size_t round_down_size(size_t value, size_t multiple) { + if (multiple == 0 || value < multiple) { + return value; + } + return value - (value % multiple); +} + +static int query_compute_capability(struct cuda_api *api, CUdevice dev, int *major, int *minor) { + int cc_major = 0; + int cc_minor = 0; + if (!check_rc(api, + "cuDeviceGetAttribute(major)", + api->cuDeviceGetAttribute(&cc_major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, dev))) { + return 0; + } + if (!check_rc(api, + "cuDeviceGetAttribute(minor)", + api->cuDeviceGetAttribute(&cc_minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, dev))) { + return 0; + } + *major = cc_major; + *minor = cc_minor; + return 1; +} + +static int query_multiprocessor_count(struct cuda_api *api, CUdevice dev, int *count) { + int mp_count = 0; + if (!check_rc(api, + "cuDeviceGetAttribute(multiprocessors)", + api->cuDeviceGetAttribute(&mp_count, CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, dev))) { + return 0; + } + *count = mp_count; + return 1; +} + +static size_t clamp_budget_to_free_memory(struct cuda_api *api, size_t requested_bytes) { + size_t free_bytes = 0; + size_t total_bytes = 0; + size_t max_bytes = requested_bytes; + + if (!api->cuMemGetInfo) { + return requested_bytes; + } + if (api->cuMemGetInfo(&free_bytes, &total_bytes) != CU_SUCCESS || free_bytes == 0) { + return requested_bytes; + } + + max_bytes = (free_bytes * 9u) / 10u; + if (max_bytes < (size_t)4u * 1024u * 1024u) { + max_bytes = (size_t)4u * 1024u * 1024u; + } + if (requested_bytes > max_bytes) { + return max_bytes; + } + return requested_bytes; +} + +static int choose_stream_count(int mp_count, int planned_profiles, size_t total_budget, int have_streams) { + int stream_count = 1; + if (!have_streams || mp_count <= 0 || planned_profiles <= 0) { + return 1; + } + + stream_count = mp_count / 8; + if (stream_count < 2) { + stream_count = 2; + } + if (stream_count > MAX_STRESS_STREAMS) { + stream_count = MAX_STRESS_STREAMS; + } + + while (stream_count > 1) { + size_t per_stream_budget = total_budget / ((size_t)planned_profiles * (size_t)stream_count); + if (per_stream_budget >= MIN_STREAM_BUDGET_BYTES) { + break; + } + stream_count--; + } + return stream_count; +} + +#if HAVE_CUBLASLT_HEADERS +static size_t clamp_single_precision_profile_budget(size_t profile_budget_bytes) { + if (profile_budget_bytes > MAX_SINGLE_PRECISION_PROFILE_BUDGET_BYTES) { + return MAX_SINGLE_PRECISION_PROFILE_BUDGET_BYTES; + } + return profile_budget_bytes; +} +#endif + +static void destroy_streams(struct cuda_api *api, CUstream *streams, int count) { + if (!api->cuStreamDestroy) { + return; + } + for (int i = 0; i < count; i++) { + if (streams[i]) { + api->cuStreamDestroy(streams[i]); + streams[i] = NULL; + } + } +} + +#if HAVE_CUBLASLT_HEADERS +static void append_detail(char *buf, size_t cap, const char *fmt, ...) { + size_t len = strlen(buf); + if (len >= cap) { + return; + } + va_list ap; + va_start(ap, fmt); + vsnprintf(buf + len, cap - len, fmt, ap); + va_end(ap); +} +#endif + +static int run_ptx_fallback(struct cuda_api *api, + CUdevice dev, + const char *device_name, + int cc_major, + int cc_minor, + int seconds, + int size_mb, + struct stress_report *report) { + CUcontext ctx = NULL; + CUmodule module = NULL; + CUfunction kernel = NULL; + uint32_t sample[256]; + CUdeviceptr device_mem[MAX_STRESS_STREAMS] = {0}; + CUstream streams[MAX_STRESS_STREAMS] = {0}; + uint32_t words[MAX_STRESS_STREAMS] = {0}; + uint32_t rounds[MAX_STRESS_STREAMS] = {0}; + void *params[MAX_STRESS_STREAMS][3]; + size_t bytes_per_stream[MAX_STRESS_STREAMS] = {0}; + unsigned long iterations = 0; + int mp_count = 0; + int stream_count = 1; + + memset(report, 0, sizeof(*report)); + snprintf(report->backend, sizeof(report->backend), "driver-ptx"); + snprintf(report->device, sizeof(report->device), "%s", device_name); + report->cc_major = cc_major; + report->cc_minor = cc_minor; + report->buffer_mb = size_mb; + + if (!check_rc(api, "cuCtxCreate", api->cuCtxCreate(&ctx, 0, dev))) { + return 0; + } + + size_t requested_bytes = (size_t)size_mb * 1024u * 1024u; + if (requested_bytes < MIN_PROFILE_BUDGET_BYTES) { + requested_bytes = MIN_PROFILE_BUDGET_BYTES; + } + size_t total_bytes = clamp_budget_to_free_memory(api, requested_bytes); + if (total_bytes < MIN_PROFILE_BUDGET_BYTES) { + total_bytes = MIN_PROFILE_BUDGET_BYTES; + } + report->buffer_mb = (int)(total_bytes / (1024u * 1024u)); + + if (query_multiprocessor_count(api, dev, &mp_count) && + api->cuStreamCreate && + api->cuStreamDestroy) { + stream_count = choose_stream_count(mp_count, 1, total_bytes, 1); + } + if (stream_count > 1) { + int created = 0; + for (; created < stream_count; created++) { + if (!check_rc(api, "cuStreamCreate", api->cuStreamCreate(&streams[created], 0))) { + destroy_streams(api, streams, created); + stream_count = 1; + break; + } + } + } + report->stream_count = stream_count; + + for (int lane = 0; lane < stream_count; lane++) { + size_t slice = total_bytes / (size_t)stream_count; + if (lane == stream_count - 1) { + slice = total_bytes - ((size_t)lane * (total_bytes / (size_t)stream_count)); + } + slice = round_down_size(slice, sizeof(uint32_t)); + if (slice < MIN_PROFILE_BUDGET_BYTES) { + slice = MIN_PROFILE_BUDGET_BYTES; + } + bytes_per_stream[lane] = slice; + words[lane] = (uint32_t)(slice / sizeof(uint32_t)); + + if (!check_rc(api, "cuMemAlloc", api->cuMemAlloc(&device_mem[lane], slice))) { + goto fail; + } + if (!check_rc(api, "cuMemsetD8", api->cuMemsetD8(device_mem[lane], 0, slice))) { + goto fail; + } + rounds[lane] = 2048; + params[lane][0] = &device_mem[lane]; + params[lane][1] = &words[lane]; + params[lane][2] = &rounds[lane]; + } + + if (!check_rc(api, + "cuModuleLoadDataEx", + api->cuModuleLoadDataEx(&module, ptx_source, 0, NULL, NULL))) { + goto fail; + } + if (!check_rc(api, "cuModuleGetFunction", api->cuModuleGetFunction(&kernel, module, "burn"))) { + goto fail; + } + + unsigned int threads = 256; + + double deadline = now_seconds() + (double)seconds; + double next_sync = now_seconds() + 1.0; + while (now_seconds() < deadline) { + int launched = 0; + for (int lane = 0; lane < stream_count; lane++) { + unsigned int blocks = (unsigned int)((words[lane] + threads - 1) / threads); + if (!check_rc(api, + "cuLaunchKernel", + api->cuLaunchKernel(kernel, + blocks, + 1, + 1, + threads, + 1, + 1, + 0, + streams[lane], + params[lane], + NULL))) { + goto fail; + } + launched++; + iterations++; + } + if (launched <= 0) { + goto fail; + } + double now = now_seconds(); + if (now >= next_sync || now >= deadline) { + if (!check_rc(api, "cuCtxSynchronize", api->cuCtxSynchronize())) { + goto fail; + } + next_sync = now + 1.0; + } + } + api->cuCtxSynchronize(); + + if (!check_rc(api, "cuMemcpyDtoH", api->cuMemcpyDtoH(sample, device_mem[0], sizeof(sample)))) { + goto fail; + } + + for (size_t i = 0; i < sizeof(sample) / sizeof(sample[0]); i++) { + report->checksum += sample[i]; + } + report->iterations = iterations; + snprintf(report->details, + sizeof(report->details), + "fallback_int32=OK requested_mb=%d actual_mb=%d streams=%d per_stream_mb=%zu iterations=%lu\n", + size_mb, + report->buffer_mb, + report->stream_count, + bytes_per_stream[0] / (1024u * 1024u), + iterations); + + for (int lane = 0; lane < stream_count; lane++) { + if (device_mem[lane]) { + api->cuMemFree(device_mem[lane]); + } + } + destroy_streams(api, streams, stream_count); + api->cuCtxDestroy(ctx); + return 1; + +fail: + for (int lane = 0; lane < MAX_STRESS_STREAMS; lane++) { + if (device_mem[lane]) { + api->cuMemFree(device_mem[lane]); + } + } + destroy_streams(api, streams, MAX_STRESS_STREAMS); + if (ctx) { + api->cuCtxDestroy(ctx); + } + return 0; +} diff --git a/iso/builder/bee-gpu-stress-main.inc b/iso/builder/bee-gpu-stress-main.inc new file mode 100644 index 0000000..08ee526 --- /dev/null +++ b/iso/builder/bee-gpu-stress-main.inc @@ -0,0 +1,202 @@ +static void print_stress_report(const struct stress_report *report, int device_index, int seconds) { + printf("device=%s\n", report->device); + printf("device_index=%d\n", device_index); + printf("compute_capability=%d.%d\n", report->cc_major, report->cc_minor); + printf("backend=%s\n", report->backend); + printf("duration_s=%d\n", seconds); + printf("buffer_mb=%d\n", report->buffer_mb); + printf("streams=%d\n", report->stream_count); + printf("iterations=%lu\n", report->iterations); + printf("checksum=%llu\n", (unsigned long long)report->checksum); + if (report->details[0] != '\0') { + printf("%s", report->details); + } + printf("status=OK\n"); +} + +int main(int argc, char **argv) { + int seconds = 5; + int size_mb = 64; + int device_index = 0; + const char *precision_filter = NULL; /* NULL = all; else block_label to match */ +#if HAVE_CUBLASLT_HEADERS + const char *precision_plan = NULL; + const char *precision_plan_seconds = NULL; +#endif + for (int i = 1; i < argc; i++) { + if ((strcmp(argv[i], "--seconds") == 0 || strcmp(argv[i], "-t") == 0) && i + 1 < argc) { + seconds = atoi(argv[++i]); + } else if ((strcmp(argv[i], "--size-mb") == 0 || strcmp(argv[i], "-m") == 0) && i + 1 < argc) { + size_mb = atoi(argv[++i]); + } else if ((strcmp(argv[i], "--device") == 0 || strcmp(argv[i], "-d") == 0) && i + 1 < argc) { + device_index = atoi(argv[++i]); + } else if (strcmp(argv[i], "--precision") == 0 && i + 1 < argc) { + precision_filter = argv[++i]; + } else if (strcmp(argv[i], "--precision-plan") == 0 && i + 1 < argc) { +#if HAVE_CUBLASLT_HEADERS + precision_plan = argv[++i]; +#else + fprintf(stderr, "--precision-plan requires a build with cuBLASLt headers\n"); + return 2; +#endif + } else if (strcmp(argv[i], "--precision-plan-seconds") == 0 && i + 1 < argc) { +#if HAVE_CUBLASLT_HEADERS + precision_plan_seconds = argv[++i]; +#else + fprintf(stderr, "--precision-plan-seconds requires a build with cuBLASLt headers\n"); + return 2; +#endif + } else { + fprintf(stderr, + "usage: %s [--seconds N] [--size-mb N] [--device N] [--precision int8|fp8|fp16|fp32|fp64|fp4] [--precision-plan p1,p2,...,mixed] [--precision-plan-seconds s1,s2,...]\n", + argv[0]); + return 2; + } + } + if (seconds <= 0) { + seconds = 5; + } + if (size_mb <= 0) { + size_mb = 64; + } + if (device_index < 0) { + device_index = 0; + } + + struct cuda_api cuda; + if (!load_cuda(&cuda)) { + fprintf(stderr, "failed to load libcuda.so.1 or required Driver API symbols\n"); + return 1; + } + load_symbol(cuda.lib, "cuGetErrorName", (void **)&cuda.cuGetErrorName); + load_symbol(cuda.lib, "cuGetErrorString", (void **)&cuda.cuGetErrorString); + + if (!check_rc(&cuda, "cuInit", cuda.cuInit(0))) { + return 1; + } + + int count = 0; + if (!check_rc(&cuda, "cuDeviceGetCount", cuda.cuDeviceGetCount(&count))) { + return 1; + } + if (count <= 0) { + fprintf(stderr, "no CUDA devices found\n"); + return 1; + } + + if (device_index >= count) { + fprintf(stderr, "device index %d out of range (found %d CUDA device(s))\n", device_index, count); + return 1; + } + + CUdevice dev = 0; + if (!check_rc(&cuda, "cuDeviceGet", cuda.cuDeviceGet(&dev, device_index))) { + return 1; + } + + char name[128] = {0}; + if (!check_rc(&cuda, "cuDeviceGetName", cuda.cuDeviceGetName(name, (int)sizeof(name), dev))) { + return 1; + } + + int cc_major = 0; + int cc_minor = 0; + if (!query_compute_capability(&cuda, dev, &cc_major, &cc_minor)) { + return 1; + } + + struct stress_report report; + int ok = 0; + +#if HAVE_CUBLASLT_HEADERS + if (precision_plan != NULL && precision_plan[0] != '\0') { + char *plan_copy = strdup(precision_plan); + char *plan_seconds_copy = NULL; + int phase_seconds[32] = {0}; + int phase_seconds_count = 0; + int phase_ok = 0; + if (plan_copy == NULL) { + fprintf(stderr, "failed to allocate precision plan buffer\n"); + return 1; + } + if (precision_plan_seconds != NULL && precision_plan_seconds[0] != '\0') { + plan_seconds_copy = strdup(precision_plan_seconds); + if (plan_seconds_copy == NULL) { + free(plan_copy); + fprintf(stderr, "failed to allocate precision plan seconds buffer\n"); + return 1; + } + for (char *sec_token = strtok(plan_seconds_copy, ","); + sec_token != NULL && phase_seconds_count < (int)(sizeof(phase_seconds) / sizeof(phase_seconds[0])); + sec_token = strtok(NULL, ",")) { + while (*sec_token == ' ' || *sec_token == '\t') { + sec_token++; + } + if (*sec_token == '\0') { + continue; + } + phase_seconds[phase_seconds_count++] = atoi(sec_token); + } + } + int phase_idx = 0; + for (char *token = strtok(plan_copy, ","); token != NULL; token = strtok(NULL, ","), phase_idx++) { + while (*token == ' ' || *token == '\t') { + token++; + } + if (*token == '\0') { + continue; + } + const char *phase_name = token; + const char *phase_filter = token; + if (strcmp(token, "mixed") == 0 || strcmp(token, "all") == 0) { + phase_filter = NULL; + } + int phase_duration = seconds; + if (phase_idx < phase_seconds_count && phase_seconds[phase_idx] > 0) { + phase_duration = phase_seconds[phase_idx]; + } + printf("phase_begin=%s\n", phase_name); + fflush(stdout); + memset(&report, 0, sizeof(report)); + ok = run_cublaslt_stress(&cuda, dev, name, cc_major, cc_minor, phase_duration, size_mb, phase_filter, &report); + if (ok) { + print_stress_report(&report, device_index, phase_duration); + phase_ok = 1; + } else { + printf("phase_error=%s\n", phase_name); + if (report.details[0] != '\0') { + printf("%s", report.details); + if (report.details[strlen(report.details) - 1] != '\n') { + printf("\n"); + } + } + printf("status=FAILED\n"); + } + printf("phase_end=%s\n", phase_name); + fflush(stdout); + } + free(plan_seconds_copy); + free(plan_copy); + return phase_ok ? 0 : 1; + } + ok = run_cublaslt_stress(&cuda, dev, name, cc_major, cc_minor, seconds, size_mb, precision_filter, &report); +#endif + if (!ok) { + if (precision_filter != NULL) { + fprintf(stderr, + "requested precision path unavailable: precision=%s device=%s cc=%d.%d\n", + precision_filter, + name, + cc_major, + cc_minor); + return 1; + } + int ptx_mb = size_mb; + if (!run_ptx_fallback(&cuda, dev, name, cc_major, cc_minor, seconds, ptx_mb, &report)) { + return 1; + } + } + + print_stress_report(&report, device_index, seconds); + return 0; +} diff --git a/iso/builder/bee-gpu-stress.c b/iso/builder/bee-gpu-stress.c index 2bc377e..8a6aad9 100644 --- a/iso/builder/bee-gpu-stress.c +++ b/iso/builder/bee-gpu-stress.c @@ -146,1493 +146,8 @@ struct stress_report { char details[16384]; }; -static int load_symbol(void *lib, const char *name, void **out) { - *out = dlsym(lib, name); - return *out != NULL; -} - -static int load_cuda(struct cuda_api *api) { - memset(api, 0, sizeof(*api)); - api->lib = dlopen("libcuda.so.1", RTLD_NOW | RTLD_LOCAL); - if (!api->lib) { - return 0; - } - if (!( - load_symbol(api->lib, "cuInit", (void **)&api->cuInit) && - load_symbol(api->lib, "cuDeviceGetCount", (void **)&api->cuDeviceGetCount) && - load_symbol(api->lib, "cuDeviceGet", (void **)&api->cuDeviceGet) && - load_symbol(api->lib, "cuDeviceGetName", (void **)&api->cuDeviceGetName) && - load_symbol(api->lib, "cuDeviceGetAttribute", (void **)&api->cuDeviceGetAttribute) && - load_symbol(api->lib, "cuCtxCreate_v2", (void **)&api->cuCtxCreate) && - load_symbol(api->lib, "cuCtxDestroy_v2", (void **)&api->cuCtxDestroy) && - load_symbol(api->lib, "cuCtxSynchronize", (void **)&api->cuCtxSynchronize) && - load_symbol(api->lib, "cuMemAlloc_v2", (void **)&api->cuMemAlloc) && - load_symbol(api->lib, "cuMemFree_v2", (void **)&api->cuMemFree) && - load_symbol(api->lib, "cuMemsetD8_v2", (void **)&api->cuMemsetD8) && - load_symbol(api->lib, "cuMemcpyHtoD_v2", (void **)&api->cuMemcpyHtoD) && - load_symbol(api->lib, "cuMemcpyDtoH_v2", (void **)&api->cuMemcpyDtoH) && - load_symbol(api->lib, "cuModuleLoadDataEx", (void **)&api->cuModuleLoadDataEx) && - load_symbol(api->lib, "cuModuleGetFunction", (void **)&api->cuModuleGetFunction) && - load_symbol(api->lib, "cuLaunchKernel", (void **)&api->cuLaunchKernel))) { - dlclose(api->lib); - memset(api, 0, sizeof(*api)); - return 0; - } - load_symbol(api->lib, "cuMemGetInfo_v2", (void **)&api->cuMemGetInfo); - load_symbol(api->lib, "cuStreamCreate", (void **)&api->cuStreamCreate); - if (!load_symbol(api->lib, "cuStreamDestroy_v2", (void **)&api->cuStreamDestroy)) { - load_symbol(api->lib, "cuStreamDestroy", (void **)&api->cuStreamDestroy); - } - return 1; -} - -static const char *cu_error_name(struct cuda_api *api, CUresult rc) { - const char *value = NULL; - if (api->cuGetErrorName && api->cuGetErrorName(rc, &value) == CU_SUCCESS && value) { - return value; - } - return "CUDA_ERROR"; -} - -static const char *cu_error_string(struct cuda_api *api, CUresult rc) { - const char *value = NULL; - if (api->cuGetErrorString && api->cuGetErrorString(rc, &value) == CU_SUCCESS && value) { - return value; - } - return "unknown"; -} - -static int check_rc(struct cuda_api *api, const char *step, CUresult rc) { - if (rc == CU_SUCCESS) { - return 1; - } - fprintf(stderr, "%s failed: %s (%s)\n", step, cu_error_name(api, rc), cu_error_string(api, rc)); - return 0; -} - -static double now_seconds(void) { - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return (double)ts.tv_sec + ((double)ts.tv_nsec / 1000000000.0); -} - -static size_t round_down_size(size_t value, size_t multiple) { - if (multiple == 0 || value < multiple) { - return value; - } - return value - (value % multiple); -} - -static int query_compute_capability(struct cuda_api *api, CUdevice dev, int *major, int *minor) { - int cc_major = 0; - int cc_minor = 0; - if (!check_rc(api, - "cuDeviceGetAttribute(major)", - api->cuDeviceGetAttribute(&cc_major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, dev))) { - return 0; - } - if (!check_rc(api, - "cuDeviceGetAttribute(minor)", - api->cuDeviceGetAttribute(&cc_minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, dev))) { - return 0; - } - *major = cc_major; - *minor = cc_minor; - return 1; -} - -static int query_multiprocessor_count(struct cuda_api *api, CUdevice dev, int *count) { - int mp_count = 0; - if (!check_rc(api, - "cuDeviceGetAttribute(multiprocessors)", - api->cuDeviceGetAttribute(&mp_count, CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, dev))) { - return 0; - } - *count = mp_count; - return 1; -} - -static size_t clamp_budget_to_free_memory(struct cuda_api *api, size_t requested_bytes) { - size_t free_bytes = 0; - size_t total_bytes = 0; - size_t max_bytes = requested_bytes; - - if (!api->cuMemGetInfo) { - return requested_bytes; - } - if (api->cuMemGetInfo(&free_bytes, &total_bytes) != CU_SUCCESS || free_bytes == 0) { - return requested_bytes; - } - - max_bytes = (free_bytes * 9u) / 10u; - if (max_bytes < (size_t)4u * 1024u * 1024u) { - max_bytes = (size_t)4u * 1024u * 1024u; - } - if (requested_bytes > max_bytes) { - return max_bytes; - } - return requested_bytes; -} - -static int choose_stream_count(int mp_count, int planned_profiles, size_t total_budget, int have_streams) { - int stream_count = 1; - if (!have_streams || mp_count <= 0 || planned_profiles <= 0) { - return 1; - } - - stream_count = mp_count / 8; - if (stream_count < 2) { - stream_count = 2; - } - if (stream_count > MAX_STRESS_STREAMS) { - stream_count = MAX_STRESS_STREAMS; - } - - while (stream_count > 1) { - size_t per_stream_budget = total_budget / ((size_t)planned_profiles * (size_t)stream_count); - if (per_stream_budget >= MIN_STREAM_BUDGET_BYTES) { - break; - } - stream_count--; - } - return stream_count; -} - -static size_t clamp_single_precision_profile_budget(size_t profile_budget_bytes) { - if (profile_budget_bytes > MAX_SINGLE_PRECISION_PROFILE_BUDGET_BYTES) { - return MAX_SINGLE_PRECISION_PROFILE_BUDGET_BYTES; - } - return profile_budget_bytes; -} - -static void destroy_streams(struct cuda_api *api, CUstream *streams, int count) { - if (!api->cuStreamDestroy) { - return; - } - for (int i = 0; i < count; i++) { - if (streams[i]) { - api->cuStreamDestroy(streams[i]); - streams[i] = NULL; - } - } -} - -#if HAVE_CUBLASLT_HEADERS -static void append_detail(char *buf, size_t cap, const char *fmt, ...) { - size_t len = strlen(buf); - if (len >= cap) { - return; - } - va_list ap; - va_start(ap, fmt); - vsnprintf(buf + len, cap - len, fmt, ap); - va_end(ap); -} -#endif - -static int run_ptx_fallback(struct cuda_api *api, - CUdevice dev, - const char *device_name, - int cc_major, - int cc_minor, - int seconds, - int size_mb, - struct stress_report *report) { - CUcontext ctx = NULL; - CUmodule module = NULL; - CUfunction kernel = NULL; - uint32_t sample[256]; - CUdeviceptr device_mem[MAX_STRESS_STREAMS] = {0}; - CUstream streams[MAX_STRESS_STREAMS] = {0}; - uint32_t words[MAX_STRESS_STREAMS] = {0}; - uint32_t rounds[MAX_STRESS_STREAMS] = {0}; - void *params[MAX_STRESS_STREAMS][3]; - size_t bytes_per_stream[MAX_STRESS_STREAMS] = {0}; - unsigned long iterations = 0; - int mp_count = 0; - int stream_count = 1; - - memset(report, 0, sizeof(*report)); - snprintf(report->backend, sizeof(report->backend), "driver-ptx"); - snprintf(report->device, sizeof(report->device), "%s", device_name); - report->cc_major = cc_major; - report->cc_minor = cc_minor; - report->buffer_mb = size_mb; - - if (!check_rc(api, "cuCtxCreate", api->cuCtxCreate(&ctx, 0, dev))) { - return 0; - } - - size_t requested_bytes = (size_t)size_mb * 1024u * 1024u; - if (requested_bytes < MIN_PROFILE_BUDGET_BYTES) { - requested_bytes = MIN_PROFILE_BUDGET_BYTES; - } - size_t total_bytes = clamp_budget_to_free_memory(api, requested_bytes); - if (total_bytes < MIN_PROFILE_BUDGET_BYTES) { - total_bytes = MIN_PROFILE_BUDGET_BYTES; - } - report->buffer_mb = (int)(total_bytes / (1024u * 1024u)); - - if (query_multiprocessor_count(api, dev, &mp_count) && - api->cuStreamCreate && - api->cuStreamDestroy) { - stream_count = choose_stream_count(mp_count, 1, total_bytes, 1); - } - if (stream_count > 1) { - int created = 0; - for (; created < stream_count; created++) { - if (!check_rc(api, "cuStreamCreate", api->cuStreamCreate(&streams[created], 0))) { - destroy_streams(api, streams, created); - stream_count = 1; - break; - } - } - } - report->stream_count = stream_count; - - for (int lane = 0; lane < stream_count; lane++) { - size_t slice = total_bytes / (size_t)stream_count; - if (lane == stream_count - 1) { - slice = total_bytes - ((size_t)lane * (total_bytes / (size_t)stream_count)); - } - slice = round_down_size(slice, sizeof(uint32_t)); - if (slice < MIN_PROFILE_BUDGET_BYTES) { - slice = MIN_PROFILE_BUDGET_BYTES; - } - bytes_per_stream[lane] = slice; - words[lane] = (uint32_t)(slice / sizeof(uint32_t)); - - if (!check_rc(api, "cuMemAlloc", api->cuMemAlloc(&device_mem[lane], slice))) { - goto fail; - } - if (!check_rc(api, "cuMemsetD8", api->cuMemsetD8(device_mem[lane], 0, slice))) { - goto fail; - } - rounds[lane] = 2048; - params[lane][0] = &device_mem[lane]; - params[lane][1] = &words[lane]; - params[lane][2] = &rounds[lane]; - } - - if (!check_rc(api, - "cuModuleLoadDataEx", - api->cuModuleLoadDataEx(&module, ptx_source, 0, NULL, NULL))) { - goto fail; - } - if (!check_rc(api, "cuModuleGetFunction", api->cuModuleGetFunction(&kernel, module, "burn"))) { - goto fail; - } - - unsigned int threads = 256; - - double deadline = now_seconds() + (double)seconds; - double next_sync = now_seconds() + 1.0; - while (now_seconds() < deadline) { - int launched = 0; - for (int lane = 0; lane < stream_count; lane++) { - unsigned int blocks = (unsigned int)((words[lane] + threads - 1) / threads); - if (!check_rc(api, - "cuLaunchKernel", - api->cuLaunchKernel(kernel, - blocks, - 1, - 1, - threads, - 1, - 1, - 0, - streams[lane], - params[lane], - NULL))) { - goto fail; - } - launched++; - iterations++; - } - if (launched <= 0) { - goto fail; - } - double now = now_seconds(); - if (now >= next_sync || now >= deadline) { - if (!check_rc(api, "cuCtxSynchronize", api->cuCtxSynchronize())) { - goto fail; - } - next_sync = now + 1.0; - } - } - api->cuCtxSynchronize(); - - if (!check_rc(api, "cuMemcpyDtoH", api->cuMemcpyDtoH(sample, device_mem[0], sizeof(sample)))) { - goto fail; - } - - for (size_t i = 0; i < sizeof(sample) / sizeof(sample[0]); i++) { - report->checksum += sample[i]; - } - report->iterations = iterations; - snprintf(report->details, - sizeof(report->details), - "fallback_int32=OK requested_mb=%d actual_mb=%d streams=%d per_stream_mb=%zu iterations=%lu\n", - size_mb, - report->buffer_mb, - report->stream_count, - bytes_per_stream[0] / (1024u * 1024u), - iterations); - - for (int lane = 0; lane < stream_count; lane++) { - if (device_mem[lane]) { - api->cuMemFree(device_mem[lane]); - } - } - destroy_streams(api, streams, stream_count); - api->cuCtxDestroy(ctx); - return 1; - -fail: - for (int lane = 0; lane < MAX_STRESS_STREAMS; lane++) { - if (device_mem[lane]) { - api->cuMemFree(device_mem[lane]); - } - } - destroy_streams(api, streams, MAX_STRESS_STREAMS); - if (ctx) { - api->cuCtxDestroy(ctx); - } - return 0; -} - -#if HAVE_CUBLASLT_HEADERS -typedef cublasStatus_t (*cublasLtCreate_fn)(cublasLtHandle_t *); -typedef cublasStatus_t (*cublasLtDestroy_fn)(cublasLtHandle_t); -typedef cublasStatus_t (*cublasLtMatmulDescCreate_fn)(cublasLtMatmulDesc_t *, - cublasComputeType_t, - cudaDataType_t); -typedef cublasStatus_t (*cublasLtMatmulDescDestroy_fn)(cublasLtMatmulDesc_t); -typedef cublasStatus_t (*cublasLtMatmulDescSetAttribute_fn)(cublasLtMatmulDesc_t, - cublasLtMatmulDescAttributes_t, - const void *, - size_t); -typedef cublasStatus_t (*cublasLtMatrixLayoutCreate_fn)(cublasLtMatrixLayout_t *, - cudaDataType_t, - uint64_t, - uint64_t, - int64_t); -typedef cublasStatus_t (*cublasLtMatrixLayoutDestroy_fn)(cublasLtMatrixLayout_t); -typedef cublasStatus_t (*cublasLtMatmulPreferenceCreate_fn)(cublasLtMatmulPreference_t *); -typedef cublasStatus_t (*cublasLtMatmulPreferenceDestroy_fn)(cublasLtMatmulPreference_t); -typedef cublasStatus_t (*cublasLtMatmulPreferenceSetAttribute_fn)(cublasLtMatmulPreference_t, - cublasLtMatmulPreferenceAttributes_t, - const void *, - size_t); -typedef cublasStatus_t (*cublasLtMatmulAlgoGetHeuristic_fn)( - cublasLtHandle_t, - cublasLtMatmulDesc_t, - cublasLtMatrixLayout_t, - cublasLtMatrixLayout_t, - cublasLtMatrixLayout_t, - cublasLtMatrixLayout_t, - cublasLtMatmulPreference_t, - int, - cublasLtMatmulHeuristicResult_t *, - int *); -typedef cublasStatus_t (*cublasLtMatmul_fn)(cublasLtHandle_t, - cublasLtMatmulDesc_t, - const void *, - const void *, - cublasLtMatrixLayout_t, - const void *, - cublasLtMatrixLayout_t, - const void *, - const void *, - cublasLtMatrixLayout_t, - void *, - cublasLtMatrixLayout_t, - const cublasLtMatmulAlgo_t *, - void *, - size_t, - cudaStream_t); - -struct cublaslt_api { - void *lib; - cublasLtCreate_fn cublasLtCreate; - cublasLtDestroy_fn cublasLtDestroy; - cublasLtMatmulDescCreate_fn cublasLtMatmulDescCreate; - cublasLtMatmulDescDestroy_fn cublasLtMatmulDescDestroy; - cublasLtMatmulDescSetAttribute_fn cublasLtMatmulDescSetAttribute; - cublasLtMatrixLayoutCreate_fn cublasLtMatrixLayoutCreate; - cublasLtMatrixLayoutDestroy_fn cublasLtMatrixLayoutDestroy; - cublasLtMatmulPreferenceCreate_fn cublasLtMatmulPreferenceCreate; - cublasLtMatmulPreferenceDestroy_fn cublasLtMatmulPreferenceDestroy; - cublasLtMatmulPreferenceSetAttribute_fn cublasLtMatmulPreferenceSetAttribute; - cublasLtMatmulAlgoGetHeuristic_fn cublasLtMatmulAlgoGetHeuristic; - cublasLtMatmul_fn cublasLtMatmul; -}; - -struct profile_desc { - const char *name; - const char *block_label; - int min_cc; - int enabled; - int needs_scalar_scale; - int needs_block_scale; - int min_multiple; - cudaDataType_t a_type; - cudaDataType_t b_type; - cudaDataType_t c_type; - cudaDataType_t d_type; - cublasComputeType_t compute_type; -}; - -struct prepared_profile { - struct profile_desc desc; - CUstream stream; - cublasLtMatmulDesc_t op_desc; - cublasLtMatrixLayout_t a_layout; - cublasLtMatrixLayout_t b_layout; - cublasLtMatrixLayout_t c_layout; - cublasLtMatrixLayout_t d_layout; - cublasLtMatmulPreference_t preference; - cublasLtMatmulHeuristicResult_t heuristic; - CUdeviceptr a_dev; - CUdeviceptr b_dev; - CUdeviceptr c_dev; - CUdeviceptr d_dev; - CUdeviceptr a_scale_dev; - CUdeviceptr b_scale_dev; - CUdeviceptr workspace_dev; - size_t workspace_size; - uint64_t m; - uint64_t n; - uint64_t k; - unsigned long iterations; - int ready; -}; - -static const struct profile_desc k_profiles[] = { - { - "fp64", - "fp64", - 80, - 1, - 0, - 0, - 8, - CUDA_R_64F, - CUDA_R_64F, - CUDA_R_64F, - CUDA_R_64F, - CUBLAS_COMPUTE_64F, - }, - { - "fp32_tf32", - "fp32", - 80, - 1, - 0, - 0, - 128, - CUDA_R_32F, - CUDA_R_32F, - CUDA_R_32F, - CUDA_R_32F, - CUBLAS_COMPUTE_32F_FAST_TF32, - }, - { - "fp16_tensor", - "fp16", - 80, - 1, - 0, - 0, - 128, - CUDA_R_16F, - CUDA_R_16F, - CUDA_R_16F, - CUDA_R_16F, - CUBLAS_COMPUTE_32F_FAST_16F, - }, - { - "int8_tensor", - "int8", - 75, - 1, - 0, - 0, - 128, - CUDA_R_8I, - CUDA_R_8I, - CUDA_R_32I, - CUDA_R_32I, - CUBLAS_COMPUTE_32I, - }, - { - "fp8_e4m3", - "fp8", - 89, - 1, - 1, - 0, - 128, - CUDA_R_8F_E4M3, - CUDA_R_8F_E4M3, - CUDA_R_16BF, - CUDA_R_16BF, - CUBLAS_COMPUTE_32F, - }, - { - "fp8_e5m2", - "fp8", - 89, - 1, - 1, - 0, - 128, - CUDA_R_8F_E5M2, - CUDA_R_8F_E5M2, - CUDA_R_16BF, - CUDA_R_16BF, - CUBLAS_COMPUTE_32F, - }, -#if defined(CUDA_R_4F_E2M1) && defined(CUBLASLT_MATMUL_MATRIX_SCALE_VEC16_UE4M3) - { - "fp4_e2m1", - "fp4", - 100, - 1, - 0, - 1, - 128, - CUDA_R_4F_E2M1, - CUDA_R_4F_E2M1, - CUDA_R_16BF, - CUDA_R_16BF, - CUBLAS_COMPUTE_32F, - }, -#endif -}; - -#define PROFILE_COUNT ((int)(sizeof(k_profiles) / sizeof(k_profiles[0]))) - -static int profile_allowed_for_run(const struct profile_desc *desc, int cc, const char *precision_filter) { - if (!(desc->enabled && cc >= desc->min_cc)) { - return 0; - } - if (precision_filter != NULL) { - return strcmp(desc->block_label, precision_filter) == 0; - } - /* Mixed/all phases intentionally exclude fp64/fp4 for now: both paths are - * unstable on the current benchmark fleet and can abort the whole mixed - * pass after earlier phases already collected useful telemetry. */ - return strcmp(desc->block_label, "fp64") != 0 && strcmp(desc->block_label, "fp4") != 0; -} - -static int load_cublaslt(struct cublaslt_api *api) { - memset(api, 0, sizeof(*api)); - api->lib = dlopen("libcublasLt.so.13", RTLD_NOW | RTLD_LOCAL); - if (!api->lib) { - api->lib = dlopen("libcublasLt.so", RTLD_NOW | RTLD_LOCAL); - } - if (!api->lib) { - return 0; - } - return - load_symbol(api->lib, "cublasLtCreate", (void **)&api->cublasLtCreate) && - load_symbol(api->lib, "cublasLtDestroy", (void **)&api->cublasLtDestroy) && - load_symbol(api->lib, "cublasLtMatmulDescCreate", (void **)&api->cublasLtMatmulDescCreate) && - load_symbol(api->lib, "cublasLtMatmulDescDestroy", (void **)&api->cublasLtMatmulDescDestroy) && - load_symbol(api->lib, - "cublasLtMatmulDescSetAttribute", - (void **)&api->cublasLtMatmulDescSetAttribute) && - load_symbol(api->lib, "cublasLtMatrixLayoutCreate", (void **)&api->cublasLtMatrixLayoutCreate) && - load_symbol(api->lib, "cublasLtMatrixLayoutDestroy", (void **)&api->cublasLtMatrixLayoutDestroy) && - load_symbol(api->lib, - "cublasLtMatmulPreferenceCreate", - (void **)&api->cublasLtMatmulPreferenceCreate) && - load_symbol(api->lib, - "cublasLtMatmulPreferenceDestroy", - (void **)&api->cublasLtMatmulPreferenceDestroy) && - load_symbol(api->lib, - "cublasLtMatmulPreferenceSetAttribute", - (void **)&api->cublasLtMatmulPreferenceSetAttribute) && - load_symbol(api->lib, - "cublasLtMatmulAlgoGetHeuristic", - (void **)&api->cublasLtMatmulAlgoGetHeuristic) && - load_symbol(api->lib, "cublasLtMatmul", (void **)&api->cublasLtMatmul); -} - -static const char *cublas_status_text(cublasStatus_t status) { - switch (status) { - case CUBLAS_STATUS_SUCCESS: - return "CUBLAS_STATUS_SUCCESS"; - case CUBLAS_STATUS_NOT_INITIALIZED: - return "CUBLAS_STATUS_NOT_INITIALIZED"; - case CUBLAS_STATUS_ALLOC_FAILED: - return "CUBLAS_STATUS_ALLOC_FAILED"; - case CUBLAS_STATUS_INVALID_VALUE: - return "CUBLAS_STATUS_INVALID_VALUE"; - case CUBLAS_STATUS_ARCH_MISMATCH: - return "CUBLAS_STATUS_ARCH_MISMATCH"; - case CUBLAS_STATUS_MAPPING_ERROR: - return "CUBLAS_STATUS_MAPPING_ERROR"; - case CUBLAS_STATUS_EXECUTION_FAILED: - return "CUBLAS_STATUS_EXECUTION_FAILED"; - case CUBLAS_STATUS_INTERNAL_ERROR: - return "CUBLAS_STATUS_INTERNAL_ERROR"; - case CUBLAS_STATUS_NOT_SUPPORTED: - return "CUBLAS_STATUS_NOT_SUPPORTED"; - default: - return "CUBLAS_STATUS_UNKNOWN"; - } -} - -static int check_cublas(const char *step, cublasStatus_t status) { - if (status == CUBLAS_STATUS_SUCCESS) { - return 1; - } - fprintf(stderr, "%s failed: %s (%d)\n", step, cublas_status_text(status), (int)status); - return 0; -} - -static size_t bytes_for_elements(cudaDataType_t type, uint64_t elements) { - switch (type) { - case CUDA_R_32F: - case CUDA_R_32I: - return (size_t)(elements * 4u); - case CUDA_R_16F: - case CUDA_R_16BF: - return (size_t)(elements * 2u); - case CUDA_R_8I: - case CUDA_R_8F_E4M3: - case CUDA_R_8F_E5M2: - return (size_t)(elements); -#if defined(CUDA_R_4F_E2M1) - case CUDA_R_4F_E2M1: - return (size_t)((elements + 1u) / 2u); -#endif - default: - return (size_t)(elements * 4u); - } -} - -static cudaDataType_t matmul_scale_type(const struct profile_desc *desc) { - if (desc->compute_type == CUBLAS_COMPUTE_32I) { - return CUDA_R_32I; - } - if (desc->compute_type == CUBLAS_COMPUTE_64F) { - return CUDA_R_64F; - } - return CUDA_R_32F; -} - -static size_t fp4_scale_bytes(uint64_t rows, uint64_t cols) { - uint64_t row_tiles = (rows + 127u) / 128u; - uint64_t col_tiles = (cols + 63u) / 64u; - return (size_t)(row_tiles * col_tiles * 128u); -} - -static uint64_t choose_square_dim(size_t budget_bytes, size_t bytes_per_cell, int multiple) { - double approx = sqrt((double)budget_bytes / (double)bytes_per_cell); - uint64_t dim = (uint64_t)approx; - if (dim < (uint64_t)multiple) { - dim = (uint64_t)multiple; - } - dim = (uint64_t)round_down_size((size_t)dim, (size_t)multiple); - if (dim < (uint64_t)multiple) { - dim = (uint64_t)multiple; - } - if (dim > 65536u) { - dim = 65536u; - } - return dim; -} - -static int device_upload(struct cuda_api *cuda, CUdeviceptr dev, const void *src, size_t bytes) { - return check_rc(cuda, "cuMemcpyHtoD", cuda->cuMemcpyHtoD(dev, src, bytes)); -} - -static int alloc_filled(struct cuda_api *cuda, CUdeviceptr *ptr, size_t bytes, unsigned char pattern) { - if (!check_rc(cuda, "cuMemAlloc", cuda->cuMemAlloc(ptr, bytes))) { - return 0; - } - if (!check_rc(cuda, "cuMemsetD8", cuda->cuMemsetD8(*ptr, pattern, bytes))) { - cuda->cuMemFree(*ptr); - *ptr = 0; - return 0; - } - return 1; -} - -static size_t profile_scale_bytes(const struct profile_desc *desc, uint64_t m, uint64_t n, uint64_t k) { - size_t bytes = 0; - if (desc->needs_scalar_scale) { - bytes += 2u * sizeof(float); - } -#if defined(CUBLASLT_MATMUL_MATRIX_SCALE_VEC16_UE4M3) - if (desc->needs_block_scale) { - bytes += fp4_scale_bytes(k, m); - bytes += fp4_scale_bytes(k, n); - } -#else - (void)m; - (void)n; - (void)k; -#endif - return bytes; -} - -static void destroy_profile(struct cublaslt_api *cublas, struct cuda_api *cuda, struct prepared_profile *profile) { - if (profile->workspace_dev) { - cuda->cuMemFree(profile->workspace_dev); - } - if (profile->a_scale_dev) { - cuda->cuMemFree(profile->a_scale_dev); - } - if (profile->b_scale_dev) { - cuda->cuMemFree(profile->b_scale_dev); - } - if (profile->d_dev) { - cuda->cuMemFree(profile->d_dev); - } - if (profile->c_dev) { - cuda->cuMemFree(profile->c_dev); - } - if (profile->b_dev) { - cuda->cuMemFree(profile->b_dev); - } - if (profile->a_dev) { - cuda->cuMemFree(profile->a_dev); - } - if (profile->preference) { - cublas->cublasLtMatmulPreferenceDestroy(profile->preference); - } - if (profile->d_layout) { - cublas->cublasLtMatrixLayoutDestroy(profile->d_layout); - } - if (profile->c_layout) { - cublas->cublasLtMatrixLayoutDestroy(profile->c_layout); - } - if (profile->b_layout) { - cublas->cublasLtMatrixLayoutDestroy(profile->b_layout); - } - if (profile->a_layout) { - cublas->cublasLtMatrixLayoutDestroy(profile->a_layout); - } - if (profile->op_desc) { - cublas->cublasLtMatmulDescDestroy(profile->op_desc); - } - memset(profile, 0, sizeof(*profile)); -} - -static int prepare_profile(struct cublaslt_api *cublas, - cublasLtHandle_t handle, - struct cuda_api *cuda, - const struct profile_desc *desc, - CUstream stream, - size_t profile_budget_bytes, - struct prepared_profile *out) { - size_t bytes_per_cell = 0; - size_t attempt_budget = profile_budget_bytes; - - bytes_per_cell += bytes_for_elements(desc->a_type, 1); - bytes_per_cell += bytes_for_elements(desc->b_type, 1); - bytes_per_cell += bytes_for_elements(desc->c_type, 1); - bytes_per_cell += bytes_for_elements(desc->d_type, 1); - if (bytes_per_cell == 0) { - return 0; - } - - while (attempt_budget >= MIN_PROFILE_BUDGET_BYTES) { - memset(out, 0, sizeof(*out)); - out->desc = *desc; - out->stream = stream; - - uint64_t dim = choose_square_dim(attempt_budget, bytes_per_cell, desc->min_multiple); - out->m = dim; - out->n = dim; - out->k = dim; - - size_t desired_workspace = attempt_budget / 8u; - if (desired_workspace > 32u * 1024u * 1024u) { - desired_workspace = 32u * 1024u * 1024u; - } - desired_workspace = round_down_size(desired_workspace, 256u); - - size_t a_bytes = 0; - size_t b_bytes = 0; - size_t c_bytes = 0; - size_t d_bytes = 0; - size_t scale_bytes = 0; - while (1) { - a_bytes = bytes_for_elements(desc->a_type, out->k * out->m); - b_bytes = bytes_for_elements(desc->b_type, out->k * out->n); - c_bytes = bytes_for_elements(desc->c_type, out->m * out->n); - d_bytes = bytes_for_elements(desc->d_type, out->m * out->n); - scale_bytes = profile_scale_bytes(desc, out->m, out->n, out->k); - - size_t matrix_bytes = a_bytes + b_bytes + c_bytes + d_bytes + scale_bytes; - if (matrix_bytes <= attempt_budget) { - size_t remaining = attempt_budget - matrix_bytes; - out->workspace_size = desired_workspace; - if (out->workspace_size > remaining) { - out->workspace_size = round_down_size(remaining, 256u); - } - break; - } - - if (out->m <= (uint64_t)desc->min_multiple) { - break; - } - out->m -= (uint64_t)desc->min_multiple; - out->n = out->m; - out->k = out->m; - } - if (out->m < (uint64_t)desc->min_multiple) { - attempt_budget /= 2u; - continue; - } - - if (!alloc_filled(cuda, &out->a_dev, a_bytes, 0x11) || - !alloc_filled(cuda, &out->b_dev, b_bytes, 0x11) || - !alloc_filled(cuda, &out->c_dev, c_bytes, 0x00) || - !alloc_filled(cuda, &out->d_dev, d_bytes, 0x00)) { - destroy_profile(cublas, cuda, out); - return 0; - } - - cudaDataType_t scale_type = matmul_scale_type(desc); - if (!check_cublas("cublasLtMatmulDescCreate", - cublas->cublasLtMatmulDescCreate(&out->op_desc, desc->compute_type, scale_type))) { - destroy_profile(cublas, cuda, out); - return 0; - } - - cublasOperation_t transa = CUBLAS_OP_T; - cublasOperation_t transb = CUBLAS_OP_N; - if (!check_cublas("set TRANSA", - cublas->cublasLtMatmulDescSetAttribute(out->op_desc, - CUBLASLT_MATMUL_DESC_TRANSA, - &transa, - sizeof(transa))) || - !check_cublas("set TRANSB", - cublas->cublasLtMatmulDescSetAttribute(out->op_desc, - CUBLASLT_MATMUL_DESC_TRANSB, - &transb, - sizeof(transb)))) { - destroy_profile(cublas, cuda, out); - return 0; - } - - if (desc->needs_scalar_scale) { - float one = 1.0f; - if (!alloc_filled(cuda, &out->a_scale_dev, sizeof(one), 0x00) || - !alloc_filled(cuda, &out->b_scale_dev, sizeof(one), 0x00)) { - destroy_profile(cublas, cuda, out); - return 0; - } - if (!device_upload(cuda, out->a_scale_dev, &one, sizeof(one)) || - !device_upload(cuda, out->b_scale_dev, &one, sizeof(one))) { - destroy_profile(cublas, cuda, out); - return 0; - } - void *a_scale_ptr = (void *)(uintptr_t)out->a_scale_dev; - void *b_scale_ptr = (void *)(uintptr_t)out->b_scale_dev; - if (!check_cublas("set A scale ptr", - cublas->cublasLtMatmulDescSetAttribute(out->op_desc, - CUBLASLT_MATMUL_DESC_A_SCALE_POINTER, - &a_scale_ptr, - sizeof(a_scale_ptr))) || - !check_cublas("set B scale ptr", - cublas->cublasLtMatmulDescSetAttribute(out->op_desc, - CUBLASLT_MATMUL_DESC_B_SCALE_POINTER, - &b_scale_ptr, - sizeof(b_scale_ptr)))) { - destroy_profile(cublas, cuda, out); - return 0; - } - } - -#if defined(CUBLASLT_MATMUL_MATRIX_SCALE_VEC16_UE4M3) - if (desc->needs_block_scale) { - size_t a_scale_bytes = fp4_scale_bytes(out->k, out->m); - size_t b_scale_bytes = fp4_scale_bytes(out->k, out->n); - if (!alloc_filled(cuda, &out->a_scale_dev, a_scale_bytes, 0x11) || - !alloc_filled(cuda, &out->b_scale_dev, b_scale_bytes, 0x11)) { - destroy_profile(cublas, cuda, out); - return 0; - } - cublasLtMatmulMatrixScale_t scale_mode = CUBLASLT_MATMUL_MATRIX_SCALE_VEC16_UE4M3; - void *a_scale_ptr = (void *)(uintptr_t)out->a_scale_dev; - void *b_scale_ptr = (void *)(uintptr_t)out->b_scale_dev; - if (!check_cublas("set A scale mode", - cublas->cublasLtMatmulDescSetAttribute(out->op_desc, - CUBLASLT_MATMUL_DESC_A_SCALE_MODE, - &scale_mode, - sizeof(scale_mode))) || - !check_cublas("set B scale mode", - cublas->cublasLtMatmulDescSetAttribute(out->op_desc, - CUBLASLT_MATMUL_DESC_B_SCALE_MODE, - &scale_mode, - sizeof(scale_mode))) || - !check_cublas("set A block scale ptr", - cublas->cublasLtMatmulDescSetAttribute(out->op_desc, - CUBLASLT_MATMUL_DESC_A_SCALE_POINTER, - &a_scale_ptr, - sizeof(a_scale_ptr))) || - !check_cublas("set B block scale ptr", - cublas->cublasLtMatmulDescSetAttribute(out->op_desc, - CUBLASLT_MATMUL_DESC_B_SCALE_POINTER, - &b_scale_ptr, - sizeof(b_scale_ptr)))) { - destroy_profile(cublas, cuda, out); - return 0; - } - } -#endif - - if (!check_cublas("create A layout", - cublas->cublasLtMatrixLayoutCreate(&out->a_layout, desc->a_type, out->k, out->m, out->k)) || - !check_cublas("create B layout", - cublas->cublasLtMatrixLayoutCreate(&out->b_layout, desc->b_type, out->k, out->n, out->k)) || - !check_cublas("create C layout", - cublas->cublasLtMatrixLayoutCreate(&out->c_layout, desc->c_type, out->m, out->n, out->m)) || - !check_cublas("create D layout", - cublas->cublasLtMatrixLayoutCreate(&out->d_layout, desc->d_type, out->m, out->n, out->m))) { - destroy_profile(cublas, cuda, out); - return 0; - } - - if (!check_cublas("create preference", cublas->cublasLtMatmulPreferenceCreate(&out->preference))) { - destroy_profile(cublas, cuda, out); - return 0; - } - - if (out->workspace_size > 0) { - if (!alloc_filled(cuda, &out->workspace_dev, out->workspace_size, 0x00)) { - destroy_profile(cublas, cuda, out); - return 0; - } - } - - if (!check_cublas("set workspace", - cublas->cublasLtMatmulPreferenceSetAttribute( - out->preference, - CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, - &out->workspace_size, - sizeof(out->workspace_size)))) { - destroy_profile(cublas, cuda, out); - return 0; - } - - int found = 0; - if (check_cublas("heuristic", - cublas->cublasLtMatmulAlgoGetHeuristic(handle, - out->op_desc, - out->a_layout, - out->b_layout, - out->c_layout, - out->d_layout, - out->preference, - 1, - &out->heuristic, - &found)) && - found > 0) { - out->ready = 1; - return 1; - } - - destroy_profile(cublas, cuda, out); - attempt_budget = round_down_size(attempt_budget * 3u / 4u, 256u); - if (attempt_budget < MIN_PROFILE_BUDGET_BYTES) { - break; - } - } - - return 0; -} - -static int run_cublas_profile(cublasLtHandle_t handle, - struct cublaslt_api *cublas, - struct prepared_profile *profile) { - int32_t alpha_i32 = 1; - int32_t beta_i32 = 0; - double alpha_f64 = 1.0; - double beta_f64 = 0.0; - float alpha = 1.0f; - float beta = 0.0f; - const void *alpha_ptr = α - const void *beta_ptr = β - if (profile->desc.compute_type == CUBLAS_COMPUTE_32I) { - alpha_ptr = &alpha_i32; - beta_ptr = &beta_i32; - } else if (profile->desc.compute_type == CUBLAS_COMPUTE_64F) { - alpha_ptr = &alpha_f64; - beta_ptr = &beta_f64; - } - return check_cublas(profile->desc.name, - cublas->cublasLtMatmul(handle, - profile->op_desc, - alpha_ptr, - (const void *)(uintptr_t)profile->a_dev, - profile->a_layout, - (const void *)(uintptr_t)profile->b_dev, - profile->b_layout, - beta_ptr, - (const void *)(uintptr_t)profile->c_dev, - profile->c_layout, - (void *)(uintptr_t)profile->d_dev, - profile->d_layout, - &profile->heuristic.algo, - (void *)(uintptr_t)profile->workspace_dev, - profile->workspace_size, - profile->stream)); -} - -static int run_cublaslt_stress(struct cuda_api *cuda, - CUdevice dev, - const char *device_name, - int cc_major, - int cc_minor, - int seconds, - int size_mb, - const char *precision_filter, - struct stress_report *report) { - struct cublaslt_api cublas; - struct prepared_profile prepared[MAX_STRESS_STREAMS * PROFILE_COUNT]; - cublasLtHandle_t handle = NULL; - CUcontext ctx = NULL; - CUstream streams[MAX_STRESS_STREAMS] = {0}; - uint16_t sample[256]; - int cc = cc_major * 10 + cc_minor; - int planned = 0; - int active = 0; - int mp_count = 0; - int stream_count = 1; - int profile_count = PROFILE_COUNT; - int prepared_count = 0; - size_t requested_budget = 0; - size_t total_budget = 0; - size_t per_profile_budget = 0; - int budget_profiles = 0; - - memset(report, 0, sizeof(*report)); - snprintf(report->backend, sizeof(report->backend), "cublasLt"); - snprintf(report->device, sizeof(report->device), "%s", device_name); - report->cc_major = cc_major; - report->cc_minor = cc_minor; - report->buffer_mb = size_mb; - - if (!load_cublaslt(&cublas)) { - snprintf(report->details, sizeof(report->details), "cublasLt=unavailable\n"); - return 0; - } - if (!check_rc(cuda, "cuCtxCreate", cuda->cuCtxCreate(&ctx, 0, dev))) { - return 0; - } - if (!check_cublas("cublasLtCreate", cublas.cublasLtCreate(&handle))) { - cuda->cuCtxDestroy(ctx); - return 0; - } - - /* Count profiles matching the filter (for deciding what to run). */ - for (size_t i = 0; i < sizeof(k_profiles) / sizeof(k_profiles[0]); i++) { - if (profile_allowed_for_run(&k_profiles[i], cc, precision_filter)) { - planned++; - } - } - if (planned <= 0) { - snprintf(report->details, sizeof(report->details), "cublasLt_profiles=unsupported\n"); - cublas.cublasLtDestroy(handle); - cuda->cuCtxDestroy(ctx); - return 0; - } - - /* Count all profiles active on this GPU regardless of filter. - * Mixed phases still divide budget across the full precision set, while - * single-precision benchmark phases dedicate budget only to active - * profiles matching precision_filter. */ - int planned_total = 0; - for (size_t i = 0; i < sizeof(k_profiles) / sizeof(k_profiles[0]); i++) { - if (profile_allowed_for_run(&k_profiles[i], cc, precision_filter)) { - planned_total++; - } - } - if (planned_total < planned) { - planned_total = planned; - } - budget_profiles = planned_total; - if (precision_filter != NULL) { - budget_profiles = planned; - } - if (budget_profiles <= 0) { - budget_profiles = planned_total; - } - - requested_budget = (size_t)size_mb * 1024u * 1024u; - if (requested_budget < (size_t)budget_profiles * MIN_PROFILE_BUDGET_BYTES) { - requested_budget = (size_t)budget_profiles * MIN_PROFILE_BUDGET_BYTES; - } - total_budget = clamp_budget_to_free_memory(cuda, requested_budget); - if (total_budget < (size_t)budget_profiles * MIN_PROFILE_BUDGET_BYTES) { - total_budget = (size_t)budget_profiles * MIN_PROFILE_BUDGET_BYTES; - } - if (query_multiprocessor_count(cuda, dev, &mp_count) && - cuda->cuStreamCreate && - cuda->cuStreamDestroy) { - stream_count = choose_stream_count(mp_count, budget_profiles, total_budget, 1); - } - if (precision_filter != NULL && stream_count > MAX_SINGLE_PRECISION_STREAMS) { - stream_count = MAX_SINGLE_PRECISION_STREAMS; - } - if (stream_count > 1) { - int created = 0; - for (; created < stream_count; created++) { - if (!check_rc(cuda, "cuStreamCreate", cuda->cuStreamCreate(&streams[created], 0))) { - destroy_streams(cuda, streams, created); - stream_count = 1; - break; - } - } - } - report->stream_count = stream_count; - per_profile_budget = total_budget / ((size_t)budget_profiles * (size_t)stream_count); - if (per_profile_budget < MIN_PROFILE_BUDGET_BYTES) { - per_profile_budget = MIN_PROFILE_BUDGET_BYTES; - } - if (precision_filter != NULL) { - per_profile_budget = clamp_single_precision_profile_budget(per_profile_budget); - } - report->buffer_mb = (int)(total_budget / (1024u * 1024u)); - append_detail(report->details, - sizeof(report->details), - "requested_mb=%d actual_mb=%d streams=%d mp_count=%d budget_profiles=%d per_worker_mb=%zu\n", - size_mb, - report->buffer_mb, - report->stream_count, - mp_count, - budget_profiles, - per_profile_budget / (1024u * 1024u)); - - for (int i = 0; i < profile_count; i++) { - const struct profile_desc *desc = &k_profiles[i]; - if (!(desc->enabled && cc >= desc->min_cc)) { - append_detail(report->details, - sizeof(report->details), - "%s=SKIPPED cc<%d\n", - desc->name, - desc->min_cc); - continue; - } - if (!profile_allowed_for_run(desc, cc, precision_filter)) { - append_detail(report->details, - sizeof(report->details), - "%s=SKIPPED benchmark_disabled\n", - desc->name); - continue; - } - for (int lane = 0; lane < stream_count; lane++) { - CUstream stream = streams[lane]; - if (prepared_count >= (int)(sizeof(prepared) / sizeof(prepared[0]))) { - break; - } - if (prepare_profile(&cublas, handle, cuda, desc, stream, per_profile_budget, &prepared[prepared_count])) { - active++; - append_detail(report->details, - sizeof(report->details), - "%s[%d]=READY dim=%llux%llux%llu block=%s stream=%d\n", - desc->name, - lane, - (unsigned long long)prepared[prepared_count].m, - (unsigned long long)prepared[prepared_count].n, - (unsigned long long)prepared[prepared_count].k, - desc->block_label, - lane); - prepared_count++; - } else { - append_detail(report->details, - sizeof(report->details), - "%s[%d]=SKIPPED unsupported\n", - desc->name, - lane); - } - } - } - - if (active <= 0) { - cublas.cublasLtDestroy(handle); - destroy_streams(cuda, streams, stream_count); - cuda->cuCtxDestroy(ctx); - return 0; - } - - /* Keep the GPU queue continuously full by submitting kernels without - * synchronizing after every wave. A sync barrier after each small batch - * creates CPU↔GPU ping-pong gaps that prevent full TDP utilisation, - * especially when individual kernels are short. Instead we sync at most - * once per second (for error detection) and once at the very end. */ - double deadline = now_seconds() + (double)seconds; - double next_sync = now_seconds() + 1.0; - while (now_seconds() < deadline) { - int launched = 0; - for (int i = 0; i < prepared_count; i++) { - if (!prepared[i].ready) { - continue; - } - if (!run_cublas_profile(handle, &cublas, &prepared[i])) { - append_detail(report->details, - sizeof(report->details), - "%s=FAILED runtime\n", - prepared[i].desc.name); - for (int j = 0; j < prepared_count; j++) { - destroy_profile(&cublas, cuda, &prepared[j]); - } - cublas.cublasLtDestroy(handle); - destroy_streams(cuda, streams, stream_count); - cuda->cuCtxDestroy(ctx); - return 0; - } - prepared[i].iterations++; - report->iterations++; - launched++; - } - if (launched <= 0) { - break; - } - double now = now_seconds(); - if (now >= next_sync || now >= deadline) { - if (!check_rc(cuda, "cuCtxSynchronize", cuda->cuCtxSynchronize())) { - for (int i = 0; i < prepared_count; i++) { - destroy_profile(&cublas, cuda, &prepared[i]); - } - cublas.cublasLtDestroy(handle); - destroy_streams(cuda, streams, stream_count); - cuda->cuCtxDestroy(ctx); - return 0; - } - next_sync = now + 1.0; - } - } - /* Final drain — ensure all queued work finishes before we read results. */ - cuda->cuCtxSynchronize(); - - for (int i = 0; i < prepared_count; i++) { - if (!prepared[i].ready) { - continue; - } - append_detail(report->details, - sizeof(report->details), - "%s_iterations=%lu\n", - prepared[i].desc.name, - prepared[i].iterations); - } - - for (int i = 0; i < prepared_count; i++) { - if (prepared[i].ready) { - if (check_rc(cuda, "cuMemcpyDtoH", cuda->cuMemcpyDtoH(sample, prepared[i].d_dev, sizeof(sample)))) { - for (size_t j = 0; j < sizeof(sample) / sizeof(sample[0]); j++) { - report->checksum += sample[j]; - } - } - break; - } - } - - for (int i = 0; i < prepared_count; i++) { - destroy_profile(&cublas, cuda, &prepared[i]); - } - cublas.cublasLtDestroy(handle); - destroy_streams(cuda, streams, stream_count); - cuda->cuCtxDestroy(ctx); - return 1; -} -#endif - -static void print_stress_report(const struct stress_report *report, int device_index, int seconds) { - printf("device=%s\n", report->device); - printf("device_index=%d\n", device_index); - printf("compute_capability=%d.%d\n", report->cc_major, report->cc_minor); - printf("backend=%s\n", report->backend); - printf("duration_s=%d\n", seconds); - printf("buffer_mb=%d\n", report->buffer_mb); - printf("streams=%d\n", report->stream_count); - printf("iterations=%lu\n", report->iterations); - printf("checksum=%llu\n", (unsigned long long)report->checksum); - if (report->details[0] != '\0') { - printf("%s", report->details); - } - printf("status=OK\n"); -} - -int main(int argc, char **argv) { - int seconds = 5; - int size_mb = 64; - int device_index = 0; - const char *precision_filter = NULL; /* NULL = all; else block_label to match */ - const char *precision_plan = NULL; - const char *precision_plan_seconds = NULL; - for (int i = 1; i < argc; i++) { - if ((strcmp(argv[i], "--seconds") == 0 || strcmp(argv[i], "-t") == 0) && i + 1 < argc) { - seconds = atoi(argv[++i]); - } else if ((strcmp(argv[i], "--size-mb") == 0 || strcmp(argv[i], "-m") == 0) && i + 1 < argc) { - size_mb = atoi(argv[++i]); - } else if ((strcmp(argv[i], "--device") == 0 || strcmp(argv[i], "-d") == 0) && i + 1 < argc) { - device_index = atoi(argv[++i]); - } else if (strcmp(argv[i], "--precision") == 0 && i + 1 < argc) { - precision_filter = argv[++i]; - } else if (strcmp(argv[i], "--precision-plan") == 0 && i + 1 < argc) { - precision_plan = argv[++i]; - } else if (strcmp(argv[i], "--precision-plan-seconds") == 0 && i + 1 < argc) { - precision_plan_seconds = argv[++i]; - } else { - fprintf(stderr, - "usage: %s [--seconds N] [--size-mb N] [--device N] [--precision int8|fp8|fp16|fp32|fp64|fp4] [--precision-plan p1,p2,...,mixed] [--precision-plan-seconds s1,s2,...]\n", - argv[0]); - return 2; - } - } - if (seconds <= 0) { - seconds = 5; - } - if (size_mb <= 0) { - size_mb = 64; - } - if (device_index < 0) { - device_index = 0; - } - - struct cuda_api cuda; - if (!load_cuda(&cuda)) { - fprintf(stderr, "failed to load libcuda.so.1 or required Driver API symbols\n"); - return 1; - } - load_symbol(cuda.lib, "cuGetErrorName", (void **)&cuda.cuGetErrorName); - load_symbol(cuda.lib, "cuGetErrorString", (void **)&cuda.cuGetErrorString); - - if (!check_rc(&cuda, "cuInit", cuda.cuInit(0))) { - return 1; - } - - int count = 0; - if (!check_rc(&cuda, "cuDeviceGetCount", cuda.cuDeviceGetCount(&count))) { - return 1; - } - if (count <= 0) { - fprintf(stderr, "no CUDA devices found\n"); - return 1; - } - - if (device_index >= count) { - fprintf(stderr, "device index %d out of range (found %d CUDA device(s))\n", device_index, count); - return 1; - } - - CUdevice dev = 0; - if (!check_rc(&cuda, "cuDeviceGet", cuda.cuDeviceGet(&dev, device_index))) { - return 1; - } - - char name[128] = {0}; - if (!check_rc(&cuda, "cuDeviceGetName", cuda.cuDeviceGetName(name, (int)sizeof(name), dev))) { - return 1; - } - - int cc_major = 0; - int cc_minor = 0; - if (!query_compute_capability(&cuda, dev, &cc_major, &cc_minor)) { - return 1; - } - - struct stress_report report; - int ok = 0; - -#if HAVE_CUBLASLT_HEADERS - if (precision_plan != NULL && precision_plan[0] != '\0') { - char *plan_copy = strdup(precision_plan); - char *plan_seconds_copy = NULL; - int phase_seconds[32] = {0}; - int phase_seconds_count = 0; - int phase_ok = 0; - if (plan_copy == NULL) { - fprintf(stderr, "failed to allocate precision plan buffer\n"); - return 1; - } - if (precision_plan_seconds != NULL && precision_plan_seconds[0] != '\0') { - plan_seconds_copy = strdup(precision_plan_seconds); - if (plan_seconds_copy == NULL) { - free(plan_copy); - fprintf(stderr, "failed to allocate precision plan seconds buffer\n"); - return 1; - } - for (char *sec_token = strtok(plan_seconds_copy, ","); - sec_token != NULL && phase_seconds_count < (int)(sizeof(phase_seconds) / sizeof(phase_seconds[0])); - sec_token = strtok(NULL, ",")) { - while (*sec_token == ' ' || *sec_token == '\t') { - sec_token++; - } - if (*sec_token == '\0') { - continue; - } - phase_seconds[phase_seconds_count++] = atoi(sec_token); - } - } - int phase_idx = 0; - for (char *token = strtok(plan_copy, ","); token != NULL; token = strtok(NULL, ","), phase_idx++) { - while (*token == ' ' || *token == '\t') { - token++; - } - if (*token == '\0') { - continue; - } - const char *phase_name = token; - const char *phase_filter = token; - if (strcmp(token, "mixed") == 0 || strcmp(token, "all") == 0) { - phase_filter = NULL; - } - int phase_duration = seconds; - if (phase_idx < phase_seconds_count && phase_seconds[phase_idx] > 0) { - phase_duration = phase_seconds[phase_idx]; - } - printf("phase_begin=%s\n", phase_name); - fflush(stdout); - memset(&report, 0, sizeof(report)); - ok = run_cublaslt_stress(&cuda, dev, name, cc_major, cc_minor, phase_duration, size_mb, phase_filter, &report); - if (ok) { - print_stress_report(&report, device_index, phase_duration); - phase_ok = 1; - } else { - printf("phase_error=%s\n", phase_name); - if (report.details[0] != '\0') { - printf("%s", report.details); - if (report.details[strlen(report.details) - 1] != '\n') { - printf("\n"); - } - } - printf("status=FAILED\n"); - } - printf("phase_end=%s\n", phase_name); - fflush(stdout); - } - free(plan_seconds_copy); - free(plan_copy); - return phase_ok ? 0 : 1; - } - ok = run_cublaslt_stress(&cuda, dev, name, cc_major, cc_minor, seconds, size_mb, precision_filter, &report); -#endif - if (!ok) { - if (precision_filter != NULL) { - fprintf(stderr, - "requested precision path unavailable: precision=%s device=%s cc=%d.%d\n", - precision_filter, - name, - cc_major, - cc_minor); - return 1; - } - int ptx_mb = size_mb; - if (!run_ptx_fallback(&cuda, dev, name, cc_major, cc_minor, seconds, ptx_mb, &report)) { - return 1; - } - } - - print_stress_report(&report, device_index, seconds); - return 0; -} +/* Kept as one translation unit so the dynamically loaded CUDA ABI and static + * helpers remain private, while each backend can be reviewed independently. */ +#include "bee-gpu-stress-cuda.inc" +#include "bee-gpu-stress-cublaslt.inc" +#include "bee-gpu-stress-main.inc" diff --git a/iso/builder/build.sh b/iso/builder/build.sh index 08652c5..f71f0bd 100755 --- a/iso/builder/build.sh +++ b/iso/builder/build.sh @@ -1,5 +1,5 @@ #!/bin/sh -# build.sh — internal ISO build entrypoint executed inside the builder container. +# build.sh - internal ISO build entrypoint executed inside the builder container. set -e @@ -74,6 +74,12 @@ mkdir -p "${CACHE_ROOT}" : "${GOMODCACHE:=${CACHE_ROOT}/go-mod}" export GOCACHE GOMODCACHE +# Keep the entrypoint focused on build orchestration. These libraries only +# define helpers; stateful logging is loaded after its output paths are known. +. "${BUILDER_DIR}/lib/iso-validation.sh" +. "${BUILDER_DIR}/lib/bootloader.sh" +. "${BUILDER_DIR}/lib/fast-path.sh" + resolve_project_version() { if [ -n "${BEE_VERSION:-}" ]; then echo "${BEE_VERSION}" @@ -131,7 +137,9 @@ sync_builder_workdir() { rsync -a --delete \ --exclude='cache/' \ --exclude='chroot/' \ + --exclude='binary/' \ --exclude='.build/' \ + --exclude='.stage/' \ --exclude='*.iso' \ --exclude='*.packages' \ --exclude='*.contents' \ @@ -148,1096 +156,6 @@ sync_builder_workdir() { fi } -iso_list_files() { - iso_path="$1" - - if command -v bsdtar >/dev/null 2>&1; then - bsdtar -tf "$iso_path" - return $? - fi - - if command -v xorriso >/dev/null 2>&1; then - xorriso -indev "$iso_path" -find / -type f -print 2>/dev/null | sed 's#^/##' - return $? - fi - - return 127 -} - -iso_extract_file() { - iso_path="$1" - iso_member="$2" - - if command -v bsdtar >/dev/null 2>&1; then - bsdtar -xOf "$iso_path" "$iso_member" - return $? - fi - - if command -v xorriso >/dev/null 2>&1; then - xorriso -osirrox on -indev "$iso_path" -cat "/$iso_member" 2>/dev/null - return $? - fi - - return 127 -} - -iso_read_file_list() { - iso_path="$1" - out_path="$2" - - iso_list_files "$iso_path" > "$out_path" || return 1 - [ -s "$out_path" ] || return 1 - return 0 -} - -iso_read_member() { - iso_path="$1" - iso_member="$2" - out_path="$3" - - iso_extract_file "$iso_path" "$iso_member" > "$out_path" || return 1 - [ -s "$out_path" ] || return 1 - return 0 -} - -require_iso_reader() { - command -v bsdtar >/dev/null 2>&1 && return 0 - command -v xorriso >/dev/null 2>&1 && return 0 - memtest_fail "ISO reader is required for validation/debug (expected bsdtar or xorriso)" "${1:-}" -} - -dump_memtest_debug() { - phase="$1" - lb_dir="${2:-}" - iso_path="${3:-}" - phase_slug="$(printf '%s' "${phase}" | tr ' /' '__')" - memtest_log="${LOG_DIR:-}/memtest-${phase_slug}.log" - - ( - echo "=== memtest debug: ${phase} ===" - - echo "-- auto/config --" - if [ -f "${BUILDER_DIR}/auto/config" ]; then - grep -n -- '--memtest' "${BUILDER_DIR}/auto/config" || echo " (no --memtest line found)" - else - echo " (missing ${BUILDER_DIR}/auto/config)" - fi - - echo "-- source bootloader templates --" - for cfg in \ - "${BUILDER_DIR}/config/bootloaders/grub-efi/grub.cfg" \ - "${BUILDER_DIR}/config/bootloaders/isolinux/live.cfg.in"; do - if [ -f "$cfg" ]; then - echo " file: $cfg" - grep -n 'Memory Test\|memtest' "$cfg" || echo " (no memtest lines)" - fi - done - - echo "-- source binary hooks --" - for hook in \ - "${BUILDER_DIR}/config/hooks/normal/9100-memtest.hook.binary"; do - if [ -f "$hook" ]; then - echo " hook: $hook" - else - echo " (missing $hook)" - fi - done - - if [ -n "$lb_dir" ] && [ -d "$lb_dir" ]; then - echo "-- live-build workdir package lists --" - for pkg in \ - "$lb_dir/config/package-lists/bee.list.chroot" \ - "$lb_dir/config/package-lists/bee-gpu.list.chroot" \ - "$lb_dir/config/package-lists/bee-nvidia.list.chroot"; do - if [ -f "$pkg" ]; then - echo " file: $pkg" - grep -n 'memtest' "$pkg" || echo " (no memtest lines)" - fi - done - - echo "-- live-build chroot/boot --" - if [ -d "$lb_dir/chroot/boot" ]; then - find "$lb_dir/chroot/boot" -maxdepth 1 -name 'memtest*' -print | sed 's/^/ /' || true - else - echo " (missing $lb_dir/chroot/boot)" - fi - - echo "-- live-build binary/boot --" - if [ -d "$lb_dir/binary/boot" ]; then - find "$lb_dir/binary/boot" -maxdepth 1 -name 'memtest*' -print | sed 's/^/ /' || true - else - echo " (missing $lb_dir/binary/boot)" - fi - - echo "-- live-build binary grub cfg --" - if [ -f "$lb_dir/binary/boot/grub/grub.cfg" ]; then - grep -n 'Memory Test\|memtest' "$lb_dir/binary/boot/grub/grub.cfg" || echo " (no memtest lines)" - else - echo " (missing $lb_dir/binary/boot/grub/grub.cfg)" - fi - - echo "-- live-build binary isolinux cfg --" - if [ -f "$lb_dir/binary/isolinux/live.cfg" ]; then - grep -n 'Memory Test\|memtest' "$lb_dir/binary/isolinux/live.cfg" || echo " (no memtest lines)" - else - echo " (missing $lb_dir/binary/isolinux/live.cfg)" - fi - - echo "-- live-build package cache --" - if [ -d "$lb_dir/cache/packages.chroot" ]; then - find "$lb_dir/cache/packages.chroot" -maxdepth 1 -name 'memtest86+*.deb' -print | sed 's/^/ /' || true - else - echo " (missing $lb_dir/cache/packages.chroot)" - fi - fi - - if [ -n "$iso_path" ] && [ -f "$iso_path" ]; then - iso_files="$(mktemp)" - iso_grub_cfg="$(mktemp)" - iso_isolinux_cfg="$(mktemp)" - - echo "-- ISO memtest files --" - if iso_read_file_list "$iso_path" "$iso_files"; then - grep 'memtest' "$iso_files" | sed 's/^/ /' || echo " (no memtest files in ISO)" - else - echo " (failed to list ISO contents)" - fi - - echo "-- ISO GRUB memtest lines --" - if iso_read_member "$iso_path" boot/grub/grub.cfg "$iso_grub_cfg"; then - grep -n 'Memory Test\|memtest' "$iso_grub_cfg" || echo " (no memtest lines in boot/grub/grub.cfg)" - else - echo " (failed to read boot/grub/grub.cfg from ISO)" - fi - - echo "-- ISO isolinux memtest lines --" - if iso_read_member "$iso_path" isolinux/live.cfg "$iso_isolinux_cfg"; then - grep -n 'Memory Test\|memtest' "$iso_isolinux_cfg" || echo " (no memtest lines in isolinux/live.cfg)" - else - echo " (failed to read isolinux/live.cfg from ISO)" - fi - - rm -f "$iso_files" "$iso_grub_cfg" "$iso_isolinux_cfg" - fi - - echo "=== end memtest debug: ${phase} ===" - ) | { - if [ -n "${LOG_DIR:-}" ] && [ -d "${LOG_DIR}" ]; then - tee "${memtest_log}" - else - cat - fi - } -} - -memtest_fail() { - msg="$1" - iso_path="${2:-}" - level="WARNING" - if [ "${BEE_REQUIRE_MEMTEST:-0}" = "1" ]; then - level="ERROR" - fi - echo "${level}: ${msg}" >&2 - dump_memtest_debug "failure" "${LB_DIR:-}" "$iso_path" >&2 - if [ "${BEE_REQUIRE_MEMTEST:-0}" = "1" ]; then - exit 1 - fi - return 0 -} - -nvidia_runtime_fail() { - msg="$1" - echo "ERROR: ${msg}" >&2 - exit 1 -} - -iso_memtest_present() { - iso_path="$1" - iso_files="$(mktemp)" - - [ -f "$iso_path" ] || return 1 - - if command -v bsdtar >/dev/null 2>&1; then - : - elif command -v xorriso >/dev/null 2>&1; then - : - else - return 2 - fi - - iso_read_file_list "$iso_path" "$iso_files" || { - rm -f "$iso_files" - return 2 - } - - grep -q '^boot/memtest86+x64\.bin$' "$iso_files" || { - rm -f "$iso_files" - return 1 - } - grep -q '^boot/memtest86+x64\.efi$' "$iso_files" || { - rm -f "$iso_files" - return 1 - } - - grub_cfg="$(mktemp)" - isolinux_cfg="$(mktemp)" - - iso_read_member "$iso_path" boot/grub/grub.cfg "$grub_cfg" || { - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 2 - } - iso_read_member "$iso_path" isolinux/live.cfg "$isolinux_cfg" || { - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 2 - } - - grep -q 'Memory Test (memtest86+)' "$grub_cfg" || { - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 1 - } - grep -q '/boot/memtest86+x64\.efi' "$grub_cfg" || { - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 1 - } - grep -q '/boot/memtest86+x64\.bin' "$grub_cfg" || { - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 1 - } - grep -q 'Memory Test (memtest86+)' "$isolinux_cfg" || { - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 1 - } - grep -q '/boot/memtest86+x64\.bin' "$isolinux_cfg" || { - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 1 - } - - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 0 -} - -validate_iso_memtest() { - iso_path="$1" - echo "=== validating memtest in ISO ===" - - [ -f "$iso_path" ] || { - memtest_fail "ISO not found for validation: $iso_path" "$iso_path" - return 0 - } - require_iso_reader "$iso_path" || return 0 - - iso_files="$(mktemp)" - iso_read_file_list "$iso_path" "$iso_files" || { - memtest_fail "failed to list ISO contents while validating memtest" "$iso_path" - rm -f "$iso_files" - return 0 - } - - grep -q '^boot/memtest86+x64\.bin$' "$iso_files" || { - memtest_fail "memtest BIOS binary missing in ISO: boot/memtest86+x64.bin" "$iso_path" - rm -f "$iso_files" - return 0 - } - grep -q '^boot/memtest86+x64\.efi$' "$iso_files" || { - memtest_fail "memtest EFI binary missing in ISO: boot/memtest86+x64.efi" "$iso_path" - rm -f "$iso_files" - return 0 - } - - grub_cfg="$(mktemp)" - isolinux_cfg="$(mktemp)" - - iso_read_member "$iso_path" boot/grub/grub.cfg "$grub_cfg" || { - memtest_fail "failed to read boot/grub/grub.cfg from ISO" "$iso_path" - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 0 - } - iso_read_member "$iso_path" isolinux/live.cfg "$isolinux_cfg" || { - memtest_fail "failed to read isolinux/live.cfg from ISO" "$iso_path" - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 0 - } - - grep -q 'Memory Test (memtest86+)' "$grub_cfg" || { - memtest_fail "GRUB menu entry for memtest is missing" "$iso_path" - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 0 - } - grep -q '/boot/memtest86+x64\.efi' "$grub_cfg" || { - memtest_fail "GRUB memtest EFI path is missing" "$iso_path" - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 0 - } - grep -q '/boot/memtest86+x64\.bin' "$grub_cfg" || { - memtest_fail "GRUB memtest BIOS path is missing" "$iso_path" - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 0 - } - grep -q 'Memory Test (memtest86+)' "$isolinux_cfg" || { - memtest_fail "isolinux menu entry for memtest is missing" "$iso_path" - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 0 - } - grep -q '/boot/memtest86+x64\.bin' "$isolinux_cfg" || { - memtest_fail "isolinux memtest path is missing" "$iso_path" - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - return 0 - } - - rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" - echo "=== memtest validation OK ===" -} - -validate_live_cmdline_params() { - cfg="$1" - command="$2" - bootloader="$3" - - awk -v command="$command" -v bootloader="$bootloader" ' - function has(token, i) { - for (i = 1; i <= NF; i++) { - if ($i == token) { - return 1 - } - } - return 0 - } - - function reject(message) { - printf "ERROR: %s live entry at %s:%d: %s\n", bootloader, FILENAME, NR, message - bad = 1 - } - - $1 == command && has("boot=live") { - live_entries++ - - if (!has("udev.children_max=1")) { - reject("missing udev.children_max=1") - } - if (!has("intel_iommu=on")) { - reject("missing intel_iommu=on") - } - if (!has("iommu.passthrough=0")) { - reject("missing iommu.passthrough=0") - } - if (!has("efi=disable_early_pci_dma")) { - reject("missing efi=disable_early_pci_dma") - } - if (has("iommu=pt")) { - reject("contains forbidden iommu=pt") - } - - if (has("pci=realloc")) { - failsafe_entries++ - if (!has("iommu.strict=1")) { - reject("pci=realloc entry is missing iommu.strict=1") - } - } else if (has("iommu.strict=1")) { - reject("iommu.strict=1 is allowed only in the pci=realloc fail-safe entry") - } - } - - END { - if (live_entries == 0) { - printf "ERROR: %s config has no live boot entries\n", bootloader - bad = 1 - } - if (failsafe_entries != 1) { - printf "ERROR: %s config has %d pci=realloc fail-safe entries, expected 1\n", bootloader, failsafe_entries - bad = 1 - } - exit bad ? 1 : 0 - } - ' "$cfg" -} - -validate_iso_live_boot_entries() { - iso_path="$1" - echo "=== validating live boot entries in ISO ===" - - [ -f "$iso_path" ] || { - echo "ERROR: ISO not found for live boot validation: $iso_path" >&2 - exit 1 - } - require_iso_reader "$iso_path" >/dev/null 2>&1 || { - echo "ERROR: ISO reader unavailable for live boot validation" >&2 - exit 1 - } - - grub_cfg="$(mktemp)" - isolinux_cfg="$(mktemp)" - - iso_read_member "$iso_path" boot/grub/grub.cfg "$grub_cfg" || { - echo "ERROR: failed to read boot/grub/grub.cfg from ISO" >&2 - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - } - iso_read_member "$iso_path" isolinux/live.cfg "$isolinux_cfg" || { - echo "ERROR: failed to read isolinux/live.cfg from ISO" >&2 - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - } - - if grep -q '@APPEND_LIVE@\|@KERNEL_LIVE@\|@INITRD_LIVE@' "$grub_cfg" "$isolinux_cfg"; then - echo "ERROR: unresolved live-build placeholders remain in ISO bootloader config" >&2 - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - fi - - if grep -q 'iommu=pt' "$grub_cfg" "$isolinux_cfg"; then - echo "ERROR: forbidden iommu=pt remains in ISO bootloader config" >&2 - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - fi - - if ! validate_live_cmdline_params "$grub_cfg" linux GRUB; then - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - fi - if ! validate_live_cmdline_params "$isolinux_cfg" append isolinux; then - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - fi - - grep -q 'menuentry "EASY-BEE v' "$grub_cfg" || { - echo "ERROR: GRUB default EASY-BEE entry is missing" >&2 - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - } - grep -q 'menuentry "EASY-BEE v.* -- load to RAM (toram)"' "$grub_cfg" || { - echo "ERROR: GRUB toram entry is missing" >&2 - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - } - grep -q 'linux .*boot=live ' "$grub_cfg" || { - echo "ERROR: GRUB live entry is missing boot=live" >&2 - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - } - grep -q 'linux .*boot=live .*toram ' "$grub_cfg" || { - echo "ERROR: GRUB toram entry is missing boot=live or toram" >&2 - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - } - grep -q 'linux .*live-media-label=EASY_BEE_' "$grub_cfg" || { - echo "ERROR: GRUB live entry is missing live-media-label pinning" >&2 - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - } - - grep -q 'append .*boot=live ' "$isolinux_cfg" || { - echo "ERROR: isolinux live entry is missing boot=live" >&2 - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - } - grep -q 'append .*boot=live .*toram ' "$isolinux_cfg" || { - echo "ERROR: isolinux toram entry is missing boot=live or toram" >&2 - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - } - grep -q 'append .*live-media-label=EASY_BEE_' "$isolinux_cfg" || { - echo "ERROR: isolinux live entry is missing live-media-label pinning" >&2 - rm -f "$grub_cfg" "$isolinux_cfg" - exit 1 - } - - rm -f "$grub_cfg" "$isolinux_cfg" - echo "=== live boot validation OK ===" -} - -validate_iso_grub_assets() { - iso_path="$1" - echo "=== validating GRUB assets in ISO ===" - - [ -f "$iso_path" ] || { - echo "ERROR: ISO not found for GRUB asset validation: $iso_path" >&2 - exit 1 - } - require_iso_reader "$iso_path" >/dev/null 2>&1 || { - echo "ERROR: ISO reader unavailable for GRUB asset validation" >&2 - exit 1 - } - - iso_files="$(mktemp)" - iso_list_files "$iso_path" > "$iso_files" || { - echo "ERROR: failed to list ISO files for GRUB asset validation" >&2 - rm -f "$iso_files" - exit 1 - } - - for required in \ - boot/grub/config.cfg \ - boot/grub/grub.cfg; do - grep -q "^${required}$" "$iso_files" || { - echo "ERROR: missing GRUB asset in ISO: ${required}" >&2 - rm -f "$iso_files" - exit 1 - } - done - - rm -f "$iso_files" - echo "=== GRUB asset validation OK ===" -} - -validate_iso_nvidia_runtime() { - iso_path="$1" - [ "$BEE_GPU_VENDOR" = "nvidia" ] || return 0 - - echo "=== validating NVIDIA runtime in ISO ===" - - [ -f "$iso_path" ] || nvidia_runtime_fail "ISO not found for NVIDIA runtime validation: $iso_path" - require_iso_reader "$iso_path" >/dev/null 2>&1 || nvidia_runtime_fail "ISO reader unavailable for NVIDIA runtime validation" - command -v unsquashfs >/dev/null 2>&1 || nvidia_runtime_fail "unsquashfs is required for NVIDIA runtime validation" - - squashfs_tmp="$(mktemp)" - squashfs_list="$(mktemp)" - iso_files="$(mktemp)" - dpkg_status_dir="$(mktemp -d)" - iso_list_files "$iso_path" > "$iso_files" || { - rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" - rm -rf "$dpkg_status_dir" - nvidia_runtime_fail "failed to list ISO files for NVIDIA runtime validation" - } - grep '^live/.*\.squashfs$' "$iso_files" | while IFS= read -r squashfs_member; do - iso_read_member "$iso_path" "$squashfs_member" "$squashfs_tmp" || { - rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" - rm -rf "$dpkg_status_dir" - nvidia_runtime_fail "failed to extract $squashfs_member from ISO" - } - unsquashfs -ll "$squashfs_tmp" >> "$squashfs_list" 2>/dev/null || { - rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" - rm -rf "$dpkg_status_dir" - nvidia_runtime_fail "failed to inspect $squashfs_member from ISO" - } - # var/lib/dpkg/status lives in whichever squashfs layer has the base - # rootfs (not the usr/firmware split-off layers); harmless no-op on - # the others. - unsquashfs -d "${dpkg_status_dir}/extract" -f "$squashfs_tmp" var/lib/dpkg/status >/dev/null 2>&1 || true - : > "$squashfs_tmp" - done - - grep -Eq 'usr/bin/dcgmi$' "$squashfs_list" || { - rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" - rm -rf "$dpkg_status_dir" - nvidia_runtime_fail "dcgmi missing from final NVIDIA ISO" - } - grep -Eq 'usr/bin/nv-hostengine$' "$squashfs_list" || { - rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" - rm -rf "$dpkg_status_dir" - nvidia_runtime_fail "nv-hostengine missing from final NVIDIA ISO" - } - grep -Eq 'usr/bin/dcgmproftester([0-9]+)?$' "$squashfs_list" || { - rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" - rm -rf "$dpkg_status_dir" - nvidia_runtime_fail "dcgmproftester missing from final NVIDIA ISO" - } - - # Cross-check the DCGM package version actually baked into the squashfs - # against VERSIONS. dcgmi/nv-hostengine/dcgmproftester presence alone - # doesn't catch a stale squashfs served by a mis-detected fast-path build - # (dcgmi stays present across DCGM versions) — this does. - dpkg_status_file="${dpkg_status_dir}/extract/var/lib/dpkg/status" - if [ -f "$dpkg_status_file" ]; then - _installed_dcgm_version="$(awk ' - /^Package: datacenter-gpu-manager-4-core$/ { in_pkg=1; next } - /^Package: / { in_pkg=0 } - in_pkg && /^Version: / { sub(/^Version: /, ""); print; exit } - ' "$dpkg_status_file")" - # dpkg records an epoch-qualified version ("1:4.5.2-1"); VERSIONS - # pins the same value without the epoch, so match by suffix. - case "$_installed_dcgm_version" in - *"${DCGM_VERSION}") - ;; - "") - echo "=== WARNING: datacenter-gpu-manager-4-core not found in ISO dpkg status; skipping DCGM version check ===" - ;; - *) - rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" - rm -rf "$dpkg_status_dir" - nvidia_runtime_fail "DCGM version mismatch: VERSIONS pins ${DCGM_VERSION} but ISO has ${_installed_dcgm_version} (stale squashfs — retry with --clean-build)" - ;; - esac - else - echo "=== WARNING: could not read dpkg status from ISO; skipping DCGM version check ===" - fi - - rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" - rm -rf "$dpkg_status_dir" - echo "=== NVIDIA runtime validation OK ===" -} - -append_memtest_grub_entry() { - grub_cfg="$1" - [ -f "$grub_cfg" ] || return 1 - grep -q 'Memory Test (memtest86+)' "$grub_cfg" && return 0 - grep -q '### BEE MEMTEST ###' "$grub_cfg" && return 0 - - cat >> "$grub_cfg" <<'EOF' - -### BEE MEMTEST ### -if [ "${grub_platform}" = "efi" ]; then - menuentry "Memory Test (memtest86+)" { - chainloader /boot/memtest86+x64.efi - } -else - menuentry "Memory Test (memtest86+)" { - linux16 /boot/memtest86+x64.bin - } -fi -### /BEE MEMTEST ### -EOF -} - -append_memtest_isolinux_entry() { - isolinux_cfg="$1" - [ -f "$isolinux_cfg" ] || return 1 - grep -q 'Memory Test (memtest86+)' "$isolinux_cfg" && return 0 - grep -q '### BEE MEMTEST ###' "$isolinux_cfg" && return 0 - - cat >> "$isolinux_cfg" <<'EOF' - -# ### BEE MEMTEST ### -label memtest - menu label ^Memory Test (memtest86+) - linux /boot/memtest86+x64.bin -# ### /BEE MEMTEST ### -EOF -} - -extract_live_grub_entry() { - cfg="$1" - live_linux="$(awk '/^[[:space:]]*linux[[:space:]]+\/live\// { print; exit }' "$cfg")" - live_initrd="$(awk '/^[[:space:]]*initrd[[:space:]]+\/live\// { print; exit }' "$cfg")" - [ -n "$live_linux" ] || return 1 - [ -n "$live_initrd" ] || return 1 - - grub_kernel="$(printf '%s\n' "$live_linux" | awk '{print $2}')" - grub_append="$(printf '%s\n' "$live_linux" | cut -d' ' -f3-)" - grub_initrd="$(printf '%s\n' "$live_initrd" | awk '{print $2}')" - [ -n "$grub_kernel" ] || return 1 - [ -n "$grub_append" ] || return 1 - [ -n "$grub_initrd" ] || return 1 - return 0 -} - -load_live_build_append() { - lb_dir="$1" - binary_cfg="$lb_dir/config/binary" - [ -f "$binary_cfg" ] || return 1 - - # config/binary is generated by live-build and contains shell variable - # assignments such as LB_BOOTAPPEND_LIVE="boot=live ...". - # shellcheck disable=SC1090 - . "$binary_cfg" - - [ -n "${LB_BOOTAPPEND_LIVE:-}" ] || return 1 - live_build_append="$LB_BOOTAPPEND_LIVE" - return 0 -} - -extract_live_isolinux_entry() { - cfg="$1" - isolinux_linux="$(awk '/^[[:space:]]*linux[[:space:]]+\/live\// { print; exit }' "$cfg")" - isolinux_initrd="$(awk '/^[[:space:]]*initrd[[:space:]]+\/live\// { print; exit }' "$cfg")" - isolinux_append="$(awk '/^[[:space:]]*append[[:space:]]+/ { sub(/^[[:space:]]*append[[:space:]]+/, ""); print; exit }' "$cfg")" - [ -n "$isolinux_linux" ] || return 1 - [ -n "$isolinux_initrd" ] || return 1 - [ -n "$isolinux_append" ] || return 1 - - isolinux_kernel="$(printf '%s\n' "$isolinux_linux" | awk '{print $2}')" - isolinux_initrd_path="$(printf '%s\n' "$isolinux_initrd" | awk '{print $2}')" - [ -n "$isolinux_kernel" ] || return 1 - [ -n "$isolinux_initrd_path" ] || return 1 - return 0 -} - -write_canonical_grub_cfg() { - cfg="$1" - kernel="$2" - append_live="$3" - initrd="$4" - version_label="${PROJECT_VERSION_EFFECTIVE}" - - cat > "$cfg" < "$cfg" <&2 - live_build_append="" - fi - - if [ -f "$grub_cfg" ]; then - if extract_live_grub_entry "$grub_cfg"; then - cp "${BUILDER_DIR}/config/bootloaders/grub-efi/config.cfg" "$grub_dir/config.cfg" - write_canonical_grub_cfg "$grub_cfg" "$grub_kernel" "${live_build_append:-$grub_append}" "$grub_initrd" - echo "bootloader sync: rewrote binary/boot/grub/grub.cfg with canonical EASY-BEE menu" - else - echo "bootloader sync: WARNING: could not extract live entry from $grub_cfg" >&2 - fi - fi - - if [ -f "$isolinux_cfg" ]; then - if extract_live_isolinux_entry "$isolinux_cfg"; then - write_canonical_isolinux_cfg "$isolinux_cfg" "$isolinux_kernel" "$isolinux_initrd_path" "${live_build_append:-$isolinux_append}" - echo "bootloader sync: rewrote binary/isolinux/live.cfg with canonical EASY-BEE menu" - else - echo "bootloader sync: WARNING: could not extract live entry from $isolinux_cfg" >&2 - fi - fi -} - -copy_memtest_from_deb() { - deb="$1" - dst_boot="$2" - tmpdir="$(mktemp -d)" - - dpkg-deb -x "$deb" "$tmpdir" - for f in memtest86+x64.bin memtest86+x64.efi; do - if [ -f "$tmpdir/boot/$f" ]; then - cp "$tmpdir/boot/$f" "$dst_boot/$f" - fi - done - rm -rf "$tmpdir" -} - -reset_live_build_stage() { - lb_dir="$1" - stage="$2" - - for root in \ - "$lb_dir/.build" \ - "$lb_dir/.stage" \ - "$lb_dir/auto"; do - [ -d "$root" ] || continue - find "$root" -maxdepth 1 \( -name "${stage}" -o -name "${stage}.*" -o -name "*${stage}*" \) -exec rm -rf {} + 2>/dev/null || true - done -} - -# Marker + content hash written after every successful full lb build for this variant -FULL_BUILD_MARKER="${BUILD_WORK_DIR}/.bee-full-build-marker" -FULL_BUILD_HASH_FILE="${BUILD_WORK_DIR}/.bee-full-build-hash" - -# Hashes the content of every "heavy" config input (VERSIONS, package lists, -# hooks, archives, bootloaders, auto/config, Dockerfile). Deliberately content- -# based rather than mtime-based: mtimes get reset by git checkouts, rsync, and -# retried builds in ways that don't track "did this content actually change -# since the last full build", which previously let needs_full_build() silently -# take the fast-path (reusing an old squashfs built against different package -# pins) with no error. -hash_heavy_config() { - { - [ -f "${BUILDER_DIR}/VERSIONS" ] && cat "${BUILDER_DIR}/VERSIONS" - [ -f "${BUILDER_DIR}/auto/config" ] && cat "${BUILDER_DIR}/auto/config" - [ -f "${BUILDER_DIR}/Dockerfile" ] && cat "${BUILDER_DIR}/Dockerfile" - find \ - "${BUILDER_DIR}/config/package-lists" \ - "${BUILDER_DIR}/config/hooks" \ - "${BUILDER_DIR}/config/archives" \ - "${BUILDER_DIR}/config/bootloaders" \ - -type f 2>/dev/null | sort | xargs -r cat - } | sha256sum | awk '{print $1}' -} - -# Returns 0 if full lb build is needed, 1 if fast-path is safe. -# Fast-path is safe when only light files changed since the last full build -# (Go source, overlay scripts/configs). Heavy changes (VERSIONS, package lists, -# hooks, archives, Dockerfile, auto/config) require a full lb build. -needs_full_build() { - [ -f "${FULL_BUILD_MARKER}" ] || return 0 - [ -f "${FULL_BUILD_HASH_FILE}" ] || return 0 - [ -f "${BUILD_WORK_DIR}/live-image-amd64.hybrid.iso" ] || return 0 - # Accept any versioned squashfs (filesystem-v*.squashfs or legacy filesystem.squashfs) - _any_sq=$(find "${BUILD_WORK_DIR}/binary/live" -maxdepth 1 \ - -name 'filesystem*.squashfs' 2>/dev/null | head -1) - [ -n "$_any_sq" ] || return 0 - - _new_hash="$(hash_heavy_config)" - _old_hash="$(cat "${FULL_BUILD_HASH_FILE}" 2>/dev/null)" - - if [ "$_new_hash" != "$_old_hash" ]; then - echo "=== full build required: heavy config content changed since last full build ===" - return 0 - fi - - return 1 -} - -# Fast-path: unsquash existing filesystem, rsync overlay on top, repack. -# Requires ~10 GB free in BEE_CACHE_DIR for the unpacked squashfs. -fast_path_repack_squashfs() { - _old_sq=$(find "${BUILD_WORK_DIR}/binary/live" -maxdepth 1 \ - -name 'filesystem*.squashfs' | sort | head -1) - _sq="${BUILD_WORK_DIR}/binary/live/${SQUASHFS_FILENAME}" - _tmp="${BEE_CACHE_DIR}/fast-unsquash-${BUILD_VARIANT}" - echo "=== fast-path: unsquash $(basename "$_old_sq") ($(du -sh "$_old_sq" | cut -f1) compressed) ===" - rm -rf "$_tmp" - unsquashfs -d "$_tmp" "$_old_sq" - echo "=== fast-path: syncing overlay stage ===" - rsync -a --checksum "${OVERLAY_STAGE_DIR}/" "$_tmp/" - echo "=== fast-path: repacking as ${SQUASHFS_FILENAME} ===" - _sq_new="${_sq}.new" - rm -f "$_sq_new" - mksquashfs "$_tmp" "$_sq_new" -comp zstd -b 1048576 -noappend -no-progress -no-xattrs - mv "$_sq_new" "$_sq" - rm -rf "$_tmp" - [ "$_old_sq" != "$_sq" ] && rm -f "$_old_sq" - echo "=== fast-path: squashfs repacked ($(du -sh "$_sq" | cut -f1)) ===" -} - -# Fast-path: rebuild ISO replacing the squashfs via xorriso. -# Boot structure (El Torito, EFI, MBR hybrid) is replayed from the prior ISO. -fast_path_rebuild_iso() { - _sq="${BUILD_WORK_DIR}/binary/live/${SQUASHFS_FILENAME}" - _prior="${BUILD_WORK_DIR}/live-image-amd64.hybrid.iso" - _new="${BUILD_WORK_DIR}/live-image-amd64.hybrid.iso.new" - echo "=== fast-path: rebuilding ISO with xorriso ===" - rm -f "$_new" - # Remove any old squashfs entries from the prior ISO before adding the new one - _old_entries=$(xorriso -indev "$_prior" -find /live -name 'filesystem*.squashfs' -- 2>/dev/null \ - | grep -E '^/live/filesystem.*\.squashfs$' || true) - _rm_args="" - for _e in $_old_entries; do - _rm_args="$_rm_args -rm $_e --" - done - # shellcheck disable=SC2086 - xorriso \ - -indev "$_prior" \ - -outdev "$_new" \ - ${_rm_args} \ - -map "$_sq" /live/${SQUASHFS_FILENAME} \ - -boot_image any replay \ - -commit - mv "$_new" "$_prior" - echo "=== fast-path: ISO rebuilt ===" -} - -dir_has_entries() { - _dir="$1" - [ -d "$_dir" ] || return 1 - find "$_dir" -mindepth 1 -print -quit 2>/dev/null | grep -q . -} - -move_tree_to_layer() { - _src_root="$1" - _rel="$2" - _dst_root="$3" - [ -e "${_src_root}/${_rel}" ] || return 0 - mkdir -p "${_dst_root}/$(dirname "$_rel")" - mv "${_src_root}/${_rel}" "${_dst_root}/${_rel}" -} - -split_live_squashfs_layers() { - lb_dir="$1" - live_dir="${lb_dir}/binary/live" - base_sq="${live_dir}/filesystem.squashfs" - usr_sq="${live_dir}/10-usr.squashfs" - fw_sq="${live_dir}/20-firmware.squashfs" - - [ -f "$base_sq" ] || return 0 - command -v unsquashfs >/dev/null 2>&1 || return 0 - command -v mksquashfs >/dev/null 2>&1 || return 0 - - tmp_root="$(mktemp -d)" - tmp_usr="$(mktemp -d)" - tmp_fw="$(mktemp -d)" - - echo "=== splitting live squashfs into smaller layers ===" - unsquashfs -d "$tmp_root/root" "$base_sq" >/dev/null - mkdir -p "$tmp_usr/root" "$tmp_fw/root" - - move_tree_to_layer "$tmp_root/root" "usr" "$tmp_usr/root" - move_tree_to_layer "$tmp_root/root" "lib/firmware" "$tmp_fw/root" - move_tree_to_layer "$tmp_root/root" "usr/lib/firmware" "$tmp_fw/root" - move_tree_to_layer "$tmp_root/root" "boot/firmware" "$tmp_fw/root" - - rm -f "$usr_sq" "$fw_sq" - mksquashfs "$tmp_root/root" "${base_sq}.new" -comp zstd -b 1048576 -noappend -no-progress -no-xattrs >/dev/null - mv "${base_sq}.new" "$base_sq" - - if dir_has_entries "$tmp_usr/root"; then - mksquashfs "$tmp_usr/root" "${usr_sq}.new" -comp zstd -b 1048576 -noappend -no-progress -no-xattrs >/dev/null - mv "${usr_sq}.new" "$usr_sq" - fi - if dir_has_entries "$tmp_fw/root"; then - mksquashfs "$tmp_fw/root" "${fw_sq}.new" -comp zstd -b 1048576 -noappend -no-progress -no-xattrs >/dev/null - mv "${fw_sq}.new" "$fw_sq" - fi - - echo "=== live squashfs layers ===" - find "$live_dir" -maxdepth 1 -type f -name '*.squashfs' -exec du -sh {} \; | sort - rm -rf "$tmp_root" "$tmp_usr" "$tmp_fw" -} - -recover_iso_memtest() { - lb_dir="$1" - iso_path="$2" - binary_boot="$lb_dir/binary/boot" - grub_cfg="$lb_dir/binary/boot/grub/grub.cfg" - isolinux_cfg="$lb_dir/binary/isolinux/live.cfg" - - echo "=== attempting memtest recovery in binary tree ===" - - mkdir -p "$binary_boot" - - for root in \ - "$lb_dir/chroot/boot" \ - "/boot"; do - for f in memtest86+x64.bin memtest86+x64.efi; do - if [ ! -f "$binary_boot/$f" ] && [ -f "$root/$f" ]; then - cp "$root/$f" "$binary_boot/$f" - echo "memtest recovery: copied $f from $root" - fi - done - done - - if [ ! -f "$binary_boot/memtest86+x64.bin" ] || [ ! -f "$binary_boot/memtest86+x64.efi" ]; then - for dir in \ - "$lb_dir/cache/packages.binary" \ - "$lb_dir/cache/packages.chroot" \ - "$lb_dir/chroot/var/cache/apt/archives" \ - "${BEE_CACHE_DIR:-${DIST_DIR}/cache}/lb-packages" \ - "/var/cache/apt/archives"; do - [ -d "$dir" ] || continue - deb="$(find "$dir" -maxdepth 1 -type f -name 'memtest86+*.deb' 2>/dev/null | head -1)" - [ -n "$deb" ] || continue - echo "memtest recovery: extracting payload from $deb" - copy_memtest_from_deb "$deb" "$binary_boot" - break - done - fi - - if [ ! -f "$binary_boot/memtest86+x64.bin" ] || [ ! -f "$binary_boot/memtest86+x64.efi" ]; then - tmpdl="$(mktemp -d)" - if ( - cd "$tmpdl" && apt-get download memtest86+ >/dev/null 2>&1 - ); then - deb="$(find "$tmpdl" -maxdepth 1 -type f -name 'memtest86+*.deb' 2>/dev/null | head -1)" - if [ -n "$deb" ]; then - echo "memtest recovery: downloaded $deb" - copy_memtest_from_deb "$deb" "$binary_boot" - fi - fi - rm -rf "$tmpdl" - fi - - if [ -f "$grub_cfg" ]; then - append_memtest_grub_entry "$grub_cfg" && echo "memtest recovery: ensured GRUB entry" - else - echo "memtest recovery: WARNING: missing $grub_cfg" - fi - - if [ -f "$isolinux_cfg" ]; then - append_memtest_isolinux_entry "$isolinux_cfg" && echo "memtest recovery: ensured isolinux entry" - else - echo "memtest recovery: WARNING: missing $isolinux_cfg" - fi - - reset_live_build_stage "$lb_dir" "binary_checksums" - reset_live_build_stage "$lb_dir" "binary_iso" - reset_live_build_stage "$lb_dir" "binary_zsync" - - run_optional_step_sh "rebuild live-build checksums after memtest recovery" "91-lb-checksums" "lb binary_checksums 2>&1" - run_optional_step_sh "rebuild ISO after memtest recovery" "92-lb-binary-iso" "rm -f '$iso_path' && lb binary_iso 2>&1" - run_optional_step_sh "rebuild zsync after memtest recovery" "93-lb-zsync" "lb binary_zsync 2>&1" - - if [ ! -f "$iso_path" ]; then - memtest_fail "ISO rebuild was skipped or failed after memtest recovery: $iso_path" "$iso_path" - fi -} PROJECT_VERSION_EFFECTIVE="$(resolve_project_version)" SQUASHFS_FILENAME="filesystem-v${PROJECT_VERSION_EFFECTIVE}.squashfs" @@ -1245,7 +163,7 @@ ISO_BASENAME="easy-bee-${BUILD_VARIANT}-v${PROJECT_VERSION_EFFECTIVE}-amd64" # Hostname baked into the live image: same product+version scheme as # ISO_BASENAME/SQUASHFS_FILENAME, but hostname labels don't allow dots. BEE_HOSTNAME="easy-bee-${BUILD_VARIANT}-v$(printf '%s' "${PROJECT_VERSION_EFFECTIVE}" | tr '.' '-')" -# Versioned output directory: dist/easy-bee-v4.1/ — all final artefacts live here. +# Versioned output directory: dist/easy-bee-v4.1/ - all final artifacts live here. OUT_DIR="${DIST_DIR}/release/easy-bee-v${PROJECT_VERSION_EFFECTIVE}" ISO_VERSION_LABEL_TOKEN="$(printf '%s' "${PROJECT_VERSION_EFFECTIVE}" | tr '[:lower:].-' '[:upper:]__')" mkdir -p "${OUT_DIR}" @@ -1254,141 +172,7 @@ LOG_ARCHIVE="${OUT_DIR}/${ISO_BASENAME}.logs.tar.gz" ISO_OUT="${OUT_DIR}/${ISO_BASENAME}.iso" LOG_OUT="${LOG_DIR}/build.log" -cleanup_build_log() { - status="${1:-$?}" - trap - EXIT INT TERM HUP - - if [ "${STEP_LOG_ACTIVE:-0}" = "1" ]; then - cleanup_step_log "${status}" || true - fi - - if [ "${BUILD_LOG_ACTIVE:-0}" = "1" ]; then - BUILD_LOG_ACTIVE=0 - exec 1>&3 2>&4 - exec 3>&- 4>&- - if [ -n "${BUILD_TEE_PID:-}" ]; then - wait "${BUILD_TEE_PID}" 2>/dev/null || true - fi - rm -f "${BUILD_LOG_PIPE}" - fi - - if [ -n "${LOG_DIR:-}" ] && [ -d "${LOG_DIR}" ] && command -v tar >/dev/null 2>&1; then - rm -f "${LOG_ARCHIVE}" - tar -czf "${LOG_ARCHIVE}" -C "$(dirname "${LOG_DIR}")" "$(basename "${LOG_DIR}")" 2>/dev/null || true - rm -rf "${LOG_DIR}" - fi - - exit "${status}" -} - -start_build_log() { - command -v tee >/dev/null 2>&1 || { - echo "ERROR: tee is required for build logging" >&2 - exit 1 - } - - rm -rf "${LOG_DIR}" - rm -f "${LOG_ARCHIVE}" - mkdir -p "${LOG_DIR}" - BUILD_LOG_PIPE="$(mktemp -u "${TMPDIR:-/tmp}/bee-build-log.XXXXXX")" - mkfifo "${BUILD_LOG_PIPE}" - - exec 3>&1 4>&2 - tee "${LOG_OUT}" < "${BUILD_LOG_PIPE}" & - BUILD_TEE_PID=$! - exec > "${BUILD_LOG_PIPE}" 2>&1 - BUILD_LOG_ACTIVE=1 - - trap 'cleanup_build_log "$?"' EXIT INT TERM HUP - - echo "=== build log dir: ${LOG_DIR} ===" - echo "=== build log: ${LOG_OUT} ===" - echo "=== build log archive: ${LOG_ARCHIVE} ===" -} - -cleanup_step_log() { - status="${1:-$?}" - - if [ "${STEP_LOG_ACTIVE:-0}" = "1" ]; then - STEP_LOG_ACTIVE=0 - exec 1>&5 2>&6 - exec 5>&- 6>&- - if [ -n "${STEP_TEE_PID:-}" ]; then - wait "${STEP_TEE_PID}" 2>/dev/null || true - fi - rm -f "${STEP_LOG_PIPE}" - fi - - return "${status}" -} - -run_step() { - step_name="$1" - step_slug="$2" - shift 2 - - step_log="${LOG_DIR}/${step_slug}.log" - echo "" - echo "=== step: ${step_name} ===" - echo "=== step log: ${step_log} ===" - - STEP_LOG_PIPE="$(mktemp -u "${TMPDIR:-/tmp}/bee-step-log.XXXXXX")" - mkfifo "${STEP_LOG_PIPE}" - - exec 5>&1 6>&2 - tee "${step_log}" < "${STEP_LOG_PIPE}" >&5 & - STEP_TEE_PID=$! - exec > "${STEP_LOG_PIPE}" 2>&1 - STEP_LOG_ACTIVE=1 - - set +e - "$@" - step_status=$? - set -e - - cleanup_step_log "${step_status}" - if [ "${step_status}" -ne 0 ]; then - echo "ERROR: step failed: ${step_name} (see ${step_log})" >&2 - exit "${step_status}" - fi - - echo "=== step OK: ${step_name} ===" -} - -run_step_sh() { - step_name="$1" - step_slug="$2" - step_script="$3" - - run_step "${step_name}" "${step_slug}" sh -c "${step_script}" -} - -run_optional_step_sh() { - step_name="$1" - step_slug="$2" - step_script="$3" - - if [ "${BEE_REQUIRE_MEMTEST:-0}" = "1" ]; then - run_step_sh "${step_name}" "${step_slug}" "${step_script}" - return 0 - fi - - mkdir -p "${LOG_DIR}" 2>/dev/null || true - step_log="${LOG_DIR}/${step_slug}.log" - echo "" - echo "=== optional step: ${step_name} ===" - echo "=== optional step log: ${step_log} ===" - set +e - sh -c "${step_script}" > "${step_log}" 2>&1 - step_status=$? - set -e - cat "${step_log}" - if [ "${step_status}" -ne 0 ]; then - echo "WARNING: optional step failed: ${step_name} (see ${step_log})" >&2 - else - echo "=== optional step OK: ${step_name} ===" - fi -} +. "${BUILDER_DIR}/lib/build-log.sh" start_build_log @@ -1430,35 +214,26 @@ run_step "sync git submodules" "05-git-submodules" \ git -C "${REPO_ROOT}" submodule update --init --recursive # --- compile bee binary (static, Linux amd64) --- -# Shared between variants — built once, reused on second pass. +# Always rebuild: version/commit ldflags, go.mod/go.sum changes, and deleted Go +# files are not represented reliably by source mtimes. BEE_BIN="${DIST_DIR}/cache/bee-linux-amd64" -NEED_BUILD=1 -if [ -f "$BEE_BIN" ]; then - NEWEST_SRC=$(find "${REPO_ROOT}/audit" -name '*.go' -newer "$BEE_BIN" | head -1) - [ -z "$NEWEST_SRC" ] && NEED_BUILD=0 -fi - -if [ "$NEED_BUILD" = "1" ]; then - run_step_sh "build bee binary" "10-build-bee" \ - "cd '${REPO_ROOT}/audit' && \ - env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \ - go build \ - -ldflags '-s -w -X main.Version=${PROJECT_VERSION_EFFECTIVE} -X main.BuildCommit=${PROJECT_BUILD_COMMIT}' \ - -o '${BEE_BIN}' \ - ./cmd/bee" - echo "binary: $BEE_BIN" - if command -v stat >/dev/null 2>&1; then - BEE_SIZE_BYTES="$(stat -c '%s' "$BEE_BIN" 2>/dev/null || stat -f '%z' "$BEE_BIN")" - else - BEE_SIZE_BYTES="$(wc -c < "$BEE_BIN" | tr -d ' ')" - fi - if command -v numfmt >/dev/null 2>&1; then - echo "size: $(numfmt --to=iec --suffix=B "$BEE_SIZE_BYTES")" - else - echo "size: ${BEE_SIZE_BYTES} bytes" - fi +run_step_sh "build bee binary" "10-build-bee" \ + "cd '${REPO_ROOT}/audit' && \ + env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \ + go build \ + -ldflags '-s -w -X main.Version=${PROJECT_VERSION_EFFECTIVE} -X main.BuildCommit=${PROJECT_BUILD_COMMIT}' \ + -o '${BEE_BIN}' \ + ./cmd/bee" +echo "binary: $BEE_BIN" +if command -v stat >/dev/null 2>&1; then + BEE_SIZE_BYTES="$(stat -c '%s' "$BEE_BIN" 2>/dev/null || stat -f '%z' "$BEE_BIN")" else - echo "=== bee binary up to date, skipping build ===" + BEE_SIZE_BYTES="$(wc -c < "$BEE_BIN" | tr -d ' ')" +fi +if command -v numfmt >/dev/null 2>&1; then + echo "size: $(numfmt --to=iec --suffix=B "$BEE_SIZE_BYTES")" +else + echo "size: ${BEE_SIZE_BYTES} bytes" fi # --- NVIDIA-only build steps --- @@ -1494,6 +269,9 @@ if [ "$BEE_GPU_VENDOR" = "nvidia" ]; then GPU_STRESS_NEED_BUILD=0 for dep in \ "${BUILDER_DIR}/bee-gpu-stress.c" \ + "${BUILDER_DIR}/bee-gpu-stress-cuda.inc" \ + "${BUILDER_DIR}/bee-gpu-stress-cublaslt.inc" \ + "${BUILDER_DIR}/bee-gpu-stress-main.inc" \ "${BUILDER_DIR}/VERSIONS"; do if [ "$dep" -nt "$GPU_BURN_WORKER_BIN" ]; then GPU_STRESS_NEED_BUILD=1 @@ -1590,7 +368,7 @@ else chmod 600 "$AUTHORIZED_KEYS_FILE" echo "SSH authorized_keys: $FOUND key(s) from ~/.keys/*.key.pub" else - echo "WARNING: no SSH public keys found — falling back to password auth" + echo "WARNING: no SSH public keys found - falling back to password auth" echo " SSH login: bee / eeb" USE_PASSWORD_FALLBACK=1 fi @@ -1631,7 +409,7 @@ for tool in storcli64 storcli2 sas2ircu sas3ircu arcconf ssacli saa; do fi done -# saa companion directories — saa searches for these relative to CWD (/usr/local/bin) +# saa companion directories - saa searches for these relative to CWD (/usr/local/bin) for saa_subdir in acpica_bin ExternalData tool stunnel GO_SNMP; do if [ -d "${VENDOR_DIR}/${saa_subdir}" ]; then cp -r "${VENDOR_DIR}/${saa_subdir}" "${OVERLAY_STAGE_DIR}/usr/local/bin/" @@ -1766,7 +544,7 @@ if [ -f "${OVERLAY_STAGE_DIR}/etc/motd" ]; then fi # --- copy variant-specific package list, remove all other variant lists --- -# live-build picks up ALL .list.chroot files — delete other variants to avoid conflicts. +# live-build picks up ALL .list.chroot files; delete other variants to avoid conflicts. cp "${BUILD_WORK_DIR}/config/package-lists/bee-${BEE_GPU_VENDOR}.list.chroot" \ "${BUILD_WORK_DIR}/config/package-lists/bee-gpu.list.chroot" rm -f "${BUILD_WORK_DIR}/config/package-lists/bee-nvidia.list.chroot" \ @@ -1825,12 +603,37 @@ if [ -f "${LB_INCLUDES}/root/.ssh/authorized_keys" ]; then chmod 600 "${LB_INCLUDES}/root/.ssh/authorized_keys" fi +# Export the release-specific outer ISO identity before either build path. +# The fast path regenerates bootloader assets and the ISO container too. +BEE_GPU_VENDOR_UPPER="$(echo "${BUILD_VARIANT}" | tr 'a-z-' 'A-Z_')" +_vol_prefix="EASY_BEE_${BEE_GPU_VENDOR_UPPER}_V" +_max_token=$(( 32 - ${#_vol_prefix} )) +_vol_token="$(printf '%s' "${ISO_VERSION_LABEL_TOKEN}" | cut -c1-${_max_token})" +BEE_ISO_VOLUME="${_vol_prefix}${_vol_token}" +unset _vol_prefix _max_token _vol_token +export BEE_GPU_VENDOR_UPPER BEE_ISO_VOLUME BEE_HOSTNAME + # --- auto fast-path: squashfs surgery if only light files changed --- if ! needs_full_build; then echo "=== fast-path build (no heavy config changes since last full build) ===" fast_path_repack_squashfs - fast_path_rebuild_iso + # The reusable squashfs now reflects this overlay revision. Advance the + # manifest immediately, even if outer-ISO assembly later fails, so a file + # added by this build and removed before a retry cannot linger unnoticed. + write_overlay_manifest "${FULL_BUILD_OVERLAY_MANIFEST}.new" + mv "${FULL_BUILD_OVERLAY_MANIFEST}.new" "${FULL_BUILD_OVERLAY_MANIFEST}" + cd "${LB_DIR}" + run_step_sh "fast-path live-build config" "81-fast-lb-config" "lb config 2>&1 | tail -5" + echo "=== fast-path: enforcing canonical bootloader assets ===" + enforce_live_build_bootloader_assets "${LB_DIR}" + reset_live_build_stage "${LB_DIR}" "binary_checksums" + reset_live_build_stage "${LB_DIR}" "binary_iso" + reset_live_build_stage "${LB_DIR}" "binary_zsync" + run_step_sh "fast-path rebuild checksums" "82-fast-lb-checksums" "lb binary_checksums 2>&1" + run_step_sh "fast-path rebuild ISO" "83-fast-lb-binary-iso" "lb binary_iso 2>&1" + run_step_sh "fast-path rebuild zsync" "84-fast-lb-zsync" "lb binary_zsync 2>&1" ISO_RAW="${LB_DIR}/live-image-amd64.hybrid.iso" + validate_iso_memtest "$ISO_RAW" validate_iso_live_boot_entries "$ISO_RAW" validate_iso_grub_assets "$ISO_RAW" validate_iso_nvidia_runtime "$ISO_RAW" @@ -1844,16 +647,7 @@ fi # --- build ISO using live-build --- echo "" echo "=== building ISO (variant: ${BUILD_VARIANT}) ===" - -# Export for auto/config -BEE_GPU_VENDOR_UPPER="$(echo "${BUILD_VARIANT}" | tr 'a-z-' 'A-Z_')" -# ISO 9660 volume ID is limited to 32 characters; truncate the version token to fit. -_vol_prefix="EASY_BEE_${BEE_GPU_VENDOR_UPPER}_V" -_max_token=$(( 32 - ${#_vol_prefix} )) -_vol_token="$(printf '%s' "${ISO_VERSION_LABEL_TOKEN}" | cut -c1-${_max_token})" -BEE_ISO_VOLUME="${_vol_prefix}${_vol_token}" -unset _vol_prefix _max_token _vol_token -export BEE_GPU_VENDOR_UPPER BEE_ISO_VOLUME BEE_HOSTNAME +rm -f "${FULL_BUILD_MARKER}" cd "${LB_DIR}" run_step_sh "live-build clean" "80-lb-clean" "lb clean --all 2>&1 | tail -3" @@ -1869,7 +663,7 @@ _std_sq="${LB_DIR}/binary/live/filesystem.squashfs" _ver_sq="${LB_DIR}/binary/live/${SQUASHFS_FILENAME}" if [ -f "${_std_sq}" ] && [ "${_std_sq}" != "${_ver_sq}" ]; then mv "${_std_sq}" "${_ver_sq}" - echo "=== squashfs renamed: filesystem.squashfs → ${SQUASHFS_FILENAME} ===" + echo "=== squashfs renamed: filesystem.squashfs -> ${SQUASHFS_FILENAME} ===" fi reset_live_build_stage "${LB_DIR}" "binary_checksums" reset_live_build_stage "${LB_DIR}" "binary_iso" @@ -1905,8 +699,13 @@ if [ -f "$ISO_RAW" ]; then validate_iso_grub_assets "$ISO_RAW" validate_iso_nvidia_runtime "$ISO_RAW" cp "$ISO_RAW" "$ISO_OUT" + hash_heavy_config > "${FULL_BUILD_HASH_FILE}.new" + printf '%s\n' "${DEBIAN_KERNEL_ABI}" > "${FULL_BUILD_ABI_FILE}.new" + write_overlay_manifest "${FULL_BUILD_OVERLAY_MANIFEST}.new" + mv "${FULL_BUILD_HASH_FILE}.new" "${FULL_BUILD_HASH_FILE}" + mv "${FULL_BUILD_ABI_FILE}.new" "${FULL_BUILD_ABI_FILE}" + mv "${FULL_BUILD_OVERLAY_MANIFEST}.new" "${FULL_BUILD_OVERLAY_MANIFEST}" touch "${FULL_BUILD_MARKER}" - hash_heavy_config > "${FULL_BUILD_HASH_FILE}" echo "" echo "=== done (${BUILD_VARIANT}) ===" echo "ISO: $ISO_OUT" diff --git a/iso/builder/config/hooks/normal/9100-memtest.hook.binary b/iso/builder/config/hooks/normal/9100-memtest.hook.binary index 01bc7b1..8d66417 100755 --- a/iso/builder/config/hooks/normal/9100-memtest.hook.binary +++ b/iso/builder/config/hooks/normal/9100-memtest.hook.binary @@ -1,16 +1,14 @@ #!/bin/sh -# Ensure memtest is present in the final ISO even if live-build's built-in -# memtest stage does not copy the binaries or expose menu entries. +# Ensure memtest binaries are present even if live-build's built-in memtest +# stage does not copy them. Boot menu entries come from the canonical templates +# enforced by build.sh after live-build finishes. set -e : "${BEE_REQUIRE_MEMTEST:=0}" -# memtest86+ 6.x uses memtest86+.bin (no x64 suffix) for the BIOS binary, -# while 5.x used memtest86+x64.bin. We normalise both to x64 names in the ISO. +# Debian Bookworm's pinned memtest86+ package installs these exact paths. MEMTEST_FILES="memtest86+x64.bin memtest86+x64.efi" BINARY_BOOT_DIR="binary/boot" -GRUB_CFG="binary/boot/grub/grub.cfg" -ISOLINUX_CFG="binary/isolinux/live.cfg" log() { echo "memtest hook: $*" @@ -26,14 +24,6 @@ fail_or_warn() { return 0 } -# grub.cfg and live.cfg may not exist yet when binary hooks run — live-build -# creates them after this hook (lb binary_grub-efi / lb binary_syslinux). -# The template already has memtest entries hardcoded, so a missing config file -# here is not an error; validate_iso_memtest() checks the final ISO instead. -warn_only() { - log "WARNING: $1" -} - copy_memtest_file() { src="$1" dst_name="${2:-$(basename "$src")}" @@ -52,16 +42,12 @@ extract_memtest_from_deb() { log "extracting memtest payload from ${deb}" dpkg-deb -x "$deb" "$tmpdir" - # EFI binary: both 5.x and 6.x use memtest86+x64.efi if [ -f "${tmpdir}/boot/memtest86+x64.efi" ]; then copy_memtest_file "${tmpdir}/boot/memtest86+x64.efi" fi - # BIOS binary: 5.x = memtest86+x64.bin, 6.x = memtest86+.bin if [ -f "${tmpdir}/boot/memtest86+x64.bin" ]; then copy_memtest_file "${tmpdir}/boot/memtest86+x64.bin" - elif [ -f "${tmpdir}/boot/memtest86+.bin" ]; then - copy_memtest_file "${tmpdir}/boot/memtest86+.bin" "memtest86+x64.bin" fi rm -rf "$tmpdir" @@ -101,10 +87,6 @@ ensure_memtest_binaries() { for f in ${MEMTEST_FILES}; do [ -f "${BINARY_BOOT_DIR}/${f}" ] || copy_memtest_file "${root}/${f}" || true done - # 6.x BIOS binary may lack x64 in name — copy with normalised name - if [ ! -f "${BINARY_BOOT_DIR}/memtest86+x64.bin" ]; then - copy_memtest_file "${root}/memtest86+.bin" "memtest86+x64.bin" || true - fi done missing=0 @@ -141,54 +123,6 @@ ensure_memtest_binaries() { [ "$missing" -eq 0 ] || return 0 } -ensure_grub_entry() { - [ -f "$GRUB_CFG" ] || { - warn_only "missing ${GRUB_CFG} (will be created by lb binary_grub-efi from template)" - return 0 - } - - grep -q '### BEE MEMTEST ###' "$GRUB_CFG" && return 0 - - cat >> "$GRUB_CFG" <<'EOF' - -### BEE MEMTEST ### -if [ "${grub_platform}" = "efi" ]; then - menuentry "Memory Test (memtest86+)" { - chainloader /boot/memtest86+x64.efi - } -else - menuentry "Memory Test (memtest86+)" { - linux16 /boot/memtest86+x64.bin - } -fi -### /BEE MEMTEST ### -EOF - - log "appended memtest entry to ${GRUB_CFG}" -} - -ensure_isolinux_entry() { - [ -f "$ISOLINUX_CFG" ] || { - warn_only "missing ${ISOLINUX_CFG} (will be created by lb binary_syslinux from template)" - return 0 - } - - grep -q '### BEE MEMTEST ###' "$ISOLINUX_CFG" && return 0 - - cat >> "$ISOLINUX_CFG" <<'EOF' - -# ### BEE MEMTEST ### -label memtest - menu label ^Memory Test (memtest86+) - linux /boot/memtest86+x64.bin -# ### /BEE MEMTEST ### -EOF - - log "appended memtest entry to ${ISOLINUX_CFG}" -} - -log "ensuring memtest binaries and menu entries in binary image" +log "ensuring memtest binaries in binary image" ensure_memtest_binaries -ensure_grub_entry -ensure_isolinux_entry log "memtest assets ready" diff --git a/iso/builder/lib/bootloader.sh b/iso/builder/lib/bootloader.sh new file mode 100755 index 0000000..dff211b --- /dev/null +++ b/iso/builder/lib/bootloader.sh @@ -0,0 +1,146 @@ +extract_live_grub_entry() { + cfg="$1" + live_linux="$(awk '/^[[:space:]]*linux[[:space:]]+\/live\// { print; exit }' "$cfg")" + live_initrd="$(awk '/^[[:space:]]*initrd[[:space:]]+\/live\// { print; exit }' "$cfg")" + [ -n "$live_linux" ] || return 1 + [ -n "$live_initrd" ] || return 1 + + grub_kernel="$(printf '%s\n' "$live_linux" | awk '{print $2}')" + grub_append="$(printf '%s\n' "$live_linux" | cut -d' ' -f3-)" + grub_initrd="$(printf '%s\n' "$live_initrd" | awk '{print $2}')" + [ -n "$grub_kernel" ] || return 1 + [ -n "$grub_append" ] || return 1 + [ -n "$grub_initrd" ] || return 1 + return 0 +} + +load_live_build_append() { + lb_dir="$1" + binary_cfg="$lb_dir/config/binary" + [ -f "$binary_cfg" ] || return 1 + + # config/binary is generated by live-build and contains shell variable + # assignments such as LB_BOOTAPPEND_LIVE="boot=live ...". + # shellcheck disable=SC1090 + . "$binary_cfg" + + [ -n "${LB_BOOTAPPEND_LIVE:-}" ] || return 1 + live_build_append="$LB_BOOTAPPEND_LIVE" + return 0 +} + +extract_live_isolinux_entry() { + cfg="$1" + isolinux_linux="$(awk '/^[[:space:]]*linux[[:space:]]+\/live\// { print; exit }' "$cfg")" + isolinux_initrd="$(awk '/^[[:space:]]*initrd[[:space:]]+\/live\// { print; exit }' "$cfg")" + isolinux_append="$(awk '/^[[:space:]]*append[[:space:]]+/ { sub(/^[[:space:]]*append[[:space:]]+/, ""); print; exit }' "$cfg")" + [ -n "$isolinux_linux" ] || return 1 + [ -n "$isolinux_initrd" ] || return 1 + [ -n "$isolinux_append" ] || return 1 + + isolinux_kernel="$(printf '%s\n' "$isolinux_linux" | awk '{print $2}')" + isolinux_initrd_path="$(printf '%s\n' "$isolinux_initrd" | awk '{print $2}')" + [ -n "$isolinux_kernel" ] || return 1 + [ -n "$isolinux_initrd_path" ] || return 1 + return 0 +} + +write_canonical_grub_cfg() { + cfg="$1" + kernel="$2" + append_live="$3" + initrd="$4" + version_label="${PROJECT_VERSION_EFFECTIVE}" + + template="${BUILDER_DIR}/config/bootloaders/grub-efi/grub.cfg" + tmp_cfg="${cfg}.new" + render_bootloader_template "$template" "$tmp_cfg" \ + "$version_label" "@KERNEL_LIVE@" "$kernel" "$append_live" "@INITRD_LIVE@" "$initrd" + mv "$tmp_cfg" "$cfg" +} + +write_canonical_isolinux_cfg() { + cfg="$1" + kernel="$2" + initrd="$3" + append_live="$4" + version_label="${PROJECT_VERSION_EFFECTIVE}" + + template="${BUILDER_DIR}/config/bootloaders/isolinux/live.cfg.in" + tmp_cfg="${cfg}.new" + render_bootloader_template "$template" "$tmp_cfg" \ + "$version_label" "@LINUX@" "$kernel" "$append_live" "@INITRD@" "$initrd" + mv "$tmp_cfg" "$cfg" +} + +# Render literal placeholders without sed replacement-string semantics. Values +# containing '&', backslashes, or the sed delimiter must be copied unchanged. +render_bootloader_template() { + template="$1" + output="$2" + version="$3" + kernel_placeholder="$4" + kernel="$5" + append_live="$6" + initrd_placeholder="$7" + initrd="$8" + + BEE_RENDER_VERSION="$version" \ + BEE_RENDER_KERNEL_PLACEHOLDER="$kernel_placeholder" \ + BEE_RENDER_KERNEL="$kernel" \ + BEE_RENDER_APPEND="$append_live" \ + BEE_RENDER_INITRD_PLACEHOLDER="$initrd_placeholder" \ + BEE_RENDER_INITRD="$initrd" \ + awk ' + function replace_literal(text, needle, replacement, at) { + while ((at = index(text, needle)) != 0) { + text = substr(text, 1, at - 1) replacement substr(text, at + length(needle)) + } + return text + } + { + version = ENVIRON["BEE_RENDER_VERSION"] + kernel_placeholder = ENVIRON["BEE_RENDER_KERNEL_PLACEHOLDER"] + kernel = ENVIRON["BEE_RENDER_KERNEL"] + append_live = ENVIRON["BEE_RENDER_APPEND"] + initrd_placeholder = ENVIRON["BEE_RENDER_INITRD_PLACEHOLDER"] + initrd = ENVIRON["BEE_RENDER_INITRD"] + line = replace_literal($0, "@VERSION@", version) + line = replace_literal(line, kernel_placeholder, kernel) + line = replace_literal(line, "@APPEND_LIVE@", append_live) + line = replace_literal(line, initrd_placeholder, initrd) + print line + } + ' "$template" > "$output" +} + +enforce_live_build_bootloader_assets() { + lb_dir="$1" + grub_cfg="$lb_dir/binary/boot/grub/grub.cfg" + grub_dir="$lb_dir/binary/boot/grub" + isolinux_cfg="$lb_dir/binary/isolinux/live.cfg" + + if ! load_live_build_append "$lb_dir"; then + echo "bootloader sync: WARNING: could not load LB_BOOTAPPEND_LIVE from $lb_dir/config/binary" >&2 + live_build_append="" + fi + + if [ -f "$grub_cfg" ]; then + if extract_live_grub_entry "$grub_cfg"; then + cp "${BUILDER_DIR}/config/bootloaders/grub-efi/config.cfg" "$grub_dir/config.cfg" + write_canonical_grub_cfg "$grub_cfg" "$grub_kernel" "${live_build_append:-$grub_append}" "$grub_initrd" + echo "bootloader sync: rewrote binary/boot/grub/grub.cfg with canonical EASY-BEE menu" + else + echo "bootloader sync: WARNING: could not extract live entry from $grub_cfg" >&2 + fi + fi + + if [ -f "$isolinux_cfg" ]; then + if extract_live_isolinux_entry "$isolinux_cfg"; then + write_canonical_isolinux_cfg "$isolinux_cfg" "$isolinux_kernel" "$isolinux_initrd_path" "${live_build_append:-$isolinux_append}" + echo "bootloader sync: rewrote binary/isolinux/live.cfg with canonical EASY-BEE menu" + else + echo "bootloader sync: WARNING: could not extract live entry from $isolinux_cfg" >&2 + fi + fi +} diff --git a/iso/builder/lib/build-log.sh b/iso/builder/lib/build-log.sh new file mode 100755 index 0000000..0a8ff2c --- /dev/null +++ b/iso/builder/lib/build-log.sh @@ -0,0 +1,137 @@ +cleanup_build_log() { + status="${1:-$?}" + trap - EXIT INT TERM HUP + + if [ "${STEP_LOG_ACTIVE:-0}" = "1" ]; then + cleanup_step_log "${status}" || true + fi + + if [ "${BUILD_LOG_ACTIVE:-0}" = "1" ]; then + BUILD_LOG_ACTIVE=0 + exec 1>&3 2>&4 + exec 3>&- 4>&- + if [ -n "${BUILD_TEE_PID:-}" ]; then + wait "${BUILD_TEE_PID}" 2>/dev/null || true + fi + rm -rf "${BUILD_LOG_TMPDIR}" + fi + + if [ -n "${LOG_DIR:-}" ] && [ -d "${LOG_DIR}" ] && command -v tar >/dev/null 2>&1; then + rm -f "${LOG_ARCHIVE}" + tar -czf "${LOG_ARCHIVE}" -C "$(dirname "${LOG_DIR}")" "$(basename "${LOG_DIR}")" 2>/dev/null || true + rm -rf "${LOG_DIR}" + fi + + exit "${status}" +} + +start_build_log() { + command -v tee >/dev/null 2>&1 || { + echo "ERROR: tee is required for build logging" >&2 + exit 1 + } + + rm -rf "${LOG_DIR}" + rm -f "${LOG_ARCHIVE}" + mkdir -p "${LOG_DIR}" + BUILD_LOG_TMPDIR="$(mktemp -d "${TMPDIR:-/tmp}/bee-build-log.XXXXXX")" + BUILD_LOG_PIPE="${BUILD_LOG_TMPDIR}/pipe" + mkfifo "${BUILD_LOG_PIPE}" + + exec 3>&1 4>&2 + tee "${LOG_OUT}" < "${BUILD_LOG_PIPE}" & + BUILD_TEE_PID=$! + exec > "${BUILD_LOG_PIPE}" 2>&1 + BUILD_LOG_ACTIVE=1 + + trap 'cleanup_build_log "$?"' EXIT INT TERM HUP + + echo "=== build log dir: ${LOG_DIR} ===" + echo "=== build log: ${LOG_OUT} ===" + echo "=== build log archive: ${LOG_ARCHIVE} ===" +} + +cleanup_step_log() { + status="${1:-$?}" + + if [ "${STEP_LOG_ACTIVE:-0}" = "1" ]; then + STEP_LOG_ACTIVE=0 + exec 1>&5 2>&6 + exec 5>&- 6>&- + if [ -n "${STEP_TEE_PID:-}" ]; then + wait "${STEP_TEE_PID}" 2>/dev/null || true + fi + rm -rf "${STEP_LOG_TMPDIR}" + fi + + return "${status}" +} + +run_step() { + step_name="$1" + step_slug="$2" + shift 2 + + step_log="${LOG_DIR}/${step_slug}.log" + echo "" + echo "=== step: ${step_name} ===" + echo "=== step log: ${step_log} ===" + + STEP_LOG_TMPDIR="$(mktemp -d "${TMPDIR:-/tmp}/bee-step-log.XXXXXX")" + STEP_LOG_PIPE="${STEP_LOG_TMPDIR}/pipe" + mkfifo "${STEP_LOG_PIPE}" + + exec 5>&1 6>&2 + tee "${step_log}" < "${STEP_LOG_PIPE}" >&5 & + STEP_TEE_PID=$! + exec > "${STEP_LOG_PIPE}" 2>&1 + STEP_LOG_ACTIVE=1 + + set +e + "$@" + step_status=$? + set -e + + cleanup_step_log "${step_status}" + if [ "${step_status}" -ne 0 ]; then + echo "ERROR: step failed: ${step_name} (see ${step_log})" >&2 + exit "${step_status}" + fi + + echo "=== step OK: ${step_name} ===" +} + +run_step_sh() { + step_name="$1" + step_slug="$2" + step_script="$3" + + run_step "${step_name}" "${step_slug}" sh -c "${step_script}" +} + +run_optional_step_sh() { + step_name="$1" + step_slug="$2" + step_script="$3" + + if [ "${BEE_REQUIRE_MEMTEST:-0}" = "1" ]; then + run_step_sh "${step_name}" "${step_slug}" "${step_script}" + return 0 + fi + + mkdir -p "${LOG_DIR}" 2>/dev/null || true + step_log="${LOG_DIR}/${step_slug}.log" + echo "" + echo "=== optional step: ${step_name} ===" + echo "=== optional step log: ${step_log} ===" + set +e + sh -c "${step_script}" > "${step_log}" 2>&1 + step_status=$? + set -e + cat "${step_log}" + if [ "${step_status}" -ne 0 ]; then + echo "WARNING: optional step failed: ${step_name} (see ${step_log})" >&2 + else + echo "=== optional step OK: ${step_name} ===" + fi +} diff --git a/iso/builder/lib/fast-path.sh b/iso/builder/lib/fast-path.sh new file mode 100755 index 0000000..05390ae --- /dev/null +++ b/iso/builder/lib/fast-path.sh @@ -0,0 +1,206 @@ +copy_memtest_from_deb() { + deb="$1" + dst_boot="$2" + tmpdir="$(mktemp -d)" + + dpkg-deb -x "$deb" "$tmpdir" + for f in memtest86+x64.bin memtest86+x64.efi; do + if [ -f "$tmpdir/boot/$f" ]; then + cp "$tmpdir/boot/$f" "$dst_boot/$f" + fi + done + rm -rf "$tmpdir" +} + +reset_live_build_stage() { + lb_dir="$1" + stage="$2" + + for root in \ + "$lb_dir/.build" \ + "$lb_dir/.stage" \ + "$lb_dir/auto"; do + [ -d "$root" ] || continue + find "$root" -maxdepth 1 \( -name "${stage}" -o -name "${stage}.*" -o -name "*${stage}*" \) -exec rm -rf {} + 2>/dev/null || true + done +} + +# State written after every successful full lb build for this variant. Keep it +# outside the rsync-managed live-build workdir so source synchronization cannot +# delete the state that decides whether the fast path is safe. +FULL_BUILD_STATE_DIR="${CACHE_ROOT}/full-build-state-${BUILD_VARIANT}" +mkdir -p "${FULL_BUILD_STATE_DIR}" +FULL_BUILD_MARKER="${FULL_BUILD_STATE_DIR}/complete" +FULL_BUILD_HASH_FILE="${FULL_BUILD_STATE_DIR}/heavy-config.sha256" +FULL_BUILD_ABI_FILE="${FULL_BUILD_STATE_DIR}/kernel-abi" +FULL_BUILD_OVERLAY_MANIFEST="${FULL_BUILD_STATE_DIR}/overlay.manifest" + +# Hashes the content of every "heavy" config input (VERSIONS, package lists, +# hooks, archives, auto/config, Dockerfile). Bootloader templates are excluded: +# the fast path regenerates the complete outer ISO layer from them. Deliberately content- +# based rather than mtime-based: mtimes get reset by git checkouts, rsync, and +# retried builds in ways that don't track "did this content actually change +# since the last full build", which previously let needs_full_build() silently +# take the fast path (reusing an old squashfs built against different package +# pins) with no error. +hash_heavy_config() { + ( + cd "${BUILDER_DIR}" + find \ + VERSIONS auto/config Dockerfile \ + config/package-lists config/hooks config/archives \ + -type f -print0 2>/dev/null | + sort -z | + xargs -0 -r sha256sum + ) | sha256sum | awk '{print $1}' +} + +write_overlay_manifest() { + out_path="$1" + ( + cd "${OVERLAY_STAGE_DIR}" + find . -mindepth 1 -printf '%y %P\n' | sort + ) > "$out_path" +} + +overlay_paths_were_removed() { + [ -f "${FULL_BUILD_OVERLAY_MANIFEST}" ] || return 0 + current_manifest="$(mktemp)" + write_overlay_manifest "$current_manifest" + if comm -23 "${FULL_BUILD_OVERLAY_MANIFEST}" "$current_manifest" | grep -q .; then + rm -f "$current_manifest" + return 0 + fi + rm -f "$current_manifest" + return 1 +} + +# Returns 0 if full lb build is needed, 1 if fast-path is safe. +# Fast-path is safe when only light files changed since the last full build +# (Go source, overlay scripts/configs). Heavy changes (VERSIONS, package lists, +# hooks, archives, Dockerfile, auto/config) require a full lb build. +needs_full_build() { + [ -f "${FULL_BUILD_MARKER}" ] || return 0 + [ -f "${FULL_BUILD_HASH_FILE}" ] || return 0 + [ -f "${FULL_BUILD_ABI_FILE}" ] || return 0 + [ -f "${FULL_BUILD_OVERLAY_MANIFEST}" ] || return 0 + [ -f "${BUILD_WORK_DIR}/live-image-amd64.hybrid.iso" ] || return 0 + # Accept any versioned squashfs (filesystem-v*.squashfs or legacy filesystem.squashfs) + _any_sq=$(find "${BUILD_WORK_DIR}/binary/live" -maxdepth 1 \ + -name 'filesystem*.squashfs' 2>/dev/null | head -1) + [ -n "$_any_sq" ] || return 0 + + _old_abi="$(cat "${FULL_BUILD_ABI_FILE}" 2>/dev/null)" + if [ "${DEBIAN_KERNEL_ABI}" != "$_old_abi" ]; then + echo "=== full build required: kernel ABI changed (${_old_abi:-unknown} -> ${DEBIAN_KERNEL_ABI}) ===" + return 0 + fi + + if overlay_paths_were_removed; then + echo "=== full build required: overlay paths were removed or changed type ===" + return 0 + fi + + _new_hash="$(hash_heavy_config)" + _old_hash="$(cat "${FULL_BUILD_HASH_FILE}" 2>/dev/null)" + + if [ "$_new_hash" != "$_old_hash" ]; then + echo "=== full build required: heavy config content changed since last full build ===" + return 0 + fi + + return 1 +} + +# Fast path: unsquash existing filesystem, rsync overlay on top, repack. +# CACHE_ROOT must have enough free space for the extracted root filesystem. +fast_path_repack_squashfs() ( + _old_sq=$(find "${BUILD_WORK_DIR}/binary/live" -maxdepth 1 \ + -name 'filesystem*.squashfs' | sort | head -1) + _sq="${BUILD_WORK_DIR}/binary/live/${SQUASHFS_FILENAME}" + _tmp_parent="$(mktemp -d "${CACHE_ROOT}/fast-unsquash-${BUILD_VARIANT}.XXXXXX")" + _tmp="${_tmp_parent}/root" + trap 'rm -rf "$_tmp_parent"' EXIT + echo "=== fast-path: unsquash $(basename "$_old_sq") ($(du -sh "$_old_sq" | cut -f1) compressed) ===" + unsquashfs -d "$_tmp" "$_old_sq" + echo "=== fast-path: syncing overlay stage ===" + rsync -a --checksum "${OVERLAY_STAGE_DIR}/" "$_tmp/" + echo "=== fast-path: repacking as ${SQUASHFS_FILENAME} ===" + _sq_new="${_sq}.new" + rm -f "$_sq_new" + mksquashfs "$_tmp" "$_sq_new" -comp zstd -b 1048576 -noappend -no-progress -no-xattrs + mv "$_sq_new" "$_sq" + rm -rf "$_tmp_parent" + for _candidate in "${BUILD_WORK_DIR}/binary/live/"filesystem*.squashfs; do + [ -e "$_candidate" ] || continue + [ "$_candidate" = "$_sq" ] || rm -f "$_candidate" + done + echo "=== fast-path: squashfs repacked ($(du -sh "$_sq" | cut -f1)) ===" +) + +# Fast-path: rebuild ISO replacing the squashfs via xorriso. +# Boot structure (El Torito, EFI, MBR hybrid) is replayed from the prior ISO. +recover_iso_memtest() { + lb_dir="$1" + iso_path="$2" + binary_boot="$lb_dir/binary/boot" + + echo "=== attempting memtest recovery in binary tree ===" + + mkdir -p "$binary_boot" + + for root in \ + "$lb_dir/chroot/boot" \ + "/boot"; do + for f in memtest86+x64.bin memtest86+x64.efi; do + if [ ! -f "$binary_boot/$f" ] && [ -f "$root/$f" ]; then + cp "$root/$f" "$binary_boot/$f" + echo "memtest recovery: copied $f from $root" + fi + done + done + + if [ ! -f "$binary_boot/memtest86+x64.bin" ] || [ ! -f "$binary_boot/memtest86+x64.efi" ]; then + for dir in \ + "$lb_dir/cache/packages.binary" \ + "$lb_dir/cache/packages.chroot" \ + "$lb_dir/chroot/var/cache/apt/archives" \ + "${BEE_CACHE_DIR:-${DIST_DIR}/cache}/lb-packages" \ + "/var/cache/apt/archives"; do + [ -d "$dir" ] || continue + deb="$(find "$dir" -maxdepth 1 -type f -name 'memtest86+*.deb' 2>/dev/null | head -1)" + [ -n "$deb" ] || continue + echo "memtest recovery: extracting payload from $deb" + copy_memtest_from_deb "$deb" "$binary_boot" + break + done + fi + + if [ ! -f "$binary_boot/memtest86+x64.bin" ] || [ ! -f "$binary_boot/memtest86+x64.efi" ]; then + tmpdl="$(mktemp -d)" + if ( + cd "$tmpdl" && apt-get download memtest86+ >/dev/null 2>&1 + ); then + deb="$(find "$tmpdl" -maxdepth 1 -type f -name 'memtest86+*.deb' 2>/dev/null | head -1)" + if [ -n "$deb" ]; then + echo "memtest recovery: downloaded $deb" + copy_memtest_from_deb "$deb" "$binary_boot" + fi + fi + rm -rf "$tmpdl" + fi + + enforce_live_build_bootloader_assets "$lb_dir" + + reset_live_build_stage "$lb_dir" "binary_checksums" + reset_live_build_stage "$lb_dir" "binary_iso" + reset_live_build_stage "$lb_dir" "binary_zsync" + + run_optional_step_sh "rebuild live-build checksums after memtest recovery" "91-lb-checksums" "lb binary_checksums 2>&1" + run_optional_step_sh "rebuild ISO after memtest recovery" "92-lb-binary-iso" "rm -f '$iso_path' && lb binary_iso 2>&1" + run_optional_step_sh "rebuild zsync after memtest recovery" "93-lb-zsync" "lb binary_zsync 2>&1" + + if [ ! -f "$iso_path" ]; then + memtest_fail "ISO rebuild was skipped or failed after memtest recovery: $iso_path" "$iso_path" + fi +} diff --git a/iso/builder/lib/iso-validation.sh b/iso/builder/lib/iso-validation.sh new file mode 100755 index 0000000..ca4bc03 --- /dev/null +++ b/iso/builder/lib/iso-validation.sh @@ -0,0 +1,636 @@ +iso_list_files() { + iso_path="$1" + + if command -v bsdtar >/dev/null 2>&1; then + bsdtar -tf "$iso_path" + return $? + fi + + if command -v xorriso >/dev/null 2>&1; then + xorriso -indev "$iso_path" -find / -type f -print 2>/dev/null | sed 's#^/##' + return $? + fi + + return 127 +} + +iso_extract_file() { + iso_path="$1" + iso_member="$2" + + if command -v bsdtar >/dev/null 2>&1; then + bsdtar -xOf "$iso_path" "$iso_member" + return $? + fi + + if command -v xorriso >/dev/null 2>&1; then + xorriso -osirrox on -indev "$iso_path" -cat "/$iso_member" 2>/dev/null + return $? + fi + + return 127 +} + +iso_read_file_list() { + iso_path="$1" + out_path="$2" + + iso_list_files "$iso_path" > "$out_path" || return 1 + [ -s "$out_path" ] || return 1 + return 0 +} + +iso_read_member() { + iso_path="$1" + iso_member="$2" + out_path="$3" + + iso_extract_file "$iso_path" "$iso_member" > "$out_path" || return 1 + [ -s "$out_path" ] || return 1 + return 0 +} + +require_iso_reader() { + command -v bsdtar >/dev/null 2>&1 && return 0 + command -v xorriso >/dev/null 2>&1 && return 0 + memtest_fail "ISO reader is required for validation/debug (expected bsdtar or xorriso)" "${1:-}" +} + +dump_memtest_debug() { + phase="$1" + lb_dir="${2:-}" + iso_path="${3:-}" + phase_slug="$(printf '%s' "${phase}" | tr ' /' '__')" + memtest_log="${LOG_DIR:-}/memtest-${phase_slug}.log" + + ( + echo "=== memtest debug: ${phase} ===" + + echo "-- auto/config --" + if [ -f "${BUILDER_DIR}/auto/config" ]; then + grep -n -- '--memtest' "${BUILDER_DIR}/auto/config" || echo " (no --memtest line found)" + else + echo " (missing ${BUILDER_DIR}/auto/config)" + fi + + echo "-- source bootloader templates --" + for cfg in \ + "${BUILDER_DIR}/config/bootloaders/grub-efi/grub.cfg" \ + "${BUILDER_DIR}/config/bootloaders/isolinux/live.cfg.in"; do + if [ -f "$cfg" ]; then + echo " file: $cfg" + grep -n 'Memory Test\|memtest' "$cfg" || echo " (no memtest lines)" + fi + done + + echo "-- source binary hooks --" + for hook in \ + "${BUILDER_DIR}/config/hooks/normal/9100-memtest.hook.binary"; do + if [ -f "$hook" ]; then + echo " hook: $hook" + else + echo " (missing $hook)" + fi + done + + if [ -n "$lb_dir" ] && [ -d "$lb_dir" ]; then + echo "-- live-build workdir package lists --" + for pkg in \ + "$lb_dir/config/package-lists/bee.list.chroot" \ + "$lb_dir/config/package-lists/bee-gpu.list.chroot" \ + "$lb_dir/config/package-lists/bee-nvidia.list.chroot"; do + if [ -f "$pkg" ]; then + echo " file: $pkg" + grep -n 'memtest' "$pkg" || echo " (no memtest lines)" + fi + done + + echo "-- live-build chroot/boot --" + if [ -d "$lb_dir/chroot/boot" ]; then + find "$lb_dir/chroot/boot" -maxdepth 1 -name 'memtest*' -print | sed 's/^/ /' || true + else + echo " (missing $lb_dir/chroot/boot)" + fi + + echo "-- live-build binary/boot --" + if [ -d "$lb_dir/binary/boot" ]; then + find "$lb_dir/binary/boot" -maxdepth 1 -name 'memtest*' -print | sed 's/^/ /' || true + else + echo " (missing $lb_dir/binary/boot)" + fi + + echo "-- live-build binary grub cfg --" + if [ -f "$lb_dir/binary/boot/grub/grub.cfg" ]; then + grep -n 'Memory Test\|memtest' "$lb_dir/binary/boot/grub/grub.cfg" || echo " (no memtest lines)" + else + echo " (missing $lb_dir/binary/boot/grub/grub.cfg)" + fi + + echo "-- live-build binary isolinux cfg --" + if [ -f "$lb_dir/binary/isolinux/live.cfg" ]; then + grep -n 'Memory Test\|memtest' "$lb_dir/binary/isolinux/live.cfg" || echo " (no memtest lines)" + else + echo " (missing $lb_dir/binary/isolinux/live.cfg)" + fi + + echo "-- live-build package cache --" + if [ -d "$lb_dir/cache/packages.chroot" ]; then + find "$lb_dir/cache/packages.chroot" -maxdepth 1 -name 'memtest86+*.deb' -print | sed 's/^/ /' || true + else + echo " (missing $lb_dir/cache/packages.chroot)" + fi + fi + + if [ -n "$iso_path" ] && [ -f "$iso_path" ]; then + iso_files="$(mktemp)" + iso_grub_cfg="$(mktemp)" + iso_isolinux_cfg="$(mktemp)" + + echo "-- ISO memtest files --" + if iso_read_file_list "$iso_path" "$iso_files"; then + grep 'memtest' "$iso_files" | sed 's/^/ /' || echo " (no memtest files in ISO)" + else + echo " (failed to list ISO contents)" + fi + + echo "-- ISO GRUB memtest lines --" + if iso_read_member "$iso_path" boot/grub/grub.cfg "$iso_grub_cfg"; then + grep -n 'Memory Test\|memtest' "$iso_grub_cfg" || echo " (no memtest lines in boot/grub/grub.cfg)" + else + echo " (failed to read boot/grub/grub.cfg from ISO)" + fi + + echo "-- ISO isolinux memtest lines --" + if iso_read_member "$iso_path" isolinux/live.cfg "$iso_isolinux_cfg"; then + grep -n 'Memory Test\|memtest' "$iso_isolinux_cfg" || echo " (no memtest lines in isolinux/live.cfg)" + else + echo " (failed to read isolinux/live.cfg from ISO)" + fi + + rm -f "$iso_files" "$iso_grub_cfg" "$iso_isolinux_cfg" + fi + + echo "=== end memtest debug: ${phase} ===" + ) | { + if [ -n "${LOG_DIR:-}" ] && [ -d "${LOG_DIR}" ]; then + tee "${memtest_log}" + else + cat + fi + } +} + +memtest_fail() { + msg="$1" + iso_path="${2:-}" + level="WARNING" + if [ "${BEE_REQUIRE_MEMTEST:-0}" = "1" ]; then + level="ERROR" + fi + echo "${level}: ${msg}" >&2 + dump_memtest_debug "failure" "${LB_DIR:-}" "$iso_path" >&2 + if [ "${BEE_REQUIRE_MEMTEST:-0}" = "1" ]; then + exit 1 + fi + return 0 +} + +nvidia_runtime_fail() { + msg="$1" + echo "ERROR: ${msg}" >&2 + exit 1 +} + +iso_memtest_present() { + iso_path="$1" + iso_files="$(mktemp)" + + [ -f "$iso_path" ] || return 1 + + if command -v bsdtar >/dev/null 2>&1; then + : + elif command -v xorriso >/dev/null 2>&1; then + : + else + return 2 + fi + + iso_read_file_list "$iso_path" "$iso_files" || { + rm -f "$iso_files" + return 2 + } + + grep -q '^boot/memtest86+x64\.bin$' "$iso_files" || { + rm -f "$iso_files" + return 1 + } + grep -q '^boot/memtest86+x64\.efi$' "$iso_files" || { + rm -f "$iso_files" + return 1 + } + + grub_cfg="$(mktemp)" + isolinux_cfg="$(mktemp)" + + iso_read_member "$iso_path" boot/grub/grub.cfg "$grub_cfg" || { + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 2 + } + iso_read_member "$iso_path" isolinux/live.cfg "$isolinux_cfg" || { + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 2 + } + + grep -q 'Memory Test (memtest86+)' "$grub_cfg" || { + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 1 + } + grep -q '/boot/memtest86+x64\.efi' "$grub_cfg" || { + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 1 + } + grep -q '/boot/memtest86+x64\.bin' "$grub_cfg" || { + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 1 + } + grep -q 'Memory Test (memtest86+)' "$isolinux_cfg" || { + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 1 + } + grep -q '/boot/memtest86+x64\.bin' "$isolinux_cfg" || { + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 1 + } + + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 0 +} + +validate_iso_memtest() { + iso_path="$1" + echo "=== validating memtest in ISO ===" + + [ -f "$iso_path" ] || { + memtest_fail "ISO not found for validation: $iso_path" "$iso_path" + return 0 + } + require_iso_reader "$iso_path" || return 0 + + iso_files="$(mktemp)" + iso_read_file_list "$iso_path" "$iso_files" || { + memtest_fail "failed to list ISO contents while validating memtest" "$iso_path" + rm -f "$iso_files" + return 0 + } + + grep -q '^boot/memtest86+x64\.bin$' "$iso_files" || { + memtest_fail "memtest BIOS binary missing in ISO: boot/memtest86+x64.bin" "$iso_path" + rm -f "$iso_files" + return 0 + } + grep -q '^boot/memtest86+x64\.efi$' "$iso_files" || { + memtest_fail "memtest EFI binary missing in ISO: boot/memtest86+x64.efi" "$iso_path" + rm -f "$iso_files" + return 0 + } + + grub_cfg="$(mktemp)" + isolinux_cfg="$(mktemp)" + + iso_read_member "$iso_path" boot/grub/grub.cfg "$grub_cfg" || { + memtest_fail "failed to read boot/grub/grub.cfg from ISO" "$iso_path" + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 0 + } + iso_read_member "$iso_path" isolinux/live.cfg "$isolinux_cfg" || { + memtest_fail "failed to read isolinux/live.cfg from ISO" "$iso_path" + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 0 + } + + grep -q 'Memory Test (memtest86+)' "$grub_cfg" || { + memtest_fail "GRUB menu entry for memtest is missing" "$iso_path" + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 0 + } + grep -q '/boot/memtest86+x64\.efi' "$grub_cfg" || { + memtest_fail "GRUB memtest EFI path is missing" "$iso_path" + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 0 + } + grep -q '/boot/memtest86+x64\.bin' "$grub_cfg" || { + memtest_fail "GRUB memtest BIOS path is missing" "$iso_path" + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 0 + } + grep -q 'Memory Test (memtest86+)' "$isolinux_cfg" || { + memtest_fail "isolinux menu entry for memtest is missing" "$iso_path" + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 0 + } + grep -q '/boot/memtest86+x64\.bin' "$isolinux_cfg" || { + memtest_fail "isolinux memtest path is missing" "$iso_path" + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + return 0 + } + + rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" + echo "=== memtest validation OK ===" +} + +validate_live_cmdline_params() { + cfg="$1" + command="$2" + bootloader="$3" + expected_label="$4" + + awk -v command="$command" -v bootloader="$bootloader" -v expected_label="$expected_label" ' + function has(token, i) { + for (i = 1; i <= NF; i++) { + if ($i == token) { + return 1 + } + } + return 0 + } + + function reject(message) { + printf "ERROR: %s live entry at %s:%d: %s\n", bootloader, FILENAME, NR, message + bad = 1 + } + + $1 == command && has("boot=live") { + live_entries++ + + if (!has("udev.children_max=1")) { + reject("missing udev.children_max=1") + } + if (!has("intel_iommu=on")) { + reject("missing intel_iommu=on") + } + if (!has("iommu.passthrough=0")) { + reject("missing iommu.passthrough=0") + } + if (!has("efi=disable_early_pci_dma")) { + reject("missing efi=disable_early_pci_dma") + } + if (!has("live-media-label=" expected_label)) { + reject("missing expected live-media-label=" expected_label) + } + if (has("iommu=pt")) { + reject("contains forbidden iommu=pt") + } + + if (has("pci=realloc")) { + failsafe_entries++ + if (!has("iommu.strict=1")) { + reject("pci=realloc entry is missing iommu.strict=1") + } + } else if (has("iommu.strict=1")) { + reject("iommu.strict=1 is allowed only in the pci=realloc fail-safe entry") + } + } + + END { + if (live_entries == 0) { + printf "ERROR: %s config has no live boot entries\n", bootloader + bad = 1 + } + if (failsafe_entries != 1) { + printf "ERROR: %s config has %d pci=realloc fail-safe entries, expected 1\n", bootloader, failsafe_entries + bad = 1 + } + exit bad ? 1 : 0 + } + ' "$cfg" +} + +validate_iso_live_boot_entries() { + iso_path="$1" + echo "=== validating live boot entries in ISO ===" + + [ -f "$iso_path" ] || { + echo "ERROR: ISO not found for live boot validation: $iso_path" >&2 + exit 1 + } + require_iso_reader "$iso_path" >/dev/null 2>&1 || { + echo "ERROR: ISO reader unavailable for live boot validation" >&2 + exit 1 + } + + grub_cfg="$(mktemp)" + isolinux_cfg="$(mktemp)" + + iso_read_member "$iso_path" boot/grub/grub.cfg "$grub_cfg" || { + echo "ERROR: failed to read boot/grub/grub.cfg from ISO" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + } + iso_read_member "$iso_path" isolinux/live.cfg "$isolinux_cfg" || { + echo "ERROR: failed to read isolinux/live.cfg from ISO" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + } + + if grep -q '@APPEND_LIVE@\|@KERNEL_LIVE@\|@INITRD_LIVE@' "$grub_cfg" "$isolinux_cfg"; then + echo "ERROR: unresolved live-build placeholders remain in ISO bootloader config" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + fi + + if grep -q 'iommu=pt' "$grub_cfg" "$isolinux_cfg"; then + echo "ERROR: forbidden iommu=pt remains in ISO bootloader config" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + fi + + if ! validate_live_cmdline_params "$grub_cfg" linux GRUB "${BEE_ISO_VOLUME}"; then + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + fi + if ! validate_live_cmdline_params "$isolinux_cfg" append isolinux "${BEE_ISO_VOLUME}"; then + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + fi + + grep -q 'menuentry "EASY-BEE v' "$grub_cfg" || { + echo "ERROR: GRUB default EASY-BEE entry is missing" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + } + grep -Fq "menuentry \"EASY-BEE v${PROJECT_VERSION_EFFECTIVE}\"" "$grub_cfg" || { + echo "ERROR: GRUB version does not match ${PROJECT_VERSION_EFFECTIVE}" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + } + grep -Fq "menu label ^EASY-BEE v${PROJECT_VERSION_EFFECTIVE}" "$isolinux_cfg" || { + echo "ERROR: isolinux version does not match ${PROJECT_VERSION_EFFECTIVE}" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + } + + if command -v xorriso >/dev/null 2>&1; then + iso_volume="$(xorriso -indev "$iso_path" -pvd_info 2>/dev/null | awk -F: '/Volume [Ii]d/ { sub(/^[[:space:]'\''"]+/, "", $2); sub(/[[:space:]'\''"]+$/, "", $2); print $2; exit }')" + if [ "$iso_volume" != "${BEE_ISO_VOLUME}" ]; then + echo "ERROR: ISO volume ID is ${iso_volume:-unknown}, expected ${BEE_ISO_VOLUME}" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + fi + fi + grep -q 'menuentry "EASY-BEE v.* -- load to RAM (toram)"' "$grub_cfg" || { + echo "ERROR: GRUB toram entry is missing" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + } + grep -q 'linux .*boot=live ' "$grub_cfg" || { + echo "ERROR: GRUB live entry is missing boot=live" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + } + grep -q 'linux .*boot=live .*toram ' "$grub_cfg" || { + echo "ERROR: GRUB toram entry is missing boot=live or toram" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + } + grep -q 'linux .*live-media-label=EASY_BEE_' "$grub_cfg" || { + echo "ERROR: GRUB live entry is missing live-media-label pinning" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + } + + grep -q 'append .*boot=live ' "$isolinux_cfg" || { + echo "ERROR: isolinux live entry is missing boot=live" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + } + grep -q 'append .*boot=live .*toram ' "$isolinux_cfg" || { + echo "ERROR: isolinux toram entry is missing boot=live or toram" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + } + grep -q 'append .*live-media-label=EASY_BEE_' "$isolinux_cfg" || { + echo "ERROR: isolinux live entry is missing live-media-label pinning" >&2 + rm -f "$grub_cfg" "$isolinux_cfg" + exit 1 + } + + rm -f "$grub_cfg" "$isolinux_cfg" + echo "=== live boot validation OK ===" +} + +validate_iso_grub_assets() { + iso_path="$1" + echo "=== validating GRUB assets in ISO ===" + + [ -f "$iso_path" ] || { + echo "ERROR: ISO not found for GRUB asset validation: $iso_path" >&2 + exit 1 + } + require_iso_reader "$iso_path" >/dev/null 2>&1 || { + echo "ERROR: ISO reader unavailable for GRUB asset validation" >&2 + exit 1 + } + + iso_files="$(mktemp)" + iso_list_files "$iso_path" > "$iso_files" || { + echo "ERROR: failed to list ISO files for GRUB asset validation" >&2 + rm -f "$iso_files" + exit 1 + } + + for required in \ + boot/grub/config.cfg \ + boot/grub/grub.cfg; do + grep -q "^${required}$" "$iso_files" || { + echo "ERROR: missing GRUB asset in ISO: ${required}" >&2 + rm -f "$iso_files" + exit 1 + } + done + + rm -f "$iso_files" + echo "=== GRUB asset validation OK ===" +} + +validate_iso_nvidia_runtime() { + iso_path="$1" + [ "$BEE_GPU_VENDOR" = "nvidia" ] || return 0 + + echo "=== validating NVIDIA runtime in ISO ===" + + [ -f "$iso_path" ] || nvidia_runtime_fail "ISO not found for NVIDIA runtime validation: $iso_path" + require_iso_reader "$iso_path" >/dev/null 2>&1 || nvidia_runtime_fail "ISO reader unavailable for NVIDIA runtime validation" + command -v unsquashfs >/dev/null 2>&1 || nvidia_runtime_fail "unsquashfs is required for NVIDIA runtime validation" + + squashfs_tmp="$(mktemp)" + squashfs_list="$(mktemp)" + iso_files="$(mktemp)" + dpkg_status_dir="$(mktemp -d)" + iso_list_files "$iso_path" > "$iso_files" || { + rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" + nvidia_runtime_fail "failed to list ISO files for NVIDIA runtime validation" + } + grep '^live/.*\.squashfs$' "$iso_files" | while IFS= read -r squashfs_member; do + iso_read_member "$iso_path" "$squashfs_member" "$squashfs_tmp" || { + rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" + nvidia_runtime_fail "failed to extract $squashfs_member from ISO" + } + unsquashfs -ll "$squashfs_tmp" >> "$squashfs_list" 2>/dev/null || { + rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" + nvidia_runtime_fail "failed to inspect $squashfs_member from ISO" + } + # var/lib/dpkg/status lives in whichever squashfs layer has the base + # rootfs (not the usr/firmware split-off layers); harmless no-op on + # the others. + unsquashfs -d "${dpkg_status_dir}/extract" -f "$squashfs_tmp" var/lib/dpkg/status >/dev/null 2>&1 || true + : > "$squashfs_tmp" + done + + grep -Eq 'usr/bin/dcgmi$' "$squashfs_list" || { + rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" + nvidia_runtime_fail "dcgmi missing from final NVIDIA ISO" + } + grep -Eq 'usr/bin/nv-hostengine$' "$squashfs_list" || { + rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" + nvidia_runtime_fail "nv-hostengine missing from final NVIDIA ISO" + } + grep -Eq 'usr/bin/dcgmproftester([0-9]+)?$' "$squashfs_list" || { + rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" + nvidia_runtime_fail "dcgmproftester missing from final NVIDIA ISO" + } + + # Cross-check the DCGM package version actually baked into the squashfs + # against VERSIONS. dcgmi/nv-hostengine/dcgmproftester presence alone + # doesn't catch a stale squashfs served by a mis-detected fast-path build + # (dcgmi stays present across DCGM versions); this does. + dpkg_status_file="${dpkg_status_dir}/extract/var/lib/dpkg/status" + if [ -f "$dpkg_status_file" ]; then + _installed_dcgm_version="$(awk ' + /^Package: datacenter-gpu-manager-4-core$/ { in_pkg=1; next } + /^Package: / { in_pkg=0 } + in_pkg && /^Version: / { sub(/^Version: /, ""); print; exit } + ' "$dpkg_status_file")" + if [ -z "$_installed_dcgm_version" ]; then + echo "=== WARNING: datacenter-gpu-manager-4-core not found in ISO dpkg status; skipping DCGM version check ===" + else + _installed_dcgm_no_epoch="${_installed_dcgm_version#*:}" + if [ "$_installed_dcgm_no_epoch" != "${DCGM_VERSION}" ]; then + rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" + nvidia_runtime_fail "DCGM version mismatch: VERSIONS pins ${DCGM_VERSION} but ISO has ${_installed_dcgm_version} (stale squashfs; retry with --clean-build)" + fi + fi + else + echo "=== WARNING: could not read dpkg status from ISO; skipping DCGM version check ===" + fi + + rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" + echo "=== NVIDIA runtime validation OK ===" +} diff --git a/iso/builder/test-build-libs.sh b/iso/builder/test-build-libs.sh new file mode 100755 index 0000000..cbfd57a --- /dev/null +++ b/iso/builder/test-build-libs.sh @@ -0,0 +1,98 @@ +#!/bin/sh +set -eu + +BUILDER_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)" +TEST_ROOT="$(mktemp -d)" +trap 'rm -rf "$TEST_ROOT"' EXIT INT TERM HUP + +PROJECT_VERSION_EFFECTIVE="13.0-test" +BEE_ISO_VOLUME="EASY_BEE_TEST" +LOG_DIR="$TEST_ROOT/log" +mkdir -p "$LOG_DIR" + +. "$BUILDER_DIR/lib/iso-validation.sh" +. "$BUILDER_DIR/lib/bootloader.sh" + +literal_template="$TEST_ROOT/literal-template" +literal_output="$TEST_ROOT/literal-output" +printf '%s\n' '@VERSION@ @KERNEL@ @APPEND_LIVE@ @INITRD@' > "$literal_template" +render_bootloader_template "$literal_template" "$literal_output" \ + '13&0#test' '@KERNEL@' '/live/vmlinuz\\literal' 'boot=live marker=a&b#c\\d' '@INITRD@' '/live/initrd.img' +literal_expected='13&0#test /live/vmlinuz\\literal boot=live marker=a&b#c\\d /live/initrd.img' +if [ "$(cat "$literal_output")" != "$literal_expected" ]; then + echo "ERROR: bootloader renderer changed literal replacement characters" >&2 + exit 1 +fi + +rendered_grub="$TEST_ROOT/grub.cfg" +rendered_isolinux="$TEST_ROOT/live.cfg" +sed \ + -e 's#@APPEND_LIVE@#boot=live live-media-label=EASY_BEE_TEST#g' \ + -e 's#@KERNEL_LIVE@#/live/vmlinuz#g' \ + -e 's#@INITRD_LIVE@#/live/initrd.img#g' \ + -e 's#@VERSION@#13.0-test#g' \ + "$BUILDER_DIR/config/bootloaders/grub-efi/grub.cfg" > "$rendered_grub" +sed \ + -e 's#@APPEND_LIVE@#boot=live live-media-label=EASY_BEE_TEST#g' \ + -e 's#@LINUX@#/live/vmlinuz#g' \ + -e 's#@INITRD@#/live/initrd.img#g' \ + -e 's#@VERSION@#13.0-test#g' \ + "$BUILDER_DIR/config/bootloaders/isolinux/live.cfg.in" > "$rendered_isolinux" + +validate_live_cmdline_params "$rendered_grub" linux GRUB "$BEE_ISO_VOLUME" +validate_live_cmdline_params "$rendered_isolinux" append isolinux "$BEE_ISO_VOLUME" + +missing_serialization="$TEST_ROOT/missing-serialization.cfg" +sed 's/ udev\.children_max=1//' "$rendered_grub" > "$missing_serialization" +if validate_live_cmdline_params "$missing_serialization" linux GRUB "$BEE_ISO_VOLUME" >/dev/null 2>&1; then + echo "ERROR: validator accepted a live entry without udev.children_max=1" >&2 + exit 1 +fi + +CACHE_ROOT="$TEST_ROOT/cache" +BUILD_VARIANT="test" +BUILD_WORK_DIR="$TEST_ROOT/work" +OVERLAY_STAGE_DIR="$TEST_ROOT/overlay" +DEBIAN_KERNEL_ABI="6.1.0-test" +SQUASHFS_FILENAME="filesystem-v13.0-test.squashfs" +DIST_DIR="$TEST_ROOT/dist" +mkdir -p "$BUILD_WORK_DIR/binary/live" "$OVERLAY_STAGE_DIR" +touch "$BUILD_WORK_DIR/live-image-amd64.hybrid.iso" +touch "$BUILD_WORK_DIR/binary/live/$SQUASHFS_FILENAME" + +. "$BUILDER_DIR/lib/fast-path.sh" + +# Isolate the decision test from repository content and GNU find extensions. +hash_heavy_config() { printf '%s\n' test-heavy-hash; } +write_overlay_manifest() { find "$OVERLAY_STAGE_DIR" -mindepth 1 -print | sed "s#^$OVERLAY_STAGE_DIR/##" | sort > "$1"; } + +printf '%s\n' test-heavy-hash > "$FULL_BUILD_HASH_FILE" +printf '%s\n' "$DEBIAN_KERNEL_ABI" > "$FULL_BUILD_ABI_FILE" +write_overlay_manifest "$FULL_BUILD_OVERLAY_MANIFEST" +touch "$FULL_BUILD_MARKER" + +if needs_full_build; then + echo "ERROR: fast path was rejected for unchanged valid state" >&2 + exit 1 +fi + +touch "$OVERLAY_STAGE_DIR/added-by-fast-path" +if needs_full_build; then + echo "ERROR: an additive overlay change unnecessarily forced a full build" >&2 + exit 1 +fi +write_overlay_manifest "$FULL_BUILD_OVERLAY_MANIFEST" +rm "$OVERLAY_STAGE_DIR/added-by-fast-path" +if ! needs_full_build >/dev/null; then + echo "ERROR: removal from the latest fast-path overlay was not detected" >&2 + exit 1 +fi +touch "$OVERLAY_STAGE_DIR/added-by-fast-path" + +printf '%s\n' different-abi > "$FULL_BUILD_ABI_FILE" +if ! needs_full_build >/dev/null; then + echo "ERROR: kernel ABI change did not force a full build" >&2 + exit 1 +fi + +echo "build library tests: OK" diff --git a/iso/overlay/usr/local/bin/bee-nvidia-load b/iso/overlay/usr/local/bin/bee-nvidia-load index 0c06a51..b45f544 100755 --- a/iso/overlay/usr/local/bin/bee-nvidia-load +++ b/iso/overlay/usr/local/bin/bee-nvidia-load @@ -1,5 +1,5 @@ #!/bin/sh -# bee-nvidia-load — load NVIDIA kernel modules and create device nodes +# bee-nvidia-load - load NVIDIA kernel modules and create device nodes # Called by bee-nvidia.service at boot. NVIDIA_KO_DIR="/usr/local/lib/nvidia" @@ -28,7 +28,7 @@ have_nvidia_gpu() { } if ! have_nvidia_gpu; then - log "no NVIDIA GPU detected — skipping module load" + log "no NVIDIA GPU detected - skipping module load" exit 0 fi @@ -65,7 +65,8 @@ load_module() { mod="$1" shift ko="$NVIDIA_KO_DIR/${mod}.ko" - [ -f "$ko" ] || ko="$NVIDIA_KO_DIR/${mod//-/_}.ko" + mod_file="$(printf '%s' "$mod" | tr '-' '_')" + [ -f "$ko" ] || ko="$NVIDIA_KO_DIR/${mod_file}.ko" if [ ! -f "$ko" ]; then log "WARN: not found: $ko" return 1 @@ -90,7 +91,7 @@ load_module_with_gsp_fallback() { return 1 fi - # Run insmod in background — on some converted SXM→PCIe cards GSP enters an + # Run insmod in background. On some converted SXM-to-PCIe cards GSP enters an # infinite crash/reload loop and insmod never returns. We check for successful # initialization by polling /proc/devices for nvidiactl instead of waiting for # insmod to exit. @@ -114,29 +115,29 @@ load_module_with_gsp_fallback() { dmesg | tail -n 10 | sed 's/^/ dmesg: /' || true return 1 fi - # insmod exited 0 but nvidiactl not yet in /proc/devices — give it a moment + # insmod exited 0 but nvidiactl is not yet in /proc/devices; give it a moment sleep 2 if nvidia_is_functional; then log "loaded: nvidia (GSP enabled, ${_waited}s)" return 0 fi - log "insmod exited 0 but nvidiactl missing — treating as failure" + log "insmod exited 0 but nvidiactl missing - treating as failure" return 1 fi sleep 1 _waited=$((_waited + 1)) done - # GSP init timed out — kill the hanging insmod and attempt gsp-off fallback + # GSP init timed out; kill the hanging insmod and attempt gsp-off fallback. log "nvidia GSP init timed out after 90s" kill "$_insmod_pid" 2>/dev/null || true wait "$_insmod_pid" 2>/dev/null || true # Attempt to unload the partially-initialized module if ! rmmod nvidia 2>/dev/null; then - # Module is stuck in the kernel — cannot reload with different params. + # Module is stuck in the kernel; cannot reload with different params. # User must reboot and select bee.nvidia.mode=gsp-off at boot menu. - log "ERROR: rmmod nvidia failed (EBUSY) — module stuck in kernel" + log "ERROR: rmmod nvidia failed (EBUSY) - module stuck in kernel" log "ERROR: reboot and select 'EASY-BEE (advanced) -> GSP=off' in boot menu" echo "gsp-stuck" > /run/bee-nvidia-mode return 1 @@ -144,7 +145,7 @@ load_module_with_gsp_fallback() { sleep 2 log "retrying with NVreg_EnableGpuFirmware=0" - log "WARNING: GSP disabled — power management will run via CPU path, not GPU firmware" + log "WARNING: GSP disabled - power management will run via CPU path, not GPU firmware" if insmod "$ko" NVreg_EnableGpuFirmware=0; then if nvidia_is_functional; then @@ -208,7 +209,7 @@ else log "GSP-off mode: skipping nvidia-modeset and nvidia-uvm during boot" ;; nomsi|*) - # nomsi: disable MSI-X/MSI interrupts — use when RmInitAdapter fails with + # nomsi: disable MSI-X/MSI interrupts; use when RmInitAdapter fails with # "Failed to enable MSI-X" on one or more GPUs (IOMMU group interrupt limits). # NVreg_EnableMSI=0 forces legacy INTx interrupts for all GPUs. if ! load_module nvidia NVreg_EnableGpuFirmware=0 NVreg_EnableMSI=0; then @@ -230,7 +231,7 @@ if [ -n "$nvidia_major" ]; then done log "created /dev/nvidia{0-7}" else - log "WARN: nvidiactl not in /proc/devices — no GPU hardware present?" + log "WARN: nvidiactl not in /proc/devices - no GPU hardware present?" fi uvm_major=$(grep -m1 ' nvidia-uvm$' /proc/devices | awk '{print $1}') @@ -255,60 +256,40 @@ if command -v nvidia-smi >/dev/null 2>&1; then log "WARN: failed to enable NVIDIA persistence mode" fi else - log "WARN: nvidia-smi not found — cannot enable persistence mode" + log "WARN: nvidia-smi not found - cannot enable persistence mode" fi -# Bound every systemctl call below: a unit whose ExecStart/ExecCondition hangs -# (e.g. fabricmanager stuck training a bad NVSwitch fabric) must not be able to -# wedge bee-nvidia.service forever — that would keep nvidia-dcgm.service from -# ever starting, since it's ordered After= this one. 60s comfortably covers a -# normal fabricmanager/dcgm startup without blocking boot indefinitely. -SYSTEMCTL_TIMEOUT=60 -timeout_systemctl() { - timeout "${SYSTEMCTL_TIMEOUT}" systemctl "$@" +# Refresh nvidia-fabricmanager and nvidia-dcgm so they (re)enumerate against +# the device nodes we just created. +# +# These MUST NOT block. bee-nvidia.service is Type=oneshot and ordered +# Before=nvidia-fabricmanager.service nvidia-dcgm.service, so systemd will not +# run either unit until this script returns. A synchronous "systemctl restart" +# here therefore deadlocks against our own ordering and only unwedges when its +# timeout fires. "systemctl --no-block try-restart" queues a restart only for +# an active unit and returns without waiting. An inactive enabled unit remains +# in the normal boot transaction and can start after bee-nvidia.service exits. +nvidia_refresh_unit() { + unit="$1" + if ! command -v systemctl >/dev/null 2>&1 || + ! systemctl list-unit-files --no-legend 2>/dev/null | awk -v wanted="$unit" '$1 == wanted { found=1 } END { exit(found ? 0 : 1) }'; then + log "WARN: ${unit} not installed" + return + fi + if systemctl --no-block try-restart "$unit" >/dev/null 2>&1; then + log "queued refresh of ${unit} (non-blocking)" + else + log "WARN: could not queue refresh of ${unit}" + fi } -# Start or refresh Fabric Manager after the NVIDIA stack is ready. On NVSwitch -# systems CUDA/DCGM can report "system not yet initialized" until fabric -# training completes under nvidia-fabricmanager. -if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-fabricmanager\.service'; then - log "restarting nvidia-fabricmanager.service (timeout ${SYSTEMCTL_TIMEOUT}s)" - if timeout_systemctl restart nvidia-fabricmanager.service >/dev/null 2>&1; then - log "nvidia-fabricmanager restarted" - elif [ $? -eq 124 ]; then - log "WARN: systemctl restart nvidia-fabricmanager.service timed out after ${SYSTEMCTL_TIMEOUT}s" - elif timeout_systemctl start nvidia-fabricmanager.service >/dev/null 2>&1; then - log "nvidia-fabricmanager started" - else - log "WARN: failed to start nvidia-fabricmanager.service" - systemctl status nvidia-fabricmanager.service --no-pager 2>&1 | sed 's/^/ fabricmanager: /' || true - fi -else - log "WARN: nvidia-fabricmanager.service not installed" -fi +# On NVSwitch systems CUDA/DCGM can report "system not yet initialized" until +# fabric training completes under nvidia-fabricmanager; on non-NVSwitch boxes +# the unit's ExecCondition skips it and this is a no-op. +nvidia_refresh_unit nvidia-fabricmanager.service -# Restart the DCGM host engine so dcgmi can discover GPUs. nv-hostengine -# enumerates GPUs once at startup and never rescans; bee-nvidia.service now -# orders itself Before=nvidia-dcgm.service so systemd shouldn't start it until -# modules/device nodes exist, but restart here too in case the unit was -# already active from a previous boot/reload with a stale empty inventory. -# Use systemctl (not a raw nv-hostengine invocation) so systemd's own -# supervision of nvidia-dcgm.service stays authoritative and we don't end up -# with two host engines racing for the same port. -if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-dcgm\.service'; then - log "restarting nvidia-dcgm.service (timeout ${SYSTEMCTL_TIMEOUT}s)" - if timeout_systemctl restart nvidia-dcgm.service >/dev/null 2>&1; then - log "nvidia-dcgm restarted" - elif [ $? -eq 124 ]; then - log "WARN: systemctl restart nvidia-dcgm.service timed out after ${SYSTEMCTL_TIMEOUT}s" - elif timeout_systemctl start nvidia-dcgm.service >/dev/null 2>&1; then - log "nvidia-dcgm started" - else - log "WARN: failed to start nvidia-dcgm.service" - systemctl status nvidia-dcgm.service --no-pager 2>&1 | sed 's/^/ nvidia-dcgm: /' || true - fi -else - log "WARN: nvidia-dcgm.service not installed" -fi +# If nvidia-dcgm is already active, restart it after the device nodes exist so +# its hostengine refreshes its device view. Otherwise normal boot starts it. +nvidia_refresh_unit nvidia-dcgm.service log "done"