From 0a6ca8ba0f175776b7e413bf32657ed40d759118 Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Tue, 1 Sep 2026 13:01:28 +0300 Subject: [PATCH] refactor: harden diagnostics and consolidate runtime code --- audit/internal/app/app.go | 8 - audit/internal/app/app_install.go | 19 -- audit/internal/app/app_network.go | 12 -- audit/internal/app/app_packs.go | 71 +------ audit/internal/app/app_services.go | 8 - audit/internal/app/blackbox_archive.go | 12 +- audit/internal/app/component_status_db.go | 13 +- .../internal/app/component_status_db_test.go | 31 +++ audit/internal/app/support_bundle.go | 84 +++++--- audit/internal/app/support_bundle_test.go | 97 +++++++++ audit/internal/platform/benchmark.go | 13 +- audit/internal/platform/benchmark_analysis.go | 7 - .../platform/benchmark_power_autotune.go | 3 - .../platform/benchmark_power_calibration.go | 61 ++---- .../internal/platform/install_to_ram_test.go | 6 - audit/internal/platform/live_metrics.go | 6 +- .../internal/platform/nvidia_config_check.go | 193 +++++++++++++----- .../platform/nvidia_config_check_test.go | 106 +++++++++- audit/internal/platform/sat.go | 170 +++++---------- audit/internal/platform/sat_fan_stress.go | 37 +--- audit/internal/platform/sat_gpu_packs.go | 98 ++++----- .../internal/platform/sat_nccl_validation.go | 123 +++++++++++ .../platform/sat_nccl_validation_test.go | 136 ++++++++++++ audit/internal/platform/sat_nvidia_health.go | 126 ++++++++++++ audit/internal/platform/sat_test.go | 26 +++ audit/internal/webui/api_gpu_tools.go | 153 +++++++------- audit/internal/webui/api_runs.go | 17 -- audit/internal/webui/api_services_system.go | 21 +- audit/internal/webui/api_system_test.go | 50 +++++ audit/internal/webui/jobs.go | 31 --- audit/internal/webui/metrics_charts_http.go | 31 --- audit/internal/webui/page_benchmark.go | 4 - audit/internal/webui/page_topo.go | 94 +-------- audit/internal/webui/page_topo_diagram.go | 35 +--- audit/internal/webui/pages.go | 73 ++++++- audit/internal/webui/server.go | 17 ++ audit/internal/webui/tasks.go | 4 +- .../internal/webui/tasks_http_persistence.go | 22 +- audit/internal/webui/tasks_test.go | 23 +++ bible-local/architecture/api-surface.md | 6 +- bible-local/architecture/runtime-flows.md | 17 ++ ...8-31-support-bundle-private-staging-dir.md | 52 +++++ bible-local/decisions/README.md | 1 + internal/chart | 2 +- iso/builder/build.sh | 47 +++-- iso/builder/lib/bootloader.sh | 8 + iso/builder/lib/iso-validation.sh | 7 +- iso/builder/lib/template.sh | 55 +++++ iso/builder/test-build-libs.sh | 42 ++++ 49 files changed, 1441 insertions(+), 837 deletions(-) create mode 100644 audit/internal/platform/sat_nccl_validation.go create mode 100644 audit/internal/platform/sat_nccl_validation_test.go create mode 100644 audit/internal/platform/sat_nvidia_health.go create mode 100644 audit/internal/webui/api_system_test.go create mode 100644 bible-local/decisions/2026-08-31-support-bundle-private-staging-dir.md create mode 100644 iso/builder/lib/template.sh diff --git a/audit/internal/app/app.go b/audit/internal/app/app.go index 4b982ea..c0b906b 100644 --- a/audit/internal/app/app.go +++ b/audit/internal/app/app.go @@ -129,14 +129,6 @@ func (a *App) TPMPresent() bool { return a.sat.TPMPresent() } -func (a *App) IsLiveMediaInRAM() bool { - return a.installer.IsLiveMediaInRAM() -} - -func (a *App) LiveBootSource() platform.LiveBootSource { - return a.installer.LiveBootSource() -} - func (a *App) LiveMediaRAMState() platform.LiveMediaRAMState { return a.installer.LiveMediaRAMState() } diff --git a/audit/internal/app/app_install.go b/audit/internal/app/app_install.go index c13c2b4..100cb72 100644 --- a/audit/internal/app/app_install.go +++ b/audit/internal/app/app_install.go @@ -1,7 +1,6 @@ package app import ( - "context" "fmt" "os" "path/filepath" @@ -14,20 +13,6 @@ func (a *App) ListRemovableTargets() ([]platform.RemovableTarget, error) { return a.exports.ListRemovableTargets() } -// ListScenarioFilesOnRemovableMedia lists scenarios/*.json found on any -// mounted removable target (e.g. the blackbox USB stick) — see -// platform.System.ListScenarioFilesOnRemovableMedia. -func (a *App) ListScenarioFilesOnRemovableMedia() ([]platform.ScenarioFileOnRemovableMedia, error) { - return a.exports.ListScenarioFilesOnRemovableMedia() -} - -// ReadScenarioFromRemovableMedia reads scenarios/.json from whichever -// mounted removable target has it — see -// platform.System.ReadScenarioFromRemovableMedia. -func (a *App) ReadScenarioFromRemovableMedia(name string) ([]byte, error) { - return a.exports.ReadScenarioFromRemovableMedia(name) -} - // ListAvailableScenarios lists every scenario runnable via ReadScenario: // shipped with the image plus anything found on removable media — see // platform.System.ListAvailableScenarios. @@ -86,7 +71,3 @@ func (a *App) ExportSupportBundleResult(target platform.RemovableTarget) (Action func (a *App) ListInstallDisks() ([]platform.InstallDisk, error) { return a.installer.ListInstallDisks() } - -func (a *App) InstallToDisk(ctx context.Context, device string, logFile string) error { - return a.installer.InstallToDisk(ctx, device, logFile) -} diff --git a/audit/internal/app/app_network.go b/audit/internal/app/app_network.go index f6f05bd..cb6e4ec 100644 --- a/audit/internal/app/app_network.go +++ b/audit/internal/app/app_network.go @@ -15,28 +15,16 @@ func (a *App) DefaultRoute() string { return a.network.DefaultRoute() } -func (a *App) DHCPOne(iface string) (string, error) { - return a.network.DHCPOne(iface) -} - func (a *App) DHCPOneResult(iface string) (ActionResult, error) { body, err := a.network.DHCPOne(iface) return ActionResult{Title: "DHCP: " + iface, Body: bodyOr(body, "DHCP completed.")}, err } -func (a *App) DHCPAll() (string, error) { - return a.network.DHCPAll() -} - func (a *App) DHCPAllResult() (ActionResult, error) { body, err := a.network.DHCPAll() return ActionResult{Title: "DHCP: all interfaces", Body: bodyOr(body, "DHCP completed.")}, err } -func (a *App) SetStaticIPv4(cfg platform.StaticIPv4Config) (string, error) { - return a.network.SetStaticIPv4(cfg) -} - func (a *App) SetInterfaceState(iface string, up bool) error { return a.network.SetInterfaceState(iface, up) } diff --git a/audit/internal/app/app_packs.go b/audit/internal/app/app_packs.go index 310fd88..5b45729 100644 --- a/audit/internal/app/app_packs.go +++ b/audit/internal/app/app_packs.go @@ -110,36 +110,28 @@ func (a *App) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir s return a.sat.RunNvidiaTargetedStressValidatePack(ctx, baseDir, durationSec, gpuIndices, logFunc) } -func (a *App) RunNvidiaStressPack(baseDir string, opts platform.NvidiaStressOptions, logFunc func(string)) (string, error) { - return a.RunNvidiaStressPackCtx(context.Background(), baseDir, opts, logFunc) -} - func (a *App) RunNvidiaBenchmark(baseDir string, opts platform.NvidiaBenchmarkOptions, logFunc func(string)) (string, error) { return a.RunNvidiaBenchmarkCtx(context.Background(), baseDir, opts, logFunc) } func (a *App) RunNvidiaBenchmarkCtx(ctx context.Context, baseDir string, opts platform.NvidiaBenchmarkOptions, logFunc func(string)) (string, error) { - if strings.TrimSpace(baseDir) == "" { - baseDir = DefaultBeeBenchPerfDir - } - resolved, err := a.ensureBenchmarkPowerAutotune(ctx, baseDir, opts, "performance", logFunc) - if err != nil { - return "", err - } - opts.ServerPowerSource = resolved.SelectedSource - return a.sat.RunNvidiaBenchmark(ctx, baseDir, opts, logFunc) + return a.runNvidiaBenchmarkKind(ctx, baseDir, DefaultBeeBenchPerfDir, opts, "performance", logFunc, a.sat.RunNvidiaBenchmark) } func (a *App) RunNvidiaPowerBenchCtx(ctx context.Context, baseDir string, opts platform.NvidiaBenchmarkOptions, logFunc func(string)) (string, error) { + return a.runNvidiaBenchmarkKind(ctx, baseDir, DefaultBeeBenchPowerDir, opts, "power-fit", logFunc, a.sat.RunNvidiaPowerBench) +} + +func (a *App) runNvidiaBenchmarkKind(ctx context.Context, baseDir, defaultBaseDir string, opts platform.NvidiaBenchmarkOptions, benchmarkKind string, logFunc func(string), run func(context.Context, string, platform.NvidiaBenchmarkOptions, func(string)) (string, error)) (string, error) { if strings.TrimSpace(baseDir) == "" { - baseDir = DefaultBeeBenchPowerDir + baseDir = defaultBaseDir } - resolved, err := a.ensureBenchmarkPowerAutotune(ctx, baseDir, opts, "power-fit", logFunc) + resolved, err := a.ensureBenchmarkPowerAutotune(ctx, baseDir, opts, benchmarkKind, logFunc) if err != nil { return "", err } opts.ServerPowerSource = resolved.SelectedSource - return a.sat.RunNvidiaPowerBench(ctx, baseDir, opts, logFunc) + return run(ctx, baseDir, opts, logFunc) } func (a *App) RunNvidiaPowerSourceAutotuneCtx(ctx context.Context, baseDir string, opts platform.NvidiaBenchmarkOptions, benchmarkKind string, logFunc func(string)) (string, error) { @@ -226,10 +218,6 @@ func (a *App) RunMemoryAcceptancePackResult(baseDir string) (ActionResult, error return ActionResult{Title: "Memory SAT", Body: satResultBody(path)}, err } -func (a *App) RunCPUAcceptancePack(baseDir string, durationSec int, logFunc func(string)) (string, error) { - return a.RunCPUAcceptancePackCtx(context.Background(), baseDir, durationSec, logFunc) -} - func (a *App) RunCPUAcceptancePackCtx(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { if strings.TrimSpace(baseDir) == "" { baseDir = DefaultSATBaseDir @@ -267,10 +255,6 @@ func (a *App) RunNvidiaConfigCheckPackCtx(ctx context.Context, baseDir string, l return a.sat.RunNvidiaConfigCheckPack(ctx, baseDir, logFunc) } -func (a *App) RunNvidiaConfigCheckPack(baseDir string, logFunc func(string)) (string, error) { - return a.RunNvidiaConfigCheckPackCtx(context.Background(), baseDir, logFunc) -} - func (a *App) RunPCIeLinkCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { if strings.TrimSpace(baseDir) == "" { baseDir = DefaultSATBaseDir @@ -278,10 +262,6 @@ func (a *App) RunPCIeLinkCheckPackCtx(ctx context.Context, baseDir string, logFu return a.sat.RunPCIeLinkCheckPack(ctx, baseDir, logFunc) } -func (a *App) RunPCIeLinkCheckPack(baseDir string, logFunc func(string)) (string, error) { - return a.RunPCIeLinkCheckPackCtx(context.Background(), baseDir, logFunc) -} - func (a *App) RunNvidiaPCIeBandwidthPackCtx(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) { if strings.TrimSpace(baseDir) == "" { baseDir = DefaultSATBaseDir @@ -289,22 +269,6 @@ func (a *App) RunNvidiaPCIeBandwidthPackCtx(ctx context.Context, baseDir string, return a.sat.RunNvidiaPCIeBandwidthPack(ctx, baseDir, gpuIndices, logFunc) } -func (a *App) RunNvidiaPCIeBandwidthPack(baseDir string, gpuIndices []int, logFunc func(string)) (string, error) { - return a.RunNvidiaPCIeBandwidthPackCtx(context.Background(), baseDir, gpuIndices, logFunc) -} - -func (a *App) DetectGPUVendor() string { - return a.sat.DetectGPUVendor() -} - -func (a *App) ListAMDGPUs() ([]platform.AMDGPUInfo, error) { - return a.sat.ListAMDGPUs() -} - -func (a *App) RunAMDAcceptancePack(baseDir string, logFunc func(string)) (string, error) { - return a.RunAMDAcceptancePackCtx(context.Background(), baseDir, logFunc) -} - func (a *App) RunAMDAcceptancePackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { if strings.TrimSpace(baseDir) == "" { baseDir = DefaultSATBaseDir @@ -326,18 +290,6 @@ func (a *App) RunAMDMemBandwidthPackCtx(ctx context.Context, baseDir string, log return a.sat.RunAMDMemBandwidthPack(ctx, baseDir, logFunc) } -func (a *App) RunMemoryStressPack(baseDir string, durationSec int, logFunc func(string)) (string, error) { - return a.RunMemoryStressPackCtx(context.Background(), baseDir, durationSec, logFunc) -} - -func (a *App) RunSATStressPack(baseDir string, durationSec int, logFunc func(string)) (string, error) { - return a.RunSATStressPackCtx(context.Background(), baseDir, durationSec, logFunc) -} - -func (a *App) RunAMDStressPack(baseDir string, durationSec int, logFunc func(string)) (string, error) { - return a.RunAMDStressPackCtx(context.Background(), baseDir, durationSec, logFunc) -} - func (a *App) RunMemoryStressPackCtx(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { return a.sat.RunMemoryStressPack(ctx, baseDir, durationSec, logFunc) } @@ -353,13 +305,6 @@ func (a *App) RunAMDStressPackCtx(ctx context.Context, baseDir string, durationS return a.sat.RunAMDStressPack(ctx, baseDir, durationSec, logFunc) } -func (a *App) RunFanStressTest(ctx context.Context, baseDir string, opts platform.FanStressOptions) (string, error) { - if strings.TrimSpace(baseDir) == "" { - baseDir = DefaultSATBaseDir - } - return a.sat.RunFanStressTest(ctx, baseDir, opts) -} - func (a *App) RunPlatformStress(ctx context.Context, baseDir string, opts platform.PlatformStressOptions, logFunc func(string)) (string, error) { if strings.TrimSpace(baseDir) == "" { baseDir = DefaultSATBaseDir diff --git a/audit/internal/app/app_services.go b/audit/internal/app/app_services.go index ecf9f3a..ad0004d 100644 --- a/audit/internal/app/app_services.go +++ b/audit/internal/app/app_services.go @@ -24,19 +24,11 @@ func (a *App) ServiceStatusResult(name string) (ActionResult, error) { return ActionResult{Title: "service status: " + name, Body: bodyOr(body, "No status output.")}, err } -func (a *App) ServiceDo(name string, action platform.ServiceAction) (string, error) { - return a.services.ServiceDo(name, action) -} - func (a *App) ServiceActionResult(name string, action platform.ServiceAction) (ActionResult, error) { body, err := a.services.ServiceDo(name, action) return ActionResult{Title: "service " + string(action) + ": " + name, Body: bodyOr(body, "Action completed.")}, err } -func (a *App) TailFile(path string, lines int) string { - return a.tools.TailFile(path, lines) -} - func (a *App) CheckTools(names []string) []platform.ToolStatus { return a.tools.CheckTools(names) } diff --git a/audit/internal/app/blackbox_archive.go b/audit/internal/app/blackbox_archive.go index b357c9c..247b11a 100644 --- a/audit/internal/app/blackbox_archive.go +++ b/audit/internal/app/blackbox_archive.go @@ -3,6 +3,7 @@ package app import ( "archive/zip" "bufio" + "errors" "io" "io/fs" "os" @@ -61,9 +62,8 @@ func buildZipArchive(root, destPath string) error { if err != nil { return err } - defer src.Close() - _, err = io.Copy(w, src) - return err + _, copyErr := io.Copy(w, src) + return errors.Join(copyErr, src.Close()) }) if walkErr != nil { _ = zw.Close() @@ -136,7 +136,7 @@ func isEOFLike(err error) bool { // (first run, external tampering, a previous crash mid-write), it falls back // to writing the whole archive rather than risk corrupting it with a wrong // truncate point. -func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) error { +func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) (retErr error) { newInfo, err := os.Stat(newLocalZipPath) if err != nil { return err @@ -159,7 +159,7 @@ func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) error if err != nil { return err } - defer target.Close() + defer func() { retErr = errors.Join(retErr, target.Close()) }() if err := target.Truncate(prefixLen); err != nil { return err @@ -172,7 +172,7 @@ func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) error if err != nil { return err } - defer src.Close() + defer func() { retErr = errors.Join(retErr, src.Close()) }() if _, err := src.Seek(prefixLen, io.SeekStart); err != nil { return err } diff --git a/audit/internal/app/component_status_db.go b/audit/internal/app/component_status_db.go index 0f35a86..b1c7b60 100644 --- a/audit/internal/app/component_status_db.go +++ b/audit/internal/app/component_status_db.go @@ -454,10 +454,19 @@ func satFailureDetailFromKV(kv map[string]string) string { continue } job := strings.TrimSuffix(k, "_status") + detail := strings.TrimSpace(kv[job+"_detail"]) if rc, ok := kv[job+"_rc"]; ok && strings.TrimSpace(rc) != "" { - failed = append(failed, fmt.Sprintf("%s=%s (rc=%s)", job, v, rc)) + entry := fmt.Sprintf("%s=%s (rc=%s)", job, v, rc) + if detail != "" { + entry += ": " + detail + } + failed = append(failed, entry) } else { - failed = append(failed, fmt.Sprintf("%s=%s", job, v)) + entry := fmt.Sprintf("%s=%s", job, v) + if detail != "" { + entry += ": " + detail + } + failed = append(failed, entry) } } if len(failed) == 0 { diff --git a/audit/internal/app/component_status_db_test.go b/audit/internal/app/component_status_db_test.go index 143a053..9934ae5 100644 --- a/audit/internal/app/component_status_db_test.go +++ b/audit/internal/app/component_status_db_test.go @@ -3,6 +3,7 @@ package app import ( "os" "path/filepath" + "strings" "testing" "bee/audit/internal/schema" @@ -145,6 +146,21 @@ func TestSATFailureDetailFallsBackToFailedSubJobs(t *testing.T) { } } +func TestSATFailureDetailIncludesValidatorDetail(t *testing.T) { + runDir := t.TempDir() + summary := "2-all-reduce-perf_rc=0\n" + + "2-all-reduce-perf_status=FAILED\n" + + "2-all-reduce-perf_detail=NVLink state does not match selected topology: GPU0<->GPU1\n" + + "overall_status=FAILED\n" + if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil { + t.Fatal(err) + } + got := SATFailureDetail(runDir) + if !strings.Contains(got, "NVLink state does not match selected topology") { + t.Fatalf("SATFailureDetail() = %q, want validator detail", got) + } +} + // TestSATFailureDetailEmptyWhenNoReasonFound guards the ultimate fallback: // when summary.txt carries no identifiable per-job or warnings detail (e.g. // unreadable or from an older binary version), callers must get "" so they @@ -233,6 +249,21 @@ func TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets(t *tes } } +func TestApplySATResultToDBNvidiaInterconnectFailureReachesGPUComponent(t *testing.T) { + db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json")) + if err != nil { + t.Fatal(err) + } + ApplySATResultToDB(db, "nvidia-interconnect", writeSATSummary(t, "FAILED")) + rec, ok := db.Get("pcie:gpu:nvidia") + if !ok { + t.Fatal("nvidia-interconnect failure wrote no pcie:gpu:nvidia record") + } + if rec.Status != "Warning" { + t.Fatalf("status=%q want Warning", rec.Status) + } +} + // TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the // hardware-ingest-contract.md rule that status_history is a transition log // ("История переходов статусов"), not a per-poll journal. A component diff --git a/audit/internal/app/support_bundle.go b/audit/internal/app/support_bundle.go index f802993..f7aac4b 100644 --- a/audit/internal/app/support_bundle.go +++ b/audit/internal/app/support_bundle.go @@ -6,6 +6,7 @@ import ( "compress/gzip" "context" _ "embed" + "errors" "fmt" "io" "os" @@ -505,21 +506,16 @@ func BuildSupportBundle(exportDir string) (string, error) { return "", err } - archiveName := SupportBundleBaseName(now) + ".tar.gz" - archivePath := filepath.Join(os.TempDir(), archiveName) - if err := createSupportTarGz(archivePath, stageRoot); err != nil { - return "", err - } - return archivePath, nil + return createSupportTarGz(os.TempDir(), SupportBundleBaseName(now), stageRoot) } func SupportBundleBaseName(at time.Time) string { at = at.UTC() date := at.Format("2006-01-02") tod := at.Format("150405") - ver := bundleVersion() - model := serverModelForBundle() - sn := serverSerialForBundle() + ver := sanitizeFilename(bundleVersion()) + model := sanitizeFilename(serverModelForBundle()) + sn := sanitizeFilename(serverSerialForBundle()) return fmt.Sprintf("%s (BEE-SP v%s) %s %s %s", date, ver, model, sn, tod) } @@ -617,17 +613,15 @@ func copyOptionalFile(src, dst string) error { if err != nil { return err } - defer in.Close() if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { - return err + return errors.Join(err, in.Close()) } out, err := os.Create(dst) if err != nil { - return err + return errors.Join(err, in.Close()) } - defer out.Close() - _, err = io.Copy(out, in) - return err + _, copyErr := io.Copy(out, in) + return errors.Join(copyErr, in.Close(), out.Close()) } func writeManifest(dst, exportDir, stageRoot string) error { @@ -823,16 +817,12 @@ func copyPath(src, dst string) error { } return err } - defer in.Close() - out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode().Perm()) if err != nil { - return err + return errors.Join(err, in.Close()) } - defer out.Close() - - _, err = io.Copy(out, in) - return err + _, copyErr := io.Copy(out, in) + return errors.Join(copyErr, in.Close(), out.Close()) } func copyPathFiltered(rootSrc, src, dst string, keep func(rel string, info os.FileInfo) bool) error { @@ -874,21 +864,35 @@ func copyPathFiltered(rootSrc, src, dst string, keep func(rel string, info os.Fi return copyPath(src, dst) } -func createSupportTarGz(dst, srcDir string) error { - file, err := os.Create(dst) +func createSupportTarGz(dir, baseName, srcDir string) (string, error) { + archiveFile, err := os.CreateTemp(dir, baseName+"-*.partial") if err != nil { - return err + return "", err } - defer file.Close() + partialPath := archiveFile.Name() + if err := writeSupportTarGz(archiveFile, srcDir); err != nil { + _ = archiveFile.Close() + _ = os.Remove(partialPath) + return "", err + } + if err := archiveFile.Close(); err != nil { + _ = os.Remove(partialPath) + return "", err + } + archivePath := strings.TrimSuffix(partialPath, ".partial") + ".tar.gz" + if err := os.Rename(partialPath, archivePath); err != nil { + _ = os.Remove(partialPath) + return "", err + } + return archivePath, nil +} +func writeSupportTarGz(file *os.File, srcDir string) error { gz := gzip.NewWriter(file) - defer gz.Close() - tw := tar.NewWriter(gz) - defer tw.Close() base := filepath.Dir(srcDir) - return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { + walkErr := filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -912,9 +916,21 @@ func createSupportTarGz(dst, srcDir string) error { if err != nil { return err } - defer f.Close() - - _, err = io.Copy(tw, f) - return err + _, copyErr := io.Copy(tw, f) + closeErr := f.Close() + if copyErr != nil { + return copyErr + } + return closeErr }) + if walkErr != nil { + _ = tw.Close() + _ = gz.Close() + return walkErr + } + if err := tw.Close(); err != nil { + _ = gz.Close() + return err + } + return gz.Close() } diff --git a/audit/internal/app/support_bundle_test.go b/audit/internal/app/support_bundle_test.go index 9d2773b..b184097 100644 --- a/audit/internal/app/support_bundle_test.go +++ b/audit/internal/app/support_bundle_test.go @@ -1,9 +1,14 @@ package app import ( + "archive/tar" + "compress/gzip" + "errors" + "io" "os" "path/filepath" "strings" + "sync" "testing" ) @@ -29,3 +34,95 @@ func TestWriteBundleDocs(t *testing.T) { t.Fatalf("README.md should explain how to check SAT pass/fail:\n%s", readme) } } + +func TestCreateSupportTarGzUsesIndependentFilesConcurrently(t *testing.T) { + tempDir := t.TempDir() + srcDir := filepath.Join(tempDir, "bee-support-stage-test") + if err := os.Mkdir(srcDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(srcDir, "evidence.txt"), []byte("complete evidence\n"), 0644); err != nil { + t.Fatal(err) + } + + const builds = 4 + paths := make(chan string, builds) + errs := make(chan error, builds) + var wg sync.WaitGroup + for range builds { + wg.Add(1) + go func() { + defer wg.Done() + path, err := createSupportTarGz(tempDir, "bundle", srcDir) + if err != nil { + errs <- err + return + } + paths <- path + }() + } + wg.Wait() + close(paths) + close(errs) + for err := range errs { + t.Fatalf("create archive: %v", err) + } + + seen := make(map[string]struct{}, builds) + for path := range paths { + if !strings.HasSuffix(path, ".tar.gz") { + t.Fatalf("archive path %q does not end in .tar.gz", path) + } + if _, exists := seen[path]; exists { + t.Fatalf("duplicate archive path %q", path) + } + seen[path] = struct{}{} + assertSupportArchiveEntry(t, path, "bee-support-stage-test/evidence.txt", "complete evidence\n") + } + if len(seen) != builds { + t.Fatalf("archive count = %d, want %d", len(seen), builds) + } + partials, err := filepath.Glob(filepath.Join(tempDir, "*.partial")) + if err != nil { + t.Fatal(err) + } + if len(partials) != 0 { + t.Fatalf("unpublished partial archives remain: %v", partials) + } +} + +func assertSupportArchiveEntry(t *testing.T, path, wantName, wantBody string) { + t.Helper() + file, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + gz, err := gzip.NewReader(file) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + tr := tar.NewReader(gz) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatal(err) + } + if header.Name != wantName { + continue + } + body, err := io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + if string(body) != wantBody { + t.Fatalf("entry body = %q, want %q", body, wantBody) + } + return + } + t.Fatalf("archive %q is missing %q", path, wantName) +} diff --git a/audit/internal/platform/benchmark.go b/audit/internal/platform/benchmark.go index 01be3cd..736851f 100644 --- a/audit/internal/platform/benchmark.go +++ b/audit/internal/platform/benchmark.go @@ -453,18 +453,7 @@ func (s *System) RunNvidiaBenchmark(ctx context.Context, baseDir string, opts Nv Status: "FAILED", } if info, ok := infoByIndex[idx]; ok { - gpuResult.UUID = info.UUID - gpuResult.Name = info.Name - gpuResult.BusID = info.BusID - gpuResult.VBIOS = info.VBIOS - gpuResult.PowerLimitW = info.PowerLimitW - gpuResult.MultiprocessorCount = info.MultiprocessorCount - gpuResult.DefaultPowerLimitW = info.DefaultPowerLimitW - gpuResult.ShutdownTempC = info.ShutdownTempC - gpuResult.SlowdownTempC = info.SlowdownTempC - gpuResult.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz - gpuResult.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz - gpuResult.MaxMemoryClockMHz = info.MaxMemoryClockMHz + populateBenchmarkGPUInfo(&gpuResult, info) } if calib, ok := calibByIndex[idx]; ok { gpuResult.CalibratedPeakPowerW = calib.Summary.P95PowerW diff --git a/audit/internal/platform/benchmark_analysis.go b/audit/internal/platform/benchmark_analysis.go index d1d14e0..edca764 100644 --- a/audit/internal/platform/benchmark_analysis.go +++ b/audit/internal/platform/benchmark_analysis.go @@ -525,13 +525,6 @@ func scoreBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkScorecard { return score } -// compositeBenchmarkScore is kept for compatibility with legacy callers. -// CompositeScore = ComputeScore (no quality multiplier; throttling already -// reduces TOPS directly, so no additional penalty is needed). -func compositeBenchmarkScore(score BenchmarkScorecard) float64 { - return score.ComputeScore -} - // serverQualityScore returns a 0–100 score reflecting server infrastructure // quality, independent of GPU model or compute speed. // diff --git a/audit/internal/platform/benchmark_power_autotune.go b/audit/internal/platform/benchmark_power_autotune.go index 247aacd..17b8929 100644 --- a/audit/internal/platform/benchmark_power_autotune.go +++ b/audit/internal/platform/benchmark_power_autotune.go @@ -6,7 +6,6 @@ import ( "fmt" "math" "os" - "os/exec" "path/filepath" "sort" "strings" @@ -721,5 +720,3 @@ func minInt(a, b int) int { } return b } - -var _ = exec.ErrNotFound diff --git a/audit/internal/platform/benchmark_power_calibration.go b/audit/internal/platform/benchmark_power_calibration.go index 8a72501..163d870 100644 --- a/audit/internal/platform/benchmark_power_calibration.go +++ b/audit/internal/platform/benchmark_power_calibration.go @@ -406,39 +406,6 @@ func queryIPMIServerPowerW() (float64, error) { return 0, fmt.Errorf("could not parse ipmitool dcmi power reading output") } -// sampleIPMIPowerSeries collects IPMI power readings every 2 seconds for -// durationSec seconds. Returns the mean of all successful samples. -// Returns 0, false if IPMI is unavailable. -func sampleIPMIPowerSeries(ctx context.Context, durationSec int) (meanW float64, ok bool) { - if durationSec <= 0 { - return 0, false - } - deadline := time.Now().Add(time.Duration(durationSec) * time.Second) - var samples []float64 -loop: - for { - if w, err := queryIPMIServerPowerW(); err == nil { - samples = append(samples, w) - } - if time.Now().After(deadline) { - break - } - select { - case <-ctx.Done(): - break loop - case <-time.After(2 * time.Second): - } - } - if len(samples) == 0 { - return 0, false - } - var sum float64 - for _, w := range samples { - sum += w - } - return sum / float64(len(samples)), true -} - // characterizeServerPower computes BenchmarkServerPower from idle and loaded // samples plus the GPU-reported average power during steady state. func characterizeServerPower(idleW, loadedW, gpuReportedSumW float64, source string, available bool) *BenchmarkServerPower { @@ -535,18 +502,7 @@ func runNvidiaBenchmarkParallel( for _, idx := range selected { r := &BenchmarkGPUResult{Index: idx, Status: "FAILED"} if info, ok := infoByIndex[idx]; ok { - r.UUID = info.UUID - r.Name = info.Name - r.BusID = info.BusID - r.VBIOS = info.VBIOS - r.PowerLimitW = info.PowerLimitW - r.MultiprocessorCount = info.MultiprocessorCount - r.DefaultPowerLimitW = info.DefaultPowerLimitW - r.ShutdownTempC = info.ShutdownTempC - r.SlowdownTempC = info.SlowdownTempC - r.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz - r.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz - r.MaxMemoryClockMHz = info.MaxMemoryClockMHz + populateBenchmarkGPUInfo(r, info) } if calib, ok := calibByIndex[idx]; ok { r.CalibratedPeakPowerW = calib.Summary.P95PowerW @@ -753,6 +709,21 @@ func runNvidiaBenchmarkParallel( } } +func populateBenchmarkGPUInfo(result *BenchmarkGPUResult, info benchmarkGPUInfo) { + result.UUID = info.UUID + result.Name = info.Name + result.BusID = info.BusID + result.VBIOS = info.VBIOS + result.PowerLimitW = info.PowerLimitW + result.MultiprocessorCount = info.MultiprocessorCount + result.DefaultPowerLimitW = info.DefaultPowerLimitW + result.ShutdownTempC = info.ShutdownTempC + result.SlowdownTempC = info.SlowdownTempC + result.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz + result.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz + result.MaxMemoryClockMHz = info.MaxMemoryClockMHz +} + // readBenchmarkHostConfig reads static CPU and memory configuration from // /proc/cpuinfo and /proc/meminfo. Returns nil if neither source is readable. func readBenchmarkHostConfig() *BenchmarkHostConfig { diff --git a/audit/internal/platform/install_to_ram_test.go b/audit/internal/platform/install_to_ram_test.go index 73cf043..acd2793 100644 --- a/audit/internal/platform/install_to_ram_test.go +++ b/audit/internal/platform/install_to_ram_test.go @@ -129,8 +129,6 @@ func TestShouldLogCopyProgress(t *testing.T) { } func TestTryRemountLiveMedium(t *testing.T) { - t.Parallel() - orig := runRemountMedium t.Cleanup(func() { runRemountMedium = orig @@ -165,8 +163,6 @@ func TestTryRemountLiveMedium(t *testing.T) { } func TestEnsureLiveMediumAvailableRemountsSource(t *testing.T) { - t.Parallel() - origGlob := liveMediumSquashfsGlob origRemount := runRemountMedium t.Cleanup(func() { @@ -210,8 +206,6 @@ func TestEnsureLiveMediumAvailableRemountsSource(t *testing.T) { } func TestDetachInstallMedium(t *testing.T) { - t.Parallel() - origUmount := umountLiveMedium origEject := ejectDevice t.Cleanup(func() { diff --git a/audit/internal/platform/live_metrics.go b/audit/internal/platform/live_metrics.go index 5381a0f..0fe5e6a 100644 --- a/audit/internal/platform/live_metrics.go +++ b/audit/internal/platform/live_metrics.go @@ -293,6 +293,10 @@ func sampleLiveTempsViaIPMI() []TempReading { } func firstTempInputValue(feature map[string]any) (float64, bool) { + return firstSensorInputValue(feature, "temp") +} + +func firstSensorInputValue(feature map[string]any, kind string) (float64, bool) { keys := make([]string, 0, len(feature)) for key := range feature { keys = append(keys, key) @@ -300,7 +304,7 @@ func firstTempInputValue(feature map[string]any) (float64, bool) { sort.Strings(keys) for _, key := range keys { lower := strings.ToLower(key) - if !strings.Contains(lower, "temp") || !strings.HasSuffix(lower, "_input") { + if !strings.Contains(lower, kind) || !strings.HasSuffix(lower, "_input") { continue } switch value := feature[key].(type) { diff --git a/audit/internal/platform/nvidia_config_check.go b/audit/internal/platform/nvidia_config_check.go index d2cfc80..13ac28a 100644 --- a/audit/internal/platform/nvidia_config_check.go +++ b/audit/internal/platform/nvidia_config_check.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strconv" "strings" "time" @@ -71,9 +72,9 @@ func (s *System) RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, l errOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-e", []string{"nvidia-smi", "nvlink", "-e"}, nil, logFunc) _ = os.WriteFile(filepath.Join(runDir, "03-nvidia-smi-nvlink-e.log"), errOut, 0644) - pairs := parseNvidiaNVLinkBondedPairs(string(topoOut)) + pairs := ParseNvidiaNVLinkBondedPairs(string(topoOut)) linkStatus := parseNvidiaNVLinkStatus(string(statusOut)) - linkErrors := parseNvidiaNVLinkErrors(string(errOut)) + linkErrors := ParseNvidiaNVLinkErrors(string(errOut)) for _, pair := range pairs { f := evaluateNvidiaNVLinkPair(pair, linkStatus, linkErrors) status.NVLinkPairs = append(status.NVLinkPairs, f) @@ -96,6 +97,11 @@ func (s *System) RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, l dmesgOut, _ := runSATCommandCtx(ctx, verboseLog, "dmesg", []string{"dmesg"}, nil, nil) ccDmesgLines := filterConfComputeDmesgLines(dmesgOut) _ = os.WriteFile(filepath.Join(runDir, "05-dmesg-cc-relevant.log"), []byte(strings.Join(ccDmesgLines, "\n")+"\n"), 0644) + nvlinkDegraded := parseNvidiaNVLinkDegradedDmesg(dmesgOut) + _ = os.WriteFile(filepath.Join(runDir, "06-dmesg-nvlink-degraded.log"), []byte(renderNvidiaNVLinkDegradedLog(nvlinkDegraded)), 0644) + for _, event := range nvlinkDegraded { + status.Warnings = append(status.Warnings, event.Warning()) + } lowerDmesg := strings.ToLower(strings.Join(ccDmesgLines, "\n")) status.HostAMDSEVSNPActive = strings.Contains(lowerDmesg, "sev-snp enabled") status.HostIntelTDXActive = strings.Contains(lowerDmesg, "tdx module") && strings.Contains(lowerDmesg, "module initialized") || @@ -166,7 +172,11 @@ type NvidiaNVLinkPairFinding struct { GPUA int `json:"gpu_a"` GPUB int `json:"gpu_b"` ExpectedLinks int `json:"expected_links"` // from "nvidia-smi topo -m"'s NVx cell - ActiveLinks int `json:"active_links"` // from "nvidia-smi nvlink -s" + ActiveLinksA int `json:"active_links_a"` // per endpoint from "nvidia-smi nvlink -s" + ActiveLinksB int `json:"active_links_b"` + TotalLinksA int `json:"total_links_a"` + TotalLinksB int `json:"total_links_b"` + ActiveLinks int `json:"active_links"` // aggregate retained for report compatibility TotalLinks int `json:"total_links"` ErrorLinks int `json:"error_links"` // links with a nonzero replay/recovery/CRC counter Issues []string `json:"issues,omitempty"` @@ -197,28 +207,57 @@ func evaluateNvidiaGPUConfig(g NvidiaGPUSetting) []string { // evaluateNvidiaNVLinkPair compares a bonded pair's actual link state // against what the topology matrix says should be there. -func evaluateNvidiaNVLinkPair(pair nvidiaNVLinkBondedPair, status map[int][]nvidiaNVLinkPort, errors map[int]map[int][3]int64) NvidiaNVLinkPairFinding { - f := NvidiaNVLinkPairFinding{GPUA: pair.gpuA, GPUB: pair.gpuB, ExpectedLinks: pair.links} +func evaluateNvidiaNVLinkPair(pair NvidiaNVLinkBondedPair, status map[int][]nvidiaNVLinkPort, errors map[int]map[int][3]int64) NvidiaNVLinkPairFinding { + f := NvidiaNVLinkPairFinding{GPUA: pair.GPUA, GPUB: pair.GPUB, ExpectedLinks: pair.NVLinks} - active, total, errLinks := 0, 0, 0 - for _, gpu := range []int{pair.gpuA, pair.gpuB} { - for _, port := range status[gpu] { + countPorts := func(gpu int) (active, total int, present bool) { + ports, present := status[gpu] + for _, port := range ports { total++ if port.active { active++ } } + return active, total, present + } + f.ActiveLinksA, f.TotalLinksA, _ = countPorts(pair.GPUA) + f.ActiveLinksB, f.TotalLinksB, _ = countPorts(pair.GPUB) + f.ActiveLinks = f.ActiveLinksA + f.ActiveLinksB + f.TotalLinks = f.TotalLinksA + f.TotalLinksB + + _, statusASeen := status[pair.GPUA] + _, statusBSeen := status[pair.GPUB] + if !statusASeen || !statusBSeen { + var missing []string + if !statusASeen { + missing = append(missing, fmt.Sprintf("GPU%d", pair.GPUA)) + } + if !statusBSeen { + missing = append(missing, fmt.Sprintf("GPU%d", pair.GPUB)) + } + f.Issues = append(f.Issues, "NVLink status unavailable for "+strings.Join(missing, ", ")) + } + if pair.NVLinks > 0 && (f.ActiveLinksA < pair.NVLinks || f.ActiveLinksB < pair.NVLinks) { + f.Issues = append(f.Issues, fmt.Sprintf( + "GPU%d %d/%d, GPU%d %d/%d active links (topo NV%d)", + pair.GPUA, f.ActiveLinksA, pair.NVLinks, + pair.GPUB, f.ActiveLinksB, pair.NVLinks, + pair.NVLinks, + )) + } + if f.TotalLinks > 0 && f.ActiveLinks < f.TotalLinks { + f.Issues = append(f.Issues, fmt.Sprintf("%d/%d reported NVLinks inactive on a bonded pair", f.TotalLinks-f.ActiveLinks, f.TotalLinks)) + } + + errLinks := 0 + for _, gpu := range []int{pair.GPUA, pair.GPUB} { for _, counters := range errors[gpu] { if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 { errLinks++ } } } - f.ActiveLinks, f.TotalLinks, f.ErrorLinks = active, total, errLinks - - if total > 0 && active < total { - f.Issues = append(f.Issues, fmt.Sprintf("%d/%d NVLinks inactive on a bonded pair", total-active, total)) - } + f.ErrorLinks = errLinks if errLinks > 0 { f.Issues = append(f.Issues, fmt.Sprintf("%d link(s) reporting replay/recovery/CRC errors", errLinks)) } @@ -226,17 +265,14 @@ func evaluateNvidiaNVLinkPair(pair nvidiaNVLinkBondedPair, status map[int][]nvid } // --------------------------------------------------------------------------- -// NVLink parsing — deliberately self-contained rather than importing -// internal/collector or internal/webui's equivalent parsers (isNVLinkBridgeCandidate -// et al. and parseGPUPairAdjacency/parseTopoNVLinkStatus/parseTopoNVLinkErrors, -// respectively), matching this codebase's existing precedent of small, -// package-local duplication over cross-package coupling for narrow parsing -// helpers (see webui/page_topo.go's isNICDeviceClassDev/isRAIDControllerClass). +// NVLink parsing shared by diagnostics and the read-only topology page. // --------------------------------------------------------------------------- -type nvidiaNVLinkBondedPair struct { - gpuA, gpuB int - links int +// NvidiaNVLinkBondedPair identifies two GPU indices and their negotiated +// NVLink bond width as reported by nvidia-smi topo -m. +type NvidiaNVLinkBondedPair struct { + GPUA, GPUB int + NVLinks int } var ( @@ -244,37 +280,31 @@ var ( nvidiaNVLinkNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`) ) -// parseNvidiaNVLinkBondedPairs returns every GPU pair with a nonzero NVLink +// ParseNvidiaNVLinkBondedPairs returns every GPU pair with a nonzero NVLink // bond count from a "nvidia-smi topo -m" matrix, deduplicated (A,B) == (B,A). -func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair { +func ParseNvidiaNVLinkBondedPairs(raw string) []NvidiaNVLinkBondedPair { lines := strings.Split(nvidiaNVLinkANSIRe.ReplaceAllString(raw, ""), "\n") headerIdx := -1 - var gpuColIndices []int + gpuColumns := map[int]int{} for i, line := range lines { - trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "GPU0") { - parts := strings.Fields(trimmed) - for j, col := range parts { - if strings.HasPrefix(col, "GPU") { - gpuColIndices = append(gpuColIndices, j) - } - } - if len(gpuColIndices) >= 2 { - headerIdx = i + columns := map[int]int{} + for columnIndex, label := range strings.Fields(strings.TrimSpace(line)) { + gpuIndex, err := strconv.Atoi(strings.TrimPrefix(label, "GPU")) + if err == nil && strings.HasPrefix(label, "GPU") { + columns[columnIndex] = gpuIndex } + } + if len(columns) >= 2 { + headerIdx = i + gpuColumns = columns break } } if headerIdx < 0 { return nil } - colIdxToGPU := make(map[int]int, len(gpuColIndices)) - for gpuIdx, colIdx := range gpuColIndices { - colIdxToGPU[colIdx] = gpuIdx - } - seen := map[[2]int]bool{} - var pairs []nvidiaNVLinkBondedPair + var pairs []NvidiaNVLinkBondedPair for _, line := range lines[headerIdx+1:] { trimmed := strings.TrimSpace(line) if !strings.HasPrefix(trimmed, "GPU") { @@ -288,7 +318,7 @@ func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair { if err != nil { continue } - for colIdx, colGPU := range colIdxToGPU { + for colIdx, colGPU := range gpuColumns { if colGPU == rowGPU { continue } @@ -313,9 +343,15 @@ func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair { continue } seen[key] = true - pairs = append(pairs, nvidiaNVLinkBondedPair{gpuA: a, gpuB: b, links: nv}) + pairs = append(pairs, NvidiaNVLinkBondedPair{GPUA: a, GPUB: b, NVLinks: nv}) } } + sort.Slice(pairs, func(i, j int) bool { + if pairs[i].GPUA != pairs[j].GPUA { + return pairs[i].GPUA < pairs[j].GPUA + } + return pairs[i].GPUB < pairs[j].GPUB + }) return pairs } @@ -339,6 +375,11 @@ func parseNvidiaNVLinkStatus(raw string) map[int][]nvidiaNVLinkPort { trimmed := strings.TrimSpace(line) if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { currentGPU, _ = strconv.Atoi(m[1]) + if _, exists := result[currentGPU]; !exists { + // Preserve the distinction between a present GPU with no active + // links and a GPU whose block is absent from the command output. + result[currentGPU] = nil + } continue } if currentGPU < 0 { @@ -355,9 +396,67 @@ func parseNvidiaNVLinkStatus(raw string) map[int][]nvidiaNVLinkPort { return result } -// parseNvidiaNVLinkErrors parses "nvidia-smi nvlink -e" output into, per GPU +type nvidiaNVLinkDegradedEvent struct { + GPUIndex int + LinkID int + RawLine string +} + +func (e nvidiaNVLinkDegradedEvent) Warning() string { + switch { + case e.GPUIndex >= 0 && e.LinkID >= 0: + return fmt.Sprintf("GPU%d NVLink degraded mode (linkId %d): %s", e.GPUIndex, e.LinkID, e.RawLine) + case e.GPUIndex >= 0: + return fmt.Sprintf("GPU%d NVLink degraded mode: %s", e.GPUIndex, e.RawLine) + default: + return "NVIDIA NVLink degraded mode: " + e.RawLine + } +} + +var ( + nvidiaNVLinkDegradedGPURe = regexp.MustCompile(`(?i)\bGPU\s*(\d+)\b`) + nvidiaNVLinkDegradedLinkRe = regexp.MustCompile(`(?i)\blinkId\s*[:=]?\s*(\d+)\b`) +) + +func parseNvidiaNVLinkDegradedDmesg(raw []byte) []nvidiaNVLinkDegradedEvent { + var events []nvidiaNVLinkDegradedEvent + for _, lineBytes := range bytes.Split(raw, []byte("\n")) { + line := strings.TrimSpace(string(lineBytes)) + lower := strings.ToLower(line) + explicitNVLinkMarker := strings.Contains(lower, "knvlinksetdegradedmode") || + strings.Contains(lower, "nvlink_is_gpu_degraded") || + strings.Contains(lower, "error originated on linkid") + contextualDegradedMarker := strings.Contains(lower, "marked degraded") && + (strings.Contains(lower, "nvlink") || strings.Contains(lower, "nvrm") || nvidiaNVLinkDegradedGPURe.MatchString(line)) + if line == "" || (!explicitNVLinkMarker && !contextualDegradedMarker) { + continue + } + event := nvidiaNVLinkDegradedEvent{GPUIndex: -1, LinkID: -1, RawLine: line} + if match := nvidiaNVLinkDegradedGPURe.FindStringSubmatch(line); len(match) == 2 { + event.GPUIndex, _ = strconv.Atoi(match[1]) + } + if match := nvidiaNVLinkDegradedLinkRe.FindStringSubmatch(line); len(match) == 2 { + event.LinkID, _ = strconv.Atoi(match[1]) + } + events = append(events, event) + } + return events +} + +func renderNvidiaNVLinkDegradedLog(events []nvidiaNVLinkDegradedEvent) string { + if len(events) == 0 { + return "" + } + lines := make([]string, len(events)) + for i, event := range events { + lines[i] = event.RawLine + } + return strings.Join(lines, "\n") + "\n" +} + +// ParseNvidiaNVLinkErrors parses "nvidia-smi nvlink -e" output into, per GPU // then link index, [replay, recovery, crc] error counts. -func parseNvidiaNVLinkErrors(raw string) map[int]map[int][3]int64 { +func ParseNvidiaNVLinkErrors(raw string) map[int]map[int][3]int64 { result := map[int]map[int][3]int64{} currentGPU := -1 for _, line := range strings.Split(raw, "\n") { @@ -516,8 +615,8 @@ func renderNvidiaConfigCheckReport(status NvidiaConfigCheckStatus) string { if len(p.Issues) > 0 { verdict = "ISSUES FOUND" } - fmt.Fprintf(&b, " GPU%d <-> GPU%d: %d/%d links active (topology expects %d): %s\n", - p.GPUA, p.GPUB, p.ActiveLinks, p.TotalLinks, p.ExpectedLinks, verdict) + fmt.Fprintf(&b, " GPU%d <-> GPU%d: endpoints %d/%d and %d/%d active (topology expects %d each): %s\n", + p.GPUA, p.GPUB, p.ActiveLinksA, p.TotalLinksA, p.ActiveLinksB, p.TotalLinksB, p.ExpectedLinks, verdict) for _, issue := range p.Issues { fmt.Fprintf(&b, " - %s\n", issue) } diff --git a/audit/internal/platform/nvidia_config_check_test.go b/audit/internal/platform/nvidia_config_check_test.go index c8cab16..c963c59 100644 --- a/audit/internal/platform/nvidia_config_check_test.go +++ b/audit/internal/platform/nvidia_config_check_test.go @@ -14,11 +14,11 @@ func TestParseNvidiaNVLinkBondedPairsRealTwoGPUDump(t *testing.T) { "NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" + "NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n" - pairs := parseNvidiaNVLinkBondedPairs(input) + pairs := ParseNvidiaNVLinkBondedPairs(input) if len(pairs) != 1 { t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs) } - if pairs[0].gpuA != 0 || pairs[0].gpuB != 1 || pairs[0].links != 17 { + if pairs[0].GPUA != 0 || pairs[0].GPUB != 1 || pairs[0].NVLinks != 17 { t.Fatalf("pair=%#v want {0,1,17}", pairs[0]) } } @@ -34,18 +34,32 @@ func TestParseNvidiaNVLinkBondedPairsANSIUnderlinedHeader(t *testing.T) { "GPU2\tPIX\tPIX\t X \tNV18\t0-31,64-95\n" + "GPU3\tPIX\tPIX\tNV18\t X \t0-31,64-95\n" - pairs := parseNvidiaNVLinkBondedPairs(input) + pairs := ParseNvidiaNVLinkBondedPairs(input) if len(pairs) != 2 { t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs) } want := map[[2]int]int{{0, 1}: 18, {2, 3}: 18} for _, p := range pairs { - if want[[2]int{p.gpuA, p.gpuB}] != p.links { + if want[[2]int{p.GPUA, p.GPUB}] != p.NVLinks { t.Fatalf("unexpected pair %#v", p) } } } +func TestParseNvidiaNVLinkBondedPairsUsesGPUIndicesFromHeader(t *testing.T) { + input := "\tGPU2\tGPU4\tCPU Affinity\n" + + "GPU2\t X \tNV4\t0-15\n" + + "GPU4\tNV4\t X \t16-31\n" + + pairs := ParseNvidiaNVLinkBondedPairs(input) + if len(pairs) != 1 { + t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs) + } + if pairs[0] != (NvidiaNVLinkBondedPair{GPUA: 2, GPUB: 4, NVLinks: 4}) { + t.Fatalf("pair=%#v want GPU2<->GPU4 NV4", pairs[0]) + } +} + func TestParseNvidiaNVLinkStatusMarksInactiveLinks(t *testing.T) { input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140) Link 0: 26.562 GB/s @@ -71,7 +85,7 @@ func TestParseNvidiaNVLinkErrors(t *testing.T) { Link 1: Recovery Errors: 1 Link 1: CRC Errors: 2 ` - got := parseNvidiaNVLinkErrors(input) + got := ParseNvidiaNVLinkErrors(input) c := got[0][1] if c[0] != 3 || c[1] != 1 || c[2] != 2 { t.Fatalf("link1 counters=%#v want {3,1,2}", c) @@ -79,7 +93,7 @@ func TestParseNvidiaNVLinkErrors(t *testing.T) { } func TestEvaluateNvidiaNVLinkPairFlagsInactiveLinksAndErrors(t *testing.T) { - pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18} + pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 2} status := map[int][]nvidiaNVLinkPort{ 0: {{active: true}, {active: false}}, 1: {{active: true}, {active: true}}, @@ -92,16 +106,16 @@ func TestEvaluateNvidiaNVLinkPairFlagsInactiveLinksAndErrors(t *testing.T) { if f.ActiveLinks != 3 || f.TotalLinks != 4 { t.Fatalf("active=%d total=%d want 3/4", f.ActiveLinks, f.TotalLinks) } - if len(f.Issues) != 2 { - t.Fatalf("issues=%#v want 2 (inactive link + error)", f.Issues) + if len(f.Issues) != 3 { + t.Fatalf("issues=%#v want 3 (expected-count mismatch + inactive link + error)", f.Issues) } } func TestEvaluateNvidiaNVLinkPairHealthyBondHasNoIssues(t *testing.T) { - pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18} + pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 18} status := map[int][]nvidiaNVLinkPort{ - 0: {{active: true}}, - 1: {{active: true}}, + 0: activeNvidiaNVLinkPorts(18), + 1: activeNvidiaNVLinkPorts(18), } f := evaluateNvidiaNVLinkPair(pair, status, nil) if len(f.Issues) != 0 { @@ -109,6 +123,76 @@ func TestEvaluateNvidiaNVLinkPairHealthyBondHasNoIssues(t *testing.T) { } } +func TestEvaluateNvidiaNVLinkPairChecksEachEndpoint(t *testing.T) { + pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 17} + status := map[int][]nvidiaNVLinkPort{ + 0: activeNvidiaNVLinkPorts(17), + 1: {}, + } + f := evaluateNvidiaNVLinkPair(pair, status, nil) + if f.ActiveLinksA != 17 || f.ActiveLinksB != 0 { + t.Fatalf("endpoint counts=%d,%d want 17,0", f.ActiveLinksA, f.ActiveLinksB) + } + if len(f.Issues) == 0 || !strings.Contains(strings.Join(f.Issues, " "), "GPU1 0/17") { + t.Fatalf("issues=%#v want endpoint-specific 0/17 failure", f.Issues) + } +} + +func TestParseNvidiaNVLinkStatusPreservesAllInactiveGPUBlocks(t *testing.T) { + raw := `GPU 0: NVIDIA H100 +NVML: Unable to retrieve NVLink information as all links are inActive +GPU 1: NVIDIA H100 +all links are inActive +` + status := parseNvidiaNVLinkStatus(raw) + if _, ok := status[0]; !ok { + t.Fatal("GPU0 header was lost") + } + if _, ok := status[1]; !ok { + t.Fatal("GPU1 header was lost") + } + f := evaluateNvidiaNVLinkPair(NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 17}, status, nil) + if len(f.Issues) == 0 || f.ActiveLinksA != 0 || f.ActiveLinksB != 0 { + t.Fatalf("finding=%#v want failed 0/17 endpoints", f) + } +} + +func TestEvaluateNvidiaNVLinkPairFlagsMissingGPUBlock(t *testing.T) { + f := evaluateNvidiaNVLinkPair( + NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 2}, + map[int][]nvidiaNVLinkPort{0: activeNvidiaNVLinkPorts(2)}, nil, + ) + if !strings.Contains(strings.Join(f.Issues, " "), "status unavailable for GPU1") { + t.Fatalf("issues=%#v want missing GPU1 status", f.Issues) + } +} + +func TestParseNvidiaNVLinkDegradedDmesg(t *testing.T) { + raw := []byte("[ 11.0] md: array md0 marked Degraded\n[ 12.0] NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n") + events := parseNvidiaNVLinkDegradedDmesg(raw) + if len(events) != 1 { + t.Fatalf("events=%#v want one", events) + } + if events[0].GPUIndex != 1 || events[0].LinkID != 1 { + t.Fatalf("event=%#v want GPU1 linkId 1", events[0]) + } + if warning := events[0].Warning(); !strings.Contains(warning, "GPU1 NVLink degraded mode (linkId 1)") { + t.Fatalf("warning=%q", warning) + } + wantLog := "[ 12.0] NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n" + if log := renderNvidiaNVLinkDegradedLog(events); log != wantLog { + t.Fatalf("log=%q want only NVLink line %q", log, wantLog) + } +} + +func activeNvidiaNVLinkPorts(count int) []nvidiaNVLinkPort { + ports := make([]nvidiaNVLinkPort, count) + for i := range ports { + ports[i].active = true + } + return ports +} + func TestEvaluateNvidiaGPUConfigFlagsECCDisabled(t *testing.T) { g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Disabled"} issues := evaluateNvidiaGPUConfig(g) diff --git a/audit/internal/platform/sat.go b/audit/internal/platform/sat.go index da07c3a..d218b6a 100644 --- a/audit/internal/platform/sat.go +++ b/audit/internal/platform/sat.go @@ -191,13 +191,19 @@ func streamExecOutput(cmd *exec.Cmd, logFunc func(string), livePath string) ([]b return buf.Bytes(), waitErr } -// NvidiaGPU holds basic GPU info from nvidia-smi. +// satJob describes one command and the checks needed to turn its process and +// output results into a SAT verdict. type satJob struct { - name string - cmd []string - env []string // extra env vars (appended to os.Environ) - collectGPU bool // collect GPU metrics via nvidia-smi while this job runs - gpuIndices []int // GPU indices to collect metrics for (empty = all) + name string + cmd []string + env []string // extra env vars (appended to os.Environ) + // validate checks successful command output against the tool's documented + // result format. It may inspect artifacts from earlier jobs in runDir. + // FAILED means the tool proved a test failure; UNSUPPORTED means the pinned + // output contract could not be recognized safely. + validate func(runDir string, out []byte) (status, detail string) + collectGPU bool // collect GPU metrics via nvidia-smi while this job runs + gpuIndices []int // GPU indices to collect metrics for (empty = all) // informational marks a preflight/metadata job (e.g. dcgmi discovery) whose // failure shouldn't flip the pack's overall status — the diagnostic jobs // that follow it are the actual test of GPU health. @@ -314,7 +320,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa var summary strings.Builder stats := satStats{} - nvidiaPack := strings.HasPrefix(prefix, "gpu-nvidia") + nvidiaPack := isNvidiaAcceptancePack(prefix) perGPU := map[int]*nvidiaGPUStatusFile{} selectedGPUIndices := map[int]struct{}{} fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339)) @@ -338,6 +344,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa var out []byte var err error + jobDetail := "" if nvidiaPack && nvidiaJobNeedsHealthCheck(job) { if msg, healthErr := checkNvidiaJobHealth(job.gpuIndices); healthErr != nil { @@ -346,6 +353,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa } out = []byte(msg + "\n") err = healthErr + jobDetail = msg } } @@ -379,6 +387,23 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa if err == nil { err = healthErr } + jobDetail = msg + } + } + + status, rc := classifySATResult(job.name, out, err) + validationDetail := singleLineSATDetail(jobDetail) + validated := false + if status == "OK" && job.validate != nil { + status, validationDetail = validateSATJobOutput(job, runDir, out) + validated = true + } + if validated { + if validationDetail != "" { + if len(out) > 0 && !bytes.HasSuffix(out, []byte("\n")) { + out = append(out, '\n') + } + out = append(out, []byte(fmt.Sprintf("[bee-validator] %s: %s\n", status, validationDetail))...) } } @@ -392,7 +417,6 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa if ctx.Err() != nil { return "", ctx.Err() } - status, rc := classifySATResult(job.name, out, err) if job.informational && status != "OK" { stats.Informational++ } else { @@ -406,6 +430,9 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa key := strings.TrimSuffix(strings.TrimPrefix(job.name, "0"), ".log") fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc) fmt.Fprintf(&summary, "%s_status=%s\n", key, status) + if validationDetail != "" { + fmt.Fprintf(&summary, "%s_detail=%s\n", key, singleLineSATDetail(validationDetail)) + } } writeSATStats(&summary, stats) if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil { @@ -420,6 +447,25 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa return runDir, nil } +func isNvidiaAcceptancePack(prefix string) bool { + return strings.HasPrefix(prefix, "gpu-nvidia") || prefix == "nccl-tests" +} + +func validateSATJobOutput(job satJob, runDir string, out []byte) (string, string) { + status, detail := job.validate(runDir, out) + status = strings.ToUpper(strings.TrimSpace(status)) + switch status { + case "OK", "FAILED", "UNSUPPORTED", "PARTIAL": + return status, strings.TrimSpace(detail) + default: + return "UNSUPPORTED", fmt.Sprintf("validator returned invalid status %q", status) + } +} + +func singleLineSATDetail(detail string) string { + return strings.Join(strings.Fields(detail), " ") +} + func updateNvidiaGPUStatus(perGPU map[int]*nvidiaGPUStatusFile, idx int, status, jobName, detail string) { entry := perGPU[idx] if entry == nil { @@ -505,112 +551,6 @@ func writeNvidiaGPUStatusFiles(runDir, overall string, perGPU map[int]*nvidiaGPU return nil } -func nvidiaSATStatusSeverity(status string) int { - switch strings.ToUpper(strings.TrimSpace(status)) { - case "FAILED": - return 3 - case "PARTIAL", "UNSUPPORTED": - return 2 - case "OK": - return 1 - default: - return 0 - } -} - -func firstLine(s string) string { - s = strings.TrimSpace(s) - if s == "" { - return "" - } - if idx := strings.IndexByte(s, '\n'); idx >= 0 { - return strings.TrimSpace(s[:idx]) - } - return s -} - -func nvidiaJobNeedsHealthCheck(job satJob) bool { - if job.collectGPU { - return true - } - name := strings.ToLower(strings.TrimSpace(job.name)) - return strings.Contains(name, "dcgmi") || - strings.Contains(name, "gpu-burn") || - strings.Contains(name, "gpu-stress") || - strings.Contains(name, "dcgmproftester") -} - -func checkNvidiaJobHealth(selected []int) (string, error) { - health, err := readNvidiaGPUHealth() - if err != nil { - return "", nil - } - var bad []nvidiaGPUHealth - selectedSet := make(map[int]struct{}, len(selected)) - for _, idx := range selected { - selectedSet[idx] = struct{}{} - } - for _, gpu := range health { - if len(selectedSet) > 0 { - if _, ok := selectedSet[gpu.Index]; !ok { - continue - } - } - if gpu.NeedsReset { - bad = append(bad, gpu) - } - } - if len(bad) == 0 { - return "", nil - } - lines := make([]string, 0, len(bad)+1) - lines = append(lines, "NVIDIA GPU health check failed:") - for _, gpu := range bad { - lines = append(lines, fmt.Sprintf("gpu %d (%s) requires reset: %s", gpu.Index, gpu.Name, gpu.RawLine)) - } - return strings.Join(lines, "\n"), errors.New("nvidia gpu requires reset") -} - -func readNvidiaGPUHealth() ([]nvidiaGPUHealth, error) { - out, err := satExecCommand( - "nvidia-smi", - "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", - "--format=csv,noheader,nounits", - ).Output() - if err != nil { - return nil, fmt.Errorf("nvidia-smi: %w", err) - } - return parseNvidiaGPUHealth(string(out)), nil -} - -func parseNvidiaGPUHealth(raw string) []nvidiaGPUHealth { - var gpus []nvidiaGPUHealth - for _, line := range strings.Split(strings.TrimSpace(raw), "\n") { - line = strings.TrimSpace(line) - if line == "" { - continue - } - parts := strings.Split(line, ",") - if len(parts) < 2 { - gpus = append(gpus, nvidiaGPUHealth{RawLine: line, ParseFailure: true}) - continue - } - idx, err := strconv.Atoi(strings.TrimSpace(parts[0])) - if err != nil { - gpus = append(gpus, nvidiaGPUHealth{RawLine: line, ParseFailure: true}) - continue - } - upper := strings.ToUpper(line) - gpus = append(gpus, nvidiaGPUHealth{ - Index: idx, - Name: strings.TrimSpace(parts[1]), - NeedsReset: strings.Contains(upper, "GPU REQUIRES RESET"), - RawLine: line, - }) - } - return gpus -} - // runSATCommandCtx runs cmd and returns its combined output. livePath is // variadic purely so existing callers are unaffected: pass a path (job's // output file) to also stream output to disk live as it runs, so a crash @@ -736,7 +676,7 @@ func (s *satStats) Add(status string) { switch status { case "OK": s.OK++ - case "UNSUPPORTED": + case "UNSUPPORTED", "PARTIAL": s.Unsupported++ default: s.Failed++ diff --git a/audit/internal/platform/sat_fan_stress.go b/audit/internal/platform/sat_fan_stress.go index 399f7cf..5d22f8f 100644 --- a/audit/internal/platform/sat_fan_stress.go +++ b/audit/internal/platform/sat_fan_stress.go @@ -571,21 +571,6 @@ func sampleFanSpeedsViaSensorsJSON() ([]FanReading, error) { return fans, nil } -// sampleFanDutyCyclePct reads fan PWM/duty-cycle controls from lm-sensors. -// Returns the average duty cycle across all exposed PWM controls. -func sampleFanDutyCyclePct() (float64, bool, bool) { - out, err := exec.Command("sensors", "-j").Output() - if err != nil || len(out) == 0 { - fans, fanErr := sampleFanSpeeds() - if fanErr != nil { - return 0, false, false - } - return sampleFanDutyCyclePctFromFans(fans) - } - pct, ok := parseFanDutyCyclePctSensorsJSON(out) - return pct, ok, false -} - func sampleFanDutyCyclePctFromFans(fans []FanReading) (float64, bool, bool) { if len(fans) == 0 { return 0, false, false @@ -682,27 +667,7 @@ func normalizePWMAsDutyPct(raw float64) (float64, bool) { } func firstFanInputValue(feature map[string]any) (float64, bool) { - keys := make([]string, 0, len(feature)) - for key := range feature { - keys = append(keys, key) - } - sort.Strings(keys) - for _, key := range keys { - lower := strings.ToLower(key) - if !strings.Contains(lower, "fan") || !strings.HasSuffix(lower, "_input") { - continue - } - switch value := feature[key].(type) { - case float64: - return value, true - case string: - f, err := strconv.ParseFloat(value, 64) - if err == nil { - return f, true - } - } - } - return 0, false + return firstSensorInputValue(feature, "fan") } // sampleCPUMaxTemp returns the highest CPU/inlet temperature from ipmitool or sensors. diff --git a/audit/internal/platform/sat_gpu_packs.go b/audit/internal/platform/sat_gpu_packs.go index 7f016da..c820733 100644 --- a/audit/internal/platform/sat_gpu_packs.go +++ b/audit/internal/platform/sat_gpu_packs.go @@ -324,17 +324,30 @@ func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices [] if err != nil { return "", err } + return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", ncclSATJobs(selected), logFunc) +} + +func ncclSATJobs(selected []int) []satJob { gpuCount := len(selected) if gpuCount < 1 { gpuCount = 1 } - return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", withNvidiaPersistenceMode( + ncclEnv := append(nvidiaVisibleDevicesEnv(selected), + "NCCL_DEBUG=INFO", + "NCCL_DEBUG_SUBSYS=INIT,GRAPH", + ) + return withNvidiaPersistenceMode( satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, + satJob{name: "01-nvidia-smi-topo-m.log", cmd: []string{"nvidia-smi", "topo", "-m"}, informational: true}, + satJob{name: "01-nvidia-smi-nvlink-s.log", cmd: []string{"nvidia-smi", "nvlink", "-s"}, informational: true}, satJob{name: "02-all-reduce-perf.log", cmd: []string{ "all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2", "-g", strconv.Itoa(gpuCount), "--iters", "20", - }, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true}, - ), logFunc) + }, env: ncclEnv, gpuIndices: selected, syncBracket: true, + validate: func(runDir string, out []byte) (string, string) { + return validateNCCLAllReduceOutput(runDir, out, selected) + }}, + ) } func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error) { @@ -384,23 +397,27 @@ func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir strin } func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) { + return s.runNvidiaNamedDiagPack(ctx, baseDir, durationSec, gpuIndices, + "gpu-nvidia-targeted-power", "targeted_power", "03-dcgmi-targeted-power.log", logFunc) +} + +func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) { + return s.runNvidiaNamedDiagPack(ctx, baseDir, durationSec, gpuIndices, + "gpu-nvidia-pulse", "pulse_test", "03-dcgmi-pulse-test.log", logFunc) +} + +func (s *System) runNvidiaNamedDiagPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, packName, diagName, logName string, logFunc func(string)) (string, error) { selected, err := resolveDCGMGPUIndices(gpuIndices) if err != nil { return "", err } - // Kill any lingering nvvs/dcgmi processes from a previous interrupted run - // before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34). - if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil { - for _, p := range killed { - logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) - } - } - return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode( + killStaleNvidiaTestWorkers(logFunc) + return runAcceptancePackCtx(ctx, baseDir, packName, withNvidiaPersistenceMode( satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, satJob{ - name: "03-dcgmi-targeted-power.log", - cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected), + name: logName, + cmd: nvidiaDCGMNamedDiagCommand(diagName, normalizeNvidiaBurnDuration(durationSec), selected), collectGPU: true, gpuIndices: selected, syncBracket: true, @@ -409,30 +426,14 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, ), logFunc) } -func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) { - selected, err := resolveDCGMGPUIndices(gpuIndices) - if err != nil { - return "", err - } - // Kill any lingering nvvs/dcgmi processes from a previous interrupted run - // before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34). +// Kill any lingering nvvs/dcgmi processes from a previous interrupted run +// before starting; otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34). +func killStaleNvidiaTestWorkers(logFunc func(string)) { if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil { for _, p := range killed { logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) } } - return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode( - satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, - satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, - satJob{ - name: "03-dcgmi-pulse-test.log", - cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected), - collectGPU: true, - gpuIndices: selected, - syncBracket: true, - }, - satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, - ), logFunc) } // RunNvidiaBandwidthPack runs `dcgmi diag -r nvbandwidth`. The only thing @@ -449,13 +450,7 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu if err != nil { return "", err } - // Kill any lingering nvvs/dcgmi processes from a previous interrupted run - // before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34). - if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil { - for _, p := range killed { - logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) - } - } + killStaleNvidiaTestWorkers(logFunc) jobs := []satJob{ {name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, {name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, @@ -528,29 +523,8 @@ func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir } func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) { - selected, err := resolveDCGMGPUIndices(gpuIndices) - if err != nil { - return "", err - } - // Kill any lingering nvvs/dcgmi processes from a previous interrupted run - // before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34). - if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil { - for _, p := range killed { - logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) - } - } - return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode( - satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, - satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, - satJob{ - name: "03-dcgmi-targeted-stress.log", - cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected), - collectGPU: true, - gpuIndices: selected, - syncBracket: true, - }, - satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, - ), logFunc) + return s.runNvidiaNamedDiagPack(ctx, baseDir, durationSec, gpuIndices, + "gpu-nvidia-targeted-stress", "targeted_stress", "03-dcgmi-targeted-stress.log", logFunc) } func resolveDCGMGPUIndices(gpuIndices []int) ([]int, error) { diff --git a/audit/internal/platform/sat_nccl_validation.go b/audit/internal/platform/sat_nccl_validation.go new file mode 100644 index 0000000..bc579cb --- /dev/null +++ b/audit/internal/platform/sat_nccl_validation.go @@ -0,0 +1,123 @@ +package platform + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +var ( + ncclOutOfBoundsRE = regexp.MustCompile(`(?im)^#\s*Out of bounds values\s*:\s*(\d+)(?:\s+\S+)?\s*$`) + ncclAvgBusBWRE = regexp.MustCompile(`(?im)^#\s*Avg bus bandwidth\s*:\s*([0-9]+(?:\.[0-9]+)?)\s*$`) + ncclGPUHeaderRE = regexp.MustCompile(`(?i)\bGPU\d+\b`) +) + +// validateNCCLAllReduceOutput validates only facts that nccl-tests reports +// directly. It deliberately does not apply a model-independent bandwidth +// threshold: busbw depends on GPU model, topology, rank count, message size, +// and NCCL algorithm. The measured value is retained in detail for diagnosis. +func validateNCCLAllReduceOutput(runDir string, out []byte, selectedGPUIndices []int) (string, string) { + text := string(out) + oobMatch := ncclOutOfBoundsRE.FindStringSubmatch(text) + if len(oobMatch) != 2 { + return "UNSUPPORTED", "nccl-tests output has no recognized Out of bounds values summary" + } + oob, err := strconv.ParseUint(oobMatch[1], 10, 64) + if err != nil { + return "UNSUPPORTED", "nccl-tests Out of bounds values is not an unsigned integer" + } + if oob != 0 { + return "FAILED", fmt.Sprintf("nccl-tests reported %d out-of-bounds value(s)", oob) + } + + rows, wrong, rowsValid := parseNCCLWrongCounts(text) + if !rowsValid || rows == 0 { + return "UNSUPPORTED", "nccl-tests output has no recognized all_reduce_perf result rows" + } + if wrong != 0 { + return "FAILED", fmt.Sprintf("nccl-tests result rows reported %d wrong value(s)", wrong) + } + + avgMatch := ncclAvgBusBWRE.FindStringSubmatch(text) + if len(avgMatch) != 2 { + return "UNSUPPORTED", "nccl-tests output has no recognized Avg bus bandwidth value" + } + avgBusBW, err := strconv.ParseFloat(avgMatch[1], 64) + if err != nil { + return "UNSUPPORTED", "nccl-tests Avg bus bandwidth is not numeric" + } + + detail := fmt.Sprintf("validation passed; avg_bus_bandwidth=%.4g GB/s (diagnostic only, no unverified threshold)", avgBusBW) + if len(selectedGPUIndices) < 2 { + return "OK", detail + } + + topoRaw, err := os.ReadFile(filepath.Join(runDir, "01-nvidia-smi-topo-m.log")) + if err != nil || !ncclGPUHeaderRE.Match(topoRaw) { + return "UNSUPPORTED", "NVIDIA topology output unavailable or unrecognized; cannot validate expected NVLink pairs" + } + pairs := selectedNvidiaNVLinkPairs(ParseNvidiaNVLinkBondedPairs(string(topoRaw)), selectedGPUIndices) + if len(pairs) == 0 { + return "OK", detail + "; selected topology has no NVLink-bonded pair" + } + + statusRaw, err := os.ReadFile(filepath.Join(runDir, "01-nvidia-smi-nvlink-s.log")) + if err != nil { + return "UNSUPPORTED", "NVLink status output unavailable for selected NVLink-bonded pair" + } + linkStatus := parseNvidiaNVLinkStatus(string(statusRaw)) + var issues []string + for _, pair := range pairs { + finding := evaluateNvidiaNVLinkPair(pair, linkStatus, nil) + for _, issue := range finding.Issues { + issues = append(issues, fmt.Sprintf("GPU%d<->GPU%d: %s", pair.GPUA, pair.GPUB, issue)) + } + } + if len(issues) != 0 { + return "FAILED", "NVLink state does not match selected topology: " + strings.Join(issues, "; ") + } + return "OK", detail + "; selected NVLink endpoints match topology" +} + +func parseNCCLWrongCounts(raw string) (rows int, wrong uint64, valid bool) { + for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + if len(fields) < 13 { + continue + } + if _, err := strconv.ParseUint(fields[0], 10, 64); err != nil { + continue + } + outOfPlaceWrong, errA := strconv.ParseUint(fields[8], 10, 64) + inPlaceWrong, errB := strconv.ParseUint(fields[12], 10, 64) + if errA != nil || errB != nil { + return rows, wrong, false + } + rows++ + wrong += outOfPlaceWrong + inPlaceWrong + } + return rows, wrong, true +} + +func selectedNvidiaNVLinkPairs(pairs []NvidiaNVLinkBondedPair, selected []int) []NvidiaNVLinkBondedPair { + selectedSet := make(map[int]struct{}, len(selected)) + for _, index := range selected { + selectedSet[index] = struct{}{} + } + filtered := make([]NvidiaNVLinkBondedPair, 0, len(pairs)) + for _, pair := range pairs { + _, aSelected := selectedSet[pair.GPUA] + _, bSelected := selectedSet[pair.GPUB] + if aSelected && bSelected { + filtered = append(filtered, pair) + } + } + return filtered +} diff --git a/audit/internal/platform/sat_nccl_validation_test.go b/audit/internal/platform/sat_nccl_validation_test.go new file mode 100644 index 0000000..128d2e2 --- /dev/null +++ b/audit/internal/platform/sat_nccl_validation_test.go @@ -0,0 +1,136 @@ +package platform + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +const ncclValidOutput = `# size count type redop root time algbw busbw #wrong time algbw busbw #wrong +536870912 134217728 float sum -1 19547 27.47 48.07 0 19512 27.52 48.15 0 +# Out of bounds values : 0 OK +# Avg bus bandwidth : 48.1908 +` + +func TestValidateNCCLAllReduceOutputHealthyNVLinkPair(t *testing.T) { + runDir := t.TempDir() + writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tNV2\nGPU1\tNV2\tX\n") + writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-nvlink-s.log", `GPU 0: H100 +Link 0: 26.562 GB/s +Link 1: 26.562 GB/s +GPU 1: H100 +Link 0: 26.562 GB/s +Link 1: 26.562 GB/s +`) + status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1}) + if status != "OK" { + t.Fatalf("status=%q detail=%q want OK", status, detail) + } +} + +func TestValidateNCCLAllReduceOutputFailsInactiveNVLinkEndpoint(t *testing.T) { + runDir := t.TempDir() + writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tNV2\nGPU1\tNV2\tX\n") + writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-nvlink-s.log", `GPU 0: H100 +Link 0: 26.562 GB/s +Link 1: 26.562 GB/s +GPU 1: H100 +NVML: Unable to retrieve NVLink information as all links are inActive +`) + status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1}) + if status != "FAILED" || !strings.Contains(detail, "GPU1 0/2") { + t.Fatalf("status=%q detail=%q want FAILED with GPU1 0/2", status, detail) + } +} + +func TestValidateNCCLAllReduceOutputRejectsWrongValues(t *testing.T) { + bad := strings.Replace(ncclValidOutput, "48.07 0", "48.07 2", 1) + status, detail := validateNCCLAllReduceOutput(t.TempDir(), []byte(bad), []int{0}) + if status != "FAILED" || !strings.Contains(detail, "2 wrong") { + t.Fatalf("status=%q detail=%q want wrong-value failure", status, detail) + } +} + +func TestValidateNCCLAllReduceOutputPCIeOnlyDoesNotRequireNVLink(t *testing.T) { + runDir := t.TempDir() + writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tPIX\nGPU1\tPIX\tX\n") + status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1}) + if status != "OK" || !strings.Contains(detail, "no NVLink-bonded pair") { + t.Fatalf("status=%q detail=%q want PCIe-only OK", status, detail) + } +} + +func TestValidateNCCLAllReduceOutputUnknownFormatIsUnsupported(t *testing.T) { + status, _ := validateNCCLAllReduceOutput(t.TempDir(), []byte("all done\n"), []int{0}) + if status != "UNSUPPORTED" { + t.Fatalf("status=%q want UNSUPPORTED", status) + } +} + +func TestRunAcceptancePackAppliesOutputValidator(t *testing.T) { + runDir, err := runAcceptancePackCtx(context.Background(), t.TempDir(), "validator-test", []satJob{{ + name: "01-result.log", + cmd: []string{"sh", "-c", "printf command-ok"}, + validate: func(string, []byte) (string, string) { + return "FAILED", "documented output check failed" + }, + }}, nil) + if err != nil { + t.Fatal(err) + } + summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(summary), "overall_status=FAILED") || + !strings.Contains(string(summary), "1-result_detail=documented output check failed") { + t.Fatalf("summary did not retain validator failure:\n%s", summary) + } + logData, err := os.ReadFile(filepath.Join(runDir, "01-result.log")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(logData), "[bee-validator] FAILED: documented output check failed") { + t.Fatalf("job log did not retain validator result:\n%s", logData) + } +} + +func TestNCCLJobHealthAndDebugConfiguration(t *testing.T) { + jobs := ncclSATJobs([]int{0, 2}) + var job satJob + for _, candidate := range jobs { + if candidate.name == "02-all-reduce-perf.log" { + job = candidate + break + } + } + if job.name == "" || job.validate == nil { + t.Fatal("all_reduce_perf job or its output validator is missing") + } + if !nvidiaJobNeedsHealthCheck(job) { + t.Fatal("all_reduce_perf job must run NVIDIA health checks") + } + if !isNvidiaAcceptancePack("nccl-tests") { + t.Fatal("nccl-tests must produce NVIDIA per-GPU status files") + } + env := strings.Join(job.env, "\n") + for _, expected := range []string{ + "CUDA_DEVICE_ORDER=PCI_BUS_ID", + "CUDA_VISIBLE_DEVICES=0,2", + "NCCL_DEBUG=INFO", + "NCCL_DEBUG_SUBSYS=INIT,GRAPH", + } { + if !strings.Contains(env, expected) { + t.Fatalf("job env=%q missing %q", env, expected) + } + } +} + +func writeNCCLValidationArtifact(t *testing.T, dir, name, body string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0644); err != nil { + t.Fatal(err) + } +} diff --git a/audit/internal/platform/sat_nvidia_health.go b/audit/internal/platform/sat_nvidia_health.go new file mode 100644 index 0000000..5ae9d1e --- /dev/null +++ b/audit/internal/platform/sat_nvidia_health.go @@ -0,0 +1,126 @@ +package platform + +import ( + "errors" + "fmt" + "strconv" + "strings" +) + +func nvidiaSATStatusSeverity(status string) int { + switch strings.ToUpper(strings.TrimSpace(status)) { + case "FAILED": + return 3 + case "PARTIAL", "UNSUPPORTED": + return 2 + case "OK": + return 1 + default: + return 0 + } +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if s == "" { + return "" + } + if idx := strings.IndexByte(s, '\n'); idx >= 0 { + return strings.TrimSpace(s[:idx]) + } + return s +} + +func nvidiaJobNeedsHealthCheck(job satJob) bool { + if job.collectGPU { + return true + } + name := strings.ToLower(strings.TrimSpace(job.name)) + return strings.Contains(name, "dcgmi") || + strings.Contains(name, "all-reduce") || + strings.Contains(name, "gpu-burn") || + strings.Contains(name, "gpu-stress") || + strings.Contains(name, "dcgmproftester") +} + +func checkNvidiaJobHealth(selected []int) (string, error) { + health, _ := readNvidiaGPUHealth() + var bad []nvidiaGPUHealth + selectedSet := make(map[int]struct{}, len(selected)) + for _, idx := range selected { + selectedSet[idx] = struct{}{} + } + for _, gpu := range health { + if len(selectedSet) > 0 { + if _, ok := selectedSet[gpu.Index]; !ok { + continue + } + } + if gpu.NeedsReset { + bad = append(bad, gpu) + } + } + var degraded []nvidiaNVLinkDegradedEvent + if out, err := satExecCommand("dmesg").Output(); err == nil { + for _, event := range parseNvidiaNVLinkDegradedDmesg(out) { + if event.GPUIndex >= 0 && len(selectedSet) > 0 { + if _, ok := selectedSet[event.GPUIndex]; !ok { + continue + } + } + degraded = append(degraded, event) + } + } + if len(bad) == 0 && len(degraded) == 0 { + return "", nil + } + lines := make([]string, 0, len(bad)+len(degraded)+1) + lines = append(lines, "NVIDIA GPU health check failed:") + for _, gpu := range bad { + lines = append(lines, fmt.Sprintf("gpu %d (%s) requires reset: %s", gpu.Index, gpu.Name, gpu.RawLine)) + } + for _, event := range degraded { + lines = append(lines, event.Warning()) + } + return strings.Join(lines, "\n"), errors.New("nvidia gpu health check failed") +} + +func readNvidiaGPUHealth() ([]nvidiaGPUHealth, error) { + out, err := satExecCommand( + "nvidia-smi", + "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", + "--format=csv,noheader,nounits", + ).Output() + if err != nil { + return nil, fmt.Errorf("nvidia-smi: %w", err) + } + return parseNvidiaGPUHealth(string(out)), nil +} + +func parseNvidiaGPUHealth(raw string) []nvidiaGPUHealth { + var gpus []nvidiaGPUHealth + for _, line := range strings.Split(strings.TrimSpace(raw), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.Split(line, ",") + if len(parts) < 2 { + gpus = append(gpus, nvidiaGPUHealth{RawLine: line, ParseFailure: true}) + continue + } + idx, err := strconv.Atoi(strings.TrimSpace(parts[0])) + if err != nil { + gpus = append(gpus, nvidiaGPUHealth{RawLine: line, ParseFailure: true}) + continue + } + upper := strings.ToUpper(line) + gpus = append(gpus, nvidiaGPUHealth{ + Index: idx, + Name: strings.TrimSpace(parts[1]), + NeedsReset: strings.Contains(upper, "GPU REQUIRES RESET"), + RawLine: line, + }) + } + return gpus +} diff --git a/audit/internal/platform/sat_test.go b/audit/internal/platform/sat_test.go index df689c3..c1e5441 100644 --- a/audit/internal/platform/sat_test.go +++ b/audit/internal/platform/sat_test.go @@ -273,6 +273,32 @@ func TestCheckNvidiaJobHealthReturnsErrorForSelectedResetRequiredGPU(t *testing. } } +func TestCheckNvidiaJobHealthReturnsErrorForSelectedNVLinkDegradedGPU(t *testing.T) { + oldExecCommand := satExecCommand + satExecCommand = func(name string, args ...string) *exec.Cmd { + switch name { + case "nvidia-smi": + return exec.Command("sh", "-c", "printf '0, NVIDIA H100 PCIe, 38, 46.89, 0, 0, 81559\n1, NVIDIA H100 PCIe, 39, 47.00, 0, 0, 81559\n'") + case "dmesg": + return exec.Command("sh", "-c", "printf 'NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n'") + default: + return exec.Command(name, args...) + } + } + t.Cleanup(func() { satExecCommand = oldExecCommand }) + + msg, err := checkNvidiaJobHealth([]int{1}) + if err == nil { + t.Fatal("expected NVLink degraded health check error") + } + if !strings.Contains(msg, "GPU1 NVLink degraded mode (linkId 1)") { + t.Fatalf("unexpected message: %q", msg) + } + if msg, err := checkNvidiaJobHealth([]int{0}); err != nil || msg != "" { + t.Fatalf("unaffected selected GPU0 got msg=%q err=%v", msg, err) + } +} + func TestWriteNvidiaGPUStatusFilesCreatesPerGPUFiles(t *testing.T) { dir := t.TempDir() oldExecCommand := satExecCommand diff --git a/audit/internal/webui/api_gpu_tools.go b/audit/internal/webui/api_gpu_tools.go index 208b6d8..6f7e1b4 100644 --- a/audit/internal/webui/api_gpu_tools.go +++ b/audit/internal/webui/api_gpu_tools.go @@ -1,6 +1,7 @@ package webui import ( + "context" "encoding/json" "fmt" "net/http" @@ -16,35 +17,15 @@ import ( ) func (h *handler) handleAPIGNVIDIAGPUs(w http.ResponseWriter, _ *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - gpus, err := h.opts.App.ListNvidiaGPUs() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - if gpus == nil { - gpus = []platform.NvidiaGPU{} - } - writeJSON(w, gpus) + writeAPIListResponse(w, h.opts.App != nil, func() ([]platform.NvidiaGPU, error) { + return h.opts.App.ListNvidiaGPUs() + }) } func (h *handler) handleAPIGNVIDIAGPUStatuses(w http.ResponseWriter, _ *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - gpus, err := apiListNvidiaGPUStatuses(h.opts.App) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - if gpus == nil { - gpus = []platform.NvidiaGPUStatus{} - } - writeJSON(w, gpus) + writeAPIListResponse(w, h.opts.App != nil, func() ([]platform.NvidiaGPUStatus, error) { + return apiListNvidiaGPUStatuses(h.opts.App) + }) } func (h *handler) handleAPIGNVIDIAReset(w http.ResponseWriter, r *http.Request) { @@ -70,64 +51,33 @@ func (h *handler) handleAPIGNVIDIAReset(w http.ResponseWriter, r *http.Request) // ── GPU settings (ECC / power limit) ────────────────────────────────────────── func (h *handler) handleAPIGNVIDIAGPUSettings(w http.ResponseWriter, _ *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - settings, err := h.opts.App.ListNvidiaGPUSettings() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - if settings == nil { - settings = []platform.NvidiaGPUSetting{} - } - writeJSON(w, settings) + writeAPIListResponse(w, h.opts.App != nil, func() ([]platform.NvidiaGPUSetting, error) { + return h.opts.App.ListNvidiaGPUSettings() + }) } func (h *handler) handleAPIGNVIDIASetECC(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var req struct { - Index int `json:"index"` - Enabled bool `json:"enabled"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - result, err := h.opts.App.SetNvidiaGPUECC(req.Index, req.Enabled) - status := "ok" - if err != nil { - status = "error" - } - writeJSON(w, map[string]string{"status": status, "output": result.Body}) + h.handleAPIGNVIDIASetBool(w, r, func(index int, enabled bool) (string, error) { + result, err := h.opts.App.SetNvidiaGPUECC(index, enabled) + return result.Body, err + }) } func (h *handler) handleAPIGNVIDIASetMIG(w http.ResponseWriter, r *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - var req struct { - Index int `json:"index"` - Enabled bool `json:"enabled"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - result, err := h.opts.App.SetNvidiaGPUMIG(req.Index, req.Enabled) - status := "ok" - if err != nil { - status = "error" - } - writeJSON(w, map[string]string{"status": status, "output": result.Body}) + h.handleAPIGNVIDIASetBool(w, r, func(index int, enabled bool) (string, error) { + result, err := h.opts.App.SetNvidiaGPUMIG(index, enabled) + return result.Body, err + }) } func (h *handler) handleAPIGNVIDIASetCCMode(w http.ResponseWriter, r *http.Request) { + h.handleAPIGNVIDIASetBool(w, r, func(index int, enabled bool) (string, error) { + result, err := h.opts.App.SetNvidiaGPUCCMode(index, enabled) + return result.Body, err + }) +} + +func (h *handler) handleAPIGNVIDIASetBool(w http.ResponseWriter, r *http.Request, apply func(int, bool) (string, error)) { if h.opts.App == nil { writeError(w, http.StatusServiceUnavailable, "app not configured") return @@ -140,12 +90,12 @@ func (h *handler) handleAPIGNVIDIASetCCMode(w http.ResponseWriter, r *http.Reque writeError(w, http.StatusBadRequest, "invalid request body") return } - result, err := h.opts.App.SetNvidiaGPUCCMode(req.Index, req.Enabled) + output, err := apply(req.Index, req.Enabled) status := "ok" if err != nil { status = "error" } - writeJSON(w, map[string]string{"status": status, "output": result.Body}) + writeJSON(w, map[string]string{"status": status, "output": output}) } func (h *handler) handleAPIGNVIDIASetPowerLimit(w http.ResponseWriter, r *http.Request) { @@ -348,6 +298,9 @@ func validTimezoneName(tz string) bool { // values supplied by the client's browser (used when the appliance has no // network/NTP access to keep its own clock in sync). func (h *handler) handleAPISystemTimeSync(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + var req struct { Timezone string `json:"timezone"` EpochMS int64 `json:"epoch_ms"` @@ -368,19 +321,25 @@ func (h *handler) handleAPISystemTimeSync(w http.ResponseWriter, r *http.Request writeError(w, http.StatusBadRequest, "invalid timezone") return } - if b, err := exec.Command("timedatectl", "set-timezone", req.Timezone).CombinedOutput(); err != nil { - writeError(w, http.StatusInternalServerError, "set-timezone failed: "+strings.TrimSpace(string(b))) + if b, err := exec.CommandContext(ctx, "timedatectl", "set-timezone", req.Timezone).CombinedOutput(); err != nil { + writeError(w, http.StatusInternalServerError, "set-timezone failed: "+commandFailureDetail(b, err)) return } fmt.Fprintf(&out, "timezone set to %s\n", req.Timezone) } // Manual time only sticks if NTP sync is off. - _ = exec.Command("timedatectl", "set-ntp", "false").Run() + if b, err := exec.CommandContext(ctx, "timedatectl", "set-ntp", "false").CombinedOutput(); err != nil { + canNTP, canErr := exec.CommandContext(ctx, "timedatectl", "show", "-p", "CanNTP", "--value").Output() + if canErr != nil || strings.TrimSpace(string(canNTP)) != "no" { + writeError(w, http.StatusInternalServerError, "disable NTP failed: "+commandFailureDetail(b, err)) + return + } + } sec := req.EpochMS / 1000 - if b, err := exec.Command("date", "-u", "-s", "@"+strconv.FormatInt(sec, 10)).CombinedOutput(); err != nil { - writeError(w, http.StatusInternalServerError, "set-time failed: "+strings.TrimSpace(string(b))) + if b, err := exec.CommandContext(ctx, "date", "-u", "-s", "@"+strconv.FormatInt(sec, 10)).CombinedOutput(); err != nil { + writeError(w, http.StatusInternalServerError, "set-time failed: "+commandFailureDetail(b, err)) return } out.WriteString("system clock synced\n") @@ -388,6 +347,36 @@ func (h *handler) handleAPISystemTimeSync(w http.ResponseWriter, r *http.Request writeJSON(w, map[string]string{"status": "ok", "output": out.String()}) } +func commandFailureDetail(output []byte, err error) string { + if detail := strings.TrimSpace(string(output)); detail != "" { + return detail + } + return err.Error() +} + +// handleAPISystemTime reports the host's current wall-clock time and configured +// timezone so the dashboard can show them next to the browser's own clock and +// flag a drift. +func (h *handler) handleAPISystemTime(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + + tz := "" + if b, err := exec.CommandContext(ctx, "timedatectl", "show", "-p", "Timezone", "--value").Output(); err == nil { + tz = strings.TrimSpace(string(b)) + } + now := time.Now() + if tz == "" { + tz, _ = now.Zone() + } + + writeJSON(w, map[string]any{ + "epoch_ms": now.UnixMilli(), + "local_time": now.Format("2006-01-02 15:04:05"), + "timezone": tz, + }) +} + // ── Tools ───────────────────────────────────────────────────────────────────── var standardTools = []string{ diff --git a/audit/internal/webui/api_runs.go b/audit/internal/webui/api_runs.go index bbb9df9..590408d 100644 --- a/audit/internal/webui/api_runs.go +++ b/audit/internal/webui/api_runs.go @@ -35,15 +35,10 @@ func (h *handler) handleAPIAuditStream(w http.ResponseWriter, r *http.Request) { if id == "" { id = r.URL.Query().Get("task_id") } - // Try task queue first, then legacy job manager if j, ok := globalQueue.findJob(id); ok { streamJob(w, r, j) return } - if j, ok := globalJobs.get(id); ok { - streamJob(w, r, j) - return - } http.Error(w, "job not found", http.StatusNotFound) } @@ -383,10 +378,6 @@ func (h *handler) handleAPISATStream(w http.ResponseWriter, r *http.Request) { streamJob(w, r, j) return } - if j, ok := globalJobs.get(id); ok { - streamJob(w, r, j) - return - } http.Error(w, "job not found", http.StatusNotFound) } @@ -416,14 +407,6 @@ func (h *handler) handleAPISATAbort(w http.ResponseWriter, r *http.Request) { writeJSON(w, map[string]string{"status": "aborted"}) return } - if j, ok := globalJobs.get(id); ok { - if j.abort() { - writeJSON(w, map[string]string{"status": "aborted"}) - } else { - writeJSON(w, map[string]string{"status": "not_running"}) - } - return - } http.Error(w, "job not found", http.StatusNotFound) } diff --git a/audit/internal/webui/api_services_system.go b/audit/internal/webui/api_services_system.go index 8344adb..f0862ee 100644 --- a/audit/internal/webui/api_services_system.go +++ b/audit/internal/webui/api_services_system.go @@ -99,7 +99,10 @@ func (h *handler) handleAPINetworkDHCP(w http.ResponseWriter, r *http.Request) { var req struct { Interface string `json:"interface"` } - _ = json.NewDecoder(r.Body).Decode(&req) + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } result, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) { if req.Interface == "" || req.Interface == "all" { @@ -167,19 +170,9 @@ func (h *handler) handleAPIExportList(w http.ResponseWriter, r *http.Request) { } func (h *handler) handleAPIExportUSBTargets(w http.ResponseWriter, _ *http.Request) { - if h.opts.App == nil { - writeError(w, http.StatusServiceUnavailable, "app not configured") - return - } - targets, err := h.opts.App.ListRemovableTargets() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - if targets == nil { - targets = []platform.RemovableTarget{} - } - writeJSON(w, targets) + writeAPIListResponse(w, h.opts.App != nil, func() ([]platform.RemovableTarget, error) { + return h.opts.App.ListRemovableTargets() + }) } func (h *handler) handleAPIBlackboxStatus(w http.ResponseWriter, _ *http.Request) { diff --git a/audit/internal/webui/api_system_test.go b/audit/internal/webui/api_system_test.go new file mode 100644 index 0000000..6221488 --- /dev/null +++ b/audit/internal/webui/api_system_test.go @@ -0,0 +1,50 @@ +package webui + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "bee/audit/internal/app" +) + +func TestSystemTimeEndpointReportsEpochLocalTimeAndTimezone(t *testing.T) { + before := time.Now().Add(-time.Second).UnixMilli() + rec := httptest.NewRecorder() + NewHandler(HandlerOptions{}).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/system/time", nil)) + after := time.Now().Add(time.Second).UnixMilli() + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) + } + var body struct { + EpochMS int64 `json:"epoch_ms"` + LocalTime string `json:"local_time"` + Timezone string `json:"timezone"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode response: %v", err) + } + if body.EpochMS < before || body.EpochMS > after { + t.Fatalf("epoch_ms = %d, want value in [%d, %d]", body.EpochMS, before, after) + } + if _, err := time.ParseInLocation("2006-01-02 15:04:05", body.LocalTime, time.Local); err != nil { + t.Fatalf("local_time = %q: %v", body.LocalTime, err) + } + if strings.TrimSpace(body.Timezone) == "" { + t.Fatal("timezone must not be empty") + } +} + +func TestNetworkDHCPRejectsMalformedJSONBeforeChangingNetwork(t *testing.T) { + h := &handler{opts: HandlerOptions{App: &app.App{}}} + rec := httptest.NewRecorder() + h.handleAPINetworkDHCP(rec, httptest.NewRequest(http.MethodPost, "/api/network/dhcp", strings.NewReader("{"))) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest) + } +} diff --git a/audit/internal/webui/jobs.go b/audit/internal/webui/jobs.go index cab211b..c144913 100644 --- a/audit/internal/webui/jobs.go +++ b/audit/internal/webui/jobs.go @@ -7,7 +7,6 @@ import ( "os" "strings" "sync" - "time" ) // jobState holds the output lines and completion status of an async job. @@ -134,29 +133,6 @@ func (j *jobState) subscribe() ([]string, <-chan string) { return existing, ch } -// jobManager manages async jobs identified by string IDs. -type jobManager struct { - mu sync.Mutex - jobs map[string]*jobState -} - -var globalJobs = &jobManager{jobs: make(map[string]*jobState)} - -func (m *jobManager) create(id string) *jobState { - m.mu.Lock() - defer m.mu.Unlock() - j := &jobState{} - m.jobs[id] = j - // Schedule cleanup after 30 minutes - goRecoverOnce("job cleanup", func() { - time.Sleep(30 * time.Minute) - m.mu.Lock() - delete(m.jobs, id) - m.mu.Unlock() - }) - return j -} - // isDone returns true if the job has finished (either successfully or with error). func (j *jobState) isDone() bool { j.mu.Lock() @@ -164,13 +140,6 @@ func (j *jobState) isDone() bool { return j.done } -func (m *jobManager) get(id string) (*jobState, bool) { - m.mu.Lock() - defer m.mu.Unlock() - j, ok := m.jobs[id] - return j, ok -} - func newTaskJobState(logPath string, serialPrefix ...string) *jobState { j := &jobState{logPath: logPath} if len(serialPrefix) > 0 { diff --git a/audit/internal/webui/metrics_charts_http.go b/audit/internal/webui/metrics_charts_http.go index 7ffdd5d..cc4c960 100644 --- a/audit/internal/webui/metrics_charts_http.go +++ b/audit/internal/webui/metrics_charts_http.go @@ -400,37 +400,6 @@ func normalizePowerSeries(ds []float64) []float64 { return out } -// psuSlotsFromSamples returns the sorted list of PSU slot numbers seen across samples. -func psuSlotsFromSamples(samples []platform.LiveMetricSample) []int { - seen := map[int]struct{}{} - for _, s := range samples { - for _, p := range s.PSUs { - seen[p.Slot] = struct{}{} - } - } - slots := make([]int, 0, len(seen)) - for s := range seen { - slots = append(slots, s) - } - sort.Ints(slots) - return slots -} - -// psuStackedTotal returns the point-by-point sum of all PSU datasets (for scale calculation). -func psuStackedTotal(datasets [][]float64) []float64 { - if len(datasets) == 0 { - return nil - } - n := len(datasets[0]) - total := make([]float64, n) - for _, ds := range datasets { - for i, v := range ds { - total[i] += v - } - } - return total -} - func normalizeFanSeries(ds []float64) []float64 { if len(ds) == 0 { return nil diff --git a/audit/internal/webui/page_benchmark.go b/audit/internal/webui/page_benchmark.go index 676dc1a..f8b876d 100644 --- a/audit/internal/webui/page_benchmark.go +++ b/audit/internal/webui/page_benchmark.go @@ -611,7 +611,3 @@ func renderPowerBenchmarkResultsCard(exportDir string) string { b.WriteString(``) return b.String() } - -// renderSpeed and renderEndurance are legacy wrappers; canonical page is 5. Benchmark at /benchmark. -func renderSpeed(opts HandlerOptions) string { return renderBenchmark(opts) } -func renderEndurance(opts HandlerOptions) string { return renderBenchmark(opts) } diff --git a/audit/internal/webui/page_topo.go b/audit/internal/webui/page_topo.go index 19f27e3..dab6b36 100644 --- a/audit/internal/webui/page_topo.go +++ b/audit/internal/webui/page_topo.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" + "bee/audit/internal/platform" "bee/audit/internal/schema" ) @@ -246,17 +247,7 @@ func buildMemoryColumnIndex(rawNodes []int) map[int]int { // GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query) // --------------------------------------------------------------------------- -type gpuPairLink struct { - GPUA, GPUB int - NVLinks int -} - -var topoNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`) - -// nvidia-smi underlines the topo -m header row with ANSI CSI sequences -// (ESC[4m...ESC[0m) even when stdout is not a TTY, so the captured techdump -// contains them and "GPU0" is not at the start of the trimmed header line. -var topoANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]") +type gpuPairLink = platform.NvidiaNVLinkBondedPair // parseGPUPairAdjacency returns every GPU pair with a nonzero NVLink bond // count from a "nvidia-smi topo -m" matrix. Unlike parseNVIDIATopologyMatrix @@ -264,86 +255,7 @@ var topoANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]") // who is bonded to whom — required so GPU-GPU edges are drawn for actually // bonded pairs, not for adjacent boxes in the layout. func parseGPUPairAdjacency(raw string) []gpuPairLink { - lines := strings.Split(topoANSIRe.ReplaceAllString(raw, ""), "\n") - headerIdx := -1 - var gpuColIndices []int - for i, line := range lines { - trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "GPU0") { - parts := strings.Fields(trimmed) - for j, col := range parts { - if strings.HasPrefix(col, "GPU") { - gpuColIndices = append(gpuColIndices, j) - } - } - if len(gpuColIndices) >= 2 { - headerIdx = i - } - break - } - } - if headerIdx < 0 { - return nil - } - - colIdxToGPU := make(map[int]int, len(gpuColIndices)) - for gpuIdx, colIdx := range gpuColIndices { - colIdxToGPU[colIdx] = gpuIdx - } - - seen := map[[2]int]bool{} - var pairs []gpuPairLink - rowGPU := -1 - for _, line := range lines[headerIdx+1:] { - trimmed := strings.TrimSpace(line) - if !strings.HasPrefix(trimmed, "GPU") { - continue - } - cells := strings.Fields(trimmed) - if len(cells) == 0 { - continue - } - rowLabel := strings.TrimPrefix(cells[0], "GPU") - n, err := strconv.Atoi(rowLabel) - if err != nil { - continue - } - rowGPU = n - for colIdx, colGPU := range colIdxToGPU { - if colGPU == rowGPU { - continue - } - dataIdx := colIdx + 1 - if dataIdx >= len(cells) { - continue - } - m := topoNVRe.FindStringSubmatch(cells[dataIdx]) - if len(m) != 2 { - continue - } - nv, err := strconv.Atoi(m[1]) - if err != nil || nv <= 0 { - continue - } - a, bGPU := rowGPU, colGPU - if a > bGPU { - a, bGPU = bGPU, a - } - key := [2]int{a, bGPU} - if seen[key] { - continue - } - seen[key] = true - pairs = append(pairs, gpuPairLink{GPUA: a, GPUB: bGPU, NVLinks: nv}) - } - } - sort.Slice(pairs, func(i, j int) bool { - if pairs[i].GPUA != pairs[j].GPUA { - return pairs[i].GPUA < pairs[j].GPUA - } - return pairs[i].GPUB < pairs[j].GPUB - }) - return pairs + return platform.ParseNvidiaNVLinkBondedPairs(raw) } // readTopoTechDump reads a file previously captured into the persistent diff --git a/audit/internal/webui/page_topo_diagram.go b/audit/internal/webui/page_topo_diagram.go index 09ca865..b814bd5 100644 --- a/audit/internal/webui/page_topo_diagram.go +++ b/audit/internal/webui/page_topo_diagram.go @@ -10,6 +10,7 @@ import ( "strings" "bee/audit/internal/app" + "bee/audit/internal/platform" "bee/audit/internal/schema" ) @@ -499,7 +500,6 @@ var ( topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`) topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`) topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*`) - topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`) ) func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) { @@ -549,38 +549,7 @@ func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) { // parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc]. func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 { - result := map[int]map[int][3]int64{} - currentGPU := -1 - for _, line := range strings.Split(raw, "\n") { - trimmed := strings.TrimSpace(line) - if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { - currentGPU, _ = strconv.Atoi(m[1]) - continue - } - if currentGPU < 0 { - continue - } - m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed) - if m == nil { - continue - } - linkIdx, _ := strconv.Atoi(m[1]) - count, _ := strconv.ParseInt(m[3], 10, 64) - if result[currentGPU] == nil { - result[currentGPU] = map[int][3]int64{} - } - c := result[currentGPU][linkIdx] - switch m[2] { - case "Replay": - c[0] = count - case "Recovery": - c[1] = count - case "CRC": - c[2] = count - } - result[currentGPU][linkIdx] = c - } - return result + return platform.ParseNvidiaNVLinkErrors(raw) } // renderTopoNVLinkCard renders the separate NVLink topology card. Returns "" diff --git a/audit/internal/webui/pages.go b/audit/internal/webui/pages.go index 34bc638..e9a2dc5 100644 --- a/audit/internal/webui/pages.go +++ b/audit/internal/webui/pages.go @@ -150,8 +150,9 @@ setInterval(function(){ return b.String() } -// renderTimeSyncCard shows the server's current time and a button that syncs -// the host clock and timezone to whatever the client's browser reports. +// renderTimeSyncCard shows the server's current clock and timezone next to the +// browser's own, highlights any drift, and offers a button that syncs the host +// clock and timezone to whatever the client's browser reports. func renderTimeSyncCard() string { return `
@@ -161,11 +162,66 @@ func renderTimeSyncCard() string {
+ + + + + + + +
Server
Browser
+
` } diff --git a/audit/internal/webui/server.go b/audit/internal/webui/server.go index 87acb3d..4155a24 100644 --- a/audit/internal/webui/server.go +++ b/audit/internal/webui/server.go @@ -348,6 +348,7 @@ func NewHandler(opts HandlerOptions) http.Handler { mux.HandleFunc("POST /api/system/install-to-ram", h.handleAPIInstallToRAM) mux.HandleFunc("POST /api/system/reboot", h.handleAPISystemReboot) mux.HandleFunc("POST /api/system/shutdown", h.handleAPISystemShutdown) + mux.HandleFunc("GET /api/system/time", h.handleAPISystemTime) mux.HandleFunc("POST /api/system/time-sync", h.handleAPISystemTimeSync) // Preflight @@ -813,3 +814,19 @@ func writeError(w http.ResponseWriter, status int, msg string) { w.WriteHeader(status) _ = json.NewEncoder(w).Encode(map[string]string{"error": msg}) } + +func writeAPIListResponse[T any](w http.ResponseWriter, configured bool, load func() ([]T, error)) { + if !configured { + writeError(w, http.StatusServiceUnavailable, "app not configured") + return + } + items, err := load() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if items == nil { + items = []T{} + } + writeJSON(w, items) +} diff --git a/audit/internal/webui/tasks.go b/audit/internal/webui/tasks.go index 429cea1..0b080d3 100644 --- a/audit/internal/webui/tasks.go +++ b/audit/internal/webui/tasks.go @@ -519,7 +519,9 @@ func (q *taskQueue) startWorker(opts *HandlerOptions) { q.opts = opts q.statePath = filepath.Join(opts.ExportDir, "tasks-state.json") q.logsDir = filepath.Join(opts.ExportDir, "tasks") - _ = os.MkdirAll(q.logsDir, 0755) + if err := os.MkdirAll(q.logsDir, 0755); err != nil { + slog.Error("task queue: create logs directory", "path", q.logsDir, "error", err) + } if !q.started { q.loadLocked() q.started = true diff --git a/audit/internal/webui/tasks_http_persistence.go b/audit/internal/webui/tasks_http_persistence.go index df1742f..33fae9f 100644 --- a/audit/internal/webui/tasks_http_persistence.go +++ b/audit/internal/webui/tasks_http_persistence.go @@ -3,6 +3,7 @@ package webui import ( "encoding/json" "fmt" + "log/slog" "net/http" "os" "path/filepath" @@ -274,13 +275,17 @@ func (q *taskQueue) persistLocked() { } data, err := json.MarshalIndent(state, "", " ") if err != nil { + slog.Error("task queue: marshal persistent state", "path", q.statePath, "error", err) return } tmp := q.statePath + ".tmp" if err := os.WriteFile(tmp, data, 0644); err != nil { + slog.Error("task queue: write persistent state", "path", tmp, "error", err) return } - _ = os.Rename(tmp, q.statePath) + if err := os.Rename(tmp, q.statePath); err != nil { + slog.Error("task queue: publish persistent state", "source", tmp, "path", q.statePath, "error", err) + } } func taskElapsedSec(t *Task, now time.Time) int { @@ -388,7 +393,9 @@ func (q *taskQueue) ensureTaskArtifactPathsLocked(t *Task) { t.ArtifactsDir = taskArtifactsDir(q.logsDir, t, t.Status) } if t.ArtifactsDir != "" { - _ = os.MkdirAll(t.ArtifactsDir, 0755) + if err := os.MkdirAll(t.ArtifactsDir, 0755); err != nil { + slog.Error("task queue: create artifacts directory", "path", t.ArtifactsDir, "task_id", t.ID, "error", err) + } } ensureTaskReportPaths(t) } @@ -403,10 +410,15 @@ func (q *taskQueue) finalizeTaskArtifactPathsLocked(t *Task) { return } if t.ArtifactsDir != "" && t.ArtifactsDir != dstDir { - if _, err := os.Stat(dstDir); err != nil { - _ = os.Rename(t.ArtifactsDir, dstDir) + if _, err := os.Stat(dstDir); err == nil { + slog.Error("task queue: destination artifacts directory already exists", "source", t.ArtifactsDir, "destination", dstDir, "task_id", t.ID) + } else if !os.IsNotExist(err) { + slog.Error("task queue: inspect destination artifacts directory", "destination", dstDir, "task_id", t.ID, "error", err) + } else if err := os.Rename(t.ArtifactsDir, dstDir); err != nil { + slog.Error("task queue: move artifacts directory", "source", t.ArtifactsDir, "destination", dstDir, "task_id", t.ID, "error", err) + } else { + t.ArtifactsDir = dstDir } - t.ArtifactsDir = dstDir } ensureTaskReportPaths(t) } diff --git a/audit/internal/webui/tasks_test.go b/audit/internal/webui/tasks_test.go index aa1f292..5743bf4 100644 --- a/audit/internal/webui/tasks_test.go +++ b/audit/internal/webui/tasks_test.go @@ -227,6 +227,29 @@ func TestTaskArtifactsDirStartsWithTaskNumber(t *testing.T) { } } +func TestFinalizeTaskArtifactPathsKeepsSourceWhenDestinationExists(t *testing.T) { + logsDir := t.TempDir() + sourceDir := filepath.Join(logsDir, "running-artifacts") + if err := os.Mkdir(sourceDir, 0755); err != nil { + t.Fatal(err) + } + task := &Task{ID: "TASK-007", Name: "NVIDIA Benchmark", Status: TaskDone, ArtifactsDir: sourceDir} + destinationDir := taskArtifactsDir(logsDir, task, TaskDone) + if err := os.Mkdir(destinationDir, 0755); err != nil { + t.Fatal(err) + } + + q := &taskQueue{logsDir: logsDir} + q.finalizeTaskArtifactPathsLocked(task) + + if task.ArtifactsDir != sourceDir { + t.Fatalf("artifacts dir = %q, want retained source %q after destination collision", task.ArtifactsDir, sourceDir) + } + if task.ReportJSONPath != filepath.Join(sourceDir, "report.json") { + t.Fatalf("report path = %q, want source directory", task.ReportJSONPath) + } +} + func TestHandleAPITasksStreamReplaysPersistedLogWithoutLiveJob(t *testing.T) { dir := t.TempDir() logPath := filepath.Join(dir, "task.log") diff --git a/bible-local/architecture/api-surface.md b/bible-local/architecture/api-surface.md index 74c8083..5e701e5 100644 --- a/bible-local/architecture/api-surface.md +++ b/bible-local/architecture/api-surface.md @@ -125,7 +125,7 @@ All SAT run endpoints enqueue an async task. Response: `{"task_id": "..."}`. | Method | Path | Description | |--------|----------------------------|----------------------------------------------------| -| GET | `/api/gpu/presence` | `{"nvidia": bool, "amd": bool}` | +| GET | `/api/gpu/presence` | `{"nvidia", "amd", "nvidia_initializing", "amd_initializing": bool}`. `*_initializing` = PCI GPU present but driver not up yet (distinct from absent). Backed by `app.DetectGPUPresence`. | | GET | `/api/gpu/nvidia` | List NVIDIA GPUs from nvidia-smi | | GET | `/api/gpu/nvidia-status` | Per-GPU status (ECC, power, throttle) | | POST | `/api/gpu/nvidia-reset` | GPU reset by index | @@ -139,6 +139,10 @@ All SAT run endpoints enqueue an async task. Response: `{"task_id": "..."}`. |--------|------------------------------|---------------------------------------------------| | GET | `/api/system/ram-status` | toram boot state and ISO copy status | | POST | `/api/system/install-to-ram` | Copy ISO to RAM (background task) | +| GET | `/api/system/time` | Current host epoch, local wall-clock text, and timezone | +| POST | `/api/system/time-sync` | Set host time and timezone from the browser | +| POST | `/api/system/reboot` | Reboot the host | +| POST | `/api/system/shutdown` | Power off the host | | GET | `/api/install/disks` | List block devices suitable for disk installation | | POST | `/api/install/run` | Install bee to disk (background task) | diff --git a/bible-local/architecture/runtime-flows.md b/bible-local/architecture/runtime-flows.md index e2385db..08bd534 100644 --- a/bible-local/architecture/runtime-flows.md +++ b/bible-local/architecture/runtime-flows.md @@ -263,6 +263,11 @@ bee-blackbox.service (separate process from bee-web/bee-audit) a time-derived staging path: two builds in the same wall-clock second (two operators, or an on-demand build racing the blackbox worker) then share one tree and one's deferred `os.RemoveAll` truncates the other's archive. +- The resulting archive is also unique (`os.CreateTemp` with the readable + support-bundle name as its prefix) and remains a hidden `.partial` from + retention cleanup until an atomic rename publishes the closed `.tar.gz`. + A timestamp-only output path lets same-second writers share one inode and + can corrupt both gzip streams. ## NVIDIA SAT Web UI flow @@ -306,3 +311,15 @@ Web UI: "Run All" button -> POST /api/sat/run-all - `RunNvidiaBandwidthPack` runs one all-GPU `nvbandwidth` pass in Validate; the per-NUMA-node matrix is Stress-tier only (`fullMatrix` arg). See `decisions/2026-08-31-nvbandwidth-validate-single-deep-matrix.md`. +- `nvidia-config` validates the active-link count independently at both ends of + every `NVx` pair reported by `nvidia-smi topo -m`; aggregate counts must not + be compared with a per-endpoint `NVx` value. It also persists matching + NVIDIA degraded-mode kernel lines in `06-dmesg-nvlink-degraded.log`. +- Generic SAT jobs may attach an output validator. A validator runs only after + exit status 0 and returns `OK`, `FAILED`, or `UNSUPPORTED`; its detail is + persisted in both the job log and `_detail` in `summary.txt`. +- `nccl-tests` explicitly validates the pinned `all_reduce_perf` correctness + summary and both `#wrong` columns, and cross-checks selected NVLink endpoints + against the captured topology/status. `Avg bus bandwidth` is retained as a + diagnostic value only: no model-independent bandwidth threshold is valid. + NCCL `P2P` alone is not proof of NVLink because P2P can also use PCIe. diff --git a/bible-local/decisions/2026-08-31-support-bundle-private-staging-dir.md b/bible-local/decisions/2026-08-31-support-bundle-private-staging-dir.md new file mode 100644 index 0000000..f5c6c20 --- /dev/null +++ b/bible-local/decisions/2026-08-31-support-bundle-private-staging-dir.md @@ -0,0 +1,52 @@ +# Support bundle: stage under a private os.MkdirTemp parent, not a time-derived path + +**Date:** 2026-08-31 +**Status:** active + +## Context + +`BuildSupportBundle` staged into +`os.TempDir()/bee-support-stage--` and did +`defer os.RemoveAll(stageRoot)`. The name has one-second resolution and a +fixed hostname, so two builds started in the same wall-clock second resolve +to the same directory: one operator double-clicking "Download Support +Bundle", two operators, or an on-demand build racing the blackbox worker. + +When that happened, the first build to finish ran `os.RemoveAll` on the +shared tree while the second was still populating it or archiving it. The +second build's `createSupportTarGz` then walked a half-deleted tree and +produced a truncated archive - with no error. It surfaced first as a flaky +test (`TestBuildSupportBundleIncludesExportDirContents` vs the webui +endpoint test, run in parallel by `go test ./...`), but the same race is +reachable in production. + +## Decision + +`BuildSupportBundle` creates a private parent with +`os.MkdirTemp(os.TempDir(), "bee-support-build-")` and stages inside it. The +leaf directory keeps the meaningful `bee-support-stage--` name +(it becomes the archive's top-level directory). `defer os.RemoveAll` targets +the unique parent. The output archive is created with `os.CreateTemp`, retaining +the human-readable prefix while guaranteeing a unique file for each concurrent +build. It is written under a non-published `.partial` suffix and atomically +renamed to `.tar.gz` only after tar, gzip, and file closes succeed, so retention +cleanup can only see completed archives. + +Tests that exercise the real `BuildSupportBundle` set +`t.Setenv("TMPDIR", t.TempDir())` so each test's `os.TempDir()` - staging +parent, archive path, and the `cleanupOldSupportBundles` scan - is isolated +from parallel packages. + +## Consequences + +- Concurrent support-bundle builds are independent; neither can truncate the + other. +- DO NOT reintroduce a time-derived or otherwise non-unique staging path. + See `architecture/runtime-flows.md` (Blackbox sync flow invariants). +- The archive file itself still lands in `os.TempDir()` under a + human-readable `SupportBundleBaseName` prefix (callers `defer os.Remove` it), + with a random suffix preventing same-second writers from sharing one inode. +- Tar, gzip, source-file, and destination-file close failures are propagated; + an incomplete archive is removed instead of being returned as successful. +- DMI-derived version/model/serial components are sanitized before becoming + path components. diff --git a/bible-local/decisions/README.md b/bible-local/decisions/README.md index ab599a4..0fda803 100644 --- a/bible-local/decisions/README.md +++ b/bible-local/decisions/README.md @@ -15,3 +15,4 @@ One file per decision, named `YYYY-MM-DD-short-topic.md`. | 2026-08-31 | nvbandwidth: single all-GPU pass in Validate, per-socket matrix only in deep tier | active | | 2026-08-31 | bee-nvidia.service: never blocking `systemctl restart` on units ordered After= itself | active | | 2026-08-31 | "Run All" SAT planning happens on the backend, not the browser | active | +| 2026-08-31 | Support bundle uses private staging and unique atomic output | active | diff --git a/internal/chart b/internal/chart index 2a15bc8..be68069 160000 --- a/internal/chart +++ b/internal/chart @@ -1 +1 @@ -Subproject commit 2a15bc87f138451f27218d0a02d6a36a3e4afdf4 +Subproject commit be68069a513daf883c980889c9b4e52ca5083664 diff --git a/iso/builder/build.sh b/iso/builder/build.sh index f71f0bd..f28dd33 100755 --- a/iso/builder/build.sh +++ b/iso/builder/build.sh @@ -77,6 +77,7 @@ export GOCACHE GOMODCACHE # Keep the entrypoint focused on build orchestration. These libraries only # define helpers; stateful logging is loaded after its output paths are known. . "${BUILDER_DIR}/lib/iso-validation.sh" +. "${BUILDER_DIR}/lib/template.sh" . "${BUILDER_DIR}/lib/bootloader.sh" . "${BUILDER_DIR}/lib/fast-path.sh" @@ -158,11 +159,19 @@ sync_builder_workdir() { PROJECT_VERSION_EFFECTIVE="$(resolve_project_version)" +if ! validate_project_version "${PROJECT_VERSION_EFFECTIVE}"; then + echo "ERROR: project version must start and end with an ASCII letter or digit and contain only letters, digits, dots, and hyphens: ${PROJECT_VERSION_EFFECTIVE}" >&2 + exit 1 +fi SQUASHFS_FILENAME="filesystem-v${PROJECT_VERSION_EFFECTIVE}.squashfs" ISO_BASENAME="easy-bee-${BUILD_VARIANT}-v${PROJECT_VERSION_EFFECTIVE}-amd64" # Hostname baked into the live image: same product+version scheme as # ISO_BASENAME/SQUASHFS_FILENAME, but hostname labels don't allow dots. BEE_HOSTNAME="easy-bee-${BUILD_VARIANT}-v$(printf '%s' "${PROJECT_VERSION_EFFECTIVE}" | tr '.' '-')" +if [ "${#BEE_HOSTNAME}" -gt 63 ]; then + echo "ERROR: derived hostname exceeds the 63-byte DNS label limit: ${BEE_HOSTNAME}" >&2 + exit 1 +fi # Versioned output directory: dist/easy-bee-v4.1/ - all final artifacts live here. OUT_DIR="${DIST_DIR}/release/easy-bee-v${PROJECT_VERSION_EFFECTIVE}" ISO_VERSION_LABEL_TOKEN="$(printf '%s' "${PROJECT_VERSION_EFFECTIVE}" | tr '[:lower:].-' '[:upper:]__')" @@ -538,9 +547,7 @@ fi # Patch motd with build info BEE_BUILD_INFO="${BUILD_DATE} git:${GIT_COMMIT} debian:${DEBIAN_VERSION} ${GPU_BUILD_INFO}" if [ -f "${OVERLAY_STAGE_DIR}/etc/motd" ]; then - sed "s/%%BUILD_INFO%%/${BEE_BUILD_INFO}/" "${OVERLAY_STAGE_DIR}/etc/motd" \ - > "${OVERLAY_STAGE_DIR}/etc/motd.patched" - mv "${OVERLAY_STAGE_DIR}/etc/motd.patched" "${OVERLAY_STAGE_DIR}/etc/motd" + replace_literal_in_file "${OVERLAY_STAGE_DIR}/etc/motd" "%%BUILD_INFO%%" "${BEE_BUILD_INFO}" fi # --- copy variant-specific package list, remove all other variant lists --- @@ -568,26 +575,24 @@ fi # --- substitute version placeholders in package list and archive --- if [ "$BEE_GPU_VENDOR" = "nvidia" ]; then - sed -i \ - -e "s/%%NVIDIA_FABRICMANAGER_VERSION%%/${NVIDIA_FABRICMANAGER_VERSION}/g" \ - -e "s/%%DCGM_VERSION%%/${DCGM_VERSION}/g" \ - "${BUILD_WORK_DIR}/config/package-lists/bee-gpu.list.chroot" + replace_literal_in_file "${BUILD_WORK_DIR}/config/package-lists/bee-gpu.list.chroot" \ + "%%NVIDIA_FABRICMANAGER_VERSION%%" "${NVIDIA_FABRICMANAGER_VERSION}" + replace_literal_in_file "${BUILD_WORK_DIR}/config/package-lists/bee-gpu.list.chroot" \ + "%%DCGM_VERSION%%" "${DCGM_VERSION}" elif [ "$BEE_GPU_VENDOR" = "amd" ]; then - sed -i \ - -e "s/%%ROCM_VERSION%%/${ROCM_VERSION}/g" \ - -e "s/%%ROCM_SMI_VERSION%%/${ROCM_SMI_VERSION}/g" \ - -e "s/%%ROCM_BANDWIDTH_TEST_VERSION%%/${ROCM_BANDWIDTH_TEST_VERSION}/g" \ - -e "s/%%ROCM_VALIDATION_SUITE_VERSION%%/${ROCM_VALIDATION_SUITE_VERSION}/g" \ - -e "s/%%ROCBLAS_VERSION%%/${ROCBLAS_VERSION}/g" \ - -e "s/%%ROCRAND_VERSION%%/${ROCRAND_VERSION}/g" \ - -e "s/%%HIP_RUNTIME_AMD_VERSION%%/${HIP_RUNTIME_AMD_VERSION}/g" \ - -e "s/%%HIPBLASLT_VERSION%%/${HIPBLASLT_VERSION}/g" \ - -e "s/%%COMGR_VERSION%%/${COMGR_VERSION}/g" \ - "${BUILD_WORK_DIR}/config/package-lists/bee-gpu.list.chroot" + gpu_package_list="${BUILD_WORK_DIR}/config/package-lists/bee-gpu.list.chroot" + replace_literal_in_file "$gpu_package_list" "%%ROCM_VERSION%%" "${ROCM_VERSION}" + replace_literal_in_file "$gpu_package_list" "%%ROCM_SMI_VERSION%%" "${ROCM_SMI_VERSION}" + replace_literal_in_file "$gpu_package_list" "%%ROCM_BANDWIDTH_TEST_VERSION%%" "${ROCM_BANDWIDTH_TEST_VERSION}" + replace_literal_in_file "$gpu_package_list" "%%ROCM_VALIDATION_SUITE_VERSION%%" "${ROCM_VALIDATION_SUITE_VERSION}" + replace_literal_in_file "$gpu_package_list" "%%ROCBLAS_VERSION%%" "${ROCBLAS_VERSION}" + replace_literal_in_file "$gpu_package_list" "%%ROCRAND_VERSION%%" "${ROCRAND_VERSION}" + replace_literal_in_file "$gpu_package_list" "%%HIP_RUNTIME_AMD_VERSION%%" "${HIP_RUNTIME_AMD_VERSION}" + replace_literal_in_file "$gpu_package_list" "%%HIPBLASLT_VERSION%%" "${HIPBLASLT_VERSION}" + replace_literal_in_file "$gpu_package_list" "%%COMGR_VERSION%%" "${COMGR_VERSION}" if [ -f "${BUILD_WORK_DIR}/config/archives/rocm.list.chroot" ]; then - sed -i \ - -e "s/%%ROCM_VERSION%%/${ROCM_VERSION}/g" \ - "${BUILD_WORK_DIR}/config/archives/rocm.list.chroot" + replace_literal_in_file "${BUILD_WORK_DIR}/config/archives/rocm.list.chroot" \ + "%%ROCM_VERSION%%" "${ROCM_VERSION}" fi fi diff --git a/iso/builder/lib/bootloader.sh b/iso/builder/lib/bootloader.sh index dff211b..1af725d 100755 --- a/iso/builder/lib/bootloader.sh +++ b/iso/builder/lib/bootloader.sh @@ -85,6 +85,14 @@ render_bootloader_template() { initrd_placeholder="$7" initrd="$8" + for required_placeholder in \ + "@VERSION@" \ + "$kernel_placeholder" \ + "@APPEND_LIVE@" \ + "$initrd_placeholder"; do + require_template_placeholder "$template" "$required_placeholder" || return 1 + done + BEE_RENDER_VERSION="$version" \ BEE_RENDER_KERNEL_PLACEHOLDER="$kernel_placeholder" \ BEE_RENDER_KERNEL="$kernel" \ diff --git a/iso/builder/lib/iso-validation.sh b/iso/builder/lib/iso-validation.sh index ca4bc03..31f76bd 100755 --- a/iso/builder/lib/iso-validation.sh +++ b/iso/builder/lib/iso-validation.sh @@ -359,9 +359,12 @@ validate_live_cmdline_params() { bad = 1 } - $1 == command && has("boot=live") { + $1 == command { live_entries++ + if (!has("boot=live")) { + reject("missing boot=live") + } if (!has("udev.children_max=1")) { reject("missing udev.children_max=1") } @@ -432,7 +435,7 @@ validate_iso_live_boot_entries() { exit 1 } - if grep -q '@APPEND_LIVE@\|@KERNEL_LIVE@\|@INITRD_LIVE@' "$grub_cfg" "$isolinux_cfg"; then + if grep -Eq '@[A-Z][A-Z_]*@' "$grub_cfg" "$isolinux_cfg"; then echo "ERROR: unresolved live-build placeholders remain in ISO bootloader config" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 diff --git a/iso/builder/lib/template.sh b/iso/builder/lib/template.sh new file mode 100644 index 0000000..f2f4726 --- /dev/null +++ b/iso/builder/lib/template.sh @@ -0,0 +1,55 @@ +require_template_placeholder() { + file="$1" + placeholder="$2" + if [ -z "$placeholder" ]; then + echo "ERROR: template placeholder must not be empty" >&2 + return 1 + fi + if ! BEE_TEMPLATE_PLACEHOLDER="$placeholder" awk ' + index($0, ENVIRON["BEE_TEMPLATE_PLACEHOLDER"]) { found=1 } + END { exit found ? 0 : 1 } + ' "$file"; then + echo "ERROR: required placeholder $placeholder is missing from $file" >&2 + return 1 + fi +} + +replace_literal_in_file() { + file="$1" + placeholder="$2" + replacement="$3" + require_template_placeholder "$file" "$placeholder" || return 1 + tmp="$(mktemp "${TMPDIR:-/tmp}/bee-template.XXXXXX")" + + if BEE_TEMPLATE_PLACEHOLDER="$placeholder" \ + BEE_TEMPLATE_REPLACEMENT="$replacement" \ + awk ' + function replace_literal(text, needle, value, at) { + while ((at = index(text, needle)) != 0) { + text = substr(text, 1, at - 1) value substr(text, at + length(needle)) + } + return text + } + { + print replace_literal($0, + ENVIRON["BEE_TEMPLATE_PLACEHOLDER"], + ENVIRON["BEE_TEMPLATE_REPLACEMENT"]) + } + ' "$file" > "$tmp" && cat "$tmp" > "$file"; then + status=0 + else + status=$? + fi + rm -f "$tmp" + return "$status" +} + +validate_project_version() { + version="$1" + case "$version" in + ""|[!A-Za-z0-9]*|*[!A-Za-z0-9]|*[!A-Za-z0-9.-]*) + return 1 + ;; + esac + return 0 +} diff --git a/iso/builder/test-build-libs.sh b/iso/builder/test-build-libs.sh index cbfd57a..5441373 100755 --- a/iso/builder/test-build-libs.sh +++ b/iso/builder/test-build-libs.sh @@ -11,8 +11,37 @@ LOG_DIR="$TEST_ROOT/log" mkdir -p "$LOG_DIR" . "$BUILDER_DIR/lib/iso-validation.sh" +. "$BUILDER_DIR/lib/template.sh" . "$BUILDER_DIR/lib/bootloader.sh" +literal_file="$TEST_ROOT/literal-file" +printf '%s\n' 'before %%VALUE%% after %%VALUE%%' > "$literal_file" +replace_literal_in_file "$literal_file" '%%VALUE%%' 'a&b#c\\d' +if [ "$(cat "$literal_file")" != 'before a&b#c\\d after a&b#c\\d' ]; then + echo "ERROR: generic literal replacement changed replacement characters" >&2 + exit 1 +fi +if replace_literal_in_file "$literal_file" '' value >/dev/null 2>&1; then + echo "ERROR: generic literal replacement accepted an empty placeholder" >&2 + exit 1 +fi +if replace_literal_in_file "$literal_file" '%%MISSING%%' value >/dev/null 2>&1; then + echo "ERROR: generic literal replacement accepted a missing placeholder" >&2 + exit 1 +fi +for valid_version in 13 13.0 v13.0-2-gabcdef0-dirty; do + if ! validate_project_version "$valid_version"; then + echo "ERROR: rejected valid project version: $valid_version" >&2 + exit 1 + fi +done +for invalid_version in '' '.13' '13.' '13/0' '13_0' '13 0' '13&0'; do + if validate_project_version "$invalid_version"; then + echo "ERROR: accepted unsafe project version: $invalid_version" >&2 + exit 1 + fi +done + literal_template="$TEST_ROOT/literal-template" literal_output="$TEST_ROOT/literal-output" printf '%s\n' '@VERSION@ @KERNEL@ @APPEND_LIVE@ @INITRD@' > "$literal_template" @@ -23,6 +52,11 @@ if [ "$(cat "$literal_output")" != "$literal_expected" ]; then echo "ERROR: bootloader renderer changed literal replacement characters" >&2 exit 1 fi +if render_bootloader_template "$literal_template" "$literal_output" \ + '13.0-test' '@MISSING_KERNEL@' '/live/vmlinuz' 'boot=live' '@INITRD@' '/live/initrd.img' >/dev/null 2>&1; then + echo "ERROR: bootloader renderer accepted a missing required placeholder" >&2 + exit 1 +fi rendered_grub="$TEST_ROOT/grub.cfg" rendered_isolinux="$TEST_ROOT/live.cfg" @@ -49,6 +83,14 @@ if validate_live_cmdline_params "$missing_serialization" linux GRUB "$BEE_ISO_VO exit 1 fi +missing_boot_live="$TEST_ROOT/missing-boot-live.cfg" +awk 'BEGIN { removed=0 } /^ linux / && !removed { sub(/ boot=live/, ""); removed=1 } { print }' \ + "$rendered_grub" > "$missing_boot_live" +if validate_live_cmdline_params "$missing_boot_live" linux GRUB "$BEE_ISO_VOLUME" >/dev/null 2>&1; then + echo "ERROR: validator ignored a live entry without boot=live" >&2 + exit 1 +fi + CACHE_ROOT="$TEST_ROOT/cache" BUILD_VARIANT="test" BUILD_WORK_DIR="$TEST_ROOT/work"