reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

refactor: harden diagnostics and consolidate runtime code

Browse Source
This commit is contained in:
Mikhail Chusavitin
2026-09-01 13:01:28 +03:00
parent ac4bc0b2b7
commit 0a6ca8ba0f
49 changed files with 1441 additions and 837 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/app/app.go
-8
View File
@@ -129,14 +129,6 @@ func (a *App) TPMPresent() bool {
return a.sat.TPMPresent()
}
func (a *App) IsLiveMediaInRAM() bool {
return a.installer.IsLiveMediaInRAM()
}
func (a *App) LiveBootSource() platform.LiveBootSource {
return a.installer.LiveBootSource()
}
func (a *App) LiveMediaRAMState() platform.LiveMediaRAMState {
return a.installer.LiveMediaRAMState()
}
audit/internal/app/app_install.go
-19
View File
@@ -1,7 +1,6 @@
package app
import (
"context"
"fmt"
"os"
"path/filepath"
@@ -14,20 +13,6 @@ func (a *App) ListRemovableTargets() ([]platform.RemovableTarget, error) {
return a.exports.ListRemovableTargets()
}
// ListScenarioFilesOnRemovableMedia lists scenarios/*.json found on any
// mounted removable target (e.g. the blackbox USB stick) — see
// platform.System.ListScenarioFilesOnRemovableMedia.
func (a *App) ListScenarioFilesOnRemovableMedia() ([]platform.ScenarioFileOnRemovableMedia, error) {
return a.exports.ListScenarioFilesOnRemovableMedia()
}
// ReadScenarioFromRemovableMedia reads scenarios/<name>.json from whichever
// mounted removable target has it — see
// platform.System.ReadScenarioFromRemovableMedia.
func (a *App) ReadScenarioFromRemovableMedia(name string) ([]byte, error) {
return a.exports.ReadScenarioFromRemovableMedia(name)
}
// ListAvailableScenarios lists every scenario runnable via ReadScenario:
// shipped with the image plus anything found on removable media — see
// platform.System.ListAvailableScenarios.
@@ -86,7 +71,3 @@ func (a *App) ExportSupportBundleResult(target platform.RemovableTarget) (Action
func (a *App) ListInstallDisks() ([]platform.InstallDisk, error) {
return a.installer.ListInstallDisks()
}
func (a *App) InstallToDisk(ctx context.Context, device string, logFile string) error {
return a.installer.InstallToDisk(ctx, device, logFile)
}
audit/internal/app/app_network.go
-12
View File
@@ -15,28 +15,16 @@ func (a *App) DefaultRoute() string {
return a.network.DefaultRoute()
}
func (a *App) DHCPOne(iface string) (string, error) {
return a.network.DHCPOne(iface)
}
func (a *App) DHCPOneResult(iface string) (ActionResult, error) {
body, err := a.network.DHCPOne(iface)
return ActionResult{Title: "DHCP: " + iface, Body: bodyOr(body, "DHCP completed.")}, err
}
func (a *App) DHCPAll() (string, error) {
return a.network.DHCPAll()
}
func (a *App) DHCPAllResult() (ActionResult, error) {
body, err := a.network.DHCPAll()
return ActionResult{Title: "DHCP: all interfaces", Body: bodyOr(body, "DHCP completed.")}, err
}
func (a *App) SetStaticIPv4(cfg platform.StaticIPv4Config) (string, error) {
return a.network.SetStaticIPv4(cfg)
}
func (a *App) SetInterfaceState(iface string, up bool) error {
return a.network.SetInterfaceState(iface, up)
}
audit/internal/app/app_packs.go
+8 -63
View File
@@ -110,36 +110,28 @@ func (a *App) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir s
return a.sat.RunNvidiaTargetedStressValidatePack(ctx, baseDir, durationSec, gpuIndices, logFunc)
}
func (a *App) RunNvidiaStressPack(baseDir string, opts platform.NvidiaStressOptions, logFunc func(string)) (string, error) {
return a.RunNvidiaStressPackCtx(context.Background(), baseDir, opts, logFunc)
}
func (a *App) RunNvidiaBenchmark(baseDir string, opts platform.NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
return a.RunNvidiaBenchmarkCtx(context.Background(), baseDir, opts, logFunc)
}
func (a *App) RunNvidiaBenchmarkCtx(ctx context.Context, baseDir string, opts platform.NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultBeeBenchPerfDir
}
resolved, err := a.ensureBenchmarkPowerAutotune(ctx, baseDir, opts, "performance", logFunc)
if err != nil {
return "", err
}
opts.ServerPowerSource = resolved.SelectedSource
return a.sat.RunNvidiaBenchmark(ctx, baseDir, opts, logFunc)
return a.runNvidiaBenchmarkKind(ctx, baseDir, DefaultBeeBenchPerfDir, opts, "performance", logFunc, a.sat.RunNvidiaBenchmark)
}
func (a *App) RunNvidiaPowerBenchCtx(ctx context.Context, baseDir string, opts platform.NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
return a.runNvidiaBenchmarkKind(ctx, baseDir, DefaultBeeBenchPowerDir, opts, "power-fit", logFunc, a.sat.RunNvidiaPowerBench)
}
func (a *App) runNvidiaBenchmarkKind(ctx context.Context, baseDir, defaultBaseDir string, opts platform.NvidiaBenchmarkOptions, benchmarkKind string, logFunc func(string), run func(context.Context, string, platform.NvidiaBenchmarkOptions, func(string)) (string, error)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultBeeBenchPowerDir
baseDir = defaultBaseDir
}
resolved, err := a.ensureBenchmarkPowerAutotune(ctx, baseDir, opts, "power-fit", logFunc)
resolved, err := a.ensureBenchmarkPowerAutotune(ctx, baseDir, opts, benchmarkKind, logFunc)
if err != nil {
return "", err
}
opts.ServerPowerSource = resolved.SelectedSource
return a.sat.RunNvidiaPowerBench(ctx, baseDir, opts, logFunc)
return run(ctx, baseDir, opts, logFunc)
}
func (a *App) RunNvidiaPowerSourceAutotuneCtx(ctx context.Context, baseDir string, opts platform.NvidiaBenchmarkOptions, benchmarkKind string, logFunc func(string)) (string, error) {
@@ -226,10 +218,6 @@ func (a *App) RunMemoryAcceptancePackResult(baseDir string) (ActionResult, error
return ActionResult{Title: "Memory SAT", Body: satResultBody(path)}, err
}
func (a *App) RunCPUAcceptancePack(baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.RunCPUAcceptancePackCtx(context.Background(), baseDir, durationSec, logFunc)
}
func (a *App) RunCPUAcceptancePackCtx(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
@@ -267,10 +255,6 @@ func (a *App) RunNvidiaConfigCheckPackCtx(ctx context.Context, baseDir string, l
return a.sat.RunNvidiaConfigCheckPack(ctx, baseDir, logFunc)
}
func (a *App) RunNvidiaConfigCheckPack(baseDir string, logFunc func(string)) (string, error) {
return a.RunNvidiaConfigCheckPackCtx(context.Background(), baseDir, logFunc)
}
func (a *App) RunPCIeLinkCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
@@ -278,10 +262,6 @@ func (a *App) RunPCIeLinkCheckPackCtx(ctx context.Context, baseDir string, logFu
return a.sat.RunPCIeLinkCheckPack(ctx, baseDir, logFunc)
}
func (a *App) RunPCIeLinkCheckPack(baseDir string, logFunc func(string)) (string, error) {
return a.RunPCIeLinkCheckPackCtx(context.Background(), baseDir, logFunc)
}
func (a *App) RunNvidiaPCIeBandwidthPackCtx(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
@@ -289,22 +269,6 @@ func (a *App) RunNvidiaPCIeBandwidthPackCtx(ctx context.Context, baseDir string,
return a.sat.RunNvidiaPCIeBandwidthPack(ctx, baseDir, gpuIndices, logFunc)
}
func (a *App) RunNvidiaPCIeBandwidthPack(baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
return a.RunNvidiaPCIeBandwidthPackCtx(context.Background(), baseDir, gpuIndices, logFunc)
}
func (a *App) DetectGPUVendor() string {
return a.sat.DetectGPUVendor()
}
func (a *App) ListAMDGPUs() ([]platform.AMDGPUInfo, error) {
return a.sat.ListAMDGPUs()
}
func (a *App) RunAMDAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
return a.RunAMDAcceptancePackCtx(context.Background(), baseDir, logFunc)
}
func (a *App) RunAMDAcceptancePackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
@@ -326,18 +290,6 @@ func (a *App) RunAMDMemBandwidthPackCtx(ctx context.Context, baseDir string, log
return a.sat.RunAMDMemBandwidthPack(ctx, baseDir, logFunc)
}
func (a *App) RunMemoryStressPack(baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.RunMemoryStressPackCtx(context.Background(), baseDir, durationSec, logFunc)
}
func (a *App) RunSATStressPack(baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.RunSATStressPackCtx(context.Background(), baseDir, durationSec, logFunc)
}
func (a *App) RunAMDStressPack(baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.RunAMDStressPackCtx(context.Background(), baseDir, durationSec, logFunc)
}
func (a *App) RunMemoryStressPackCtx(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.sat.RunMemoryStressPack(ctx, baseDir, durationSec, logFunc)
}
@@ -353,13 +305,6 @@ func (a *App) RunAMDStressPackCtx(ctx context.Context, baseDir string, durationS
return a.sat.RunAMDStressPack(ctx, baseDir, durationSec, logFunc)
}
func (a *App) RunFanStressTest(ctx context.Context, baseDir string, opts platform.FanStressOptions) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
}
return a.sat.RunFanStressTest(ctx, baseDir, opts)
}
func (a *App) RunPlatformStress(ctx context.Context, baseDir string, opts platform.PlatformStressOptions, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
audit/internal/app/app_services.go
-8
View File
@@ -24,19 +24,11 @@ func (a *App) ServiceStatusResult(name string) (ActionResult, error) {
return ActionResult{Title: "service status: " + name, Body: bodyOr(body, "No status output.")}, err
}
func (a *App) ServiceDo(name string, action platform.ServiceAction) (string, error) {
return a.services.ServiceDo(name, action)
}
func (a *App) ServiceActionResult(name string, action platform.ServiceAction) (ActionResult, error) {
body, err := a.services.ServiceDo(name, action)
return ActionResult{Title: "service " + string(action) + ": " + name, Body: bodyOr(body, "Action completed.")}, err
}
func (a *App) TailFile(path string, lines int) string {
return a.tools.TailFile(path, lines)
}
func (a *App) CheckTools(names []string) []platform.ToolStatus {
return a.tools.CheckTools(names)
}
audit/internal/app/blackbox_archive.go
+6 -6
View File
@@ -3,6 +3,7 @@ package app
import (
"archive/zip"
"bufio"
"errors"
"io"
"io/fs"
"os"
@@ -61,9 +62,8 @@ func buildZipArchive(root, destPath string) error {
if err != nil {
return err
}
defer src.Close()
_, err = io.Copy(w, src)
return err
_, copyErr := io.Copy(w, src)
return errors.Join(copyErr, src.Close())
})
if walkErr != nil {
_ = zw.Close()
@@ -136,7 +136,7 @@ func isEOFLike(err error) bool {
// (first run, external tampering, a previous crash mid-write), it falls back
// to writing the whole archive rather than risk corrupting it with a wrong
// truncate point.
func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) error {
func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) (retErr error) {
newInfo, err := os.Stat(newLocalZipPath)
if err != nil {
return err
@@ -159,7 +159,7 @@ func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) error
if err != nil {
return err
}
defer target.Close()
defer func() { retErr = errors.Join(retErr, target.Close()) }()
if err := target.Truncate(prefixLen); err != nil {
return err
@@ -172,7 +172,7 @@ func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) error
if err != nil {
return err
}
defer src.Close()
defer func() { retErr = errors.Join(retErr, src.Close()) }()
if _, err := src.Seek(prefixLen, io.SeekStart); err != nil {
return err
}
audit/internal/app/component_status_db.go
+11 -2
View File
@@ -454,10 +454,19 @@ func satFailureDetailFromKV(kv map[string]string) string {
continue
}
job := strings.TrimSuffix(k, "_status")
detail := strings.TrimSpace(kv[job+"_detail"])
if rc, ok := kv[job+"_rc"]; ok && strings.TrimSpace(rc) != "" {
failed = append(failed, fmt.Sprintf("%s=%s (rc=%s)", job, v, rc))
entry := fmt.Sprintf("%s=%s (rc=%s)", job, v, rc)
if detail != "" {
entry += ": " + detail
}
failed = append(failed, entry)
} else {
failed = append(failed, fmt.Sprintf("%s=%s", job, v))
entry := fmt.Sprintf("%s=%s", job, v)
if detail != "" {
entry += ": " + detail
}
failed = append(failed, entry)
}
}
if len(failed) == 0 {
audit/internal/app/component_status_db_test.go
+31
View File
@@ -3,6 +3,7 @@ package app
import (
"os"
"path/filepath"
"strings"
"testing"
"bee/audit/internal/schema"
@@ -145,6 +146,21 @@ func TestSATFailureDetailFallsBackToFailedSubJobs(t *testing.T) {
}
}
func TestSATFailureDetailIncludesValidatorDetail(t *testing.T) {
runDir := t.TempDir()
summary := "2-all-reduce-perf_rc=0\n" +
"2-all-reduce-perf_status=FAILED\n" +
"2-all-reduce-perf_detail=NVLink state does not match selected topology: GPU0<->GPU1\n" +
"overall_status=FAILED\n"
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
t.Fatal(err)
}
got := SATFailureDetail(runDir)
if !strings.Contains(got, "NVLink state does not match selected topology") {
t.Fatalf("SATFailureDetail() = %q, want validator detail", got)
}
}
// TestSATFailureDetailEmptyWhenNoReasonFound guards the ultimate fallback:
// when summary.txt carries no identifiable per-job or warnings detail (e.g.
// unreadable or from an older binary version), callers must get "" so they
@@ -233,6 +249,21 @@ func TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets(t *tes
}
}
func TestApplySATResultToDBNvidiaInterconnectFailureReachesGPUComponent(t *testing.T) {
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
if err != nil {
t.Fatal(err)
}
ApplySATResultToDB(db, "nvidia-interconnect", writeSATSummary(t, "FAILED"))
rec, ok := db.Get("pcie:gpu:nvidia")
if !ok {
t.Fatal("nvidia-interconnect failure wrote no pcie:gpu:nvidia record")
}
if rec.Status != "Warning" {
t.Fatalf("status=%q want Warning", rec.Status)
}
}
// TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the
// hardware-ingest-contract.md rule that status_history is a transition log
// ("История переходов статусов"), not a per-poll journal. A component
audit/internal/app/support_bundle.go
+50 -34
View File
@@ -6,6 +6,7 @@ import (
"compress/gzip"
"context"
_ "embed"
"errors"
"fmt"
"io"
"os"
@@ -505,21 +506,16 @@ func BuildSupportBundle(exportDir string) (string, error) {
return "", err
}
archiveName := SupportBundleBaseName(now) + ".tar.gz"
archivePath := filepath.Join(os.TempDir(), archiveName)
if err := createSupportTarGz(archivePath, stageRoot); err != nil {
return "", err
}
return archivePath, nil
return createSupportTarGz(os.TempDir(), SupportBundleBaseName(now), stageRoot)
}
func SupportBundleBaseName(at time.Time) string {
at = at.UTC()
date := at.Format("2006-01-02")
tod := at.Format("150405")
ver := bundleVersion()
model := serverModelForBundle()
sn := serverSerialForBundle()
ver := sanitizeFilename(bundleVersion())
model := sanitizeFilename(serverModelForBundle())
sn := sanitizeFilename(serverSerialForBundle())
return fmt.Sprintf("%s (BEE-SP v%s) %s %s %s", date, ver, model, sn, tod)
}
@@ -617,17 +613,15 @@ func copyOptionalFile(src, dst string) error {
if err != nil {
return err
}
defer in.Close()
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return err
return errors.Join(err, in.Close())
}
out, err := os.Create(dst)
if err != nil {
return err
return errors.Join(err, in.Close())
}
defer out.Close()
_, err = io.Copy(out, in)
return err
_, copyErr := io.Copy(out, in)
return errors.Join(copyErr, in.Close(), out.Close())
}
func writeManifest(dst, exportDir, stageRoot string) error {
@@ -823,16 +817,12 @@ func copyPath(src, dst string) error {
}
return err
}
defer in.Close()
out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode().Perm())
if err != nil {
return err
return errors.Join(err, in.Close())
}
defer out.Close()
_, err = io.Copy(out, in)
return err
_, copyErr := io.Copy(out, in)
return errors.Join(copyErr, in.Close(), out.Close())
}
func copyPathFiltered(rootSrc, src, dst string, keep func(rel string, info os.FileInfo) bool) error {
@@ -874,21 +864,35 @@ func copyPathFiltered(rootSrc, src, dst string, keep func(rel string, info os.Fi
return copyPath(src, dst)
}
func createSupportTarGz(dst, srcDir string) error {
file, err := os.Create(dst)
func createSupportTarGz(dir, baseName, srcDir string) (string, error) {
archiveFile, err := os.CreateTemp(dir, baseName+"-*.partial")
if err != nil {
return err
return "", err
}
defer file.Close()
partialPath := archiveFile.Name()
if err := writeSupportTarGz(archiveFile, srcDir); err != nil {
_ = archiveFile.Close()
_ = os.Remove(partialPath)
return "", err
}
if err := archiveFile.Close(); err != nil {
_ = os.Remove(partialPath)
return "", err
}
archivePath := strings.TrimSuffix(partialPath, ".partial") + ".tar.gz"
if err := os.Rename(partialPath, archivePath); err != nil {
_ = os.Remove(partialPath)
return "", err
}
return archivePath, nil
}
func writeSupportTarGz(file *os.File, srcDir string) error {
gz := gzip.NewWriter(file)
defer gz.Close()
tw := tar.NewWriter(gz)
defer tw.Close()
base := filepath.Dir(srcDir)
return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
walkErr := filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
@@ -912,9 +916,21 @@ func createSupportTarGz(dst, srcDir string) error {
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(tw, f)
return err
_, copyErr := io.Copy(tw, f)
closeErr := f.Close()
if copyErr != nil {
return copyErr
}
return closeErr
})
if walkErr != nil {
_ = tw.Close()
_ = gz.Close()
return walkErr
}
if err := tw.Close(); err != nil {
_ = gz.Close()
return err
}
return gz.Close()
}
audit/internal/app/support_bundle_test.go
+97
View File
@@ -1,9 +1,14 @@
package app
import (
"archive/tar"
"compress/gzip"
"errors"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"
)
@@ -29,3 +34,95 @@ func TestWriteBundleDocs(t *testing.T) {
t.Fatalf("README.md should explain how to check SAT pass/fail:\n%s", readme)
}
}
func TestCreateSupportTarGzUsesIndependentFilesConcurrently(t *testing.T) {
tempDir := t.TempDir()
srcDir := filepath.Join(tempDir, "bee-support-stage-test")
if err := os.Mkdir(srcDir, 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(srcDir, "evidence.txt"), []byte("complete evidence\n"), 0644); err != nil {
t.Fatal(err)
}
const builds = 4
paths := make(chan string, builds)
errs := make(chan error, builds)
var wg sync.WaitGroup
for range builds {
wg.Add(1)
go func() {
defer wg.Done()
path, err := createSupportTarGz(tempDir, "bundle", srcDir)
if err != nil {
errs <- err
return
}
paths <- path
}()
}
wg.Wait()
close(paths)
close(errs)
for err := range errs {
t.Fatalf("create archive: %v", err)
}
seen := make(map[string]struct{}, builds)
for path := range paths {
if !strings.HasSuffix(path, ".tar.gz") {
t.Fatalf("archive path %q does not end in .tar.gz", path)
}
if _, exists := seen[path]; exists {
t.Fatalf("duplicate archive path %q", path)
}
seen[path] = struct{}{}
assertSupportArchiveEntry(t, path, "bee-support-stage-test/evidence.txt", "complete evidence\n")
}
if len(seen) != builds {
t.Fatalf("archive count = %d, want %d", len(seen), builds)
}
partials, err := filepath.Glob(filepath.Join(tempDir, "*.partial"))
if err != nil {
t.Fatal(err)
}
if len(partials) != 0 {
t.Fatalf("unpublished partial archives remain: %v", partials)
}
}
func assertSupportArchiveEntry(t *testing.T, path, wantName, wantBody string) {
t.Helper()
file, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer file.Close()
gz, err := gzip.NewReader(file)
if err != nil {
t.Fatal(err)
}
defer gz.Close()
tr := tar.NewReader(gz)
for {
header, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
t.Fatal(err)
}
if header.Name != wantName {
continue
}
body, err := io.ReadAll(tr)
if err != nil {
t.Fatal(err)
}
if string(body) != wantBody {
t.Fatalf("entry body = %q, want %q", body, wantBody)
}
return
}
t.Fatalf("archive %q is missing %q", path, wantName)
}
audit/internal/platform/benchmark.go
+1 -12
View File
@@ -453,18 +453,7 @@ func (s *System) RunNvidiaBenchmark(ctx context.Context, baseDir string, opts Nv
Status: "FAILED",
}
if info, ok := infoByIndex[idx]; ok {
gpuResult.UUID = info.UUID
gpuResult.Name = info.Name
gpuResult.BusID = info.BusID
gpuResult.VBIOS = info.VBIOS
gpuResult.PowerLimitW = info.PowerLimitW
gpuResult.MultiprocessorCount = info.MultiprocessorCount
gpuResult.DefaultPowerLimitW = info.DefaultPowerLimitW
gpuResult.ShutdownTempC = info.ShutdownTempC
gpuResult.SlowdownTempC = info.SlowdownTempC
gpuResult.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz
gpuResult.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz
gpuResult.MaxMemoryClockMHz = info.MaxMemoryClockMHz
populateBenchmarkGPUInfo(&gpuResult, info)
}
if calib, ok := calibByIndex[idx]; ok {
gpuResult.CalibratedPeakPowerW = calib.Summary.P95PowerW
audit/internal/platform/benchmark_analysis.go
-7
View File
@@ -525,13 +525,6 @@ func scoreBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkScorecard {
return score
}
// compositeBenchmarkScore is kept for compatibility with legacy callers.
// CompositeScore = ComputeScore (no quality multiplier; throttling already
// reduces TOPS directly, so no additional penalty is needed).
func compositeBenchmarkScore(score BenchmarkScorecard) float64 {
return score.ComputeScore
}
// serverQualityScore returns a 0–100 score reflecting server infrastructure
// quality, independent of GPU model or compute speed.
//
audit/internal/platform/benchmark_power_autotune.go
-3
View File
@@ -6,7 +6,6 @@ import (
"fmt"
"math"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
@@ -721,5 +720,3 @@ func minInt(a, b int) int {
}
return b
}
var _ = exec.ErrNotFound
audit/internal/platform/benchmark_power_calibration.go
+16 -45
View File
@@ -406,39 +406,6 @@ func queryIPMIServerPowerW() (float64, error) {
return 0, fmt.Errorf("could not parse ipmitool dcmi power reading output")
}
// sampleIPMIPowerSeries collects IPMI power readings every 2 seconds for
// durationSec seconds. Returns the mean of all successful samples.
// Returns 0, false if IPMI is unavailable.
func sampleIPMIPowerSeries(ctx context.Context, durationSec int) (meanW float64, ok bool) {
if durationSec <= 0 {
return 0, false
}
deadline := time.Now().Add(time.Duration(durationSec) * time.Second)
var samples []float64
loop:
for {
if w, err := queryIPMIServerPowerW(); err == nil {
samples = append(samples, w)
}
if time.Now().After(deadline) {
break
}
select {
case <-ctx.Done():
break loop
case <-time.After(2 * time.Second):
}
}
if len(samples) == 0 {
return 0, false
}
var sum float64
for _, w := range samples {
sum += w
}
return sum / float64(len(samples)), true
}
// characterizeServerPower computes BenchmarkServerPower from idle and loaded
// samples plus the GPU-reported average power during steady state.
func characterizeServerPower(idleW, loadedW, gpuReportedSumW float64, source string, available bool) *BenchmarkServerPower {
@@ -535,18 +502,7 @@ func runNvidiaBenchmarkParallel(
for _, idx := range selected {
r := &BenchmarkGPUResult{Index: idx, Status: "FAILED"}
if info, ok := infoByIndex[idx]; ok {
r.UUID = info.UUID
r.Name = info.Name
r.BusID = info.BusID
r.VBIOS = info.VBIOS
r.PowerLimitW = info.PowerLimitW
r.MultiprocessorCount = info.MultiprocessorCount
r.DefaultPowerLimitW = info.DefaultPowerLimitW
r.ShutdownTempC = info.ShutdownTempC
r.SlowdownTempC = info.SlowdownTempC
r.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz
r.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz
r.MaxMemoryClockMHz = info.MaxMemoryClockMHz
populateBenchmarkGPUInfo(r, info)
}
if calib, ok := calibByIndex[idx]; ok {
r.CalibratedPeakPowerW = calib.Summary.P95PowerW
@@ -753,6 +709,21 @@ func runNvidiaBenchmarkParallel(
}
}
func populateBenchmarkGPUInfo(result *BenchmarkGPUResult, info benchmarkGPUInfo) {
result.UUID = info.UUID
result.Name = info.Name
result.BusID = info.BusID
result.VBIOS = info.VBIOS
result.PowerLimitW = info.PowerLimitW
result.MultiprocessorCount = info.MultiprocessorCount
result.DefaultPowerLimitW = info.DefaultPowerLimitW
result.ShutdownTempC = info.ShutdownTempC
result.SlowdownTempC = info.SlowdownTempC
result.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz
result.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz
result.MaxMemoryClockMHz = info.MaxMemoryClockMHz
}
// readBenchmarkHostConfig reads static CPU and memory configuration from
// /proc/cpuinfo and /proc/meminfo. Returns nil if neither source is readable.
func readBenchmarkHostConfig() *BenchmarkHostConfig {
audit/internal/platform/install_to_ram_test.go
-6
View File
@@ -129,8 +129,6 @@ func TestShouldLogCopyProgress(t *testing.T) {
}
func TestTryRemountLiveMedium(t *testing.T) {
t.Parallel()
orig := runRemountMedium
t.Cleanup(func() {
runRemountMedium = orig
@@ -165,8 +163,6 @@ func TestTryRemountLiveMedium(t *testing.T) {
}
func TestEnsureLiveMediumAvailableRemountsSource(t *testing.T) {
t.Parallel()
origGlob := liveMediumSquashfsGlob
origRemount := runRemountMedium
t.Cleanup(func() {
@@ -210,8 +206,6 @@ func TestEnsureLiveMediumAvailableRemountsSource(t *testing.T) {
}
func TestDetachInstallMedium(t *testing.T) {
t.Parallel()
origUmount := umountLiveMedium
origEject := ejectDevice
t.Cleanup(func() {
audit/internal/platform/live_metrics.go
+5 -1
View File
@@ -293,6 +293,10 @@ func sampleLiveTempsViaIPMI() []TempReading {
}
func firstTempInputValue(feature map[string]any) (float64, bool) {
return firstSensorInputValue(feature, "temp")
}
func firstSensorInputValue(feature map[string]any, kind string) (float64, bool) {
keys := make([]string, 0, len(feature))
for key := range feature {
keys = append(keys, key)
@@ -300,7 +304,7 @@ func firstTempInputValue(feature map[string]any) (float64, bool) {
sort.Strings(keys)
for _, key := range keys {
lower := strings.ToLower(key)
if !strings.Contains(lower, "temp") || !strings.HasSuffix(lower, "_input") {
if !strings.Contains(lower, kind) || !strings.HasSuffix(lower, "_input") {
continue
}
switch value := feature[key].(type) {
audit/internal/platform/nvidia_config_check.go
+146 -47
View File
@@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
@@ -71,9 +72,9 @@ func (s *System) RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, l
errOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-e", []string{"nvidia-smi", "nvlink", "-e"}, nil, logFunc)
_ = os.WriteFile(filepath.Join(runDir, "03-nvidia-smi-nvlink-e.log"), errOut, 0644)
pairs := parseNvidiaNVLinkBondedPairs(string(topoOut))
pairs := ParseNvidiaNVLinkBondedPairs(string(topoOut))
linkStatus := parseNvidiaNVLinkStatus(string(statusOut))
linkErrors := parseNvidiaNVLinkErrors(string(errOut))
linkErrors := ParseNvidiaNVLinkErrors(string(errOut))
for _, pair := range pairs {
f := evaluateNvidiaNVLinkPair(pair, linkStatus, linkErrors)
status.NVLinkPairs = append(status.NVLinkPairs, f)
@@ -96,6 +97,11 @@ func (s *System) RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, l
dmesgOut, _ := runSATCommandCtx(ctx, verboseLog, "dmesg", []string{"dmesg"}, nil, nil)
ccDmesgLines := filterConfComputeDmesgLines(dmesgOut)
_ = os.WriteFile(filepath.Join(runDir, "05-dmesg-cc-relevant.log"), []byte(strings.Join(ccDmesgLines, "\n")+"\n"), 0644)
nvlinkDegraded := parseNvidiaNVLinkDegradedDmesg(dmesgOut)
_ = os.WriteFile(filepath.Join(runDir, "06-dmesg-nvlink-degraded.log"), []byte(renderNvidiaNVLinkDegradedLog(nvlinkDegraded)), 0644)
for _, event := range nvlinkDegraded {
status.Warnings = append(status.Warnings, event.Warning())
}
lowerDmesg := strings.ToLower(strings.Join(ccDmesgLines, "\n"))
status.HostAMDSEVSNPActive = strings.Contains(lowerDmesg, "sev-snp enabled")
status.HostIntelTDXActive = strings.Contains(lowerDmesg, "tdx module") && strings.Contains(lowerDmesg, "module initialized") ||
@@ -166,7 +172,11 @@ type NvidiaNVLinkPairFinding struct {
GPUA int `json:"gpu_a"`
GPUB int `json:"gpu_b"`
ExpectedLinks int `json:"expected_links"` // from "nvidia-smi topo -m"'s NVx cell
ActiveLinks int `json:"active_links"` // from "nvidia-smi nvlink -s"
ActiveLinksA int `json:"active_links_a"` // per endpoint from "nvidia-smi nvlink -s"
ActiveLinksB int `json:"active_links_b"`
TotalLinksA int `json:"total_links_a"`
TotalLinksB int `json:"total_links_b"`
ActiveLinks int `json:"active_links"` // aggregate retained for report compatibility
TotalLinks int `json:"total_links"`
ErrorLinks int `json:"error_links"` // links with a nonzero replay/recovery/CRC counter
Issues []string `json:"issues,omitempty"`
@@ -197,28 +207,57 @@ func evaluateNvidiaGPUConfig(g NvidiaGPUSetting) []string {
// evaluateNvidiaNVLinkPair compares a bonded pair's actual link state
// against what the topology matrix says should be there.
func evaluateNvidiaNVLinkPair(pair nvidiaNVLinkBondedPair, status map[int][]nvidiaNVLinkPort, errors map[int]map[int][3]int64) NvidiaNVLinkPairFinding {
f := NvidiaNVLinkPairFinding{GPUA: pair.gpuA, GPUB: pair.gpuB, ExpectedLinks: pair.links}
func evaluateNvidiaNVLinkPair(pair NvidiaNVLinkBondedPair, status map[int][]nvidiaNVLinkPort, errors map[int]map[int][3]int64) NvidiaNVLinkPairFinding {
f := NvidiaNVLinkPairFinding{GPUA: pair.GPUA, GPUB: pair.GPUB, ExpectedLinks: pair.NVLinks}
active, total, errLinks := 0, 0, 0
for _, gpu := range []int{pair.gpuA, pair.gpuB} {
for _, port := range status[gpu] {
countPorts := func(gpu int) (active, total int, present bool) {
ports, present := status[gpu]
for _, port := range ports {
total++
if port.active {
active++
}
}
return active, total, present
}
f.ActiveLinksA, f.TotalLinksA, _ = countPorts(pair.GPUA)
f.ActiveLinksB, f.TotalLinksB, _ = countPorts(pair.GPUB)
f.ActiveLinks = f.ActiveLinksA + f.ActiveLinksB
f.TotalLinks = f.TotalLinksA + f.TotalLinksB
_, statusASeen := status[pair.GPUA]
_, statusBSeen := status[pair.GPUB]
if !statusASeen || !statusBSeen {
var missing []string
if !statusASeen {
missing = append(missing, fmt.Sprintf("GPU%d", pair.GPUA))
}
if !statusBSeen {
missing = append(missing, fmt.Sprintf("GPU%d", pair.GPUB))
}
f.Issues = append(f.Issues, "NVLink status unavailable for "+strings.Join(missing, ", "))
}
if pair.NVLinks > 0 && (f.ActiveLinksA < pair.NVLinks || f.ActiveLinksB < pair.NVLinks) {
f.Issues = append(f.Issues, fmt.Sprintf(
"GPU%d %d/%d, GPU%d %d/%d active links (topo NV%d)",
pair.GPUA, f.ActiveLinksA, pair.NVLinks,
pair.GPUB, f.ActiveLinksB, pair.NVLinks,
pair.NVLinks,
))
}
if f.TotalLinks > 0 && f.ActiveLinks < f.TotalLinks {
f.Issues = append(f.Issues, fmt.Sprintf("%d/%d reported NVLinks inactive on a bonded pair", f.TotalLinks-f.ActiveLinks, f.TotalLinks))
}
errLinks := 0
for _, gpu := range []int{pair.GPUA, pair.GPUB} {
for _, counters := range errors[gpu] {
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
errLinks++
}
}
}
f.ActiveLinks, f.TotalLinks, f.ErrorLinks = active, total, errLinks
if total > 0 && active < total {
f.Issues = append(f.Issues, fmt.Sprintf("%d/%d NVLinks inactive on a bonded pair", total-active, total))
}
f.ErrorLinks = errLinks
if errLinks > 0 {
f.Issues = append(f.Issues, fmt.Sprintf("%d link(s) reporting replay/recovery/CRC errors", errLinks))
}
@@ -226,17 +265,14 @@ func evaluateNvidiaNVLinkPair(pair nvidiaNVLinkBondedPair, status map[int][]nvid
}
// ---------------------------------------------------------------------------
// NVLink parsing — deliberately self-contained rather than importing
// internal/collector or internal/webui's equivalent parsers (isNVLinkBridgeCandidate
// et al. and parseGPUPairAdjacency/parseTopoNVLinkStatus/parseTopoNVLinkErrors,
// respectively), matching this codebase's existing precedent of small,
// package-local duplication over cross-package coupling for narrow parsing
// helpers (see webui/page_topo.go's isNICDeviceClassDev/isRAIDControllerClass).
// NVLink parsing shared by diagnostics and the read-only topology page.
// ---------------------------------------------------------------------------
type nvidiaNVLinkBondedPair struct {
gpuA, gpuB int
links int
// NvidiaNVLinkBondedPair identifies two GPU indices and their negotiated
// NVLink bond width as reported by nvidia-smi topo -m.
type NvidiaNVLinkBondedPair struct {
GPUA, GPUB int
NVLinks int
}
var (
@@ -244,37 +280,31 @@ var (
nvidiaNVLinkNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
)
// parseNvidiaNVLinkBondedPairs returns every GPU pair with a nonzero NVLink
// ParseNvidiaNVLinkBondedPairs returns every GPU pair with a nonzero NVLink
// bond count from a "nvidia-smi topo -m" matrix, deduplicated (A,B) == (B,A).
func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair {
func ParseNvidiaNVLinkBondedPairs(raw string) []NvidiaNVLinkBondedPair {
lines := strings.Split(nvidiaNVLinkANSIRe.ReplaceAllString(raw, ""), "\n")
headerIdx := -1
var gpuColIndices []int
gpuColumns := map[int]int{}
for i, line := range lines {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "GPU0") {
parts := strings.Fields(trimmed)
for j, col := range parts {
if strings.HasPrefix(col, "GPU") {
gpuColIndices = append(gpuColIndices, j)
}
}
if len(gpuColIndices) >= 2 {
headerIdx = i
columns := map[int]int{}
for columnIndex, label := range strings.Fields(strings.TrimSpace(line)) {
gpuIndex, err := strconv.Atoi(strings.TrimPrefix(label, "GPU"))
if err == nil && strings.HasPrefix(label, "GPU") {
columns[columnIndex] = gpuIndex
}
}
if len(columns) >= 2 {
headerIdx = i
gpuColumns = columns
break
}
}
if headerIdx < 0 {
return nil
}
colIdxToGPU := make(map[int]int, len(gpuColIndices))
for gpuIdx, colIdx := range gpuColIndices {
colIdxToGPU[colIdx] = gpuIdx
}
seen := map[[2]int]bool{}
var pairs []nvidiaNVLinkBondedPair
var pairs []NvidiaNVLinkBondedPair
for _, line := range lines[headerIdx+1:] {
trimmed := strings.TrimSpace(line)
if !strings.HasPrefix(trimmed, "GPU") {
@@ -288,7 +318,7 @@ func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair {
if err != nil {
continue
}
for colIdx, colGPU := range colIdxToGPU {
for colIdx, colGPU := range gpuColumns {
if colGPU == rowGPU {
continue
}
@@ -313,9 +343,15 @@ func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair {
continue
}
seen[key] = true
pairs = append(pairs, nvidiaNVLinkBondedPair{gpuA: a, gpuB: b, links: nv})
pairs = append(pairs, NvidiaNVLinkBondedPair{GPUA: a, GPUB: b, NVLinks: nv})
}
}
sort.Slice(pairs, func(i, j int) bool {
if pairs[i].GPUA != pairs[j].GPUA {
return pairs[i].GPUA < pairs[j].GPUA
}
return pairs[i].GPUB < pairs[j].GPUB
})
return pairs
}
@@ -339,6 +375,11 @@ func parseNvidiaNVLinkStatus(raw string) map[int][]nvidiaNVLinkPort {
trimmed := strings.TrimSpace(line)
if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
currentGPU, _ = strconv.Atoi(m[1])
if _, exists := result[currentGPU]; !exists {
// Preserve the distinction between a present GPU with no active
// links and a GPU whose block is absent from the command output.
result[currentGPU] = nil
}
continue
}
if currentGPU < 0 {
@@ -355,9 +396,67 @@ func parseNvidiaNVLinkStatus(raw string) map[int][]nvidiaNVLinkPort {
return result
}
// parseNvidiaNVLinkErrors parses "nvidia-smi nvlink -e" output into, per GPU
type nvidiaNVLinkDegradedEvent struct {
GPUIndex int
LinkID int
RawLine string
}
func (e nvidiaNVLinkDegradedEvent) Warning() string {
switch {
case e.GPUIndex >= 0 && e.LinkID >= 0:
return fmt.Sprintf("GPU%d NVLink degraded mode (linkId %d): %s", e.GPUIndex, e.LinkID, e.RawLine)
case e.GPUIndex >= 0:
return fmt.Sprintf("GPU%d NVLink degraded mode: %s", e.GPUIndex, e.RawLine)
default:
return "NVIDIA NVLink degraded mode: " + e.RawLine
}
}
var (
nvidiaNVLinkDegradedGPURe = regexp.MustCompile(`(?i)\bGPU\s*(\d+)\b`)
nvidiaNVLinkDegradedLinkRe = regexp.MustCompile(`(?i)\blinkId\s*[:=]?\s*(\d+)\b`)
)
func parseNvidiaNVLinkDegradedDmesg(raw []byte) []nvidiaNVLinkDegradedEvent {
var events []nvidiaNVLinkDegradedEvent
for _, lineBytes := range bytes.Split(raw, []byte("\n")) {
line := strings.TrimSpace(string(lineBytes))
lower := strings.ToLower(line)
explicitNVLinkMarker := strings.Contains(lower, "knvlinksetdegradedmode") ||
strings.Contains(lower, "nvlink_is_gpu_degraded") ||
strings.Contains(lower, "error originated on linkid")
contextualDegradedMarker := strings.Contains(lower, "marked degraded") &&
(strings.Contains(lower, "nvlink") || strings.Contains(lower, "nvrm") || nvidiaNVLinkDegradedGPURe.MatchString(line))
if line == "" || (!explicitNVLinkMarker && !contextualDegradedMarker) {
continue
}
event := nvidiaNVLinkDegradedEvent{GPUIndex: -1, LinkID: -1, RawLine: line}
if match := nvidiaNVLinkDegradedGPURe.FindStringSubmatch(line); len(match) == 2 {
event.GPUIndex, _ = strconv.Atoi(match[1])
}
if match := nvidiaNVLinkDegradedLinkRe.FindStringSubmatch(line); len(match) == 2 {
event.LinkID, _ = strconv.Atoi(match[1])
}
events = append(events, event)
}
return events
}
func renderNvidiaNVLinkDegradedLog(events []nvidiaNVLinkDegradedEvent) string {
if len(events) == 0 {
return ""
}
lines := make([]string, len(events))
for i, event := range events {
lines[i] = event.RawLine
}
return strings.Join(lines, "\n") + "\n"
}
// ParseNvidiaNVLinkErrors parses "nvidia-smi nvlink -e" output into, per GPU
// then link index, [replay, recovery, crc] error counts.
func parseNvidiaNVLinkErrors(raw string) map[int]map[int][3]int64 {
func ParseNvidiaNVLinkErrors(raw string) map[int]map[int][3]int64 {
result := map[int]map[int][3]int64{}
currentGPU := -1
for _, line := range strings.Split(raw, "\n") {
@@ -516,8 +615,8 @@ func renderNvidiaConfigCheckReport(status NvidiaConfigCheckStatus) string {
if len(p.Issues) > 0 {
verdict = "ISSUES FOUND"
}
fmt.Fprintf(&b, " GPU%d <-> GPU%d: %d/%d links active (topology expects %d): %s\n",
p.GPUA, p.GPUB, p.ActiveLinks, p.TotalLinks, p.ExpectedLinks, verdict)
fmt.Fprintf(&b, " GPU%d <-> GPU%d: endpoints %d/%d and %d/%d active (topology expects %d each): %s\n",
p.GPUA, p.GPUB, p.ActiveLinksA, p.TotalLinksA, p.ActiveLinksB, p.TotalLinksB, p.ExpectedLinks, verdict)
for _, issue := range p.Issues {
fmt.Fprintf(&b, " - %s\n", issue)
}
audit/internal/platform/nvidia_config_check_test.go
+95 -11
View File
@@ -14,11 +14,11 @@ func TestParseNvidiaNVLinkBondedPairsRealTwoGPUDump(t *testing.T) {
"NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" +
"NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n"
pairs := parseNvidiaNVLinkBondedPairs(input)
pairs := ParseNvidiaNVLinkBondedPairs(input)
if len(pairs) != 1 {
t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
}
if pairs[0].gpuA != 0 || pairs[0].gpuB != 1 || pairs[0].links != 17 {
if pairs[0].GPUA != 0 || pairs[0].GPUB != 1 || pairs[0].NVLinks != 17 {
t.Fatalf("pair=%#v want {0,1,17}", pairs[0])
}
}
@@ -34,18 +34,32 @@ func TestParseNvidiaNVLinkBondedPairsANSIUnderlinedHeader(t *testing.T) {
"GPU2\tPIX\tPIX\t X \tNV18\t0-31,64-95\n" +
"GPU3\tPIX\tPIX\tNV18\t X \t0-31,64-95\n"
pairs := parseNvidiaNVLinkBondedPairs(input)
pairs := ParseNvidiaNVLinkBondedPairs(input)
if len(pairs) != 2 {
t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs)
}
want := map[[2]int]int{{0, 1}: 18, {2, 3}: 18}
for _, p := range pairs {
if want[[2]int{p.gpuA, p.gpuB}] != p.links {
if want[[2]int{p.GPUA, p.GPUB}] != p.NVLinks {
t.Fatalf("unexpected pair %#v", p)
}
}
}
func TestParseNvidiaNVLinkBondedPairsUsesGPUIndicesFromHeader(t *testing.T) {
input := "\tGPU2\tGPU4\tCPU Affinity\n" +
"GPU2\t X \tNV4\t0-15\n" +
"GPU4\tNV4\t X \t16-31\n"
pairs := ParseNvidiaNVLinkBondedPairs(input)
if len(pairs) != 1 {
t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
}
if pairs[0] != (NvidiaNVLinkBondedPair{GPUA: 2, GPUB: 4, NVLinks: 4}) {
t.Fatalf("pair=%#v want GPU2<->GPU4 NV4", pairs[0])
}
}
func TestParseNvidiaNVLinkStatusMarksInactiveLinks(t *testing.T) {
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
Link 0: 26.562 GB/s
@@ -71,7 +85,7 @@ func TestParseNvidiaNVLinkErrors(t *testing.T) {
Link 1: Recovery Errors: 1
Link 1: CRC Errors: 2
`
got := parseNvidiaNVLinkErrors(input)
got := ParseNvidiaNVLinkErrors(input)
c := got[0][1]
if c[0] != 3 || c[1] != 1 || c[2] != 2 {
t.Fatalf("link1 counters=%#v want {3,1,2}", c)
@@ -79,7 +93,7 @@ func TestParseNvidiaNVLinkErrors(t *testing.T) {
}
func TestEvaluateNvidiaNVLinkPairFlagsInactiveLinksAndErrors(t *testing.T) {
pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18}
pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 2}
status := map[int][]nvidiaNVLinkPort{
0: {{active: true}, {active: false}},
1: {{active: true}, {active: true}},
@@ -92,16 +106,16 @@ func TestEvaluateNvidiaNVLinkPairFlagsInactiveLinksAndErrors(t *testing.T) {
if f.ActiveLinks != 3 || f.TotalLinks != 4 {
t.Fatalf("active=%d total=%d want 3/4", f.ActiveLinks, f.TotalLinks)
}
if len(f.Issues) != 2 {
t.Fatalf("issues=%#v want 2 (inactive link + error)", f.Issues)
if len(f.Issues) != 3 {
t.Fatalf("issues=%#v want 3 (expected-count mismatch + inactive link + error)", f.Issues)
}
}
func TestEvaluateNvidiaNVLinkPairHealthyBondHasNoIssues(t *testing.T) {
pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18}
pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 18}
status := map[int][]nvidiaNVLinkPort{
0: {{active: true}},
1: {{active: true}},
0: activeNvidiaNVLinkPorts(18),
1: activeNvidiaNVLinkPorts(18),
}
f := evaluateNvidiaNVLinkPair(pair, status, nil)
if len(f.Issues) != 0 {
@@ -109,6 +123,76 @@ func TestEvaluateNvidiaNVLinkPairHealthyBondHasNoIssues(t *testing.T) {
}
}
func TestEvaluateNvidiaNVLinkPairChecksEachEndpoint(t *testing.T) {
pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 17}
status := map[int][]nvidiaNVLinkPort{
0: activeNvidiaNVLinkPorts(17),
1: {},
}
f := evaluateNvidiaNVLinkPair(pair, status, nil)
if f.ActiveLinksA != 17 || f.ActiveLinksB != 0 {
t.Fatalf("endpoint counts=%d,%d want 17,0", f.ActiveLinksA, f.ActiveLinksB)
}
if len(f.Issues) == 0 || !strings.Contains(strings.Join(f.Issues, " "), "GPU1 0/17") {
t.Fatalf("issues=%#v want endpoint-specific 0/17 failure", f.Issues)
}
}
func TestParseNvidiaNVLinkStatusPreservesAllInactiveGPUBlocks(t *testing.T) {
raw := `GPU 0: NVIDIA H100
NVML: Unable to retrieve NVLink information as all links are inActive
GPU 1: NVIDIA H100
all links are inActive
`
status := parseNvidiaNVLinkStatus(raw)
if _, ok := status[0]; !ok {
t.Fatal("GPU0 header was lost")
}
if _, ok := status[1]; !ok {
t.Fatal("GPU1 header was lost")
}
f := evaluateNvidiaNVLinkPair(NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 17}, status, nil)
if len(f.Issues) == 0 || f.ActiveLinksA != 0 || f.ActiveLinksB != 0 {
t.Fatalf("finding=%#v want failed 0/17 endpoints", f)
}
}
func TestEvaluateNvidiaNVLinkPairFlagsMissingGPUBlock(t *testing.T) {
f := evaluateNvidiaNVLinkPair(
NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 2},
map[int][]nvidiaNVLinkPort{0: activeNvidiaNVLinkPorts(2)}, nil,
)
if !strings.Contains(strings.Join(f.Issues, " "), "status unavailable for GPU1") {
t.Fatalf("issues=%#v want missing GPU1 status", f.Issues)
}
}
func TestParseNvidiaNVLinkDegradedDmesg(t *testing.T) {
raw := []byte("[ 11.0] md: array md0 marked Degraded\n[ 12.0] NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n")
events := parseNvidiaNVLinkDegradedDmesg(raw)
if len(events) != 1 {
t.Fatalf("events=%#v want one", events)
}
if events[0].GPUIndex != 1 || events[0].LinkID != 1 {
t.Fatalf("event=%#v want GPU1 linkId 1", events[0])
}
if warning := events[0].Warning(); !strings.Contains(warning, "GPU1 NVLink degraded mode (linkId 1)") {
t.Fatalf("warning=%q", warning)
}
wantLog := "[ 12.0] NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n"
if log := renderNvidiaNVLinkDegradedLog(events); log != wantLog {
t.Fatalf("log=%q want only NVLink line %q", log, wantLog)
}
}
func activeNvidiaNVLinkPorts(count int) []nvidiaNVLinkPort {
ports := make([]nvidiaNVLinkPort, count)
for i := range ports {
ports[i].active = true
}
return ports
}
func TestEvaluateNvidiaGPUConfigFlagsECCDisabled(t *testing.T) {
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Disabled"}
issues := evaluateNvidiaGPUConfig(g)
audit/internal/platform/sat.go
+55 -115
View File
@@ -191,13 +191,19 @@ func streamExecOutput(cmd *exec.Cmd, logFunc func(string), livePath string) ([]b
return buf.Bytes(), waitErr
}
// NvidiaGPU holds basic GPU info from nvidia-smi.
// satJob describes one command and the checks needed to turn its process and
// output results into a SAT verdict.
type satJob struct {
name string
cmd []string
env []string // extra env vars (appended to os.Environ)
collectGPU bool // collect GPU metrics via nvidia-smi while this job runs
gpuIndices []int // GPU indices to collect metrics for (empty = all)
name string
cmd []string
env []string // extra env vars (appended to os.Environ)
// validate checks successful command output against the tool's documented
// result format. It may inspect artifacts from earlier jobs in runDir.
// FAILED means the tool proved a test failure; UNSUPPORTED means the pinned
// output contract could not be recognized safely.
validate func(runDir string, out []byte) (status, detail string)
collectGPU bool // collect GPU metrics via nvidia-smi while this job runs
gpuIndices []int // GPU indices to collect metrics for (empty = all)
// informational marks a preflight/metadata job (e.g. dcgmi discovery) whose
// failure shouldn't flip the pack's overall status — the diagnostic jobs
// that follow it are the actual test of GPU health.
@@ -314,7 +320,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
var summary strings.Builder
stats := satStats{}
nvidiaPack := strings.HasPrefix(prefix, "gpu-nvidia")
nvidiaPack := isNvidiaAcceptancePack(prefix)
perGPU := map[int]*nvidiaGPUStatusFile{}
selectedGPUIndices := map[int]struct{}{}
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
@@ -338,6 +344,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
var out []byte
var err error
jobDetail := ""
if nvidiaPack && nvidiaJobNeedsHealthCheck(job) {
if msg, healthErr := checkNvidiaJobHealth(job.gpuIndices); healthErr != nil {
@@ -346,6 +353,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
}
out = []byte(msg + "\n")
err = healthErr
jobDetail = msg
}
}
@@ -379,6 +387,23 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
if err == nil {
err = healthErr
}
jobDetail = msg
}
}
status, rc := classifySATResult(job.name, out, err)
validationDetail := singleLineSATDetail(jobDetail)
validated := false
if status == "OK" && job.validate != nil {
status, validationDetail = validateSATJobOutput(job, runDir, out)
validated = true
}
if validated {
if validationDetail != "" {
if len(out) > 0 && !bytes.HasSuffix(out, []byte("\n")) {
out = append(out, '\n')
}
out = append(out, []byte(fmt.Sprintf("[bee-validator] %s: %s\n", status, validationDetail))...)
}
}
@@ -392,7 +417,6 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
if ctx.Err() != nil {
return "", ctx.Err()
}
status, rc := classifySATResult(job.name, out, err)
if job.informational && status != "OK" {
stats.Informational++
} else {
@@ -406,6 +430,9 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
key := strings.TrimSuffix(strings.TrimPrefix(job.name, "0"), ".log")
fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
if validationDetail != "" {
fmt.Fprintf(&summary, "%s_detail=%s\n", key, singleLineSATDetail(validationDetail))
}
}
writeSATStats(&summary, stats)
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
@@ -420,6 +447,25 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
return runDir, nil
}
func isNvidiaAcceptancePack(prefix string) bool {
return strings.HasPrefix(prefix, "gpu-nvidia") || prefix == "nccl-tests"
}
func validateSATJobOutput(job satJob, runDir string, out []byte) (string, string) {
status, detail := job.validate(runDir, out)
status = strings.ToUpper(strings.TrimSpace(status))
switch status {
case "OK", "FAILED", "UNSUPPORTED", "PARTIAL":
return status, strings.TrimSpace(detail)
default:
return "UNSUPPORTED", fmt.Sprintf("validator returned invalid status %q", status)
}
}
func singleLineSATDetail(detail string) string {
return strings.Join(strings.Fields(detail), " ")
}
func updateNvidiaGPUStatus(perGPU map[int]*nvidiaGPUStatusFile, idx int, status, jobName, detail string) {
entry := perGPU[idx]
if entry == nil {
@@ -505,112 +551,6 @@ func writeNvidiaGPUStatusFiles(runDir, overall string, perGPU map[int]*nvidiaGPU
return nil
}
func nvidiaSATStatusSeverity(status string) int {
switch strings.ToUpper(strings.TrimSpace(status)) {
case "FAILED":
return 3
case "PARTIAL", "UNSUPPORTED":
return 2
case "OK":
return 1
default:
return 0
}
}
func firstLine(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return ""
}
if idx := strings.IndexByte(s, '\n'); idx >= 0 {
return strings.TrimSpace(s[:idx])
}
return s
}
func nvidiaJobNeedsHealthCheck(job satJob) bool {
if job.collectGPU {
return true
}
name := strings.ToLower(strings.TrimSpace(job.name))
return strings.Contains(name, "dcgmi") ||
strings.Contains(name, "gpu-burn") ||
strings.Contains(name, "gpu-stress") ||
strings.Contains(name, "dcgmproftester")
}
func checkNvidiaJobHealth(selected []int) (string, error) {
health, err := readNvidiaGPUHealth()
if err != nil {
return "", nil
}
var bad []nvidiaGPUHealth
selectedSet := make(map[int]struct{}, len(selected))
for _, idx := range selected {
selectedSet[idx] = struct{}{}
}
for _, gpu := range health {
if len(selectedSet) > 0 {
if _, ok := selectedSet[gpu.Index]; !ok {
continue
}
}
if gpu.NeedsReset {
bad = append(bad, gpu)
}
}
if len(bad) == 0 {
return "", nil
}
lines := make([]string, 0, len(bad)+1)
lines = append(lines, "NVIDIA GPU health check failed:")
for _, gpu := range bad {
lines = append(lines, fmt.Sprintf("gpu %d (%s) requires reset: %s", gpu.Index, gpu.Name, gpu.RawLine))
}
return strings.Join(lines, "\n"), errors.New("nvidia gpu requires reset")
}
func readNvidiaGPUHealth() ([]nvidiaGPUHealth, error) {
out, err := satExecCommand(
"nvidia-smi",
"--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total",
"--format=csv,noheader,nounits",
).Output()
if err != nil {
return nil, fmt.Errorf("nvidia-smi: %w", err)
}
return parseNvidiaGPUHealth(string(out)), nil
}
func parseNvidiaGPUHealth(raw string) []nvidiaGPUHealth {
var gpus []nvidiaGPUHealth
for _, line := range strings.Split(strings.TrimSpace(raw), "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
parts := strings.Split(line, ",")
if len(parts) < 2 {
gpus = append(gpus, nvidiaGPUHealth{RawLine: line, ParseFailure: true})
continue
}
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
if err != nil {
gpus = append(gpus, nvidiaGPUHealth{RawLine: line, ParseFailure: true})
continue
}
upper := strings.ToUpper(line)
gpus = append(gpus, nvidiaGPUHealth{
Index: idx,
Name: strings.TrimSpace(parts[1]),
NeedsReset: strings.Contains(upper, "GPU REQUIRES RESET"),
RawLine: line,
})
}
return gpus
}
// runSATCommandCtx runs cmd and returns its combined output. livePath is
// variadic purely so existing callers are unaffected: pass a path (job's
// output file) to also stream output to disk live as it runs, so a crash
@@ -736,7 +676,7 @@ func (s *satStats) Add(status string) {
switch status {
case "OK":
s.OK++
case "UNSUPPORTED":
case "UNSUPPORTED", "PARTIAL":
s.Unsupported++
default:
s.Failed++
audit/internal/platform/sat_fan_stress.go
+1 -36
View File
@@ -571,21 +571,6 @@ func sampleFanSpeedsViaSensorsJSON() ([]FanReading, error) {
return fans, nil
}
// sampleFanDutyCyclePct reads fan PWM/duty-cycle controls from lm-sensors.
// Returns the average duty cycle across all exposed PWM controls.
func sampleFanDutyCyclePct() (float64, bool, bool) {
out, err := exec.Command("sensors", "-j").Output()
if err != nil || len(out) == 0 {
fans, fanErr := sampleFanSpeeds()
if fanErr != nil {
return 0, false, false
}
return sampleFanDutyCyclePctFromFans(fans)
}
pct, ok := parseFanDutyCyclePctSensorsJSON(out)
return pct, ok, false
}
func sampleFanDutyCyclePctFromFans(fans []FanReading) (float64, bool, bool) {
if len(fans) == 0 {
return 0, false, false
@@ -682,27 +667,7 @@ func normalizePWMAsDutyPct(raw float64) (float64, bool) {
}
func firstFanInputValue(feature map[string]any) (float64, bool) {
keys := make([]string, 0, len(feature))
for key := range feature {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
lower := strings.ToLower(key)
if !strings.Contains(lower, "fan") || !strings.HasSuffix(lower, "_input") {
continue
}
switch value := feature[key].(type) {
case float64:
return value, true
case string:
f, err := strconv.ParseFloat(value, 64)
if err == nil {
return f, true
}
}
}
return 0, false
return firstSensorInputValue(feature, "fan")
}
// sampleCPUMaxTemp returns the highest CPU/inlet temperature from ipmitool or sensors.
audit/internal/platform/sat_gpu_packs.go
+36 -62
View File
@@ -324,17 +324,30 @@ func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []
if err != nil {
return "", err
}
return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", ncclSATJobs(selected), logFunc)
}
func ncclSATJobs(selected []int) []satJob {
gpuCount := len(selected)
if gpuCount < 1 {
gpuCount = 1
}
return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", withNvidiaPersistenceMode(
ncclEnv := append(nvidiaVisibleDevicesEnv(selected),
"NCCL_DEBUG=INFO",
"NCCL_DEBUG_SUBSYS=INIT,GRAPH",
)
return withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "01-nvidia-smi-topo-m.log", cmd: []string{"nvidia-smi", "topo", "-m"}, informational: true},
satJob{name: "01-nvidia-smi-nvlink-s.log", cmd: []string{"nvidia-smi", "nvlink", "-s"}, informational: true},
satJob{name: "02-all-reduce-perf.log", cmd: []string{
"all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2",
"-g", strconv.Itoa(gpuCount), "--iters", "20",
}, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true},
), logFunc)
}, env: ncclEnv, gpuIndices: selected, syncBracket: true,
validate: func(runDir string, out []byte) (string, string) {
return validateNCCLAllReduceOutput(runDir, out, selected)
}},
)
}
func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error) {
@@ -384,23 +397,27 @@ func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir strin
}
func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
return s.runNvidiaNamedDiagPack(ctx, baseDir, durationSec, gpuIndices,
"gpu-nvidia-targeted-power", "targeted_power", "03-dcgmi-targeted-power.log", logFunc)
}
func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
return s.runNvidiaNamedDiagPack(ctx, baseDir, durationSec, gpuIndices,
"gpu-nvidia-pulse", "pulse_test", "03-dcgmi-pulse-test.log", logFunc)
}
func (s *System) runNvidiaNamedDiagPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, packName, diagName, logName string, logFunc func(string)) (string, error) {
selected, err := resolveDCGMGPUIndices(gpuIndices)
if err != nil {
return "", err
}
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
for _, p := range killed {
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
}
}
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode(
killStaleNvidiaTestWorkers(logFunc)
return runAcceptancePackCtx(ctx, baseDir, packName, withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{
name: "03-dcgmi-targeted-power.log",
cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
name: logName,
cmd: nvidiaDCGMNamedDiagCommand(diagName, normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
@@ -409,30 +426,14 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string,
), logFunc)
}
func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
selected, err := resolveDCGMGPUIndices(gpuIndices)
if err != nil {
return "", err
}
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
// before starting; otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
func killStaleNvidiaTestWorkers(logFunc func(string)) {
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
for _, p := range killed {
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
}
}
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{
name: "03-dcgmi-pulse-test.log",
cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
},
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
), logFunc)
}
// RunNvidiaBandwidthPack runs `dcgmi diag -r nvbandwidth`. The only thing
@@ -449,13 +450,7 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
if err != nil {
return "", err
}
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
for _, p := range killed {
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
}
}
killStaleNvidiaTestWorkers(logFunc)
jobs := []satJob{
{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
@@ -528,29 +523,8 @@ func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir
}
func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
selected, err := resolveDCGMGPUIndices(gpuIndices)
if err != nil {
return "", err
}
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
for _, p := range killed {
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
}
}
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{
name: "03-dcgmi-targeted-stress.log",
cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
},
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
), logFunc)
return s.runNvidiaNamedDiagPack(ctx, baseDir, durationSec, gpuIndices,
"gpu-nvidia-targeted-stress", "targeted_stress", "03-dcgmi-targeted-stress.log", logFunc)
}
func resolveDCGMGPUIndices(gpuIndices []int) ([]int, error) {
audit/internal/platform/sat_nccl_validation.go
+123
View File
@@ -0,0 +1,123 @@
package platform
import (
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
)
var (
ncclOutOfBoundsRE = regexp.MustCompile(`(?im)^#\s*Out of bounds values\s*:\s*(\d+)(?:\s+\S+)?\s*$`)
ncclAvgBusBWRE = regexp.MustCompile(`(?im)^#\s*Avg bus bandwidth\s*:\s*([0-9]+(?:\.[0-9]+)?)\s*$`)
ncclGPUHeaderRE = regexp.MustCompile(`(?i)\bGPU\d+\b`)
)
// validateNCCLAllReduceOutput validates only facts that nccl-tests reports
// directly. It deliberately does not apply a model-independent bandwidth
// threshold: busbw depends on GPU model, topology, rank count, message size,
// and NCCL algorithm. The measured value is retained in detail for diagnosis.
func validateNCCLAllReduceOutput(runDir string, out []byte, selectedGPUIndices []int) (string, string) {
text := string(out)
oobMatch := ncclOutOfBoundsRE.FindStringSubmatch(text)
if len(oobMatch) != 2 {
return "UNSUPPORTED", "nccl-tests output has no recognized Out of bounds values summary"
}
oob, err := strconv.ParseUint(oobMatch[1], 10, 64)
if err != nil {
return "UNSUPPORTED", "nccl-tests Out of bounds values is not an unsigned integer"
}
if oob != 0 {
return "FAILED", fmt.Sprintf("nccl-tests reported %d out-of-bounds value(s)", oob)
}
rows, wrong, rowsValid := parseNCCLWrongCounts(text)
if !rowsValid || rows == 0 {
return "UNSUPPORTED", "nccl-tests output has no recognized all_reduce_perf result rows"
}
if wrong != 0 {
return "FAILED", fmt.Sprintf("nccl-tests result rows reported %d wrong value(s)", wrong)
}
avgMatch := ncclAvgBusBWRE.FindStringSubmatch(text)
if len(avgMatch) != 2 {
return "UNSUPPORTED", "nccl-tests output has no recognized Avg bus bandwidth value"
}
avgBusBW, err := strconv.ParseFloat(avgMatch[1], 64)
if err != nil {
return "UNSUPPORTED", "nccl-tests Avg bus bandwidth is not numeric"
}
detail := fmt.Sprintf("validation passed; avg_bus_bandwidth=%.4g GB/s (diagnostic only, no unverified threshold)", avgBusBW)
if len(selectedGPUIndices) < 2 {
return "OK", detail
}
topoRaw, err := os.ReadFile(filepath.Join(runDir, "01-nvidia-smi-topo-m.log"))
if err != nil || !ncclGPUHeaderRE.Match(topoRaw) {
return "UNSUPPORTED", "NVIDIA topology output unavailable or unrecognized; cannot validate expected NVLink pairs"
}
pairs := selectedNvidiaNVLinkPairs(ParseNvidiaNVLinkBondedPairs(string(topoRaw)), selectedGPUIndices)
if len(pairs) == 0 {
return "OK", detail + "; selected topology has no NVLink-bonded pair"
}
statusRaw, err := os.ReadFile(filepath.Join(runDir, "01-nvidia-smi-nvlink-s.log"))
if err != nil {
return "UNSUPPORTED", "NVLink status output unavailable for selected NVLink-bonded pair"
}
linkStatus := parseNvidiaNVLinkStatus(string(statusRaw))
var issues []string
for _, pair := range pairs {
finding := evaluateNvidiaNVLinkPair(pair, linkStatus, nil)
for _, issue := range finding.Issues {
issues = append(issues, fmt.Sprintf("GPU%d<->GPU%d: %s", pair.GPUA, pair.GPUB, issue))
}
}
if len(issues) != 0 {
return "FAILED", "NVLink state does not match selected topology: " + strings.Join(issues, "; ")
}
return "OK", detail + "; selected NVLink endpoints match topology"
}
func parseNCCLWrongCounts(raw string) (rows int, wrong uint64, valid bool) {
for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
if len(fields) < 13 {
continue
}
if _, err := strconv.ParseUint(fields[0], 10, 64); err != nil {
continue
}
outOfPlaceWrong, errA := strconv.ParseUint(fields[8], 10, 64)
inPlaceWrong, errB := strconv.ParseUint(fields[12], 10, 64)
if errA != nil || errB != nil {
return rows, wrong, false
}
rows++
wrong += outOfPlaceWrong + inPlaceWrong
}
return rows, wrong, true
}
func selectedNvidiaNVLinkPairs(pairs []NvidiaNVLinkBondedPair, selected []int) []NvidiaNVLinkBondedPair {
selectedSet := make(map[int]struct{}, len(selected))
for _, index := range selected {
selectedSet[index] = struct{}{}
}
filtered := make([]NvidiaNVLinkBondedPair, 0, len(pairs))
for _, pair := range pairs {
_, aSelected := selectedSet[pair.GPUA]
_, bSelected := selectedSet[pair.GPUB]
if aSelected && bSelected {
filtered = append(filtered, pair)
}
}
return filtered
}
audit/internal/platform/sat_nccl_validation_test.go
+136
View File
@@ -0,0 +1,136 @@
package platform
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
)
const ncclValidOutput = `# size count type redop root time algbw busbw #wrong time algbw busbw #wrong
536870912 134217728 float sum -1 19547 27.47 48.07 0 19512 27.52 48.15 0
# Out of bounds values : 0 OK
# Avg bus bandwidth : 48.1908
`
func TestValidateNCCLAllReduceOutputHealthyNVLinkPair(t *testing.T) {
runDir := t.TempDir()
writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tNV2\nGPU1\tNV2\tX\n")
writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-nvlink-s.log", `GPU 0: H100
Link 0: 26.562 GB/s
Link 1: 26.562 GB/s
GPU 1: H100
Link 0: 26.562 GB/s
Link 1: 26.562 GB/s
`)
status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1})
if status != "OK" {
t.Fatalf("status=%q detail=%q want OK", status, detail)
}
}
func TestValidateNCCLAllReduceOutputFailsInactiveNVLinkEndpoint(t *testing.T) {
runDir := t.TempDir()
writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tNV2\nGPU1\tNV2\tX\n")
writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-nvlink-s.log", `GPU 0: H100
Link 0: 26.562 GB/s
Link 1: 26.562 GB/s
GPU 1: H100
NVML: Unable to retrieve NVLink information as all links are inActive
`)
status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1})
if status != "FAILED" || !strings.Contains(detail, "GPU1 0/2") {
t.Fatalf("status=%q detail=%q want FAILED with GPU1 0/2", status, detail)
}
}
func TestValidateNCCLAllReduceOutputRejectsWrongValues(t *testing.T) {
bad := strings.Replace(ncclValidOutput, "48.07 0", "48.07 2", 1)
status, detail := validateNCCLAllReduceOutput(t.TempDir(), []byte(bad), []int{0})
if status != "FAILED" || !strings.Contains(detail, "2 wrong") {
t.Fatalf("status=%q detail=%q want wrong-value failure", status, detail)
}
}
func TestValidateNCCLAllReduceOutputPCIeOnlyDoesNotRequireNVLink(t *testing.T) {
runDir := t.TempDir()
writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tPIX\nGPU1\tPIX\tX\n")
status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1})
if status != "OK" || !strings.Contains(detail, "no NVLink-bonded pair") {
t.Fatalf("status=%q detail=%q want PCIe-only OK", status, detail)
}
}
func TestValidateNCCLAllReduceOutputUnknownFormatIsUnsupported(t *testing.T) {
status, _ := validateNCCLAllReduceOutput(t.TempDir(), []byte("all done\n"), []int{0})
if status != "UNSUPPORTED" {
t.Fatalf("status=%q want UNSUPPORTED", status)
}
}
func TestRunAcceptancePackAppliesOutputValidator(t *testing.T) {
runDir, err := runAcceptancePackCtx(context.Background(), t.TempDir(), "validator-test", []satJob{{
name: "01-result.log",
cmd: []string{"sh", "-c", "printf command-ok"},
validate: func(string, []byte) (string, string) {
return "FAILED", "documented output check failed"
},
}}, nil)
if err != nil {
t.Fatal(err)
}
summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(summary), "overall_status=FAILED") ||
!strings.Contains(string(summary), "1-result_detail=documented output check failed") {
t.Fatalf("summary did not retain validator failure:\n%s", summary)
}
logData, err := os.ReadFile(filepath.Join(runDir, "01-result.log"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(logData), "[bee-validator] FAILED: documented output check failed") {
t.Fatalf("job log did not retain validator result:\n%s", logData)
}
}
func TestNCCLJobHealthAndDebugConfiguration(t *testing.T) {
jobs := ncclSATJobs([]int{0, 2})
var job satJob
for _, candidate := range jobs {
if candidate.name == "02-all-reduce-perf.log" {
job = candidate
break
}
}
if job.name == "" || job.validate == nil {
t.Fatal("all_reduce_perf job or its output validator is missing")
}
if !nvidiaJobNeedsHealthCheck(job) {
t.Fatal("all_reduce_perf job must run NVIDIA health checks")
}
if !isNvidiaAcceptancePack("nccl-tests") {
t.Fatal("nccl-tests must produce NVIDIA per-GPU status files")
}
env := strings.Join(job.env, "\n")
for _, expected := range []string{
"CUDA_DEVICE_ORDER=PCI_BUS_ID",
"CUDA_VISIBLE_DEVICES=0,2",
"NCCL_DEBUG=INFO",
"NCCL_DEBUG_SUBSYS=INIT,GRAPH",
} {
if !strings.Contains(env, expected) {
t.Fatalf("job env=%q missing %q", env, expected)
}
}
}
func writeNCCLValidationArtifact(t *testing.T, dir, name, body string) {
t.Helper()
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0644); err != nil {
t.Fatal(err)
}
}
audit/internal/platform/sat_nvidia_health.go
+126
View File
@@ -0,0 +1,126 @@
package platform
import (
"errors"
"fmt"
"strconv"
"strings"
)
func nvidiaSATStatusSeverity(status string) int {
switch strings.ToUpper(strings.TrimSpace(status)) {
case "FAILED":
return 3
case "PARTIAL", "UNSUPPORTED":
return 2
case "OK":
return 1
default:
return 0
}
}
func firstLine(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return ""
}
if idx := strings.IndexByte(s, '\n'); idx >= 0 {
return strings.TrimSpace(s[:idx])
}
return s
}
func nvidiaJobNeedsHealthCheck(job satJob) bool {
if job.collectGPU {
return true
}
name := strings.ToLower(strings.TrimSpace(job.name))
return strings.Contains(name, "dcgmi") ||
strings.Contains(name, "all-reduce") ||
strings.Contains(name, "gpu-burn") ||
strings.Contains(name, "gpu-stress") ||
strings.Contains(name, "dcgmproftester")
}
func checkNvidiaJobHealth(selected []int) (string, error) {
health, _ := readNvidiaGPUHealth()
var bad []nvidiaGPUHealth
selectedSet := make(map[int]struct{}, len(selected))
for _, idx := range selected {
selectedSet[idx] = struct{}{}
}
for _, gpu := range health {
if len(selectedSet) > 0 {
if _, ok := selectedSet[gpu.Index]; !ok {
continue
}
}
if gpu.NeedsReset {
bad = append(bad, gpu)
}
}
var degraded []nvidiaNVLinkDegradedEvent
if out, err := satExecCommand("dmesg").Output(); err == nil {
for _, event := range parseNvidiaNVLinkDegradedDmesg(out) {
if event.GPUIndex >= 0 && len(selectedSet) > 0 {
if _, ok := selectedSet[event.GPUIndex]; !ok {
continue
}
}
degraded = append(degraded, event)
}
}
if len(bad) == 0 && len(degraded) == 0 {
return "", nil
}
lines := make([]string, 0, len(bad)+len(degraded)+1)
lines = append(lines, "NVIDIA GPU health check failed:")
for _, gpu := range bad {
lines = append(lines, fmt.Sprintf("gpu %d (%s) requires reset: %s", gpu.Index, gpu.Name, gpu.RawLine))
}
for _, event := range degraded {
lines = append(lines, event.Warning())
}
return strings.Join(lines, "\n"), errors.New("nvidia gpu health check failed")
}
func readNvidiaGPUHealth() ([]nvidiaGPUHealth, error) {
out, err := satExecCommand(
"nvidia-smi",
"--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total",
"--format=csv,noheader,nounits",
).Output()
if err != nil {
return nil, fmt.Errorf("nvidia-smi: %w", err)
}
return parseNvidiaGPUHealth(string(out)), nil
}
func parseNvidiaGPUHealth(raw string) []nvidiaGPUHealth {
var gpus []nvidiaGPUHealth
for _, line := range strings.Split(strings.TrimSpace(raw), "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
parts := strings.Split(line, ",")
if len(parts) < 2 {
gpus = append(gpus, nvidiaGPUHealth{RawLine: line, ParseFailure: true})
continue
}
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
if err != nil {
gpus = append(gpus, nvidiaGPUHealth{RawLine: line, ParseFailure: true})
continue
}
upper := strings.ToUpper(line)
gpus = append(gpus, nvidiaGPUHealth{
Index: idx,
Name: strings.TrimSpace(parts[1]),
NeedsReset: strings.Contains(upper, "GPU REQUIRES RESET"),
RawLine: line,
})
}
return gpus
}
audit/internal/platform/sat_test.go
+26
View File
@@ -273,6 +273,32 @@ func TestCheckNvidiaJobHealthReturnsErrorForSelectedResetRequiredGPU(t *testing.
}
}
func TestCheckNvidiaJobHealthReturnsErrorForSelectedNVLinkDegradedGPU(t *testing.T) {
oldExecCommand := satExecCommand
satExecCommand = func(name string, args ...string) *exec.Cmd {
switch name {
case "nvidia-smi":
return exec.Command("sh", "-c", "printf '0, NVIDIA H100 PCIe, 38, 46.89, 0, 0, 81559\n1, NVIDIA H100 PCIe, 39, 47.00, 0, 0, 81559\n'")
case "dmesg":
return exec.Command("sh", "-c", "printf 'NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n'")
default:
return exec.Command(name, args...)
}
}
t.Cleanup(func() { satExecCommand = oldExecCommand })
msg, err := checkNvidiaJobHealth([]int{1})
if err == nil {
t.Fatal("expected NVLink degraded health check error")
}
if !strings.Contains(msg, "GPU1 NVLink degraded mode (linkId 1)") {
t.Fatalf("unexpected message: %q", msg)
}
if msg, err := checkNvidiaJobHealth([]int{0}); err != nil || msg != "" {
t.Fatalf("unaffected selected GPU0 got msg=%q err=%v", msg, err)
}
}
func TestWriteNvidiaGPUStatusFilesCreatesPerGPUFiles(t *testing.T) {
dir := t.TempDir()
oldExecCommand := satExecCommand
audit/internal/webui/api_gpu_tools.go
+71 -82
View File
@@ -1,6 +1,7 @@
package webui
import (
"context"
"encoding/json"
"fmt"
"net/http"
@@ -16,35 +17,15 @@ import (
)
func (h *handler) handleAPIGNVIDIAGPUs(w http.ResponseWriter, _ *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
gpus, err := h.opts.App.ListNvidiaGPUs()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if gpus == nil {
gpus = []platform.NvidiaGPU{}
}
writeJSON(w, gpus)
writeAPIListResponse(w, h.opts.App != nil, func() ([]platform.NvidiaGPU, error) {
return h.opts.App.ListNvidiaGPUs()
})
}
func (h *handler) handleAPIGNVIDIAGPUStatuses(w http.ResponseWriter, _ *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
gpus, err := apiListNvidiaGPUStatuses(h.opts.App)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if gpus == nil {
gpus = []platform.NvidiaGPUStatus{}
}
writeJSON(w, gpus)
writeAPIListResponse(w, h.opts.App != nil, func() ([]platform.NvidiaGPUStatus, error) {
return apiListNvidiaGPUStatuses(h.opts.App)
})
}
func (h *handler) handleAPIGNVIDIAReset(w http.ResponseWriter, r *http.Request) {
@@ -70,64 +51,33 @@ func (h *handler) handleAPIGNVIDIAReset(w http.ResponseWriter, r *http.Request)
// ── GPU settings (ECC / power limit) ──────────────────────────────────────────
func (h *handler) handleAPIGNVIDIAGPUSettings(w http.ResponseWriter, _ *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
settings, err := h.opts.App.ListNvidiaGPUSettings()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if settings == nil {
settings = []platform.NvidiaGPUSetting{}
}
writeJSON(w, settings)
writeAPIListResponse(w, h.opts.App != nil, func() ([]platform.NvidiaGPUSetting, error) {
return h.opts.App.ListNvidiaGPUSettings()
})
}
func (h *handler) handleAPIGNVIDIASetECC(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var req struct {
Index int `json:"index"`
Enabled bool `json:"enabled"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
result, err := h.opts.App.SetNvidiaGPUECC(req.Index, req.Enabled)
status := "ok"
if err != nil {
status = "error"
}
writeJSON(w, map[string]string{"status": status, "output": result.Body})
h.handleAPIGNVIDIASetBool(w, r, func(index int, enabled bool) (string, error) {
result, err := h.opts.App.SetNvidiaGPUECC(index, enabled)
return result.Body, err
})
}
func (h *handler) handleAPIGNVIDIASetMIG(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var req struct {
Index int `json:"index"`
Enabled bool `json:"enabled"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
result, err := h.opts.App.SetNvidiaGPUMIG(req.Index, req.Enabled)
status := "ok"
if err != nil {
status = "error"
}
writeJSON(w, map[string]string{"status": status, "output": result.Body})
h.handleAPIGNVIDIASetBool(w, r, func(index int, enabled bool) (string, error) {
result, err := h.opts.App.SetNvidiaGPUMIG(index, enabled)
return result.Body, err
})
}
func (h *handler) handleAPIGNVIDIASetCCMode(w http.ResponseWriter, r *http.Request) {
h.handleAPIGNVIDIASetBool(w, r, func(index int, enabled bool) (string, error) {
result, err := h.opts.App.SetNvidiaGPUCCMode(index, enabled)
return result.Body, err
})
}
func (h *handler) handleAPIGNVIDIASetBool(w http.ResponseWriter, r *http.Request, apply func(int, bool) (string, error)) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
@@ -140,12 +90,12 @@ func (h *handler) handleAPIGNVIDIASetCCMode(w http.ResponseWriter, r *http.Reque
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
result, err := h.opts.App.SetNvidiaGPUCCMode(req.Index, req.Enabled)
output, err := apply(req.Index, req.Enabled)
status := "ok"
if err != nil {
status = "error"
}
writeJSON(w, map[string]string{"status": status, "output": result.Body})
writeJSON(w, map[string]string{"status": status, "output": output})
}
func (h *handler) handleAPIGNVIDIASetPowerLimit(w http.ResponseWriter, r *http.Request) {
@@ -348,6 +298,9 @@ func validTimezoneName(tz string) bool {
// values supplied by the client's browser (used when the appliance has no
// network/NTP access to keep its own clock in sync).
func (h *handler) handleAPISystemTimeSync(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
defer cancel()
var req struct {
Timezone string `json:"timezone"`
EpochMS int64 `json:"epoch_ms"`
@@ -368,19 +321,25 @@ func (h *handler) handleAPISystemTimeSync(w http.ResponseWriter, r *http.Request
writeError(w, http.StatusBadRequest, "invalid timezone")
return
}
if b, err := exec.Command("timedatectl", "set-timezone", req.Timezone).CombinedOutput(); err != nil {
writeError(w, http.StatusInternalServerError, "set-timezone failed: "+strings.TrimSpace(string(b)))
if b, err := exec.CommandContext(ctx, "timedatectl", "set-timezone", req.Timezone).CombinedOutput(); err != nil {
writeError(w, http.StatusInternalServerError, "set-timezone failed: "+commandFailureDetail(b, err))
return
}
fmt.Fprintf(&out, "timezone set to %s\n", req.Timezone)
}
// Manual time only sticks if NTP sync is off.
_ = exec.Command("timedatectl", "set-ntp", "false").Run()
if b, err := exec.CommandContext(ctx, "timedatectl", "set-ntp", "false").CombinedOutput(); err != nil {
canNTP, canErr := exec.CommandContext(ctx, "timedatectl", "show", "-p", "CanNTP", "--value").Output()
if canErr != nil || strings.TrimSpace(string(canNTP)) != "no" {
writeError(w, http.StatusInternalServerError, "disable NTP failed: "+commandFailureDetail(b, err))
return
}
}
sec := req.EpochMS / 1000
if b, err := exec.Command("date", "-u", "-s", "@"+strconv.FormatInt(sec, 10)).CombinedOutput(); err != nil {
writeError(w, http.StatusInternalServerError, "set-time failed: "+strings.TrimSpace(string(b)))
if b, err := exec.CommandContext(ctx, "date", "-u", "-s", "@"+strconv.FormatInt(sec, 10)).CombinedOutput(); err != nil {
writeError(w, http.StatusInternalServerError, "set-time failed: "+commandFailureDetail(b, err))
return
}
out.WriteString("system clock synced\n")
@@ -388,6 +347,36 @@ func (h *handler) handleAPISystemTimeSync(w http.ResponseWriter, r *http.Request
writeJSON(w, map[string]string{"status": "ok", "output": out.String()})
}
func commandFailureDetail(output []byte, err error) string {
if detail := strings.TrimSpace(string(output)); detail != "" {
return detail
}
return err.Error()
}
// handleAPISystemTime reports the host's current wall-clock time and configured
// timezone so the dashboard can show them next to the browser's own clock and
// flag a drift.
func (h *handler) handleAPISystemTime(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
defer cancel()
tz := ""
if b, err := exec.CommandContext(ctx, "timedatectl", "show", "-p", "Timezone", "--value").Output(); err == nil {
tz = strings.TrimSpace(string(b))
}
now := time.Now()
if tz == "" {
tz, _ = now.Zone()
}
writeJSON(w, map[string]any{
"epoch_ms": now.UnixMilli(),
"local_time": now.Format("2006-01-02 15:04:05"),
"timezone": tz,
})
}
// ── Tools ─────────────────────────────────────────────────────────────────────
var standardTools = []string{
audit/internal/webui/api_runs.go
-17
View File
@@ -35,15 +35,10 @@ func (h *handler) handleAPIAuditStream(w http.ResponseWriter, r *http.Request) {
if id == "" {
id = r.URL.Query().Get("task_id")
}
// Try task queue first, then legacy job manager
if j, ok := globalQueue.findJob(id); ok {
streamJob(w, r, j)
return
}
if j, ok := globalJobs.get(id); ok {
streamJob(w, r, j)
return
}
http.Error(w, "job not found", http.StatusNotFound)
}
@@ -383,10 +378,6 @@ func (h *handler) handleAPISATStream(w http.ResponseWriter, r *http.Request) {
streamJob(w, r, j)
return
}
if j, ok := globalJobs.get(id); ok {
streamJob(w, r, j)
return
}
http.Error(w, "job not found", http.StatusNotFound)
}
@@ -416,14 +407,6 @@ func (h *handler) handleAPISATAbort(w http.ResponseWriter, r *http.Request) {
writeJSON(w, map[string]string{"status": "aborted"})
return
}
if j, ok := globalJobs.get(id); ok {
if j.abort() {
writeJSON(w, map[string]string{"status": "aborted"})
} else {
writeJSON(w, map[string]string{"status": "not_running"})
}
return
}
http.Error(w, "job not found", http.StatusNotFound)
}
audit/internal/webui/api_services_system.go
+7 -14
View File
@@ -99,7 +99,10 @@ func (h *handler) handleAPINetworkDHCP(w http.ResponseWriter, r *http.Request) {
var req struct {
Interface string `json:"interface"`
}
_ = json.NewDecoder(r.Body).Decode(&req)
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
result, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) {
if req.Interface == "" || req.Interface == "all" {
@@ -167,19 +170,9 @@ func (h *handler) handleAPIExportList(w http.ResponseWriter, r *http.Request) {
}
func (h *handler) handleAPIExportUSBTargets(w http.ResponseWriter, _ *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
targets, err := h.opts.App.ListRemovableTargets()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if targets == nil {
targets = []platform.RemovableTarget{}
}
writeJSON(w, targets)
writeAPIListResponse(w, h.opts.App != nil, func() ([]platform.RemovableTarget, error) {
return h.opts.App.ListRemovableTargets()
})
}
func (h *handler) handleAPIBlackboxStatus(w http.ResponseWriter, _ *http.Request) {
audit/internal/webui/api_system_test.go
+50
View File
@@ -0,0 +1,50 @@
package webui
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"bee/audit/internal/app"
)
func TestSystemTimeEndpointReportsEpochLocalTimeAndTimezone(t *testing.T) {
before := time.Now().Add(-time.Second).UnixMilli()
rec := httptest.NewRecorder()
NewHandler(HandlerOptions{}).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/system/time", nil))
after := time.Now().Add(time.Second).UnixMilli()
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
}
var body struct {
EpochMS int64 `json:"epoch_ms"`
LocalTime string `json:"local_time"`
Timezone string `json:"timezone"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatalf("decode response: %v", err)
}
if body.EpochMS < before || body.EpochMS > after {
t.Fatalf("epoch_ms = %d, want value in [%d, %d]", body.EpochMS, before, after)
}
if _, err := time.ParseInLocation("2006-01-02 15:04:05", body.LocalTime, time.Local); err != nil {
t.Fatalf("local_time = %q: %v", body.LocalTime, err)
}
if strings.TrimSpace(body.Timezone) == "" {
t.Fatal("timezone must not be empty")
}
}
func TestNetworkDHCPRejectsMalformedJSONBeforeChangingNetwork(t *testing.T) {
h := &handler{opts: HandlerOptions{App: &app.App{}}}
rec := httptest.NewRecorder()
h.handleAPINetworkDHCP(rec, httptest.NewRequest(http.MethodPost, "/api/network/dhcp", strings.NewReader("{")))
if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest)
}
}
audit/internal/webui/jobs.go
-31
View File
@@ -7,7 +7,6 @@ import (
"os"
"strings"
"sync"
"time"
)
// jobState holds the output lines and completion status of an async job.
@@ -134,29 +133,6 @@ func (j *jobState) subscribe() ([]string, <-chan string) {
return existing, ch
}
// jobManager manages async jobs identified by string IDs.
type jobManager struct {
mu sync.Mutex
jobs map[string]*jobState
}
var globalJobs = &jobManager{jobs: make(map[string]*jobState)}
func (m *jobManager) create(id string) *jobState {
m.mu.Lock()
defer m.mu.Unlock()
j := &jobState{}
m.jobs[id] = j
// Schedule cleanup after 30 minutes
goRecoverOnce("job cleanup", func() {
time.Sleep(30 * time.Minute)
m.mu.Lock()
delete(m.jobs, id)
m.mu.Unlock()
})
return j
}
// isDone returns true if the job has finished (either successfully or with error).
func (j *jobState) isDone() bool {
j.mu.Lock()
@@ -164,13 +140,6 @@ func (j *jobState) isDone() bool {
return j.done
}
func (m *jobManager) get(id string) (*jobState, bool) {
m.mu.Lock()
defer m.mu.Unlock()
j, ok := m.jobs[id]
return j, ok
}
func newTaskJobState(logPath string, serialPrefix ...string) *jobState {
j := &jobState{logPath: logPath}
if len(serialPrefix) > 0 {
audit/internal/webui/metrics_charts_http.go
-31
View File
@@ -400,37 +400,6 @@ func normalizePowerSeries(ds []float64) []float64 {
return out
}
// psuSlotsFromSamples returns the sorted list of PSU slot numbers seen across samples.
func psuSlotsFromSamples(samples []platform.LiveMetricSample) []int {
seen := map[int]struct{}{}
for _, s := range samples {
for _, p := range s.PSUs {
seen[p.Slot] = struct{}{}
}
}
slots := make([]int, 0, len(seen))
for s := range seen {
slots = append(slots, s)
}
sort.Ints(slots)
return slots
}
// psuStackedTotal returns the point-by-point sum of all PSU datasets (for scale calculation).
func psuStackedTotal(datasets [][]float64) []float64 {
if len(datasets) == 0 {
return nil
}
n := len(datasets[0])
total := make([]float64, n)
for _, ds := range datasets {
for i, v := range ds {
total[i] += v
}
}
return total
}
func normalizeFanSeries(ds []float64) []float64 {
if len(ds) == 0 {
return nil
audit/internal/webui/page_benchmark.go
-4
View File
@@ -611,7 +611,3 @@ func renderPowerBenchmarkResultsCard(exportDir string) string {
b.WriteString(`</div></div>`)
return b.String()
}
// renderSpeed and renderEndurance are legacy wrappers; canonical page is 5. Benchmark at /benchmark.
func renderSpeed(opts HandlerOptions) string { return renderBenchmark(opts) }
func renderEndurance(opts HandlerOptions) string { return renderBenchmark(opts) }
audit/internal/webui/page_topo.go
+3 -91
View File
@@ -11,6 +11,7 @@ import (
"strconv"
"strings"
"bee/audit/internal/platform"
"bee/audit/internal/schema"
)
@@ -246,17 +247,7 @@ func buildMemoryColumnIndex(rawNodes []int) map[int]int {
// GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query)
// ---------------------------------------------------------------------------
type gpuPairLink struct {
GPUA, GPUB int
NVLinks int
}
var topoNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
// nvidia-smi underlines the topo -m header row with ANSI CSI sequences
// (ESC[4m...ESC[0m) even when stdout is not a TTY, so the captured techdump
// contains them and "GPU0" is not at the start of the trimmed header line.
var topoANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]")
type gpuPairLink = platform.NvidiaNVLinkBondedPair
// parseGPUPairAdjacency returns every GPU pair with a nonzero NVLink bond
// count from a "nvidia-smi topo -m" matrix. Unlike parseNVIDIATopologyMatrix
@@ -264,86 +255,7 @@ var topoANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]")
// who is bonded to whom — required so GPU-GPU edges are drawn for actually
// bonded pairs, not for adjacent boxes in the layout.
func parseGPUPairAdjacency(raw string) []gpuPairLink {
lines := strings.Split(topoANSIRe.ReplaceAllString(raw, ""), "\n")
headerIdx := -1
var gpuColIndices []int
for i, line := range lines {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "GPU0") {
parts := strings.Fields(trimmed)
for j, col := range parts {
if strings.HasPrefix(col, "GPU") {
gpuColIndices = append(gpuColIndices, j)
}
}
if len(gpuColIndices) >= 2 {
headerIdx = i
}
break
}
}
if headerIdx < 0 {
return nil
}
colIdxToGPU := make(map[int]int, len(gpuColIndices))
for gpuIdx, colIdx := range gpuColIndices {
colIdxToGPU[colIdx] = gpuIdx
}
seen := map[[2]int]bool{}
var pairs []gpuPairLink
rowGPU := -1
for _, line := range lines[headerIdx+1:] {
trimmed := strings.TrimSpace(line)
if !strings.HasPrefix(trimmed, "GPU") {
continue
}
cells := strings.Fields(trimmed)
if len(cells) == 0 {
continue
}
rowLabel := strings.TrimPrefix(cells[0], "GPU")
n, err := strconv.Atoi(rowLabel)
if err != nil {
continue
}
rowGPU = n
for colIdx, colGPU := range colIdxToGPU {
if colGPU == rowGPU {
continue
}
dataIdx := colIdx + 1
if dataIdx >= len(cells) {
continue
}
m := topoNVRe.FindStringSubmatch(cells[dataIdx])
if len(m) != 2 {
continue
}
nv, err := strconv.Atoi(m[1])
if err != nil || nv <= 0 {
continue
}
a, bGPU := rowGPU, colGPU
if a > bGPU {
a, bGPU = bGPU, a
}
key := [2]int{a, bGPU}
if seen[key] {
continue
}
seen[key] = true
pairs = append(pairs, gpuPairLink{GPUA: a, GPUB: bGPU, NVLinks: nv})
}
}
sort.Slice(pairs, func(i, j int) bool {
if pairs[i].GPUA != pairs[j].GPUA {
return pairs[i].GPUA < pairs[j].GPUA
}
return pairs[i].GPUB < pairs[j].GPUB
})
return pairs
return platform.ParseNvidiaNVLinkBondedPairs(raw)
}
// readTopoTechDump reads a file previously captured into the persistent
audit/internal/webui/page_topo_diagram.go
+2 -33
View File
@@ -10,6 +10,7 @@ import (
"strings"
"bee/audit/internal/app"
"bee/audit/internal/platform"
"bee/audit/internal/schema"
)
@@ -499,7 +500,6 @@ var (
topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
)
func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) {
@@ -549,38 +549,7 @@ func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) {
// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc].
func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 {
result := map[int]map[int][3]int64{}
currentGPU := -1
for _, line := range strings.Split(raw, "\n") {
trimmed := strings.TrimSpace(line)
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
currentGPU, _ = strconv.Atoi(m[1])
continue
}
if currentGPU < 0 {
continue
}
m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed)
if m == nil {
continue
}
linkIdx, _ := strconv.Atoi(m[1])
count, _ := strconv.ParseInt(m[3], 10, 64)
if result[currentGPU] == nil {
result[currentGPU] = map[int][3]int64{}
}
c := result[currentGPU][linkIdx]
switch m[2] {
case "Replay":
c[0] = count
case "Recovery":
c[1] = count
case "CRC":
c[2] = count
}
result[currentGPU][linkIdx] = c
}
return result
return platform.ParseNvidiaNVLinkErrors(raw)
}
// renderTopoNVLinkCard renders the separate NVLink topology card. Returns ""
audit/internal/webui/pages.go
+67 -6
View File
@@ -150,8 +150,9 @@ setInterval(function(){
return b.String()
}
// renderTimeSyncCard shows the server's current time and a button that syncs
// the host clock and timezone to whatever the client's browser reports.
// renderTimeSyncCard shows the server's current clock and timezone next to the
// browser's own, highlights any drift, and offers a button that syncs the host
// clock and timezone to whatever the client's browser reports.
func renderTimeSyncCard() string {
return `<div class="card" style="margin-bottom:16px">
<div class="card-head card-head-actions">
@@ -161,11 +162,66 @@ func renderTimeSyncCard() string {
</div>
</div>
<div class="card-body">
<table style="font-size:13px;border-collapse:collapse">
<tr><td style="padding:2px 16px 2px 0;color:var(--muted)">Server</td>
<td style="padding:2px 24px 2px 0" id="time-server-clock">—</td>
<td style="padding:2px 0" id="time-server-tz">—</td></tr>
<tr><td style="padding:2px 16px 2px 0;color:var(--muted)">Browser</td>
<td style="padding:2px 24px 2px 0" id="time-browser-clock">—</td>
<td style="padding:2px 0" id="time-browser-tz">—</td></tr>
</table>
<div id="time-drift-note" style="font-size:13px;margin-top:8px"></div>
<span id="time-sync-status" style="font-size:13px;color:var(--muted)"></span>
</div>
</div>
<script>
function timeSyncRun() {
(function(){
var CRIT = 'var(--crit-fg,#9f3a38)';
var OK = 'var(--ok-fg,#2c662d)';
var refreshPending = false;
function refreshTime() {
var browserTz = Intl.DateTimeFormat().resolvedOptions().timeZone || '';
document.getElementById('time-browser-clock').textContent = new Date().toLocaleString();
document.getElementById('time-browser-tz').textContent = browserTz;
if (refreshPending) return Promise.resolve();
refreshPending = true;
return fetch('/api/system/time', {cache: 'no-store'})
.then(function(r){ if(!r.ok) throw new Error(r.statusText); return r.json(); })
.then(function(d){
var serverMs = d.epoch_ms;
var serverTz = d.timezone || '';
var skewMs = Math.abs(Date.now() - serverMs);
document.getElementById('time-server-clock').textContent = d.local_time || new Date(serverMs).toISOString();
document.getElementById('time-server-tz').textContent = serverTz;
var clockBad = skewMs > 60000;
var tzBad = serverTz !== browserTz;
document.getElementById('time-server-clock').style.color = clockBad ? CRIT : '';
document.getElementById('time-browser-clock').style.color = clockBad ? CRIT : '';
document.getElementById('time-server-tz').style.color = tzBad ? CRIT : '';
document.getElementById('time-browser-tz').style.color = tzBad ? CRIT : '';
var note = document.getElementById('time-drift-note');
if (clockBad || tzBad) {
var parts = [];
if (clockBad) parts.push('clock differs by ' + Math.round(skewMs/1000) + 's');
if (tzBad) parts.push('timezone mismatch');
note.style.color = CRIT;
note.textContent = '⚠ ' + parts.join(', ') + ' — click "Sync with this browser"';
} else {
note.style.color = OK;
note.textContent = '✓ server clock and timezone match this browser';
}
})
.catch(function(){
document.getElementById('time-server-clock').textContent = 'unavailable';
})
.finally(function(){ refreshPending = false; });
}
window.timeSyncRun = function() {
var btn = document.getElementById('time-sync-btn');
var status = document.getElementById('time-sync-status');
btn.disabled = true;
@@ -177,15 +233,20 @@ function timeSyncRun() {
})
.then(function(r) { if (!r.ok) return r.text().then(function(t){throw new Error(t || r.statusText);}); return r.json(); })
.then(function(d) {
status.style.color = 'var(--ok-fg,#2c662d)';
status.style.color = OK;
status.textContent = '✓ Synced to ' + tz + ' at ' + new Date().toLocaleString();
refreshTime();
})
.catch(function(err) {
status.style.color = 'var(--crit-fg,#9f3a38)';
status.style.color = CRIT;
status.textContent = '✗ Sync failed: ' + err.message;
})
.finally(function() { btn.disabled = false; });
}
};
refreshTime();
setInterval(refreshTime, 5000);
})();
</script>`
}
audit/internal/webui/server.go
+17
View File
@@ -348,6 +348,7 @@ func NewHandler(opts HandlerOptions) http.Handler {
mux.HandleFunc("POST /api/system/install-to-ram", h.handleAPIInstallToRAM)
mux.HandleFunc("POST /api/system/reboot", h.handleAPISystemReboot)
mux.HandleFunc("POST /api/system/shutdown", h.handleAPISystemShutdown)
mux.HandleFunc("GET /api/system/time", h.handleAPISystemTime)
mux.HandleFunc("POST /api/system/time-sync", h.handleAPISystemTimeSync)
// Preflight
@@ -813,3 +814,19 @@ func writeError(w http.ResponseWriter, status int, msg string) {
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
}
func writeAPIListResponse[T any](w http.ResponseWriter, configured bool, load func() ([]T, error)) {
if !configured {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
items, err := load()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if items == nil {
items = []T{}
}
writeJSON(w, items)
}
audit/internal/webui/tasks.go
+3 -1
View File
@@ -519,7 +519,9 @@ func (q *taskQueue) startWorker(opts *HandlerOptions) {
q.opts = opts
q.statePath = filepath.Join(opts.ExportDir, "tasks-state.json")
q.logsDir = filepath.Join(opts.ExportDir, "tasks")
_ = os.MkdirAll(q.logsDir, 0755)
if err := os.MkdirAll(q.logsDir, 0755); err != nil {
slog.Error("task queue: create logs directory", "path", q.logsDir, "error", err)
}
if !q.started {
q.loadLocked()
q.started = true
audit/internal/webui/tasks_http_persistence.go
+17 -5
View File
@@ -3,6 +3,7 @@ package webui
import (
"encoding/json"
"fmt"
"log/slog"
"net/http"
"os"
"path/filepath"
@@ -274,13 +275,17 @@ func (q *taskQueue) persistLocked() {
}
data, err := json.MarshalIndent(state, "", " ")
if err != nil {
slog.Error("task queue: marshal persistent state", "path", q.statePath, "error", err)
return
}
tmp := q.statePath + ".tmp"
if err := os.WriteFile(tmp, data, 0644); err != nil {
slog.Error("task queue: write persistent state", "path", tmp, "error", err)
return
}
_ = os.Rename(tmp, q.statePath)
if err := os.Rename(tmp, q.statePath); err != nil {
slog.Error("task queue: publish persistent state", "source", tmp, "path", q.statePath, "error", err)
}
}
func taskElapsedSec(t *Task, now time.Time) int {
@@ -388,7 +393,9 @@ func (q *taskQueue) ensureTaskArtifactPathsLocked(t *Task) {
t.ArtifactsDir = taskArtifactsDir(q.logsDir, t, t.Status)
}
if t.ArtifactsDir != "" {
_ = os.MkdirAll(t.ArtifactsDir, 0755)
if err := os.MkdirAll(t.ArtifactsDir, 0755); err != nil {
slog.Error("task queue: create artifacts directory", "path", t.ArtifactsDir, "task_id", t.ID, "error", err)
}
}
ensureTaskReportPaths(t)
}
@@ -403,10 +410,15 @@ func (q *taskQueue) finalizeTaskArtifactPathsLocked(t *Task) {
return
}
if t.ArtifactsDir != "" && t.ArtifactsDir != dstDir {
if _, err := os.Stat(dstDir); err != nil {
_ = os.Rename(t.ArtifactsDir, dstDir)
if _, err := os.Stat(dstDir); err == nil {
slog.Error("task queue: destination artifacts directory already exists", "source", t.ArtifactsDir, "destination", dstDir, "task_id", t.ID)
} else if !os.IsNotExist(err) {
slog.Error("task queue: inspect destination artifacts directory", "destination", dstDir, "task_id", t.ID, "error", err)
} else if err := os.Rename(t.ArtifactsDir, dstDir); err != nil {
slog.Error("task queue: move artifacts directory", "source", t.ArtifactsDir, "destination", dstDir, "task_id", t.ID, "error", err)
} else {
t.ArtifactsDir = dstDir
}
t.ArtifactsDir = dstDir
}
ensureTaskReportPaths(t)
}
audit/internal/webui/tasks_test.go
+23
View File
@@ -227,6 +227,29 @@ func TestTaskArtifactsDirStartsWithTaskNumber(t *testing.T) {
}
}
func TestFinalizeTaskArtifactPathsKeepsSourceWhenDestinationExists(t *testing.T) {
logsDir := t.TempDir()
sourceDir := filepath.Join(logsDir, "running-artifacts")
if err := os.Mkdir(sourceDir, 0755); err != nil {
t.Fatal(err)
}
task := &Task{ID: "TASK-007", Name: "NVIDIA Benchmark", Status: TaskDone, ArtifactsDir: sourceDir}
destinationDir := taskArtifactsDir(logsDir, task, TaskDone)
if err := os.Mkdir(destinationDir, 0755); err != nil {
t.Fatal(err)
}
q := &taskQueue{logsDir: logsDir}
q.finalizeTaskArtifactPathsLocked(task)
if task.ArtifactsDir != sourceDir {
t.Fatalf("artifacts dir = %q, want retained source %q after destination collision", task.ArtifactsDir, sourceDir)
}
if task.ReportJSONPath != filepath.Join(sourceDir, "report.json") {
t.Fatalf("report path = %q, want source directory", task.ReportJSONPath)
}
}
func TestHandleAPITasksStreamReplaysPersistedLogWithoutLiveJob(t *testing.T) {
dir := t.TempDir()
logPath := filepath.Join(dir, "task.log")
bible-local/architecture/api-surface.md
+5 -1
View File
@@ -125,7 +125,7 @@ All SAT run endpoints enqueue an async task. Response: `{"task_id": "..."}`.
| Method | Path | Description |
|--------|----------------------------|----------------------------------------------------|
| GET | `/api/gpu/presence` | `{"nvidia": bool, "amd": bool}` |
| GET | `/api/gpu/presence` | `{"nvidia", "amd", "nvidia_initializing", "amd_initializing": bool}`. `*_initializing` = PCI GPU present but driver not up yet (distinct from absent). Backed by `app.DetectGPUPresence`. |
| GET | `/api/gpu/nvidia` | List NVIDIA GPUs from nvidia-smi |
| GET | `/api/gpu/nvidia-status` | Per-GPU status (ECC, power, throttle) |
| POST | `/api/gpu/nvidia-reset` | GPU reset by index |
@@ -139,6 +139,10 @@ All SAT run endpoints enqueue an async task. Response: `{"task_id": "..."}`.
|--------|------------------------------|---------------------------------------------------|
| GET | `/api/system/ram-status` | toram boot state and ISO copy status |
| POST | `/api/system/install-to-ram` | Copy ISO to RAM (background task) |
| GET | `/api/system/time` | Current host epoch, local wall-clock text, and timezone |
| POST | `/api/system/time-sync` | Set host time and timezone from the browser |
| POST | `/api/system/reboot` | Reboot the host |
| POST | `/api/system/shutdown` | Power off the host |
| GET | `/api/install/disks` | List block devices suitable for disk installation |
| POST | `/api/install/run` | Install bee to disk (background task) |
bible-local/architecture/runtime-flows.md
+17
View File
@@ -263,6 +263,11 @@ bee-blackbox.service (separate process from bee-web/bee-audit)
a time-derived staging path: two builds in the same wall-clock second (two
operators, or an on-demand build racing the blackbox worker) then share one
tree and one's deferred `os.RemoveAll` truncates the other's archive.
- The resulting archive is also unique (`os.CreateTemp` with the readable
support-bundle name as its prefix) and remains a hidden `.partial` from
retention cleanup until an atomic rename publishes the closed `.tar.gz`.
A timestamp-only output path lets same-second writers share one inode and
can corrupt both gzip streams.
## NVIDIA SAT Web UI flow
@@ -306,3 +311,15 @@ Web UI: "Run All" button -> POST /api/sat/run-all
- `RunNvidiaBandwidthPack` runs one all-GPU `nvbandwidth` pass in Validate; the
per-NUMA-node matrix is Stress-tier only (`fullMatrix` arg). See
`decisions/2026-08-31-nvbandwidth-validate-single-deep-matrix.md`.
- `nvidia-config` validates the active-link count independently at both ends of
every `NVx` pair reported by `nvidia-smi topo -m`; aggregate counts must not
be compared with a per-endpoint `NVx` value. It also persists matching
NVIDIA degraded-mode kernel lines in `06-dmesg-nvlink-degraded.log`.
- Generic SAT jobs may attach an output validator. A validator runs only after
exit status 0 and returns `OK`, `FAILED`, or `UNSUPPORTED`; its detail is
persisted in both the job log and `<job>_detail` in `summary.txt`.
- `nccl-tests` explicitly validates the pinned `all_reduce_perf` correctness
summary and both `#wrong` columns, and cross-checks selected NVLink endpoints
against the captured topology/status. `Avg bus bandwidth` is retained as a
diagnostic value only: no model-independent bandwidth threshold is valid.
NCCL `P2P` alone is not proof of NVLink because P2P can also use PCIe.
bible-local/decisions/2026-08-31-support-bundle-private-staging-dir.md
+52
View File
@@ -0,0 +1,52 @@
# Support bundle: stage under a private os.MkdirTemp parent, not a time-derived path
**Date:** 2026-08-31
**Status:** active
## Context
`BuildSupportBundle` staged into
`os.TempDir()/bee-support-stage-<hostname>-<YYYYMMDD-HHMMSS>` and did
`defer os.RemoveAll(stageRoot)`. The name has one-second resolution and a
fixed hostname, so two builds started in the same wall-clock second resolve
to the same directory: one operator double-clicking "Download Support
Bundle", two operators, or an on-demand build racing the blackbox worker.
When that happened, the first build to finish ran `os.RemoveAll` on the
shared tree while the second was still populating it or archiving it. The
second build's `createSupportTarGz` then walked a half-deleted tree and
produced a truncated archive - with no error. It surfaced first as a flaky
test (`TestBuildSupportBundleIncludesExportDirContents` vs the webui
endpoint test, run in parallel by `go test ./...`), but the same race is
reachable in production.
## Decision
`BuildSupportBundle` creates a private parent with
`os.MkdirTemp(os.TempDir(), "bee-support-build-")` and stages inside it. The
leaf directory keeps the meaningful `bee-support-stage-<hostname>-<ts>` name
(it becomes the archive's top-level directory). `defer os.RemoveAll` targets
the unique parent. The output archive is created with `os.CreateTemp`, retaining
the human-readable prefix while guaranteeing a unique file for each concurrent
build. It is written under a non-published `.partial` suffix and atomically
renamed to `.tar.gz` only after tar, gzip, and file closes succeed, so retention
cleanup can only see completed archives.
Tests that exercise the real `BuildSupportBundle` set
`t.Setenv("TMPDIR", t.TempDir())` so each test's `os.TempDir()` - staging
parent, archive path, and the `cleanupOldSupportBundles` scan - is isolated
from parallel packages.
## Consequences
- Concurrent support-bundle builds are independent; neither can truncate the
other.
- DO NOT reintroduce a time-derived or otherwise non-unique staging path.
See `architecture/runtime-flows.md` (Blackbox sync flow invariants).
- The archive file itself still lands in `os.TempDir()` under a
human-readable `SupportBundleBaseName` prefix (callers `defer os.Remove` it),
with a random suffix preventing same-second writers from sharing one inode.
- Tar, gzip, source-file, and destination-file close failures are propagated;
an incomplete archive is removed instead of being returned as successful.
- DMI-derived version/model/serial components are sanitized before becoming
path components.
bible-local/decisions/README.md
+1
View File
@@ -15,3 +15,4 @@ One file per decision, named `YYYY-MM-DD-short-topic.md`.
| 2026-08-31 | nvbandwidth: single all-GPU pass in Validate, per-socket matrix only in deep tier | active |
| 2026-08-31 | bee-nvidia.service: never blocking `systemctl restart` on units ordered After= itself | active |
| 2026-08-31 | "Run All" SAT planning happens on the backend, not the browser | active |
| 2026-08-31 | Support bundle uses private staging and unique atomic output | active |
internal/chart
+1 -1
Submodule internal/chart updated: 2a15bc87f1...be68069a51
iso/builder/build.sh
+26 -21
View File
@@ -77,6 +77,7 @@ export GOCACHE GOMODCACHE
# Keep the entrypoint focused on build orchestration. These libraries only
# define helpers; stateful logging is loaded after its output paths are known.
. "${BUILDER_DIR}/lib/iso-validation.sh"
. "${BUILDER_DIR}/lib/template.sh"
. "${BUILDER_DIR}/lib/bootloader.sh"
. "${BUILDER_DIR}/lib/fast-path.sh"
@@ -158,11 +159,19 @@ sync_builder_workdir() {
PROJECT_VERSION_EFFECTIVE="$(resolve_project_version)"
if ! validate_project_version "${PROJECT_VERSION_EFFECTIVE}"; then
echo "ERROR: project version must start and end with an ASCII letter or digit and contain only letters, digits, dots, and hyphens: ${PROJECT_VERSION_EFFECTIVE}" >&2
exit 1
fi
SQUASHFS_FILENAME="filesystem-v${PROJECT_VERSION_EFFECTIVE}.squashfs"
ISO_BASENAME="easy-bee-${BUILD_VARIANT}-v${PROJECT_VERSION_EFFECTIVE}-amd64"
# Hostname baked into the live image: same product+version scheme as
# ISO_BASENAME/SQUASHFS_FILENAME, but hostname labels don't allow dots.
BEE_HOSTNAME="easy-bee-${BUILD_VARIANT}-v$(printf '%s' "${PROJECT_VERSION_EFFECTIVE}" | tr '.' '-')"
if [ "${#BEE_HOSTNAME}" -gt 63 ]; then
echo "ERROR: derived hostname exceeds the 63-byte DNS label limit: ${BEE_HOSTNAME}" >&2
exit 1
fi
# Versioned output directory: dist/easy-bee-v4.1/ - all final artifacts live here.
OUT_DIR="${DIST_DIR}/release/easy-bee-v${PROJECT_VERSION_EFFECTIVE}"
ISO_VERSION_LABEL_TOKEN="$(printf '%s' "${PROJECT_VERSION_EFFECTIVE}" | tr '[:lower:].-' '[:upper:]__')"
@@ -538,9 +547,7 @@ fi
# Patch motd with build info
BEE_BUILD_INFO="${BUILD_DATE} git:${GIT_COMMIT} debian:${DEBIAN_VERSION} ${GPU_BUILD_INFO}"
if [ -f "${OVERLAY_STAGE_DIR}/etc/motd" ]; then
sed "s/%%BUILD_INFO%%/${BEE_BUILD_INFO}/" "${OVERLAY_STAGE_DIR}/etc/motd" \
> "${OVERLAY_STAGE_DIR}/etc/motd.patched"
mv "${OVERLAY_STAGE_DIR}/etc/motd.patched" "${OVERLAY_STAGE_DIR}/etc/motd"
replace_literal_in_file "${OVERLAY_STAGE_DIR}/etc/motd" "%%BUILD_INFO%%" "${BEE_BUILD_INFO}"
fi
# --- copy variant-specific package list, remove all other variant lists ---
@@ -568,26 +575,24 @@ fi
# --- substitute version placeholders in package list and archive ---
if [ "$BEE_GPU_VENDOR" = "nvidia" ]; then
sed -i \
-e "s/%%NVIDIA_FABRICMANAGER_VERSION%%/${NVIDIA_FABRICMANAGER_VERSION}/g" \
-e "s/%%DCGM_VERSION%%/${DCGM_VERSION}/g" \
"${BUILD_WORK_DIR}/config/package-lists/bee-gpu.list.chroot"
replace_literal_in_file "${BUILD_WORK_DIR}/config/package-lists/bee-gpu.list.chroot" \
"%%NVIDIA_FABRICMANAGER_VERSION%%" "${NVIDIA_FABRICMANAGER_VERSION}"
replace_literal_in_file "${BUILD_WORK_DIR}/config/package-lists/bee-gpu.list.chroot" \
"%%DCGM_VERSION%%" "${DCGM_VERSION}"
elif [ "$BEE_GPU_VENDOR" = "amd" ]; then
sed -i \
-e "s/%%ROCM_VERSION%%/${ROCM_VERSION}/g" \
-e "s/%%ROCM_SMI_VERSION%%/${ROCM_SMI_VERSION}/g" \
-e "s/%%ROCM_BANDWIDTH_TEST_VERSION%%/${ROCM_BANDWIDTH_TEST_VERSION}/g" \
-e "s/%%ROCM_VALIDATION_SUITE_VERSION%%/${ROCM_VALIDATION_SUITE_VERSION}/g" \
-e "s/%%ROCBLAS_VERSION%%/${ROCBLAS_VERSION}/g" \
-e "s/%%ROCRAND_VERSION%%/${ROCRAND_VERSION}/g" \
-e "s/%%HIP_RUNTIME_AMD_VERSION%%/${HIP_RUNTIME_AMD_VERSION}/g" \
-e "s/%%HIPBLASLT_VERSION%%/${HIPBLASLT_VERSION}/g" \
-e "s/%%COMGR_VERSION%%/${COMGR_VERSION}/g" \
"${BUILD_WORK_DIR}/config/package-lists/bee-gpu.list.chroot"
gpu_package_list="${BUILD_WORK_DIR}/config/package-lists/bee-gpu.list.chroot"
replace_literal_in_file "$gpu_package_list" "%%ROCM_VERSION%%" "${ROCM_VERSION}"
replace_literal_in_file "$gpu_package_list" "%%ROCM_SMI_VERSION%%" "${ROCM_SMI_VERSION}"
replace_literal_in_file "$gpu_package_list" "%%ROCM_BANDWIDTH_TEST_VERSION%%" "${ROCM_BANDWIDTH_TEST_VERSION}"
replace_literal_in_file "$gpu_package_list" "%%ROCM_VALIDATION_SUITE_VERSION%%" "${ROCM_VALIDATION_SUITE_VERSION}"
replace_literal_in_file "$gpu_package_list" "%%ROCBLAS_VERSION%%" "${ROCBLAS_VERSION}"
replace_literal_in_file "$gpu_package_list" "%%ROCRAND_VERSION%%" "${ROCRAND_VERSION}"
replace_literal_in_file "$gpu_package_list" "%%HIP_RUNTIME_AMD_VERSION%%" "${HIP_RUNTIME_AMD_VERSION}"
replace_literal_in_file "$gpu_package_list" "%%HIPBLASLT_VERSION%%" "${HIPBLASLT_VERSION}"
replace_literal_in_file "$gpu_package_list" "%%COMGR_VERSION%%" "${COMGR_VERSION}"
if [ -f "${BUILD_WORK_DIR}/config/archives/rocm.list.chroot" ]; then
sed -i \
-e "s/%%ROCM_VERSION%%/${ROCM_VERSION}/g" \
"${BUILD_WORK_DIR}/config/archives/rocm.list.chroot"
replace_literal_in_file "${BUILD_WORK_DIR}/config/archives/rocm.list.chroot" \
"%%ROCM_VERSION%%" "${ROCM_VERSION}"
fi
fi
iso/builder/lib/bootloader.sh
+8
View File
@@ -85,6 +85,14 @@ render_bootloader_template() {
initrd_placeholder="$7"
initrd="$8"
for required_placeholder in \
"@VERSION@" \
"$kernel_placeholder" \
"@APPEND_LIVE@" \
"$initrd_placeholder"; do
require_template_placeholder "$template" "$required_placeholder" || return 1
done
BEE_RENDER_VERSION="$version" \
BEE_RENDER_KERNEL_PLACEHOLDER="$kernel_placeholder" \
BEE_RENDER_KERNEL="$kernel" \
iso/builder/lib/iso-validation.sh
+5 -2
View File
@@ -359,9 +359,12 @@ validate_live_cmdline_params() {
bad = 1
}
$1 == command && has("boot=live") {
$1 == command {
live_entries++
if (!has("boot=live")) {
reject("missing boot=live")
}
if (!has("udev.children_max=1")) {
reject("missing udev.children_max=1")
}
@@ -432,7 +435,7 @@ validate_iso_live_boot_entries() {
exit 1
}
if grep -q '@APPEND_LIVE@\|@KERNEL_LIVE@\|@INITRD_LIVE@' "$grub_cfg" "$isolinux_cfg"; then
if grep -Eq '@[A-Z][A-Z_]*@' "$grub_cfg" "$isolinux_cfg"; then
echo "ERROR: unresolved live-build placeholders remain in ISO bootloader config" >&2
rm -f "$grub_cfg" "$isolinux_cfg"
exit 1
iso/builder/lib/template.sh
+55
View File
@@ -0,0 +1,55 @@
require_template_placeholder() {
file="$1"
placeholder="$2"
if [ -z "$placeholder" ]; then
echo "ERROR: template placeholder must not be empty" >&2
return 1
fi
if ! BEE_TEMPLATE_PLACEHOLDER="$placeholder" awk '
index($0, ENVIRON["BEE_TEMPLATE_PLACEHOLDER"]) { found=1 }
END { exit found ? 0 : 1 }
' "$file"; then
echo "ERROR: required placeholder $placeholder is missing from $file" >&2
return 1
fi
}
replace_literal_in_file() {
file="$1"
placeholder="$2"
replacement="$3"
require_template_placeholder "$file" "$placeholder" || return 1
tmp="$(mktemp "${TMPDIR:-/tmp}/bee-template.XXXXXX")"
if BEE_TEMPLATE_PLACEHOLDER="$placeholder" \
BEE_TEMPLATE_REPLACEMENT="$replacement" \
awk '
function replace_literal(text, needle, value, at) {
while ((at = index(text, needle)) != 0) {
text = substr(text, 1, at - 1) value substr(text, at + length(needle))
}
return text
}
{
print replace_literal($0,
ENVIRON["BEE_TEMPLATE_PLACEHOLDER"],
ENVIRON["BEE_TEMPLATE_REPLACEMENT"])
}
' "$file" > "$tmp" && cat "$tmp" > "$file"; then
status=0
else
status=$?
fi
rm -f "$tmp"
return "$status"
}
validate_project_version() {
version="$1"
case "$version" in
""|[!A-Za-z0-9]*|*[!A-Za-z0-9]|*[!A-Za-z0-9.-]*)
return 1
;;
esac
return 0
}
iso/builder/test-build-libs.sh
+42
View File
@@ -11,8 +11,37 @@ LOG_DIR="$TEST_ROOT/log"
mkdir -p "$LOG_DIR"
. "$BUILDER_DIR/lib/iso-validation.sh"
. "$BUILDER_DIR/lib/template.sh"
. "$BUILDER_DIR/lib/bootloader.sh"
literal_file="$TEST_ROOT/literal-file"
printf '%s\n' 'before %%VALUE%% after %%VALUE%%' > "$literal_file"
replace_literal_in_file "$literal_file" '%%VALUE%%' 'a&b#c\\d'
if [ "$(cat "$literal_file")" != 'before a&b#c\\d after a&b#c\\d' ]; then
echo "ERROR: generic literal replacement changed replacement characters" >&2
exit 1
fi
if replace_literal_in_file "$literal_file" '' value >/dev/null 2>&1; then
echo "ERROR: generic literal replacement accepted an empty placeholder" >&2
exit 1
fi
if replace_literal_in_file "$literal_file" '%%MISSING%%' value >/dev/null 2>&1; then
echo "ERROR: generic literal replacement accepted a missing placeholder" >&2
exit 1
fi
for valid_version in 13 13.0 v13.0-2-gabcdef0-dirty; do
if ! validate_project_version "$valid_version"; then
echo "ERROR: rejected valid project version: $valid_version" >&2
exit 1
fi
done
for invalid_version in '' '.13' '13.' '13/0' '13_0' '13 0' '13&0'; do
if validate_project_version "$invalid_version"; then
echo "ERROR: accepted unsafe project version: $invalid_version" >&2
exit 1
fi
done
literal_template="$TEST_ROOT/literal-template"
literal_output="$TEST_ROOT/literal-output"
printf '%s\n' '@VERSION@ @KERNEL@ @APPEND_LIVE@ @INITRD@' > "$literal_template"
@@ -23,6 +52,11 @@ if [ "$(cat "$literal_output")" != "$literal_expected" ]; then
echo "ERROR: bootloader renderer changed literal replacement characters" >&2
exit 1
fi
if render_bootloader_template "$literal_template" "$literal_output" \
'13.0-test' '@MISSING_KERNEL@' '/live/vmlinuz' 'boot=live' '@INITRD@' '/live/initrd.img' >/dev/null 2>&1; then
echo "ERROR: bootloader renderer accepted a missing required placeholder" >&2
exit 1
fi
rendered_grub="$TEST_ROOT/grub.cfg"
rendered_isolinux="$TEST_ROOT/live.cfg"
@@ -49,6 +83,14 @@ if validate_live_cmdline_params "$missing_serialization" linux GRUB "$BEE_ISO_VO
exit 1
fi
missing_boot_live="$TEST_ROOT/missing-boot-live.cfg"
awk 'BEGIN { removed=0 } /^ linux / && !removed { sub(/ boot=live/, ""); removed=1 } { print }' \
"$rendered_grub" > "$missing_boot_live"
if validate_live_cmdline_params "$missing_boot_live" linux GRUB "$BEE_ISO_VOLUME" >/dev/null 2>&1; then
echo "ERROR: validator ignored a live entry without boot=live" >&2
exit 1
fi
CACHE_ROOT="$TEST_ROOT/cache"
BUILD_VARIANT="test"
BUILD_WORK_DIR="$TEST_ROOT/work"