webui/topo: read from techdump instead of live nvidia-smi, fix socket/NUMA mismatch, fix cross-tier SAT status merge
/topo blocked the HTTP request on live nvidia-smi calls with no timeout
(topo -m, nvlink -s/-e run on every page load), so a wedged driver hung
the page indefinitely. CaptureTechnicalDump now persists these dumps
once per audit cycle; the page reads them from techdump/ instead.
buildSocketIndex mapped NUMA node number to CPU by treating dmidecode's
Socket Designation (often 1-indexed, "CPU1"/"CPU2") as equal to the
NUMA node number (always 0-indexed) — GPUs/NICs on NUMA node 0 fell
into the "unknown" column, others attached to the wrong CPU box. Now
ranks CPUs by Socket value instead of assuming a shared numbering base.
Fixed a bug in ComponentStatusDB/applyComponentStatusDB where GPU SAT
results were keyed per-target ("pcie:gpu:nvidia-stress") instead of
per-vendor, which both broke cross-tier severity tracking (a later
clean "2. Check" run and an earlier failing "3. Load" run never
compared severities) and silently failed to match any real BDF, so the
DB overlay never reached the topology graph at all. GPU keys are now
normalized to vendor ("pcie:gpu:nvidia"/"pcie:gpu:amd"). Also skip
writing to the DB when a SAT task was aborted by the user (ctx
canceled), so a partial run can't stomp a previously recorded status.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
f46fc98110
commit
2d84ddb577
@@ -177,12 +177,21 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
|
||||
source := "sat:" + target
|
||||
dbStatus := satStatusToDBStatus(overall)
|
||||
|
||||
// Map SAT target to component keys.
|
||||
// Map SAT target to component keys. GPU targets are keyed by vendor, not
|
||||
// by the raw target string: "nvidia" (Check tier) and "nvidia-stress" /
|
||||
// "nvidia-targeted-stress" (Load/Burn tier) all exercise the same
|
||||
// physical GPUs, so they must share one severity-tracked record — a
|
||||
// severity-1 Check run after a severity-3 Load failure must not lose
|
||||
// that failure just because it ran more recently. Recording each target
|
||||
// under its own key (the previous behavior) also silently broke
|
||||
// applyComponentStatusDB below, which expects "pcie:gpu:<vendor>" and
|
||||
// otherwise fails to match any real BDF.
|
||||
switch target {
|
||||
case "nvidia", "nvidia-targeted-stress", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
|
||||
"nvidia-interconnect", "nvidia-bandwidth", "amd", "nvidia-stress",
|
||||
"amd-stress", "amd-mem", "amd-bandwidth":
|
||||
db.Record("pcie:gpu:"+target, source, dbStatus, target+" SAT: "+overall)
|
||||
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress":
|
||||
db.Record("pcie:gpu:nvidia", source, dbStatus, target+" SAT: "+overall)
|
||||
case "amd", "amd-stress", "amd-mem", "amd-bandwidth":
|
||||
db.Record("pcie:gpu:amd", source, dbStatus, target+" SAT: "+overall)
|
||||
case "memory", "memory-stress", "sat-stress":
|
||||
db.Record("memory:all", source, dbStatus, target+" SAT: "+overall)
|
||||
case "cpu", "platform-stress":
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"bee/audit/internal/schema"
|
||||
)
|
||||
|
||||
func TestExtractArchivePath(t *testing.T) {
|
||||
@@ -40,3 +42,58 @@ func TestReadSATOverallStatus_HandlesActionResultPrefix(t *testing.T) {
|
||||
t.Errorf("ReadSATOverallStatus(bare) = %q, want FAILED", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writeSATSummary(t *testing.T, overall string) string {
|
||||
t.Helper()
|
||||
runDir := t.TempDir()
|
||||
summary := "run_at_utc=2026-07-06T17:47:22Z\noverall_status=" + overall + "\n"
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return runDir
|
||||
}
|
||||
|
||||
func TestApplySATResultToDBNormalizesGPUKeyByVendor(t *testing.T) {
|
||||
// "nvidia" (Check tier) and "nvidia-stress" (Load/Burn tier) exercise the
|
||||
// same physical GPUs and must collapse onto one component key so a later
|
||||
// clean Check run can't erase an earlier Load-tier failure.
|
||||
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ApplySATResultToDB(db, "nvidia-stress", writeSATSummary(t, "FAILED"))
|
||||
ApplySATResultToDB(db, "nvidia", writeSATSummary(t, "OK"))
|
||||
|
||||
rec, ok := db.Get("pcie:gpu:nvidia")
|
||||
if !ok {
|
||||
t.Fatalf("expected pcie:gpu:nvidia record to exist")
|
||||
}
|
||||
if rec.Status != "Warning" {
|
||||
t.Fatalf("status=%q, want Warning (FAILED) to survive the later OK Check run", rec.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyComponentStatusDBMatchesGPUByVendor(t *testing.T) {
|
||||
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.Record("pcie:gpu:nvidia", "sat:nvidia-stress", "Critical", "nvidia-stress SAT: FAILED")
|
||||
|
||||
class := "VideoController"
|
||||
vendor := 0x10de // collector.NvidiaVendorID
|
||||
snap := &schema.HardwareSnapshot{
|
||||
PCIeDevices: []schema.HardwarePCIeDevice{
|
||||
{DeviceClass: &class, VendorID: &vendor, BDF: strPtr("0000:c8:00.0")},
|
||||
},
|
||||
}
|
||||
|
||||
applyComponentStatusDB(snap, db)
|
||||
|
||||
if snap.PCIeDevices[0].Status == nil || *snap.PCIeDevices[0].Status != "Critical" {
|
||||
t.Fatalf("expected GPU device status Critical, got %v", snap.PCIeDevices[0].Status)
|
||||
}
|
||||
}
|
||||
|
||||
func strPtr(s string) *string { return &s }
|
||||
|
||||
@@ -347,14 +347,19 @@ func applyComponentStatusDB(snap *schema.HardwareSnapshot, db *ComponentStatusDB
|
||||
ts := rec.LastChangedAt.UTC().Format("2006-01-02T15:04:05Z")
|
||||
|
||||
switch {
|
||||
case key == "pcie:gpu:nvidia" || key == "pcie:gpu:amd":
|
||||
vendor := strings.TrimPrefix(key, "pcie:gpu:")
|
||||
for i := range snap.PCIeDevices {
|
||||
if matchesGPUVendor(snap.PCIeDevices[i], vendor) {
|
||||
mergeComponentStatus(&snap.PCIeDevices[i].HardwareComponentStatus, ts, status, detail)
|
||||
}
|
||||
}
|
||||
case strings.HasPrefix(key, "pcie:"):
|
||||
bdf := strings.TrimPrefix(key, "pcie:")
|
||||
bdf = strings.TrimPrefix(bdf, "gpu:") // strip sub-type if present
|
||||
// bdf may be empty (e.g. "pcie:gpu:nvidia") — skip BDF matching
|
||||
if sanitizeBDFForLookup(bdf) == "" {
|
||||
normalized := sanitizeBDFForLookup(bdf)
|
||||
if normalized == "" {
|
||||
break
|
||||
}
|
||||
normalized := sanitizeBDFForLookup(bdf)
|
||||
for i := range snap.PCIeDevices {
|
||||
if snap.PCIeDevices[i].BDF == nil {
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user