/topo blocked the HTTP request on live nvidia-smi calls with no timeout
(topo -m, nvlink -s/-e run on every page load), so a wedged driver hung
the page indefinitely. CaptureTechnicalDump now persists these dumps
once per audit cycle; the page reads them from techdump/ instead.
buildSocketIndex mapped NUMA node number to CPU by treating dmidecode's
Socket Designation (often 1-indexed, "CPU1"/"CPU2") as equal to the
NUMA node number (always 0-indexed) — GPUs/NICs on NUMA node 0 fell
into the "unknown" column, others attached to the wrong CPU box. Now
ranks CPUs by Socket value instead of assuming a shared numbering base.
Fixed a bug in ComponentStatusDB/applyComponentStatusDB where GPU SAT
results were keyed per-target ("pcie:gpu:nvidia-stress") instead of
per-vendor, which both broke cross-tier severity tracking (a later
clean "2. Check" run and an earlier failing "3. Load" run never
compared severities) and silently failed to match any real BDF, so the
DB overlay never reached the topology graph at all. GPU keys are now
normalized to vendor ("pcie:gpu:nvidia"/"pcie:gpu:amd"). Also skip
writing to the DB when a SAT task was aborted by the user (ctx
canceled), so a partial run can't stomp a previously recorded status.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
259 lines
7.8 KiB
Go
259 lines
7.8 KiB
Go
package webui
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"bee/audit/internal/schema"
|
|
)
|
|
|
|
func TestTopoPageNoAuditDataGracefulFallback(t *testing.T) {
|
|
handler := NewHandler(HandlerOptions{})
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("status=%d", rec.Code)
|
|
}
|
|
body := rec.Body.String()
|
|
if !strings.Contains(body, "No audit data") {
|
|
t.Fatalf("topo page missing no-audit-data fallback: %s", body)
|
|
}
|
|
}
|
|
|
|
func TestTopoPageRendersCPUAndDegradedPCIeLink(t *testing.T) {
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "audit.json")
|
|
|
|
socket := 0
|
|
numaNode := 0
|
|
gen3, gen4 := "Gen3", "Gen4"
|
|
deviceClass := "VideoController"
|
|
model := "NVIDIA H100 80GB HBM3"
|
|
cpuModel := "Intel Xeon 6530"
|
|
okStatus := "OK"
|
|
|
|
ingest := schema.HardwareIngestRequest{
|
|
CollectedAt: "2026-03-15T00:00:00Z",
|
|
Hardware: schema.HardwareSnapshot{
|
|
CPUs: []schema.HardwareCPU{
|
|
{
|
|
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus},
|
|
Socket: &socket,
|
|
Model: &cpuModel,
|
|
},
|
|
},
|
|
PCIeDevices: []schema.HardwarePCIeDevice{
|
|
{
|
|
DeviceClass: &deviceClass,
|
|
Model: &model,
|
|
NUMANode: &numaNode,
|
|
LinkSpeed: &gen3,
|
|
MaxLinkSpeed: &gen4,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
data, err := json.Marshal(ingest)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(path, data, 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
handler := NewHandler(HandlerOptions{AuditPath: path})
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("status=%d", rec.Code)
|
|
}
|
|
body := rec.Body.String()
|
|
if !strings.Contains(body, "CPU 0") {
|
|
t.Fatalf("topo page missing CPU 0 box: %s", body)
|
|
}
|
|
if !strings.Contains(body, "GPU") {
|
|
t.Fatalf("topo page missing GPU box: %s", body)
|
|
}
|
|
if !strings.Contains(body, "var(--warn-fg)") {
|
|
t.Fatalf("topo page missing degraded-link warn edge color: %s", body)
|
|
}
|
|
}
|
|
|
|
func TestTopoPageLinkedFromNav(t *testing.T) {
|
|
handler := NewHandler(HandlerOptions{})
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("status=%d", rec.Code)
|
|
}
|
|
if !strings.Contains(rec.Body.String(), `href="/topo"`) {
|
|
t.Fatalf("nav missing /topo link")
|
|
}
|
|
}
|
|
|
|
func TestParseGPUPairAdjacencyRealTwoGPUDump(t *testing.T) {
|
|
// Real system/nvidia-smi-topo.txt from a support bundle for this exact
|
|
// server: two H100s directly bridged (NV17), spanning two NUMA nodes.
|
|
input := "\tGPU0\tGPU1\tNIC0\tNIC1\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\n" +
|
|
"GPU0\t X \tNV17\tSYS\tSYS\t0-23,48-71\t0\t\tN/A\n" +
|
|
"GPU1\tNV17\t X \tNODE\tNODE\t24-47,72-95\t1\t\tN/A\n" +
|
|
"NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" +
|
|
"NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n"
|
|
|
|
pairs := parseGPUPairAdjacency(input)
|
|
if len(pairs) != 1 {
|
|
t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
|
|
}
|
|
if pairs[0].GPUA != 0 || pairs[0].GPUB != 1 || pairs[0].NVLinks != 17 {
|
|
t.Fatalf("pair=%#v want {0,1,17}", pairs[0])
|
|
}
|
|
}
|
|
|
|
func TestParseGPUPairAdjacencyDoesNotChainUnrelatedGPUs(t *testing.T) {
|
|
// 4 GPUs: only (0,1) and (2,3) are actually bonded. GPU1 and GPU2 must
|
|
// NOT get an edge just because they're adjacent in the layout.
|
|
input := "\tGPU0\tGPU1\tGPU2\tGPU3\n" +
|
|
"GPU0\t X \tNV18\tSYS\tSYS\n" +
|
|
"GPU1\tNV18\t X \tSYS\tSYS\n" +
|
|
"GPU2\tSYS\tSYS\t X \tNV18\n" +
|
|
"GPU3\tSYS\tSYS\tNV18\t X \n"
|
|
|
|
pairs := parseGPUPairAdjacency(input)
|
|
if len(pairs) != 2 {
|
|
t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs)
|
|
}
|
|
want := map[[2]int]bool{{0, 1}: true, {2, 3}: true}
|
|
for _, p := range pairs {
|
|
if !want[[2]int{p.GPUA, p.GPUB}] {
|
|
t.Fatalf("unexpected pair %#v (GPU1-GPU2 chaining bug?)", p)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestParseGPUPairAdjacencyEmptyOnNoMatrix(t *testing.T) {
|
|
if pairs := parseGPUPairAdjacency("no gpus here"); pairs != nil {
|
|
t.Fatalf("pairs=%v want nil", pairs)
|
|
}
|
|
}
|
|
|
|
func TestPcieGenRank(t *testing.T) {
|
|
if pcieGenRank("Gen5") <= pcieGenRank("Gen4") {
|
|
t.Fatalf("Gen5 should rank higher than Gen4")
|
|
}
|
|
if pcieGenRank("bogus") != 0 {
|
|
t.Fatalf("unparseable gen should rank 0")
|
|
}
|
|
}
|
|
|
|
func TestTopoEdgeColorVar(t *testing.T) {
|
|
gen3, gen4 := "Gen3", "Gen4"
|
|
degraded := schema.HardwarePCIeDevice{LinkSpeed: &gen3, MaxLinkSpeed: &gen4}
|
|
if got := topoEdgeColorVar(degraded); got != "var(--warn-fg)" {
|
|
t.Fatalf("degraded color=%q want warn", got)
|
|
}
|
|
full := schema.HardwarePCIeDevice{LinkSpeed: &gen4, MaxLinkSpeed: &gen4}
|
|
if got := topoEdgeColorVar(full); got != "var(--ok-fg)" {
|
|
t.Fatalf("full-speed color=%q want ok", got)
|
|
}
|
|
unknown := schema.HardwarePCIeDevice{}
|
|
if got := topoEdgeColorVar(unknown); got != "var(--muted)" {
|
|
t.Fatalf("unknown color=%q want muted", got)
|
|
}
|
|
}
|
|
|
|
func TestBuildSocketIndex(t *testing.T) {
|
|
s0, s1 := 0, 1
|
|
cpus := []schema.HardwareCPU{{Socket: &s1}, {Socket: &s0}}
|
|
idx := buildSocketIndex(cpus)
|
|
if idx[0] != 1 || idx[1] != 0 {
|
|
t.Fatalf("idx=%#v want {0:1, 1:0}", idx)
|
|
}
|
|
}
|
|
|
|
func TestBuildSocketIndexOneIndexedSocketDesignation(t *testing.T) {
|
|
// dmidecode "Socket Designation" is frequently 1-indexed ("CPU1", "CPU2")
|
|
// while Linux NUMA nodes are always 0-indexed. NUMA node 0 must still
|
|
// resolve to the first CPU in Socket order, not fall through to the
|
|
// "unknown" column (the bug reported against the /topo page).
|
|
s1, s2 := 1, 2
|
|
cpus := []schema.HardwareCPU{{Socket: &s1}, {Socket: &s2}}
|
|
idx := buildSocketIndex(cpus)
|
|
if idx[0] != 0 || idx[1] != 1 {
|
|
t.Fatalf("idx=%#v want {0:0, 1:1}", idx)
|
|
}
|
|
}
|
|
|
|
func TestIsRAIDControllerClass(t *testing.T) {
|
|
if !isRAIDControllerClass("StorageController") || !isRAIDControllerClass("MassStorageController") {
|
|
t.Fatalf("expected known RAID/storage classes to match")
|
|
}
|
|
if isRAIDControllerClass("VideoController") {
|
|
t.Fatalf("GPU class should not match RAID classifier")
|
|
}
|
|
}
|
|
|
|
func TestIsNICDeviceClassDev(t *testing.T) {
|
|
class := "EthernetController"
|
|
nic := schema.HardwarePCIeDevice{DeviceClass: &class}
|
|
if !isNICDeviceClassDev(nic) {
|
|
t.Fatalf("expected EthernetController to classify as NIC")
|
|
}
|
|
withMac := schema.HardwarePCIeDevice{MacAddresses: []string{"aa:bb:cc:dd:ee:ff"}}
|
|
if !isNICDeviceClassDev(withMac) {
|
|
t.Fatalf("expected device with MAC address to classify as NIC")
|
|
}
|
|
other := schema.HardwarePCIeDevice{}
|
|
if isNICDeviceClassDev(other) {
|
|
t.Fatalf("expected empty device to not classify as NIC")
|
|
}
|
|
}
|
|
|
|
func TestTopoStatusBadgeClassNilIsUnknown(t *testing.T) {
|
|
label, cls := topoStatusBadgeClass(nil)
|
|
if label != "?" || cls != "badge-unknown" {
|
|
t.Fatalf("nil status = (%q,%q) want (?, badge-unknown)", label, cls)
|
|
}
|
|
ok := "OK"
|
|
label, cls = topoStatusBadgeClass(&ok)
|
|
if label != "OK" || cls != "badge-ok" {
|
|
t.Fatalf("OK status = (%q,%q) want (OK, badge-ok)", label, cls)
|
|
}
|
|
}
|
|
|
|
func TestParseTopoNVLinkStatus(t *testing.T) {
|
|
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
|
Link 0: 26.562 GB/s
|
|
Link 15: <inactive>
|
|
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
|
|
Link 0: 26.562 GB/s
|
|
`
|
|
got := parseTopoNVLinkStatus(input)
|
|
if len(got[0]) != 2 || got[0][1].Active {
|
|
t.Fatalf("gpu0=%#v want link15 inactive", got[0])
|
|
}
|
|
if len(got[1]) != 1 || got[1][0].SpeedGBs == nil || *got[1][0].SpeedGBs != 26.562 {
|
|
t.Fatalf("gpu1=%#v want link0 26.562 GB/s", got[1])
|
|
}
|
|
}
|
|
|
|
func TestParseTopoNVLinkErrors(t *testing.T) {
|
|
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
|
Link 0: Replay Errors: 0
|
|
Link 0: Recovery Errors: 0
|
|
Link 0: CRC Errors: 0
|
|
Link 1: Replay Errors: 3
|
|
Link 1: Recovery Errors: 1
|
|
Link 1: CRC Errors: 2
|
|
`
|
|
got := parseTopoNVLinkErrors(input)
|
|
c := got[0][1]
|
|
if c[0] != 3 || c[1] != 1 || c[2] != 2 {
|
|
t.Fatalf("link1 counters=%#v want {3,1,2}", c)
|
|
}
|
|
}
|