Mikhail ChusavitinandClaude Sonnet 5
20cd317c87
Investigating the CG480-S6053 reboot needed a way to run an ad-hoc load
(nvbandwidth across a specific GPU set) while sampling IPMI/nvidia-smi
telemetry in the background — without hardcoding a one-off test into the
SAT pack code for a single investigation.
- audit/internal/platform/scenario.go: ScenarioSpec/ScenarioJob (JSON,
no new dependency) + System.RunScenario. "command" jobs run sequential
or parallel (per-job "parallel" flag); "sampler" jobs run concurrently
in the background on their own interval until every command job
finishes or the scenario's timeout elapses. "{{gpus}}" in a command's
cmd is substituted from that job's gpu_indices. Command jobs are wired
through the same satJobBoundaryHook/satSyncBracketHook seams the SAT
job runner uses, so a scenario run gets the same durability treatment
(evidence that a risky command started/finished reaches blackbox before
a possible crash, not just whatever streamed to the RAM-backed export
dir).
- export.go: ReadScenarioFromRemovableMedia mounts each removable target
looking for scenarios/<name>.json — an air-gapped engineer can author a
scenario elsewhere, drop it under scenarios/ on the same USB stick
already plugged in for blackbox, and run it with no network path onto
the host.
- cmd/bee: new `bee run <file.json|name>` (bare name = looked up on
removable media); `bee scenario run <arg>` kept as a longer alias.
- scenarios/nvbandwidth-all-gpu-power-watch.json: the scenario that
reproduced the actual reboot (full nvbandwidth across all GPUs, which
crashed, vs. clean per-socket passes), with IPMI sensor + GPU power/temp
sampling for a power-delivery correlation check.
Also: webui/page_topo.go — the /topo page's component-status-detail modal
(GET /api/component-detail/{type}) showed "No status data recorded yet"
for any component type ComponentStatusDB has no history for yet (e.g. GPU
before a SAT run this boot), even though the topology card for the same
component already showed "N OK" from the audit inventory snapshot.
inventoryFallbackRecords now synthesizes records from that same inventory
snapshot when StatusDB is empty, using the same device classifiers
(isGPUDeviceClass etc.) and severity mapping (classifyTopoSeverity) the
topology card itself uses, so the two views never disagree.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
652 lines
21 KiB
Go
652 lines
21 KiB
Go
package webui
|
||
|
||
import (
|
||
"encoding/json"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"strconv"
|
||
"strings"
|
||
"testing"
|
||
|
||
"bee/audit/internal/schema"
|
||
)
|
||
|
||
func TestTopoPageNoAuditDataGracefulFallback(t *testing.T) {
|
||
handler := NewHandler(HandlerOptions{})
|
||
rec := httptest.NewRecorder()
|
||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
|
||
if rec.Code != http.StatusOK {
|
||
t.Fatalf("status=%d", rec.Code)
|
||
}
|
||
body := rec.Body.String()
|
||
if !strings.Contains(body, "No audit data") {
|
||
t.Fatalf("topo page missing no-audit-data fallback: %s", body)
|
||
}
|
||
}
|
||
|
||
func TestTopoPageRendersCPUAndDegradedPCIeLink(t *testing.T) {
|
||
dir := t.TempDir()
|
||
path := filepath.Join(dir, "audit.json")
|
||
|
||
socket := 0
|
||
numaNode := 0
|
||
gen3, gen4 := "Gen3", "Gen4"
|
||
deviceClass := "VideoController"
|
||
model := "NVIDIA H100 80GB HBM3"
|
||
cpuModel := "Intel Xeon 6530"
|
||
okStatus := "OK"
|
||
|
||
ingest := schema.HardwareIngestRequest{
|
||
CollectedAt: "2026-03-15T00:00:00Z",
|
||
Hardware: schema.HardwareSnapshot{
|
||
CPUs: []schema.HardwareCPU{
|
||
{
|
||
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus},
|
||
Socket: &socket,
|
||
Model: &cpuModel,
|
||
},
|
||
},
|
||
PCIeDevices: []schema.HardwarePCIeDevice{
|
||
{
|
||
DeviceClass: &deviceClass,
|
||
Model: &model,
|
||
NUMANode: &numaNode,
|
||
LinkSpeed: &gen3,
|
||
MaxLinkSpeed: &gen4,
|
||
},
|
||
},
|
||
},
|
||
}
|
||
data, err := json.Marshal(ingest)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
handler := NewHandler(HandlerOptions{AuditPath: path})
|
||
rec := httptest.NewRecorder()
|
||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
|
||
if rec.Code != http.StatusOK {
|
||
t.Fatalf("status=%d", rec.Code)
|
||
}
|
||
body := rec.Body.String()
|
||
if !strings.Contains(body, "CPU 0") {
|
||
t.Fatalf("topo page missing CPU 0 box: %s", body)
|
||
}
|
||
if !strings.Contains(body, "GPU") {
|
||
t.Fatalf("topo page missing GPU box: %s", body)
|
||
}
|
||
if !strings.Contains(body, "var(--warn-fg)") {
|
||
t.Fatalf("topo page missing degraded-link warn edge color: %s", body)
|
||
}
|
||
}
|
||
|
||
func TestTopoPageRendersArbitraryPSUAndFirmwareCountsAsFlexRows(t *testing.T) {
|
||
dir := t.TempDir()
|
||
path := filepath.Join(dir, "audit.json")
|
||
|
||
okStatus := "OK"
|
||
watt := 3000
|
||
|
||
var psus []schema.HardwarePowerSupply
|
||
for i := 0; i < 6; i++ {
|
||
slot := strconv.Itoa(i)
|
||
psus = append(psus, schema.HardwarePowerSupply{
|
||
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus},
|
||
Slot: &slot,
|
||
WattageW: &watt,
|
||
})
|
||
}
|
||
|
||
ingest := schema.HardwareIngestRequest{
|
||
CollectedAt: "2026-03-15T00:00:00Z",
|
||
Hardware: schema.HardwareSnapshot{
|
||
Firmware: []schema.HardwareFirmwareRecord{
|
||
{DeviceName: "BIOS", Version: "2.1.0"},
|
||
{DeviceName: "BMC", Version: "5.17.00"},
|
||
},
|
||
PowerSupplies: psus,
|
||
},
|
||
}
|
||
data, err := json.Marshal(ingest)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
handler := NewHandler(HandlerOptions{AuditPath: path})
|
||
rec := httptest.NewRecorder()
|
||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
|
||
if rec.Code != http.StatusOK {
|
||
t.Fatalf("status=%d", rec.Code)
|
||
}
|
||
body := rec.Body.String()
|
||
|
||
// Firmware row must render every record (BIOS + BMC, not just BMC).
|
||
if !strings.Contains(body, "BIOS") || !strings.Contains(body, "BMC") {
|
||
t.Fatalf("topo page missing BIOS/BMC firmware boxes: %s", body)
|
||
}
|
||
// All 6 PSUs must be represented, grouped into one stacked card with a
|
||
// count rather than 6 separate boxes.
|
||
if !strings.Contains(body, "Power Supplies ×6") {
|
||
t.Fatalf("topo page missing grouped Power Supplies x6 card: %s", body)
|
||
}
|
||
if strings.Count(body, `onclick="openComponentDetail('psu')"`) != 1 &&
|
||
strings.Count(body, `onclick="openComponentDetail('psu')"`) != 1 {
|
||
t.Fatalf("expected exactly one clickable PSU group card, not one per PSU: %s", body)
|
||
}
|
||
// Firmware/PSU rows must be flex-wrap HTML (arbitrary count, no overlap),
|
||
// not absolutely-positioned SVG rects sharing fixed x/y coordinates.
|
||
if !strings.Contains(body, "flex-wrap:wrap") {
|
||
t.Fatalf("topo page missing flex-wrap layout for firmware/PSU rows: %s", body)
|
||
}
|
||
// Firmware row must come before the PSU row.
|
||
if strings.Index(body, "BIOS") > strings.Index(body, "Power Supplies") {
|
||
t.Fatalf("firmware row should render before PSU row: %s", body)
|
||
}
|
||
}
|
||
|
||
func TestTopoPageLinkedFromNav(t *testing.T) {
|
||
handler := NewHandler(HandlerOptions{})
|
||
rec := httptest.NewRecorder()
|
||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil))
|
||
if rec.Code != http.StatusOK {
|
||
t.Fatalf("status=%d", rec.Code)
|
||
}
|
||
if !strings.Contains(rec.Body.String(), `href="/topo"`) {
|
||
t.Fatalf("nav missing /topo link")
|
||
}
|
||
}
|
||
|
||
func TestParseGPUPairAdjacencyRealTwoGPUDump(t *testing.T) {
|
||
// Real system/nvidia-smi-topo.txt from a support bundle for this exact
|
||
// server: two H100s directly bridged (NV17), spanning two NUMA nodes.
|
||
input := "\tGPU0\tGPU1\tNIC0\tNIC1\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\n" +
|
||
"GPU0\t X \tNV17\tSYS\tSYS\t0-23,48-71\t0\t\tN/A\n" +
|
||
"GPU1\tNV17\t X \tNODE\tNODE\t24-47,72-95\t1\t\tN/A\n" +
|
||
"NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" +
|
||
"NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n"
|
||
|
||
pairs := parseGPUPairAdjacency(input)
|
||
if len(pairs) != 1 {
|
||
t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
|
||
}
|
||
if pairs[0].GPUA != 0 || pairs[0].GPUB != 1 || pairs[0].NVLinks != 17 {
|
||
t.Fatalf("pair=%#v want {0,1,17}", pairs[0])
|
||
}
|
||
}
|
||
|
||
func TestParseGPUPairAdjacencyDoesNotChainUnrelatedGPUs(t *testing.T) {
|
||
// 4 GPUs: only (0,1) and (2,3) are actually bonded. GPU1 and GPU2 must
|
||
// NOT get an edge just because they're adjacent in the layout.
|
||
input := "\tGPU0\tGPU1\tGPU2\tGPU3\n" +
|
||
"GPU0\t X \tNV18\tSYS\tSYS\n" +
|
||
"GPU1\tNV18\t X \tSYS\tSYS\n" +
|
||
"GPU2\tSYS\tSYS\t X \tNV18\n" +
|
||
"GPU3\tSYS\tSYS\tNV18\t X \n"
|
||
|
||
pairs := parseGPUPairAdjacency(input)
|
||
if len(pairs) != 2 {
|
||
t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs)
|
||
}
|
||
want := map[[2]int]bool{{0, 1}: true, {2, 3}: true}
|
||
for _, p := range pairs {
|
||
if !want[[2]int{p.GPUA, p.GPUB}] {
|
||
t.Fatalf("unexpected pair %#v (GPU1-GPU2 chaining bug?)", p)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestParseGPUPairAdjacencyANSIUnderlinedHeader(t *testing.T) {
|
||
// Real techdump capture: nvidia-smi underlines the header row with ANSI
|
||
// escapes even when writing to a file, so the header line starts with
|
||
// ESC[4m, not "GPU0".
|
||
input := "\x1b[4m\tGPU0\tGPU1\tGPU2\tGPU3\tNIC0\tNIC1\tCPU Affinity\x1b[0m\n" +
|
||
"GPU0\t X \tNV18\tPIX\tPIX\tNODE\tNODE\t0-31,64-95\t0\n" +
|
||
"GPU1\tNV18\t X \tPIX\tPIX\tNODE\tNODE\t0-31,64-95\t0\n" +
|
||
"GPU2\tPIX\tPIX\t X \tNV18\tNODE\tNODE\t0-31,64-95\t0\n" +
|
||
"GPU3\tPIX\tPIX\tNV18\t X \tNODE\tNODE\t0-31,64-95\t0\n" +
|
||
"NIC0\tNODE\tNODE\tNODE\tNODE\t X \tPIX\n" +
|
||
"NIC1\tNODE\tNODE\tNODE\tNODE\tPIX\t X \n"
|
||
|
||
pairs := parseGPUPairAdjacency(input)
|
||
if len(pairs) != 2 {
|
||
t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs)
|
||
}
|
||
want := map[[2]int]bool{{0, 1}: true, {2, 3}: true}
|
||
for _, p := range pairs {
|
||
if !want[[2]int{p.GPUA, p.GPUB}] || p.NVLinks != 18 {
|
||
t.Fatalf("unexpected pair %#v", p)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestParseGPUPairAdjacencyEmptyOnNoMatrix(t *testing.T) {
|
||
if pairs := parseGPUPairAdjacency("no gpus here"); pairs != nil {
|
||
t.Fatalf("pairs=%v want nil", pairs)
|
||
}
|
||
}
|
||
|
||
func TestPcieGenRank(t *testing.T) {
|
||
if pcieGenRank("Gen5") <= pcieGenRank("Gen4") {
|
||
t.Fatalf("Gen5 should rank higher than Gen4")
|
||
}
|
||
if pcieGenRank("bogus") != 0 {
|
||
t.Fatalf("unparseable gen should rank 0")
|
||
}
|
||
}
|
||
|
||
func TestTopoEdgeColorVar(t *testing.T) {
|
||
gen3, gen4 := "Gen3", "Gen4"
|
||
degraded := schema.HardwarePCIeDevice{LinkSpeed: &gen3, MaxLinkSpeed: &gen4}
|
||
if got := topoEdgeColorVar(degraded); got != "var(--warn-fg)" {
|
||
t.Fatalf("degraded color=%q want warn", got)
|
||
}
|
||
full := schema.HardwarePCIeDevice{LinkSpeed: &gen4, MaxLinkSpeed: &gen4}
|
||
if got := topoEdgeColorVar(full); got != "var(--ok-fg)" {
|
||
t.Fatalf("full-speed color=%q want ok", got)
|
||
}
|
||
unknown := schema.HardwarePCIeDevice{}
|
||
if got := topoEdgeColorVar(unknown); got != "var(--muted)" {
|
||
t.Fatalf("unknown color=%q want muted", got)
|
||
}
|
||
}
|
||
|
||
func TestBuildSocketIndex(t *testing.T) {
|
||
s0, s1 := 0, 1
|
||
cpus := []schema.HardwareCPU{{Socket: &s1}, {Socket: &s0}}
|
||
idx := buildSocketIndex(cpus)
|
||
if idx[0] != 1 || idx[1] != 0 {
|
||
t.Fatalf("idx=%#v want {0:1, 1:0}", idx)
|
||
}
|
||
}
|
||
|
||
func TestBuildSocketIndexOneIndexedSocketDesignation(t *testing.T) {
|
||
// dmidecode "Socket Designation" is frequently 1-indexed ("CPU1", "CPU2")
|
||
// while Linux NUMA nodes are always 0-indexed. NUMA node 0 must still
|
||
// resolve to the first CPU in Socket order, not fall through to the
|
||
// "unknown" column (the bug reported against the /topo page).
|
||
s1, s2 := 1, 2
|
||
cpus := []schema.HardwareCPU{{Socket: &s1}, {Socket: &s2}}
|
||
idx := buildSocketIndex(cpus)
|
||
if idx[0] != 0 || idx[1] != 1 {
|
||
t.Fatalf("idx=%#v want {0:0, 1:1}", idx)
|
||
}
|
||
}
|
||
|
||
func TestIsRAIDControllerClass(t *testing.T) {
|
||
if !isRAIDControllerClass("StorageController") || !isRAIDControllerClass("MassStorageController") {
|
||
t.Fatalf("expected known RAID/storage classes to match")
|
||
}
|
||
if isRAIDControllerClass("VideoController") {
|
||
t.Fatalf("GPU class should not match RAID classifier")
|
||
}
|
||
}
|
||
|
||
func TestIsNICDeviceClassDev(t *testing.T) {
|
||
class := "EthernetController"
|
||
nic := schema.HardwarePCIeDevice{DeviceClass: &class}
|
||
if !isNICDeviceClassDev(nic) {
|
||
t.Fatalf("expected EthernetController to classify as NIC")
|
||
}
|
||
withMac := schema.HardwarePCIeDevice{MacAddresses: []string{"aa:bb:cc:dd:ee:ff"}}
|
||
if !isNICDeviceClassDev(withMac) {
|
||
t.Fatalf("expected device with MAC address to classify as NIC")
|
||
}
|
||
other := schema.HardwarePCIeDevice{}
|
||
if isNICDeviceClassDev(other) {
|
||
t.Fatalf("expected empty device to not classify as NIC")
|
||
}
|
||
}
|
||
|
||
func TestClassifyTopoSeverityNilIsUnknown(t *testing.T) {
|
||
if sev := classifyTopoSeverity(nil); sev != 0 {
|
||
t.Fatalf("nil status severity=%d want 0 (unknown)", sev)
|
||
}
|
||
ok := "OK"
|
||
if sev := classifyTopoSeverity(&ok); sev != 1 {
|
||
t.Fatalf("OK status severity=%d want 1", sev)
|
||
}
|
||
warn := "Warning"
|
||
if sev := classifyTopoSeverity(&warn); sev != 2 {
|
||
t.Fatalf("Warning status severity=%d want 2", sev)
|
||
}
|
||
crit := "Critical"
|
||
if sev := classifyTopoSeverity(&crit); sev != 3 {
|
||
t.Fatalf("Critical status severity=%d want 3", sev)
|
||
}
|
||
}
|
||
|
||
func TestTopoStatusTallyLine(t *testing.T) {
|
||
var t1 topoStatusTally
|
||
t1.add(1)
|
||
if got := t1.line(); got != "OK" {
|
||
t.Fatalf("single-OK line=%q want %q", got, "OK")
|
||
}
|
||
var t2 topoStatusTally
|
||
t2.add(1)
|
||
t2.add(1)
|
||
t2.add(1)
|
||
t2.add(2)
|
||
if got := t2.line(); got != "1 Warning, 3 OK" {
|
||
t.Fatalf("mixed line=%q want %q", got, "1 Warning, 3 OK")
|
||
}
|
||
}
|
||
|
||
func TestTopoMainDiagramGroupsSameKindSameColumnIntoOneStackedCard(t *testing.T) {
|
||
dir := t.TempDir()
|
||
path := filepath.Join(dir, "audit.json")
|
||
|
||
socket := 0
|
||
numaNode := 0
|
||
deviceClass := "VideoController"
|
||
model := "NVIDIA H100 80GB HBM3"
|
||
okStatus := "OK"
|
||
|
||
var gpus []schema.HardwarePCIeDevice
|
||
for i := 0; i < 4; i++ {
|
||
gpus = append(gpus, schema.HardwarePCIeDevice{
|
||
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus},
|
||
DeviceClass: &deviceClass,
|
||
Model: &model,
|
||
NUMANode: &numaNode,
|
||
})
|
||
}
|
||
|
||
ingest := schema.HardwareIngestRequest{
|
||
CollectedAt: "2026-03-15T00:00:00Z",
|
||
Hardware: schema.HardwareSnapshot{
|
||
CPUs: []schema.HardwareCPU{
|
||
{HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus}, Socket: &socket},
|
||
},
|
||
PCIeDevices: gpus,
|
||
},
|
||
}
|
||
data, err := json.Marshal(ingest)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
handler := NewHandler(HandlerOptions{AuditPath: path})
|
||
rec := httptest.NewRecorder()
|
||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
|
||
body := rec.Body.String()
|
||
|
||
if !strings.Contains(body, "GPU ×4") {
|
||
t.Fatalf("expected one grouped GPU x4 card, got: %s", body)
|
||
}
|
||
if strings.Count(body, "openComponentDetail('gpu')") != 1 {
|
||
t.Fatalf("expected exactly one clickable GPU card, not one per GPU: %s", body)
|
||
}
|
||
if !strings.Contains(body, "4 OK") {
|
||
t.Fatalf("expected group status line '4 OK': %s", body)
|
||
}
|
||
}
|
||
|
||
func TestDimmRawNodeFromCPULocatorPrefix(t *testing.T) {
|
||
slot := "CPU1_DIMM_A1"
|
||
mem := schema.HardwareMemory{Slot: &slot}
|
||
node, ok := dimmRawNode(mem, nil)
|
||
if !ok || node != 1 {
|
||
t.Fatalf("dimmRawNode() = (%d, %v), want (1, true)", node, ok)
|
||
}
|
||
}
|
||
|
||
func TestDimmRawNodeFromBankLocatorFallback(t *testing.T) {
|
||
// Boards whose Locator carries no CPU number of its own (e.g.
|
||
// "DIMM000(A)") still need to be attached to a column via the node
|
||
// number in Bank Locator, read separately from techdump.
|
||
slot := "DIMM100(A)"
|
||
mem := schema.HardwareMemory{Slot: &slot}
|
||
bankNodes := map[string]int{"DIMM100(A)": 2}
|
||
node, ok := dimmRawNode(mem, bankNodes)
|
||
if !ok || node != 2 {
|
||
t.Fatalf("dimmRawNode() = (%d, %v), want (2, true)", node, ok)
|
||
}
|
||
}
|
||
|
||
func TestDimmRawNodeUnmatched(t *testing.T) {
|
||
slot := "SOMETHING_UNRECOGNIZED"
|
||
mem := schema.HardwareMemory{Slot: &slot}
|
||
if _, ok := dimmRawNode(mem, nil); ok {
|
||
t.Fatalf("expected no match for an unrecognized locator")
|
||
}
|
||
if _, ok := dimmRawNode(schema.HardwareMemory{}, nil); ok {
|
||
t.Fatalf("expected no match when Slot is nil")
|
||
}
|
||
}
|
||
|
||
func TestParseDIMMBankLocatorNodes(t *testing.T) {
|
||
// Real dmidecode -t 17 shape: "Locator" precedes "Bank Locator" within
|
||
// each "Memory Device" section.
|
||
raw := `Handle 0x0017, DMI type 17, 92 bytes
|
||
Memory Device
|
||
Size: 64 GB
|
||
Locator: DIMM000(A)
|
||
Bank Locator: _Node1_Channel0_Dimm0
|
||
Type: DDR5
|
||
Handle 0x0018, DMI type 17, 92 bytes
|
||
Memory Device
|
||
Size: No Module Installed
|
||
Locator: DIMM001(I)
|
||
Bank Locator: _Node1_Channel0_Dimm1
|
||
Handle 0x0019, DMI type 17, 92 bytes
|
||
Memory Device
|
||
Size: 64 GB
|
||
Locator: DIMM100(A)
|
||
Bank Locator: _Node2_Channel0_Dimm0
|
||
`
|
||
got := parseDIMMBankLocatorNodes(raw)
|
||
want := map[string]int{"DIMM000(A)": 1, "DIMM001(I)": 1, "DIMM100(A)": 2}
|
||
if len(got) != len(want) {
|
||
t.Fatalf("got=%#v want=%#v", got, want)
|
||
}
|
||
for k, v := range want {
|
||
if got[k] != v {
|
||
t.Fatalf("got[%q]=%d want %d (full: %#v)", k, got[k], v, got)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestBuildMemoryColumnIndex(t *testing.T) {
|
||
// Node numbers observed on real boards are not guaranteed 0-based (Bank
|
||
// Locator "NodeN" has been seen starting at 1) — this must rank by
|
||
// order, not treat the raw value as a column index.
|
||
idx := buildMemoryColumnIndex([]int{1, 2, 1, 2})
|
||
if idx[1] != 0 || idx[2] != 1 {
|
||
t.Fatalf("idx=%#v want {1:0, 2:1}", idx)
|
||
}
|
||
}
|
||
|
||
// TestTopoMainDiagramAttachesMemoryToItsCPUColumn is the regression test for
|
||
// the actual feature request: memory used to render as one unattached
|
||
// "MEMORY" row below the whole diagram regardless of which CPU it belonged
|
||
// to. DIMMs whose Locator carries a CPU number must now render as their own
|
||
// box directly under that CPU, wired to it with an edge, the same as
|
||
// GPU/NIC/RAID — and must NOT also show up in the leftover flex row.
|
||
func TestTopoMainDiagramAttachesMemoryToItsCPUColumn(t *testing.T) {
|
||
dir := t.TempDir()
|
||
path := filepath.Join(dir, "audit.json")
|
||
|
||
socket0, socket1 := 0, 1
|
||
okStatus := "OK"
|
||
slot0, slot1 := "CPU0_DIMM_A1", "CPU1_DIMM_A1"
|
||
sizeMB := 98304
|
||
|
||
ingest := schema.HardwareIngestRequest{
|
||
CollectedAt: "2026-03-15T00:00:00Z",
|
||
Hardware: schema.HardwareSnapshot{
|
||
CPUs: []schema.HardwareCPU{
|
||
{HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus}, Socket: &socket0},
|
||
{HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus}, Socket: &socket1},
|
||
},
|
||
Memory: []schema.HardwareMemory{
|
||
{HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus}, Slot: &slot0, SizeMB: &sizeMB},
|
||
{HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus}, Slot: &slot1, SizeMB: &sizeMB},
|
||
},
|
||
},
|
||
}
|
||
data, err := json.Marshal(ingest)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
handler := NewHandler(HandlerOptions{AuditPath: path})
|
||
rec := httptest.NewRecorder()
|
||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
|
||
body := rec.Body.String()
|
||
|
||
if strings.Count(body, `>Memory<`) != 2 {
|
||
t.Fatalf("expected 2 attached Memory boxes (one per CPU column), got body: %s", body)
|
||
}
|
||
if strings.Contains(body, "Memory</div>") {
|
||
t.Fatalf("both DIMMs matched a CPU column — the leftover unattached Memory row must not render: %s", body)
|
||
}
|
||
if strings.Count(body, "openComponentDetail('memory')") != 2 {
|
||
t.Fatalf("expected 2 clickable Memory boxes, got body: %s", body)
|
||
}
|
||
}
|
||
|
||
// TestTopoMainDiagramUnattachableMemoryFallsBackToFlexRow guards that a DIMM
|
||
// whose Locator can't be parsed into a CPU/node number still shows up
|
||
// somewhere (the old unattached row) instead of silently disappearing.
|
||
func TestTopoMainDiagramUnattachableMemoryFallsBackToFlexRow(t *testing.T) {
|
||
dir := t.TempDir()
|
||
path := filepath.Join(dir, "audit.json")
|
||
|
||
socket0 := 0
|
||
okStatus := "OK"
|
||
slot := "SOMETHING_UNRECOGNIZED"
|
||
sizeMB := 32768
|
||
|
||
ingest := schema.HardwareIngestRequest{
|
||
CollectedAt: "2026-03-15T00:00:00Z",
|
||
Hardware: schema.HardwareSnapshot{
|
||
CPUs: []schema.HardwareCPU{
|
||
{HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus}, Socket: &socket0},
|
||
},
|
||
Memory: []schema.HardwareMemory{
|
||
{HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus}, Slot: &slot, SizeMB: &sizeMB},
|
||
},
|
||
},
|
||
}
|
||
data, err := json.Marshal(ingest)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
handler := NewHandler(HandlerOptions{AuditPath: path})
|
||
rec := httptest.NewRecorder()
|
||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
|
||
body := rec.Body.String()
|
||
|
||
if !strings.Contains(body, "Memory</div>") {
|
||
t.Fatalf("unattachable DIMM should still appear in the leftover flex row: %s", body)
|
||
}
|
||
}
|
||
|
||
func TestParseTopoNVLinkStatus(t *testing.T) {
|
||
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
||
Link 0: 26.562 GB/s
|
||
Link 15: <inactive>
|
||
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
|
||
Link 0: 26.562 GB/s
|
||
`
|
||
got := parseTopoNVLinkStatus(input)
|
||
if len(got[0]) != 2 || got[0][1].Active {
|
||
t.Fatalf("gpu0=%#v want link15 inactive", got[0])
|
||
}
|
||
if len(got[1]) != 1 || got[1][0].SpeedGBs == nil || *got[1][0].SpeedGBs != 26.562 {
|
||
t.Fatalf("gpu1=%#v want link0 26.562 GB/s", got[1])
|
||
}
|
||
}
|
||
|
||
// TestComponentDetailFallsBackToInventoryWhenStatusDBEmpty covers the bug where
|
||
// a /topo card shows "3 OK" (from schema.HardwareComponentStatus.Status in the
|
||
// audit snapshot) but clicking it opens a modal saying "No status data recorded
|
||
// yet" (because ComponentStatusDB has no pcie:gpu:* entries — nothing has run a
|
||
// SAT test on this boot yet). The modal must show the same 3 devices/status the
|
||
// card does, not an empty state.
|
||
func TestComponentDetailFallsBackToInventoryWhenStatusDBEmpty(t *testing.T) {
|
||
dir := t.TempDir()
|
||
path := filepath.Join(dir, "audit.json")
|
||
|
||
okStatus := "OK"
|
||
warnStatus := "Warning"
|
||
deviceClass := "VideoController"
|
||
var gpus []schema.HardwarePCIeDevice
|
||
for i, st := range []*string{&okStatus, &okStatus, &warnStatus} {
|
||
slot := "0000:c" + strconv.Itoa(i) + ":00.0"
|
||
gpus = append(gpus, schema.HardwarePCIeDevice{
|
||
HardwareComponentStatus: schema.HardwareComponentStatus{Status: st},
|
||
DeviceClass: &deviceClass,
|
||
Slot: &slot,
|
||
})
|
||
}
|
||
|
||
ingest := schema.HardwareIngestRequest{
|
||
Hardware: schema.HardwareSnapshot{PCIeDevices: gpus},
|
||
}
|
||
data, err := json.Marshal(ingest)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
// No HandlerOptions.App / StatusDB set — matches a host where nothing has
|
||
// written to ComponentStatusDB yet.
|
||
handler := NewHandler(HandlerOptions{AuditPath: path})
|
||
rec := httptest.NewRecorder()
|
||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/components/gpu", nil))
|
||
if rec.Code != http.StatusOK {
|
||
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
|
||
}
|
||
body := rec.Body.String()
|
||
|
||
if strings.Contains(body, "No status data recorded yet") {
|
||
t.Fatalf("modal should not show empty state when inventory has GPUs: %s", body)
|
||
}
|
||
if strings.Count(body, "chip-ok") != 2 {
|
||
t.Fatalf("expected 2 OK chips from inventory fallback: %s", body)
|
||
}
|
||
if strings.Count(body, "chip-warn") != 1 {
|
||
t.Fatalf("expected 1 Warning chip from inventory fallback: %s", body)
|
||
}
|
||
if !strings.Contains(body, "No SAT-test history yet") {
|
||
t.Fatalf("expected fallback marker text: %s", body)
|
||
}
|
||
}
|
||
|
||
func TestParseTopoNVLinkErrors(t *testing.T) {
|
||
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
||
Link 0: Replay Errors: 0
|
||
Link 0: Recovery Errors: 0
|
||
Link 0: CRC Errors: 0
|
||
Link 1: Replay Errors: 3
|
||
Link 1: Recovery Errors: 1
|
||
Link 1: CRC Errors: 2
|
||
`
|
||
got := parseTopoNVLinkErrors(input)
|
||
c := got[0][1]
|
||
if c[0] != 3 || c[1] != 1 || c[2] != 2 {
|
||
t.Fatalf("link1 counters=%#v want {3,1,2}", c)
|
||
}
|
||
}
|