- nvidia-smi underlines the topo -m header with ANSI CSI codes even when writing to a file; both NVLink matrix parsers failed to find the header and /topo showed "No NVLink-bonded GPU pairs found" on bonded systems. - raid-lsi-create-mirror failed with "resources already in use" (exit 11) on JBOD drives; the task now reads drive states and auto-converts JBOD/UBad (set good force), releases hotspares, refuses Frgn/Onln with actionable messages, and dumps preservedcache info when add vd fails. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
283 lines
8.7 KiB
Go
283 lines
8.7 KiB
Go
package webui
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"bee/audit/internal/schema"
|
|
)
|
|
|
|
func TestTopoPageNoAuditDataGracefulFallback(t *testing.T) {
|
|
handler := NewHandler(HandlerOptions{})
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("status=%d", rec.Code)
|
|
}
|
|
body := rec.Body.String()
|
|
if !strings.Contains(body, "No audit data") {
|
|
t.Fatalf("topo page missing no-audit-data fallback: %s", body)
|
|
}
|
|
}
|
|
|
|
func TestTopoPageRendersCPUAndDegradedPCIeLink(t *testing.T) {
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "audit.json")
|
|
|
|
socket := 0
|
|
numaNode := 0
|
|
gen3, gen4 := "Gen3", "Gen4"
|
|
deviceClass := "VideoController"
|
|
model := "NVIDIA H100 80GB HBM3"
|
|
cpuModel := "Intel Xeon 6530"
|
|
okStatus := "OK"
|
|
|
|
ingest := schema.HardwareIngestRequest{
|
|
CollectedAt: "2026-03-15T00:00:00Z",
|
|
Hardware: schema.HardwareSnapshot{
|
|
CPUs: []schema.HardwareCPU{
|
|
{
|
|
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus},
|
|
Socket: &socket,
|
|
Model: &cpuModel,
|
|
},
|
|
},
|
|
PCIeDevices: []schema.HardwarePCIeDevice{
|
|
{
|
|
DeviceClass: &deviceClass,
|
|
Model: &model,
|
|
NUMANode: &numaNode,
|
|
LinkSpeed: &gen3,
|
|
MaxLinkSpeed: &gen4,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
data, err := json.Marshal(ingest)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(path, data, 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
handler := NewHandler(HandlerOptions{AuditPath: path})
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("status=%d", rec.Code)
|
|
}
|
|
body := rec.Body.String()
|
|
if !strings.Contains(body, "CPU 0") {
|
|
t.Fatalf("topo page missing CPU 0 box: %s", body)
|
|
}
|
|
if !strings.Contains(body, "GPU") {
|
|
t.Fatalf("topo page missing GPU box: %s", body)
|
|
}
|
|
if !strings.Contains(body, "var(--warn-fg)") {
|
|
t.Fatalf("topo page missing degraded-link warn edge color: %s", body)
|
|
}
|
|
}
|
|
|
|
func TestTopoPageLinkedFromNav(t *testing.T) {
|
|
handler := NewHandler(HandlerOptions{})
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("status=%d", rec.Code)
|
|
}
|
|
if !strings.Contains(rec.Body.String(), `href="/topo"`) {
|
|
t.Fatalf("nav missing /topo link")
|
|
}
|
|
}
|
|
|
|
func TestParseGPUPairAdjacencyRealTwoGPUDump(t *testing.T) {
|
|
// Real system/nvidia-smi-topo.txt from a support bundle for this exact
|
|
// server: two H100s directly bridged (NV17), spanning two NUMA nodes.
|
|
input := "\tGPU0\tGPU1\tNIC0\tNIC1\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\n" +
|
|
"GPU0\t X \tNV17\tSYS\tSYS\t0-23,48-71\t0\t\tN/A\n" +
|
|
"GPU1\tNV17\t X \tNODE\tNODE\t24-47,72-95\t1\t\tN/A\n" +
|
|
"NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" +
|
|
"NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n"
|
|
|
|
pairs := parseGPUPairAdjacency(input)
|
|
if len(pairs) != 1 {
|
|
t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
|
|
}
|
|
if pairs[0].GPUA != 0 || pairs[0].GPUB != 1 || pairs[0].NVLinks != 17 {
|
|
t.Fatalf("pair=%#v want {0,1,17}", pairs[0])
|
|
}
|
|
}
|
|
|
|
func TestParseGPUPairAdjacencyDoesNotChainUnrelatedGPUs(t *testing.T) {
|
|
// 4 GPUs: only (0,1) and (2,3) are actually bonded. GPU1 and GPU2 must
|
|
// NOT get an edge just because they're adjacent in the layout.
|
|
input := "\tGPU0\tGPU1\tGPU2\tGPU3\n" +
|
|
"GPU0\t X \tNV18\tSYS\tSYS\n" +
|
|
"GPU1\tNV18\t X \tSYS\tSYS\n" +
|
|
"GPU2\tSYS\tSYS\t X \tNV18\n" +
|
|
"GPU3\tSYS\tSYS\tNV18\t X \n"
|
|
|
|
pairs := parseGPUPairAdjacency(input)
|
|
if len(pairs) != 2 {
|
|
t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs)
|
|
}
|
|
want := map[[2]int]bool{{0, 1}: true, {2, 3}: true}
|
|
for _, p := range pairs {
|
|
if !want[[2]int{p.GPUA, p.GPUB}] {
|
|
t.Fatalf("unexpected pair %#v (GPU1-GPU2 chaining bug?)", p)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestParseGPUPairAdjacencyANSIUnderlinedHeader(t *testing.T) {
|
|
// Real techdump capture: nvidia-smi underlines the header row with ANSI
|
|
// escapes even when writing to a file, so the header line starts with
|
|
// ESC[4m, not "GPU0".
|
|
input := "\x1b[4m\tGPU0\tGPU1\tGPU2\tGPU3\tNIC0\tNIC1\tCPU Affinity\x1b[0m\n" +
|
|
"GPU0\t X \tNV18\tPIX\tPIX\tNODE\tNODE\t0-31,64-95\t0\n" +
|
|
"GPU1\tNV18\t X \tPIX\tPIX\tNODE\tNODE\t0-31,64-95\t0\n" +
|
|
"GPU2\tPIX\tPIX\t X \tNV18\tNODE\tNODE\t0-31,64-95\t0\n" +
|
|
"GPU3\tPIX\tPIX\tNV18\t X \tNODE\tNODE\t0-31,64-95\t0\n" +
|
|
"NIC0\tNODE\tNODE\tNODE\tNODE\t X \tPIX\n" +
|
|
"NIC1\tNODE\tNODE\tNODE\tNODE\tPIX\t X \n"
|
|
|
|
pairs := parseGPUPairAdjacency(input)
|
|
if len(pairs) != 2 {
|
|
t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs)
|
|
}
|
|
want := map[[2]int]bool{{0, 1}: true, {2, 3}: true}
|
|
for _, p := range pairs {
|
|
if !want[[2]int{p.GPUA, p.GPUB}] || p.NVLinks != 18 {
|
|
t.Fatalf("unexpected pair %#v", p)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestParseGPUPairAdjacencyEmptyOnNoMatrix(t *testing.T) {
|
|
if pairs := parseGPUPairAdjacency("no gpus here"); pairs != nil {
|
|
t.Fatalf("pairs=%v want nil", pairs)
|
|
}
|
|
}
|
|
|
|
func TestPcieGenRank(t *testing.T) {
|
|
if pcieGenRank("Gen5") <= pcieGenRank("Gen4") {
|
|
t.Fatalf("Gen5 should rank higher than Gen4")
|
|
}
|
|
if pcieGenRank("bogus") != 0 {
|
|
t.Fatalf("unparseable gen should rank 0")
|
|
}
|
|
}
|
|
|
|
func TestTopoEdgeColorVar(t *testing.T) {
|
|
gen3, gen4 := "Gen3", "Gen4"
|
|
degraded := schema.HardwarePCIeDevice{LinkSpeed: &gen3, MaxLinkSpeed: &gen4}
|
|
if got := topoEdgeColorVar(degraded); got != "var(--warn-fg)" {
|
|
t.Fatalf("degraded color=%q want warn", got)
|
|
}
|
|
full := schema.HardwarePCIeDevice{LinkSpeed: &gen4, MaxLinkSpeed: &gen4}
|
|
if got := topoEdgeColorVar(full); got != "var(--ok-fg)" {
|
|
t.Fatalf("full-speed color=%q want ok", got)
|
|
}
|
|
unknown := schema.HardwarePCIeDevice{}
|
|
if got := topoEdgeColorVar(unknown); got != "var(--muted)" {
|
|
t.Fatalf("unknown color=%q want muted", got)
|
|
}
|
|
}
|
|
|
|
func TestBuildSocketIndex(t *testing.T) {
|
|
s0, s1 := 0, 1
|
|
cpus := []schema.HardwareCPU{{Socket: &s1}, {Socket: &s0}}
|
|
idx := buildSocketIndex(cpus)
|
|
if idx[0] != 1 || idx[1] != 0 {
|
|
t.Fatalf("idx=%#v want {0:1, 1:0}", idx)
|
|
}
|
|
}
|
|
|
|
func TestBuildSocketIndexOneIndexedSocketDesignation(t *testing.T) {
|
|
// dmidecode "Socket Designation" is frequently 1-indexed ("CPU1", "CPU2")
|
|
// while Linux NUMA nodes are always 0-indexed. NUMA node 0 must still
|
|
// resolve to the first CPU in Socket order, not fall through to the
|
|
// "unknown" column (the bug reported against the /topo page).
|
|
s1, s2 := 1, 2
|
|
cpus := []schema.HardwareCPU{{Socket: &s1}, {Socket: &s2}}
|
|
idx := buildSocketIndex(cpus)
|
|
if idx[0] != 0 || idx[1] != 1 {
|
|
t.Fatalf("idx=%#v want {0:0, 1:1}", idx)
|
|
}
|
|
}
|
|
|
|
func TestIsRAIDControllerClass(t *testing.T) {
|
|
if !isRAIDControllerClass("StorageController") || !isRAIDControllerClass("MassStorageController") {
|
|
t.Fatalf("expected known RAID/storage classes to match")
|
|
}
|
|
if isRAIDControllerClass("VideoController") {
|
|
t.Fatalf("GPU class should not match RAID classifier")
|
|
}
|
|
}
|
|
|
|
func TestIsNICDeviceClassDev(t *testing.T) {
|
|
class := "EthernetController"
|
|
nic := schema.HardwarePCIeDevice{DeviceClass: &class}
|
|
if !isNICDeviceClassDev(nic) {
|
|
t.Fatalf("expected EthernetController to classify as NIC")
|
|
}
|
|
withMac := schema.HardwarePCIeDevice{MacAddresses: []string{"aa:bb:cc:dd:ee:ff"}}
|
|
if !isNICDeviceClassDev(withMac) {
|
|
t.Fatalf("expected device with MAC address to classify as NIC")
|
|
}
|
|
other := schema.HardwarePCIeDevice{}
|
|
if isNICDeviceClassDev(other) {
|
|
t.Fatalf("expected empty device to not classify as NIC")
|
|
}
|
|
}
|
|
|
|
func TestTopoStatusBadgeClassNilIsUnknown(t *testing.T) {
|
|
label, cls := topoStatusBadgeClass(nil)
|
|
if label != "?" || cls != "badge-unknown" {
|
|
t.Fatalf("nil status = (%q,%q) want (?, badge-unknown)", label, cls)
|
|
}
|
|
ok := "OK"
|
|
label, cls = topoStatusBadgeClass(&ok)
|
|
if label != "OK" || cls != "badge-ok" {
|
|
t.Fatalf("OK status = (%q,%q) want (OK, badge-ok)", label, cls)
|
|
}
|
|
}
|
|
|
|
func TestParseTopoNVLinkStatus(t *testing.T) {
|
|
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
|
Link 0: 26.562 GB/s
|
|
Link 15: <inactive>
|
|
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
|
|
Link 0: 26.562 GB/s
|
|
`
|
|
got := parseTopoNVLinkStatus(input)
|
|
if len(got[0]) != 2 || got[0][1].Active {
|
|
t.Fatalf("gpu0=%#v want link15 inactive", got[0])
|
|
}
|
|
if len(got[1]) != 1 || got[1][0].SpeedGBs == nil || *got[1][0].SpeedGBs != 26.562 {
|
|
t.Fatalf("gpu1=%#v want link0 26.562 GB/s", got[1])
|
|
}
|
|
}
|
|
|
|
func TestParseTopoNVLinkErrors(t *testing.T) {
|
|
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
|
Link 0: Replay Errors: 0
|
|
Link 0: Recovery Errors: 0
|
|
Link 0: CRC Errors: 0
|
|
Link 1: Replay Errors: 3
|
|
Link 1: Recovery Errors: 1
|
|
Link 1: CRC Errors: 2
|
|
`
|
|
got := parseTopoNVLinkErrors(input)
|
|
c := got[0][1]
|
|
if c[0] != 3 || c[1] != 1 || c[2] != 2 {
|
|
t.Fatalf("link1 counters=%#v want {3,1,2}", c)
|
|
}
|
|
}
|