reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

refactor: harden diagnostics and consolidate runtime code

Browse Source
This commit is contained in:
Mikhail Chusavitin
2026-09-01 13:01:28 +03:00
parent ac4bc0b2b7
commit 0a6ca8ba0f
49 changed files with 1441 additions and 837 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/platform/nvidia_config_check_test.go
+95 -11
View File
@@ -14,11 +14,11 @@ func TestParseNvidiaNVLinkBondedPairsRealTwoGPUDump(t *testing.T) {
"NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" +
"NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n"
pairs := parseNvidiaNVLinkBondedPairs(input)
pairs := ParseNvidiaNVLinkBondedPairs(input)
if len(pairs) != 1 {
t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
}
if pairs[0].gpuA != 0 || pairs[0].gpuB != 1 || pairs[0].links != 17 {
if pairs[0].GPUA != 0 || pairs[0].GPUB != 1 || pairs[0].NVLinks != 17 {
t.Fatalf("pair=%#v want {0,1,17}", pairs[0])
}
}
@@ -34,18 +34,32 @@ func TestParseNvidiaNVLinkBondedPairsANSIUnderlinedHeader(t *testing.T) {
"GPU2\tPIX\tPIX\t X \tNV18\t0-31,64-95\n" +
"GPU3\tPIX\tPIX\tNV18\t X \t0-31,64-95\n"
pairs := parseNvidiaNVLinkBondedPairs(input)
pairs := ParseNvidiaNVLinkBondedPairs(input)
if len(pairs) != 2 {
t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs)
}
want := map[[2]int]int{{0, 1}: 18, {2, 3}: 18}
for _, p := range pairs {
if want[[2]int{p.gpuA, p.gpuB}] != p.links {
if want[[2]int{p.GPUA, p.GPUB}] != p.NVLinks {
t.Fatalf("unexpected pair %#v", p)
}
}
}
func TestParseNvidiaNVLinkBondedPairsUsesGPUIndicesFromHeader(t *testing.T) {
input := "\tGPU2\tGPU4\tCPU Affinity\n" +
"GPU2\t X \tNV4\t0-15\n" +
"GPU4\tNV4\t X \t16-31\n"
pairs := ParseNvidiaNVLinkBondedPairs(input)
if len(pairs) != 1 {
t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
}
if pairs[0] != (NvidiaNVLinkBondedPair{GPUA: 2, GPUB: 4, NVLinks: 4}) {
t.Fatalf("pair=%#v want GPU2<->GPU4 NV4", pairs[0])
}
}
func TestParseNvidiaNVLinkStatusMarksInactiveLinks(t *testing.T) {
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
Link 0: 26.562 GB/s
@@ -71,7 +85,7 @@ func TestParseNvidiaNVLinkErrors(t *testing.T) {
Link 1: Recovery Errors: 1
Link 1: CRC Errors: 2
`
got := parseNvidiaNVLinkErrors(input)
got := ParseNvidiaNVLinkErrors(input)
c := got[0][1]
if c[0] != 3 || c[1] != 1 || c[2] != 2 {
t.Fatalf("link1 counters=%#v want {3,1,2}", c)
@@ -79,7 +93,7 @@ func TestParseNvidiaNVLinkErrors(t *testing.T) {
}
func TestEvaluateNvidiaNVLinkPairFlagsInactiveLinksAndErrors(t *testing.T) {
pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18}
pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 2}
status := map[int][]nvidiaNVLinkPort{
0: {{active: true}, {active: false}},
1: {{active: true}, {active: true}},
@@ -92,16 +106,16 @@ func TestEvaluateNvidiaNVLinkPairFlagsInactiveLinksAndErrors(t *testing.T) {
if f.ActiveLinks != 3 || f.TotalLinks != 4 {
t.Fatalf("active=%d total=%d want 3/4", f.ActiveLinks, f.TotalLinks)
}
if len(f.Issues) != 2 {
t.Fatalf("issues=%#v want 2 (inactive link + error)", f.Issues)
if len(f.Issues) != 3 {
t.Fatalf("issues=%#v want 3 (expected-count mismatch + inactive link + error)", f.Issues)
}
}
func TestEvaluateNvidiaNVLinkPairHealthyBondHasNoIssues(t *testing.T) {
pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18}
pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 18}
status := map[int][]nvidiaNVLinkPort{
0: {{active: true}},
1: {{active: true}},
0: activeNvidiaNVLinkPorts(18),
1: activeNvidiaNVLinkPorts(18),
}
f := evaluateNvidiaNVLinkPair(pair, status, nil)
if len(f.Issues) != 0 {
@@ -109,6 +123,76 @@ func TestEvaluateNvidiaNVLinkPairHealthyBondHasNoIssues(t *testing.T) {
}
}
func TestEvaluateNvidiaNVLinkPairChecksEachEndpoint(t *testing.T) {
pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 17}
status := map[int][]nvidiaNVLinkPort{
0: activeNvidiaNVLinkPorts(17),
1: {},
}
f := evaluateNvidiaNVLinkPair(pair, status, nil)
if f.ActiveLinksA != 17 || f.ActiveLinksB != 0 {
t.Fatalf("endpoint counts=%d,%d want 17,0", f.ActiveLinksA, f.ActiveLinksB)
}
if len(f.Issues) == 0 || !strings.Contains(strings.Join(f.Issues, " "), "GPU1 0/17") {
t.Fatalf("issues=%#v want endpoint-specific 0/17 failure", f.Issues)
}
}
func TestParseNvidiaNVLinkStatusPreservesAllInactiveGPUBlocks(t *testing.T) {
raw := `GPU 0: NVIDIA H100
NVML: Unable to retrieve NVLink information as all links are inActive
GPU 1: NVIDIA H100
all links are inActive
`
status := parseNvidiaNVLinkStatus(raw)
if _, ok := status[0]; !ok {
t.Fatal("GPU0 header was lost")
}
if _, ok := status[1]; !ok {
t.Fatal("GPU1 header was lost")
}
f := evaluateNvidiaNVLinkPair(NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 17}, status, nil)
if len(f.Issues) == 0 || f.ActiveLinksA != 0 || f.ActiveLinksB != 0 {
t.Fatalf("finding=%#v want failed 0/17 endpoints", f)
}
}
func TestEvaluateNvidiaNVLinkPairFlagsMissingGPUBlock(t *testing.T) {
f := evaluateNvidiaNVLinkPair(
NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 2},
map[int][]nvidiaNVLinkPort{0: activeNvidiaNVLinkPorts(2)}, nil,
)
if !strings.Contains(strings.Join(f.Issues, " "), "status unavailable for GPU1") {
t.Fatalf("issues=%#v want missing GPU1 status", f.Issues)
}
}
func TestParseNvidiaNVLinkDegradedDmesg(t *testing.T) {
raw := []byte("[ 11.0] md: array md0 marked Degraded\n[ 12.0] NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n")
events := parseNvidiaNVLinkDegradedDmesg(raw)
if len(events) != 1 {
t.Fatalf("events=%#v want one", events)
}
if events[0].GPUIndex != 1 || events[0].LinkID != 1 {
t.Fatalf("event=%#v want GPU1 linkId 1", events[0])
}
if warning := events[0].Warning(); !strings.Contains(warning, "GPU1 NVLink degraded mode (linkId 1)") {
t.Fatalf("warning=%q", warning)
}
wantLog := "[ 12.0] NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n"
if log := renderNvidiaNVLinkDegradedLog(events); log != wantLog {
t.Fatalf("log=%q want only NVLink line %q", log, wantLog)
}
}
func activeNvidiaNVLinkPorts(count int) []nvidiaNVLinkPort {
ports := make([]nvidiaNVLinkPort, count)
for i := range ports {
ports[i].active = true
}
return ports
}
func TestEvaluateNvidiaGPUConfigFlagsECCDisabled(t *testing.T) {
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Disabled"}
issues := evaluateNvidiaGPUConfig(g)