279 lines
10 KiB
Go
279 lines
10 KiB
Go
package platform
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestParseNvidiaNVLinkBondedPairsRealTwoGPUDump(t *testing.T) {
|
|
// Real system/nvidia-smi-topo.txt shape: two H100s directly bridged
|
|
// (NV17).
|
|
input := "\tGPU0\tGPU1\tNIC0\tNIC1\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\n" +
|
|
"GPU0\t X \tNV17\tSYS\tSYS\t0-23,48-71\t0\t\tN/A\n" +
|
|
"GPU1\tNV17\t X \tNODE\tNODE\t24-47,72-95\t1\t\tN/A\n" +
|
|
"NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" +
|
|
"NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n"
|
|
|
|
pairs := ParseNvidiaNVLinkBondedPairs(input)
|
|
if len(pairs) != 1 {
|
|
t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
|
|
}
|
|
if pairs[0].GPUA != 0 || pairs[0].GPUB != 1 || pairs[0].NVLinks != 17 {
|
|
t.Fatalf("pair=%#v want {0,1,17}", pairs[0])
|
|
}
|
|
}
|
|
|
|
func TestParseNvidiaNVLinkBondedPairsANSIUnderlinedHeader(t *testing.T) {
|
|
// nvidia-smi underlines the topo -m header with ANSI CSI codes even when
|
|
// writing to a file — the same real-world quirk fixed for the /topo page
|
|
// parser must be handled here too, since this uses a fresh live query
|
|
// rather than reading the persisted techdump.
|
|
input := "\x1b[4m\tGPU0\tGPU1\tGPU2\tGPU3\tCPU Affinity\x1b[0m\n" +
|
|
"GPU0\t X \tNV18\tPIX\tPIX\t0-31,64-95\n" +
|
|
"GPU1\tNV18\t X \tPIX\tPIX\t0-31,64-95\n" +
|
|
"GPU2\tPIX\tPIX\t X \tNV18\t0-31,64-95\n" +
|
|
"GPU3\tPIX\tPIX\tNV18\t X \t0-31,64-95\n"
|
|
|
|
pairs := ParseNvidiaNVLinkBondedPairs(input)
|
|
if len(pairs) != 2 {
|
|
t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs)
|
|
}
|
|
want := map[[2]int]int{{0, 1}: 18, {2, 3}: 18}
|
|
for _, p := range pairs {
|
|
if want[[2]int{p.GPUA, p.GPUB}] != p.NVLinks {
|
|
t.Fatalf("unexpected pair %#v", p)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestParseNvidiaNVLinkBondedPairsUsesGPUIndicesFromHeader(t *testing.T) {
|
|
input := "\tGPU2\tGPU4\tCPU Affinity\n" +
|
|
"GPU2\t X \tNV4\t0-15\n" +
|
|
"GPU4\tNV4\t X \t16-31\n"
|
|
|
|
pairs := ParseNvidiaNVLinkBondedPairs(input)
|
|
if len(pairs) != 1 {
|
|
t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
|
|
}
|
|
if pairs[0] != (NvidiaNVLinkBondedPair{GPUA: 2, GPUB: 4, NVLinks: 4}) {
|
|
t.Fatalf("pair=%#v want GPU2<->GPU4 NV4", pairs[0])
|
|
}
|
|
}
|
|
|
|
func TestParseNvidiaNVLinkStatusMarksInactiveLinks(t *testing.T) {
|
|
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
|
Link 0: 26.562 GB/s
|
|
Link 1: <inactive>
|
|
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
|
|
Link 0: 26.562 GB/s
|
|
`
|
|
got := parseNvidiaNVLinkStatus(input)
|
|
if len(got[0]) != 2 || got[0][0].active != true || got[0][1].active != false {
|
|
t.Fatalf("gpu0=%#v want [active, inactive]", got[0])
|
|
}
|
|
if len(got[1]) != 1 || !got[1][0].active {
|
|
t.Fatalf("gpu1=%#v want [active]", got[1])
|
|
}
|
|
}
|
|
|
|
func TestParseNvidiaNVLinkErrors(t *testing.T) {
|
|
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
|
Link 0: Replay Errors: 0
|
|
Link 0: Recovery Errors: 0
|
|
Link 0: CRC Errors: 0
|
|
Link 1: Replay Errors: 3
|
|
Link 1: Recovery Errors: 1
|
|
Link 1: CRC Errors: 2
|
|
`
|
|
got := ParseNvidiaNVLinkErrors(input)
|
|
c := got[0][1]
|
|
if c[0] != 3 || c[1] != 1 || c[2] != 2 {
|
|
t.Fatalf("link1 counters=%#v want {3,1,2}", c)
|
|
}
|
|
}
|
|
|
|
func TestEvaluateNvidiaNVLinkPairFlagsInactiveLinksAndErrors(t *testing.T) {
|
|
pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 2}
|
|
status := map[int][]nvidiaNVLinkPort{
|
|
0: {{active: true}, {active: false}},
|
|
1: {{active: true}, {active: true}},
|
|
}
|
|
errors := map[int]map[int][3]int64{
|
|
0: {1: {3, 0, 0}},
|
|
}
|
|
|
|
f := evaluateNvidiaNVLinkPair(pair, status, errors)
|
|
if f.ActiveLinks != 3 || f.TotalLinks != 4 {
|
|
t.Fatalf("active=%d total=%d want 3/4", f.ActiveLinks, f.TotalLinks)
|
|
}
|
|
if len(f.Issues) != 3 {
|
|
t.Fatalf("issues=%#v want 3 (expected-count mismatch + inactive link + error)", f.Issues)
|
|
}
|
|
}
|
|
|
|
func TestEvaluateNvidiaNVLinkPairHealthyBondHasNoIssues(t *testing.T) {
|
|
pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 18}
|
|
status := map[int][]nvidiaNVLinkPort{
|
|
0: activeNvidiaNVLinkPorts(18),
|
|
1: activeNvidiaNVLinkPorts(18),
|
|
}
|
|
f := evaluateNvidiaNVLinkPair(pair, status, nil)
|
|
if len(f.Issues) != 0 {
|
|
t.Fatalf("issues=%#v want none for a fully active, error-free bond", f.Issues)
|
|
}
|
|
}
|
|
|
|
func TestEvaluateNvidiaNVLinkPairChecksEachEndpoint(t *testing.T) {
|
|
pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 17}
|
|
status := map[int][]nvidiaNVLinkPort{
|
|
0: activeNvidiaNVLinkPorts(17),
|
|
1: {},
|
|
}
|
|
f := evaluateNvidiaNVLinkPair(pair, status, nil)
|
|
if f.ActiveLinksA != 17 || f.ActiveLinksB != 0 {
|
|
t.Fatalf("endpoint counts=%d,%d want 17,0", f.ActiveLinksA, f.ActiveLinksB)
|
|
}
|
|
if len(f.Issues) == 0 || !strings.Contains(strings.Join(f.Issues, " "), "GPU1 0/17") {
|
|
t.Fatalf("issues=%#v want endpoint-specific 0/17 failure", f.Issues)
|
|
}
|
|
}
|
|
|
|
func TestParseNvidiaNVLinkStatusPreservesAllInactiveGPUBlocks(t *testing.T) {
|
|
raw := `GPU 0: NVIDIA H100
|
|
NVML: Unable to retrieve NVLink information as all links are inActive
|
|
GPU 1: NVIDIA H100
|
|
all links are inActive
|
|
`
|
|
status := parseNvidiaNVLinkStatus(raw)
|
|
if _, ok := status[0]; !ok {
|
|
t.Fatal("GPU0 header was lost")
|
|
}
|
|
if _, ok := status[1]; !ok {
|
|
t.Fatal("GPU1 header was lost")
|
|
}
|
|
f := evaluateNvidiaNVLinkPair(NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 17}, status, nil)
|
|
if len(f.Issues) == 0 || f.ActiveLinksA != 0 || f.ActiveLinksB != 0 {
|
|
t.Fatalf("finding=%#v want failed 0/17 endpoints", f)
|
|
}
|
|
}
|
|
|
|
func TestEvaluateNvidiaNVLinkPairFlagsMissingGPUBlock(t *testing.T) {
|
|
f := evaluateNvidiaNVLinkPair(
|
|
NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 2},
|
|
map[int][]nvidiaNVLinkPort{0: activeNvidiaNVLinkPorts(2)}, nil,
|
|
)
|
|
if !strings.Contains(strings.Join(f.Issues, " "), "status unavailable for GPU1") {
|
|
t.Fatalf("issues=%#v want missing GPU1 status", f.Issues)
|
|
}
|
|
}
|
|
|
|
func TestParseNvidiaNVLinkDegradedDmesg(t *testing.T) {
|
|
raw := []byte("[ 11.0] md: array md0 marked Degraded\n[ 12.0] NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n")
|
|
events := parseNvidiaNVLinkDegradedDmesg(raw)
|
|
if len(events) != 1 {
|
|
t.Fatalf("events=%#v want one", events)
|
|
}
|
|
if events[0].GPUIndex != 1 || events[0].LinkID != 1 {
|
|
t.Fatalf("event=%#v want GPU1 linkId 1", events[0])
|
|
}
|
|
if warning := events[0].Warning(); !strings.Contains(warning, "GPU1 NVLink degraded mode (linkId 1)") {
|
|
t.Fatalf("warning=%q", warning)
|
|
}
|
|
wantLog := "[ 12.0] NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n"
|
|
if log := renderNvidiaNVLinkDegradedLog(events); log != wantLog {
|
|
t.Fatalf("log=%q want only NVLink line %q", log, wantLog)
|
|
}
|
|
}
|
|
|
|
func activeNvidiaNVLinkPorts(count int) []nvidiaNVLinkPort {
|
|
ports := make([]nvidiaNVLinkPort, count)
|
|
for i := range ports {
|
|
ports[i].active = true
|
|
}
|
|
return ports
|
|
}
|
|
|
|
func TestEvaluateNvidiaGPUConfigFlagsECCDisabled(t *testing.T) {
|
|
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Disabled"}
|
|
issues := evaluateNvidiaGPUConfig(g)
|
|
if len(issues) != 1 {
|
|
t.Fatalf("issues=%#v want 1 (ECC disabled)", issues)
|
|
}
|
|
}
|
|
|
|
func TestEvaluateNvidiaGPUConfigFlagsStuckMIGPending(t *testing.T) {
|
|
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", MIGCurrent: "Disabled", MIGPending: "Enabled"}
|
|
issues := evaluateNvidiaGPUConfig(g)
|
|
if len(issues) != 1 {
|
|
t.Fatalf("issues=%#v want 1 (MIG pending != current)", issues)
|
|
}
|
|
}
|
|
|
|
func TestEvaluateNvidiaGPUConfigFlagsPowerLimitBelowDefault(t *testing.T) {
|
|
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", PowerLimitW: 300, PowerDefaultLimitW: 700}
|
|
issues := evaluateNvidiaGPUConfig(g)
|
|
if len(issues) != 1 {
|
|
t.Fatalf("issues=%#v want 1 (power limit far below default)", issues)
|
|
}
|
|
}
|
|
|
|
func TestEvaluateNvidiaGPUConfigNominalHasNoIssues(t *testing.T) {
|
|
g := NvidiaGPUSetting{
|
|
Index: 0, Name: "H100", ECCCurrent: "Enabled",
|
|
MIGCurrent: "Disabled", MIGPending: "Disabled",
|
|
PowerLimitW: 700, PowerDefaultLimitW: 700,
|
|
}
|
|
issues := evaluateNvidiaGPUConfig(g)
|
|
if len(issues) != 0 {
|
|
t.Fatalf("issues=%#v want none for nominal config", issues)
|
|
}
|
|
}
|
|
|
|
func TestEvaluateNvidiaGPUConfigSmallPowerLimitDeviationIsNotFlagged(t *testing.T) {
|
|
// Power limits set slightly below default (e.g. rounding, minor
|
|
// operator tuning) should not be treated as an anomaly — only a
|
|
// meaningful cap (>5%) is.
|
|
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", PowerLimitW: 690, PowerDefaultLimitW: 700}
|
|
issues := evaluateNvidiaGPUConfig(g)
|
|
if len(issues) != 0 {
|
|
t.Fatalf("issues=%#v want none for a <2%% power-limit deviation", issues)
|
|
}
|
|
}
|
|
|
|
func TestRenderNvidiaConfigCheckSummaryOverallStatus(t *testing.T) {
|
|
clean := NvidiaConfigCheckStatus{}
|
|
if got := renderNvidiaConfigCheckSummary(clean); !strings.Contains(got, "overall_status=OK") {
|
|
t.Fatalf("clean status summary missing overall_status=OK:\n%s", got)
|
|
}
|
|
withWarning := NvidiaConfigCheckStatus{Warnings: []string{"GPU 0: ECC is disabled"}}
|
|
if got := renderNvidiaConfigCheckSummary(withWarning); !strings.Contains(got, "overall_status=FAILED") {
|
|
t.Fatalf("status with warnings missing overall_status=FAILED:\n%s", got)
|
|
}
|
|
noGPU := NvidiaConfigCheckStatus{NoGPUs: true}
|
|
if got := renderNvidiaConfigCheckSummary(noGPU); !strings.Contains(got, "overall_status=UNSUPPORTED") {
|
|
t.Fatalf("no-GPU summary must be unsupported:\n%s", got)
|
|
}
|
|
}
|
|
|
|
// TestRenderNvidiaConfigCheckSummaryIncludesWarningsField guards that a
|
|
// failed run's summary.txt carries the actual reason in prose (a "warnings"
|
|
// field), not just a bare overall_status=FAILED — app.SATFailureDetail reads
|
|
// this field so a failed task's error message names the specific GPU/NVLink
|
|
// problem instead of telling the engineer to go read summary.txt themselves.
|
|
func TestRenderNvidiaConfigCheckSummaryIncludesWarningsField(t *testing.T) {
|
|
status := NvidiaConfigCheckStatus{Warnings: []string{
|
|
"GPU 0 (H100): ECC is disabled",
|
|
"NVLink GPU0<->GPU1: 2/36 NVLinks inactive on a bonded pair",
|
|
}}
|
|
got := renderNvidiaConfigCheckSummary(status)
|
|
want := "warnings=GPU 0 (H100): ECC is disabled; NVLink GPU0<->GPU1: 2/36 NVLinks inactive on a bonded pair\n"
|
|
if !strings.Contains(got, want) {
|
|
t.Fatalf("summary missing combined warnings field:\ngot:\n%s\nwant substring:\n%s", got, want)
|
|
}
|
|
|
|
clean := NvidiaConfigCheckStatus{}
|
|
if got := renderNvidiaConfigCheckSummary(clean); strings.Contains(got, "warnings=") {
|
|
t.Fatalf("clean status summary should omit warnings field entirely:\n%s", got)
|
|
}
|
|
}
|