sat: collect NVLink status/errors, fix nv-hostengine restart via systemd
Add NVLink port status (nvidia-smi nvlink -s), error counters (nvidia-smi nvlink -e), and dcgmi nvlink status to the support bundle, and enrich HardwarePCIeDevice entries with per-link telemetry. Replace the raw nv-hostengine pkill/restart dance in bee-nvidia-load with systemctl restart/start of nvidia-dcgm.service, and order bee-nvidia.service Before= nvidia-dcgm.service and nvidia-fabricmanager.service so modules/device nodes exist before those units start.
This commit is contained in:
@@ -126,3 +126,92 @@ func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) {
|
||||
|
||||
func ptrInt64(v int64) *int64 { return &v }
|
||||
func ptrFloat(v float64) *float64 { return &v }
|
||||
|
||||
func TestParseNVIDIANVLinkStatusByGPU(t *testing.T) {
|
||||
// Real-world 2-GPU direct-bridge H100 SXM output: link 15 inactive on both GPUs.
|
||||
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
||||
Link 0: 26.562 GB/s
|
||||
Link 15: <inactive>
|
||||
Link 17: 26.562 GB/s
|
||||
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
|
||||
Link 0: 26.562 GB/s
|
||||
Link 15: <inactive>
|
||||
`
|
||||
got := parseNVIDIANVLinkStatusByGPU(input)
|
||||
|
||||
if len(got[0]) != 3 {
|
||||
t.Fatalf("gpu0 ports=%d want 3 (%#v)", len(got[0]), got[0])
|
||||
}
|
||||
if got[0][1].Index != 15 || got[0][1].Active {
|
||||
t.Fatalf("gpu0 link15=%#v want inactive", got[0][1])
|
||||
}
|
||||
if got[0][0].SpeedGBs == nil || *got[0][0].SpeedGBs != 26.562 {
|
||||
t.Fatalf("gpu0 link0 speed=%#v want 26.562", got[0][0].SpeedGBs)
|
||||
}
|
||||
if len(got[1]) != 2 {
|
||||
t.Fatalf("gpu1 ports=%d want 2 (%#v)", len(got[1]), got[1])
|
||||
}
|
||||
if got[1][1].Active {
|
||||
t.Fatalf("gpu1 link15 should be inactive: %#v", got[1][1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNVIDIANVLinkErrorsByGPU(t *testing.T) {
|
||||
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
||||
Link 0: Replay Errors: 0
|
||||
Link 0: Recovery Errors: 0
|
||||
Link 0: CRC Errors: 0
|
||||
Link 1: Replay Errors: 3
|
||||
Link 1: Recovery Errors: 1
|
||||
Link 1: CRC Errors: 2
|
||||
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
|
||||
Link 0: Replay Errors: 0
|
||||
Link 0: Recovery Errors: 0
|
||||
Link 0: CRC Errors: 0
|
||||
`
|
||||
got := parseNVIDIANVLinkErrorsByGPU(input)
|
||||
|
||||
c := got[0][1]
|
||||
if c.Replay != 3 || c.Recovery != 1 || c.CRC != 2 {
|
||||
t.Fatalf("gpu0 link1 counters=%#v want {3,1,2}", c)
|
||||
}
|
||||
zero := got[0][0]
|
||||
if zero.Replay != 0 || zero.Recovery != 0 || zero.CRC != 0 {
|
||||
t.Fatalf("gpu0 link0 counters=%#v want all zero", zero)
|
||||
}
|
||||
if _, ok := got[1][0]; !ok {
|
||||
t.Fatalf("expected gpu1 link0 entry present")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichPCIeWithNVIDIANVLinksAttachesPortsByIndex(t *testing.T) {
|
||||
oldStatus, oldErrors := nvlinkStatusFn, nvlinkErrorsFn
|
||||
t.Cleanup(func() { nvlinkStatusFn, nvlinkErrorsFn = oldStatus, oldErrors })
|
||||
|
||||
nvlinkStatusFn = func() (map[int][]schema.HardwareNVLinkPort, error) {
|
||||
return map[int][]schema.HardwareNVLinkPort{
|
||||
0: {{Index: 0, Active: true, SpeedGBs: ptrFloat(26.562)}, {Index: 15, Active: false}},
|
||||
}, nil
|
||||
}
|
||||
nvlinkErrorsFn = func() (map[int]map[int]nvlinkErrorCounters, error) {
|
||||
return map[int]map[int]nvlinkErrorCounters{
|
||||
0: {0: {Replay: 1}},
|
||||
}, nil
|
||||
}
|
||||
|
||||
devices := []schema.HardwarePCIeDevice{
|
||||
{Telemetry: map[string]any{"nvidia_gpu_index": 0}},
|
||||
}
|
||||
|
||||
out := enrichPCIeWithNVIDIANVLinks(devices)
|
||||
|
||||
if len(out[0].NVLinks) != 2 {
|
||||
t.Fatalf("nvlinks=%d want 2", len(out[0].NVLinks))
|
||||
}
|
||||
if out[0].NVLinks[0].ReplayErrors == nil || *out[0].NVLinks[0].ReplayErrors != 1 {
|
||||
t.Fatalf("link0 replay errors=%#v want 1", out[0].NVLinks[0].ReplayErrors)
|
||||
}
|
||||
if out[0].NVLinks[1].Active {
|
||||
t.Fatalf("link15 should stay inactive")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user