sat: collect NVLink status/errors, fix nv-hostengine restart via systemd

Add NVLink port status (nvidia-smi nvlink -s), error counters
(nvidia-smi nvlink -e), and dcgmi nvlink status to the support bundle,
and enrich HardwarePCIeDevice entries with per-link telemetry.

Replace the raw nv-hostengine pkill/restart dance in bee-nvidia-load
with systemctl restart/start of nvidia-dcgm.service, and order
bee-nvidia.service Before= nvidia-dcgm.service and
nvidia-fabricmanager.service so modules/device nodes exist before
those units start.
This commit is contained in:
Mikhail Chusavitin
2026-07-07 10:21:56 +03:00
parent a98721b038
commit 33d6eee9cf
6 changed files with 339 additions and 81 deletions
+89
View File
@@ -126,3 +126,92 @@ func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) {
func ptrInt64(v int64) *int64 { return &v }
func ptrFloat(v float64) *float64 { return &v }
func TestParseNVIDIANVLinkStatusByGPU(t *testing.T) {
// Real-world 2-GPU direct-bridge H100 SXM output: link 15 inactive on both GPUs.
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
Link 0: 26.562 GB/s
Link 15: <inactive>
Link 17: 26.562 GB/s
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
Link 0: 26.562 GB/s
Link 15: <inactive>
`
got := parseNVIDIANVLinkStatusByGPU(input)
if len(got[0]) != 3 {
t.Fatalf("gpu0 ports=%d want 3 (%#v)", len(got[0]), got[0])
}
if got[0][1].Index != 15 || got[0][1].Active {
t.Fatalf("gpu0 link15=%#v want inactive", got[0][1])
}
if got[0][0].SpeedGBs == nil || *got[0][0].SpeedGBs != 26.562 {
t.Fatalf("gpu0 link0 speed=%#v want 26.562", got[0][0].SpeedGBs)
}
if len(got[1]) != 2 {
t.Fatalf("gpu1 ports=%d want 2 (%#v)", len(got[1]), got[1])
}
if got[1][1].Active {
t.Fatalf("gpu1 link15 should be inactive: %#v", got[1][1])
}
}
func TestParseNVIDIANVLinkErrorsByGPU(t *testing.T) {
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
Link 0: Replay Errors: 0
Link 0: Recovery Errors: 0
Link 0: CRC Errors: 0
Link 1: Replay Errors: 3
Link 1: Recovery Errors: 1
Link 1: CRC Errors: 2
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
Link 0: Replay Errors: 0
Link 0: Recovery Errors: 0
Link 0: CRC Errors: 0
`
got := parseNVIDIANVLinkErrorsByGPU(input)
c := got[0][1]
if c.Replay != 3 || c.Recovery != 1 || c.CRC != 2 {
t.Fatalf("gpu0 link1 counters=%#v want {3,1,2}", c)
}
zero := got[0][0]
if zero.Replay != 0 || zero.Recovery != 0 || zero.CRC != 0 {
t.Fatalf("gpu0 link0 counters=%#v want all zero", zero)
}
if _, ok := got[1][0]; !ok {
t.Fatalf("expected gpu1 link0 entry present")
}
}
func TestEnrichPCIeWithNVIDIANVLinksAttachesPortsByIndex(t *testing.T) {
oldStatus, oldErrors := nvlinkStatusFn, nvlinkErrorsFn
t.Cleanup(func() { nvlinkStatusFn, nvlinkErrorsFn = oldStatus, oldErrors })
nvlinkStatusFn = func() (map[int][]schema.HardwareNVLinkPort, error) {
return map[int][]schema.HardwareNVLinkPort{
0: {{Index: 0, Active: true, SpeedGBs: ptrFloat(26.562)}, {Index: 15, Active: false}},
}, nil
}
nvlinkErrorsFn = func() (map[int]map[int]nvlinkErrorCounters, error) {
return map[int]map[int]nvlinkErrorCounters{
0: {0: {Replay: 1}},
}, nil
}
devices := []schema.HardwarePCIeDevice{
{Telemetry: map[string]any{"nvidia_gpu_index": 0}},
}
out := enrichPCIeWithNVIDIANVLinks(devices)
if len(out[0].NVLinks) != 2 {
t.Fatalf("nvlinks=%d want 2", len(out[0].NVLinks))
}
if out[0].NVLinks[0].ReplayErrors == nil || *out[0].NVLinks[0].ReplayErrors != 1 {
t.Fatalf("link0 replay errors=%#v want 1", out[0].NVLinks[0].ReplayErrors)
}
if out[0].NVLinks[1].Active {
t.Fatalf("link15 should stay inactive")
}
}