Mikhail Chusavitin
33d6eee9cf
Add NVLink port status (nvidia-smi nvlink -s), error counters (nvidia-smi nvlink -e), and dcgmi nvlink status to the support bundle, and enrich HardwarePCIeDevice entries with per-link telemetry. Replace the raw nv-hostengine pkill/restart dance in bee-nvidia-load with systemctl restart/start of nvidia-dcgm.service, and order bee-nvidia.service Before= nvidia-dcgm.service and nvidia-fabricmanager.service so modules/device nodes exist before those units start.
218 lines
6.5 KiB
Go
218 lines
6.5 KiB
Go
package collector
|
|
|
|
import (
|
|
"bee/audit/internal/schema"
|
|
"testing"
|
|
)
|
|
|
|
func TestParseNVIDIASMIQuery(t *testing.T) {
|
|
raw := "0, 00000000:65:00.0, NVIDIA H100 80GB HBM3, GPU-SERIAL-1, 96.00.1F.00.02, 54, 210.33, 0, 5, Not Active, 4, 4, 16, 16\n"
|
|
byBDF, err := parseNVIDIASMIQuery(raw)
|
|
if err != nil {
|
|
t.Fatalf("parse failed: %v", err)
|
|
}
|
|
|
|
gpu, ok := byBDF["0000:65:00.0"]
|
|
if !ok {
|
|
t.Fatalf("gpu by normalized bdf not found")
|
|
}
|
|
if gpu.Name != "NVIDIA H100 80GB HBM3" {
|
|
t.Fatalf("name: got %q", gpu.Name)
|
|
}
|
|
if gpu.Serial != "GPU-SERIAL-1" {
|
|
t.Fatalf("serial: got %q", gpu.Serial)
|
|
}
|
|
if gpu.VBIOS != "96.00.1F.00.02" {
|
|
t.Fatalf("vbios: got %q", gpu.VBIOS)
|
|
}
|
|
if gpu.ECCUncorrected == nil || *gpu.ECCUncorrected != 0 {
|
|
t.Fatalf("ecc uncorrected: got %v", gpu.ECCUncorrected)
|
|
}
|
|
if gpu.HWSlowdown == nil || *gpu.HWSlowdown {
|
|
t.Fatalf("hw slowdown: got %v, want false", gpu.HWSlowdown)
|
|
}
|
|
if gpu.PCIeLinkGenCurrent == nil || *gpu.PCIeLinkGenCurrent != 4 {
|
|
t.Fatalf("pcie link gen current: got %v, want 4", gpu.PCIeLinkGenCurrent)
|
|
}
|
|
if gpu.PCIeLinkGenMax == nil || *gpu.PCIeLinkGenMax != 4 {
|
|
t.Fatalf("pcie link gen max: got %v, want 4", gpu.PCIeLinkGenMax)
|
|
}
|
|
}
|
|
|
|
func TestNormalizePCIeBDF(t *testing.T) {
|
|
tests := []struct {
|
|
in string
|
|
want string
|
|
}{
|
|
{"00000000:17:00.0", "0000:17:00.0"},
|
|
{"0000:17:00.0", "0000:17:00.0"},
|
|
{"17:00.0", "0000:17:00.0"},
|
|
}
|
|
for _, tt := range tests {
|
|
got := normalizePCIeBDF(tt.in)
|
|
if got != tt.want {
|
|
t.Fatalf("normalizePCIeBDF(%q)=%q want %q", tt.in, got, tt.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEnrichPCIeWithNVIDIAData_driverLoaded(t *testing.T) {
|
|
vendorID := NvidiaVendorID
|
|
bdf := "0000:65:00.0"
|
|
manufacturer := "NVIDIA Corporation"
|
|
status := "OK"
|
|
devices := []schema.HardwarePCIeDevice{
|
|
{
|
|
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status},
|
|
VendorID: &vendorID,
|
|
BDF: &bdf,
|
|
Manufacturer: &manufacturer,
|
|
},
|
|
}
|
|
|
|
byBDF := map[string]nvidiaGPUInfo{
|
|
"0000:65:00.0": {
|
|
BDF: "0000:65:00.0",
|
|
Serial: "GPU-ABC",
|
|
VBIOS: "96.00.1F.00.02",
|
|
ECCUncorrected: ptrInt64(2),
|
|
ECCCorrected: ptrInt64(10),
|
|
TemperatureC: ptrFloat(55.5),
|
|
PowerW: ptrFloat(230.2),
|
|
},
|
|
}
|
|
|
|
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
|
|
if out[0].SerialNumber == nil || *out[0].SerialNumber != "GPU-ABC" {
|
|
t.Fatalf("serial: got %v", out[0].SerialNumber)
|
|
}
|
|
if out[0].Firmware == nil || *out[0].Firmware != "96.00.1F.00.02" {
|
|
t.Fatalf("firmware: got %v", out[0].Firmware)
|
|
}
|
|
if out[0].Telemetry == nil || out[0].Telemetry["nvidia_gpu_index"] != 0 {
|
|
t.Fatalf("telemetry nvidia_gpu_index: got %#v", out[0].Telemetry)
|
|
}
|
|
if out[0].Status == nil || *out[0].Status != statusWarning {
|
|
t.Fatalf("status: got %v", out[0].Status)
|
|
}
|
|
if out[0].ECCUncorrectedTotal == nil || *out[0].ECCUncorrectedTotal != 2 {
|
|
t.Fatalf("ecc_uncorrected_total: got %#v", out[0].ECCUncorrectedTotal)
|
|
}
|
|
if out[0].TemperatureC == nil || *out[0].TemperatureC != 55.5 {
|
|
t.Fatalf("temperature_c: got %#v", out[0].TemperatureC)
|
|
}
|
|
}
|
|
|
|
func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) {
|
|
vendorID := NvidiaVendorID
|
|
bdf := "0000:17:00.0"
|
|
manufacturer := "NVIDIA Corporation"
|
|
devices := []schema.HardwarePCIeDevice{
|
|
{
|
|
VendorID: &vendorID,
|
|
BDF: &bdf,
|
|
Manufacturer: &manufacturer,
|
|
},
|
|
}
|
|
|
|
out := enrichPCIeWithNVIDIAData(devices, nil, false)
|
|
if out[0].SerialNumber != nil {
|
|
t.Fatalf("serial should stay nil without source data, got %v", out[0].SerialNumber)
|
|
}
|
|
if out[0].Status == nil || *out[0].Status != statusUnknown {
|
|
t.Fatalf("fallback status: got %v", out[0].Status)
|
|
}
|
|
}
|
|
|
|
func ptrInt64(v int64) *int64 { return &v }
|
|
func ptrFloat(v float64) *float64 { return &v }
|
|
|
|
func TestParseNVIDIANVLinkStatusByGPU(t *testing.T) {
|
|
// Real-world 2-GPU direct-bridge H100 SXM output: link 15 inactive on both GPUs.
|
|
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
|
Link 0: 26.562 GB/s
|
|
Link 15: <inactive>
|
|
Link 17: 26.562 GB/s
|
|
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
|
|
Link 0: 26.562 GB/s
|
|
Link 15: <inactive>
|
|
`
|
|
got := parseNVIDIANVLinkStatusByGPU(input)
|
|
|
|
if len(got[0]) != 3 {
|
|
t.Fatalf("gpu0 ports=%d want 3 (%#v)", len(got[0]), got[0])
|
|
}
|
|
if got[0][1].Index != 15 || got[0][1].Active {
|
|
t.Fatalf("gpu0 link15=%#v want inactive", got[0][1])
|
|
}
|
|
if got[0][0].SpeedGBs == nil || *got[0][0].SpeedGBs != 26.562 {
|
|
t.Fatalf("gpu0 link0 speed=%#v want 26.562", got[0][0].SpeedGBs)
|
|
}
|
|
if len(got[1]) != 2 {
|
|
t.Fatalf("gpu1 ports=%d want 2 (%#v)", len(got[1]), got[1])
|
|
}
|
|
if got[1][1].Active {
|
|
t.Fatalf("gpu1 link15 should be inactive: %#v", got[1][1])
|
|
}
|
|
}
|
|
|
|
func TestParseNVIDIANVLinkErrorsByGPU(t *testing.T) {
|
|
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
|
Link 0: Replay Errors: 0
|
|
Link 0: Recovery Errors: 0
|
|
Link 0: CRC Errors: 0
|
|
Link 1: Replay Errors: 3
|
|
Link 1: Recovery Errors: 1
|
|
Link 1: CRC Errors: 2
|
|
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
|
|
Link 0: Replay Errors: 0
|
|
Link 0: Recovery Errors: 0
|
|
Link 0: CRC Errors: 0
|
|
`
|
|
got := parseNVIDIANVLinkErrorsByGPU(input)
|
|
|
|
c := got[0][1]
|
|
if c.Replay != 3 || c.Recovery != 1 || c.CRC != 2 {
|
|
t.Fatalf("gpu0 link1 counters=%#v want {3,1,2}", c)
|
|
}
|
|
zero := got[0][0]
|
|
if zero.Replay != 0 || zero.Recovery != 0 || zero.CRC != 0 {
|
|
t.Fatalf("gpu0 link0 counters=%#v want all zero", zero)
|
|
}
|
|
if _, ok := got[1][0]; !ok {
|
|
t.Fatalf("expected gpu1 link0 entry present")
|
|
}
|
|
}
|
|
|
|
func TestEnrichPCIeWithNVIDIANVLinksAttachesPortsByIndex(t *testing.T) {
|
|
oldStatus, oldErrors := nvlinkStatusFn, nvlinkErrorsFn
|
|
t.Cleanup(func() { nvlinkStatusFn, nvlinkErrorsFn = oldStatus, oldErrors })
|
|
|
|
nvlinkStatusFn = func() (map[int][]schema.HardwareNVLinkPort, error) {
|
|
return map[int][]schema.HardwareNVLinkPort{
|
|
0: {{Index: 0, Active: true, SpeedGBs: ptrFloat(26.562)}, {Index: 15, Active: false}},
|
|
}, nil
|
|
}
|
|
nvlinkErrorsFn = func() (map[int]map[int]nvlinkErrorCounters, error) {
|
|
return map[int]map[int]nvlinkErrorCounters{
|
|
0: {0: {Replay: 1}},
|
|
}, nil
|
|
}
|
|
|
|
devices := []schema.HardwarePCIeDevice{
|
|
{Telemetry: map[string]any{"nvidia_gpu_index": 0}},
|
|
}
|
|
|
|
out := enrichPCIeWithNVIDIANVLinks(devices)
|
|
|
|
if len(out[0].NVLinks) != 2 {
|
|
t.Fatalf("nvlinks=%d want 2", len(out[0].NVLinks))
|
|
}
|
|
if out[0].NVLinks[0].ReplayErrors == nil || *out[0].NVLinks[0].ReplayErrors != 1 {
|
|
t.Fatalf("link0 replay errors=%#v want 1", out[0].NVLinks[0].ReplayErrors)
|
|
}
|
|
if out[0].NVLinks[1].Active {
|
|
t.Fatalf("link15 should stay inactive")
|
|
}
|
|
}
|