Files
bee/audit/internal/collector/nvidia_test.go
T
Mikhail ChusavitinandClaude Sonnet 5 1175e6ccd8 gpu: collect reset-required/row-remap status and classify Xid codes by severity
nvidia-smi exposes reset_status.reset_required and remapped_rows.* only on
newer drivers for Ampere+ GPUs; queried via a separate exec call since an
unrecognized field name fails the whole --query-gpu command and would have
wiped out unrelated telemetry (temp/ECC/power) on older drivers otherwise.

Also refines Xid severity in both the ingest dmesg collector and the
always-on kmsg watcher: Xid 64 (row-remap InfoROM write failure) now
escalates to Critical instead of the generic warning every other Xid got,
and fixes the SAT-window flush path which previously hardcoded "Warning"
and ignored pattern severity entirely.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-07 19:28:59 +03:00

198 lines
5.9 KiB
Go

package collector
import (
"bee/audit/internal/schema"
"testing"
)
func TestParseNVIDIASMIQuery(t *testing.T) {
raw := "0, 00000000:65:00.0, NVIDIA H100 80GB HBM3, GPU-SERIAL-1, 96.00.1F.00.02, 54, 210.33, 0, 5, Not Active, 4, 4, 16, 16\n"
byBDF, err := parseNVIDIASMIQuery(raw)
if err != nil {
t.Fatalf("parse failed: %v", err)
}
gpu, ok := byBDF["0000:65:00.0"]
if !ok {
t.Fatalf("gpu by normalized bdf not found")
}
if gpu.Name != "NVIDIA H100 80GB HBM3" {
t.Fatalf("name: got %q", gpu.Name)
}
if gpu.Serial != "GPU-SERIAL-1" {
t.Fatalf("serial: got %q", gpu.Serial)
}
if gpu.VBIOS != "96.00.1F.00.02" {
t.Fatalf("vbios: got %q", gpu.VBIOS)
}
if gpu.ECCUncorrected == nil || *gpu.ECCUncorrected != 0 {
t.Fatalf("ecc uncorrected: got %v", gpu.ECCUncorrected)
}
if gpu.HWSlowdown == nil || *gpu.HWSlowdown {
t.Fatalf("hw slowdown: got %v, want false", gpu.HWSlowdown)
}
if gpu.PCIeLinkGenCurrent == nil || *gpu.PCIeLinkGenCurrent != 4 {
t.Fatalf("pcie link gen current: got %v, want 4", gpu.PCIeLinkGenCurrent)
}
if gpu.PCIeLinkGenMax == nil || *gpu.PCIeLinkGenMax != 4 {
t.Fatalf("pcie link gen max: got %v, want 4", gpu.PCIeLinkGenMax)
}
}
func TestNormalizePCIeBDF(t *testing.T) {
tests := []struct {
in string
want string
}{
{"00000000:17:00.0", "0000:17:00.0"},
{"0000:17:00.0", "0000:17:00.0"},
{"17:00.0", "0000:17:00.0"},
}
for _, tt := range tests {
got := normalizePCIeBDF(tt.in)
if got != tt.want {
t.Fatalf("normalizePCIeBDF(%q)=%q want %q", tt.in, got, tt.want)
}
}
}
func TestEnrichPCIeWithNVIDIAData_driverLoaded(t *testing.T) {
vendorID := NvidiaVendorID
bdf := "0000:65:00.0"
manufacturer := "NVIDIA Corporation"
status := "OK"
devices := []schema.HardwarePCIeDevice{
{
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status},
VendorID: &vendorID,
BDF: &bdf,
Manufacturer: &manufacturer,
},
}
byBDF := map[string]nvidiaGPUInfo{
"0000:65:00.0": {
BDF: "0000:65:00.0",
Serial: "GPU-ABC",
VBIOS: "96.00.1F.00.02",
ECCUncorrected: ptrInt64(2),
ECCCorrected: ptrInt64(10),
TemperatureC: ptrFloat(55.5),
PowerW: ptrFloat(230.2),
},
}
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
if out[0].SerialNumber == nil || *out[0].SerialNumber != "GPU-ABC" {
t.Fatalf("serial: got %v", out[0].SerialNumber)
}
if out[0].Firmware == nil || *out[0].Firmware != "96.00.1F.00.02" {
t.Fatalf("firmware: got %v", out[0].Firmware)
}
if out[0].Telemetry == nil || out[0].Telemetry["nvidia_gpu_index"] != 0 {
t.Fatalf("telemetry nvidia_gpu_index: got %#v", out[0].Telemetry)
}
if out[0].Status == nil || *out[0].Status != statusWarning {
t.Fatalf("status: got %v", out[0].Status)
}
if out[0].ECCUncorrectedTotal == nil || *out[0].ECCUncorrectedTotal != 2 {
t.Fatalf("ecc_uncorrected_total: got %#v", out[0].ECCUncorrectedTotal)
}
if out[0].TemperatureC == nil || *out[0].TemperatureC != 55.5 {
t.Fatalf("temperature_c: got %#v", out[0].TemperatureC)
}
}
func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) {
vendorID := NvidiaVendorID
bdf := "0000:17:00.0"
manufacturer := "NVIDIA Corporation"
devices := []schema.HardwarePCIeDevice{
{
VendorID: &vendorID,
BDF: &bdf,
Manufacturer: &manufacturer,
},
}
out := enrichPCIeWithNVIDIAData(devices, nil, false)
if out[0].SerialNumber != nil {
t.Fatalf("serial should stay nil without source data, got %v", out[0].SerialNumber)
}
if out[0].Status == nil || *out[0].Status != statusUnknown {
t.Fatalf("fallback status: got %v", out[0].Status)
}
}
func TestParseNVIDIAReliability(t *testing.T) {
raw := "0000:65:00.0, No, 0, 2, No, Yes\n"
byBDF, err := parseNVIDIAReliabilityCSV(raw)
if err != nil {
t.Fatalf("parse failed: %v", err)
}
gpu, ok := byBDF["0000:65:00.0"]
if !ok {
t.Fatalf("gpu by normalized bdf not found")
}
if gpu.ResetRequired == nil || *gpu.ResetRequired {
t.Fatalf("reset_required: got %v, want false", gpu.ResetRequired)
}
if gpu.RemapCorrectable == nil || *gpu.RemapCorrectable != 0 {
t.Fatalf("remap correctable: got %v", gpu.RemapCorrectable)
}
if gpu.RemapUncorrectable == nil || *gpu.RemapUncorrectable != 2 {
t.Fatalf("remap uncorrectable: got %v", gpu.RemapUncorrectable)
}
if gpu.RemapPending == nil || *gpu.RemapPending {
t.Fatalf("remap pending: got %v, want false", gpu.RemapPending)
}
if gpu.RemapFailure == nil || !*gpu.RemapFailure {
t.Fatalf("remap failure: got %v, want true", gpu.RemapFailure)
}
}
func TestEnrichPCIeWithNVIDIAData_resetRequiredCritical(t *testing.T) {
vendorID := NvidiaVendorID
bdf := "0000:65:00.0"
devices := []schema.HardwarePCIeDevice{
{VendorID: &vendorID, BDF: &bdf},
}
byBDF := map[string]nvidiaGPUInfo{
"0000:65:00.0": {ResetRequired: ptrBool(true)},
}
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
if out[0].Status == nil || *out[0].Status != statusCritical {
t.Fatalf("status: got %v, want %v", out[0].Status, statusCritical)
}
if out[0].ResetRequired == nil || !*out[0].ResetRequired {
t.Fatalf("reset_required: got %v", out[0].ResetRequired)
}
}
func TestEnrichPCIeWithNVIDIAData_remapFailureCritical(t *testing.T) {
vendorID := NvidiaVendorID
bdf := "0000:65:00.0"
devices := []schema.HardwarePCIeDevice{
{VendorID: &vendorID, BDF: &bdf},
}
byBDF := map[string]nvidiaGPUInfo{
"0000:65:00.0": {RemapFailure: ptrBool(true)},
}
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
if out[0].Status == nil || *out[0].Status != statusCritical {
t.Fatalf("status: got %v, want %v", out[0].Status, statusCritical)
}
if out[0].RemappedRowsFailure == nil || !*out[0].RemappedRowsFailure {
t.Fatalf("remapped_rows_failure: got %v", out[0].RemappedRowsFailure)
}
}
func ptrInt64(v int64) *int64 { return &v }
func ptrFloat(v float64) *float64 { return &v }
func ptrBool(v bool) *bool { return &v }