package collector import ( "bee/audit/internal/schema" "encoding/csv" "fmt" "log/slog" "os/exec" "strconv" "strings" ) type nvidiaGPUInfo struct { Index int BDF string Name string Serial string VBIOS string TemperatureC *float64 PowerW *float64 ECCUncorrected *int64 ECCCorrected *int64 HWSlowdown *bool PCIeLinkGenCurrent *int PCIeLinkGenMax *int PCIeLinkWidthCur *int PCIeLinkWidthMax *int ResetRequired *bool RemapCorrectable *int64 RemapUncorrectable *int64 RemapPending *bool RemapFailure *bool } // enrichPCIeWithNVIDIA enriches NVIDIA PCIe devices with data from nvidia-smi. // If the driver/tool is unavailable, NVIDIA devices get Unknown status. func enrichPCIeWithNVIDIA(devs []schema.HardwarePCIeDevice) []schema.HardwarePCIeDevice { if !hasNVIDIADevices(devs) { return devs } gpuByBDF, err := queryNVIDIAGPUs() if err != nil { slog.Info("nvidia: enrichment skipped", "err", err) return enrichPCIeWithNVIDIAData(devs, nil, false) } return enrichPCIeWithNVIDIAData(devs, gpuByBDF, true) } func hasNVIDIADevices(devs []schema.HardwarePCIeDevice) bool { for _, dev := range devs { if isNVIDIADevice(dev) { return true } } return false } func enrichPCIeWithNVIDIAData(devs []schema.HardwarePCIeDevice, gpuByBDF map[string]nvidiaGPUInfo, driverLoaded bool) []schema.HardwarePCIeDevice { enriched := 0 for i := range devs { if !isNVIDIADevice(devs[i]) { continue } if !driverLoaded { setPCIeFallback(&devs[i]) continue } bdf := "" if devs[i].BDF != nil { bdf = normalizePCIeBDF(*devs[i].BDF) } info, ok := gpuByBDF[bdf] if !ok { setPCIeFallback(&devs[i]) continue } if v := strings.TrimSpace(info.Name); v != "" { devs[i].Model = &v } if v := strings.TrimSpace(info.Serial); v != "" { devs[i].SerialNumber = &v } if v := strings.TrimSpace(info.VBIOS); v != "" { devs[i].Firmware = &v } status := statusOK if info.ECCUncorrected != nil && *info.ECCUncorrected > 0 { status = statusWarning devs[i].ErrorDescription = stringPtr("GPU reports uncorrected ECC errors") } if info.RemapUncorrectable != nil && *info.RemapUncorrectable > 0 { status = statusWarning devs[i].ErrorDescription = stringPtr("GPU has uncorrectable row remap events (bad HBM cell repair scheduled)") } if info.RemapFailure != nil && *info.RemapFailure { status = statusCritical devs[i].ErrorDescription = stringPtr("GPU row remap failed to commit to InfoROM (XID 64)") } if info.ResetRequired != nil && *info.ResetRequired { status = statusCritical devs[i].ErrorDescription = stringPtr("GPU requires a reset") } devs[i].Status = &status injectNVIDIATelemetry(&devs[i], info) enriched++ } if driverLoaded { slog.Info("nvidia: enriched", "count", enriched) } return devs } func queryNVIDIAGPUs() (map[string]nvidiaGPUInfo, error) { out, err := exec.Command( "nvidia-smi", "--query-gpu=index,pci.bus_id,name,serial,vbios_version,temperature.gpu,power.draw,ecc.errors.uncorrected.aggregate.total,ecc.errors.corrected.aggregate.total,clocks_throttle_reasons.hw_slowdown,pcie.link.gen.current,pcie.link.gen.max,pcie.link.width.current,pcie.link.width.max", "--format=csv,noheader,nounits", ).Output() if err != nil { return nil, err } result, err := parseNVIDIASMIQuery(string(out)) if err != nil { return nil, err } // reset_status.* and remapped_rows.* are only recognized by newer drivers // on Ampere+ GPUs; an unrecognized field name fails the whole nvidia-smi // call, so this is queried separately to avoid losing the fields above. if reliability, err := queryNVIDIAReliability(); err != nil { slog.Info("nvidia: reliability fields skipped", "err", err) } else { for bdf, r := range reliability { if info, ok := result[bdf]; ok { info.ResetRequired = r.ResetRequired info.RemapCorrectable = r.RemapCorrectable info.RemapUncorrectable = r.RemapUncorrectable info.RemapPending = r.RemapPending info.RemapFailure = r.RemapFailure result[bdf] = info } } } return result, nil } func queryNVIDIAReliability() (map[string]nvidiaGPUInfo, error) { out, err := exec.Command( "nvidia-smi", "--query-gpu=pci.bus_id,reset_status.reset_required,remapped_rows.correctable,remapped_rows.uncorrectable,remapped_rows.pending,remapped_rows.failure", "--format=csv,noheader,nounits", ).Output() if err != nil { return nil, err } return parseNVIDIAReliabilityCSV(string(out)) } func parseNVIDIAReliabilityCSV(raw string) (map[string]nvidiaGPUInfo, error) { r := csv.NewReader(strings.NewReader(raw)) r.TrimLeadingSpace = true r.FieldsPerRecord = -1 records, err := r.ReadAll() if err != nil { return nil, err } result := make(map[string]nvidiaGPUInfo) for _, rec := range records { if len(rec) == 0 { continue } if len(rec) < 6 { return nil, fmt.Errorf("unexpected nvidia-smi reliability columns: got %d, want 6", len(rec)) } bdf := normalizePCIeBDF(rec[0]) if bdf == "" { continue } result[bdf] = nvidiaGPUInfo{ ResetRequired: parseMaybeBool(rec[1]), RemapCorrectable: parseMaybeInt64(rec[2]), RemapUncorrectable: parseMaybeInt64(rec[3]), RemapPending: parseMaybeBool(rec[4]), RemapFailure: parseMaybeBool(rec[5]), } } return result, nil } func parseNVIDIASMIQuery(raw string) (map[string]nvidiaGPUInfo, error) { r := csv.NewReader(strings.NewReader(raw)) r.TrimLeadingSpace = true r.FieldsPerRecord = -1 records, err := r.ReadAll() if err != nil { return nil, err } result := make(map[string]nvidiaGPUInfo) for _, rec := range records { if len(rec) == 0 { continue } if len(rec) < 14 { return nil, fmt.Errorf("unexpected nvidia-smi columns: got %d, want 14", len(rec)) } bdf := normalizePCIeBDF(rec[1]) if bdf == "" { continue } info := nvidiaGPUInfo{ Index: parseRequiredInt(rec[0]), BDF: bdf, Name: strings.TrimSpace(rec[2]), Serial: strings.TrimSpace(rec[3]), VBIOS: strings.TrimSpace(rec[4]), TemperatureC: parseMaybeFloat(rec[5]), PowerW: parseMaybeFloat(rec[6]), ECCUncorrected: parseMaybeInt64(rec[7]), ECCCorrected: parseMaybeInt64(rec[8]), HWSlowdown: parseMaybeBool(rec[9]), PCIeLinkGenCurrent: parseMaybeInt(rec[10]), PCIeLinkGenMax: parseMaybeInt(rec[11]), PCIeLinkWidthCur: parseMaybeInt(rec[12]), PCIeLinkWidthMax: parseMaybeInt(rec[13]), } result[bdf] = info } return result, nil } func parseMaybeFloat(v string) *float64 { v = strings.TrimSpace(v) if v == "" || strings.EqualFold(v, "n/a") || strings.EqualFold(v, "not supported") || strings.EqualFold(v, "[not supported]") { return nil } n, err := strconv.ParseFloat(v, 64) if err != nil { return nil } return &n } func parseMaybeInt64(v string) *int64 { v = strings.TrimSpace(v) if v == "" || strings.EqualFold(v, "n/a") || strings.EqualFold(v, "not supported") || strings.EqualFold(v, "[not supported]") { return nil } n, err := strconv.ParseInt(v, 10, 64) if err != nil { return nil } return &n } func parseMaybeInt(v string) *int { v = strings.TrimSpace(v) if v == "" || strings.EqualFold(v, "n/a") || strings.EqualFold(v, "not supported") || strings.EqualFold(v, "[not supported]") { return nil } n, err := strconv.Atoi(v) if err != nil { return nil } return &n } func parseRequiredInt(v string) int { n, err := strconv.Atoi(strings.TrimSpace(v)) if err != nil { return 0 } return n } func pcieLinkGenLabel(gen int) string { return fmt.Sprintf("Gen%d", gen) } func parseMaybeBool(v string) *bool { v = strings.TrimSpace(strings.ToLower(v)) switch v { case "active", "enabled", "true", "1", "yes": b := true return &b case "not active", "disabled", "false", "0", "no": b := false return &b default: return nil } } func normalizePCIeBDF(bdf string) string { bdf = strings.TrimSpace(strings.ToLower(bdf)) if bdf == "" { return "" } parts := strings.Split(bdf, ":") if len(parts) == 3 { domain := parts[0] if len(domain) > 4 { domain = domain[len(domain)-4:] } return domain + ":" + parts[1] + ":" + parts[2] } if len(parts) == 2 { return "0000:" + parts[0] + ":" + parts[1] } return bdf } func isNVIDIADevice(dev schema.HardwarePCIeDevice) bool { return dev.VendorID != nil && *dev.VendorID == NvidiaVendorID } func setPCIeFallback(dev *schema.HardwarePCIeDevice) { status := statusUnknown dev.Status = &status } func injectNVIDIATelemetry(dev *schema.HardwarePCIeDevice, info nvidiaGPUInfo) { if dev.Telemetry == nil { dev.Telemetry = map[string]any{} } dev.Telemetry["nvidia_gpu_index"] = info.Index if info.TemperatureC != nil { dev.TemperatureC = info.TemperatureC } if info.PowerW != nil { dev.PowerW = info.PowerW } if info.ECCUncorrected != nil { dev.ECCUncorrectedTotal = info.ECCUncorrected } if info.ECCCorrected != nil { dev.ECCCorrectedTotal = info.ECCCorrected } if info.HWSlowdown != nil { dev.HWSlowdown = info.HWSlowdown } if info.ResetRequired != nil { dev.ResetRequired = info.ResetRequired } if info.RemapCorrectable != nil { dev.RemappedRowsCorrectable = info.RemapCorrectable } if info.RemapUncorrectable != nil { dev.RemappedRowsUncorrectable = info.RemapUncorrectable } if info.RemapPending != nil { dev.RemappedRowsPending = info.RemapPending } if info.RemapFailure != nil { dev.RemappedRowsFailure = info.RemapFailure } // Override PCIe link speed/width with nvidia-smi driver values. // sysfs current_link_speed reflects the instantaneous physical link state and // can show Gen1 when the GPU is idle due to ASPM power management. The driver // knows the negotiated speed regardless of the current power state. if info.PCIeLinkGenCurrent != nil { s := pcieLinkGenLabel(*info.PCIeLinkGenCurrent) dev.LinkSpeed = &s } if info.PCIeLinkGenMax != nil { s := pcieLinkGenLabel(*info.PCIeLinkGenMax) dev.MaxLinkSpeed = &s } if info.PCIeLinkWidthCur != nil { dev.LinkWidth = info.PCIeLinkWidthCur } if info.PCIeLinkWidthMax != nil { dev.MaxLinkWidth = info.PCIeLinkWidthMax } }