bee/audit/internal/collector/nvidia.go

package collector

import (
	"bee/audit/internal/schema"
	"encoding/csv"
	"fmt"
	"log/slog"
	"os/exec"
	"strconv"
	"strings"
)

type nvidiaGPUInfo struct {
	Index              int
	BDF                string
	Name               string
	Serial             string
	VBIOS              string
	TemperatureC       *float64
	PowerW             *float64
	ECCUncorrected     *int64
	ECCCorrected       *int64
	HWSlowdown         *bool
	PCIeLinkGenCurrent *int
	PCIeLinkGenMax     *int
	PCIeLinkWidthCur   *int
	PCIeLinkWidthMax   *int
	ResetRequired      *bool
	RemapCorrectable   *int64
	RemapUncorrectable *int64
	RemapPending       *bool
	RemapFailure       *bool
}

// enrichPCIeWithNVIDIA enriches NVIDIA PCIe devices with data from nvidia-smi.
// If the driver/tool is unavailable, NVIDIA devices get Unknown status.
func enrichPCIeWithNVIDIA(devs []schema.HardwarePCIeDevice) []schema.HardwarePCIeDevice {
	if !hasNVIDIADevices(devs) {
		return devs
	}
	gpuByBDF, err := queryNVIDIAGPUs()
	if err != nil {
		slog.Info("nvidia: enrichment skipped", "err", err)
		return enrichPCIeWithNVIDIAData(devs, nil, false)
	}
	return enrichPCIeWithNVIDIAData(devs, gpuByBDF, true)
}

func hasNVIDIADevices(devs []schema.HardwarePCIeDevice) bool {
	for _, dev := range devs {
		if isNVIDIADevice(dev) {
			return true
		}
	}
	return false
}

func enrichPCIeWithNVIDIAData(devs []schema.HardwarePCIeDevice, gpuByBDF map[string]nvidiaGPUInfo, driverLoaded bool) []schema.HardwarePCIeDevice {
	enriched := 0
	for i := range devs {
		if !isNVIDIADevice(devs[i]) {
			continue
		}

		if !driverLoaded {
			setPCIeFallback(&devs[i])
			continue
		}

		bdf := ""
		if devs[i].BDF != nil {
			bdf = normalizePCIeBDF(*devs[i].BDF)
		}
		info, ok := gpuByBDF[bdf]
		if !ok {
			setPCIeFallback(&devs[i])
			continue
		}

		if v := strings.TrimSpace(info.Name); v != "" {
			devs[i].Model = &v
		}
		if v := strings.TrimSpace(info.Serial); v != "" {
			devs[i].SerialNumber = &v
		}
		if v := strings.TrimSpace(info.VBIOS); v != "" {
			devs[i].Firmware = &v
		}

		status := statusOK
		desc := ""
		if info.ECCUncorrected != nil && *info.ECCUncorrected > 0 {
			status = statusWarning
			desc = "GPU reports uncorrected ECC errors"
		}
		if info.RemapUncorrectable != nil && *info.RemapUncorrectable > 0 {
			status = statusWarning
			desc = "GPU has uncorrectable row remap events (bad HBM cell repair scheduled)"
		}
		if info.RemapFailure != nil && *info.RemapFailure {
			status = statusCritical
			desc = "GPU row remap failed to commit to InfoROM (XID 64)"
		}
		if info.ResetRequired != nil && *info.ResetRequired {
			status = statusCritical
			desc = "GPU requires a reset"
		}
		mergeDeviceStatus(&devs[i], status, desc)
		injectNVIDIATelemetry(&devs[i], info)
		enriched++
	}

	if driverLoaded {
		slog.Info("nvidia: enriched", "count", enriched)
	}
	return devs
}

func queryNVIDIAGPUs() (map[string]nvidiaGPUInfo, error) {
	out, err := exec.Command(
		"nvidia-smi",
		"--query-gpu=index,pci.bus_id,name,serial,vbios_version,temperature.gpu,power.draw,ecc.errors.uncorrected.aggregate.total,ecc.errors.corrected.aggregate.total,clocks_throttle_reasons.hw_slowdown,pcie.link.gen.current,pcie.link.gen.max,pcie.link.width.current,pcie.link.width.max",
		"--format=csv,noheader,nounits",
	).Output()
	if err != nil {
		return nil, err
	}
	result, err := parseNVIDIASMIQuery(string(out))
	if err != nil {
		return nil, err
	}

	// reset_status.* and remapped_rows.* are only recognized by newer drivers
	// on Ampere+ GPUs; an unrecognized field name fails the whole nvidia-smi
	// call, so this is queried separately to avoid losing the fields above.
	if reliability, err := queryNVIDIAReliability(); err != nil {
		slog.Info("nvidia: reliability fields skipped", "err", err)
	} else {
		for bdf, r := range reliability {
			if info, ok := result[bdf]; ok {
				info.ResetRequired = r.ResetRequired
				info.RemapCorrectable = r.RemapCorrectable
				info.RemapUncorrectable = r.RemapUncorrectable
				info.RemapPending = r.RemapPending
				info.RemapFailure = r.RemapFailure
				result[bdf] = info
			}
		}
	}

	return result, nil
}

func queryNVIDIAReliability() (map[string]nvidiaGPUInfo, error) {
	out, err := exec.Command(
		"nvidia-smi",
		"--query-gpu=pci.bus_id,reset_status.reset_required,remapped_rows.correctable,remapped_rows.uncorrectable,remapped_rows.pending,remapped_rows.failure",
		"--format=csv,noheader,nounits",
	).Output()
	if err != nil {
		return nil, err
	}
	return parseNVIDIAReliabilityCSV(string(out))
}

func parseNVIDIAReliabilityCSV(raw string) (map[string]nvidiaGPUInfo, error) {
	r := csv.NewReader(strings.NewReader(raw))
	r.TrimLeadingSpace = true
	r.FieldsPerRecord = -1
	records, err := r.ReadAll()
	if err != nil {
		return nil, err
	}

	result := make(map[string]nvidiaGPUInfo)
	for _, rec := range records {
		if len(rec) == 0 {
			continue
		}
		if len(rec) < 6 {
			return nil, fmt.Errorf("unexpected nvidia-smi reliability columns: got %d, want 6", len(rec))
		}
		bdf := normalizePCIeBDF(rec[0])
		if bdf == "" {
			continue
		}
		result[bdf] = nvidiaGPUInfo{
			ResetRequired:      parseMaybeBool(rec[1]),
			RemapCorrectable:   parseMaybeInt64(rec[2]),
			RemapUncorrectable: parseMaybeInt64(rec[3]),
			RemapPending:       parseMaybeBool(rec[4]),
			RemapFailure:       parseMaybeBool(rec[5]),
		}
	}
	return result, nil
}

func parseNVIDIASMIQuery(raw string) (map[string]nvidiaGPUInfo, error) {
	r := csv.NewReader(strings.NewReader(raw))
	r.TrimLeadingSpace = true
	r.FieldsPerRecord = -1

	records, err := r.ReadAll()
	if err != nil {
		return nil, err
	}

	result := make(map[string]nvidiaGPUInfo)
	for _, rec := range records {
		if len(rec) == 0 {
			continue
		}
		if len(rec) < 14 {
			return nil, fmt.Errorf("unexpected nvidia-smi columns: got %d, want 14", len(rec))
		}

		bdf := normalizePCIeBDF(rec[1])
		if bdf == "" {
			continue
		}

		info := nvidiaGPUInfo{
			Index:              parseRequiredInt(rec[0]),
			BDF:                bdf,
			Name:               strings.TrimSpace(rec[2]),
			Serial:             strings.TrimSpace(rec[3]),
			VBIOS:              strings.TrimSpace(rec[4]),
			TemperatureC:       parseMaybeFloat(rec[5]),
			PowerW:             parseMaybeFloat(rec[6]),
			ECCUncorrected:     parseMaybeInt64(rec[7]),
			ECCCorrected:       parseMaybeInt64(rec[8]),
			HWSlowdown:         parseMaybeBool(rec[9]),
			PCIeLinkGenCurrent: parseMaybeInt(rec[10]),
			PCIeLinkGenMax:     parseMaybeInt(rec[11]),
			PCIeLinkWidthCur:   parseMaybeInt(rec[12]),
			PCIeLinkWidthMax:   parseMaybeInt(rec[13]),
		}
		result[bdf] = info
	}

	return result, nil
}

func parseMaybeFloat(v string) *float64 {
	v = strings.TrimSpace(v)
	if v == "" || strings.EqualFold(v, "n/a") || strings.EqualFold(v, "not supported") || strings.EqualFold(v, "[not supported]") {
		return nil
	}
	n, err := strconv.ParseFloat(v, 64)
	if err != nil {
		return nil
	}
	return &n
}

func parseMaybeInt64(v string) *int64 {
	v = strings.TrimSpace(v)
	if v == "" || strings.EqualFold(v, "n/a") || strings.EqualFold(v, "not supported") || strings.EqualFold(v, "[not supported]") {
		return nil
	}
	n, err := strconv.ParseInt(v, 10, 64)
	if err != nil {
		return nil
	}
	return &n
}

func parseMaybeInt(v string) *int {
	v = strings.TrimSpace(v)
	if v == "" || strings.EqualFold(v, "n/a") || strings.EqualFold(v, "not supported") || strings.EqualFold(v, "[not supported]") {
		return nil
	}
	n, err := strconv.Atoi(v)
	if err != nil {
		return nil
	}
	return &n
}

func parseRequiredInt(v string) int {
	n, err := strconv.Atoi(strings.TrimSpace(v))
	if err != nil {
		return 0
	}
	return n
}

func pcieLinkGenLabel(gen int) string {
	return fmt.Sprintf("Gen%d", gen)
}

func parseMaybeBool(v string) *bool {
	v = strings.TrimSpace(strings.ToLower(v))
	switch v {
	case "active", "enabled", "true", "1", "yes":
		b := true
		return &b
	case "not active", "disabled", "false", "0", "no":
		b := false
		return &b
	default:
		return nil
	}
}

func normalizePCIeBDF(bdf string) string {
	bdf = strings.TrimSpace(strings.ToLower(bdf))
	if bdf == "" {
		return ""
	}
	parts := strings.Split(bdf, ":")
	if len(parts) == 3 {
		domain := parts[0]
		if len(domain) > 4 {
			domain = domain[len(domain)-4:]
		}
		return domain + ":" + parts[1] + ":" + parts[2]
	}
	if len(parts) == 2 {
		return "0000:" + parts[0] + ":" + parts[1]
	}
	return bdf
}

func isNVIDIADevice(dev schema.HardwarePCIeDevice) bool {
	return dev.VendorID != nil && *dev.VendorID == NvidiaVendorID
}

func setPCIeFallback(dev *schema.HardwarePCIeDevice) {
	mergeDeviceStatus(dev, statusUnknown, "")
}

func injectNVIDIATelemetry(dev *schema.HardwarePCIeDevice, info nvidiaGPUInfo) {
	if dev.Telemetry == nil {
		dev.Telemetry = map[string]any{}
	}
	dev.Telemetry["nvidia_gpu_index"] = info.Index
	if info.TemperatureC != nil {
		dev.TemperatureC = info.TemperatureC
	}
	if info.PowerW != nil {
		dev.PowerW = info.PowerW
	}
	if info.ECCUncorrected != nil {
		dev.ECCUncorrectedTotal = info.ECCUncorrected
	}
	if info.ECCCorrected != nil {
		dev.ECCCorrectedTotal = info.ECCCorrected
	}
	if info.HWSlowdown != nil {
		dev.HWSlowdown = info.HWSlowdown
	}
	if info.ResetRequired != nil {
		dev.ResetRequired = info.ResetRequired
	}
	if info.RemapCorrectable != nil {
		dev.RemappedRowsCorrectable = info.RemapCorrectable
	}
	if info.RemapUncorrectable != nil {
		dev.RemappedRowsUncorrectable = info.RemapUncorrectable
	}
	if info.RemapPending != nil {
		dev.RemappedRowsPending = info.RemapPending
	}
	if info.RemapFailure != nil {
		dev.RemappedRowsFailure = info.RemapFailure
	}
	// Override PCIe link speed/width with nvidia-smi driver values.
	// sysfs current_link_speed reflects the instantaneous physical link state and
	// can show Gen1 when the GPU is idle due to ASPM power management. The driver
	// knows the negotiated speed regardless of the current power state.
	if info.PCIeLinkGenCurrent != nil {
		s := pcieLinkGenLabel(*info.PCIeLinkGenCurrent)
		dev.LinkSpeed = &s
	}
	if info.PCIeLinkGenMax != nil {
		s := pcieLinkGenLabel(*info.PCIeLinkGenMax)
		dev.MaxLinkSpeed = &s
	}
	if info.PCIeLinkWidthCur != nil {
		dev.LinkWidth = info.PCIeLinkWidthCur
	}
	if info.PCIeLinkWidthMax != nil {
		dev.MaxLinkWidth = info.PCIeLinkWidthMax
	}
}