fix(collector): stop pinning GPU PCIe status on an unverified idle reading
NVIDIA GPUs deliberately downclock PCIe to Gen1 at idle for power saving, and applyPCIeLinkSpeedWarning fired on every idle collector pass regardless - since component-status DB records never downgrade (Record() only ever raises severity), one boot-time idle sample permanently pinned pcie:gpu:nvidia to Warning for the rest of the session even after every load-bearing GPU SAT test passed clean. Two prior fixes (nvidia-smi-sourced link speed, pcie_aspm=off boot flag) didn't hold up against this hardware/driver combination - see bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md for the full history. Rather than add a downgrade path, stop writing an unverified status in the first place: parseLspciDevice no longer calls applyPCIeLinkSpeedWarning on the idle path. LinkSpeed/MaxLinkSpeed stay populated as plain descriptive fields; only a verified-under-load caller may now turn them into a status verdict. Two new SAT targets provide that verified signal: - pcie-link (platform/pcie_link_check.go): forces every enabled PCIe device - not just GPUs - to retrain via the PCIe spec's Link Control "Retrain Link" bit, then compares the negotiated speed against the device's max. Covers NICs/HBAs/switches that have no bee-gpu-burn equivalent load tool. Classifies by PCI class code + vendor ID, not name substrings. Routes gpu_nvidia/gpu_amd/other sub-verdicts into their own component-status keys so a degraded NIC never reads as a GPU fault. - nvidia-pcie-bandwidth (platform/nvidia_pcie_bandwidth.go): drives real host<->device traffic via dcgmi diag -r nvbandwidth and resamples link speed immediately after, independent of nvbandwidth's own pass/fail. Both wired into the task queue/webui the same way as nvidia-config (routes, dispatch, priority, Validate page cards, Run All Check SAT). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
7aa276320b
commit
b11018ac5e
@@ -0,0 +1,191 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// nvidiaPCIeBandwidthFinding is one GPU's post-load PCIe link-speed result.
|
||||
type nvidiaPCIeBandwidthFinding struct {
|
||||
Index int
|
||||
BDF string
|
||||
BeforeSpeed string
|
||||
AfterSpeed string
|
||||
MaxSpeed string
|
||||
Width int
|
||||
MaxWidth int
|
||||
Degraded bool
|
||||
}
|
||||
|
||||
// RunNvidiaPCIeBandwidthPack drives real host<->device traffic across the
|
||||
// GPUs' PCIe links (via `dcgmi diag -r nvbandwidth`, the same tool the
|
||||
// existing nvidia-bandwidth SAT uses for GPU-to-GPU throughput) and then
|
||||
// resamples each GPU's PCIe link speed from sysfs immediately afterward.
|
||||
//
|
||||
// This is deliberately a separate, narrower check from "nvidia-bandwidth":
|
||||
// that SAT's overall_status reflects nvbandwidth's own pass/fail (did the
|
||||
// measured GB/s clear its internal threshold), never PCIe link speed
|
||||
// itself. This pack exists purely to answer "did sustained real traffic
|
||||
// make the link train up to its negotiated maximum" — the load-bearing
|
||||
// counterpart to the idle-time collector reading that used to (falsely)
|
||||
// drive pcie:gpu:nvidia's status. See
|
||||
// bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning-unresolved.md.
|
||||
func (s *System) RunNvidiaPCIeBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
runDir := filepath.Join(baseDir, "nvidia-pcie-bandwidth-"+ts)
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||
|
||||
bdfByIndex, err := gpuIndexToBDF(gpuIndices)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve GPU BDFs: %w", err)
|
||||
}
|
||||
indices := make([]int, 0, len(bdfByIndex))
|
||||
for idx := range bdfByIndex {
|
||||
indices = append(indices, idx)
|
||||
}
|
||||
sort.Ints(indices)
|
||||
|
||||
findings := make([]nvidiaPCIeBandwidthFinding, 0, len(indices))
|
||||
for _, idx := range indices {
|
||||
bdf := bdfByIndex[idx]
|
||||
before, _ := readPCIeSysfsString(bdf, "current_link_speed")
|
||||
max, _ := readPCIeSysfsString(bdf, "max_link_speed")
|
||||
maxWidth, _ := readPCIeSysfsInt(bdf, "max_link_width")
|
||||
findings = append(findings, nvidiaPCIeBandwidthFinding{
|
||||
Index: idx, BDF: bdf, BeforeSpeed: before, MaxSpeed: max, MaxWidth: maxWidth,
|
||||
})
|
||||
}
|
||||
|
||||
cmd := []string{"dcgmi", "diag", "-r", "nvbandwidth"}
|
||||
if len(indices) > 0 {
|
||||
cmd = append(cmd, "-i", joinIndexList(indices))
|
||||
}
|
||||
out, runErr := runSATCommandCtx(ctx, verboseLog, "dcgmi-nvbandwidth", cmd, nil, logFunc)
|
||||
_ = os.WriteFile(filepath.Join(runDir, "01-dcgmi-nvbandwidth.log"), out, 0644)
|
||||
|
||||
for i := range findings {
|
||||
bdf := findings[i].BDF
|
||||
after, _ := readPCIeSysfsString(bdf, "current_link_speed")
|
||||
width, _ := readPCIeSysfsInt(bdf, "current_link_width")
|
||||
findings[i].AfterSpeed = after
|
||||
findings[i].Width = width
|
||||
findings[i].Degraded = after != findings[i].MaxSpeed
|
||||
}
|
||||
|
||||
summary := renderNvidiaPCIeBandwidthSummary(findings, runErr)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
report := renderNvidiaPCIeBandwidthReport(findings, runErr)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "nvidia-pcie-bandwidth-report.txt"), []byte(report), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
|
||||
// gpuIndexToBDF resolves each of gpuIndices to its PCI BDF via nvidia-smi.
|
||||
// An empty gpuIndices resolves every GPU nvidia-smi reports.
|
||||
func gpuIndexToBDF(gpuIndices []int) (map[int]string, error) {
|
||||
out, err := satExecCommand("nvidia-smi", "--query-gpu=index,pci.bus_id", "--format=csv,noheader,nounits").Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nvidia-smi: %w", err)
|
||||
}
|
||||
want := make(map[int]struct{}, len(gpuIndices))
|
||||
for _, idx := range gpuIndices {
|
||||
want[idx] = struct{}{}
|
||||
}
|
||||
filterByIndex := len(want) > 0
|
||||
|
||||
result := make(map[int]string)
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
fields := strings.SplitN(line, ",", 2)
|
||||
if len(fields) != 2 {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(strings.TrimSpace(fields[0]))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if filterByIndex {
|
||||
if _, ok := want[idx]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
result[idx] = normalizeNvidiaBDF(strings.TrimSpace(fields[1]))
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func renderNvidiaPCIeBandwidthSummary(findings []nvidiaPCIeBandwidthFinding, runErr error) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
||||
fmt.Fprintf(&b, "gpu_count=%d\n", len(findings))
|
||||
degraded := 0
|
||||
for _, f := range findings {
|
||||
fmt.Fprintf(&b, "gpu%d_status=%s\n", f.Index, statusLabel(!f.Degraded))
|
||||
if f.Degraded {
|
||||
degraded++
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "degraded=%d\n", degraded)
|
||||
if runErr != nil {
|
||||
fmt.Fprintf(&b, "dcgmi_nvbandwidth_error=%s\n", runErr.Error())
|
||||
}
|
||||
if degraded > 0 {
|
||||
fmt.Fprintln(&b, "overall_status=FAILED")
|
||||
var reasons []string
|
||||
for _, f := range findings {
|
||||
if f.Degraded {
|
||||
reasons = append(reasons, fmt.Sprintf("GPU%d (%s): still at %s under load, capable of %s",
|
||||
f.Index, f.BDF, f.AfterSpeed, f.MaxSpeed))
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "warnings=%s\n", strings.Join(reasons, "; "))
|
||||
} else {
|
||||
fmt.Fprintln(&b, "overall_status=OK")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func statusLabel(ok bool) string {
|
||||
if ok {
|
||||
return "OK"
|
||||
}
|
||||
return "FAILED"
|
||||
}
|
||||
|
||||
func renderNvidiaPCIeBandwidthReport(findings []nvidiaPCIeBandwidthFinding, runErr error) string {
|
||||
var b strings.Builder
|
||||
line := strings.Repeat("=", 80)
|
||||
b.WriteString(line + "\n")
|
||||
b.WriteString("NVIDIA GPU PCIe Bandwidth / Link-Under-Load Check\n")
|
||||
b.WriteString(line + "\n\n")
|
||||
if runErr != nil {
|
||||
fmt.Fprintf(&b, "dcgmi diag -r nvbandwidth: %s\n\n", runErr.Error())
|
||||
}
|
||||
for _, f := range findings {
|
||||
verdict := "OK"
|
||||
if f.Degraded {
|
||||
verdict = "DEGRADED"
|
||||
}
|
||||
fmt.Fprintf(&b, "GPU%d (%s): %s\n", f.Index, f.BDF, verdict)
|
||||
fmt.Fprintf(&b, " before=%s after=%s max=%s width=%d/%d\n",
|
||||
f.BeforeSpeed, f.AfterSpeed, f.MaxSpeed, f.Width, f.MaxWidth)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bee/audit/internal/collector"
|
||||
)
|
||||
|
||||
// pcieLinkRetrainTimeout bounds how long we wait for a device to finish
|
||||
// retraining (clear the Link Status "Link Training" bit) before giving up
|
||||
// and reading whatever speed it settled on anyway. The PCIe spec allows up
|
||||
// to 100ms for Gen1-3 and longer for higher generations with equalization;
|
||||
// this is generous headroom above that.
|
||||
const pcieLinkRetrainTimeout = 2 * time.Second
|
||||
|
||||
// pcieLinkFinding is one device's before/after link-speed retrain result.
|
||||
type pcieLinkFinding struct {
|
||||
BDF string
|
||||
Description string
|
||||
VendorID string
|
||||
ClassCode string
|
||||
IsGPU bool
|
||||
GPUVendor string // "nvidia" or "amd", only set when IsGPU
|
||||
Skipped string // non-empty reason this device wasn't retrained
|
||||
BeforeSpeed string
|
||||
AfterSpeed string
|
||||
MaxSpeed string
|
||||
Width int
|
||||
MaxWidth int
|
||||
Degraded bool
|
||||
}
|
||||
|
||||
// RunPCIeLinkCheckPack forces every enabled PCIe device to retrain its link
|
||||
// (via the PCIe Link Control register's spec-defined Retrain Link bit — see
|
||||
// PCIe base spec, Link Control Register, bit 5) and compares the
|
||||
// post-retrain negotiated speed against the device's own reported maximum.
|
||||
//
|
||||
// This exists because a plain idle-time sysfs read of current_link_speed is
|
||||
// not a reliable fault signal: NVIDIA GPUs (and other devices with runtime
|
||||
// power management) deliberately downclock their PCIe link to save power
|
||||
// while idle, which is indistinguishable from a real degraded slot/riser/
|
||||
// cable without either sustained traffic or a forced retrain. Forcing a
|
||||
// retrain sidesteps needing a device-specific load generator (bee-gpu-burn
|
||||
// exists for GPUs; nothing plays that role for NICs, HBAs, or PCIe
|
||||
// switches) — retraining is a PCIe-spec mechanism every endpoint supports,
|
||||
// so this one check covers every PCIe device in the machine, not just
|
||||
// GPUs. See bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning-unresolved.md
|
||||
// for the history of narrower attempts that didn't generalize.
|
||||
//
|
||||
// Disabled devices (sysfs enable==0 — e.g. PCIe fabric-management endpoints
|
||||
// the kernel never activates, per the 2026-06-12 decision) are left alone:
|
||||
// they carry no data traffic, so there is nothing to verify and no reason
|
||||
// to poke them.
|
||||
func (s *System) RunPCIeLinkCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
runDir := filepath.Join(baseDir, "pcie-link-"+ts)
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||
|
||||
bdfs, err := listPCIDeviceBDFs()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("list PCI devices: %w", err)
|
||||
}
|
||||
|
||||
var findings []pcieLinkFinding
|
||||
for _, bdf := range bdfs {
|
||||
if logFunc != nil {
|
||||
logFunc(fmt.Sprintf("=== %s ===", bdf))
|
||||
}
|
||||
f := retrainAndSamplePCIeDevice(ctx, verboseLog, bdf, logFunc)
|
||||
findings = append(findings, f)
|
||||
}
|
||||
|
||||
summary := renderPCIeLinkCheckSummary(findings)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
report := renderPCIeLinkCheckReport(findings)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "pcie-link-report.txt"), []byte(report), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
|
||||
// listPCIDeviceBDFs returns every BDF under /sys/bus/pci/devices, sorted for
|
||||
// deterministic report ordering.
|
||||
func listPCIDeviceBDFs() ([]string, error) {
|
||||
entries, err := os.ReadDir("/sys/bus/pci/devices")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bdfs := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
bdfs = append(bdfs, e.Name())
|
||||
}
|
||||
sort.Strings(bdfs)
|
||||
return bdfs, nil
|
||||
}
|
||||
|
||||
func retrainAndSamplePCIeDevice(ctx context.Context, verboseLog, bdf string, logFunc func(string)) pcieLinkFinding {
|
||||
f := pcieLinkFinding{BDF: bdf}
|
||||
|
||||
vendor, _ := readPCIeSysfsHex(bdf, "vendor")
|
||||
class, _ := readPCIeSysfsHex(bdf, "class")
|
||||
f.VendorID = vendor
|
||||
f.ClassCode = class
|
||||
f.IsGPU, f.GPUVendor = classifyGPUFromVendorClass(vendor, class)
|
||||
f.Description = pcieDeviceDescription(ctx, verboseLog, bdf, logFunc)
|
||||
|
||||
if enabled, ok := readPCIeSysfsInt(bdf, "enable"); ok && enabled == 0 {
|
||||
f.Skipped = "device disabled (no data traffic; link state has no operational impact)"
|
||||
return f
|
||||
}
|
||||
|
||||
before, beforeOK := readPCIeSysfsString(bdf, "current_link_speed")
|
||||
maxSpeed, maxOK := readPCIeSysfsString(bdf, "max_link_speed")
|
||||
width, _ := readPCIeSysfsInt(bdf, "current_link_width")
|
||||
maxWidth, _ := readPCIeSysfsInt(bdf, "max_link_width")
|
||||
f.BeforeSpeed = before
|
||||
f.MaxSpeed = maxSpeed
|
||||
f.MaxWidth = maxWidth
|
||||
if !beforeOK || !maxOK {
|
||||
f.Skipped = "no PCIe link-speed attributes in sysfs (not a link-trained endpoint)"
|
||||
return f
|
||||
}
|
||||
|
||||
if err := retrainPCIeLink(ctx, verboseLog, bdf, logFunc); err != nil {
|
||||
f.Skipped = "retrain failed: " + err.Error()
|
||||
f.AfterSpeed = before
|
||||
f.Width = width
|
||||
// before/maxSpeed are already normalized "GenN" labels (see
|
||||
// readPCIeSysfsString) — compare directly, don't re-normalize.
|
||||
f.Degraded = before != maxSpeed
|
||||
return f
|
||||
}
|
||||
|
||||
after, _ := readPCIeSysfsString(bdf, "current_link_speed")
|
||||
widthAfter, _ := readPCIeSysfsInt(bdf, "current_link_width")
|
||||
f.AfterSpeed = after
|
||||
f.Width = widthAfter
|
||||
f.Degraded = after != maxSpeed
|
||||
return f
|
||||
}
|
||||
|
||||
// retrainPCIeLink sets the Retrain Link bit (bit 5) of the PCI Express
|
||||
// Capability's Link Control register via setpci, then polls the Link
|
||||
// Status register's Link Training bit (bit 11) until it clears or
|
||||
// pcieLinkRetrainTimeout elapses.
|
||||
func retrainPCIeLink(ctx context.Context, verboseLog, bdf string, logFunc func(string)) error {
|
||||
linkCtrlOut, err := runSATCommandCtx(ctx, verboseLog, "setpci-read-"+bdf,
|
||||
[]string{"setpci", "-s", bdf, "CAP_EXP+0x10.w"}, nil, logFunc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read Link Control: %w", err)
|
||||
}
|
||||
cur, err := strconv.ParseUint(strings.TrimSpace(string(linkCtrlOut)), 16, 16)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse Link Control %q: %w", linkCtrlOut, err)
|
||||
}
|
||||
const retrainLinkBit = 0x0020
|
||||
newVal := uint16(cur) | retrainLinkBit
|
||||
|
||||
if _, err := runSATCommandCtx(ctx, verboseLog, "setpci-retrain-"+bdf,
|
||||
[]string{"setpci", "-s", bdf, fmt.Sprintf("CAP_EXP+0x10.w=%04x", newVal)}, nil, logFunc); err != nil {
|
||||
return fmt.Errorf("write Retrain Link bit: %w", err)
|
||||
}
|
||||
|
||||
const linkTrainingBit = 0x0800
|
||||
deadline := time.Now().Add(pcieLinkRetrainTimeout)
|
||||
for time.Now().Before(deadline) {
|
||||
statusOut, err := runSATCommandCtx(ctx, verboseLog, "setpci-status-"+bdf,
|
||||
[]string{"setpci", "-s", bdf, "CAP_EXP+0x12.w"}, nil, nil)
|
||||
if err == nil {
|
||||
if status, perr := strconv.ParseUint(strings.TrimSpace(string(statusOut)), 16, 16); perr == nil {
|
||||
if status&linkTrainingBit == 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
// Timed out waiting for training to clear; the caller still samples
|
||||
// whatever speed sysfs reports, which is the honest answer either way.
|
||||
return nil
|
||||
}
|
||||
|
||||
func pcieDeviceDescription(ctx context.Context, verboseLog, bdf string, logFunc func(string)) string {
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, "lspci-"+bdf, []string{"lspci", "-s", bdf}, nil, logFunc)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
line := strings.TrimSpace(string(out))
|
||||
if idx := strings.Index(line, "\n"); idx >= 0 {
|
||||
line = line[:idx]
|
||||
}
|
||||
if idx := strings.Index(line, " "); idx >= 0 {
|
||||
return strings.TrimSpace(line[idx+1:])
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func readPCIeSysfsString(bdf, attr string) (string, bool) {
|
||||
raw, err := os.ReadFile(filepath.Join("/sys/bus/pci/devices", bdf, attr))
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
v := strings.TrimSpace(string(raw))
|
||||
if v == "" {
|
||||
return "", false
|
||||
}
|
||||
return collector.NormalizePCILinkSpeed(v), true
|
||||
}
|
||||
|
||||
func readPCIeSysfsInt(bdf, attr string) (int, bool) {
|
||||
raw, err := os.ReadFile(filepath.Join("/sys/bus/pci/devices", bdf, attr))
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
v, err := strconv.Atoi(strings.TrimSpace(string(raw)))
|
||||
if err != nil || v < 0 {
|
||||
return 0, false
|
||||
}
|
||||
return v, true
|
||||
}
|
||||
|
||||
func readPCIeSysfsHex(bdf, attr string) (string, bool) {
|
||||
raw, err := os.ReadFile(filepath.Join("/sys/bus/pci/devices", bdf, attr))
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimSpace(string(raw)), true
|
||||
}
|
||||
|
||||
// classifyGPUFromVendorClass reports whether a device is a GPU die itself
|
||||
// (PCI base class 0x03 — Display Controller — under NVIDIA/AMD's vendor
|
||||
// ID), as opposed to a same-vendor companion device (NIC, storage
|
||||
// controller, NVLink bridge) that shares the GPU's PCI vendor ID. Class-code
|
||||
// based, not name-substring based, per the same reasoning as
|
||||
// collector.IsGPUClass.
|
||||
func classifyGPUFromVendorClass(vendorHex, classHex string) (isGPU bool, vendor string) {
|
||||
v := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(vendorHex)), "0x")
|
||||
c := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(classHex)), "0x")
|
||||
if len(c) < 2 || c[:2] != "03" {
|
||||
return false, ""
|
||||
}
|
||||
switch v {
|
||||
case "10de":
|
||||
return true, "nvidia"
|
||||
case "1002":
|
||||
return true, "amd"
|
||||
default:
|
||||
return false, ""
|
||||
}
|
||||
}
|
||||
|
||||
func renderPCIeLinkCheckSummary(findings []pcieLinkFinding) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
||||
fmt.Fprintf(&b, "devices_tested=%d\n", len(findings))
|
||||
|
||||
gpuStatus := map[string]string{} // vendor -> OK/FAILED
|
||||
otherDegraded := 0
|
||||
otherTested := 0
|
||||
anyDegraded := false
|
||||
for _, f := range findings {
|
||||
if f.Skipped != "" && f.AfterSpeed == "" {
|
||||
continue
|
||||
}
|
||||
if f.IsGPU {
|
||||
if _, ok := gpuStatus[f.GPUVendor]; !ok {
|
||||
gpuStatus[f.GPUVendor] = "OK"
|
||||
}
|
||||
if f.Degraded {
|
||||
gpuStatus[f.GPUVendor] = "FAILED"
|
||||
anyDegraded = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
otherTested++
|
||||
if f.Degraded {
|
||||
otherDegraded++
|
||||
anyDegraded = true
|
||||
}
|
||||
}
|
||||
for _, vendor := range []string{"nvidia", "amd"} {
|
||||
if status, ok := gpuStatus[vendor]; ok {
|
||||
fmt.Fprintf(&b, "gpu_%s_status=%s\n", vendor, status)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "other_devices_tested=%d\n", otherTested)
|
||||
fmt.Fprintf(&b, "other_devices_degraded=%d\n", otherDegraded)
|
||||
if otherTested > 0 {
|
||||
if otherDegraded > 0 {
|
||||
fmt.Fprintln(&b, "other_status=FAILED")
|
||||
} else {
|
||||
fmt.Fprintln(&b, "other_status=OK")
|
||||
}
|
||||
}
|
||||
|
||||
if anyDegraded {
|
||||
fmt.Fprintln(&b, "overall_status=FAILED")
|
||||
var reasons []string
|
||||
for _, f := range findings {
|
||||
if f.Degraded {
|
||||
reasons = append(reasons, fmt.Sprintf("%s (%s): retrained to %s, capable of %s",
|
||||
f.BDF, nonEmptyOr(f.Description, "unknown device"), f.AfterSpeed, f.MaxSpeed))
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "warnings=%s\n", strings.Join(reasons, "; "))
|
||||
} else {
|
||||
fmt.Fprintln(&b, "overall_status=OK")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func renderPCIeLinkCheckReport(findings []pcieLinkFinding) string {
|
||||
var b strings.Builder
|
||||
line := strings.Repeat("=", 80)
|
||||
b.WriteString(line + "\n")
|
||||
b.WriteString("PCIe Link Retrain Check\n")
|
||||
b.WriteString(line + "\n\n")
|
||||
for _, f := range findings {
|
||||
fmt.Fprintf(&b, "%s %s\n", f.BDF, nonEmptyOr(f.Description, "(unknown device)"))
|
||||
if f.Skipped != "" && f.AfterSpeed == "" {
|
||||
fmt.Fprintf(&b, " skipped: %s\n", f.Skipped)
|
||||
continue
|
||||
}
|
||||
verdict := "OK"
|
||||
if f.Degraded {
|
||||
verdict = "DEGRADED"
|
||||
}
|
||||
fmt.Fprintf(&b, " %s: before=%s after=%s max=%s width=%d/%d\n",
|
||||
verdict, f.BeforeSpeed, f.AfterSpeed, f.MaxSpeed, f.Width, f.MaxWidth)
|
||||
if f.Skipped != "" {
|
||||
fmt.Fprintf(&b, " note: %s\n", f.Skipped)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestClassifyGPUFromVendorClass(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
vendor string
|
||||
class string
|
||||
wantIsGPU bool
|
||||
wantVendor string
|
||||
}{
|
||||
{"nvidia GPU", "0x10de", "0x030200", true, "nvidia"},
|
||||
{"amd GPU", "0x1002", "0x030000", true, "amd"},
|
||||
{"nvidia NIC (same vendor, not display class)", "0x10de", "0x020000", false, ""},
|
||||
{"intel NIC", "0x8086", "0x020000", false, ""},
|
||||
{"unrelated display-class vendor", "0x1234", "0x030000", false, ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
gotIsGPU, gotVendor := classifyGPUFromVendorClass(tc.vendor, tc.class)
|
||||
if gotIsGPU != tc.wantIsGPU || gotVendor != tc.wantVendor {
|
||||
t.Fatalf("classifyGPUFromVendorClass(%q,%q) = (%v,%q), want (%v,%q)",
|
||||
tc.vendor, tc.class, gotIsGPU, gotVendor, tc.wantIsGPU, tc.wantVendor)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderPCIeLinkCheckSummaryDegradedGPU(t *testing.T) {
|
||||
findings := []pcieLinkFinding{
|
||||
{BDF: "0000:0d:00.0", Description: "NVIDIA GPU", IsGPU: true, GPUVendor: "nvidia",
|
||||
BeforeSpeed: "Gen1", AfterSpeed: "Gen1", MaxSpeed: "Gen5", Degraded: true},
|
||||
{BDF: "0000:37:00.0", Description: "NVIDIA GPU", IsGPU: true, GPUVendor: "nvidia",
|
||||
BeforeSpeed: "Gen1", AfterSpeed: "Gen5", MaxSpeed: "Gen5", Degraded: false},
|
||||
{BDF: "0000:01:00.0", Description: "Mellanox NIC", IsGPU: false,
|
||||
BeforeSpeed: "Gen3", AfterSpeed: "Gen4", MaxSpeed: "Gen4", Degraded: false},
|
||||
}
|
||||
summary := renderPCIeLinkCheckSummary(findings)
|
||||
if !strings.Contains(summary, "gpu_nvidia_status=FAILED") {
|
||||
t.Fatalf("expected gpu_nvidia_status=FAILED (one degraded GPU should fail the vendor group), got:\n%s", summary)
|
||||
}
|
||||
if !strings.Contains(summary, "other_status=OK") {
|
||||
t.Fatalf("expected other_status=OK, got:\n%s", summary)
|
||||
}
|
||||
if !strings.Contains(summary, "overall_status=FAILED") {
|
||||
t.Fatalf("expected overall_status=FAILED, got:\n%s", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderPCIeLinkCheckSummaryAllClean(t *testing.T) {
|
||||
findings := []pcieLinkFinding{
|
||||
{BDF: "0000:0d:00.0", IsGPU: true, GPUVendor: "nvidia", BeforeSpeed: "Gen1", AfterSpeed: "Gen5", MaxSpeed: "Gen5"},
|
||||
{BDF: "0000:2c:00.0", Skipped: "device disabled (no data traffic; link state has no operational impact)"},
|
||||
}
|
||||
summary := renderPCIeLinkCheckSummary(findings)
|
||||
if !strings.Contains(summary, "overall_status=OK") {
|
||||
t.Fatalf("expected overall_status=OK, got:\n%s", summary)
|
||||
}
|
||||
if strings.Contains(summary, "other_status=") {
|
||||
t.Fatalf("disabled/skipped-only device should not produce an other_status line, got:\n%s", summary)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user