fix(collector): stop pinning GPU PCIe status on an unverified idle reading
NVIDIA GPUs deliberately downclock PCIe to Gen1 at idle for power saving, and applyPCIeLinkSpeedWarning fired on every idle collector pass regardless - since component-status DB records never downgrade (Record() only ever raises severity), one boot-time idle sample permanently pinned pcie:gpu:nvidia to Warning for the rest of the session even after every load-bearing GPU SAT test passed clean. Two prior fixes (nvidia-smi-sourced link speed, pcie_aspm=off boot flag) didn't hold up against this hardware/driver combination - see bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md for the full history. Rather than add a downgrade path, stop writing an unverified status in the first place: parseLspciDevice no longer calls applyPCIeLinkSpeedWarning on the idle path. LinkSpeed/MaxLinkSpeed stay populated as plain descriptive fields; only a verified-under-load caller may now turn them into a status verdict. Two new SAT targets provide that verified signal: - pcie-link (platform/pcie_link_check.go): forces every enabled PCIe device - not just GPUs - to retrain via the PCIe spec's Link Control "Retrain Link" bit, then compares the negotiated speed against the device's max. Covers NICs/HBAs/switches that have no bee-gpu-burn equivalent load tool. Classifies by PCI class code + vendor ID, not name substrings. Routes gpu_nvidia/gpu_amd/other sub-verdicts into their own component-status keys so a degraded NIC never reads as a GPU fault. - nvidia-pcie-bandwidth (platform/nvidia_pcie_bandwidth.go): drives real host<->device traffic via dcgmi diag -r nvbandwidth and resamples link speed immediately after, independent of nvbandwidth's own pass/fail. Both wired into the task queue/webui the same way as nvidia-config (routes, dispatch, priority, Validate page cards, Run All Check SAT). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
7aa276320b
commit
b11018ac5e
@@ -0,0 +1,191 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// nvidiaPCIeBandwidthFinding is one GPU's post-load PCIe link-speed result.
|
||||
type nvidiaPCIeBandwidthFinding struct {
|
||||
Index int
|
||||
BDF string
|
||||
BeforeSpeed string
|
||||
AfterSpeed string
|
||||
MaxSpeed string
|
||||
Width int
|
||||
MaxWidth int
|
||||
Degraded bool
|
||||
}
|
||||
|
||||
// RunNvidiaPCIeBandwidthPack drives real host<->device traffic across the
|
||||
// GPUs' PCIe links (via `dcgmi diag -r nvbandwidth`, the same tool the
|
||||
// existing nvidia-bandwidth SAT uses for GPU-to-GPU throughput) and then
|
||||
// resamples each GPU's PCIe link speed from sysfs immediately afterward.
|
||||
//
|
||||
// This is deliberately a separate, narrower check from "nvidia-bandwidth":
|
||||
// that SAT's overall_status reflects nvbandwidth's own pass/fail (did the
|
||||
// measured GB/s clear its internal threshold), never PCIe link speed
|
||||
// itself. This pack exists purely to answer "did sustained real traffic
|
||||
// make the link train up to its negotiated maximum" — the load-bearing
|
||||
// counterpart to the idle-time collector reading that used to (falsely)
|
||||
// drive pcie:gpu:nvidia's status. See
|
||||
// bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning-unresolved.md.
|
||||
func (s *System) RunNvidiaPCIeBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
runDir := filepath.Join(baseDir, "nvidia-pcie-bandwidth-"+ts)
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||
|
||||
bdfByIndex, err := gpuIndexToBDF(gpuIndices)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve GPU BDFs: %w", err)
|
||||
}
|
||||
indices := make([]int, 0, len(bdfByIndex))
|
||||
for idx := range bdfByIndex {
|
||||
indices = append(indices, idx)
|
||||
}
|
||||
sort.Ints(indices)
|
||||
|
||||
findings := make([]nvidiaPCIeBandwidthFinding, 0, len(indices))
|
||||
for _, idx := range indices {
|
||||
bdf := bdfByIndex[idx]
|
||||
before, _ := readPCIeSysfsString(bdf, "current_link_speed")
|
||||
max, _ := readPCIeSysfsString(bdf, "max_link_speed")
|
||||
maxWidth, _ := readPCIeSysfsInt(bdf, "max_link_width")
|
||||
findings = append(findings, nvidiaPCIeBandwidthFinding{
|
||||
Index: idx, BDF: bdf, BeforeSpeed: before, MaxSpeed: max, MaxWidth: maxWidth,
|
||||
})
|
||||
}
|
||||
|
||||
cmd := []string{"dcgmi", "diag", "-r", "nvbandwidth"}
|
||||
if len(indices) > 0 {
|
||||
cmd = append(cmd, "-i", joinIndexList(indices))
|
||||
}
|
||||
out, runErr := runSATCommandCtx(ctx, verboseLog, "dcgmi-nvbandwidth", cmd, nil, logFunc)
|
||||
_ = os.WriteFile(filepath.Join(runDir, "01-dcgmi-nvbandwidth.log"), out, 0644)
|
||||
|
||||
for i := range findings {
|
||||
bdf := findings[i].BDF
|
||||
after, _ := readPCIeSysfsString(bdf, "current_link_speed")
|
||||
width, _ := readPCIeSysfsInt(bdf, "current_link_width")
|
||||
findings[i].AfterSpeed = after
|
||||
findings[i].Width = width
|
||||
findings[i].Degraded = after != findings[i].MaxSpeed
|
||||
}
|
||||
|
||||
summary := renderNvidiaPCIeBandwidthSummary(findings, runErr)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
report := renderNvidiaPCIeBandwidthReport(findings, runErr)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "nvidia-pcie-bandwidth-report.txt"), []byte(report), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
|
||||
// gpuIndexToBDF resolves each of gpuIndices to its PCI BDF via nvidia-smi.
|
||||
// An empty gpuIndices resolves every GPU nvidia-smi reports.
|
||||
func gpuIndexToBDF(gpuIndices []int) (map[int]string, error) {
|
||||
out, err := satExecCommand("nvidia-smi", "--query-gpu=index,pci.bus_id", "--format=csv,noheader,nounits").Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nvidia-smi: %w", err)
|
||||
}
|
||||
want := make(map[int]struct{}, len(gpuIndices))
|
||||
for _, idx := range gpuIndices {
|
||||
want[idx] = struct{}{}
|
||||
}
|
||||
filterByIndex := len(want) > 0
|
||||
|
||||
result := make(map[int]string)
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
fields := strings.SplitN(line, ",", 2)
|
||||
if len(fields) != 2 {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(strings.TrimSpace(fields[0]))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if filterByIndex {
|
||||
if _, ok := want[idx]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
result[idx] = normalizeNvidiaBDF(strings.TrimSpace(fields[1]))
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func renderNvidiaPCIeBandwidthSummary(findings []nvidiaPCIeBandwidthFinding, runErr error) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
||||
fmt.Fprintf(&b, "gpu_count=%d\n", len(findings))
|
||||
degraded := 0
|
||||
for _, f := range findings {
|
||||
fmt.Fprintf(&b, "gpu%d_status=%s\n", f.Index, statusLabel(!f.Degraded))
|
||||
if f.Degraded {
|
||||
degraded++
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "degraded=%d\n", degraded)
|
||||
if runErr != nil {
|
||||
fmt.Fprintf(&b, "dcgmi_nvbandwidth_error=%s\n", runErr.Error())
|
||||
}
|
||||
if degraded > 0 {
|
||||
fmt.Fprintln(&b, "overall_status=FAILED")
|
||||
var reasons []string
|
||||
for _, f := range findings {
|
||||
if f.Degraded {
|
||||
reasons = append(reasons, fmt.Sprintf("GPU%d (%s): still at %s under load, capable of %s",
|
||||
f.Index, f.BDF, f.AfterSpeed, f.MaxSpeed))
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "warnings=%s\n", strings.Join(reasons, "; "))
|
||||
} else {
|
||||
fmt.Fprintln(&b, "overall_status=OK")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func statusLabel(ok bool) string {
|
||||
if ok {
|
||||
return "OK"
|
||||
}
|
||||
return "FAILED"
|
||||
}
|
||||
|
||||
func renderNvidiaPCIeBandwidthReport(findings []nvidiaPCIeBandwidthFinding, runErr error) string {
|
||||
var b strings.Builder
|
||||
line := strings.Repeat("=", 80)
|
||||
b.WriteString(line + "\n")
|
||||
b.WriteString("NVIDIA GPU PCIe Bandwidth / Link-Under-Load Check\n")
|
||||
b.WriteString(line + "\n\n")
|
||||
if runErr != nil {
|
||||
fmt.Fprintf(&b, "dcgmi diag -r nvbandwidth: %s\n\n", runErr.Error())
|
||||
}
|
||||
for _, f := range findings {
|
||||
verdict := "OK"
|
||||
if f.Degraded {
|
||||
verdict = "DEGRADED"
|
||||
}
|
||||
fmt.Fprintf(&b, "GPU%d (%s): %s\n", f.Index, f.BDF, verdict)
|
||||
fmt.Fprintf(&b, " before=%s after=%s max=%s width=%d/%d\n",
|
||||
f.BeforeSpeed, f.AfterSpeed, f.MaxSpeed, f.Width, f.MaxWidth)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
Reference in New Issue
Block a user