reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(pcie): verify GPU links under real bandwidth load

Browse Source
This commit is contained in:
Mikhail Chusavitin
2026-09-03 10:23:39 +03:00
parent 642e68631d
commit 347bc8310a
27 changed files with 384 additions and 902 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/platform/nvidia_pcie_bandwidth.go
+98 -77
View File
@@ -1,14 +1,14 @@
package platform
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"bee/audit/internal/collector"
)
// nvidiaPCIeBandwidthFinding is one GPU's post-load PCIe link-speed result.
@@ -21,38 +21,18 @@ type nvidiaPCIeBandwidthFinding struct {
Width int
MaxWidth int
Degraded bool
Supported bool
Sampled bool
}
// RunNvidiaPCIeBandwidthPack drives real host<->device traffic across the
// GPUs' PCIe links (via `dcgmi diag -r nvbandwidth`, the same tool the
// existing nvidia-bandwidth SAT uses for GPU-to-GPU throughput) and then
// resamples each GPU's PCIe link speed from sysfs immediately afterward.
//
// This is deliberately a separate, narrower check from "nvidia-bandwidth":
// that SAT's overall_status reflects nvbandwidth's own pass/fail (did the
// measured GB/s clear its internal threshold), never PCIe link speed
// itself. This pack exists purely to answer "did sustained real traffic
// make the link train up to its negotiated maximum" — the load-bearing
// counterpart to the idle-time collector reading that used to (falsely)
// drive pcie:gpu:nvidia's status. See
// bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning-unresolved.md.
func (s *System) RunNvidiaPCIeBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
if ctx == nil {
ctx = context.Background()
}
if baseDir == "" {
baseDir = "/var/log/bee-sat"
}
ts := time.Now().UTC().Format("20060102-150405")
runDir := filepath.Join(baseDir, "nvidia-pcie-bandwidth-"+ts)
if err := os.MkdirAll(runDir, 0755); err != nil {
return "", err
}
verboseLog := filepath.Join(runDir, "verbose.log")
// captureNvidiaPCIeLinkBaseline records link capabilities immediately before
// the existing nvbandwidth SAT. The post-load half is completed by
// finishNvidiaPCIeLinkCheck as soon as that same SAT returns, so one real PCIe
// traffic pass provides both the DCGM verdict and the negotiated-link verdict.
func captureNvidiaPCIeLinkBaseline(gpuIndices []int) ([]nvidiaPCIeBandwidthFinding, error) {
bdfByIndex, err := gpuIndexToBDF(gpuIndices)
if err != nil {
return "", fmt.Errorf("resolve GPU BDFs: %w", err)
return nil, fmt.Errorf("resolve GPU BDFs: %w", err)
}
indices := make([]int, 0, len(bdfByIndex))
for idx := range bdfByIndex {
@@ -63,39 +43,37 @@ func (s *System) RunNvidiaPCIeBandwidthPack(ctx context.Context, baseDir string,
findings := make([]nvidiaPCIeBandwidthFinding, 0, len(indices))
for _, idx := range indices {
bdf := bdfByIndex[idx]
before, _ := readPCIeSysfsString(bdf, "current_link_speed")
max, _ := readPCIeSysfsString(bdf, "max_link_speed")
maxWidth, _ := readPCIeSysfsInt(bdf, "max_link_width")
before, beforeOK := readPCIeSysfsString(bdf, "current_link_speed")
max, maxOK := readPCIeSysfsString(bdf, "max_link_speed")
maxWidth, widthOK := readPCIeSysfsInt(bdf, "max_link_width")
findings = append(findings, nvidiaPCIeBandwidthFinding{
Index: idx, BDF: bdf, BeforeSpeed: before, MaxSpeed: max, MaxWidth: maxWidth,
Supported: beforeOK && maxOK && widthOK,
})
}
return findings, nil
}
cmd := []string{"dcgmi", "diag", "-r", "nvbandwidth"}
if len(indices) > 0 {
cmd = append(cmd, "-i", joinIndexList(indices))
func finishNvidiaPCIeLinkCheck(runDir string, findings []nvidiaPCIeBandwidthFinding) error {
report := renderNvidiaPCIeBandwidthReport(findings)
if err := os.WriteFile(filepath.Join(runDir, "nvidia-pcie-link-under-load-report.txt"), []byte(report), 0644); err != nil {
return err
}
out, runErr := runSATCommandCtx(ctx, verboseLog, "dcgmi-nvbandwidth", cmd, nil, logFunc)
_ = os.WriteFile(filepath.Join(runDir, "01-dcgmi-nvbandwidth.log"), out, 0644)
return appendNvidiaPCIeLinkSummary(filepath.Join(runDir, "summary.txt"), findings)
}
func sampleNvidiaPCIeLinkAfterLoad(findings []nvidiaPCIeBandwidthFinding) {
for i := range findings {
bdf := findings[i].BDF
after, _ := readPCIeSysfsString(bdf, "current_link_speed")
width, _ := readPCIeSysfsInt(bdf, "current_link_width")
after, afterOK := readPCIeSysfsString(bdf, "current_link_speed")
width, widthOK := readPCIeSysfsInt(bdf, "current_link_width")
findings[i].AfterSpeed = after
findings[i].Width = width
findings[i].Degraded = after != findings[i].MaxSpeed
findings[i].Sampled = true
findings[i].Supported = findings[i].Supported && afterOK && widthOK
findings[i].Degraded = findings[i].Supported &&
(after != findings[i].MaxSpeed || width != findings[i].MaxWidth)
}
summary := renderNvidiaPCIeBandwidthSummary(findings, runErr)
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
return "", err
}
report := renderNvidiaPCIeBandwidthReport(findings, runErr)
if err := os.WriteFile(filepath.Join(runDir, "nvidia-pcie-bandwidth-report.txt"), []byte(report), 0644); err != nil {
return "", err
}
return runDir, nil
}
// gpuIndexToBDF resolves each of gpuIndices to its PCI BDF via nvidia-smi.
@@ -131,35 +109,79 @@ func gpuIndexToBDF(gpuIndices []int) (map[int]string, error) {
return result, nil
}
func renderNvidiaPCIeBandwidthSummary(findings []nvidiaPCIeBandwidthFinding, runErr error) string {
var b strings.Builder
fmt.Fprintf(&b, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
fmt.Fprintf(&b, "gpu_count=%d\n", len(findings))
func readPCIeSysfsString(bdf, attr string) (string, bool) {
raw, err := satReadFile(filepath.Join("/sys/bus/pci/devices", bdf, attr))
if err != nil {
return "", false
}
v := strings.TrimSpace(string(raw))
if v == "" {
return "", false
}
return collector.NormalizePCILinkSpeed(v), true
}
func readPCIeSysfsInt(bdf, attr string) (int, bool) {
raw, err := satReadFile(filepath.Join("/sys/bus/pci/devices", bdf, attr))
if err != nil {
return 0, false
}
v, err := strconv.Atoi(strings.TrimSpace(string(raw)))
if err != nil || v < 0 {
return 0, false
}
return v, true
}
func appendNvidiaPCIeLinkSummary(path string, findings []nvidiaPCIeBandwidthFinding) error {
raw, err := os.ReadFile(path)
if err != nil {
return err
}
var extra strings.Builder
fmt.Fprintf(&extra, "pcie_gpu_count=%d\n", len(findings))
degraded := 0
supported := 0
var reasons []string
for _, f := range findings {
fmt.Fprintf(&b, "gpu%d_status=%s\n", f.Index, statusLabel(!f.Degraded))
status := "UNSUPPORTED"
if f.Supported && f.Sampled {
supported++
status = statusLabel(!f.Degraded)
}
fmt.Fprintf(&extra, "pcie_gpu%d_status=%s\n", f.Index, status)
if f.Degraded {
degraded++
reasons = append(reasons, fmt.Sprintf("GPU%d (%s): under load at %s x%d, capable of %s x%d",
f.Index, f.BDF, f.AfterSpeed, f.Width, f.MaxSpeed, f.MaxWidth))
}
}
fmt.Fprintf(&b, "degraded=%d\n", degraded)
if runErr != nil {
fmt.Fprintf(&b, "dcgmi_nvbandwidth_error=%s\n", runErr.Error())
}
fmt.Fprintf(&extra, "pcie_degraded=%d\n", degraded)
linkStatus := "UNSUPPORTED"
if degraded > 0 {
fmt.Fprintln(&b, "overall_status=FAILED")
var reasons []string
for _, f := range findings {
if f.Degraded {
reasons = append(reasons, fmt.Sprintf("GPU%d (%s): still at %s under load, capable of %s",
f.Index, f.BDF, f.AfterSpeed, f.MaxSpeed))
linkStatus = "FAILED"
} else if supported == len(findings) && supported > 0 {
linkStatus = "OK"
}
fmt.Fprintf(&extra, "pcie_link_under_load_status=%s\n", linkStatus)
if len(reasons) > 0 {
fmt.Fprintf(&extra, "pcie_link_under_load_detail=%s\n", strings.Join(reasons, "; "))
}
if linkStatus == "FAILED" {
lines := strings.Split(string(raw), "\n")
for i := range lines {
if strings.HasPrefix(lines[i], "overall_status=") {
lines[i] = "overall_status=FAILED"
break
}
}
fmt.Fprintf(&b, "warnings=%s\n", strings.Join(reasons, "; "))
} else {
fmt.Fprintln(&b, "overall_status=OK")
raw = []byte(strings.Join(lines, "\n"))
}
return b.String()
if len(raw) > 0 && raw[len(raw)-1] != '\n' {
raw = append(raw, '\n')
}
raw = append(raw, extra.String()...)
return os.WriteFile(path, raw, 0644)
}
func statusLabel(ok bool) string {
@@ -169,18 +191,17 @@ func statusLabel(ok bool) string {
return "FAILED"
}
func renderNvidiaPCIeBandwidthReport(findings []nvidiaPCIeBandwidthFinding, runErr error) string {
func renderNvidiaPCIeBandwidthReport(findings []nvidiaPCIeBandwidthFinding) string {
var b strings.Builder
line := strings.Repeat("=", 80)
b.WriteString(line + "\n")
b.WriteString("NVIDIA GPU PCIe Bandwidth / Link-Under-Load Check\n")
b.WriteString("NVIDIA GPU PCIe Link-Under-Load Check\n")
b.WriteString(line + "\n\n")
if runErr != nil {
fmt.Fprintf(&b, "dcgmi diag -r nvbandwidth: %s\n\n", runErr.Error())
}
for _, f := range findings {
verdict := "OK"
if f.Degraded {
verdict := "UNSUPPORTED"
if f.Supported && f.Sampled && !f.Degraded {
verdict = "OK"
} else if f.Degraded {
verdict = "DEGRADED"
}
fmt.Fprintf(&b, "GPU%d (%s): %s\n", f.Index, f.BDF, verdict)