reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(pcie): verify GPU links under real bandwidth load

Browse Source
This commit is contained in:
Mikhail Chusavitin
2026-09-03 10:23:39 +03:00
parent 642e68631d
commit 347bc8310a
27 changed files with 384 additions and 902 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/app/app.go
-2
View File
@@ -156,8 +156,6 @@ type satRunner interface {
RunTPMValidationPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
TPMPresent() bool
RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
RunPCIeLinkCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
RunNvidiaPCIeBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error)
RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
ListNvidiaGPUs() ([]platform.NvidiaGPU, error)
ListNvidiaGPUSettings() ([]platform.NvidiaGPUSetting, error)
audit/internal/app/app_packs.go
-14
View File
@@ -255,20 +255,6 @@ func (a *App) RunNvidiaConfigCheckPackCtx(ctx context.Context, baseDir string, l
return a.sat.RunNvidiaConfigCheckPack(ctx, baseDir, logFunc)
}
func (a *App) RunPCIeLinkCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
}
return a.sat.RunPCIeLinkCheckPack(ctx, baseDir, logFunc)
}
func (a *App) RunNvidiaPCIeBandwidthPackCtx(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
}
return a.sat.RunNvidiaPCIeBandwidthPack(ctx, baseDir, gpuIndices, logFunc)
}
func (a *App) RunAMDAcceptancePackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
audit/internal/app/app_test.go
-8
View File
@@ -295,14 +295,6 @@ func (f fakeSAT) RunNvidiaConfigCheckPack(_ context.Context, baseDir string, _ f
return "", nil
}
func (f fakeSAT) RunPCIeLinkCheckPack(_ context.Context, baseDir string, _ func(string)) (string, error) {
return "", nil
}
func (f fakeSAT) RunNvidiaPCIeBandwidthPack(_ context.Context, baseDir string, _ []int, _ func(string)) (string, error) {
return "", nil
}
func (f fakeSAT) RunCPUAcceptancePack(_ context.Context, baseDir string, durationSec int, _ func(string)) (string, error) {
if f.runCPUFn != nil {
return f.runCPUFn(baseDir, durationSec)
audit/internal/app/assets/README.md
+15 -18
View File
@@ -45,8 +45,7 @@ which bee component happened to produce it:
`UNSUPPORTED`).
- **"What's the *current* rolled-up health per component?"** →
`status/component-status.json`. One record per component key (e.g.
`pcie:gpu:nvidia`, `pcie:link:other`, `cpu:all`, `psu:all`,
`sat:pcie-link`), each with a `status` and a
`pcie:gpu:nvidia`, `cpu:all`, `psu:all`), each with a `status` and a
`history` array. This is what feeds `reanimator.json`'s `status` fields —
it's the source, not a duplicate. **The history is a transition log, not a
per-check journal** — a component whose status never changed only has one
@@ -82,16 +81,15 @@ which bee component happened to produce it:
for NVLink). If NVLink is expected, one `<inactive>` lane next to active
ones on an otherwise-bonded pair is the real fault signature — a degraded
link, not an absent one.
- **"PCIe link speeds / retrain check?"** → `tasks/bee-sat/pcie-link-*/`:
`summary.txt` (`overall_status`, `gpu_<vendor>_status`, `other_status`,
`devices_tested`, `other_devices_degraded`, `warnings`) and
`pcie-link-report.txt` (one line per BDF — `OK` / `DEGRADED` /
`FELL OFF BUS` / `skipped: <reason>`, with `before`/`after`/`max` speed
and width). Both endpoints and the bridges above them are listed; a
bridge's `max` is its own port capability capped to what the device
downstream of it supports, so `before == after == max` — even with a
`(port capability GenX, limited by downstream device)` note — means the
link runs at the fastest rate that pair supports, not degraded.
- **"NVIDIA PCIe link speed under real traffic?"** ->
`tasks/bee-sat/gpu-nvidia-bandwidth-*/summary.txt`: inspect
`pcie_link_under_load_status`, `pcie_gpuN_status`, and `pcie_degraded`.
`nvidia-pcie-link-under-load-report.txt` records each GPU's idle baseline,
post-`nvbandwidth` speed, maximum speed, and negotiated/max width. A Gen1
baseline is informational; only a link still below maximum immediately
after real traffic fails. For NICs/HBAs and other non-GPU PCIe devices,
use the read-only inventory and AER logs; there is no generic link-speed
health verdict without a device-specific traffic generator.
- **"TPM state / validation?"** → `tasks/bee-sat/tpm-*/summary.txt`
(`overall_status` plus per-step `N-<step>_status`); raw dumps in
`export/platform/tpm-*.txt`. Read-only — it never writes the TPM.
@@ -122,11 +120,10 @@ which bee component happened to produce it:
`storcli2-show-all.json`'s `"Number of Controllers"` and `lspci`/
`nvme-list.json` for actual storage hardware. Same pattern as
`nvlink_pairs_checked=0` above.
- `pcie-link-report.txt` bridge line `DEGRADED` with `before == after ==
max` and `(port capability GenX, limited by downstream device)`, while
the endpoint below it on the same link reads `OK` → link is at the
downstream device's native maximum, not a fault. A real bridge fault has
`after` below the (already capped) `max`, or `FELL OFF BUS`.
- `nvidia-pcie-link-under-load-report.txt` shows `before=Gen1` and
`after=max` at full width -> normal GPU idle power management, not a PCIe
downgrade. Treat `after < max` or `width < max_width` as a fault only when
the paired `nvbandwidth` command actually ran.
- `systemctl restart ... timed out` in a `tasks/_services/bee-*.log` →
implicates a SAT run only if its timestamp is inside that run's
`run_at_utc`/`started_at`–`done_at` window in
@@ -209,7 +206,7 @@ current-state answer.
| Path | What it is |
|---|---|
| `bee-sat/<target>-<timestamp>/` | One directory per acceptance-test run — `target` is what was tested (`gpu-nvidia`, `cpu`, `memory`, `storage`, `nccl-tests`, `gpu-nvidia-bandwidth`, `nvidia-config` covering GPU config/NVLink/Confidential-Computing readiness, `pcie-link` (whole-bus link-retrain check — see "Fastest answers"), `tpm` (read-only TPM validation), etc). Each contains `summary.txt` (key=value, always has `overall_status`), a full human-readable report (e.g. `storage`'s per-drive `disk-NN-<dev>-report.txt` — a narrative verdict, not raw tool output; the raw data behind it lives in `export/storage/`), numbered per-job logs, and `verbose.log` (every subprocess invocation + exit code — the place to look when a summary doesn't explain *why* something failed). |
| `bee-sat/<target>-<timestamp>/` | One directory per acceptance-test run — `target` is what was tested (`gpu-nvidia`, `cpu`, `memory`, `storage`, `nccl-tests`, `gpu-nvidia-bandwidth` including the NVIDIA PCIe link-under-load verdict, `nvidia-config` covering GPU config/NVLink/Confidential-Computing readiness, `tpm` (read-only TPM validation), etc). Each contains `summary.txt` (key=value, always has `overall_status`), a full human-readable report (e.g. `storage`'s per-drive `disk-NN-<dev>-report.txt` — a narrative verdict, not raw tool output; the raw data behind it lives in `export/storage/`), numbered per-job logs, and `verbose.log` (every subprocess invocation + exit code — the place to look when a summary doesn't explain *why* something failed). |
| `bee-bench/` | Benchmark run artifacts, if any have been run. |
| `<NNN>_<slug>_<done\|failed>/` | One directory per task launched from the web UI's task queue — see "Fastest answers" above. |
| `_state/bee-audit.json` | The full hardware-audit document as bee last generated it — same content as `export/reanimator.json`, kept here as bee's own internal record. |
audit/internal/app/component_status_db.go
+53 -23
View File
@@ -125,6 +125,55 @@ func lastEntryFromSource(history []ComponentStatusEntry, source string) *Compone
return nil
}
// recomputeIgnoringSources retires obsolete producers from the current
// component verdict without deleting their audit history. It selects the
// highest-severity latest observation from every remaining source.
func (db *ComponentStatusDB) recomputeIgnoringSources(key string, sources ...string) {
if db == nil {
return
}
ignored := make(map[string]struct{}, len(sources))
for _, source := range sources {
ignored[source] = struct{}{}
}
db.mu.Lock()
defer db.mu.Unlock()
db.reloadLocked()
rec := db.records[key]
if rec == nil {
return
}
latest := make(map[string]ComponentStatusEntry)
for i := len(rec.History) - 1; i >= 0; i-- {
entry := rec.History[i]
if _, skip := ignored[entry.Source]; skip {
continue
}
if _, seen := latest[entry.Source]; !seen {
latest[entry.Source] = entry
}
}
var winner *ComponentStatusEntry
for _, entry := range latest {
candidate := entry
if winner == nil || componentSeverity(candidate.Status) > componentSeverity(winner.Status) ||
(componentSeverity(candidate.Status) == componentSeverity(winner.Status) && candidate.At.After(winner.At)) {
winner = &candidate
}
}
if winner == nil {
return
}
if rec.Status != winner.Status || rec.ErrorSummary != winner.Detail {
rec.LastChangedAt = time.Now().UTC()
}
rec.Status = winner.Status
rec.ErrorSummary = winner.Detail
_ = db.saveLocked()
}
// Get returns the current record for a component key.
func (db *ComponentStatusDB) Get(key string) (ComponentStatusRecord, bool) {
if db == nil {
@@ -258,32 +307,13 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
// otherwise fails to match any real BDF.
switch target {
case "nvidia", "nvidia-targeted-stress", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "nvidia-config", "nvidia-pcie-bandwidth":
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "nvidia-config":
db.Record("pcie:gpu:nvidia", source, dbStatus, detail)
if target == "nvidia-bandwidth" {
db.recomputeIgnoringSources("pcie:gpu:nvidia", "sat:pcie-link", "sat:nvidia-pcie-bandwidth")
}
case "amd", "amd-stress", "amd-mem", "amd-bandwidth":
db.Record("pcie:gpu:amd", source, dbStatus, detail)
case "pcie-link":
// Forced-retrain PCIe link check (audit/internal/platform/pcie_link_check.go):
// the only verified (non-idle-sampled) source for PCIe link-speed
// status. summary.txt carries up to three independent sub-verdicts
// — record each into its own component key rather than collapsing
// them into one, since a degraded NIC/HBA shouldn't be reported as
// a GPU fault or vice versa.
recordPCIeLinkSubStatus := func(key, kvKey string) {
v, ok := kv[kvKey]
if !ok {
return
}
st := strings.ToUpper(strings.TrimSpace(v))
d := "pcie-link SAT: " + st
if st != "OK" && kv["warnings"] != "" {
d += " — " + kv["warnings"]
}
db.Record(key, source, satStatusToDBStatus(st), d)
}
recordPCIeLinkSubStatus("pcie:gpu:nvidia", "gpu_nvidia_status")
recordPCIeLinkSubStatus("pcie:gpu:amd", "gpu_amd_status")
recordPCIeLinkSubStatus("pcie:link:other", "other_status")
case "memory", "memory-stress", "sat-stress":
db.Record("memory:all", source, dbStatus, detail)
case "cpu", "platform-stress":
audit/internal/app/component_status_db_test.go
+40
View File
@@ -75,6 +75,46 @@ func TestApplySATResultToDBNormalizesGPUKeyByVendor(t *testing.T) {
}
}
func TestNvidiaBandwidthRetiresForcedRetrainWarning(t *testing.T) {
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
if err != nil {
t.Fatal(err)
}
db.Record("pcie:gpu:nvidia", "sat:pcie-link", "Warning", "idle forced retrain stayed at Gen1")
ApplySATResultToDB(db, "nvidia-bandwidth", writeSATSummary(t, "OK"))
rec, ok := db.Get("pcie:gpu:nvidia")
if !ok {
t.Fatal("expected pcie:gpu:nvidia record")
}
if rec.Status != "OK" {
t.Fatalf("status=%q want OK after real-traffic check supersedes forced retrain", rec.Status)
}
if lastEntryFromSource(rec.History, "sat:pcie-link") == nil {
t.Fatal("retired forced-retrain observation must remain in audit history")
}
}
func TestNvidiaBandwidthDoesNotClearIndependentGPUWarning(t *testing.T) {
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
if err != nil {
t.Fatal(err)
}
db.Record("pcie:gpu:nvidia", "sat:pcie-link", "Warning", "idle forced retrain stayed at Gen1")
db.Record("pcie:gpu:nvidia", "sat:nvidia-config", "Warning", "NVLink error counters are non-zero")
ApplySATResultToDB(db, "nvidia-bandwidth", writeSATSummary(t, "OK"))
rec, ok := db.Get("pcie:gpu:nvidia")
if !ok {
t.Fatal("expected pcie:gpu:nvidia record")
}
if rec.Status != "Warning" || !strings.Contains(rec.ErrorSummary, "NVLink") {
t.Fatalf("record=%+v want independent nvidia-config Warning to survive", rec)
}
}
func TestApplyComponentStatusDBMatchesGPUByVendor(t *testing.T) {
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
if err != nil {