fix(app): capture PCIe link / AER diagnostics at boot, not just on demand

pcie-nvidia-link.txt, pcie-nvidia-link-under-load.txt, and
kernel-aer-nvidia.txt only ever lived in supportBundleCommands, which
exclusively runs inside the on-demand "Download Support Bundle" web UI
action. The blackbox USB auto-sync worker never calls that function -
it only mirrors whatever CaptureTechnicalDump already wrote into the
live export tree at boot. So these three files were structurally
unreachable from a blackbox pull no matter how fresh the build was;
earlier analysis of a real blackbox misattributed their absence to
build/version drift instead.

Move the underlying scripts into shared exported constants
(platform.PCIeNvidiaLinkScript, PCIeNvidiaLinkUnderLoadScript,
KernelAERNvidiaScript) and add them to techDumpNvidiaCommands, so
CaptureTechnicalDump captures all three once at boot (bee-audit is
oneshot, so the ~8s bee-gpu-burn cost for the under-load sample is a
one-time boot cost, not a per-sync-cycle one) alongside the existing
nvidia-smi-* dumps. supportBundleCommands still re-runs the same
scripts on demand for a fresher sample - that's intentional, not a
duplicate to clean up.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-08-24 18:39:28 +03:00
co-authored by Claude Sonnet 5
parent b11018ac5e
commit ba250330d5
4 changed files with 81 additions and 49 deletions
+2 -1
View File
@@ -27,7 +27,8 @@ func techdumpBucketFor(name string) string {
return "storage" return "storage"
case name == "nvidia-smi-q.txt", name == "nvidia-smi-query.csv", name == "nvidia-smi-conf-compute-q.txt", case name == "nvidia-smi-q.txt", name == "nvidia-smi-query.csv", name == "nvidia-smi-conf-compute-q.txt",
name == "nvidia-smi-topo.txt", name == "nvidia-smi-nvlink-status.txt", name == "nvidia-smi-nvlink-errors.txt", name == "nvidia-smi-topo.txt", name == "nvidia-smi-nvlink-status.txt", name == "nvidia-smi-nvlink-errors.txt",
name == "rocm-smi.txt", name == "rocm-smi-showallinfo.txt": name == "rocm-smi.txt", name == "rocm-smi-showallinfo.txt",
name == "pcie-nvidia-link.txt", name == "pcie-nvidia-link-under-load.txt", name == "kernel-aer-nvidia.txt":
return "gpu" return "gpu"
default: default:
// dmidecode-type0/1/2, ipmitool-*, sensors.json, lspci-vmm/vvv, and // dmidecode-type0/1/2, ipmitool-*, sensors.json, lspci-vmm/vvv, and
+16
View File
@@ -0,0 +1,16 @@
package app
import "testing"
func TestTechdumpBucketForPCIeLinkFiles(t *testing.T) {
cases := map[string]string{
"pcie-nvidia-link.txt": "gpu",
"pcie-nvidia-link-under-load.txt": "gpu",
"kernel-aer-nvidia.txt": "gpu",
}
for name, want := range cases {
if got := techdumpBucketFor(name); got != want {
t.Errorf("techdumpBucketFor(%q) = %q, want %q", name, got, want)
}
}
}
+7 -48
View File
@@ -98,13 +98,7 @@ else
echo "dmesg not found" echo "dmesg not found"
fi fi
`}}, `}},
{name: "export/gpu/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", ` {name: "export/gpu/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", platform.KernelAERNvidiaScript}},
if command -v dmesg >/dev/null 2>&1; then
dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
else
echo "dmesg not found"
fi
`}},
{name: "livecd/gui/loginctl-sessions.txt", cmd: []string{"sh", "-c", ` {name: "livecd/gui/loginctl-sessions.txt", cmd: []string{"sh", "-c", `
if command -v loginctl >/dev/null 2>&1; then if command -v loginctl >/dev/null 2>&1; then
loginctl list-sessions 2>&1 || true loginctl list-sessions 2>&1 || true
@@ -310,22 +304,11 @@ if [ "$found" -eq 0 ]; then
echo "no NVIDIA PCI devices found" echo "no NVIDIA PCI devices found"
fi fi
`}}, `}},
{name: "export/gpu/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", ` // Also captured once at boot by platform.CaptureTechnicalDump (which the
for d in /sys/bus/pci/devices/*/; do // blackbox mirror picks up automatically); re-run here so an on-demand
vendor=$(cat "$d/vendor" 2>/dev/null) // support bundle gets a fresh sample instead of a possibly-stale boot
[ "$vendor" = "0x10de" ] || continue // one. See bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md.
class=$(cat "$d/class" 2>/dev/null) {name: "export/gpu/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", platform.PCIeNvidiaLinkScript}},
case "$class" in
0x030000|0x030200) ;;
*) continue ;;
esac
dev=$(basename "$d")
echo "=== $dev ==="
for f in current_link_speed current_link_width max_link_speed max_link_width; do
printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
done
done
`}},
// A Gen1-vs-Gen4 link speed reading at idle is ambiguous: NVIDIA drivers // A Gen1-vs-Gen4 link speed reading at idle is ambiguous: NVIDIA drivers
// deliberately downclock PCIe in low power states and re-train to full // deliberately downclock PCIe in low power states and re-train to full
// speed under load, so pcie-nvidia-link.txt alone can't tell a real // speed under load, so pcie-nvidia-link.txt alone can't tell a real
@@ -333,31 +316,7 @@ done
// sysfs attributes while bee-gpu-burn is actively loading the GPUs — if // sysfs attributes while bee-gpu-burn is actively loading the GPUs — if
// the link comes up here, the idle Gen1 reading above was power saving, // the link comes up here, the idle Gen1 reading above was power saving,
// not a fault. // not a fault.
{name: "export/gpu/pcie-nvidia-link-under-load.txt", cmd: []string{"sh", "-c", ` {name: "export/gpu/pcie-nvidia-link-under-load.txt", cmd: []string{"sh", "-c", platform.PCIeNvidiaLinkUnderLoadScript}},
if ! command -v bee-gpu-burn >/dev/null 2>&1; then
echo "bee-gpu-burn not found; cannot sample PCIe link speed under load"
exit 0
fi
bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 &
burn_pid=$!
sleep 3
for d in /sys/bus/pci/devices/*/; do
vendor=$(cat "$d/vendor" 2>/dev/null)
[ "$vendor" = "0x10de" ] || continue
class=$(cat "$d/class" 2>/dev/null)
case "$class" in
0x030000|0x030200) ;;
*) continue ;;
esac
dev=$(basename "$d")
echo "=== $dev ==="
for f in current_link_speed current_link_width max_link_speed max_link_width; do
printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
done
done
wait "$burn_pid" 2>/dev/null || true
rm -f /tmp/bee-pcie-load-burn.log
`}},
{name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", ` {name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
found=0 found=0
for dev in /sys/bus/pci/devices/*; do for dev in /sys/bus/pci/devices/*; do
+56
View File
@@ -39,6 +39,59 @@ var techDumpFixedCommands = []struct {
{Name: "storcli2", Args: []string{"show", "all", "J"}, File: "storcli2-show-all.json"}, {Name: "storcli2", Args: []string{"show", "all", "J"}, File: "storcli2-show-all.json"},
} }
// PCIeNvidiaLinkScript samples each NVIDIA GPU's current/max PCIe link
// speed and width from sysfs. Exported so both the boot-time techdump
// (below) and the on-demand support bundle (app/support_bundle.go, which
// re-samples for freshness at bundle-build time) run the identical script
// instead of two copies that can drift.
const PCIeNvidiaLinkScript = `
for d in /sys/bus/pci/devices/*/; do
vendor=$(cat "$d/vendor" 2>/dev/null)
[ "$vendor" = "0x10de" ] || continue
class=$(cat "$d/class" 2>/dev/null)
case "$class" in
0x030000|0x030200) ;;
*) continue ;;
esac
dev=$(basename "$d")
echo "=== $dev ==="
for f in current_link_speed current_link_width max_link_speed max_link_width; do
printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
done
done
`
// PCIeNvidiaLinkUnderLoadScript re-samples PCIeNvidiaLinkScript's sysfs
// attributes while bee-gpu-burn briefly loads the GPUs. An idle-only
// reading can't tell a real degraded slot/riser from NVIDIA's normal
// idle power-management downclock; this is the load-bearing counterpart
// that lets a reader (human or agent) tell the two apart by comparing
// pcie-nvidia-link.txt against pcie-nvidia-link-under-load.txt. See
// bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md.
const PCIeNvidiaLinkUnderLoadScript = `
if ! command -v bee-gpu-burn >/dev/null 2>&1; then
echo "bee-gpu-burn not found; cannot sample PCIe link speed under load"
exit 0
fi
bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 &
burn_pid=$!
sleep 3
` + PCIeNvidiaLinkScript + `
wait "$burn_pid" 2>/dev/null || true
rm -f /tmp/bee-pcie-load-burn.log
`
// KernelAERNvidiaScript filters dmesg for PCIe AER, NVRM, and Xid lines —
// cheap (dmesg is already in memory) and the fastest way to tell a real
// PCIe/GPU hardware fault from power-management noise.
const KernelAERNvidiaScript = `
if command -v dmesg >/dev/null 2>&1; then
dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
else
echo "dmesg not found"
fi
`
var techDumpNvidiaCommands = []struct { var techDumpNvidiaCommands = []struct {
Name string Name string
Args []string Args []string
@@ -53,6 +106,9 @@ var techDumpNvidiaCommands = []struct {
{Name: "nvidia-smi", Args: []string{"topo", "-m"}, File: "nvidia-smi-topo.txt"}, {Name: "nvidia-smi", Args: []string{"topo", "-m"}, File: "nvidia-smi-topo.txt"},
{Name: "nvidia-smi", Args: []string{"nvlink", "-s"}, File: "nvidia-smi-nvlink-status.txt"}, {Name: "nvidia-smi", Args: []string{"nvlink", "-s"}, File: "nvidia-smi-nvlink-status.txt"},
{Name: "nvidia-smi", Args: []string{"nvlink", "-e"}, File: "nvidia-smi-nvlink-errors.txt"}, {Name: "nvidia-smi", Args: []string{"nvlink", "-e"}, File: "nvidia-smi-nvlink-errors.txt"},
{Name: "sh", Args: []string{"-c", PCIeNvidiaLinkScript}, File: "pcie-nvidia-link.txt"},
{Name: "sh", Args: []string{"-c", PCIeNvidiaLinkUnderLoadScript}, File: "pcie-nvidia-link-under-load.txt"},
{Name: "sh", Args: []string{"-c", KernelAERNvidiaScript}, File: "kernel-aer-nvidia.txt"},
} }
type lsblkDumpRoot struct { type lsblkDumpRoot struct {