fix(app): capture PCIe link / AER diagnostics at boot, not just on demand

pcie-nvidia-link.txt, pcie-nvidia-link-under-load.txt, and
kernel-aer-nvidia.txt only ever lived in supportBundleCommands, which
exclusively runs inside the on-demand "Download Support Bundle" web UI
action. The blackbox USB auto-sync worker never calls that function -
it only mirrors whatever CaptureTechnicalDump already wrote into the
live export tree at boot. So these three files were structurally
unreachable from a blackbox pull no matter how fresh the build was;
earlier analysis of a real blackbox misattributed their absence to
build/version drift instead.

Move the underlying scripts into shared exported constants
(platform.PCIeNvidiaLinkScript, PCIeNvidiaLinkUnderLoadScript,
KernelAERNvidiaScript) and add them to techDumpNvidiaCommands, so
CaptureTechnicalDump captures all three once at boot (bee-audit is
oneshot, so the ~8s bee-gpu-burn cost for the under-load sample is a
one-time boot cost, not a per-sync-cycle one) alongside the existing
nvidia-smi-* dumps. supportBundleCommands still re-runs the same
scripts on demand for a fresher sample - that's intentional, not a
duplicate to clean up.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-08-24 18:39:28 +03:00
co-authored by Claude Sonnet 5
parent b11018ac5e
commit ba250330d5
4 changed files with 81 additions and 49 deletions
+56
View File
@@ -39,6 +39,59 @@ var techDumpFixedCommands = []struct {
{Name: "storcli2", Args: []string{"show", "all", "J"}, File: "storcli2-show-all.json"},
}
// PCIeNvidiaLinkScript samples each NVIDIA GPU's current/max PCIe link
// speed and width from sysfs. Exported so both the boot-time techdump
// (below) and the on-demand support bundle (app/support_bundle.go, which
// re-samples for freshness at bundle-build time) run the identical script
// instead of two copies that can drift.
const PCIeNvidiaLinkScript = `
for d in /sys/bus/pci/devices/*/; do
vendor=$(cat "$d/vendor" 2>/dev/null)
[ "$vendor" = "0x10de" ] || continue
class=$(cat "$d/class" 2>/dev/null)
case "$class" in
0x030000|0x030200) ;;
*) continue ;;
esac
dev=$(basename "$d")
echo "=== $dev ==="
for f in current_link_speed current_link_width max_link_speed max_link_width; do
printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
done
done
`
// PCIeNvidiaLinkUnderLoadScript re-samples PCIeNvidiaLinkScript's sysfs
// attributes while bee-gpu-burn briefly loads the GPUs. An idle-only
// reading can't tell a real degraded slot/riser from NVIDIA's normal
// idle power-management downclock; this is the load-bearing counterpart
// that lets a reader (human or agent) tell the two apart by comparing
// pcie-nvidia-link.txt against pcie-nvidia-link-under-load.txt. See
// bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md.
const PCIeNvidiaLinkUnderLoadScript = `
if ! command -v bee-gpu-burn >/dev/null 2>&1; then
echo "bee-gpu-burn not found; cannot sample PCIe link speed under load"
exit 0
fi
bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 &
burn_pid=$!
sleep 3
` + PCIeNvidiaLinkScript + `
wait "$burn_pid" 2>/dev/null || true
rm -f /tmp/bee-pcie-load-burn.log
`
// KernelAERNvidiaScript filters dmesg for PCIe AER, NVRM, and Xid lines —
// cheap (dmesg is already in memory) and the fastest way to tell a real
// PCIe/GPU hardware fault from power-management noise.
const KernelAERNvidiaScript = `
if command -v dmesg >/dev/null 2>&1; then
dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
else
echo "dmesg not found"
fi
`
var techDumpNvidiaCommands = []struct {
Name string
Args []string
@@ -53,6 +106,9 @@ var techDumpNvidiaCommands = []struct {
{Name: "nvidia-smi", Args: []string{"topo", "-m"}, File: "nvidia-smi-topo.txt"},
{Name: "nvidia-smi", Args: []string{"nvlink", "-s"}, File: "nvidia-smi-nvlink-status.txt"},
{Name: "nvidia-smi", Args: []string{"nvlink", "-e"}, File: "nvidia-smi-nvlink-errors.txt"},
{Name: "sh", Args: []string{"-c", PCIeNvidiaLinkScript}, File: "pcie-nvidia-link.txt"},
{Name: "sh", Args: []string{"-c", PCIeNvidiaLinkUnderLoadScript}, File: "pcie-nvidia-link-under-load.txt"},
{Name: "sh", Args: []string{"-c", KernelAERNvidiaScript}, File: "kernel-aer-nvidia.txt"},
}
type lsblkDumpRoot struct {