fix(app): capture PCIe link / AER diagnostics at boot, not just on demand
pcie-nvidia-link.txt, pcie-nvidia-link-under-load.txt, and kernel-aer-nvidia.txt only ever lived in supportBundleCommands, which exclusively runs inside the on-demand "Download Support Bundle" web UI action. The blackbox USB auto-sync worker never calls that function - it only mirrors whatever CaptureTechnicalDump already wrote into the live export tree at boot. So these three files were structurally unreachable from a blackbox pull no matter how fresh the build was; earlier analysis of a real blackbox misattributed their absence to build/version drift instead. Move the underlying scripts into shared exported constants (platform.PCIeNvidiaLinkScript, PCIeNvidiaLinkUnderLoadScript, KernelAERNvidiaScript) and add them to techDumpNvidiaCommands, so CaptureTechnicalDump captures all three once at boot (bee-audit is oneshot, so the ~8s bee-gpu-burn cost for the under-load sample is a one-time boot cost, not a per-sync-cycle one) alongside the existing nvidia-smi-* dumps. supportBundleCommands still re-runs the same scripts on demand for a fresher sample - that's intentional, not a duplicate to clean up. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
b11018ac5e
commit
ba250330d5
@@ -39,6 +39,59 @@ var techDumpFixedCommands = []struct {
|
||||
{Name: "storcli2", Args: []string{"show", "all", "J"}, File: "storcli2-show-all.json"},
|
||||
}
|
||||
|
||||
// PCIeNvidiaLinkScript samples each NVIDIA GPU's current/max PCIe link
|
||||
// speed and width from sysfs. Exported so both the boot-time techdump
|
||||
// (below) and the on-demand support bundle (app/support_bundle.go, which
|
||||
// re-samples for freshness at bundle-build time) run the identical script
|
||||
// instead of two copies that can drift.
|
||||
const PCIeNvidiaLinkScript = `
|
||||
for d in /sys/bus/pci/devices/*/; do
|
||||
vendor=$(cat "$d/vendor" 2>/dev/null)
|
||||
[ "$vendor" = "0x10de" ] || continue
|
||||
class=$(cat "$d/class" 2>/dev/null)
|
||||
case "$class" in
|
||||
0x030000|0x030200) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
dev=$(basename "$d")
|
||||
echo "=== $dev ==="
|
||||
for f in current_link_speed current_link_width max_link_speed max_link_width; do
|
||||
printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
|
||||
done
|
||||
done
|
||||
`
|
||||
|
||||
// PCIeNvidiaLinkUnderLoadScript re-samples PCIeNvidiaLinkScript's sysfs
|
||||
// attributes while bee-gpu-burn briefly loads the GPUs. An idle-only
|
||||
// reading can't tell a real degraded slot/riser from NVIDIA's normal
|
||||
// idle power-management downclock; this is the load-bearing counterpart
|
||||
// that lets a reader (human or agent) tell the two apart by comparing
|
||||
// pcie-nvidia-link.txt against pcie-nvidia-link-under-load.txt. See
|
||||
// bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md.
|
||||
const PCIeNvidiaLinkUnderLoadScript = `
|
||||
if ! command -v bee-gpu-burn >/dev/null 2>&1; then
|
||||
echo "bee-gpu-burn not found; cannot sample PCIe link speed under load"
|
||||
exit 0
|
||||
fi
|
||||
bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 &
|
||||
burn_pid=$!
|
||||
sleep 3
|
||||
` + PCIeNvidiaLinkScript + `
|
||||
wait "$burn_pid" 2>/dev/null || true
|
||||
rm -f /tmp/bee-pcie-load-burn.log
|
||||
`
|
||||
|
||||
// KernelAERNvidiaScript filters dmesg for PCIe AER, NVRM, and Xid lines —
|
||||
// cheap (dmesg is already in memory) and the fastest way to tell a real
|
||||
// PCIe/GPU hardware fault from power-management noise.
|
||||
const KernelAERNvidiaScript = `
|
||||
if command -v dmesg >/dev/null 2>&1; then
|
||||
dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
|
||||
else
|
||||
echo "dmesg not found"
|
||||
fi
|
||||
`
|
||||
|
||||
var techDumpNvidiaCommands = []struct {
|
||||
Name string
|
||||
Args []string
|
||||
@@ -53,6 +106,9 @@ var techDumpNvidiaCommands = []struct {
|
||||
{Name: "nvidia-smi", Args: []string{"topo", "-m"}, File: "nvidia-smi-topo.txt"},
|
||||
{Name: "nvidia-smi", Args: []string{"nvlink", "-s"}, File: "nvidia-smi-nvlink-status.txt"},
|
||||
{Name: "nvidia-smi", Args: []string{"nvlink", "-e"}, File: "nvidia-smi-nvlink-errors.txt"},
|
||||
{Name: "sh", Args: []string{"-c", PCIeNvidiaLinkScript}, File: "pcie-nvidia-link.txt"},
|
||||
{Name: "sh", Args: []string{"-c", PCIeNvidiaLinkUnderLoadScript}, File: "pcie-nvidia-link-under-load.txt"},
|
||||
{Name: "sh", Args: []string{"-c", KernelAERNvidiaScript}, File: "kernel-aer-nvidia.txt"},
|
||||
}
|
||||
|
||||
type lsblkDumpRoot struct {
|
||||
|
||||
Reference in New Issue
Block a user