fix(app): capture PCIe link / AER diagnostics at boot, not just on demand
pcie-nvidia-link.txt, pcie-nvidia-link-under-load.txt, and kernel-aer-nvidia.txt only ever lived in supportBundleCommands, which exclusively runs inside the on-demand "Download Support Bundle" web UI action. The blackbox USB auto-sync worker never calls that function - it only mirrors whatever CaptureTechnicalDump already wrote into the live export tree at boot. So these three files were structurally unreachable from a blackbox pull no matter how fresh the build was; earlier analysis of a real blackbox misattributed their absence to build/version drift instead. Move the underlying scripts into shared exported constants (platform.PCIeNvidiaLinkScript, PCIeNvidiaLinkUnderLoadScript, KernelAERNvidiaScript) and add them to techDumpNvidiaCommands, so CaptureTechnicalDump captures all three once at boot (bee-audit is oneshot, so the ~8s bee-gpu-burn cost for the under-load sample is a one-time boot cost, not a per-sync-cycle one) alongside the existing nvidia-smi-* dumps. supportBundleCommands still re-runs the same scripts on demand for a fresher sample - that's intentional, not a duplicate to clean up. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
b11018ac5e
commit
ba250330d5
@@ -27,7 +27,8 @@ func techdumpBucketFor(name string) string {
|
|||||||
return "storage"
|
return "storage"
|
||||||
case name == "nvidia-smi-q.txt", name == "nvidia-smi-query.csv", name == "nvidia-smi-conf-compute-q.txt",
|
case name == "nvidia-smi-q.txt", name == "nvidia-smi-query.csv", name == "nvidia-smi-conf-compute-q.txt",
|
||||||
name == "nvidia-smi-topo.txt", name == "nvidia-smi-nvlink-status.txt", name == "nvidia-smi-nvlink-errors.txt",
|
name == "nvidia-smi-topo.txt", name == "nvidia-smi-nvlink-status.txt", name == "nvidia-smi-nvlink-errors.txt",
|
||||||
name == "rocm-smi.txt", name == "rocm-smi-showallinfo.txt":
|
name == "rocm-smi.txt", name == "rocm-smi-showallinfo.txt",
|
||||||
|
name == "pcie-nvidia-link.txt", name == "pcie-nvidia-link-under-load.txt", name == "kernel-aer-nvidia.txt":
|
||||||
return "gpu"
|
return "gpu"
|
||||||
default:
|
default:
|
||||||
// dmidecode-type0/1/2, ipmitool-*, sensors.json, lspci-vmm/vvv, and
|
// dmidecode-type0/1/2, ipmitool-*, sensors.json, lspci-vmm/vvv, and
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestTechdumpBucketForPCIeLinkFiles(t *testing.T) {
|
||||||
|
cases := map[string]string{
|
||||||
|
"pcie-nvidia-link.txt": "gpu",
|
||||||
|
"pcie-nvidia-link-under-load.txt": "gpu",
|
||||||
|
"kernel-aer-nvidia.txt": "gpu",
|
||||||
|
}
|
||||||
|
for name, want := range cases {
|
||||||
|
if got := techdumpBucketFor(name); got != want {
|
||||||
|
t.Errorf("techdumpBucketFor(%q) = %q, want %q", name, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -98,13 +98,7 @@ else
|
|||||||
echo "dmesg not found"
|
echo "dmesg not found"
|
||||||
fi
|
fi
|
||||||
`}},
|
`}},
|
||||||
{name: "export/gpu/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", `
|
{name: "export/gpu/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", platform.KernelAERNvidiaScript}},
|
||||||
if command -v dmesg >/dev/null 2>&1; then
|
|
||||||
dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
|
|
||||||
else
|
|
||||||
echo "dmesg not found"
|
|
||||||
fi
|
|
||||||
`}},
|
|
||||||
{name: "livecd/gui/loginctl-sessions.txt", cmd: []string{"sh", "-c", `
|
{name: "livecd/gui/loginctl-sessions.txt", cmd: []string{"sh", "-c", `
|
||||||
if command -v loginctl >/dev/null 2>&1; then
|
if command -v loginctl >/dev/null 2>&1; then
|
||||||
loginctl list-sessions 2>&1 || true
|
loginctl list-sessions 2>&1 || true
|
||||||
@@ -310,22 +304,11 @@ if [ "$found" -eq 0 ]; then
|
|||||||
echo "no NVIDIA PCI devices found"
|
echo "no NVIDIA PCI devices found"
|
||||||
fi
|
fi
|
||||||
`}},
|
`}},
|
||||||
{name: "export/gpu/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", `
|
// Also captured once at boot by platform.CaptureTechnicalDump (which the
|
||||||
for d in /sys/bus/pci/devices/*/; do
|
// blackbox mirror picks up automatically); re-run here so an on-demand
|
||||||
vendor=$(cat "$d/vendor" 2>/dev/null)
|
// support bundle gets a fresh sample instead of a possibly-stale boot
|
||||||
[ "$vendor" = "0x10de" ] || continue
|
// one. See bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md.
|
||||||
class=$(cat "$d/class" 2>/dev/null)
|
{name: "export/gpu/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", platform.PCIeNvidiaLinkScript}},
|
||||||
case "$class" in
|
|
||||||
0x030000|0x030200) ;;
|
|
||||||
*) continue ;;
|
|
||||||
esac
|
|
||||||
dev=$(basename "$d")
|
|
||||||
echo "=== $dev ==="
|
|
||||||
for f in current_link_speed current_link_width max_link_speed max_link_width; do
|
|
||||||
printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
|
|
||||||
done
|
|
||||||
done
|
|
||||||
`}},
|
|
||||||
// A Gen1-vs-Gen4 link speed reading at idle is ambiguous: NVIDIA drivers
|
// A Gen1-vs-Gen4 link speed reading at idle is ambiguous: NVIDIA drivers
|
||||||
// deliberately downclock PCIe in low power states and re-train to full
|
// deliberately downclock PCIe in low power states and re-train to full
|
||||||
// speed under load, so pcie-nvidia-link.txt alone can't tell a real
|
// speed under load, so pcie-nvidia-link.txt alone can't tell a real
|
||||||
@@ -333,31 +316,7 @@ done
|
|||||||
// sysfs attributes while bee-gpu-burn is actively loading the GPUs — if
|
// sysfs attributes while bee-gpu-burn is actively loading the GPUs — if
|
||||||
// the link comes up here, the idle Gen1 reading above was power saving,
|
// the link comes up here, the idle Gen1 reading above was power saving,
|
||||||
// not a fault.
|
// not a fault.
|
||||||
{name: "export/gpu/pcie-nvidia-link-under-load.txt", cmd: []string{"sh", "-c", `
|
{name: "export/gpu/pcie-nvidia-link-under-load.txt", cmd: []string{"sh", "-c", platform.PCIeNvidiaLinkUnderLoadScript}},
|
||||||
if ! command -v bee-gpu-burn >/dev/null 2>&1; then
|
|
||||||
echo "bee-gpu-burn not found; cannot sample PCIe link speed under load"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 &
|
|
||||||
burn_pid=$!
|
|
||||||
sleep 3
|
|
||||||
for d in /sys/bus/pci/devices/*/; do
|
|
||||||
vendor=$(cat "$d/vendor" 2>/dev/null)
|
|
||||||
[ "$vendor" = "0x10de" ] || continue
|
|
||||||
class=$(cat "$d/class" 2>/dev/null)
|
|
||||||
case "$class" in
|
|
||||||
0x030000|0x030200) ;;
|
|
||||||
*) continue ;;
|
|
||||||
esac
|
|
||||||
dev=$(basename "$d")
|
|
||||||
echo "=== $dev ==="
|
|
||||||
for f in current_link_speed current_link_width max_link_speed max_link_width; do
|
|
||||||
printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
|
|
||||||
done
|
|
||||||
done
|
|
||||||
wait "$burn_pid" 2>/dev/null || true
|
|
||||||
rm -f /tmp/bee-pcie-load-burn.log
|
|
||||||
`}},
|
|
||||||
{name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
|
{name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
|
||||||
found=0
|
found=0
|
||||||
for dev in /sys/bus/pci/devices/*; do
|
for dev in /sys/bus/pci/devices/*; do
|
||||||
|
|||||||
@@ -39,6 +39,59 @@ var techDumpFixedCommands = []struct {
|
|||||||
{Name: "storcli2", Args: []string{"show", "all", "J"}, File: "storcli2-show-all.json"},
|
{Name: "storcli2", Args: []string{"show", "all", "J"}, File: "storcli2-show-all.json"},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PCIeNvidiaLinkScript samples each NVIDIA GPU's current/max PCIe link
|
||||||
|
// speed and width from sysfs. Exported so both the boot-time techdump
|
||||||
|
// (below) and the on-demand support bundle (app/support_bundle.go, which
|
||||||
|
// re-samples for freshness at bundle-build time) run the identical script
|
||||||
|
// instead of two copies that can drift.
|
||||||
|
const PCIeNvidiaLinkScript = `
|
||||||
|
for d in /sys/bus/pci/devices/*/; do
|
||||||
|
vendor=$(cat "$d/vendor" 2>/dev/null)
|
||||||
|
[ "$vendor" = "0x10de" ] || continue
|
||||||
|
class=$(cat "$d/class" 2>/dev/null)
|
||||||
|
case "$class" in
|
||||||
|
0x030000|0x030200) ;;
|
||||||
|
*) continue ;;
|
||||||
|
esac
|
||||||
|
dev=$(basename "$d")
|
||||||
|
echo "=== $dev ==="
|
||||||
|
for f in current_link_speed current_link_width max_link_speed max_link_width; do
|
||||||
|
printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
|
||||||
|
done
|
||||||
|
done
|
||||||
|
`
|
||||||
|
|
||||||
|
// PCIeNvidiaLinkUnderLoadScript re-samples PCIeNvidiaLinkScript's sysfs
|
||||||
|
// attributes while bee-gpu-burn briefly loads the GPUs. An idle-only
|
||||||
|
// reading can't tell a real degraded slot/riser from NVIDIA's normal
|
||||||
|
// idle power-management downclock; this is the load-bearing counterpart
|
||||||
|
// that lets a reader (human or agent) tell the two apart by comparing
|
||||||
|
// pcie-nvidia-link.txt against pcie-nvidia-link-under-load.txt. See
|
||||||
|
// bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md.
|
||||||
|
const PCIeNvidiaLinkUnderLoadScript = `
|
||||||
|
if ! command -v bee-gpu-burn >/dev/null 2>&1; then
|
||||||
|
echo "bee-gpu-burn not found; cannot sample PCIe link speed under load"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 &
|
||||||
|
burn_pid=$!
|
||||||
|
sleep 3
|
||||||
|
` + PCIeNvidiaLinkScript + `
|
||||||
|
wait "$burn_pid" 2>/dev/null || true
|
||||||
|
rm -f /tmp/bee-pcie-load-burn.log
|
||||||
|
`
|
||||||
|
|
||||||
|
// KernelAERNvidiaScript filters dmesg for PCIe AER, NVRM, and Xid lines —
|
||||||
|
// cheap (dmesg is already in memory) and the fastest way to tell a real
|
||||||
|
// PCIe/GPU hardware fault from power-management noise.
|
||||||
|
const KernelAERNvidiaScript = `
|
||||||
|
if command -v dmesg >/dev/null 2>&1; then
|
||||||
|
dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
|
||||||
|
else
|
||||||
|
echo "dmesg not found"
|
||||||
|
fi
|
||||||
|
`
|
||||||
|
|
||||||
var techDumpNvidiaCommands = []struct {
|
var techDumpNvidiaCommands = []struct {
|
||||||
Name string
|
Name string
|
||||||
Args []string
|
Args []string
|
||||||
@@ -53,6 +106,9 @@ var techDumpNvidiaCommands = []struct {
|
|||||||
{Name: "nvidia-smi", Args: []string{"topo", "-m"}, File: "nvidia-smi-topo.txt"},
|
{Name: "nvidia-smi", Args: []string{"topo", "-m"}, File: "nvidia-smi-topo.txt"},
|
||||||
{Name: "nvidia-smi", Args: []string{"nvlink", "-s"}, File: "nvidia-smi-nvlink-status.txt"},
|
{Name: "nvidia-smi", Args: []string{"nvlink", "-s"}, File: "nvidia-smi-nvlink-status.txt"},
|
||||||
{Name: "nvidia-smi", Args: []string{"nvlink", "-e"}, File: "nvidia-smi-nvlink-errors.txt"},
|
{Name: "nvidia-smi", Args: []string{"nvlink", "-e"}, File: "nvidia-smi-nvlink-errors.txt"},
|
||||||
|
{Name: "sh", Args: []string{"-c", PCIeNvidiaLinkScript}, File: "pcie-nvidia-link.txt"},
|
||||||
|
{Name: "sh", Args: []string{"-c", PCIeNvidiaLinkUnderLoadScript}, File: "pcie-nvidia-link-under-load.txt"},
|
||||||
|
{Name: "sh", Args: []string{"-c", KernelAERNvidiaScript}, File: "kernel-aer-nvidia.txt"},
|
||||||
}
|
}
|
||||||
|
|
||||||
type lsblkDumpRoot struct {
|
type lsblkDumpRoot struct {
|
||||||
|
|||||||
Reference in New Issue
Block a user