diff --git a/audit/internal/app/bundle_layout.go b/audit/internal/app/bundle_layout.go index 8d054ee..eadcbe4 100644 --- a/audit/internal/app/bundle_layout.go +++ b/audit/internal/app/bundle_layout.go @@ -27,7 +27,8 @@ func techdumpBucketFor(name string) string { return "storage" case name == "nvidia-smi-q.txt", name == "nvidia-smi-query.csv", name == "nvidia-smi-conf-compute-q.txt", name == "nvidia-smi-topo.txt", name == "nvidia-smi-nvlink-status.txt", name == "nvidia-smi-nvlink-errors.txt", - name == "rocm-smi.txt", name == "rocm-smi-showallinfo.txt": + name == "rocm-smi.txt", name == "rocm-smi-showallinfo.txt", + name == "pcie-nvidia-link.txt", name == "pcie-nvidia-link-under-load.txt", name == "kernel-aer-nvidia.txt": return "gpu" default: // dmidecode-type0/1/2, ipmitool-*, sensors.json, lspci-vmm/vvv, and diff --git a/audit/internal/app/bundle_layout_test.go b/audit/internal/app/bundle_layout_test.go new file mode 100644 index 0000000..3f518e1 --- /dev/null +++ b/audit/internal/app/bundle_layout_test.go @@ -0,0 +1,16 @@ +package app + +import "testing" + +func TestTechdumpBucketForPCIeLinkFiles(t *testing.T) { + cases := map[string]string{ + "pcie-nvidia-link.txt": "gpu", + "pcie-nvidia-link-under-load.txt": "gpu", + "kernel-aer-nvidia.txt": "gpu", + } + for name, want := range cases { + if got := techdumpBucketFor(name); got != want { + t.Errorf("techdumpBucketFor(%q) = %q, want %q", name, got, want) + } + } +} diff --git a/audit/internal/app/support_bundle.go b/audit/internal/app/support_bundle.go index e11758f..1243c0d 100644 --- a/audit/internal/app/support_bundle.go +++ b/audit/internal/app/support_bundle.go @@ -98,13 +98,7 @@ else echo "dmesg not found" fi `}}, - {name: "export/gpu/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", ` -if command -v dmesg >/dev/null 2>&1; then - dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found" -else - echo "dmesg not found" -fi -`}}, + {name: "export/gpu/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", platform.KernelAERNvidiaScript}}, {name: "livecd/gui/loginctl-sessions.txt", cmd: []string{"sh", "-c", ` if command -v loginctl >/dev/null 2>&1; then loginctl list-sessions 2>&1 || true @@ -310,22 +304,11 @@ if [ "$found" -eq 0 ]; then echo "no NVIDIA PCI devices found" fi `}}, - {name: "export/gpu/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", ` -for d in /sys/bus/pci/devices/*/; do - vendor=$(cat "$d/vendor" 2>/dev/null) - [ "$vendor" = "0x10de" ] || continue - class=$(cat "$d/class" 2>/dev/null) - case "$class" in - 0x030000|0x030200) ;; - *) continue ;; - esac - dev=$(basename "$d") - echo "=== $dev ===" - for f in current_link_speed current_link_width max_link_speed max_link_width; do - printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)" - done -done -`}}, + // Also captured once at boot by platform.CaptureTechnicalDump (which the + // blackbox mirror picks up automatically); re-run here so an on-demand + // support bundle gets a fresh sample instead of a possibly-stale boot + // one. See bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md. + {name: "export/gpu/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", platform.PCIeNvidiaLinkScript}}, // A Gen1-vs-Gen4 link speed reading at idle is ambiguous: NVIDIA drivers // deliberately downclock PCIe in low power states and re-train to full // speed under load, so pcie-nvidia-link.txt alone can't tell a real @@ -333,31 +316,7 @@ done // sysfs attributes while bee-gpu-burn is actively loading the GPUs — if // the link comes up here, the idle Gen1 reading above was power saving, // not a fault. - {name: "export/gpu/pcie-nvidia-link-under-load.txt", cmd: []string{"sh", "-c", ` -if ! command -v bee-gpu-burn >/dev/null 2>&1; then - echo "bee-gpu-burn not found; cannot sample PCIe link speed under load" - exit 0 -fi -bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 & -burn_pid=$! -sleep 3 -for d in /sys/bus/pci/devices/*/; do - vendor=$(cat "$d/vendor" 2>/dev/null) - [ "$vendor" = "0x10de" ] || continue - class=$(cat "$d/class" 2>/dev/null) - case "$class" in - 0x030000|0x030200) ;; - *) continue ;; - esac - dev=$(basename "$d") - echo "=== $dev ===" - for f in current_link_speed current_link_width max_link_speed max_link_width; do - printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)" - done -done -wait "$burn_pid" 2>/dev/null || true -rm -f /tmp/bee-pcie-load-burn.log -`}}, + {name: "export/gpu/pcie-nvidia-link-under-load.txt", cmd: []string{"sh", "-c", platform.PCIeNvidiaLinkUnderLoadScript}}, {name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", ` found=0 for dev in /sys/bus/pci/devices/*; do diff --git a/audit/internal/platform/techdump.go b/audit/internal/platform/techdump.go index 2b95080..b66c95b 100644 --- a/audit/internal/platform/techdump.go +++ b/audit/internal/platform/techdump.go @@ -39,6 +39,59 @@ var techDumpFixedCommands = []struct { {Name: "storcli2", Args: []string{"show", "all", "J"}, File: "storcli2-show-all.json"}, } +// PCIeNvidiaLinkScript samples each NVIDIA GPU's current/max PCIe link +// speed and width from sysfs. Exported so both the boot-time techdump +// (below) and the on-demand support bundle (app/support_bundle.go, which +// re-samples for freshness at bundle-build time) run the identical script +// instead of two copies that can drift. +const PCIeNvidiaLinkScript = ` +for d in /sys/bus/pci/devices/*/; do + vendor=$(cat "$d/vendor" 2>/dev/null) + [ "$vendor" = "0x10de" ] || continue + class=$(cat "$d/class" 2>/dev/null) + case "$class" in + 0x030000|0x030200) ;; + *) continue ;; + esac + dev=$(basename "$d") + echo "=== $dev ===" + for f in current_link_speed current_link_width max_link_speed max_link_width; do + printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)" + done +done +` + +// PCIeNvidiaLinkUnderLoadScript re-samples PCIeNvidiaLinkScript's sysfs +// attributes while bee-gpu-burn briefly loads the GPUs. An idle-only +// reading can't tell a real degraded slot/riser from NVIDIA's normal +// idle power-management downclock; this is the load-bearing counterpart +// that lets a reader (human or agent) tell the two apart by comparing +// pcie-nvidia-link.txt against pcie-nvidia-link-under-load.txt. See +// bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md. +const PCIeNvidiaLinkUnderLoadScript = ` +if ! command -v bee-gpu-burn >/dev/null 2>&1; then + echo "bee-gpu-burn not found; cannot sample PCIe link speed under load" + exit 0 +fi +bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 & +burn_pid=$! +sleep 3 +` + PCIeNvidiaLinkScript + ` +wait "$burn_pid" 2>/dev/null || true +rm -f /tmp/bee-pcie-load-burn.log +` + +// KernelAERNvidiaScript filters dmesg for PCIe AER, NVRM, and Xid lines — +// cheap (dmesg is already in memory) and the fastest way to tell a real +// PCIe/GPU hardware fault from power-management noise. +const KernelAERNvidiaScript = ` +if command -v dmesg >/dev/null 2>&1; then + dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found" +else + echo "dmesg not found" +fi +` + var techDumpNvidiaCommands = []struct { Name string Args []string @@ -53,6 +106,9 @@ var techDumpNvidiaCommands = []struct { {Name: "nvidia-smi", Args: []string{"topo", "-m"}, File: "nvidia-smi-topo.txt"}, {Name: "nvidia-smi", Args: []string{"nvlink", "-s"}, File: "nvidia-smi-nvlink-status.txt"}, {Name: "nvidia-smi", Args: []string{"nvlink", "-e"}, File: "nvidia-smi-nvlink-errors.txt"}, + {Name: "sh", Args: []string{"-c", PCIeNvidiaLinkScript}, File: "pcie-nvidia-link.txt"}, + {Name: "sh", Args: []string{"-c", PCIeNvidiaLinkUnderLoadScript}, File: "pcie-nvidia-link-under-load.txt"}, + {Name: "sh", Args: []string{"-c", KernelAERNvidiaScript}, File: "kernel-aer-nvidia.txt"}, } type lsblkDumpRoot struct {