package platform import ( "encoding/json" "os" "os/exec" "path/filepath" "sort" "strings" ) var techDumpFixedCommands = []struct { Name string Args []string File string }{ {Name: "dmidecode", Args: []string{"-t", "0"}, File: "dmidecode-type0.txt"}, {Name: "dmidecode", Args: []string{"-t", "1"}, File: "dmidecode-type1.txt"}, {Name: "dmidecode", Args: []string{"-t", "2"}, File: "dmidecode-type2.txt"}, {Name: "dmidecode", Args: []string{"-t", "4"}, File: "dmidecode-type4.txt"}, {Name: "dmidecode", Args: []string{"-t", "17"}, File: "dmidecode-type17.txt"}, {Name: "lspci", Args: []string{"-vmm", "-D"}, File: "lspci-vmm.txt"}, {Name: "lspci", Args: []string{"-vvv"}, File: "lspci-vvv.txt"}, {Name: "lscpu", Args: nil, File: "lscpu.txt"}, {Name: "lsblk", Args: []string{"-J", "-d", "-o", "NAME,TYPE,SIZE,SERIAL,MODEL,TRAN,HCTL"}, File: "lsblk.json"}, {Name: "sensors", Args: []string{"-j"}, File: "sensors.json"}, {Name: "ipmitool", Args: []string{"fru", "print"}, File: "ipmitool-fru.txt"}, {Name: "ipmitool", Args: []string{"sdr"}, File: "ipmitool-sdr.txt"}, {Name: "ipmitool", Args: []string{"sensor"}, File: "ipmitool-sensor.txt"}, {Name: "ipmitool", Args: []string{"sel", "list"}, File: "ipmitool-sel.txt"}, {Name: "ipmitool", Args: []string{"sel", "time", "get"}, File: "ipmitool-sel-time.txt"}, {Name: "nvme", Args: []string{"list", "-o", "json"}, File: "nvme-list.json"}, {Name: "storcli64", Args: []string{"/call/eall/sall", "show", "all", "J"}, File: "storcli64-drives.json"}, // storcli2 (Tri-Mode controllers, e.g. SAS3808-iMR/9500 series) needs an // explicit /cN target for drive listing, not a /call wildcard — this // system-level dump alone (no per-controller loop) is still useful raw // diagnostics; structured per-drive data is collected separately by // collectStorcli2Drives in the collector package. {Name: "storcli2", Args: []string{"show", "all", "J"}, File: "storcli2-show-all.json"}, } // PCIeNvidiaLinkScript samples each NVIDIA GPU's current/max PCIe link // speed and width from sysfs. Exported so both the boot-time techdump // (below) and the on-demand support bundle (app/support_bundle.go, which // re-samples for freshness at bundle-build time) run the identical script // instead of two copies that can drift. const PCIeNvidiaLinkScript = ` for d in /sys/bus/pci/devices/*/; do vendor=$(cat "$d/vendor" 2>/dev/null) [ "$vendor" = "0x10de" ] || continue class=$(cat "$d/class" 2>/dev/null) case "$class" in 0x030000|0x030200) ;; *) continue ;; esac dev=$(basename "$d") echo "=== $dev ===" for f in current_link_speed current_link_width max_link_speed max_link_width; do printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)" done done ` // PCIeNvidiaLinkUnderLoadScript re-samples PCIeNvidiaLinkScript's sysfs // attributes while bee-gpu-burn briefly loads the GPUs. An idle-only // reading can't tell a real degraded slot/riser from NVIDIA's normal // idle power-management downclock; this is the load-bearing counterpart // that lets a reader (human or agent) tell the two apart by comparing // pcie-nvidia-link.txt against pcie-nvidia-link-under-load.txt. See // bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md. const PCIeNvidiaLinkUnderLoadScript = ` if ! command -v bee-gpu-burn >/dev/null 2>&1; then echo "bee-gpu-burn not found; cannot sample PCIe link speed under load" exit 0 fi bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 & burn_pid=$! sleep 3 ` + PCIeNvidiaLinkScript + ` wait "$burn_pid" 2>/dev/null || true rm -f /tmp/bee-pcie-load-burn.log ` // KernelAERNvidiaScript filters dmesg for PCIe AER, NVRM, and Xid lines — // cheap (dmesg is already in memory) and the fastest way to tell a real // PCIe/GPU hardware fault from power-management noise. const KernelAERNvidiaScript = ` if command -v dmesg >/dev/null 2>&1; then dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found" else echo "dmesg not found" fi ` var techDumpNvidiaCommands = []struct { Name string Args []string File string }{ {Name: "nvidia-smi", Args: []string{"-q"}, File: "nvidia-smi-q.txt"}, {Name: "nvidia-smi", Args: []string{"--query-gpu=index,pci.bus_id,serial,vbios_version,temperature.gpu,power.draw,ecc.errors.uncorrected.aggregate.total,ecc.errors.corrected.aggregate.total,clocks_throttle_reasons.hw_slowdown", "--format=csv,noheader,nounits"}, File: "nvidia-smi-query.csv"}, {Name: "nvidia-smi", Args: []string{"conf-compute", "-q"}, File: "nvidia-smi-conf-compute-q.txt"}, // Consumed by webui's /topo page (GPU-GPU NVLink adjacency and per-link // status/errors) so that page can render from this persisted dump instead // of shelling out to nvidia-smi on every HTTP request. {Name: "nvidia-smi", Args: []string{"topo", "-m"}, File: "nvidia-smi-topo.txt"}, {Name: "nvidia-smi", Args: []string{"nvlink", "-s"}, File: "nvidia-smi-nvlink-status.txt"}, {Name: "nvidia-smi", Args: []string{"nvlink", "-e"}, File: "nvidia-smi-nvlink-errors.txt"}, {Name: "sh", Args: []string{"-c", PCIeNvidiaLinkScript}, File: "pcie-nvidia-link.txt"}, {Name: "sh", Args: []string{"-c", PCIeNvidiaLinkUnderLoadScript}, File: "pcie-nvidia-link-under-load.txt"}, {Name: "sh", Args: []string{"-c", KernelAERNvidiaScript}, File: "kernel-aer-nvidia.txt"}, } type lsblkDumpRoot struct { Blockdevices []struct { Name string `json:"name"` Type string `json:"type"` Tran string `json:"tran"` } `json:"blockdevices"` } type nvmeDumpRoot struct { Devices []struct { DevicePath string `json:"DevicePath"` } `json:"Devices"` } func (s *System) CaptureTechnicalDump(baseDir string) error { if err := os.MkdirAll(baseDir, 0755); err != nil { return err } for _, cmd := range techDumpFixedCommands { writeCommandDump(filepath.Join(baseDir, cmd.File), cmd.Name, cmd.Args...) } switch s.DetectGPUVendor() { case "nvidia": for _, cmd := range techDumpNvidiaCommands { writeCommandDump(filepath.Join(baseDir, cmd.File), cmd.Name, cmd.Args...) } case "amd": writeROCmSMIDump(filepath.Join(baseDir, "rocm-smi.txt")) writeROCmSMIDump(filepath.Join(baseDir, "rocm-smi-showallinfo.txt"), "--showallinfo") } for _, dev := range lsblkDumpDevices(filepath.Join(baseDir, "lsblk.json")) { writeCommandDump(filepath.Join(baseDir, "smartctl-"+sanitizeDumpName(dev)+".json"), "smartctl", "-j", "-a", "/dev/"+dev) } for _, dev := range nvmeDumpDevices(filepath.Join(baseDir, "nvme-list.json")) { writeCommandDump(filepath.Join(baseDir, "nvme-id-ctrl-"+sanitizeDumpName(dev)+".json"), "nvme", "id-ctrl", dev, "-o", "json") writeCommandDump(filepath.Join(baseDir, "nvme-smart-log-"+sanitizeDumpName(dev)+".json"), "nvme", "smart-log", dev, "-o", "json") } return nil } func writeCommandDump(path, name string, args ...string) { out, err := exec.Command(name, args...).CombinedOutput() if err != nil && len(out) == 0 { return } _ = os.WriteFile(path, out, 0644) } func writeROCmSMIDump(path string, args ...string) { out, err := runROCmSMI(args...) if err != nil && len(out) == 0 { return } _ = os.WriteFile(path, out, 0644) } func lsblkDumpDevices(path string) []string { raw, err := os.ReadFile(path) if err != nil { return nil } var root lsblkDumpRoot if err := json.Unmarshal(raw, &root); err != nil { return nil } var devices []string for _, dev := range root.Blockdevices { if strings.EqualFold(strings.TrimSpace(dev.Tran), "usb") { continue } if dev.Type == "disk" && strings.TrimSpace(dev.Name) != "" { devices = append(devices, strings.TrimSpace(dev.Name)) } } sort.Strings(devices) return devices } func nvmeDumpDevices(path string) []string { raw, err := os.ReadFile(path) if err != nil { return nil } var root nvmeDumpRoot if err := json.Unmarshal(raw, &root); err != nil { return nil } seen := map[string]bool{} var devices []string for _, dev := range root.Devices { name := strings.TrimSpace(dev.DevicePath) if name == "" || seen[name] { continue } seen[name] = true devices = append(devices, name) } sort.Strings(devices) return devices } func sanitizeDumpName(value string) string { value = strings.TrimSpace(value) value = strings.TrimPrefix(value, "/dev/") value = strings.ReplaceAll(value, "/", "_") if value == "" { return "unknown" } return value }