bee/audit/internal/platform/techdump.go

package platform

import (
	"encoding/json"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
)

var techDumpFixedCommands = []struct {
	Name string
	Args []string
	File string
}{
	{Name: "dmidecode", Args: []string{"-t", "0"}, File: "dmidecode-type0.txt"},
	{Name: "dmidecode", Args: []string{"-t", "1"}, File: "dmidecode-type1.txt"},
	{Name: "dmidecode", Args: []string{"-t", "2"}, File: "dmidecode-type2.txt"},
	{Name: "dmidecode", Args: []string{"-t", "4"}, File: "dmidecode-type4.txt"},
	{Name: "dmidecode", Args: []string{"-t", "17"}, File: "dmidecode-type17.txt"},
	{Name: "lspci", Args: []string{"-vmm", "-D"}, File: "lspci-vmm.txt"},
	{Name: "lspci", Args: []string{"-vvv"}, File: "lspci-vvv.txt"},
	{Name: "lscpu", Args: nil, File: "lscpu.txt"},
	{Name: "lsblk", Args: []string{"-J", "-d", "-o", "NAME,TYPE,SIZE,SERIAL,MODEL,TRAN,HCTL"}, File: "lsblk.json"},
	{Name: "sensors", Args: []string{"-j"}, File: "sensors.json"},
	{Name: "ipmitool", Args: []string{"fru", "print"}, File: "ipmitool-fru.txt"},
	{Name: "ipmitool", Args: []string{"sdr"}, File: "ipmitool-sdr.txt"},
	{Name: "ipmitool", Args: []string{"sensor"}, File: "ipmitool-sensor.txt"},
	{Name: "ipmitool", Args: []string{"sel", "list"}, File: "ipmitool-sel.txt"},
	{Name: "ipmitool", Args: []string{"sel", "time", "get"}, File: "ipmitool-sel-time.txt"},
	{Name: "nvme", Args: []string{"list", "-o", "json"}, File: "nvme-list.json"},
	{Name: "storcli64", Args: []string{"/call/eall/sall", "show", "all", "J"}, File: "storcli64-drives.json"},
	// storcli2 (Tri-Mode controllers, e.g. SAS3808-iMR/9500 series) needs an
	// explicit /cN target for drive listing, not a /call wildcard — this
	// system-level dump alone (no per-controller loop) is still useful raw
	// diagnostics; structured per-drive data is collected separately by
	// collectStorcli2Drives in the collector package.
	{Name: "storcli2", Args: []string{"show", "all", "J"}, File: "storcli2-show-all.json"},
}

// PCIeNvidiaLinkScript samples each NVIDIA GPU's current/max PCIe link
// speed and width from sysfs. Exported so both the boot-time techdump
// (below) and the on-demand support bundle (app/support_bundle.go, which
// re-samples for freshness at bundle-build time) run the identical script
// instead of two copies that can drift.
const PCIeNvidiaLinkScript = `
for d in /sys/bus/pci/devices/*/; do
  vendor=$(cat "$d/vendor" 2>/dev/null)
  [ "$vendor" = "0x10de" ] || continue
  class=$(cat "$d/class" 2>/dev/null)
  case "$class" in
    0x030000|0x030200) ;;
    *) continue ;;
  esac
  dev=$(basename "$d")
  echo "=== $dev ==="
  for f in current_link_speed current_link_width max_link_speed max_link_width; do
    printf "  %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
  done
done
`

// PCIeNvidiaLinkUnderLoadScript re-samples PCIeNvidiaLinkScript's sysfs
// attributes while bee-gpu-burn briefly loads the GPUs. An idle-only
// reading can't tell a real degraded slot/riser from NVIDIA's normal
// idle power-management downclock; this is the load-bearing counterpart
// that lets a reader (human or agent) tell the two apart by comparing
// pcie-nvidia-link.txt against pcie-nvidia-link-under-load.txt. See
// bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md.
const PCIeNvidiaLinkUnderLoadScript = `
if ! command -v bee-gpu-burn >/dev/null 2>&1; then
  echo "bee-gpu-burn not found; cannot sample PCIe link speed under load"
  exit 0
fi
bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 &
burn_pid=$!
sleep 3
` + PCIeNvidiaLinkScript + `
wait "$burn_pid" 2>/dev/null || true
rm -f /tmp/bee-pcie-load-burn.log
`

// KernelAERNvidiaScript filters dmesg for PCIe AER, NVRM, and Xid lines —
// cheap (dmesg is already in memory) and the fastest way to tell a real
// PCIe/GPU hardware fault from power-management noise.
const KernelAERNvidiaScript = `
if command -v dmesg >/dev/null 2>&1; then
  dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
else
  echo "dmesg not found"
fi
`

var techDumpNvidiaCommands = []struct {
	Name string
	Args []string
	File string
}{
	{Name: "nvidia-smi", Args: []string{"-q"}, File: "nvidia-smi-q.txt"},
	{Name: "nvidia-smi", Args: []string{"--query-gpu=index,pci.bus_id,serial,vbios_version,temperature.gpu,power.draw,ecc.errors.uncorrected.aggregate.total,ecc.errors.corrected.aggregate.total,clocks_throttle_reasons.hw_slowdown", "--format=csv,noheader,nounits"}, File: "nvidia-smi-query.csv"},
	{Name: "nvidia-smi", Args: []string{"conf-compute", "-q"}, File: "nvidia-smi-conf-compute-q.txt"},
	// Consumed by webui's /topo page (GPU-GPU NVLink adjacency and per-link
	// status/errors) so that page can render from this persisted dump instead
	// of shelling out to nvidia-smi on every HTTP request.
	{Name: "nvidia-smi", Args: []string{"topo", "-m"}, File: "nvidia-smi-topo.txt"},
	{Name: "nvidia-smi", Args: []string{"nvlink", "-s"}, File: "nvidia-smi-nvlink-status.txt"},
	{Name: "nvidia-smi", Args: []string{"nvlink", "-e"}, File: "nvidia-smi-nvlink-errors.txt"},
	{Name: "sh", Args: []string{"-c", PCIeNvidiaLinkScript}, File: "pcie-nvidia-link.txt"},
	{Name: "sh", Args: []string{"-c", PCIeNvidiaLinkUnderLoadScript}, File: "pcie-nvidia-link-under-load.txt"},
	{Name: "sh", Args: []string{"-c", KernelAERNvidiaScript}, File: "kernel-aer-nvidia.txt"},
}

type lsblkDumpRoot struct {
	Blockdevices []struct {
		Name string `json:"name"`
		Type string `json:"type"`
		Tran string `json:"tran"`
	} `json:"blockdevices"`
}

type nvmeDumpRoot struct {
	Devices []struct {
		DevicePath string `json:"DevicePath"`
	} `json:"Devices"`
}

func (s *System) CaptureTechnicalDump(baseDir string) error {
	if err := os.MkdirAll(baseDir, 0755); err != nil {
		return err
	}

	for _, cmd := range techDumpFixedCommands {
		writeCommandDump(filepath.Join(baseDir, cmd.File), cmd.Name, cmd.Args...)
	}
	switch s.DetectGPUVendor() {
	case "nvidia":
		for _, cmd := range techDumpNvidiaCommands {
			writeCommandDump(filepath.Join(baseDir, cmd.File), cmd.Name, cmd.Args...)
		}
	case "amd":
		writeROCmSMIDump(filepath.Join(baseDir, "rocm-smi.txt"))
		writeROCmSMIDump(filepath.Join(baseDir, "rocm-smi-showallinfo.txt"), "--showallinfo")
	}

	for _, dev := range lsblkDumpDevices(filepath.Join(baseDir, "lsblk.json")) {
		writeCommandDump(filepath.Join(baseDir, "smartctl-"+sanitizeDumpName(dev)+".json"), "smartctl", "-j", "-a", "/dev/"+dev)
	}
	for _, dev := range nvmeDumpDevices(filepath.Join(baseDir, "nvme-list.json")) {
		writeCommandDump(filepath.Join(baseDir, "nvme-id-ctrl-"+sanitizeDumpName(dev)+".json"), "nvme", "id-ctrl", dev, "-o", "json")
		writeCommandDump(filepath.Join(baseDir, "nvme-smart-log-"+sanitizeDumpName(dev)+".json"), "nvme", "smart-log", dev, "-o", "json")
	}
	return nil
}

func writeCommandDump(path, name string, args ...string) {
	out, err := exec.Command(name, args...).CombinedOutput()
	if err != nil && len(out) == 0 {
		return
	}
	_ = os.WriteFile(path, out, 0644)
}

func writeROCmSMIDump(path string, args ...string) {
	out, err := runROCmSMI(args...)
	if err != nil && len(out) == 0 {
		return
	}
	_ = os.WriteFile(path, out, 0644)
}

func lsblkDumpDevices(path string) []string {
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil
	}
	var root lsblkDumpRoot
	if err := json.Unmarshal(raw, &root); err != nil {
		return nil
	}
	var devices []string
	for _, dev := range root.Blockdevices {
		if strings.EqualFold(strings.TrimSpace(dev.Tran), "usb") {
			continue
		}
		if dev.Type == "disk" && strings.TrimSpace(dev.Name) != "" {
			devices = append(devices, strings.TrimSpace(dev.Name))
		}
	}
	sort.Strings(devices)
	return devices
}

func nvmeDumpDevices(path string) []string {
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil
	}
	var root nvmeDumpRoot
	if err := json.Unmarshal(raw, &root); err != nil {
		return nil
	}
	seen := map[string]bool{}
	var devices []string
	for _, dev := range root.Devices {
		name := strings.TrimSpace(dev.DevicePath)
		if name == "" || seen[name] {
			continue
		}
		seen[name] = true
		devices = append(devices, name)
	}
	sort.Strings(devices)
	return devices
}

func sanitizeDumpName(value string) string {
	value = strings.TrimSpace(value)
	value = strings.TrimPrefix(value, "/dev/")
	value = strings.ReplaceAll(value, "/", "_")
	if value == "" {
		return "unknown"
	}
	return value
}