/topo blocked the HTTP request on live nvidia-smi calls with no timeout
(topo -m, nvlink -s/-e run on every page load), so a wedged driver hung
the page indefinitely. CaptureTechnicalDump now persists these dumps
once per audit cycle; the page reads them from techdump/ instead.
buildSocketIndex mapped NUMA node number to CPU by treating dmidecode's
Socket Designation (often 1-indexed, "CPU1"/"CPU2") as equal to the
NUMA node number (always 0-indexed) — GPUs/NICs on NUMA node 0 fell
into the "unknown" column, others attached to the wrong CPU box. Now
ranks CPUs by Socket value instead of assuming a shared numbering base.
Fixed a bug in ComponentStatusDB/applyComponentStatusDB where GPU SAT
results were keyed per-target ("pcie:gpu:nvidia-stress") instead of
per-vendor, which both broke cross-tier severity tracking (a later
clean "2. Check" run and an earlier failing "3. Load" run never
compared severities) and silently failed to match any real BDF, so the
DB overlay never reached the topology graph at all. GPU keys are now
normalized to vendor ("pcie:gpu:nvidia"/"pcie:gpu:amd"). Also skip
writing to the DB when a SAT task was aborted by the user (ctx
canceled), so a partial run can't stomp a previously recorded status.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
170 lines
5.9 KiB
Go
170 lines
5.9 KiB
Go
package platform
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
var techDumpFixedCommands = []struct {
|
|
Name string
|
|
Args []string
|
|
File string
|
|
}{
|
|
{Name: "dmidecode", Args: []string{"-t", "0"}, File: "dmidecode-type0.txt"},
|
|
{Name: "dmidecode", Args: []string{"-t", "1"}, File: "dmidecode-type1.txt"},
|
|
{Name: "dmidecode", Args: []string{"-t", "2"}, File: "dmidecode-type2.txt"},
|
|
{Name: "dmidecode", Args: []string{"-t", "4"}, File: "dmidecode-type4.txt"},
|
|
{Name: "dmidecode", Args: []string{"-t", "17"}, File: "dmidecode-type17.txt"},
|
|
{Name: "lspci", Args: []string{"-vmm", "-D"}, File: "lspci-vmm.txt"},
|
|
{Name: "lspci", Args: []string{"-vvv"}, File: "lspci-vvv.txt"},
|
|
{Name: "lscpu", Args: nil, File: "lscpu.txt"},
|
|
{Name: "lsblk", Args: []string{"-J", "-d", "-o", "NAME,TYPE,SIZE,SERIAL,MODEL,TRAN,HCTL"}, File: "lsblk.json"},
|
|
{Name: "sensors", Args: []string{"-j"}, File: "sensors.json"},
|
|
{Name: "ipmitool", Args: []string{"fru", "print"}, File: "ipmitool-fru.txt"},
|
|
{Name: "ipmitool", Args: []string{"sdr"}, File: "ipmitool-sdr.txt"},
|
|
{Name: "ipmitool", Args: []string{"sensor"}, File: "ipmitool-sensor.txt"},
|
|
{Name: "ipmitool", Args: []string{"sel", "list"}, File: "ipmitool-sel.txt"},
|
|
{Name: "ipmitool", Args: []string{"sel", "time", "get"}, File: "ipmitool-sel-time.txt"},
|
|
{Name: "nvme", Args: []string{"list", "-o", "json"}, File: "nvme-list.json"},
|
|
{Name: "storcli64", Args: []string{"/call/eall/sall", "show", "all", "J"}, File: "storcli64-drives.json"},
|
|
// storcli2 (Tri-Mode controllers, e.g. SAS3808-iMR/9500 series) needs an
|
|
// explicit /cN target for drive listing, not a /call wildcard — this
|
|
// system-level dump alone (no per-controller loop) is still useful raw
|
|
// diagnostics; structured per-drive data is collected separately by
|
|
// collectStorcli2Drives in the collector package.
|
|
{Name: "storcli2", Args: []string{"show", "all", "J"}, File: "storcli2-show-all.json"},
|
|
}
|
|
|
|
var techDumpNvidiaCommands = []struct {
|
|
Name string
|
|
Args []string
|
|
File string
|
|
}{
|
|
{Name: "nvidia-smi", Args: []string{"-q"}, File: "nvidia-smi-q.txt"},
|
|
{Name: "nvidia-smi", Args: []string{"--query-gpu=index,pci.bus_id,serial,vbios_version,temperature.gpu,power.draw,ecc.errors.uncorrected.aggregate.total,ecc.errors.corrected.aggregate.total,clocks_throttle_reasons.hw_slowdown", "--format=csv,noheader,nounits"}, File: "nvidia-smi-query.csv"},
|
|
{Name: "nvidia-smi", Args: []string{"conf-compute", "-q"}, File: "nvidia-smi-conf-compute-q.txt"},
|
|
// Consumed by webui's /topo page (GPU-GPU NVLink adjacency and per-link
|
|
// status/errors) so that page can render from this persisted dump instead
|
|
// of shelling out to nvidia-smi on every HTTP request.
|
|
{Name: "nvidia-smi", Args: []string{"topo", "-m"}, File: "nvidia-smi-topo.txt"},
|
|
{Name: "nvidia-smi", Args: []string{"nvlink", "-s"}, File: "nvidia-smi-nvlink-status.txt"},
|
|
{Name: "nvidia-smi", Args: []string{"nvlink", "-e"}, File: "nvidia-smi-nvlink-errors.txt"},
|
|
}
|
|
|
|
type lsblkDumpRoot struct {
|
|
Blockdevices []struct {
|
|
Name string `json:"name"`
|
|
Type string `json:"type"`
|
|
Tran string `json:"tran"`
|
|
} `json:"blockdevices"`
|
|
}
|
|
|
|
type nvmeDumpRoot struct {
|
|
Devices []struct {
|
|
DevicePath string `json:"DevicePath"`
|
|
} `json:"Devices"`
|
|
}
|
|
|
|
func (s *System) CaptureTechnicalDump(baseDir string) error {
|
|
if err := os.MkdirAll(baseDir, 0755); err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, cmd := range techDumpFixedCommands {
|
|
writeCommandDump(filepath.Join(baseDir, cmd.File), cmd.Name, cmd.Args...)
|
|
}
|
|
switch s.DetectGPUVendor() {
|
|
case "nvidia":
|
|
for _, cmd := range techDumpNvidiaCommands {
|
|
writeCommandDump(filepath.Join(baseDir, cmd.File), cmd.Name, cmd.Args...)
|
|
}
|
|
case "amd":
|
|
writeROCmSMIDump(filepath.Join(baseDir, "rocm-smi.txt"))
|
|
writeROCmSMIDump(filepath.Join(baseDir, "rocm-smi-showallinfo.txt"), "--showallinfo")
|
|
}
|
|
|
|
for _, dev := range lsblkDumpDevices(filepath.Join(baseDir, "lsblk.json")) {
|
|
writeCommandDump(filepath.Join(baseDir, "smartctl-"+sanitizeDumpName(dev)+".json"), "smartctl", "-j", "-a", "/dev/"+dev)
|
|
}
|
|
for _, dev := range nvmeDumpDevices(filepath.Join(baseDir, "nvme-list.json")) {
|
|
writeCommandDump(filepath.Join(baseDir, "nvme-id-ctrl-"+sanitizeDumpName(dev)+".json"), "nvme", "id-ctrl", dev, "-o", "json")
|
|
writeCommandDump(filepath.Join(baseDir, "nvme-smart-log-"+sanitizeDumpName(dev)+".json"), "nvme", "smart-log", dev, "-o", "json")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func writeCommandDump(path, name string, args ...string) {
|
|
out, err := exec.Command(name, args...).CombinedOutput()
|
|
if err != nil && len(out) == 0 {
|
|
return
|
|
}
|
|
_ = os.WriteFile(path, out, 0644)
|
|
}
|
|
|
|
func writeROCmSMIDump(path string, args ...string) {
|
|
out, err := runROCmSMI(args...)
|
|
if err != nil && len(out) == 0 {
|
|
return
|
|
}
|
|
_ = os.WriteFile(path, out, 0644)
|
|
}
|
|
|
|
func lsblkDumpDevices(path string) []string {
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
var root lsblkDumpRoot
|
|
if err := json.Unmarshal(raw, &root); err != nil {
|
|
return nil
|
|
}
|
|
var devices []string
|
|
for _, dev := range root.Blockdevices {
|
|
if strings.EqualFold(strings.TrimSpace(dev.Tran), "usb") {
|
|
continue
|
|
}
|
|
if dev.Type == "disk" && strings.TrimSpace(dev.Name) != "" {
|
|
devices = append(devices, strings.TrimSpace(dev.Name))
|
|
}
|
|
}
|
|
sort.Strings(devices)
|
|
return devices
|
|
}
|
|
|
|
func nvmeDumpDevices(path string) []string {
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
var root nvmeDumpRoot
|
|
if err := json.Unmarshal(raw, &root); err != nil {
|
|
return nil
|
|
}
|
|
seen := map[string]bool{}
|
|
var devices []string
|
|
for _, dev := range root.Devices {
|
|
name := strings.TrimSpace(dev.DevicePath)
|
|
if name == "" || seen[name] {
|
|
continue
|
|
}
|
|
seen[name] = true
|
|
devices = append(devices, name)
|
|
}
|
|
sort.Strings(devices)
|
|
return devices
|
|
}
|
|
|
|
func sanitizeDumpName(value string) string {
|
|
value = strings.TrimSpace(value)
|
|
value = strings.TrimPrefix(value, "/dev/")
|
|
value = strings.ReplaceAll(value, "/", "_")
|
|
if value == "" {
|
|
return "unknown"
|
|
}
|
|
return value
|
|
}
|