From ced2175fb0b8379b7a75166f44f2508b0eddd267 Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Mon, 27 Jul 2026 17:01:39 +0300 Subject: [PATCH] platform: split nvbandwidth SAT into per-socket + all-GPU passes On multi-socket systems, run the NVIDIA bandwidth diagnostic once per CPU socket before the all-GPU pass, so a crash confined to the all-GPU run (with clean per-socket passes preceding it) isolates a cross-socket peer-to-peer fault instead of leaving it conflated with a general GPU/PCIe issue. Single-socket systems keep the original one-pass shape. Also expand the support-bundle README with reference notes distilled from a real analysis pass (BMC clock drift, "0/empty" tool output meaning absent hardware rather than a fault, timestamp-matching before assigning causality, and a normal-power-cycle SEL signature), plus a step-by-step recipe for diagnosing an unexpected reboot/crash during a specific test. --- .gitignore | 1 + audit/internal/app/assets/README.md | 57 ++++- .../platform/nvidia_bandwidth_socket.go | 123 ++++++++++ .../platform/nvidia_bandwidth_socket_test.go | 213 ++++++++++++++++++ audit/internal/platform/sat.go | 56 ++++- 5 files changed, 433 insertions(+), 17 deletions(-) create mode 100644 audit/internal/platform/nvidia_bandwidth_socket.go create mode 100644 audit/internal/platform/nvidia_bandwidth_socket_test.go diff --git a/.gitignore b/.gitignore index c9989e7..ff8e0dc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dist/ build-cache/ audit/bee +audit/.tmp/ diff --git a/audit/internal/app/assets/README.md b/audit/internal/app/assets/README.md index a3d307e..9ce63d4 100644 --- a/audit/internal/app/assets/README.md +++ b/audit/internal/app/assets/README.md @@ -40,14 +40,18 @@ diagnostic command output, and logging its own systemd services. drive in `JBOD`/`UBad` state cannot join a new virtual disk without first being converted (`set good force`). - **"GPU topology / NVLink health?"** → - `export/techdump/nvidia-smi-topo.txt` (which GPUs are NVLink-bonded to - which, and how many links), `nvidia-smi-nvlink-status.txt` (per-link - active/inactive — only present in bundles built after this capture was - added; older bundles only have the topo -m aggregate), `nvidia-smi-nvlink-errors.txt` - (replay/recovery/CRC error counters, should be zero). All lanes of a - bonded pair are expected to show active; even one `` lane next - to otherwise-active ones is a real fault signature, not benign — "no - NVLink present" instead shows *all* lanes inactive. + `export/techdump/nvidia-smi-topo.txt` (`NV#` = bonded NVLink pair, `PIX`/ + `NODE`/`SYS` = PCIe-only, no NVLink), `nvidia-smi-nvlink-status.txt` + (per-link active/inactive), `nvidia-smi-nvlink-errors.txt` (replay/ + recovery/CRC counters, should be zero), and `bee-sat/nvidia-config-*/summary.txt`'s + `nvlink_pairs_checked` (how many pairs it found to check, not how many + passed). **Not every GPU config has NVLink bridges — check whether this + SKU/order is supposed to have them before calling their absence a fault** + (all-`PIX` topology, empty nvlink-status/errors files, and + `nvlink_pairs_checked=0` together mean "none detected," which is only a + problem if the config calls for NVLink). If NVLink is expected, one + `` lane next to active ones on an otherwise-bonded pair is the + real fault signature — a degraded link, not an absent one. - **"What tasks were run from the web UI, in what order, with what result?"** → `export/tasks-state.json` is the index (id, target, status, timestamps, paths). Each task also has its own directory @@ -60,6 +64,43 @@ diagnostic command output, and logging its own systemd services. release tags if you're checking whether a specific fix shipped in this build. +## Signatures + +- `ipmitool-sel-time.txt` disagrees with `manifest.txt`'s `generated_at_utc` + by more than minutes → BMC RTC drifted → all `ipmitool-sel.txt` timestamps + unreliable; use record-ID order (hex counter, column 1), not printed date. +- `storcli64` → `"Status": "Failure", "Description": "No Controller found"` + → no legacy MegaRAID controller present, not a drive/RAID fault. Cross-ref + `storcli2-show-all.json`'s `"Number of Controllers"` and `lspci`/ + `nvme-list.json` for actual storage hardware. Same pattern as + `nvlink_pairs_checked=0` above. +- `systemctl restart ... timed out` in a `bee-*.log` → implicates a SAT run + only if its timestamp is inside that run's `run_at_utc`/`started_at`– + `done_at` window in `tasks-state.json`; outside that window (e.g. boot + bring-up) it's unrelated. +- SEL sequence `Power Supply Failure detected`/`AC lost` (all PSUs) → + `ACPI Legacy OFF` → `Chassis intrusion` → `Power Button pressed` → + `Legacy ON` → full power-cycle, not a failing PSU. Isolated failure + assertion with no OFF/ON bracket, or one PSU failing while siblings stay + healthy → real PSU fault. +- Every conclusion needs a file/line/key citation. `Failure`/`Critical`/ + `timeout`/`Error` matched without reading the surrounding context is not + a citation. + +## Diagnosing "it rebooted/crashed during test X" + +1. `tasks-state.json` → task stuck `pending`/`running`, or last + `bee-sat/-*/` with no `summary.txt` = check running at crash time. +2. That directory's `verbose.log` → last subprocess with no matching + `finish`/exit-code line = the trigger. +3. `system/dmesg.txt` starting at uptime 0, ending after a few hundred + seconds = fresh-boot log captured after an unclean reset (confirms + unclean reboot; does not contain the crash itself). +4. Diff against a step that passed cleanly → isolates what's specific to + the failing workload (e.g. GPU-to-GPU/NVLink traffic vs. per-GPU + compute-only) instead of a generic cause (thermal, power) that would + also hit the passing step. + ## Top-level layout ``` diff --git a/audit/internal/platform/nvidia_bandwidth_socket.go b/audit/internal/platform/nvidia_bandwidth_socket.go new file mode 100644 index 0000000..b3a4072 --- /dev/null +++ b/audit/internal/platform/nvidia_bandwidth_socket.go @@ -0,0 +1,123 @@ +package platform + +import ( + "fmt" + "os" + "sort" + "strconv" + "strings" +) + +// satReadFile is a seam for tests to fake sysfs reads (numa_node files). +var satReadFile = os.ReadFile + +// gpuBandwidthSocketGroups splits gpuIndices into per-socket groups (ordered +// by ascending NUMA node ID) for RunNvidiaBandwidthPack. A cross-socket +// peer-to-peer path is a different (and, on platforms without NVLink, far +// less exercised) fault domain than a same-socket one, so testing each +// socket's GPUs in isolation before testing all of them together isolates +// whether a failure is specific to the cross-socket path. +// +// Falls back to a single group containing all of gpuIndices — i.e. no split +// — whenever the NUMA node can't be resolved for every GPU, or all resolve +// to the same node: there's nothing meaningful to split in that case. +func gpuBandwidthSocketGroups(gpuIndices []int, logFunc func(string)) [][]int { + nodes, err := gpuNUMANodes(gpuIndices) + if err != nil { + if logFunc != nil { + logFunc(fmt.Sprintf("nvbandwidth: could not resolve GPU NUMA nodes (%v); running all GPUs as one group", err)) + } + return [][]int{gpuIndices} + } + + byNode := map[int][]int{} + for _, idx := range gpuIndices { + node, ok := nodes[idx] + if !ok { + if logFunc != nil { + logFunc(fmt.Sprintf("nvbandwidth: no NUMA node resolved for GPU %d; running all GPUs as one group", idx)) + } + return [][]int{gpuIndices} + } + byNode[node] = append(byNode[node], idx) + } + if len(byNode) < 2 { + return [][]int{gpuIndices} + } + + sortedNodes := make([]int, 0, len(byNode)) + for node := range byNode { + sortedNodes = append(sortedNodes, node) + } + sort.Ints(sortedNodes) + + groups := make([][]int, 0, len(sortedNodes)) + for _, node := range sortedNodes { + groups = append(groups, dedupeSortedIndices(byNode[node])) + } + return groups +} + +// gpuNUMANodes resolves the NUMA node each of gpuIndices' GPU is attached to, +// via nvidia-smi's PCI bus ID and the device's sysfs numa_node attribute. +// A GPU missing from the returned map means its node couldn't be resolved. +func gpuNUMANodes(gpuIndices []int) (map[int]int, error) { + out, err := satExecCommand("nvidia-smi", "--query-gpu=index,pci.bus_id", "--format=csv,noheader,nounits").Output() + if err != nil { + return nil, fmt.Errorf("nvidia-smi: %w", err) + } + + want := make(map[int]struct{}, len(gpuIndices)) + for _, idx := range gpuIndices { + want[idx] = struct{}{} + } + + nodes := make(map[int]int, len(gpuIndices)) + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + fields := strings.SplitN(line, ",", 2) + if len(fields) != 2 { + continue + } + idx, err := strconv.Atoi(strings.TrimSpace(fields[0])) + if err != nil { + continue + } + if _, ok := want[idx]; !ok { + continue + } + bdf := normalizeNvidiaBDF(strings.TrimSpace(fields[1])) + if node, ok := readPCINumaNode(bdf); ok { + nodes[idx] = node + } + } + return nodes, nil +} + +// normalizeNvidiaBDF converts nvidia-smi's 8-hex-digit-domain PCI bus ID +// ("00000000:05:00.0") to the 4-hex-digit-domain form sysfs paths use +// ("0000:05:00.0"). +func normalizeNvidiaBDF(busID string) string { + domain, rest, ok := strings.Cut(busID, ":") + if !ok { + return busID + } + if len(domain) > 4 { + domain = domain[len(domain)-4:] + } + return domain + ":" + rest +} + +// readPCINumaNode reads a PCI device's NUMA affinity from sysfs. Returns +// false if the attribute is missing/unreadable or reports -1 (no affinity — +// common on single-socket or non-NUMA systems). +func readPCINumaNode(bdf string) (int, bool) { + data, err := satReadFile("/sys/bus/pci/devices/" + bdf + "/numa_node") + if err != nil { + return 0, false + } + node, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil || node < 0 { + return 0, false + } + return node, true +} diff --git a/audit/internal/platform/nvidia_bandwidth_socket_test.go b/audit/internal/platform/nvidia_bandwidth_socket_test.go new file mode 100644 index 0000000..9b7bfef --- /dev/null +++ b/audit/internal/platform/nvidia_bandwidth_socket_test.go @@ -0,0 +1,213 @@ +package platform + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +func fakeNvidiaSmiBusIDs(t *testing.T, csv string) { + t.Helper() + old := satExecCommand + satExecCommand = func(name string, args ...string) *exec.Cmd { + if name == "nvidia-smi" { + return exec.Command("printf", csv) + } + return exec.Command(name, args...) + } + t.Cleanup(func() { satExecCommand = old }) +} + +func fakeNUMANodes(t *testing.T, byBDF map[string]string) { + t.Helper() + old := satReadFile + satReadFile = func(path string) ([]byte, error) { + bdf := filepath.Base(filepath.Dir(path)) + if node, ok := byBDF[bdf]; ok { + return []byte(node), nil + } + return nil, os.ErrNotExist + } + t.Cleanup(func() { satReadFile = old }) +} + +func TestGPUNUMANodesResolvesFromPCIBusID(t *testing.T) { + fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:F4:00.0\n") + fakeNUMANodes(t, map[string]string{ + "0000:05:00.0": "0\n", + "0000:F4:00.0": "1\n", + }) + + nodes, err := gpuNUMANodes([]int{0, 1}) + if err != nil { + t.Fatalf("gpuNUMANodes error: %v", err) + } + if nodes[0] != 0 || nodes[1] != 1 { + t.Fatalf("nodes=%v want {0:0, 1:1}", nodes) + } +} + +func TestGPUNUMANodesSkipsUnresolvableNode(t *testing.T) { + fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n") + fakeNUMANodes(t, map[string]string{ + "0000:05:00.0": "0\n", + // GPU 1's numa_node deliberately missing, and node -1 (no affinity). + "0000:06:00.0": "-1\n", + }) + + nodes, err := gpuNUMANodes([]int{0, 1}) + if err != nil { + t.Fatalf("gpuNUMANodes error: %v", err) + } + if _, ok := nodes[1]; ok { + t.Fatalf("nodes=%v want GPU 1 absent (node -1 means no affinity)", nodes) + } + if nodes[0] != 0 { + t.Fatalf("nodes[0]=%d want 0", nodes[0]) + } +} + +func TestGPUBandwidthSocketGroupsSplitsBySocket(t *testing.T) { + fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:F4:00.0\n5, 00000000:F5:00.0\n") + fakeNUMANodes(t, map[string]string{ + "0000:05:00.0": "0\n", + "0000:06:00.0": "0\n", + "0000:76:00.0": "0\n", + "0000:77:00.0": "0\n", + "0000:F4:00.0": "1\n", + "0000:F5:00.0": "1\n", + }) + + groups := gpuBandwidthSocketGroups([]int{0, 1, 2, 3, 4, 5}, nil) + if len(groups) != 2 { + t.Fatalf("groups=%v want 2 groups", groups) + } + if joinIndexList(groups[0]) != "0,1,2,3" { + t.Fatalf("groups[0]=%v want 0,1,2,3", groups[0]) + } + if joinIndexList(groups[1]) != "4,5" { + t.Fatalf("groups[1]=%v want 4,5", groups[1]) + } +} + +func TestGPUBandwidthSocketGroupsFallsBackToSingleGroup(t *testing.T) { + t.Run("single NUMA node", func(t *testing.T) { + fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n") + fakeNUMANodes(t, map[string]string{ + "0000:05:00.0": "0\n", + "0000:06:00.0": "0\n", + }) + groups := gpuBandwidthSocketGroups([]int{0, 1}, nil) + if len(groups) != 1 || joinIndexList(groups[0]) != "0,1" { + t.Fatalf("groups=%v want single group [0,1]", groups) + } + }) + + t.Run("unresolvable NUMA node", func(t *testing.T) { + fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n") + fakeNUMANodes(t, map[string]string{ + "0000:05:00.0": "0\n", + // GPU 1 missing entirely. + }) + groups := gpuBandwidthSocketGroups([]int{0, 1}, nil) + if len(groups) != 1 || joinIndexList(groups[0]) != "0,1" { + t.Fatalf("groups=%v want single fallback group [0,1]", groups) + } + }) + + t.Run("nvidia-smi command failure", func(t *testing.T) { + old := satExecCommand + satExecCommand = func(name string, args ...string) *exec.Cmd { + return exec.Command("false") + } + t.Cleanup(func() { satExecCommand = old }) + + groups := gpuBandwidthSocketGroups([]int{0, 1}, nil) + if len(groups) != 1 || joinIndexList(groups[0]) != "0,1" { + t.Fatalf("groups=%v want single fallback group [0,1]", groups) + } + }) +} + +func TestNormalizeNvidiaBDF(t *testing.T) { + cases := map[string]string{ + "00000000:05:00.0": "0000:05:00.0", + "0000:05:00.0": "0000:05:00.0", + "garbage": "garbage", + } + for in, want := range cases { + if got := normalizeNvidiaBDF(in); got != want { + t.Fatalf("normalizeNvidiaBDF(%q)=%q want %q", in, got, want) + } + } +} + +func TestRunNvidiaBandwidthPackSplitsPerSocketThenAll(t *testing.T) { + fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:F4:00.0\n3, 00000000:F5:00.0\n") + fakeNUMANodes(t, map[string]string{ + "0000:05:00.0": "0\n", + "0000:06:00.0": "0\n", + "0000:F4:00.0": "1\n", + "0000:F5:00.0": "1\n", + }) + + dir := t.TempDir() + s := &System{} + _, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1, 2, 3}, nil) + if err != nil { + t.Fatalf("RunNvidiaBandwidthPack error: %v", err) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + if len(entries) != 1 { + t.Fatalf("want exactly one run dir, got %v", entries) + } + runDir := filepath.Join(dir, entries[0].Name()) + + wantFiles := []string{ + "00-nvidia-smi-persistence-mode.log", + "01-nvidia-smi-q.log", + "02-dcgmi-discovery.log", + "03-dcgmi-nvbandwidth-socket0.log", + "04-dcgmi-nvbandwidth-socket1.log", + "05-dcgmi-nvbandwidth-all.log", + "06-nvidia-smi-after.log", + } + for _, name := range wantFiles { + if _, err := os.Stat(filepath.Join(runDir, name)); err != nil { + t.Fatalf("missing expected job output %s: %v", name, err) + } + } +} + +func TestRunNvidiaBandwidthPackSinglePassWhenOneSocket(t *testing.T) { + fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n") + fakeNUMANodes(t, map[string]string{ + "0000:05:00.0": "0\n", + "0000:06:00.0": "0\n", + }) + + dir := t.TempDir() + s := &System{} + _, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1}, nil) + if err != nil { + t.Fatalf("RunNvidiaBandwidthPack error: %v", err) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + runDir := filepath.Join(dir, entries[0].Name()) + + if _, err := os.Stat(filepath.Join(runDir, "03-dcgmi-nvbandwidth.log")); err != nil { + t.Fatalf("missing single-pass job output: %v", err) + } + if _, err := os.Stat(filepath.Join(runDir, "03-dcgmi-nvbandwidth-socket0.log")); err == nil { + t.Fatalf("did not expect a per-socket split for a single-socket system") + } +} diff --git a/audit/internal/platform/sat.go b/audit/internal/platform/sat.go index 6855141..d36a46e 100644 --- a/audit/internal/platform/sat.go +++ b/audit/internal/platform/sat.go @@ -36,7 +36,9 @@ import ( // - SATEstimatedNvidiaTargetedPowerSec: MSI v8.22 / xFusion v8.6 — 346–351 s/GPU (measured per-GPU; re-measure after switch to all-GPU simultaneous) // - SATEstimatedNvidiaPulseTestSec: xFusion v8.6 — 4 926 s / 8 GPU (all simultaneous) // - SATEstimatedNvidiaInterconnectSec: xFusion v8.6/v8.22 — 210–384 s / 8 GPU (all simultaneous) -// - SATEstimatedNvidiaBandwidthSec: xFusion v8.6/v8.22 — 2 664–2 688 s / 8 GPU (all simultaneous) +// - SATEstimatedNvidiaBandwidthSec: xFusion v8.6/v8.22 — 2 664–2 688 s / 8 GPU (all simultaneous); +// on multi-socket systems now runs as up to 3 passes (per-socket + all-GPU) — re-measure and bump this once +// real multi-socket task logs exist, current value only covers the single-pass/single-socket case. const ( // CPU stress: stress-ng 60 s + lscpu/sensors overhead. SATEstimatedCPUValidateSec = 65 @@ -541,17 +543,53 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name)) } } - return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode( - satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, - satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, - satJob{ - name: "03-dcgmi-nvbandwidth.log", + jobs := []satJob{ + {name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, + {name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, + } + + // On a system with GPUs on more than one CPU socket, run each socket's + // GPUs through nvbandwidth in isolation before the all-GPU pass. Without + // NVLink, cross-socket peer-to-peer traffic is a distinct fault domain + // from same-socket traffic; if the single-socket passes log clean and + // only the all-GPU pass doesn't complete, that isolates the cross-socket + // path as the trigger instead of leaving it conflated with a general + // GPU/PCIe fault. Systems with one socket (or no resolvable NUMA + // affinity) get a single group back and keep the original one-pass shape. + step := 3 + socketGroups := gpuBandwidthSocketGroups(selected, logFunc) + if len(socketGroups) <= 1 { + jobs = append(jobs, satJob{ + name: fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step), cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected), collectGPU: true, gpuIndices: selected, - }, - satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, - ), logFunc) + }) + step++ + } else { + for i, group := range socketGroups { + jobs = append(jobs, satJob{ + name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i), + cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group), + collectGPU: true, + gpuIndices: group, + }) + step++ + } + jobs = append(jobs, satJob{ + name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step), + cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected), + collectGPU: true, + gpuIndices: selected, + }) + step++ + } + jobs = append(jobs, satJob{ + name: fmt.Sprintf("%02d-nvidia-smi-after.log", step), + cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}, + }) + + return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(jobs...), logFunc) } func (s *System) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) {