platform: split nvbandwidth SAT into per-socket + all-GPU passes

On multi-socket systems, run the NVIDIA bandwidth diagnostic once per
CPU socket before the all-GPU pass, so a crash confined to the all-GPU
run (with clean per-socket passes preceding it) isolates a cross-socket
peer-to-peer fault instead of leaving it conflated with a general
GPU/PCIe issue. Single-socket systems keep the original one-pass shape.

Also expand the support-bundle README with reference notes distilled
from a real analysis pass (BMC clock drift, "0/empty" tool output
meaning absent hardware rather than a fault, timestamp-matching before
assigning causality, and a normal-power-cycle SEL signature), plus a
step-by-step recipe for diagnosing an unexpected reboot/crash during a
specific test.

audit/internal/platform/sat.go

@@ -36,7 +36,9 @@ import (
 //   - SATEstimatedNvidiaTargetedPowerSec:          MSI v8.22 / xFusion v8.6 — 346–351 s/GPU (measured per-GPU; re-measure after switch to all-GPU simultaneous)
 //   - SATEstimatedNvidiaPulseTestSec:              xFusion v8.6 — 4 926 s / 8 GPU (all simultaneous)
 //   - SATEstimatedNvidiaInterconnectSec:           xFusion v8.6/v8.22 — 210–384 s / 8 GPU (all simultaneous)
-//   - SATEstimatedNvidiaBandwidthSec:              xFusion v8.6/v8.22 — 2 664–2 688 s / 8 GPU (all simultaneous)
+//   - SATEstimatedNvidiaBandwidthSec:              xFusion v8.6/v8.22 — 2 664–2 688 s / 8 GPU (all simultaneous);
+//     on multi-socket systems now runs as up to 3 passes (per-socket + all-GPU) — re-measure and bump this once
+//     real multi-socket task logs exist, current value only covers the single-pass/single-socket case.
 const (
 	// CPU stress: stress-ng 60 s + lscpu/sensors overhead.
 	SATEstimatedCPUValidateSec = 65
@@ -541,17 +543,53 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
 			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
 		}
 	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(
-		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
-		satJob{
-			name:       "03-dcgmi-nvbandwidth.log",
+	jobs := []satJob{
+		{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
+		{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
+	}
+
+	// On a system with GPUs on more than one CPU socket, run each socket's
+	// GPUs through nvbandwidth in isolation before the all-GPU pass. Without
+	// NVLink, cross-socket peer-to-peer traffic is a distinct fault domain
+	// from same-socket traffic; if the single-socket passes log clean and
+	// only the all-GPU pass doesn't complete, that isolates the cross-socket
+	// path as the trigger instead of leaving it conflated with a general
+	// GPU/PCIe fault. Systems with one socket (or no resolvable NUMA
+	// affinity) get a single group back and keep the original one-pass shape.
+	step := 3
+	socketGroups := gpuBandwidthSocketGroups(selected, logFunc)
+	if len(socketGroups) <= 1 {
+		jobs = append(jobs, satJob{
+			name:       fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step),
 			cmd:        nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
 			collectGPU: true,
 			gpuIndices: selected,
-		},
-		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
-	), logFunc)
+		})
+		step++
+	} else {
+		for i, group := range socketGroups {
+			jobs = append(jobs, satJob{
+				name:       fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i),
+				cmd:        nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group),
+				collectGPU: true,
+				gpuIndices: group,
+			})
+			step++
+		}
+		jobs = append(jobs, satJob{
+			name:       fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step),
+			cmd:        nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
+			collectGPU: true,
+			gpuIndices: selected,
+		})
+		step++
+	}
+	jobs = append(jobs, satJob{
+		name: fmt.Sprintf("%02d-nvidia-smi-after.log", step),
+		cmd:  []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"},
+	})
+
+	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(jobs...), logFunc)
 }
 
 func (s *System) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) {