refactor: harden diagnostics and consolidate runtime code

audit/internal/platform/benchmark.go

@@ -453,18 +453,7 @@ func (s *System) RunNvidiaBenchmark(ctx context.Context, baseDir string, opts Nv
 				Status: "FAILED",
 			}
 			if info, ok := infoByIndex[idx]; ok {
-				gpuResult.UUID = info.UUID
-				gpuResult.Name = info.Name
-				gpuResult.BusID = info.BusID
-				gpuResult.VBIOS = info.VBIOS
-				gpuResult.PowerLimitW = info.PowerLimitW
-				gpuResult.MultiprocessorCount = info.MultiprocessorCount
-				gpuResult.DefaultPowerLimitW = info.DefaultPowerLimitW
-				gpuResult.ShutdownTempC = info.ShutdownTempC
-				gpuResult.SlowdownTempC = info.SlowdownTempC
-				gpuResult.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz
-				gpuResult.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz
-				gpuResult.MaxMemoryClockMHz = info.MaxMemoryClockMHz
+				populateBenchmarkGPUInfo(&gpuResult, info)
 			}
 			if calib, ok := calibByIndex[idx]; ok {
 				gpuResult.CalibratedPeakPowerW = calib.Summary.P95PowerW

audit/internal/platform/benchmark_analysis.go

@@ -525,13 +525,6 @@ func scoreBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkScorecard {
 	return score
 }
 
-// compositeBenchmarkScore is kept for compatibility with legacy callers.
-// CompositeScore = ComputeScore (no quality multiplier; throttling already
-// reduces TOPS directly, so no additional penalty is needed).
-func compositeBenchmarkScore(score BenchmarkScorecard) float64 {
-	return score.ComputeScore
-}
-
 // serverQualityScore returns a 0–100 score reflecting server infrastructure
 // quality, independent of GPU model or compute speed.
 //

audit/internal/platform/benchmark_power_autotune.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"math"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -721,5 +720,3 @@ func minInt(a, b int) int {
 	}
 	return b
 }
-
-var _ = exec.ErrNotFound

audit/internal/platform/benchmark_power_calibration.go

@@ -406,39 +406,6 @@ func queryIPMIServerPowerW() (float64, error) {
 	return 0, fmt.Errorf("could not parse ipmitool dcmi power reading output")
 }
 
-// sampleIPMIPowerSeries collects IPMI power readings every 2 seconds for
-// durationSec seconds. Returns the mean of all successful samples.
-// Returns 0, false if IPMI is unavailable.
-func sampleIPMIPowerSeries(ctx context.Context, durationSec int) (meanW float64, ok bool) {
-	if durationSec <= 0 {
-		return 0, false
-	}
-	deadline := time.Now().Add(time.Duration(durationSec) * time.Second)
-	var samples []float64
-loop:
-	for {
-		if w, err := queryIPMIServerPowerW(); err == nil {
-			samples = append(samples, w)
-		}
-		if time.Now().After(deadline) {
-			break
-		}
-		select {
-		case <-ctx.Done():
-			break loop
-		case <-time.After(2 * time.Second):
-		}
-	}
-	if len(samples) == 0 {
-		return 0, false
-	}
-	var sum float64
-	for _, w := range samples {
-		sum += w
-	}
-	return sum / float64(len(samples)), true
-}
-
 // characterizeServerPower computes BenchmarkServerPower from idle and loaded
 // samples plus the GPU-reported average power during steady state.
 func characterizeServerPower(idleW, loadedW, gpuReportedSumW float64, source string, available bool) *BenchmarkServerPower {
@@ -535,18 +502,7 @@ func runNvidiaBenchmarkParallel(
 	for _, idx := range selected {
 		r := &BenchmarkGPUResult{Index: idx, Status: "FAILED"}
 		if info, ok := infoByIndex[idx]; ok {
-			r.UUID = info.UUID
-			r.Name = info.Name
-			r.BusID = info.BusID
-			r.VBIOS = info.VBIOS
-			r.PowerLimitW = info.PowerLimitW
-			r.MultiprocessorCount = info.MultiprocessorCount
-			r.DefaultPowerLimitW = info.DefaultPowerLimitW
-			r.ShutdownTempC = info.ShutdownTempC
-			r.SlowdownTempC = info.SlowdownTempC
-			r.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz
-			r.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz
-			r.MaxMemoryClockMHz = info.MaxMemoryClockMHz
+			populateBenchmarkGPUInfo(r, info)
 		}
 		if calib, ok := calibByIndex[idx]; ok {
 			r.CalibratedPeakPowerW = calib.Summary.P95PowerW
@@ -753,6 +709,21 @@ func runNvidiaBenchmarkParallel(
 	}
 }
 
+func populateBenchmarkGPUInfo(result *BenchmarkGPUResult, info benchmarkGPUInfo) {
+	result.UUID = info.UUID
+	result.Name = info.Name
+	result.BusID = info.BusID
+	result.VBIOS = info.VBIOS
+	result.PowerLimitW = info.PowerLimitW
+	result.MultiprocessorCount = info.MultiprocessorCount
+	result.DefaultPowerLimitW = info.DefaultPowerLimitW
+	result.ShutdownTempC = info.ShutdownTempC
+	result.SlowdownTempC = info.SlowdownTempC
+	result.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz
+	result.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz
+	result.MaxMemoryClockMHz = info.MaxMemoryClockMHz
+}
+
 // readBenchmarkHostConfig reads static CPU and memory configuration from
 // /proc/cpuinfo and /proc/meminfo. Returns nil if neither source is readable.
 func readBenchmarkHostConfig() *BenchmarkHostConfig {

audit/internal/platform/install_to_ram_test.go

@@ -129,8 +129,6 @@ func TestShouldLogCopyProgress(t *testing.T) {
 }
 
 func TestTryRemountLiveMedium(t *testing.T) {
-	t.Parallel()
-
 	orig := runRemountMedium
 	t.Cleanup(func() {
 		runRemountMedium = orig
@@ -165,8 +163,6 @@ func TestTryRemountLiveMedium(t *testing.T) {
 }
 
 func TestEnsureLiveMediumAvailableRemountsSource(t *testing.T) {
-	t.Parallel()
-
 	origGlob := liveMediumSquashfsGlob
 	origRemount := runRemountMedium
 	t.Cleanup(func() {
@@ -210,8 +206,6 @@ func TestEnsureLiveMediumAvailableRemountsSource(t *testing.T) {
 }
 
 func TestDetachInstallMedium(t *testing.T) {
-	t.Parallel()
-
 	origUmount := umountLiveMedium
 	origEject := ejectDevice
 	t.Cleanup(func() {

audit/internal/platform/live_metrics.go

@@ -293,6 +293,10 @@ func sampleLiveTempsViaIPMI() []TempReading {
 }
 
 func firstTempInputValue(feature map[string]any) (float64, bool) {
+	return firstSensorInputValue(feature, "temp")
+}
+
+func firstSensorInputValue(feature map[string]any, kind string) (float64, bool) {
 	keys := make([]string, 0, len(feature))
 	for key := range feature {
 		keys = append(keys, key)
@@ -300,7 +304,7 @@ func firstTempInputValue(feature map[string]any) (float64, bool) {
 	sort.Strings(keys)
 	for _, key := range keys {
 		lower := strings.ToLower(key)
-		if !strings.Contains(lower, "temp") || !strings.HasSuffix(lower, "_input") {
+		if !strings.Contains(lower, kind) || !strings.HasSuffix(lower, "_input") {
 			continue
 		}
 		switch value := feature[key].(type) {

audit/internal/platform/nvidia_config_check.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -71,9 +72,9 @@ func (s *System) RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, l
 	errOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-e", []string{"nvidia-smi", "nvlink", "-e"}, nil, logFunc)
 	_ = os.WriteFile(filepath.Join(runDir, "03-nvidia-smi-nvlink-e.log"), errOut, 0644)
 
-	pairs := parseNvidiaNVLinkBondedPairs(string(topoOut))
+	pairs := ParseNvidiaNVLinkBondedPairs(string(topoOut))
 	linkStatus := parseNvidiaNVLinkStatus(string(statusOut))
-	linkErrors := parseNvidiaNVLinkErrors(string(errOut))
+	linkErrors := ParseNvidiaNVLinkErrors(string(errOut))
 	for _, pair := range pairs {
 		f := evaluateNvidiaNVLinkPair(pair, linkStatus, linkErrors)
 		status.NVLinkPairs = append(status.NVLinkPairs, f)
@@ -96,6 +97,11 @@ func (s *System) RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, l
 	dmesgOut, _ := runSATCommandCtx(ctx, verboseLog, "dmesg", []string{"dmesg"}, nil, nil)
 	ccDmesgLines := filterConfComputeDmesgLines(dmesgOut)
 	_ = os.WriteFile(filepath.Join(runDir, "05-dmesg-cc-relevant.log"), []byte(strings.Join(ccDmesgLines, "\n")+"\n"), 0644)
+	nvlinkDegraded := parseNvidiaNVLinkDegradedDmesg(dmesgOut)
+	_ = os.WriteFile(filepath.Join(runDir, "06-dmesg-nvlink-degraded.log"), []byte(renderNvidiaNVLinkDegradedLog(nvlinkDegraded)), 0644)
+	for _, event := range nvlinkDegraded {
+		status.Warnings = append(status.Warnings, event.Warning())
+	}
 	lowerDmesg := strings.ToLower(strings.Join(ccDmesgLines, "\n"))
 	status.HostAMDSEVSNPActive = strings.Contains(lowerDmesg, "sev-snp enabled")
 	status.HostIntelTDXActive = strings.Contains(lowerDmesg, "tdx module") && strings.Contains(lowerDmesg, "module initialized") ||
@@ -166,7 +172,11 @@ type NvidiaNVLinkPairFinding struct {
 	GPUA          int      `json:"gpu_a"`
 	GPUB          int      `json:"gpu_b"`
 	ExpectedLinks int      `json:"expected_links"` // from "nvidia-smi topo -m"'s NVx cell
-	ActiveLinks   int      `json:"active_links"`   // from "nvidia-smi nvlink -s"
+	ActiveLinksA  int      `json:"active_links_a"` // per endpoint from "nvidia-smi nvlink -s"
+	ActiveLinksB  int      `json:"active_links_b"`
+	TotalLinksA   int      `json:"total_links_a"`
+	TotalLinksB   int      `json:"total_links_b"`
+	ActiveLinks   int      `json:"active_links"` // aggregate retained for report compatibility
 	TotalLinks    int      `json:"total_links"`
 	ErrorLinks    int      `json:"error_links"` // links with a nonzero replay/recovery/CRC counter
 	Issues        []string `json:"issues,omitempty"`
@@ -197,28 +207,57 @@ func evaluateNvidiaGPUConfig(g NvidiaGPUSetting) []string {
 
 // evaluateNvidiaNVLinkPair compares a bonded pair's actual link state
 // against what the topology matrix says should be there.
-func evaluateNvidiaNVLinkPair(pair nvidiaNVLinkBondedPair, status map[int][]nvidiaNVLinkPort, errors map[int]map[int][3]int64) NvidiaNVLinkPairFinding {
-	f := NvidiaNVLinkPairFinding{GPUA: pair.gpuA, GPUB: pair.gpuB, ExpectedLinks: pair.links}
+func evaluateNvidiaNVLinkPair(pair NvidiaNVLinkBondedPair, status map[int][]nvidiaNVLinkPort, errors map[int]map[int][3]int64) NvidiaNVLinkPairFinding {
+	f := NvidiaNVLinkPairFinding{GPUA: pair.GPUA, GPUB: pair.GPUB, ExpectedLinks: pair.NVLinks}
 
-	active, total, errLinks := 0, 0, 0
-	for _, gpu := range []int{pair.gpuA, pair.gpuB} {
-		for _, port := range status[gpu] {
+	countPorts := func(gpu int) (active, total int, present bool) {
+		ports, present := status[gpu]
+		for _, port := range ports {
 			total++
 			if port.active {
 				active++
 			}
 		}
+		return active, total, present
+	}
+	f.ActiveLinksA, f.TotalLinksA, _ = countPorts(pair.GPUA)
+	f.ActiveLinksB, f.TotalLinksB, _ = countPorts(pair.GPUB)
+	f.ActiveLinks = f.ActiveLinksA + f.ActiveLinksB
+	f.TotalLinks = f.TotalLinksA + f.TotalLinksB
+
+	_, statusASeen := status[pair.GPUA]
+	_, statusBSeen := status[pair.GPUB]
+	if !statusASeen || !statusBSeen {
+		var missing []string
+		if !statusASeen {
+			missing = append(missing, fmt.Sprintf("GPU%d", pair.GPUA))
+		}
+		if !statusBSeen {
+			missing = append(missing, fmt.Sprintf("GPU%d", pair.GPUB))
+		}
+		f.Issues = append(f.Issues, "NVLink status unavailable for "+strings.Join(missing, ", "))
+	}
+	if pair.NVLinks > 0 && (f.ActiveLinksA < pair.NVLinks || f.ActiveLinksB < pair.NVLinks) {
+		f.Issues = append(f.Issues, fmt.Sprintf(
+			"GPU%d %d/%d, GPU%d %d/%d active links (topo NV%d)",
+			pair.GPUA, f.ActiveLinksA, pair.NVLinks,
+			pair.GPUB, f.ActiveLinksB, pair.NVLinks,
+			pair.NVLinks,
+		))
+	}
+	if f.TotalLinks > 0 && f.ActiveLinks < f.TotalLinks {
+		f.Issues = append(f.Issues, fmt.Sprintf("%d/%d reported NVLinks inactive on a bonded pair", f.TotalLinks-f.ActiveLinks, f.TotalLinks))
+	}
+
+	errLinks := 0
+	for _, gpu := range []int{pair.GPUA, pair.GPUB} {
 		for _, counters := range errors[gpu] {
 			if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
 				errLinks++
 			}
 		}
 	}
-	f.ActiveLinks, f.TotalLinks, f.ErrorLinks = active, total, errLinks
-
-	if total > 0 && active < total {
-		f.Issues = append(f.Issues, fmt.Sprintf("%d/%d NVLinks inactive on a bonded pair", total-active, total))
-	}
+	f.ErrorLinks = errLinks
 	if errLinks > 0 {
 		f.Issues = append(f.Issues, fmt.Sprintf("%d link(s) reporting replay/recovery/CRC errors", errLinks))
 	}
@@ -226,17 +265,14 @@ func evaluateNvidiaNVLinkPair(pair nvidiaNVLinkBondedPair, status map[int][]nvid
 }
 
 // ---------------------------------------------------------------------------
-// NVLink parsing — deliberately self-contained rather than importing
-// internal/collector or internal/webui's equivalent parsers (isNVLinkBridgeCandidate
-// et al. and parseGPUPairAdjacency/parseTopoNVLinkStatus/parseTopoNVLinkErrors,
-// respectively), matching this codebase's existing precedent of small,
-// package-local duplication over cross-package coupling for narrow parsing
-// helpers (see webui/page_topo.go's isNICDeviceClassDev/isRAIDControllerClass).
+// NVLink parsing shared by diagnostics and the read-only topology page.
 // ---------------------------------------------------------------------------
 
-type nvidiaNVLinkBondedPair struct {
-	gpuA, gpuB int
-	links      int
+// NvidiaNVLinkBondedPair identifies two GPU indices and their negotiated
+// NVLink bond width as reported by nvidia-smi topo -m.
+type NvidiaNVLinkBondedPair struct {
+	GPUA, GPUB int
+	NVLinks    int
 }
 
 var (
@@ -244,37 +280,31 @@ var (
 	nvidiaNVLinkNVRe   = regexp.MustCompile(`(?i)^NV(\d+)$`)
 )
 
-// parseNvidiaNVLinkBondedPairs returns every GPU pair with a nonzero NVLink
+// ParseNvidiaNVLinkBondedPairs returns every GPU pair with a nonzero NVLink
 // bond count from a "nvidia-smi topo -m" matrix, deduplicated (A,B) == (B,A).
-func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair {
+func ParseNvidiaNVLinkBondedPairs(raw string) []NvidiaNVLinkBondedPair {
 	lines := strings.Split(nvidiaNVLinkANSIRe.ReplaceAllString(raw, ""), "\n")
 	headerIdx := -1
-	var gpuColIndices []int
+	gpuColumns := map[int]int{}
 	for i, line := range lines {
-		trimmed := strings.TrimSpace(line)
-		if strings.HasPrefix(trimmed, "GPU0") {
-			parts := strings.Fields(trimmed)
-			for j, col := range parts {
-				if strings.HasPrefix(col, "GPU") {
-					gpuColIndices = append(gpuColIndices, j)
-				}
-			}
-			if len(gpuColIndices) >= 2 {
-				headerIdx = i
+		columns := map[int]int{}
+		for columnIndex, label := range strings.Fields(strings.TrimSpace(line)) {
+			gpuIndex, err := strconv.Atoi(strings.TrimPrefix(label, "GPU"))
+			if err == nil && strings.HasPrefix(label, "GPU") {
+				columns[columnIndex] = gpuIndex
 			}
+		}
+		if len(columns) >= 2 {
+			headerIdx = i
+			gpuColumns = columns
 			break
 		}
 	}
 	if headerIdx < 0 {
 		return nil
 	}
-	colIdxToGPU := make(map[int]int, len(gpuColIndices))
-	for gpuIdx, colIdx := range gpuColIndices {
-		colIdxToGPU[colIdx] = gpuIdx
-	}
-
 	seen := map[[2]int]bool{}
-	var pairs []nvidiaNVLinkBondedPair
+	var pairs []NvidiaNVLinkBondedPair
 	for _, line := range lines[headerIdx+1:] {
 		trimmed := strings.TrimSpace(line)
 		if !strings.HasPrefix(trimmed, "GPU") {
@@ -288,7 +318,7 @@ func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair {
 		if err != nil {
 			continue
 		}
-		for colIdx, colGPU := range colIdxToGPU {
+		for colIdx, colGPU := range gpuColumns {
 			if colGPU == rowGPU {
 				continue
 			}
@@ -313,9 +343,15 @@ func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair {
 				continue
 			}
 			seen[key] = true
-			pairs = append(pairs, nvidiaNVLinkBondedPair{gpuA: a, gpuB: b, links: nv})
+			pairs = append(pairs, NvidiaNVLinkBondedPair{GPUA: a, GPUB: b, NVLinks: nv})
 		}
 	}
+	sort.Slice(pairs, func(i, j int) bool {
+		if pairs[i].GPUA != pairs[j].GPUA {
+			return pairs[i].GPUA < pairs[j].GPUA
+		}
+		return pairs[i].GPUB < pairs[j].GPUB
+	})
 	return pairs
 }
 
@@ -339,6 +375,11 @@ func parseNvidiaNVLinkStatus(raw string) map[int][]nvidiaNVLinkPort {
 		trimmed := strings.TrimSpace(line)
 		if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
 			currentGPU, _ = strconv.Atoi(m[1])
+			if _, exists := result[currentGPU]; !exists {
+				// Preserve the distinction between a present GPU with no active
+				// links and a GPU whose block is absent from the command output.
+				result[currentGPU] = nil
+			}
 			continue
 		}
 		if currentGPU < 0 {
@@ -355,9 +396,67 @@ func parseNvidiaNVLinkStatus(raw string) map[int][]nvidiaNVLinkPort {
 	return result
 }
 
-// parseNvidiaNVLinkErrors parses "nvidia-smi nvlink -e" output into, per GPU
+type nvidiaNVLinkDegradedEvent struct {
+	GPUIndex int
+	LinkID   int
+	RawLine  string
+}
+
+func (e nvidiaNVLinkDegradedEvent) Warning() string {
+	switch {
+	case e.GPUIndex >= 0 && e.LinkID >= 0:
+		return fmt.Sprintf("GPU%d NVLink degraded mode (linkId %d): %s", e.GPUIndex, e.LinkID, e.RawLine)
+	case e.GPUIndex >= 0:
+		return fmt.Sprintf("GPU%d NVLink degraded mode: %s", e.GPUIndex, e.RawLine)
+	default:
+		return "NVIDIA NVLink degraded mode: " + e.RawLine
+	}
+}
+
+var (
+	nvidiaNVLinkDegradedGPURe  = regexp.MustCompile(`(?i)\bGPU\s*(\d+)\b`)
+	nvidiaNVLinkDegradedLinkRe = regexp.MustCompile(`(?i)\blinkId\s*[:=]?\s*(\d+)\b`)
+)
+
+func parseNvidiaNVLinkDegradedDmesg(raw []byte) []nvidiaNVLinkDegradedEvent {
+	var events []nvidiaNVLinkDegradedEvent
+	for _, lineBytes := range bytes.Split(raw, []byte("\n")) {
+		line := strings.TrimSpace(string(lineBytes))
+		lower := strings.ToLower(line)
+		explicitNVLinkMarker := strings.Contains(lower, "knvlinksetdegradedmode") ||
+			strings.Contains(lower, "nvlink_is_gpu_degraded") ||
+			strings.Contains(lower, "error originated on linkid")
+		contextualDegradedMarker := strings.Contains(lower, "marked degraded") &&
+			(strings.Contains(lower, "nvlink") || strings.Contains(lower, "nvrm") || nvidiaNVLinkDegradedGPURe.MatchString(line))
+		if line == "" || (!explicitNVLinkMarker && !contextualDegradedMarker) {
+			continue
+		}
+		event := nvidiaNVLinkDegradedEvent{GPUIndex: -1, LinkID: -1, RawLine: line}
+		if match := nvidiaNVLinkDegradedGPURe.FindStringSubmatch(line); len(match) == 2 {
+			event.GPUIndex, _ = strconv.Atoi(match[1])
+		}
+		if match := nvidiaNVLinkDegradedLinkRe.FindStringSubmatch(line); len(match) == 2 {
+			event.LinkID, _ = strconv.Atoi(match[1])
+		}
+		events = append(events, event)
+	}
+	return events
+}
+
+func renderNvidiaNVLinkDegradedLog(events []nvidiaNVLinkDegradedEvent) string {
+	if len(events) == 0 {
+		return ""
+	}
+	lines := make([]string, len(events))
+	for i, event := range events {
+		lines[i] = event.RawLine
+	}
+	return strings.Join(lines, "\n") + "\n"
+}
+
+// ParseNvidiaNVLinkErrors parses "nvidia-smi nvlink -e" output into, per GPU
 // then link index, [replay, recovery, crc] error counts.
-func parseNvidiaNVLinkErrors(raw string) map[int]map[int][3]int64 {
+func ParseNvidiaNVLinkErrors(raw string) map[int]map[int][3]int64 {
 	result := map[int]map[int][3]int64{}
 	currentGPU := -1
 	for _, line := range strings.Split(raw, "\n") {
@@ -516,8 +615,8 @@ func renderNvidiaConfigCheckReport(status NvidiaConfigCheckStatus) string {
 		if len(p.Issues) > 0 {
 			verdict = "ISSUES FOUND"
 		}
-		fmt.Fprintf(&b, "  GPU%d <-> GPU%d: %d/%d links active (topology expects %d): %s\n",
-			p.GPUA, p.GPUB, p.ActiveLinks, p.TotalLinks, p.ExpectedLinks, verdict)
+		fmt.Fprintf(&b, "  GPU%d <-> GPU%d: endpoints %d/%d and %d/%d active (topology expects %d each): %s\n",
+			p.GPUA, p.GPUB, p.ActiveLinksA, p.TotalLinksA, p.ActiveLinksB, p.TotalLinksB, p.ExpectedLinks, verdict)
 		for _, issue := range p.Issues {
 			fmt.Fprintf(&b, "    - %s\n", issue)
 		}

audit/internal/platform/nvidia_config_check_test.go

@@ -14,11 +14,11 @@ func TestParseNvidiaNVLinkBondedPairsRealTwoGPUDump(t *testing.T) {
 		"NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" +
 		"NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n"
 
-	pairs := parseNvidiaNVLinkBondedPairs(input)
+	pairs := ParseNvidiaNVLinkBondedPairs(input)
 	if len(pairs) != 1 {
 		t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
 	}
-	if pairs[0].gpuA != 0 || pairs[0].gpuB != 1 || pairs[0].links != 17 {
+	if pairs[0].GPUA != 0 || pairs[0].GPUB != 1 || pairs[0].NVLinks != 17 {
 		t.Fatalf("pair=%#v want {0,1,17}", pairs[0])
 	}
 }
@@ -34,18 +34,32 @@ func TestParseNvidiaNVLinkBondedPairsANSIUnderlinedHeader(t *testing.T) {
 		"GPU2\tPIX\tPIX\t X \tNV18\t0-31,64-95\n" +
 		"GPU3\tPIX\tPIX\tNV18\t X \t0-31,64-95\n"
 
-	pairs := parseNvidiaNVLinkBondedPairs(input)
+	pairs := ParseNvidiaNVLinkBondedPairs(input)
 	if len(pairs) != 2 {
 		t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs)
 	}
 	want := map[[2]int]int{{0, 1}: 18, {2, 3}: 18}
 	for _, p := range pairs {
-		if want[[2]int{p.gpuA, p.gpuB}] != p.links {
+		if want[[2]int{p.GPUA, p.GPUB}] != p.NVLinks {
 			t.Fatalf("unexpected pair %#v", p)
 		}
 	}
 }
 
+func TestParseNvidiaNVLinkBondedPairsUsesGPUIndicesFromHeader(t *testing.T) {
+	input := "\tGPU2\tGPU4\tCPU Affinity\n" +
+		"GPU2\t X \tNV4\t0-15\n" +
+		"GPU4\tNV4\t X \t16-31\n"
+
+	pairs := ParseNvidiaNVLinkBondedPairs(input)
+	if len(pairs) != 1 {
+		t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
+	}
+	if pairs[0] != (NvidiaNVLinkBondedPair{GPUA: 2, GPUB: 4, NVLinks: 4}) {
+		t.Fatalf("pair=%#v want GPU2<->GPU4 NV4", pairs[0])
+	}
+}
+
 func TestParseNvidiaNVLinkStatusMarksInactiveLinks(t *testing.T) {
 	input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
 	 Link 0: 26.562 GB/s
@@ -71,7 +85,7 @@ func TestParseNvidiaNVLinkErrors(t *testing.T) {
 	 Link 1: Recovery Errors: 1
 	 Link 1: CRC Errors: 2
 `
-	got := parseNvidiaNVLinkErrors(input)
+	got := ParseNvidiaNVLinkErrors(input)
 	c := got[0][1]
 	if c[0] != 3 || c[1] != 1 || c[2] != 2 {
 		t.Fatalf("link1 counters=%#v want {3,1,2}", c)
@@ -79,7 +93,7 @@ func TestParseNvidiaNVLinkErrors(t *testing.T) {
 }
 
 func TestEvaluateNvidiaNVLinkPairFlagsInactiveLinksAndErrors(t *testing.T) {
-	pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18}
+	pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 2}
 	status := map[int][]nvidiaNVLinkPort{
 		0: {{active: true}, {active: false}},
 		1: {{active: true}, {active: true}},
@@ -92,16 +106,16 @@ func TestEvaluateNvidiaNVLinkPairFlagsInactiveLinksAndErrors(t *testing.T) {
 	if f.ActiveLinks != 3 || f.TotalLinks != 4 {
 		t.Fatalf("active=%d total=%d want 3/4", f.ActiveLinks, f.TotalLinks)
 	}
-	if len(f.Issues) != 2 {
-		t.Fatalf("issues=%#v want 2 (inactive link + error)", f.Issues)
+	if len(f.Issues) != 3 {
+		t.Fatalf("issues=%#v want 3 (expected-count mismatch + inactive link + error)", f.Issues)
 	}
 }
 
 func TestEvaluateNvidiaNVLinkPairHealthyBondHasNoIssues(t *testing.T) {
-	pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18}
+	pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 18}
 	status := map[int][]nvidiaNVLinkPort{
-		0: {{active: true}},
-		1: {{active: true}},
+		0: activeNvidiaNVLinkPorts(18),
+		1: activeNvidiaNVLinkPorts(18),
 	}
 	f := evaluateNvidiaNVLinkPair(pair, status, nil)
 	if len(f.Issues) != 0 {
@@ -109,6 +123,76 @@ func TestEvaluateNvidiaNVLinkPairHealthyBondHasNoIssues(t *testing.T) {
 	}
 }
 
+func TestEvaluateNvidiaNVLinkPairChecksEachEndpoint(t *testing.T) {
+	pair := NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 17}
+	status := map[int][]nvidiaNVLinkPort{
+		0: activeNvidiaNVLinkPorts(17),
+		1: {},
+	}
+	f := evaluateNvidiaNVLinkPair(pair, status, nil)
+	if f.ActiveLinksA != 17 || f.ActiveLinksB != 0 {
+		t.Fatalf("endpoint counts=%d,%d want 17,0", f.ActiveLinksA, f.ActiveLinksB)
+	}
+	if len(f.Issues) == 0 || !strings.Contains(strings.Join(f.Issues, " "), "GPU1 0/17") {
+		t.Fatalf("issues=%#v want endpoint-specific 0/17 failure", f.Issues)
+	}
+}
+
+func TestParseNvidiaNVLinkStatusPreservesAllInactiveGPUBlocks(t *testing.T) {
+	raw := `GPU 0: NVIDIA H100
+NVML: Unable to retrieve NVLink information as all links are inActive
+GPU 1: NVIDIA H100
+all links are inActive
+`
+	status := parseNvidiaNVLinkStatus(raw)
+	if _, ok := status[0]; !ok {
+		t.Fatal("GPU0 header was lost")
+	}
+	if _, ok := status[1]; !ok {
+		t.Fatal("GPU1 header was lost")
+	}
+	f := evaluateNvidiaNVLinkPair(NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 17}, status, nil)
+	if len(f.Issues) == 0 || f.ActiveLinksA != 0 || f.ActiveLinksB != 0 {
+		t.Fatalf("finding=%#v want failed 0/17 endpoints", f)
+	}
+}
+
+func TestEvaluateNvidiaNVLinkPairFlagsMissingGPUBlock(t *testing.T) {
+	f := evaluateNvidiaNVLinkPair(
+		NvidiaNVLinkBondedPair{GPUA: 0, GPUB: 1, NVLinks: 2},
+		map[int][]nvidiaNVLinkPort{0: activeNvidiaNVLinkPorts(2)}, nil,
+	)
+	if !strings.Contains(strings.Join(f.Issues, " "), "status unavailable for GPU1") {
+		t.Fatalf("issues=%#v want missing GPU1 status", f.Issues)
+	}
+}
+
+func TestParseNvidiaNVLinkDegradedDmesg(t *testing.T) {
+	raw := []byte("[ 11.0] md: array md0 marked Degraded\n[ 12.0] NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n")
+	events := parseNvidiaNVLinkDegradedDmesg(raw)
+	if len(events) != 1 {
+		t.Fatalf("events=%#v want one", events)
+	}
+	if events[0].GPUIndex != 1 || events[0].LinkID != 1 {
+		t.Fatalf("event=%#v want GPU1 linkId 1", events[0])
+	}
+	if warning := events[0].Warning(); !strings.Contains(warning, "GPU1 NVLink degraded mode (linkId 1)") {
+		t.Fatalf("warning=%q", warning)
+	}
+	wantLog := "[ 12.0] NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n"
+	if log := renderNvidiaNVLinkDegradedLog(events); log != wantLog {
+		t.Fatalf("log=%q want only NVLink line %q", log, wantLog)
+	}
+}
+
+func activeNvidiaNVLinkPorts(count int) []nvidiaNVLinkPort {
+	ports := make([]nvidiaNVLinkPort, count)
+	for i := range ports {
+		ports[i].active = true
+	}
+	return ports
+}
+
 func TestEvaluateNvidiaGPUConfigFlagsECCDisabled(t *testing.T) {
 	g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Disabled"}
 	issues := evaluateNvidiaGPUConfig(g)

audit/internal/platform/sat.go

@@ -191,13 +191,19 @@ func streamExecOutput(cmd *exec.Cmd, logFunc func(string), livePath string) ([]b
 	return buf.Bytes(), waitErr
 }
 
-// NvidiaGPU holds basic GPU info from nvidia-smi.
+// satJob describes one command and the checks needed to turn its process and
+// output results into a SAT verdict.
 type satJob struct {
-	name       string
-	cmd        []string
-	env        []string // extra env vars (appended to os.Environ)
-	collectGPU bool     // collect GPU metrics via nvidia-smi while this job runs
-	gpuIndices []int    // GPU indices to collect metrics for (empty = all)
+	name string
+	cmd  []string
+	env  []string // extra env vars (appended to os.Environ)
+	// validate checks successful command output against the tool's documented
+	// result format. It may inspect artifacts from earlier jobs in runDir.
+	// FAILED means the tool proved a test failure; UNSUPPORTED means the pinned
+	// output contract could not be recognized safely.
+	validate   func(runDir string, out []byte) (status, detail string)
+	collectGPU bool  // collect GPU metrics via nvidia-smi while this job runs
+	gpuIndices []int // GPU indices to collect metrics for (empty = all)
 	// informational marks a preflight/metadata job (e.g. dcgmi discovery) whose
 	// failure shouldn't flip the pack's overall status — the diagnostic jobs
 	// that follow it are the actual test of GPU health.
@@ -314,7 +320,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
 
 	var summary strings.Builder
 	stats := satStats{}
-	nvidiaPack := strings.HasPrefix(prefix, "gpu-nvidia")
+	nvidiaPack := isNvidiaAcceptancePack(prefix)
 	perGPU := map[int]*nvidiaGPUStatusFile{}
 	selectedGPUIndices := map[int]struct{}{}
 	fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
@@ -338,6 +344,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
 
 		var out []byte
 		var err error
+		jobDetail := ""
 
 		if nvidiaPack && nvidiaJobNeedsHealthCheck(job) {
 			if msg, healthErr := checkNvidiaJobHealth(job.gpuIndices); healthErr != nil {
@@ -346,6 +353,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
 				}
 				out = []byte(msg + "\n")
 				err = healthErr
+				jobDetail = msg
 			}
 		}
 
@@ -379,6 +387,23 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
 				if err == nil {
 					err = healthErr
 				}
+				jobDetail = msg
+			}
+		}
+
+		status, rc := classifySATResult(job.name, out, err)
+		validationDetail := singleLineSATDetail(jobDetail)
+		validated := false
+		if status == "OK" && job.validate != nil {
+			status, validationDetail = validateSATJobOutput(job, runDir, out)
+			validated = true
+		}
+		if validated {
+			if validationDetail != "" {
+				if len(out) > 0 && !bytes.HasSuffix(out, []byte("\n")) {
+					out = append(out, '\n')
+				}
+				out = append(out, []byte(fmt.Sprintf("[bee-validator] %s: %s\n", status, validationDetail))...)
 			}
 		}
 
@@ -392,7 +417,6 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
 		if ctx.Err() != nil {
 			return "", ctx.Err()
 		}
-		status, rc := classifySATResult(job.name, out, err)
 		if job.informational && status != "OK" {
 			stats.Informational++
 		} else {
@@ -406,6 +430,9 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
 		key := strings.TrimSuffix(strings.TrimPrefix(job.name, "0"), ".log")
 		fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
 		fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
+		if validationDetail != "" {
+			fmt.Fprintf(&summary, "%s_detail=%s\n", key, singleLineSATDetail(validationDetail))
+		}
 	}
 	writeSATStats(&summary, stats)
 	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
@@ -420,6 +447,25 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
 	return runDir, nil
 }
 
+func isNvidiaAcceptancePack(prefix string) bool {
+	return strings.HasPrefix(prefix, "gpu-nvidia") || prefix == "nccl-tests"
+}
+
+func validateSATJobOutput(job satJob, runDir string, out []byte) (string, string) {
+	status, detail := job.validate(runDir, out)
+	status = strings.ToUpper(strings.TrimSpace(status))
+	switch status {
+	case "OK", "FAILED", "UNSUPPORTED", "PARTIAL":
+		return status, strings.TrimSpace(detail)
+	default:
+		return "UNSUPPORTED", fmt.Sprintf("validator returned invalid status %q", status)
+	}
+}
+
+func singleLineSATDetail(detail string) string {
+	return strings.Join(strings.Fields(detail), " ")
+}
+
 func updateNvidiaGPUStatus(perGPU map[int]*nvidiaGPUStatusFile, idx int, status, jobName, detail string) {
 	entry := perGPU[idx]
 	if entry == nil {
@@ -505,112 +551,6 @@ func writeNvidiaGPUStatusFiles(runDir, overall string, perGPU map[int]*nvidiaGPU
 	return nil
 }
 
-func nvidiaSATStatusSeverity(status string) int {
-	switch strings.ToUpper(strings.TrimSpace(status)) {
-	case "FAILED":
-		return 3
-	case "PARTIAL", "UNSUPPORTED":
-		return 2
-	case "OK":
-		return 1
-	default:
-		return 0
-	}
-}
-
-func firstLine(s string) string {
-	s = strings.TrimSpace(s)
-	if s == "" {
-		return ""
-	}
-	if idx := strings.IndexByte(s, '\n'); idx >= 0 {
-		return strings.TrimSpace(s[:idx])
-	}
-	return s
-}
-
-func nvidiaJobNeedsHealthCheck(job satJob) bool {
-	if job.collectGPU {
-		return true
-	}
-	name := strings.ToLower(strings.TrimSpace(job.name))
-	return strings.Contains(name, "dcgmi") ||
-		strings.Contains(name, "gpu-burn") ||
-		strings.Contains(name, "gpu-stress") ||
-		strings.Contains(name, "dcgmproftester")
-}
-
-func checkNvidiaJobHealth(selected []int) (string, error) {
-	health, err := readNvidiaGPUHealth()
-	if err != nil {
-		return "", nil
-	}
-	var bad []nvidiaGPUHealth
-	selectedSet := make(map[int]struct{}, len(selected))
-	for _, idx := range selected {
-		selectedSet[idx] = struct{}{}
-	}
-	for _, gpu := range health {
-		if len(selectedSet) > 0 {
-			if _, ok := selectedSet[gpu.Index]; !ok {
-				continue
-			}
-		}
-		if gpu.NeedsReset {
-			bad = append(bad, gpu)
-		}
-	}
-	if len(bad) == 0 {
-		return "", nil
-	}
-	lines := make([]string, 0, len(bad)+1)
-	lines = append(lines, "NVIDIA GPU health check failed:")
-	for _, gpu := range bad {
-		lines = append(lines, fmt.Sprintf("gpu %d (%s) requires reset: %s", gpu.Index, gpu.Name, gpu.RawLine))
-	}
-	return strings.Join(lines, "\n"), errors.New("nvidia gpu requires reset")
-}
-
-func readNvidiaGPUHealth() ([]nvidiaGPUHealth, error) {
-	out, err := satExecCommand(
-		"nvidia-smi",
-		"--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total",
-		"--format=csv,noheader,nounits",
-	).Output()
-	if err != nil {
-		return nil, fmt.Errorf("nvidia-smi: %w", err)
-	}
-	return parseNvidiaGPUHealth(string(out)), nil
-}
-
-func parseNvidiaGPUHealth(raw string) []nvidiaGPUHealth {
-	var gpus []nvidiaGPUHealth
-	for _, line := range strings.Split(strings.TrimSpace(raw), "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" {
-			continue
-		}
-		parts := strings.Split(line, ",")
-		if len(parts) < 2 {
-			gpus = append(gpus, nvidiaGPUHealth{RawLine: line, ParseFailure: true})
-			continue
-		}
-		idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
-		if err != nil {
-			gpus = append(gpus, nvidiaGPUHealth{RawLine: line, ParseFailure: true})
-			continue
-		}
-		upper := strings.ToUpper(line)
-		gpus = append(gpus, nvidiaGPUHealth{
-			Index:      idx,
-			Name:       strings.TrimSpace(parts[1]),
-			NeedsReset: strings.Contains(upper, "GPU REQUIRES RESET"),
-			RawLine:    line,
-		})
-	}
-	return gpus
-}
-
 // runSATCommandCtx runs cmd and returns its combined output. livePath is
 // variadic purely so existing callers are unaffected: pass a path (job's
 // output file) to also stream output to disk live as it runs, so a crash
@@ -736,7 +676,7 @@ func (s *satStats) Add(status string) {
 	switch status {
 	case "OK":
 		s.OK++
-	case "UNSUPPORTED":
+	case "UNSUPPORTED", "PARTIAL":
 		s.Unsupported++
 	default:
 		s.Failed++

audit/internal/platform/sat_fan_stress.go

@@ -571,21 +571,6 @@ func sampleFanSpeedsViaSensorsJSON() ([]FanReading, error) {
 	return fans, nil
 }
 
-// sampleFanDutyCyclePct reads fan PWM/duty-cycle controls from lm-sensors.
-// Returns the average duty cycle across all exposed PWM controls.
-func sampleFanDutyCyclePct() (float64, bool, bool) {
-	out, err := exec.Command("sensors", "-j").Output()
-	if err != nil || len(out) == 0 {
-		fans, fanErr := sampleFanSpeeds()
-		if fanErr != nil {
-			return 0, false, false
-		}
-		return sampleFanDutyCyclePctFromFans(fans)
-	}
-	pct, ok := parseFanDutyCyclePctSensorsJSON(out)
-	return pct, ok, false
-}
-
 func sampleFanDutyCyclePctFromFans(fans []FanReading) (float64, bool, bool) {
 	if len(fans) == 0 {
 		return 0, false, false
@@ -682,27 +667,7 @@ func normalizePWMAsDutyPct(raw float64) (float64, bool) {
 }
 
 func firstFanInputValue(feature map[string]any) (float64, bool) {
-	keys := make([]string, 0, len(feature))
-	for key := range feature {
-		keys = append(keys, key)
-	}
-	sort.Strings(keys)
-	for _, key := range keys {
-		lower := strings.ToLower(key)
-		if !strings.Contains(lower, "fan") || !strings.HasSuffix(lower, "_input") {
-			continue
-		}
-		switch value := feature[key].(type) {
-		case float64:
-			return value, true
-		case string:
-			f, err := strconv.ParseFloat(value, 64)
-			if err == nil {
-				return f, true
-			}
-		}
-	}
-	return 0, false
+	return firstSensorInputValue(feature, "fan")
 }
 
 // sampleCPUMaxTemp returns the highest CPU/inlet temperature from ipmitool or sensors.

audit/internal/platform/sat_gpu_packs.go

@@ -324,17 +324,30 @@ func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []
 	if err != nil {
 		return "", err
 	}
+	return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", ncclSATJobs(selected), logFunc)
+}
+
+func ncclSATJobs(selected []int) []satJob {
 	gpuCount := len(selected)
 	if gpuCount < 1 {
 		gpuCount = 1
 	}
-	return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", withNvidiaPersistenceMode(
+	ncclEnv := append(nvidiaVisibleDevicesEnv(selected),
+		"NCCL_DEBUG=INFO",
+		"NCCL_DEBUG_SUBSYS=INIT,GRAPH",
+	)
+	return withNvidiaPersistenceMode(
 		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
+		satJob{name: "01-nvidia-smi-topo-m.log", cmd: []string{"nvidia-smi", "topo", "-m"}, informational: true},
+		satJob{name: "01-nvidia-smi-nvlink-s.log", cmd: []string{"nvidia-smi", "nvlink", "-s"}, informational: true},
 		satJob{name: "02-all-reduce-perf.log", cmd: []string{
 			"all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2",
 			"-g", strconv.Itoa(gpuCount), "--iters", "20",
-		}, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true},
-	), logFunc)
+		}, env: ncclEnv, gpuIndices: selected, syncBracket: true,
+			validate: func(runDir string, out []byte) (string, string) {
+				return validateNCCLAllReduceOutput(runDir, out, selected)
+			}},
+	)
 }
 
 func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error) {
@@ -384,23 +397,27 @@ func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir strin
 }
 
 func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
+	return s.runNvidiaNamedDiagPack(ctx, baseDir, durationSec, gpuIndices,
+		"gpu-nvidia-targeted-power", "targeted_power", "03-dcgmi-targeted-power.log", logFunc)
+}
+
+func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
+	return s.runNvidiaNamedDiagPack(ctx, baseDir, durationSec, gpuIndices,
+		"gpu-nvidia-pulse", "pulse_test", "03-dcgmi-pulse-test.log", logFunc)
+}
+
+func (s *System) runNvidiaNamedDiagPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, packName, diagName, logName string, logFunc func(string)) (string, error) {
 	selected, err := resolveDCGMGPUIndices(gpuIndices)
 	if err != nil {
 		return "", err
 	}
-	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
-	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
-	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
-		for _, p := range killed {
-			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
-		}
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode(
+	killStaleNvidiaTestWorkers(logFunc)
+	return runAcceptancePackCtx(ctx, baseDir, packName, withNvidiaPersistenceMode(
 		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
 		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
 		satJob{
-			name:        "03-dcgmi-targeted-power.log",
-			cmd:         nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
+			name:        logName,
+			cmd:         nvidiaDCGMNamedDiagCommand(diagName, normalizeNvidiaBurnDuration(durationSec), selected),
 			collectGPU:  true,
 			gpuIndices:  selected,
 			syncBracket: true,
@@ -409,30 +426,14 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string,
 	), logFunc)
 }
 
-func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
-	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
+// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
+// before starting; otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
+func killStaleNvidiaTestWorkers(logFunc func(string)) {
 	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
 		for _, p := range killed {
 			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
 		}
 	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode(
-		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
-		satJob{
-			name:        "03-dcgmi-pulse-test.log",
-			cmd:         nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		},
-		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
-	), logFunc)
 }
 
 // RunNvidiaBandwidthPack runs `dcgmi diag -r nvbandwidth`. The only thing
@@ -449,13 +450,7 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
 	if err != nil {
 		return "", err
 	}
-	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
-	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
-	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
-		for _, p := range killed {
-			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
-		}
-	}
+	killStaleNvidiaTestWorkers(logFunc)
 	jobs := []satJob{
 		{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
 		{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
@@ -528,29 +523,8 @@ func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir
 }
 
 func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
-	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
-	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
-		for _, p := range killed {
-			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
-		}
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode(
-		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
-		satJob{
-			name:        "03-dcgmi-targeted-stress.log",
-			cmd:         nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		},
-		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
-	), logFunc)
+	return s.runNvidiaNamedDiagPack(ctx, baseDir, durationSec, gpuIndices,
+		"gpu-nvidia-targeted-stress", "targeted_stress", "03-dcgmi-targeted-stress.log", logFunc)
 }
 
 func resolveDCGMGPUIndices(gpuIndices []int) ([]int, error) {

audit/internal/platform/sat_nccl_validation.go

@@ -0,0 +1,123 @@
+package platform
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var (
+	ncclOutOfBoundsRE = regexp.MustCompile(`(?im)^#\s*Out of bounds values\s*:\s*(\d+)(?:\s+\S+)?\s*$`)
+	ncclAvgBusBWRE    = regexp.MustCompile(`(?im)^#\s*Avg bus bandwidth\s*:\s*([0-9]+(?:\.[0-9]+)?)\s*$`)
+	ncclGPUHeaderRE   = regexp.MustCompile(`(?i)\bGPU\d+\b`)
+)
+
+// validateNCCLAllReduceOutput validates only facts that nccl-tests reports
+// directly. It deliberately does not apply a model-independent bandwidth
+// threshold: busbw depends on GPU model, topology, rank count, message size,
+// and NCCL algorithm. The measured value is retained in detail for diagnosis.
+func validateNCCLAllReduceOutput(runDir string, out []byte, selectedGPUIndices []int) (string, string) {
+	text := string(out)
+	oobMatch := ncclOutOfBoundsRE.FindStringSubmatch(text)
+	if len(oobMatch) != 2 {
+		return "UNSUPPORTED", "nccl-tests output has no recognized Out of bounds values summary"
+	}
+	oob, err := strconv.ParseUint(oobMatch[1], 10, 64)
+	if err != nil {
+		return "UNSUPPORTED", "nccl-tests Out of bounds values is not an unsigned integer"
+	}
+	if oob != 0 {
+		return "FAILED", fmt.Sprintf("nccl-tests reported %d out-of-bounds value(s)", oob)
+	}
+
+	rows, wrong, rowsValid := parseNCCLWrongCounts(text)
+	if !rowsValid || rows == 0 {
+		return "UNSUPPORTED", "nccl-tests output has no recognized all_reduce_perf result rows"
+	}
+	if wrong != 0 {
+		return "FAILED", fmt.Sprintf("nccl-tests result rows reported %d wrong value(s)", wrong)
+	}
+
+	avgMatch := ncclAvgBusBWRE.FindStringSubmatch(text)
+	if len(avgMatch) != 2 {
+		return "UNSUPPORTED", "nccl-tests output has no recognized Avg bus bandwidth value"
+	}
+	avgBusBW, err := strconv.ParseFloat(avgMatch[1], 64)
+	if err != nil {
+		return "UNSUPPORTED", "nccl-tests Avg bus bandwidth is not numeric"
+	}
+
+	detail := fmt.Sprintf("validation passed; avg_bus_bandwidth=%.4g GB/s (diagnostic only, no unverified threshold)", avgBusBW)
+	if len(selectedGPUIndices) < 2 {
+		return "OK", detail
+	}
+
+	topoRaw, err := os.ReadFile(filepath.Join(runDir, "01-nvidia-smi-topo-m.log"))
+	if err != nil || !ncclGPUHeaderRE.Match(topoRaw) {
+		return "UNSUPPORTED", "NVIDIA topology output unavailable or unrecognized; cannot validate expected NVLink pairs"
+	}
+	pairs := selectedNvidiaNVLinkPairs(ParseNvidiaNVLinkBondedPairs(string(topoRaw)), selectedGPUIndices)
+	if len(pairs) == 0 {
+		return "OK", detail + "; selected topology has no NVLink-bonded pair"
+	}
+
+	statusRaw, err := os.ReadFile(filepath.Join(runDir, "01-nvidia-smi-nvlink-s.log"))
+	if err != nil {
+		return "UNSUPPORTED", "NVLink status output unavailable for selected NVLink-bonded pair"
+	}
+	linkStatus := parseNvidiaNVLinkStatus(string(statusRaw))
+	var issues []string
+	for _, pair := range pairs {
+		finding := evaluateNvidiaNVLinkPair(pair, linkStatus, nil)
+		for _, issue := range finding.Issues {
+			issues = append(issues, fmt.Sprintf("GPU%d<->GPU%d: %s", pair.GPUA, pair.GPUB, issue))
+		}
+	}
+	if len(issues) != 0 {
+		return "FAILED", "NVLink state does not match selected topology: " + strings.Join(issues, "; ")
+	}
+	return "OK", detail + "; selected NVLink endpoints match topology"
+}
+
+func parseNCCLWrongCounts(raw string) (rows int, wrong uint64, valid bool) {
+	for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) < 13 {
+			continue
+		}
+		if _, err := strconv.ParseUint(fields[0], 10, 64); err != nil {
+			continue
+		}
+		outOfPlaceWrong, errA := strconv.ParseUint(fields[8], 10, 64)
+		inPlaceWrong, errB := strconv.ParseUint(fields[12], 10, 64)
+		if errA != nil || errB != nil {
+			return rows, wrong, false
+		}
+		rows++
+		wrong += outOfPlaceWrong + inPlaceWrong
+	}
+	return rows, wrong, true
+}
+
+func selectedNvidiaNVLinkPairs(pairs []NvidiaNVLinkBondedPair, selected []int) []NvidiaNVLinkBondedPair {
+	selectedSet := make(map[int]struct{}, len(selected))
+	for _, index := range selected {
+		selectedSet[index] = struct{}{}
+	}
+	filtered := make([]NvidiaNVLinkBondedPair, 0, len(pairs))
+	for _, pair := range pairs {
+		_, aSelected := selectedSet[pair.GPUA]
+		_, bSelected := selectedSet[pair.GPUB]
+		if aSelected && bSelected {
+			filtered = append(filtered, pair)
+		}
+	}
+	return filtered
+}

audit/internal/platform/sat_nccl_validation_test.go

@@ -0,0 +1,136 @@
+package platform
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+const ncclValidOutput = `# size count type redop root time algbw busbw #wrong time algbw busbw #wrong
+536870912 134217728 float sum -1 19547 27.47 48.07 0 19512 27.52 48.15 0
+# Out of bounds values : 0 OK
+# Avg bus bandwidth    : 48.1908
+`
+
+func TestValidateNCCLAllReduceOutputHealthyNVLinkPair(t *testing.T) {
+	runDir := t.TempDir()
+	writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tNV2\nGPU1\tNV2\tX\n")
+	writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-nvlink-s.log", `GPU 0: H100
+Link 0: 26.562 GB/s
+Link 1: 26.562 GB/s
+GPU 1: H100
+Link 0: 26.562 GB/s
+Link 1: 26.562 GB/s
+`)
+	status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1})
+	if status != "OK" {
+		t.Fatalf("status=%q detail=%q want OK", status, detail)
+	}
+}
+
+func TestValidateNCCLAllReduceOutputFailsInactiveNVLinkEndpoint(t *testing.T) {
+	runDir := t.TempDir()
+	writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tNV2\nGPU1\tNV2\tX\n")
+	writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-nvlink-s.log", `GPU 0: H100
+Link 0: 26.562 GB/s
+Link 1: 26.562 GB/s
+GPU 1: H100
+NVML: Unable to retrieve NVLink information as all links are inActive
+`)
+	status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1})
+	if status != "FAILED" || !strings.Contains(detail, "GPU1 0/2") {
+		t.Fatalf("status=%q detail=%q want FAILED with GPU1 0/2", status, detail)
+	}
+}
+
+func TestValidateNCCLAllReduceOutputRejectsWrongValues(t *testing.T) {
+	bad := strings.Replace(ncclValidOutput, "48.07 0", "48.07 2", 1)
+	status, detail := validateNCCLAllReduceOutput(t.TempDir(), []byte(bad), []int{0})
+	if status != "FAILED" || !strings.Contains(detail, "2 wrong") {
+		t.Fatalf("status=%q detail=%q want wrong-value failure", status, detail)
+	}
+}
+
+func TestValidateNCCLAllReduceOutputPCIeOnlyDoesNotRequireNVLink(t *testing.T) {
+	runDir := t.TempDir()
+	writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tPIX\nGPU1\tPIX\tX\n")
+	status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1})
+	if status != "OK" || !strings.Contains(detail, "no NVLink-bonded pair") {
+		t.Fatalf("status=%q detail=%q want PCIe-only OK", status, detail)
+	}
+}
+
+func TestValidateNCCLAllReduceOutputUnknownFormatIsUnsupported(t *testing.T) {
+	status, _ := validateNCCLAllReduceOutput(t.TempDir(), []byte("all done\n"), []int{0})
+	if status != "UNSUPPORTED" {
+		t.Fatalf("status=%q want UNSUPPORTED", status)
+	}
+}
+
+func TestRunAcceptancePackAppliesOutputValidator(t *testing.T) {
+	runDir, err := runAcceptancePackCtx(context.Background(), t.TempDir(), "validator-test", []satJob{{
+		name: "01-result.log",
+		cmd:  []string{"sh", "-c", "printf command-ok"},
+		validate: func(string, []byte) (string, string) {
+			return "FAILED", "documented output check failed"
+		},
+	}}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(summary), "overall_status=FAILED") ||
+		!strings.Contains(string(summary), "1-result_detail=documented output check failed") {
+		t.Fatalf("summary did not retain validator failure:\n%s", summary)
+	}
+	logData, err := os.ReadFile(filepath.Join(runDir, "01-result.log"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(logData), "[bee-validator] FAILED: documented output check failed") {
+		t.Fatalf("job log did not retain validator result:\n%s", logData)
+	}
+}
+
+func TestNCCLJobHealthAndDebugConfiguration(t *testing.T) {
+	jobs := ncclSATJobs([]int{0, 2})
+	var job satJob
+	for _, candidate := range jobs {
+		if candidate.name == "02-all-reduce-perf.log" {
+			job = candidate
+			break
+		}
+	}
+	if job.name == "" || job.validate == nil {
+		t.Fatal("all_reduce_perf job or its output validator is missing")
+	}
+	if !nvidiaJobNeedsHealthCheck(job) {
+		t.Fatal("all_reduce_perf job must run NVIDIA health checks")
+	}
+	if !isNvidiaAcceptancePack("nccl-tests") {
+		t.Fatal("nccl-tests must produce NVIDIA per-GPU status files")
+	}
+	env := strings.Join(job.env, "\n")
+	for _, expected := range []string{
+		"CUDA_DEVICE_ORDER=PCI_BUS_ID",
+		"CUDA_VISIBLE_DEVICES=0,2",
+		"NCCL_DEBUG=INFO",
+		"NCCL_DEBUG_SUBSYS=INIT,GRAPH",
+	} {
+		if !strings.Contains(env, expected) {
+			t.Fatalf("job env=%q missing %q", env, expected)
+		}
+	}
+}
+
+func writeNCCLValidationArtifact(t *testing.T, dir, name, body string) {
+	t.Helper()
+	if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0644); err != nil {
+		t.Fatal(err)
+	}
+}

audit/internal/platform/sat_nvidia_health.go

@@ -0,0 +1,126 @@
+package platform
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+func nvidiaSATStatusSeverity(status string) int {
+	switch strings.ToUpper(strings.TrimSpace(status)) {
+	case "FAILED":
+		return 3
+	case "PARTIAL", "UNSUPPORTED":
+		return 2
+	case "OK":
+		return 1
+	default:
+		return 0
+	}
+}
+
+func firstLine(s string) string {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return ""
+	}
+	if idx := strings.IndexByte(s, '\n'); idx >= 0 {
+		return strings.TrimSpace(s[:idx])
+	}
+	return s
+}
+
+func nvidiaJobNeedsHealthCheck(job satJob) bool {
+	if job.collectGPU {
+		return true
+	}
+	name := strings.ToLower(strings.TrimSpace(job.name))
+	return strings.Contains(name, "dcgmi") ||
+		strings.Contains(name, "all-reduce") ||
+		strings.Contains(name, "gpu-burn") ||
+		strings.Contains(name, "gpu-stress") ||
+		strings.Contains(name, "dcgmproftester")
+}
+
+func checkNvidiaJobHealth(selected []int) (string, error) {
+	health, _ := readNvidiaGPUHealth()
+	var bad []nvidiaGPUHealth
+	selectedSet := make(map[int]struct{}, len(selected))
+	for _, idx := range selected {
+		selectedSet[idx] = struct{}{}
+	}
+	for _, gpu := range health {
+		if len(selectedSet) > 0 {
+			if _, ok := selectedSet[gpu.Index]; !ok {
+				continue
+			}
+		}
+		if gpu.NeedsReset {
+			bad = append(bad, gpu)
+		}
+	}
+	var degraded []nvidiaNVLinkDegradedEvent
+	if out, err := satExecCommand("dmesg").Output(); err == nil {
+		for _, event := range parseNvidiaNVLinkDegradedDmesg(out) {
+			if event.GPUIndex >= 0 && len(selectedSet) > 0 {
+				if _, ok := selectedSet[event.GPUIndex]; !ok {
+					continue
+				}
+			}
+			degraded = append(degraded, event)
+		}
+	}
+	if len(bad) == 0 && len(degraded) == 0 {
+		return "", nil
+	}
+	lines := make([]string, 0, len(bad)+len(degraded)+1)
+	lines = append(lines, "NVIDIA GPU health check failed:")
+	for _, gpu := range bad {
+		lines = append(lines, fmt.Sprintf("gpu %d (%s) requires reset: %s", gpu.Index, gpu.Name, gpu.RawLine))
+	}
+	for _, event := range degraded {
+		lines = append(lines, event.Warning())
+	}
+	return strings.Join(lines, "\n"), errors.New("nvidia gpu health check failed")
+}
+
+func readNvidiaGPUHealth() ([]nvidiaGPUHealth, error) {
+	out, err := satExecCommand(
+		"nvidia-smi",
+		"--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total",
+		"--format=csv,noheader,nounits",
+	).Output()
+	if err != nil {
+		return nil, fmt.Errorf("nvidia-smi: %w", err)
+	}
+	return parseNvidiaGPUHealth(string(out)), nil
+}
+
+func parseNvidiaGPUHealth(raw string) []nvidiaGPUHealth {
+	var gpus []nvidiaGPUHealth
+	for _, line := range strings.Split(strings.TrimSpace(raw), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		parts := strings.Split(line, ",")
+		if len(parts) < 2 {
+			gpus = append(gpus, nvidiaGPUHealth{RawLine: line, ParseFailure: true})
+			continue
+		}
+		idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
+		if err != nil {
+			gpus = append(gpus, nvidiaGPUHealth{RawLine: line, ParseFailure: true})
+			continue
+		}
+		upper := strings.ToUpper(line)
+		gpus = append(gpus, nvidiaGPUHealth{
+			Index:      idx,
+			Name:       strings.TrimSpace(parts[1]),
+			NeedsReset: strings.Contains(upper, "GPU REQUIRES RESET"),
+			RawLine:    line,
+		})
+	}
+	return gpus
+}

audit/internal/platform/sat_test.go

@@ -273,6 +273,32 @@ func TestCheckNvidiaJobHealthReturnsErrorForSelectedResetRequiredGPU(t *testing.
 	}
 }
 
+func TestCheckNvidiaJobHealthReturnsErrorForSelectedNVLinkDegradedGPU(t *testing.T) {
+	oldExecCommand := satExecCommand
+	satExecCommand = func(name string, args ...string) *exec.Cmd {
+		switch name {
+		case "nvidia-smi":
+			return exec.Command("sh", "-c", "printf '0, NVIDIA H100 PCIe, 38, 46.89, 0, 0, 81559\n1, NVIDIA H100 PCIe, 39, 47.00, 0, 0, 81559\n'")
+		case "dmesg":
+			return exec.Command("sh", "-c", "printf 'NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n'")
+		default:
+			return exec.Command(name, args...)
+		}
+	}
+	t.Cleanup(func() { satExecCommand = oldExecCommand })
+
+	msg, err := checkNvidiaJobHealth([]int{1})
+	if err == nil {
+		t.Fatal("expected NVLink degraded health check error")
+	}
+	if !strings.Contains(msg, "GPU1 NVLink degraded mode (linkId 1)") {
+		t.Fatalf("unexpected message: %q", msg)
+	}
+	if msg, err := checkNvidiaJobHealth([]int{0}); err != nil || msg != "" {
+		t.Fatalf("unaffected selected GPU0 got msg=%q err=%v", msg, err)
+	}
+}
+
 func TestWriteNvidiaGPUStatusFilesCreatesPerGPUFiles(t *testing.T) {
 	dir := t.TempDir()
 	oldExecCommand := satExecCommand