bee/audit/internal/platform/benchmark_power_calibration.go

package platform

import (
	"bee/audit/internal/collector"
	"context"
	"fmt"
	"math"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"strings"
	"time"
)

func detectDCMIPartialCoverage(sp *BenchmarkServerPower) bool {
	if sp == nil || !sp.Available {
		return false
	}
	if sp.PSUInputIdleW > 0 && sp.IdleW > 0 {
		return sp.IdleW/sp.PSUInputIdleW < 0.7
	}
	if sp.PSUInputLoadedW > 0 && sp.LoadedW > 0 {
		return sp.LoadedW/sp.PSUInputLoadedW < 0.7
	}
	return false
}

// detectIPMISaturationFallback is the heuristic used when SDR PSU sensors are
// absent. It analyses the power ramp: if 2+ of the last 3 per-step incremental
// DCMI/GPU ratios fall below 25% of the first-step ratio, DCMI has likely
// plateaued while GPU load continued to grow (saturation proxy).
// Prefer detectDCMIPartialCoverage when SDR data is available.
func detectIPMISaturationFallback(steps []NvidiaPowerBenchStep) bool {
	type pt struct{ incIPMI, incGPU float64 }
	var pts []pt
	for i := 1; i < len(steps); i++ {
		if steps[i].ServerDeltaW <= 0 || steps[i-1].ServerDeltaW <= 0 {
			continue
		}
		incIPMI := steps[i].ServerDeltaW - steps[i-1].ServerDeltaW
		incGPU := steps[i].TotalObservedPowerW - steps[i-1].TotalObservedPowerW
		if incGPU <= 0 {
			continue
		}
		pts = append(pts, pt{incIPMI, incGPU})
	}
	if len(pts) < 3 {
		return false
	}
	refRatio := pts[0].incIPMI / pts[0].incGPU
	if refRatio <= 0 {
		return false
	}
	saturated := 0
	for _, p := range pts[len(pts)-3:] {
		if p.incIPMI/p.incGPU < refRatio*0.25 {
			saturated++
		}
	}
	return saturated >= 2
}

// psuStatusSnapshot samples PSU health sensor states via
// `ipmitool sdr type "Power Supply"`. Returns a map of sensor name → reading
// string (e.g. "Presence detected", "Failure detected"). Returns nil when IPMI
// is unavailable or no Power Supply entity sensors are present.
func psuStatusSnapshot() map[string]string {
	out, err := exec.Command("ipmitool", "sdr", "type", "Power Supply").Output()
	if err != nil || len(out) == 0 {
		return nil
	}
	result := make(map[string]string)
	for _, line := range strings.Split(string(out), "\n") {
		parts := strings.Split(line, "|")
		if len(parts) < 5 {
			continue
		}
		name := strings.TrimSpace(parts[0])
		reading := strings.TrimSpace(parts[4])
		if name == "" {
			continue
		}
		result[name] = reading
	}
	return result
}

// diffPSUStatus compares PSU sensor snapshots taken before and after a test.
// Returns human-readable fault strings for sensors that entered a fault state
// during the test. Pre-existing faults (present in both snapshots) are excluded
// so that only new anomalies caused by the test are reported.
func diffPSUStatus(before, after map[string]string) []string {
	if len(after) == 0 {
		return nil
	}
	isFault := func(s string) bool {
		lower := strings.ToLower(s)
		return strings.Contains(lower, "failure") ||
			strings.Contains(lower, "fault") ||
			strings.Contains(lower, "warning") ||
			strings.Contains(lower, "predictive") ||
			strings.Contains(lower, "absent") ||
			strings.Contains(lower, "ac lost")
	}
	var issues []string
	for name, afterReading := range after {
		if !isFault(afterReading) {
			continue
		}
		if beforeReading, had := before[name]; had && isFault(beforeReading) {
			continue // pre-existing fault, not caused by this test
		}
		if prev, had := before[name]; had {
			issues = append(issues, fmt.Sprintf("%s: changed from %q to %q during test", name, prev, afterReading))
		} else {
			issues = append(issues, fmt.Sprintf("%s: %s (appeared after test start)", name, afterReading))
		}
	}
	sort.Strings(issues)
	return issues
}

// sdrPowerSnapshot holds per-source power sums from a single `ipmitool sdr` read.
type sdrPowerSnapshot struct {
	PSUInW   float64 // sum of PSU AC input across all slots
	PSUOutW  float64 // sum of PSU DC output across all slots
	GPUSlotW float64 // sum of GPU slot/GPU power sensors

	// Per-slot PSU data from collector.PSUSlotsFromSDR — same slot keys as
	// audit HardwarePowerSupply.Slot (0-based strings).
	PSUSlots map[string]BenchmarkPSUSlotPower

	SkippedSensors []string // sensors rejected during self-healing
}

type benchmarkSDRSeriesSummary struct {
	PSUInW   float64
	PSUOutW  float64
	GPUSlotW float64
	PSUSlots map[string]BenchmarkPSUSlotPower
	Samples  int

	SkippedSensors []string
}

// sdrSensor is a name+watts pair used for GPU slot self-healing filtering.
type sdrSensor struct {
	name  string
	watts float64
}

// filterSensorGroup removes physically implausible readings from a group.
// Hard bounds: 0 < watts ≤ maxPerSensorW. Within groups of 2+ sensors,
// values more than 5× the group median are rejected as stuck/fault sensors.
func filterSensorGroup(sensors []sdrSensor, maxPerSensorW float64) (valid []sdrSensor, skipped []string) {
	var inBounds []sdrSensor
	for _, s := range sensors {
		if s.watts <= 0 || s.watts > maxPerSensorW {
			skipped = append(skipped, fmt.Sprintf("%s (%.0f W: out of range 0–%.0f W)", s.name, s.watts, maxPerSensorW))
		} else {
			inBounds = append(inBounds, s)
		}
	}
	if len(inBounds) < 2 {
		return inBounds, skipped
	}
	vals := make([]float64, len(inBounds))
	for i, s := range inBounds {
		vals[i] = s.watts
	}
	sort.Float64s(vals)
	mid := len(vals) / 2
	var median float64
	if len(vals)%2 == 0 {
		median = (vals[mid-1] + vals[mid]) / 2
	} else {
		median = vals[mid]
	}
	for _, s := range inBounds {
		if median > 0 && s.watts > median*5 {
			skipped = append(skipped, fmt.Sprintf("%s (%.0f W: >5× median %.0f W, likely sensor fault)", s.name, s.watts, median))
		} else {
			valid = append(valid, s)
		}
	}
	return valid, skipped
}

// sampleIPMISDRPowerSensors reads power sensors from `ipmitool sdr` in a single
// invocation and returns self-healed grouped sums.
//
// PSU identification delegates to collector.PSUSlotsFromSDR which uses the same
// slot-detection regexes as the hardware audit (PSU1_POWER_IN, PSU1_PIN, PS1 POut,
// Power1…). Self-healing: bounds checking + 5× median outlier rejection.
//
// GPU slot sensors (GPU_POWER_SLOTx, GPU1 Power, …) are classified separately
// since the audit collector does not track GPU PCIe slot power.
func sampleIPMISDRPowerSensors() sdrPowerSnapshot {
	// Reads from the shared hardware telemetry cache (hwSnapshot) rather than
	// shelling out to `ipmitool sdr` itself — see hw_telemetry.go for why.
	sdrStr := hwSnapshot().Raw
	if sdrStr == "" {
		return sdrPowerSnapshot{}
	}
	var snap sdrPowerSnapshot

	// ── PSU data via audit collector ─────────────────────────────────────────
	// collector.PSUSlotsFromSDR handles all vendor naming variants and applies
	// bounds checking inside parseBoundedFloat (0 < w ≤ 6000 W).
	collectorSlots := collector.PSUSlotsFromSDR(sdrStr)

	// Convert to benchmark type and apply cross-slot median filtering.
	var psuInSensors, psuOutSensors []sdrSensor
	for slotKey, sp := range collectorSlots {
		bsp := BenchmarkPSUSlotPower{Status: sp.Status}
		if sp.InputW != nil {
			bsp.InputW = sp.InputW
			psuInSensors = append(psuInSensors, sdrSensor{name: "PSU-slot-" + slotKey, watts: *sp.InputW})
		}
		if sp.OutputW != nil {
			bsp.OutputW = sp.OutputW
			psuOutSensors = append(psuOutSensors, sdrSensor{name: "PSU-slot-" + slotKey + "-out", watts: *sp.OutputW})
		}
		if snap.PSUSlots == nil {
			snap.PSUSlots = make(map[string]BenchmarkPSUSlotPower)
		}
		snap.PSUSlots[slotKey] = bsp
	}

	// Apply cross-slot outlier filter and sum.
	validIn, skIn := filterSensorGroup(psuInSensors, 6000)
	for _, s := range validIn {
		snap.PSUInW += s.watts
	}
	snap.SkippedSensors = append(snap.SkippedSensors, skIn...)

	validOut, skOut := filterSensorGroup(psuOutSensors, 6000)
	for _, s := range validOut {
		snap.PSUOutW += s.watts
	}
	snap.SkippedSensors = append(snap.SkippedSensors, skOut...)

	// ── GPU slot sensors ─────────────────────────────────────────────────────
	// collector does not track GPU PCIe slot power; classify here.
	// Matches: GPU_POWER_SLOTx (MSI), GPU1 Power (xFusion), GPU_PWR_x (generic).
	var gpuSensors []sdrSensor
	for _, line := range strings.Split(sdrStr, "\n") {
		parts := strings.Split(line, "|")
		if len(parts) < 2 {
			continue
		}
		name := strings.TrimSpace(parts[0])
		nameLower := strings.ToLower(name)
		if !strings.Contains(nameLower, "gpu") {
			continue
		}
		if !strings.Contains(nameLower, "slot") && !strings.Contains(nameLower, "power") &&
			!strings.Contains(nameLower, "pwr") {
			continue
		}
		var w float64
		if n, _ := fmt.Sscanf(strings.TrimSpace(parts[1]), "%f Watts", &w); n != 1 {
			continue
		}
		gpuSensors = append(gpuSensors, sdrSensor{name: name, watts: w})
	}
	validGPU, skGPU := filterSensorGroup(gpuSensors, 2000)
	for _, s := range validGPU {
		snap.GPUSlotW += s.watts
	}
	snap.SkippedSensors = append(snap.SkippedSensors, skGPU...)

	return snap
}

func startIPMISDRSampler(stopCh <-chan struct{}, intervalSec int) <-chan []sdrPowerSnapshot {
	if intervalSec <= 0 {
		intervalSec = benchmarkPowerAutotuneSampleInterval
	}
	ch := make(chan []sdrPowerSnapshot, 1)
	go func() {
		defer close(ch)
		var samples []sdrPowerSnapshot
		record := func() {
			snap := sampleIPMISDRPowerSensors()
			if snap.PSUInW <= 0 && snap.PSUOutW <= 0 && snap.GPUSlotW <= 0 && len(snap.PSUSlots) == 0 {
				return
			}
			samples = append(samples, snap)
		}
		record()
		ticker := time.NewTicker(time.Duration(intervalSec) * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-stopCh:
				ch <- samples
				return
			case <-ticker.C:
				record()
			}
		}
	}()
	return ch
}

func summarizeSDRPowerSeries(samples []sdrPowerSnapshot) benchmarkSDRSeriesSummary {
	var summary benchmarkSDRSeriesSummary
	if len(samples) == 0 {
		return summary
	}

	type slotAggregate struct {
		inputs  []float64
		outputs []float64
		status  string
	}

	slotAgg := make(map[string]*slotAggregate)
	skippedSet := make(map[string]struct{})
	var inputTotals []float64
	var outputTotals []float64
	var gpuSlotTotals []float64

	for _, sample := range samples {
		if sample.PSUInW > 0 {
			inputTotals = append(inputTotals, sample.PSUInW)
		}
		if sample.PSUOutW > 0 {
			outputTotals = append(outputTotals, sample.PSUOutW)
		}
		if sample.GPUSlotW > 0 {
			gpuSlotTotals = append(gpuSlotTotals, sample.GPUSlotW)
		}
		for _, skipped := range sample.SkippedSensors {
			if skipped != "" {
				skippedSet[skipped] = struct{}{}
			}
		}
		for slot, reading := range sample.PSUSlots {
			agg := slotAgg[slot]
			if agg == nil {
				agg = &slotAggregate{}
				slotAgg[slot] = agg
			}
			if reading.InputW != nil && *reading.InputW > 0 {
				agg.inputs = append(agg.inputs, *reading.InputW)
			}
			if reading.OutputW != nil && *reading.OutputW > 0 {
				agg.outputs = append(agg.outputs, *reading.OutputW)
			}
			switch {
			case reading.Status == "":
			case agg.status == "":
				agg.status = reading.Status
			case agg.status == "OK" && reading.Status != "OK":
				agg.status = reading.Status
			}
		}
	}

	summary.PSUInW = benchmarkMean(inputTotals)
	summary.PSUOutW = benchmarkMean(outputTotals)
	summary.GPUSlotW = benchmarkMean(gpuSlotTotals)
	summary.Samples = len(samples)

	if len(slotAgg) > 0 {
		summary.PSUSlots = make(map[string]BenchmarkPSUSlotPower, len(slotAgg))
		for slot, agg := range slotAgg {
			reading := BenchmarkPSUSlotPower{Status: agg.status}
			if mean := benchmarkMean(agg.inputs); mean > 0 {
				v := mean
				reading.InputW = &v
			}
			if mean := benchmarkMean(agg.outputs); mean > 0 {
				v := mean
				reading.OutputW = &v
			}
			summary.PSUSlots[slot] = reading
		}
	}
	if len(skippedSet) > 0 {
		summary.SkippedSensors = make([]string, 0, len(skippedSet))
		for skipped := range skippedSet {
			summary.SkippedSensors = append(summary.SkippedSensors, skipped)
		}
		sort.Strings(summary.SkippedSensors)
	}

	return summary
}

// queryIPMIServerPowerW reads the current server power draw from the shared
// hardware telemetry cache (hwSnapshot), which runs `ipmitool dcmi power
// reading` itself on its own cadence — see hw_telemetry.go for why this
// doesn't shell out directly. Returns 0 and an error if IPMI dcmi power is
// unavailable on this BMC.
func queryIPMIServerPowerW() (float64, error) {
	if w := hwSnapshot().DCMIPowerW; w > 0 {
		return w, nil
	}
	return 0, fmt.Errorf("ipmitool dcmi power reading unavailable")
}

// characterizeServerPower computes BenchmarkServerPower from idle and loaded
// samples plus the GPU-reported average power during steady state.
func characterizeServerPower(idleW, loadedW, gpuReportedSumW float64, source string, available bool) *BenchmarkServerPower {
	sp := &BenchmarkServerPower{
		Available:         available,
		Source:            normalizeBenchmarkPowerSource(source),
		SampleIntervalSec: benchmarkPowerAutotuneSampleInterval,
	}
	if !available {
		sp.Notes = append(sp.Notes, "IPMI power reading unavailable; server-side power characterization skipped")
		return sp
	}
	sp.IdleW = idleW
	sp.LoadedW = loadedW
	sp.DeltaW = loadedW - idleW
	sp.GPUReportedSumW = gpuReportedSumW
	if gpuReportedSumW > 0 && sp.DeltaW > 0 {
		sp.ReportingRatio = sp.DeltaW / gpuReportedSumW
	}
	return sp
}

// readServerModel returns the DMI system product name (e.g. "SuperMicro SYS-421GE-TNRT").
// Returns empty string if unavailable (non-Linux or missing DMI entry).
func readServerModel() string {
	data, err := os.ReadFile("/sys/class/dmi/id/product_name")
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(data))
}

// filterRowsByGPU returns only the metric rows for a specific GPU index.
func filterRowsByGPU(rows []GPUMetricRow, gpuIndex int) []GPUMetricRow {
	var out []GPUMetricRow
	for _, r := range rows {
		if r.GPUIndex == gpuIndex {
			out = append(out, r)
		}
	}
	return out
}

// parseBenchmarkBurnLogByGPU splits a multi-GPU bee-gpu-burn output by [gpu N] prefix
// and returns a per-GPU parse result map.
func parseBenchmarkBurnLogByGPU(raw string) map[int]benchmarkBurnParseResult {
	gpuLines := make(map[int][]string)
	for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") {
		line = strings.TrimSpace(line)
		if !strings.HasPrefix(line, "[gpu ") {
			continue
		}
		end := strings.Index(line, "] ")
		if end < 0 {
			continue
		}
		gpuIdx, err := strconv.Atoi(strings.TrimSpace(line[5:end]))
		if err != nil {
			continue
		}
		gpuLines[gpuIdx] = append(gpuLines[gpuIdx], line[end+2:])
	}
	results := make(map[int]benchmarkBurnParseResult, len(gpuLines))
	for gpuIdx, lines := range gpuLines {
		// Lines are already stripped of the [gpu N] prefix; parseBenchmarkBurnLog
		// calls stripBenchmarkPrefix which is a no-op on already-stripped lines.
		results[gpuIdx] = parseBenchmarkBurnLog(strings.Join(lines, "\n"))
	}
	return results
}

// runNvidiaBenchmarkParallel runs warmup and steady compute on all selected GPUs
// simultaneously using a single bee-gpu-burn invocation per phase.
func runNvidiaBenchmarkParallel(
	ctx context.Context,
	verboseLog, runDir string,
	selected []int,
	infoByIndex map[int]benchmarkGPUInfo,
	opts NvidiaBenchmarkOptions,
	spec benchmarkProfileSpec,
	logFunc func(string),
	result *NvidiaBenchmarkResult,
	calibByIndex map[int]benchmarkPowerCalibrationResult,
	serverIdleW *float64, serverLoadedWSum *float64,
	serverIdleOK *bool, serverLoadedOK *bool, serverLoadedSamples *int,
	allMetricRows *[]GPUMetricRow,
	metricTimelineSec *float64,
	gpuBurnLog string,
) {
	allDevices := joinIndexList(selected)

	// Build per-GPU result stubs.
	gpuResults := make(map[int]*BenchmarkGPUResult, len(selected))
	for _, idx := range selected {
		r := &BenchmarkGPUResult{Index: idx, Status: "FAILED"}
		if info, ok := infoByIndex[idx]; ok {
			populateBenchmarkGPUInfo(r, info)
		}
		if calib, ok := calibByIndex[idx]; ok {
			r.CalibratedPeakPowerW = calib.Summary.P95PowerW
			r.CalibratedPeakTempC = calib.Summary.P95TempC
			r.PowerCalibrationTries = calib.Attempts
			r.PowerLimitDerated = calib.Derated
			r.Notes = append(r.Notes, calib.Notes...)
			if calib.CoolingWarning != "" {
				r.CoolingWarning = calib.CoolingWarning
			}
		}
		if norm := findBenchmarkNormalization(result.Normalization.GPUs, idx); norm != nil {
			r.LockedGraphicsClockMHz = norm.GPUClockLockMHz
			r.LockedMemoryClockMHz = norm.MemoryClockLockMHz
		}
		gpuResults[idx] = r
	}

	// Baseline: sample all GPUs together.
	baselineRows, err := collectBenchmarkSamples(ctx, spec.BaselineSec, selected)
	if err != nil && err != context.Canceled {
		for _, idx := range selected {
			gpuResults[idx].Notes = append(gpuResults[idx].Notes, "baseline sampling failed: "+err.Error())
		}
	}
	for _, idx := range selected {
		perGPU := filterRowsByGPU(baselineRows, idx)
		gpuResults[idx].Baseline = summarizeBenchmarkTelemetry(perGPU)
	}
	appendBenchmarkMetrics(allMetricRows, baselineRows, "baseline", metricTimelineSec, float64(spec.BaselineSec))

	// Sample server idle power once.
	if !*serverIdleOK {
		if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, maxInt(spec.BaselineSec, 10), benchmarkPowerAutotuneSampleInterval); ok {
			*serverIdleW = w
			*serverIdleOK = true
			logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w))
		}
	}

	// Warmup: all GPUs simultaneously.
	warmupCmd := []string{
		"bee-gpu-burn",
		"--seconds", strconv.Itoa(spec.WarmupSec),
		"--size-mb", strconv.Itoa(opts.SizeMB),
		"--devices", allDevices,
	}
	logFunc(fmt.Sprintf("GPUs %s: parallel warmup (%ds)", allDevices, spec.WarmupSec))
	warmupOut, warmupRows, warmupErr := runBenchmarkCommandWithMetrics(ctx, verboseLog, "gpu-all-warmup.log", warmupCmd, nil, selected, logFunc)
	appendBenchmarkMetrics(allMetricRows, warmupRows, "warmup", metricTimelineSec, float64(spec.WarmupSec))
	appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", "warmup", warmupOut)
	if warmupErr != nil {
		for _, idx := range selected {
			gpuResults[idx].Notes = append(gpuResults[idx].Notes, "parallel warmup failed: "+warmupErr.Error())
		}
	}
	warmupParseByGPU := parseBenchmarkBurnLogByGPU(string(warmupOut))
	supportedPrecisions := append([]string(nil), benchmarkPrecisionPhases...)
	for _, idx := range selected {
		if pr, ok := warmupParseByGPU[idx]; ok && pr.ComputeCapability != "" {
			if gpuResults[idx].ComputeCapability == "" {
				gpuResults[idx].ComputeCapability = pr.ComputeCapability
			}
			if ccPrecisions := benchmarkSupportedPrecisions(pr.ComputeCapability); len(ccPrecisions) < len(supportedPrecisions) {
				supportedPrecisions = ccPrecisions
			}
		}
	}

	// Run synthetic precision phases and the combined steady phase as one
	// uninterrupted command so the GPUs stay hot between windows.
	eccBase := make(map[int]BenchmarkECCCounters, len(selected))
	for _, idx := range selected {
		eccBase[idx], _ = queryECCCounters(idx)
	}
	planLabels, planPhases, basePhaseSec, mixedPhaseSec := buildBenchmarkSteadyPlan(spec, supportedPrecisions, func(label string) string {
		if label == "mixed" {
			return "steady"
		}
		return "gpu-all-steady-" + label
	})
	planCmd := []string{
		"bee-gpu-burn",
		"--seconds", strconv.Itoa(basePhaseSec),
		"--size-mb", strconv.Itoa(opts.SizeMB),
		"--devices", allDevices,
		"--precision-plan", strings.Join(planLabels, ","),
		"--precision-plan-seconds", benchmarkPlanDurationsCSV(planPhases),
	}
	logFunc(fmt.Sprintf("GPUs %s: uninterrupted precision plan (%d precision phases x %ds, mixed %ds)", allDevices, len(supportedPrecisions), basePhaseSec, mixedPhaseSec))
	serverPowerStopCh := make(chan struct{})
	serverPowerCh := startSelectedPowerSourceSampler(serverPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
	_, phaseRowsByStage, phaseLogs, planErr := runBenchmarkPlannedCommandWithMetrics(ctx, verboseLog, "gpu-all-precision-plan.log", planCmd, nil, selected, planPhases, logFunc)
	close(serverPowerStopCh)
	if serverPowerSamples := <-serverPowerCh; len(serverPowerSamples) > 0 {
		*serverLoadedWSum += benchmarkMean(serverPowerSamples)
		(*serverLoadedSamples)++
		*serverLoadedOK = true
		logFunc(fmt.Sprintf("GPUs %s: server loaded power (%s avg): %.0f W", allDevices, opts.ServerPowerSource, benchmarkMean(serverPowerSamples)))
	}
	for _, phaseSpec := range planPhases {
		if rows := phaseRowsByStage[phaseSpec.MetricStage]; len(rows) > 0 {
			appendBenchmarkMetrics(allMetricRows, rows, phaseSpec.MetricStage, metricTimelineSec, float64(phaseSpec.DurationSec))
		}
		appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", phaseSpec.MetricStage, phaseLogs[phaseSpec.PlanLabel])
	}
	for _, prec := range supportedPrecisions {
		phaseLogName := "gpu-all-steady-" + prec
		phaseRows := phaseRowsByStage[phaseLogName]
		parseByGPU := parseBenchmarkBurnLogByGPU(string(phaseLogs[prec]))
		for _, idx := range selected {
			perGPU := filterRowsByGPU(phaseRows, idx)
			phase := BenchmarkPrecisionSteadyPhase{
				Precision: prec,
				Status:    "OK",
				Steady:    summarizeBenchmarkTelemetry(perGPU),
			}
			if status, note := benchmarkPlannedPhaseStatus(phaseLogs[prec]); status != "OK" {
				phase.Status = status
				phase.Notes = note
				gpuResults[idx].PrecisionFailures = append(gpuResults[idx].PrecisionFailures, prec+":"+status)
			}
			if pr, ok := parseByGPU[idx]; ok {
				for _, p := range pr.Profiles {
					if p.Supported {
						phase.TeraOpsPerSec += p.TeraOpsPerSec
						phase.WeightedTeraOpsPerSec += p.WeightedTeraOpsPerSec
					}
				}
			}
			gpuResults[idx].PrecisionSteady = append(gpuResults[idx].PrecisionSteady, phase)
		}
	}

	// Snapshot throttle counters before steady.
	beforeThrottle := make(map[int]BenchmarkThrottleCounters, len(selected))
	for _, idx := range selected {
		beforeThrottle[idx], _ = queryThrottleCounters(idx)
	}

	logFunc(fmt.Sprintf("GPUs %s: parallel steady compute (combined, %ds)", allDevices, mixedPhaseSec))
	afterThrottle := make(map[int]BenchmarkThrottleCounters, len(selected))
	for _, idx := range selected {
		afterThrottle[idx], _ = queryThrottleCounters(idx)
	}

	steadyRows := phaseRowsByStage["steady"]
	parseResults := parseBenchmarkBurnLogByGPU(string(phaseLogs["mixed"]))

	for _, idx := range selected {
		perGPU := filterRowsByGPU(steadyRows, idx)
		gpuResults[idx].Steady = summarizeBenchmarkTelemetry(perGPU)
		gpuResults[idx].Throttle = diffThrottleCounters(beforeThrottle[idx], afterThrottle[idx])
		if eccFinal, err := queryECCCounters(idx); err == nil {
			gpuResults[idx].ECC = diffECCCounters(eccBase[idx], eccFinal)
		}

		if pr, ok := parseResults[idx]; ok {
			gpuResults[idx].ComputeCapability = pr.ComputeCapability
			gpuResults[idx].Backend = pr.Backend
			gpuResults[idx].PrecisionResults = pr.Profiles
			if pr.Fallback {
				gpuResults[idx].Notes = append(gpuResults[idx].Notes, "benchmark used driver PTX fallback; tensor throughput score is not comparable")
			}
		}
		if planErr != nil {
			gpuResults[idx].Notes = append(gpuResults[idx].Notes, "precision plan failed: "+planErr.Error())
		}
	}

	// Cooldown: all GPUs together.
	if spec.CooldownSec > 0 {
		cooldownRows, err := collectBenchmarkSamples(ctx, spec.CooldownSec, selected)
		if err != nil && err != context.Canceled {
			for _, idx := range selected {
				gpuResults[idx].Notes = append(gpuResults[idx].Notes, "cooldown sampling failed: "+err.Error())
			}
		}
		for _, idx := range selected {
			perGPU := filterRowsByGPU(cooldownRows, idx)
			gpuResults[idx].Cooldown = summarizeBenchmarkTelemetry(perGPU)
		}
		appendBenchmarkMetrics(allMetricRows, cooldownRows, "cooldown", metricTimelineSec, float64(spec.CooldownSec))
	}

	// Score and finalize each GPU.
	for _, idx := range selected {
		r := gpuResults[idx]
		applyBenchmarkSteadyFallback(r)
		r.Scores = scoreBenchmarkGPUResult(*r)
		r.DegradationReasons = detectBenchmarkDegradationReasons(*r, result.Normalization.Status)
		pr := parseResults[idx]
		switch {
		case planErr != nil:
			r.Status = classifySATErrorStatus(phaseLogs["mixed"], planErr)
		case len(r.PrecisionFailures) > 0:
			r.Status = "PARTIAL"
		case pr.Fallback:
			r.Status = "PARTIAL"
		default:
			r.Status = "OK"
		}
		result.GPUs = append(result.GPUs, finalizeBenchmarkGPUResult(*r))
	}
}

func populateBenchmarkGPUInfo(result *BenchmarkGPUResult, info benchmarkGPUInfo) {
	result.UUID = info.UUID
	result.Name = info.Name
	result.BusID = info.BusID
	result.VBIOS = info.VBIOS
	result.PowerLimitW = info.PowerLimitW
	result.MultiprocessorCount = info.MultiprocessorCount
	result.DefaultPowerLimitW = info.DefaultPowerLimitW
	result.ShutdownTempC = info.ShutdownTempC
	result.SlowdownTempC = info.SlowdownTempC
	result.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz
	result.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz
	result.MaxMemoryClockMHz = info.MaxMemoryClockMHz
}

// readBenchmarkHostConfig reads static CPU and memory configuration from
// /proc/cpuinfo and /proc/meminfo. Returns nil if neither source is readable.
func readBenchmarkHostConfig() *BenchmarkHostConfig {
	cfg := &BenchmarkHostConfig{}
	populated := false

	// Parse /proc/cpuinfo for CPU model, sockets, cores, threads.
	if data, err := os.ReadFile("/proc/cpuinfo"); err == nil {
		socketIDs := map[string]struct{}{}
		coresPerSocket := map[string]int{}
		var modelName string
		threads := 0
		for _, line := range strings.Split(string(data), "\n") {
			kv := strings.SplitN(line, ":", 2)
			if len(kv) != 2 {
				continue
			}
			key := strings.TrimSpace(kv[0])
			val := strings.TrimSpace(kv[1])
			switch key {
			case "processor":
				threads++
			case "model name":
				if modelName == "" {
					modelName = val
				}
			case "physical id":
				socketIDs[val] = struct{}{}
			case "cpu cores":
				// Overwrite per-socket core count (last wins per socket, but all
				// entries for the same socket report the same value).
				if physLine := ""; physLine == "" {
					// We accumulate below by treating cpu cores as a per-thread
					// field; sum by socket requires a two-pass approach. Use the
					// simpler approximation: totalCores = threads / (threads per core).
					_ = val
				}
			}
		}
		// Second pass: per-socket core count.
		var curSocket string
		for _, line := range strings.Split(string(data), "\n") {
			kv := strings.SplitN(line, ":", 2)
			if len(kv) != 2 {
				continue
			}
			key := strings.TrimSpace(kv[0])
			val := strings.TrimSpace(kv[1])
			switch key {
			case "physical id":
				curSocket = val
			case "cpu cores":
				if curSocket != "" {
					if _, seen := coresPerSocket[curSocket]; !seen {
						v, _ := strconv.Atoi(val)
						coresPerSocket[curSocket] = v
					}
				}
			}
		}
		totalCores := 0
		for _, c := range coresPerSocket {
			totalCores += c
		}
		cfg.CPUModel = modelName
		cfg.CPUSockets = len(socketIDs)
		if cfg.CPUSockets == 0 && threads > 0 {
			cfg.CPUSockets = 1
		}
		cfg.CPUCores = totalCores
		cfg.CPUThreads = threads
		if modelName != "" || threads > 0 {
			populated = true
		}
	}

	// Parse /proc/meminfo for total physical RAM.
	if data, err := os.ReadFile("/proc/meminfo"); err == nil {
		for _, line := range strings.Split(string(data), "\n") {
			if strings.HasPrefix(line, "MemTotal:") {
				fields := strings.Fields(line)
				if len(fields) >= 2 {
					kb, _ := strconv.ParseUint(fields[1], 10, 64)
					cfg.MemTotalGiB = float64(kb) / (1024 * 1024)
					populated = true
				}
				break
			}
		}
	}

	if !populated {
		return nil
	}
	return cfg
}

// startCPULoadSampler starts a goroutine that samples host CPU load every
// intervalSec seconds until stopCh is closed, then sends the collected
// samples on the returned channel.
func startCPULoadSampler(stopCh <-chan struct{}, intervalSec int) <-chan []float64 {
	ch := make(chan []float64, 1)
	go func() {
		var samples []float64
		ticker := time.NewTicker(time.Duration(intervalSec) * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-stopCh:
				ch <- samples
				return
			case <-ticker.C:
				if pct := sampleCPULoadPct(); pct > 0 {
					samples = append(samples, pct)
				}
			}
		}
	}()
	return ch
}

// summarizeCPULoad computes stats over sampled CPU load values and assigns
// a health status.
func summarizeCPULoad(samples []float64) *BenchmarkCPULoad {
	if len(samples) == 0 {
		return nil
	}
	sorted := append([]float64(nil), samples...)
	sort.Float64s(sorted)
	var sum float64
	for _, v := range sorted {
		sum += v
	}
	avg := sum / float64(len(sorted))
	p95 := sorted[int(float64(len(sorted))*0.95)]
	max := sorted[len(sorted)-1]

	cl := &BenchmarkCPULoad{
		AvgPct:  math.Round(avg*10) / 10,
		MaxPct:  math.Round(max*10) / 10,
		P95Pct:  math.Round(p95*10) / 10,
		Samples: len(sorted),
	}

	// Compute standard deviation to detect instability.
	var variance float64
	for _, v := range sorted {
		d := v - avg
		variance += d * d
	}
	stdDev := math.Sqrt(variance / float64(len(sorted)))

	switch {
	case avg > 20 || max > 40:
		cl.Status = "high"
		cl.Note = fmt.Sprintf("avg %.1f%% max %.1f%% — elevated host CPU load may interfere with GPU benchmark results", avg, max)
	case stdDev > 12:
		cl.Status = "unstable"
		cl.Note = fmt.Sprintf("avg %.1f%% stddev %.1f%% — host CPU load was erratic during the benchmark", avg, stdDev)
	default:
		cl.Status = "ok"
	}
	return cl
}

// runBenchmarkPowerCalibration runs the configured power-fit load for the supplied
// GPU set and actively watches throttle counters. seedLimits, when provided, are treated as
// the starting point for this calibration pass rather than as immutable fixed
// limits. This matters during cumulative ramp-up: once an additional GPU is
// introduced, every already-active GPU must be revalidated under the new
// thermal state instead of assuming its previous single-step limit is still
// valid. The selected reduced power limits stay active for the main benchmark
// and are restored by the caller afterwards.