package platform

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const benchmarkVersion = "2"

type benchmarkProfileSpec struct {
	Name        string
	BaselineSec int
	WarmupSec   int
	SteadySec   int
	NCCLSec     int
	CooldownSec int
}

type benchmarkGPUInfo struct {
	Index                int
	UUID                 string
	Name                 string
	BusID                string
	VBIOS                string
	PowerLimitW          float64
	DefaultPowerLimitW   float64
	MinPowerLimitW       float64
	MaxPowerLimitW       float64
	MaxGraphicsClockMHz  float64
	MaxMemoryClockMHz    float64
	BaseGraphicsClockMHz float64
	MultiprocessorCount  int
	// Temperature limits sourced from nvidia-smi -q verbose output.
	// ShutdownTempC is the hardware thermal shutdown threshold.
	// SlowdownTempC is the software throttle onset threshold.
	// Both fall back to safe conservative defaults when not available.
	ShutdownTempC float64 // fallback: 90°C
	SlowdownTempC float64 // fallback: 80°C
}

type benchmarkPowerCalibrationResult struct {
	Summary            BenchmarkTelemetrySummary
	AppliedPowerLimitW float64
	Attempts           int
	Derated            bool
	Completed          bool
	Notes              []string
	// CoolingWarning is set when the GPU throttled thermally with a clock drop
	// ≥20% while server fans were below 100% duty cycle — a signal that the
	// cooling system may not be correctly configured for full GPU load.
	CoolingWarning string
	// MetricRows holds the telemetry rows from the final (converged) attempt
	// for this GPU. Used to build per-run gpu-metrics.csv.
	MetricRows []GPUMetricRow
}

type benchmarkPowerCalibrationRunSummary struct {
	LoadedSDR          benchmarkSDRSeriesSummary
	AvgFanRPM          float64
	AvgFanDutyCyclePct float64
	FanSamples         int
}

type benchmarkBurnProfile struct {
	name       string
	category   string
	supported  bool
	lanes      int
	m          uint64
	n          uint64
	k          uint64
	iterations uint64
	notes      string
}

type benchmarkBurnParseResult struct {
	Device            string
	ComputeCapability string
	Backend           string
	DurationSec       int
	Profiles          []BenchmarkPrecisionResult
	Fallback          bool
}

type benchmarkRestoreAction struct {
	name string
	fn   func()
}

var (
	benchmarkReadyPattern      = regexp.MustCompile(`^([a-z0-9_]+)\[(\d+)\]=READY dim=(\d+)x(\d+)x(\d+)\b`)
	benchmarkSkippedPattern    = regexp.MustCompile(`^([a-z0-9_]+)(?:\[\d+\])?=SKIPPED (.+)$`)
	benchmarkIterationsPattern = regexp.MustCompile(`^([a-z0-9_]+)_iterations=(\d+)$`)
	benchmarkGeteuid           = os.Geteuid
	benchmarkResetNvidiaGPU    = resetNvidiaGPU
	benchmarkSleep             = time.Sleep
)

// benchmarkPrecisionPhases lists the precision categories run as individual
// steady-state windows before the combined steady pass. Order is from lowest
// to highest power draw so thermal ramp-up is gradual.
//
// fp64 and fp4 are intentionally disabled for now: both are currently unstable
// on the target fleet and can abort the mixed steady stage after the earlier
// phases already collected useful telemetry.
var benchmarkPrecisionPhases = []string{"int8", "fp8", "fp16", "fp32"}

func computeCapabilityCode(raw string) int {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return 0
	}
	parts := strings.SplitN(raw, ".", 2)
	major, _ := strconv.Atoi(strings.TrimSpace(parts[0]))
	minor := 0
	if len(parts) > 1 {
		minor, _ = strconv.Atoi(strings.TrimSpace(parts[1]))
	}
	return major*10 + minor
}

func benchmarkSupportedPrecisions(computeCapability string) []string {
	cc := computeCapabilityCode(computeCapability)
	out := make([]string, 0, len(benchmarkPrecisionPhases))
	for _, prec := range benchmarkPrecisionPhases {
		if prec == "fp4" && cc > 0 && cc < 100 {
			continue
		}
		out = append(out, prec)
	}
	return out
}

func benchmarkPrecisionEnabled(category string) bool {
	switch category {
	case "int8", "fp8", "fp16", "fp16_bf16", "fp32", "fp32_tf32":
		return true
	default:
		return false
	}
}

func buildBenchmarkSteadyPlan(spec benchmarkProfileSpec, precisions []string, metricStage func(string) string) (planLabels []string, planPhases []benchmarkPlannedPhase, basePhaseSec int, mixedPhaseSec int) {
	if len(precisions) == 0 {
		precisions = append([]string(nil), benchmarkPrecisionPhases...)
	}
	switch spec.Name {
	case NvidiaBenchmarkProfileStandard:
		basePhaseSec = 60
		mixedPhaseSec = 300
	case NvidiaBenchmarkProfileStability:
		basePhaseSec = 300
		mixedPhaseSec = 3600
	case NvidiaBenchmarkProfileOvernight:
		basePhaseSec = 3600
		mixedPhaseSec = 14400
	default:
		totalWeight := len(precisions) + 5
		if totalWeight <= 0 {
			return nil, nil, 0, 0
		}
		basePhaseSec = spec.SteadySec / totalWeight
		if basePhaseSec <= 0 {
			basePhaseSec = 1
		}
		mixedPhaseSec = basePhaseSec * 5
	}
	planLabels = make([]string, 0, len(precisions)+1)
	planPhases = make([]benchmarkPlannedPhase, 0, len(precisions)+1)
	for _, prec := range precisions {
		planLabels = append(planLabels, prec)
		planPhases = append(planPhases, benchmarkPlannedPhase{
			PlanLabel:   prec,
			MetricStage: metricStage(prec),
			DurationSec: basePhaseSec,
		})
	}
	planLabels = append(planLabels, "mixed")
	planPhases = append(planPhases, benchmarkPlannedPhase{
		PlanLabel:   "mixed",
		MetricStage: metricStage("mixed"),
		DurationSec: mixedPhaseSec,
	})
	return planLabels, planPhases, basePhaseSec, mixedPhaseSec
}

func benchmarkPlanDurationsCSV(phases []benchmarkPlannedPhase) string {
	values := make([]string, 0, len(phases))
	for _, phase := range phases {
		values = append(values, strconv.Itoa(phase.DurationSec))
	}
	return strings.Join(values, ",")
}

func benchmarkPlannedPhaseStatus(raw []byte) (string, string) {
	text := strings.ToLower(strings.TrimSpace(string(raw)))
	switch {
	case text == "":
		return "FAILED", "phase produced no output"
	case strings.Contains(text, "phase_error="):
		if strings.Contains(text, "unsupported") || strings.Contains(text, "not supported") || strings.Contains(text, "cublaslt_profiles=unsupported") {
			return "UNSUPPORTED", "precision phase unsupported on this GPU/userspace path"
		}
		return "FAILED", "precision phase failed"
	case strings.Contains(text, "status=failed"):
		if strings.Contains(text, "unsupported") || strings.Contains(text, "not supported") {
			return "UNSUPPORTED", "precision phase unsupported on this GPU/userspace path"
		}
		return "FAILED", "precision phase failed"
	default:
		return "OK", ""
	}
}

func benchmarkCalibrationThrottleReason(before, after BenchmarkThrottleCounters) string {
	diff := diffThrottleCounters(before, after)
	switch {
	case diff.HWThermalSlowdownUS > 0:
		return "hw_thermal"
	case diff.SWThermalSlowdownUS > 0:
		return "sw_thermal"
	default:
		return ""
	}
}

func setBenchmarkPowerLimit(ctx context.Context, verboseLog string, gpuIndex, powerLimitW int) error {
	if powerLimitW <= 0 {
		return fmt.Errorf("invalid power limit %d", powerLimitW)
	}
	out, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("gpu-%d-set-power-limit-%dw", gpuIndex, powerLimitW), []string{
		"nvidia-smi", "-i", strconv.Itoa(gpuIndex), "-pl", strconv.Itoa(powerLimitW),
	}, nil, nil)
	if err != nil {
		return fmt.Errorf("set power limit gpu=%d limit=%dw: %w (%s)", gpuIndex, powerLimitW, err, strings.TrimSpace(string(out)))
	}
	return nil
}

func resetBenchmarkGPU(ctx context.Context, verboseLog string, gpuIndex int, logFunc func(string)) error {
	if logFunc != nil {
		logFunc(fmt.Sprintf("power benchmark pre-flight: GPU %d reset via shared NVIDIA recover path", gpuIndex))
	}
	out, err := benchmarkResetNvidiaGPU(gpuIndex)
	appendSATVerboseLog(verboseLog,
		fmt.Sprintf("[%s] start power-preflight-gpu-%d-reset.log", time.Now().UTC().Format(time.RFC3339), gpuIndex),
		"cmd: bee-nvidia-recover reset-gpu "+strconv.Itoa(gpuIndex),
	)
	if trimmed := strings.TrimSpace(out); trimmed != "" && logFunc != nil {
		for _, line := range strings.Split(trimmed, "\n") {
			line = strings.TrimSpace(line)
			if line != "" {
				logFunc(line)
			}
		}
	}
	rc := 0
	if err != nil {
		rc = 1
	}
	appendSATVerboseLog(verboseLog,
		fmt.Sprintf("[%s] finish power-preflight-gpu-%d-reset.log", time.Now().UTC().Format(time.RFC3339), gpuIndex),
		fmt.Sprintf("rc: %d", rc),
		"",
	)
	return err
}

func resetBenchmarkGPUs(ctx context.Context, verboseLog string, gpuIndices []int, logFunc func(string)) []int {
	if len(gpuIndices) == 0 {
		return nil
	}
	if benchmarkGeteuid() != 0 {
		if logFunc != nil {
			logFunc("power benchmark pre-flight: root privileges unavailable, GPU reset skipped")
		}
		return append([]int(nil), gpuIndices...)
	}
	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
		for _, p := range killed {
			logFunc(fmt.Sprintf("power benchmark pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
		}
	}
	var failed []int
	for _, idx := range gpuIndices {
		if err := resetBenchmarkGPU(ctx, verboseLog, idx, logFunc); err != nil {
			failed = append(failed, idx)
			if logFunc != nil {
				logFunc(fmt.Sprintf("power benchmark pre-flight: GPU %d reset failed: %v", idx, err))
			}
			continue
		}
		if logFunc != nil {
			logFunc(fmt.Sprintf("power benchmark pre-flight: GPU %d reset completed", idx))
		}
		benchmarkSleep(time.Second)
	}
	return failed
}

func benchmarkPowerEngine() string {
	switch strings.TrimSpace(strings.ToLower(os.Getenv("BEE_BENCH_POWER_ENGINE"))) {
	case BenchmarkPowerEngineTargetedPower:
		return BenchmarkPowerEngineTargetedPower
	default:
		return BenchmarkPowerEngineDCGMProfTester
	}
}

func benchmarkPowerEngineLabel(engine string) string {
	switch strings.TrimSpace(strings.ToLower(engine)) {
	case BenchmarkPowerEngineTargetedPower:
		return "dcgmi diag targeted_power"
	default:
		return "dcgmproftester"
	}
}

func resolveBenchmarkPowerLoadCommand(durationSec int, gpuIndices []int) ([]string, []string, error) {
	engine := benchmarkPowerEngine()
	durationSec = normalizeNvidiaBurnDuration(durationSec)
	switch engine {
	case BenchmarkPowerEngineTargetedPower:
		return nvidiaDCGMNamedDiagCommand("targeted_power", durationSec, gpuIndices), nil, nil
	default:
		if len(gpuIndices) > 1 {
			return []string{
				"bee-dcgmproftester-staggered",
				"--seconds", strconv.Itoa(durationSec),
				"--stagger-seconds", "0",
				"--devices", joinIndexList(gpuIndices),
			}, nil, nil
		}
		cmd, err := resolveDCGMProfTesterCommand("--no-dcgm-validation", "-t", "1004", "-d", strconv.Itoa(durationSec))
		if err != nil {
			return nil, nil, err
		}
		return cmd, nvidiaVisibleDevicesEnv(gpuIndices), nil
	}
}

func (s *System) RunNvidiaBenchmark(ctx context.Context, baseDir string, opts NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
	if ctx == nil {
		ctx = context.Background()
	}
	if logFunc == nil {
		logFunc = func(string) {}
	}
	if strings.TrimSpace(baseDir) == "" {
		baseDir = "/var/log/bee-bench/perf"
	}
	spec := resolveBenchmarkProfile(opts.Profile)
	opts = normalizeNvidiaBenchmarkOptionsForBenchmark(opts)

	selected, err := resolveNvidiaGPUSelection(opts.GPUIndices, opts.ExcludeGPUIndices)
	if err != nil {
		return "", err
	}
	if len(selected) == 0 {
		return "", fmt.Errorf("no NVIDIA GPUs selected")
	}

	ts := time.Now().UTC().Format("20060102-150405")
	runDir := filepath.Join(baseDir, "perf-"+ts)
	if err := os.MkdirAll(runDir, 0755); err != nil {
		return "", fmt.Errorf("mkdir %s: %w", runDir, err)
	}
	verboseLog := filepath.Join(runDir, "verbose.log")

	hostname, _ := os.Hostname()
	result := NvidiaBenchmarkResult{
		BenchmarkVersion:   benchmarkVersion,
		GeneratedAt:        time.Now().UTC(),
		Hostname:           hostname,
		ServerModel:        readServerModel(),
		BenchmarkProfile:   spec.Name,
		ParallelGPUs:       opts.ParallelGPUs,
		RampStep:           opts.RampStep,
		RampTotal:          opts.RampTotal,
		RampRunID:          opts.RampRunID,
		SelectedGPUIndices: append([]int(nil), selected...),
		HostConfig:         readBenchmarkHostConfig(),
		Normalization: BenchmarkNormalization{
			Status: "full",
		},
	}

	logFunc(fmt.Sprintf("NVIDIA benchmark profile=%s gpus=%s", spec.Name, joinIndexList(selected)))
	psuBefore := psuStatusSnapshot()
	var metricRows []GPUMetricRow
	metricTimelineSec := 0.0
	gpuBurnLog := filepath.Join(runDir, "gpu-burn.log")

	// Server power characterization state — populated during per-GPU phases.
	var serverIdleW, serverLoadedWSum float64
	var serverIdleOK, serverLoadedOK bool
	var serverLoadedSamples int

	// Run nvidia-smi -q first: used both for the log file and as a fallback
	// source of max clock values when CSV clock fields are unsupported.
	var nvsmiQOut []byte
	if out, err := runSATCommandCtx(ctx, verboseLog, "00-nvidia-smi-q.log", []string{"nvidia-smi", "-q"}, nil, nil); err == nil {
		nvsmiQOut = out
		_ = os.WriteFile(filepath.Join(runDir, "00-nvidia-smi-q.log"), out, 0644)
	}

	infoByIndex, infoErr := queryBenchmarkGPUInfo(selected)
	if infoErr != nil {
		result.Warnings = append(result.Warnings, "gpu inventory query failed: "+infoErr.Error())
		result.Normalization.Status = "partial"
	}
	// Enrich with verbose nvidia-smi data — covers GPUs where some CSV fields
	// are unsupported (e.g. clocks.max.* on Blackwell / driver 98.x).
	enrichGPUInfoWithNvidiaSMIQ(infoByIndex, nvsmiQOut)

	activeApps, err := queryActiveComputeApps(selected)
	if err == nil && len(activeApps) > 0 {
		result.Warnings = append(result.Warnings, "active GPU compute processes detected before benchmark")
		result.Normalization.Notes = append(result.Normalization.Notes, activeApps...)
		result.Normalization.Status = "partial"
	}

	restoreActions := applyBenchmarkNormalization(ctx, verboseLog, selected, infoByIndex, &result)
	defer func() {
		for i := len(restoreActions) - 1; i >= 0; i-- {
			restoreActions[i].fn()
		}
	}()

	// No power calibration before performance benchmark — GPUs run at their
	// default power limits. PowerSustainScore is derived from steady-state power
	// observed during the benchmark itself.
	calibByIndex := make(map[int]benchmarkPowerCalibrationResult)

	// Start background CPU load sampler — samples every 10s during GPU phases.
	cpuStopCh := make(chan struct{})
	cpuSamplesCh := startCPULoadSampler(cpuStopCh, 10)

	if opts.ParallelGPUs {
		runNvidiaBenchmarkParallel(ctx, verboseLog, runDir, selected, infoByIndex, opts, spec, logFunc, &result, calibByIndex, &serverIdleW, &serverLoadedWSum, &serverIdleOK, &serverLoadedOK, &serverLoadedSamples, &metricRows, &metricTimelineSec, gpuBurnLog)
	} else {

		for _, idx := range selected {
			gpuResult := BenchmarkGPUResult{
				Index:  idx,
				Status: "FAILED",
			}
			if info, ok := infoByIndex[idx]; ok {
				gpuResult.UUID = info.UUID
				gpuResult.Name = info.Name
				gpuResult.BusID = info.BusID
				gpuResult.VBIOS = info.VBIOS
				gpuResult.PowerLimitW = info.PowerLimitW
				gpuResult.MultiprocessorCount = info.MultiprocessorCount
				gpuResult.DefaultPowerLimitW = info.DefaultPowerLimitW
				gpuResult.ShutdownTempC = info.ShutdownTempC
				gpuResult.SlowdownTempC = info.SlowdownTempC
				gpuResult.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz
				gpuResult.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz
				gpuResult.MaxMemoryClockMHz = info.MaxMemoryClockMHz
			}
			if calib, ok := calibByIndex[idx]; ok {
				gpuResult.CalibratedPeakPowerW = calib.Summary.P95PowerW
				gpuResult.CalibratedPeakTempC = calib.Summary.P95TempC
				gpuResult.PowerCalibrationTries = calib.Attempts
				gpuResult.PowerLimitDerated = calib.Derated
				gpuResult.Notes = append(gpuResult.Notes, calib.Notes...)
				if calib.CoolingWarning != "" {
					gpuResult.CoolingWarning = calib.CoolingWarning
				}
			}
			if norm := findBenchmarkNormalization(result.Normalization.GPUs, idx); norm != nil {
				gpuResult.LockedGraphicsClockMHz = norm.GPUClockLockMHz
				gpuResult.LockedMemoryClockMHz = norm.MemoryClockLockMHz
			}

			baselineRows, err := collectBenchmarkSamples(ctx, spec.BaselineSec, []int{idx})
			if err != nil && err != context.Canceled {
				gpuResult.Notes = append(gpuResult.Notes, "baseline sampling failed: "+err.Error())
			}
			gpuResult.Baseline = summarizeBenchmarkTelemetry(baselineRows)
			appendBenchmarkMetrics(&metricRows, baselineRows, fmt.Sprintf("gpu-%d-baseline", idx), &metricTimelineSec, float64(spec.BaselineSec))

			// Sample server idle power once (first GPU only — server state is global).
			if !serverIdleOK {
				if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, maxInt(spec.BaselineSec, 10), benchmarkPowerAutotuneSampleInterval); ok {
					serverIdleW = w
					serverIdleOK = true
					logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w))
				}
			}

			warmupCmd := []string{
				"bee-gpu-burn",
				"--seconds", strconv.Itoa(spec.WarmupSec),
				"--size-mb", strconv.Itoa(opts.SizeMB),
				"--devices", strconv.Itoa(idx),
			}
			logFunc(fmt.Sprintf("GPU %d: warmup (%ds)", idx, spec.WarmupSec))
			warmupOut, warmupRows, warmupErr := runBenchmarkCommandWithMetrics(ctx, verboseLog, fmt.Sprintf("gpu-%d-warmup.log", idx), warmupCmd, nil, []int{idx}, logFunc)
			appendBenchmarkMetrics(&metricRows, warmupRows, fmt.Sprintf("gpu-%d-warmup", idx), &metricTimelineSec, float64(spec.WarmupSec))
			appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", fmt.Sprintf("gpu-%d-warmup", idx), warmupOut)
			if warmupErr != nil {
				gpuResult.Notes = append(gpuResult.Notes, "warmup failed: "+warmupErr.Error())
				result.GPUs = append(result.GPUs, finalizeBenchmarkGPUResult(gpuResult))
				continue
			}
			warmupParse := parseBenchmarkBurnLog(string(warmupOut))
			if gpuResult.ComputeCapability == "" {
				gpuResult.ComputeCapability = warmupParse.ComputeCapability
			}

			// Run synthetic precision phases and the combined steady phase as one
			// uninterrupted command so the GPU stays hot between windows.
			eccBase, _ := queryECCCounters(idx)
			supportedPrecisions := benchmarkSupportedPrecisions(gpuResult.ComputeCapability)
			planLabels, planPhases, basePhaseSec, mixedPhaseSec := buildBenchmarkSteadyPlan(spec, supportedPrecisions, func(label string) string {
				if label == "mixed" {
					return fmt.Sprintf("gpu-%d-steady", idx)
				}
				return fmt.Sprintf("gpu-%d-steady-%s", idx, label)
			})
			planCmd := []string{
				"bee-gpu-burn",
				"--seconds", strconv.Itoa(basePhaseSec),
				"--size-mb", strconv.Itoa(opts.SizeMB),
				"--devices", strconv.Itoa(idx),
				"--precision-plan", strings.Join(planLabels, ","),
				"--precision-plan-seconds", benchmarkPlanDurationsCSV(planPhases),
			}
			logFunc(fmt.Sprintf("GPU %d: uninterrupted precision plan (%d precision phases x %ds, mixed %ds)", idx, len(supportedPrecisions), basePhaseSec, mixedPhaseSec))
			serverPowerStopCh := make(chan struct{})
			serverPowerCh := startSelectedPowerSourceSampler(serverPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
			_, phaseRowsByStage, phaseLogs, planErr := runBenchmarkPlannedCommandWithMetrics(ctx, verboseLog, fmt.Sprintf("gpu-%d-precision-plan.log", idx), planCmd, nil, []int{idx}, planPhases, logFunc)
			close(serverPowerStopCh)
			if serverPowerSamples := <-serverPowerCh; len(serverPowerSamples) > 0 {
				serverLoadedWSum += benchmarkMean(serverPowerSamples)
				serverLoadedSamples++
				serverLoadedOK = true
				logFunc(fmt.Sprintf("GPU %d: server loaded power (%s avg): %.0f W", idx, opts.ServerPowerSource, benchmarkMean(serverPowerSamples)))
			}
			for _, phaseSpec := range planPhases {
				if rows := phaseRowsByStage[phaseSpec.MetricStage]; len(rows) > 0 {
					appendBenchmarkMetrics(&metricRows, rows, phaseSpec.MetricStage, &metricTimelineSec, float64(phaseSpec.DurationSec))
				}
				appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", phaseSpec.MetricStage, phaseLogs[phaseSpec.PlanLabel])
			}
			for _, prec := range supportedPrecisions {
				stageName := fmt.Sprintf("gpu-%d-steady-%s", idx, prec)
				phaseRows := phaseRowsByStage[stageName]
				phase := BenchmarkPrecisionSteadyPhase{
					Precision: prec,
					Status:    "OK",
					Steady:    summarizeBenchmarkTelemetry(phaseRows),
				}
				if status, note := benchmarkPlannedPhaseStatus(phaseLogs[prec]); status != "OK" {
					phase.Status = status
					phase.Notes = note
					gpuResult.PrecisionFailures = append(gpuResult.PrecisionFailures, prec+":"+status)
				}
				for _, p := range parseBenchmarkBurnLog(string(phaseLogs[prec])).Profiles {
					if p.Supported {
						phase.TeraOpsPerSec += p.TeraOpsPerSec
						phase.WeightedTeraOpsPerSec += p.WeightedTeraOpsPerSec
					}
				}
				gpuResult.PrecisionSteady = append(gpuResult.PrecisionSteady, phase)
			}

			beforeThrottle, _ := queryThrottleCounters(idx)
			logFunc(fmt.Sprintf("GPU %d: steady compute (combined, %ds)", idx, mixedPhaseSec))
			afterThrottle, _ := queryThrottleCounters(idx)
			if planErr != nil {
				gpuResult.Notes = append(gpuResult.Notes, "precision plan failed: "+planErr.Error())
			}

			steadyRows := phaseRowsByStage[fmt.Sprintf("gpu-%d-steady", idx)]
			parseResult := parseBenchmarkBurnLog(string(phaseLogs["mixed"]))
			gpuResult.ComputeCapability = parseResult.ComputeCapability
			gpuResult.Backend = parseResult.Backend
			gpuResult.PrecisionResults = parseResult.Profiles
			if parseResult.Fallback {
				gpuResult.Notes = append(gpuResult.Notes, "benchmark used driver PTX fallback; tensor throughput score is not comparable")
			}

			gpuResult.Steady = summarizeBenchmarkTelemetry(steadyRows)
			gpuResult.Throttle = diffThrottleCounters(beforeThrottle, afterThrottle)
			if eccFinal, err := queryECCCounters(idx); err == nil {
				gpuResult.ECC = diffECCCounters(eccBase, eccFinal)
			}

			if spec.CooldownSec > 0 {
				cooldownRows, err := collectBenchmarkSamples(ctx, spec.CooldownSec, []int{idx})
				if err != nil && err != context.Canceled {
					gpuResult.Notes = append(gpuResult.Notes, "cooldown sampling failed: "+err.Error())
				}
				gpuResult.Cooldown = summarizeBenchmarkTelemetry(cooldownRows)
				appendBenchmarkMetrics(&metricRows, cooldownRows, fmt.Sprintf("gpu-%d-cooldown", idx), &metricTimelineSec, float64(spec.CooldownSec))
			}

			applyBenchmarkSteadyFallback(&gpuResult)
			gpuResult.Scores = scoreBenchmarkGPUResult(gpuResult)
			gpuResult.DegradationReasons = detectBenchmarkDegradationReasons(gpuResult, result.Normalization.Status)
			if anomaly := detectPowerAnomaly(metricRows, idx); anomaly != "" {
				gpuResult.Notes = append(gpuResult.Notes,
					fmt.Sprintf("[HARD STOP] GPU %d: %s", idx, anomaly))
			}
			if warn := detectSlowdownTempExceedance(metricRows, idx, gpuResult.SlowdownTempC); warn != "" {
				gpuResult.Notes = append(gpuResult.Notes,
					fmt.Sprintf("[WARNING] GPU %d: %s", idx, warn))
				if gpuResult.Status == "OK" {
					gpuResult.Status = "PARTIAL"
				}
			}
			if planErr != nil {
				gpuResult.Status = classifySATErrorStatus(phaseLogs["mixed"], planErr)
			} else if len(gpuResult.PrecisionFailures) > 0 {
				gpuResult.Status = "PARTIAL"
			} else if parseResult.Fallback {
				gpuResult.Status = "PARTIAL"
			} else {
				gpuResult.Status = "OK"
			}

			result.GPUs = append(result.GPUs, finalizeBenchmarkGPUResult(gpuResult))
		}

	} // end sequential path

	// Performance scalability ramp-up: run parallel benchmarks for k=2..N GPUs
	// and compute compute scalability relative to the best single-GPU result.
	// Only runs in sequential mode (each GPU was tested individually above) and
	// when there are at least 2 GPUs.
	if !opts.ParallelGPUs && len(selected) >= 2 {
		// Find the best single-card SyntheticScore as the 1-GPU baseline.
		var bestTOPS float64
		for _, g := range result.GPUs {
			if g.Scores.SyntheticScore > bestTOPS {
				bestTOPS = g.Scores.SyntheticScore
			}
		}
		if bestTOPS > 0 {
			var rampSteps []NvidiaPerformanceRampStep
			var scalabilityPcts []float64
			for k := 2; k <= len(selected); k++ {
				subset := append([]int(nil), selected[:k]...)
				rampDir := filepath.Join(runDir, fmt.Sprintf("ramp-%02d", k))
				_ = os.MkdirAll(rampDir, 0755)
				logFunc(fmt.Sprintf("performance ramp: step %d/%d — running %d GPUs in parallel", k, len(selected), k))

				var rampResult NvidiaBenchmarkResult
				var rampIdleW, rampLoadedWSum float64
				var rampIdleOK, rampLoadedOK bool
				var rampLoadedSamples int
				var rampMetricRows []GPUMetricRow
				var rampTimelineSec float64
				emptyCalib := make(map[int]benchmarkPowerCalibrationResult)

				runNvidiaBenchmarkParallel(ctx, verboseLog, rampDir, subset, infoByIndex, opts, spec, logFunc,
					&rampResult, emptyCalib,
					&rampIdleW, &rampLoadedWSum, &rampIdleOK, &rampLoadedOK, &rampLoadedSamples,
					&rampMetricRows, &rampTimelineSec, "")

				var totalSynth, totalMixed float64
				for _, g := range rampResult.GPUs {
					totalSynth += g.Scores.SyntheticScore
					totalMixed += g.Scores.MixedScore
				}
				scalPct := totalSynth / (float64(k) * bestTOPS) * 100
				scalabilityPcts = append(scalabilityPcts, scalPct)

				stepStatus := "OK"
				if len(rampResult.GPUs) < k {
					stepStatus = "PARTIAL"
				}
				rampSteps = append(rampSteps, NvidiaPerformanceRampStep{
					StepIndex:          k,
					GPUIndices:         subset,
					TotalSyntheticTOPS: totalSynth,
					TotalMixedTOPS:     totalMixed,
					ScalabilityPct:     scalPct,
					Status:             stepStatus,
				})
			}
			result.PerformanceRampSteps = rampSteps
			result.PlatformPowerScore = benchmarkMean(scalabilityPcts)
			if len(scalabilityPcts) > 0 {
				result.ScalabilityScore = scalabilityPcts[len(scalabilityPcts)-1]
			}
		}
	}

	if len(selected) > 1 && opts.RunNCCL {
		result.Interconnect = runBenchmarkInterconnect(ctx, verboseLog, runDir, selected, spec, logFunc)
		if result.Interconnect != nil && result.Interconnect.Supported {
			for i := range result.GPUs {
				result.GPUs[i].Scores.InterconnectScore = result.Interconnect.MaxBusBWGBps
			}
		}
	}

	// Stop CPU load sampler and attach results.
	close(cpuStopCh)
	if cpuSamples := <-cpuSamplesCh; len(cpuSamples) > 0 {
		result.CPULoad = summarizeCPULoad(cpuSamples)
		if result.CPULoad != nil && result.CPULoad.Status != "ok" {
			logFunc(fmt.Sprintf("host CPU load during benchmark: avg=%.1f%% max=%.1f%% status=%s",
				result.CPULoad.AvgPct, result.CPULoad.MaxPct, result.CPULoad.Status))
		}
	}

	// Compute server power characterization from accumulated IPMI samples.
	var gpuReportedSumW float64
	for _, gpu := range result.GPUs {
		gpuReportedSumW += gpu.Steady.AvgPowerW
	}
	var serverLoadedW float64
	if serverLoadedSamples > 0 {
		serverLoadedW = serverLoadedWSum / float64(serverLoadedSamples)
	}
	result.ServerPower = characterizeServerPower(serverIdleW, serverLoadedW, gpuReportedSumW, opts.ServerPowerSource, serverIdleOK && serverLoadedOK)
	result.Cooling = summarizeBenchmarkCooling(metricRows)

	// Apply server-power penalty when IPMI reports the server delta is much
	// lower than GPU-reported sum: GPU power telemetry is over-stated, making
	// CalibratedPeakPowerW and PowerSustainScore unreliable.
	// Penalty factor scales from 1.0 (ratio ≥ 0.75, no penalty) down to 0.
	if sp := result.ServerPower; sp != nil && sp.Available && sp.ReportingRatio > 0 && sp.ReportingRatio < 0.75 {
		factor := sp.ReportingRatio / 0.75
		for i := range result.GPUs {
			result.GPUs[i].Scores.CompositeScore *= factor
			result.GPUs[i].Notes = append(result.GPUs[i].Notes,
				fmt.Sprintf("server-power penalty applied (reporting_ratio=%.2f < 0.75): composite score reduced to %.1f%%",
					sp.ReportingRatio, factor*100))
		}
	}

	result.Findings = buildBenchmarkFindings(result)
	result.OverallStatus = benchmarkOverallStatus(result)
	result.PSUIssues = diffPSUStatus(psuBefore, psuStatusSnapshot())
	writeBenchmarkMetricsFiles(runDir, metricRows)

	resultJSON, err := json.MarshalIndent(result, "", "  ")
	if err != nil {
		return "", fmt.Errorf("marshal benchmark result: %w", err)
	}
	if err := os.WriteFile(filepath.Join(runDir, "result.json"), resultJSON, 0644); err != nil {
		return "", fmt.Errorf("write result.json: %w", err)
	}

	report := renderBenchmarkReportWithCharts(result)
	if err := os.WriteFile(filepath.Join(runDir, "report.md"), []byte(report), 0644); err != nil {
		return "", fmt.Errorf("write report.md: %w", err)
	}

	summary := renderBenchmarkSummary(result)
	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
		return "", fmt.Errorf("write summary.txt: %w", err)
	}

	return runDir, nil
}