bee/audit/internal/platform/benchmark_analysis.go

package platform

import (
	"context"
	"fmt"
	"math"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
)

type benchmarkPlannedPhase struct {
	PlanLabel   string
	MetricStage string
	DurationSec int
}

func runBenchmarkPlannedCommandWithMetrics(
	ctx context.Context,
	verboseLog, name string,
	cmd []string,
	env []string,
	gpuIndices []int,
	phases []benchmarkPlannedPhase,
	logFunc func(string),
) ([]byte, map[string][]GPUMetricRow, map[string][]byte, error) {
	out, rows, err := runBenchmarkCommandWithMetrics(ctx, verboseLog, name, cmd, env, gpuIndices, logFunc)
	return out, splitBenchmarkRowsByPlannedPhase(rows, phases), splitBenchmarkLogByPlannedPhase(out), err
}

func splitBenchmarkRowsByPlannedPhase(rows []GPUMetricRow, phases []benchmarkPlannedPhase) map[string][]GPUMetricRow {
	out := make(map[string][]GPUMetricRow, len(phases))
	if len(rows) == 0 || len(phases) == 0 {
		return out
	}
	for _, row := range rows {
		idx := len(phases) - 1
		var elapsed float64
		for i, phase := range phases {
			durationSec := phase.DurationSec
			if durationSec <= 0 {
				durationSec = 1
			}
			elapsed += float64(durationSec)
			if row.ElapsedSec < elapsed {
				idx = i
				break
			}
		}
		out[phases[idx].MetricStage] = append(out[phases[idx].MetricStage], row)
	}
	return out
}

func splitBenchmarkLogByPlannedPhase(raw []byte) map[string][]byte {
	out := make(map[string][]byte)
	var current string
	for _, line := range strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") {
		trimmed := strings.TrimSpace(stripBenchmarkPrefix(line))
		switch {
		case strings.HasPrefix(trimmed, "phase_begin="):
			current = strings.TrimSpace(strings.TrimPrefix(trimmed, "phase_begin="))
		case strings.HasPrefix(trimmed, "phase_end="):
			current = ""
		case current != "":
			out[current] = append(out[current], []byte(line+"\n")...)
		}
	}
	return out
}

type benchmarkCoolingSample struct {
	AvgFanRPM             float64
	AvgFanDutyCyclePct    float64
	FanDutyCycleAvailable bool
	FanDutyCycleEstimated bool
}

func sampleBenchmarkTelemetry(gpuIndices []int) ([]GPUMetricRow, error) {
	samples, err := sampleGPUMetrics(gpuIndices)
	if err != nil {
		return nil, err
	}
	fanSample := sampleBenchmarkCoolingSample()
	for i := range samples {
		samples[i].FanAvgRPM = fanSample.AvgFanRPM
		samples[i].FanDutyCyclePct = fanSample.AvgFanDutyCyclePct
		samples[i].FanDutyCycleAvailable = fanSample.FanDutyCycleAvailable
		samples[i].FanDutyCycleEstimated = fanSample.FanDutyCycleEstimated
	}
	return samples, nil
}

func sampleBenchmarkCoolingSample() benchmarkCoolingSample {
	fans, _ := sampleFanSpeeds()
	avgRPM, _, _ := fanRPMStats(fans)
	dutyPct, dutyAvailable, dutyEstimated := sampleFanDutyCyclePctFromFans(fans)
	return benchmarkCoolingSample{
		AvgFanRPM:             avgRPM,
		AvgFanDutyCyclePct:    dutyPct,
		FanDutyCycleAvailable: dutyAvailable,
		FanDutyCycleEstimated: dutyEstimated,
	}
}

func annotateBenchmarkMetricRows(rows []GPUMetricRow, stage string, offset, durationSec float64) []GPUMetricRow {
	if len(rows) == 0 {
		return nil
	}
	stageEnd := offset + durationSec
	if stageEnd <= offset {
		stageEnd = offset
		for _, row := range rows {
			if row.ElapsedSec+offset > stageEnd {
				stageEnd = row.ElapsedSec + offset
			}
		}
	}
	out := make([]GPUMetricRow, len(rows))
	for i, row := range rows {
		row.Stage = stage
		row.ElapsedSec += offset
		row.StageStartSec = offset
		row.StageEndSec = stageEnd
		out[i] = row
	}
	return out
}

func appendBenchmarkMetrics(allRows *[]GPUMetricRow, rows []GPUMetricRow, stage string, cursor *float64, durationSec float64) {
	annotated := annotateBenchmarkMetricRows(rows, stage, *cursor, durationSec)
	*allRows = append(*allRows, annotated...)
	*cursor += durationSec
}

func writeBenchmarkMetricsFiles(runDir string, rows []GPUMetricRow) {
	if len(rows) == 0 {
		return
	}
	_ = WriteGPUMetricsCSV(filepath.Join(runDir, "gpu-metrics.csv"), rows)
	_ = WriteGPUMetricsHTML(filepath.Join(runDir, "gpu-metrics.html"), rows)
}

func appendBenchmarkStageLog(path, source, stage string, raw []byte) {
	if path == "" || len(raw) == 0 {
		return
	}
	f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
	if err != nil {
		return
	}
	defer f.Close()
	header := fmt.Sprintf("\n========== %s | stage=%s ==========\n", source, stage)
	_, _ = f.WriteString(header)
	if len(raw) > 0 {
		_, _ = f.Write(raw)
		if raw[len(raw)-1] != '\n' {
			_, _ = f.WriteString("\n")
		}
	}
}

func parseBenchmarkBurnLog(raw string) benchmarkBurnParseResult {
	result := benchmarkBurnParseResult{}
	lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
	profiles := make(map[string]*benchmarkBurnProfile)
	for _, line := range lines {
		line = stripBenchmarkPrefix(strings.TrimSpace(line))
		if line == "" {
			continue
		}
		switch {
		case strings.HasPrefix(line, "device="):
			result.Device = strings.TrimSpace(strings.TrimPrefix(line, "device="))
		case strings.HasPrefix(line, "compute_capability="):
			result.ComputeCapability = strings.TrimSpace(strings.TrimPrefix(line, "compute_capability="))
		case strings.HasPrefix(line, "backend="):
			result.Backend = strings.TrimSpace(strings.TrimPrefix(line, "backend="))
			result.Fallback = result.Backend == "driver-ptx"
		case strings.HasPrefix(line, "duration_s="):
			result.DurationSec, _ = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "duration_s=")))
		default:
			if m := benchmarkReadyPattern.FindStringSubmatch(line); len(m) == 6 {
				profile := ensureBenchmarkProfile(profiles, m[1])
				profile.supported = true
				profile.lanes++
				profile.m, _ = strconv.ParseUint(m[3], 10, 64)
				profile.n, _ = strconv.ParseUint(m[4], 10, 64)
				profile.k, _ = strconv.ParseUint(m[5], 10, 64)
				continue
			}
			if m := benchmarkSkippedPattern.FindStringSubmatch(line); len(m) == 3 {
				profile := ensureBenchmarkProfile(profiles, m[1])
				profile.supported = false
				profile.notes = strings.TrimSpace(m[2])
				continue
			}
			if m := benchmarkIterationsPattern.FindStringSubmatch(line); len(m) == 3 {
				profile := ensureBenchmarkProfile(profiles, m[1])
				iters, _ := strconv.ParseUint(m[2], 10, 64)
				profile.iterations += iters
			}
		}
	}

	keys := make([]string, 0, len(profiles))
	for key := range profiles {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	for _, key := range keys {
		profile := profiles[key]
		precision := BenchmarkPrecisionResult{
			Name:       profile.name,
			Category:   profile.category,
			Supported:  profile.supported,
			Lanes:      profile.lanes,
			M:          profile.m,
			N:          profile.n,
			K:          profile.k,
			Iterations: profile.iterations,
			Notes:      profile.notes,
		}
		w := precisionWeight(profile.category)
		precision.Weight = w
		if profile.supported && result.DurationSec > 0 && profile.m > 0 && profile.n > 0 && profile.k > 0 && profile.iterations > 0 {
			precision.TeraOpsPerSec = (2.0 * float64(profile.m) * float64(profile.n) * float64(profile.k) * float64(profile.iterations)) / float64(result.DurationSec) / 1e12
			precision.WeightedTeraOpsPerSec = precision.TeraOpsPerSec * w
		}
		result.Profiles = append(result.Profiles, precision)
	}
	return result
}

func ensureBenchmarkProfile(profiles map[string]*benchmarkBurnProfile, name string) *benchmarkBurnProfile {
	if profile, ok := profiles[name]; ok {
		return profile
	}
	category := "other"
	switch {
	case strings.HasPrefix(name, "fp64"):
		category = "fp64"
	case strings.HasPrefix(name, "fp32"):
		category = "fp32_tf32"
	case strings.HasPrefix(name, "fp16"):
		category = "fp16_bf16"
	case strings.HasPrefix(name, "int8"):
		category = "int8"
	case strings.HasPrefix(name, "fp8"):
		category = "fp8"
	case strings.HasPrefix(name, "fp4"):
		category = "fp4"
	}
	profile := &benchmarkBurnProfile{name: name, category: category, supported: true}
	profiles[name] = profile
	return profile
}

// precisionWeight returns the fp32-equivalence factor for a precision category.
// Each factor represents how much "real" numeric work one operation of that
// type performs relative to fp32 (single precision = 1.0 baseline):
//
//	fp64  = 2.0  — double precision, 2× more bits per operand
//	fp32  = 1.0  — single precision baseline
//	fp16  = 0.5  — half precision
//	int8  = 0.25 — quarter precision
//	fp8   = 0.25 — quarter precision
//	fp4   = 0.125 — eighth precision
//
// Multiplying raw TOPS by the weight gives fp32-equivalent TOPS, enabling
// cross-precision comparison on the same numeric scale.
func precisionWeight(category string) float64 {
	switch category {
	case "fp64":
		return 2.0
	case "fp32_tf32":
		return 1.0
	case "fp16_bf16":
		return 0.5
	case "int8":
		return 0.25
	case "fp8":
		return 0.25
	case "fp4":
		return 0.125
	default:
		return 1.0
	}
}

func stripBenchmarkPrefix(line string) string {
	if strings.HasPrefix(line, "[gpu ") {
		if idx := strings.Index(line, "] "); idx >= 0 {
			return line[idx+2:]
		}
	}
	return line
}

func summarizeBenchmarkTelemetry(rows []GPUMetricRow) BenchmarkTelemetrySummary {
	summary := BenchmarkTelemetrySummary{}
	if len(rows) == 0 {
		return summary
	}
	temps := make([]float64, 0, len(rows))
	powers := make([]float64, 0, len(rows))
	clocks := make([]float64, 0, len(rows))
	memClocks := make([]float64, 0, len(rows))
	usages := make([]float64, 0, len(rows))
	memUsages := make([]float64, 0, len(rows))
	summary.DurationSec = rows[len(rows)-1].ElapsedSec
	summary.Samples = len(rows)
	for _, row := range rows {
		temps = append(temps, row.TempC)
		powers = append(powers, row.PowerW)
		clocks = append(clocks, row.ClockMHz)
		memClocks = append(memClocks, row.MemClockMHz)
		usages = append(usages, row.UsagePct)
		memUsages = append(memUsages, row.MemUsagePct)
	}
	summary.AvgTempC = benchmarkMean(temps)
	summary.P95TempC = benchmarkPercentile(temps, 95)
	summary.AvgPowerW = benchmarkMean(powers)
	summary.P95PowerW = benchmarkPercentile(powers, 95)
	summary.AvgGraphicsClockMHz = benchmarkMean(clocks)
	summary.P95GraphicsClockMHz = benchmarkPercentile(clocks, 95)
	summary.AvgMemoryClockMHz = benchmarkMean(memClocks)
	summary.P95MemoryClockMHz = benchmarkPercentile(memClocks, 95)
	summary.AvgUsagePct = benchmarkMean(usages)
	summary.AvgMemUsagePct = benchmarkMean(memUsages)
	summary.ClockCVPct = benchmarkCV(clocks)
	summary.PowerCVPct = benchmarkCV(powers)
	summary.TempCVPct = benchmarkCV(temps)
	summary.ClockDriftPct = benchmarkClockDrift(clocks)
	return summary
}

func summarizeBenchmarkCooling(rows []GPUMetricRow) *BenchmarkCoolingSummary {
	if len(rows) == 0 {
		return nil
	}
	var rpmValues []float64
	var dutyValues []float64
	var dutyEstimated bool
	for _, row := range rows {
		if row.FanAvgRPM > 0 {
			rpmValues = append(rpmValues, row.FanAvgRPM)
		}
		if row.FanDutyCycleAvailable {
			dutyValues = append(dutyValues, row.FanDutyCyclePct)
			if row.FanDutyCycleEstimated {
				dutyEstimated = true
			}
		}
	}
	if len(rpmValues) == 0 && len(dutyValues) == 0 {
		return nil
	}
	summary := &BenchmarkCoolingSummary{
		Available:             true,
		AvgFanRPM:             benchmarkMean(rpmValues),
		FanDutyCycleEstimated: dutyEstimated,
	}
	if len(dutyValues) > 0 {
		summary.FanDutyCycleAvailable = true
		summary.AvgFanDutyCyclePct = benchmarkMean(dutyValues)
		summary.P95FanDutyCyclePct = benchmarkPercentile(dutyValues, 95)
		if summary.FanDutyCycleEstimated {
			summary.Notes = append(summary.Notes, "fan duty cycle is estimated from the highest fan RPM observed since boot; treat it as an approximation, not a direct PWM reading")
		}
	} else {
		summary.Notes = append(summary.Notes, "fan duty cycle unavailable on this host; RPM-only fan telemetry was collected")
	}
	return summary
}

func benchmarkTelemetryAvailable(summary BenchmarkTelemetrySummary) bool {
	return summary.Samples > 0 || summary.DurationSec > 0
}

func benchmarkPrecisionSteadyFallback(phases []BenchmarkPrecisionSteadyPhase) (BenchmarkTelemetrySummary, string, bool) {
	var (
		best      BenchmarkTelemetrySummary
		bestLabel string
		found     bool
	)
	for _, phase := range phases {
		if !benchmarkTelemetryAvailable(phase.Steady) {
			continue
		}
		if !found ||
			phase.Steady.DurationSec > best.DurationSec ||
			(phase.Steady.DurationSec == best.DurationSec && phase.Steady.P95PowerW > best.P95PowerW) {
			best = phase.Steady
			bestLabel = phase.Precision
			found = true
		}
	}
	return best, bestLabel, found
}

func applyBenchmarkSteadyFallback(gpu *BenchmarkGPUResult) {
	if gpu == nil || benchmarkTelemetryAvailable(gpu.Steady) {
		return
	}
	if fallback, label, ok := benchmarkPrecisionSteadyFallback(gpu.PrecisionSteady); ok {
		gpu.Steady = fallback
		gpu.Notes = append(gpu.Notes,
			fmt.Sprintf("mixed steady telemetry unavailable; reporting steady-state fallback from %s precision phase", label))
	}
}

func scoreBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkScorecard {
	score := BenchmarkScorecard{}

	// SyntheticScore: sum of fp32-equivalent TOPS from per-precision phases.
	// Each precision ran alone with full GPU dedicated — peak capability.
	for _, p := range gpu.PrecisionSteady {
		if !benchmarkPrecisionEnabled(p.Precision) {
			continue
		}
		score.SyntheticScore += p.WeightedTeraOpsPerSec
	}

	// MixedScore: sum of fp32-equivalent TOPS from the combined phase.
	// All precisions compete simultaneously — closer to real inference workloads.
	for _, p := range gpu.PrecisionResults {
		if p.Supported && benchmarkPrecisionEnabled(p.Category) {
			score.MixedScore += p.WeightedTeraOpsPerSec
		}
	}

	// MixedEfficiency = MixedScore / SyntheticScore.
	// Measures how well the GPU sustains throughput under concurrent mixed load.
	// A healthy GPU scores ~0.8–0.95; severe degradation suggests bandwidth
	// contention or scheduler inefficiency.
	if score.SyntheticScore > 0 && score.MixedScore > 0 {
		score.MixedEfficiency = score.MixedScore / score.SyntheticScore
	}

	// ComputeScore = SyntheticScore × (1 + MixedEfficiency × 0.3).
	// SyntheticScore is the primary signal; MixedEfficiency adds up to +30%
	// bonus for GPUs that handle mixed-precision concurrency well.
	// Falls back to MixedScore alone when per-precision data is absent.
	switch {
	case score.SyntheticScore > 0:
		score.ComputeScore = score.SyntheticScore * (1 + score.MixedEfficiency*0.3)
	case score.MixedScore > 0:
		score.ComputeScore = score.MixedScore
	}
	// PowerSustainScore: how stable is GPU power draw during the benchmark?
	// High variance means the workload is bursting or the power delivery is
	// unstable. Score = max(0, 100 − PowerCVPct × 3).
	// At 10% CV → score 70; at 33%+ CV → score 0.
	// Uses per-precision windows when available (each runs a single kernel,
	// so CV reflects genuine power regulation, not workload switching).
	if len(gpu.PrecisionSteady) > 0 {
		var sum float64
		var count int
		for _, p := range gpu.PrecisionSteady {
			if !benchmarkPrecisionEnabled(p.Precision) {
				continue
			}
			sum += clampScore(100 - p.Steady.PowerCVPct*3)
			count++
		}
		if count > 0 {
			score.PowerSustainScore = sum / float64(count)
		}
	} else if gpu.Steady.PowerCVPct > 0 {
		score.PowerSustainScore = clampScore(100 - gpu.Steady.PowerCVPct*3)
	}

	// ThermalSustainScore: how stable is GPU temperature during the benchmark?
	// High variance means cooling is inconsistent (fan bursts, liquid flow
	// instability, or frequent transitions in and out of throttle).
	// Score = max(0, 100 − TempCVPct × 3).
	if gpu.Steady.TempCVPct > 0 {
		score.ThermalSustainScore = clampScore(100 - gpu.Steady.TempCVPct*3)
	} else {
		// TempCV not recorded — fall back to 100 (no penalty).
		score.ThermalSustainScore = 100
	}

	// Throttle breakdown: compute per-type percentages for diagnosis.
	// Each counter measures microseconds spent in that throttle state during
	// the steady-state window. Counters can overlap (e.g. thermal + power cap
	// simultaneously), so they are reported independently, not summed.
	runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6)
	score.ThermalThrottlePct = math.Min(100,
		float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS*100)
	score.PowerCapThrottlePct = math.Min(100,
		float64(gpu.Throttle.SWPowerCapUS)/runtimeUS*100)
	score.SyncBoostThrottlePct = math.Min(100,
		float64(gpu.Throttle.SyncBoostUS)/runtimeUS*100)

	// StabilityScore: combined throttle signal (thermal + power cap).
	// Score = max(0, 100 − combined_throttle_pct).
	// 1% throttle → 99; 10% → 90; any throttle > 0 is penalised.
	combinedThrottlePct := math.Min(100,
		float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS+gpu.Throttle.SWPowerCapUS)/runtimeUS*100)
	score.StabilityScore = clampScore(100 - combinedThrottlePct)

	// TempHeadroomC: distance from p95 temperature to the GPU's hardware
	// shutdown threshold (sourced from nvidia-smi -q "GPU Shutdown Temp").
	// Fallback: 90°C when not available.
	// Assessed independently of throttle — a GPU at 86°C without any throttle
	// counter still has limited headroom and operates in degraded reliability zone.
	// Warning zone: headroom < (shutdownTemp - slowdownTemp), i.e. past slowdown onset.
	// Critical zone: headroom < 10°C from shutdown.
	if gpu.Steady.P95TempC > 0 {
		shutdownTemp := gpu.ShutdownTempC
		if shutdownTemp <= 0 {
			shutdownTemp = 90
		}
		score.TempHeadroomC = shutdownTemp - gpu.Steady.P95TempC
	}
	score.ServerQualityScore = serverQualityScore(score)
	score.CompositeScore = score.ComputeScore
	if gpu.MultiprocessorCount > 0 && gpu.Steady.AvgGraphicsClockMHz > 0 && score.ComputeScore > 0 {
		score.TOPSPerSMPerGHz = score.ComputeScore / float64(gpu.MultiprocessorCount) / (gpu.Steady.AvgGraphicsClockMHz / 1000.0)
	}
	return score
}

// serverQualityScore returns a 0–100 score reflecting server infrastructure
// quality, independent of GPU model or compute speed.
//
//	StabilityScore (throttle time)   0.40 — heaviest: direct evidence GPU can't sustain load
//	PowerSustainScore (power CV)     0.30 — unstable draw hints at PSU/VRM issues
//	ThermalSustainScore (temp CV)    0.30 — unstable temp hints at airflow/cooling issues
func serverQualityScore(score BenchmarkScorecard) float64 {
	q := 0.40*(score.StabilityScore/100.0) +
		0.30*(score.PowerSustainScore/100.0) +
		0.30*(score.ThermalSustainScore/100.0)
	return clampScore(q * 100)
}

// detectPowerAnomaly scans per-GPU steady-state metric rows for a sudden
// power drop — a symptom of bad cable contact, VRM fault, or thermal event
// on the power delivery path. Returns a non-empty string if an anomaly is found.
//
// Algorithm: uses a 5-sample rolling baseline; flags any sample that falls
// more than 30% below the baseline while the GPU was otherwise loaded
// (usage > 50%). A sustained throttle (power cap) is not flagged here —
// that is already captured by PowerCapThrottlePct.
func detectPowerAnomaly(rows []GPUMetricRow, gpuIndex int) string {
	const windowSize = 5
	const dropThresholdPct = 30.0
	const minUsagePct = 50.0

	// Filter rows for this GPU during steady state only.
	var steady []GPUMetricRow
	for _, r := range rows {
		if r.GPUIndex == gpuIndex && r.Stage != "" && strings.Contains(r.Stage, "steady") {
			steady = append(steady, r)
		}
	}
	if len(steady) < windowSize+2 {
		return ""
	}

	// Compute initial baseline from the first window.
	var baseSum float64
	for i := 0; i < windowSize; i++ {
		baseSum += steady[i].PowerW
	}

	for i := windowSize; i < len(steady); i++ {
		baseline := baseSum / float64(windowSize)
		sample := steady[i]
		if baseline > 0 && sample.UsagePct >= minUsagePct {
			dropPct := (baseline - sample.PowerW) / baseline * 100
			if dropPct >= dropThresholdPct {
				return fmt.Sprintf("sudden power drop detected at t=%.0fs: %.0f W → %.0f W (%.0f%% below rolling baseline) — possible bad cable contact or VRM fault",
					sample.ElapsedSec, baseline, sample.PowerW, dropPct)
			}
		}
		// Slide the window baseline.
		baseSum -= steady[i-windowSize].PowerW
		baseSum += sample.PowerW
	}
	return ""
}

// detectSlowdownTempExceedance scans steady-state metric rows for a GPU and
// returns a warning string if any temperature sample exceeded the GPU's
// SlowdownTempC threshold. Uses fallback 80°C when SlowdownTempC is zero.
// This is a real-time signal distinct from p95 stats — even a single spike
// above the slowdown threshold is worth flagging.
func detectSlowdownTempExceedance(rows []GPUMetricRow, gpuIndex int, slowdownTempC float64) string {
	if slowdownTempC <= 0 {
		slowdownTempC = 80
	}
	var maxTemp float64
	var exceedCount int
	for _, r := range rows {
		if r.GPUIndex != gpuIndex {
			continue
		}
		if !strings.Contains(r.Stage, "steady") {
			continue
		}
		if r.TempC > maxTemp {
			maxTemp = r.TempC
		}
		if r.TempC >= slowdownTempC {
			exceedCount++
		}
	}
	if exceedCount == 0 {
		return ""
	}
	return fmt.Sprintf(
		"temperature exceeded slowdown threshold (%.0f°C) in %d sample(s) during steady state — peak %.1f°C",
		slowdownTempC, exceedCount, maxTemp)
}

func detectBenchmarkDegradationReasons(gpu BenchmarkGPUResult, normalizationStatus string) []string {
	var reasons []string
	runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6)
	if float64(gpu.Throttle.SWPowerCapUS)/runtimeUS >= 0.05 {
		reasons = append(reasons, "power_capped")
	}
	if float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS >= 0.01 {
		reasons = append(reasons, "thermal_limited")
	}
	if float64(gpu.Throttle.SyncBoostUS)/runtimeUS >= 0.01 {
		reasons = append(reasons, "sync_boost_limited")
	}
	if gpu.LockedGraphicsClockMHz > 0 && gpu.Steady.AvgGraphicsClockMHz < gpu.LockedGraphicsClockMHz*0.90 {
		reasons = append(reasons, "low_sm_clock_vs_target")
	}
	if gpu.Scores.StabilityScore > 0 && gpu.Scores.StabilityScore < 85 {
		reasons = append(reasons, "variance_too_high")
	}
	if normalizationStatus != "full" {
		reasons = append(reasons, "normalization_partial")
	}
	if gpu.PowerLimitDerated {
		reasons = append(reasons, "power_limit_derated")
	}
	if gpu.ECC.Uncorrected > 0 {
		reasons = append(reasons, "ecc_uncorrected_errors")
	}
	if gpu.ECC.Corrected > 0 {
		reasons = append(reasons, "ecc_corrected_errors")
	}
	return dedupeStrings(reasons)
}