package platform import ( "context" "fmt" "math" "os" "path/filepath" "sort" "strconv" "strings" ) type benchmarkPlannedPhase struct { PlanLabel string MetricStage string DurationSec int } func runBenchmarkPlannedCommandWithMetrics( ctx context.Context, verboseLog, name string, cmd []string, env []string, gpuIndices []int, phases []benchmarkPlannedPhase, logFunc func(string), ) ([]byte, map[string][]GPUMetricRow, map[string][]byte, error) { out, rows, err := runBenchmarkCommandWithMetrics(ctx, verboseLog, name, cmd, env, gpuIndices, logFunc) return out, splitBenchmarkRowsByPlannedPhase(rows, phases), splitBenchmarkLogByPlannedPhase(out), err } func splitBenchmarkRowsByPlannedPhase(rows []GPUMetricRow, phases []benchmarkPlannedPhase) map[string][]GPUMetricRow { out := make(map[string][]GPUMetricRow, len(phases)) if len(rows) == 0 || len(phases) == 0 { return out } for _, row := range rows { idx := len(phases) - 1 var elapsed float64 for i, phase := range phases { durationSec := phase.DurationSec if durationSec <= 0 { durationSec = 1 } elapsed += float64(durationSec) if row.ElapsedSec < elapsed { idx = i break } } out[phases[idx].MetricStage] = append(out[phases[idx].MetricStage], row) } return out } func splitBenchmarkLogByPlannedPhase(raw []byte) map[string][]byte { out := make(map[string][]byte) var current string for _, line := range strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") { trimmed := strings.TrimSpace(stripBenchmarkPrefix(line)) switch { case strings.HasPrefix(trimmed, "phase_begin="): current = strings.TrimSpace(strings.TrimPrefix(trimmed, "phase_begin=")) case strings.HasPrefix(trimmed, "phase_end="): current = "" case current != "": out[current] = append(out[current], []byte(line+"\n")...) } } return out } type benchmarkCoolingSample struct { AvgFanRPM float64 AvgFanDutyCyclePct float64 FanDutyCycleAvailable bool FanDutyCycleEstimated bool } func sampleBenchmarkTelemetry(gpuIndices []int) ([]GPUMetricRow, error) { samples, err := sampleGPUMetrics(gpuIndices) if err != nil { return nil, err } fanSample := sampleBenchmarkCoolingSample() for i := range samples { samples[i].FanAvgRPM = fanSample.AvgFanRPM samples[i].FanDutyCyclePct = fanSample.AvgFanDutyCyclePct samples[i].FanDutyCycleAvailable = fanSample.FanDutyCycleAvailable samples[i].FanDutyCycleEstimated = fanSample.FanDutyCycleEstimated } return samples, nil } func sampleBenchmarkCoolingSample() benchmarkCoolingSample { fans, _ := sampleFanSpeeds() avgRPM, _, _ := fanRPMStats(fans) dutyPct, dutyAvailable, dutyEstimated := sampleFanDutyCyclePctFromFans(fans) return benchmarkCoolingSample{ AvgFanRPM: avgRPM, AvgFanDutyCyclePct: dutyPct, FanDutyCycleAvailable: dutyAvailable, FanDutyCycleEstimated: dutyEstimated, } } func annotateBenchmarkMetricRows(rows []GPUMetricRow, stage string, offset, durationSec float64) []GPUMetricRow { if len(rows) == 0 { return nil } stageEnd := offset + durationSec if stageEnd <= offset { stageEnd = offset for _, row := range rows { if row.ElapsedSec+offset > stageEnd { stageEnd = row.ElapsedSec + offset } } } out := make([]GPUMetricRow, len(rows)) for i, row := range rows { row.Stage = stage row.ElapsedSec += offset row.StageStartSec = offset row.StageEndSec = stageEnd out[i] = row } return out } func appendBenchmarkMetrics(allRows *[]GPUMetricRow, rows []GPUMetricRow, stage string, cursor *float64, durationSec float64) { annotated := annotateBenchmarkMetricRows(rows, stage, *cursor, durationSec) *allRows = append(*allRows, annotated...) *cursor += durationSec } func writeBenchmarkMetricsFiles(runDir string, rows []GPUMetricRow) { if len(rows) == 0 { return } _ = WriteGPUMetricsCSV(filepath.Join(runDir, "gpu-metrics.csv"), rows) _ = WriteGPUMetricsHTML(filepath.Join(runDir, "gpu-metrics.html"), rows) } func appendBenchmarkStageLog(path, source, stage string, raw []byte) { if path == "" || len(raw) == 0 { return } f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) if err != nil { return } defer f.Close() header := fmt.Sprintf("\n========== %s | stage=%s ==========\n", source, stage) _, _ = f.WriteString(header) if len(raw) > 0 { _, _ = f.Write(raw) if raw[len(raw)-1] != '\n' { _, _ = f.WriteString("\n") } } } func parseBenchmarkBurnLog(raw string) benchmarkBurnParseResult { result := benchmarkBurnParseResult{} lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") profiles := make(map[string]*benchmarkBurnProfile) for _, line := range lines { line = stripBenchmarkPrefix(strings.TrimSpace(line)) if line == "" { continue } switch { case strings.HasPrefix(line, "device="): result.Device = strings.TrimSpace(strings.TrimPrefix(line, "device=")) case strings.HasPrefix(line, "compute_capability="): result.ComputeCapability = strings.TrimSpace(strings.TrimPrefix(line, "compute_capability=")) case strings.HasPrefix(line, "backend="): result.Backend = strings.TrimSpace(strings.TrimPrefix(line, "backend=")) result.Fallback = result.Backend == "driver-ptx" case strings.HasPrefix(line, "duration_s="): result.DurationSec, _ = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "duration_s="))) default: if m := benchmarkReadyPattern.FindStringSubmatch(line); len(m) == 6 { profile := ensureBenchmarkProfile(profiles, m[1]) profile.supported = true profile.lanes++ profile.m, _ = strconv.ParseUint(m[3], 10, 64) profile.n, _ = strconv.ParseUint(m[4], 10, 64) profile.k, _ = strconv.ParseUint(m[5], 10, 64) continue } if m := benchmarkSkippedPattern.FindStringSubmatch(line); len(m) == 3 { profile := ensureBenchmarkProfile(profiles, m[1]) profile.supported = false profile.notes = strings.TrimSpace(m[2]) continue } if m := benchmarkIterationsPattern.FindStringSubmatch(line); len(m) == 3 { profile := ensureBenchmarkProfile(profiles, m[1]) iters, _ := strconv.ParseUint(m[2], 10, 64) profile.iterations += iters } } } keys := make([]string, 0, len(profiles)) for key := range profiles { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { profile := profiles[key] precision := BenchmarkPrecisionResult{ Name: profile.name, Category: profile.category, Supported: profile.supported, Lanes: profile.lanes, M: profile.m, N: profile.n, K: profile.k, Iterations: profile.iterations, Notes: profile.notes, } w := precisionWeight(profile.category) precision.Weight = w if profile.supported && result.DurationSec > 0 && profile.m > 0 && profile.n > 0 && profile.k > 0 && profile.iterations > 0 { precision.TeraOpsPerSec = (2.0 * float64(profile.m) * float64(profile.n) * float64(profile.k) * float64(profile.iterations)) / float64(result.DurationSec) / 1e12 precision.WeightedTeraOpsPerSec = precision.TeraOpsPerSec * w } result.Profiles = append(result.Profiles, precision) } return result } func ensureBenchmarkProfile(profiles map[string]*benchmarkBurnProfile, name string) *benchmarkBurnProfile { if profile, ok := profiles[name]; ok { return profile } category := "other" switch { case strings.HasPrefix(name, "fp64"): category = "fp64" case strings.HasPrefix(name, "fp32"): category = "fp32_tf32" case strings.HasPrefix(name, "fp16"): category = "fp16_bf16" case strings.HasPrefix(name, "int8"): category = "int8" case strings.HasPrefix(name, "fp8"): category = "fp8" case strings.HasPrefix(name, "fp4"): category = "fp4" } profile := &benchmarkBurnProfile{name: name, category: category, supported: true} profiles[name] = profile return profile } // precisionWeight returns the fp32-equivalence factor for a precision category. // Each factor represents how much "real" numeric work one operation of that // type performs relative to fp32 (single precision = 1.0 baseline): // // fp64 = 2.0 — double precision, 2× more bits per operand // fp32 = 1.0 — single precision baseline // fp16 = 0.5 — half precision // int8 = 0.25 — quarter precision // fp8 = 0.25 — quarter precision // fp4 = 0.125 — eighth precision // // Multiplying raw TOPS by the weight gives fp32-equivalent TOPS, enabling // cross-precision comparison on the same numeric scale. func precisionWeight(category string) float64 { switch category { case "fp64": return 2.0 case "fp32_tf32": return 1.0 case "fp16_bf16": return 0.5 case "int8": return 0.25 case "fp8": return 0.25 case "fp4": return 0.125 default: return 1.0 } } func stripBenchmarkPrefix(line string) string { if strings.HasPrefix(line, "[gpu ") { if idx := strings.Index(line, "] "); idx >= 0 { return line[idx+2:] } } return line } func summarizeBenchmarkTelemetry(rows []GPUMetricRow) BenchmarkTelemetrySummary { summary := BenchmarkTelemetrySummary{} if len(rows) == 0 { return summary } temps := make([]float64, 0, len(rows)) powers := make([]float64, 0, len(rows)) clocks := make([]float64, 0, len(rows)) memClocks := make([]float64, 0, len(rows)) usages := make([]float64, 0, len(rows)) memUsages := make([]float64, 0, len(rows)) summary.DurationSec = rows[len(rows)-1].ElapsedSec summary.Samples = len(rows) for _, row := range rows { temps = append(temps, row.TempC) powers = append(powers, row.PowerW) clocks = append(clocks, row.ClockMHz) memClocks = append(memClocks, row.MemClockMHz) usages = append(usages, row.UsagePct) memUsages = append(memUsages, row.MemUsagePct) } summary.AvgTempC = benchmarkMean(temps) summary.P95TempC = benchmarkPercentile(temps, 95) summary.AvgPowerW = benchmarkMean(powers) summary.P95PowerW = benchmarkPercentile(powers, 95) summary.AvgGraphicsClockMHz = benchmarkMean(clocks) summary.P95GraphicsClockMHz = benchmarkPercentile(clocks, 95) summary.AvgMemoryClockMHz = benchmarkMean(memClocks) summary.P95MemoryClockMHz = benchmarkPercentile(memClocks, 95) summary.AvgUsagePct = benchmarkMean(usages) summary.AvgMemUsagePct = benchmarkMean(memUsages) summary.ClockCVPct = benchmarkCV(clocks) summary.PowerCVPct = benchmarkCV(powers) summary.TempCVPct = benchmarkCV(temps) summary.ClockDriftPct = benchmarkClockDrift(clocks) return summary } func summarizeBenchmarkCooling(rows []GPUMetricRow) *BenchmarkCoolingSummary { if len(rows) == 0 { return nil } var rpmValues []float64 var dutyValues []float64 var dutyEstimated bool for _, row := range rows { if row.FanAvgRPM > 0 { rpmValues = append(rpmValues, row.FanAvgRPM) } if row.FanDutyCycleAvailable { dutyValues = append(dutyValues, row.FanDutyCyclePct) if row.FanDutyCycleEstimated { dutyEstimated = true } } } if len(rpmValues) == 0 && len(dutyValues) == 0 { return nil } summary := &BenchmarkCoolingSummary{ Available: true, AvgFanRPM: benchmarkMean(rpmValues), FanDutyCycleEstimated: dutyEstimated, } if len(dutyValues) > 0 { summary.FanDutyCycleAvailable = true summary.AvgFanDutyCyclePct = benchmarkMean(dutyValues) summary.P95FanDutyCyclePct = benchmarkPercentile(dutyValues, 95) if summary.FanDutyCycleEstimated { summary.Notes = append(summary.Notes, "fan duty cycle is estimated from the highest fan RPM observed since boot; treat it as an approximation, not a direct PWM reading") } } else { summary.Notes = append(summary.Notes, "fan duty cycle unavailable on this host; RPM-only fan telemetry was collected") } return summary } func benchmarkTelemetryAvailable(summary BenchmarkTelemetrySummary) bool { return summary.Samples > 0 || summary.DurationSec > 0 } func benchmarkPrecisionSteadyFallback(phases []BenchmarkPrecisionSteadyPhase) (BenchmarkTelemetrySummary, string, bool) { var ( best BenchmarkTelemetrySummary bestLabel string found bool ) for _, phase := range phases { if !benchmarkTelemetryAvailable(phase.Steady) { continue } if !found || phase.Steady.DurationSec > best.DurationSec || (phase.Steady.DurationSec == best.DurationSec && phase.Steady.P95PowerW > best.P95PowerW) { best = phase.Steady bestLabel = phase.Precision found = true } } return best, bestLabel, found } func applyBenchmarkSteadyFallback(gpu *BenchmarkGPUResult) { if gpu == nil || benchmarkTelemetryAvailable(gpu.Steady) { return } if fallback, label, ok := benchmarkPrecisionSteadyFallback(gpu.PrecisionSteady); ok { gpu.Steady = fallback gpu.Notes = append(gpu.Notes, fmt.Sprintf("mixed steady telemetry unavailable; reporting steady-state fallback from %s precision phase", label)) } } func scoreBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkScorecard { score := BenchmarkScorecard{} // SyntheticScore: sum of fp32-equivalent TOPS from per-precision phases. // Each precision ran alone with full GPU dedicated — peak capability. for _, p := range gpu.PrecisionSteady { if !benchmarkPrecisionEnabled(p.Precision) { continue } score.SyntheticScore += p.WeightedTeraOpsPerSec } // MixedScore: sum of fp32-equivalent TOPS from the combined phase. // All precisions compete simultaneously — closer to real inference workloads. for _, p := range gpu.PrecisionResults { if p.Supported && benchmarkPrecisionEnabled(p.Category) { score.MixedScore += p.WeightedTeraOpsPerSec } } // MixedEfficiency = MixedScore / SyntheticScore. // Measures how well the GPU sustains throughput under concurrent mixed load. // A healthy GPU scores ~0.8–0.95; severe degradation suggests bandwidth // contention or scheduler inefficiency. if score.SyntheticScore > 0 && score.MixedScore > 0 { score.MixedEfficiency = score.MixedScore / score.SyntheticScore } // ComputeScore = SyntheticScore × (1 + MixedEfficiency × 0.3). // SyntheticScore is the primary signal; MixedEfficiency adds up to +30% // bonus for GPUs that handle mixed-precision concurrency well. // Falls back to MixedScore alone when per-precision data is absent. switch { case score.SyntheticScore > 0: score.ComputeScore = score.SyntheticScore * (1 + score.MixedEfficiency*0.3) case score.MixedScore > 0: score.ComputeScore = score.MixedScore } // PowerSustainScore: how stable is GPU power draw during the benchmark? // High variance means the workload is bursting or the power delivery is // unstable. Score = max(0, 100 − PowerCVPct × 3). // At 10% CV → score 70; at 33%+ CV → score 0. // Uses per-precision windows when available (each runs a single kernel, // so CV reflects genuine power regulation, not workload switching). if len(gpu.PrecisionSteady) > 0 { var sum float64 var count int for _, p := range gpu.PrecisionSteady { if !benchmarkPrecisionEnabled(p.Precision) { continue } sum += clampScore(100 - p.Steady.PowerCVPct*3) count++ } if count > 0 { score.PowerSustainScore = sum / float64(count) } } else if gpu.Steady.PowerCVPct > 0 { score.PowerSustainScore = clampScore(100 - gpu.Steady.PowerCVPct*3) } // ThermalSustainScore: how stable is GPU temperature during the benchmark? // High variance means cooling is inconsistent (fan bursts, liquid flow // instability, or frequent transitions in and out of throttle). // Score = max(0, 100 − TempCVPct × 3). if gpu.Steady.TempCVPct > 0 { score.ThermalSustainScore = clampScore(100 - gpu.Steady.TempCVPct*3) } else { // TempCV not recorded — fall back to 100 (no penalty). score.ThermalSustainScore = 100 } // Throttle breakdown: compute per-type percentages for diagnosis. // Each counter measures microseconds spent in that throttle state during // the steady-state window. Counters can overlap (e.g. thermal + power cap // simultaneously), so they are reported independently, not summed. runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6) score.ThermalThrottlePct = math.Min(100, float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS*100) score.PowerCapThrottlePct = math.Min(100, float64(gpu.Throttle.SWPowerCapUS)/runtimeUS*100) score.SyncBoostThrottlePct = math.Min(100, float64(gpu.Throttle.SyncBoostUS)/runtimeUS*100) // StabilityScore: combined throttle signal (thermal + power cap). // Score = max(0, 100 − combined_throttle_pct). // 1% throttle → 99; 10% → 90; any throttle > 0 is penalised. combinedThrottlePct := math.Min(100, float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS+gpu.Throttle.SWPowerCapUS)/runtimeUS*100) score.StabilityScore = clampScore(100 - combinedThrottlePct) // TempHeadroomC: distance from p95 temperature to the GPU's hardware // shutdown threshold (sourced from nvidia-smi -q "GPU Shutdown Temp"). // Fallback: 90°C when not available. // Assessed independently of throttle — a GPU at 86°C without any throttle // counter still has limited headroom and operates in degraded reliability zone. // Warning zone: headroom < (shutdownTemp - slowdownTemp), i.e. past slowdown onset. // Critical zone: headroom < 10°C from shutdown. if gpu.Steady.P95TempC > 0 { shutdownTemp := gpu.ShutdownTempC if shutdownTemp <= 0 { shutdownTemp = 90 } score.TempHeadroomC = shutdownTemp - gpu.Steady.P95TempC } score.ServerQualityScore = serverQualityScore(score) score.CompositeScore = score.ComputeScore if gpu.MultiprocessorCount > 0 && gpu.Steady.AvgGraphicsClockMHz > 0 && score.ComputeScore > 0 { score.TOPSPerSMPerGHz = score.ComputeScore / float64(gpu.MultiprocessorCount) / (gpu.Steady.AvgGraphicsClockMHz / 1000.0) } return score } // compositeBenchmarkScore is kept for compatibility with legacy callers. // CompositeScore = ComputeScore (no quality multiplier; throttling already // reduces TOPS directly, so no additional penalty is needed). func compositeBenchmarkScore(score BenchmarkScorecard) float64 { return score.ComputeScore } // serverQualityScore returns a 0–100 score reflecting server infrastructure // quality, independent of GPU model or compute speed. // // StabilityScore (throttle time) 0.40 — heaviest: direct evidence GPU can't sustain load // PowerSustainScore (power CV) 0.30 — unstable draw hints at PSU/VRM issues // ThermalSustainScore (temp CV) 0.30 — unstable temp hints at airflow/cooling issues func serverQualityScore(score BenchmarkScorecard) float64 { q := 0.40*(score.StabilityScore/100.0) + 0.30*(score.PowerSustainScore/100.0) + 0.30*(score.ThermalSustainScore/100.0) return clampScore(q * 100) } // detectPowerAnomaly scans per-GPU steady-state metric rows for a sudden // power drop — a symptom of bad cable contact, VRM fault, or thermal event // on the power delivery path. Returns a non-empty string if an anomaly is found. // // Algorithm: uses a 5-sample rolling baseline; flags any sample that falls // more than 30% below the baseline while the GPU was otherwise loaded // (usage > 50%). A sustained throttle (power cap) is not flagged here — // that is already captured by PowerCapThrottlePct. func detectPowerAnomaly(rows []GPUMetricRow, gpuIndex int) string { const windowSize = 5 const dropThresholdPct = 30.0 const minUsagePct = 50.0 // Filter rows for this GPU during steady state only. var steady []GPUMetricRow for _, r := range rows { if r.GPUIndex == gpuIndex && r.Stage != "" && strings.Contains(r.Stage, "steady") { steady = append(steady, r) } } if len(steady) < windowSize+2 { return "" } // Compute initial baseline from the first window. var baseSum float64 for i := 0; i < windowSize; i++ { baseSum += steady[i].PowerW } for i := windowSize; i < len(steady); i++ { baseline := baseSum / float64(windowSize) sample := steady[i] if baseline > 0 && sample.UsagePct >= minUsagePct { dropPct := (baseline - sample.PowerW) / baseline * 100 if dropPct >= dropThresholdPct { return fmt.Sprintf("sudden power drop detected at t=%.0fs: %.0f W → %.0f W (%.0f%% below rolling baseline) — possible bad cable contact or VRM fault", sample.ElapsedSec, baseline, sample.PowerW, dropPct) } } // Slide the window baseline. baseSum -= steady[i-windowSize].PowerW baseSum += sample.PowerW } return "" } // detectSlowdownTempExceedance scans steady-state metric rows for a GPU and // returns a warning string if any temperature sample exceeded the GPU's // SlowdownTempC threshold. Uses fallback 80°C when SlowdownTempC is zero. // This is a real-time signal distinct from p95 stats — even a single spike // above the slowdown threshold is worth flagging. func detectSlowdownTempExceedance(rows []GPUMetricRow, gpuIndex int, slowdownTempC float64) string { if slowdownTempC <= 0 { slowdownTempC = 80 } var maxTemp float64 var exceedCount int for _, r := range rows { if r.GPUIndex != gpuIndex { continue } if !strings.Contains(r.Stage, "steady") { continue } if r.TempC > maxTemp { maxTemp = r.TempC } if r.TempC >= slowdownTempC { exceedCount++ } } if exceedCount == 0 { return "" } return fmt.Sprintf( "temperature exceeded slowdown threshold (%.0f°C) in %d sample(s) during steady state — peak %.1f°C", slowdownTempC, exceedCount, maxTemp) } func detectBenchmarkDegradationReasons(gpu BenchmarkGPUResult, normalizationStatus string) []string { var reasons []string runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6) if float64(gpu.Throttle.SWPowerCapUS)/runtimeUS >= 0.05 { reasons = append(reasons, "power_capped") } if float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS >= 0.01 { reasons = append(reasons, "thermal_limited") } if float64(gpu.Throttle.SyncBoostUS)/runtimeUS >= 0.01 { reasons = append(reasons, "sync_boost_limited") } if gpu.LockedGraphicsClockMHz > 0 && gpu.Steady.AvgGraphicsClockMHz < gpu.LockedGraphicsClockMHz*0.90 { reasons = append(reasons, "low_sm_clock_vs_target") } if gpu.Scores.StabilityScore > 0 && gpu.Scores.StabilityScore < 85 { reasons = append(reasons, "variance_too_high") } if normalizationStatus != "full" { reasons = append(reasons, "normalization_partial") } if gpu.PowerLimitDerated { reasons = append(reasons, "power_limit_derated") } if gpu.ECC.Uncorrected > 0 { reasons = append(reasons, "ecc_uncorrected_errors") } if gpu.ECC.Corrected > 0 { reasons = append(reasons, "ecc_corrected_errors") } return dedupeStrings(reasons) }