refactor: modularize audit and harden build validation
This commit is contained in:
Mikhail Chusavitin
@@ -0,0 +1,659 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type benchmarkPlannedPhase struct {
|
||||
PlanLabel string
|
||||
MetricStage string
|
||||
DurationSec int
|
||||
}
|
||||
|
||||
func runBenchmarkPlannedCommandWithMetrics(
|
||||
ctx context.Context,
|
||||
verboseLog, name string,
|
||||
cmd []string,
|
||||
env []string,
|
||||
gpuIndices []int,
|
||||
phases []benchmarkPlannedPhase,
|
||||
logFunc func(string),
|
||||
) ([]byte, map[string][]GPUMetricRow, map[string][]byte, error) {
|
||||
out, rows, err := runBenchmarkCommandWithMetrics(ctx, verboseLog, name, cmd, env, gpuIndices, logFunc)
|
||||
return out, splitBenchmarkRowsByPlannedPhase(rows, phases), splitBenchmarkLogByPlannedPhase(out), err
|
||||
}
|
||||
|
||||
func splitBenchmarkRowsByPlannedPhase(rows []GPUMetricRow, phases []benchmarkPlannedPhase) map[string][]GPUMetricRow {
|
||||
out := make(map[string][]GPUMetricRow, len(phases))
|
||||
if len(rows) == 0 || len(phases) == 0 {
|
||||
return out
|
||||
}
|
||||
for _, row := range rows {
|
||||
idx := len(phases) - 1
|
||||
var elapsed float64
|
||||
for i, phase := range phases {
|
||||
durationSec := phase.DurationSec
|
||||
if durationSec <= 0 {
|
||||
durationSec = 1
|
||||
}
|
||||
elapsed += float64(durationSec)
|
||||
if row.ElapsedSec < elapsed {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
out[phases[idx].MetricStage] = append(out[phases[idx].MetricStage], row)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func splitBenchmarkLogByPlannedPhase(raw []byte) map[string][]byte {
|
||||
out := make(map[string][]byte)
|
||||
var current string
|
||||
for _, line := range strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") {
|
||||
trimmed := strings.TrimSpace(stripBenchmarkPrefix(line))
|
||||
switch {
|
||||
case strings.HasPrefix(trimmed, "phase_begin="):
|
||||
current = strings.TrimSpace(strings.TrimPrefix(trimmed, "phase_begin="))
|
||||
case strings.HasPrefix(trimmed, "phase_end="):
|
||||
current = ""
|
||||
case current != "":
|
||||
out[current] = append(out[current], []byte(line+"\n")...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type benchmarkCoolingSample struct {
|
||||
AvgFanRPM float64
|
||||
AvgFanDutyCyclePct float64
|
||||
FanDutyCycleAvailable bool
|
||||
FanDutyCycleEstimated bool
|
||||
}
|
||||
|
||||
func sampleBenchmarkTelemetry(gpuIndices []int) ([]GPUMetricRow, error) {
|
||||
samples, err := sampleGPUMetrics(gpuIndices)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fanSample := sampleBenchmarkCoolingSample()
|
||||
for i := range samples {
|
||||
samples[i].FanAvgRPM = fanSample.AvgFanRPM
|
||||
samples[i].FanDutyCyclePct = fanSample.AvgFanDutyCyclePct
|
||||
samples[i].FanDutyCycleAvailable = fanSample.FanDutyCycleAvailable
|
||||
samples[i].FanDutyCycleEstimated = fanSample.FanDutyCycleEstimated
|
||||
}
|
||||
return samples, nil
|
||||
}
|
||||
|
||||
func sampleBenchmarkCoolingSample() benchmarkCoolingSample {
|
||||
fans, _ := sampleFanSpeeds()
|
||||
avgRPM, _, _ := fanRPMStats(fans)
|
||||
dutyPct, dutyAvailable, dutyEstimated := sampleFanDutyCyclePctFromFans(fans)
|
||||
return benchmarkCoolingSample{
|
||||
AvgFanRPM: avgRPM,
|
||||
AvgFanDutyCyclePct: dutyPct,
|
||||
FanDutyCycleAvailable: dutyAvailable,
|
||||
FanDutyCycleEstimated: dutyEstimated,
|
||||
}
|
||||
}
|
||||
|
||||
func annotateBenchmarkMetricRows(rows []GPUMetricRow, stage string, offset, durationSec float64) []GPUMetricRow {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
stageEnd := offset + durationSec
|
||||
if stageEnd <= offset {
|
||||
stageEnd = offset
|
||||
for _, row := range rows {
|
||||
if row.ElapsedSec+offset > stageEnd {
|
||||
stageEnd = row.ElapsedSec + offset
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]GPUMetricRow, len(rows))
|
||||
for i, row := range rows {
|
||||
row.Stage = stage
|
||||
row.ElapsedSec += offset
|
||||
row.StageStartSec = offset
|
||||
row.StageEndSec = stageEnd
|
||||
out[i] = row
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func appendBenchmarkMetrics(allRows *[]GPUMetricRow, rows []GPUMetricRow, stage string, cursor *float64, durationSec float64) {
|
||||
annotated := annotateBenchmarkMetricRows(rows, stage, *cursor, durationSec)
|
||||
*allRows = append(*allRows, annotated...)
|
||||
*cursor += durationSec
|
||||
}
|
||||
|
||||
func writeBenchmarkMetricsFiles(runDir string, rows []GPUMetricRow) {
|
||||
if len(rows) == 0 {
|
||||
return
|
||||
}
|
||||
_ = WriteGPUMetricsCSV(filepath.Join(runDir, "gpu-metrics.csv"), rows)
|
||||
_ = WriteGPUMetricsHTML(filepath.Join(runDir, "gpu-metrics.html"), rows)
|
||||
}
|
||||
|
||||
func appendBenchmarkStageLog(path, source, stage string, raw []byte) {
|
||||
if path == "" || len(raw) == 0 {
|
||||
return
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
header := fmt.Sprintf("\n========== %s | stage=%s ==========\n", source, stage)
|
||||
_, _ = f.WriteString(header)
|
||||
if len(raw) > 0 {
|
||||
_, _ = f.Write(raw)
|
||||
if raw[len(raw)-1] != '\n' {
|
||||
_, _ = f.WriteString("\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseBenchmarkBurnLog(raw string) benchmarkBurnParseResult {
|
||||
result := benchmarkBurnParseResult{}
|
||||
lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
|
||||
profiles := make(map[string]*benchmarkBurnProfile)
|
||||
for _, line := range lines {
|
||||
line = stripBenchmarkPrefix(strings.TrimSpace(line))
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(line, "device="):
|
||||
result.Device = strings.TrimSpace(strings.TrimPrefix(line, "device="))
|
||||
case strings.HasPrefix(line, "compute_capability="):
|
||||
result.ComputeCapability = strings.TrimSpace(strings.TrimPrefix(line, "compute_capability="))
|
||||
case strings.HasPrefix(line, "backend="):
|
||||
result.Backend = strings.TrimSpace(strings.TrimPrefix(line, "backend="))
|
||||
result.Fallback = result.Backend == "driver-ptx"
|
||||
case strings.HasPrefix(line, "duration_s="):
|
||||
result.DurationSec, _ = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "duration_s=")))
|
||||
default:
|
||||
if m := benchmarkReadyPattern.FindStringSubmatch(line); len(m) == 6 {
|
||||
profile := ensureBenchmarkProfile(profiles, m[1])
|
||||
profile.supported = true
|
||||
profile.lanes++
|
||||
profile.m, _ = strconv.ParseUint(m[3], 10, 64)
|
||||
profile.n, _ = strconv.ParseUint(m[4], 10, 64)
|
||||
profile.k, _ = strconv.ParseUint(m[5], 10, 64)
|
||||
continue
|
||||
}
|
||||
if m := benchmarkSkippedPattern.FindStringSubmatch(line); len(m) == 3 {
|
||||
profile := ensureBenchmarkProfile(profiles, m[1])
|
||||
profile.supported = false
|
||||
profile.notes = strings.TrimSpace(m[2])
|
||||
continue
|
||||
}
|
||||
if m := benchmarkIterationsPattern.FindStringSubmatch(line); len(m) == 3 {
|
||||
profile := ensureBenchmarkProfile(profiles, m[1])
|
||||
iters, _ := strconv.ParseUint(m[2], 10, 64)
|
||||
profile.iterations += iters
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(profiles))
|
||||
for key := range profiles {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
profile := profiles[key]
|
||||
precision := BenchmarkPrecisionResult{
|
||||
Name: profile.name,
|
||||
Category: profile.category,
|
||||
Supported: profile.supported,
|
||||
Lanes: profile.lanes,
|
||||
M: profile.m,
|
||||
N: profile.n,
|
||||
K: profile.k,
|
||||
Iterations: profile.iterations,
|
||||
Notes: profile.notes,
|
||||
}
|
||||
w := precisionWeight(profile.category)
|
||||
precision.Weight = w
|
||||
if profile.supported && result.DurationSec > 0 && profile.m > 0 && profile.n > 0 && profile.k > 0 && profile.iterations > 0 {
|
||||
precision.TeraOpsPerSec = (2.0 * float64(profile.m) * float64(profile.n) * float64(profile.k) * float64(profile.iterations)) / float64(result.DurationSec) / 1e12
|
||||
precision.WeightedTeraOpsPerSec = precision.TeraOpsPerSec * w
|
||||
}
|
||||
result.Profiles = append(result.Profiles, precision)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func ensureBenchmarkProfile(profiles map[string]*benchmarkBurnProfile, name string) *benchmarkBurnProfile {
|
||||
if profile, ok := profiles[name]; ok {
|
||||
return profile
|
||||
}
|
||||
category := "other"
|
||||
switch {
|
||||
case strings.HasPrefix(name, "fp64"):
|
||||
category = "fp64"
|
||||
case strings.HasPrefix(name, "fp32"):
|
||||
category = "fp32_tf32"
|
||||
case strings.HasPrefix(name, "fp16"):
|
||||
category = "fp16_bf16"
|
||||
case strings.HasPrefix(name, "int8"):
|
||||
category = "int8"
|
||||
case strings.HasPrefix(name, "fp8"):
|
||||
category = "fp8"
|
||||
case strings.HasPrefix(name, "fp4"):
|
||||
category = "fp4"
|
||||
}
|
||||
profile := &benchmarkBurnProfile{name: name, category: category, supported: true}
|
||||
profiles[name] = profile
|
||||
return profile
|
||||
}
|
||||
|
||||
// precisionWeight returns the fp32-equivalence factor for a precision category.
|
||||
// Each factor represents how much "real" numeric work one operation of that
|
||||
// type performs relative to fp32 (single precision = 1.0 baseline):
|
||||
//
|
||||
// fp64 = 2.0 — double precision, 2× more bits per operand
|
||||
// fp32 = 1.0 — single precision baseline
|
||||
// fp16 = 0.5 — half precision
|
||||
// int8 = 0.25 — quarter precision
|
||||
// fp8 = 0.25 — quarter precision
|
||||
// fp4 = 0.125 — eighth precision
|
||||
//
|
||||
// Multiplying raw TOPS by the weight gives fp32-equivalent TOPS, enabling
|
||||
// cross-precision comparison on the same numeric scale.
|
||||
func precisionWeight(category string) float64 {
|
||||
switch category {
|
||||
case "fp64":
|
||||
return 2.0
|
||||
case "fp32_tf32":
|
||||
return 1.0
|
||||
case "fp16_bf16":
|
||||
return 0.5
|
||||
case "int8":
|
||||
return 0.25
|
||||
case "fp8":
|
||||
return 0.25
|
||||
case "fp4":
|
||||
return 0.125
|
||||
default:
|
||||
return 1.0
|
||||
}
|
||||
}
|
||||
|
||||
func stripBenchmarkPrefix(line string) string {
|
||||
if strings.HasPrefix(line, "[gpu ") {
|
||||
if idx := strings.Index(line, "] "); idx >= 0 {
|
||||
return line[idx+2:]
|
||||
}
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func summarizeBenchmarkTelemetry(rows []GPUMetricRow) BenchmarkTelemetrySummary {
|
||||
summary := BenchmarkTelemetrySummary{}
|
||||
if len(rows) == 0 {
|
||||
return summary
|
||||
}
|
||||
temps := make([]float64, 0, len(rows))
|
||||
powers := make([]float64, 0, len(rows))
|
||||
clocks := make([]float64, 0, len(rows))
|
||||
memClocks := make([]float64, 0, len(rows))
|
||||
usages := make([]float64, 0, len(rows))
|
||||
memUsages := make([]float64, 0, len(rows))
|
||||
summary.DurationSec = rows[len(rows)-1].ElapsedSec
|
||||
summary.Samples = len(rows)
|
||||
for _, row := range rows {
|
||||
temps = append(temps, row.TempC)
|
||||
powers = append(powers, row.PowerW)
|
||||
clocks = append(clocks, row.ClockMHz)
|
||||
memClocks = append(memClocks, row.MemClockMHz)
|
||||
usages = append(usages, row.UsagePct)
|
||||
memUsages = append(memUsages, row.MemUsagePct)
|
||||
}
|
||||
summary.AvgTempC = benchmarkMean(temps)
|
||||
summary.P95TempC = benchmarkPercentile(temps, 95)
|
||||
summary.AvgPowerW = benchmarkMean(powers)
|
||||
summary.P95PowerW = benchmarkPercentile(powers, 95)
|
||||
summary.AvgGraphicsClockMHz = benchmarkMean(clocks)
|
||||
summary.P95GraphicsClockMHz = benchmarkPercentile(clocks, 95)
|
||||
summary.AvgMemoryClockMHz = benchmarkMean(memClocks)
|
||||
summary.P95MemoryClockMHz = benchmarkPercentile(memClocks, 95)
|
||||
summary.AvgUsagePct = benchmarkMean(usages)
|
||||
summary.AvgMemUsagePct = benchmarkMean(memUsages)
|
||||
summary.ClockCVPct = benchmarkCV(clocks)
|
||||
summary.PowerCVPct = benchmarkCV(powers)
|
||||
summary.TempCVPct = benchmarkCV(temps)
|
||||
summary.ClockDriftPct = benchmarkClockDrift(clocks)
|
||||
return summary
|
||||
}
|
||||
|
||||
func summarizeBenchmarkCooling(rows []GPUMetricRow) *BenchmarkCoolingSummary {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
var rpmValues []float64
|
||||
var dutyValues []float64
|
||||
var dutyEstimated bool
|
||||
for _, row := range rows {
|
||||
if row.FanAvgRPM > 0 {
|
||||
rpmValues = append(rpmValues, row.FanAvgRPM)
|
||||
}
|
||||
if row.FanDutyCycleAvailable {
|
||||
dutyValues = append(dutyValues, row.FanDutyCyclePct)
|
||||
if row.FanDutyCycleEstimated {
|
||||
dutyEstimated = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(rpmValues) == 0 && len(dutyValues) == 0 {
|
||||
return nil
|
||||
}
|
||||
summary := &BenchmarkCoolingSummary{
|
||||
Available: true,
|
||||
AvgFanRPM: benchmarkMean(rpmValues),
|
||||
FanDutyCycleEstimated: dutyEstimated,
|
||||
}
|
||||
if len(dutyValues) > 0 {
|
||||
summary.FanDutyCycleAvailable = true
|
||||
summary.AvgFanDutyCyclePct = benchmarkMean(dutyValues)
|
||||
summary.P95FanDutyCyclePct = benchmarkPercentile(dutyValues, 95)
|
||||
if summary.FanDutyCycleEstimated {
|
||||
summary.Notes = append(summary.Notes, "fan duty cycle is estimated from the highest fan RPM observed since boot; treat it as an approximation, not a direct PWM reading")
|
||||
}
|
||||
} else {
|
||||
summary.Notes = append(summary.Notes, "fan duty cycle unavailable on this host; RPM-only fan telemetry was collected")
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
func benchmarkTelemetryAvailable(summary BenchmarkTelemetrySummary) bool {
|
||||
return summary.Samples > 0 || summary.DurationSec > 0
|
||||
}
|
||||
|
||||
func benchmarkPrecisionSteadyFallback(phases []BenchmarkPrecisionSteadyPhase) (BenchmarkTelemetrySummary, string, bool) {
|
||||
var (
|
||||
best BenchmarkTelemetrySummary
|
||||
bestLabel string
|
||||
found bool
|
||||
)
|
||||
for _, phase := range phases {
|
||||
if !benchmarkTelemetryAvailable(phase.Steady) {
|
||||
continue
|
||||
}
|
||||
if !found ||
|
||||
phase.Steady.DurationSec > best.DurationSec ||
|
||||
(phase.Steady.DurationSec == best.DurationSec && phase.Steady.P95PowerW > best.P95PowerW) {
|
||||
best = phase.Steady
|
||||
bestLabel = phase.Precision
|
||||
found = true
|
||||
}
|
||||
}
|
||||
return best, bestLabel, found
|
||||
}
|
||||
|
||||
func applyBenchmarkSteadyFallback(gpu *BenchmarkGPUResult) {
|
||||
if gpu == nil || benchmarkTelemetryAvailable(gpu.Steady) {
|
||||
return
|
||||
}
|
||||
if fallback, label, ok := benchmarkPrecisionSteadyFallback(gpu.PrecisionSteady); ok {
|
||||
gpu.Steady = fallback
|
||||
gpu.Notes = append(gpu.Notes,
|
||||
fmt.Sprintf("mixed steady telemetry unavailable; reporting steady-state fallback from %s precision phase", label))
|
||||
}
|
||||
}
|
||||
|
||||
func scoreBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkScorecard {
|
||||
score := BenchmarkScorecard{}
|
||||
|
||||
// SyntheticScore: sum of fp32-equivalent TOPS from per-precision phases.
|
||||
// Each precision ran alone with full GPU dedicated — peak capability.
|
||||
for _, p := range gpu.PrecisionSteady {
|
||||
if !benchmarkPrecisionEnabled(p.Precision) {
|
||||
continue
|
||||
}
|
||||
score.SyntheticScore += p.WeightedTeraOpsPerSec
|
||||
}
|
||||
|
||||
// MixedScore: sum of fp32-equivalent TOPS from the combined phase.
|
||||
// All precisions compete simultaneously — closer to real inference workloads.
|
||||
for _, p := range gpu.PrecisionResults {
|
||||
if p.Supported && benchmarkPrecisionEnabled(p.Category) {
|
||||
score.MixedScore += p.WeightedTeraOpsPerSec
|
||||
}
|
||||
}
|
||||
|
||||
// MixedEfficiency = MixedScore / SyntheticScore.
|
||||
// Measures how well the GPU sustains throughput under concurrent mixed load.
|
||||
// A healthy GPU scores ~0.8–0.95; severe degradation suggests bandwidth
|
||||
// contention or scheduler inefficiency.
|
||||
if score.SyntheticScore > 0 && score.MixedScore > 0 {
|
||||
score.MixedEfficiency = score.MixedScore / score.SyntheticScore
|
||||
}
|
||||
|
||||
// ComputeScore = SyntheticScore × (1 + MixedEfficiency × 0.3).
|
||||
// SyntheticScore is the primary signal; MixedEfficiency adds up to +30%
|
||||
// bonus for GPUs that handle mixed-precision concurrency well.
|
||||
// Falls back to MixedScore alone when per-precision data is absent.
|
||||
switch {
|
||||
case score.SyntheticScore > 0:
|
||||
score.ComputeScore = score.SyntheticScore * (1 + score.MixedEfficiency*0.3)
|
||||
case score.MixedScore > 0:
|
||||
score.ComputeScore = score.MixedScore
|
||||
}
|
||||
// PowerSustainScore: how stable is GPU power draw during the benchmark?
|
||||
// High variance means the workload is bursting or the power delivery is
|
||||
// unstable. Score = max(0, 100 − PowerCVPct × 3).
|
||||
// At 10% CV → score 70; at 33%+ CV → score 0.
|
||||
// Uses per-precision windows when available (each runs a single kernel,
|
||||
// so CV reflects genuine power regulation, not workload switching).
|
||||
if len(gpu.PrecisionSteady) > 0 {
|
||||
var sum float64
|
||||
var count int
|
||||
for _, p := range gpu.PrecisionSteady {
|
||||
if !benchmarkPrecisionEnabled(p.Precision) {
|
||||
continue
|
||||
}
|
||||
sum += clampScore(100 - p.Steady.PowerCVPct*3)
|
||||
count++
|
||||
}
|
||||
if count > 0 {
|
||||
score.PowerSustainScore = sum / float64(count)
|
||||
}
|
||||
} else if gpu.Steady.PowerCVPct > 0 {
|
||||
score.PowerSustainScore = clampScore(100 - gpu.Steady.PowerCVPct*3)
|
||||
}
|
||||
|
||||
// ThermalSustainScore: how stable is GPU temperature during the benchmark?
|
||||
// High variance means cooling is inconsistent (fan bursts, liquid flow
|
||||
// instability, or frequent transitions in and out of throttle).
|
||||
// Score = max(0, 100 − TempCVPct × 3).
|
||||
if gpu.Steady.TempCVPct > 0 {
|
||||
score.ThermalSustainScore = clampScore(100 - gpu.Steady.TempCVPct*3)
|
||||
} else {
|
||||
// TempCV not recorded — fall back to 100 (no penalty).
|
||||
score.ThermalSustainScore = 100
|
||||
}
|
||||
|
||||
// Throttle breakdown: compute per-type percentages for diagnosis.
|
||||
// Each counter measures microseconds spent in that throttle state during
|
||||
// the steady-state window. Counters can overlap (e.g. thermal + power cap
|
||||
// simultaneously), so they are reported independently, not summed.
|
||||
runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6)
|
||||
score.ThermalThrottlePct = math.Min(100,
|
||||
float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS*100)
|
||||
score.PowerCapThrottlePct = math.Min(100,
|
||||
float64(gpu.Throttle.SWPowerCapUS)/runtimeUS*100)
|
||||
score.SyncBoostThrottlePct = math.Min(100,
|
||||
float64(gpu.Throttle.SyncBoostUS)/runtimeUS*100)
|
||||
|
||||
// StabilityScore: combined throttle signal (thermal + power cap).
|
||||
// Score = max(0, 100 − combined_throttle_pct).
|
||||
// 1% throttle → 99; 10% → 90; any throttle > 0 is penalised.
|
||||
combinedThrottlePct := math.Min(100,
|
||||
float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS+gpu.Throttle.SWPowerCapUS)/runtimeUS*100)
|
||||
score.StabilityScore = clampScore(100 - combinedThrottlePct)
|
||||
|
||||
// TempHeadroomC: distance from p95 temperature to the GPU's hardware
|
||||
// shutdown threshold (sourced from nvidia-smi -q "GPU Shutdown Temp").
|
||||
// Fallback: 90°C when not available.
|
||||
// Assessed independently of throttle — a GPU at 86°C without any throttle
|
||||
// counter still has limited headroom and operates in degraded reliability zone.
|
||||
// Warning zone: headroom < (shutdownTemp - slowdownTemp), i.e. past slowdown onset.
|
||||
// Critical zone: headroom < 10°C from shutdown.
|
||||
if gpu.Steady.P95TempC > 0 {
|
||||
shutdownTemp := gpu.ShutdownTempC
|
||||
if shutdownTemp <= 0 {
|
||||
shutdownTemp = 90
|
||||
}
|
||||
score.TempHeadroomC = shutdownTemp - gpu.Steady.P95TempC
|
||||
}
|
||||
score.ServerQualityScore = serverQualityScore(score)
|
||||
score.CompositeScore = score.ComputeScore
|
||||
if gpu.MultiprocessorCount > 0 && gpu.Steady.AvgGraphicsClockMHz > 0 && score.ComputeScore > 0 {
|
||||
score.TOPSPerSMPerGHz = score.ComputeScore / float64(gpu.MultiprocessorCount) / (gpu.Steady.AvgGraphicsClockMHz / 1000.0)
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
// compositeBenchmarkScore is kept for compatibility with legacy callers.
|
||||
// CompositeScore = ComputeScore (no quality multiplier; throttling already
|
||||
// reduces TOPS directly, so no additional penalty is needed).
|
||||
func compositeBenchmarkScore(score BenchmarkScorecard) float64 {
|
||||
return score.ComputeScore
|
||||
}
|
||||
|
||||
// serverQualityScore returns a 0–100 score reflecting server infrastructure
|
||||
// quality, independent of GPU model or compute speed.
|
||||
//
|
||||
// StabilityScore (throttle time) 0.40 — heaviest: direct evidence GPU can't sustain load
|
||||
// PowerSustainScore (power CV) 0.30 — unstable draw hints at PSU/VRM issues
|
||||
// ThermalSustainScore (temp CV) 0.30 — unstable temp hints at airflow/cooling issues
|
||||
func serverQualityScore(score BenchmarkScorecard) float64 {
|
||||
q := 0.40*(score.StabilityScore/100.0) +
|
||||
0.30*(score.PowerSustainScore/100.0) +
|
||||
0.30*(score.ThermalSustainScore/100.0)
|
||||
return clampScore(q * 100)
|
||||
}
|
||||
|
||||
// detectPowerAnomaly scans per-GPU steady-state metric rows for a sudden
|
||||
// power drop — a symptom of bad cable contact, VRM fault, or thermal event
|
||||
// on the power delivery path. Returns a non-empty string if an anomaly is found.
|
||||
//
|
||||
// Algorithm: uses a 5-sample rolling baseline; flags any sample that falls
|
||||
// more than 30% below the baseline while the GPU was otherwise loaded
|
||||
// (usage > 50%). A sustained throttle (power cap) is not flagged here —
|
||||
// that is already captured by PowerCapThrottlePct.
|
||||
func detectPowerAnomaly(rows []GPUMetricRow, gpuIndex int) string {
|
||||
const windowSize = 5
|
||||
const dropThresholdPct = 30.0
|
||||
const minUsagePct = 50.0
|
||||
|
||||
// Filter rows for this GPU during steady state only.
|
||||
var steady []GPUMetricRow
|
||||
for _, r := range rows {
|
||||
if r.GPUIndex == gpuIndex && r.Stage != "" && strings.Contains(r.Stage, "steady") {
|
||||
steady = append(steady, r)
|
||||
}
|
||||
}
|
||||
if len(steady) < windowSize+2 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Compute initial baseline from the first window.
|
||||
var baseSum float64
|
||||
for i := 0; i < windowSize; i++ {
|
||||
baseSum += steady[i].PowerW
|
||||
}
|
||||
|
||||
for i := windowSize; i < len(steady); i++ {
|
||||
baseline := baseSum / float64(windowSize)
|
||||
sample := steady[i]
|
||||
if baseline > 0 && sample.UsagePct >= minUsagePct {
|
||||
dropPct := (baseline - sample.PowerW) / baseline * 100
|
||||
if dropPct >= dropThresholdPct {
|
||||
return fmt.Sprintf("sudden power drop detected at t=%.0fs: %.0f W → %.0f W (%.0f%% below rolling baseline) — possible bad cable contact or VRM fault",
|
||||
sample.ElapsedSec, baseline, sample.PowerW, dropPct)
|
||||
}
|
||||
}
|
||||
// Slide the window baseline.
|
||||
baseSum -= steady[i-windowSize].PowerW
|
||||
baseSum += sample.PowerW
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// detectSlowdownTempExceedance scans steady-state metric rows for a GPU and
|
||||
// returns a warning string if any temperature sample exceeded the GPU's
|
||||
// SlowdownTempC threshold. Uses fallback 80°C when SlowdownTempC is zero.
|
||||
// This is a real-time signal distinct from p95 stats — even a single spike
|
||||
// above the slowdown threshold is worth flagging.
|
||||
func detectSlowdownTempExceedance(rows []GPUMetricRow, gpuIndex int, slowdownTempC float64) string {
|
||||
if slowdownTempC <= 0 {
|
||||
slowdownTempC = 80
|
||||
}
|
||||
var maxTemp float64
|
||||
var exceedCount int
|
||||
for _, r := range rows {
|
||||
if r.GPUIndex != gpuIndex {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(r.Stage, "steady") {
|
||||
continue
|
||||
}
|
||||
if r.TempC > maxTemp {
|
||||
maxTemp = r.TempC
|
||||
}
|
||||
if r.TempC >= slowdownTempC {
|
||||
exceedCount++
|
||||
}
|
||||
}
|
||||
if exceedCount == 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"temperature exceeded slowdown threshold (%.0f°C) in %d sample(s) during steady state — peak %.1f°C",
|
||||
slowdownTempC, exceedCount, maxTemp)
|
||||
}
|
||||
|
||||
func detectBenchmarkDegradationReasons(gpu BenchmarkGPUResult, normalizationStatus string) []string {
|
||||
var reasons []string
|
||||
runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6)
|
||||
if float64(gpu.Throttle.SWPowerCapUS)/runtimeUS >= 0.05 {
|
||||
reasons = append(reasons, "power_capped")
|
||||
}
|
||||
if float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS >= 0.01 {
|
||||
reasons = append(reasons, "thermal_limited")
|
||||
}
|
||||
if float64(gpu.Throttle.SyncBoostUS)/runtimeUS >= 0.01 {
|
||||
reasons = append(reasons, "sync_boost_limited")
|
||||
}
|
||||
if gpu.LockedGraphicsClockMHz > 0 && gpu.Steady.AvgGraphicsClockMHz < gpu.LockedGraphicsClockMHz*0.90 {
|
||||
reasons = append(reasons, "low_sm_clock_vs_target")
|
||||
}
|
||||
if gpu.Scores.StabilityScore > 0 && gpu.Scores.StabilityScore < 85 {
|
||||
reasons = append(reasons, "variance_too_high")
|
||||
}
|
||||
if normalizationStatus != "full" {
|
||||
reasons = append(reasons, "normalization_partial")
|
||||
}
|
||||
if gpu.PowerLimitDerated {
|
||||
reasons = append(reasons, "power_limit_derated")
|
||||
}
|
||||
if gpu.ECC.Uncorrected > 0 {
|
||||
reasons = append(reasons, "ecc_uncorrected_errors")
|
||||
}
|
||||
if gpu.ECC.Corrected > 0 {
|
||||
reasons = append(reasons, "ecc_corrected_errors")
|
||||
}
|
||||
return dedupeStrings(reasons)
|
||||
}
|
||||
Reference in New Issue
Block a user