refactor: modularize audit and harden build validation
This commit is contained in:
Mikhail Chusavitin
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,659 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type benchmarkPlannedPhase struct {
|
||||
PlanLabel string
|
||||
MetricStage string
|
||||
DurationSec int
|
||||
}
|
||||
|
||||
func runBenchmarkPlannedCommandWithMetrics(
|
||||
ctx context.Context,
|
||||
verboseLog, name string,
|
||||
cmd []string,
|
||||
env []string,
|
||||
gpuIndices []int,
|
||||
phases []benchmarkPlannedPhase,
|
||||
logFunc func(string),
|
||||
) ([]byte, map[string][]GPUMetricRow, map[string][]byte, error) {
|
||||
out, rows, err := runBenchmarkCommandWithMetrics(ctx, verboseLog, name, cmd, env, gpuIndices, logFunc)
|
||||
return out, splitBenchmarkRowsByPlannedPhase(rows, phases), splitBenchmarkLogByPlannedPhase(out), err
|
||||
}
|
||||
|
||||
func splitBenchmarkRowsByPlannedPhase(rows []GPUMetricRow, phases []benchmarkPlannedPhase) map[string][]GPUMetricRow {
|
||||
out := make(map[string][]GPUMetricRow, len(phases))
|
||||
if len(rows) == 0 || len(phases) == 0 {
|
||||
return out
|
||||
}
|
||||
for _, row := range rows {
|
||||
idx := len(phases) - 1
|
||||
var elapsed float64
|
||||
for i, phase := range phases {
|
||||
durationSec := phase.DurationSec
|
||||
if durationSec <= 0 {
|
||||
durationSec = 1
|
||||
}
|
||||
elapsed += float64(durationSec)
|
||||
if row.ElapsedSec < elapsed {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
out[phases[idx].MetricStage] = append(out[phases[idx].MetricStage], row)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func splitBenchmarkLogByPlannedPhase(raw []byte) map[string][]byte {
|
||||
out := make(map[string][]byte)
|
||||
var current string
|
||||
for _, line := range strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") {
|
||||
trimmed := strings.TrimSpace(stripBenchmarkPrefix(line))
|
||||
switch {
|
||||
case strings.HasPrefix(trimmed, "phase_begin="):
|
||||
current = strings.TrimSpace(strings.TrimPrefix(trimmed, "phase_begin="))
|
||||
case strings.HasPrefix(trimmed, "phase_end="):
|
||||
current = ""
|
||||
case current != "":
|
||||
out[current] = append(out[current], []byte(line+"\n")...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type benchmarkCoolingSample struct {
|
||||
AvgFanRPM float64
|
||||
AvgFanDutyCyclePct float64
|
||||
FanDutyCycleAvailable bool
|
||||
FanDutyCycleEstimated bool
|
||||
}
|
||||
|
||||
func sampleBenchmarkTelemetry(gpuIndices []int) ([]GPUMetricRow, error) {
|
||||
samples, err := sampleGPUMetrics(gpuIndices)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fanSample := sampleBenchmarkCoolingSample()
|
||||
for i := range samples {
|
||||
samples[i].FanAvgRPM = fanSample.AvgFanRPM
|
||||
samples[i].FanDutyCyclePct = fanSample.AvgFanDutyCyclePct
|
||||
samples[i].FanDutyCycleAvailable = fanSample.FanDutyCycleAvailable
|
||||
samples[i].FanDutyCycleEstimated = fanSample.FanDutyCycleEstimated
|
||||
}
|
||||
return samples, nil
|
||||
}
|
||||
|
||||
func sampleBenchmarkCoolingSample() benchmarkCoolingSample {
|
||||
fans, _ := sampleFanSpeeds()
|
||||
avgRPM, _, _ := fanRPMStats(fans)
|
||||
dutyPct, dutyAvailable, dutyEstimated := sampleFanDutyCyclePctFromFans(fans)
|
||||
return benchmarkCoolingSample{
|
||||
AvgFanRPM: avgRPM,
|
||||
AvgFanDutyCyclePct: dutyPct,
|
||||
FanDutyCycleAvailable: dutyAvailable,
|
||||
FanDutyCycleEstimated: dutyEstimated,
|
||||
}
|
||||
}
|
||||
|
||||
func annotateBenchmarkMetricRows(rows []GPUMetricRow, stage string, offset, durationSec float64) []GPUMetricRow {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
stageEnd := offset + durationSec
|
||||
if stageEnd <= offset {
|
||||
stageEnd = offset
|
||||
for _, row := range rows {
|
||||
if row.ElapsedSec+offset > stageEnd {
|
||||
stageEnd = row.ElapsedSec + offset
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]GPUMetricRow, len(rows))
|
||||
for i, row := range rows {
|
||||
row.Stage = stage
|
||||
row.ElapsedSec += offset
|
||||
row.StageStartSec = offset
|
||||
row.StageEndSec = stageEnd
|
||||
out[i] = row
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func appendBenchmarkMetrics(allRows *[]GPUMetricRow, rows []GPUMetricRow, stage string, cursor *float64, durationSec float64) {
|
||||
annotated := annotateBenchmarkMetricRows(rows, stage, *cursor, durationSec)
|
||||
*allRows = append(*allRows, annotated...)
|
||||
*cursor += durationSec
|
||||
}
|
||||
|
||||
func writeBenchmarkMetricsFiles(runDir string, rows []GPUMetricRow) {
|
||||
if len(rows) == 0 {
|
||||
return
|
||||
}
|
||||
_ = WriteGPUMetricsCSV(filepath.Join(runDir, "gpu-metrics.csv"), rows)
|
||||
_ = WriteGPUMetricsHTML(filepath.Join(runDir, "gpu-metrics.html"), rows)
|
||||
}
|
||||
|
||||
func appendBenchmarkStageLog(path, source, stage string, raw []byte) {
|
||||
if path == "" || len(raw) == 0 {
|
||||
return
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
header := fmt.Sprintf("\n========== %s | stage=%s ==========\n", source, stage)
|
||||
_, _ = f.WriteString(header)
|
||||
if len(raw) > 0 {
|
||||
_, _ = f.Write(raw)
|
||||
if raw[len(raw)-1] != '\n' {
|
||||
_, _ = f.WriteString("\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseBenchmarkBurnLog(raw string) benchmarkBurnParseResult {
|
||||
result := benchmarkBurnParseResult{}
|
||||
lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
|
||||
profiles := make(map[string]*benchmarkBurnProfile)
|
||||
for _, line := range lines {
|
||||
line = stripBenchmarkPrefix(strings.TrimSpace(line))
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(line, "device="):
|
||||
result.Device = strings.TrimSpace(strings.TrimPrefix(line, "device="))
|
||||
case strings.HasPrefix(line, "compute_capability="):
|
||||
result.ComputeCapability = strings.TrimSpace(strings.TrimPrefix(line, "compute_capability="))
|
||||
case strings.HasPrefix(line, "backend="):
|
||||
result.Backend = strings.TrimSpace(strings.TrimPrefix(line, "backend="))
|
||||
result.Fallback = result.Backend == "driver-ptx"
|
||||
case strings.HasPrefix(line, "duration_s="):
|
||||
result.DurationSec, _ = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "duration_s=")))
|
||||
default:
|
||||
if m := benchmarkReadyPattern.FindStringSubmatch(line); len(m) == 6 {
|
||||
profile := ensureBenchmarkProfile(profiles, m[1])
|
||||
profile.supported = true
|
||||
profile.lanes++
|
||||
profile.m, _ = strconv.ParseUint(m[3], 10, 64)
|
||||
profile.n, _ = strconv.ParseUint(m[4], 10, 64)
|
||||
profile.k, _ = strconv.ParseUint(m[5], 10, 64)
|
||||
continue
|
||||
}
|
||||
if m := benchmarkSkippedPattern.FindStringSubmatch(line); len(m) == 3 {
|
||||
profile := ensureBenchmarkProfile(profiles, m[1])
|
||||
profile.supported = false
|
||||
profile.notes = strings.TrimSpace(m[2])
|
||||
continue
|
||||
}
|
||||
if m := benchmarkIterationsPattern.FindStringSubmatch(line); len(m) == 3 {
|
||||
profile := ensureBenchmarkProfile(profiles, m[1])
|
||||
iters, _ := strconv.ParseUint(m[2], 10, 64)
|
||||
profile.iterations += iters
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(profiles))
|
||||
for key := range profiles {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
profile := profiles[key]
|
||||
precision := BenchmarkPrecisionResult{
|
||||
Name: profile.name,
|
||||
Category: profile.category,
|
||||
Supported: profile.supported,
|
||||
Lanes: profile.lanes,
|
||||
M: profile.m,
|
||||
N: profile.n,
|
||||
K: profile.k,
|
||||
Iterations: profile.iterations,
|
||||
Notes: profile.notes,
|
||||
}
|
||||
w := precisionWeight(profile.category)
|
||||
precision.Weight = w
|
||||
if profile.supported && result.DurationSec > 0 && profile.m > 0 && profile.n > 0 && profile.k > 0 && profile.iterations > 0 {
|
||||
precision.TeraOpsPerSec = (2.0 * float64(profile.m) * float64(profile.n) * float64(profile.k) * float64(profile.iterations)) / float64(result.DurationSec) / 1e12
|
||||
precision.WeightedTeraOpsPerSec = precision.TeraOpsPerSec * w
|
||||
}
|
||||
result.Profiles = append(result.Profiles, precision)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func ensureBenchmarkProfile(profiles map[string]*benchmarkBurnProfile, name string) *benchmarkBurnProfile {
|
||||
if profile, ok := profiles[name]; ok {
|
||||
return profile
|
||||
}
|
||||
category := "other"
|
||||
switch {
|
||||
case strings.HasPrefix(name, "fp64"):
|
||||
category = "fp64"
|
||||
case strings.HasPrefix(name, "fp32"):
|
||||
category = "fp32_tf32"
|
||||
case strings.HasPrefix(name, "fp16"):
|
||||
category = "fp16_bf16"
|
||||
case strings.HasPrefix(name, "int8"):
|
||||
category = "int8"
|
||||
case strings.HasPrefix(name, "fp8"):
|
||||
category = "fp8"
|
||||
case strings.HasPrefix(name, "fp4"):
|
||||
category = "fp4"
|
||||
}
|
||||
profile := &benchmarkBurnProfile{name: name, category: category, supported: true}
|
||||
profiles[name] = profile
|
||||
return profile
|
||||
}
|
||||
|
||||
// precisionWeight returns the fp32-equivalence factor for a precision category.
|
||||
// Each factor represents how much "real" numeric work one operation of that
|
||||
// type performs relative to fp32 (single precision = 1.0 baseline):
|
||||
//
|
||||
// fp64 = 2.0 — double precision, 2× more bits per operand
|
||||
// fp32 = 1.0 — single precision baseline
|
||||
// fp16 = 0.5 — half precision
|
||||
// int8 = 0.25 — quarter precision
|
||||
// fp8 = 0.25 — quarter precision
|
||||
// fp4 = 0.125 — eighth precision
|
||||
//
|
||||
// Multiplying raw TOPS by the weight gives fp32-equivalent TOPS, enabling
|
||||
// cross-precision comparison on the same numeric scale.
|
||||
func precisionWeight(category string) float64 {
|
||||
switch category {
|
||||
case "fp64":
|
||||
return 2.0
|
||||
case "fp32_tf32":
|
||||
return 1.0
|
||||
case "fp16_bf16":
|
||||
return 0.5
|
||||
case "int8":
|
||||
return 0.25
|
||||
case "fp8":
|
||||
return 0.25
|
||||
case "fp4":
|
||||
return 0.125
|
||||
default:
|
||||
return 1.0
|
||||
}
|
||||
}
|
||||
|
||||
func stripBenchmarkPrefix(line string) string {
|
||||
if strings.HasPrefix(line, "[gpu ") {
|
||||
if idx := strings.Index(line, "] "); idx >= 0 {
|
||||
return line[idx+2:]
|
||||
}
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func summarizeBenchmarkTelemetry(rows []GPUMetricRow) BenchmarkTelemetrySummary {
|
||||
summary := BenchmarkTelemetrySummary{}
|
||||
if len(rows) == 0 {
|
||||
return summary
|
||||
}
|
||||
temps := make([]float64, 0, len(rows))
|
||||
powers := make([]float64, 0, len(rows))
|
||||
clocks := make([]float64, 0, len(rows))
|
||||
memClocks := make([]float64, 0, len(rows))
|
||||
usages := make([]float64, 0, len(rows))
|
||||
memUsages := make([]float64, 0, len(rows))
|
||||
summary.DurationSec = rows[len(rows)-1].ElapsedSec
|
||||
summary.Samples = len(rows)
|
||||
for _, row := range rows {
|
||||
temps = append(temps, row.TempC)
|
||||
powers = append(powers, row.PowerW)
|
||||
clocks = append(clocks, row.ClockMHz)
|
||||
memClocks = append(memClocks, row.MemClockMHz)
|
||||
usages = append(usages, row.UsagePct)
|
||||
memUsages = append(memUsages, row.MemUsagePct)
|
||||
}
|
||||
summary.AvgTempC = benchmarkMean(temps)
|
||||
summary.P95TempC = benchmarkPercentile(temps, 95)
|
||||
summary.AvgPowerW = benchmarkMean(powers)
|
||||
summary.P95PowerW = benchmarkPercentile(powers, 95)
|
||||
summary.AvgGraphicsClockMHz = benchmarkMean(clocks)
|
||||
summary.P95GraphicsClockMHz = benchmarkPercentile(clocks, 95)
|
||||
summary.AvgMemoryClockMHz = benchmarkMean(memClocks)
|
||||
summary.P95MemoryClockMHz = benchmarkPercentile(memClocks, 95)
|
||||
summary.AvgUsagePct = benchmarkMean(usages)
|
||||
summary.AvgMemUsagePct = benchmarkMean(memUsages)
|
||||
summary.ClockCVPct = benchmarkCV(clocks)
|
||||
summary.PowerCVPct = benchmarkCV(powers)
|
||||
summary.TempCVPct = benchmarkCV(temps)
|
||||
summary.ClockDriftPct = benchmarkClockDrift(clocks)
|
||||
return summary
|
||||
}
|
||||
|
||||
func summarizeBenchmarkCooling(rows []GPUMetricRow) *BenchmarkCoolingSummary {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
var rpmValues []float64
|
||||
var dutyValues []float64
|
||||
var dutyEstimated bool
|
||||
for _, row := range rows {
|
||||
if row.FanAvgRPM > 0 {
|
||||
rpmValues = append(rpmValues, row.FanAvgRPM)
|
||||
}
|
||||
if row.FanDutyCycleAvailable {
|
||||
dutyValues = append(dutyValues, row.FanDutyCyclePct)
|
||||
if row.FanDutyCycleEstimated {
|
||||
dutyEstimated = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(rpmValues) == 0 && len(dutyValues) == 0 {
|
||||
return nil
|
||||
}
|
||||
summary := &BenchmarkCoolingSummary{
|
||||
Available: true,
|
||||
AvgFanRPM: benchmarkMean(rpmValues),
|
||||
FanDutyCycleEstimated: dutyEstimated,
|
||||
}
|
||||
if len(dutyValues) > 0 {
|
||||
summary.FanDutyCycleAvailable = true
|
||||
summary.AvgFanDutyCyclePct = benchmarkMean(dutyValues)
|
||||
summary.P95FanDutyCyclePct = benchmarkPercentile(dutyValues, 95)
|
||||
if summary.FanDutyCycleEstimated {
|
||||
summary.Notes = append(summary.Notes, "fan duty cycle is estimated from the highest fan RPM observed since boot; treat it as an approximation, not a direct PWM reading")
|
||||
}
|
||||
} else {
|
||||
summary.Notes = append(summary.Notes, "fan duty cycle unavailable on this host; RPM-only fan telemetry was collected")
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
func benchmarkTelemetryAvailable(summary BenchmarkTelemetrySummary) bool {
|
||||
return summary.Samples > 0 || summary.DurationSec > 0
|
||||
}
|
||||
|
||||
func benchmarkPrecisionSteadyFallback(phases []BenchmarkPrecisionSteadyPhase) (BenchmarkTelemetrySummary, string, bool) {
|
||||
var (
|
||||
best BenchmarkTelemetrySummary
|
||||
bestLabel string
|
||||
found bool
|
||||
)
|
||||
for _, phase := range phases {
|
||||
if !benchmarkTelemetryAvailable(phase.Steady) {
|
||||
continue
|
||||
}
|
||||
if !found ||
|
||||
phase.Steady.DurationSec > best.DurationSec ||
|
||||
(phase.Steady.DurationSec == best.DurationSec && phase.Steady.P95PowerW > best.P95PowerW) {
|
||||
best = phase.Steady
|
||||
bestLabel = phase.Precision
|
||||
found = true
|
||||
}
|
||||
}
|
||||
return best, bestLabel, found
|
||||
}
|
||||
|
||||
func applyBenchmarkSteadyFallback(gpu *BenchmarkGPUResult) {
|
||||
if gpu == nil || benchmarkTelemetryAvailable(gpu.Steady) {
|
||||
return
|
||||
}
|
||||
if fallback, label, ok := benchmarkPrecisionSteadyFallback(gpu.PrecisionSteady); ok {
|
||||
gpu.Steady = fallback
|
||||
gpu.Notes = append(gpu.Notes,
|
||||
fmt.Sprintf("mixed steady telemetry unavailable; reporting steady-state fallback from %s precision phase", label))
|
||||
}
|
||||
}
|
||||
|
||||
func scoreBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkScorecard {
|
||||
score := BenchmarkScorecard{}
|
||||
|
||||
// SyntheticScore: sum of fp32-equivalent TOPS from per-precision phases.
|
||||
// Each precision ran alone with full GPU dedicated — peak capability.
|
||||
for _, p := range gpu.PrecisionSteady {
|
||||
if !benchmarkPrecisionEnabled(p.Precision) {
|
||||
continue
|
||||
}
|
||||
score.SyntheticScore += p.WeightedTeraOpsPerSec
|
||||
}
|
||||
|
||||
// MixedScore: sum of fp32-equivalent TOPS from the combined phase.
|
||||
// All precisions compete simultaneously — closer to real inference workloads.
|
||||
for _, p := range gpu.PrecisionResults {
|
||||
if p.Supported && benchmarkPrecisionEnabled(p.Category) {
|
||||
score.MixedScore += p.WeightedTeraOpsPerSec
|
||||
}
|
||||
}
|
||||
|
||||
// MixedEfficiency = MixedScore / SyntheticScore.
|
||||
// Measures how well the GPU sustains throughput under concurrent mixed load.
|
||||
// A healthy GPU scores ~0.8–0.95; severe degradation suggests bandwidth
|
||||
// contention or scheduler inefficiency.
|
||||
if score.SyntheticScore > 0 && score.MixedScore > 0 {
|
||||
score.MixedEfficiency = score.MixedScore / score.SyntheticScore
|
||||
}
|
||||
|
||||
// ComputeScore = SyntheticScore × (1 + MixedEfficiency × 0.3).
|
||||
// SyntheticScore is the primary signal; MixedEfficiency adds up to +30%
|
||||
// bonus for GPUs that handle mixed-precision concurrency well.
|
||||
// Falls back to MixedScore alone when per-precision data is absent.
|
||||
switch {
|
||||
case score.SyntheticScore > 0:
|
||||
score.ComputeScore = score.SyntheticScore * (1 + score.MixedEfficiency*0.3)
|
||||
case score.MixedScore > 0:
|
||||
score.ComputeScore = score.MixedScore
|
||||
}
|
||||
// PowerSustainScore: how stable is GPU power draw during the benchmark?
|
||||
// High variance means the workload is bursting or the power delivery is
|
||||
// unstable. Score = max(0, 100 − PowerCVPct × 3).
|
||||
// At 10% CV → score 70; at 33%+ CV → score 0.
|
||||
// Uses per-precision windows when available (each runs a single kernel,
|
||||
// so CV reflects genuine power regulation, not workload switching).
|
||||
if len(gpu.PrecisionSteady) > 0 {
|
||||
var sum float64
|
||||
var count int
|
||||
for _, p := range gpu.PrecisionSteady {
|
||||
if !benchmarkPrecisionEnabled(p.Precision) {
|
||||
continue
|
||||
}
|
||||
sum += clampScore(100 - p.Steady.PowerCVPct*3)
|
||||
count++
|
||||
}
|
||||
if count > 0 {
|
||||
score.PowerSustainScore = sum / float64(count)
|
||||
}
|
||||
} else if gpu.Steady.PowerCVPct > 0 {
|
||||
score.PowerSustainScore = clampScore(100 - gpu.Steady.PowerCVPct*3)
|
||||
}
|
||||
|
||||
// ThermalSustainScore: how stable is GPU temperature during the benchmark?
|
||||
// High variance means cooling is inconsistent (fan bursts, liquid flow
|
||||
// instability, or frequent transitions in and out of throttle).
|
||||
// Score = max(0, 100 − TempCVPct × 3).
|
||||
if gpu.Steady.TempCVPct > 0 {
|
||||
score.ThermalSustainScore = clampScore(100 - gpu.Steady.TempCVPct*3)
|
||||
} else {
|
||||
// TempCV not recorded — fall back to 100 (no penalty).
|
||||
score.ThermalSustainScore = 100
|
||||
}
|
||||
|
||||
// Throttle breakdown: compute per-type percentages for diagnosis.
|
||||
// Each counter measures microseconds spent in that throttle state during
|
||||
// the steady-state window. Counters can overlap (e.g. thermal + power cap
|
||||
// simultaneously), so they are reported independently, not summed.
|
||||
runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6)
|
||||
score.ThermalThrottlePct = math.Min(100,
|
||||
float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS*100)
|
||||
score.PowerCapThrottlePct = math.Min(100,
|
||||
float64(gpu.Throttle.SWPowerCapUS)/runtimeUS*100)
|
||||
score.SyncBoostThrottlePct = math.Min(100,
|
||||
float64(gpu.Throttle.SyncBoostUS)/runtimeUS*100)
|
||||
|
||||
// StabilityScore: combined throttle signal (thermal + power cap).
|
||||
// Score = max(0, 100 − combined_throttle_pct).
|
||||
// 1% throttle → 99; 10% → 90; any throttle > 0 is penalised.
|
||||
combinedThrottlePct := math.Min(100,
|
||||
float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS+gpu.Throttle.SWPowerCapUS)/runtimeUS*100)
|
||||
score.StabilityScore = clampScore(100 - combinedThrottlePct)
|
||||
|
||||
// TempHeadroomC: distance from p95 temperature to the GPU's hardware
|
||||
// shutdown threshold (sourced from nvidia-smi -q "GPU Shutdown Temp").
|
||||
// Fallback: 90°C when not available.
|
||||
// Assessed independently of throttle — a GPU at 86°C without any throttle
|
||||
// counter still has limited headroom and operates in degraded reliability zone.
|
||||
// Warning zone: headroom < (shutdownTemp - slowdownTemp), i.e. past slowdown onset.
|
||||
// Critical zone: headroom < 10°C from shutdown.
|
||||
if gpu.Steady.P95TempC > 0 {
|
||||
shutdownTemp := gpu.ShutdownTempC
|
||||
if shutdownTemp <= 0 {
|
||||
shutdownTemp = 90
|
||||
}
|
||||
score.TempHeadroomC = shutdownTemp - gpu.Steady.P95TempC
|
||||
}
|
||||
score.ServerQualityScore = serverQualityScore(score)
|
||||
score.CompositeScore = score.ComputeScore
|
||||
if gpu.MultiprocessorCount > 0 && gpu.Steady.AvgGraphicsClockMHz > 0 && score.ComputeScore > 0 {
|
||||
score.TOPSPerSMPerGHz = score.ComputeScore / float64(gpu.MultiprocessorCount) / (gpu.Steady.AvgGraphicsClockMHz / 1000.0)
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
// compositeBenchmarkScore is kept for compatibility with legacy callers.
|
||||
// CompositeScore = ComputeScore (no quality multiplier; throttling already
|
||||
// reduces TOPS directly, so no additional penalty is needed).
|
||||
func compositeBenchmarkScore(score BenchmarkScorecard) float64 {
|
||||
return score.ComputeScore
|
||||
}
|
||||
|
||||
// serverQualityScore returns a 0–100 score reflecting server infrastructure
|
||||
// quality, independent of GPU model or compute speed.
|
||||
//
|
||||
// StabilityScore (throttle time) 0.40 — heaviest: direct evidence GPU can't sustain load
|
||||
// PowerSustainScore (power CV) 0.30 — unstable draw hints at PSU/VRM issues
|
||||
// ThermalSustainScore (temp CV) 0.30 — unstable temp hints at airflow/cooling issues
|
||||
func serverQualityScore(score BenchmarkScorecard) float64 {
|
||||
q := 0.40*(score.StabilityScore/100.0) +
|
||||
0.30*(score.PowerSustainScore/100.0) +
|
||||
0.30*(score.ThermalSustainScore/100.0)
|
||||
return clampScore(q * 100)
|
||||
}
|
||||
|
||||
// detectPowerAnomaly scans per-GPU steady-state metric rows for a sudden
|
||||
// power drop — a symptom of bad cable contact, VRM fault, or thermal event
|
||||
// on the power delivery path. Returns a non-empty string if an anomaly is found.
|
||||
//
|
||||
// Algorithm: uses a 5-sample rolling baseline; flags any sample that falls
|
||||
// more than 30% below the baseline while the GPU was otherwise loaded
|
||||
// (usage > 50%). A sustained throttle (power cap) is not flagged here —
|
||||
// that is already captured by PowerCapThrottlePct.
|
||||
func detectPowerAnomaly(rows []GPUMetricRow, gpuIndex int) string {
|
||||
const windowSize = 5
|
||||
const dropThresholdPct = 30.0
|
||||
const minUsagePct = 50.0
|
||||
|
||||
// Filter rows for this GPU during steady state only.
|
||||
var steady []GPUMetricRow
|
||||
for _, r := range rows {
|
||||
if r.GPUIndex == gpuIndex && r.Stage != "" && strings.Contains(r.Stage, "steady") {
|
||||
steady = append(steady, r)
|
||||
}
|
||||
}
|
||||
if len(steady) < windowSize+2 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Compute initial baseline from the first window.
|
||||
var baseSum float64
|
||||
for i := 0; i < windowSize; i++ {
|
||||
baseSum += steady[i].PowerW
|
||||
}
|
||||
|
||||
for i := windowSize; i < len(steady); i++ {
|
||||
baseline := baseSum / float64(windowSize)
|
||||
sample := steady[i]
|
||||
if baseline > 0 && sample.UsagePct >= minUsagePct {
|
||||
dropPct := (baseline - sample.PowerW) / baseline * 100
|
||||
if dropPct >= dropThresholdPct {
|
||||
return fmt.Sprintf("sudden power drop detected at t=%.0fs: %.0f W → %.0f W (%.0f%% below rolling baseline) — possible bad cable contact or VRM fault",
|
||||
sample.ElapsedSec, baseline, sample.PowerW, dropPct)
|
||||
}
|
||||
}
|
||||
// Slide the window baseline.
|
||||
baseSum -= steady[i-windowSize].PowerW
|
||||
baseSum += sample.PowerW
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// detectSlowdownTempExceedance scans steady-state metric rows for a GPU and
|
||||
// returns a warning string if any temperature sample exceeded the GPU's
|
||||
// SlowdownTempC threshold. Uses fallback 80°C when SlowdownTempC is zero.
|
||||
// This is a real-time signal distinct from p95 stats — even a single spike
|
||||
// above the slowdown threshold is worth flagging.
|
||||
func detectSlowdownTempExceedance(rows []GPUMetricRow, gpuIndex int, slowdownTempC float64) string {
|
||||
if slowdownTempC <= 0 {
|
||||
slowdownTempC = 80
|
||||
}
|
||||
var maxTemp float64
|
||||
var exceedCount int
|
||||
for _, r := range rows {
|
||||
if r.GPUIndex != gpuIndex {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(r.Stage, "steady") {
|
||||
continue
|
||||
}
|
||||
if r.TempC > maxTemp {
|
||||
maxTemp = r.TempC
|
||||
}
|
||||
if r.TempC >= slowdownTempC {
|
||||
exceedCount++
|
||||
}
|
||||
}
|
||||
if exceedCount == 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"temperature exceeded slowdown threshold (%.0f°C) in %d sample(s) during steady state — peak %.1f°C",
|
||||
slowdownTempC, exceedCount, maxTemp)
|
||||
}
|
||||
|
||||
func detectBenchmarkDegradationReasons(gpu BenchmarkGPUResult, normalizationStatus string) []string {
|
||||
var reasons []string
|
||||
runtimeUS := math.Max(1, gpu.Steady.DurationSec*1e6)
|
||||
if float64(gpu.Throttle.SWPowerCapUS)/runtimeUS >= 0.05 {
|
||||
reasons = append(reasons, "power_capped")
|
||||
}
|
||||
if float64(gpu.Throttle.HWThermalSlowdownUS+gpu.Throttle.SWThermalSlowdownUS)/runtimeUS >= 0.01 {
|
||||
reasons = append(reasons, "thermal_limited")
|
||||
}
|
||||
if float64(gpu.Throttle.SyncBoostUS)/runtimeUS >= 0.01 {
|
||||
reasons = append(reasons, "sync_boost_limited")
|
||||
}
|
||||
if gpu.LockedGraphicsClockMHz > 0 && gpu.Steady.AvgGraphicsClockMHz < gpu.LockedGraphicsClockMHz*0.90 {
|
||||
reasons = append(reasons, "low_sm_clock_vs_target")
|
||||
}
|
||||
if gpu.Scores.StabilityScore > 0 && gpu.Scores.StabilityScore < 85 {
|
||||
reasons = append(reasons, "variance_too_high")
|
||||
}
|
||||
if normalizationStatus != "full" {
|
||||
reasons = append(reasons, "normalization_partial")
|
||||
}
|
||||
if gpu.PowerLimitDerated {
|
||||
reasons = append(reasons, "power_limit_derated")
|
||||
}
|
||||
if gpu.ECC.Uncorrected > 0 {
|
||||
reasons = append(reasons, "ecc_uncorrected_errors")
|
||||
}
|
||||
if gpu.ECC.Corrected > 0 {
|
||||
reasons = append(reasons, "ecc_corrected_errors")
|
||||
}
|
||||
return dedupeStrings(reasons)
|
||||
}
|
||||
@@ -69,16 +69,6 @@ func LoadSystemPowerSourceConfig(exportDir string) (*BenchmarkPowerAutotuneConfi
|
||||
return LoadBenchmarkPowerAutotuneConfig(BenchmarkPowerSourceConfigPath(exportDir))
|
||||
}
|
||||
|
||||
func ResetBenchmarkPowerAutotuneConfig(path string) error {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return fmt.Errorf("empty autotune config path")
|
||||
}
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeBenchmarkPowerSource(source string) string {
|
||||
switch strings.TrimSpace(strings.ToLower(source)) {
|
||||
case BenchmarkPowerSourceSDRPSUInput:
|
||||
|
||||
@@ -0,0 +1,928 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"bee/audit/internal/collector"
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func detectDCMIPartialCoverage(sp *BenchmarkServerPower) bool {
|
||||
if sp == nil || !sp.Available {
|
||||
return false
|
||||
}
|
||||
if sp.PSUInputIdleW > 0 && sp.IdleW > 0 {
|
||||
return sp.IdleW/sp.PSUInputIdleW < 0.7
|
||||
}
|
||||
if sp.PSUInputLoadedW > 0 && sp.LoadedW > 0 {
|
||||
return sp.LoadedW/sp.PSUInputLoadedW < 0.7
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// detectIPMISaturationFallback is the heuristic used when SDR PSU sensors are
|
||||
// absent. It analyses the power ramp: if 2+ of the last 3 per-step incremental
|
||||
// DCMI/GPU ratios fall below 25% of the first-step ratio, DCMI has likely
|
||||
// plateaued while GPU load continued to grow (saturation proxy).
|
||||
// Prefer detectDCMIPartialCoverage when SDR data is available.
|
||||
func detectIPMISaturationFallback(steps []NvidiaPowerBenchStep) bool {
|
||||
type pt struct{ incIPMI, incGPU float64 }
|
||||
var pts []pt
|
||||
for i := 1; i < len(steps); i++ {
|
||||
if steps[i].ServerDeltaW <= 0 || steps[i-1].ServerDeltaW <= 0 {
|
||||
continue
|
||||
}
|
||||
incIPMI := steps[i].ServerDeltaW - steps[i-1].ServerDeltaW
|
||||
incGPU := steps[i].TotalObservedPowerW - steps[i-1].TotalObservedPowerW
|
||||
if incGPU <= 0 {
|
||||
continue
|
||||
}
|
||||
pts = append(pts, pt{incIPMI, incGPU})
|
||||
}
|
||||
if len(pts) < 3 {
|
||||
return false
|
||||
}
|
||||
refRatio := pts[0].incIPMI / pts[0].incGPU
|
||||
if refRatio <= 0 {
|
||||
return false
|
||||
}
|
||||
saturated := 0
|
||||
for _, p := range pts[len(pts)-3:] {
|
||||
if p.incIPMI/p.incGPU < refRatio*0.25 {
|
||||
saturated++
|
||||
}
|
||||
}
|
||||
return saturated >= 2
|
||||
}
|
||||
|
||||
// psuStatusSnapshot samples PSU health sensor states via
|
||||
// `ipmitool sdr type "Power Supply"`. Returns a map of sensor name → reading
|
||||
// string (e.g. "Presence detected", "Failure detected"). Returns nil when IPMI
|
||||
// is unavailable or no Power Supply entity sensors are present.
|
||||
func psuStatusSnapshot() map[string]string {
|
||||
out, err := exec.Command("ipmitool", "sdr", "type", "Power Supply").Output()
|
||||
if err != nil || len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
result := make(map[string]string)
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
parts := strings.Split(line, "|")
|
||||
if len(parts) < 5 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(parts[0])
|
||||
reading := strings.TrimSpace(parts[4])
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
result[name] = reading
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// diffPSUStatus compares PSU sensor snapshots taken before and after a test.
|
||||
// Returns human-readable fault strings for sensors that entered a fault state
|
||||
// during the test. Pre-existing faults (present in both snapshots) are excluded
|
||||
// so that only new anomalies caused by the test are reported.
|
||||
func diffPSUStatus(before, after map[string]string) []string {
|
||||
if len(after) == 0 {
|
||||
return nil
|
||||
}
|
||||
isFault := func(s string) bool {
|
||||
lower := strings.ToLower(s)
|
||||
return strings.Contains(lower, "failure") ||
|
||||
strings.Contains(lower, "fault") ||
|
||||
strings.Contains(lower, "warning") ||
|
||||
strings.Contains(lower, "predictive") ||
|
||||
strings.Contains(lower, "absent") ||
|
||||
strings.Contains(lower, "ac lost")
|
||||
}
|
||||
var issues []string
|
||||
for name, afterReading := range after {
|
||||
if !isFault(afterReading) {
|
||||
continue
|
||||
}
|
||||
if beforeReading, had := before[name]; had && isFault(beforeReading) {
|
||||
continue // pre-existing fault, not caused by this test
|
||||
}
|
||||
if prev, had := before[name]; had {
|
||||
issues = append(issues, fmt.Sprintf("%s: changed from %q to %q during test", name, prev, afterReading))
|
||||
} else {
|
||||
issues = append(issues, fmt.Sprintf("%s: %s (appeared after test start)", name, afterReading))
|
||||
}
|
||||
}
|
||||
sort.Strings(issues)
|
||||
return issues
|
||||
}
|
||||
|
||||
// sdrPowerSnapshot holds per-source power sums from a single `ipmitool sdr` read.
|
||||
type sdrPowerSnapshot struct {
|
||||
PSUInW float64 // sum of PSU AC input across all slots
|
||||
PSUOutW float64 // sum of PSU DC output across all slots
|
||||
GPUSlotW float64 // sum of GPU slot/GPU power sensors
|
||||
|
||||
// Per-slot PSU data from collector.PSUSlotsFromSDR — same slot keys as
|
||||
// audit HardwarePowerSupply.Slot (0-based strings).
|
||||
PSUSlots map[string]BenchmarkPSUSlotPower
|
||||
|
||||
SkippedSensors []string // sensors rejected during self-healing
|
||||
}
|
||||
|
||||
type benchmarkSDRSeriesSummary struct {
|
||||
PSUInW float64
|
||||
PSUOutW float64
|
||||
GPUSlotW float64
|
||||
PSUSlots map[string]BenchmarkPSUSlotPower
|
||||
Samples int
|
||||
|
||||
SkippedSensors []string
|
||||
}
|
||||
|
||||
// sdrSensor is a name+watts pair used for GPU slot self-healing filtering.
|
||||
type sdrSensor struct {
|
||||
name string
|
||||
watts float64
|
||||
}
|
||||
|
||||
// filterSensorGroup removes physically implausible readings from a group.
|
||||
// Hard bounds: 0 < watts ≤ maxPerSensorW. Within groups of 2+ sensors,
|
||||
// values more than 5× the group median are rejected as stuck/fault sensors.
|
||||
func filterSensorGroup(sensors []sdrSensor, maxPerSensorW float64) (valid []sdrSensor, skipped []string) {
|
||||
var inBounds []sdrSensor
|
||||
for _, s := range sensors {
|
||||
if s.watts <= 0 || s.watts > maxPerSensorW {
|
||||
skipped = append(skipped, fmt.Sprintf("%s (%.0f W: out of range 0–%.0f W)", s.name, s.watts, maxPerSensorW))
|
||||
} else {
|
||||
inBounds = append(inBounds, s)
|
||||
}
|
||||
}
|
||||
if len(inBounds) < 2 {
|
||||
return inBounds, skipped
|
||||
}
|
||||
vals := make([]float64, len(inBounds))
|
||||
for i, s := range inBounds {
|
||||
vals[i] = s.watts
|
||||
}
|
||||
sort.Float64s(vals)
|
||||
mid := len(vals) / 2
|
||||
var median float64
|
||||
if len(vals)%2 == 0 {
|
||||
median = (vals[mid-1] + vals[mid]) / 2
|
||||
} else {
|
||||
median = vals[mid]
|
||||
}
|
||||
for _, s := range inBounds {
|
||||
if median > 0 && s.watts > median*5 {
|
||||
skipped = append(skipped, fmt.Sprintf("%s (%.0f W: >5× median %.0f W, likely sensor fault)", s.name, s.watts, median))
|
||||
} else {
|
||||
valid = append(valid, s)
|
||||
}
|
||||
}
|
||||
return valid, skipped
|
||||
}
|
||||
|
||||
// sampleIPMISDRPowerSensors reads power sensors from `ipmitool sdr` in a single
|
||||
// invocation and returns self-healed grouped sums.
|
||||
//
|
||||
// PSU identification delegates to collector.PSUSlotsFromSDR which uses the same
|
||||
// slot-detection regexes as the hardware audit (PSU1_POWER_IN, PSU1_PIN, PS1 POut,
|
||||
// Power1…). Self-healing: bounds checking + 5× median outlier rejection.
|
||||
//
|
||||
// GPU slot sensors (GPU_POWER_SLOTx, GPU1 Power, …) are classified separately
|
||||
// since the audit collector does not track GPU PCIe slot power.
|
||||
func sampleIPMISDRPowerSensors() sdrPowerSnapshot {
|
||||
raw, err := exec.Command("ipmitool", "sdr").Output()
|
||||
if err != nil || len(raw) == 0 {
|
||||
return sdrPowerSnapshot{}
|
||||
}
|
||||
sdrStr := string(raw)
|
||||
var snap sdrPowerSnapshot
|
||||
|
||||
// ── PSU data via audit collector ─────────────────────────────────────────
|
||||
// collector.PSUSlotsFromSDR handles all vendor naming variants and applies
|
||||
// bounds checking inside parseBoundedFloat (0 < w ≤ 6000 W).
|
||||
collectorSlots := collector.PSUSlotsFromSDR(sdrStr)
|
||||
|
||||
// Convert to benchmark type and apply cross-slot median filtering.
|
||||
var psuInSensors, psuOutSensors []sdrSensor
|
||||
for slotKey, sp := range collectorSlots {
|
||||
bsp := BenchmarkPSUSlotPower{Status: sp.Status}
|
||||
if sp.InputW != nil {
|
||||
bsp.InputW = sp.InputW
|
||||
psuInSensors = append(psuInSensors, sdrSensor{name: "PSU-slot-" + slotKey, watts: *sp.InputW})
|
||||
}
|
||||
if sp.OutputW != nil {
|
||||
bsp.OutputW = sp.OutputW
|
||||
psuOutSensors = append(psuOutSensors, sdrSensor{name: "PSU-slot-" + slotKey + "-out", watts: *sp.OutputW})
|
||||
}
|
||||
if snap.PSUSlots == nil {
|
||||
snap.PSUSlots = make(map[string]BenchmarkPSUSlotPower)
|
||||
}
|
||||
snap.PSUSlots[slotKey] = bsp
|
||||
}
|
||||
|
||||
// Apply cross-slot outlier filter and sum.
|
||||
validIn, skIn := filterSensorGroup(psuInSensors, 6000)
|
||||
for _, s := range validIn {
|
||||
snap.PSUInW += s.watts
|
||||
}
|
||||
snap.SkippedSensors = append(snap.SkippedSensors, skIn...)
|
||||
|
||||
validOut, skOut := filterSensorGroup(psuOutSensors, 6000)
|
||||
for _, s := range validOut {
|
||||
snap.PSUOutW += s.watts
|
||||
}
|
||||
snap.SkippedSensors = append(snap.SkippedSensors, skOut...)
|
||||
|
||||
// ── GPU slot sensors ─────────────────────────────────────────────────────
|
||||
// collector does not track GPU PCIe slot power; classify here.
|
||||
// Matches: GPU_POWER_SLOTx (MSI), GPU1 Power (xFusion), GPU_PWR_x (generic).
|
||||
var gpuSensors []sdrSensor
|
||||
for _, line := range strings.Split(sdrStr, "\n") {
|
||||
parts := strings.Split(line, "|")
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(parts[0])
|
||||
nameLower := strings.ToLower(name)
|
||||
if !strings.Contains(nameLower, "gpu") {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(nameLower, "slot") && !strings.Contains(nameLower, "power") &&
|
||||
!strings.Contains(nameLower, "pwr") {
|
||||
continue
|
||||
}
|
||||
var w float64
|
||||
if n, _ := fmt.Sscanf(strings.TrimSpace(parts[1]), "%f Watts", &w); n != 1 {
|
||||
continue
|
||||
}
|
||||
gpuSensors = append(gpuSensors, sdrSensor{name: name, watts: w})
|
||||
}
|
||||
validGPU, skGPU := filterSensorGroup(gpuSensors, 2000)
|
||||
for _, s := range validGPU {
|
||||
snap.GPUSlotW += s.watts
|
||||
}
|
||||
snap.SkippedSensors = append(snap.SkippedSensors, skGPU...)
|
||||
|
||||
return snap
|
||||
}
|
||||
|
||||
func startIPMISDRSampler(stopCh <-chan struct{}, intervalSec int) <-chan []sdrPowerSnapshot {
|
||||
if intervalSec <= 0 {
|
||||
intervalSec = benchmarkPowerAutotuneSampleInterval
|
||||
}
|
||||
ch := make(chan []sdrPowerSnapshot, 1)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
var samples []sdrPowerSnapshot
|
||||
record := func() {
|
||||
snap := sampleIPMISDRPowerSensors()
|
||||
if snap.PSUInW <= 0 && snap.PSUOutW <= 0 && snap.GPUSlotW <= 0 && len(snap.PSUSlots) == 0 {
|
||||
return
|
||||
}
|
||||
samples = append(samples, snap)
|
||||
}
|
||||
record()
|
||||
ticker := time.NewTicker(time.Duration(intervalSec) * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stopCh:
|
||||
ch <- samples
|
||||
return
|
||||
case <-ticker.C:
|
||||
record()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
func summarizeSDRPowerSeries(samples []sdrPowerSnapshot) benchmarkSDRSeriesSummary {
|
||||
var summary benchmarkSDRSeriesSummary
|
||||
if len(samples) == 0 {
|
||||
return summary
|
||||
}
|
||||
|
||||
type slotAggregate struct {
|
||||
inputs []float64
|
||||
outputs []float64
|
||||
status string
|
||||
}
|
||||
|
||||
slotAgg := make(map[string]*slotAggregate)
|
||||
skippedSet := make(map[string]struct{})
|
||||
var inputTotals []float64
|
||||
var outputTotals []float64
|
||||
var gpuSlotTotals []float64
|
||||
|
||||
for _, sample := range samples {
|
||||
if sample.PSUInW > 0 {
|
||||
inputTotals = append(inputTotals, sample.PSUInW)
|
||||
}
|
||||
if sample.PSUOutW > 0 {
|
||||
outputTotals = append(outputTotals, sample.PSUOutW)
|
||||
}
|
||||
if sample.GPUSlotW > 0 {
|
||||
gpuSlotTotals = append(gpuSlotTotals, sample.GPUSlotW)
|
||||
}
|
||||
for _, skipped := range sample.SkippedSensors {
|
||||
if skipped != "" {
|
||||
skippedSet[skipped] = struct{}{}
|
||||
}
|
||||
}
|
||||
for slot, reading := range sample.PSUSlots {
|
||||
agg := slotAgg[slot]
|
||||
if agg == nil {
|
||||
agg = &slotAggregate{}
|
||||
slotAgg[slot] = agg
|
||||
}
|
||||
if reading.InputW != nil && *reading.InputW > 0 {
|
||||
agg.inputs = append(agg.inputs, *reading.InputW)
|
||||
}
|
||||
if reading.OutputW != nil && *reading.OutputW > 0 {
|
||||
agg.outputs = append(agg.outputs, *reading.OutputW)
|
||||
}
|
||||
switch {
|
||||
case reading.Status == "":
|
||||
case agg.status == "":
|
||||
agg.status = reading.Status
|
||||
case agg.status == "OK" && reading.Status != "OK":
|
||||
agg.status = reading.Status
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
summary.PSUInW = benchmarkMean(inputTotals)
|
||||
summary.PSUOutW = benchmarkMean(outputTotals)
|
||||
summary.GPUSlotW = benchmarkMean(gpuSlotTotals)
|
||||
summary.Samples = len(samples)
|
||||
|
||||
if len(slotAgg) > 0 {
|
||||
summary.PSUSlots = make(map[string]BenchmarkPSUSlotPower, len(slotAgg))
|
||||
for slot, agg := range slotAgg {
|
||||
reading := BenchmarkPSUSlotPower{Status: agg.status}
|
||||
if mean := benchmarkMean(agg.inputs); mean > 0 {
|
||||
v := mean
|
||||
reading.InputW = &v
|
||||
}
|
||||
if mean := benchmarkMean(agg.outputs); mean > 0 {
|
||||
v := mean
|
||||
reading.OutputW = &v
|
||||
}
|
||||
summary.PSUSlots[slot] = reading
|
||||
}
|
||||
}
|
||||
if len(skippedSet) > 0 {
|
||||
summary.SkippedSensors = make([]string, 0, len(skippedSet))
|
||||
for skipped := range skippedSet {
|
||||
summary.SkippedSensors = append(summary.SkippedSensors, skipped)
|
||||
}
|
||||
sort.Strings(summary.SkippedSensors)
|
||||
}
|
||||
|
||||
return summary
|
||||
}
|
||||
|
||||
// queryIPMIServerPowerW reads the current server power draw via ipmitool dcmi.
|
||||
// Returns 0 and an error if IPMI is unavailable or the output cannot be parsed.
|
||||
func queryIPMIServerPowerW() (float64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "ipmitool", "dcmi", "power", "reading")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("ipmitool dcmi power reading: %w", err)
|
||||
}
|
||||
if w := parseDCMIPowerReading(string(out)); w > 0 {
|
||||
return w, nil
|
||||
}
|
||||
return 0, fmt.Errorf("could not parse ipmitool dcmi power reading output")
|
||||
}
|
||||
|
||||
// sampleIPMIPowerSeries collects IPMI power readings every 2 seconds for
|
||||
// durationSec seconds. Returns the mean of all successful samples.
|
||||
// Returns 0, false if IPMI is unavailable.
|
||||
func sampleIPMIPowerSeries(ctx context.Context, durationSec int) (meanW float64, ok bool) {
|
||||
if durationSec <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
deadline := time.Now().Add(time.Duration(durationSec) * time.Second)
|
||||
var samples []float64
|
||||
loop:
|
||||
for {
|
||||
if w, err := queryIPMIServerPowerW(); err == nil {
|
||||
samples = append(samples, w)
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break loop
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
if len(samples) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
var sum float64
|
||||
for _, w := range samples {
|
||||
sum += w
|
||||
}
|
||||
return sum / float64(len(samples)), true
|
||||
}
|
||||
|
||||
// characterizeServerPower computes BenchmarkServerPower from idle and loaded
|
||||
// samples plus the GPU-reported average power during steady state.
|
||||
func characterizeServerPower(idleW, loadedW, gpuReportedSumW float64, source string, available bool) *BenchmarkServerPower {
|
||||
sp := &BenchmarkServerPower{
|
||||
Available: available,
|
||||
Source: normalizeBenchmarkPowerSource(source),
|
||||
SampleIntervalSec: benchmarkPowerAutotuneSampleInterval,
|
||||
}
|
||||
if !available {
|
||||
sp.Notes = append(sp.Notes, "IPMI power reading unavailable; server-side power characterization skipped")
|
||||
return sp
|
||||
}
|
||||
sp.IdleW = idleW
|
||||
sp.LoadedW = loadedW
|
||||
sp.DeltaW = loadedW - idleW
|
||||
sp.GPUReportedSumW = gpuReportedSumW
|
||||
if gpuReportedSumW > 0 && sp.DeltaW > 0 {
|
||||
sp.ReportingRatio = sp.DeltaW / gpuReportedSumW
|
||||
}
|
||||
return sp
|
||||
}
|
||||
|
||||
// readServerModel returns the DMI system product name (e.g. "SuperMicro SYS-421GE-TNRT").
|
||||
// Returns empty string if unavailable (non-Linux or missing DMI entry).
|
||||
func readServerModel() string {
|
||||
data, err := os.ReadFile("/sys/class/dmi/id/product_name")
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(data))
|
||||
}
|
||||
|
||||
// filterRowsByGPU returns only the metric rows for a specific GPU index.
|
||||
func filterRowsByGPU(rows []GPUMetricRow, gpuIndex int) []GPUMetricRow {
|
||||
var out []GPUMetricRow
|
||||
for _, r := range rows {
|
||||
if r.GPUIndex == gpuIndex {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// parseBenchmarkBurnLogByGPU splits a multi-GPU bee-gpu-burn output by [gpu N] prefix
|
||||
// and returns a per-GPU parse result map.
|
||||
func parseBenchmarkBurnLogByGPU(raw string) map[int]benchmarkBurnParseResult {
|
||||
gpuLines := make(map[int][]string)
|
||||
for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(line, "[gpu ") {
|
||||
continue
|
||||
}
|
||||
end := strings.Index(line, "] ")
|
||||
if end < 0 {
|
||||
continue
|
||||
}
|
||||
gpuIdx, err := strconv.Atoi(strings.TrimSpace(line[5:end]))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
gpuLines[gpuIdx] = append(gpuLines[gpuIdx], line[end+2:])
|
||||
}
|
||||
results := make(map[int]benchmarkBurnParseResult, len(gpuLines))
|
||||
for gpuIdx, lines := range gpuLines {
|
||||
// Lines are already stripped of the [gpu N] prefix; parseBenchmarkBurnLog
|
||||
// calls stripBenchmarkPrefix which is a no-op on already-stripped lines.
|
||||
results[gpuIdx] = parseBenchmarkBurnLog(strings.Join(lines, "\n"))
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// runNvidiaBenchmarkParallel runs warmup and steady compute on all selected GPUs
|
||||
// simultaneously using a single bee-gpu-burn invocation per phase.
|
||||
func runNvidiaBenchmarkParallel(
|
||||
ctx context.Context,
|
||||
verboseLog, runDir string,
|
||||
selected []int,
|
||||
infoByIndex map[int]benchmarkGPUInfo,
|
||||
opts NvidiaBenchmarkOptions,
|
||||
spec benchmarkProfileSpec,
|
||||
logFunc func(string),
|
||||
result *NvidiaBenchmarkResult,
|
||||
calibByIndex map[int]benchmarkPowerCalibrationResult,
|
||||
serverIdleW *float64, serverLoadedWSum *float64,
|
||||
serverIdleOK *bool, serverLoadedOK *bool, serverLoadedSamples *int,
|
||||
allMetricRows *[]GPUMetricRow,
|
||||
metricTimelineSec *float64,
|
||||
gpuBurnLog string,
|
||||
) {
|
||||
allDevices := joinIndexList(selected)
|
||||
|
||||
// Build per-GPU result stubs.
|
||||
gpuResults := make(map[int]*BenchmarkGPUResult, len(selected))
|
||||
for _, idx := range selected {
|
||||
r := &BenchmarkGPUResult{Index: idx, Status: "FAILED"}
|
||||
if info, ok := infoByIndex[idx]; ok {
|
||||
r.UUID = info.UUID
|
||||
r.Name = info.Name
|
||||
r.BusID = info.BusID
|
||||
r.VBIOS = info.VBIOS
|
||||
r.PowerLimitW = info.PowerLimitW
|
||||
r.MultiprocessorCount = info.MultiprocessorCount
|
||||
r.DefaultPowerLimitW = info.DefaultPowerLimitW
|
||||
r.ShutdownTempC = info.ShutdownTempC
|
||||
r.SlowdownTempC = info.SlowdownTempC
|
||||
r.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz
|
||||
r.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz
|
||||
r.MaxMemoryClockMHz = info.MaxMemoryClockMHz
|
||||
}
|
||||
if calib, ok := calibByIndex[idx]; ok {
|
||||
r.CalibratedPeakPowerW = calib.Summary.P95PowerW
|
||||
r.CalibratedPeakTempC = calib.Summary.P95TempC
|
||||
r.PowerCalibrationTries = calib.Attempts
|
||||
r.PowerLimitDerated = calib.Derated
|
||||
r.Notes = append(r.Notes, calib.Notes...)
|
||||
if calib.CoolingWarning != "" {
|
||||
r.CoolingWarning = calib.CoolingWarning
|
||||
}
|
||||
}
|
||||
if norm := findBenchmarkNormalization(result.Normalization.GPUs, idx); norm != nil {
|
||||
r.LockedGraphicsClockMHz = norm.GPUClockLockMHz
|
||||
r.LockedMemoryClockMHz = norm.MemoryClockLockMHz
|
||||
}
|
||||
gpuResults[idx] = r
|
||||
}
|
||||
|
||||
// Baseline: sample all GPUs together.
|
||||
baselineRows, err := collectBenchmarkSamples(ctx, spec.BaselineSec, selected)
|
||||
if err != nil && err != context.Canceled {
|
||||
for _, idx := range selected {
|
||||
gpuResults[idx].Notes = append(gpuResults[idx].Notes, "baseline sampling failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
for _, idx := range selected {
|
||||
perGPU := filterRowsByGPU(baselineRows, idx)
|
||||
gpuResults[idx].Baseline = summarizeBenchmarkTelemetry(perGPU)
|
||||
}
|
||||
appendBenchmarkMetrics(allMetricRows, baselineRows, "baseline", metricTimelineSec, float64(spec.BaselineSec))
|
||||
|
||||
// Sample server idle power once.
|
||||
if !*serverIdleOK {
|
||||
if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, maxInt(spec.BaselineSec, 10), benchmarkPowerAutotuneSampleInterval); ok {
|
||||
*serverIdleW = w
|
||||
*serverIdleOK = true
|
||||
logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w))
|
||||
}
|
||||
}
|
||||
|
||||
// Warmup: all GPUs simultaneously.
|
||||
warmupCmd := []string{
|
||||
"bee-gpu-burn",
|
||||
"--seconds", strconv.Itoa(spec.WarmupSec),
|
||||
"--size-mb", strconv.Itoa(opts.SizeMB),
|
||||
"--devices", allDevices,
|
||||
}
|
||||
logFunc(fmt.Sprintf("GPUs %s: parallel warmup (%ds)", allDevices, spec.WarmupSec))
|
||||
warmupOut, warmupRows, warmupErr := runBenchmarkCommandWithMetrics(ctx, verboseLog, "gpu-all-warmup.log", warmupCmd, nil, selected, logFunc)
|
||||
appendBenchmarkMetrics(allMetricRows, warmupRows, "warmup", metricTimelineSec, float64(spec.WarmupSec))
|
||||
appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", "warmup", warmupOut)
|
||||
if warmupErr != nil {
|
||||
for _, idx := range selected {
|
||||
gpuResults[idx].Notes = append(gpuResults[idx].Notes, "parallel warmup failed: "+warmupErr.Error())
|
||||
}
|
||||
}
|
||||
warmupParseByGPU := parseBenchmarkBurnLogByGPU(string(warmupOut))
|
||||
supportedPrecisions := append([]string(nil), benchmarkPrecisionPhases...)
|
||||
for _, idx := range selected {
|
||||
if pr, ok := warmupParseByGPU[idx]; ok && pr.ComputeCapability != "" {
|
||||
if gpuResults[idx].ComputeCapability == "" {
|
||||
gpuResults[idx].ComputeCapability = pr.ComputeCapability
|
||||
}
|
||||
if ccPrecisions := benchmarkSupportedPrecisions(pr.ComputeCapability); len(ccPrecisions) < len(supportedPrecisions) {
|
||||
supportedPrecisions = ccPrecisions
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run synthetic precision phases and the combined steady phase as one
|
||||
// uninterrupted command so the GPUs stay hot between windows.
|
||||
eccBase := make(map[int]BenchmarkECCCounters, len(selected))
|
||||
for _, idx := range selected {
|
||||
eccBase[idx], _ = queryECCCounters(idx)
|
||||
}
|
||||
planLabels, planPhases, basePhaseSec, mixedPhaseSec := buildBenchmarkSteadyPlan(spec, supportedPrecisions, func(label string) string {
|
||||
if label == "mixed" {
|
||||
return "steady"
|
||||
}
|
||||
return "gpu-all-steady-" + label
|
||||
})
|
||||
planCmd := []string{
|
||||
"bee-gpu-burn",
|
||||
"--seconds", strconv.Itoa(basePhaseSec),
|
||||
"--size-mb", strconv.Itoa(opts.SizeMB),
|
||||
"--devices", allDevices,
|
||||
"--precision-plan", strings.Join(planLabels, ","),
|
||||
"--precision-plan-seconds", benchmarkPlanDurationsCSV(planPhases),
|
||||
}
|
||||
logFunc(fmt.Sprintf("GPUs %s: uninterrupted precision plan (%d precision phases x %ds, mixed %ds)", allDevices, len(supportedPrecisions), basePhaseSec, mixedPhaseSec))
|
||||
serverPowerStopCh := make(chan struct{})
|
||||
serverPowerCh := startSelectedPowerSourceSampler(serverPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
|
||||
_, phaseRowsByStage, phaseLogs, planErr := runBenchmarkPlannedCommandWithMetrics(ctx, verboseLog, "gpu-all-precision-plan.log", planCmd, nil, selected, planPhases, logFunc)
|
||||
close(serverPowerStopCh)
|
||||
if serverPowerSamples := <-serverPowerCh; len(serverPowerSamples) > 0 {
|
||||
*serverLoadedWSum += benchmarkMean(serverPowerSamples)
|
||||
(*serverLoadedSamples)++
|
||||
*serverLoadedOK = true
|
||||
logFunc(fmt.Sprintf("GPUs %s: server loaded power (%s avg): %.0f W", allDevices, opts.ServerPowerSource, benchmarkMean(serverPowerSamples)))
|
||||
}
|
||||
for _, phaseSpec := range planPhases {
|
||||
if rows := phaseRowsByStage[phaseSpec.MetricStage]; len(rows) > 0 {
|
||||
appendBenchmarkMetrics(allMetricRows, rows, phaseSpec.MetricStage, metricTimelineSec, float64(phaseSpec.DurationSec))
|
||||
}
|
||||
appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", phaseSpec.MetricStage, phaseLogs[phaseSpec.PlanLabel])
|
||||
}
|
||||
for _, prec := range supportedPrecisions {
|
||||
phaseLogName := "gpu-all-steady-" + prec
|
||||
phaseRows := phaseRowsByStage[phaseLogName]
|
||||
parseByGPU := parseBenchmarkBurnLogByGPU(string(phaseLogs[prec]))
|
||||
for _, idx := range selected {
|
||||
perGPU := filterRowsByGPU(phaseRows, idx)
|
||||
phase := BenchmarkPrecisionSteadyPhase{
|
||||
Precision: prec,
|
||||
Status: "OK",
|
||||
Steady: summarizeBenchmarkTelemetry(perGPU),
|
||||
}
|
||||
if status, note := benchmarkPlannedPhaseStatus(phaseLogs[prec]); status != "OK" {
|
||||
phase.Status = status
|
||||
phase.Notes = note
|
||||
gpuResults[idx].PrecisionFailures = append(gpuResults[idx].PrecisionFailures, prec+":"+status)
|
||||
}
|
||||
if pr, ok := parseByGPU[idx]; ok {
|
||||
for _, p := range pr.Profiles {
|
||||
if p.Supported {
|
||||
phase.TeraOpsPerSec += p.TeraOpsPerSec
|
||||
phase.WeightedTeraOpsPerSec += p.WeightedTeraOpsPerSec
|
||||
}
|
||||
}
|
||||
}
|
||||
gpuResults[idx].PrecisionSteady = append(gpuResults[idx].PrecisionSteady, phase)
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot throttle counters before steady.
|
||||
beforeThrottle := make(map[int]BenchmarkThrottleCounters, len(selected))
|
||||
for _, idx := range selected {
|
||||
beforeThrottle[idx], _ = queryThrottleCounters(idx)
|
||||
}
|
||||
|
||||
logFunc(fmt.Sprintf("GPUs %s: parallel steady compute (combined, %ds)", allDevices, mixedPhaseSec))
|
||||
afterThrottle := make(map[int]BenchmarkThrottleCounters, len(selected))
|
||||
for _, idx := range selected {
|
||||
afterThrottle[idx], _ = queryThrottleCounters(idx)
|
||||
}
|
||||
|
||||
steadyRows := phaseRowsByStage["steady"]
|
||||
parseResults := parseBenchmarkBurnLogByGPU(string(phaseLogs["mixed"]))
|
||||
|
||||
for _, idx := range selected {
|
||||
perGPU := filterRowsByGPU(steadyRows, idx)
|
||||
gpuResults[idx].Steady = summarizeBenchmarkTelemetry(perGPU)
|
||||
gpuResults[idx].Throttle = diffThrottleCounters(beforeThrottle[idx], afterThrottle[idx])
|
||||
if eccFinal, err := queryECCCounters(idx); err == nil {
|
||||
gpuResults[idx].ECC = diffECCCounters(eccBase[idx], eccFinal)
|
||||
}
|
||||
|
||||
if pr, ok := parseResults[idx]; ok {
|
||||
gpuResults[idx].ComputeCapability = pr.ComputeCapability
|
||||
gpuResults[idx].Backend = pr.Backend
|
||||
gpuResults[idx].PrecisionResults = pr.Profiles
|
||||
if pr.Fallback {
|
||||
gpuResults[idx].Notes = append(gpuResults[idx].Notes, "benchmark used driver PTX fallback; tensor throughput score is not comparable")
|
||||
}
|
||||
}
|
||||
if planErr != nil {
|
||||
gpuResults[idx].Notes = append(gpuResults[idx].Notes, "precision plan failed: "+planErr.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// Cooldown: all GPUs together.
|
||||
if spec.CooldownSec > 0 {
|
||||
cooldownRows, err := collectBenchmarkSamples(ctx, spec.CooldownSec, selected)
|
||||
if err != nil && err != context.Canceled {
|
||||
for _, idx := range selected {
|
||||
gpuResults[idx].Notes = append(gpuResults[idx].Notes, "cooldown sampling failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
for _, idx := range selected {
|
||||
perGPU := filterRowsByGPU(cooldownRows, idx)
|
||||
gpuResults[idx].Cooldown = summarizeBenchmarkTelemetry(perGPU)
|
||||
}
|
||||
appendBenchmarkMetrics(allMetricRows, cooldownRows, "cooldown", metricTimelineSec, float64(spec.CooldownSec))
|
||||
}
|
||||
|
||||
// Score and finalize each GPU.
|
||||
for _, idx := range selected {
|
||||
r := gpuResults[idx]
|
||||
applyBenchmarkSteadyFallback(r)
|
||||
r.Scores = scoreBenchmarkGPUResult(*r)
|
||||
r.DegradationReasons = detectBenchmarkDegradationReasons(*r, result.Normalization.Status)
|
||||
pr := parseResults[idx]
|
||||
switch {
|
||||
case planErr != nil:
|
||||
r.Status = classifySATErrorStatus(phaseLogs["mixed"], planErr)
|
||||
case len(r.PrecisionFailures) > 0:
|
||||
r.Status = "PARTIAL"
|
||||
case pr.Fallback:
|
||||
r.Status = "PARTIAL"
|
||||
default:
|
||||
r.Status = "OK"
|
||||
}
|
||||
result.GPUs = append(result.GPUs, finalizeBenchmarkGPUResult(*r))
|
||||
}
|
||||
}
|
||||
|
||||
// readBenchmarkHostConfig reads static CPU and memory configuration from
|
||||
// /proc/cpuinfo and /proc/meminfo. Returns nil if neither source is readable.
|
||||
func readBenchmarkHostConfig() *BenchmarkHostConfig {
|
||||
cfg := &BenchmarkHostConfig{}
|
||||
populated := false
|
||||
|
||||
// Parse /proc/cpuinfo for CPU model, sockets, cores, threads.
|
||||
if data, err := os.ReadFile("/proc/cpuinfo"); err == nil {
|
||||
socketIDs := map[string]struct{}{}
|
||||
coresPerSocket := map[string]int{}
|
||||
var modelName string
|
||||
threads := 0
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
kv := strings.SplitN(line, ":", 2)
|
||||
if len(kv) != 2 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(kv[0])
|
||||
val := strings.TrimSpace(kv[1])
|
||||
switch key {
|
||||
case "processor":
|
||||
threads++
|
||||
case "model name":
|
||||
if modelName == "" {
|
||||
modelName = val
|
||||
}
|
||||
case "physical id":
|
||||
socketIDs[val] = struct{}{}
|
||||
case "cpu cores":
|
||||
// Overwrite per-socket core count (last wins per socket, but all
|
||||
// entries for the same socket report the same value).
|
||||
if physLine := ""; physLine == "" {
|
||||
// We accumulate below by treating cpu cores as a per-thread
|
||||
// field; sum by socket requires a two-pass approach. Use the
|
||||
// simpler approximation: totalCores = threads / (threads per core).
|
||||
_ = val
|
||||
}
|
||||
}
|
||||
}
|
||||
// Second pass: per-socket core count.
|
||||
var curSocket string
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
kv := strings.SplitN(line, ":", 2)
|
||||
if len(kv) != 2 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(kv[0])
|
||||
val := strings.TrimSpace(kv[1])
|
||||
switch key {
|
||||
case "physical id":
|
||||
curSocket = val
|
||||
case "cpu cores":
|
||||
if curSocket != "" {
|
||||
if _, seen := coresPerSocket[curSocket]; !seen {
|
||||
v, _ := strconv.Atoi(val)
|
||||
coresPerSocket[curSocket] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
totalCores := 0
|
||||
for _, c := range coresPerSocket {
|
||||
totalCores += c
|
||||
}
|
||||
cfg.CPUModel = modelName
|
||||
cfg.CPUSockets = len(socketIDs)
|
||||
if cfg.CPUSockets == 0 && threads > 0 {
|
||||
cfg.CPUSockets = 1
|
||||
}
|
||||
cfg.CPUCores = totalCores
|
||||
cfg.CPUThreads = threads
|
||||
if modelName != "" || threads > 0 {
|
||||
populated = true
|
||||
}
|
||||
}
|
||||
|
||||
// Parse /proc/meminfo for total physical RAM.
|
||||
if data, err := os.ReadFile("/proc/meminfo"); err == nil {
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
if strings.HasPrefix(line, "MemTotal:") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 2 {
|
||||
kb, _ := strconv.ParseUint(fields[1], 10, 64)
|
||||
cfg.MemTotalGiB = float64(kb) / (1024 * 1024)
|
||||
populated = true
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !populated {
|
||||
return nil
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// startCPULoadSampler starts a goroutine that samples host CPU load every
|
||||
// intervalSec seconds until stopCh is closed, then sends the collected
|
||||
// samples on the returned channel.
|
||||
func startCPULoadSampler(stopCh <-chan struct{}, intervalSec int) <-chan []float64 {
|
||||
ch := make(chan []float64, 1)
|
||||
go func() {
|
||||
var samples []float64
|
||||
ticker := time.NewTicker(time.Duration(intervalSec) * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stopCh:
|
||||
ch <- samples
|
||||
return
|
||||
case <-ticker.C:
|
||||
if pct := sampleCPULoadPct(); pct > 0 {
|
||||
samples = append(samples, pct)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
// summarizeCPULoad computes stats over sampled CPU load values and assigns
|
||||
// a health status.
|
||||
func summarizeCPULoad(samples []float64) *BenchmarkCPULoad {
|
||||
if len(samples) == 0 {
|
||||
return nil
|
||||
}
|
||||
sorted := append([]float64(nil), samples...)
|
||||
sort.Float64s(sorted)
|
||||
var sum float64
|
||||
for _, v := range sorted {
|
||||
sum += v
|
||||
}
|
||||
avg := sum / float64(len(sorted))
|
||||
p95 := sorted[int(float64(len(sorted))*0.95)]
|
||||
max := sorted[len(sorted)-1]
|
||||
|
||||
cl := &BenchmarkCPULoad{
|
||||
AvgPct: math.Round(avg*10) / 10,
|
||||
MaxPct: math.Round(max*10) / 10,
|
||||
P95Pct: math.Round(p95*10) / 10,
|
||||
Samples: len(sorted),
|
||||
}
|
||||
|
||||
// Compute standard deviation to detect instability.
|
||||
var variance float64
|
||||
for _, v := range sorted {
|
||||
d := v - avg
|
||||
variance += d * d
|
||||
}
|
||||
stdDev := math.Sqrt(variance / float64(len(sorted)))
|
||||
|
||||
switch {
|
||||
case avg > 20 || max > 40:
|
||||
cl.Status = "high"
|
||||
cl.Note = fmt.Sprintf("avg %.1f%% max %.1f%% — elevated host CPU load may interfere with GPU benchmark results", avg, max)
|
||||
case stdDev > 12:
|
||||
cl.Status = "unstable"
|
||||
cl.Note = fmt.Sprintf("avg %.1f%% stddev %.1f%% — host CPU load was erratic during the benchmark", avg, stdDev)
|
||||
default:
|
||||
cl.Status = "ok"
|
||||
}
|
||||
return cl
|
||||
}
|
||||
|
||||
// runBenchmarkPowerCalibration runs the configured power-fit load for the supplied
|
||||
// GPU set and actively watches throttle counters. seedLimits, when provided, are treated as
|
||||
// the starting point for this calibration pass rather than as immutable fixed
|
||||
// limits. This matters during cumulative ramp-up: once an additional GPU is
|
||||
// introduced, every already-active GPU must be revalidated under the new
|
||||
// thermal state instead of assuming its previous single-step limit is still
|
||||
// valid. The selected reduced power limits stay active for the main benchmark
|
||||
// and are restored by the caller afterwards.
|
||||
@@ -0,0 +1,558 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func runBenchmarkPowerCalibration(
|
||||
ctx context.Context,
|
||||
verboseLog, runDir string,
|
||||
gpuIndices []int,
|
||||
infoByIndex map[int]benchmarkGPUInfo,
|
||||
logFunc func(string),
|
||||
seedLimits map[int]int,
|
||||
durationSec int,
|
||||
) (map[int]benchmarkPowerCalibrationResult, []benchmarkRestoreAction, []GPUMetricRow, benchmarkPowerCalibrationRunSummary) {
|
||||
calibDurationSec := durationSec
|
||||
var runSummary benchmarkPowerCalibrationRunSummary
|
||||
if calibDurationSec <= 0 {
|
||||
calibDurationSec = 120
|
||||
}
|
||||
// calibSearchTolerance is the binary-search convergence threshold in watts.
|
||||
// When hi-lo ≤ this, the highest verified-stable limit (lo) is used.
|
||||
const calibSearchTolerance = 10
|
||||
// dcgmResourceBusyMaxDelaySec caps the exponential back-off when DCGM
|
||||
// returns DCGM_ST_IN_USE (exit 222). The sequence is 1 s, 2 s, 4 s, …
|
||||
// doubling each retry until it would exceed the cap, at which point the
|
||||
// next busy response fails the calibration immediately.
|
||||
const dcgmResourceBusyMaxDelaySec = 300
|
||||
engine := benchmarkPowerEngine()
|
||||
engineLabel := benchmarkPowerEngineLabel(engine)
|
||||
|
||||
if engine == BenchmarkPowerEngineTargetedPower {
|
||||
if _, err := exec.LookPath("dcgmi"); err != nil {
|
||||
logFunc("power calibration: dcgmi not found, skipping (will use default power limit)")
|
||||
return map[int]benchmarkPowerCalibrationResult{}, nil, nil, runSummary
|
||||
}
|
||||
} else {
|
||||
if _, _, err := resolveBenchmarkPowerLoadCommand(calibDurationSec, gpuIndices); err != nil {
|
||||
logFunc("power calibration: dcgmproftester not found, skipping (will use default power limit)")
|
||||
return map[int]benchmarkPowerCalibrationResult{}, nil, nil, runSummary
|
||||
}
|
||||
}
|
||||
if killed := KillTestWorkers(); len(killed) > 0 {
|
||||
for _, p := range killed {
|
||||
logFunc(fmt.Sprintf("power calibration pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
|
||||
}
|
||||
}
|
||||
|
||||
canDerate := os.Geteuid() == 0
|
||||
if !canDerate {
|
||||
logFunc("power calibration: root privileges unavailable, adaptive power-limit derating disabled")
|
||||
}
|
||||
|
||||
type calibrationAttemptResult struct {
|
||||
out []byte
|
||||
rows []GPUMetricRow
|
||||
err error
|
||||
}
|
||||
|
||||
// gpuCalibState holds per-GPU binary search state during parallel calibration.
|
||||
type gpuCalibState struct {
|
||||
idx int
|
||||
info benchmarkGPUInfo
|
||||
originalLimitW int
|
||||
appliedLimitW int
|
||||
minLimitW int
|
||||
lo int // highest verified-stable limit
|
||||
hi int // lowest verified-unstable limit (exclusive sentinel above start)
|
||||
loVerified bool
|
||||
calib benchmarkPowerCalibrationResult
|
||||
converged bool
|
||||
}
|
||||
|
||||
results := make(map[int]benchmarkPowerCalibrationResult, len(gpuIndices))
|
||||
var restore []benchmarkRestoreAction
|
||||
var allCalibRows []GPUMetricRow // accumulated telemetry across all attempts
|
||||
var calibCursor float64
|
||||
|
||||
// Initialise per-GPU state.
|
||||
states := make([]*gpuCalibState, 0, len(gpuIndices))
|
||||
for _, idx := range gpuIndices {
|
||||
info := infoByIndex[idx]
|
||||
originalLimitW := int(math.Round(info.PowerLimitW))
|
||||
if originalLimitW <= 0 {
|
||||
originalLimitW = int(math.Round(info.DefaultPowerLimitW))
|
||||
}
|
||||
defaultLimitW := int(math.Round(info.DefaultPowerLimitW))
|
||||
if defaultLimitW <= 0 {
|
||||
defaultLimitW = originalLimitW
|
||||
}
|
||||
appliedLimitW := initialBenchmarkCalibrationLimitW(info)
|
||||
if appliedLimitW <= 0 {
|
||||
appliedLimitW = defaultLimitW
|
||||
}
|
||||
minLimitW := int(math.Round(info.MinPowerLimitW))
|
||||
if minLimitW <= 0 {
|
||||
minLimitW = appliedLimitW
|
||||
}
|
||||
maxLimitW := int(math.Round(info.MaxPowerLimitW))
|
||||
if maxLimitW > 0 && appliedLimitW > maxLimitW {
|
||||
appliedLimitW = maxLimitW
|
||||
}
|
||||
s := &gpuCalibState{
|
||||
idx: idx,
|
||||
info: info,
|
||||
originalLimitW: originalLimitW,
|
||||
appliedLimitW: appliedLimitW,
|
||||
minLimitW: minLimitW,
|
||||
lo: minLimitW,
|
||||
hi: appliedLimitW + 1, // not yet tested, not yet confirmed unstable
|
||||
calib: benchmarkPowerCalibrationResult{AppliedPowerLimitW: float64(appliedLimitW)},
|
||||
}
|
||||
if minLimitW > 0 && appliedLimitW > 0 && minLimitW >= appliedLimitW {
|
||||
s.appliedLimitW = minLimitW
|
||||
s.hi = minLimitW + 1
|
||||
}
|
||||
if info.MinPowerLimitW <= 0 {
|
||||
s.calib.Notes = append(s.calib.Notes, "minimum power limit was not reported by nvidia-smi; calibration can only validate the current/default power limit")
|
||||
}
|
||||
if seedLimits != nil {
|
||||
if seedW, ok := seedLimits[idx]; ok && seedW > 0 {
|
||||
// A previously validated limit is only a starting point. Re-run
|
||||
// targeted_power under the current multi-GPU thermal load and derate
|
||||
// again if this step shows new throttling.
|
||||
if seedW < s.minLimitW {
|
||||
seedW = s.minLimitW
|
||||
}
|
||||
if maxLimitW > 0 && seedW > maxLimitW {
|
||||
seedW = maxLimitW
|
||||
}
|
||||
if canDerate {
|
||||
_ = setBenchmarkPowerLimit(ctx, verboseLog, idx, seedW)
|
||||
}
|
||||
s.appliedLimitW = seedW
|
||||
s.hi = seedW + 1
|
||||
s.calib.AppliedPowerLimitW = float64(seedW)
|
||||
s.calib.Derated = seedW < s.originalLimitW
|
||||
s.calib.Notes = append(s.calib.Notes,
|
||||
fmt.Sprintf("seed limit: %d W (revalidating under current thermal load)", seedW))
|
||||
}
|
||||
}
|
||||
states = append(states, s)
|
||||
if canDerate && originalLimitW > 0 {
|
||||
idxCopy := idx
|
||||
orig := originalLimitW
|
||||
restore = append(restore, benchmarkRestoreAction{
|
||||
name: fmt.Sprintf("gpu-%d-restore-power-limit", idxCopy),
|
||||
fn: func() {
|
||||
_ = setBenchmarkPowerLimit(context.Background(), verboseLog, idxCopy, orig)
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Shared DCGM resource-busy back-off state (single diagnostic session).
|
||||
busyRetries := 0
|
||||
busyDelaySec := 1
|
||||
sharedAttempt := 0
|
||||
|
||||
type sharedAttemptResult struct {
|
||||
out []byte
|
||||
rows []GPUMetricRow
|
||||
err error
|
||||
}
|
||||
|
||||
calibDone:
|
||||
for {
|
||||
// Collect non-converged GPUs.
|
||||
var active []*gpuCalibState
|
||||
for _, s := range states {
|
||||
if !s.converged {
|
||||
active = append(active, s)
|
||||
}
|
||||
}
|
||||
if len(active) == 0 || ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
sharedAttempt++
|
||||
for _, s := range active {
|
||||
s.calib.Attempts++
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d %s attempt %d at %d W for %ds", s.idx, engineLabel, s.calib.Attempts, s.appliedLimitW, calibDurationSec))
|
||||
}
|
||||
|
||||
// Snapshot throttle counters for all active GPUs before the run.
|
||||
beforeThrottle := make(map[int]BenchmarkThrottleCounters, len(active))
|
||||
for _, s := range active {
|
||||
beforeThrottle[s.idx], _ = queryThrottleCounters(s.idx)
|
||||
}
|
||||
|
||||
// Run the selected power-fit load for ALL gpuIndices simultaneously so every card
|
||||
// is under load during calibration — this reflects real server thermals.
|
||||
logName := fmt.Sprintf("power-calibration-attempt-%d.log", sharedAttempt)
|
||||
cmd, env, err := resolveBenchmarkPowerLoadCommand(calibDurationSec, gpuIndices)
|
||||
if err != nil {
|
||||
for _, s := range active {
|
||||
s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("failed to resolve %s command: %v", engineLabel, err))
|
||||
s.converged = true
|
||||
}
|
||||
logFunc(fmt.Sprintf("power calibration: failed to resolve %s command: %v", engineLabel, err))
|
||||
break calibDone
|
||||
}
|
||||
attemptCtx, cancelAttempt := context.WithCancel(ctx)
|
||||
doneCh := make(chan sharedAttemptResult, 1)
|
||||
sdrStopCh := make(chan struct{})
|
||||
sdrDoneCh := startIPMISDRSampler(sdrStopCh, benchmarkPowerAutotuneSampleInterval)
|
||||
fanStopCh := make(chan struct{})
|
||||
fanDoneCh := startBenchmarkFanSampler(fanStopCh, benchmarkPowerAutotuneSampleInterval)
|
||||
go func() {
|
||||
out, rows, err := runBenchmarkCommandWithMetrics(attemptCtx, verboseLog, logName, cmd, env, gpuIndices, logFunc)
|
||||
doneCh <- sharedAttemptResult{out: out, rows: rows, err: err}
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(time.Second)
|
||||
throttleReasons := make(map[int]string, len(active))
|
||||
var ar sharedAttemptResult
|
||||
|
||||
attemptLoop:
|
||||
for {
|
||||
select {
|
||||
case ar = <-doneCh:
|
||||
break attemptLoop
|
||||
case <-ticker.C:
|
||||
// Poll throttle counters for each active GPU independently.
|
||||
for _, s := range active {
|
||||
if throttleReasons[s.idx] != "" {
|
||||
continue // already detected for this GPU
|
||||
}
|
||||
after, err := queryThrottleCounters(s.idx)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Record throttle but do NOT cancel — let the load command finish so
|
||||
// runtime resources release cleanly before the next attempt.
|
||||
if reason := benchmarkCalibrationThrottleReason(beforeThrottle[s.idx], after); reason != "" {
|
||||
throttleReasons[s.idx] = reason
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d detected %s throttle at %d W, waiting for run to finish", s.idx, reason, s.appliedLimitW))
|
||||
}
|
||||
}
|
||||
case <-ctx.Done():
|
||||
cancelAttempt()
|
||||
ar = <-doneCh
|
||||
break attemptLoop
|
||||
}
|
||||
}
|
||||
ticker.Stop()
|
||||
cancelAttempt()
|
||||
close(sdrStopCh)
|
||||
close(fanStopCh)
|
||||
attemptSDRSummary := summarizeSDRPowerSeries(<-sdrDoneCh)
|
||||
attemptFanSummary := <-fanDoneCh
|
||||
_ = os.WriteFile(filepath.Join(runDir, logName), ar.out, 0644)
|
||||
// Accumulate telemetry rows with attempt stage label.
|
||||
appendBenchmarkMetrics(&allCalibRows, ar.rows, fmt.Sprintf("attempt-%d", sharedAttempt), &calibCursor, float64(calibDurationSec))
|
||||
|
||||
// Resource busy: retry with exponential back-off (shared — one DCGM session).
|
||||
if ar.err != nil && isDCGMResourceBusy(ar.err) {
|
||||
if busyDelaySec > dcgmResourceBusyMaxDelaySec {
|
||||
for _, s := range active {
|
||||
s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("DCGM resource busy after %d retries, giving up", busyRetries))
|
||||
s.converged = true
|
||||
}
|
||||
logFunc(fmt.Sprintf("power calibration: DCGM resource persistently busy after %d retries, stopping", busyRetries))
|
||||
break calibDone
|
||||
}
|
||||
busyRetries++
|
||||
// Undo attempt counter: busy retries don't count as real attempts.
|
||||
for _, s := range active {
|
||||
s.calib.Attempts--
|
||||
}
|
||||
logFunc(fmt.Sprintf("power calibration: DCGM resource busy (attempt %d), retrying in %ds", sharedAttempt, busyDelaySec))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break calibDone
|
||||
case <-time.After(time.Duration(busyDelaySec) * time.Second):
|
||||
}
|
||||
next := busyDelaySec * 2
|
||||
if next > dcgmResourceBusyMaxDelaySec {
|
||||
next = dcgmResourceBusyMaxDelaySec + 1
|
||||
}
|
||||
busyDelaySec = next
|
||||
sharedAttempt-- // retry same logical attempt number
|
||||
continue
|
||||
}
|
||||
busyRetries = 0
|
||||
busyDelaySec = 1
|
||||
|
||||
// Per-GPU analysis and binary search update.
|
||||
attemptStable := ar.err == nil
|
||||
for _, s := range active {
|
||||
perGPU := filterRowsByGPU(ar.rows, s.idx)
|
||||
summary := summarizeBenchmarkTelemetry(perGPU)
|
||||
throttle := throttleReasons[s.idx]
|
||||
if throttle != "" || summary.P95PowerW <= 0 {
|
||||
attemptStable = false
|
||||
}
|
||||
|
||||
// Cooling warning: thermal throttle with fans not at maximum.
|
||||
if strings.Contains(throttle, "thermal") && s.calib.CoolingWarning == "" {
|
||||
clocks := make([]float64, 0, len(perGPU))
|
||||
var fanDutyValues []float64
|
||||
fanDutyAvail := false
|
||||
for _, r := range perGPU {
|
||||
if r.ClockMHz > 0 {
|
||||
clocks = append(clocks, r.ClockMHz)
|
||||
}
|
||||
if r.FanDutyCycleAvailable {
|
||||
fanDutyAvail = true
|
||||
fanDutyValues = append(fanDutyValues, r.FanDutyCyclePct)
|
||||
}
|
||||
}
|
||||
dropPct := benchmarkClockDrift(clocks)
|
||||
p95FanDuty := benchmarkPercentile(fanDutyValues, 95)
|
||||
if dropPct >= 20 && fanDutyAvail && p95FanDuty < 98 {
|
||||
s.calib.CoolingWarning = fmt.Sprintf(
|
||||
"thermal throttle (%s) caused a %.0f%% clock drop while fans were at %.0f%% duty cycle — server cooling may not be configured for full GPU load",
|
||||
throttle, dropPct, p95FanDuty,
|
||||
)
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d cooling warning: %s", s.idx, s.calib.CoolingWarning))
|
||||
}
|
||||
}
|
||||
|
||||
if throttle == "" && ar.err == nil && summary.P95PowerW > 0 {
|
||||
// Stable at current limit — update lo and binary-search upward.
|
||||
s.calib.Summary = summary
|
||||
s.calib.Completed = true
|
||||
s.calib.AppliedPowerLimitW = float64(s.appliedLimitW)
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d stable at %d W, p95=%.0f W p95_temp=%.1f C (%d samples)", s.idx, s.appliedLimitW, summary.P95PowerW, summary.P95TempC, summary.Samples))
|
||||
s.lo = s.appliedLimitW
|
||||
s.loVerified = true
|
||||
if canDerate && s.hi-s.lo > calibSearchTolerance {
|
||||
next := roundTo5W((s.lo + s.hi) / 2)
|
||||
if next > s.lo && next < s.hi {
|
||||
if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, next); err == nil {
|
||||
s.appliedLimitW = next
|
||||
s.calib.AppliedPowerLimitW = float64(next)
|
||||
s.calib.Completed = false // keep searching
|
||||
s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search: stable at %d W, trying %d W (lo=%d hi=%d)", s.lo, next, s.lo, s.hi))
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d binary search up: stable at %d W, trying %d W", s.idx, s.lo, next))
|
||||
continue // next GPU in active list
|
||||
}
|
||||
}
|
||||
}
|
||||
s.calib.MetricRows = filterRowsByGPU(ar.rows, s.idx)
|
||||
s.converged = true
|
||||
continue
|
||||
}
|
||||
|
||||
// Failed or throttled — log and binary-search downward.
|
||||
switch {
|
||||
case throttle != "":
|
||||
s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("targeted_power attempt %d: %s throttle at %d W", s.calib.Attempts, throttle, s.appliedLimitW))
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d throttled (%s) at %d W, reducing power limit", s.idx, throttle, s.appliedLimitW))
|
||||
case ar.err != nil:
|
||||
s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("targeted_power attempt %d failed at %d W: %v", s.calib.Attempts, s.appliedLimitW, ar.err))
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d %s failed at %d W: %v", s.idx, engineLabel, s.appliedLimitW, ar.err))
|
||||
default:
|
||||
s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("%s attempt %d at %d W: no valid power telemetry", engineLabel, s.calib.Attempts, s.appliedLimitW))
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d attempt %d at %d W: no valid telemetry", s.idx, s.calib.Attempts, s.appliedLimitW))
|
||||
}
|
||||
|
||||
if !canDerate || s.appliedLimitW <= 0 {
|
||||
s.converged = true
|
||||
continue
|
||||
}
|
||||
s.hi = s.appliedLimitW
|
||||
|
||||
if s.hi-s.lo <= calibSearchTolerance {
|
||||
if !s.loVerified && s.minLimitW > 0 && s.appliedLimitW != s.minLimitW {
|
||||
if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, s.minLimitW); err != nil {
|
||||
s.calib.Notes = append(s.calib.Notes, "failed to set power limit: "+err.Error())
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d failed to set minimum power limit %d W: %v", s.idx, s.minLimitW, err))
|
||||
s.converged = true
|
||||
continue
|
||||
}
|
||||
s.appliedLimitW = s.minLimitW
|
||||
s.calib.AppliedPowerLimitW = float64(s.minLimitW)
|
||||
s.calib.Derated = s.minLimitW < s.originalLimitW
|
||||
s.info.PowerLimitW = float64(s.minLimitW)
|
||||
infoByIndex[s.idx] = s.info
|
||||
s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search: validating minimum settable limit %d W before concluding failure", s.minLimitW))
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d binary search: validating minimum settable limit %d W", s.idx, s.minLimitW))
|
||||
continue
|
||||
}
|
||||
if s.loVerified {
|
||||
s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search converged: using %d W (lo=%d hi=%d)", s.lo, s.lo, s.hi))
|
||||
if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, s.lo); err == nil {
|
||||
s.appliedLimitW = s.lo
|
||||
s.calib.AppliedPowerLimitW = float64(s.lo)
|
||||
s.calib.Derated = s.lo < s.originalLimitW
|
||||
// Summary was captured when we last verified stability at s.lo,
|
||||
// so the result is valid — mark as completed even though we
|
||||
// converged from the failure path (tried higher, failed, fell back).
|
||||
s.calib.Completed = true
|
||||
}
|
||||
} else {
|
||||
s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("could not find a stable %s limit down to the minimum settable power limit %d W", engineLabel, s.minLimitW))
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d no stable limit found down to minimum settable power limit %d W", s.idx, s.minLimitW))
|
||||
}
|
||||
s.calib.MetricRows = filterRowsByGPU(ar.rows, s.idx)
|
||||
s.converged = true
|
||||
continue
|
||||
}
|
||||
|
||||
next := roundTo5W((s.lo + s.hi) / 2)
|
||||
if next <= s.lo {
|
||||
next = s.lo + calibSearchTolerance
|
||||
}
|
||||
if next >= s.hi {
|
||||
next = (s.lo + s.hi) / 2
|
||||
}
|
||||
if next < s.minLimitW {
|
||||
next = s.minLimitW
|
||||
}
|
||||
if err := setBenchmarkPowerLimit(ctx, verboseLog, s.idx, next); err != nil {
|
||||
s.calib.Notes = append(s.calib.Notes, "failed to set power limit: "+err.Error())
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d failed to set power limit %d W: %v", s.idx, next, err))
|
||||
s.converged = true
|
||||
continue
|
||||
}
|
||||
s.appliedLimitW = next
|
||||
s.calib.AppliedPowerLimitW = float64(next)
|
||||
s.calib.Derated = next < s.originalLimitW
|
||||
s.info.PowerLimitW = float64(next)
|
||||
infoByIndex[s.idx] = s.info
|
||||
s.calib.Notes = append(s.calib.Notes, fmt.Sprintf("binary search: trying %d W (lo=%d hi=%d)", next, s.lo, s.hi))
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d binary search: trying %d W (lo=%d hi=%d)", s.idx, next, s.lo, s.hi))
|
||||
}
|
||||
if attemptStable {
|
||||
if attemptSDRSummary.Samples > 0 {
|
||||
runSummary.LoadedSDR = attemptSDRSummary
|
||||
}
|
||||
if attemptFanSummary.FanSamples > 0 {
|
||||
runSummary.AvgFanRPM = attemptFanSummary.AvgFanRPM
|
||||
runSummary.AvgFanDutyCyclePct = attemptFanSummary.AvgFanDutyCyclePct
|
||||
runSummary.FanSamples = attemptFanSummary.FanSamples
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, s := range states {
|
||||
if s.calib.Completed || s.calib.Attempts > 0 || len(s.calib.Notes) > 0 {
|
||||
results[s.idx] = s.calib
|
||||
}
|
||||
}
|
||||
writeBenchmarkMetricsFiles(runDir, allCalibRows)
|
||||
return results, restore, allCalibRows, runSummary
|
||||
}
|
||||
|
||||
// isDCGMResourceBusy returns true when dcgmi exits with DCGM_ST_IN_USE (222),
|
||||
// meaning nv-hostengine still holds the diagnostic slot from a prior run.
|
||||
func isDCGMResourceBusy(err error) bool {
|
||||
var exitErr *exec.ExitError
|
||||
return errors.As(err, &exitErr) && exitErr.ExitCode() == 222
|
||||
}
|
||||
|
||||
// roundTo5W rounds w to the nearest 5 W boundary.
|
||||
func roundTo5W(w int) int {
|
||||
return ((w + 2) / 5) * 5
|
||||
}
|
||||
|
||||
func initialBenchmarkCalibrationLimitW(info benchmarkGPUInfo) int {
|
||||
defaultLimitW := int(math.Round(info.DefaultPowerLimitW))
|
||||
currentLimitW := int(math.Round(info.PowerLimitW))
|
||||
maxLimitW := int(math.Round(info.MaxPowerLimitW))
|
||||
|
||||
startW := defaultLimitW
|
||||
if startW <= 0 {
|
||||
startW = currentLimitW
|
||||
}
|
||||
if startW <= 0 {
|
||||
startW = maxLimitW
|
||||
}
|
||||
if maxLimitW > 0 && startW > maxLimitW {
|
||||
startW = maxLimitW
|
||||
}
|
||||
return startW
|
||||
}
|
||||
|
||||
// meanFanRPM returns the average RPM across a set of fan readings.
|
||||
func meanFanRPM(fans []FanReading) float64 {
|
||||
if len(fans) == 0 {
|
||||
return 0
|
||||
}
|
||||
var sum float64
|
||||
for _, f := range fans {
|
||||
sum += f.RPM
|
||||
}
|
||||
return sum / float64(len(fans))
|
||||
}
|
||||
|
||||
func startBenchmarkFanSampler(stopCh <-chan struct{}, intervalSec int) <-chan benchmarkPowerCalibrationRunSummary {
|
||||
if intervalSec <= 0 {
|
||||
intervalSec = benchmarkPowerAutotuneSampleInterval
|
||||
}
|
||||
ch := make(chan benchmarkPowerCalibrationRunSummary, 1)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
var rpmSamples []float64
|
||||
var dutySamples []float64
|
||||
record := func() {
|
||||
fans, err := sampleFanSpeeds()
|
||||
if err != nil || len(fans) == 0 {
|
||||
return
|
||||
}
|
||||
if rpm := meanFanRPM(fans); rpm > 0 {
|
||||
rpmSamples = append(rpmSamples, rpm)
|
||||
}
|
||||
if duty, ok, _ := sampleFanDutyCyclePctFromFans(fans); ok && duty > 0 {
|
||||
dutySamples = append(dutySamples, duty)
|
||||
}
|
||||
}
|
||||
record()
|
||||
ticker := time.NewTicker(time.Duration(intervalSec) * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stopCh:
|
||||
ch <- benchmarkPowerCalibrationRunSummary{
|
||||
AvgFanRPM: benchmarkMean(rpmSamples),
|
||||
AvgFanDutyCyclePct: benchmarkMean(dutySamples),
|
||||
FanSamples: len(rpmSamples),
|
||||
}
|
||||
return
|
||||
case <-ticker.C:
|
||||
record()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
func powerBenchDurationSec(profile string) int {
|
||||
switch strings.TrimSpace(strings.ToLower(profile)) {
|
||||
case NvidiaBenchmarkProfileStability:
|
||||
return 300
|
||||
case NvidiaBenchmarkProfileOvernight:
|
||||
return 600
|
||||
default:
|
||||
return 120
|
||||
}
|
||||
}
|
||||
|
||||
func cloneBenchmarkGPUInfoMap(src map[int]benchmarkGPUInfo) map[int]benchmarkGPUInfo {
|
||||
out := make(map[int]benchmarkGPUInfo, len(src))
|
||||
for k, v := range src {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,624 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func renderPowerBenchReport(result NvidiaPowerBenchResult) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Bee Bench Power Report\n\n")
|
||||
fmt.Fprintf(&b, "**Benchmark version:** %s \n", result.BenchmarkVersion)
|
||||
fmt.Fprintf(&b, "**Profile:** %s \n", result.BenchmarkProfile)
|
||||
fmt.Fprintf(&b, "**Generated:** %s \n", result.GeneratedAt.Format("2006-01-02 15:04:05 UTC"))
|
||||
fmt.Fprintf(&b, "**Overall status:** %s \n", result.OverallStatus)
|
||||
fmt.Fprintf(&b, "**Platform max TDP (GPU-reported):** %.0f W \n", result.PlatformMaxTDPW)
|
||||
if sp := result.ServerPower; sp != nil && sp.Available {
|
||||
sourceLabel := "autotuned source"
|
||||
switch normalizeBenchmarkPowerSource(sp.Source) {
|
||||
case BenchmarkPowerSourceSDRPSUInput:
|
||||
sourceLabel = "autotuned source (SDR PSU AC input)"
|
||||
case BenchmarkPowerSourceDCMI:
|
||||
sourceLabel = "autotuned source (DCMI)"
|
||||
}
|
||||
fmt.Fprintf(&b, "**Server power delta (%s):** %.0f W \n", sourceLabel, sp.DeltaW)
|
||||
fmt.Fprintf(&b, "**Reporting ratio:** %.2f \n", sp.ReportingRatio)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
// Server power comparison table.
|
||||
if sp := result.ServerPower; sp != nil {
|
||||
b.WriteString("## Server vs GPU Power Comparison\n\n")
|
||||
selectedSource := normalizeBenchmarkPowerSource(sp.Source)
|
||||
selectedSourceLabel := "Selected source"
|
||||
if selectedSource == BenchmarkPowerSourceSDRPSUInput {
|
||||
selectedSourceLabel = "Selected source (SDR PSU AC input)"
|
||||
} else if selectedSource == BenchmarkPowerSourceDCMI {
|
||||
selectedSourceLabel = "Selected source (DCMI)"
|
||||
}
|
||||
var spRows [][]string
|
||||
spRows = append(spRows, []string{"GPU actual power sum (p95, last step)", fmt.Sprintf("%.0f W", sp.GPUReportedSumW)})
|
||||
if sp.Available {
|
||||
spRows = append(spRows, []string{selectedSourceLabel + " idle power", fmt.Sprintf("%.0f W", sp.IdleW)})
|
||||
spRows = append(spRows, []string{selectedSourceLabel + " loaded power", fmt.Sprintf("%.0f W", sp.LoadedW)})
|
||||
spRows = append(spRows, []string{selectedSourceLabel + " Δ power (loaded − idle)", fmt.Sprintf("%.0f W", sp.DeltaW)})
|
||||
}
|
||||
if selectedSource == BenchmarkPowerSourceSDRPSUInput && sp.PSUInputLoadedW > 0 {
|
||||
spRows = append(spRows, []string{"PSU AC input (idle avg, pre-load phase)", fmt.Sprintf("%.0f W", sp.PSUInputIdleW)})
|
||||
spRows = append(spRows, []string{"PSU AC input (loaded avg, final phase)", fmt.Sprintf("%.0f W", sp.PSUInputLoadedW)})
|
||||
psuDelta := sp.PSUInputLoadedW - sp.PSUInputIdleW
|
||||
spRows = append(spRows, []string{"PSU AC input Δ (loaded − idle)", fmt.Sprintf("%.0f W", psuDelta)})
|
||||
}
|
||||
if sp.Available {
|
||||
ratio := sp.ReportingRatio
|
||||
dcmiPartial := detectDCMIPartialCoverage(sp) ||
|
||||
(sp.PSUInputIdleW == 0 && detectIPMISaturationFallback(result.RampSteps))
|
||||
ratioNote := ""
|
||||
switch {
|
||||
case dcmiPartial:
|
||||
ratioNote = "⚠ IPMI DCMI covers partial PSU set; use SDR ratio below for accuracy assessment"
|
||||
case ratio >= 0.9:
|
||||
ratioNote = "✓ GPU telemetry matches server power"
|
||||
case ratio >= 0.75:
|
||||
ratioNote = "⚠ minor discrepancy — GPU may slightly over-report TDP"
|
||||
default:
|
||||
ratioNote = "✗ significant discrepancy — GPU over-reports TDP vs wall power"
|
||||
}
|
||||
spRows = append(spRows, []string{"Reporting ratio", fmt.Sprintf("%.2f — %s", ratio, ratioNote)})
|
||||
if selectedSource == BenchmarkPowerSourceSDRPSUInput && sp.PSUInputLoadedW > 0 && sp.GPUReportedSumW > 0 {
|
||||
psuDelta := sp.PSUInputLoadedW - sp.PSUInputIdleW
|
||||
sdrRatio := psuDelta / sp.GPUReportedSumW
|
||||
sdrNote := ""
|
||||
switch {
|
||||
case sdrRatio >= 0.9:
|
||||
sdrNote = "✓ GPU telemetry matches wall power"
|
||||
case sdrRatio >= 0.75:
|
||||
sdrNote = "⚠ minor discrepancy"
|
||||
default:
|
||||
sdrNote = "✗ significant discrepancy"
|
||||
}
|
||||
spRows = append(spRows, []string{"PSU AC input reporting ratio", fmt.Sprintf("%.2f — %s", sdrRatio, sdrNote)})
|
||||
}
|
||||
} else {
|
||||
spRows = append(spRows, []string{"IPMI availability", "not available — IPMI not supported or ipmitool not found"})
|
||||
}
|
||||
b.WriteString(fmtMDTable([]string{"Metric", "Value"}, spRows))
|
||||
for _, note := range sp.Notes {
|
||||
fmt.Fprintf(&b, "\n> %s\n", note)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
|
||||
if len(sp.PSUSlotReadingsIdle) > 0 || len(sp.PSUSlotReadingsLoaded) > 0 {
|
||||
b.WriteString("## PSU Load Distribution\n\n")
|
||||
|
||||
slotSet := map[string]struct{}{}
|
||||
for k := range sp.PSUSlotReadingsIdle {
|
||||
slotSet[k] = struct{}{}
|
||||
}
|
||||
for k := range sp.PSUSlotReadingsLoaded {
|
||||
slotSet[k] = struct{}{}
|
||||
}
|
||||
slots := make([]string, 0, len(slotSet))
|
||||
for k := range slotSet {
|
||||
slots = append(slots, k)
|
||||
}
|
||||
sort.Strings(slots)
|
||||
|
||||
fmtW := func(v *float64) string {
|
||||
if v == nil {
|
||||
return "—"
|
||||
}
|
||||
return fmt.Sprintf("%.0f W", *v)
|
||||
}
|
||||
|
||||
var psuDistRows [][]string
|
||||
for _, slot := range slots {
|
||||
idle := sp.PSUSlotReadingsIdle[slot]
|
||||
loaded := sp.PSUSlotReadingsLoaded[slot]
|
||||
|
||||
var deltaStr string
|
||||
if idle.InputW != nil && loaded.InputW != nil {
|
||||
deltaStr = fmt.Sprintf("%+.0f W", *loaded.InputW-*idle.InputW)
|
||||
} else {
|
||||
deltaStr = "—"
|
||||
}
|
||||
|
||||
status := loaded.Status
|
||||
if status == "" {
|
||||
status = idle.Status
|
||||
}
|
||||
if status == "" {
|
||||
status = "—"
|
||||
}
|
||||
|
||||
psuDistRows = append(psuDistRows, []string{
|
||||
slot,
|
||||
fmtW(idle.InputW), fmtW(loaded.InputW),
|
||||
deltaStr, status,
|
||||
})
|
||||
}
|
||||
b.WriteString(fmtMDTable([]string{"Slot", "AC Input (idle avg)", "AC Input (loaded avg)", "Load Δ", "Status"}, psuDistRows))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
|
||||
if len(result.Findings) > 0 {
|
||||
b.WriteString("## Summary\n\n")
|
||||
for _, finding := range result.Findings {
|
||||
fmt.Fprintf(&b, "- %s\n", finding)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
// ── Single GPU section ───────────────────────────────────────────────────
|
||||
b.WriteString("## Single GPU\n\n")
|
||||
{
|
||||
var sgRows [][]string
|
||||
for _, gpu := range result.GPUs {
|
||||
clk := "—"
|
||||
mem := "—"
|
||||
temp := "—"
|
||||
pwr := "—"
|
||||
if gpu.Telemetry != nil {
|
||||
clk = fmt.Sprintf("%.0f", gpu.Telemetry.AvgGraphicsClockMHz)
|
||||
mem = fmt.Sprintf("%.0f", gpu.Telemetry.AvgMemoryClockMHz)
|
||||
temp = fmt.Sprintf("%.1f", gpu.Telemetry.AvgTempC)
|
||||
pwr = fmt.Sprintf("%.0f W", gpu.Telemetry.AvgPowerW)
|
||||
}
|
||||
serverDelta := "—"
|
||||
if gpu.ServerDeltaW > 0 {
|
||||
serverDelta = fmt.Sprintf("%.0f W", gpu.ServerDeltaW)
|
||||
}
|
||||
fan := "—"
|
||||
if gpu.AvgFanRPM > 0 {
|
||||
if gpu.AvgFanDutyCyclePct > 0 {
|
||||
fan = fmt.Sprintf("%.0f RPM (%.0f%%)", gpu.AvgFanRPM, gpu.AvgFanDutyCyclePct)
|
||||
} else {
|
||||
fan = fmt.Sprintf("%.0f RPM", gpu.AvgFanRPM)
|
||||
}
|
||||
}
|
||||
sgRows = append(sgRows, []string{
|
||||
fmt.Sprintf("GPU %d", gpu.Index),
|
||||
fmt.Sprintf("%s (%s)", clk, mem),
|
||||
temp,
|
||||
pwr,
|
||||
serverDelta,
|
||||
fan,
|
||||
})
|
||||
}
|
||||
b.WriteString(fmtMDTable([]string{"GPU", "Clock MHz (Mem MHz)", "Avg Temp °C", "Power W", "Server Δ W", "Avg Fan RPM (duty%)"}, sgRows))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
if len(result.RecommendedSlotOrder) > 0 {
|
||||
fmt.Fprintf(&b, "Recommended slot order for best single-card power realization: `%s`\n\n", joinIndexList(result.RecommendedSlotOrder))
|
||||
}
|
||||
|
||||
// ── Ramp Sequence ────────────────────────────────────────────────────────
|
||||
// Rows = run number; Cols = per-GPU power (from step telemetry) + aggregates.
|
||||
if len(result.RampSteps) > 0 {
|
||||
b.WriteString("## Ramp Sequence\n\n")
|
||||
|
||||
// Collect all GPU indices that appear across all steps (ordered by first appearance).
|
||||
allGPUIndices := make([]int, 0, len(result.GPUs))
|
||||
seen := map[int]bool{}
|
||||
for _, step := range result.RampSteps {
|
||||
for _, idx := range step.GPUIndices {
|
||||
if !seen[idx] {
|
||||
seen[idx] = true
|
||||
allGPUIndices = append(allGPUIndices, idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var idleW float64
|
||||
if result.ServerPower != nil {
|
||||
idleW = result.ServerPower.IdleW
|
||||
}
|
||||
|
||||
// Build header: Run | GPU 0 | GPU 1 | ... | GPU total W | Server itself W | Server wall W | Per GPU wall W | Platform eff.
|
||||
headers := []string{"Run"}
|
||||
for _, idx := range allGPUIndices {
|
||||
headers = append(headers, fmt.Sprintf("GPU %d W", idx))
|
||||
}
|
||||
headers = append(headers, "GPU total W", "Server itself W", "Server wall W", "Per GPU wall W", "Platform eff.")
|
||||
|
||||
var rampRows [][]string
|
||||
if idleW > 0 {
|
||||
idleRow := []string{"0 (idle)"}
|
||||
for range allGPUIndices {
|
||||
idleRow = append(idleRow, "—")
|
||||
}
|
||||
// No load: GPU total is negligible, all draw is the server's own baseline.
|
||||
idleRow = append(idleRow, "—", fmt.Sprintf("%.0f", idleW), fmt.Sprintf("%.0f", idleW), "—", "—")
|
||||
rampRows = append(rampRows, idleRow)
|
||||
}
|
||||
for _, step := range result.RampSteps {
|
||||
row := []string{fmt.Sprintf("%d", step.StepIndex)}
|
||||
for _, idx := range allGPUIndices {
|
||||
inStep := false
|
||||
for _, si := range step.GPUIndices {
|
||||
if si == idx {
|
||||
inStep = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !inStep {
|
||||
row = append(row, "—")
|
||||
continue
|
||||
}
|
||||
gpuPwr := "—"
|
||||
if t, ok := step.PerGPUTelemetry[idx]; ok && t != nil && t.AvgPowerW > 0 {
|
||||
gpuPwr = fmt.Sprintf("%.0f", t.AvgPowerW)
|
||||
}
|
||||
row = append(row, gpuPwr)
|
||||
}
|
||||
// GPU total W = sum of observed GPU power (nvidia-smi)
|
||||
gpuTotal := "—"
|
||||
if step.TotalObservedPowerW > 0 {
|
||||
gpuTotal = fmt.Sprintf("%.0f", step.TotalObservedPowerW)
|
||||
}
|
||||
// Server itself W = server wall power minus GPU total (non-GPU baseline draw)
|
||||
serverItself := "—"
|
||||
if step.ServerLoadedW > 0 && step.TotalObservedPowerW > 0 {
|
||||
serverItself = fmt.Sprintf("%.0f", step.ServerLoadedW-step.TotalObservedPowerW)
|
||||
}
|
||||
// Server wall W
|
||||
serverWall := "—"
|
||||
if step.ServerLoadedW > 0 {
|
||||
serverWall = fmt.Sprintf("%.0f", step.ServerLoadedW)
|
||||
}
|
||||
// Per GPU wall W = ServerDeltaW / len(GPUIndices)
|
||||
perGPUWall := "—"
|
||||
if step.ServerDeltaW > 0 && len(step.GPUIndices) > 0 {
|
||||
perGPUWall = fmt.Sprintf("%.0f", step.ServerDeltaW/float64(len(step.GPUIndices)))
|
||||
}
|
||||
// Platform eff. = (ServerLoadedW − idleW) / TotalObservedPowerW
|
||||
platEff := "—"
|
||||
if step.TotalObservedPowerW > 0 {
|
||||
eff := step.ServerDeltaW / step.TotalObservedPowerW
|
||||
if idleW > 0 && step.ServerLoadedW > 0 {
|
||||
eff = (step.ServerLoadedW - idleW) / step.TotalObservedPowerW
|
||||
}
|
||||
platEff = fmt.Sprintf("%.2f", eff)
|
||||
}
|
||||
row = append(row, gpuTotal, serverItself, serverWall, perGPUWall, platEff)
|
||||
rampRows = append(rampRows, row)
|
||||
}
|
||||
b.WriteString(fmtMDTable(headers, rampRows))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
// ── PSU Performance ───────────────────────────────────────────────────────
|
||||
{
|
||||
// Collect all PSU slot keys from any ramp step.
|
||||
psuSlotSet := map[string]struct{}{}
|
||||
for _, step := range result.RampSteps {
|
||||
for k := range step.PSUSlotReadings {
|
||||
psuSlotSet[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(psuSlotSet) > 0 {
|
||||
b.WriteString("## PSU Performance\n\n")
|
||||
psuSlots := make([]string, 0, len(psuSlotSet))
|
||||
for k := range psuSlotSet {
|
||||
psuSlots = append(psuSlots, k)
|
||||
}
|
||||
sort.Strings(psuSlots)
|
||||
|
||||
var idleW float64
|
||||
if result.ServerPower != nil {
|
||||
idleW = result.ServerPower.IdleW
|
||||
}
|
||||
|
||||
psuHeaders := []string{"Run"}
|
||||
for _, slot := range psuSlots {
|
||||
psuHeaders = append(psuHeaders, fmt.Sprintf("PSU %s W", slot))
|
||||
}
|
||||
psuHeaders = append(psuHeaders, "PSU Total W", "Platform eff.", "Avg Fan RPM (duty%)")
|
||||
|
||||
var psuRows [][]string
|
||||
for _, step := range result.RampSteps {
|
||||
row := []string{fmt.Sprintf("%d", step.StepIndex)}
|
||||
var psuTotal float64
|
||||
for _, slot := range psuSlots {
|
||||
sp, ok := step.PSUSlotReadings[slot]
|
||||
if !ok || sp.InputW == nil {
|
||||
row = append(row, "—")
|
||||
continue
|
||||
}
|
||||
row = append(row, fmt.Sprintf("%.0f", *sp.InputW))
|
||||
psuTotal += *sp.InputW
|
||||
}
|
||||
totalStr := "—"
|
||||
if psuTotal > 0 {
|
||||
totalStr = fmt.Sprintf("%.0f", psuTotal)
|
||||
}
|
||||
platEff := "—"
|
||||
if step.TotalObservedPowerW > 0 {
|
||||
eff := step.ServerDeltaW / step.TotalObservedPowerW
|
||||
if idleW > 0 && step.ServerLoadedW > 0 {
|
||||
eff = (step.ServerLoadedW - idleW) / step.TotalObservedPowerW
|
||||
}
|
||||
platEff = fmt.Sprintf("%.2f", eff)
|
||||
}
|
||||
fan := "—"
|
||||
if step.AvgFanRPM > 0 {
|
||||
if step.AvgFanDutyCyclePct > 0 {
|
||||
fan = fmt.Sprintf("%.0f (%.0f%%)", step.AvgFanRPM, step.AvgFanDutyCyclePct)
|
||||
} else {
|
||||
fan = fmt.Sprintf("%.0f", step.AvgFanRPM)
|
||||
}
|
||||
}
|
||||
row = append(row, totalStr, platEff, fan)
|
||||
psuRows = append(psuRows, row)
|
||||
}
|
||||
b.WriteString(fmtMDTable(psuHeaders, psuRows))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
|
||||
// ── PSU Issues ────────────────────────────────────────────────────────────
|
||||
if len(result.PSUIssues) > 0 {
|
||||
b.WriteString("## PSU Issues\n\n")
|
||||
b.WriteString("The following power supply anomalies were detected during the test:\n\n")
|
||||
for _, issue := range result.PSUIssues {
|
||||
fmt.Fprintf(&b, "- ⛔ %s\n", issue)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
// ── Power Distribution Summary ────────────────────────────────────────────
|
||||
b.WriteString("## Power Distribution Summary\n\n")
|
||||
{
|
||||
var totalDefault, totalStable float64
|
||||
for _, gpu := range result.GPUs {
|
||||
stable := gpu.StablePowerLimitW
|
||||
if stable <= 0 {
|
||||
stable = gpu.AppliedPowerLimitW
|
||||
}
|
||||
totalDefault += gpu.DefaultPowerLimitW
|
||||
totalStable += stable
|
||||
}
|
||||
var pdRows [][]string
|
||||
for _, gpu := range result.GPUs {
|
||||
stable := gpu.StablePowerLimitW
|
||||
if stable <= 0 {
|
||||
stable = gpu.AppliedPowerLimitW
|
||||
}
|
||||
realization := "-"
|
||||
if gpu.DefaultPowerLimitW > 0 && stable > 0 {
|
||||
realization = fmt.Sprintf("%.1f%%", stable/gpu.DefaultPowerLimitW*100)
|
||||
}
|
||||
derated := "-"
|
||||
if gpu.Derated {
|
||||
derated = "⚠ yes"
|
||||
}
|
||||
pdRows = append(pdRows, []string{
|
||||
fmt.Sprintf("GPU %d", gpu.Index),
|
||||
fmt.Sprintf("%.0f W", gpu.AppliedPowerLimitW),
|
||||
fmt.Sprintf("%.0f W", stable),
|
||||
realization,
|
||||
derated,
|
||||
})
|
||||
}
|
||||
platformReal := "-"
|
||||
if totalDefault > 0 && totalStable > 0 {
|
||||
platformReal = fmt.Sprintf("%.1f%%", totalStable/totalDefault*100)
|
||||
}
|
||||
pdRows = append(pdRows, []string{
|
||||
"**Platform**",
|
||||
"—",
|
||||
fmt.Sprintf("**%.0f W**", totalStable),
|
||||
fmt.Sprintf("**%s**", platformReal),
|
||||
"",
|
||||
})
|
||||
b.WriteString(fmtMDTable([]string{"GPU", "Single-card limit", "Stable limit", "Realization", "Derated"}, pdRows))
|
||||
b.WriteString("\n")
|
||||
|
||||
// Balance across GPUs — only meaningful with 2+ GPUs.
|
||||
if len(result.GPUs) > 1 {
|
||||
var minS, maxS, sumS float64
|
||||
var cnt int
|
||||
for _, gpu := range result.GPUs {
|
||||
s := gpu.StablePowerLimitW
|
||||
if s <= 0 {
|
||||
s = gpu.AppliedPowerLimitW
|
||||
}
|
||||
if s <= 0 {
|
||||
continue
|
||||
}
|
||||
sumS += s
|
||||
cnt++
|
||||
if cnt == 1 || s < minS {
|
||||
minS = s
|
||||
}
|
||||
if s > maxS {
|
||||
maxS = s
|
||||
}
|
||||
}
|
||||
if cnt > 0 {
|
||||
avg := sumS / float64(cnt)
|
||||
spread := (maxS - minS) / avg * 100
|
||||
balanceNote := "✓ balanced"
|
||||
switch {
|
||||
case spread > 20:
|
||||
balanceNote = "⚠ significant imbalance — check slot thermals"
|
||||
case spread > 10:
|
||||
balanceNote = "— minor imbalance"
|
||||
}
|
||||
fmt.Fprintf(&b, "**GPU power balance:** avg %.0f W · min %.0f W · max %.0f W · spread %.1f%% — %s\n\n",
|
||||
avg, minS, maxS, spread, balanceNote)
|
||||
}
|
||||
}
|
||||
|
||||
// Ramp scalability table — power efficiency of adding each GPU.
|
||||
if len(result.RampSteps) > 1 {
|
||||
b.WriteString("**Ramp power scalability** (stable TDP per step):\n\n")
|
||||
var firstStable float64
|
||||
if len(result.GPUs) > 0 {
|
||||
firstStable = result.GPUs[0].StablePowerLimitW
|
||||
if firstStable <= 0 {
|
||||
firstStable = result.GPUs[0].AppliedPowerLimitW
|
||||
}
|
||||
}
|
||||
var prevCumulative float64
|
||||
var scalRows [][]string
|
||||
for _, step := range result.RampSteps {
|
||||
var cumulative float64
|
||||
for _, gpuIdx := range step.GPUIndices {
|
||||
for _, g := range result.GPUs {
|
||||
if g.Index != gpuIdx {
|
||||
continue
|
||||
}
|
||||
s := g.StablePowerLimitW
|
||||
if s <= 0 {
|
||||
s = g.AppliedPowerLimitW
|
||||
}
|
||||
cumulative += s
|
||||
}
|
||||
}
|
||||
incremental := cumulative - prevCumulative
|
||||
efficiency := "—"
|
||||
if step.StepIndex > 1 && firstStable > 0 {
|
||||
efficiency = fmt.Sprintf("%.1f%%", incremental/firstStable*100)
|
||||
}
|
||||
scalRows = append(scalRows, []string{
|
||||
fmt.Sprintf("%d", step.StepIndex),
|
||||
joinIndexList(step.GPUIndices),
|
||||
fmt.Sprintf("%.0f W", cumulative),
|
||||
fmt.Sprintf("%.0f W", incremental),
|
||||
efficiency,
|
||||
})
|
||||
prevCumulative = cumulative
|
||||
}
|
||||
b.WriteString(fmtMDTable([]string{"Step", "GPUs", "Cumulative stable TDP", "Incremental", "Efficiency vs GPU 1"}, scalRows))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
|
||||
// ── Per-GPU sections ──────────────────────────────────────────────────────
|
||||
var lastStep *NvidiaPowerBenchStep
|
||||
if n := len(result.RampSteps); n > 0 {
|
||||
lastStep = &result.RampSteps[n-1]
|
||||
}
|
||||
for _, gpu := range result.GPUs {
|
||||
fmt.Fprintf(&b, "### GPU %d — %s\n\n", gpu.Index, gpu.Name)
|
||||
|
||||
// Transposed comparison table: Single Run vs All GPU Run.
|
||||
singleClk := "—"
|
||||
singleMem := "—"
|
||||
singleTemp := "—"
|
||||
singlePwr := "—"
|
||||
singleWall := "—"
|
||||
singleFan := "—"
|
||||
if gpu.Telemetry != nil {
|
||||
singleClk = fmt.Sprintf("%.0f", gpu.Telemetry.AvgGraphicsClockMHz)
|
||||
singleMem = fmt.Sprintf("%.0f", gpu.Telemetry.AvgMemoryClockMHz)
|
||||
singleTemp = fmt.Sprintf("%.1f", gpu.Telemetry.AvgTempC)
|
||||
singlePwr = fmt.Sprintf("%.0f W", gpu.Telemetry.AvgPowerW)
|
||||
}
|
||||
if gpu.ServerDeltaW > 0 {
|
||||
singleWall = fmt.Sprintf("%.0f W", gpu.ServerDeltaW)
|
||||
}
|
||||
if gpu.AvgFanRPM > 0 {
|
||||
if gpu.AvgFanDutyCyclePct > 0 {
|
||||
singleFan = fmt.Sprintf("%.0f RPM (%.0f%%)", gpu.AvgFanRPM, gpu.AvgFanDutyCyclePct)
|
||||
} else {
|
||||
singleFan = fmt.Sprintf("%.0f RPM", gpu.AvgFanRPM)
|
||||
}
|
||||
}
|
||||
|
||||
allClk := "—"
|
||||
allMem := "—"
|
||||
allTemp := "—"
|
||||
allPwr := "—"
|
||||
allWall := "—"
|
||||
allFan := "—"
|
||||
if lastStep != nil {
|
||||
if t, ok := lastStep.PerGPUTelemetry[gpu.Index]; ok && t != nil {
|
||||
allClk = fmt.Sprintf("%.0f", t.AvgGraphicsClockMHz)
|
||||
allMem = fmt.Sprintf("%.0f", t.AvgMemoryClockMHz)
|
||||
allTemp = fmt.Sprintf("%.1f", t.AvgTempC)
|
||||
allPwr = fmt.Sprintf("%.0f W", t.AvgPowerW)
|
||||
}
|
||||
if lastStep.ServerDeltaW > 0 && len(lastStep.GPUIndices) > 0 {
|
||||
allWall = fmt.Sprintf("%.0f W", lastStep.ServerDeltaW/float64(len(lastStep.GPUIndices)))
|
||||
}
|
||||
if lastStep.AvgFanRPM > 0 {
|
||||
if lastStep.AvgFanDutyCyclePct > 0 {
|
||||
allFan = fmt.Sprintf("%.0f RPM (%.0f%%)", lastStep.AvgFanRPM, lastStep.AvgFanDutyCyclePct)
|
||||
} else {
|
||||
allFan = fmt.Sprintf("%.0f RPM", lastStep.AvgFanRPM)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tableHeaders := []string{"", "Single Run"}
|
||||
if lastStep != nil {
|
||||
tableHeaders = append(tableHeaders, "All GPU Run")
|
||||
}
|
||||
compRows := [][]string{
|
||||
{"Clock MHz (Mem MHz)", fmt.Sprintf("%s (%s)", singleClk, singleMem)},
|
||||
{"Avg Temp °C", singleTemp},
|
||||
{"Power W", singlePwr},
|
||||
{"Per GPU wall W", singleWall},
|
||||
{"Avg Fan RPM (duty%)", singleFan},
|
||||
}
|
||||
if lastStep != nil {
|
||||
compRows[0] = append(compRows[0], fmt.Sprintf("%s (%s)", allClk, allMem))
|
||||
compRows[1] = append(compRows[1], allTemp)
|
||||
compRows[2] = append(compRows[2], allPwr)
|
||||
compRows[3] = append(compRows[3], allWall)
|
||||
compRows[4] = append(compRows[4], allFan)
|
||||
}
|
||||
b.WriteString(fmtMDTable(tableHeaders, compRows))
|
||||
b.WriteString("\n")
|
||||
|
||||
for _, note := range gpu.Notes {
|
||||
fmt.Fprintf(&b, "- %s\n", note)
|
||||
}
|
||||
if len(gpu.Notes) > 0 {
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func renderPowerBenchSummary(result NvidiaPowerBenchResult) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "run_at_utc=%s\n", result.GeneratedAt.Format(time.RFC3339))
|
||||
fmt.Fprintf(&b, "benchmark_version=%s\n", result.BenchmarkVersion)
|
||||
fmt.Fprintf(&b, "benchmark_profile=%s\n", result.BenchmarkProfile)
|
||||
fmt.Fprintf(&b, "overall_status=%s\n", result.OverallStatus)
|
||||
fmt.Fprintf(&b, "platform_max_tdp_w=%.0f\n", result.PlatformMaxTDPW)
|
||||
fmt.Fprintf(&b, "gpu_count=%d\n", len(result.GPUs))
|
||||
if len(result.RecommendedSlotOrder) > 0 {
|
||||
fmt.Fprintf(&b, "recommended_slot_order=%s\n", joinIndexList(result.RecommendedSlotOrder))
|
||||
}
|
||||
for _, step := range result.RampSteps {
|
||||
fmt.Fprintf(&b, "ramp_step_%d_gpus=%s\n", step.StepIndex, joinIndexList(step.GPUIndices))
|
||||
fmt.Fprintf(&b, "ramp_step_%d_new_gpu=%d\n", step.StepIndex, step.NewGPUIndex)
|
||||
fmt.Fprintf(&b, "ramp_step_%d_stable_limit_w=%.0f\n", step.StepIndex, step.NewGPUStableLimitW)
|
||||
fmt.Fprintf(&b, "ramp_step_%d_total_power_w=%.0f\n", step.StepIndex, step.TotalObservedPowerW)
|
||||
if step.ServerLoadedW > 0 {
|
||||
fmt.Fprintf(&b, "ramp_step_%d_server_loaded_w=%.0f\n", step.StepIndex, step.ServerLoadedW)
|
||||
fmt.Fprintf(&b, "ramp_step_%d_server_delta_w=%.0f\n", step.StepIndex, step.ServerDeltaW)
|
||||
}
|
||||
}
|
||||
for _, gpu := range result.GPUs {
|
||||
if gpu.StablePowerLimitW > 0 {
|
||||
fmt.Fprintf(&b, "gpu_%d_stable_limit_w=%.0f\n", gpu.Index, gpu.StablePowerLimitW)
|
||||
}
|
||||
if gpu.ServerLoadedW > 0 {
|
||||
fmt.Fprintf(&b, "gpu_%d_server_loaded_w=%.0f\n", gpu.Index, gpu.ServerLoadedW)
|
||||
fmt.Fprintf(&b, "gpu_%d_server_delta_w=%.0f\n", gpu.Index, gpu.ServerDeltaW)
|
||||
}
|
||||
}
|
||||
if sp := result.ServerPower; sp != nil && sp.Available {
|
||||
fmt.Fprintf(&b, "server_idle_w=%.0f\n", sp.IdleW)
|
||||
fmt.Fprintf(&b, "server_loaded_w=%.0f\n", sp.LoadedW)
|
||||
fmt.Fprintf(&b, "server_delta_w=%.0f\n", sp.DeltaW)
|
||||
fmt.Fprintf(&b, "server_reporting_ratio=%.2f\n", sp.ReportingRatio)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,495 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (s *System) RunNvidiaPowerBench(ctx context.Context, baseDir string, opts NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if logFunc == nil {
|
||||
logFunc = func(string) {}
|
||||
}
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = "/var/log/bee-bench/power"
|
||||
}
|
||||
opts = normalizeNvidiaBenchmarkOptionsForBenchmark(opts)
|
||||
selected, err := resolveNvidiaGPUSelection(opts.GPUIndices, opts.ExcludeGPUIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(selected) == 0 {
|
||||
return "", fmt.Errorf("no NVIDIA GPUs selected")
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
runDir := filepath.Join(baseDir, "power-"+ts)
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", fmt.Errorf("mkdir %s: %w", runDir, err)
|
||||
}
|
||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||
hostname, _ := os.Hostname()
|
||||
result := NvidiaPowerBenchResult{
|
||||
BenchmarkVersion: benchmarkVersion,
|
||||
GeneratedAt: time.Now().UTC(),
|
||||
Hostname: hostname,
|
||||
ServerModel: readServerModel(),
|
||||
BenchmarkProfile: opts.Profile,
|
||||
SelectedGPUIndices: append([]int(nil), selected...),
|
||||
OverallStatus: "OK",
|
||||
}
|
||||
infoByIndex, infoErr := queryBenchmarkGPUInfo(selected)
|
||||
if infoErr != nil {
|
||||
return "", infoErr
|
||||
}
|
||||
// Capture full nvidia-smi -q snapshot at the start of the run.
|
||||
if out, err := runSATCommandCtx(ctx, verboseLog, "00-nvidia-smi-q.log", []string{"nvidia-smi", "-q"}, nil, nil); err == nil {
|
||||
_ = os.WriteFile(filepath.Join(runDir, "00-nvidia-smi-q.log"), out, 0644)
|
||||
}
|
||||
durationSec := powerBenchDurationSec(opts.Profile)
|
||||
|
||||
// Sample server idle power before any GPU load.
|
||||
var serverIdleW float64
|
||||
var serverIdleOK bool
|
||||
idleSDRStopCh := make(chan struct{})
|
||||
idleSDRCh := startIPMISDRSampler(idleSDRStopCh, benchmarkPowerAutotuneSampleInterval)
|
||||
if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, 10, benchmarkPowerAutotuneSampleInterval); ok {
|
||||
serverIdleW = w
|
||||
serverIdleOK = true
|
||||
logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w))
|
||||
}
|
||||
close(idleSDRStopCh)
|
||||
sdrIdle := summarizeSDRPowerSeries(<-idleSDRCh)
|
||||
psuBefore := psuStatusSnapshot()
|
||||
|
||||
// Phase 1: calibrate each GPU individually (sequentially, one at a time) to
|
||||
// establish a true single-card power baseline unaffected by neighbour heat.
|
||||
calibByIndex := make(map[int]benchmarkPowerCalibrationResult, len(selected))
|
||||
singleIPMILoadedW := make(map[int]float64, len(selected))
|
||||
singleRunSummaryByIndex := make(map[int]benchmarkPowerCalibrationRunSummary, len(selected))
|
||||
var allRestoreActions []benchmarkRestoreAction
|
||||
// allPowerRows accumulates telemetry from all phases for the top-level gpu-metrics.csv.
|
||||
var allPowerRows []GPUMetricRow
|
||||
var powerCursor float64
|
||||
for _, idx := range selected {
|
||||
singleDir := filepath.Join(runDir, fmt.Sprintf("single-%02d", idx))
|
||||
_ = os.MkdirAll(singleDir, 0755)
|
||||
singleInfo := cloneBenchmarkGPUInfoMap(infoByIndex)
|
||||
if failed := resetBenchmarkGPUs(ctx, verboseLog, []int{idx}, logFunc); len(failed) > 0 {
|
||||
return "", fmt.Errorf("power benchmark pre-flight: failed to reset GPU %d; benchmark aborted to keep measurements clean", idx)
|
||||
}
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d single-card baseline", idx))
|
||||
singlePowerStopCh := make(chan struct{})
|
||||
singlePowerCh := startSelectedPowerSourceSampler(singlePowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
|
||||
c, restore, singleRows, singleRun := runBenchmarkPowerCalibration(ctx, verboseLog, singleDir, []int{idx}, singleInfo, logFunc, nil, durationSec)
|
||||
appendBenchmarkMetrics(&allPowerRows, singleRows, fmt.Sprintf("single-gpu-%d", idx), &powerCursor, 0)
|
||||
close(singlePowerStopCh)
|
||||
if samples := <-singlePowerCh; len(samples) > 0 {
|
||||
singleIPMILoadedW[idx] = benchmarkMean(samples)
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d single-card server power (%s avg): %.0f W", idx, opts.ServerPowerSource, singleIPMILoadedW[idx]))
|
||||
} else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && singleRun.LoadedSDR.PSUInW > 0 {
|
||||
singleIPMILoadedW[idx] = singleRun.LoadedSDR.PSUInW
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d single-card fallback server power (SDR avg): %.0f W", idx, singleRun.LoadedSDR.PSUInW))
|
||||
}
|
||||
allRestoreActions = append(allRestoreActions, restore...)
|
||||
if r, ok := c[idx]; ok {
|
||||
calibByIndex[idx] = r
|
||||
}
|
||||
singleRunSummaryByIndex[idx] = singleRun
|
||||
}
|
||||
defer func() {
|
||||
for i := len(allRestoreActions) - 1; i >= 0; i-- {
|
||||
allRestoreActions[i].fn()
|
||||
}
|
||||
}()
|
||||
gpus := make([]NvidiaPowerBenchGPU, 0, len(selected))
|
||||
for _, idx := range selected {
|
||||
info := infoByIndex[idx]
|
||||
calib := calibByIndex[idx]
|
||||
status := "OK"
|
||||
if !calib.Completed {
|
||||
status = "FAILED"
|
||||
result.OverallStatus = "PARTIAL"
|
||||
} else if calib.Derated {
|
||||
status = "PARTIAL"
|
||||
if result.OverallStatus == "OK" {
|
||||
result.OverallStatus = "PARTIAL"
|
||||
}
|
||||
}
|
||||
gpu := NvidiaPowerBenchGPU{
|
||||
Index: idx,
|
||||
Name: info.Name,
|
||||
BusID: info.BusID,
|
||||
DefaultPowerLimitW: info.DefaultPowerLimitW,
|
||||
AppliedPowerLimitW: calib.AppliedPowerLimitW,
|
||||
MaxObservedPowerW: calib.Summary.P95PowerW,
|
||||
MaxObservedTempC: calib.Summary.P95TempC,
|
||||
CalibrationAttempts: calib.Attempts,
|
||||
Derated: calib.Derated,
|
||||
Status: status,
|
||||
Notes: append([]string(nil), calib.Notes...),
|
||||
CoolingWarning: calib.CoolingWarning,
|
||||
}
|
||||
if w, ok := singleIPMILoadedW[idx]; ok && serverIdleOK && w > 0 {
|
||||
gpu.ServerLoadedW = w
|
||||
gpu.ServerDeltaW = w - serverIdleW
|
||||
}
|
||||
if len(calib.MetricRows) > 0 {
|
||||
t := summarizeBenchmarkTelemetry(calib.MetricRows)
|
||||
gpu.Telemetry = &t
|
||||
}
|
||||
if singleRun := singleRunSummaryByIndex[idx]; singleRun.AvgFanRPM > 0 {
|
||||
gpu.AvgFanRPM = singleRun.AvgFanRPM
|
||||
gpu.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct
|
||||
}
|
||||
gpus = append(gpus, gpu)
|
||||
}
|
||||
sort.Slice(gpus, func(i, j int) bool {
|
||||
if gpus[i].MaxObservedPowerW != gpus[j].MaxObservedPowerW {
|
||||
return gpus[i].MaxObservedPowerW > gpus[j].MaxObservedPowerW
|
||||
}
|
||||
if gpus[i].AppliedPowerLimitW != gpus[j].AppliedPowerLimitW {
|
||||
return gpus[i].AppliedPowerLimitW > gpus[j].AppliedPowerLimitW
|
||||
}
|
||||
if gpus[i].Derated != gpus[j].Derated {
|
||||
return !gpus[i].Derated
|
||||
}
|
||||
return gpus[i].Index < gpus[j].Index
|
||||
})
|
||||
result.GPUs = gpus
|
||||
result.RecommendedSlotOrder = make([]int, 0, len(gpus))
|
||||
for _, gpu := range gpus {
|
||||
result.RecommendedSlotOrder = append(result.RecommendedSlotOrder, gpu.Index)
|
||||
}
|
||||
if len(result.RecommendedSlotOrder) > 0 {
|
||||
result.Findings = append(result.Findings, fmt.Sprintf("Recommended slot order for installation based on single-card %s: %s.", benchmarkPowerEngineLabel(benchmarkPowerEngine()), joinIndexList(result.RecommendedSlotOrder)))
|
||||
}
|
||||
for _, gpu := range gpus {
|
||||
if gpu.Derated {
|
||||
result.Findings = append(result.Findings, fmt.Sprintf("GPU %d required reduced power limit %.0f W to complete %s.", gpu.Index, gpu.AppliedPowerLimitW, benchmarkPowerEngineLabel(benchmarkPowerEngine())))
|
||||
}
|
||||
if gpu.CoolingWarning != "" {
|
||||
result.Findings = append(result.Findings, fmt.Sprintf(
|
||||
"GPU %d: %s. Operator action: rerun the benchmark with fan speed manually fixed at 100%% to confirm actual thermal headroom.",
|
||||
gpu.Index, gpu.CoolingWarning,
|
||||
))
|
||||
}
|
||||
}
|
||||
singleByIndex := make(map[int]NvidiaPowerBenchGPU, len(gpus))
|
||||
for _, gpu := range gpus {
|
||||
singleByIndex[gpu.Index] = gpu
|
||||
}
|
||||
|
||||
// Phase 2: cumulative thermal ramp.
|
||||
// Each step introduces one new GPU into an environment where all previously
|
||||
// calibrated GPUs are already running at their fixed stable limits. The new
|
||||
// GPU's stable TDP is searched via binary search under real
|
||||
// multi-GPU thermal load. Once found, its limit is fixed permanently for all
|
||||
// subsequent steps. This ensures each GPU's limit reflects actual sustained
|
||||
// power in the final full-system thermal state.
|
||||
//
|
||||
// stableLimits accumulates GPU index → fixed stable limit (W) across steps.
|
||||
stableLimits := make(map[int]int, len(result.RecommendedSlotOrder))
|
||||
|
||||
// serverLoadedW tracks the IPMI server power from the final ramp step
|
||||
// (all GPUs simultaneously loaded). Earlier steps' values are stored
|
||||
// per-step in NvidiaPowerBenchStep.ServerLoadedW.
|
||||
var serverLoadedW float64
|
||||
var serverLoadedOK bool
|
||||
// sdrLastStep retains the phase-averaged SDR readings from the last ramp step
|
||||
// while GPUs are loaded. Used in the summary instead of re-sampling after the
|
||||
// test when GPUs have already returned to idle.
|
||||
var sdrLastStep benchmarkSDRSeriesSummary
|
||||
|
||||
// Step 1: reuse single-card calibration result directly.
|
||||
if len(result.RecommendedSlotOrder) > 0 {
|
||||
firstIdx := result.RecommendedSlotOrder[0]
|
||||
firstCalib := calibByIndex[firstIdx]
|
||||
stableLimits[firstIdx] = int(math.Round(firstCalib.AppliedPowerLimitW))
|
||||
ramp := NvidiaPowerBenchStep{
|
||||
StepIndex: 1,
|
||||
GPUIndices: []int{firstIdx},
|
||||
NewGPUIndex: firstIdx,
|
||||
NewGPUStableLimitW: firstCalib.AppliedPowerLimitW,
|
||||
TotalObservedPowerW: firstCalib.Summary.P95PowerW,
|
||||
AvgObservedPowerW: firstCalib.Summary.P95PowerW,
|
||||
Derated: firstCalib.Derated,
|
||||
Status: "OK",
|
||||
}
|
||||
if w, ok := singleIPMILoadedW[firstIdx]; ok && serverIdleOK && w > 0 {
|
||||
ramp.ServerLoadedW = w
|
||||
ramp.ServerDeltaW = w - serverIdleW
|
||||
}
|
||||
if singleRun := singleRunSummaryByIndex[firstIdx]; singleRun.AvgFanRPM > 0 {
|
||||
ramp.AvgFanRPM = singleRun.AvgFanRPM
|
||||
ramp.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct
|
||||
}
|
||||
firstSummary := firstCalib.Summary
|
||||
ramp.PerGPUTelemetry = map[int]*BenchmarkTelemetrySummary{firstIdx: &firstSummary}
|
||||
if !firstCalib.Completed {
|
||||
ramp.Status = "FAILED"
|
||||
ramp.Notes = append(ramp.Notes, fmt.Sprintf("GPU %d did not complete single-card %s", firstIdx, benchmarkPowerEngineLabel(benchmarkPowerEngine())))
|
||||
result.OverallStatus = "PARTIAL"
|
||||
} else if firstCalib.Derated {
|
||||
ramp.Status = "PARTIAL"
|
||||
if result.OverallStatus == "OK" {
|
||||
result.OverallStatus = "PARTIAL"
|
||||
}
|
||||
result.Findings = append(result.Findings, fmt.Sprintf("Ramp step 1 (GPU %d) required derating to %.0f W.", firstIdx, firstCalib.AppliedPowerLimitW))
|
||||
}
|
||||
result.RampSteps = append(result.RampSteps, ramp)
|
||||
logFunc(fmt.Sprintf("power ramp: step 1/%d — reused single-card calibration for GPU %d, stable limit %.0f W",
|
||||
len(result.RecommendedSlotOrder), firstIdx, firstCalib.AppliedPowerLimitW))
|
||||
}
|
||||
|
||||
// Steps 2..N: each step revalidates every already-active GPU under the new
|
||||
// cumulative thermal environment and also calibrates the newly introduced
|
||||
// GPU. Previously found limits are used only as seeds for the search.
|
||||
for stepNum := 1; stepNum < len(result.RecommendedSlotOrder); stepNum++ {
|
||||
step := stepNum + 1
|
||||
subset := append([]int(nil), result.RecommendedSlotOrder[:step]...)
|
||||
newGPUIdx := result.RecommendedSlotOrder[stepNum]
|
||||
stepDir := filepath.Join(runDir, fmt.Sprintf("step-%02d", step))
|
||||
_ = os.MkdirAll(stepDir, 0755)
|
||||
|
||||
// Reuse the latest stable limits as starting points, but re-check every
|
||||
// active GPU in this hotter configuration. For the newly introduced GPU,
|
||||
// seed from its single-card calibration so we do not restart from the
|
||||
// default TDP when a prior derated limit is already known.
|
||||
seedForStep := make(map[int]int, len(subset))
|
||||
for _, idx := range subset {
|
||||
if lim, ok := stableLimits[idx]; ok && lim > 0 {
|
||||
seedForStep[idx] = lim
|
||||
continue
|
||||
}
|
||||
if base, ok := calibByIndex[idx]; ok {
|
||||
lim := int(math.Round(base.AppliedPowerLimitW))
|
||||
if lim > 0 {
|
||||
seedForStep[idx] = lim
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logFunc(fmt.Sprintf("power ramp: step %d/%d — revalidating %d active GPU(s) including new GPU %d",
|
||||
step, len(result.RecommendedSlotOrder), len(subset), newGPUIdx))
|
||||
|
||||
stepInfo := cloneBenchmarkGPUInfoMap(infoByIndex)
|
||||
stepPowerStopCh := make(chan struct{})
|
||||
stepPowerCh := startSelectedPowerSourceSampler(stepPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
|
||||
stepCalib, stepRestore, stepRows, stepRun := runBenchmarkPowerCalibration(ctx, verboseLog, stepDir, subset, stepInfo, logFunc, seedForStep, durationSec)
|
||||
appendBenchmarkMetrics(&allPowerRows, stepRows, fmt.Sprintf("ramp-step-%d", step), &powerCursor, 0)
|
||||
close(stepPowerStopCh)
|
||||
var stepIPMILoadedW float64
|
||||
var stepIPMIOK bool
|
||||
if samples := <-stepPowerCh; len(samples) > 0 {
|
||||
stepIPMILoadedW = benchmarkMean(samples)
|
||||
stepIPMIOK = true
|
||||
}
|
||||
// Accumulate restore actions; they all run in the outer defer.
|
||||
allRestoreActions = append(allRestoreActions, stepRestore...)
|
||||
|
||||
ramp := NvidiaPowerBenchStep{
|
||||
StepIndex: step,
|
||||
GPUIndices: subset,
|
||||
NewGPUIndex: newGPUIdx,
|
||||
Status: "OK",
|
||||
}
|
||||
|
||||
// Total observed power = sum of p95 across all GPUs in this step.
|
||||
for _, idx := range subset {
|
||||
if c, ok := stepCalib[idx]; ok {
|
||||
ramp.TotalObservedPowerW += c.Summary.P95PowerW
|
||||
}
|
||||
}
|
||||
if len(subset) > 0 {
|
||||
ramp.AvgObservedPowerW = ramp.TotalObservedPowerW / float64(len(subset))
|
||||
}
|
||||
|
||||
for _, idx := range subset {
|
||||
c, ok := stepCalib[idx]
|
||||
if !ok || !c.Completed {
|
||||
fallback := 0
|
||||
if lim, ok := stableLimits[idx]; ok && lim > 0 {
|
||||
fallback = lim
|
||||
} else if fb, ok := calibByIndex[idx]; ok {
|
||||
fallback = int(math.Round(fb.AppliedPowerLimitW))
|
||||
}
|
||||
if fallback > 0 {
|
||||
stableLimits[idx] = fallback
|
||||
}
|
||||
ramp.Status = "FAILED"
|
||||
ramp.Notes = append(ramp.Notes,
|
||||
fmt.Sprintf("GPU %d did not complete %s in ramp step %d; keeping previous stable limit %d W", idx, benchmarkPowerEngineLabel(benchmarkPowerEngine()), step, fallback))
|
||||
result.OverallStatus = "PARTIAL"
|
||||
continue
|
||||
}
|
||||
|
||||
prevLimit, hadPrev := stableLimits[idx]
|
||||
newLimit := int(math.Round(c.AppliedPowerLimitW))
|
||||
stableLimits[idx] = newLimit
|
||||
if idx == newGPUIdx {
|
||||
ramp.NewGPUStableLimitW = c.AppliedPowerLimitW
|
||||
ramp.Derated = c.Derated
|
||||
}
|
||||
if c.Derated {
|
||||
ramp.Status = "PARTIAL"
|
||||
if result.OverallStatus == "OK" {
|
||||
result.OverallStatus = "PARTIAL"
|
||||
}
|
||||
}
|
||||
if hadPrev && newLimit < prevLimit {
|
||||
ramp.Notes = append(ramp.Notes,
|
||||
fmt.Sprintf("GPU %d was re-derated from %d W to %d W under combined thermal load.", idx, prevLimit, newLimit))
|
||||
}
|
||||
}
|
||||
|
||||
if c, ok := stepCalib[newGPUIdx]; ok && c.Completed && c.Derated {
|
||||
result.Findings = append(result.Findings, fmt.Sprintf("Ramp step %d (GPU %d) required derating to %.0f W under combined thermal load.", step, newGPUIdx, c.AppliedPowerLimitW))
|
||||
}
|
||||
|
||||
// Per-step PSU slot readings are averaged over the whole load phase rather
|
||||
// than captured as a single end-of-phase snapshot.
|
||||
sdrStep := stepRun.LoadedSDR
|
||||
if len(sdrStep.PSUSlots) > 0 {
|
||||
ramp.PSUSlotReadings = sdrStep.PSUSlots
|
||||
}
|
||||
|
||||
if stepIPMIOK && serverIdleOK && stepIPMILoadedW > 0 {
|
||||
ramp.ServerLoadedW = stepIPMILoadedW
|
||||
ramp.ServerDeltaW = stepIPMILoadedW - serverIdleW
|
||||
logFunc(fmt.Sprintf("power ramp: step %d server loaded power (%s avg): %.0f W", step, opts.ServerPowerSource, stepIPMILoadedW))
|
||||
// The last step has all GPUs loaded — use it as the top-level loaded_w.
|
||||
if step == len(result.RecommendedSlotOrder) {
|
||||
serverLoadedW = stepIPMILoadedW
|
||||
serverLoadedOK = true
|
||||
sdrLastStep = sdrStep
|
||||
}
|
||||
} else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && sdrStep.PSUInW > 0 {
|
||||
ramp.ServerLoadedW = sdrStep.PSUInW
|
||||
ramp.ServerDeltaW = sdrStep.PSUInW - sdrIdle.PSUInW
|
||||
logFunc(fmt.Sprintf("power ramp: step %d fallback server loaded power (SDR avg): %.0f W", step, sdrStep.PSUInW))
|
||||
if step == len(result.RecommendedSlotOrder) {
|
||||
serverLoadedW = sdrStep.PSUInW
|
||||
serverLoadedOK = true
|
||||
sdrLastStep = sdrStep
|
||||
}
|
||||
}
|
||||
|
||||
// Fan values are phase averages over the same load window.
|
||||
if stepRun.AvgFanRPM > 0 {
|
||||
ramp.AvgFanRPM = stepRun.AvgFanRPM
|
||||
ramp.AvgFanDutyCyclePct = stepRun.AvgFanDutyCyclePct
|
||||
}
|
||||
|
||||
// Per-GPU telemetry from this ramp step's calibration.
|
||||
ramp.PerGPUTelemetry = make(map[int]*BenchmarkTelemetrySummary, len(subset))
|
||||
for _, gpuIdx := range subset {
|
||||
if c, ok := stepCalib[gpuIdx]; ok {
|
||||
s := c.Summary
|
||||
ramp.PerGPUTelemetry[gpuIdx] = &s
|
||||
}
|
||||
}
|
||||
|
||||
result.RampSteps = append(result.RampSteps, ramp)
|
||||
}
|
||||
|
||||
// Populate StablePowerLimitW on each GPU entry from the accumulated stable limits.
|
||||
for i := range result.GPUs {
|
||||
if lim, ok := stableLimits[result.GPUs[i].Index]; ok {
|
||||
result.GPUs[i].StablePowerLimitW = float64(lim)
|
||||
}
|
||||
if result.GPUs[i].StablePowerLimitW > 0 && result.GPUs[i].AppliedPowerLimitW > 0 &&
|
||||
result.GPUs[i].StablePowerLimitW < result.GPUs[i].AppliedPowerLimitW {
|
||||
result.GPUs[i].Derated = true
|
||||
result.Findings = append(result.Findings, fmt.Sprintf(
|
||||
"GPU %d required additional derating from %.0f W (single-card) to %.0f W under full-system thermal load.",
|
||||
result.GPUs[i].Index, result.GPUs[i].AppliedPowerLimitW, result.GPUs[i].StablePowerLimitW,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
// PlatformMaxTDPW = sum of all stable limits — the actual sustained power
|
||||
// budget of this server with all GPUs running simultaneously without throttling.
|
||||
for _, lim := range stableLimits {
|
||||
result.PlatformMaxTDPW += float64(lim)
|
||||
}
|
||||
|
||||
// Characterize server power from IPMI idle/loaded samples.
|
||||
// gpuActualSumW = sum of p95 GPU power from the last ramp step — actual
|
||||
// measured consumption, not the stable limit cap. This is the correct
|
||||
// denominator for the reporting ratio: limit caps (PlatformMaxTDPW) inflate
|
||||
// the denominator and make the ratio appear artificially low.
|
||||
var gpuActualSumW float64
|
||||
if n := len(result.RampSteps); n > 0 {
|
||||
gpuActualSumW = result.RampSteps[n-1].TotalObservedPowerW
|
||||
}
|
||||
if gpuActualSumW <= 0 {
|
||||
gpuActualSumW = result.PlatformMaxTDPW
|
||||
}
|
||||
_ = serverIdleOK // used implicitly via characterizeServerPower
|
||||
result.ServerPower = characterizeServerPower(serverIdleW, serverLoadedW, gpuActualSumW, opts.ServerPowerSource, serverIdleOK && serverLoadedOK)
|
||||
// Supplement DCMI with SDR multi-source data via collector's PSU slot patterns.
|
||||
// Per-slot readings enable correlation with audit HardwarePowerSupply entries.
|
||||
if result.ServerPower != nil {
|
||||
// Use the SDR phase average from the last ramp step (GPUs still loaded)
|
||||
// rather than re-sampling here, which would capture post-test idle state.
|
||||
sdrLoaded := sdrLastStep
|
||||
result.ServerPower.PSUInputIdleW = sdrIdle.PSUInW
|
||||
result.ServerPower.PSUInputLoadedW = sdrLoaded.PSUInW
|
||||
result.ServerPower.PSUOutputIdleW = sdrIdle.PSUOutW
|
||||
result.ServerPower.PSUOutputLoadedW = sdrLoaded.PSUOutW
|
||||
result.ServerPower.GPUSlotTotalW = sdrLoaded.GPUSlotW
|
||||
if len(sdrIdle.PSUSlots) > 0 {
|
||||
result.ServerPower.PSUSlotReadingsIdle = sdrIdle.PSUSlots
|
||||
}
|
||||
if len(sdrLoaded.PSUSlots) > 0 {
|
||||
result.ServerPower.PSUSlotReadingsLoaded = sdrLoaded.PSUSlots
|
||||
}
|
||||
if sdrIdle.PSUInW > 0 && result.ServerPower.IdleW > 0 {
|
||||
result.ServerPower.DCMICoverageRatio = result.ServerPower.IdleW / sdrIdle.PSUInW
|
||||
}
|
||||
if len(sdrLoaded.SkippedSensors) > 0 {
|
||||
result.ServerPower.Notes = append(result.ServerPower.Notes,
|
||||
"SDR sensors skipped (self-healed): "+strings.Join(sdrLoaded.SkippedSensors, "; "))
|
||||
}
|
||||
if sdrLoaded.Samples > 0 {
|
||||
result.ServerPower.Notes = append(result.ServerPower.Notes,
|
||||
fmt.Sprintf("Final SDR PSU loaded values are phase averages across %d sample(s) from the last full-load step.", sdrLoaded.Samples))
|
||||
}
|
||||
// Detect DCMI partial coverage: direct SDR comparison first,
|
||||
// ramp heuristic as fallback when SDR PSU sensors are absent.
|
||||
dcmiUnreliable := detectDCMIPartialCoverage(result.ServerPower) ||
|
||||
(sdrIdle.PSUInW == 0 && detectIPMISaturationFallback(result.RampSteps))
|
||||
if dcmiUnreliable {
|
||||
result.ServerPower.Notes = append(result.ServerPower.Notes,
|
||||
fmt.Sprintf("IPMI DCMI covers only a subset of installed PSUs (coverage %.0f%%). "+
|
||||
"Use SDR PSU Δ ratio for GPU accuracy assessment; DCMI ratio is not reliable.",
|
||||
result.ServerPower.DCMICoverageRatio*100))
|
||||
}
|
||||
}
|
||||
result.PSUIssues = diffPSUStatus(psuBefore, psuStatusSnapshot())
|
||||
// Write top-level gpu-metrics.csv/.html aggregating all phases.
|
||||
writeBenchmarkMetricsFiles(runDir, allPowerRows)
|
||||
resultJSON, err := json.MarshalIndent(result, "", " ")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal power result: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(runDir, "result.json"), resultJSON, 0644); err != nil {
|
||||
return "", fmt.Errorf("write result.json: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(runDir, "report.md"), []byte(renderPowerBenchReport(result)), 0644); err != nil {
|
||||
return "", fmt.Errorf("write report.md: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(renderPowerBenchSummary(result)), 0644); err != nil {
|
||||
return "", fmt.Errorf("write summary.txt: %w", err)
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
@@ -0,0 +1,537 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func runBenchmarkInterconnect(ctx context.Context, verboseLog, runDir string, gpuIndices []int, spec benchmarkProfileSpec, logFunc func(string)) *BenchmarkInterconnectResult {
|
||||
result := &BenchmarkInterconnectResult{
|
||||
Status: "UNSUPPORTED",
|
||||
Attempted: true,
|
||||
SelectedGPUIndices: append([]int(nil), gpuIndices...),
|
||||
}
|
||||
cmd := []string{
|
||||
"all_reduce_perf",
|
||||
"-b", "512M",
|
||||
"-e", "4G",
|
||||
"-f", "2",
|
||||
"-g", strconv.Itoa(len(gpuIndices)),
|
||||
"--iters", strconv.Itoa(maxInt(20, spec.NCCLSec/10)),
|
||||
}
|
||||
env := []string{
|
||||
"CUDA_DEVICE_ORDER=PCI_BUS_ID",
|
||||
"CUDA_VISIBLE_DEVICES=" + joinIndexList(gpuIndices),
|
||||
}
|
||||
logFunc(fmt.Sprintf("NCCL interconnect: gpus=%s", joinIndexList(gpuIndices)))
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, "nccl-all-reduce.log", cmd, env, logFunc)
|
||||
_ = os.WriteFile(filepath.Join(runDir, "nccl-all-reduce.log"), out, 0644)
|
||||
if err != nil {
|
||||
result.Notes = append(result.Notes, strings.TrimSpace(string(out)))
|
||||
return result
|
||||
}
|
||||
avgAlg, maxAlg, avgBus, maxBus := parseNCCLAllReduceOutput(string(out))
|
||||
result.Status = "OK"
|
||||
result.Supported = true
|
||||
result.AvgAlgBWGBps = avgAlg
|
||||
result.MaxAlgBWGBps = maxAlg
|
||||
result.AvgBusBWGBps = avgBus
|
||||
result.MaxBusBWGBps = maxBus
|
||||
return result
|
||||
}
|
||||
|
||||
func parseNCCLAllReduceOutput(raw string) (avgAlg, maxAlg, avgBus, maxBus float64) {
|
||||
lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
|
||||
var algs []float64
|
||||
var buses []float64
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 8 {
|
||||
continue
|
||||
}
|
||||
for i := 0; i+2 < len(fields); i++ {
|
||||
timeVal, err1 := strconv.ParseFloat(fields[i], 64)
|
||||
algVal, err2 := strconv.ParseFloat(fields[i+1], 64)
|
||||
busVal, err3 := strconv.ParseFloat(fields[i+2], 64)
|
||||
if err1 == nil && err2 == nil && err3 == nil && timeVal > 0 {
|
||||
algs = append(algs, algVal)
|
||||
buses = append(buses, busVal)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(algs) == 0 {
|
||||
return 0, 0, 0, 0
|
||||
}
|
||||
return benchmarkMean(algs), benchmarkMax(algs), benchmarkMean(buses), benchmarkMax(buses)
|
||||
}
|
||||
|
||||
func queryThrottleCounters(gpuIndex int) (BenchmarkThrottleCounters, error) {
|
||||
out, err := satExecCommand(
|
||||
"nvidia-smi",
|
||||
"--id="+strconv.Itoa(gpuIndex),
|
||||
"--query-gpu=clocks_event_reasons_counters.sw_power_cap,clocks_event_reasons_counters.sw_thermal_slowdown,clocks_event_reasons_counters.sync_boost,clocks_event_reasons_counters.hw_thermal_slowdown,clocks_event_reasons_counters.hw_power_brake_slowdown",
|
||||
"--format=csv,noheader,nounits",
|
||||
).Output()
|
||||
if err != nil {
|
||||
return BenchmarkThrottleCounters{}, err
|
||||
}
|
||||
fields := strings.Split(strings.TrimSpace(string(out)), ",")
|
||||
if len(fields) < 5 {
|
||||
return BenchmarkThrottleCounters{}, fmt.Errorf("unexpected throttle counter columns: %q", strings.TrimSpace(string(out)))
|
||||
}
|
||||
return BenchmarkThrottleCounters{
|
||||
SWPowerCapUS: parseBenchmarkUint64(fields[0]),
|
||||
SWThermalSlowdownUS: parseBenchmarkUint64(fields[1]),
|
||||
SyncBoostUS: parseBenchmarkUint64(fields[2]),
|
||||
HWThermalSlowdownUS: parseBenchmarkUint64(fields[3]),
|
||||
HWPowerBrakeSlowdownUS: parseBenchmarkUint64(fields[4]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func diffThrottleCounters(before, after BenchmarkThrottleCounters) BenchmarkThrottleCounters {
|
||||
return BenchmarkThrottleCounters{
|
||||
SWPowerCapUS: saturatingSub(after.SWPowerCapUS, before.SWPowerCapUS),
|
||||
SWThermalSlowdownUS: saturatingSub(after.SWThermalSlowdownUS, before.SWThermalSlowdownUS),
|
||||
SyncBoostUS: saturatingSub(after.SyncBoostUS, before.SyncBoostUS),
|
||||
HWThermalSlowdownUS: saturatingSub(after.HWThermalSlowdownUS, before.HWThermalSlowdownUS),
|
||||
HWPowerBrakeSlowdownUS: saturatingSub(after.HWPowerBrakeSlowdownUS, before.HWPowerBrakeSlowdownUS),
|
||||
}
|
||||
}
|
||||
|
||||
func queryECCCounters(gpuIndex int) (BenchmarkECCCounters, error) {
|
||||
out, err := satExecCommand(
|
||||
"nvidia-smi",
|
||||
"--id="+strconv.Itoa(gpuIndex),
|
||||
"--query-gpu=ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total",
|
||||
"--format=csv,noheader,nounits",
|
||||
).Output()
|
||||
if err != nil {
|
||||
return BenchmarkECCCounters{}, err
|
||||
}
|
||||
fields := strings.Split(strings.TrimSpace(string(out)), ",")
|
||||
if len(fields) < 2 {
|
||||
return BenchmarkECCCounters{}, fmt.Errorf("unexpected ECC counter columns: %q", strings.TrimSpace(string(out)))
|
||||
}
|
||||
corrected, err1 := strconv.ParseUint(strings.TrimSpace(fields[0]), 10, 64)
|
||||
uncorrected, err2 := strconv.ParseUint(strings.TrimSpace(fields[1]), 10, 64)
|
||||
if err1 != nil || err2 != nil {
|
||||
// ECC may be disabled on this GPU — return zero counters silently.
|
||||
return BenchmarkECCCounters{}, nil
|
||||
}
|
||||
return BenchmarkECCCounters{Corrected: corrected, Uncorrected: uncorrected}, nil
|
||||
}
|
||||
|
||||
func diffECCCounters(before, after BenchmarkECCCounters) BenchmarkECCCounters {
|
||||
return BenchmarkECCCounters{
|
||||
Corrected: saturatingSub(after.Corrected, before.Corrected),
|
||||
Uncorrected: saturatingSub(after.Uncorrected, before.Uncorrected),
|
||||
}
|
||||
}
|
||||
|
||||
func queryActiveComputeApps(gpuIndices []int) ([]string, error) {
|
||||
args := []string{
|
||||
"--query-compute-apps=gpu_uuid,pid,process_name",
|
||||
"--format=csv,noheader,nounits",
|
||||
}
|
||||
if len(gpuIndices) > 0 {
|
||||
args = append([]string{"--id=" + joinIndexList(gpuIndices)}, args...)
|
||||
}
|
||||
out, err := satExecCommand("nvidia-smi", args...).Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lines []string
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
func finalizeBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkGPUResult {
|
||||
if gpu.Status == "" {
|
||||
gpu.Status = "OK"
|
||||
}
|
||||
if gpu.Scores.CompositeScore == 0 {
|
||||
gpu.Scores.CompositeScore = gpu.Scores.ComputeScore
|
||||
}
|
||||
return gpu
|
||||
}
|
||||
|
||||
func buildBenchmarkFindings(result NvidiaBenchmarkResult) []string {
|
||||
var findings []string
|
||||
|
||||
passed := 0
|
||||
for _, gpu := range result.GPUs {
|
||||
if gpu.Status == "OK" {
|
||||
passed++
|
||||
}
|
||||
}
|
||||
total := len(result.GPUs)
|
||||
if total > 0 {
|
||||
if passed == total {
|
||||
findings = append(findings, fmt.Sprintf("All %d GPU(s) passed the benchmark.", total))
|
||||
} else {
|
||||
findings = append(findings, fmt.Sprintf("%d of %d GPU(s) passed the benchmark.", passed, total))
|
||||
}
|
||||
}
|
||||
|
||||
if result.Normalization.Status != "full" {
|
||||
findings = append(findings, "Environment normalization was partial; compare results with caution.")
|
||||
}
|
||||
for _, gpu := range result.GPUs {
|
||||
if gpu.Status == "FAILED" && len(gpu.DegradationReasons) == 0 {
|
||||
findings = append(findings, fmt.Sprintf("GPU %d failed the benchmark (check verbose.log for details).", gpu.Index))
|
||||
continue
|
||||
}
|
||||
if len(gpu.DegradationReasons) == 0 && gpu.Status == "OK" {
|
||||
findings = append(findings, fmt.Sprintf("GPU %d held clocks without observable throttle counters during steady state.", gpu.Index))
|
||||
continue
|
||||
}
|
||||
for _, reason := range gpu.DegradationReasons {
|
||||
switch reason {
|
||||
case "power_capped":
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[POWER] GPU %d: power cap throttle %.1f%% of steady state — server is not delivering full TDP to the GPU.",
|
||||
gpu.Index, gpu.Scores.PowerCapThrottlePct))
|
||||
case "thermal_limited":
|
||||
// Hard stop check: thermal throttle while fans are not at maximum.
|
||||
// This means the server does not see GPU thermals — incompatible config.
|
||||
if result.Cooling != nil && result.Cooling.FanDutyCycleAvailable &&
|
||||
result.Cooling.P95FanDutyCyclePct < 95 {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[HARD STOP] GPU %d: thermal throttle (%.1f%% of time) while fans peaked at only %.0f%% duty cycle — server cooling is not responding to GPU heat load. Configuration is likely incompatible.",
|
||||
gpu.Index, gpu.Scores.ThermalThrottlePct, result.Cooling.P95FanDutyCyclePct))
|
||||
} else {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[THERMAL] GPU %d: thermal throttle %.1f%% of steady state.",
|
||||
gpu.Index, gpu.Scores.ThermalThrottlePct))
|
||||
}
|
||||
case "sync_boost_limited":
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[SYNC] GPU %d: sync boost throttle %.1f%% of steady state — GPUs are constraining each other's clocks.",
|
||||
gpu.Index, gpu.Scores.SyncBoostThrottlePct))
|
||||
case "low_sm_clock_vs_target":
|
||||
findings = append(findings, fmt.Sprintf("GPU %d average SM clock stayed below the requested lock target.", gpu.Index))
|
||||
case "variance_too_high":
|
||||
findings = append(findings, fmt.Sprintf("GPU %d showed unstable clocks/power over the benchmark window.", gpu.Index))
|
||||
case "normalization_partial":
|
||||
findings = append(findings, fmt.Sprintf("GPU %d ran without full benchmark normalization.", gpu.Index))
|
||||
case "power_limit_derated":
|
||||
findings = append(findings, fmt.Sprintf("[POWER] GPU %d could not sustain full TDP in this server; benchmark ran at reduced limit %.0f W.", gpu.Index, gpu.PowerLimitW))
|
||||
case "ecc_uncorrected_errors":
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[HARD STOP] GPU %d: %d uncorrected ECC error(s) detected — possible hardware fault. Do not use in production.",
|
||||
gpu.Index, gpu.ECC.Uncorrected))
|
||||
case "ecc_corrected_errors":
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[WARNING] GPU %d: %d corrected ECC error(s) — possible DRAM degradation, monitor closely.",
|
||||
gpu.Index, gpu.ECC.Corrected))
|
||||
}
|
||||
}
|
||||
// Temperature headroom checks — independent of throttle counters.
|
||||
// Shutdown and slowdown thresholds are per-GPU from nvidia-smi -q;
|
||||
// fall back to 90°C / 80°C when unavailable.
|
||||
if gpu.Steady.P95TempC > 0 {
|
||||
shutdownTemp := gpu.ShutdownTempC
|
||||
if shutdownTemp <= 0 {
|
||||
shutdownTemp = 90
|
||||
}
|
||||
slowdownTemp := gpu.SlowdownTempC
|
||||
if slowdownTemp <= 0 {
|
||||
slowdownTemp = 80
|
||||
}
|
||||
headroom := shutdownTemp - gpu.Steady.P95TempC
|
||||
switch {
|
||||
case headroom < 10:
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[HARD STOP] GPU %d: p95 temperature %.1f°C — only %.1f°C from shutdown threshold (%.0f°C). Do not operate.",
|
||||
gpu.Index, gpu.Steady.P95TempC, headroom, shutdownTemp))
|
||||
case gpu.Steady.P95TempC >= slowdownTemp:
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[THERMAL] GPU %d: p95 temperature %.1f°C exceeds slowdown threshold (%.0f°C) — %.1f°C headroom to shutdown. Operating in degraded reliability zone.",
|
||||
gpu.Index, gpu.Steady.P95TempC, slowdownTemp, headroom))
|
||||
}
|
||||
}
|
||||
if gpu.CoolingWarning != "" {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"GPU %d: %s. Operator action: rerun the benchmark with fan speed manually fixed at 100%% to confirm actual thermal headroom.",
|
||||
gpu.Index, gpu.CoolingWarning,
|
||||
))
|
||||
}
|
||||
if len(gpu.PrecisionFailures) > 0 {
|
||||
findings = append(findings, fmt.Sprintf("GPU %d had incomplete precision coverage: %s.", gpu.Index, strings.Join(gpu.PrecisionFailures, ", ")))
|
||||
}
|
||||
if gpu.Backend == "driver-ptx" {
|
||||
findings = append(findings, fmt.Sprintf("GPU %d used driver PTX fallback; tensor score is intentionally degraded.", gpu.Index))
|
||||
}
|
||||
if gpu.DefaultPowerLimitW > 0 && gpu.PowerLimitW > 0 && gpu.PowerLimitW < gpu.DefaultPowerLimitW*0.95 {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"GPU %d power limit %.0f W is below default %.0f W (%.0f%%). Performance may be artificially reduced.",
|
||||
gpu.Index, gpu.PowerLimitW, gpu.DefaultPowerLimitW, gpu.PowerLimitW/gpu.DefaultPowerLimitW*100,
|
||||
))
|
||||
}
|
||||
// Flag significant TDP deviation (over or under) from calibration.
|
||||
if gpu.CalibratedPeakPowerW > 0 {
|
||||
ref := gpu.DefaultPowerLimitW
|
||||
if ref <= 0 {
|
||||
ref = gpu.PowerLimitW
|
||||
}
|
||||
if ref > 0 {
|
||||
deviationPct := (gpu.CalibratedPeakPowerW - ref) / ref * 100
|
||||
switch {
|
||||
case deviationPct < -10:
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"GPU %d reached only %.0f W (%.0f%% of rated %.0f W) under targeted_power. Check power delivery or cooling.",
|
||||
gpu.Index, gpu.CalibratedPeakPowerW, gpu.CalibratedPeakPowerW/ref*100, ref,
|
||||
))
|
||||
case deviationPct > 5:
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"GPU %d exceeded rated TDP: %.0f W measured vs %.0f W rated (+%.0f%%). Power limit may not be enforced correctly.",
|
||||
gpu.Index, gpu.CalibratedPeakPowerW, ref, deviationPct,
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if result.Interconnect != nil && result.Interconnect.Supported {
|
||||
findings = append(findings, fmt.Sprintf("Multi-GPU all_reduce max bus bandwidth: %.1f GB/s.", result.Interconnect.MaxBusBWGBps))
|
||||
}
|
||||
if cl := result.CPULoad; cl != nil {
|
||||
switch cl.Status {
|
||||
case "high":
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"Host CPU load was elevated during the benchmark (avg %.1f%%, max %.1f%%). A competing CPU workload may skew GPU results.",
|
||||
cl.AvgPct, cl.MaxPct,
|
||||
))
|
||||
case "unstable":
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"Host CPU load was erratic during the benchmark (avg %.1f%%, p95 %.1f%%). Results may be less reproducible.",
|
||||
cl.AvgPct, cl.P95Pct,
|
||||
))
|
||||
}
|
||||
}
|
||||
if sp := result.ServerPower; sp != nil && sp.Available && sp.GPUReportedSumW > 0 {
|
||||
dcmiPartial := detectDCMIPartialCoverage(sp)
|
||||
if sp.ReportingRatio < 0.75 && !dcmiPartial {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"GPU power reporting may be unreliable: server delta %.0f W vs GPU-reported %.0f W (ratio %.2f). GPU telemetry likely over-reports actual consumption. Composite scores have been penalized accordingly.",
|
||||
sp.DeltaW, sp.GPUReportedSumW, sp.ReportingRatio,
|
||||
))
|
||||
} else if sp.ReportingRatio < 0.75 && dcmiPartial {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"IPMI DCMI covers partial PSU set (DCMI/SDR coverage %.0f%%): ratio %.2f reflects DCMI under-reporting, not GPU inaccuracy. GPU telemetry is the reliable power source; use SDR-based ratio for server-side accuracy.",
|
||||
sp.DCMICoverageRatio*100, sp.ReportingRatio,
|
||||
))
|
||||
} else if sp.ReportingRatio > 1.25 {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"Server power delta %.0f W exceeds GPU-reported sum %.0f W by %.0f%%. Other components (CPU, NVMe, networking) may be drawing substantial power under GPU load.",
|
||||
sp.DeltaW, sp.GPUReportedSumW, (sp.ReportingRatio-1)*100,
|
||||
))
|
||||
}
|
||||
}
|
||||
return dedupeStrings(findings)
|
||||
}
|
||||
|
||||
func benchmarkOverallStatus(result NvidiaBenchmarkResult) string {
|
||||
if len(result.GPUs) == 0 {
|
||||
return "FAILED"
|
||||
}
|
||||
hasOK := false
|
||||
hasPartial := result.Normalization.Status != "full"
|
||||
for _, gpu := range result.GPUs {
|
||||
switch gpu.Status {
|
||||
case "OK":
|
||||
hasOK = true
|
||||
case "PARTIAL", "UNSUPPORTED":
|
||||
hasPartial = true
|
||||
}
|
||||
}
|
||||
if !hasOK {
|
||||
return "FAILED"
|
||||
}
|
||||
if hasPartial {
|
||||
return "PARTIAL"
|
||||
}
|
||||
return "OK"
|
||||
}
|
||||
|
||||
func findBenchmarkNormalization(items []BenchmarkNormalizationGPU, idx int) *BenchmarkNormalizationGPU {
|
||||
for i := range items {
|
||||
if items[i].Index == idx {
|
||||
return &items[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func classifySATErrorStatus(out []byte, err error) string {
|
||||
status, _ := classifySATResult("benchmark", out, err)
|
||||
if status == "UNSUPPORTED" {
|
||||
return "UNSUPPORTED"
|
||||
}
|
||||
return "FAILED"
|
||||
}
|
||||
|
||||
func parseBenchmarkFloat(raw string) float64 {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" || strings.EqualFold(raw, "n/a") || strings.EqualFold(raw, "[not supported]") {
|
||||
return 0
|
||||
}
|
||||
value, _ := strconv.ParseFloat(raw, 64)
|
||||
return value
|
||||
}
|
||||
|
||||
func parseBenchmarkUint64(raw string) uint64 {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" || strings.EqualFold(raw, "n/a") || strings.EqualFold(raw, "[not supported]") {
|
||||
return 0
|
||||
}
|
||||
value, _ := strconv.ParseUint(raw, 10, 64)
|
||||
return value
|
||||
}
|
||||
|
||||
func benchmarkMean(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
var sum float64
|
||||
for _, value := range values {
|
||||
sum += value
|
||||
}
|
||||
return sum / float64(len(values))
|
||||
}
|
||||
|
||||
func benchmarkPercentile(values []float64, p float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
copyValues := append([]float64(nil), values...)
|
||||
sort.Float64s(copyValues)
|
||||
if len(copyValues) == 1 {
|
||||
return copyValues[0]
|
||||
}
|
||||
rank := (p / 100.0) * float64(len(copyValues)-1)
|
||||
lower := int(math.Floor(rank))
|
||||
upper := int(math.Ceil(rank))
|
||||
if lower == upper {
|
||||
return copyValues[lower]
|
||||
}
|
||||
frac := rank - float64(lower)
|
||||
return copyValues[lower] + (copyValues[upper]-copyValues[lower])*frac
|
||||
}
|
||||
|
||||
func benchmarkCV(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
mean := benchmarkMean(values)
|
||||
if mean == 0 {
|
||||
return 0
|
||||
}
|
||||
var variance float64
|
||||
for _, value := range values {
|
||||
diff := value - mean
|
||||
variance += diff * diff
|
||||
}
|
||||
variance /= float64(len(values))
|
||||
return math.Sqrt(variance) / mean * 100
|
||||
}
|
||||
|
||||
func benchmarkClockDrift(values []float64) float64 {
|
||||
if len(values) < 4 {
|
||||
return 0
|
||||
}
|
||||
window := len(values) / 4
|
||||
if window < 1 {
|
||||
window = 1
|
||||
}
|
||||
head := benchmarkMean(values[:window])
|
||||
tail := benchmarkMean(values[len(values)-window:])
|
||||
if head <= 0 || tail >= head {
|
||||
return 0
|
||||
}
|
||||
return ((head - tail) / head) * 100
|
||||
}
|
||||
|
||||
func benchmarkMax(values []float64) float64 {
|
||||
var max float64
|
||||
for i, value := range values {
|
||||
if i == 0 || value > max {
|
||||
max = value
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
func clampScore(value float64) float64 {
|
||||
switch {
|
||||
case value < 0:
|
||||
return 0
|
||||
case value > 100:
|
||||
return 100
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func dedupeStrings(values []string) []string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[value]; ok {
|
||||
continue
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
out = append(out, value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func saturatingSub(after, before uint64) uint64 {
|
||||
if after <= before {
|
||||
return 0
|
||||
}
|
||||
return after - before
|
||||
}
|
||||
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// detectDCMIPartialCoverage returns true when IPMI DCMI under-reports actual
|
||||
// server power by comparing DCMI readings against SDR PSUx_POWER_IN sensor sums.
|
||||
//
|
||||
// Primary check: DCMI_idle / SDR_PSU_IN_idle — most reliable because GPU load
|
||||
// is zero, so both sources measure the same server state. A ratio below 0.7
|
||||
// means DCMI misses ≥30% of installed PSUs (e.g. 0.50 = sees 2 of 4 PSUs).
|
||||
//
|
||||
// Fallback: DCMI_loaded / SDR_PSU_IN_loaded — less precise (GPU load may
|
||||
// affect different PSUs differently) but still useful when idle SDR is absent.
|
||||
//
|
||||
// Returns false when SDR data is unavailable (server has no PSUx_POWER_IN
|
||||
// sensors); the heuristic detectIPMISaturationFallback is used in that case.
|
||||
@@ -0,0 +1,396 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func normalizeNvidiaBenchmarkOptionsForBenchmark(opts NvidiaBenchmarkOptions) NvidiaBenchmarkOptions {
|
||||
switch strings.TrimSpace(strings.ToLower(opts.Profile)) {
|
||||
case NvidiaBenchmarkProfileStability:
|
||||
opts.Profile = NvidiaBenchmarkProfileStability
|
||||
case NvidiaBenchmarkProfileOvernight:
|
||||
opts.Profile = NvidiaBenchmarkProfileOvernight
|
||||
default:
|
||||
opts.Profile = NvidiaBenchmarkProfileStandard
|
||||
}
|
||||
if opts.SizeMB < 0 {
|
||||
opts.SizeMB = 0
|
||||
}
|
||||
opts.ServerPowerSource = normalizeBenchmarkPowerSource(opts.ServerPowerSource)
|
||||
opts.GPUIndices = dedupeSortedIndices(opts.GPUIndices)
|
||||
opts.ExcludeGPUIndices = dedupeSortedIndices(opts.ExcludeGPUIndices)
|
||||
return opts
|
||||
}
|
||||
|
||||
func resolveBenchmarkProfile(profile string) benchmarkProfileSpec {
|
||||
switch strings.TrimSpace(strings.ToLower(profile)) {
|
||||
case NvidiaBenchmarkProfileStability:
|
||||
return benchmarkProfileSpec{Name: NvidiaBenchmarkProfileStability, BaselineSec: 30, WarmupSec: 120, SteadySec: 3600, NCCLSec: 300, CooldownSec: 0}
|
||||
case NvidiaBenchmarkProfileOvernight:
|
||||
return benchmarkProfileSpec{Name: NvidiaBenchmarkProfileOvernight, BaselineSec: 60, WarmupSec: 180, SteadySec: 27000, NCCLSec: 600, CooldownSec: 0}
|
||||
default:
|
||||
return benchmarkProfileSpec{Name: NvidiaBenchmarkProfileStandard, BaselineSec: 15, WarmupSec: 45, SteadySec: 480, NCCLSec: 180, CooldownSec: 0}
|
||||
}
|
||||
}
|
||||
|
||||
// benchmarkGPUInfoQuery describes a nvidia-smi --query-gpu field set to try.
|
||||
// Fields are tried in order; the first successful query wins. Extended fields
|
||||
// (attribute.multiprocessor_count, power.default_limit) are not supported on
|
||||
// all driver versions, so we fall back to the base set if the full query fails.
|
||||
// The minimal fallback omits clock fields entirely — clocks.max.* returns
|
||||
// exit status 2 on some GPU generations (e.g. Blackwell); missing data is
|
||||
// then recovered from nvidia-smi -q.
|
||||
var benchmarkGPUInfoQueries = []struct {
|
||||
fields string
|
||||
extended bool // whether this query includes optional extended fields
|
||||
minimal bool // clock fields omitted; max clocks must be filled separately
|
||||
}{
|
||||
{
|
||||
fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics,attribute.multiprocessor_count,power.default_limit",
|
||||
extended: true,
|
||||
},
|
||||
{
|
||||
fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics",
|
||||
extended: false,
|
||||
},
|
||||
{
|
||||
fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit",
|
||||
minimal: true,
|
||||
},
|
||||
}
|
||||
|
||||
// enrichGPUInfoWithNvidiaSMIQ fills benchmark GPU metadata from nvidia-smi -q
|
||||
// for fields that may be missing from --query-gpu on some driver versions.
|
||||
func enrichGPUInfoWithNvidiaSMIQ(infoByIndex map[int]benchmarkGPUInfo, nvsmiQ []byte) {
|
||||
if len(infoByIndex) == 0 || len(nvsmiQ) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Build bus_id → index map for matching verbose sections to GPU indices.
|
||||
busToBenchIdx := make(map[string]int, len(infoByIndex))
|
||||
for idx, info := range infoByIndex {
|
||||
if info.BusID != "" {
|
||||
// nvidia-smi -q uses "GPU 00000000:4E:00.0" (8-digit domain),
|
||||
// while --query-gpu returns the same format; normalise to lower.
|
||||
busToBenchIdx[strings.ToLower(strings.TrimSpace(info.BusID))] = idx
|
||||
}
|
||||
}
|
||||
|
||||
// Split the verbose output into per-GPU sections on "^GPU " lines.
|
||||
gpuSectionRe := regexp.MustCompile(`(?m)^GPU\s+([\dA-Fa-f:\.]+)`)
|
||||
maxGfxRe := regexp.MustCompile(`(?i)Max Clocks[\s\S]*?Graphics\s*:\s*(\d+)\s*MHz`)
|
||||
maxMemRe := regexp.MustCompile(`(?i)Max Clocks[\s\S]*?Memory\s*:\s*(\d+)\s*MHz`)
|
||||
defaultPwrRe := regexp.MustCompile(`(?i)Default Power Limit\s*:\s*([0-9.]+)\s*W`)
|
||||
currentPwrRe := regexp.MustCompile(`(?i)Current Power Limit\s*:\s*([0-9.]+)\s*W`)
|
||||
minPwrRe := regexp.MustCompile(`(?i)Min Power Limit\s*:\s*([0-9.]+)\s*W`)
|
||||
maxPwrRe := regexp.MustCompile(`(?i)Max Power Limit\s*:\s*([0-9.]+)\s*W`)
|
||||
smCountRe := regexp.MustCompile(`(?i)Multiprocessor Count\s*:\s*(\d+)`)
|
||||
shutdownTempRe := regexp.MustCompile(`(?i)GPU Shutdown Temp\s*:\s*(\d+)\s*C`)
|
||||
slowdownTempRe := regexp.MustCompile(`(?i)GPU Slowdown Temp\s*:\s*(\d+)\s*C`)
|
||||
|
||||
sectionStarts := gpuSectionRe.FindAllSubmatchIndex(nvsmiQ, -1)
|
||||
for i, loc := range sectionStarts {
|
||||
busID := strings.ToLower(string(nvsmiQ[loc[2]:loc[3]]))
|
||||
benchIdx, ok := busToBenchIdx[busID]
|
||||
if !ok {
|
||||
// Bus IDs from verbose output may have a different domain prefix;
|
||||
// try suffix match on the slot portion (XX:XX.X).
|
||||
for k, v := range busToBenchIdx {
|
||||
if strings.HasSuffix(k, busID) || strings.HasSuffix(busID, k) {
|
||||
benchIdx = v
|
||||
ok = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
end := len(nvsmiQ)
|
||||
if i+1 < len(sectionStarts) {
|
||||
end = sectionStarts[i+1][0]
|
||||
}
|
||||
section := nvsmiQ[loc[0]:end]
|
||||
|
||||
info := infoByIndex[benchIdx]
|
||||
|
||||
if info.MaxGraphicsClockMHz == 0 {
|
||||
if m := maxGfxRe.FindSubmatch(section); m != nil {
|
||||
if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil {
|
||||
info.MaxGraphicsClockMHz = v
|
||||
}
|
||||
}
|
||||
}
|
||||
if info.MaxMemoryClockMHz == 0 {
|
||||
if m := maxMemRe.FindSubmatch(section); m != nil {
|
||||
if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil {
|
||||
info.MaxMemoryClockMHz = v
|
||||
}
|
||||
}
|
||||
}
|
||||
if info.DefaultPowerLimitW == 0 {
|
||||
if m := defaultPwrRe.FindSubmatch(section); m != nil {
|
||||
if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 {
|
||||
info.DefaultPowerLimitW = v
|
||||
}
|
||||
}
|
||||
}
|
||||
if info.PowerLimitW == 0 {
|
||||
if m := currentPwrRe.FindSubmatch(section); m != nil {
|
||||
if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 {
|
||||
info.PowerLimitW = v
|
||||
}
|
||||
}
|
||||
}
|
||||
if info.MinPowerLimitW == 0 {
|
||||
if m := minPwrRe.FindSubmatch(section); m != nil {
|
||||
if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 {
|
||||
info.MinPowerLimitW = v
|
||||
}
|
||||
}
|
||||
}
|
||||
if info.MaxPowerLimitW == 0 {
|
||||
if m := maxPwrRe.FindSubmatch(section); m != nil {
|
||||
if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 {
|
||||
info.MaxPowerLimitW = v
|
||||
}
|
||||
}
|
||||
}
|
||||
if info.MultiprocessorCount == 0 {
|
||||
if m := smCountRe.FindSubmatch(section); m != nil {
|
||||
if v, err := strconv.Atoi(string(m[1])); err == nil && v > 0 {
|
||||
info.MultiprocessorCount = v
|
||||
}
|
||||
}
|
||||
}
|
||||
if info.ShutdownTempC == 0 {
|
||||
if m := shutdownTempRe.FindSubmatch(section); m != nil {
|
||||
if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 {
|
||||
info.ShutdownTempC = v
|
||||
}
|
||||
}
|
||||
}
|
||||
if info.SlowdownTempC == 0 {
|
||||
if m := slowdownTempRe.FindSubmatch(section); m != nil {
|
||||
if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil && v > 0 {
|
||||
info.SlowdownTempC = v
|
||||
}
|
||||
}
|
||||
}
|
||||
infoByIndex[benchIdx] = info
|
||||
}
|
||||
}
|
||||
|
||||
func queryBenchmarkGPUInfo(gpuIndices []int) (map[int]benchmarkGPUInfo, error) {
|
||||
var lastErr error
|
||||
for _, q := range benchmarkGPUInfoQueries {
|
||||
args := []string{
|
||||
"--query-gpu=" + q.fields,
|
||||
"--format=csv,noheader,nounits",
|
||||
}
|
||||
if len(gpuIndices) > 0 {
|
||||
args = append([]string{"--id=" + joinIndexList(gpuIndices)}, args...)
|
||||
}
|
||||
out, err := satExecCommand("nvidia-smi", args...).Output()
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("nvidia-smi gpu info (%s): %w", q.fields[:min(len(q.fields), 40)], err)
|
||||
continue
|
||||
}
|
||||
|
||||
r := csv.NewReader(strings.NewReader(string(out)))
|
||||
r.TrimLeadingSpace = true
|
||||
r.FieldsPerRecord = -1
|
||||
rows, err := r.ReadAll()
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("parse nvidia-smi gpu info: %w", err)
|
||||
continue
|
||||
}
|
||||
|
||||
minFields := 6
|
||||
if !q.minimal {
|
||||
minFields = 9
|
||||
}
|
||||
infoByIndex := make(map[int]benchmarkGPUInfo, len(rows))
|
||||
for _, row := range rows {
|
||||
if len(row) < minFields {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(strings.TrimSpace(row[0]))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
info := benchmarkGPUInfo{
|
||||
Index: idx,
|
||||
UUID: strings.TrimSpace(row[1]),
|
||||
Name: strings.TrimSpace(row[2]),
|
||||
BusID: strings.TrimSpace(row[3]),
|
||||
VBIOS: strings.TrimSpace(row[4]),
|
||||
PowerLimitW: parseBenchmarkFloat(row[5]),
|
||||
}
|
||||
if !q.minimal {
|
||||
info.MaxGraphicsClockMHz = parseBenchmarkFloat(row[6])
|
||||
info.MaxMemoryClockMHz = parseBenchmarkFloat(row[7])
|
||||
if len(row) >= 9 {
|
||||
info.BaseGraphicsClockMHz = parseBenchmarkFloat(row[8])
|
||||
}
|
||||
if q.extended {
|
||||
if len(row) >= 10 {
|
||||
info.MultiprocessorCount = int(parseBenchmarkFloat(row[9]))
|
||||
}
|
||||
if len(row) >= 11 {
|
||||
info.DefaultPowerLimitW = parseBenchmarkFloat(row[10])
|
||||
}
|
||||
}
|
||||
}
|
||||
infoByIndex[idx] = info
|
||||
}
|
||||
return infoByIndex, nil
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func applyBenchmarkNormalization(ctx context.Context, verboseLog string, gpuIndices []int, infoByIndex map[int]benchmarkGPUInfo, result *NvidiaBenchmarkResult) []benchmarkRestoreAction {
|
||||
if os.Geteuid() != 0 {
|
||||
result.Normalization.Status = "partial"
|
||||
result.Normalization.Notes = append(result.Normalization.Notes, "benchmark normalization skipped: root privileges are required for persistence mode and clock locks")
|
||||
for _, idx := range gpuIndices {
|
||||
result.Normalization.GPUs = append(result.Normalization.GPUs, BenchmarkNormalizationGPU{
|
||||
Index: idx,
|
||||
Notes: []string{"normalization skipped: root privileges are required"},
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var restore []benchmarkRestoreAction
|
||||
for _, idx := range gpuIndices {
|
||||
rec := BenchmarkNormalizationGPU{Index: idx}
|
||||
if _, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("normalize-gpu-%d-pm", idx), []string{"nvidia-smi", "-i", strconv.Itoa(idx), "-pm", "1"}, nil, nil); err != nil {
|
||||
rec.PersistenceMode = "failed"
|
||||
rec.Notes = append(rec.Notes, "failed to enable persistence mode")
|
||||
result.Normalization.Status = "partial"
|
||||
} else {
|
||||
rec.PersistenceMode = "applied"
|
||||
}
|
||||
|
||||
if info, ok := infoByIndex[idx]; ok && info.MaxGraphicsClockMHz > 0 {
|
||||
target := int(math.Round(info.MaxGraphicsClockMHz))
|
||||
if out, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("normalize-gpu-%d-lgc", idx), []string{"nvidia-smi", "-i", strconv.Itoa(idx), "-lgc", strconv.Itoa(target)}, nil, nil); err != nil {
|
||||
rec.GPUClockLockStatus = "failed"
|
||||
rec.Notes = append(rec.Notes, "graphics clock lock failed: "+strings.TrimSpace(string(out)))
|
||||
result.Normalization.Status = "partial"
|
||||
} else {
|
||||
rec.GPUClockLockStatus = "applied"
|
||||
rec.GPUClockLockMHz = float64(target)
|
||||
idxCopy := idx
|
||||
restore = append(restore, benchmarkRestoreAction{name: fmt.Sprintf("gpu-%d-rgc", idxCopy), fn: func() {
|
||||
_, _ = runSATCommandCtx(context.Background(), verboseLog, fmt.Sprintf("restore-gpu-%d-rgc", idxCopy), []string{"nvidia-smi", "-i", strconv.Itoa(idxCopy), "-rgc"}, nil, nil)
|
||||
}})
|
||||
}
|
||||
} else {
|
||||
rec.GPUClockLockStatus = "skipped"
|
||||
rec.Notes = append(rec.Notes, "graphics clock lock skipped: gpu inventory unavailable or MaxGraphicsClockMHz=0")
|
||||
result.Normalization.Status = "partial"
|
||||
}
|
||||
|
||||
if info, ok := infoByIndex[idx]; ok && info.MaxMemoryClockMHz > 0 {
|
||||
target := int(math.Round(info.MaxMemoryClockMHz))
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("normalize-gpu-%d-lmc", idx), []string{"nvidia-smi", "-i", strconv.Itoa(idx), "-lmc", strconv.Itoa(target)}, nil, nil)
|
||||
switch {
|
||||
case err == nil:
|
||||
rec.MemoryClockLockStatus = "applied"
|
||||
rec.MemoryClockLockMHz = float64(target)
|
||||
idxCopy := idx
|
||||
restore = append(restore, benchmarkRestoreAction{name: fmt.Sprintf("gpu-%d-rmc", idxCopy), fn: func() {
|
||||
_, _ = runSATCommandCtx(context.Background(), verboseLog, fmt.Sprintf("restore-gpu-%d-rmc", idxCopy), []string{"nvidia-smi", "-i", strconv.Itoa(idxCopy), "-rmc"}, nil, nil)
|
||||
}})
|
||||
case strings.Contains(strings.ToLower(string(out)), "deferred") || strings.Contains(strings.ToLower(string(out)), "not supported"):
|
||||
rec.MemoryClockLockStatus = "unsupported"
|
||||
rec.Notes = append(rec.Notes, "memory clock lock unsupported on this GPU/driver path")
|
||||
result.Normalization.Status = "partial"
|
||||
default:
|
||||
rec.MemoryClockLockStatus = "failed"
|
||||
rec.Notes = append(rec.Notes, "memory clock lock failed: "+strings.TrimSpace(string(out)))
|
||||
result.Normalization.Status = "partial"
|
||||
}
|
||||
}
|
||||
|
||||
result.Normalization.GPUs = append(result.Normalization.GPUs, rec)
|
||||
}
|
||||
return restore
|
||||
}
|
||||
|
||||
func collectBenchmarkSamples(ctx context.Context, durationSec int, gpuIndices []int) ([]GPUMetricRow, error) {
|
||||
if durationSec <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
deadline := time.Now().Add(time.Duration(durationSec) * time.Second)
|
||||
var rows []GPUMetricRow
|
||||
start := time.Now()
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return rows, ctx.Err()
|
||||
}
|
||||
samples, err := sampleBenchmarkTelemetry(gpuIndices)
|
||||
if err == nil {
|
||||
elapsed := time.Since(start).Seconds()
|
||||
for i := range samples {
|
||||
samples[i].ElapsedSec = elapsed
|
||||
}
|
||||
rows = append(rows, samples...)
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return rows, ctx.Err()
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func runBenchmarkCommandWithMetrics(ctx context.Context, verboseLog, name string, cmd []string, env []string, gpuIndices []int, logFunc func(string)) ([]byte, []GPUMetricRow, error) {
|
||||
stopCh := make(chan struct{})
|
||||
doneCh := make(chan struct{})
|
||||
var metricRows []GPUMetricRow
|
||||
start := time.Now()
|
||||
|
||||
go func() {
|
||||
defer close(doneCh)
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stopCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
samples, err := sampleBenchmarkTelemetry(gpuIndices)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
elapsed := time.Since(start).Seconds()
|
||||
for i := range samples {
|
||||
samples[i].ElapsedSec = elapsed
|
||||
}
|
||||
metricRows = append(metricRows, samples...)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, name, cmd, env, logFunc)
|
||||
close(stopCh)
|
||||
<-doneCh
|
||||
|
||||
return out, metricRows, err
|
||||
}
|
||||
@@ -11,12 +11,8 @@ import (
|
||||
// satReadFile is a seam for tests to fake sysfs reads (numa_node files).
|
||||
var satReadFile = os.ReadFile
|
||||
|
||||
// gpuBandwidthSocketGroups splits gpuIndices into per-socket groups (ordered
|
||||
// by ascending NUMA node ID) for RunNvidiaBandwidthPack. A cross-socket
|
||||
// peer-to-peer path is a different (and, on platforms without NVLink, far
|
||||
// less exercised) fault domain than a same-socket one, so testing each
|
||||
// socket's GPUs in isolation before testing all of them together isolates
|
||||
// whether a failure is specific to the cross-socket path.
|
||||
// gpuBandwidthSocketGroups splits gpuIndices into NUMA-locality groups,
|
||||
// ordered by ascending Linux NUMA node ID, for RunNvidiaBandwidthPack.
|
||||
//
|
||||
// Falls back to a single group containing all of gpuIndices — i.e. no split
|
||||
// — whenever the NUMA node can't be resolved for every GPU, or all resolve
|
||||
@@ -31,22 +27,17 @@ func gpuBandwidthSocketGroups(gpuIndices []int, logFunc func(string)) [][]int {
|
||||
}
|
||||
|
||||
byNode := map[int][]int{}
|
||||
var unresolved []int
|
||||
for _, idx := range gpuIndices {
|
||||
node, ok := nodes[idx]
|
||||
if !ok {
|
||||
if logFunc != nil {
|
||||
logFunc(fmt.Sprintf("nvbandwidth: no NUMA node resolved for GPU %d; will fold it into a resolved socket group instead of dropping the split", idx))
|
||||
logFunc(fmt.Sprintf("nvbandwidth: no NUMA node resolved for GPU %d; running all GPUs as one group", idx))
|
||||
}
|
||||
unresolved = append(unresolved, idx)
|
||||
continue
|
||||
return [][]int{gpuIndices}
|
||||
}
|
||||
byNode[node] = append(byNode[node], idx)
|
||||
}
|
||||
// Fewer than two resolved sockets means there's nothing to split either
|
||||
// way: every GPU's node is unknown, or every resolved GPU shares one
|
||||
// socket. A single unresolved GPU among an otherwise clean multi-socket
|
||||
// system shouldn't cost us the split, so only bail out here.
|
||||
// Fewer than two NUMA nodes means there is nothing meaningful to split.
|
||||
if len(byNode) < 2 {
|
||||
return [][]int{gpuIndices}
|
||||
}
|
||||
@@ -61,14 +52,6 @@ func gpuBandwidthSocketGroups(gpuIndices []int, logFunc func(string)) [][]int {
|
||||
for _, node := range sortedNodes {
|
||||
groups = append(groups, dedupeSortedIndices(byNode[node]))
|
||||
}
|
||||
if len(unresolved) > 0 {
|
||||
// Fold into the last group rather than running unresolved GPUs in a
|
||||
// group of their own — a lone GPU can't run a GPU-to-GPU bandwidth
|
||||
// test by itself, and the point of the split is to isolate the
|
||||
// sockets we *do* know about, not to also isolate the unknown one.
|
||||
last := len(groups) - 1
|
||||
groups[last] = dedupeSortedIndices(append(groups[last], unresolved...))
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
@@ -107,10 +90,16 @@ func gpuNUMANodes(gpuIndices []int) (map[int]int, error) {
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
// normalizeNvidiaBDF converts nvidia-smi's 8-hex-digit-domain PCI bus ID
|
||||
// ("00000000:05:00.0") to the 4-hex-digit-domain form sysfs paths use
|
||||
// ("0000:05:00.0").
|
||||
// normalizeNvidiaBDF converts nvidia-smi's PCI bus ID to the exact form the
|
||||
// sysfs paths under /sys/bus/pci/devices use: an 8-hex-digit domain is
|
||||
// narrowed to 4 digits ("00000000:05:00.0" -> "0000:05:00.0"), and the hex
|
||||
// is lower-cased ("0000:CB:00.0" -> "0000:cb:00.0"). nvidia-smi upper-cases
|
||||
// the bus/device hex; sysfs directory names are always lower-case, so
|
||||
// without this a BDF containing a hex letter (e.g. GPUs on bus 4b/cb/cf)
|
||||
// would never match a sysfs entry and every numa_node / link-speed read
|
||||
// would silently fail.
|
||||
func normalizeNvidiaBDF(busID string) string {
|
||||
busID = strings.ToLower(strings.TrimSpace(busID))
|
||||
domain, rest, ok := strings.Cut(busID, ":")
|
||||
if !ok {
|
||||
return busID
|
||||
|
||||
@@ -33,10 +33,10 @@ func fakeNUMANodes(t *testing.T, byBDF map[string]string) {
|
||||
}
|
||||
|
||||
func TestGPUNUMANodesResolvesFromPCIBusID(t *testing.T) {
|
||||
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:F4:00.0\n")
|
||||
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:f4:00.0\n")
|
||||
fakeNUMANodes(t, map[string]string{
|
||||
"0000:05:00.0": "0\n",
|
||||
"0000:F4:00.0": "1\n",
|
||||
"0000:f4:00.0": "1\n",
|
||||
})
|
||||
|
||||
nodes, err := gpuNUMANodes([]int{0, 1})
|
||||
@@ -69,14 +69,14 @@ func TestGPUNUMANodesSkipsUnresolvableNode(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGPUBandwidthSocketGroupsSplitsBySocket(t *testing.T) {
|
||||
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:F4:00.0\n5, 00000000:F5:00.0\n")
|
||||
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:f4:00.0\n5, 00000000:f5:00.0\n")
|
||||
fakeNUMANodes(t, map[string]string{
|
||||
"0000:05:00.0": "0\n",
|
||||
"0000:06:00.0": "0\n",
|
||||
"0000:76:00.0": "0\n",
|
||||
"0000:77:00.0": "0\n",
|
||||
"0000:F4:00.0": "1\n",
|
||||
"0000:F5:00.0": "1\n",
|
||||
"0000:f4:00.0": "1\n",
|
||||
"0000:f5:00.0": "1\n",
|
||||
})
|
||||
|
||||
groups := gpuBandwidthSocketGroups([]int{0, 1, 2, 3, 4, 5}, nil)
|
||||
@@ -91,30 +91,20 @@ func TestGPUBandwidthSocketGroupsSplitsBySocket(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPUBandwidthSocketGroupsFoldsUnresolvedIntoLastGroup(t *testing.T) {
|
||||
// GPU 4's NUMA node fails to resolve (e.g. a flaky sysfs read), but the
|
||||
// other 5 GPUs still clearly span two sockets — the split should survive
|
||||
// and GPU 4 should ride along with the last group rather than being
|
||||
// tested alone or collapsing the whole thing to one pass.
|
||||
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:F4:00.0\n5, 00000000:F5:00.0\n")
|
||||
func TestGPUBandwidthSocketGroupsFallsBackWhenAnyNodeIsUnresolved(t *testing.T) {
|
||||
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:f4:00.0\n5, 00000000:f5:00.0\n")
|
||||
fakeNUMANodes(t, map[string]string{
|
||||
"0000:05:00.0": "0\n",
|
||||
"0000:06:00.0": "0\n",
|
||||
"0000:76:00.0": "0\n",
|
||||
"0000:77:00.0": "0\n",
|
||||
// GPU 4 (F4:00.0) deliberately missing.
|
||||
"0000:F5:00.0": "1\n",
|
||||
"0000:f5:00.0": "1\n",
|
||||
})
|
||||
|
||||
groups := gpuBandwidthSocketGroups([]int{0, 1, 2, 3, 4, 5}, nil)
|
||||
if len(groups) != 2 {
|
||||
t.Fatalf("groups=%v want 2 groups", groups)
|
||||
}
|
||||
if joinIndexList(groups[0]) != "0,1,2,3" {
|
||||
t.Fatalf("groups[0]=%v want 0,1,2,3", groups[0])
|
||||
}
|
||||
if joinIndexList(groups[1]) != "4,5" {
|
||||
t.Fatalf("groups[1]=%v want 4,5 (unresolved GPU 4 folded into last group)", groups[1])
|
||||
if len(groups) != 1 || joinIndexList(groups[0]) != "0,1,2,3,4,5" {
|
||||
t.Fatalf("groups=%v want single fallback group", groups)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,6 +151,8 @@ func TestNormalizeNvidiaBDF(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"00000000:05:00.0": "0000:05:00.0",
|
||||
"0000:05:00.0": "0000:05:00.0",
|
||||
"00000000:CB:00.0": "0000:cb:00.0",
|
||||
"0000:4F:00.0": "0000:4f:00.0",
|
||||
"garbage": "garbage",
|
||||
}
|
||||
for in, want := range cases {
|
||||
@@ -171,17 +163,17 @@ func TestNormalizeNvidiaBDF(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunNvidiaBandwidthPackSplitsPerSocketThenAll(t *testing.T) {
|
||||
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:F4:00.0\n3, 00000000:F5:00.0\n")
|
||||
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:f4:00.0\n3, 00000000:f5:00.0\n")
|
||||
fakeNUMANodes(t, map[string]string{
|
||||
"0000:05:00.0": "0\n",
|
||||
"0000:06:00.0": "0\n",
|
||||
"0000:F4:00.0": "1\n",
|
||||
"0000:F5:00.0": "1\n",
|
||||
"0000:f4:00.0": "1\n",
|
||||
"0000:f5:00.0": "1\n",
|
||||
})
|
||||
|
||||
dir := t.TempDir()
|
||||
s := &System{}
|
||||
_, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1, 2, 3}, nil)
|
||||
_, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1, 2, 3}, true, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("RunNvidiaBandwidthPack error: %v", err)
|
||||
}
|
||||
@@ -211,6 +203,34 @@ func TestRunNvidiaBandwidthPackSplitsPerSocketThenAll(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNvidiaBandwidthPackValidateNeverSplits(t *testing.T) {
|
||||
// Multi-socket system, but fullMatrix=false (Validate tier): still one
|
||||
// nvbandwidth pass across every GPU, no per-socket split.
|
||||
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:f4:00.0\n3, 00000000:f5:00.0\n")
|
||||
fakeNUMANodes(t, map[string]string{
|
||||
"0000:05:00.0": "0\n",
|
||||
"0000:06:00.0": "0\n",
|
||||
"0000:f4:00.0": "1\n",
|
||||
"0000:f5:00.0": "1\n",
|
||||
})
|
||||
|
||||
dir := t.TempDir()
|
||||
if _, err := (&System{}).RunNvidiaBandwidthPack(nil, dir, []int{0, 1, 2, 3}, false, nil); err != nil {
|
||||
t.Fatalf("RunNvidiaBandwidthPack error: %v", err)
|
||||
}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir: %v", err)
|
||||
}
|
||||
runDir := filepath.Join(dir, entries[0].Name())
|
||||
if _, err := os.Stat(filepath.Join(runDir, "03-dcgmi-nvbandwidth.log")); err != nil {
|
||||
t.Fatalf("missing single-pass job output: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(runDir, "03-dcgmi-nvbandwidth-socket0.log")); err == nil {
|
||||
t.Fatalf("Validate tier must not split per socket")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNvidiaBandwidthPackSinglePassWhenOneSocket(t *testing.T) {
|
||||
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n")
|
||||
fakeNUMANodes(t, map[string]string{
|
||||
@@ -220,7 +240,7 @@ func TestRunNvidiaBandwidthPackSinglePassWhenOneSocket(t *testing.T) {
|
||||
|
||||
dir := t.TempDir()
|
||||
s := &System{}
|
||||
_, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1}, nil)
|
||||
_, err := s.RunNvidiaBandwidthPack(nil, dir, []int{0, 1}, true, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("RunNvidiaBandwidthPack error: %v", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPhysicalGPUVendors(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
lspci string
|
||||
wantNvidia bool
|
||||
wantAMD bool
|
||||
}{
|
||||
{
|
||||
name: "nvidia 3D controllers",
|
||||
lspci: "4b:00.0 3D controller [0302]: NVIDIA Corporation GH100 [H200 NVL] [10de:233b] (rev a1)\n" +
|
||||
"02:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED [1a03:2000]\n",
|
||||
wantNvidia: true,
|
||||
},
|
||||
{
|
||||
name: "amd VGA by vendor id",
|
||||
lspci: "63:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Navi [1002:744c]\n",
|
||||
wantAMD: true,
|
||||
},
|
||||
{
|
||||
name: "amd cpu root complex is not a GPU",
|
||||
lspci: "00:00.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14a4]\n" +
|
||||
"00:01.0 IOMMU [0806]: Advanced Micro Devices, Inc. [AMD] Device [1022:14a1]\n",
|
||||
},
|
||||
{
|
||||
name: "no GPU",
|
||||
lspci: "02:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED [1a03:2000]\n",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
old := satExecCommand
|
||||
satExecCommand = func(name string, args ...string) *exec.Cmd {
|
||||
if name == "lspci" {
|
||||
return exec.Command("printf", "%s", tt.lspci)
|
||||
}
|
||||
return exec.Command(name, args...)
|
||||
}
|
||||
t.Cleanup(func() { satExecCommand = old })
|
||||
|
||||
nvidia, amd := (&System{}).PhysicalGPUVendors()
|
||||
if nvidia != tt.wantNvidia || amd != tt.wantAMD {
|
||||
t.Fatalf("got nvidia=%v amd=%v, want nvidia=%v amd=%v", nvidia, amd, tt.wantNvidia, tt.wantAMD)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,7 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
@@ -532,44 +530,3 @@ func containsComponent(components []string, name string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func packPlatformDir(dir, dest string) error {
|
||||
f, err := os.Create(dest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
gz := gzip.NewWriter(f)
|
||||
defer gz.Close()
|
||||
tw := tar.NewWriter(gz)
|
||||
defer tw.Close()
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
base := filepath.Base(dir)
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
fpath := filepath.Join(dir, e.Name())
|
||||
data, err := os.ReadFile(fpath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
hdr := &tar.Header{
|
||||
Name: filepath.Join(base, e.Name()),
|
||||
Size: int64(len(data)),
|
||||
Mode: 0644,
|
||||
ModTime: time.Now(),
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tw.Write(data); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -194,774 +192,6 @@ func streamExecOutput(cmd *exec.Cmd, logFunc func(string), livePath string) ([]b
|
||||
}
|
||||
|
||||
// NvidiaGPU holds basic GPU info from nvidia-smi.
|
||||
type NvidiaGPU struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
MemoryMB int `json:"memory_mb"`
|
||||
}
|
||||
|
||||
type NvidiaGPUStatus struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
BDF string `json:"bdf,omitempty"`
|
||||
Serial string `json:"serial,omitempty"`
|
||||
Status string `json:"status"`
|
||||
RawLine string `json:"raw_line,omitempty"`
|
||||
NeedsReset bool `json:"needs_reset"`
|
||||
ParseFailure bool `json:"parse_failure,omitempty"`
|
||||
}
|
||||
|
||||
type nvidiaGPUHealth struct {
|
||||
Index int
|
||||
Name string
|
||||
NeedsReset bool
|
||||
RawLine string
|
||||
ParseFailure bool
|
||||
}
|
||||
|
||||
type nvidiaGPUStatusFile struct {
|
||||
Index int
|
||||
Name string
|
||||
RunStatus string
|
||||
Reason string
|
||||
Health string
|
||||
HealthRaw string
|
||||
Observed bool
|
||||
Selected bool
|
||||
FailingJob string
|
||||
}
|
||||
|
||||
// AMDGPUInfo holds basic info about an AMD GPU from rocm-smi.
|
||||
type AMDGPUInfo struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// DetectGPUVendor returns "nvidia" if /dev/nvidia0 exists, "amd" if /dev/kfd exists, or "" otherwise.
|
||||
func (s *System) DetectGPUVendor() string {
|
||||
if _, err := os.Stat("/dev/nvidia0"); err == nil {
|
||||
return "nvidia"
|
||||
}
|
||||
if _, err := os.Stat("/dev/kfd"); err == nil {
|
||||
return "amd"
|
||||
}
|
||||
if raw, err := exec.Command("lspci", "-nn").Output(); err == nil {
|
||||
// Only match AMD GPU device classes [0300]=VGA, [0302]=3D controller, [0380]=Display.
|
||||
// AMD CPUs also appear in lspci as "Advanced Micro Devices" (Root Complex, IOMMU, etc.)
|
||||
// so matching vendor alone causes false positives on AMD CPU servers without GPUs.
|
||||
for _, line := range strings.Split(strings.ToLower(string(raw)), "\n") {
|
||||
if !strings.Contains(line, "advanced micro devices") && !strings.Contains(line, "amd/ati") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(line, "[0300]") || strings.Contains(line, "[0302]") || strings.Contains(line, "[0380]") {
|
||||
return "amd"
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ListAMDGPUs returns AMD GPUs visible to rocm-smi.
|
||||
func (s *System) ListAMDGPUs() ([]AMDGPUInfo, error) {
|
||||
out, err := runROCmSMI("--showproductname", "--csv")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rocm-smi: %w", err)
|
||||
}
|
||||
var gpus []AMDGPUInfo
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(strings.ToLower(line), "device") {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(line, ",", 2)
|
||||
name := ""
|
||||
if len(parts) >= 2 {
|
||||
name = strings.TrimSpace(parts[1])
|
||||
}
|
||||
idx := len(gpus)
|
||||
gpus = append(gpus, AMDGPUInfo{Index: idx, Name: name})
|
||||
}
|
||||
return gpus, nil
|
||||
}
|
||||
|
||||
// RunAMDAcceptancePack runs an AMD GPU diagnostic pack using rocm-smi.
|
||||
func (s *System) RunAMDAcceptancePack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-amd", []satJob{
|
||||
{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
|
||||
{name: "02-rocm-smi-showallinfo.log", cmd: []string{"rocm-smi", "--showallinfo"}},
|
||||
{name: "03-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
|
||||
{name: "04-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
// RunAMDMemIntegrityPack runs the official RVS MEM module as a validate-style memory integrity test.
|
||||
func (s *System) RunAMDMemIntegrityPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
if err := ensureAMDRuntimeReady(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
cfgFile := "/tmp/bee-amd-mem.conf"
|
||||
cfg := `actions:
|
||||
- name: mem_integrity
|
||||
device: all
|
||||
module: mem
|
||||
parallel: true
|
||||
duration: 60000
|
||||
copy_matrix: false
|
||||
target_stress: 90
|
||||
matrix_size: 8640
|
||||
`
|
||||
_ = os.WriteFile(cfgFile, []byte(cfg), 0644)
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-mem", []satJob{
|
||||
{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
|
||||
{name: "02-rvs-mem.log", cmd: []string{"rvs", "-c", cfgFile}},
|
||||
{name: "03-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
// RunAMDMemBandwidthPack runs AMD's memory/interconnect bandwidth-oriented tools.
|
||||
func (s *System) RunAMDMemBandwidthPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
if err := ensureAMDRuntimeReady(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
cfgFile := "/tmp/bee-amd-babel.conf"
|
||||
cfg := `actions:
|
||||
- name: babel_mem_bw
|
||||
device: all
|
||||
module: babel
|
||||
parallel: true
|
||||
copy_matrix: true
|
||||
target_stress: 90
|
||||
matrix_size: 134217728
|
||||
`
|
||||
_ = os.WriteFile(cfgFile, []byte(cfg), 0644)
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-bandwidth", []satJob{
|
||||
{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
|
||||
{name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}},
|
||||
{name: "03-rvs-babel.log", cmd: []string{"rvs", "-c", cfgFile}},
|
||||
{name: "04-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
// RunAMDStressPack runs an AMD GPU burn-in pack.
|
||||
// Missing tools are reported as UNSUPPORTED, consistent with the existing SAT pattern.
|
||||
func (s *System) RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
seconds := durationSec
|
||||
if seconds <= 0 {
|
||||
seconds = envInt("BEE_AMD_STRESS_SECONDS", 300)
|
||||
}
|
||||
if err := ensureAMDRuntimeReady(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Enable copy_matrix so the same GST run drives VRAM traffic in addition to compute.
|
||||
rvsCfg := amdStressRVSConfig(seconds)
|
||||
cfgFile := "/tmp/bee-amd-gst.conf"
|
||||
_ = os.WriteFile(cfgFile, []byte(rvsCfg), 0644)
|
||||
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-stress", amdStressJobs(seconds, cfgFile), logFunc)
|
||||
}
|
||||
|
||||
func amdStressRVSConfig(seconds int) string {
|
||||
return fmt.Sprintf(`actions:
|
||||
- name: gst_stress
|
||||
device: all
|
||||
module: gst
|
||||
parallel: true
|
||||
duration: %d
|
||||
copy_matrix: false
|
||||
target_stress: 90
|
||||
matrix_size_a: 8640
|
||||
matrix_size_b: 8640
|
||||
matrix_size_c: 8640
|
||||
`, seconds*1000)
|
||||
}
|
||||
|
||||
func amdStressJobs(seconds int, cfgFile string) []satJob {
|
||||
return []satJob{
|
||||
{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
|
||||
{name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}},
|
||||
{name: fmt.Sprintf("03-rvs-gst-%ds.log", seconds), cmd: []string{"rvs", "-c", cfgFile}},
|
||||
{name: fmt.Sprintf("04-rocm-smi-after.log"), cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--csv"}},
|
||||
}
|
||||
}
|
||||
|
||||
// ListNvidiaGPUs returns GPUs visible to nvidia-smi.
|
||||
func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) {
|
||||
out, err := exec.Command("nvidia-smi",
|
||||
"--query-gpu=index,name,memory.total",
|
||||
"--format=csv,noheader,nounits").Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nvidia-smi: %w", err)
|
||||
}
|
||||
var gpus []NvidiaGPU
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(line, ", ", 3)
|
||||
if len(parts) != 3 {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
memMB, _ := strconv.Atoi(strings.TrimSpace(parts[2]))
|
||||
gpus = append(gpus, NvidiaGPU{
|
||||
Index: idx,
|
||||
Name: strings.TrimSpace(parts[1]),
|
||||
MemoryMB: memMB,
|
||||
})
|
||||
}
|
||||
sort.Slice(gpus, func(i, j int) bool {
|
||||
return gpus[i].Index < gpus[j].Index
|
||||
})
|
||||
return gpus, nil
|
||||
}
|
||||
|
||||
func (s *System) ListNvidiaGPUStatuses() ([]NvidiaGPUStatus, error) {
|
||||
out, err := satExecCommand(
|
||||
"nvidia-smi",
|
||||
"--query-gpu=index,name,pci.bus_id,serial,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total",
|
||||
"--format=csv,noheader,nounits",
|
||||
).Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nvidia-smi: %w", err)
|
||||
}
|
||||
var gpus []NvidiaGPUStatus
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(line, ",")
|
||||
if len(parts) < 4 {
|
||||
gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true})
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||||
if err != nil {
|
||||
gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true})
|
||||
continue
|
||||
}
|
||||
upper := strings.ToUpper(line)
|
||||
needsReset := strings.Contains(upper, "GPU REQUIRES RESET")
|
||||
status := "OK"
|
||||
if needsReset {
|
||||
status = "RESET_REQUIRED"
|
||||
}
|
||||
gpus = append(gpus, NvidiaGPUStatus{
|
||||
Index: idx,
|
||||
Name: strings.TrimSpace(parts[1]),
|
||||
BDF: normalizeNvidiaBusID(strings.TrimSpace(parts[2])),
|
||||
Serial: strings.TrimSpace(parts[3]),
|
||||
Status: status,
|
||||
RawLine: line,
|
||||
NeedsReset: needsReset,
|
||||
})
|
||||
}
|
||||
sort.Slice(gpus, func(i, j int) bool { return gpus[i].Index < gpus[j].Index })
|
||||
return gpus, nil
|
||||
}
|
||||
|
||||
func normalizeNvidiaBusID(v string) string {
|
||||
v = strings.TrimSpace(strings.ToLower(v))
|
||||
parts := strings.Split(v, ":")
|
||||
if len(parts) == 3 && len(parts[0]) > 4 {
|
||||
parts[0] = parts[0][len(parts[0])-4:]
|
||||
return strings.Join(parts, ":")
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (s *System) ResetNvidiaGPU(index int) (string, error) {
|
||||
return resetNvidiaGPU(index)
|
||||
}
|
||||
|
||||
// RunNCCLTests runs nccl-tests all_reduce_perf across the selected NVIDIA GPUs.
|
||||
// Measures collective communication bandwidth over NVLink/PCIe.
|
||||
func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
selected, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gpuCount := len(selected)
|
||||
if gpuCount < 1 {
|
||||
gpuCount = 1
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-all-reduce-perf.log", cmd: []string{
|
||||
"all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2",
|
||||
"-g", strconv.Itoa(gpuCount), "--iters", "20",
|
||||
}, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true},
|
||||
), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error) {
|
||||
selected, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var (
|
||||
profCmd []string
|
||||
profEnv []string
|
||||
)
|
||||
if len(selected) > 1 {
|
||||
// For multiple GPUs, always spawn one dcgmproftester process per GPU via
|
||||
// bee-dcgmproftester-staggered (stagger=0 means all start simultaneously).
|
||||
// A single dcgmproftester process without -i only loads GPU 0 regardless
|
||||
// of CUDA_VISIBLE_DEVICES.
|
||||
stagger := staggerSec
|
||||
if stagger < 0 {
|
||||
stagger = 0
|
||||
}
|
||||
profCmd = []string{
|
||||
"bee-dcgmproftester-staggered",
|
||||
"--seconds", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec)),
|
||||
"--stagger-seconds", strconv.Itoa(stagger),
|
||||
"--devices", joinIndexList(selected),
|
||||
}
|
||||
} else {
|
||||
profCmd, err = resolveDCGMProfTesterCommand("--no-dcgm-validation", "-t", "1004", "-d", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
profEnv = nvidiaVisibleDevicesEnv(selected)
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-compute", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-version.log", cmd: []string{"dcgmi", "-v"}},
|
||||
satJob{
|
||||
name: "03-dcgmproftester.log",
|
||||
cmd: profCmd,
|
||||
env: profEnv,
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
selected, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
|
||||
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
|
||||
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
|
||||
for _, p := range killed {
|
||||
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
|
||||
}
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-targeted-power.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
selected, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
|
||||
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
|
||||
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
|
||||
for _, p := range killed {
|
||||
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
|
||||
}
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-pulse-test.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
selected, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
|
||||
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
|
||||
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
|
||||
for _, p := range killed {
|
||||
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
|
||||
}
|
||||
}
|
||||
jobs := []satJob{
|
||||
{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
}
|
||||
|
||||
// On a system with GPUs on more than one CPU socket, run each socket's
|
||||
// GPUs through nvbandwidth in isolation before the all-GPU pass. Without
|
||||
// NVLink, cross-socket peer-to-peer traffic is a distinct fault domain
|
||||
// from same-socket traffic; if the single-socket passes log clean and
|
||||
// only the all-GPU pass doesn't complete, that isolates the cross-socket
|
||||
// path as the trigger instead of leaving it conflated with a general
|
||||
// GPU/PCIe fault. Systems with one socket (or no resolvable NUMA
|
||||
// affinity) get a single group back and keep the original one-pass shape.
|
||||
step := 3
|
||||
socketGroups := gpuBandwidthSocketGroups(selected, logFunc)
|
||||
if len(socketGroups) <= 1 {
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
})
|
||||
step++
|
||||
} else {
|
||||
for i, group := range socketGroups {
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group),
|
||||
collectGPU: true,
|
||||
gpuIndices: group,
|
||||
syncBracket: true,
|
||||
})
|
||||
step++
|
||||
}
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
})
|
||||
step++
|
||||
}
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-nvidia-smi-after.log", step),
|
||||
cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"},
|
||||
})
|
||||
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(jobs...), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
||||
return runAcceptancePackCtx(context.Background(), baseDir, "gpu-nvidia", nvidiaSATJobs(), logFunc)
|
||||
}
|
||||
|
||||
// RunNvidiaAcceptancePackWithOptions runs the NVIDIA diagnostics via DCGM.
|
||||
// diagLevel: 1=quick, 2=medium, 3=targeted stress, 4=extended stress.
|
||||
// gpuIndices: specific GPU indices to test (empty = all GPUs).
|
||||
// ctx cancellation kills the running job.
|
||||
func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
resolvedGPUIndices, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia", nvidiaDCGMJobs(diagLevel, resolvedGPUIndices), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
selected, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
|
||||
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
|
||||
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
|
||||
for _, p := range killed {
|
||||
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
|
||||
}
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-targeted-stress.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
}
|
||||
|
||||
func resolveDCGMGPUIndices(gpuIndices []int) ([]int, error) {
|
||||
if len(gpuIndices) > 0 {
|
||||
return dedupeSortedIndices(gpuIndices), nil
|
||||
}
|
||||
all, err := listNvidiaGPUIndices()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(all) == 0 {
|
||||
return nil, fmt.Errorf("nvidia-smi found no NVIDIA GPUs")
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
func memoryStressSizeArg() string {
|
||||
if mb := envInt("BEE_VM_STRESS_SIZE_MB", 0); mb > 0 {
|
||||
return fmt.Sprintf("%dM", mb)
|
||||
}
|
||||
availBytes := satFreeMemBytes()
|
||||
if availBytes <= 0 {
|
||||
return "80%"
|
||||
}
|
||||
availMB := availBytes / (1024 * 1024)
|
||||
targetMB := (availMB * 2) / 3
|
||||
if targetMB >= 256 {
|
||||
targetMB = (targetMB / 256) * 256
|
||||
}
|
||||
if targetMB <= 0 {
|
||||
return "80%"
|
||||
}
|
||||
return fmt.Sprintf("%dM", targetMB)
|
||||
}
|
||||
|
||||
func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error) {
|
||||
if sizeMB <= 0 {
|
||||
sizeMB = 256
|
||||
}
|
||||
if passes <= 0 {
|
||||
passes = 1
|
||||
}
|
||||
// Keep Validate Memory bounded to a quick diagnostic window. The timeout is
|
||||
// intentionally conservative enough for healthy systems while avoiding the
|
||||
// prior 30-80 minute hangs caused by memtester spinning on a bad subtest.
|
||||
timeoutSec := sizeMB*passes*20/100 + 60
|
||||
if timeoutSec < 180 {
|
||||
timeoutSec = 180
|
||||
}
|
||||
if timeoutSec > 900 {
|
||||
timeoutSec = 900
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}, syncBracket: true},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
seconds := durationSec
|
||||
if seconds <= 0 {
|
||||
seconds = envInt("BEE_VM_STRESS_SECONDS", 300)
|
||||
}
|
||||
// Base the default on current MemAvailable and keep headroom for the OS and
|
||||
// concurrent stressors so mixed burn runs do not trip the OOM killer.
|
||||
sizeArg := memoryStressSizeArg()
|
||||
return runAcceptancePackCtx(ctx, baseDir, "memory-stress", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-stress-ng-vm.log", cmd: []string{
|
||||
"stress-ng", "--vm", "1",
|
||||
"--vm-bytes", sizeArg,
|
||||
"--vm-method", "all",
|
||||
"--timeout", fmt.Sprintf("%d", seconds),
|
||||
"--metrics-brief",
|
||||
}, syncBracket: true},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
seconds := durationSec
|
||||
if seconds <= 0 {
|
||||
seconds = envInt("BEE_SAT_STRESS_SECONDS", 300)
|
||||
}
|
||||
cmd := []string{"stressapptest", "-s", fmt.Sprintf("%d", seconds), "-W", "--cc_test"}
|
||||
if mb := envInt("BEE_SAT_STRESS_MB", 0); mb > 0 {
|
||||
cmd = append(cmd, "-M", fmt.Sprintf("%d", mb))
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "sat-stress", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-stressapptest.log", cmd: cmd},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
// cpuThermalThrottleSysDir is the sysfs root the throttle-check scripts glob
|
||||
// under. Overridden in tests so they can point at a fake directory tree
|
||||
// instead of the real /sys.
|
||||
var cpuThermalThrottleSysDir = "/sys/devices/system/cpu"
|
||||
|
||||
// cpuThrottleSumScript is the shell fragment both before/after scripts use to
|
||||
// sum the kernel's cumulative-since-boot thermal throttle counters across
|
||||
// every CPU.
|
||||
func cpuThrottleSumScript() string {
|
||||
return fmt.Sprintf(`
|
||||
sum=0
|
||||
for f in %[1]s/cpu*/thermal_throttle/core_throttle_count %[1]s/cpu*/thermal_throttle/package_throttle_count; do
|
||||
[ -f "$f" ] || continue
|
||||
v=$(cat "$f" 2>/dev/null)
|
||||
case "$v" in ''|*[!0-9]*) continue ;; esac
|
||||
sum=$((sum + v))
|
||||
done
|
||||
`, cpuThermalThrottleSysDir)
|
||||
}
|
||||
|
||||
// cpuThrottleBeforeScript snapshots the throttle counter sum into a file in
|
||||
// {{run_dir}} so cpuThrottleCheckScript can later diff before/after despite
|
||||
// each satJob running as an independent process.
|
||||
func cpuThrottleBeforeScript() string {
|
||||
return cpuThrottleSumScript() + `echo "$sum" | tee {{run_dir}}/.cpu-throttle-before` + "\n"
|
||||
}
|
||||
|
||||
// cpuThrottleCheckScript compares the after-run throttle counter sum against
|
||||
// the snapshot cpuThrottleBeforeScript took, and fails (non-zero exit) if it
|
||||
// increased — i.e. the CPU actually hit thermal throttling during this
|
||||
// specific run, not just at some earlier point this boot. classifySATResult
|
||||
// maps a failed job here to SAT status FAILED, which ApplySATResultToDB
|
||||
// records as component status "Warning" for cpu:all — without this, the
|
||||
// "cpu" SAT pack only checks stress-ng's exit code, which is 0 whether or
|
||||
// not the CPU throttled while running it.
|
||||
func cpuThrottleCheckScript() string {
|
||||
return `before=$(cat {{run_dir}}/.cpu-throttle-before 2>/dev/null)
|
||||
case "$before" in ''|*[!0-9]*) before=0 ;; esac
|
||||
` + cpuThrottleSumScript() + `after=$sum
|
||||
echo "throttle_count_before=$before"
|
||||
echo "throttle_count_after=$after"
|
||||
if [ "$after" -gt "$before" ]; then
|
||||
echo "THROTTLE DETECTED: CPU package/core hit thermal throttling during this stress-ng run ($before -> $after)"
|
||||
exit 1
|
||||
fi
|
||||
echo "no new thermal throttling detected during this run"
|
||||
`
|
||||
}
|
||||
|
||||
func cpuSATJobs(durationSec int) []satJob {
|
||||
return []satJob{
|
||||
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
|
||||
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
|
||||
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
|
||||
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
|
||||
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
|
||||
{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
if durationSec <= 0 {
|
||||
durationSec = 60
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "cpu", cpuSATJobs(durationSec), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) {
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
runDir := filepath.Join(baseDir, "storage-"+ts)
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||
|
||||
devices, err := listStorageDevices()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sort.Strings(devices)
|
||||
|
||||
var summary strings.Builder
|
||||
stats := satStats{}
|
||||
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
||||
if len(devices) == 0 {
|
||||
fmt.Fprintln(&summary, "devices=0")
|
||||
stats.Unsupported++
|
||||
} else {
|
||||
fmt.Fprintf(&summary, "devices=%d\n", len(devices))
|
||||
}
|
||||
|
||||
for index, devPath := range devices {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
prefix := fmt.Sprintf("%02d-%s", index+1, filepath.Base(devPath))
|
||||
commands := storageSATCommands(devPath, extended)
|
||||
deviceOutputs := make(map[string][]byte, len(commands))
|
||||
for cmdIndex, job := range commands {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
|
||||
livePath := filepath.Join(runDir, name)
|
||||
runSyncBracketHook(job, "before", logFunc)
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc, livePath)
|
||||
deviceOutputs[job.name] = out
|
||||
if writeErr := os.WriteFile(livePath, out, 0644); writeErr != nil {
|
||||
return "", writeErr
|
||||
}
|
||||
if satJobBoundaryHook != nil {
|
||||
satJobBoundaryHook(name)
|
||||
}
|
||||
// smartctl -t short only launches the self-test on the drive firmware and
|
||||
// returns immediately ("Testing has begun"); unlike `nvme device-self-test
|
||||
// --wait`, smartctl has no blocking mode, so we must poll the drive
|
||||
// ourselves until the self-test actually finishes. Hold the "after" sync
|
||||
// until that poll completes — the self-test itself, not just its launch,
|
||||
// is the load worth having durable evidence of.
|
||||
deferSyncBracketAfter := job.name == "smartctl-self-test-short" && err == nil
|
||||
if !deferSyncBracketAfter {
|
||||
runSyncBracketHook(job, "after", logFunc)
|
||||
}
|
||||
status, rc := classifySATResult(job.name, out, err)
|
||||
// A zero smartctl exit status only proves the command ran. If the
|
||||
// drive did not return its overall-health verdict, it must not turn
|
||||
// the storage SAT green.
|
||||
if job.name == "smartctl-health" && status == "OK" && !hasSMARTOverallHealth(out) {
|
||||
status = "UNSUPPORTED"
|
||||
}
|
||||
stats.Add(status)
|
||||
key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_")
|
||||
fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
|
||||
fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
|
||||
|
||||
if deferSyncBracketAfter {
|
||||
statusName := "smartctl-self-test-status"
|
||||
statusOut := waitForSmartctlSelfTest(ctx, verboseLog, devPath, logFunc)
|
||||
deviceOutputs[statusName] = statusOut
|
||||
statusFile := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+2, statusName)
|
||||
if writeErr := os.WriteFile(filepath.Join(runDir, statusFile), statusOut, 0644); writeErr != nil {
|
||||
return "", writeErr
|
||||
}
|
||||
runSyncBracketHook(job, "after", logFunc)
|
||||
sStatus, sRC := classifySATResult(statusName, statusOut, nil)
|
||||
stats.Add(sStatus)
|
||||
sKey := filepath.Base(devPath) + "_" + strings.ReplaceAll(statusName, "-", "_")
|
||||
fmt.Fprintf(&summary, "%s_rc=%d\n", sKey, sRC)
|
||||
fmt.Fprintf(&summary, "%s_status=%s\n", sKey, sStatus)
|
||||
}
|
||||
}
|
||||
reportText := GenerateDiskReportText(index+1, devPath, deviceOutputs, time.Now().UTC())
|
||||
reportName := "disk-" + prefix + "-report.txt"
|
||||
_ = os.WriteFile(filepath.Join(runDir, reportName), []byte(reportText), 0644)
|
||||
}
|
||||
|
||||
writeSATStats(&summary, stats)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
|
||||
type satJob struct {
|
||||
name string
|
||||
cmd []string
|
||||
@@ -1574,41 +804,6 @@ func hasSMARTOverallHealth(out []byte) bool {
|
||||
return len(m) > 1 && strings.TrimSpace(m[1]) != ""
|
||||
}
|
||||
|
||||
func runSATCommand(verboseLog, name string, cmd []string, logFunc func(string)) ([]byte, error) {
|
||||
start := time.Now().UTC()
|
||||
resolvedCmd, err := resolveSATCommand(cmd)
|
||||
appendSATVerboseLog(verboseLog,
|
||||
fmt.Sprintf("[%s] start %s", start.Format(time.RFC3339), name),
|
||||
"cmd: "+strings.Join(resolvedCmd, " "),
|
||||
)
|
||||
if logFunc != nil {
|
||||
logFunc(fmt.Sprintf("=== %s ===", name))
|
||||
}
|
||||
if err != nil {
|
||||
appendSATVerboseLog(verboseLog,
|
||||
fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
|
||||
"rc: 1",
|
||||
fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()),
|
||||
"",
|
||||
)
|
||||
return []byte(err.Error() + "\n"), err
|
||||
}
|
||||
|
||||
out, err := streamExecOutput(satExecCommand(resolvedCmd[0], resolvedCmd[1:]...), logFunc, "")
|
||||
|
||||
rc := 0
|
||||
if err != nil {
|
||||
rc = 1
|
||||
}
|
||||
appendSATVerboseLog(verboseLog,
|
||||
fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
|
||||
fmt.Sprintf("rc: %d", rc),
|
||||
fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()),
|
||||
"",
|
||||
)
|
||||
return out, err
|
||||
}
|
||||
|
||||
func runROCmSMI(args ...string) ([]byte, error) {
|
||||
cmd, err := resolveROCmSMICommand(args...)
|
||||
if err != nil {
|
||||
@@ -1802,46 +997,3 @@ func envInt(name string, fallback int) int {
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func createTarGz(dst, srcDir string) error {
|
||||
file, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
gz := gzip.NewWriter(file)
|
||||
defer gz.Close()
|
||||
|
||||
tw := tar.NewWriter(gz)
|
||||
defer tw.Close()
|
||||
|
||||
base := filepath.Dir(srcDir)
|
||||
return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
header, err := tar.FileInfoHeader(info, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(base, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header.Name = rel
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
_, err = io.Copy(tw, file)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,568 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type NvidiaGPU struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
MemoryMB int `json:"memory_mb"`
|
||||
}
|
||||
|
||||
type NvidiaGPUStatus struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
BDF string `json:"bdf,omitempty"`
|
||||
Serial string `json:"serial,omitempty"`
|
||||
Status string `json:"status"`
|
||||
RawLine string `json:"raw_line,omitempty"`
|
||||
NeedsReset bool `json:"needs_reset"`
|
||||
ParseFailure bool `json:"parse_failure,omitempty"`
|
||||
}
|
||||
|
||||
type nvidiaGPUHealth struct {
|
||||
Index int
|
||||
Name string
|
||||
NeedsReset bool
|
||||
RawLine string
|
||||
ParseFailure bool
|
||||
}
|
||||
|
||||
type nvidiaGPUStatusFile struct {
|
||||
Index int
|
||||
Name string
|
||||
RunStatus string
|
||||
Reason string
|
||||
Health string
|
||||
HealthRaw string
|
||||
Observed bool
|
||||
Selected bool
|
||||
FailingJob string
|
||||
}
|
||||
|
||||
// AMDGPUInfo holds basic info about an AMD GPU from rocm-smi.
|
||||
type AMDGPUInfo struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// DetectGPUVendor returns "nvidia" if /dev/nvidia0 exists, "amd" if /dev/kfd exists, or "" otherwise.
|
||||
func (s *System) DetectGPUVendor() string {
|
||||
if _, err := os.Stat("/dev/nvidia0"); err == nil {
|
||||
return "nvidia"
|
||||
}
|
||||
if _, err := os.Stat("/dev/kfd"); err == nil {
|
||||
return "amd"
|
||||
}
|
||||
if raw, err := exec.Command("lspci", "-nn").Output(); err == nil {
|
||||
// Only match AMD GPU device classes [0300]=VGA, [0302]=3D controller, [0380]=Display.
|
||||
// AMD CPUs also appear in lspci as "Advanced Micro Devices" (Root Complex, IOMMU, etc.)
|
||||
// so matching vendor alone causes false positives on AMD CPU servers without GPUs.
|
||||
for _, line := range strings.Split(strings.ToLower(string(raw)), "\n") {
|
||||
if !strings.Contains(line, "advanced micro devices") && !strings.Contains(line, "amd/ati") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(line, "[0300]") || strings.Contains(line, "[0302]") || strings.Contains(line, "[0380]") {
|
||||
return "amd"
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// PhysicalGPUVendors reports which supported vendors have a display-class PCI
|
||||
// function, regardless of driver state. It is used to distinguish absent
|
||||
// hardware from a PCI function whose runtime is not operational yet.
|
||||
func (s *System) PhysicalGPUVendors() (nvidia bool, amd bool) {
|
||||
raw, err := satExecCommand("lspci", "-nn").Output()
|
||||
if err != nil {
|
||||
return false, false
|
||||
}
|
||||
for _, line := range strings.Split(strings.ToLower(string(raw)), "\n") {
|
||||
// [0300]=VGA, [0302]=3D controller, [0380]=Display controller.
|
||||
if !strings.Contains(line, "[0300]") && !strings.Contains(line, "[0302]") && !strings.Contains(line, "[0380]") {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case strings.Contains(line, "[10de:"):
|
||||
nvidia = true
|
||||
case strings.Contains(line, "[1002:"), strings.Contains(line, "advanced micro devices"), strings.Contains(line, "amd/ati"):
|
||||
amd = true
|
||||
}
|
||||
}
|
||||
return nvidia, amd
|
||||
}
|
||||
|
||||
// ListAMDGPUs returns AMD GPUs visible to rocm-smi.
|
||||
func (s *System) ListAMDGPUs() ([]AMDGPUInfo, error) {
|
||||
out, err := runROCmSMI("--showproductname", "--csv")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rocm-smi: %w", err)
|
||||
}
|
||||
var gpus []AMDGPUInfo
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(strings.ToLower(line), "device") {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(line, ",", 2)
|
||||
name := ""
|
||||
if len(parts) >= 2 {
|
||||
name = strings.TrimSpace(parts[1])
|
||||
}
|
||||
idx := len(gpus)
|
||||
gpus = append(gpus, AMDGPUInfo{Index: idx, Name: name})
|
||||
}
|
||||
return gpus, nil
|
||||
}
|
||||
|
||||
// RunAMDAcceptancePack runs an AMD GPU diagnostic pack using rocm-smi.
|
||||
func (s *System) RunAMDAcceptancePack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-amd", []satJob{
|
||||
{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
|
||||
{name: "02-rocm-smi-showallinfo.log", cmd: []string{"rocm-smi", "--showallinfo"}},
|
||||
{name: "03-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
|
||||
{name: "04-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
// RunAMDMemIntegrityPack runs the official RVS MEM module as a validate-style memory integrity test.
|
||||
func (s *System) RunAMDMemIntegrityPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
if err := ensureAMDRuntimeReady(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
cfgFile := "/tmp/bee-amd-mem.conf"
|
||||
cfg := `actions:
|
||||
- name: mem_integrity
|
||||
device: all
|
||||
module: mem
|
||||
parallel: true
|
||||
duration: 60000
|
||||
copy_matrix: false
|
||||
target_stress: 90
|
||||
matrix_size: 8640
|
||||
`
|
||||
_ = os.WriteFile(cfgFile, []byte(cfg), 0644)
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-mem", []satJob{
|
||||
{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
|
||||
{name: "02-rvs-mem.log", cmd: []string{"rvs", "-c", cfgFile}},
|
||||
{name: "03-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
// RunAMDMemBandwidthPack runs AMD's memory/interconnect bandwidth-oriented tools.
|
||||
func (s *System) RunAMDMemBandwidthPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
if err := ensureAMDRuntimeReady(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
cfgFile := "/tmp/bee-amd-babel.conf"
|
||||
cfg := `actions:
|
||||
- name: babel_mem_bw
|
||||
device: all
|
||||
module: babel
|
||||
parallel: true
|
||||
copy_matrix: true
|
||||
target_stress: 90
|
||||
matrix_size: 134217728
|
||||
`
|
||||
_ = os.WriteFile(cfgFile, []byte(cfg), 0644)
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-bandwidth", []satJob{
|
||||
{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
|
||||
{name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}},
|
||||
{name: "03-rvs-babel.log", cmd: []string{"rvs", "-c", cfgFile}},
|
||||
{name: "04-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
// RunAMDStressPack runs an AMD GPU burn-in pack.
|
||||
// Missing tools are reported as UNSUPPORTED, consistent with the existing SAT pattern.
|
||||
func (s *System) RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
seconds := durationSec
|
||||
if seconds <= 0 {
|
||||
seconds = envInt("BEE_AMD_STRESS_SECONDS", 300)
|
||||
}
|
||||
if err := ensureAMDRuntimeReady(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Enable copy_matrix so the same GST run drives VRAM traffic in addition to compute.
|
||||
rvsCfg := amdStressRVSConfig(seconds)
|
||||
cfgFile := "/tmp/bee-amd-gst.conf"
|
||||
_ = os.WriteFile(cfgFile, []byte(rvsCfg), 0644)
|
||||
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-stress", amdStressJobs(seconds, cfgFile), logFunc)
|
||||
}
|
||||
|
||||
func amdStressRVSConfig(seconds int) string {
|
||||
return fmt.Sprintf(`actions:
|
||||
- name: gst_stress
|
||||
device: all
|
||||
module: gst
|
||||
parallel: true
|
||||
duration: %d
|
||||
copy_matrix: false
|
||||
target_stress: 90
|
||||
matrix_size_a: 8640
|
||||
matrix_size_b: 8640
|
||||
matrix_size_c: 8640
|
||||
`, seconds*1000)
|
||||
}
|
||||
|
||||
func amdStressJobs(seconds int, cfgFile string) []satJob {
|
||||
return []satJob{
|
||||
{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
|
||||
{name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}},
|
||||
{name: fmt.Sprintf("03-rvs-gst-%ds.log", seconds), cmd: []string{"rvs", "-c", cfgFile}},
|
||||
{name: fmt.Sprintf("04-rocm-smi-after.log"), cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--csv"}},
|
||||
}
|
||||
}
|
||||
|
||||
// ListNvidiaGPUs returns GPUs visible to nvidia-smi.
|
||||
func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) {
|
||||
out, err := exec.Command("nvidia-smi",
|
||||
"--query-gpu=index,name,memory.total",
|
||||
"--format=csv,noheader,nounits").Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nvidia-smi: %w", err)
|
||||
}
|
||||
var gpus []NvidiaGPU
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(line, ", ", 3)
|
||||
if len(parts) != 3 {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
memMB, _ := strconv.Atoi(strings.TrimSpace(parts[2]))
|
||||
gpus = append(gpus, NvidiaGPU{
|
||||
Index: idx,
|
||||
Name: strings.TrimSpace(parts[1]),
|
||||
MemoryMB: memMB,
|
||||
})
|
||||
}
|
||||
sort.Slice(gpus, func(i, j int) bool {
|
||||
return gpus[i].Index < gpus[j].Index
|
||||
})
|
||||
return gpus, nil
|
||||
}
|
||||
|
||||
func (s *System) ListNvidiaGPUStatuses() ([]NvidiaGPUStatus, error) {
|
||||
out, err := satExecCommand(
|
||||
"nvidia-smi",
|
||||
"--query-gpu=index,name,pci.bus_id,serial,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total",
|
||||
"--format=csv,noheader,nounits",
|
||||
).Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nvidia-smi: %w", err)
|
||||
}
|
||||
var gpus []NvidiaGPUStatus
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(line, ",")
|
||||
if len(parts) < 4 {
|
||||
gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true})
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||||
if err != nil {
|
||||
gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true})
|
||||
continue
|
||||
}
|
||||
upper := strings.ToUpper(line)
|
||||
needsReset := strings.Contains(upper, "GPU REQUIRES RESET")
|
||||
status := "OK"
|
||||
if needsReset {
|
||||
status = "RESET_REQUIRED"
|
||||
}
|
||||
gpus = append(gpus, NvidiaGPUStatus{
|
||||
Index: idx,
|
||||
Name: strings.TrimSpace(parts[1]),
|
||||
BDF: normalizeNvidiaBusID(strings.TrimSpace(parts[2])),
|
||||
Serial: strings.TrimSpace(parts[3]),
|
||||
Status: status,
|
||||
RawLine: line,
|
||||
NeedsReset: needsReset,
|
||||
})
|
||||
}
|
||||
sort.Slice(gpus, func(i, j int) bool { return gpus[i].Index < gpus[j].Index })
|
||||
return gpus, nil
|
||||
}
|
||||
|
||||
func normalizeNvidiaBusID(v string) string {
|
||||
v = strings.TrimSpace(strings.ToLower(v))
|
||||
parts := strings.Split(v, ":")
|
||||
if len(parts) == 3 && len(parts[0]) > 4 {
|
||||
parts[0] = parts[0][len(parts[0])-4:]
|
||||
return strings.Join(parts, ":")
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (s *System) ResetNvidiaGPU(index int) (string, error) {
|
||||
return resetNvidiaGPU(index)
|
||||
}
|
||||
|
||||
// RunNCCLTests runs nccl-tests all_reduce_perf across the selected NVIDIA GPUs.
|
||||
// Measures collective communication bandwidth over NVLink/PCIe.
|
||||
func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
selected, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gpuCount := len(selected)
|
||||
if gpuCount < 1 {
|
||||
gpuCount = 1
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-all-reduce-perf.log", cmd: []string{
|
||||
"all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2",
|
||||
"-g", strconv.Itoa(gpuCount), "--iters", "20",
|
||||
}, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true},
|
||||
), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error) {
|
||||
selected, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var (
|
||||
profCmd []string
|
||||
profEnv []string
|
||||
)
|
||||
if len(selected) > 1 {
|
||||
// For multiple GPUs, always spawn one dcgmproftester process per GPU via
|
||||
// bee-dcgmproftester-staggered (stagger=0 means all start simultaneously).
|
||||
// A single dcgmproftester process without -i only loads GPU 0 regardless
|
||||
// of CUDA_VISIBLE_DEVICES.
|
||||
stagger := staggerSec
|
||||
if stagger < 0 {
|
||||
stagger = 0
|
||||
}
|
||||
profCmd = []string{
|
||||
"bee-dcgmproftester-staggered",
|
||||
"--seconds", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec)),
|
||||
"--stagger-seconds", strconv.Itoa(stagger),
|
||||
"--devices", joinIndexList(selected),
|
||||
}
|
||||
} else {
|
||||
profCmd, err = resolveDCGMProfTesterCommand("--no-dcgm-validation", "-t", "1004", "-d", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
profEnv = nvidiaVisibleDevicesEnv(selected)
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-compute", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-version.log", cmd: []string{"dcgmi", "-v"}},
|
||||
satJob{
|
||||
name: "03-dcgmproftester.log",
|
||||
cmd: profCmd,
|
||||
env: profEnv,
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
selected, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
|
||||
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
|
||||
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
|
||||
for _, p := range killed {
|
||||
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
|
||||
}
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-targeted-power.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
selected, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
|
||||
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
|
||||
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
|
||||
for _, p := range killed {
|
||||
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
|
||||
}
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-pulse-test.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
}
|
||||
|
||||
// RunNvidiaBandwidthPack runs `dcgmi diag -r nvbandwidth`. The only thing
|
||||
// fullMatrix changes is which GPU sets each invocation gets via `-i`:
|
||||
//
|
||||
// - fullMatrix=false (Validate): a single pass across every selected GPU.
|
||||
// - fullMatrix=true (deep/Stress): on a system whose GPUs span more than
|
||||
// one CPU socket, one pass per socket group and then one pass across all
|
||||
// of them, isolating the cross-socket peer-to-peer path as its own
|
||||
// fault domain (see bible-local/decisions/2026-07-27-nvbandwidth-per-socket-split.md).
|
||||
// Single-socket systems collapse back to one pass.
|
||||
func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, fullMatrix bool, logFunc func(string)) (string, error) {
|
||||
selected, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
|
||||
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
|
||||
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
|
||||
for _, p := range killed {
|
||||
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
|
||||
}
|
||||
}
|
||||
jobs := []satJob{
|
||||
{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
}
|
||||
|
||||
// On a system with GPUs on more than one CPU socket, run each socket's
|
||||
// GPUs through nvbandwidth in isolation before the all-GPU pass. Without
|
||||
// NVLink, cross-socket peer-to-peer traffic is a distinct fault domain
|
||||
// from same-socket traffic; if the single-socket passes log clean and
|
||||
// only the all-GPU pass doesn't complete, that isolates the cross-socket
|
||||
// path as the trigger instead of leaving it conflated with a general
|
||||
// GPU/PCIe fault. Systems with one socket (or no resolvable NUMA
|
||||
// affinity) get a single group back and keep the original one-pass shape.
|
||||
step := 3
|
||||
socketGroups := [][]int{selected}
|
||||
if fullMatrix {
|
||||
socketGroups = gpuBandwidthSocketGroups(selected, logFunc)
|
||||
}
|
||||
if len(socketGroups) <= 1 {
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
})
|
||||
step++
|
||||
} else {
|
||||
for i, group := range socketGroups {
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group),
|
||||
collectGPU: true,
|
||||
gpuIndices: group,
|
||||
syncBracket: true,
|
||||
})
|
||||
step++
|
||||
}
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
})
|
||||
step++
|
||||
}
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-nvidia-smi-after.log", step),
|
||||
cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"},
|
||||
})
|
||||
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(jobs...), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
||||
return runAcceptancePackCtx(context.Background(), baseDir, "gpu-nvidia", nvidiaSATJobs(), logFunc)
|
||||
}
|
||||
|
||||
// RunNvidiaAcceptancePackWithOptions runs the NVIDIA diagnostics via DCGM.
|
||||
// diagLevel: 1=quick, 2=medium, 3=targeted stress, 4=extended stress.
|
||||
// gpuIndices: specific GPU indices to test (empty = all GPUs).
|
||||
// ctx cancellation kills the running job.
|
||||
func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
resolvedGPUIndices, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia", nvidiaDCGMJobs(diagLevel, resolvedGPUIndices), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
selected, err := resolveDCGMGPUIndices(gpuIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
|
||||
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
|
||||
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
|
||||
for _, p := range killed {
|
||||
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
|
||||
}
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-targeted-stress.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
}
|
||||
|
||||
func resolveDCGMGPUIndices(gpuIndices []int) ([]int, error) {
|
||||
if len(gpuIndices) > 0 {
|
||||
return dedupeSortedIndices(gpuIndices), nil
|
||||
}
|
||||
all, err := listNvidiaGPUIndices()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(all) == 0 {
|
||||
return nil, fmt.Errorf("nvidia-smi found no NVIDIA GPUs")
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func memoryStressSizeArg() string {
|
||||
if mb := envInt("BEE_VM_STRESS_SIZE_MB", 0); mb > 0 {
|
||||
return fmt.Sprintf("%dM", mb)
|
||||
}
|
||||
availBytes := satFreeMemBytes()
|
||||
if availBytes <= 0 {
|
||||
return "80%"
|
||||
}
|
||||
availMB := availBytes / (1024 * 1024)
|
||||
targetMB := (availMB * 2) / 3
|
||||
if targetMB >= 256 {
|
||||
targetMB = (targetMB / 256) * 256
|
||||
}
|
||||
if targetMB <= 0 {
|
||||
return "80%"
|
||||
}
|
||||
return fmt.Sprintf("%dM", targetMB)
|
||||
}
|
||||
|
||||
func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error) {
|
||||
if sizeMB <= 0 {
|
||||
sizeMB = 256
|
||||
}
|
||||
if passes <= 0 {
|
||||
passes = 1
|
||||
}
|
||||
// Keep Validate Memory bounded to a quick diagnostic window. The timeout is
|
||||
// intentionally conservative enough for healthy systems while avoiding the
|
||||
// prior 30-80 minute hangs caused by memtester spinning on a bad subtest.
|
||||
timeoutSec := sizeMB*passes*20/100 + 60
|
||||
if timeoutSec < 180 {
|
||||
timeoutSec = 180
|
||||
}
|
||||
if timeoutSec > 900 {
|
||||
timeoutSec = 900
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}, syncBracket: true},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
seconds := durationSec
|
||||
if seconds <= 0 {
|
||||
seconds = envInt("BEE_VM_STRESS_SECONDS", 300)
|
||||
}
|
||||
// Base the default on current MemAvailable and keep headroom for the OS and
|
||||
// concurrent stressors so mixed burn runs do not trip the OOM killer.
|
||||
sizeArg := memoryStressSizeArg()
|
||||
return runAcceptancePackCtx(ctx, baseDir, "memory-stress", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-stress-ng-vm.log", cmd: []string{
|
||||
"stress-ng", "--vm", "1",
|
||||
"--vm-bytes", sizeArg,
|
||||
"--vm-method", "all",
|
||||
"--timeout", fmt.Sprintf("%d", seconds),
|
||||
"--metrics-brief",
|
||||
}, syncBracket: true},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
seconds := durationSec
|
||||
if seconds <= 0 {
|
||||
seconds = envInt("BEE_SAT_STRESS_SECONDS", 300)
|
||||
}
|
||||
cmd := []string{"stressapptest", "-s", fmt.Sprintf("%d", seconds), "-W", "--cc_test"}
|
||||
if mb := envInt("BEE_SAT_STRESS_MB", 0); mb > 0 {
|
||||
cmd = append(cmd, "-M", fmt.Sprintf("%d", mb))
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "sat-stress", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-stressapptest.log", cmd: cmd},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
// cpuThermalThrottleSysDir is the sysfs root the throttle-check scripts glob
|
||||
// under. Overridden in tests so they can point at a fake directory tree
|
||||
// instead of the real /sys.
|
||||
var cpuThermalThrottleSysDir = "/sys/devices/system/cpu"
|
||||
|
||||
// cpuThrottleSumScript is the shell fragment both before/after scripts use to
|
||||
// sum the kernel's cumulative-since-boot thermal throttle counters across
|
||||
// every CPU.
|
||||
func cpuThrottleSumScript() string {
|
||||
return fmt.Sprintf(`
|
||||
sum=0
|
||||
for f in %[1]s/cpu*/thermal_throttle/core_throttle_count %[1]s/cpu*/thermal_throttle/package_throttle_count; do
|
||||
[ -f "$f" ] || continue
|
||||
v=$(cat "$f" 2>/dev/null)
|
||||
case "$v" in ''|*[!0-9]*) continue ;; esac
|
||||
sum=$((sum + v))
|
||||
done
|
||||
`, cpuThermalThrottleSysDir)
|
||||
}
|
||||
|
||||
// cpuThrottleBeforeScript snapshots the throttle counter sum into a file in
|
||||
// {{run_dir}} so cpuThrottleCheckScript can later diff before/after despite
|
||||
// each satJob running as an independent process.
|
||||
func cpuThrottleBeforeScript() string {
|
||||
return cpuThrottleSumScript() + `echo "$sum" | tee {{run_dir}}/.cpu-throttle-before` + "\n"
|
||||
}
|
||||
|
||||
// cpuThrottleCheckScript compares the after-run throttle counter sum against
|
||||
// the snapshot cpuThrottleBeforeScript took, and fails (non-zero exit) if it
|
||||
// increased — i.e. the CPU actually hit thermal throttling during this
|
||||
// specific run, not just at some earlier point this boot. classifySATResult
|
||||
// maps a failed job here to SAT status FAILED, which ApplySATResultToDB
|
||||
// records as component status "Warning" for cpu:all — without this, the
|
||||
// "cpu" SAT pack only checks stress-ng's exit code, which is 0 whether or
|
||||
// not the CPU throttled while running it.
|
||||
func cpuThrottleCheckScript() string {
|
||||
return `before=$(cat {{run_dir}}/.cpu-throttle-before 2>/dev/null)
|
||||
case "$before" in ''|*[!0-9]*) before=0 ;; esac
|
||||
` + cpuThrottleSumScript() + `after=$sum
|
||||
echo "throttle_count_before=$before"
|
||||
echo "throttle_count_after=$after"
|
||||
if [ "$after" -gt "$before" ]; then
|
||||
echo "THROTTLE DETECTED: CPU package/core hit thermal throttling during this stress-ng run ($before -> $after)"
|
||||
exit 1
|
||||
fi
|
||||
echo "no new thermal throttling detected during this run"
|
||||
`
|
||||
}
|
||||
|
||||
func cpuSATJobs(durationSec int) []satJob {
|
||||
return []satJob{
|
||||
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
|
||||
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
|
||||
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
|
||||
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
|
||||
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
|
||||
{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
if durationSec <= 0 {
|
||||
durationSec = 60
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "cpu", cpuSATJobs(durationSec), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) {
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
runDir := filepath.Join(baseDir, "storage-"+ts)
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||
|
||||
devices, err := listStorageDevices()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sort.Strings(devices)
|
||||
|
||||
var summary strings.Builder
|
||||
stats := satStats{}
|
||||
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
||||
if len(devices) == 0 {
|
||||
fmt.Fprintln(&summary, "devices=0")
|
||||
stats.Unsupported++
|
||||
} else {
|
||||
fmt.Fprintf(&summary, "devices=%d\n", len(devices))
|
||||
}
|
||||
|
||||
for index, devPath := range devices {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
prefix := fmt.Sprintf("%02d-%s", index+1, filepath.Base(devPath))
|
||||
commands := storageSATCommands(devPath, extended)
|
||||
deviceOutputs := make(map[string][]byte, len(commands))
|
||||
for cmdIndex, job := range commands {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
|
||||
livePath := filepath.Join(runDir, name)
|
||||
runSyncBracketHook(job, "before", logFunc)
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc, livePath)
|
||||
deviceOutputs[job.name] = out
|
||||
if writeErr := os.WriteFile(livePath, out, 0644); writeErr != nil {
|
||||
return "", writeErr
|
||||
}
|
||||
if satJobBoundaryHook != nil {
|
||||
satJobBoundaryHook(name)
|
||||
}
|
||||
// smartctl -t short only launches the self-test on the drive firmware and
|
||||
// returns immediately ("Testing has begun"); unlike `nvme device-self-test
|
||||
// --wait`, smartctl has no blocking mode, so we must poll the drive
|
||||
// ourselves until the self-test actually finishes. Hold the "after" sync
|
||||
// until that poll completes — the self-test itself, not just its launch,
|
||||
// is the load worth having durable evidence of.
|
||||
deferSyncBracketAfter := job.name == "smartctl-self-test-short" && err == nil
|
||||
if !deferSyncBracketAfter {
|
||||
runSyncBracketHook(job, "after", logFunc)
|
||||
}
|
||||
status, rc := classifySATResult(job.name, out, err)
|
||||
// A zero smartctl exit status only proves the command ran. If the
|
||||
// drive did not return its overall-health verdict, it must not turn
|
||||
// the storage SAT green.
|
||||
if job.name == "smartctl-health" && status == "OK" && !hasSMARTOverallHealth(out) {
|
||||
status = "UNSUPPORTED"
|
||||
}
|
||||
stats.Add(status)
|
||||
key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_")
|
||||
fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
|
||||
fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
|
||||
|
||||
if deferSyncBracketAfter {
|
||||
statusName := "smartctl-self-test-status"
|
||||
statusOut := waitForSmartctlSelfTest(ctx, verboseLog, devPath, logFunc)
|
||||
deviceOutputs[statusName] = statusOut
|
||||
statusFile := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+2, statusName)
|
||||
if writeErr := os.WriteFile(filepath.Join(runDir, statusFile), statusOut, 0644); writeErr != nil {
|
||||
return "", writeErr
|
||||
}
|
||||
runSyncBracketHook(job, "after", logFunc)
|
||||
sStatus, sRC := classifySATResult(statusName, statusOut, nil)
|
||||
stats.Add(sStatus)
|
||||
sKey := filepath.Base(devPath) + "_" + strings.ReplaceAll(statusName, "-", "_")
|
||||
fmt.Fprintf(&summary, "%s_rc=%d\n", sKey, sRC)
|
||||
fmt.Fprintf(&summary, "%s_status=%s\n", sKey, sStatus)
|
||||
}
|
||||
}
|
||||
reportText := GenerateDiskReportText(index+1, devPath, deviceOutputs, time.Now().UTC())
|
||||
reportName := "disk-" + prefix + "-report.txt"
|
||||
_ = os.WriteFile(filepath.Join(runDir, reportName), []byte(reportText), 0644)
|
||||
}
|
||||
|
||||
writeSATStats(&summary, stats)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
@@ -145,8 +145,6 @@ func TestNvidiaDCGMJobsEnablePersistenceModeBeforeDiag(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestBuildNvidiaStressJobUsesSelectedLoaderAndDevices(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
oldExecCommand := satExecCommand
|
||||
satExecCommand = func(name string, args ...string) *exec.Cmd {
|
||||
if name == "nvidia-smi" {
|
||||
@@ -179,8 +177,6 @@ func TestBuildNvidiaStressJobUsesSelectedLoaderAndDevices(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestBuildNvidiaStressJobUsesNCCLLoader(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
oldExecCommand := satExecCommand
|
||||
satExecCommand = func(name string, args ...string) *exec.Cmd {
|
||||
if name == "nvidia-smi" {
|
||||
@@ -213,8 +209,6 @@ func TestBuildNvidiaStressJobUsesNCCLLoader(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestResolveDCGMGPUIndicesUsesDetectedGPUsWhenUnset(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
oldExecCommand := satExecCommand
|
||||
satExecCommand = func(name string, args ...string) *exec.Cmd {
|
||||
if name == "nvidia-smi" {
|
||||
|
||||
@@ -1,14 +1,82 @@
|
||||
package platform
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// tpmDeviceGlob is a seam for tests. It reports the sysfs TPM device nodes
|
||||
// the kernel has registered; an empty result means the platform exposes no
|
||||
// TPM at all (no discrete chip, or firmware/BIOS has it disabled).
|
||||
var tpmDeviceGlob = func() []string {
|
||||
matches, _ := filepath.Glob("/sys/class/tpm/tpm*")
|
||||
return matches
|
||||
}
|
||||
|
||||
var tpmReadFile = os.ReadFile
|
||||
|
||||
// TPMPresent reports whether sysfs identifies a registered device as TPM 2.x.
|
||||
// The validation pack uses tpm2-tools, so a TPM 1.2 device is not sufficient.
|
||||
func (s *System) TPMPresent() bool {
|
||||
for _, device := range tpmDeviceGlob() {
|
||||
version, err := tpmReadFile(filepath.Join(device, "tpm_version_major"))
|
||||
if err == nil && strings.TrimSpace(string(version)) == "2" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// RunTPMValidationPack verifies TPM 2.0 communication using read-only
|
||||
// commands. It deliberately excludes SelfTest, provisioning, NV writes, PCR
|
||||
// changes, key creation, and ownership operations.
|
||||
//
|
||||
// When the platform exposes no TPM device at all, the pack does not run the
|
||||
// tpm2_* tools: without a TCTI device they only ever emit a wall of
|
||||
// "Failed to open ... /dev/tpmrm0" errors that read as a hard failure when
|
||||
// the real situation is "this machine has no TPM". Instead it writes an
|
||||
// UNSUPPORTED summary and returns, the same way the storage pack handles a
|
||||
// host with no drives.
|
||||
func (s *System) RunTPMValidationPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
if !s.TPMPresent() {
|
||||
return writeTPMUnsupportedRun(baseDir, logFunc)
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "tpm", tpmValidationJobs(), logFunc)
|
||||
}
|
||||
|
||||
func writeTPMUnsupportedRun(baseDir string, logFunc func(string)) (string, error) {
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
runDir := filepath.Join(baseDir, "tpm-"+now.Format("20060102-150405"))
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if logFunc != nil {
|
||||
logFunc("no TPM 2.x device reported by sysfs; skipping read-only TPM checks")
|
||||
}
|
||||
|
||||
var summary strings.Builder
|
||||
fmt.Fprintf(&summary, "run_at_utc=%s\n", now.Format(time.RFC3339))
|
||||
summary.WriteString("tpm_present=false\n")
|
||||
summary.WriteString("skip_reason=no TPM 2.x device reported by sysfs; tpm2-tools are not applicable\n")
|
||||
summary.WriteString("tpm_check_status=UNSUPPORTED\n")
|
||||
summary.WriteString("overall_status=UNSUPPORTED\n")
|
||||
summary.WriteString("job_ok=0\n")
|
||||
summary.WriteString("job_failed=0\n")
|
||||
summary.WriteString("job_unsupported=1\n")
|
||||
summary.WriteString("job_informational_failed=0\n")
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
|
||||
func tpmValidationJobs() []satJob {
|
||||
return []satJob{
|
||||
{name: "01-properties-fixed.log", cmd: []string{"tpm2_getcap", "properties-fixed"}},
|
||||
|
||||
@@ -1,11 +1,66 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunTPMValidationPackSkipsWhenNoTPMDevice(t *testing.T) {
|
||||
old := tpmDeviceGlob
|
||||
tpmDeviceGlob = func() []string { return nil }
|
||||
t.Cleanup(func() { tpmDeviceGlob = old })
|
||||
|
||||
dir := t.TempDir()
|
||||
runDir, err := (&System{}).RunTPMValidationPack(nil, dir, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("RunTPMValidationPack: %v", err)
|
||||
}
|
||||
for _, name := range []string{"01-properties-fixed.log", "02-pcr-banks.log", "03-pcr-values.log", "04-test-result.log"} {
|
||||
if _, err := os.Stat(filepath.Join(runDir, name)); err == nil {
|
||||
t.Fatalf("tpm2 job %q ran despite no TPM device", name)
|
||||
}
|
||||
}
|
||||
summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("read summary: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(summary), "overall_status=UNSUPPORTED") ||
|
||||
!strings.Contains(string(summary), "tpm_present=false") {
|
||||
t.Fatalf("summary missing UNSUPPORTED/tpm_present markers:\n%s", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTPMPresentRequiresVersion2(t *testing.T) {
|
||||
oldGlob, oldRead := tpmDeviceGlob, tpmReadFile
|
||||
t.Cleanup(func() {
|
||||
tpmDeviceGlob = oldGlob
|
||||
tpmReadFile = oldRead
|
||||
})
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
version string
|
||||
readErr error
|
||||
want bool
|
||||
}{
|
||||
{name: "TPM 2", version: "2\n", want: true},
|
||||
{name: "TPM 1.2", version: "1\n", want: false},
|
||||
{name: "missing version attribute", readErr: os.ErrNotExist, want: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
tpmDeviceGlob = func() []string { return []string{"/sys/class/tpm/tpm0"} }
|
||||
tpmReadFile = func(string) ([]byte, error) { return []byte(test.version), test.readErr }
|
||||
if got := (&System{}).TPMPresent(); got != test.want {
|
||||
t.Fatalf("TPMPresent()=%v want %v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTPMValidationJobsAreReadOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user