759 lines
27 KiB
Go
759 lines
27 KiB
Go
package platform
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const benchmarkVersion = "2"
|
|
|
|
type benchmarkProfileSpec struct {
|
|
Name string
|
|
BaselineSec int
|
|
WarmupSec int
|
|
SteadySec int
|
|
NCCLSec int
|
|
CooldownSec int
|
|
}
|
|
|
|
type benchmarkGPUInfo struct {
|
|
Index int
|
|
UUID string
|
|
Name string
|
|
BusID string
|
|
VBIOS string
|
|
PowerLimitW float64
|
|
DefaultPowerLimitW float64
|
|
MinPowerLimitW float64
|
|
MaxPowerLimitW float64
|
|
MaxGraphicsClockMHz float64
|
|
MaxMemoryClockMHz float64
|
|
BaseGraphicsClockMHz float64
|
|
MultiprocessorCount int
|
|
// Temperature limits sourced from nvidia-smi -q verbose output.
|
|
// ShutdownTempC is the hardware thermal shutdown threshold.
|
|
// SlowdownTempC is the software throttle onset threshold.
|
|
// Both fall back to safe conservative defaults when not available.
|
|
ShutdownTempC float64 // fallback: 90°C
|
|
SlowdownTempC float64 // fallback: 80°C
|
|
}
|
|
|
|
type benchmarkPowerCalibrationResult struct {
|
|
Summary BenchmarkTelemetrySummary
|
|
AppliedPowerLimitW float64
|
|
Attempts int
|
|
Derated bool
|
|
Completed bool
|
|
Notes []string
|
|
// CoolingWarning is set when the GPU throttled thermally with a clock drop
|
|
// ≥20% while server fans were below 100% duty cycle — a signal that the
|
|
// cooling system may not be correctly configured for full GPU load.
|
|
CoolingWarning string
|
|
// MetricRows holds the telemetry rows from the final (converged) attempt
|
|
// for this GPU. Used to build per-run gpu-metrics.csv.
|
|
MetricRows []GPUMetricRow
|
|
}
|
|
|
|
type benchmarkPowerCalibrationRunSummary struct {
|
|
LoadedSDR benchmarkSDRSeriesSummary
|
|
AvgFanRPM float64
|
|
AvgFanDutyCyclePct float64
|
|
FanSamples int
|
|
}
|
|
|
|
type benchmarkBurnProfile struct {
|
|
name string
|
|
category string
|
|
supported bool
|
|
lanes int
|
|
m uint64
|
|
n uint64
|
|
k uint64
|
|
iterations uint64
|
|
notes string
|
|
}
|
|
|
|
type benchmarkBurnParseResult struct {
|
|
Device string
|
|
ComputeCapability string
|
|
Backend string
|
|
DurationSec int
|
|
Profiles []BenchmarkPrecisionResult
|
|
Fallback bool
|
|
}
|
|
|
|
type benchmarkRestoreAction struct {
|
|
name string
|
|
fn func()
|
|
}
|
|
|
|
var (
|
|
benchmarkReadyPattern = regexp.MustCompile(`^([a-z0-9_]+)\[(\d+)\]=READY dim=(\d+)x(\d+)x(\d+)\b`)
|
|
benchmarkSkippedPattern = regexp.MustCompile(`^([a-z0-9_]+)(?:\[\d+\])?=SKIPPED (.+)$`)
|
|
benchmarkIterationsPattern = regexp.MustCompile(`^([a-z0-9_]+)_iterations=(\d+)$`)
|
|
benchmarkGeteuid = os.Geteuid
|
|
benchmarkResetNvidiaGPU = resetNvidiaGPU
|
|
benchmarkSleep = time.Sleep
|
|
)
|
|
|
|
// benchmarkPrecisionPhases lists the precision categories run as individual
|
|
// steady-state windows before the combined steady pass. Order is from lowest
|
|
// to highest power draw so thermal ramp-up is gradual.
|
|
//
|
|
// fp64 and fp4 are intentionally disabled for now: both are currently unstable
|
|
// on the target fleet and can abort the mixed steady stage after the earlier
|
|
// phases already collected useful telemetry.
|
|
var benchmarkPrecisionPhases = []string{"int8", "fp8", "fp16", "fp32"}
|
|
|
|
func computeCapabilityCode(raw string) int {
|
|
raw = strings.TrimSpace(raw)
|
|
if raw == "" {
|
|
return 0
|
|
}
|
|
parts := strings.SplitN(raw, ".", 2)
|
|
major, _ := strconv.Atoi(strings.TrimSpace(parts[0]))
|
|
minor := 0
|
|
if len(parts) > 1 {
|
|
minor, _ = strconv.Atoi(strings.TrimSpace(parts[1]))
|
|
}
|
|
return major*10 + minor
|
|
}
|
|
|
|
func benchmarkSupportedPrecisions(computeCapability string) []string {
|
|
cc := computeCapabilityCode(computeCapability)
|
|
out := make([]string, 0, len(benchmarkPrecisionPhases))
|
|
for _, prec := range benchmarkPrecisionPhases {
|
|
if prec == "fp4" && cc > 0 && cc < 100 {
|
|
continue
|
|
}
|
|
out = append(out, prec)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func benchmarkPrecisionEnabled(category string) bool {
|
|
switch category {
|
|
case "int8", "fp8", "fp16", "fp16_bf16", "fp32", "fp32_tf32":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func buildBenchmarkSteadyPlan(spec benchmarkProfileSpec, precisions []string, metricStage func(string) string) (planLabels []string, planPhases []benchmarkPlannedPhase, basePhaseSec int, mixedPhaseSec int) {
|
|
if len(precisions) == 0 {
|
|
precisions = append([]string(nil), benchmarkPrecisionPhases...)
|
|
}
|
|
switch spec.Name {
|
|
case NvidiaBenchmarkProfileStandard:
|
|
basePhaseSec = 60
|
|
mixedPhaseSec = 300
|
|
case NvidiaBenchmarkProfileStability:
|
|
basePhaseSec = 300
|
|
mixedPhaseSec = 3600
|
|
case NvidiaBenchmarkProfileOvernight:
|
|
basePhaseSec = 3600
|
|
mixedPhaseSec = 14400
|
|
default:
|
|
totalWeight := len(precisions) + 5
|
|
if totalWeight <= 0 {
|
|
return nil, nil, 0, 0
|
|
}
|
|
basePhaseSec = spec.SteadySec / totalWeight
|
|
if basePhaseSec <= 0 {
|
|
basePhaseSec = 1
|
|
}
|
|
mixedPhaseSec = basePhaseSec * 5
|
|
}
|
|
planLabels = make([]string, 0, len(precisions)+1)
|
|
planPhases = make([]benchmarkPlannedPhase, 0, len(precisions)+1)
|
|
for _, prec := range precisions {
|
|
planLabels = append(planLabels, prec)
|
|
planPhases = append(planPhases, benchmarkPlannedPhase{
|
|
PlanLabel: prec,
|
|
MetricStage: metricStage(prec),
|
|
DurationSec: basePhaseSec,
|
|
})
|
|
}
|
|
planLabels = append(planLabels, "mixed")
|
|
planPhases = append(planPhases, benchmarkPlannedPhase{
|
|
PlanLabel: "mixed",
|
|
MetricStage: metricStage("mixed"),
|
|
DurationSec: mixedPhaseSec,
|
|
})
|
|
return planLabels, planPhases, basePhaseSec, mixedPhaseSec
|
|
}
|
|
|
|
func benchmarkPlanDurationsCSV(phases []benchmarkPlannedPhase) string {
|
|
values := make([]string, 0, len(phases))
|
|
for _, phase := range phases {
|
|
values = append(values, strconv.Itoa(phase.DurationSec))
|
|
}
|
|
return strings.Join(values, ",")
|
|
}
|
|
|
|
func benchmarkPlannedPhaseStatus(raw []byte) (string, string) {
|
|
text := strings.ToLower(strings.TrimSpace(string(raw)))
|
|
switch {
|
|
case text == "":
|
|
return "FAILED", "phase produced no output"
|
|
case strings.Contains(text, "phase_error="):
|
|
if strings.Contains(text, "unsupported") || strings.Contains(text, "not supported") || strings.Contains(text, "cublaslt_profiles=unsupported") {
|
|
return "UNSUPPORTED", "precision phase unsupported on this GPU/userspace path"
|
|
}
|
|
return "FAILED", "precision phase failed"
|
|
case strings.Contains(text, "status=failed"):
|
|
if strings.Contains(text, "unsupported") || strings.Contains(text, "not supported") {
|
|
return "UNSUPPORTED", "precision phase unsupported on this GPU/userspace path"
|
|
}
|
|
return "FAILED", "precision phase failed"
|
|
default:
|
|
return "OK", ""
|
|
}
|
|
}
|
|
|
|
func benchmarkCalibrationThrottleReason(before, after BenchmarkThrottleCounters) string {
|
|
diff := diffThrottleCounters(before, after)
|
|
switch {
|
|
case diff.HWThermalSlowdownUS > 0:
|
|
return "hw_thermal"
|
|
case diff.SWThermalSlowdownUS > 0:
|
|
return "sw_thermal"
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func setBenchmarkPowerLimit(ctx context.Context, verboseLog string, gpuIndex, powerLimitW int) error {
|
|
if powerLimitW <= 0 {
|
|
return fmt.Errorf("invalid power limit %d", powerLimitW)
|
|
}
|
|
out, err := runSATCommandCtx(ctx, verboseLog, fmt.Sprintf("gpu-%d-set-power-limit-%dw", gpuIndex, powerLimitW), []string{
|
|
"nvidia-smi", "-i", strconv.Itoa(gpuIndex), "-pl", strconv.Itoa(powerLimitW),
|
|
}, nil, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("set power limit gpu=%d limit=%dw: %w (%s)", gpuIndex, powerLimitW, err, strings.TrimSpace(string(out)))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func resetBenchmarkGPU(ctx context.Context, verboseLog string, gpuIndex int, logFunc func(string)) error {
|
|
if logFunc != nil {
|
|
logFunc(fmt.Sprintf("power benchmark pre-flight: GPU %d reset via shared NVIDIA recover path", gpuIndex))
|
|
}
|
|
out, err := benchmarkResetNvidiaGPU(gpuIndex)
|
|
appendSATVerboseLog(verboseLog,
|
|
fmt.Sprintf("[%s] start power-preflight-gpu-%d-reset.log", time.Now().UTC().Format(time.RFC3339), gpuIndex),
|
|
"cmd: bee-nvidia-recover reset-gpu "+strconv.Itoa(gpuIndex),
|
|
)
|
|
if trimmed := strings.TrimSpace(out); trimmed != "" && logFunc != nil {
|
|
for _, line := range strings.Split(trimmed, "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line != "" {
|
|
logFunc(line)
|
|
}
|
|
}
|
|
}
|
|
rc := 0
|
|
if err != nil {
|
|
rc = 1
|
|
}
|
|
appendSATVerboseLog(verboseLog,
|
|
fmt.Sprintf("[%s] finish power-preflight-gpu-%d-reset.log", time.Now().UTC().Format(time.RFC3339), gpuIndex),
|
|
fmt.Sprintf("rc: %d", rc),
|
|
"",
|
|
)
|
|
return err
|
|
}
|
|
|
|
func resetBenchmarkGPUs(ctx context.Context, verboseLog string, gpuIndices []int, logFunc func(string)) []int {
|
|
if len(gpuIndices) == 0 {
|
|
return nil
|
|
}
|
|
if benchmarkGeteuid() != 0 {
|
|
if logFunc != nil {
|
|
logFunc("power benchmark pre-flight: root privileges unavailable, GPU reset skipped")
|
|
}
|
|
return append([]int(nil), gpuIndices...)
|
|
}
|
|
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
|
|
for _, p := range killed {
|
|
logFunc(fmt.Sprintf("power benchmark pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
|
|
}
|
|
}
|
|
var failed []int
|
|
for _, idx := range gpuIndices {
|
|
if err := resetBenchmarkGPU(ctx, verboseLog, idx, logFunc); err != nil {
|
|
failed = append(failed, idx)
|
|
if logFunc != nil {
|
|
logFunc(fmt.Sprintf("power benchmark pre-flight: GPU %d reset failed: %v", idx, err))
|
|
}
|
|
continue
|
|
}
|
|
if logFunc != nil {
|
|
logFunc(fmt.Sprintf("power benchmark pre-flight: GPU %d reset completed", idx))
|
|
}
|
|
benchmarkSleep(time.Second)
|
|
}
|
|
return failed
|
|
}
|
|
|
|
func benchmarkPowerEngine() string {
|
|
switch strings.TrimSpace(strings.ToLower(os.Getenv("BEE_BENCH_POWER_ENGINE"))) {
|
|
case BenchmarkPowerEngineTargetedPower:
|
|
return BenchmarkPowerEngineTargetedPower
|
|
default:
|
|
return BenchmarkPowerEngineDCGMProfTester
|
|
}
|
|
}
|
|
|
|
func benchmarkPowerEngineLabel(engine string) string {
|
|
switch strings.TrimSpace(strings.ToLower(engine)) {
|
|
case BenchmarkPowerEngineTargetedPower:
|
|
return "dcgmi diag targeted_power"
|
|
default:
|
|
return "dcgmproftester"
|
|
}
|
|
}
|
|
|
|
func resolveBenchmarkPowerLoadCommand(durationSec int, gpuIndices []int) ([]string, []string, error) {
|
|
engine := benchmarkPowerEngine()
|
|
durationSec = normalizeNvidiaBurnDuration(durationSec)
|
|
switch engine {
|
|
case BenchmarkPowerEngineTargetedPower:
|
|
return nvidiaDCGMNamedDiagCommand("targeted_power", durationSec, gpuIndices), nil, nil
|
|
default:
|
|
if len(gpuIndices) > 1 {
|
|
return []string{
|
|
"bee-dcgmproftester-staggered",
|
|
"--seconds", strconv.Itoa(durationSec),
|
|
"--stagger-seconds", "0",
|
|
"--devices", joinIndexList(gpuIndices),
|
|
}, nil, nil
|
|
}
|
|
cmd, err := resolveDCGMProfTesterCommand("--no-dcgm-validation", "-t", "1004", "-d", strconv.Itoa(durationSec))
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return cmd, nvidiaVisibleDevicesEnv(gpuIndices), nil
|
|
}
|
|
}
|
|
|
|
func (s *System) RunNvidiaBenchmark(ctx context.Context, baseDir string, opts NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
|
|
if ctx == nil {
|
|
ctx = context.Background()
|
|
}
|
|
if logFunc == nil {
|
|
logFunc = func(string) {}
|
|
}
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = "/var/log/bee-bench/perf"
|
|
}
|
|
spec := resolveBenchmarkProfile(opts.Profile)
|
|
opts = normalizeNvidiaBenchmarkOptionsForBenchmark(opts)
|
|
|
|
selected, err := resolveNvidiaGPUSelection(opts.GPUIndices, opts.ExcludeGPUIndices)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(selected) == 0 {
|
|
return "", fmt.Errorf("no NVIDIA GPUs selected")
|
|
}
|
|
|
|
ts := time.Now().UTC().Format("20060102-150405")
|
|
runDir := filepath.Join(baseDir, "perf-"+ts)
|
|
if err := os.MkdirAll(runDir, 0755); err != nil {
|
|
return "", fmt.Errorf("mkdir %s: %w", runDir, err)
|
|
}
|
|
verboseLog := filepath.Join(runDir, "verbose.log")
|
|
|
|
hostname, _ := os.Hostname()
|
|
result := NvidiaBenchmarkResult{
|
|
BenchmarkVersion: benchmarkVersion,
|
|
GeneratedAt: time.Now().UTC(),
|
|
Hostname: hostname,
|
|
ServerModel: readServerModel(),
|
|
BenchmarkProfile: spec.Name,
|
|
ParallelGPUs: opts.ParallelGPUs,
|
|
RampStep: opts.RampStep,
|
|
RampTotal: opts.RampTotal,
|
|
RampRunID: opts.RampRunID,
|
|
SelectedGPUIndices: append([]int(nil), selected...),
|
|
HostConfig: readBenchmarkHostConfig(),
|
|
Normalization: BenchmarkNormalization{
|
|
Status: "full",
|
|
},
|
|
}
|
|
|
|
logFunc(fmt.Sprintf("NVIDIA benchmark profile=%s gpus=%s", spec.Name, joinIndexList(selected)))
|
|
psuBefore := psuStatusSnapshot()
|
|
var metricRows []GPUMetricRow
|
|
metricTimelineSec := 0.0
|
|
gpuBurnLog := filepath.Join(runDir, "gpu-burn.log")
|
|
|
|
// Server power characterization state — populated during per-GPU phases.
|
|
var serverIdleW, serverLoadedWSum float64
|
|
var serverIdleOK, serverLoadedOK bool
|
|
var serverLoadedSamples int
|
|
|
|
// Run nvidia-smi -q first: used both for the log file and as a fallback
|
|
// source of max clock values when CSV clock fields are unsupported.
|
|
var nvsmiQOut []byte
|
|
if out, err := runSATCommandCtx(ctx, verboseLog, "00-nvidia-smi-q.log", []string{"nvidia-smi", "-q"}, nil, nil); err == nil {
|
|
nvsmiQOut = out
|
|
_ = os.WriteFile(filepath.Join(runDir, "00-nvidia-smi-q.log"), out, 0644)
|
|
}
|
|
|
|
infoByIndex, infoErr := queryBenchmarkGPUInfo(selected)
|
|
if infoErr != nil {
|
|
result.Warnings = append(result.Warnings, "gpu inventory query failed: "+infoErr.Error())
|
|
result.Normalization.Status = "partial"
|
|
}
|
|
// Enrich with verbose nvidia-smi data — covers GPUs where some CSV fields
|
|
// are unsupported (e.g. clocks.max.* on Blackwell / driver 98.x).
|
|
enrichGPUInfoWithNvidiaSMIQ(infoByIndex, nvsmiQOut)
|
|
|
|
activeApps, err := queryActiveComputeApps(selected)
|
|
if err == nil && len(activeApps) > 0 {
|
|
result.Warnings = append(result.Warnings, "active GPU compute processes detected before benchmark")
|
|
result.Normalization.Notes = append(result.Normalization.Notes, activeApps...)
|
|
result.Normalization.Status = "partial"
|
|
}
|
|
|
|
restoreActions := applyBenchmarkNormalization(ctx, verboseLog, selected, infoByIndex, &result)
|
|
defer func() {
|
|
for i := len(restoreActions) - 1; i >= 0; i-- {
|
|
restoreActions[i].fn()
|
|
}
|
|
}()
|
|
|
|
// No power calibration before performance benchmark — GPUs run at their
|
|
// default power limits. PowerSustainScore is derived from steady-state power
|
|
// observed during the benchmark itself.
|
|
calibByIndex := make(map[int]benchmarkPowerCalibrationResult)
|
|
|
|
// Start background CPU load sampler — samples every 10s during GPU phases.
|
|
cpuStopCh := make(chan struct{})
|
|
cpuSamplesCh := startCPULoadSampler(cpuStopCh, 10)
|
|
|
|
if opts.ParallelGPUs {
|
|
runNvidiaBenchmarkParallel(ctx, verboseLog, runDir, selected, infoByIndex, opts, spec, logFunc, &result, calibByIndex, &serverIdleW, &serverLoadedWSum, &serverIdleOK, &serverLoadedOK, &serverLoadedSamples, &metricRows, &metricTimelineSec, gpuBurnLog)
|
|
} else {
|
|
|
|
for _, idx := range selected {
|
|
gpuResult := BenchmarkGPUResult{
|
|
Index: idx,
|
|
Status: "FAILED",
|
|
}
|
|
if info, ok := infoByIndex[idx]; ok {
|
|
populateBenchmarkGPUInfo(&gpuResult, info)
|
|
}
|
|
if calib, ok := calibByIndex[idx]; ok {
|
|
gpuResult.CalibratedPeakPowerW = calib.Summary.P95PowerW
|
|
gpuResult.CalibratedPeakTempC = calib.Summary.P95TempC
|
|
gpuResult.PowerCalibrationTries = calib.Attempts
|
|
gpuResult.PowerLimitDerated = calib.Derated
|
|
gpuResult.Notes = append(gpuResult.Notes, calib.Notes...)
|
|
if calib.CoolingWarning != "" {
|
|
gpuResult.CoolingWarning = calib.CoolingWarning
|
|
}
|
|
}
|
|
if norm := findBenchmarkNormalization(result.Normalization.GPUs, idx); norm != nil {
|
|
gpuResult.LockedGraphicsClockMHz = norm.GPUClockLockMHz
|
|
gpuResult.LockedMemoryClockMHz = norm.MemoryClockLockMHz
|
|
}
|
|
|
|
baselineRows, err := collectBenchmarkSamples(ctx, spec.BaselineSec, []int{idx})
|
|
if err != nil && err != context.Canceled {
|
|
gpuResult.Notes = append(gpuResult.Notes, "baseline sampling failed: "+err.Error())
|
|
}
|
|
gpuResult.Baseline = summarizeBenchmarkTelemetry(baselineRows)
|
|
appendBenchmarkMetrics(&metricRows, baselineRows, fmt.Sprintf("gpu-%d-baseline", idx), &metricTimelineSec, float64(spec.BaselineSec))
|
|
|
|
// Sample server idle power once (first GPU only — server state is global).
|
|
if !serverIdleOK {
|
|
if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, maxInt(spec.BaselineSec, 10), benchmarkPowerAutotuneSampleInterval); ok {
|
|
serverIdleW = w
|
|
serverIdleOK = true
|
|
logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w))
|
|
}
|
|
}
|
|
|
|
warmupCmd := []string{
|
|
"bee-gpu-burn",
|
|
"--seconds", strconv.Itoa(spec.WarmupSec),
|
|
"--size-mb", strconv.Itoa(opts.SizeMB),
|
|
"--devices", strconv.Itoa(idx),
|
|
}
|
|
logFunc(fmt.Sprintf("GPU %d: warmup (%ds)", idx, spec.WarmupSec))
|
|
warmupOut, warmupRows, warmupErr := runBenchmarkCommandWithMetrics(ctx, verboseLog, fmt.Sprintf("gpu-%d-warmup.log", idx), warmupCmd, nil, []int{idx}, logFunc)
|
|
appendBenchmarkMetrics(&metricRows, warmupRows, fmt.Sprintf("gpu-%d-warmup", idx), &metricTimelineSec, float64(spec.WarmupSec))
|
|
appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", fmt.Sprintf("gpu-%d-warmup", idx), warmupOut)
|
|
if warmupErr != nil {
|
|
gpuResult.Notes = append(gpuResult.Notes, "warmup failed: "+warmupErr.Error())
|
|
result.GPUs = append(result.GPUs, finalizeBenchmarkGPUResult(gpuResult))
|
|
continue
|
|
}
|
|
warmupParse := parseBenchmarkBurnLog(string(warmupOut))
|
|
if gpuResult.ComputeCapability == "" {
|
|
gpuResult.ComputeCapability = warmupParse.ComputeCapability
|
|
}
|
|
|
|
// Run synthetic precision phases and the combined steady phase as one
|
|
// uninterrupted command so the GPU stays hot between windows.
|
|
eccBase, _ := queryECCCounters(idx)
|
|
supportedPrecisions := benchmarkSupportedPrecisions(gpuResult.ComputeCapability)
|
|
planLabels, planPhases, basePhaseSec, mixedPhaseSec := buildBenchmarkSteadyPlan(spec, supportedPrecisions, func(label string) string {
|
|
if label == "mixed" {
|
|
return fmt.Sprintf("gpu-%d-steady", idx)
|
|
}
|
|
return fmt.Sprintf("gpu-%d-steady-%s", idx, label)
|
|
})
|
|
planCmd := []string{
|
|
"bee-gpu-burn",
|
|
"--seconds", strconv.Itoa(basePhaseSec),
|
|
"--size-mb", strconv.Itoa(opts.SizeMB),
|
|
"--devices", strconv.Itoa(idx),
|
|
"--precision-plan", strings.Join(planLabels, ","),
|
|
"--precision-plan-seconds", benchmarkPlanDurationsCSV(planPhases),
|
|
}
|
|
logFunc(fmt.Sprintf("GPU %d: uninterrupted precision plan (%d precision phases x %ds, mixed %ds)", idx, len(supportedPrecisions), basePhaseSec, mixedPhaseSec))
|
|
serverPowerStopCh := make(chan struct{})
|
|
serverPowerCh := startSelectedPowerSourceSampler(serverPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
|
|
_, phaseRowsByStage, phaseLogs, planErr := runBenchmarkPlannedCommandWithMetrics(ctx, verboseLog, fmt.Sprintf("gpu-%d-precision-plan.log", idx), planCmd, nil, []int{idx}, planPhases, logFunc)
|
|
close(serverPowerStopCh)
|
|
if serverPowerSamples := <-serverPowerCh; len(serverPowerSamples) > 0 {
|
|
serverLoadedWSum += benchmarkMean(serverPowerSamples)
|
|
serverLoadedSamples++
|
|
serverLoadedOK = true
|
|
logFunc(fmt.Sprintf("GPU %d: server loaded power (%s avg): %.0f W", idx, opts.ServerPowerSource, benchmarkMean(serverPowerSamples)))
|
|
}
|
|
for _, phaseSpec := range planPhases {
|
|
if rows := phaseRowsByStage[phaseSpec.MetricStage]; len(rows) > 0 {
|
|
appendBenchmarkMetrics(&metricRows, rows, phaseSpec.MetricStage, &metricTimelineSec, float64(phaseSpec.DurationSec))
|
|
}
|
|
appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", phaseSpec.MetricStage, phaseLogs[phaseSpec.PlanLabel])
|
|
}
|
|
for _, prec := range supportedPrecisions {
|
|
stageName := fmt.Sprintf("gpu-%d-steady-%s", idx, prec)
|
|
phaseRows := phaseRowsByStage[stageName]
|
|
phase := BenchmarkPrecisionSteadyPhase{
|
|
Precision: prec,
|
|
Status: "OK",
|
|
Steady: summarizeBenchmarkTelemetry(phaseRows),
|
|
}
|
|
if status, note := benchmarkPlannedPhaseStatus(phaseLogs[prec]); status != "OK" {
|
|
phase.Status = status
|
|
phase.Notes = note
|
|
gpuResult.PrecisionFailures = append(gpuResult.PrecisionFailures, prec+":"+status)
|
|
}
|
|
for _, p := range parseBenchmarkBurnLog(string(phaseLogs[prec])).Profiles {
|
|
if p.Supported {
|
|
phase.TeraOpsPerSec += p.TeraOpsPerSec
|
|
phase.WeightedTeraOpsPerSec += p.WeightedTeraOpsPerSec
|
|
}
|
|
}
|
|
gpuResult.PrecisionSteady = append(gpuResult.PrecisionSteady, phase)
|
|
}
|
|
|
|
beforeThrottle, _ := queryThrottleCounters(idx)
|
|
logFunc(fmt.Sprintf("GPU %d: steady compute (combined, %ds)", idx, mixedPhaseSec))
|
|
afterThrottle, _ := queryThrottleCounters(idx)
|
|
if planErr != nil {
|
|
gpuResult.Notes = append(gpuResult.Notes, "precision plan failed: "+planErr.Error())
|
|
}
|
|
|
|
steadyRows := phaseRowsByStage[fmt.Sprintf("gpu-%d-steady", idx)]
|
|
parseResult := parseBenchmarkBurnLog(string(phaseLogs["mixed"]))
|
|
gpuResult.ComputeCapability = parseResult.ComputeCapability
|
|
gpuResult.Backend = parseResult.Backend
|
|
gpuResult.PrecisionResults = parseResult.Profiles
|
|
if parseResult.Fallback {
|
|
gpuResult.Notes = append(gpuResult.Notes, "benchmark used driver PTX fallback; tensor throughput score is not comparable")
|
|
}
|
|
|
|
gpuResult.Steady = summarizeBenchmarkTelemetry(steadyRows)
|
|
gpuResult.Throttle = diffThrottleCounters(beforeThrottle, afterThrottle)
|
|
if eccFinal, err := queryECCCounters(idx); err == nil {
|
|
gpuResult.ECC = diffECCCounters(eccBase, eccFinal)
|
|
}
|
|
|
|
if spec.CooldownSec > 0 {
|
|
cooldownRows, err := collectBenchmarkSamples(ctx, spec.CooldownSec, []int{idx})
|
|
if err != nil && err != context.Canceled {
|
|
gpuResult.Notes = append(gpuResult.Notes, "cooldown sampling failed: "+err.Error())
|
|
}
|
|
gpuResult.Cooldown = summarizeBenchmarkTelemetry(cooldownRows)
|
|
appendBenchmarkMetrics(&metricRows, cooldownRows, fmt.Sprintf("gpu-%d-cooldown", idx), &metricTimelineSec, float64(spec.CooldownSec))
|
|
}
|
|
|
|
applyBenchmarkSteadyFallback(&gpuResult)
|
|
gpuResult.Scores = scoreBenchmarkGPUResult(gpuResult)
|
|
gpuResult.DegradationReasons = detectBenchmarkDegradationReasons(gpuResult, result.Normalization.Status)
|
|
if anomaly := detectPowerAnomaly(metricRows, idx); anomaly != "" {
|
|
gpuResult.Notes = append(gpuResult.Notes,
|
|
fmt.Sprintf("[HARD STOP] GPU %d: %s", idx, anomaly))
|
|
}
|
|
if warn := detectSlowdownTempExceedance(metricRows, idx, gpuResult.SlowdownTempC); warn != "" {
|
|
gpuResult.Notes = append(gpuResult.Notes,
|
|
fmt.Sprintf("[WARNING] GPU %d: %s", idx, warn))
|
|
if gpuResult.Status == "OK" {
|
|
gpuResult.Status = "PARTIAL"
|
|
}
|
|
}
|
|
if planErr != nil {
|
|
gpuResult.Status = classifySATErrorStatus(phaseLogs["mixed"], planErr)
|
|
} else if len(gpuResult.PrecisionFailures) > 0 {
|
|
gpuResult.Status = "PARTIAL"
|
|
} else if parseResult.Fallback {
|
|
gpuResult.Status = "PARTIAL"
|
|
} else {
|
|
gpuResult.Status = "OK"
|
|
}
|
|
|
|
result.GPUs = append(result.GPUs, finalizeBenchmarkGPUResult(gpuResult))
|
|
}
|
|
|
|
} // end sequential path
|
|
|
|
// Performance scalability ramp-up: run parallel benchmarks for k=2..N GPUs
|
|
// and compute compute scalability relative to the best single-GPU result.
|
|
// Only runs in sequential mode (each GPU was tested individually above) and
|
|
// when there are at least 2 GPUs.
|
|
if !opts.ParallelGPUs && len(selected) >= 2 {
|
|
// Find the best single-card SyntheticScore as the 1-GPU baseline.
|
|
var bestTOPS float64
|
|
for _, g := range result.GPUs {
|
|
if g.Scores.SyntheticScore > bestTOPS {
|
|
bestTOPS = g.Scores.SyntheticScore
|
|
}
|
|
}
|
|
if bestTOPS > 0 {
|
|
var rampSteps []NvidiaPerformanceRampStep
|
|
var scalabilityPcts []float64
|
|
for k := 2; k <= len(selected); k++ {
|
|
subset := append([]int(nil), selected[:k]...)
|
|
rampDir := filepath.Join(runDir, fmt.Sprintf("ramp-%02d", k))
|
|
_ = os.MkdirAll(rampDir, 0755)
|
|
logFunc(fmt.Sprintf("performance ramp: step %d/%d — running %d GPUs in parallel", k, len(selected), k))
|
|
|
|
var rampResult NvidiaBenchmarkResult
|
|
var rampIdleW, rampLoadedWSum float64
|
|
var rampIdleOK, rampLoadedOK bool
|
|
var rampLoadedSamples int
|
|
var rampMetricRows []GPUMetricRow
|
|
var rampTimelineSec float64
|
|
emptyCalib := make(map[int]benchmarkPowerCalibrationResult)
|
|
|
|
runNvidiaBenchmarkParallel(ctx, verboseLog, rampDir, subset, infoByIndex, opts, spec, logFunc,
|
|
&rampResult, emptyCalib,
|
|
&rampIdleW, &rampLoadedWSum, &rampIdleOK, &rampLoadedOK, &rampLoadedSamples,
|
|
&rampMetricRows, &rampTimelineSec, "")
|
|
|
|
var totalSynth, totalMixed float64
|
|
for _, g := range rampResult.GPUs {
|
|
totalSynth += g.Scores.SyntheticScore
|
|
totalMixed += g.Scores.MixedScore
|
|
}
|
|
scalPct := totalSynth / (float64(k) * bestTOPS) * 100
|
|
scalabilityPcts = append(scalabilityPcts, scalPct)
|
|
|
|
stepStatus := "OK"
|
|
if len(rampResult.GPUs) < k {
|
|
stepStatus = "PARTIAL"
|
|
}
|
|
rampSteps = append(rampSteps, NvidiaPerformanceRampStep{
|
|
StepIndex: k,
|
|
GPUIndices: subset,
|
|
TotalSyntheticTOPS: totalSynth,
|
|
TotalMixedTOPS: totalMixed,
|
|
ScalabilityPct: scalPct,
|
|
Status: stepStatus,
|
|
})
|
|
}
|
|
result.PerformanceRampSteps = rampSteps
|
|
result.PlatformPowerScore = benchmarkMean(scalabilityPcts)
|
|
if len(scalabilityPcts) > 0 {
|
|
result.ScalabilityScore = scalabilityPcts[len(scalabilityPcts)-1]
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(selected) > 1 && opts.RunNCCL {
|
|
result.Interconnect = runBenchmarkInterconnect(ctx, verboseLog, runDir, selected, spec, logFunc)
|
|
if result.Interconnect != nil && result.Interconnect.Supported {
|
|
for i := range result.GPUs {
|
|
result.GPUs[i].Scores.InterconnectScore = result.Interconnect.MaxBusBWGBps
|
|
}
|
|
}
|
|
}
|
|
|
|
// Stop CPU load sampler and attach results.
|
|
close(cpuStopCh)
|
|
if cpuSamples := <-cpuSamplesCh; len(cpuSamples) > 0 {
|
|
result.CPULoad = summarizeCPULoad(cpuSamples)
|
|
if result.CPULoad != nil && result.CPULoad.Status != "ok" {
|
|
logFunc(fmt.Sprintf("host CPU load during benchmark: avg=%.1f%% max=%.1f%% status=%s",
|
|
result.CPULoad.AvgPct, result.CPULoad.MaxPct, result.CPULoad.Status))
|
|
}
|
|
}
|
|
|
|
// Compute server power characterization from accumulated IPMI samples.
|
|
var gpuReportedSumW float64
|
|
for _, gpu := range result.GPUs {
|
|
gpuReportedSumW += gpu.Steady.AvgPowerW
|
|
}
|
|
var serverLoadedW float64
|
|
if serverLoadedSamples > 0 {
|
|
serverLoadedW = serverLoadedWSum / float64(serverLoadedSamples)
|
|
}
|
|
result.ServerPower = characterizeServerPower(serverIdleW, serverLoadedW, gpuReportedSumW, opts.ServerPowerSource, serverIdleOK && serverLoadedOK)
|
|
result.Cooling = summarizeBenchmarkCooling(metricRows)
|
|
|
|
// Apply server-power penalty when IPMI reports the server delta is much
|
|
// lower than GPU-reported sum: GPU power telemetry is over-stated, making
|
|
// CalibratedPeakPowerW and PowerSustainScore unreliable.
|
|
// Penalty factor scales from 1.0 (ratio ≥ 0.75, no penalty) down to 0.
|
|
if sp := result.ServerPower; sp != nil && sp.Available && sp.ReportingRatio > 0 && sp.ReportingRatio < 0.75 {
|
|
factor := sp.ReportingRatio / 0.75
|
|
for i := range result.GPUs {
|
|
result.GPUs[i].Scores.CompositeScore *= factor
|
|
result.GPUs[i].Notes = append(result.GPUs[i].Notes,
|
|
fmt.Sprintf("server-power penalty applied (reporting_ratio=%.2f < 0.75): composite score reduced to %.1f%%",
|
|
sp.ReportingRatio, factor*100))
|
|
}
|
|
}
|
|
|
|
result.Findings = buildBenchmarkFindings(result)
|
|
result.OverallStatus = benchmarkOverallStatus(result)
|
|
result.PSUIssues = diffPSUStatus(psuBefore, psuStatusSnapshot())
|
|
writeBenchmarkMetricsFiles(runDir, metricRows)
|
|
|
|
resultJSON, err := json.MarshalIndent(result, "", " ")
|
|
if err != nil {
|
|
return "", fmt.Errorf("marshal benchmark result: %w", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(runDir, "result.json"), resultJSON, 0644); err != nil {
|
|
return "", fmt.Errorf("write result.json: %w", err)
|
|
}
|
|
|
|
report := renderBenchmarkReportWithCharts(result)
|
|
if err := os.WriteFile(filepath.Join(runDir, "report.md"), []byte(report), 0644); err != nil {
|
|
return "", fmt.Errorf("write report.md: %w", err)
|
|
}
|
|
|
|
summary := renderBenchmarkSummary(result)
|
|
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
|
return "", fmt.Errorf("write summary.txt: %w", err)
|
|
}
|
|
|
|
return runDir, nil
|
|
}
|