package platform import ( "context" "encoding/json" "fmt" "math" "os" "path/filepath" "sort" "strings" "time" ) func (s *System) RunNvidiaPowerBench(ctx context.Context, baseDir string, opts NvidiaBenchmarkOptions, logFunc func(string)) (string, error) { if ctx == nil { ctx = context.Background() } if logFunc == nil { logFunc = func(string) {} } if strings.TrimSpace(baseDir) == "" { baseDir = "/var/log/bee-bench/power" } opts = normalizeNvidiaBenchmarkOptionsForBenchmark(opts) selected, err := resolveNvidiaGPUSelection(opts.GPUIndices, opts.ExcludeGPUIndices) if err != nil { return "", err } if len(selected) == 0 { return "", fmt.Errorf("no NVIDIA GPUs selected") } ts := time.Now().UTC().Format("20060102-150405") runDir := filepath.Join(baseDir, "power-"+ts) if err := os.MkdirAll(runDir, 0755); err != nil { return "", fmt.Errorf("mkdir %s: %w", runDir, err) } verboseLog := filepath.Join(runDir, "verbose.log") hostname, _ := os.Hostname() result := NvidiaPowerBenchResult{ BenchmarkVersion: benchmarkVersion, GeneratedAt: time.Now().UTC(), Hostname: hostname, ServerModel: readServerModel(), BenchmarkProfile: opts.Profile, SelectedGPUIndices: append([]int(nil), selected...), OverallStatus: "OK", } infoByIndex, infoErr := queryBenchmarkGPUInfo(selected) if infoErr != nil { return "", infoErr } // Capture full nvidia-smi -q snapshot at the start of the run. if out, err := runSATCommandCtx(ctx, verboseLog, "00-nvidia-smi-q.log", []string{"nvidia-smi", "-q"}, nil, nil); err == nil { _ = os.WriteFile(filepath.Join(runDir, "00-nvidia-smi-q.log"), out, 0644) } durationSec := powerBenchDurationSec(opts.Profile) // Sample server idle power before any GPU load. var serverIdleW float64 var serverIdleOK bool idleSDRStopCh := make(chan struct{}) idleSDRCh := startIPMISDRSampler(idleSDRStopCh, benchmarkPowerAutotuneSampleInterval) if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, 10, benchmarkPowerAutotuneSampleInterval); ok { serverIdleW = w serverIdleOK = true logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w)) } close(idleSDRStopCh) sdrIdle := summarizeSDRPowerSeries(<-idleSDRCh) psuBefore := psuStatusSnapshot() // Phase 1: calibrate each GPU individually (sequentially, one at a time) to // establish a true single-card power baseline unaffected by neighbour heat. calibByIndex := make(map[int]benchmarkPowerCalibrationResult, len(selected)) singleIPMILoadedW := make(map[int]float64, len(selected)) singleRunSummaryByIndex := make(map[int]benchmarkPowerCalibrationRunSummary, len(selected)) var allRestoreActions []benchmarkRestoreAction // allPowerRows accumulates telemetry from all phases for the top-level gpu-metrics.csv. var allPowerRows []GPUMetricRow var powerCursor float64 for _, idx := range selected { singleDir := filepath.Join(runDir, fmt.Sprintf("single-%02d", idx)) _ = os.MkdirAll(singleDir, 0755) singleInfo := cloneBenchmarkGPUInfoMap(infoByIndex) if failed := resetBenchmarkGPUs(ctx, verboseLog, []int{idx}, logFunc); len(failed) > 0 { return "", fmt.Errorf("power benchmark pre-flight: failed to reset GPU %d; benchmark aborted to keep measurements clean", idx) } logFunc(fmt.Sprintf("power calibration: GPU %d single-card baseline", idx)) singlePowerStopCh := make(chan struct{}) singlePowerCh := startSelectedPowerSourceSampler(singlePowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval) c, restore, singleRows, singleRun := runBenchmarkPowerCalibration(ctx, verboseLog, singleDir, []int{idx}, singleInfo, logFunc, nil, durationSec) appendBenchmarkMetrics(&allPowerRows, singleRows, fmt.Sprintf("single-gpu-%d", idx), &powerCursor, 0) close(singlePowerStopCh) if samples := <-singlePowerCh; len(samples) > 0 { singleIPMILoadedW[idx] = benchmarkMean(samples) logFunc(fmt.Sprintf("power calibration: GPU %d single-card server power (%s avg): %.0f W", idx, opts.ServerPowerSource, singleIPMILoadedW[idx])) } else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && singleRun.LoadedSDR.PSUInW > 0 { singleIPMILoadedW[idx] = singleRun.LoadedSDR.PSUInW logFunc(fmt.Sprintf("power calibration: GPU %d single-card fallback server power (SDR avg): %.0f W", idx, singleRun.LoadedSDR.PSUInW)) } allRestoreActions = append(allRestoreActions, restore...) if r, ok := c[idx]; ok { calibByIndex[idx] = r } singleRunSummaryByIndex[idx] = singleRun } defer func() { for i := len(allRestoreActions) - 1; i >= 0; i-- { allRestoreActions[i].fn() } }() gpus := make([]NvidiaPowerBenchGPU, 0, len(selected)) for _, idx := range selected { info := infoByIndex[idx] calib := calibByIndex[idx] status := "OK" if !calib.Completed { status = "FAILED" result.OverallStatus = "PARTIAL" } else if calib.Derated { status = "PARTIAL" if result.OverallStatus == "OK" { result.OverallStatus = "PARTIAL" } } gpu := NvidiaPowerBenchGPU{ Index: idx, Name: info.Name, BusID: info.BusID, DefaultPowerLimitW: info.DefaultPowerLimitW, AppliedPowerLimitW: calib.AppliedPowerLimitW, MaxObservedPowerW: calib.Summary.P95PowerW, MaxObservedTempC: calib.Summary.P95TempC, CalibrationAttempts: calib.Attempts, Derated: calib.Derated, Status: status, Notes: append([]string(nil), calib.Notes...), CoolingWarning: calib.CoolingWarning, } if w, ok := singleIPMILoadedW[idx]; ok && serverIdleOK && w > 0 { gpu.ServerLoadedW = w gpu.ServerDeltaW = w - serverIdleW } if len(calib.MetricRows) > 0 { t := summarizeBenchmarkTelemetry(calib.MetricRows) gpu.Telemetry = &t } if singleRun := singleRunSummaryByIndex[idx]; singleRun.AvgFanRPM > 0 { gpu.AvgFanRPM = singleRun.AvgFanRPM gpu.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct } gpus = append(gpus, gpu) } sort.Slice(gpus, func(i, j int) bool { if gpus[i].MaxObservedPowerW != gpus[j].MaxObservedPowerW { return gpus[i].MaxObservedPowerW > gpus[j].MaxObservedPowerW } if gpus[i].AppliedPowerLimitW != gpus[j].AppliedPowerLimitW { return gpus[i].AppliedPowerLimitW > gpus[j].AppliedPowerLimitW } if gpus[i].Derated != gpus[j].Derated { return !gpus[i].Derated } return gpus[i].Index < gpus[j].Index }) result.GPUs = gpus result.RecommendedSlotOrder = make([]int, 0, len(gpus)) for _, gpu := range gpus { result.RecommendedSlotOrder = append(result.RecommendedSlotOrder, gpu.Index) } if len(result.RecommendedSlotOrder) > 0 { result.Findings = append(result.Findings, fmt.Sprintf("Recommended slot order for installation based on single-card %s: %s.", benchmarkPowerEngineLabel(benchmarkPowerEngine()), joinIndexList(result.RecommendedSlotOrder))) } for _, gpu := range gpus { if gpu.Derated { result.Findings = append(result.Findings, fmt.Sprintf("GPU %d required reduced power limit %.0f W to complete %s.", gpu.Index, gpu.AppliedPowerLimitW, benchmarkPowerEngineLabel(benchmarkPowerEngine()))) } if gpu.CoolingWarning != "" { result.Findings = append(result.Findings, fmt.Sprintf( "GPU %d: %s. Operator action: rerun the benchmark with fan speed manually fixed at 100%% to confirm actual thermal headroom.", gpu.Index, gpu.CoolingWarning, )) } } singleByIndex := make(map[int]NvidiaPowerBenchGPU, len(gpus)) for _, gpu := range gpus { singleByIndex[gpu.Index] = gpu } // Phase 2: cumulative thermal ramp. // Each step introduces one new GPU into an environment where all previously // calibrated GPUs are already running at their fixed stable limits. The new // GPU's stable TDP is searched via binary search under real // multi-GPU thermal load. Once found, its limit is fixed permanently for all // subsequent steps. This ensures each GPU's limit reflects actual sustained // power in the final full-system thermal state. // // stableLimits accumulates GPU index → fixed stable limit (W) across steps. stableLimits := make(map[int]int, len(result.RecommendedSlotOrder)) // serverLoadedW tracks the IPMI server power from the final ramp step // (all GPUs simultaneously loaded). Earlier steps' values are stored // per-step in NvidiaPowerBenchStep.ServerLoadedW. var serverLoadedW float64 var serverLoadedOK bool // sdrLastStep retains the phase-averaged SDR readings from the last ramp step // while GPUs are loaded. Used in the summary instead of re-sampling after the // test when GPUs have already returned to idle. var sdrLastStep benchmarkSDRSeriesSummary // Step 1: reuse single-card calibration result directly. if len(result.RecommendedSlotOrder) > 0 { firstIdx := result.RecommendedSlotOrder[0] firstCalib := calibByIndex[firstIdx] stableLimits[firstIdx] = int(math.Round(firstCalib.AppliedPowerLimitW)) ramp := NvidiaPowerBenchStep{ StepIndex: 1, GPUIndices: []int{firstIdx}, NewGPUIndex: firstIdx, NewGPUStableLimitW: firstCalib.AppliedPowerLimitW, TotalObservedPowerW: firstCalib.Summary.P95PowerW, AvgObservedPowerW: firstCalib.Summary.P95PowerW, Derated: firstCalib.Derated, Status: "OK", } if w, ok := singleIPMILoadedW[firstIdx]; ok && serverIdleOK && w > 0 { ramp.ServerLoadedW = w ramp.ServerDeltaW = w - serverIdleW } if singleRun := singleRunSummaryByIndex[firstIdx]; singleRun.AvgFanRPM > 0 { ramp.AvgFanRPM = singleRun.AvgFanRPM ramp.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct } firstSummary := firstCalib.Summary ramp.PerGPUTelemetry = map[int]*BenchmarkTelemetrySummary{firstIdx: &firstSummary} if !firstCalib.Completed { ramp.Status = "FAILED" ramp.Notes = append(ramp.Notes, fmt.Sprintf("GPU %d did not complete single-card %s", firstIdx, benchmarkPowerEngineLabel(benchmarkPowerEngine()))) result.OverallStatus = "PARTIAL" } else if firstCalib.Derated { ramp.Status = "PARTIAL" if result.OverallStatus == "OK" { result.OverallStatus = "PARTIAL" } result.Findings = append(result.Findings, fmt.Sprintf("Ramp step 1 (GPU %d) required derating to %.0f W.", firstIdx, firstCalib.AppliedPowerLimitW)) } result.RampSteps = append(result.RampSteps, ramp) logFunc(fmt.Sprintf("power ramp: step 1/%d — reused single-card calibration for GPU %d, stable limit %.0f W", len(result.RecommendedSlotOrder), firstIdx, firstCalib.AppliedPowerLimitW)) } // Steps 2..N: each step revalidates every already-active GPU under the new // cumulative thermal environment and also calibrates the newly introduced // GPU. Previously found limits are used only as seeds for the search. for stepNum := 1; stepNum < len(result.RecommendedSlotOrder); stepNum++ { step := stepNum + 1 subset := append([]int(nil), result.RecommendedSlotOrder[:step]...) newGPUIdx := result.RecommendedSlotOrder[stepNum] stepDir := filepath.Join(runDir, fmt.Sprintf("step-%02d", step)) _ = os.MkdirAll(stepDir, 0755) // Reuse the latest stable limits as starting points, but re-check every // active GPU in this hotter configuration. For the newly introduced GPU, // seed from its single-card calibration so we do not restart from the // default TDP when a prior derated limit is already known. seedForStep := make(map[int]int, len(subset)) for _, idx := range subset { if lim, ok := stableLimits[idx]; ok && lim > 0 { seedForStep[idx] = lim continue } if base, ok := calibByIndex[idx]; ok { lim := int(math.Round(base.AppliedPowerLimitW)) if lim > 0 { seedForStep[idx] = lim } } } logFunc(fmt.Sprintf("power ramp: step %d/%d — revalidating %d active GPU(s) including new GPU %d", step, len(result.RecommendedSlotOrder), len(subset), newGPUIdx)) stepInfo := cloneBenchmarkGPUInfoMap(infoByIndex) stepPowerStopCh := make(chan struct{}) stepPowerCh := startSelectedPowerSourceSampler(stepPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval) stepCalib, stepRestore, stepRows, stepRun := runBenchmarkPowerCalibration(ctx, verboseLog, stepDir, subset, stepInfo, logFunc, seedForStep, durationSec) appendBenchmarkMetrics(&allPowerRows, stepRows, fmt.Sprintf("ramp-step-%d", step), &powerCursor, 0) close(stepPowerStopCh) var stepIPMILoadedW float64 var stepIPMIOK bool if samples := <-stepPowerCh; len(samples) > 0 { stepIPMILoadedW = benchmarkMean(samples) stepIPMIOK = true } // Accumulate restore actions; they all run in the outer defer. allRestoreActions = append(allRestoreActions, stepRestore...) ramp := NvidiaPowerBenchStep{ StepIndex: step, GPUIndices: subset, NewGPUIndex: newGPUIdx, Status: "OK", } // Total observed power = sum of p95 across all GPUs in this step. for _, idx := range subset { if c, ok := stepCalib[idx]; ok { ramp.TotalObservedPowerW += c.Summary.P95PowerW } } if len(subset) > 0 { ramp.AvgObservedPowerW = ramp.TotalObservedPowerW / float64(len(subset)) } for _, idx := range subset { c, ok := stepCalib[idx] if !ok || !c.Completed { fallback := 0 if lim, ok := stableLimits[idx]; ok && lim > 0 { fallback = lim } else if fb, ok := calibByIndex[idx]; ok { fallback = int(math.Round(fb.AppliedPowerLimitW)) } if fallback > 0 { stableLimits[idx] = fallback } ramp.Status = "FAILED" ramp.Notes = append(ramp.Notes, fmt.Sprintf("GPU %d did not complete %s in ramp step %d; keeping previous stable limit %d W", idx, benchmarkPowerEngineLabel(benchmarkPowerEngine()), step, fallback)) result.OverallStatus = "PARTIAL" continue } prevLimit, hadPrev := stableLimits[idx] newLimit := int(math.Round(c.AppliedPowerLimitW)) stableLimits[idx] = newLimit if idx == newGPUIdx { ramp.NewGPUStableLimitW = c.AppliedPowerLimitW ramp.Derated = c.Derated } if c.Derated { ramp.Status = "PARTIAL" if result.OverallStatus == "OK" { result.OverallStatus = "PARTIAL" } } if hadPrev && newLimit < prevLimit { ramp.Notes = append(ramp.Notes, fmt.Sprintf("GPU %d was re-derated from %d W to %d W under combined thermal load.", idx, prevLimit, newLimit)) } } if c, ok := stepCalib[newGPUIdx]; ok && c.Completed && c.Derated { result.Findings = append(result.Findings, fmt.Sprintf("Ramp step %d (GPU %d) required derating to %.0f W under combined thermal load.", step, newGPUIdx, c.AppliedPowerLimitW)) } // Per-step PSU slot readings are averaged over the whole load phase rather // than captured as a single end-of-phase snapshot. sdrStep := stepRun.LoadedSDR if len(sdrStep.PSUSlots) > 0 { ramp.PSUSlotReadings = sdrStep.PSUSlots } if stepIPMIOK && serverIdleOK && stepIPMILoadedW > 0 { ramp.ServerLoadedW = stepIPMILoadedW ramp.ServerDeltaW = stepIPMILoadedW - serverIdleW logFunc(fmt.Sprintf("power ramp: step %d server loaded power (%s avg): %.0f W", step, opts.ServerPowerSource, stepIPMILoadedW)) // The last step has all GPUs loaded — use it as the top-level loaded_w. if step == len(result.RecommendedSlotOrder) { serverLoadedW = stepIPMILoadedW serverLoadedOK = true sdrLastStep = sdrStep } } else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && sdrStep.PSUInW > 0 { ramp.ServerLoadedW = sdrStep.PSUInW ramp.ServerDeltaW = sdrStep.PSUInW - sdrIdle.PSUInW logFunc(fmt.Sprintf("power ramp: step %d fallback server loaded power (SDR avg): %.0f W", step, sdrStep.PSUInW)) if step == len(result.RecommendedSlotOrder) { serverLoadedW = sdrStep.PSUInW serverLoadedOK = true sdrLastStep = sdrStep } } // Fan values are phase averages over the same load window. if stepRun.AvgFanRPM > 0 { ramp.AvgFanRPM = stepRun.AvgFanRPM ramp.AvgFanDutyCyclePct = stepRun.AvgFanDutyCyclePct } // Per-GPU telemetry from this ramp step's calibration. ramp.PerGPUTelemetry = make(map[int]*BenchmarkTelemetrySummary, len(subset)) for _, gpuIdx := range subset { if c, ok := stepCalib[gpuIdx]; ok { s := c.Summary ramp.PerGPUTelemetry[gpuIdx] = &s } } result.RampSteps = append(result.RampSteps, ramp) } // Populate StablePowerLimitW on each GPU entry from the accumulated stable limits. for i := range result.GPUs { if lim, ok := stableLimits[result.GPUs[i].Index]; ok { result.GPUs[i].StablePowerLimitW = float64(lim) } if result.GPUs[i].StablePowerLimitW > 0 && result.GPUs[i].AppliedPowerLimitW > 0 && result.GPUs[i].StablePowerLimitW < result.GPUs[i].AppliedPowerLimitW { result.GPUs[i].Derated = true result.Findings = append(result.Findings, fmt.Sprintf( "GPU %d required additional derating from %.0f W (single-card) to %.0f W under full-system thermal load.", result.GPUs[i].Index, result.GPUs[i].AppliedPowerLimitW, result.GPUs[i].StablePowerLimitW, )) } } // PlatformMaxTDPW = sum of all stable limits — the actual sustained power // budget of this server with all GPUs running simultaneously without throttling. for _, lim := range stableLimits { result.PlatformMaxTDPW += float64(lim) } // Characterize server power from IPMI idle/loaded samples. // gpuActualSumW = sum of p95 GPU power from the last ramp step — actual // measured consumption, not the stable limit cap. This is the correct // denominator for the reporting ratio: limit caps (PlatformMaxTDPW) inflate // the denominator and make the ratio appear artificially low. var gpuActualSumW float64 if n := len(result.RampSteps); n > 0 { gpuActualSumW = result.RampSteps[n-1].TotalObservedPowerW } if gpuActualSumW <= 0 { gpuActualSumW = result.PlatformMaxTDPW } _ = serverIdleOK // used implicitly via characterizeServerPower result.ServerPower = characterizeServerPower(serverIdleW, serverLoadedW, gpuActualSumW, opts.ServerPowerSource, serverIdleOK && serverLoadedOK) // Supplement DCMI with SDR multi-source data via collector's PSU slot patterns. // Per-slot readings enable correlation with audit HardwarePowerSupply entries. if result.ServerPower != nil { // Use the SDR phase average from the last ramp step (GPUs still loaded) // rather than re-sampling here, which would capture post-test idle state. sdrLoaded := sdrLastStep result.ServerPower.PSUInputIdleW = sdrIdle.PSUInW result.ServerPower.PSUInputLoadedW = sdrLoaded.PSUInW result.ServerPower.PSUOutputIdleW = sdrIdle.PSUOutW result.ServerPower.PSUOutputLoadedW = sdrLoaded.PSUOutW result.ServerPower.GPUSlotTotalW = sdrLoaded.GPUSlotW if len(sdrIdle.PSUSlots) > 0 { result.ServerPower.PSUSlotReadingsIdle = sdrIdle.PSUSlots } if len(sdrLoaded.PSUSlots) > 0 { result.ServerPower.PSUSlotReadingsLoaded = sdrLoaded.PSUSlots } if sdrIdle.PSUInW > 0 && result.ServerPower.IdleW > 0 { result.ServerPower.DCMICoverageRatio = result.ServerPower.IdleW / sdrIdle.PSUInW } if len(sdrLoaded.SkippedSensors) > 0 { result.ServerPower.Notes = append(result.ServerPower.Notes, "SDR sensors skipped (self-healed): "+strings.Join(sdrLoaded.SkippedSensors, "; ")) } if sdrLoaded.Samples > 0 { result.ServerPower.Notes = append(result.ServerPower.Notes, fmt.Sprintf("Final SDR PSU loaded values are phase averages across %d sample(s) from the last full-load step.", sdrLoaded.Samples)) } // Detect DCMI partial coverage: direct SDR comparison first, // ramp heuristic as fallback when SDR PSU sensors are absent. dcmiUnreliable := detectDCMIPartialCoverage(result.ServerPower) || (sdrIdle.PSUInW == 0 && detectIPMISaturationFallback(result.RampSteps)) if dcmiUnreliable { result.ServerPower.Notes = append(result.ServerPower.Notes, fmt.Sprintf("IPMI DCMI covers only a subset of installed PSUs (coverage %.0f%%). "+ "Use SDR PSU Δ ratio for GPU accuracy assessment; DCMI ratio is not reliable.", result.ServerPower.DCMICoverageRatio*100)) } } result.PSUIssues = diffPSUStatus(psuBefore, psuStatusSnapshot()) // Write top-level gpu-metrics.csv/.html aggregating all phases. writeBenchmarkMetricsFiles(runDir, allPowerRows) resultJSON, err := json.MarshalIndent(result, "", " ") if err != nil { return "", fmt.Errorf("marshal power result: %w", err) } if err := os.WriteFile(filepath.Join(runDir, "result.json"), resultJSON, 0644); err != nil { return "", fmt.Errorf("write result.json: %w", err) } if err := os.WriteFile(filepath.Join(runDir, "report.md"), []byte(renderPowerBenchReport(result)), 0644); err != nil { return "", fmt.Errorf("write report.md: %w", err) } if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(renderPowerBenchSummary(result)), 0644); err != nil { return "", fmt.Errorf("write summary.txt: %w", err) } return runDir, nil }