496 lines
20 KiB
Go
496 lines
20 KiB
Go
package platform
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"math"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
func (s *System) RunNvidiaPowerBench(ctx context.Context, baseDir string, opts NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
|
|
if ctx == nil {
|
|
ctx = context.Background()
|
|
}
|
|
if logFunc == nil {
|
|
logFunc = func(string) {}
|
|
}
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = "/var/log/bee-bench/power"
|
|
}
|
|
opts = normalizeNvidiaBenchmarkOptionsForBenchmark(opts)
|
|
selected, err := resolveNvidiaGPUSelection(opts.GPUIndices, opts.ExcludeGPUIndices)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(selected) == 0 {
|
|
return "", fmt.Errorf("no NVIDIA GPUs selected")
|
|
}
|
|
ts := time.Now().UTC().Format("20060102-150405")
|
|
runDir := filepath.Join(baseDir, "power-"+ts)
|
|
if err := os.MkdirAll(runDir, 0755); err != nil {
|
|
return "", fmt.Errorf("mkdir %s: %w", runDir, err)
|
|
}
|
|
verboseLog := filepath.Join(runDir, "verbose.log")
|
|
hostname, _ := os.Hostname()
|
|
result := NvidiaPowerBenchResult{
|
|
BenchmarkVersion: benchmarkVersion,
|
|
GeneratedAt: time.Now().UTC(),
|
|
Hostname: hostname,
|
|
ServerModel: readServerModel(),
|
|
BenchmarkProfile: opts.Profile,
|
|
SelectedGPUIndices: append([]int(nil), selected...),
|
|
OverallStatus: "OK",
|
|
}
|
|
infoByIndex, infoErr := queryBenchmarkGPUInfo(selected)
|
|
if infoErr != nil {
|
|
return "", infoErr
|
|
}
|
|
// Capture full nvidia-smi -q snapshot at the start of the run.
|
|
if out, err := runSATCommandCtx(ctx, verboseLog, "00-nvidia-smi-q.log", []string{"nvidia-smi", "-q"}, nil, nil); err == nil {
|
|
_ = os.WriteFile(filepath.Join(runDir, "00-nvidia-smi-q.log"), out, 0644)
|
|
}
|
|
durationSec := powerBenchDurationSec(opts.Profile)
|
|
|
|
// Sample server idle power before any GPU load.
|
|
var serverIdleW float64
|
|
var serverIdleOK bool
|
|
idleSDRStopCh := make(chan struct{})
|
|
idleSDRCh := startIPMISDRSampler(idleSDRStopCh, benchmarkPowerAutotuneSampleInterval)
|
|
if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, 10, benchmarkPowerAutotuneSampleInterval); ok {
|
|
serverIdleW = w
|
|
serverIdleOK = true
|
|
logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w))
|
|
}
|
|
close(idleSDRStopCh)
|
|
sdrIdle := summarizeSDRPowerSeries(<-idleSDRCh)
|
|
psuBefore := psuStatusSnapshot()
|
|
|
|
// Phase 1: calibrate each GPU individually (sequentially, one at a time) to
|
|
// establish a true single-card power baseline unaffected by neighbour heat.
|
|
calibByIndex := make(map[int]benchmarkPowerCalibrationResult, len(selected))
|
|
singleIPMILoadedW := make(map[int]float64, len(selected))
|
|
singleRunSummaryByIndex := make(map[int]benchmarkPowerCalibrationRunSummary, len(selected))
|
|
var allRestoreActions []benchmarkRestoreAction
|
|
// allPowerRows accumulates telemetry from all phases for the top-level gpu-metrics.csv.
|
|
var allPowerRows []GPUMetricRow
|
|
var powerCursor float64
|
|
for _, idx := range selected {
|
|
singleDir := filepath.Join(runDir, fmt.Sprintf("single-%02d", idx))
|
|
_ = os.MkdirAll(singleDir, 0755)
|
|
singleInfo := cloneBenchmarkGPUInfoMap(infoByIndex)
|
|
if failed := resetBenchmarkGPUs(ctx, verboseLog, []int{idx}, logFunc); len(failed) > 0 {
|
|
return "", fmt.Errorf("power benchmark pre-flight: failed to reset GPU %d; benchmark aborted to keep measurements clean", idx)
|
|
}
|
|
logFunc(fmt.Sprintf("power calibration: GPU %d single-card baseline", idx))
|
|
singlePowerStopCh := make(chan struct{})
|
|
singlePowerCh := startSelectedPowerSourceSampler(singlePowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
|
|
c, restore, singleRows, singleRun := runBenchmarkPowerCalibration(ctx, verboseLog, singleDir, []int{idx}, singleInfo, logFunc, nil, durationSec)
|
|
appendBenchmarkMetrics(&allPowerRows, singleRows, fmt.Sprintf("single-gpu-%d", idx), &powerCursor, 0)
|
|
close(singlePowerStopCh)
|
|
if samples := <-singlePowerCh; len(samples) > 0 {
|
|
singleIPMILoadedW[idx] = benchmarkMean(samples)
|
|
logFunc(fmt.Sprintf("power calibration: GPU %d single-card server power (%s avg): %.0f W", idx, opts.ServerPowerSource, singleIPMILoadedW[idx]))
|
|
} else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && singleRun.LoadedSDR.PSUInW > 0 {
|
|
singleIPMILoadedW[idx] = singleRun.LoadedSDR.PSUInW
|
|
logFunc(fmt.Sprintf("power calibration: GPU %d single-card fallback server power (SDR avg): %.0f W", idx, singleRun.LoadedSDR.PSUInW))
|
|
}
|
|
allRestoreActions = append(allRestoreActions, restore...)
|
|
if r, ok := c[idx]; ok {
|
|
calibByIndex[idx] = r
|
|
}
|
|
singleRunSummaryByIndex[idx] = singleRun
|
|
}
|
|
defer func() {
|
|
for i := len(allRestoreActions) - 1; i >= 0; i-- {
|
|
allRestoreActions[i].fn()
|
|
}
|
|
}()
|
|
gpus := make([]NvidiaPowerBenchGPU, 0, len(selected))
|
|
for _, idx := range selected {
|
|
info := infoByIndex[idx]
|
|
calib := calibByIndex[idx]
|
|
status := "OK"
|
|
if !calib.Completed {
|
|
status = "FAILED"
|
|
result.OverallStatus = "PARTIAL"
|
|
} else if calib.Derated {
|
|
status = "PARTIAL"
|
|
if result.OverallStatus == "OK" {
|
|
result.OverallStatus = "PARTIAL"
|
|
}
|
|
}
|
|
gpu := NvidiaPowerBenchGPU{
|
|
Index: idx,
|
|
Name: info.Name,
|
|
BusID: info.BusID,
|
|
DefaultPowerLimitW: info.DefaultPowerLimitW,
|
|
AppliedPowerLimitW: calib.AppliedPowerLimitW,
|
|
MaxObservedPowerW: calib.Summary.P95PowerW,
|
|
MaxObservedTempC: calib.Summary.P95TempC,
|
|
CalibrationAttempts: calib.Attempts,
|
|
Derated: calib.Derated,
|
|
Status: status,
|
|
Notes: append([]string(nil), calib.Notes...),
|
|
CoolingWarning: calib.CoolingWarning,
|
|
}
|
|
if w, ok := singleIPMILoadedW[idx]; ok && serverIdleOK && w > 0 {
|
|
gpu.ServerLoadedW = w
|
|
gpu.ServerDeltaW = w - serverIdleW
|
|
}
|
|
if len(calib.MetricRows) > 0 {
|
|
t := summarizeBenchmarkTelemetry(calib.MetricRows)
|
|
gpu.Telemetry = &t
|
|
}
|
|
if singleRun := singleRunSummaryByIndex[idx]; singleRun.AvgFanRPM > 0 {
|
|
gpu.AvgFanRPM = singleRun.AvgFanRPM
|
|
gpu.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct
|
|
}
|
|
gpus = append(gpus, gpu)
|
|
}
|
|
sort.Slice(gpus, func(i, j int) bool {
|
|
if gpus[i].MaxObservedPowerW != gpus[j].MaxObservedPowerW {
|
|
return gpus[i].MaxObservedPowerW > gpus[j].MaxObservedPowerW
|
|
}
|
|
if gpus[i].AppliedPowerLimitW != gpus[j].AppliedPowerLimitW {
|
|
return gpus[i].AppliedPowerLimitW > gpus[j].AppliedPowerLimitW
|
|
}
|
|
if gpus[i].Derated != gpus[j].Derated {
|
|
return !gpus[i].Derated
|
|
}
|
|
return gpus[i].Index < gpus[j].Index
|
|
})
|
|
result.GPUs = gpus
|
|
result.RecommendedSlotOrder = make([]int, 0, len(gpus))
|
|
for _, gpu := range gpus {
|
|
result.RecommendedSlotOrder = append(result.RecommendedSlotOrder, gpu.Index)
|
|
}
|
|
if len(result.RecommendedSlotOrder) > 0 {
|
|
result.Findings = append(result.Findings, fmt.Sprintf("Recommended slot order for installation based on single-card %s: %s.", benchmarkPowerEngineLabel(benchmarkPowerEngine()), joinIndexList(result.RecommendedSlotOrder)))
|
|
}
|
|
for _, gpu := range gpus {
|
|
if gpu.Derated {
|
|
result.Findings = append(result.Findings, fmt.Sprintf("GPU %d required reduced power limit %.0f W to complete %s.", gpu.Index, gpu.AppliedPowerLimitW, benchmarkPowerEngineLabel(benchmarkPowerEngine())))
|
|
}
|
|
if gpu.CoolingWarning != "" {
|
|
result.Findings = append(result.Findings, fmt.Sprintf(
|
|
"GPU %d: %s. Operator action: rerun the benchmark with fan speed manually fixed at 100%% to confirm actual thermal headroom.",
|
|
gpu.Index, gpu.CoolingWarning,
|
|
))
|
|
}
|
|
}
|
|
singleByIndex := make(map[int]NvidiaPowerBenchGPU, len(gpus))
|
|
for _, gpu := range gpus {
|
|
singleByIndex[gpu.Index] = gpu
|
|
}
|
|
|
|
// Phase 2: cumulative thermal ramp.
|
|
// Each step introduces one new GPU into an environment where all previously
|
|
// calibrated GPUs are already running at their fixed stable limits. The new
|
|
// GPU's stable TDP is searched via binary search under real
|
|
// multi-GPU thermal load. Once found, its limit is fixed permanently for all
|
|
// subsequent steps. This ensures each GPU's limit reflects actual sustained
|
|
// power in the final full-system thermal state.
|
|
//
|
|
// stableLimits accumulates GPU index → fixed stable limit (W) across steps.
|
|
stableLimits := make(map[int]int, len(result.RecommendedSlotOrder))
|
|
|
|
// serverLoadedW tracks the IPMI server power from the final ramp step
|
|
// (all GPUs simultaneously loaded). Earlier steps' values are stored
|
|
// per-step in NvidiaPowerBenchStep.ServerLoadedW.
|
|
var serverLoadedW float64
|
|
var serverLoadedOK bool
|
|
// sdrLastStep retains the phase-averaged SDR readings from the last ramp step
|
|
// while GPUs are loaded. Used in the summary instead of re-sampling after the
|
|
// test when GPUs have already returned to idle.
|
|
var sdrLastStep benchmarkSDRSeriesSummary
|
|
|
|
// Step 1: reuse single-card calibration result directly.
|
|
if len(result.RecommendedSlotOrder) > 0 {
|
|
firstIdx := result.RecommendedSlotOrder[0]
|
|
firstCalib := calibByIndex[firstIdx]
|
|
stableLimits[firstIdx] = int(math.Round(firstCalib.AppliedPowerLimitW))
|
|
ramp := NvidiaPowerBenchStep{
|
|
StepIndex: 1,
|
|
GPUIndices: []int{firstIdx},
|
|
NewGPUIndex: firstIdx,
|
|
NewGPUStableLimitW: firstCalib.AppliedPowerLimitW,
|
|
TotalObservedPowerW: firstCalib.Summary.P95PowerW,
|
|
AvgObservedPowerW: firstCalib.Summary.P95PowerW,
|
|
Derated: firstCalib.Derated,
|
|
Status: "OK",
|
|
}
|
|
if w, ok := singleIPMILoadedW[firstIdx]; ok && serverIdleOK && w > 0 {
|
|
ramp.ServerLoadedW = w
|
|
ramp.ServerDeltaW = w - serverIdleW
|
|
}
|
|
if singleRun := singleRunSummaryByIndex[firstIdx]; singleRun.AvgFanRPM > 0 {
|
|
ramp.AvgFanRPM = singleRun.AvgFanRPM
|
|
ramp.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct
|
|
}
|
|
firstSummary := firstCalib.Summary
|
|
ramp.PerGPUTelemetry = map[int]*BenchmarkTelemetrySummary{firstIdx: &firstSummary}
|
|
if !firstCalib.Completed {
|
|
ramp.Status = "FAILED"
|
|
ramp.Notes = append(ramp.Notes, fmt.Sprintf("GPU %d did not complete single-card %s", firstIdx, benchmarkPowerEngineLabel(benchmarkPowerEngine())))
|
|
result.OverallStatus = "PARTIAL"
|
|
} else if firstCalib.Derated {
|
|
ramp.Status = "PARTIAL"
|
|
if result.OverallStatus == "OK" {
|
|
result.OverallStatus = "PARTIAL"
|
|
}
|
|
result.Findings = append(result.Findings, fmt.Sprintf("Ramp step 1 (GPU %d) required derating to %.0f W.", firstIdx, firstCalib.AppliedPowerLimitW))
|
|
}
|
|
result.RampSteps = append(result.RampSteps, ramp)
|
|
logFunc(fmt.Sprintf("power ramp: step 1/%d — reused single-card calibration for GPU %d, stable limit %.0f W",
|
|
len(result.RecommendedSlotOrder), firstIdx, firstCalib.AppliedPowerLimitW))
|
|
}
|
|
|
|
// Steps 2..N: each step revalidates every already-active GPU under the new
|
|
// cumulative thermal environment and also calibrates the newly introduced
|
|
// GPU. Previously found limits are used only as seeds for the search.
|
|
for stepNum := 1; stepNum < len(result.RecommendedSlotOrder); stepNum++ {
|
|
step := stepNum + 1
|
|
subset := append([]int(nil), result.RecommendedSlotOrder[:step]...)
|
|
newGPUIdx := result.RecommendedSlotOrder[stepNum]
|
|
stepDir := filepath.Join(runDir, fmt.Sprintf("step-%02d", step))
|
|
_ = os.MkdirAll(stepDir, 0755)
|
|
|
|
// Reuse the latest stable limits as starting points, but re-check every
|
|
// active GPU in this hotter configuration. For the newly introduced GPU,
|
|
// seed from its single-card calibration so we do not restart from the
|
|
// default TDP when a prior derated limit is already known.
|
|
seedForStep := make(map[int]int, len(subset))
|
|
for _, idx := range subset {
|
|
if lim, ok := stableLimits[idx]; ok && lim > 0 {
|
|
seedForStep[idx] = lim
|
|
continue
|
|
}
|
|
if base, ok := calibByIndex[idx]; ok {
|
|
lim := int(math.Round(base.AppliedPowerLimitW))
|
|
if lim > 0 {
|
|
seedForStep[idx] = lim
|
|
}
|
|
}
|
|
}
|
|
|
|
logFunc(fmt.Sprintf("power ramp: step %d/%d — revalidating %d active GPU(s) including new GPU %d",
|
|
step, len(result.RecommendedSlotOrder), len(subset), newGPUIdx))
|
|
|
|
stepInfo := cloneBenchmarkGPUInfoMap(infoByIndex)
|
|
stepPowerStopCh := make(chan struct{})
|
|
stepPowerCh := startSelectedPowerSourceSampler(stepPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
|
|
stepCalib, stepRestore, stepRows, stepRun := runBenchmarkPowerCalibration(ctx, verboseLog, stepDir, subset, stepInfo, logFunc, seedForStep, durationSec)
|
|
appendBenchmarkMetrics(&allPowerRows, stepRows, fmt.Sprintf("ramp-step-%d", step), &powerCursor, 0)
|
|
close(stepPowerStopCh)
|
|
var stepIPMILoadedW float64
|
|
var stepIPMIOK bool
|
|
if samples := <-stepPowerCh; len(samples) > 0 {
|
|
stepIPMILoadedW = benchmarkMean(samples)
|
|
stepIPMIOK = true
|
|
}
|
|
// Accumulate restore actions; they all run in the outer defer.
|
|
allRestoreActions = append(allRestoreActions, stepRestore...)
|
|
|
|
ramp := NvidiaPowerBenchStep{
|
|
StepIndex: step,
|
|
GPUIndices: subset,
|
|
NewGPUIndex: newGPUIdx,
|
|
Status: "OK",
|
|
}
|
|
|
|
// Total observed power = sum of p95 across all GPUs in this step.
|
|
for _, idx := range subset {
|
|
if c, ok := stepCalib[idx]; ok {
|
|
ramp.TotalObservedPowerW += c.Summary.P95PowerW
|
|
}
|
|
}
|
|
if len(subset) > 0 {
|
|
ramp.AvgObservedPowerW = ramp.TotalObservedPowerW / float64(len(subset))
|
|
}
|
|
|
|
for _, idx := range subset {
|
|
c, ok := stepCalib[idx]
|
|
if !ok || !c.Completed {
|
|
fallback := 0
|
|
if lim, ok := stableLimits[idx]; ok && lim > 0 {
|
|
fallback = lim
|
|
} else if fb, ok := calibByIndex[idx]; ok {
|
|
fallback = int(math.Round(fb.AppliedPowerLimitW))
|
|
}
|
|
if fallback > 0 {
|
|
stableLimits[idx] = fallback
|
|
}
|
|
ramp.Status = "FAILED"
|
|
ramp.Notes = append(ramp.Notes,
|
|
fmt.Sprintf("GPU %d did not complete %s in ramp step %d; keeping previous stable limit %d W", idx, benchmarkPowerEngineLabel(benchmarkPowerEngine()), step, fallback))
|
|
result.OverallStatus = "PARTIAL"
|
|
continue
|
|
}
|
|
|
|
prevLimit, hadPrev := stableLimits[idx]
|
|
newLimit := int(math.Round(c.AppliedPowerLimitW))
|
|
stableLimits[idx] = newLimit
|
|
if idx == newGPUIdx {
|
|
ramp.NewGPUStableLimitW = c.AppliedPowerLimitW
|
|
ramp.Derated = c.Derated
|
|
}
|
|
if c.Derated {
|
|
ramp.Status = "PARTIAL"
|
|
if result.OverallStatus == "OK" {
|
|
result.OverallStatus = "PARTIAL"
|
|
}
|
|
}
|
|
if hadPrev && newLimit < prevLimit {
|
|
ramp.Notes = append(ramp.Notes,
|
|
fmt.Sprintf("GPU %d was re-derated from %d W to %d W under combined thermal load.", idx, prevLimit, newLimit))
|
|
}
|
|
}
|
|
|
|
if c, ok := stepCalib[newGPUIdx]; ok && c.Completed && c.Derated {
|
|
result.Findings = append(result.Findings, fmt.Sprintf("Ramp step %d (GPU %d) required derating to %.0f W under combined thermal load.", step, newGPUIdx, c.AppliedPowerLimitW))
|
|
}
|
|
|
|
// Per-step PSU slot readings are averaged over the whole load phase rather
|
|
// than captured as a single end-of-phase snapshot.
|
|
sdrStep := stepRun.LoadedSDR
|
|
if len(sdrStep.PSUSlots) > 0 {
|
|
ramp.PSUSlotReadings = sdrStep.PSUSlots
|
|
}
|
|
|
|
if stepIPMIOK && serverIdleOK && stepIPMILoadedW > 0 {
|
|
ramp.ServerLoadedW = stepIPMILoadedW
|
|
ramp.ServerDeltaW = stepIPMILoadedW - serverIdleW
|
|
logFunc(fmt.Sprintf("power ramp: step %d server loaded power (%s avg): %.0f W", step, opts.ServerPowerSource, stepIPMILoadedW))
|
|
// The last step has all GPUs loaded — use it as the top-level loaded_w.
|
|
if step == len(result.RecommendedSlotOrder) {
|
|
serverLoadedW = stepIPMILoadedW
|
|
serverLoadedOK = true
|
|
sdrLastStep = sdrStep
|
|
}
|
|
} else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && sdrStep.PSUInW > 0 {
|
|
ramp.ServerLoadedW = sdrStep.PSUInW
|
|
ramp.ServerDeltaW = sdrStep.PSUInW - sdrIdle.PSUInW
|
|
logFunc(fmt.Sprintf("power ramp: step %d fallback server loaded power (SDR avg): %.0f W", step, sdrStep.PSUInW))
|
|
if step == len(result.RecommendedSlotOrder) {
|
|
serverLoadedW = sdrStep.PSUInW
|
|
serverLoadedOK = true
|
|
sdrLastStep = sdrStep
|
|
}
|
|
}
|
|
|
|
// Fan values are phase averages over the same load window.
|
|
if stepRun.AvgFanRPM > 0 {
|
|
ramp.AvgFanRPM = stepRun.AvgFanRPM
|
|
ramp.AvgFanDutyCyclePct = stepRun.AvgFanDutyCyclePct
|
|
}
|
|
|
|
// Per-GPU telemetry from this ramp step's calibration.
|
|
ramp.PerGPUTelemetry = make(map[int]*BenchmarkTelemetrySummary, len(subset))
|
|
for _, gpuIdx := range subset {
|
|
if c, ok := stepCalib[gpuIdx]; ok {
|
|
s := c.Summary
|
|
ramp.PerGPUTelemetry[gpuIdx] = &s
|
|
}
|
|
}
|
|
|
|
result.RampSteps = append(result.RampSteps, ramp)
|
|
}
|
|
|
|
// Populate StablePowerLimitW on each GPU entry from the accumulated stable limits.
|
|
for i := range result.GPUs {
|
|
if lim, ok := stableLimits[result.GPUs[i].Index]; ok {
|
|
result.GPUs[i].StablePowerLimitW = float64(lim)
|
|
}
|
|
if result.GPUs[i].StablePowerLimitW > 0 && result.GPUs[i].AppliedPowerLimitW > 0 &&
|
|
result.GPUs[i].StablePowerLimitW < result.GPUs[i].AppliedPowerLimitW {
|
|
result.GPUs[i].Derated = true
|
|
result.Findings = append(result.Findings, fmt.Sprintf(
|
|
"GPU %d required additional derating from %.0f W (single-card) to %.0f W under full-system thermal load.",
|
|
result.GPUs[i].Index, result.GPUs[i].AppliedPowerLimitW, result.GPUs[i].StablePowerLimitW,
|
|
))
|
|
}
|
|
}
|
|
|
|
// PlatformMaxTDPW = sum of all stable limits — the actual sustained power
|
|
// budget of this server with all GPUs running simultaneously without throttling.
|
|
for _, lim := range stableLimits {
|
|
result.PlatformMaxTDPW += float64(lim)
|
|
}
|
|
|
|
// Characterize server power from IPMI idle/loaded samples.
|
|
// gpuActualSumW = sum of p95 GPU power from the last ramp step — actual
|
|
// measured consumption, not the stable limit cap. This is the correct
|
|
// denominator for the reporting ratio: limit caps (PlatformMaxTDPW) inflate
|
|
// the denominator and make the ratio appear artificially low.
|
|
var gpuActualSumW float64
|
|
if n := len(result.RampSteps); n > 0 {
|
|
gpuActualSumW = result.RampSteps[n-1].TotalObservedPowerW
|
|
}
|
|
if gpuActualSumW <= 0 {
|
|
gpuActualSumW = result.PlatformMaxTDPW
|
|
}
|
|
_ = serverIdleOK // used implicitly via characterizeServerPower
|
|
result.ServerPower = characterizeServerPower(serverIdleW, serverLoadedW, gpuActualSumW, opts.ServerPowerSource, serverIdleOK && serverLoadedOK)
|
|
// Supplement DCMI with SDR multi-source data via collector's PSU slot patterns.
|
|
// Per-slot readings enable correlation with audit HardwarePowerSupply entries.
|
|
if result.ServerPower != nil {
|
|
// Use the SDR phase average from the last ramp step (GPUs still loaded)
|
|
// rather than re-sampling here, which would capture post-test idle state.
|
|
sdrLoaded := sdrLastStep
|
|
result.ServerPower.PSUInputIdleW = sdrIdle.PSUInW
|
|
result.ServerPower.PSUInputLoadedW = sdrLoaded.PSUInW
|
|
result.ServerPower.PSUOutputIdleW = sdrIdle.PSUOutW
|
|
result.ServerPower.PSUOutputLoadedW = sdrLoaded.PSUOutW
|
|
result.ServerPower.GPUSlotTotalW = sdrLoaded.GPUSlotW
|
|
if len(sdrIdle.PSUSlots) > 0 {
|
|
result.ServerPower.PSUSlotReadingsIdle = sdrIdle.PSUSlots
|
|
}
|
|
if len(sdrLoaded.PSUSlots) > 0 {
|
|
result.ServerPower.PSUSlotReadingsLoaded = sdrLoaded.PSUSlots
|
|
}
|
|
if sdrIdle.PSUInW > 0 && result.ServerPower.IdleW > 0 {
|
|
result.ServerPower.DCMICoverageRatio = result.ServerPower.IdleW / sdrIdle.PSUInW
|
|
}
|
|
if len(sdrLoaded.SkippedSensors) > 0 {
|
|
result.ServerPower.Notes = append(result.ServerPower.Notes,
|
|
"SDR sensors skipped (self-healed): "+strings.Join(sdrLoaded.SkippedSensors, "; "))
|
|
}
|
|
if sdrLoaded.Samples > 0 {
|
|
result.ServerPower.Notes = append(result.ServerPower.Notes,
|
|
fmt.Sprintf("Final SDR PSU loaded values are phase averages across %d sample(s) from the last full-load step.", sdrLoaded.Samples))
|
|
}
|
|
// Detect DCMI partial coverage: direct SDR comparison first,
|
|
// ramp heuristic as fallback when SDR PSU sensors are absent.
|
|
dcmiUnreliable := detectDCMIPartialCoverage(result.ServerPower) ||
|
|
(sdrIdle.PSUInW == 0 && detectIPMISaturationFallback(result.RampSteps))
|
|
if dcmiUnreliable {
|
|
result.ServerPower.Notes = append(result.ServerPower.Notes,
|
|
fmt.Sprintf("IPMI DCMI covers only a subset of installed PSUs (coverage %.0f%%). "+
|
|
"Use SDR PSU Δ ratio for GPU accuracy assessment; DCMI ratio is not reliable.",
|
|
result.ServerPower.DCMICoverageRatio*100))
|
|
}
|
|
}
|
|
result.PSUIssues = diffPSUStatus(psuBefore, psuStatusSnapshot())
|
|
// Write top-level gpu-metrics.csv/.html aggregating all phases.
|
|
writeBenchmarkMetricsFiles(runDir, allPowerRows)
|
|
resultJSON, err := json.MarshalIndent(result, "", " ")
|
|
if err != nil {
|
|
return "", fmt.Errorf("marshal power result: %w", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(runDir, "result.json"), resultJSON, 0644); err != nil {
|
|
return "", fmt.Errorf("write result.json: %w", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(runDir, "report.md"), []byte(renderPowerBenchReport(result)), 0644); err != nil {
|
|
return "", fmt.Errorf("write report.md: %w", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(renderPowerBenchSummary(result)), 0644); err != nil {
|
|
return "", fmt.Errorf("write summary.txt: %w", err)
|
|
}
|
|
return runDir, nil
|
|
}
|