refactor: modularize audit and harden build validation
This commit is contained in:
Mikhail Chusavitin
@@ -0,0 +1,495 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (s *System) RunNvidiaPowerBench(ctx context.Context, baseDir string, opts NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if logFunc == nil {
|
||||
logFunc = func(string) {}
|
||||
}
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = "/var/log/bee-bench/power"
|
||||
}
|
||||
opts = normalizeNvidiaBenchmarkOptionsForBenchmark(opts)
|
||||
selected, err := resolveNvidiaGPUSelection(opts.GPUIndices, opts.ExcludeGPUIndices)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(selected) == 0 {
|
||||
return "", fmt.Errorf("no NVIDIA GPUs selected")
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
runDir := filepath.Join(baseDir, "power-"+ts)
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", fmt.Errorf("mkdir %s: %w", runDir, err)
|
||||
}
|
||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||
hostname, _ := os.Hostname()
|
||||
result := NvidiaPowerBenchResult{
|
||||
BenchmarkVersion: benchmarkVersion,
|
||||
GeneratedAt: time.Now().UTC(),
|
||||
Hostname: hostname,
|
||||
ServerModel: readServerModel(),
|
||||
BenchmarkProfile: opts.Profile,
|
||||
SelectedGPUIndices: append([]int(nil), selected...),
|
||||
OverallStatus: "OK",
|
||||
}
|
||||
infoByIndex, infoErr := queryBenchmarkGPUInfo(selected)
|
||||
if infoErr != nil {
|
||||
return "", infoErr
|
||||
}
|
||||
// Capture full nvidia-smi -q snapshot at the start of the run.
|
||||
if out, err := runSATCommandCtx(ctx, verboseLog, "00-nvidia-smi-q.log", []string{"nvidia-smi", "-q"}, nil, nil); err == nil {
|
||||
_ = os.WriteFile(filepath.Join(runDir, "00-nvidia-smi-q.log"), out, 0644)
|
||||
}
|
||||
durationSec := powerBenchDurationSec(opts.Profile)
|
||||
|
||||
// Sample server idle power before any GPU load.
|
||||
var serverIdleW float64
|
||||
var serverIdleOK bool
|
||||
idleSDRStopCh := make(chan struct{})
|
||||
idleSDRCh := startIPMISDRSampler(idleSDRStopCh, benchmarkPowerAutotuneSampleInterval)
|
||||
if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, 10, benchmarkPowerAutotuneSampleInterval); ok {
|
||||
serverIdleW = w
|
||||
serverIdleOK = true
|
||||
logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w))
|
||||
}
|
||||
close(idleSDRStopCh)
|
||||
sdrIdle := summarizeSDRPowerSeries(<-idleSDRCh)
|
||||
psuBefore := psuStatusSnapshot()
|
||||
|
||||
// Phase 1: calibrate each GPU individually (sequentially, one at a time) to
|
||||
// establish a true single-card power baseline unaffected by neighbour heat.
|
||||
calibByIndex := make(map[int]benchmarkPowerCalibrationResult, len(selected))
|
||||
singleIPMILoadedW := make(map[int]float64, len(selected))
|
||||
singleRunSummaryByIndex := make(map[int]benchmarkPowerCalibrationRunSummary, len(selected))
|
||||
var allRestoreActions []benchmarkRestoreAction
|
||||
// allPowerRows accumulates telemetry from all phases for the top-level gpu-metrics.csv.
|
||||
var allPowerRows []GPUMetricRow
|
||||
var powerCursor float64
|
||||
for _, idx := range selected {
|
||||
singleDir := filepath.Join(runDir, fmt.Sprintf("single-%02d", idx))
|
||||
_ = os.MkdirAll(singleDir, 0755)
|
||||
singleInfo := cloneBenchmarkGPUInfoMap(infoByIndex)
|
||||
if failed := resetBenchmarkGPUs(ctx, verboseLog, []int{idx}, logFunc); len(failed) > 0 {
|
||||
return "", fmt.Errorf("power benchmark pre-flight: failed to reset GPU %d; benchmark aborted to keep measurements clean", idx)
|
||||
}
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d single-card baseline", idx))
|
||||
singlePowerStopCh := make(chan struct{})
|
||||
singlePowerCh := startSelectedPowerSourceSampler(singlePowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
|
||||
c, restore, singleRows, singleRun := runBenchmarkPowerCalibration(ctx, verboseLog, singleDir, []int{idx}, singleInfo, logFunc, nil, durationSec)
|
||||
appendBenchmarkMetrics(&allPowerRows, singleRows, fmt.Sprintf("single-gpu-%d", idx), &powerCursor, 0)
|
||||
close(singlePowerStopCh)
|
||||
if samples := <-singlePowerCh; len(samples) > 0 {
|
||||
singleIPMILoadedW[idx] = benchmarkMean(samples)
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d single-card server power (%s avg): %.0f W", idx, opts.ServerPowerSource, singleIPMILoadedW[idx]))
|
||||
} else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && singleRun.LoadedSDR.PSUInW > 0 {
|
||||
singleIPMILoadedW[idx] = singleRun.LoadedSDR.PSUInW
|
||||
logFunc(fmt.Sprintf("power calibration: GPU %d single-card fallback server power (SDR avg): %.0f W", idx, singleRun.LoadedSDR.PSUInW))
|
||||
}
|
||||
allRestoreActions = append(allRestoreActions, restore...)
|
||||
if r, ok := c[idx]; ok {
|
||||
calibByIndex[idx] = r
|
||||
}
|
||||
singleRunSummaryByIndex[idx] = singleRun
|
||||
}
|
||||
defer func() {
|
||||
for i := len(allRestoreActions) - 1; i >= 0; i-- {
|
||||
allRestoreActions[i].fn()
|
||||
}
|
||||
}()
|
||||
gpus := make([]NvidiaPowerBenchGPU, 0, len(selected))
|
||||
for _, idx := range selected {
|
||||
info := infoByIndex[idx]
|
||||
calib := calibByIndex[idx]
|
||||
status := "OK"
|
||||
if !calib.Completed {
|
||||
status = "FAILED"
|
||||
result.OverallStatus = "PARTIAL"
|
||||
} else if calib.Derated {
|
||||
status = "PARTIAL"
|
||||
if result.OverallStatus == "OK" {
|
||||
result.OverallStatus = "PARTIAL"
|
||||
}
|
||||
}
|
||||
gpu := NvidiaPowerBenchGPU{
|
||||
Index: idx,
|
||||
Name: info.Name,
|
||||
BusID: info.BusID,
|
||||
DefaultPowerLimitW: info.DefaultPowerLimitW,
|
||||
AppliedPowerLimitW: calib.AppliedPowerLimitW,
|
||||
MaxObservedPowerW: calib.Summary.P95PowerW,
|
||||
MaxObservedTempC: calib.Summary.P95TempC,
|
||||
CalibrationAttempts: calib.Attempts,
|
||||
Derated: calib.Derated,
|
||||
Status: status,
|
||||
Notes: append([]string(nil), calib.Notes...),
|
||||
CoolingWarning: calib.CoolingWarning,
|
||||
}
|
||||
if w, ok := singleIPMILoadedW[idx]; ok && serverIdleOK && w > 0 {
|
||||
gpu.ServerLoadedW = w
|
||||
gpu.ServerDeltaW = w - serverIdleW
|
||||
}
|
||||
if len(calib.MetricRows) > 0 {
|
||||
t := summarizeBenchmarkTelemetry(calib.MetricRows)
|
||||
gpu.Telemetry = &t
|
||||
}
|
||||
if singleRun := singleRunSummaryByIndex[idx]; singleRun.AvgFanRPM > 0 {
|
||||
gpu.AvgFanRPM = singleRun.AvgFanRPM
|
||||
gpu.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct
|
||||
}
|
||||
gpus = append(gpus, gpu)
|
||||
}
|
||||
sort.Slice(gpus, func(i, j int) bool {
|
||||
if gpus[i].MaxObservedPowerW != gpus[j].MaxObservedPowerW {
|
||||
return gpus[i].MaxObservedPowerW > gpus[j].MaxObservedPowerW
|
||||
}
|
||||
if gpus[i].AppliedPowerLimitW != gpus[j].AppliedPowerLimitW {
|
||||
return gpus[i].AppliedPowerLimitW > gpus[j].AppliedPowerLimitW
|
||||
}
|
||||
if gpus[i].Derated != gpus[j].Derated {
|
||||
return !gpus[i].Derated
|
||||
}
|
||||
return gpus[i].Index < gpus[j].Index
|
||||
})
|
||||
result.GPUs = gpus
|
||||
result.RecommendedSlotOrder = make([]int, 0, len(gpus))
|
||||
for _, gpu := range gpus {
|
||||
result.RecommendedSlotOrder = append(result.RecommendedSlotOrder, gpu.Index)
|
||||
}
|
||||
if len(result.RecommendedSlotOrder) > 0 {
|
||||
result.Findings = append(result.Findings, fmt.Sprintf("Recommended slot order for installation based on single-card %s: %s.", benchmarkPowerEngineLabel(benchmarkPowerEngine()), joinIndexList(result.RecommendedSlotOrder)))
|
||||
}
|
||||
for _, gpu := range gpus {
|
||||
if gpu.Derated {
|
||||
result.Findings = append(result.Findings, fmt.Sprintf("GPU %d required reduced power limit %.0f W to complete %s.", gpu.Index, gpu.AppliedPowerLimitW, benchmarkPowerEngineLabel(benchmarkPowerEngine())))
|
||||
}
|
||||
if gpu.CoolingWarning != "" {
|
||||
result.Findings = append(result.Findings, fmt.Sprintf(
|
||||
"GPU %d: %s. Operator action: rerun the benchmark with fan speed manually fixed at 100%% to confirm actual thermal headroom.",
|
||||
gpu.Index, gpu.CoolingWarning,
|
||||
))
|
||||
}
|
||||
}
|
||||
singleByIndex := make(map[int]NvidiaPowerBenchGPU, len(gpus))
|
||||
for _, gpu := range gpus {
|
||||
singleByIndex[gpu.Index] = gpu
|
||||
}
|
||||
|
||||
// Phase 2: cumulative thermal ramp.
|
||||
// Each step introduces one new GPU into an environment where all previously
|
||||
// calibrated GPUs are already running at their fixed stable limits. The new
|
||||
// GPU's stable TDP is searched via binary search under real
|
||||
// multi-GPU thermal load. Once found, its limit is fixed permanently for all
|
||||
// subsequent steps. This ensures each GPU's limit reflects actual sustained
|
||||
// power in the final full-system thermal state.
|
||||
//
|
||||
// stableLimits accumulates GPU index → fixed stable limit (W) across steps.
|
||||
stableLimits := make(map[int]int, len(result.RecommendedSlotOrder))
|
||||
|
||||
// serverLoadedW tracks the IPMI server power from the final ramp step
|
||||
// (all GPUs simultaneously loaded). Earlier steps' values are stored
|
||||
// per-step in NvidiaPowerBenchStep.ServerLoadedW.
|
||||
var serverLoadedW float64
|
||||
var serverLoadedOK bool
|
||||
// sdrLastStep retains the phase-averaged SDR readings from the last ramp step
|
||||
// while GPUs are loaded. Used in the summary instead of re-sampling after the
|
||||
// test when GPUs have already returned to idle.
|
||||
var sdrLastStep benchmarkSDRSeriesSummary
|
||||
|
||||
// Step 1: reuse single-card calibration result directly.
|
||||
if len(result.RecommendedSlotOrder) > 0 {
|
||||
firstIdx := result.RecommendedSlotOrder[0]
|
||||
firstCalib := calibByIndex[firstIdx]
|
||||
stableLimits[firstIdx] = int(math.Round(firstCalib.AppliedPowerLimitW))
|
||||
ramp := NvidiaPowerBenchStep{
|
||||
StepIndex: 1,
|
||||
GPUIndices: []int{firstIdx},
|
||||
NewGPUIndex: firstIdx,
|
||||
NewGPUStableLimitW: firstCalib.AppliedPowerLimitW,
|
||||
TotalObservedPowerW: firstCalib.Summary.P95PowerW,
|
||||
AvgObservedPowerW: firstCalib.Summary.P95PowerW,
|
||||
Derated: firstCalib.Derated,
|
||||
Status: "OK",
|
||||
}
|
||||
if w, ok := singleIPMILoadedW[firstIdx]; ok && serverIdleOK && w > 0 {
|
||||
ramp.ServerLoadedW = w
|
||||
ramp.ServerDeltaW = w - serverIdleW
|
||||
}
|
||||
if singleRun := singleRunSummaryByIndex[firstIdx]; singleRun.AvgFanRPM > 0 {
|
||||
ramp.AvgFanRPM = singleRun.AvgFanRPM
|
||||
ramp.AvgFanDutyCyclePct = singleRun.AvgFanDutyCyclePct
|
||||
}
|
||||
firstSummary := firstCalib.Summary
|
||||
ramp.PerGPUTelemetry = map[int]*BenchmarkTelemetrySummary{firstIdx: &firstSummary}
|
||||
if !firstCalib.Completed {
|
||||
ramp.Status = "FAILED"
|
||||
ramp.Notes = append(ramp.Notes, fmt.Sprintf("GPU %d did not complete single-card %s", firstIdx, benchmarkPowerEngineLabel(benchmarkPowerEngine())))
|
||||
result.OverallStatus = "PARTIAL"
|
||||
} else if firstCalib.Derated {
|
||||
ramp.Status = "PARTIAL"
|
||||
if result.OverallStatus == "OK" {
|
||||
result.OverallStatus = "PARTIAL"
|
||||
}
|
||||
result.Findings = append(result.Findings, fmt.Sprintf("Ramp step 1 (GPU %d) required derating to %.0f W.", firstIdx, firstCalib.AppliedPowerLimitW))
|
||||
}
|
||||
result.RampSteps = append(result.RampSteps, ramp)
|
||||
logFunc(fmt.Sprintf("power ramp: step 1/%d — reused single-card calibration for GPU %d, stable limit %.0f W",
|
||||
len(result.RecommendedSlotOrder), firstIdx, firstCalib.AppliedPowerLimitW))
|
||||
}
|
||||
|
||||
// Steps 2..N: each step revalidates every already-active GPU under the new
|
||||
// cumulative thermal environment and also calibrates the newly introduced
|
||||
// GPU. Previously found limits are used only as seeds for the search.
|
||||
for stepNum := 1; stepNum < len(result.RecommendedSlotOrder); stepNum++ {
|
||||
step := stepNum + 1
|
||||
subset := append([]int(nil), result.RecommendedSlotOrder[:step]...)
|
||||
newGPUIdx := result.RecommendedSlotOrder[stepNum]
|
||||
stepDir := filepath.Join(runDir, fmt.Sprintf("step-%02d", step))
|
||||
_ = os.MkdirAll(stepDir, 0755)
|
||||
|
||||
// Reuse the latest stable limits as starting points, but re-check every
|
||||
// active GPU in this hotter configuration. For the newly introduced GPU,
|
||||
// seed from its single-card calibration so we do not restart from the
|
||||
// default TDP when a prior derated limit is already known.
|
||||
seedForStep := make(map[int]int, len(subset))
|
||||
for _, idx := range subset {
|
||||
if lim, ok := stableLimits[idx]; ok && lim > 0 {
|
||||
seedForStep[idx] = lim
|
||||
continue
|
||||
}
|
||||
if base, ok := calibByIndex[idx]; ok {
|
||||
lim := int(math.Round(base.AppliedPowerLimitW))
|
||||
if lim > 0 {
|
||||
seedForStep[idx] = lim
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logFunc(fmt.Sprintf("power ramp: step %d/%d — revalidating %d active GPU(s) including new GPU %d",
|
||||
step, len(result.RecommendedSlotOrder), len(subset), newGPUIdx))
|
||||
|
||||
stepInfo := cloneBenchmarkGPUInfoMap(infoByIndex)
|
||||
stepPowerStopCh := make(chan struct{})
|
||||
stepPowerCh := startSelectedPowerSourceSampler(stepPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
|
||||
stepCalib, stepRestore, stepRows, stepRun := runBenchmarkPowerCalibration(ctx, verboseLog, stepDir, subset, stepInfo, logFunc, seedForStep, durationSec)
|
||||
appendBenchmarkMetrics(&allPowerRows, stepRows, fmt.Sprintf("ramp-step-%d", step), &powerCursor, 0)
|
||||
close(stepPowerStopCh)
|
||||
var stepIPMILoadedW float64
|
||||
var stepIPMIOK bool
|
||||
if samples := <-stepPowerCh; len(samples) > 0 {
|
||||
stepIPMILoadedW = benchmarkMean(samples)
|
||||
stepIPMIOK = true
|
||||
}
|
||||
// Accumulate restore actions; they all run in the outer defer.
|
||||
allRestoreActions = append(allRestoreActions, stepRestore...)
|
||||
|
||||
ramp := NvidiaPowerBenchStep{
|
||||
StepIndex: step,
|
||||
GPUIndices: subset,
|
||||
NewGPUIndex: newGPUIdx,
|
||||
Status: "OK",
|
||||
}
|
||||
|
||||
// Total observed power = sum of p95 across all GPUs in this step.
|
||||
for _, idx := range subset {
|
||||
if c, ok := stepCalib[idx]; ok {
|
||||
ramp.TotalObservedPowerW += c.Summary.P95PowerW
|
||||
}
|
||||
}
|
||||
if len(subset) > 0 {
|
||||
ramp.AvgObservedPowerW = ramp.TotalObservedPowerW / float64(len(subset))
|
||||
}
|
||||
|
||||
for _, idx := range subset {
|
||||
c, ok := stepCalib[idx]
|
||||
if !ok || !c.Completed {
|
||||
fallback := 0
|
||||
if lim, ok := stableLimits[idx]; ok && lim > 0 {
|
||||
fallback = lim
|
||||
} else if fb, ok := calibByIndex[idx]; ok {
|
||||
fallback = int(math.Round(fb.AppliedPowerLimitW))
|
||||
}
|
||||
if fallback > 0 {
|
||||
stableLimits[idx] = fallback
|
||||
}
|
||||
ramp.Status = "FAILED"
|
||||
ramp.Notes = append(ramp.Notes,
|
||||
fmt.Sprintf("GPU %d did not complete %s in ramp step %d; keeping previous stable limit %d W", idx, benchmarkPowerEngineLabel(benchmarkPowerEngine()), step, fallback))
|
||||
result.OverallStatus = "PARTIAL"
|
||||
continue
|
||||
}
|
||||
|
||||
prevLimit, hadPrev := stableLimits[idx]
|
||||
newLimit := int(math.Round(c.AppliedPowerLimitW))
|
||||
stableLimits[idx] = newLimit
|
||||
if idx == newGPUIdx {
|
||||
ramp.NewGPUStableLimitW = c.AppliedPowerLimitW
|
||||
ramp.Derated = c.Derated
|
||||
}
|
||||
if c.Derated {
|
||||
ramp.Status = "PARTIAL"
|
||||
if result.OverallStatus == "OK" {
|
||||
result.OverallStatus = "PARTIAL"
|
||||
}
|
||||
}
|
||||
if hadPrev && newLimit < prevLimit {
|
||||
ramp.Notes = append(ramp.Notes,
|
||||
fmt.Sprintf("GPU %d was re-derated from %d W to %d W under combined thermal load.", idx, prevLimit, newLimit))
|
||||
}
|
||||
}
|
||||
|
||||
if c, ok := stepCalib[newGPUIdx]; ok && c.Completed && c.Derated {
|
||||
result.Findings = append(result.Findings, fmt.Sprintf("Ramp step %d (GPU %d) required derating to %.0f W under combined thermal load.", step, newGPUIdx, c.AppliedPowerLimitW))
|
||||
}
|
||||
|
||||
// Per-step PSU slot readings are averaged over the whole load phase rather
|
||||
// than captured as a single end-of-phase snapshot.
|
||||
sdrStep := stepRun.LoadedSDR
|
||||
if len(sdrStep.PSUSlots) > 0 {
|
||||
ramp.PSUSlotReadings = sdrStep.PSUSlots
|
||||
}
|
||||
|
||||
if stepIPMIOK && serverIdleOK && stepIPMILoadedW > 0 {
|
||||
ramp.ServerLoadedW = stepIPMILoadedW
|
||||
ramp.ServerDeltaW = stepIPMILoadedW - serverIdleW
|
||||
logFunc(fmt.Sprintf("power ramp: step %d server loaded power (%s avg): %.0f W", step, opts.ServerPowerSource, stepIPMILoadedW))
|
||||
// The last step has all GPUs loaded — use it as the top-level loaded_w.
|
||||
if step == len(result.RecommendedSlotOrder) {
|
||||
serverLoadedW = stepIPMILoadedW
|
||||
serverLoadedOK = true
|
||||
sdrLastStep = sdrStep
|
||||
}
|
||||
} else if opts.ServerPowerSource == BenchmarkPowerSourceSDRPSUInput && sdrStep.PSUInW > 0 {
|
||||
ramp.ServerLoadedW = sdrStep.PSUInW
|
||||
ramp.ServerDeltaW = sdrStep.PSUInW - sdrIdle.PSUInW
|
||||
logFunc(fmt.Sprintf("power ramp: step %d fallback server loaded power (SDR avg): %.0f W", step, sdrStep.PSUInW))
|
||||
if step == len(result.RecommendedSlotOrder) {
|
||||
serverLoadedW = sdrStep.PSUInW
|
||||
serverLoadedOK = true
|
||||
sdrLastStep = sdrStep
|
||||
}
|
||||
}
|
||||
|
||||
// Fan values are phase averages over the same load window.
|
||||
if stepRun.AvgFanRPM > 0 {
|
||||
ramp.AvgFanRPM = stepRun.AvgFanRPM
|
||||
ramp.AvgFanDutyCyclePct = stepRun.AvgFanDutyCyclePct
|
||||
}
|
||||
|
||||
// Per-GPU telemetry from this ramp step's calibration.
|
||||
ramp.PerGPUTelemetry = make(map[int]*BenchmarkTelemetrySummary, len(subset))
|
||||
for _, gpuIdx := range subset {
|
||||
if c, ok := stepCalib[gpuIdx]; ok {
|
||||
s := c.Summary
|
||||
ramp.PerGPUTelemetry[gpuIdx] = &s
|
||||
}
|
||||
}
|
||||
|
||||
result.RampSteps = append(result.RampSteps, ramp)
|
||||
}
|
||||
|
||||
// Populate StablePowerLimitW on each GPU entry from the accumulated stable limits.
|
||||
for i := range result.GPUs {
|
||||
if lim, ok := stableLimits[result.GPUs[i].Index]; ok {
|
||||
result.GPUs[i].StablePowerLimitW = float64(lim)
|
||||
}
|
||||
if result.GPUs[i].StablePowerLimitW > 0 && result.GPUs[i].AppliedPowerLimitW > 0 &&
|
||||
result.GPUs[i].StablePowerLimitW < result.GPUs[i].AppliedPowerLimitW {
|
||||
result.GPUs[i].Derated = true
|
||||
result.Findings = append(result.Findings, fmt.Sprintf(
|
||||
"GPU %d required additional derating from %.0f W (single-card) to %.0f W under full-system thermal load.",
|
||||
result.GPUs[i].Index, result.GPUs[i].AppliedPowerLimitW, result.GPUs[i].StablePowerLimitW,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
// PlatformMaxTDPW = sum of all stable limits — the actual sustained power
|
||||
// budget of this server with all GPUs running simultaneously without throttling.
|
||||
for _, lim := range stableLimits {
|
||||
result.PlatformMaxTDPW += float64(lim)
|
||||
}
|
||||
|
||||
// Characterize server power from IPMI idle/loaded samples.
|
||||
// gpuActualSumW = sum of p95 GPU power from the last ramp step — actual
|
||||
// measured consumption, not the stable limit cap. This is the correct
|
||||
// denominator for the reporting ratio: limit caps (PlatformMaxTDPW) inflate
|
||||
// the denominator and make the ratio appear artificially low.
|
||||
var gpuActualSumW float64
|
||||
if n := len(result.RampSteps); n > 0 {
|
||||
gpuActualSumW = result.RampSteps[n-1].TotalObservedPowerW
|
||||
}
|
||||
if gpuActualSumW <= 0 {
|
||||
gpuActualSumW = result.PlatformMaxTDPW
|
||||
}
|
||||
_ = serverIdleOK // used implicitly via characterizeServerPower
|
||||
result.ServerPower = characterizeServerPower(serverIdleW, serverLoadedW, gpuActualSumW, opts.ServerPowerSource, serverIdleOK && serverLoadedOK)
|
||||
// Supplement DCMI with SDR multi-source data via collector's PSU slot patterns.
|
||||
// Per-slot readings enable correlation with audit HardwarePowerSupply entries.
|
||||
if result.ServerPower != nil {
|
||||
// Use the SDR phase average from the last ramp step (GPUs still loaded)
|
||||
// rather than re-sampling here, which would capture post-test idle state.
|
||||
sdrLoaded := sdrLastStep
|
||||
result.ServerPower.PSUInputIdleW = sdrIdle.PSUInW
|
||||
result.ServerPower.PSUInputLoadedW = sdrLoaded.PSUInW
|
||||
result.ServerPower.PSUOutputIdleW = sdrIdle.PSUOutW
|
||||
result.ServerPower.PSUOutputLoadedW = sdrLoaded.PSUOutW
|
||||
result.ServerPower.GPUSlotTotalW = sdrLoaded.GPUSlotW
|
||||
if len(sdrIdle.PSUSlots) > 0 {
|
||||
result.ServerPower.PSUSlotReadingsIdle = sdrIdle.PSUSlots
|
||||
}
|
||||
if len(sdrLoaded.PSUSlots) > 0 {
|
||||
result.ServerPower.PSUSlotReadingsLoaded = sdrLoaded.PSUSlots
|
||||
}
|
||||
if sdrIdle.PSUInW > 0 && result.ServerPower.IdleW > 0 {
|
||||
result.ServerPower.DCMICoverageRatio = result.ServerPower.IdleW / sdrIdle.PSUInW
|
||||
}
|
||||
if len(sdrLoaded.SkippedSensors) > 0 {
|
||||
result.ServerPower.Notes = append(result.ServerPower.Notes,
|
||||
"SDR sensors skipped (self-healed): "+strings.Join(sdrLoaded.SkippedSensors, "; "))
|
||||
}
|
||||
if sdrLoaded.Samples > 0 {
|
||||
result.ServerPower.Notes = append(result.ServerPower.Notes,
|
||||
fmt.Sprintf("Final SDR PSU loaded values are phase averages across %d sample(s) from the last full-load step.", sdrLoaded.Samples))
|
||||
}
|
||||
// Detect DCMI partial coverage: direct SDR comparison first,
|
||||
// ramp heuristic as fallback when SDR PSU sensors are absent.
|
||||
dcmiUnreliable := detectDCMIPartialCoverage(result.ServerPower) ||
|
||||
(sdrIdle.PSUInW == 0 && detectIPMISaturationFallback(result.RampSteps))
|
||||
if dcmiUnreliable {
|
||||
result.ServerPower.Notes = append(result.ServerPower.Notes,
|
||||
fmt.Sprintf("IPMI DCMI covers only a subset of installed PSUs (coverage %.0f%%). "+
|
||||
"Use SDR PSU Δ ratio for GPU accuracy assessment; DCMI ratio is not reliable.",
|
||||
result.ServerPower.DCMICoverageRatio*100))
|
||||
}
|
||||
}
|
||||
result.PSUIssues = diffPSUStatus(psuBefore, psuStatusSnapshot())
|
||||
// Write top-level gpu-metrics.csv/.html aggregating all phases.
|
||||
writeBenchmarkMetricsFiles(runDir, allPowerRows)
|
||||
resultJSON, err := json.MarshalIndent(result, "", " ")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal power result: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(runDir, "result.json"), resultJSON, 0644); err != nil {
|
||||
return "", fmt.Errorf("write result.json: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(runDir, "report.md"), []byte(renderPowerBenchReport(result)), 0644); err != nil {
|
||||
return "", fmt.Errorf("write report.md: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(renderPowerBenchSummary(result)), 0644); err != nil {
|
||||
return "", fmt.Errorf("write summary.txt: %w", err)
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
Reference in New Issue
Block a user