refactor: harden diagnostics and consolidate runtime code

This commit is contained in:
Mikhail Chusavitin
2026-09-01 13:01:28 +03:00
parent ac4bc0b2b7
commit 0a6ca8ba0f
49 changed files with 1441 additions and 837 deletions
@@ -406,39 +406,6 @@ func queryIPMIServerPowerW() (float64, error) {
return 0, fmt.Errorf("could not parse ipmitool dcmi power reading output")
}
// sampleIPMIPowerSeries collects IPMI power readings every 2 seconds for
// durationSec seconds. Returns the mean of all successful samples.
// Returns 0, false if IPMI is unavailable.
func sampleIPMIPowerSeries(ctx context.Context, durationSec int) (meanW float64, ok bool) {
if durationSec <= 0 {
return 0, false
}
deadline := time.Now().Add(time.Duration(durationSec) * time.Second)
var samples []float64
loop:
for {
if w, err := queryIPMIServerPowerW(); err == nil {
samples = append(samples, w)
}
if time.Now().After(deadline) {
break
}
select {
case <-ctx.Done():
break loop
case <-time.After(2 * time.Second):
}
}
if len(samples) == 0 {
return 0, false
}
var sum float64
for _, w := range samples {
sum += w
}
return sum / float64(len(samples)), true
}
// characterizeServerPower computes BenchmarkServerPower from idle and loaded
// samples plus the GPU-reported average power during steady state.
func characterizeServerPower(idleW, loadedW, gpuReportedSumW float64, source string, available bool) *BenchmarkServerPower {
@@ -535,18 +502,7 @@ func runNvidiaBenchmarkParallel(
for _, idx := range selected {
r := &BenchmarkGPUResult{Index: idx, Status: "FAILED"}
if info, ok := infoByIndex[idx]; ok {
r.UUID = info.UUID
r.Name = info.Name
r.BusID = info.BusID
r.VBIOS = info.VBIOS
r.PowerLimitW = info.PowerLimitW
r.MultiprocessorCount = info.MultiprocessorCount
r.DefaultPowerLimitW = info.DefaultPowerLimitW
r.ShutdownTempC = info.ShutdownTempC
r.SlowdownTempC = info.SlowdownTempC
r.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz
r.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz
r.MaxMemoryClockMHz = info.MaxMemoryClockMHz
populateBenchmarkGPUInfo(r, info)
}
if calib, ok := calibByIndex[idx]; ok {
r.CalibratedPeakPowerW = calib.Summary.P95PowerW
@@ -753,6 +709,21 @@ func runNvidiaBenchmarkParallel(
}
}
func populateBenchmarkGPUInfo(result *BenchmarkGPUResult, info benchmarkGPUInfo) {
result.UUID = info.UUID
result.Name = info.Name
result.BusID = info.BusID
result.VBIOS = info.VBIOS
result.PowerLimitW = info.PowerLimitW
result.MultiprocessorCount = info.MultiprocessorCount
result.DefaultPowerLimitW = info.DefaultPowerLimitW
result.ShutdownTempC = info.ShutdownTempC
result.SlowdownTempC = info.SlowdownTempC
result.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz
result.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz
result.MaxMemoryClockMHz = info.MaxMemoryClockMHz
}
// readBenchmarkHostConfig reads static CPU and memory configuration from
// /proc/cpuinfo and /proc/meminfo. Returns nil if neither source is readable.
func readBenchmarkHostConfig() *BenchmarkHostConfig {