refactor: modularize audit and harden build validation
This commit is contained in:
Mikhail Chusavitin
@@ -0,0 +1,537 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func runBenchmarkInterconnect(ctx context.Context, verboseLog, runDir string, gpuIndices []int, spec benchmarkProfileSpec, logFunc func(string)) *BenchmarkInterconnectResult {
|
||||
result := &BenchmarkInterconnectResult{
|
||||
Status: "UNSUPPORTED",
|
||||
Attempted: true,
|
||||
SelectedGPUIndices: append([]int(nil), gpuIndices...),
|
||||
}
|
||||
cmd := []string{
|
||||
"all_reduce_perf",
|
||||
"-b", "512M",
|
||||
"-e", "4G",
|
||||
"-f", "2",
|
||||
"-g", strconv.Itoa(len(gpuIndices)),
|
||||
"--iters", strconv.Itoa(maxInt(20, spec.NCCLSec/10)),
|
||||
}
|
||||
env := []string{
|
||||
"CUDA_DEVICE_ORDER=PCI_BUS_ID",
|
||||
"CUDA_VISIBLE_DEVICES=" + joinIndexList(gpuIndices),
|
||||
}
|
||||
logFunc(fmt.Sprintf("NCCL interconnect: gpus=%s", joinIndexList(gpuIndices)))
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, "nccl-all-reduce.log", cmd, env, logFunc)
|
||||
_ = os.WriteFile(filepath.Join(runDir, "nccl-all-reduce.log"), out, 0644)
|
||||
if err != nil {
|
||||
result.Notes = append(result.Notes, strings.TrimSpace(string(out)))
|
||||
return result
|
||||
}
|
||||
avgAlg, maxAlg, avgBus, maxBus := parseNCCLAllReduceOutput(string(out))
|
||||
result.Status = "OK"
|
||||
result.Supported = true
|
||||
result.AvgAlgBWGBps = avgAlg
|
||||
result.MaxAlgBWGBps = maxAlg
|
||||
result.AvgBusBWGBps = avgBus
|
||||
result.MaxBusBWGBps = maxBus
|
||||
return result
|
||||
}
|
||||
|
||||
func parseNCCLAllReduceOutput(raw string) (avgAlg, maxAlg, avgBus, maxBus float64) {
|
||||
lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
|
||||
var algs []float64
|
||||
var buses []float64
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 8 {
|
||||
continue
|
||||
}
|
||||
for i := 0; i+2 < len(fields); i++ {
|
||||
timeVal, err1 := strconv.ParseFloat(fields[i], 64)
|
||||
algVal, err2 := strconv.ParseFloat(fields[i+1], 64)
|
||||
busVal, err3 := strconv.ParseFloat(fields[i+2], 64)
|
||||
if err1 == nil && err2 == nil && err3 == nil && timeVal > 0 {
|
||||
algs = append(algs, algVal)
|
||||
buses = append(buses, busVal)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(algs) == 0 {
|
||||
return 0, 0, 0, 0
|
||||
}
|
||||
return benchmarkMean(algs), benchmarkMax(algs), benchmarkMean(buses), benchmarkMax(buses)
|
||||
}
|
||||
|
||||
func queryThrottleCounters(gpuIndex int) (BenchmarkThrottleCounters, error) {
|
||||
out, err := satExecCommand(
|
||||
"nvidia-smi",
|
||||
"--id="+strconv.Itoa(gpuIndex),
|
||||
"--query-gpu=clocks_event_reasons_counters.sw_power_cap,clocks_event_reasons_counters.sw_thermal_slowdown,clocks_event_reasons_counters.sync_boost,clocks_event_reasons_counters.hw_thermal_slowdown,clocks_event_reasons_counters.hw_power_brake_slowdown",
|
||||
"--format=csv,noheader,nounits",
|
||||
).Output()
|
||||
if err != nil {
|
||||
return BenchmarkThrottleCounters{}, err
|
||||
}
|
||||
fields := strings.Split(strings.TrimSpace(string(out)), ",")
|
||||
if len(fields) < 5 {
|
||||
return BenchmarkThrottleCounters{}, fmt.Errorf("unexpected throttle counter columns: %q", strings.TrimSpace(string(out)))
|
||||
}
|
||||
return BenchmarkThrottleCounters{
|
||||
SWPowerCapUS: parseBenchmarkUint64(fields[0]),
|
||||
SWThermalSlowdownUS: parseBenchmarkUint64(fields[1]),
|
||||
SyncBoostUS: parseBenchmarkUint64(fields[2]),
|
||||
HWThermalSlowdownUS: parseBenchmarkUint64(fields[3]),
|
||||
HWPowerBrakeSlowdownUS: parseBenchmarkUint64(fields[4]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func diffThrottleCounters(before, after BenchmarkThrottleCounters) BenchmarkThrottleCounters {
|
||||
return BenchmarkThrottleCounters{
|
||||
SWPowerCapUS: saturatingSub(after.SWPowerCapUS, before.SWPowerCapUS),
|
||||
SWThermalSlowdownUS: saturatingSub(after.SWThermalSlowdownUS, before.SWThermalSlowdownUS),
|
||||
SyncBoostUS: saturatingSub(after.SyncBoostUS, before.SyncBoostUS),
|
||||
HWThermalSlowdownUS: saturatingSub(after.HWThermalSlowdownUS, before.HWThermalSlowdownUS),
|
||||
HWPowerBrakeSlowdownUS: saturatingSub(after.HWPowerBrakeSlowdownUS, before.HWPowerBrakeSlowdownUS),
|
||||
}
|
||||
}
|
||||
|
||||
func queryECCCounters(gpuIndex int) (BenchmarkECCCounters, error) {
|
||||
out, err := satExecCommand(
|
||||
"nvidia-smi",
|
||||
"--id="+strconv.Itoa(gpuIndex),
|
||||
"--query-gpu=ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total",
|
||||
"--format=csv,noheader,nounits",
|
||||
).Output()
|
||||
if err != nil {
|
||||
return BenchmarkECCCounters{}, err
|
||||
}
|
||||
fields := strings.Split(strings.TrimSpace(string(out)), ",")
|
||||
if len(fields) < 2 {
|
||||
return BenchmarkECCCounters{}, fmt.Errorf("unexpected ECC counter columns: %q", strings.TrimSpace(string(out)))
|
||||
}
|
||||
corrected, err1 := strconv.ParseUint(strings.TrimSpace(fields[0]), 10, 64)
|
||||
uncorrected, err2 := strconv.ParseUint(strings.TrimSpace(fields[1]), 10, 64)
|
||||
if err1 != nil || err2 != nil {
|
||||
// ECC may be disabled on this GPU — return zero counters silently.
|
||||
return BenchmarkECCCounters{}, nil
|
||||
}
|
||||
return BenchmarkECCCounters{Corrected: corrected, Uncorrected: uncorrected}, nil
|
||||
}
|
||||
|
||||
func diffECCCounters(before, after BenchmarkECCCounters) BenchmarkECCCounters {
|
||||
return BenchmarkECCCounters{
|
||||
Corrected: saturatingSub(after.Corrected, before.Corrected),
|
||||
Uncorrected: saturatingSub(after.Uncorrected, before.Uncorrected),
|
||||
}
|
||||
}
|
||||
|
||||
func queryActiveComputeApps(gpuIndices []int) ([]string, error) {
|
||||
args := []string{
|
||||
"--query-compute-apps=gpu_uuid,pid,process_name",
|
||||
"--format=csv,noheader,nounits",
|
||||
}
|
||||
if len(gpuIndices) > 0 {
|
||||
args = append([]string{"--id=" + joinIndexList(gpuIndices)}, args...)
|
||||
}
|
||||
out, err := satExecCommand("nvidia-smi", args...).Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lines []string
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
func finalizeBenchmarkGPUResult(gpu BenchmarkGPUResult) BenchmarkGPUResult {
|
||||
if gpu.Status == "" {
|
||||
gpu.Status = "OK"
|
||||
}
|
||||
if gpu.Scores.CompositeScore == 0 {
|
||||
gpu.Scores.CompositeScore = gpu.Scores.ComputeScore
|
||||
}
|
||||
return gpu
|
||||
}
|
||||
|
||||
func buildBenchmarkFindings(result NvidiaBenchmarkResult) []string {
|
||||
var findings []string
|
||||
|
||||
passed := 0
|
||||
for _, gpu := range result.GPUs {
|
||||
if gpu.Status == "OK" {
|
||||
passed++
|
||||
}
|
||||
}
|
||||
total := len(result.GPUs)
|
||||
if total > 0 {
|
||||
if passed == total {
|
||||
findings = append(findings, fmt.Sprintf("All %d GPU(s) passed the benchmark.", total))
|
||||
} else {
|
||||
findings = append(findings, fmt.Sprintf("%d of %d GPU(s) passed the benchmark.", passed, total))
|
||||
}
|
||||
}
|
||||
|
||||
if result.Normalization.Status != "full" {
|
||||
findings = append(findings, "Environment normalization was partial; compare results with caution.")
|
||||
}
|
||||
for _, gpu := range result.GPUs {
|
||||
if gpu.Status == "FAILED" && len(gpu.DegradationReasons) == 0 {
|
||||
findings = append(findings, fmt.Sprintf("GPU %d failed the benchmark (check verbose.log for details).", gpu.Index))
|
||||
continue
|
||||
}
|
||||
if len(gpu.DegradationReasons) == 0 && gpu.Status == "OK" {
|
||||
findings = append(findings, fmt.Sprintf("GPU %d held clocks without observable throttle counters during steady state.", gpu.Index))
|
||||
continue
|
||||
}
|
||||
for _, reason := range gpu.DegradationReasons {
|
||||
switch reason {
|
||||
case "power_capped":
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[POWER] GPU %d: power cap throttle %.1f%% of steady state — server is not delivering full TDP to the GPU.",
|
||||
gpu.Index, gpu.Scores.PowerCapThrottlePct))
|
||||
case "thermal_limited":
|
||||
// Hard stop check: thermal throttle while fans are not at maximum.
|
||||
// This means the server does not see GPU thermals — incompatible config.
|
||||
if result.Cooling != nil && result.Cooling.FanDutyCycleAvailable &&
|
||||
result.Cooling.P95FanDutyCyclePct < 95 {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[HARD STOP] GPU %d: thermal throttle (%.1f%% of time) while fans peaked at only %.0f%% duty cycle — server cooling is not responding to GPU heat load. Configuration is likely incompatible.",
|
||||
gpu.Index, gpu.Scores.ThermalThrottlePct, result.Cooling.P95FanDutyCyclePct))
|
||||
} else {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[THERMAL] GPU %d: thermal throttle %.1f%% of steady state.",
|
||||
gpu.Index, gpu.Scores.ThermalThrottlePct))
|
||||
}
|
||||
case "sync_boost_limited":
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[SYNC] GPU %d: sync boost throttle %.1f%% of steady state — GPUs are constraining each other's clocks.",
|
||||
gpu.Index, gpu.Scores.SyncBoostThrottlePct))
|
||||
case "low_sm_clock_vs_target":
|
||||
findings = append(findings, fmt.Sprintf("GPU %d average SM clock stayed below the requested lock target.", gpu.Index))
|
||||
case "variance_too_high":
|
||||
findings = append(findings, fmt.Sprintf("GPU %d showed unstable clocks/power over the benchmark window.", gpu.Index))
|
||||
case "normalization_partial":
|
||||
findings = append(findings, fmt.Sprintf("GPU %d ran without full benchmark normalization.", gpu.Index))
|
||||
case "power_limit_derated":
|
||||
findings = append(findings, fmt.Sprintf("[POWER] GPU %d could not sustain full TDP in this server; benchmark ran at reduced limit %.0f W.", gpu.Index, gpu.PowerLimitW))
|
||||
case "ecc_uncorrected_errors":
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[HARD STOP] GPU %d: %d uncorrected ECC error(s) detected — possible hardware fault. Do not use in production.",
|
||||
gpu.Index, gpu.ECC.Uncorrected))
|
||||
case "ecc_corrected_errors":
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[WARNING] GPU %d: %d corrected ECC error(s) — possible DRAM degradation, monitor closely.",
|
||||
gpu.Index, gpu.ECC.Corrected))
|
||||
}
|
||||
}
|
||||
// Temperature headroom checks — independent of throttle counters.
|
||||
// Shutdown and slowdown thresholds are per-GPU from nvidia-smi -q;
|
||||
// fall back to 90°C / 80°C when unavailable.
|
||||
if gpu.Steady.P95TempC > 0 {
|
||||
shutdownTemp := gpu.ShutdownTempC
|
||||
if shutdownTemp <= 0 {
|
||||
shutdownTemp = 90
|
||||
}
|
||||
slowdownTemp := gpu.SlowdownTempC
|
||||
if slowdownTemp <= 0 {
|
||||
slowdownTemp = 80
|
||||
}
|
||||
headroom := shutdownTemp - gpu.Steady.P95TempC
|
||||
switch {
|
||||
case headroom < 10:
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[HARD STOP] GPU %d: p95 temperature %.1f°C — only %.1f°C from shutdown threshold (%.0f°C). Do not operate.",
|
||||
gpu.Index, gpu.Steady.P95TempC, headroom, shutdownTemp))
|
||||
case gpu.Steady.P95TempC >= slowdownTemp:
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"[THERMAL] GPU %d: p95 temperature %.1f°C exceeds slowdown threshold (%.0f°C) — %.1f°C headroom to shutdown. Operating in degraded reliability zone.",
|
||||
gpu.Index, gpu.Steady.P95TempC, slowdownTemp, headroom))
|
||||
}
|
||||
}
|
||||
if gpu.CoolingWarning != "" {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"GPU %d: %s. Operator action: rerun the benchmark with fan speed manually fixed at 100%% to confirm actual thermal headroom.",
|
||||
gpu.Index, gpu.CoolingWarning,
|
||||
))
|
||||
}
|
||||
if len(gpu.PrecisionFailures) > 0 {
|
||||
findings = append(findings, fmt.Sprintf("GPU %d had incomplete precision coverage: %s.", gpu.Index, strings.Join(gpu.PrecisionFailures, ", ")))
|
||||
}
|
||||
if gpu.Backend == "driver-ptx" {
|
||||
findings = append(findings, fmt.Sprintf("GPU %d used driver PTX fallback; tensor score is intentionally degraded.", gpu.Index))
|
||||
}
|
||||
if gpu.DefaultPowerLimitW > 0 && gpu.PowerLimitW > 0 && gpu.PowerLimitW < gpu.DefaultPowerLimitW*0.95 {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"GPU %d power limit %.0f W is below default %.0f W (%.0f%%). Performance may be artificially reduced.",
|
||||
gpu.Index, gpu.PowerLimitW, gpu.DefaultPowerLimitW, gpu.PowerLimitW/gpu.DefaultPowerLimitW*100,
|
||||
))
|
||||
}
|
||||
// Flag significant TDP deviation (over or under) from calibration.
|
||||
if gpu.CalibratedPeakPowerW > 0 {
|
||||
ref := gpu.DefaultPowerLimitW
|
||||
if ref <= 0 {
|
||||
ref = gpu.PowerLimitW
|
||||
}
|
||||
if ref > 0 {
|
||||
deviationPct := (gpu.CalibratedPeakPowerW - ref) / ref * 100
|
||||
switch {
|
||||
case deviationPct < -10:
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"GPU %d reached only %.0f W (%.0f%% of rated %.0f W) under targeted_power. Check power delivery or cooling.",
|
||||
gpu.Index, gpu.CalibratedPeakPowerW, gpu.CalibratedPeakPowerW/ref*100, ref,
|
||||
))
|
||||
case deviationPct > 5:
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"GPU %d exceeded rated TDP: %.0f W measured vs %.0f W rated (+%.0f%%). Power limit may not be enforced correctly.",
|
||||
gpu.Index, gpu.CalibratedPeakPowerW, ref, deviationPct,
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if result.Interconnect != nil && result.Interconnect.Supported {
|
||||
findings = append(findings, fmt.Sprintf("Multi-GPU all_reduce max bus bandwidth: %.1f GB/s.", result.Interconnect.MaxBusBWGBps))
|
||||
}
|
||||
if cl := result.CPULoad; cl != nil {
|
||||
switch cl.Status {
|
||||
case "high":
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"Host CPU load was elevated during the benchmark (avg %.1f%%, max %.1f%%). A competing CPU workload may skew GPU results.",
|
||||
cl.AvgPct, cl.MaxPct,
|
||||
))
|
||||
case "unstable":
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"Host CPU load was erratic during the benchmark (avg %.1f%%, p95 %.1f%%). Results may be less reproducible.",
|
||||
cl.AvgPct, cl.P95Pct,
|
||||
))
|
||||
}
|
||||
}
|
||||
if sp := result.ServerPower; sp != nil && sp.Available && sp.GPUReportedSumW > 0 {
|
||||
dcmiPartial := detectDCMIPartialCoverage(sp)
|
||||
if sp.ReportingRatio < 0.75 && !dcmiPartial {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"GPU power reporting may be unreliable: server delta %.0f W vs GPU-reported %.0f W (ratio %.2f). GPU telemetry likely over-reports actual consumption. Composite scores have been penalized accordingly.",
|
||||
sp.DeltaW, sp.GPUReportedSumW, sp.ReportingRatio,
|
||||
))
|
||||
} else if sp.ReportingRatio < 0.75 && dcmiPartial {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"IPMI DCMI covers partial PSU set (DCMI/SDR coverage %.0f%%): ratio %.2f reflects DCMI under-reporting, not GPU inaccuracy. GPU telemetry is the reliable power source; use SDR-based ratio for server-side accuracy.",
|
||||
sp.DCMICoverageRatio*100, sp.ReportingRatio,
|
||||
))
|
||||
} else if sp.ReportingRatio > 1.25 {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"Server power delta %.0f W exceeds GPU-reported sum %.0f W by %.0f%%. Other components (CPU, NVMe, networking) may be drawing substantial power under GPU load.",
|
||||
sp.DeltaW, sp.GPUReportedSumW, (sp.ReportingRatio-1)*100,
|
||||
))
|
||||
}
|
||||
}
|
||||
return dedupeStrings(findings)
|
||||
}
|
||||
|
||||
func benchmarkOverallStatus(result NvidiaBenchmarkResult) string {
|
||||
if len(result.GPUs) == 0 {
|
||||
return "FAILED"
|
||||
}
|
||||
hasOK := false
|
||||
hasPartial := result.Normalization.Status != "full"
|
||||
for _, gpu := range result.GPUs {
|
||||
switch gpu.Status {
|
||||
case "OK":
|
||||
hasOK = true
|
||||
case "PARTIAL", "UNSUPPORTED":
|
||||
hasPartial = true
|
||||
}
|
||||
}
|
||||
if !hasOK {
|
||||
return "FAILED"
|
||||
}
|
||||
if hasPartial {
|
||||
return "PARTIAL"
|
||||
}
|
||||
return "OK"
|
||||
}
|
||||
|
||||
func findBenchmarkNormalization(items []BenchmarkNormalizationGPU, idx int) *BenchmarkNormalizationGPU {
|
||||
for i := range items {
|
||||
if items[i].Index == idx {
|
||||
return &items[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func classifySATErrorStatus(out []byte, err error) string {
|
||||
status, _ := classifySATResult("benchmark", out, err)
|
||||
if status == "UNSUPPORTED" {
|
||||
return "UNSUPPORTED"
|
||||
}
|
||||
return "FAILED"
|
||||
}
|
||||
|
||||
func parseBenchmarkFloat(raw string) float64 {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" || strings.EqualFold(raw, "n/a") || strings.EqualFold(raw, "[not supported]") {
|
||||
return 0
|
||||
}
|
||||
value, _ := strconv.ParseFloat(raw, 64)
|
||||
return value
|
||||
}
|
||||
|
||||
func parseBenchmarkUint64(raw string) uint64 {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" || strings.EqualFold(raw, "n/a") || strings.EqualFold(raw, "[not supported]") {
|
||||
return 0
|
||||
}
|
||||
value, _ := strconv.ParseUint(raw, 10, 64)
|
||||
return value
|
||||
}
|
||||
|
||||
func benchmarkMean(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
var sum float64
|
||||
for _, value := range values {
|
||||
sum += value
|
||||
}
|
||||
return sum / float64(len(values))
|
||||
}
|
||||
|
||||
func benchmarkPercentile(values []float64, p float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
copyValues := append([]float64(nil), values...)
|
||||
sort.Float64s(copyValues)
|
||||
if len(copyValues) == 1 {
|
||||
return copyValues[0]
|
||||
}
|
||||
rank := (p / 100.0) * float64(len(copyValues)-1)
|
||||
lower := int(math.Floor(rank))
|
||||
upper := int(math.Ceil(rank))
|
||||
if lower == upper {
|
||||
return copyValues[lower]
|
||||
}
|
||||
frac := rank - float64(lower)
|
||||
return copyValues[lower] + (copyValues[upper]-copyValues[lower])*frac
|
||||
}
|
||||
|
||||
func benchmarkCV(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
mean := benchmarkMean(values)
|
||||
if mean == 0 {
|
||||
return 0
|
||||
}
|
||||
var variance float64
|
||||
for _, value := range values {
|
||||
diff := value - mean
|
||||
variance += diff * diff
|
||||
}
|
||||
variance /= float64(len(values))
|
||||
return math.Sqrt(variance) / mean * 100
|
||||
}
|
||||
|
||||
func benchmarkClockDrift(values []float64) float64 {
|
||||
if len(values) < 4 {
|
||||
return 0
|
||||
}
|
||||
window := len(values) / 4
|
||||
if window < 1 {
|
||||
window = 1
|
||||
}
|
||||
head := benchmarkMean(values[:window])
|
||||
tail := benchmarkMean(values[len(values)-window:])
|
||||
if head <= 0 || tail >= head {
|
||||
return 0
|
||||
}
|
||||
return ((head - tail) / head) * 100
|
||||
}
|
||||
|
||||
func benchmarkMax(values []float64) float64 {
|
||||
var max float64
|
||||
for i, value := range values {
|
||||
if i == 0 || value > max {
|
||||
max = value
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
func clampScore(value float64) float64 {
|
||||
switch {
|
||||
case value < 0:
|
||||
return 0
|
||||
case value > 100:
|
||||
return 100
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func dedupeStrings(values []string) []string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[value]; ok {
|
||||
continue
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
out = append(out, value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func saturatingSub(after, before uint64) uint64 {
|
||||
if after <= before {
|
||||
return 0
|
||||
}
|
||||
return after - before
|
||||
}
|
||||
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// detectDCMIPartialCoverage returns true when IPMI DCMI under-reports actual
|
||||
// server power by comparing DCMI readings against SDR PSUx_POWER_IN sensor sums.
|
||||
//
|
||||
// Primary check: DCMI_idle / SDR_PSU_IN_idle — most reliable because GPU load
|
||||
// is zero, so both sources measure the same server state. A ratio below 0.7
|
||||
// means DCMI misses ≥30% of installed PSUs (e.g. 0.50 = sees 2 of 4 PSUs).
|
||||
//
|
||||
// Fallback: DCMI_loaded / SDR_PSU_IN_loaded — less precise (GPU load may
|
||||
// affect different PSUs differently) but still useful when idle SDR is absent.
|
||||
//
|
||||
// Returns false when SDR data is unavailable (server has no PSUx_POWER_IN
|
||||
// sensors); the heuristic detectIPMISaturationFallback is used in that case.
|
||||
Reference in New Issue
Block a user