refactor: modularize audit and harden build validation
This commit is contained in:
Mikhail Chusavitin
@@ -0,0 +1,928 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"bee/audit/internal/collector"
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func detectDCMIPartialCoverage(sp *BenchmarkServerPower) bool {
|
||||
if sp == nil || !sp.Available {
|
||||
return false
|
||||
}
|
||||
if sp.PSUInputIdleW > 0 && sp.IdleW > 0 {
|
||||
return sp.IdleW/sp.PSUInputIdleW < 0.7
|
||||
}
|
||||
if sp.PSUInputLoadedW > 0 && sp.LoadedW > 0 {
|
||||
return sp.LoadedW/sp.PSUInputLoadedW < 0.7
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// detectIPMISaturationFallback is the heuristic used when SDR PSU sensors are
|
||||
// absent. It analyses the power ramp: if 2+ of the last 3 per-step incremental
|
||||
// DCMI/GPU ratios fall below 25% of the first-step ratio, DCMI has likely
|
||||
// plateaued while GPU load continued to grow (saturation proxy).
|
||||
// Prefer detectDCMIPartialCoverage when SDR data is available.
|
||||
func detectIPMISaturationFallback(steps []NvidiaPowerBenchStep) bool {
|
||||
type pt struct{ incIPMI, incGPU float64 }
|
||||
var pts []pt
|
||||
for i := 1; i < len(steps); i++ {
|
||||
if steps[i].ServerDeltaW <= 0 || steps[i-1].ServerDeltaW <= 0 {
|
||||
continue
|
||||
}
|
||||
incIPMI := steps[i].ServerDeltaW - steps[i-1].ServerDeltaW
|
||||
incGPU := steps[i].TotalObservedPowerW - steps[i-1].TotalObservedPowerW
|
||||
if incGPU <= 0 {
|
||||
continue
|
||||
}
|
||||
pts = append(pts, pt{incIPMI, incGPU})
|
||||
}
|
||||
if len(pts) < 3 {
|
||||
return false
|
||||
}
|
||||
refRatio := pts[0].incIPMI / pts[0].incGPU
|
||||
if refRatio <= 0 {
|
||||
return false
|
||||
}
|
||||
saturated := 0
|
||||
for _, p := range pts[len(pts)-3:] {
|
||||
if p.incIPMI/p.incGPU < refRatio*0.25 {
|
||||
saturated++
|
||||
}
|
||||
}
|
||||
return saturated >= 2
|
||||
}
|
||||
|
||||
// psuStatusSnapshot samples PSU health sensor states via
|
||||
// `ipmitool sdr type "Power Supply"`. Returns a map of sensor name → reading
|
||||
// string (e.g. "Presence detected", "Failure detected"). Returns nil when IPMI
|
||||
// is unavailable or no Power Supply entity sensors are present.
|
||||
func psuStatusSnapshot() map[string]string {
|
||||
out, err := exec.Command("ipmitool", "sdr", "type", "Power Supply").Output()
|
||||
if err != nil || len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
result := make(map[string]string)
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
parts := strings.Split(line, "|")
|
||||
if len(parts) < 5 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(parts[0])
|
||||
reading := strings.TrimSpace(parts[4])
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
result[name] = reading
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// diffPSUStatus compares PSU sensor snapshots taken before and after a test.
|
||||
// Returns human-readable fault strings for sensors that entered a fault state
|
||||
// during the test. Pre-existing faults (present in both snapshots) are excluded
|
||||
// so that only new anomalies caused by the test are reported.
|
||||
func diffPSUStatus(before, after map[string]string) []string {
|
||||
if len(after) == 0 {
|
||||
return nil
|
||||
}
|
||||
isFault := func(s string) bool {
|
||||
lower := strings.ToLower(s)
|
||||
return strings.Contains(lower, "failure") ||
|
||||
strings.Contains(lower, "fault") ||
|
||||
strings.Contains(lower, "warning") ||
|
||||
strings.Contains(lower, "predictive") ||
|
||||
strings.Contains(lower, "absent") ||
|
||||
strings.Contains(lower, "ac lost")
|
||||
}
|
||||
var issues []string
|
||||
for name, afterReading := range after {
|
||||
if !isFault(afterReading) {
|
||||
continue
|
||||
}
|
||||
if beforeReading, had := before[name]; had && isFault(beforeReading) {
|
||||
continue // pre-existing fault, not caused by this test
|
||||
}
|
||||
if prev, had := before[name]; had {
|
||||
issues = append(issues, fmt.Sprintf("%s: changed from %q to %q during test", name, prev, afterReading))
|
||||
} else {
|
||||
issues = append(issues, fmt.Sprintf("%s: %s (appeared after test start)", name, afterReading))
|
||||
}
|
||||
}
|
||||
sort.Strings(issues)
|
||||
return issues
|
||||
}
|
||||
|
||||
// sdrPowerSnapshot holds per-source power sums from a single `ipmitool sdr` read.
|
||||
type sdrPowerSnapshot struct {
|
||||
PSUInW float64 // sum of PSU AC input across all slots
|
||||
PSUOutW float64 // sum of PSU DC output across all slots
|
||||
GPUSlotW float64 // sum of GPU slot/GPU power sensors
|
||||
|
||||
// Per-slot PSU data from collector.PSUSlotsFromSDR — same slot keys as
|
||||
// audit HardwarePowerSupply.Slot (0-based strings).
|
||||
PSUSlots map[string]BenchmarkPSUSlotPower
|
||||
|
||||
SkippedSensors []string // sensors rejected during self-healing
|
||||
}
|
||||
|
||||
type benchmarkSDRSeriesSummary struct {
|
||||
PSUInW float64
|
||||
PSUOutW float64
|
||||
GPUSlotW float64
|
||||
PSUSlots map[string]BenchmarkPSUSlotPower
|
||||
Samples int
|
||||
|
||||
SkippedSensors []string
|
||||
}
|
||||
|
||||
// sdrSensor is a name+watts pair used for GPU slot self-healing filtering.
|
||||
type sdrSensor struct {
|
||||
name string
|
||||
watts float64
|
||||
}
|
||||
|
||||
// filterSensorGroup removes physically implausible readings from a group.
|
||||
// Hard bounds: 0 < watts ≤ maxPerSensorW. Within groups of 2+ sensors,
|
||||
// values more than 5× the group median are rejected as stuck/fault sensors.
|
||||
func filterSensorGroup(sensors []sdrSensor, maxPerSensorW float64) (valid []sdrSensor, skipped []string) {
|
||||
var inBounds []sdrSensor
|
||||
for _, s := range sensors {
|
||||
if s.watts <= 0 || s.watts > maxPerSensorW {
|
||||
skipped = append(skipped, fmt.Sprintf("%s (%.0f W: out of range 0–%.0f W)", s.name, s.watts, maxPerSensorW))
|
||||
} else {
|
||||
inBounds = append(inBounds, s)
|
||||
}
|
||||
}
|
||||
if len(inBounds) < 2 {
|
||||
return inBounds, skipped
|
||||
}
|
||||
vals := make([]float64, len(inBounds))
|
||||
for i, s := range inBounds {
|
||||
vals[i] = s.watts
|
||||
}
|
||||
sort.Float64s(vals)
|
||||
mid := len(vals) / 2
|
||||
var median float64
|
||||
if len(vals)%2 == 0 {
|
||||
median = (vals[mid-1] + vals[mid]) / 2
|
||||
} else {
|
||||
median = vals[mid]
|
||||
}
|
||||
for _, s := range inBounds {
|
||||
if median > 0 && s.watts > median*5 {
|
||||
skipped = append(skipped, fmt.Sprintf("%s (%.0f W: >5× median %.0f W, likely sensor fault)", s.name, s.watts, median))
|
||||
} else {
|
||||
valid = append(valid, s)
|
||||
}
|
||||
}
|
||||
return valid, skipped
|
||||
}
|
||||
|
||||
// sampleIPMISDRPowerSensors reads power sensors from `ipmitool sdr` in a single
|
||||
// invocation and returns self-healed grouped sums.
|
||||
//
|
||||
// PSU identification delegates to collector.PSUSlotsFromSDR which uses the same
|
||||
// slot-detection regexes as the hardware audit (PSU1_POWER_IN, PSU1_PIN, PS1 POut,
|
||||
// Power1…). Self-healing: bounds checking + 5× median outlier rejection.
|
||||
//
|
||||
// GPU slot sensors (GPU_POWER_SLOTx, GPU1 Power, …) are classified separately
|
||||
// since the audit collector does not track GPU PCIe slot power.
|
||||
func sampleIPMISDRPowerSensors() sdrPowerSnapshot {
|
||||
raw, err := exec.Command("ipmitool", "sdr").Output()
|
||||
if err != nil || len(raw) == 0 {
|
||||
return sdrPowerSnapshot{}
|
||||
}
|
||||
sdrStr := string(raw)
|
||||
var snap sdrPowerSnapshot
|
||||
|
||||
// ── PSU data via audit collector ─────────────────────────────────────────
|
||||
// collector.PSUSlotsFromSDR handles all vendor naming variants and applies
|
||||
// bounds checking inside parseBoundedFloat (0 < w ≤ 6000 W).
|
||||
collectorSlots := collector.PSUSlotsFromSDR(sdrStr)
|
||||
|
||||
// Convert to benchmark type and apply cross-slot median filtering.
|
||||
var psuInSensors, psuOutSensors []sdrSensor
|
||||
for slotKey, sp := range collectorSlots {
|
||||
bsp := BenchmarkPSUSlotPower{Status: sp.Status}
|
||||
if sp.InputW != nil {
|
||||
bsp.InputW = sp.InputW
|
||||
psuInSensors = append(psuInSensors, sdrSensor{name: "PSU-slot-" + slotKey, watts: *sp.InputW})
|
||||
}
|
||||
if sp.OutputW != nil {
|
||||
bsp.OutputW = sp.OutputW
|
||||
psuOutSensors = append(psuOutSensors, sdrSensor{name: "PSU-slot-" + slotKey + "-out", watts: *sp.OutputW})
|
||||
}
|
||||
if snap.PSUSlots == nil {
|
||||
snap.PSUSlots = make(map[string]BenchmarkPSUSlotPower)
|
||||
}
|
||||
snap.PSUSlots[slotKey] = bsp
|
||||
}
|
||||
|
||||
// Apply cross-slot outlier filter and sum.
|
||||
validIn, skIn := filterSensorGroup(psuInSensors, 6000)
|
||||
for _, s := range validIn {
|
||||
snap.PSUInW += s.watts
|
||||
}
|
||||
snap.SkippedSensors = append(snap.SkippedSensors, skIn...)
|
||||
|
||||
validOut, skOut := filterSensorGroup(psuOutSensors, 6000)
|
||||
for _, s := range validOut {
|
||||
snap.PSUOutW += s.watts
|
||||
}
|
||||
snap.SkippedSensors = append(snap.SkippedSensors, skOut...)
|
||||
|
||||
// ── GPU slot sensors ─────────────────────────────────────────────────────
|
||||
// collector does not track GPU PCIe slot power; classify here.
|
||||
// Matches: GPU_POWER_SLOTx (MSI), GPU1 Power (xFusion), GPU_PWR_x (generic).
|
||||
var gpuSensors []sdrSensor
|
||||
for _, line := range strings.Split(sdrStr, "\n") {
|
||||
parts := strings.Split(line, "|")
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(parts[0])
|
||||
nameLower := strings.ToLower(name)
|
||||
if !strings.Contains(nameLower, "gpu") {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(nameLower, "slot") && !strings.Contains(nameLower, "power") &&
|
||||
!strings.Contains(nameLower, "pwr") {
|
||||
continue
|
||||
}
|
||||
var w float64
|
||||
if n, _ := fmt.Sscanf(strings.TrimSpace(parts[1]), "%f Watts", &w); n != 1 {
|
||||
continue
|
||||
}
|
||||
gpuSensors = append(gpuSensors, sdrSensor{name: name, watts: w})
|
||||
}
|
||||
validGPU, skGPU := filterSensorGroup(gpuSensors, 2000)
|
||||
for _, s := range validGPU {
|
||||
snap.GPUSlotW += s.watts
|
||||
}
|
||||
snap.SkippedSensors = append(snap.SkippedSensors, skGPU...)
|
||||
|
||||
return snap
|
||||
}
|
||||
|
||||
func startIPMISDRSampler(stopCh <-chan struct{}, intervalSec int) <-chan []sdrPowerSnapshot {
|
||||
if intervalSec <= 0 {
|
||||
intervalSec = benchmarkPowerAutotuneSampleInterval
|
||||
}
|
||||
ch := make(chan []sdrPowerSnapshot, 1)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
var samples []sdrPowerSnapshot
|
||||
record := func() {
|
||||
snap := sampleIPMISDRPowerSensors()
|
||||
if snap.PSUInW <= 0 && snap.PSUOutW <= 0 && snap.GPUSlotW <= 0 && len(snap.PSUSlots) == 0 {
|
||||
return
|
||||
}
|
||||
samples = append(samples, snap)
|
||||
}
|
||||
record()
|
||||
ticker := time.NewTicker(time.Duration(intervalSec) * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stopCh:
|
||||
ch <- samples
|
||||
return
|
||||
case <-ticker.C:
|
||||
record()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
func summarizeSDRPowerSeries(samples []sdrPowerSnapshot) benchmarkSDRSeriesSummary {
|
||||
var summary benchmarkSDRSeriesSummary
|
||||
if len(samples) == 0 {
|
||||
return summary
|
||||
}
|
||||
|
||||
type slotAggregate struct {
|
||||
inputs []float64
|
||||
outputs []float64
|
||||
status string
|
||||
}
|
||||
|
||||
slotAgg := make(map[string]*slotAggregate)
|
||||
skippedSet := make(map[string]struct{})
|
||||
var inputTotals []float64
|
||||
var outputTotals []float64
|
||||
var gpuSlotTotals []float64
|
||||
|
||||
for _, sample := range samples {
|
||||
if sample.PSUInW > 0 {
|
||||
inputTotals = append(inputTotals, sample.PSUInW)
|
||||
}
|
||||
if sample.PSUOutW > 0 {
|
||||
outputTotals = append(outputTotals, sample.PSUOutW)
|
||||
}
|
||||
if sample.GPUSlotW > 0 {
|
||||
gpuSlotTotals = append(gpuSlotTotals, sample.GPUSlotW)
|
||||
}
|
||||
for _, skipped := range sample.SkippedSensors {
|
||||
if skipped != "" {
|
||||
skippedSet[skipped] = struct{}{}
|
||||
}
|
||||
}
|
||||
for slot, reading := range sample.PSUSlots {
|
||||
agg := slotAgg[slot]
|
||||
if agg == nil {
|
||||
agg = &slotAggregate{}
|
||||
slotAgg[slot] = agg
|
||||
}
|
||||
if reading.InputW != nil && *reading.InputW > 0 {
|
||||
agg.inputs = append(agg.inputs, *reading.InputW)
|
||||
}
|
||||
if reading.OutputW != nil && *reading.OutputW > 0 {
|
||||
agg.outputs = append(agg.outputs, *reading.OutputW)
|
||||
}
|
||||
switch {
|
||||
case reading.Status == "":
|
||||
case agg.status == "":
|
||||
agg.status = reading.Status
|
||||
case agg.status == "OK" && reading.Status != "OK":
|
||||
agg.status = reading.Status
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
summary.PSUInW = benchmarkMean(inputTotals)
|
||||
summary.PSUOutW = benchmarkMean(outputTotals)
|
||||
summary.GPUSlotW = benchmarkMean(gpuSlotTotals)
|
||||
summary.Samples = len(samples)
|
||||
|
||||
if len(slotAgg) > 0 {
|
||||
summary.PSUSlots = make(map[string]BenchmarkPSUSlotPower, len(slotAgg))
|
||||
for slot, agg := range slotAgg {
|
||||
reading := BenchmarkPSUSlotPower{Status: agg.status}
|
||||
if mean := benchmarkMean(agg.inputs); mean > 0 {
|
||||
v := mean
|
||||
reading.InputW = &v
|
||||
}
|
||||
if mean := benchmarkMean(agg.outputs); mean > 0 {
|
||||
v := mean
|
||||
reading.OutputW = &v
|
||||
}
|
||||
summary.PSUSlots[slot] = reading
|
||||
}
|
||||
}
|
||||
if len(skippedSet) > 0 {
|
||||
summary.SkippedSensors = make([]string, 0, len(skippedSet))
|
||||
for skipped := range skippedSet {
|
||||
summary.SkippedSensors = append(summary.SkippedSensors, skipped)
|
||||
}
|
||||
sort.Strings(summary.SkippedSensors)
|
||||
}
|
||||
|
||||
return summary
|
||||
}
|
||||
|
||||
// queryIPMIServerPowerW reads the current server power draw via ipmitool dcmi.
|
||||
// Returns 0 and an error if IPMI is unavailable or the output cannot be parsed.
|
||||
func queryIPMIServerPowerW() (float64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "ipmitool", "dcmi", "power", "reading")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("ipmitool dcmi power reading: %w", err)
|
||||
}
|
||||
if w := parseDCMIPowerReading(string(out)); w > 0 {
|
||||
return w, nil
|
||||
}
|
||||
return 0, fmt.Errorf("could not parse ipmitool dcmi power reading output")
|
||||
}
|
||||
|
||||
// sampleIPMIPowerSeries collects IPMI power readings every 2 seconds for
|
||||
// durationSec seconds. Returns the mean of all successful samples.
|
||||
// Returns 0, false if IPMI is unavailable.
|
||||
func sampleIPMIPowerSeries(ctx context.Context, durationSec int) (meanW float64, ok bool) {
|
||||
if durationSec <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
deadline := time.Now().Add(time.Duration(durationSec) * time.Second)
|
||||
var samples []float64
|
||||
loop:
|
||||
for {
|
||||
if w, err := queryIPMIServerPowerW(); err == nil {
|
||||
samples = append(samples, w)
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break loop
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
if len(samples) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
var sum float64
|
||||
for _, w := range samples {
|
||||
sum += w
|
||||
}
|
||||
return sum / float64(len(samples)), true
|
||||
}
|
||||
|
||||
// characterizeServerPower computes BenchmarkServerPower from idle and loaded
|
||||
// samples plus the GPU-reported average power during steady state.
|
||||
func characterizeServerPower(idleW, loadedW, gpuReportedSumW float64, source string, available bool) *BenchmarkServerPower {
|
||||
sp := &BenchmarkServerPower{
|
||||
Available: available,
|
||||
Source: normalizeBenchmarkPowerSource(source),
|
||||
SampleIntervalSec: benchmarkPowerAutotuneSampleInterval,
|
||||
}
|
||||
if !available {
|
||||
sp.Notes = append(sp.Notes, "IPMI power reading unavailable; server-side power characterization skipped")
|
||||
return sp
|
||||
}
|
||||
sp.IdleW = idleW
|
||||
sp.LoadedW = loadedW
|
||||
sp.DeltaW = loadedW - idleW
|
||||
sp.GPUReportedSumW = gpuReportedSumW
|
||||
if gpuReportedSumW > 0 && sp.DeltaW > 0 {
|
||||
sp.ReportingRatio = sp.DeltaW / gpuReportedSumW
|
||||
}
|
||||
return sp
|
||||
}
|
||||
|
||||
// readServerModel returns the DMI system product name (e.g. "SuperMicro SYS-421GE-TNRT").
|
||||
// Returns empty string if unavailable (non-Linux or missing DMI entry).
|
||||
func readServerModel() string {
|
||||
data, err := os.ReadFile("/sys/class/dmi/id/product_name")
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(data))
|
||||
}
|
||||
|
||||
// filterRowsByGPU returns only the metric rows for a specific GPU index.
|
||||
func filterRowsByGPU(rows []GPUMetricRow, gpuIndex int) []GPUMetricRow {
|
||||
var out []GPUMetricRow
|
||||
for _, r := range rows {
|
||||
if r.GPUIndex == gpuIndex {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// parseBenchmarkBurnLogByGPU splits a multi-GPU bee-gpu-burn output by [gpu N] prefix
|
||||
// and returns a per-GPU parse result map.
|
||||
func parseBenchmarkBurnLogByGPU(raw string) map[int]benchmarkBurnParseResult {
|
||||
gpuLines := make(map[int][]string)
|
||||
for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(line, "[gpu ") {
|
||||
continue
|
||||
}
|
||||
end := strings.Index(line, "] ")
|
||||
if end < 0 {
|
||||
continue
|
||||
}
|
||||
gpuIdx, err := strconv.Atoi(strings.TrimSpace(line[5:end]))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
gpuLines[gpuIdx] = append(gpuLines[gpuIdx], line[end+2:])
|
||||
}
|
||||
results := make(map[int]benchmarkBurnParseResult, len(gpuLines))
|
||||
for gpuIdx, lines := range gpuLines {
|
||||
// Lines are already stripped of the [gpu N] prefix; parseBenchmarkBurnLog
|
||||
// calls stripBenchmarkPrefix which is a no-op on already-stripped lines.
|
||||
results[gpuIdx] = parseBenchmarkBurnLog(strings.Join(lines, "\n"))
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// runNvidiaBenchmarkParallel runs warmup and steady compute on all selected GPUs
|
||||
// simultaneously using a single bee-gpu-burn invocation per phase.
|
||||
func runNvidiaBenchmarkParallel(
|
||||
ctx context.Context,
|
||||
verboseLog, runDir string,
|
||||
selected []int,
|
||||
infoByIndex map[int]benchmarkGPUInfo,
|
||||
opts NvidiaBenchmarkOptions,
|
||||
spec benchmarkProfileSpec,
|
||||
logFunc func(string),
|
||||
result *NvidiaBenchmarkResult,
|
||||
calibByIndex map[int]benchmarkPowerCalibrationResult,
|
||||
serverIdleW *float64, serverLoadedWSum *float64,
|
||||
serverIdleOK *bool, serverLoadedOK *bool, serverLoadedSamples *int,
|
||||
allMetricRows *[]GPUMetricRow,
|
||||
metricTimelineSec *float64,
|
||||
gpuBurnLog string,
|
||||
) {
|
||||
allDevices := joinIndexList(selected)
|
||||
|
||||
// Build per-GPU result stubs.
|
||||
gpuResults := make(map[int]*BenchmarkGPUResult, len(selected))
|
||||
for _, idx := range selected {
|
||||
r := &BenchmarkGPUResult{Index: idx, Status: "FAILED"}
|
||||
if info, ok := infoByIndex[idx]; ok {
|
||||
r.UUID = info.UUID
|
||||
r.Name = info.Name
|
||||
r.BusID = info.BusID
|
||||
r.VBIOS = info.VBIOS
|
||||
r.PowerLimitW = info.PowerLimitW
|
||||
r.MultiprocessorCount = info.MultiprocessorCount
|
||||
r.DefaultPowerLimitW = info.DefaultPowerLimitW
|
||||
r.ShutdownTempC = info.ShutdownTempC
|
||||
r.SlowdownTempC = info.SlowdownTempC
|
||||
r.MaxGraphicsClockMHz = info.MaxGraphicsClockMHz
|
||||
r.BaseGraphicsClockMHz = info.BaseGraphicsClockMHz
|
||||
r.MaxMemoryClockMHz = info.MaxMemoryClockMHz
|
||||
}
|
||||
if calib, ok := calibByIndex[idx]; ok {
|
||||
r.CalibratedPeakPowerW = calib.Summary.P95PowerW
|
||||
r.CalibratedPeakTempC = calib.Summary.P95TempC
|
||||
r.PowerCalibrationTries = calib.Attempts
|
||||
r.PowerLimitDerated = calib.Derated
|
||||
r.Notes = append(r.Notes, calib.Notes...)
|
||||
if calib.CoolingWarning != "" {
|
||||
r.CoolingWarning = calib.CoolingWarning
|
||||
}
|
||||
}
|
||||
if norm := findBenchmarkNormalization(result.Normalization.GPUs, idx); norm != nil {
|
||||
r.LockedGraphicsClockMHz = norm.GPUClockLockMHz
|
||||
r.LockedMemoryClockMHz = norm.MemoryClockLockMHz
|
||||
}
|
||||
gpuResults[idx] = r
|
||||
}
|
||||
|
||||
// Baseline: sample all GPUs together.
|
||||
baselineRows, err := collectBenchmarkSamples(ctx, spec.BaselineSec, selected)
|
||||
if err != nil && err != context.Canceled {
|
||||
for _, idx := range selected {
|
||||
gpuResults[idx].Notes = append(gpuResults[idx].Notes, "baseline sampling failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
for _, idx := range selected {
|
||||
perGPU := filterRowsByGPU(baselineRows, idx)
|
||||
gpuResults[idx].Baseline = summarizeBenchmarkTelemetry(perGPU)
|
||||
}
|
||||
appendBenchmarkMetrics(allMetricRows, baselineRows, "baseline", metricTimelineSec, float64(spec.BaselineSec))
|
||||
|
||||
// Sample server idle power once.
|
||||
if !*serverIdleOK {
|
||||
if w, ok := sampleBenchmarkPowerSourceSeries(ctx, opts.ServerPowerSource, maxInt(spec.BaselineSec, 10), benchmarkPowerAutotuneSampleInterval); ok {
|
||||
*serverIdleW = w
|
||||
*serverIdleOK = true
|
||||
logFunc(fmt.Sprintf("server idle power (%s): %.0f W", opts.ServerPowerSource, w))
|
||||
}
|
||||
}
|
||||
|
||||
// Warmup: all GPUs simultaneously.
|
||||
warmupCmd := []string{
|
||||
"bee-gpu-burn",
|
||||
"--seconds", strconv.Itoa(spec.WarmupSec),
|
||||
"--size-mb", strconv.Itoa(opts.SizeMB),
|
||||
"--devices", allDevices,
|
||||
}
|
||||
logFunc(fmt.Sprintf("GPUs %s: parallel warmup (%ds)", allDevices, spec.WarmupSec))
|
||||
warmupOut, warmupRows, warmupErr := runBenchmarkCommandWithMetrics(ctx, verboseLog, "gpu-all-warmup.log", warmupCmd, nil, selected, logFunc)
|
||||
appendBenchmarkMetrics(allMetricRows, warmupRows, "warmup", metricTimelineSec, float64(spec.WarmupSec))
|
||||
appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", "warmup", warmupOut)
|
||||
if warmupErr != nil {
|
||||
for _, idx := range selected {
|
||||
gpuResults[idx].Notes = append(gpuResults[idx].Notes, "parallel warmup failed: "+warmupErr.Error())
|
||||
}
|
||||
}
|
||||
warmupParseByGPU := parseBenchmarkBurnLogByGPU(string(warmupOut))
|
||||
supportedPrecisions := append([]string(nil), benchmarkPrecisionPhases...)
|
||||
for _, idx := range selected {
|
||||
if pr, ok := warmupParseByGPU[idx]; ok && pr.ComputeCapability != "" {
|
||||
if gpuResults[idx].ComputeCapability == "" {
|
||||
gpuResults[idx].ComputeCapability = pr.ComputeCapability
|
||||
}
|
||||
if ccPrecisions := benchmarkSupportedPrecisions(pr.ComputeCapability); len(ccPrecisions) < len(supportedPrecisions) {
|
||||
supportedPrecisions = ccPrecisions
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run synthetic precision phases and the combined steady phase as one
|
||||
// uninterrupted command so the GPUs stay hot between windows.
|
||||
eccBase := make(map[int]BenchmarkECCCounters, len(selected))
|
||||
for _, idx := range selected {
|
||||
eccBase[idx], _ = queryECCCounters(idx)
|
||||
}
|
||||
planLabels, planPhases, basePhaseSec, mixedPhaseSec := buildBenchmarkSteadyPlan(spec, supportedPrecisions, func(label string) string {
|
||||
if label == "mixed" {
|
||||
return "steady"
|
||||
}
|
||||
return "gpu-all-steady-" + label
|
||||
})
|
||||
planCmd := []string{
|
||||
"bee-gpu-burn",
|
||||
"--seconds", strconv.Itoa(basePhaseSec),
|
||||
"--size-mb", strconv.Itoa(opts.SizeMB),
|
||||
"--devices", allDevices,
|
||||
"--precision-plan", strings.Join(planLabels, ","),
|
||||
"--precision-plan-seconds", benchmarkPlanDurationsCSV(planPhases),
|
||||
}
|
||||
logFunc(fmt.Sprintf("GPUs %s: uninterrupted precision plan (%d precision phases x %ds, mixed %ds)", allDevices, len(supportedPrecisions), basePhaseSec, mixedPhaseSec))
|
||||
serverPowerStopCh := make(chan struct{})
|
||||
serverPowerCh := startSelectedPowerSourceSampler(serverPowerStopCh, opts.ServerPowerSource, benchmarkPowerAutotuneSampleInterval)
|
||||
_, phaseRowsByStage, phaseLogs, planErr := runBenchmarkPlannedCommandWithMetrics(ctx, verboseLog, "gpu-all-precision-plan.log", planCmd, nil, selected, planPhases, logFunc)
|
||||
close(serverPowerStopCh)
|
||||
if serverPowerSamples := <-serverPowerCh; len(serverPowerSamples) > 0 {
|
||||
*serverLoadedWSum += benchmarkMean(serverPowerSamples)
|
||||
(*serverLoadedSamples)++
|
||||
*serverLoadedOK = true
|
||||
logFunc(fmt.Sprintf("GPUs %s: server loaded power (%s avg): %.0f W", allDevices, opts.ServerPowerSource, benchmarkMean(serverPowerSamples)))
|
||||
}
|
||||
for _, phaseSpec := range planPhases {
|
||||
if rows := phaseRowsByStage[phaseSpec.MetricStage]; len(rows) > 0 {
|
||||
appendBenchmarkMetrics(allMetricRows, rows, phaseSpec.MetricStage, metricTimelineSec, float64(phaseSpec.DurationSec))
|
||||
}
|
||||
appendBenchmarkStageLog(gpuBurnLog, "bee-gpu-burn", phaseSpec.MetricStage, phaseLogs[phaseSpec.PlanLabel])
|
||||
}
|
||||
for _, prec := range supportedPrecisions {
|
||||
phaseLogName := "gpu-all-steady-" + prec
|
||||
phaseRows := phaseRowsByStage[phaseLogName]
|
||||
parseByGPU := parseBenchmarkBurnLogByGPU(string(phaseLogs[prec]))
|
||||
for _, idx := range selected {
|
||||
perGPU := filterRowsByGPU(phaseRows, idx)
|
||||
phase := BenchmarkPrecisionSteadyPhase{
|
||||
Precision: prec,
|
||||
Status: "OK",
|
||||
Steady: summarizeBenchmarkTelemetry(perGPU),
|
||||
}
|
||||
if status, note := benchmarkPlannedPhaseStatus(phaseLogs[prec]); status != "OK" {
|
||||
phase.Status = status
|
||||
phase.Notes = note
|
||||
gpuResults[idx].PrecisionFailures = append(gpuResults[idx].PrecisionFailures, prec+":"+status)
|
||||
}
|
||||
if pr, ok := parseByGPU[idx]; ok {
|
||||
for _, p := range pr.Profiles {
|
||||
if p.Supported {
|
||||
phase.TeraOpsPerSec += p.TeraOpsPerSec
|
||||
phase.WeightedTeraOpsPerSec += p.WeightedTeraOpsPerSec
|
||||
}
|
||||
}
|
||||
}
|
||||
gpuResults[idx].PrecisionSteady = append(gpuResults[idx].PrecisionSteady, phase)
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot throttle counters before steady.
|
||||
beforeThrottle := make(map[int]BenchmarkThrottleCounters, len(selected))
|
||||
for _, idx := range selected {
|
||||
beforeThrottle[idx], _ = queryThrottleCounters(idx)
|
||||
}
|
||||
|
||||
logFunc(fmt.Sprintf("GPUs %s: parallel steady compute (combined, %ds)", allDevices, mixedPhaseSec))
|
||||
afterThrottle := make(map[int]BenchmarkThrottleCounters, len(selected))
|
||||
for _, idx := range selected {
|
||||
afterThrottle[idx], _ = queryThrottleCounters(idx)
|
||||
}
|
||||
|
||||
steadyRows := phaseRowsByStage["steady"]
|
||||
parseResults := parseBenchmarkBurnLogByGPU(string(phaseLogs["mixed"]))
|
||||
|
||||
for _, idx := range selected {
|
||||
perGPU := filterRowsByGPU(steadyRows, idx)
|
||||
gpuResults[idx].Steady = summarizeBenchmarkTelemetry(perGPU)
|
||||
gpuResults[idx].Throttle = diffThrottleCounters(beforeThrottle[idx], afterThrottle[idx])
|
||||
if eccFinal, err := queryECCCounters(idx); err == nil {
|
||||
gpuResults[idx].ECC = diffECCCounters(eccBase[idx], eccFinal)
|
||||
}
|
||||
|
||||
if pr, ok := parseResults[idx]; ok {
|
||||
gpuResults[idx].ComputeCapability = pr.ComputeCapability
|
||||
gpuResults[idx].Backend = pr.Backend
|
||||
gpuResults[idx].PrecisionResults = pr.Profiles
|
||||
if pr.Fallback {
|
||||
gpuResults[idx].Notes = append(gpuResults[idx].Notes, "benchmark used driver PTX fallback; tensor throughput score is not comparable")
|
||||
}
|
||||
}
|
||||
if planErr != nil {
|
||||
gpuResults[idx].Notes = append(gpuResults[idx].Notes, "precision plan failed: "+planErr.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// Cooldown: all GPUs together.
|
||||
if spec.CooldownSec > 0 {
|
||||
cooldownRows, err := collectBenchmarkSamples(ctx, spec.CooldownSec, selected)
|
||||
if err != nil && err != context.Canceled {
|
||||
for _, idx := range selected {
|
||||
gpuResults[idx].Notes = append(gpuResults[idx].Notes, "cooldown sampling failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
for _, idx := range selected {
|
||||
perGPU := filterRowsByGPU(cooldownRows, idx)
|
||||
gpuResults[idx].Cooldown = summarizeBenchmarkTelemetry(perGPU)
|
||||
}
|
||||
appendBenchmarkMetrics(allMetricRows, cooldownRows, "cooldown", metricTimelineSec, float64(spec.CooldownSec))
|
||||
}
|
||||
|
||||
// Score and finalize each GPU.
|
||||
for _, idx := range selected {
|
||||
r := gpuResults[idx]
|
||||
applyBenchmarkSteadyFallback(r)
|
||||
r.Scores = scoreBenchmarkGPUResult(*r)
|
||||
r.DegradationReasons = detectBenchmarkDegradationReasons(*r, result.Normalization.Status)
|
||||
pr := parseResults[idx]
|
||||
switch {
|
||||
case planErr != nil:
|
||||
r.Status = classifySATErrorStatus(phaseLogs["mixed"], planErr)
|
||||
case len(r.PrecisionFailures) > 0:
|
||||
r.Status = "PARTIAL"
|
||||
case pr.Fallback:
|
||||
r.Status = "PARTIAL"
|
||||
default:
|
||||
r.Status = "OK"
|
||||
}
|
||||
result.GPUs = append(result.GPUs, finalizeBenchmarkGPUResult(*r))
|
||||
}
|
||||
}
|
||||
|
||||
// readBenchmarkHostConfig reads static CPU and memory configuration from
|
||||
// /proc/cpuinfo and /proc/meminfo. Returns nil if neither source is readable.
|
||||
func readBenchmarkHostConfig() *BenchmarkHostConfig {
|
||||
cfg := &BenchmarkHostConfig{}
|
||||
populated := false
|
||||
|
||||
// Parse /proc/cpuinfo for CPU model, sockets, cores, threads.
|
||||
if data, err := os.ReadFile("/proc/cpuinfo"); err == nil {
|
||||
socketIDs := map[string]struct{}{}
|
||||
coresPerSocket := map[string]int{}
|
||||
var modelName string
|
||||
threads := 0
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
kv := strings.SplitN(line, ":", 2)
|
||||
if len(kv) != 2 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(kv[0])
|
||||
val := strings.TrimSpace(kv[1])
|
||||
switch key {
|
||||
case "processor":
|
||||
threads++
|
||||
case "model name":
|
||||
if modelName == "" {
|
||||
modelName = val
|
||||
}
|
||||
case "physical id":
|
||||
socketIDs[val] = struct{}{}
|
||||
case "cpu cores":
|
||||
// Overwrite per-socket core count (last wins per socket, but all
|
||||
// entries for the same socket report the same value).
|
||||
if physLine := ""; physLine == "" {
|
||||
// We accumulate below by treating cpu cores as a per-thread
|
||||
// field; sum by socket requires a two-pass approach. Use the
|
||||
// simpler approximation: totalCores = threads / (threads per core).
|
||||
_ = val
|
||||
}
|
||||
}
|
||||
}
|
||||
// Second pass: per-socket core count.
|
||||
var curSocket string
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
kv := strings.SplitN(line, ":", 2)
|
||||
if len(kv) != 2 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(kv[0])
|
||||
val := strings.TrimSpace(kv[1])
|
||||
switch key {
|
||||
case "physical id":
|
||||
curSocket = val
|
||||
case "cpu cores":
|
||||
if curSocket != "" {
|
||||
if _, seen := coresPerSocket[curSocket]; !seen {
|
||||
v, _ := strconv.Atoi(val)
|
||||
coresPerSocket[curSocket] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
totalCores := 0
|
||||
for _, c := range coresPerSocket {
|
||||
totalCores += c
|
||||
}
|
||||
cfg.CPUModel = modelName
|
||||
cfg.CPUSockets = len(socketIDs)
|
||||
if cfg.CPUSockets == 0 && threads > 0 {
|
||||
cfg.CPUSockets = 1
|
||||
}
|
||||
cfg.CPUCores = totalCores
|
||||
cfg.CPUThreads = threads
|
||||
if modelName != "" || threads > 0 {
|
||||
populated = true
|
||||
}
|
||||
}
|
||||
|
||||
// Parse /proc/meminfo for total physical RAM.
|
||||
if data, err := os.ReadFile("/proc/meminfo"); err == nil {
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
if strings.HasPrefix(line, "MemTotal:") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 2 {
|
||||
kb, _ := strconv.ParseUint(fields[1], 10, 64)
|
||||
cfg.MemTotalGiB = float64(kb) / (1024 * 1024)
|
||||
populated = true
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !populated {
|
||||
return nil
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// startCPULoadSampler starts a goroutine that samples host CPU load every
|
||||
// intervalSec seconds until stopCh is closed, then sends the collected
|
||||
// samples on the returned channel.
|
||||
func startCPULoadSampler(stopCh <-chan struct{}, intervalSec int) <-chan []float64 {
|
||||
ch := make(chan []float64, 1)
|
||||
go func() {
|
||||
var samples []float64
|
||||
ticker := time.NewTicker(time.Duration(intervalSec) * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stopCh:
|
||||
ch <- samples
|
||||
return
|
||||
case <-ticker.C:
|
||||
if pct := sampleCPULoadPct(); pct > 0 {
|
||||
samples = append(samples, pct)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
// summarizeCPULoad computes stats over sampled CPU load values and assigns
|
||||
// a health status.
|
||||
func summarizeCPULoad(samples []float64) *BenchmarkCPULoad {
|
||||
if len(samples) == 0 {
|
||||
return nil
|
||||
}
|
||||
sorted := append([]float64(nil), samples...)
|
||||
sort.Float64s(sorted)
|
||||
var sum float64
|
||||
for _, v := range sorted {
|
||||
sum += v
|
||||
}
|
||||
avg := sum / float64(len(sorted))
|
||||
p95 := sorted[int(float64(len(sorted))*0.95)]
|
||||
max := sorted[len(sorted)-1]
|
||||
|
||||
cl := &BenchmarkCPULoad{
|
||||
AvgPct: math.Round(avg*10) / 10,
|
||||
MaxPct: math.Round(max*10) / 10,
|
||||
P95Pct: math.Round(p95*10) / 10,
|
||||
Samples: len(sorted),
|
||||
}
|
||||
|
||||
// Compute standard deviation to detect instability.
|
||||
var variance float64
|
||||
for _, v := range sorted {
|
||||
d := v - avg
|
||||
variance += d * d
|
||||
}
|
||||
stdDev := math.Sqrt(variance / float64(len(sorted)))
|
||||
|
||||
switch {
|
||||
case avg > 20 || max > 40:
|
||||
cl.Status = "high"
|
||||
cl.Note = fmt.Sprintf("avg %.1f%% max %.1f%% — elevated host CPU load may interfere with GPU benchmark results", avg, max)
|
||||
case stdDev > 12:
|
||||
cl.Status = "unstable"
|
||||
cl.Note = fmt.Sprintf("avg %.1f%% stddev %.1f%% — host CPU load was erratic during the benchmark", avg, stdDev)
|
||||
default:
|
||||
cl.Status = "ok"
|
||||
}
|
||||
return cl
|
||||
}
|
||||
|
||||
// runBenchmarkPowerCalibration runs the configured power-fit load for the supplied
|
||||
// GPU set and actively watches throttle counters. seedLimits, when provided, are treated as
|
||||
// the starting point for this calibration pass rather than as immutable fixed
|
||||
// limits. This matters during cumulative ramp-up: once an additional GPU is
|
||||
// introduced, every already-active GPU must be revalidated under the new
|
||||
// thermal state instead of assuming its previous single-step limit is still
|
||||
// valid. The selected reduced power limits stay active for the main benchmark
|
||||
// and are restored by the caller afterwards.
|
||||
Reference in New Issue
Block a user