refactor: modularize audit and harden build validation
This commit is contained in:
Mikhail Chusavitin
@@ -0,0 +1,624 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func renderPowerBenchReport(result NvidiaPowerBenchResult) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Bee Bench Power Report\n\n")
|
||||
fmt.Fprintf(&b, "**Benchmark version:** %s \n", result.BenchmarkVersion)
|
||||
fmt.Fprintf(&b, "**Profile:** %s \n", result.BenchmarkProfile)
|
||||
fmt.Fprintf(&b, "**Generated:** %s \n", result.GeneratedAt.Format("2006-01-02 15:04:05 UTC"))
|
||||
fmt.Fprintf(&b, "**Overall status:** %s \n", result.OverallStatus)
|
||||
fmt.Fprintf(&b, "**Platform max TDP (GPU-reported):** %.0f W \n", result.PlatformMaxTDPW)
|
||||
if sp := result.ServerPower; sp != nil && sp.Available {
|
||||
sourceLabel := "autotuned source"
|
||||
switch normalizeBenchmarkPowerSource(sp.Source) {
|
||||
case BenchmarkPowerSourceSDRPSUInput:
|
||||
sourceLabel = "autotuned source (SDR PSU AC input)"
|
||||
case BenchmarkPowerSourceDCMI:
|
||||
sourceLabel = "autotuned source (DCMI)"
|
||||
}
|
||||
fmt.Fprintf(&b, "**Server power delta (%s):** %.0f W \n", sourceLabel, sp.DeltaW)
|
||||
fmt.Fprintf(&b, "**Reporting ratio:** %.2f \n", sp.ReportingRatio)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
// Server power comparison table.
|
||||
if sp := result.ServerPower; sp != nil {
|
||||
b.WriteString("## Server vs GPU Power Comparison\n\n")
|
||||
selectedSource := normalizeBenchmarkPowerSource(sp.Source)
|
||||
selectedSourceLabel := "Selected source"
|
||||
if selectedSource == BenchmarkPowerSourceSDRPSUInput {
|
||||
selectedSourceLabel = "Selected source (SDR PSU AC input)"
|
||||
} else if selectedSource == BenchmarkPowerSourceDCMI {
|
||||
selectedSourceLabel = "Selected source (DCMI)"
|
||||
}
|
||||
var spRows [][]string
|
||||
spRows = append(spRows, []string{"GPU actual power sum (p95, last step)", fmt.Sprintf("%.0f W", sp.GPUReportedSumW)})
|
||||
if sp.Available {
|
||||
spRows = append(spRows, []string{selectedSourceLabel + " idle power", fmt.Sprintf("%.0f W", sp.IdleW)})
|
||||
spRows = append(spRows, []string{selectedSourceLabel + " loaded power", fmt.Sprintf("%.0f W", sp.LoadedW)})
|
||||
spRows = append(spRows, []string{selectedSourceLabel + " Δ power (loaded − idle)", fmt.Sprintf("%.0f W", sp.DeltaW)})
|
||||
}
|
||||
if selectedSource == BenchmarkPowerSourceSDRPSUInput && sp.PSUInputLoadedW > 0 {
|
||||
spRows = append(spRows, []string{"PSU AC input (idle avg, pre-load phase)", fmt.Sprintf("%.0f W", sp.PSUInputIdleW)})
|
||||
spRows = append(spRows, []string{"PSU AC input (loaded avg, final phase)", fmt.Sprintf("%.0f W", sp.PSUInputLoadedW)})
|
||||
psuDelta := sp.PSUInputLoadedW - sp.PSUInputIdleW
|
||||
spRows = append(spRows, []string{"PSU AC input Δ (loaded − idle)", fmt.Sprintf("%.0f W", psuDelta)})
|
||||
}
|
||||
if sp.Available {
|
||||
ratio := sp.ReportingRatio
|
||||
dcmiPartial := detectDCMIPartialCoverage(sp) ||
|
||||
(sp.PSUInputIdleW == 0 && detectIPMISaturationFallback(result.RampSteps))
|
||||
ratioNote := ""
|
||||
switch {
|
||||
case dcmiPartial:
|
||||
ratioNote = "⚠ IPMI DCMI covers partial PSU set; use SDR ratio below for accuracy assessment"
|
||||
case ratio >= 0.9:
|
||||
ratioNote = "✓ GPU telemetry matches server power"
|
||||
case ratio >= 0.75:
|
||||
ratioNote = "⚠ minor discrepancy — GPU may slightly over-report TDP"
|
||||
default:
|
||||
ratioNote = "✗ significant discrepancy — GPU over-reports TDP vs wall power"
|
||||
}
|
||||
spRows = append(spRows, []string{"Reporting ratio", fmt.Sprintf("%.2f — %s", ratio, ratioNote)})
|
||||
if selectedSource == BenchmarkPowerSourceSDRPSUInput && sp.PSUInputLoadedW > 0 && sp.GPUReportedSumW > 0 {
|
||||
psuDelta := sp.PSUInputLoadedW - sp.PSUInputIdleW
|
||||
sdrRatio := psuDelta / sp.GPUReportedSumW
|
||||
sdrNote := ""
|
||||
switch {
|
||||
case sdrRatio >= 0.9:
|
||||
sdrNote = "✓ GPU telemetry matches wall power"
|
||||
case sdrRatio >= 0.75:
|
||||
sdrNote = "⚠ minor discrepancy"
|
||||
default:
|
||||
sdrNote = "✗ significant discrepancy"
|
||||
}
|
||||
spRows = append(spRows, []string{"PSU AC input reporting ratio", fmt.Sprintf("%.2f — %s", sdrRatio, sdrNote)})
|
||||
}
|
||||
} else {
|
||||
spRows = append(spRows, []string{"IPMI availability", "not available — IPMI not supported or ipmitool not found"})
|
||||
}
|
||||
b.WriteString(fmtMDTable([]string{"Metric", "Value"}, spRows))
|
||||
for _, note := range sp.Notes {
|
||||
fmt.Fprintf(&b, "\n> %s\n", note)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
|
||||
if len(sp.PSUSlotReadingsIdle) > 0 || len(sp.PSUSlotReadingsLoaded) > 0 {
|
||||
b.WriteString("## PSU Load Distribution\n\n")
|
||||
|
||||
slotSet := map[string]struct{}{}
|
||||
for k := range sp.PSUSlotReadingsIdle {
|
||||
slotSet[k] = struct{}{}
|
||||
}
|
||||
for k := range sp.PSUSlotReadingsLoaded {
|
||||
slotSet[k] = struct{}{}
|
||||
}
|
||||
slots := make([]string, 0, len(slotSet))
|
||||
for k := range slotSet {
|
||||
slots = append(slots, k)
|
||||
}
|
||||
sort.Strings(slots)
|
||||
|
||||
fmtW := func(v *float64) string {
|
||||
if v == nil {
|
||||
return "—"
|
||||
}
|
||||
return fmt.Sprintf("%.0f W", *v)
|
||||
}
|
||||
|
||||
var psuDistRows [][]string
|
||||
for _, slot := range slots {
|
||||
idle := sp.PSUSlotReadingsIdle[slot]
|
||||
loaded := sp.PSUSlotReadingsLoaded[slot]
|
||||
|
||||
var deltaStr string
|
||||
if idle.InputW != nil && loaded.InputW != nil {
|
||||
deltaStr = fmt.Sprintf("%+.0f W", *loaded.InputW-*idle.InputW)
|
||||
} else {
|
||||
deltaStr = "—"
|
||||
}
|
||||
|
||||
status := loaded.Status
|
||||
if status == "" {
|
||||
status = idle.Status
|
||||
}
|
||||
if status == "" {
|
||||
status = "—"
|
||||
}
|
||||
|
||||
psuDistRows = append(psuDistRows, []string{
|
||||
slot,
|
||||
fmtW(idle.InputW), fmtW(loaded.InputW),
|
||||
deltaStr, status,
|
||||
})
|
||||
}
|
||||
b.WriteString(fmtMDTable([]string{"Slot", "AC Input (idle avg)", "AC Input (loaded avg)", "Load Δ", "Status"}, psuDistRows))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
|
||||
if len(result.Findings) > 0 {
|
||||
b.WriteString("## Summary\n\n")
|
||||
for _, finding := range result.Findings {
|
||||
fmt.Fprintf(&b, "- %s\n", finding)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
// ── Single GPU section ───────────────────────────────────────────────────
|
||||
b.WriteString("## Single GPU\n\n")
|
||||
{
|
||||
var sgRows [][]string
|
||||
for _, gpu := range result.GPUs {
|
||||
clk := "—"
|
||||
mem := "—"
|
||||
temp := "—"
|
||||
pwr := "—"
|
||||
if gpu.Telemetry != nil {
|
||||
clk = fmt.Sprintf("%.0f", gpu.Telemetry.AvgGraphicsClockMHz)
|
||||
mem = fmt.Sprintf("%.0f", gpu.Telemetry.AvgMemoryClockMHz)
|
||||
temp = fmt.Sprintf("%.1f", gpu.Telemetry.AvgTempC)
|
||||
pwr = fmt.Sprintf("%.0f W", gpu.Telemetry.AvgPowerW)
|
||||
}
|
||||
serverDelta := "—"
|
||||
if gpu.ServerDeltaW > 0 {
|
||||
serverDelta = fmt.Sprintf("%.0f W", gpu.ServerDeltaW)
|
||||
}
|
||||
fan := "—"
|
||||
if gpu.AvgFanRPM > 0 {
|
||||
if gpu.AvgFanDutyCyclePct > 0 {
|
||||
fan = fmt.Sprintf("%.0f RPM (%.0f%%)", gpu.AvgFanRPM, gpu.AvgFanDutyCyclePct)
|
||||
} else {
|
||||
fan = fmt.Sprintf("%.0f RPM", gpu.AvgFanRPM)
|
||||
}
|
||||
}
|
||||
sgRows = append(sgRows, []string{
|
||||
fmt.Sprintf("GPU %d", gpu.Index),
|
||||
fmt.Sprintf("%s (%s)", clk, mem),
|
||||
temp,
|
||||
pwr,
|
||||
serverDelta,
|
||||
fan,
|
||||
})
|
||||
}
|
||||
b.WriteString(fmtMDTable([]string{"GPU", "Clock MHz (Mem MHz)", "Avg Temp °C", "Power W", "Server Δ W", "Avg Fan RPM (duty%)"}, sgRows))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
if len(result.RecommendedSlotOrder) > 0 {
|
||||
fmt.Fprintf(&b, "Recommended slot order for best single-card power realization: `%s`\n\n", joinIndexList(result.RecommendedSlotOrder))
|
||||
}
|
||||
|
||||
// ── Ramp Sequence ────────────────────────────────────────────────────────
|
||||
// Rows = run number; Cols = per-GPU power (from step telemetry) + aggregates.
|
||||
if len(result.RampSteps) > 0 {
|
||||
b.WriteString("## Ramp Sequence\n\n")
|
||||
|
||||
// Collect all GPU indices that appear across all steps (ordered by first appearance).
|
||||
allGPUIndices := make([]int, 0, len(result.GPUs))
|
||||
seen := map[int]bool{}
|
||||
for _, step := range result.RampSteps {
|
||||
for _, idx := range step.GPUIndices {
|
||||
if !seen[idx] {
|
||||
seen[idx] = true
|
||||
allGPUIndices = append(allGPUIndices, idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var idleW float64
|
||||
if result.ServerPower != nil {
|
||||
idleW = result.ServerPower.IdleW
|
||||
}
|
||||
|
||||
// Build header: Run | GPU 0 | GPU 1 | ... | GPU total W | Server itself W | Server wall W | Per GPU wall W | Platform eff.
|
||||
headers := []string{"Run"}
|
||||
for _, idx := range allGPUIndices {
|
||||
headers = append(headers, fmt.Sprintf("GPU %d W", idx))
|
||||
}
|
||||
headers = append(headers, "GPU total W", "Server itself W", "Server wall W", "Per GPU wall W", "Platform eff.")
|
||||
|
||||
var rampRows [][]string
|
||||
if idleW > 0 {
|
||||
idleRow := []string{"0 (idle)"}
|
||||
for range allGPUIndices {
|
||||
idleRow = append(idleRow, "—")
|
||||
}
|
||||
// No load: GPU total is negligible, all draw is the server's own baseline.
|
||||
idleRow = append(idleRow, "—", fmt.Sprintf("%.0f", idleW), fmt.Sprintf("%.0f", idleW), "—", "—")
|
||||
rampRows = append(rampRows, idleRow)
|
||||
}
|
||||
for _, step := range result.RampSteps {
|
||||
row := []string{fmt.Sprintf("%d", step.StepIndex)}
|
||||
for _, idx := range allGPUIndices {
|
||||
inStep := false
|
||||
for _, si := range step.GPUIndices {
|
||||
if si == idx {
|
||||
inStep = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !inStep {
|
||||
row = append(row, "—")
|
||||
continue
|
||||
}
|
||||
gpuPwr := "—"
|
||||
if t, ok := step.PerGPUTelemetry[idx]; ok && t != nil && t.AvgPowerW > 0 {
|
||||
gpuPwr = fmt.Sprintf("%.0f", t.AvgPowerW)
|
||||
}
|
||||
row = append(row, gpuPwr)
|
||||
}
|
||||
// GPU total W = sum of observed GPU power (nvidia-smi)
|
||||
gpuTotal := "—"
|
||||
if step.TotalObservedPowerW > 0 {
|
||||
gpuTotal = fmt.Sprintf("%.0f", step.TotalObservedPowerW)
|
||||
}
|
||||
// Server itself W = server wall power minus GPU total (non-GPU baseline draw)
|
||||
serverItself := "—"
|
||||
if step.ServerLoadedW > 0 && step.TotalObservedPowerW > 0 {
|
||||
serverItself = fmt.Sprintf("%.0f", step.ServerLoadedW-step.TotalObservedPowerW)
|
||||
}
|
||||
// Server wall W
|
||||
serverWall := "—"
|
||||
if step.ServerLoadedW > 0 {
|
||||
serverWall = fmt.Sprintf("%.0f", step.ServerLoadedW)
|
||||
}
|
||||
// Per GPU wall W = ServerDeltaW / len(GPUIndices)
|
||||
perGPUWall := "—"
|
||||
if step.ServerDeltaW > 0 && len(step.GPUIndices) > 0 {
|
||||
perGPUWall = fmt.Sprintf("%.0f", step.ServerDeltaW/float64(len(step.GPUIndices)))
|
||||
}
|
||||
// Platform eff. = (ServerLoadedW − idleW) / TotalObservedPowerW
|
||||
platEff := "—"
|
||||
if step.TotalObservedPowerW > 0 {
|
||||
eff := step.ServerDeltaW / step.TotalObservedPowerW
|
||||
if idleW > 0 && step.ServerLoadedW > 0 {
|
||||
eff = (step.ServerLoadedW - idleW) / step.TotalObservedPowerW
|
||||
}
|
||||
platEff = fmt.Sprintf("%.2f", eff)
|
||||
}
|
||||
row = append(row, gpuTotal, serverItself, serverWall, perGPUWall, platEff)
|
||||
rampRows = append(rampRows, row)
|
||||
}
|
||||
b.WriteString(fmtMDTable(headers, rampRows))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
// ── PSU Performance ───────────────────────────────────────────────────────
|
||||
{
|
||||
// Collect all PSU slot keys from any ramp step.
|
||||
psuSlotSet := map[string]struct{}{}
|
||||
for _, step := range result.RampSteps {
|
||||
for k := range step.PSUSlotReadings {
|
||||
psuSlotSet[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(psuSlotSet) > 0 {
|
||||
b.WriteString("## PSU Performance\n\n")
|
||||
psuSlots := make([]string, 0, len(psuSlotSet))
|
||||
for k := range psuSlotSet {
|
||||
psuSlots = append(psuSlots, k)
|
||||
}
|
||||
sort.Strings(psuSlots)
|
||||
|
||||
var idleW float64
|
||||
if result.ServerPower != nil {
|
||||
idleW = result.ServerPower.IdleW
|
||||
}
|
||||
|
||||
psuHeaders := []string{"Run"}
|
||||
for _, slot := range psuSlots {
|
||||
psuHeaders = append(psuHeaders, fmt.Sprintf("PSU %s W", slot))
|
||||
}
|
||||
psuHeaders = append(psuHeaders, "PSU Total W", "Platform eff.", "Avg Fan RPM (duty%)")
|
||||
|
||||
var psuRows [][]string
|
||||
for _, step := range result.RampSteps {
|
||||
row := []string{fmt.Sprintf("%d", step.StepIndex)}
|
||||
var psuTotal float64
|
||||
for _, slot := range psuSlots {
|
||||
sp, ok := step.PSUSlotReadings[slot]
|
||||
if !ok || sp.InputW == nil {
|
||||
row = append(row, "—")
|
||||
continue
|
||||
}
|
||||
row = append(row, fmt.Sprintf("%.0f", *sp.InputW))
|
||||
psuTotal += *sp.InputW
|
||||
}
|
||||
totalStr := "—"
|
||||
if psuTotal > 0 {
|
||||
totalStr = fmt.Sprintf("%.0f", psuTotal)
|
||||
}
|
||||
platEff := "—"
|
||||
if step.TotalObservedPowerW > 0 {
|
||||
eff := step.ServerDeltaW / step.TotalObservedPowerW
|
||||
if idleW > 0 && step.ServerLoadedW > 0 {
|
||||
eff = (step.ServerLoadedW - idleW) / step.TotalObservedPowerW
|
||||
}
|
||||
platEff = fmt.Sprintf("%.2f", eff)
|
||||
}
|
||||
fan := "—"
|
||||
if step.AvgFanRPM > 0 {
|
||||
if step.AvgFanDutyCyclePct > 0 {
|
||||
fan = fmt.Sprintf("%.0f (%.0f%%)", step.AvgFanRPM, step.AvgFanDutyCyclePct)
|
||||
} else {
|
||||
fan = fmt.Sprintf("%.0f", step.AvgFanRPM)
|
||||
}
|
||||
}
|
||||
row = append(row, totalStr, platEff, fan)
|
||||
psuRows = append(psuRows, row)
|
||||
}
|
||||
b.WriteString(fmtMDTable(psuHeaders, psuRows))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
|
||||
// ── PSU Issues ────────────────────────────────────────────────────────────
|
||||
if len(result.PSUIssues) > 0 {
|
||||
b.WriteString("## PSU Issues\n\n")
|
||||
b.WriteString("The following power supply anomalies were detected during the test:\n\n")
|
||||
for _, issue := range result.PSUIssues {
|
||||
fmt.Fprintf(&b, "- ⛔ %s\n", issue)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
// ── Power Distribution Summary ────────────────────────────────────────────
|
||||
b.WriteString("## Power Distribution Summary\n\n")
|
||||
{
|
||||
var totalDefault, totalStable float64
|
||||
for _, gpu := range result.GPUs {
|
||||
stable := gpu.StablePowerLimitW
|
||||
if stable <= 0 {
|
||||
stable = gpu.AppliedPowerLimitW
|
||||
}
|
||||
totalDefault += gpu.DefaultPowerLimitW
|
||||
totalStable += stable
|
||||
}
|
||||
var pdRows [][]string
|
||||
for _, gpu := range result.GPUs {
|
||||
stable := gpu.StablePowerLimitW
|
||||
if stable <= 0 {
|
||||
stable = gpu.AppliedPowerLimitW
|
||||
}
|
||||
realization := "-"
|
||||
if gpu.DefaultPowerLimitW > 0 && stable > 0 {
|
||||
realization = fmt.Sprintf("%.1f%%", stable/gpu.DefaultPowerLimitW*100)
|
||||
}
|
||||
derated := "-"
|
||||
if gpu.Derated {
|
||||
derated = "⚠ yes"
|
||||
}
|
||||
pdRows = append(pdRows, []string{
|
||||
fmt.Sprintf("GPU %d", gpu.Index),
|
||||
fmt.Sprintf("%.0f W", gpu.AppliedPowerLimitW),
|
||||
fmt.Sprintf("%.0f W", stable),
|
||||
realization,
|
||||
derated,
|
||||
})
|
||||
}
|
||||
platformReal := "-"
|
||||
if totalDefault > 0 && totalStable > 0 {
|
||||
platformReal = fmt.Sprintf("%.1f%%", totalStable/totalDefault*100)
|
||||
}
|
||||
pdRows = append(pdRows, []string{
|
||||
"**Platform**",
|
||||
"—",
|
||||
fmt.Sprintf("**%.0f W**", totalStable),
|
||||
fmt.Sprintf("**%s**", platformReal),
|
||||
"",
|
||||
})
|
||||
b.WriteString(fmtMDTable([]string{"GPU", "Single-card limit", "Stable limit", "Realization", "Derated"}, pdRows))
|
||||
b.WriteString("\n")
|
||||
|
||||
// Balance across GPUs — only meaningful with 2+ GPUs.
|
||||
if len(result.GPUs) > 1 {
|
||||
var minS, maxS, sumS float64
|
||||
var cnt int
|
||||
for _, gpu := range result.GPUs {
|
||||
s := gpu.StablePowerLimitW
|
||||
if s <= 0 {
|
||||
s = gpu.AppliedPowerLimitW
|
||||
}
|
||||
if s <= 0 {
|
||||
continue
|
||||
}
|
||||
sumS += s
|
||||
cnt++
|
||||
if cnt == 1 || s < minS {
|
||||
minS = s
|
||||
}
|
||||
if s > maxS {
|
||||
maxS = s
|
||||
}
|
||||
}
|
||||
if cnt > 0 {
|
||||
avg := sumS / float64(cnt)
|
||||
spread := (maxS - minS) / avg * 100
|
||||
balanceNote := "✓ balanced"
|
||||
switch {
|
||||
case spread > 20:
|
||||
balanceNote = "⚠ significant imbalance — check slot thermals"
|
||||
case spread > 10:
|
||||
balanceNote = "— minor imbalance"
|
||||
}
|
||||
fmt.Fprintf(&b, "**GPU power balance:** avg %.0f W · min %.0f W · max %.0f W · spread %.1f%% — %s\n\n",
|
||||
avg, minS, maxS, spread, balanceNote)
|
||||
}
|
||||
}
|
||||
|
||||
// Ramp scalability table — power efficiency of adding each GPU.
|
||||
if len(result.RampSteps) > 1 {
|
||||
b.WriteString("**Ramp power scalability** (stable TDP per step):\n\n")
|
||||
var firstStable float64
|
||||
if len(result.GPUs) > 0 {
|
||||
firstStable = result.GPUs[0].StablePowerLimitW
|
||||
if firstStable <= 0 {
|
||||
firstStable = result.GPUs[0].AppliedPowerLimitW
|
||||
}
|
||||
}
|
||||
var prevCumulative float64
|
||||
var scalRows [][]string
|
||||
for _, step := range result.RampSteps {
|
||||
var cumulative float64
|
||||
for _, gpuIdx := range step.GPUIndices {
|
||||
for _, g := range result.GPUs {
|
||||
if g.Index != gpuIdx {
|
||||
continue
|
||||
}
|
||||
s := g.StablePowerLimitW
|
||||
if s <= 0 {
|
||||
s = g.AppliedPowerLimitW
|
||||
}
|
||||
cumulative += s
|
||||
}
|
||||
}
|
||||
incremental := cumulative - prevCumulative
|
||||
efficiency := "—"
|
||||
if step.StepIndex > 1 && firstStable > 0 {
|
||||
efficiency = fmt.Sprintf("%.1f%%", incremental/firstStable*100)
|
||||
}
|
||||
scalRows = append(scalRows, []string{
|
||||
fmt.Sprintf("%d", step.StepIndex),
|
||||
joinIndexList(step.GPUIndices),
|
||||
fmt.Sprintf("%.0f W", cumulative),
|
||||
fmt.Sprintf("%.0f W", incremental),
|
||||
efficiency,
|
||||
})
|
||||
prevCumulative = cumulative
|
||||
}
|
||||
b.WriteString(fmtMDTable([]string{"Step", "GPUs", "Cumulative stable TDP", "Incremental", "Efficiency vs GPU 1"}, scalRows))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
|
||||
// ── Per-GPU sections ──────────────────────────────────────────────────────
|
||||
var lastStep *NvidiaPowerBenchStep
|
||||
if n := len(result.RampSteps); n > 0 {
|
||||
lastStep = &result.RampSteps[n-1]
|
||||
}
|
||||
for _, gpu := range result.GPUs {
|
||||
fmt.Fprintf(&b, "### GPU %d — %s\n\n", gpu.Index, gpu.Name)
|
||||
|
||||
// Transposed comparison table: Single Run vs All GPU Run.
|
||||
singleClk := "—"
|
||||
singleMem := "—"
|
||||
singleTemp := "—"
|
||||
singlePwr := "—"
|
||||
singleWall := "—"
|
||||
singleFan := "—"
|
||||
if gpu.Telemetry != nil {
|
||||
singleClk = fmt.Sprintf("%.0f", gpu.Telemetry.AvgGraphicsClockMHz)
|
||||
singleMem = fmt.Sprintf("%.0f", gpu.Telemetry.AvgMemoryClockMHz)
|
||||
singleTemp = fmt.Sprintf("%.1f", gpu.Telemetry.AvgTempC)
|
||||
singlePwr = fmt.Sprintf("%.0f W", gpu.Telemetry.AvgPowerW)
|
||||
}
|
||||
if gpu.ServerDeltaW > 0 {
|
||||
singleWall = fmt.Sprintf("%.0f W", gpu.ServerDeltaW)
|
||||
}
|
||||
if gpu.AvgFanRPM > 0 {
|
||||
if gpu.AvgFanDutyCyclePct > 0 {
|
||||
singleFan = fmt.Sprintf("%.0f RPM (%.0f%%)", gpu.AvgFanRPM, gpu.AvgFanDutyCyclePct)
|
||||
} else {
|
||||
singleFan = fmt.Sprintf("%.0f RPM", gpu.AvgFanRPM)
|
||||
}
|
||||
}
|
||||
|
||||
allClk := "—"
|
||||
allMem := "—"
|
||||
allTemp := "—"
|
||||
allPwr := "—"
|
||||
allWall := "—"
|
||||
allFan := "—"
|
||||
if lastStep != nil {
|
||||
if t, ok := lastStep.PerGPUTelemetry[gpu.Index]; ok && t != nil {
|
||||
allClk = fmt.Sprintf("%.0f", t.AvgGraphicsClockMHz)
|
||||
allMem = fmt.Sprintf("%.0f", t.AvgMemoryClockMHz)
|
||||
allTemp = fmt.Sprintf("%.1f", t.AvgTempC)
|
||||
allPwr = fmt.Sprintf("%.0f W", t.AvgPowerW)
|
||||
}
|
||||
if lastStep.ServerDeltaW > 0 && len(lastStep.GPUIndices) > 0 {
|
||||
allWall = fmt.Sprintf("%.0f W", lastStep.ServerDeltaW/float64(len(lastStep.GPUIndices)))
|
||||
}
|
||||
if lastStep.AvgFanRPM > 0 {
|
||||
if lastStep.AvgFanDutyCyclePct > 0 {
|
||||
allFan = fmt.Sprintf("%.0f RPM (%.0f%%)", lastStep.AvgFanRPM, lastStep.AvgFanDutyCyclePct)
|
||||
} else {
|
||||
allFan = fmt.Sprintf("%.0f RPM", lastStep.AvgFanRPM)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tableHeaders := []string{"", "Single Run"}
|
||||
if lastStep != nil {
|
||||
tableHeaders = append(tableHeaders, "All GPU Run")
|
||||
}
|
||||
compRows := [][]string{
|
||||
{"Clock MHz (Mem MHz)", fmt.Sprintf("%s (%s)", singleClk, singleMem)},
|
||||
{"Avg Temp °C", singleTemp},
|
||||
{"Power W", singlePwr},
|
||||
{"Per GPU wall W", singleWall},
|
||||
{"Avg Fan RPM (duty%)", singleFan},
|
||||
}
|
||||
if lastStep != nil {
|
||||
compRows[0] = append(compRows[0], fmt.Sprintf("%s (%s)", allClk, allMem))
|
||||
compRows[1] = append(compRows[1], allTemp)
|
||||
compRows[2] = append(compRows[2], allPwr)
|
||||
compRows[3] = append(compRows[3], allWall)
|
||||
compRows[4] = append(compRows[4], allFan)
|
||||
}
|
||||
b.WriteString(fmtMDTable(tableHeaders, compRows))
|
||||
b.WriteString("\n")
|
||||
|
||||
for _, note := range gpu.Notes {
|
||||
fmt.Fprintf(&b, "- %s\n", note)
|
||||
}
|
||||
if len(gpu.Notes) > 0 {
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func renderPowerBenchSummary(result NvidiaPowerBenchResult) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "run_at_utc=%s\n", result.GeneratedAt.Format(time.RFC3339))
|
||||
fmt.Fprintf(&b, "benchmark_version=%s\n", result.BenchmarkVersion)
|
||||
fmt.Fprintf(&b, "benchmark_profile=%s\n", result.BenchmarkProfile)
|
||||
fmt.Fprintf(&b, "overall_status=%s\n", result.OverallStatus)
|
||||
fmt.Fprintf(&b, "platform_max_tdp_w=%.0f\n", result.PlatformMaxTDPW)
|
||||
fmt.Fprintf(&b, "gpu_count=%d\n", len(result.GPUs))
|
||||
if len(result.RecommendedSlotOrder) > 0 {
|
||||
fmt.Fprintf(&b, "recommended_slot_order=%s\n", joinIndexList(result.RecommendedSlotOrder))
|
||||
}
|
||||
for _, step := range result.RampSteps {
|
||||
fmt.Fprintf(&b, "ramp_step_%d_gpus=%s\n", step.StepIndex, joinIndexList(step.GPUIndices))
|
||||
fmt.Fprintf(&b, "ramp_step_%d_new_gpu=%d\n", step.StepIndex, step.NewGPUIndex)
|
||||
fmt.Fprintf(&b, "ramp_step_%d_stable_limit_w=%.0f\n", step.StepIndex, step.NewGPUStableLimitW)
|
||||
fmt.Fprintf(&b, "ramp_step_%d_total_power_w=%.0f\n", step.StepIndex, step.TotalObservedPowerW)
|
||||
if step.ServerLoadedW > 0 {
|
||||
fmt.Fprintf(&b, "ramp_step_%d_server_loaded_w=%.0f\n", step.StepIndex, step.ServerLoadedW)
|
||||
fmt.Fprintf(&b, "ramp_step_%d_server_delta_w=%.0f\n", step.StepIndex, step.ServerDeltaW)
|
||||
}
|
||||
}
|
||||
for _, gpu := range result.GPUs {
|
||||
if gpu.StablePowerLimitW > 0 {
|
||||
fmt.Fprintf(&b, "gpu_%d_stable_limit_w=%.0f\n", gpu.Index, gpu.StablePowerLimitW)
|
||||
}
|
||||
if gpu.ServerLoadedW > 0 {
|
||||
fmt.Fprintf(&b, "gpu_%d_server_loaded_w=%.0f\n", gpu.Index, gpu.ServerLoadedW)
|
||||
fmt.Fprintf(&b, "gpu_%d_server_delta_w=%.0f\n", gpu.Index, gpu.ServerDeltaW)
|
||||
}
|
||||
}
|
||||
if sp := result.ServerPower; sp != nil && sp.Available {
|
||||
fmt.Fprintf(&b, "server_idle_w=%.0f\n", sp.IdleW)
|
||||
fmt.Fprintf(&b, "server_loaded_w=%.0f\n", sp.LoadedW)
|
||||
fmt.Fprintf(&b, "server_delta_w=%.0f\n", sp.DeltaW)
|
||||
fmt.Fprintf(&b, "server_reporting_ratio=%.2f\n", sp.ReportingRatio)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
Reference in New Issue
Block a user