625 lines
21 KiB
Go
625 lines
21 KiB
Go
package platform
|
||
|
||
import (
|
||
"fmt"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
)
|
||
|
||
func renderPowerBenchReport(result NvidiaPowerBenchResult) string {
|
||
var b strings.Builder
|
||
b.WriteString("# Bee Bench Power Report\n\n")
|
||
fmt.Fprintf(&b, "**Benchmark version:** %s \n", result.BenchmarkVersion)
|
||
fmt.Fprintf(&b, "**Profile:** %s \n", result.BenchmarkProfile)
|
||
fmt.Fprintf(&b, "**Generated:** %s \n", result.GeneratedAt.Format("2006-01-02 15:04:05 UTC"))
|
||
fmt.Fprintf(&b, "**Overall status:** %s \n", result.OverallStatus)
|
||
fmt.Fprintf(&b, "**Platform max TDP (GPU-reported):** %.0f W \n", result.PlatformMaxTDPW)
|
||
if sp := result.ServerPower; sp != nil && sp.Available {
|
||
sourceLabel := "autotuned source"
|
||
switch normalizeBenchmarkPowerSource(sp.Source) {
|
||
case BenchmarkPowerSourceSDRPSUInput:
|
||
sourceLabel = "autotuned source (SDR PSU AC input)"
|
||
case BenchmarkPowerSourceDCMI:
|
||
sourceLabel = "autotuned source (DCMI)"
|
||
}
|
||
fmt.Fprintf(&b, "**Server power delta (%s):** %.0f W \n", sourceLabel, sp.DeltaW)
|
||
fmt.Fprintf(&b, "**Reporting ratio:** %.2f \n", sp.ReportingRatio)
|
||
}
|
||
b.WriteString("\n")
|
||
// Server power comparison table.
|
||
if sp := result.ServerPower; sp != nil {
|
||
b.WriteString("## Server vs GPU Power Comparison\n\n")
|
||
selectedSource := normalizeBenchmarkPowerSource(sp.Source)
|
||
selectedSourceLabel := "Selected source"
|
||
if selectedSource == BenchmarkPowerSourceSDRPSUInput {
|
||
selectedSourceLabel = "Selected source (SDR PSU AC input)"
|
||
} else if selectedSource == BenchmarkPowerSourceDCMI {
|
||
selectedSourceLabel = "Selected source (DCMI)"
|
||
}
|
||
var spRows [][]string
|
||
spRows = append(spRows, []string{"GPU actual power sum (p95, last step)", fmt.Sprintf("%.0f W", sp.GPUReportedSumW)})
|
||
if sp.Available {
|
||
spRows = append(spRows, []string{selectedSourceLabel + " idle power", fmt.Sprintf("%.0f W", sp.IdleW)})
|
||
spRows = append(spRows, []string{selectedSourceLabel + " loaded power", fmt.Sprintf("%.0f W", sp.LoadedW)})
|
||
spRows = append(spRows, []string{selectedSourceLabel + " Δ power (loaded − idle)", fmt.Sprintf("%.0f W", sp.DeltaW)})
|
||
}
|
||
if selectedSource == BenchmarkPowerSourceSDRPSUInput && sp.PSUInputLoadedW > 0 {
|
||
spRows = append(spRows, []string{"PSU AC input (idle avg, pre-load phase)", fmt.Sprintf("%.0f W", sp.PSUInputIdleW)})
|
||
spRows = append(spRows, []string{"PSU AC input (loaded avg, final phase)", fmt.Sprintf("%.0f W", sp.PSUInputLoadedW)})
|
||
psuDelta := sp.PSUInputLoadedW - sp.PSUInputIdleW
|
||
spRows = append(spRows, []string{"PSU AC input Δ (loaded − idle)", fmt.Sprintf("%.0f W", psuDelta)})
|
||
}
|
||
if sp.Available {
|
||
ratio := sp.ReportingRatio
|
||
dcmiPartial := detectDCMIPartialCoverage(sp) ||
|
||
(sp.PSUInputIdleW == 0 && detectIPMISaturationFallback(result.RampSteps))
|
||
ratioNote := ""
|
||
switch {
|
||
case dcmiPartial:
|
||
ratioNote = "⚠ IPMI DCMI covers partial PSU set; use SDR ratio below for accuracy assessment"
|
||
case ratio >= 0.9:
|
||
ratioNote = "✓ GPU telemetry matches server power"
|
||
case ratio >= 0.75:
|
||
ratioNote = "⚠ minor discrepancy — GPU may slightly over-report TDP"
|
||
default:
|
||
ratioNote = "✗ significant discrepancy — GPU over-reports TDP vs wall power"
|
||
}
|
||
spRows = append(spRows, []string{"Reporting ratio", fmt.Sprintf("%.2f — %s", ratio, ratioNote)})
|
||
if selectedSource == BenchmarkPowerSourceSDRPSUInput && sp.PSUInputLoadedW > 0 && sp.GPUReportedSumW > 0 {
|
||
psuDelta := sp.PSUInputLoadedW - sp.PSUInputIdleW
|
||
sdrRatio := psuDelta / sp.GPUReportedSumW
|
||
sdrNote := ""
|
||
switch {
|
||
case sdrRatio >= 0.9:
|
||
sdrNote = "✓ GPU telemetry matches wall power"
|
||
case sdrRatio >= 0.75:
|
||
sdrNote = "⚠ minor discrepancy"
|
||
default:
|
||
sdrNote = "✗ significant discrepancy"
|
||
}
|
||
spRows = append(spRows, []string{"PSU AC input reporting ratio", fmt.Sprintf("%.2f — %s", sdrRatio, sdrNote)})
|
||
}
|
||
} else {
|
||
spRows = append(spRows, []string{"IPMI availability", "not available — IPMI not supported or ipmitool not found"})
|
||
}
|
||
b.WriteString(fmtMDTable([]string{"Metric", "Value"}, spRows))
|
||
for _, note := range sp.Notes {
|
||
fmt.Fprintf(&b, "\n> %s\n", note)
|
||
}
|
||
b.WriteString("\n")
|
||
|
||
if len(sp.PSUSlotReadingsIdle) > 0 || len(sp.PSUSlotReadingsLoaded) > 0 {
|
||
b.WriteString("## PSU Load Distribution\n\n")
|
||
|
||
slotSet := map[string]struct{}{}
|
||
for k := range sp.PSUSlotReadingsIdle {
|
||
slotSet[k] = struct{}{}
|
||
}
|
||
for k := range sp.PSUSlotReadingsLoaded {
|
||
slotSet[k] = struct{}{}
|
||
}
|
||
slots := make([]string, 0, len(slotSet))
|
||
for k := range slotSet {
|
||
slots = append(slots, k)
|
||
}
|
||
sort.Strings(slots)
|
||
|
||
fmtW := func(v *float64) string {
|
||
if v == nil {
|
||
return "—"
|
||
}
|
||
return fmt.Sprintf("%.0f W", *v)
|
||
}
|
||
|
||
var psuDistRows [][]string
|
||
for _, slot := range slots {
|
||
idle := sp.PSUSlotReadingsIdle[slot]
|
||
loaded := sp.PSUSlotReadingsLoaded[slot]
|
||
|
||
var deltaStr string
|
||
if idle.InputW != nil && loaded.InputW != nil {
|
||
deltaStr = fmt.Sprintf("%+.0f W", *loaded.InputW-*idle.InputW)
|
||
} else {
|
||
deltaStr = "—"
|
||
}
|
||
|
||
status := loaded.Status
|
||
if status == "" {
|
||
status = idle.Status
|
||
}
|
||
if status == "" {
|
||
status = "—"
|
||
}
|
||
|
||
psuDistRows = append(psuDistRows, []string{
|
||
slot,
|
||
fmtW(idle.InputW), fmtW(loaded.InputW),
|
||
deltaStr, status,
|
||
})
|
||
}
|
||
b.WriteString(fmtMDTable([]string{"Slot", "AC Input (idle avg)", "AC Input (loaded avg)", "Load Δ", "Status"}, psuDistRows))
|
||
b.WriteString("\n")
|
||
}
|
||
}
|
||
|
||
if len(result.Findings) > 0 {
|
||
b.WriteString("## Summary\n\n")
|
||
for _, finding := range result.Findings {
|
||
fmt.Fprintf(&b, "- %s\n", finding)
|
||
}
|
||
b.WriteString("\n")
|
||
}
|
||
// ── Single GPU section ───────────────────────────────────────────────────
|
||
b.WriteString("## Single GPU\n\n")
|
||
{
|
||
var sgRows [][]string
|
||
for _, gpu := range result.GPUs {
|
||
clk := "—"
|
||
mem := "—"
|
||
temp := "—"
|
||
pwr := "—"
|
||
if gpu.Telemetry != nil {
|
||
clk = fmt.Sprintf("%.0f", gpu.Telemetry.AvgGraphicsClockMHz)
|
||
mem = fmt.Sprintf("%.0f", gpu.Telemetry.AvgMemoryClockMHz)
|
||
temp = fmt.Sprintf("%.1f", gpu.Telemetry.AvgTempC)
|
||
pwr = fmt.Sprintf("%.0f W", gpu.Telemetry.AvgPowerW)
|
||
}
|
||
serverDelta := "—"
|
||
if gpu.ServerDeltaW > 0 {
|
||
serverDelta = fmt.Sprintf("%.0f W", gpu.ServerDeltaW)
|
||
}
|
||
fan := "—"
|
||
if gpu.AvgFanRPM > 0 {
|
||
if gpu.AvgFanDutyCyclePct > 0 {
|
||
fan = fmt.Sprintf("%.0f RPM (%.0f%%)", gpu.AvgFanRPM, gpu.AvgFanDutyCyclePct)
|
||
} else {
|
||
fan = fmt.Sprintf("%.0f RPM", gpu.AvgFanRPM)
|
||
}
|
||
}
|
||
sgRows = append(sgRows, []string{
|
||
fmt.Sprintf("GPU %d", gpu.Index),
|
||
fmt.Sprintf("%s (%s)", clk, mem),
|
||
temp,
|
||
pwr,
|
||
serverDelta,
|
||
fan,
|
||
})
|
||
}
|
||
b.WriteString(fmtMDTable([]string{"GPU", "Clock MHz (Mem MHz)", "Avg Temp °C", "Power W", "Server Δ W", "Avg Fan RPM (duty%)"}, sgRows))
|
||
b.WriteString("\n")
|
||
}
|
||
if len(result.RecommendedSlotOrder) > 0 {
|
||
fmt.Fprintf(&b, "Recommended slot order for best single-card power realization: `%s`\n\n", joinIndexList(result.RecommendedSlotOrder))
|
||
}
|
||
|
||
// ── Ramp Sequence ────────────────────────────────────────────────────────
|
||
// Rows = run number; Cols = per-GPU power (from step telemetry) + aggregates.
|
||
if len(result.RampSteps) > 0 {
|
||
b.WriteString("## Ramp Sequence\n\n")
|
||
|
||
// Collect all GPU indices that appear across all steps (ordered by first appearance).
|
||
allGPUIndices := make([]int, 0, len(result.GPUs))
|
||
seen := map[int]bool{}
|
||
for _, step := range result.RampSteps {
|
||
for _, idx := range step.GPUIndices {
|
||
if !seen[idx] {
|
||
seen[idx] = true
|
||
allGPUIndices = append(allGPUIndices, idx)
|
||
}
|
||
}
|
||
}
|
||
|
||
var idleW float64
|
||
if result.ServerPower != nil {
|
||
idleW = result.ServerPower.IdleW
|
||
}
|
||
|
||
// Build header: Run | GPU 0 | GPU 1 | ... | GPU total W | Server itself W | Server wall W | Per GPU wall W | Platform eff.
|
||
headers := []string{"Run"}
|
||
for _, idx := range allGPUIndices {
|
||
headers = append(headers, fmt.Sprintf("GPU %d W", idx))
|
||
}
|
||
headers = append(headers, "GPU total W", "Server itself W", "Server wall W", "Per GPU wall W", "Platform eff.")
|
||
|
||
var rampRows [][]string
|
||
if idleW > 0 {
|
||
idleRow := []string{"0 (idle)"}
|
||
for range allGPUIndices {
|
||
idleRow = append(idleRow, "—")
|
||
}
|
||
// No load: GPU total is negligible, all draw is the server's own baseline.
|
||
idleRow = append(idleRow, "—", fmt.Sprintf("%.0f", idleW), fmt.Sprintf("%.0f", idleW), "—", "—")
|
||
rampRows = append(rampRows, idleRow)
|
||
}
|
||
for _, step := range result.RampSteps {
|
||
row := []string{fmt.Sprintf("%d", step.StepIndex)}
|
||
for _, idx := range allGPUIndices {
|
||
inStep := false
|
||
for _, si := range step.GPUIndices {
|
||
if si == idx {
|
||
inStep = true
|
||
break
|
||
}
|
||
}
|
||
if !inStep {
|
||
row = append(row, "—")
|
||
continue
|
||
}
|
||
gpuPwr := "—"
|
||
if t, ok := step.PerGPUTelemetry[idx]; ok && t != nil && t.AvgPowerW > 0 {
|
||
gpuPwr = fmt.Sprintf("%.0f", t.AvgPowerW)
|
||
}
|
||
row = append(row, gpuPwr)
|
||
}
|
||
// GPU total W = sum of observed GPU power (nvidia-smi)
|
||
gpuTotal := "—"
|
||
if step.TotalObservedPowerW > 0 {
|
||
gpuTotal = fmt.Sprintf("%.0f", step.TotalObservedPowerW)
|
||
}
|
||
// Server itself W = server wall power minus GPU total (non-GPU baseline draw)
|
||
serverItself := "—"
|
||
if step.ServerLoadedW > 0 && step.TotalObservedPowerW > 0 {
|
||
serverItself = fmt.Sprintf("%.0f", step.ServerLoadedW-step.TotalObservedPowerW)
|
||
}
|
||
// Server wall W
|
||
serverWall := "—"
|
||
if step.ServerLoadedW > 0 {
|
||
serverWall = fmt.Sprintf("%.0f", step.ServerLoadedW)
|
||
}
|
||
// Per GPU wall W = ServerDeltaW / len(GPUIndices)
|
||
perGPUWall := "—"
|
||
if step.ServerDeltaW > 0 && len(step.GPUIndices) > 0 {
|
||
perGPUWall = fmt.Sprintf("%.0f", step.ServerDeltaW/float64(len(step.GPUIndices)))
|
||
}
|
||
// Platform eff. = (ServerLoadedW − idleW) / TotalObservedPowerW
|
||
platEff := "—"
|
||
if step.TotalObservedPowerW > 0 {
|
||
eff := step.ServerDeltaW / step.TotalObservedPowerW
|
||
if idleW > 0 && step.ServerLoadedW > 0 {
|
||
eff = (step.ServerLoadedW - idleW) / step.TotalObservedPowerW
|
||
}
|
||
platEff = fmt.Sprintf("%.2f", eff)
|
||
}
|
||
row = append(row, gpuTotal, serverItself, serverWall, perGPUWall, platEff)
|
||
rampRows = append(rampRows, row)
|
||
}
|
||
b.WriteString(fmtMDTable(headers, rampRows))
|
||
b.WriteString("\n")
|
||
}
|
||
|
||
// ── PSU Performance ───────────────────────────────────────────────────────
|
||
{
|
||
// Collect all PSU slot keys from any ramp step.
|
||
psuSlotSet := map[string]struct{}{}
|
||
for _, step := range result.RampSteps {
|
||
for k := range step.PSUSlotReadings {
|
||
psuSlotSet[k] = struct{}{}
|
||
}
|
||
}
|
||
if len(psuSlotSet) > 0 {
|
||
b.WriteString("## PSU Performance\n\n")
|
||
psuSlots := make([]string, 0, len(psuSlotSet))
|
||
for k := range psuSlotSet {
|
||
psuSlots = append(psuSlots, k)
|
||
}
|
||
sort.Strings(psuSlots)
|
||
|
||
var idleW float64
|
||
if result.ServerPower != nil {
|
||
idleW = result.ServerPower.IdleW
|
||
}
|
||
|
||
psuHeaders := []string{"Run"}
|
||
for _, slot := range psuSlots {
|
||
psuHeaders = append(psuHeaders, fmt.Sprintf("PSU %s W", slot))
|
||
}
|
||
psuHeaders = append(psuHeaders, "PSU Total W", "Platform eff.", "Avg Fan RPM (duty%)")
|
||
|
||
var psuRows [][]string
|
||
for _, step := range result.RampSteps {
|
||
row := []string{fmt.Sprintf("%d", step.StepIndex)}
|
||
var psuTotal float64
|
||
for _, slot := range psuSlots {
|
||
sp, ok := step.PSUSlotReadings[slot]
|
||
if !ok || sp.InputW == nil {
|
||
row = append(row, "—")
|
||
continue
|
||
}
|
||
row = append(row, fmt.Sprintf("%.0f", *sp.InputW))
|
||
psuTotal += *sp.InputW
|
||
}
|
||
totalStr := "—"
|
||
if psuTotal > 0 {
|
||
totalStr = fmt.Sprintf("%.0f", psuTotal)
|
||
}
|
||
platEff := "—"
|
||
if step.TotalObservedPowerW > 0 {
|
||
eff := step.ServerDeltaW / step.TotalObservedPowerW
|
||
if idleW > 0 && step.ServerLoadedW > 0 {
|
||
eff = (step.ServerLoadedW - idleW) / step.TotalObservedPowerW
|
||
}
|
||
platEff = fmt.Sprintf("%.2f", eff)
|
||
}
|
||
fan := "—"
|
||
if step.AvgFanRPM > 0 {
|
||
if step.AvgFanDutyCyclePct > 0 {
|
||
fan = fmt.Sprintf("%.0f (%.0f%%)", step.AvgFanRPM, step.AvgFanDutyCyclePct)
|
||
} else {
|
||
fan = fmt.Sprintf("%.0f", step.AvgFanRPM)
|
||
}
|
||
}
|
||
row = append(row, totalStr, platEff, fan)
|
||
psuRows = append(psuRows, row)
|
||
}
|
||
b.WriteString(fmtMDTable(psuHeaders, psuRows))
|
||
b.WriteString("\n")
|
||
}
|
||
}
|
||
|
||
// ── PSU Issues ────────────────────────────────────────────────────────────
|
||
if len(result.PSUIssues) > 0 {
|
||
b.WriteString("## PSU Issues\n\n")
|
||
b.WriteString("The following power supply anomalies were detected during the test:\n\n")
|
||
for _, issue := range result.PSUIssues {
|
||
fmt.Fprintf(&b, "- ⛔ %s\n", issue)
|
||
}
|
||
b.WriteString("\n")
|
||
}
|
||
|
||
// ── Power Distribution Summary ────────────────────────────────────────────
|
||
b.WriteString("## Power Distribution Summary\n\n")
|
||
{
|
||
var totalDefault, totalStable float64
|
||
for _, gpu := range result.GPUs {
|
||
stable := gpu.StablePowerLimitW
|
||
if stable <= 0 {
|
||
stable = gpu.AppliedPowerLimitW
|
||
}
|
||
totalDefault += gpu.DefaultPowerLimitW
|
||
totalStable += stable
|
||
}
|
||
var pdRows [][]string
|
||
for _, gpu := range result.GPUs {
|
||
stable := gpu.StablePowerLimitW
|
||
if stable <= 0 {
|
||
stable = gpu.AppliedPowerLimitW
|
||
}
|
||
realization := "-"
|
||
if gpu.DefaultPowerLimitW > 0 && stable > 0 {
|
||
realization = fmt.Sprintf("%.1f%%", stable/gpu.DefaultPowerLimitW*100)
|
||
}
|
||
derated := "-"
|
||
if gpu.Derated {
|
||
derated = "⚠ yes"
|
||
}
|
||
pdRows = append(pdRows, []string{
|
||
fmt.Sprintf("GPU %d", gpu.Index),
|
||
fmt.Sprintf("%.0f W", gpu.AppliedPowerLimitW),
|
||
fmt.Sprintf("%.0f W", stable),
|
||
realization,
|
||
derated,
|
||
})
|
||
}
|
||
platformReal := "-"
|
||
if totalDefault > 0 && totalStable > 0 {
|
||
platformReal = fmt.Sprintf("%.1f%%", totalStable/totalDefault*100)
|
||
}
|
||
pdRows = append(pdRows, []string{
|
||
"**Platform**",
|
||
"—",
|
||
fmt.Sprintf("**%.0f W**", totalStable),
|
||
fmt.Sprintf("**%s**", platformReal),
|
||
"",
|
||
})
|
||
b.WriteString(fmtMDTable([]string{"GPU", "Single-card limit", "Stable limit", "Realization", "Derated"}, pdRows))
|
||
b.WriteString("\n")
|
||
|
||
// Balance across GPUs — only meaningful with 2+ GPUs.
|
||
if len(result.GPUs) > 1 {
|
||
var minS, maxS, sumS float64
|
||
var cnt int
|
||
for _, gpu := range result.GPUs {
|
||
s := gpu.StablePowerLimitW
|
||
if s <= 0 {
|
||
s = gpu.AppliedPowerLimitW
|
||
}
|
||
if s <= 0 {
|
||
continue
|
||
}
|
||
sumS += s
|
||
cnt++
|
||
if cnt == 1 || s < minS {
|
||
minS = s
|
||
}
|
||
if s > maxS {
|
||
maxS = s
|
||
}
|
||
}
|
||
if cnt > 0 {
|
||
avg := sumS / float64(cnt)
|
||
spread := (maxS - minS) / avg * 100
|
||
balanceNote := "✓ balanced"
|
||
switch {
|
||
case spread > 20:
|
||
balanceNote = "⚠ significant imbalance — check slot thermals"
|
||
case spread > 10:
|
||
balanceNote = "— minor imbalance"
|
||
}
|
||
fmt.Fprintf(&b, "**GPU power balance:** avg %.0f W · min %.0f W · max %.0f W · spread %.1f%% — %s\n\n",
|
||
avg, minS, maxS, spread, balanceNote)
|
||
}
|
||
}
|
||
|
||
// Ramp scalability table — power efficiency of adding each GPU.
|
||
if len(result.RampSteps) > 1 {
|
||
b.WriteString("**Ramp power scalability** (stable TDP per step):\n\n")
|
||
var firstStable float64
|
||
if len(result.GPUs) > 0 {
|
||
firstStable = result.GPUs[0].StablePowerLimitW
|
||
if firstStable <= 0 {
|
||
firstStable = result.GPUs[0].AppliedPowerLimitW
|
||
}
|
||
}
|
||
var prevCumulative float64
|
||
var scalRows [][]string
|
||
for _, step := range result.RampSteps {
|
||
var cumulative float64
|
||
for _, gpuIdx := range step.GPUIndices {
|
||
for _, g := range result.GPUs {
|
||
if g.Index != gpuIdx {
|
||
continue
|
||
}
|
||
s := g.StablePowerLimitW
|
||
if s <= 0 {
|
||
s = g.AppliedPowerLimitW
|
||
}
|
||
cumulative += s
|
||
}
|
||
}
|
||
incremental := cumulative - prevCumulative
|
||
efficiency := "—"
|
||
if step.StepIndex > 1 && firstStable > 0 {
|
||
efficiency = fmt.Sprintf("%.1f%%", incremental/firstStable*100)
|
||
}
|
||
scalRows = append(scalRows, []string{
|
||
fmt.Sprintf("%d", step.StepIndex),
|
||
joinIndexList(step.GPUIndices),
|
||
fmt.Sprintf("%.0f W", cumulative),
|
||
fmt.Sprintf("%.0f W", incremental),
|
||
efficiency,
|
||
})
|
||
prevCumulative = cumulative
|
||
}
|
||
b.WriteString(fmtMDTable([]string{"Step", "GPUs", "Cumulative stable TDP", "Incremental", "Efficiency vs GPU 1"}, scalRows))
|
||
b.WriteString("\n")
|
||
}
|
||
}
|
||
|
||
// ── Per-GPU sections ──────────────────────────────────────────────────────
|
||
var lastStep *NvidiaPowerBenchStep
|
||
if n := len(result.RampSteps); n > 0 {
|
||
lastStep = &result.RampSteps[n-1]
|
||
}
|
||
for _, gpu := range result.GPUs {
|
||
fmt.Fprintf(&b, "### GPU %d — %s\n\n", gpu.Index, gpu.Name)
|
||
|
||
// Transposed comparison table: Single Run vs All GPU Run.
|
||
singleClk := "—"
|
||
singleMem := "—"
|
||
singleTemp := "—"
|
||
singlePwr := "—"
|
||
singleWall := "—"
|
||
singleFan := "—"
|
||
if gpu.Telemetry != nil {
|
||
singleClk = fmt.Sprintf("%.0f", gpu.Telemetry.AvgGraphicsClockMHz)
|
||
singleMem = fmt.Sprintf("%.0f", gpu.Telemetry.AvgMemoryClockMHz)
|
||
singleTemp = fmt.Sprintf("%.1f", gpu.Telemetry.AvgTempC)
|
||
singlePwr = fmt.Sprintf("%.0f W", gpu.Telemetry.AvgPowerW)
|
||
}
|
||
if gpu.ServerDeltaW > 0 {
|
||
singleWall = fmt.Sprintf("%.0f W", gpu.ServerDeltaW)
|
||
}
|
||
if gpu.AvgFanRPM > 0 {
|
||
if gpu.AvgFanDutyCyclePct > 0 {
|
||
singleFan = fmt.Sprintf("%.0f RPM (%.0f%%)", gpu.AvgFanRPM, gpu.AvgFanDutyCyclePct)
|
||
} else {
|
||
singleFan = fmt.Sprintf("%.0f RPM", gpu.AvgFanRPM)
|
||
}
|
||
}
|
||
|
||
allClk := "—"
|
||
allMem := "—"
|
||
allTemp := "—"
|
||
allPwr := "—"
|
||
allWall := "—"
|
||
allFan := "—"
|
||
if lastStep != nil {
|
||
if t, ok := lastStep.PerGPUTelemetry[gpu.Index]; ok && t != nil {
|
||
allClk = fmt.Sprintf("%.0f", t.AvgGraphicsClockMHz)
|
||
allMem = fmt.Sprintf("%.0f", t.AvgMemoryClockMHz)
|
||
allTemp = fmt.Sprintf("%.1f", t.AvgTempC)
|
||
allPwr = fmt.Sprintf("%.0f W", t.AvgPowerW)
|
||
}
|
||
if lastStep.ServerDeltaW > 0 && len(lastStep.GPUIndices) > 0 {
|
||
allWall = fmt.Sprintf("%.0f W", lastStep.ServerDeltaW/float64(len(lastStep.GPUIndices)))
|
||
}
|
||
if lastStep.AvgFanRPM > 0 {
|
||
if lastStep.AvgFanDutyCyclePct > 0 {
|
||
allFan = fmt.Sprintf("%.0f RPM (%.0f%%)", lastStep.AvgFanRPM, lastStep.AvgFanDutyCyclePct)
|
||
} else {
|
||
allFan = fmt.Sprintf("%.0f RPM", lastStep.AvgFanRPM)
|
||
}
|
||
}
|
||
}
|
||
|
||
tableHeaders := []string{"", "Single Run"}
|
||
if lastStep != nil {
|
||
tableHeaders = append(tableHeaders, "All GPU Run")
|
||
}
|
||
compRows := [][]string{
|
||
{"Clock MHz (Mem MHz)", fmt.Sprintf("%s (%s)", singleClk, singleMem)},
|
||
{"Avg Temp °C", singleTemp},
|
||
{"Power W", singlePwr},
|
||
{"Per GPU wall W", singleWall},
|
||
{"Avg Fan RPM (duty%)", singleFan},
|
||
}
|
||
if lastStep != nil {
|
||
compRows[0] = append(compRows[0], fmt.Sprintf("%s (%s)", allClk, allMem))
|
||
compRows[1] = append(compRows[1], allTemp)
|
||
compRows[2] = append(compRows[2], allPwr)
|
||
compRows[3] = append(compRows[3], allWall)
|
||
compRows[4] = append(compRows[4], allFan)
|
||
}
|
||
b.WriteString(fmtMDTable(tableHeaders, compRows))
|
||
b.WriteString("\n")
|
||
|
||
for _, note := range gpu.Notes {
|
||
fmt.Fprintf(&b, "- %s\n", note)
|
||
}
|
||
if len(gpu.Notes) > 0 {
|
||
b.WriteString("\n")
|
||
}
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func renderPowerBenchSummary(result NvidiaPowerBenchResult) string {
|
||
var b strings.Builder
|
||
fmt.Fprintf(&b, "run_at_utc=%s\n", result.GeneratedAt.Format(time.RFC3339))
|
||
fmt.Fprintf(&b, "benchmark_version=%s\n", result.BenchmarkVersion)
|
||
fmt.Fprintf(&b, "benchmark_profile=%s\n", result.BenchmarkProfile)
|
||
fmt.Fprintf(&b, "overall_status=%s\n", result.OverallStatus)
|
||
fmt.Fprintf(&b, "platform_max_tdp_w=%.0f\n", result.PlatformMaxTDPW)
|
||
fmt.Fprintf(&b, "gpu_count=%d\n", len(result.GPUs))
|
||
if len(result.RecommendedSlotOrder) > 0 {
|
||
fmt.Fprintf(&b, "recommended_slot_order=%s\n", joinIndexList(result.RecommendedSlotOrder))
|
||
}
|
||
for _, step := range result.RampSteps {
|
||
fmt.Fprintf(&b, "ramp_step_%d_gpus=%s\n", step.StepIndex, joinIndexList(step.GPUIndices))
|
||
fmt.Fprintf(&b, "ramp_step_%d_new_gpu=%d\n", step.StepIndex, step.NewGPUIndex)
|
||
fmt.Fprintf(&b, "ramp_step_%d_stable_limit_w=%.0f\n", step.StepIndex, step.NewGPUStableLimitW)
|
||
fmt.Fprintf(&b, "ramp_step_%d_total_power_w=%.0f\n", step.StepIndex, step.TotalObservedPowerW)
|
||
if step.ServerLoadedW > 0 {
|
||
fmt.Fprintf(&b, "ramp_step_%d_server_loaded_w=%.0f\n", step.StepIndex, step.ServerLoadedW)
|
||
fmt.Fprintf(&b, "ramp_step_%d_server_delta_w=%.0f\n", step.StepIndex, step.ServerDeltaW)
|
||
}
|
||
}
|
||
for _, gpu := range result.GPUs {
|
||
if gpu.StablePowerLimitW > 0 {
|
||
fmt.Fprintf(&b, "gpu_%d_stable_limit_w=%.0f\n", gpu.Index, gpu.StablePowerLimitW)
|
||
}
|
||
if gpu.ServerLoadedW > 0 {
|
||
fmt.Fprintf(&b, "gpu_%d_server_loaded_w=%.0f\n", gpu.Index, gpu.ServerLoadedW)
|
||
fmt.Fprintf(&b, "gpu_%d_server_delta_w=%.0f\n", gpu.Index, gpu.ServerDeltaW)
|
||
}
|
||
}
|
||
if sp := result.ServerPower; sp != nil && sp.Available {
|
||
fmt.Fprintf(&b, "server_idle_w=%.0f\n", sp.IdleW)
|
||
fmt.Fprintf(&b, "server_loaded_w=%.0f\n", sp.LoadedW)
|
||
fmt.Fprintf(&b, "server_delta_w=%.0f\n", sp.DeltaW)
|
||
fmt.Fprintf(&b, "server_reporting_ratio=%.2f\n", sp.ReportingRatio)
|
||
}
|
||
return b.String()
|
||
}
|