@@ -22,18 +22,53 @@ var ansiEscapePattern = regexp.MustCompile(`\x1b\[[0-9;]*m`)
func renderBenchmarkReportWithCharts ( result NvidiaBenchmarkResult , charts [ ] benchmarkReportChart ) string {
var b strings . Builder
fmt . Fprintf ( & b , "Bee NVIDIA Benchmark Report\n" )
fmt . Fprintf ( & b , "===========================\n\n" )
fmt . Fprintf ( & b , "Generated: %s\n" , result . GeneratedAt . Format ( "2006-01-02 15:04:05 UTC" ) )
fmt . Fprintf ( & b , "Host: %s\n" , result . Hostname )
fmt . Fprintf ( & b , "Profile: %s\n" , result . BenchmarkProfile )
fmt . Fprintf ( & b , "Overall status: %s\n" , result . OverallStatus )
fmt . Fprintf ( & b , "Selected GPUs: %s\n" , joinIndexList ( result . SelectedGPUIndices ) )
fmt . Fprintf ( & b , "Normalization: %s\n\n" , result . Normalization . Status )
// ── Header ────────────────────────────────────────────────────────────────
b . WriteString ( "# Bee NVIDIA Benchmark Report\n\n" )
// System identity block
if result . ServerModel != "" {
fmt . Fprintf ( & b , "**Server:** %s \n" , result . ServerModel )
}
if result . Hostname != "" {
fmt . Fprintf ( & b , "**Host:** %s \n" , result . Hostname )
}
// GPU models summary
if len ( result . GPUs ) > 0 {
modelCount := make ( map [ string ] int )
var modelOrder [ ] string
for _ , g := range result . GPUs {
m := strings . TrimSpace ( g . Name )
if m == "" {
m = "Unknown GPU"
}
if modelCount [ m ] == 0 {
modelOrder = append ( modelOrder , m )
}
modelCount [ m ] ++
}
var parts [ ] string
for _ , m := range modelOrder {
if modelCount [ m ] == 1 {
parts = append ( parts , m )
} else {
parts = append ( parts , fmt . Sprintf ( "%d× %s" , modelCount [ m ] , m ) )
}
}
fmt . Fprintf ( & b , "**GPU(s):** %s \n" , strings . Join ( parts , ", " ) )
}
fmt . Fprintf ( & b , "**Profile:** %s \n" , result . BenchmarkProfile )
fmt . Fprintf ( & b , "**App version:** %s \n" , result . BenchmarkVersion )
fmt . Fprintf ( & b , "**Generated:** %s \n" , result . GeneratedAt . Format ( "2006-01-02 15:04:05 UTC" ) )
if result . ParallelGPUs {
fmt . Fprintf ( & b , "**Mode:** parallel (all GPUs simultaneously) \n" )
}
fmt . Fprintf ( & b , "**Overall status:** %s \n" , result . OverallStatus )
b . WriteString ( "\n" )
// ── Executive Summary ─────────────────────────────────────────────────────
if len ( result . Findings ) > 0 {
fmt . Fprintf ( & b , " Executive Summary\n")
fmt . Fprintf ( & b , "-----------------\n" )
b . WriteString ( "## Executive Summary\n\n ")
for _ , finding := range result . Findings {
fmt . Fprintf ( & b , "- %s\n" , finding )
}
@@ -41,149 +76,206 @@ func renderBenchmarkReportWithCharts(result NvidiaBenchmarkResult, charts []benc
}
if len ( result . Warnings ) > 0 {
fmt . Fprintf ( & b , " Warnings\n")
fmt . Fprintf ( & b , "--------\n" )
b . WriteString ( "## Warnings\n\n ")
for _ , warning := range result . Warnings {
fmt . Fprintf ( & b , "- %s\n" , warning )
}
b . WriteString ( "\n" )
}
fmt . Fprintf ( & b , "Per GPU Scorecard\n" )
fmt . Fprintf ( & b , "----------------- \n")
// ── Scorecard table ───────────────────────────────────────────────────────
b . WriteString ( "## Scorecard\n \n")
b . WriteString ( "| GPU | Status | Composite | Compute | TOPS/SM/GHz | Power Sustain | Thermal Sustain | Stability | Interconnect |\n" )
b . WriteString ( "|-----|--------|-----------|---------|-------------|---------------|-----------------|-----------|-------------|\n" )
for _ , gpu := range result . GPUs {
fmt . Fprintf ( & b , "GPU %d %s\n" , gpu . Index , gpu . Name )
fmt . Fprintf ( & b , " Status: %s\n" , gpu . Status )
fmt . Fprintf ( & b , " Composite score: %.2f\n" , gpu . Scores . CompositeScore )
fmt . Fprintf ( & b , " Compute score: %.2f\n" , gpu . Scores . ComputeScore )
if gpu . Scores . TOPSPerSMPerGHz > 0 {
fmt . Fprintf ( & b , " Compute efficiency: %.3f TOPS/SM/GHz\n" , gpu . Scores . TOPSPerSMPerGHz )
name := strings . TrimSpace ( gpu . Name )
if name == "" {
name = "Unknown"
}
fmt . Fprintf ( & b , " Power sustain: %.1f\n" , gpu . Scores . PowerSustainScore )
fmt . Fprintf ( & b , " Thermal sustain: %.1f\n" , gpu . Scores . ThermalSustainScore )
fmt . Fprintf ( & b , " Stability: %.1f\n" , gpu . Scores . StabilityScore )
interconnect := "-"
if gpu . Scores . InterconnectScore > 0 {
fmt . Fprintf ( & b , " Interconnect: %.1f\n ", gpu . Scores . InterconnectScore )
interconnect = fmt . Sprintf ( " %.1f", gpu . Scores . InterconnectScore )
}
if len ( gpu . DegradationReasons ) > 0 {
fmt . Fprintf ( & b , " Degradation reasons: %s\n" , strings . Join ( gpu . DegradationReasons , ", " ) )
topsPerSM := "-"
if gpu . Scores . TOPSPerSMPerGHz > 0 {
topsPerSM = fmt . Sprintf ( "%.3f" , gpu . Scores . TOPSPerSMPerGHz )
}
fmt . Fprintf ( & b , " Avg power/temp/clock: %.1f W / %.1f C / %.0 f MHz\n" , gpu . Steady . AvgPowerW , gpu . Steady . AvgTempC , gpu . Steady . AvgGraphicsClockMHz )
fmt . Fprintf ( & b , " P95 power/temp/clock: %.1f W / %.1f C / %.0f MHz\n" , gpu . Steady . P95PowerW , gpu . Steady . P95TempC , gpu . Steady . P95GraphicsClockMHz )
if len ( gpu . PrecisionResults ) > 0 {
fmt . Fprintf ( & b , " Precision results:\n" )
for _ , precision := range gpu . PrecisionResults {
if precision . Supported {
fmt . Fprintf ( & b , " - %s: %.2f TOPS lanes=%d iterations=%d\n" , precision . Name , precision . TeraOpsPerSec , precision . Lanes , precision . Iterations )
} else {
fmt . Fprintf ( & b , " - %s: unsupported (%s)\n" , precision . Name , precision . Notes )
}
}
fmt . Fprintf ( & b , "| GPU %d %s | %s | **%.2f** | %.2f | %s | %.1f | %.1f | %.1 f | %s |\n" ,
gpu . Index , name ,
gpu . Status ,
gpu . Scores . CompositeScore ,
gpu . Scores . ComputeScore ,
topsPerSM ,
gpu . Scores . PowerSustainScore ,
gpu . Scores . ThermalSustainScore ,
gpu . Scores . StabilityScore ,
interconnect ,
)
}
b . WriteString ( "\n" )
// ── Per GPU detail ────────────────────────────────────────────────────────
b . WriteString ( "## Per-GPU Details\n\n" )
for _ , gpu := range result . GPUs {
name := strings . TrimSpace ( gpu . Name )
if name == "" {
name = "Unknown GPU"
}
fmt . Fprintf ( & b , " Throttle: %s\n" , formatThrottleLine ( gpu . Throttle , gpu . Steady . DurationSec ) )
if len ( gpu . Notes ) > 0 {
fmt . Fprintf ( & b , " Notes:\n" )
for _ , note : = range gpu . Notes {
fmt . Fprintf ( & b , " - %s\n" , note )
}
fmt . Fprintf ( & b , "### GPU %d — %s\n\n " , gpu . Index , name )
// Identity
if gpu . BusID ! = "" {
fmt . Fprintf ( & b , "- **Bus ID:** %s\n" , gpu . BusID )
}
if gpu . VBIOS != "" {
fmt . Fprintf ( & b , "- **vBIOS:** %s\n" , gpu . VBIOS )
}
if gpu . ComputeCapability != "" {
fmt . Fprintf ( & b , "- **Compute capability:** %s\n" , gpu . ComputeCapability )
}
if gpu . MultiprocessorCount > 0 {
fmt . Fprintf ( & b , "- **SMs:** %d\n" , gpu . MultiprocessorCount )
}
if gpu . PowerLimitW > 0 {
fmt . Fprintf ( & b , "- **Power limit:** %.0f W (default %.0f W)\n" , gpu . PowerLimitW , gpu . DefaultPowerLimitW )
}
if gpu . LockedGraphicsClockMHz > 0 {
fmt . Fprintf ( & b , "- **Locked clocks:** GPU %.0f MHz / Mem %.0f MHz\n" , gpu . LockedGraphicsClockMHz , gpu . LockedMemoryClockMHz )
}
b . WriteString ( "\n" )
// Steady-state telemetry
fmt . Fprintf ( & b , "**Steady-state telemetry** (%ds):\n\n" , int ( gpu . Steady . DurationSec ) )
b . WriteString ( "| | Avg | P95 |\n|---|---|---|\n" )
fmt . Fprintf ( & b , "| Power | %.1f W | %.1f W |\n" , gpu . Steady . AvgPowerW , gpu . Steady . P95PowerW )
fmt . Fprintf ( & b , "| Temperature | %.1f °C | %.1f °C |\n" , gpu . Steady . AvgTempC , gpu . Steady . P95TempC )
fmt . Fprintf ( & b , "| GPU clock | %.0f MHz | %.0f MHz |\n" , gpu . Steady . AvgGraphicsClockMHz , gpu . Steady . P95GraphicsClockMHz )
fmt . Fprintf ( & b , "| Memory clock | %.0f MHz | %.0f MHz |\n" , gpu . Steady . AvgMemoryClockMHz , gpu . Steady . P95MemoryClockMHz )
fmt . Fprintf ( & b , "| GPU utilisation | %.1f %% | — |\n" , gpu . Steady . AvgUsagePct )
b . WriteString ( "\n" )
// Throttle
throttle := formatThrottleLine ( gpu . Throttle , gpu . Steady . DurationSec )
if throttle != "none" {
fmt . Fprintf ( & b , "**Throttle:** %s\n\n" , throttle )
}
// Precision results
if len ( gpu . PrecisionResults ) > 0 {
b . WriteString ( "**Precision results:**\n\n" )
b . WriteString ( "| Precision | TOPS | Lanes | Iterations |\n|-----------|------|-------|------------|\n" )
for _ , p := range gpu . PrecisionResults {
if p . Supported {
fmt . Fprintf ( & b , "| %s | %.2f | %d | %d |\n" , p . Name , p . TeraOpsPerSec , p . Lanes , p . Iterations )
} else {
fmt . Fprintf ( & b , "| %s | — (unsupported) | — | — |\n" , p . Name )
}
}
b . WriteString ( "\n" )
}
// Degradation / Notes
if len ( gpu . DegradationReasons ) > 0 {
fmt . Fprintf ( & b , "**Degradation reasons:** %s\n\n" , strings . Join ( gpu . DegradationReasons , ", " ) )
}
if len ( gpu . Notes ) > 0 {
b . WriteString ( "**Notes:**\n\n" )
for _ , note := range gpu . Notes {
fmt . Fprintf ( & b , "- %s\n" , note )
}
b . WriteString ( "\n" )
}
}
// ── Interconnect ──────────────────────────────────────────────────────────
if result . Interconnect != nil {
fmt . Fprintf ( & b , " Interconnect\n")
fmt . Fprintf ( & b , "------------\n" )
fmt . Fprintf ( & b , "Status: %s\n" , result . Interconnect . Status )
b . WriteString ( "## Interconnect (NCCL)\n \n")
fmt . Fprintf ( & b , "**Status:** %s\n\n" , result . Interconnect . Status )
if result . Interconnect . Supported {
fmt . Fprintf ( & b , "Avg algbw / busbw: %.1f / %.1f GB/s\n" , result . Interconnect . AvgAlgBWGBps , result . Interconnect . AvgBusBWGBps )
fmt . Fprintf ( & b , "Max algbw / busbw: %.1f / %.1f GB/s\n" , result . Interconnect . MaxAlgBWGBps , result . Interconnect . MaxBusBWGBps )
b . WriteString ( "| Metric | Avg | Max |\n|--------|-----|-----|\n" )
fmt . Fprintf ( & b , "| Alg BW | %.1f GB/s | %.1f GB/s | \n" , result . Interconnect . AvgAlgBWGBps , result . Interconnect . MaxAlgBWGBps )
fmt . Fprintf ( & b , "| Bus BW | %.1f GB/s | %.1f GB/s |\n" , result . Interconnect . AvgBusBWGBps , result . Interconnect . MaxBusBWGBps )
b . WriteString ( "\n" )
}
for _ , note := range result . Interconnect . Notes {
fmt . Fprintf ( & b , "- %s\n" , note )
}
b . WriteString ( "\n" )
if len ( result . Interconnect . Notes ) > 0 {
b . WriteString ( "\n" )
}
}
// ── Server Power (IPMI) ───────────────────────────────────────────────────
if sp := result . ServerPower ; sp != nil {
b . WriteString ( "## Server Power (IPMI)\n\n" )
if ! sp . Available {
b . WriteString ( "IPMI power measurement unavailable.\n\n" )
} else {
b . WriteString ( "| | Value |\n|---|---|\n" )
fmt . Fprintf ( & b , "| Server idle | %.0f W |\n" , sp . IdleW )
fmt . Fprintf ( & b , "| Server under load | %.0f W |\n" , sp . LoadedW )
fmt . Fprintf ( & b , "| Server delta (load − idle) | %.0f W |\n" , sp . DeltaW )
fmt . Fprintf ( & b , "| GPU-reported sum | %.0f W |\n" , sp . GPUReportedSumW )
if sp . ReportingRatio > 0 {
fmt . Fprintf ( & b , "| Reporting ratio | %.2f (1.0 = accurate, <0.75 = GPU over-reports) |\n" , sp . ReportingRatio )
}
b . WriteString ( "\n" )
}
for _ , note := range sp . Notes {
fmt . Fprintf ( & b , "- %s\n" , note )
}
if len ( sp . Notes ) > 0 {
b . WriteString ( "\n" )
}
}
// ── Terminal charts (steady-state only) ───────────────────────────────────
if len ( charts ) > 0 {
fmt . Fprintf ( & b , "Terminal Charts\n")
fmt . Fprintf ( & b , "---------------\n" )
b . WriteString ( "## Steady-State Charts\n\n ")
for _ , chart := range charts {
content := strings . TrimSpace ( stripANSIEscapeSequences ( chart . Content ) )
if content == "" {
continue
}
fmt . Fprintf ( & b , "%s \n" , chart . Title )
fmt . Fprintf ( & b , "%s\n" , strings . Repeat ( "~" , len ( chart . Title ) ) )
fmt . Fprintf ( & b , "%s\n\n" , content )
fmt . Fprintf ( & b , "### %s\n\n```\n%s\n```\n \n" , chart . Title , content )
}
}
if sp := result . ServerPower ; sp != nil {
fmt . Fprintf ( & b , "Server Power (IPMI) \n")
fmt . Fprintf ( & b , "-------------------\n" )
if ! sp . Available {
fmt . Fprintf ( & b , "Unavailable \n")
} else {
fmt . Fprintf ( & b , " Server idle: %.0f W\n" , sp . IdleW )
fmt . Fprintf ( & b , " Server under load: %.0f W\n" , sp . LoadedW )
fmt . Fprintf ( & b , " Server delta: %.0f W\n" , sp . DeltaW )
fmt . Fprintf ( & b , " GPU reported (sum): %.0f W\n" , sp . GPUReportedSumW )
if sp . ReportingRatio > 0 {
fmt . Fprintf ( & b , " Reporting ratio: %.2f (1.0 = accurate, <0.75 = GPU over-reports)\n" , sp . ReportingRatio )
}
}
for _ , note := range sp . Notes {
fmt . Fprintf ( & b , " Note: %s\n" , note )
}
b . WriteString ( "\n" )
}
// ── Methodology ───────────────────────────────────────────────────────────
b . WriteString ( "## Methodology\n \n")
fmt . Fprintf ( & b , "- Profile `%s` uses standardized baseline → warmup → steady-state → interconnect → cooldown phases.\n" , result . BenchmarkProfile )
b . WriteString ( "- Single-GPU compute score from bee-gpu-burn cuBLASLt when available.\n" )
b . WriteString ( "- Thermal and power limitations inferred from NVIDIA clock event reason counters and sustained telemetry. \n")
b . WriteString ( "- `result.json` is the canonical machine-readable source for this benchmark run.\n\n" )
fmt . Fprintf ( & b , "Methodology\n" )
fmt . Fprintf ( & b , "----------- \n")
fmt . Fprintf ( & b , "- Profile %s uses standardized baseline, warmup, steady-state, interconnect, and cooldown phases.\n" , result . BenchmarkProfile )
fmt . Fprintf ( & b , "- Single-GPU compute score comes from bee-gpu-burn cuBLASLt output when available. \n")
fmt . Fprintf ( & b , "- Thermal and power limitations are inferred from NVIDIA clock event reason counters and sustained telemetry. \n")
fmt . Fprintf ( & b , "- result.json is the canonical machine-readable source for this benchmark run.\n \n")
fmt . Fprintf ( & b , "Raw Files \n")
fmt . Fprintf ( & b , "---------\n" )
fmt . Fprintf ( & b , "- result.json\n" )
fmt . Fprintf ( & b , "- report.txt\n" )
fmt . Fprintf ( & b , "- summary.txt\n" )
fmt . Fprintf ( & b , "- verbose.log\n" )
fmt . Fprintf ( & b , "- gpu-*-baseline-metrics.csv/html/term.txt\n" )
fmt . Fprintf ( & b , "- gpu-*-warmup.log\n" )
fmt . Fprintf ( & b , "- gpu-*-steady.log\n" )
fmt . Fprintf ( & b , "- gpu-*-steady-metrics.csv/html/term.txt\n" )
fmt . Fprintf ( & b , "- gpu-*-cooldown-metrics.csv/html/term.txt\n" )
// ── Raw files ─────────────────────────────────────────────────────────────
b . WriteString ( "## Raw Files\n \n")
b . WriteString ( "- `result.json`\n- `report.md`\n- `summary.txt`\n- `verbose.log`\n" )
b . WriteString ( "- `gpu-*-baseline-metrics.csv/html/term.txt` \n")
b . WriteString ( "- `gpu-*-warmup.log` \n")
b . WriteString ( "- `gpu-*-steady.log` \n")
b . WriteString ( "- `gpu-*-steady-metrics.csv/html/term.txt`\n" )
b . WriteString ( "- `gpu-*-cooldown-metrics.csv/html/term.txt` \n")
if result . Interconnect != nil {
fmt . Fprintf ( & b , "- nccl-all-reduce.log\n" )
b . WriteString ( "- ` nccl-all-reduce.log` \n" )
}
return b . String ( )
}
// loadBenchmarkReportCharts loads only steady-state terminal charts (baseline and
// cooldown charts are not useful for human review).
func loadBenchmarkReportCharts ( runDir string , gpuIndices [ ] int ) [ ] benchmarkReportChart {
phases := [ ] struct {
name string
label string
} {
{ name : "baseline" , label : "Baseline" } ,
{ name : "steady" , label : "Steady State" } ,
{ name : "cooldown" , label : "Cooldown" } ,
}
var charts [ ] benchmarkReportChart
for _ , idx := range gpuIndices {
for _ , phase := range phases {
path := filepath . Join ( runDir , fmt . Sprintf ( "gpu-%d-%s-metrics-term.txt" , idx , phase . name ) )
raw , err : = os . ReadFile ( path )
if err != nil || len ( raw ) == 0 {
continue
}
charts = append ( charts , benchmarkReportChart {
Title : fmt . Sprintf ( "GPU %d %s" , idx , phase . label ) ,
Content : string ( raw ) ,
} )
path := filepath . Join ( runDir , fmt . Sprintf ( "gpu-%d-steady-metrics-term.txt" , idx ) )
raw , err := os . ReadFile ( path )
if err ! = nil || len ( raw ) == 0 {
continue
}
charts = append ( charts , benchmarkReportChart {
Title : fmt . Sprintf ( "GPU %d — Steady State" , idx ) ,
Content : string ( raw ) ,
} )
}
return charts
}