fix(sat): fan check uses hottest GPU load + IPMI-hang-proof polling
- GPU load switches from bee-gpu-burn (compute burn, ~88% TDP) to dcgmproftester -t 1004 / targeted_power via resolveBenchmarkPowerLoadCommand — the same engine Power/Thermal Fit uses and the hottest sustained NVIDIA load we have, so fans are actually pushed toward their ceiling. - Sample loop is now IPMI-hang-proof: every ipmitool read is time-boxed in an abandonable goroutine, and the poll interval backs off geometrically (1s→30s) when reads are slow, tightening again on recovery. A plateau is only trusted while telemetry is healthy; degraded runs ride out to MaxLoadSec. Summary gains fan_samples / telemetry_degraded. Drops the per-second nvidia-smi+power+cpu-temp sampling from the hot loop. - Dead code removed: FanStressRow, GPUStressMetric, sampleFanStressRow, sampleGPUStressMetrics, WriteFanStressCSV/WriteFanSensorsCSV, analyzeMaxTemp, sampleSystemPowerResolved. Topology fan tiles: - size encodes the fan's ceiling RPM (its class), not current speed; coloured fill rising from the bottom encodes live duty cycle (current / ceiling), shown only when the ceiling was measured. - glyph spin rate now maps absolute RPM into a human-perceptible band (fanSpinPeriodSec: 2.2s/turn at <=1000 RPM, 0.35s at >=13000). - the "N fans · N OK · tile size ∝ …" caption line is gone. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
bb2a501a28
commit
8cb250f3f4
@@ -12,6 +12,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -35,29 +36,6 @@ type FanReading struct {
|
||||
RPM float64
|
||||
}
|
||||
|
||||
// GPUStressMetric holds per-GPU metrics during the stress test.
|
||||
type GPUStressMetric struct {
|
||||
Index int
|
||||
TempC float64
|
||||
UsagePct float64
|
||||
PowerW float64
|
||||
ClockMHz float64
|
||||
Throttled bool // true if any throttle reason is active
|
||||
}
|
||||
|
||||
// FanStressRow is one second-interval telemetry sample covering all monitored dimensions.
|
||||
type FanStressRow struct {
|
||||
TimestampUTC string
|
||||
ElapsedSec float64
|
||||
Phase string // "baseline", "load1", "pause", "load2", "cooldown"
|
||||
GPUs []GPUStressMetric
|
||||
Fans []FanReading
|
||||
CPUMaxTempC float64 // highest CPU temperature from ipmitool / sensors
|
||||
SysPowerW float64
|
||||
SysPowerSource string
|
||||
SysPowerMode string
|
||||
}
|
||||
|
||||
type cachedPowerReading struct {
|
||||
Value float64
|
||||
Source string
|
||||
@@ -76,8 +54,6 @@ type fanPeakCandidate struct {
|
||||
}
|
||||
|
||||
var (
|
||||
systemPowerCacheMu sync.Mutex
|
||||
systemPowerCache cachedPowerReading
|
||||
fanObservationMu sync.Mutex
|
||||
fanObservation fanObservationState
|
||||
fanObservationInit bool
|
||||
@@ -157,10 +133,16 @@ func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckO
|
||||
logFunc(fmt.Sprintf("Fan check: %d fans; load = CPU/mem:%v + GPU:%s", len(baseFans), haveCPU, orNone(haveGPU, vendor)))
|
||||
|
||||
// ── Load: every source runs at the same time, each in its own goroutine.
|
||||
// Sources can take different amounts of time to actually reach full load
|
||||
// (stressapptest is near-instant; a GPU burn kernel needs to compile and
|
||||
// ramp), so the plateau clock does not start until every launched source
|
||||
// has reported its process running, plus a fixed GPU ramp grace.
|
||||
// GPU load is the hottest sustained NVIDIA load we have — dcgmproftester
|
||||
// -t 1004 / targeted_power, the same engine the Power/Thermal Fit
|
||||
// benchmark uses (resolveBenchmarkPowerLoadCommand) — not the
|
||||
// compute-throughput bee-gpu-burn, which tops out well below TDP and so
|
||||
// never demands the fans' true ceiling.
|
||||
//
|
||||
// Sources reach full load at different times (stressapptest is instant; a
|
||||
// dcgmproftester kernel compiles and ramps), so the plateau clock does not
|
||||
// start until every launched source reports its process running, plus a
|
||||
// fixed GPU ramp grace.
|
||||
loadCtx, loadCancel := context.WithTimeout(ctx, time.Duration(opts.MaxLoadSec)*time.Second)
|
||||
defer loadCancel()
|
||||
|
||||
@@ -190,14 +172,20 @@ func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckO
|
||||
loadWG.Add(1)
|
||||
go func() {
|
||||
defer loadWG.Done()
|
||||
cmd := buildGPUStressCmd(loadCtx, vendor, opts.MaxLoadSec)
|
||||
if cmd == nil {
|
||||
logFunc("GPU load unavailable (no burn tool for " + vendor + ")")
|
||||
appendSATVerboseLog(verboseLog, "gpu load: no burn tool")
|
||||
cmd, label, err := buildFanCheckGPULoadCmd(loadCtx, vendor, opts.MaxLoadSec, opts.GPUIndices)
|
||||
if err != nil || cmd == nil {
|
||||
logFunc("GPU load unavailable: " + errString(err))
|
||||
appendSATVerboseLog(verboseLog, "gpu load unavailable: "+errString(err))
|
||||
started <- false
|
||||
return
|
||||
}
|
||||
logFunc("GPU load running (" + vendor + ")")
|
||||
if err := cmd.Start(); err != nil {
|
||||
logFunc("GPU load failed to start: " + err.Error())
|
||||
appendSATVerboseLog(verboseLog, "gpu load start error: "+err.Error())
|
||||
started <- false
|
||||
return
|
||||
}
|
||||
logFunc("GPU load running (" + label + ")")
|
||||
started <- true
|
||||
_ = cmd.Wait()
|
||||
}()
|
||||
@@ -226,19 +214,39 @@ func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckO
|
||||
appendSATVerboseLog(verboseLog, fmt.Sprintf("%d load source(s) active; plateau clock effective from +%.0fs",
|
||||
activeLoads, readyAt.Sub(start).Seconds()))
|
||||
|
||||
// ── Sample loop: one row/second, per-fan plateau tracking.
|
||||
// ── Sample loop with IPMI-hang protection.
|
||||
//
|
||||
// Under full load ipmitool over KCS can take tens of seconds per call or
|
||||
// wedge outright. So: every fan read is time-boxed (readFansBounded runs
|
||||
// it in a goroutine we abandon on timeout — a wedged KCS read can never
|
||||
// block this loop), and the polling interval backs off geometrically when
|
||||
// reads are slow and tightens again when they recover. A plateau is only
|
||||
// declared while telemetry is healthy (interval near the floor); a
|
||||
// degraded run just rides out to MaxLoadSec and records the peak it saw.
|
||||
const (
|
||||
fanPollFloor = 1 * time.Second
|
||||
fanPollCeil = 30 * time.Second
|
||||
fanReadTMO = 8 * time.Second
|
||||
)
|
||||
type fanState struct {
|
||||
peak float64
|
||||
lastRiseSec float64
|
||||
}
|
||||
fanBy := map[string]*fanState{}
|
||||
var rows []FanStressRow
|
||||
rampConfirmed := false
|
||||
plateauReached := false
|
||||
aborted := false
|
||||
degraded := false
|
||||
goodSamples := 0
|
||||
poll := fanPollFloor
|
||||
|
||||
csvPath := filepath.Join(runDir, "fan-sensors.csv")
|
||||
_ = os.WriteFile(csvPath, []byte("elapsed_sec,fan_name,rpm\n"), 0644)
|
||||
csvFile, _ := os.OpenFile(csvPath, os.O_APPEND|os.O_WRONLY, 0644)
|
||||
if csvFile != nil {
|
||||
defer csvFile.Close()
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
loop:
|
||||
for {
|
||||
select {
|
||||
@@ -247,13 +255,36 @@ loop:
|
||||
break loop
|
||||
case <-loadCtx.Done():
|
||||
break loop // MaxLoadSec reached
|
||||
case <-ticker.C:
|
||||
case <-time.After(poll):
|
||||
}
|
||||
elapsed := time.Since(start).Seconds()
|
||||
row := sampleFanStressRow(opts.GPUIndices, "load", elapsed)
|
||||
rows = append(rows, row)
|
||||
|
||||
for _, f := range row.Fans {
|
||||
readStart := time.Now()
|
||||
fans, ok := readFansBounded(fanReadTMO)
|
||||
readDur := time.Since(readStart)
|
||||
|
||||
// Adapt the interval to how ipmitool is behaving.
|
||||
switch {
|
||||
case !ok || readDur > fanReadTMO*3/4:
|
||||
if poll < fanPollCeil {
|
||||
poll = minDuration(poll*2, fanPollCeil)
|
||||
degraded = true
|
||||
appendSATVerboseLog(verboseLog, fmt.Sprintf("[%.0fs] ipmitool slow (%.1fs, ok=%v) — polling backed off to %s",
|
||||
elapsed, readDur.Seconds(), ok, poll))
|
||||
logFunc(fmt.Sprintf("IPMI slow — fan polling backed off to %s", poll))
|
||||
}
|
||||
case poll > fanPollFloor && readDur < fanPollFloor:
|
||||
poll = maxDuration(poll*2/3, fanPollFloor)
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
goodSamples++
|
||||
|
||||
for _, f := range fans {
|
||||
if csvFile != nil {
|
||||
fmt.Fprintf(csvFile, "%.0f,%s,%.0f\n", elapsed, f.Name, f.RPM)
|
||||
}
|
||||
st := fanBy[f.Name]
|
||||
if st == nil {
|
||||
fanBy[f.Name] = &fanState{peak: f.RPM, lastRiseSec: elapsed}
|
||||
@@ -270,7 +301,11 @@ loop:
|
||||
}
|
||||
}
|
||||
|
||||
if elapsed >= float64(opts.MinLoadSec) && time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 {
|
||||
// Only trust a plateau while telemetry is healthy and we have enough
|
||||
// recent samples to have actually seen a flat window.
|
||||
healthy := poll <= 2*fanPollFloor && goodSamples >= 5
|
||||
if healthy && rampConfirmed && elapsed >= float64(opts.MinLoadSec) &&
|
||||
time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 {
|
||||
allFlat := true
|
||||
for _, st := range fanBy {
|
||||
if elapsed-st.lastRiseSec < float64(opts.PlateauHoldSec) {
|
||||
@@ -278,7 +313,7 @@ loop:
|
||||
break
|
||||
}
|
||||
}
|
||||
if allFlat && rampConfirmed {
|
||||
if allFlat {
|
||||
plateauReached = true
|
||||
logFunc(fmt.Sprintf("All %d fans plateaued at %.0fs of load", len(fanBy), elapsed))
|
||||
break loop
|
||||
@@ -295,6 +330,8 @@ loop:
|
||||
return runDir, ctx.Err()
|
||||
}
|
||||
|
||||
loadDur := time.Since(start).Seconds()
|
||||
|
||||
// ── Verdict.
|
||||
statuses := readFanStatuses()
|
||||
var summary strings.Builder
|
||||
@@ -304,19 +341,12 @@ loop:
|
||||
fmt.Fprintf(&summary, "gpu_vendor=%s\n", orNone(haveGPU, vendor))
|
||||
fmt.Fprintf(&summary, "plateau_reached=%v\n", plateauReached)
|
||||
fmt.Fprintf(&summary, "ramp_confirmed=%v\n", rampConfirmed)
|
||||
if len(rows) > 0 {
|
||||
fmt.Fprintf(&summary, "load_duration_sec=%.0f\n", rows[len(rows)-1].ElapsedSec)
|
||||
fmt.Fprintf(&summary, "load_duration_sec=%.0f\n", loadDur)
|
||||
fmt.Fprintf(&summary, "fan_samples=%d\n", goodSamples)
|
||||
fmt.Fprintf(&summary, "telemetry_degraded=%v\n", degraded)
|
||||
if t := boundedGPUMaxTemp(opts.GPUIndices); t > 0 {
|
||||
fmt.Fprintf(&summary, "gpu_temp_c=%.0f\n", t)
|
||||
}
|
||||
fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 {
|
||||
var m float64
|
||||
for _, g := range r.GPUs {
|
||||
if g.TempC > m {
|
||||
m = g.TempC
|
||||
}
|
||||
}
|
||||
return m
|
||||
}))
|
||||
fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 { return r.CPUMaxTempC }))
|
||||
|
||||
stats := satStats{}
|
||||
names := make([]string, 0, len(baselineRPM))
|
||||
@@ -349,14 +379,126 @@ loop:
|
||||
}
|
||||
writeSATStats(&summary, stats)
|
||||
|
||||
_ = WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices)
|
||||
_ = WriteFanSensorsCSV(filepath.Join(runDir, "fan-sensors.csv"), rows)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
|
||||
func errString(err error) string {
|
||||
if err == nil {
|
||||
return "no GPU stress tool"
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
func minDuration(a, b time.Duration) time.Duration {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func maxDuration(a, b time.Duration) time.Duration {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// readFansBounded runs "ipmitool sdr type Fan" but never blocks the caller
|
||||
// longer than timeout: the read happens in a goroutine that is abandoned if it
|
||||
// does not return in time (a KCS read wedged in uninterruptible I/O cannot be
|
||||
// killed, so we leave it and move on). ok=false means "no usable sample this
|
||||
// tick" — the caller must treat that as missing data, not as a flat fan.
|
||||
func readFansBounded(timeout time.Duration) ([]FanReading, bool) {
|
||||
type result struct {
|
||||
fans []FanReading
|
||||
ok bool
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
|
||||
if err != nil {
|
||||
ch <- result{}
|
||||
return
|
||||
}
|
||||
fans := parseFanSpeeds(string(out))
|
||||
if len(fans) == 0 {
|
||||
ch <- result{}
|
||||
return
|
||||
}
|
||||
ch <- result{fans, true}
|
||||
}()
|
||||
select {
|
||||
case r := <-ch:
|
||||
if r.ok {
|
||||
updateFanObservation(r.fans, time.Now())
|
||||
}
|
||||
return r.fans, r.ok
|
||||
case <-time.After(timeout):
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// boundedGPUMaxTemp returns the hottest GPU temperature via a single
|
||||
// time-boxed nvidia-smi call, or 0 if unavailable.
|
||||
func boundedGPUMaxTemp(gpuIndices []int) float64 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
|
||||
defer cancel()
|
||||
args := []string{"--query-gpu=temperature.gpu", "--format=csv,noheader,nounits"}
|
||||
if len(gpuIndices) > 0 {
|
||||
ids := make([]string, len(gpuIndices))
|
||||
for i, idx := range gpuIndices {
|
||||
ids[i] = strconv.Itoa(idx)
|
||||
}
|
||||
args = append([]string{"--id=" + strings.Join(ids, ",")}, args...)
|
||||
}
|
||||
out, err := exec.CommandContext(ctx, "nvidia-smi", args...).Output()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
var max float64
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
if v, err := strconv.ParseFloat(strings.TrimSpace(line), 64); err == nil && v > max {
|
||||
max = v
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
// buildFanCheckGPULoadCmd builds the hottest sustained GPU load for the fan
|
||||
// check. NVIDIA uses the Power/Thermal Fit engine (dcgmproftester -t 1004 /
|
||||
// targeted_power); AMD uses the RVS gst stressor.
|
||||
func buildFanCheckGPULoadCmd(ctx context.Context, vendor string, durSec int, gpuIndices []int) (*exec.Cmd, string, error) {
|
||||
switch strings.ToLower(vendor) {
|
||||
case "nvidia":
|
||||
argv, env, err := resolveBenchmarkPowerLoadCommand(durSec, gpuIndices)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
|
||||
if len(env) > 0 {
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
}
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
cmd.Cancel = func() error {
|
||||
if cmd.Process != nil {
|
||||
_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return cmd, "dcgmproftester targeted_power", nil
|
||||
case "amd":
|
||||
cmd := buildAMDGPUStressCmd(ctx, durSec)
|
||||
if cmd == nil {
|
||||
return nil, "", nil
|
||||
}
|
||||
return cmd, "rvs gst", nil
|
||||
}
|
||||
return nil, "", nil
|
||||
}
|
||||
|
||||
func applyFanCheckDefaults(o *FanCheckOptions) {
|
||||
if o.PlateauHoldSec <= 0 {
|
||||
o.PlateauHoldSec = 60
|
||||
@@ -459,68 +601,6 @@ func ResolveFanMaxRPM(current map[string]float64) map[string]float64 {
|
||||
return out
|
||||
}
|
||||
|
||||
// sampleFanStressRow collects all metrics for one telemetry sample.
|
||||
func sampleFanStressRow(gpuIndices []int, phase string, elapsed float64) FanStressRow {
|
||||
row := FanStressRow{
|
||||
TimestampUTC: time.Now().UTC().Format(time.RFC3339),
|
||||
ElapsedSec: elapsed,
|
||||
Phase: phase,
|
||||
}
|
||||
row.GPUs = sampleGPUStressMetrics(gpuIndices)
|
||||
row.Fans, _ = sampleFanSpeeds()
|
||||
row.CPUMaxTempC = sampleCPUMaxTemp()
|
||||
row.SysPowerW, row.SysPowerSource, row.SysPowerMode = sampleSystemPowerResolved()
|
||||
return row
|
||||
}
|
||||
|
||||
// sampleGPUStressMetrics queries nvidia-smi for temperature, utilization, power,
|
||||
// clock frequency, and active throttle reasons for each GPU.
|
||||
func sampleGPUStressMetrics(gpuIndices []int) []GPUStressMetric {
|
||||
args := []string{
|
||||
"--query-gpu=index,temperature.gpu,utilization.gpu,power.draw,clocks.current.graphics,clocks_throttle_reasons.active",
|
||||
"--format=csv,noheader,nounits",
|
||||
}
|
||||
if len(gpuIndices) > 0 {
|
||||
ids := make([]string, len(gpuIndices))
|
||||
for i, idx := range gpuIndices {
|
||||
ids[i] = strconv.Itoa(idx)
|
||||
}
|
||||
args = append([]string{"--id=" + strings.Join(ids, ",")}, args...)
|
||||
}
|
||||
out, err := exec.Command("nvidia-smi", args...).Output()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var metrics []GPUStressMetric
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(line, ", ")
|
||||
if len(parts) < 6 {
|
||||
continue
|
||||
}
|
||||
idx, _ := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||||
throttleVal := strings.TrimSpace(parts[5])
|
||||
// Throttled if active reasons bitmask is non-zero.
|
||||
throttled := throttleVal != "0x0000000000000000" &&
|
||||
throttleVal != "0x0" &&
|
||||
throttleVal != "0" &&
|
||||
throttleVal != "" &&
|
||||
throttleVal != "N/A"
|
||||
metrics = append(metrics, GPUStressMetric{
|
||||
Index: idx,
|
||||
TempC: parseGPUFloat(parts[1]),
|
||||
UsagePct: parseGPUFloat(parts[2]),
|
||||
PowerW: parseGPUFloat(parts[3]),
|
||||
ClockMHz: parseGPUFloat(parts[4]),
|
||||
Throttled: throttled,
|
||||
})
|
||||
}
|
||||
return metrics
|
||||
}
|
||||
|
||||
// sampleFanSpeeds reads fan RPM values from ipmitool sdr.
|
||||
func sampleFanSpeeds() ([]FanReading, error) {
|
||||
out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
|
||||
@@ -940,21 +1020,6 @@ func sampleCPUTempViaSensors() float64 {
|
||||
return max
|
||||
}
|
||||
|
||||
// sampleSystemPowerResolved reads system power via the global autotune source,
|
||||
// falling back to the historical heuristic before autotune or when degraded.
|
||||
func sampleSystemPowerResolved() (float64, string, string) {
|
||||
now := time.Now()
|
||||
current, decision, err := SampleSystemPowerResolved("")
|
||||
systemPowerCacheMu.Lock()
|
||||
defer systemPowerCacheMu.Unlock()
|
||||
if err != nil {
|
||||
current = 0
|
||||
}
|
||||
value, updated := effectiveSystemPowerReading(systemPowerCache, current, decision.EffectiveSource, decision.Mode, decision.Reason, now)
|
||||
systemPowerCache = updated
|
||||
return value, updated.Source, updated.Mode
|
||||
}
|
||||
|
||||
// parseDCMIPowerReading extracts the instantaneous power reading from ipmitool dcmi output.
|
||||
// Sample: " Instantaneous power reading: 500 Watts"
|
||||
func parseDCMIPowerReading(raw string) float64 {
|
||||
@@ -986,76 +1051,6 @@ func effectiveSystemPowerReading(cache cachedPowerReading, current float64, sour
|
||||
return 0, cache
|
||||
}
|
||||
|
||||
// analyzeMaxTemp returns the maximum value of the given extractor across all rows.
|
||||
func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) float64 {
|
||||
var max float64
|
||||
for _, row := range rows {
|
||||
if v := extract(row); v > max {
|
||||
max = v
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
// WriteFanStressCSV writes the wide-format metrics CSV with one row per second.
|
||||
// GPU columns are generated per index in gpuIndices order.
|
||||
func WriteFanStressCSV(path string, rows []FanStressRow, gpuIndices []int) error {
|
||||
if len(rows) == 0 {
|
||||
return os.WriteFile(path, []byte("no data\n"), 0644)
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
|
||||
// Header: fixed system columns + per-GPU columns.
|
||||
b.WriteString("timestamp_utc,elapsed_sec,phase,fan_avg_rpm,fan_min_rpm,fan_max_rpm,cpu_max_temp_c,sys_power_w")
|
||||
for _, idx := range gpuIndices {
|
||||
fmt.Fprintf(&b, ",gpu%d_temp_c,gpu%d_usage_pct,gpu%d_power_w,gpu%d_clock_mhz,gpu%d_throttled",
|
||||
idx, idx, idx, idx, idx)
|
||||
}
|
||||
b.WriteRune('\n')
|
||||
|
||||
for _, row := range rows {
|
||||
favg, fmin, fmax := fanRPMStats(row.Fans)
|
||||
fmt.Fprintf(&b, "%s,%.1f,%s,%.0f,%.0f,%.0f,%.1f,%.1f",
|
||||
row.TimestampUTC,
|
||||
row.ElapsedSec,
|
||||
row.Phase,
|
||||
favg, fmin, fmax,
|
||||
row.CPUMaxTempC,
|
||||
row.SysPowerW,
|
||||
)
|
||||
gpuByIdx := make(map[int]GPUStressMetric, len(row.GPUs))
|
||||
for _, g := range row.GPUs {
|
||||
gpuByIdx[g.Index] = g
|
||||
}
|
||||
for _, idx := range gpuIndices {
|
||||
g := gpuByIdx[idx]
|
||||
throttled := 0
|
||||
if g.Throttled {
|
||||
throttled = 1
|
||||
}
|
||||
fmt.Fprintf(&b, ",%.1f,%.1f,%.1f,%.0f,%d",
|
||||
g.TempC, g.UsagePct, g.PowerW, g.ClockMHz, throttled)
|
||||
}
|
||||
b.WriteRune('\n')
|
||||
}
|
||||
|
||||
return os.WriteFile(path, []byte(b.String()), 0644)
|
||||
}
|
||||
|
||||
// WriteFanSensorsCSV writes individual fan sensor readings in long (tidy) format.
|
||||
func WriteFanSensorsCSV(path string, rows []FanStressRow) error {
|
||||
var b strings.Builder
|
||||
b.WriteString("timestamp_utc,elapsed_sec,phase,fan_name,rpm\n")
|
||||
for _, row := range rows {
|
||||
for _, f := range row.Fans {
|
||||
fmt.Fprintf(&b, "%s,%.1f,%s,%s,%.0f\n",
|
||||
row.TimestampUTC, row.ElapsedSec, row.Phase, f.Name, f.RPM)
|
||||
}
|
||||
}
|
||||
return os.WriteFile(path, []byte(b.String()), 0644)
|
||||
}
|
||||
|
||||
// fanRPMStats computes average, min, max RPM across all fans in a sample row.
|
||||
func fanRPMStats(fans []FanReading) (avg, min, max float64) {
|
||||
if len(fans) == 0 {
|
||||
|
||||
Reference in New Issue
Block a user