fix: surface CPU thermal throttling in status and SAT results

A CPU that had thermally throttled (sysfs thermal_throttle counter >
0) still reported status "OK" everywhere: dmidecode-derived CPU status
only distinguishes populated/enabled/disabled and never looked at the
throttle flag the collector already recorded next to it, and neither
SAT path meant to catch this actually could:

- The routine "cpu" SAT pack (RunCPUAcceptancePack) only checked
  lscpu/sensors/stress-ng exit codes — stress-ng exits 0 whether or
  not the CPU throttled while running it, so an 89°C/throttled CPU
  right after a "successful" run still showed cpu:all as OK in
  component-status.json.
- The more thorough platform-stress test already detected throttling
  and fan-spindown correctly, but wrote its verdict as "Overall: FAIL
  — ..." with no "=", which parseSATKV can't parse — so even a real
  detected throttle event never reached the component-status DB.

Fixes:
- cpu_telemetry.go: escalate a CPU's status to Warning (only-escalate,
  same severity ranking already used elsewhere) when Throttled is set.
- sat.go: add a before/after thermal-throttle-counter check job around
  the "cpu" pack's stress-ng run, so a throttle event during the run
  fails that job and (via the existing FAILED->Warning DB mapping)
  flips cpu:all to Warning.
- platform_stress.go: emit a machine-readable overall_status= line
  alongside the human-readable verdict so platform-stress results
  actually reach ApplySATResultToDB.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-08-24 17:41:40 +03:00
co-authored by Claude Sonnet 5
parent 56d12b1f3c
commit 198567dffe
6 changed files with 264 additions and 6 deletions
+61 -6
View File
@@ -793,16 +793,71 @@ func (s *System) RunSATStressPack(ctx context.Context, baseDir string, durationS
}, logFunc)
}
// cpuThermalThrottleSysDir is the sysfs root the throttle-check scripts glob
// under. Overridden in tests so they can point at a fake directory tree
// instead of the real /sys.
var cpuThermalThrottleSysDir = "/sys/devices/system/cpu"
// cpuThrottleSumScript is the shell fragment both before/after scripts use to
// sum the kernel's cumulative-since-boot thermal throttle counters across
// every CPU.
func cpuThrottleSumScript() string {
return fmt.Sprintf(`
sum=0
for f in %[1]s/cpu*/thermal_throttle/core_throttle_count %[1]s/cpu*/thermal_throttle/package_throttle_count; do
[ -f "$f" ] || continue
v=$(cat "$f" 2>/dev/null)
case "$v" in ''|*[!0-9]*) continue ;; esac
sum=$((sum + v))
done
`, cpuThermalThrottleSysDir)
}
// cpuThrottleBeforeScript snapshots the throttle counter sum into a file in
// {{run_dir}} so cpuThrottleCheckScript can later diff before/after despite
// each satJob running as an independent process.
func cpuThrottleBeforeScript() string {
return cpuThrottleSumScript() + `echo "$sum" | tee {{run_dir}}/.cpu-throttle-before` + "\n"
}
// cpuThrottleCheckScript compares the after-run throttle counter sum against
// the snapshot cpuThrottleBeforeScript took, and fails (non-zero exit) if it
// increased — i.e. the CPU actually hit thermal throttling during this
// specific run, not just at some earlier point this boot. classifySATResult
// maps a failed job here to SAT status FAILED, which ApplySATResultToDB
// records as component status "Warning" for cpu:all — without this, the
// "cpu" SAT pack only checks stress-ng's exit code, which is 0 whether or
// not the CPU throttled while running it.
func cpuThrottleCheckScript() string {
return `before=$(cat {{run_dir}}/.cpu-throttle-before 2>/dev/null)
case "$before" in ''|*[!0-9]*) before=0 ;; esac
` + cpuThrottleSumScript() + `after=$sum
echo "throttle_count_before=$before"
echo "throttle_count_after=$after"
if [ "$after" -gt "$before" ]; then
echo "THROTTLE DETECTED: CPU package/core hit thermal throttling during this stress-ng run ($before -> $after)"
exit 1
fi
echo "no new thermal throttling detected during this run"
`
}
func cpuSATJobs(durationSec int) []satJob {
return []satJob{
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
}
}
func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
if durationSec <= 0 {
durationSec = 60
}
return runAcceptancePackCtx(ctx, baseDir, "cpu", []satJob{
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
}, logFunc)
return runAcceptancePackCtx(ctx, baseDir, "cpu", cpuSATJobs(durationSec), logFunc)
}
func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) {