fix: surface CPU thermal throttling in status and SAT results

A CPU that had thermally throttled (sysfs thermal_throttle counter >
0) still reported status "OK" everywhere: dmidecode-derived CPU status
only distinguishes populated/enabled/disabled and never looked at the
throttle flag the collector already recorded next to it, and neither
SAT path meant to catch this actually could:

- The routine "cpu" SAT pack (RunCPUAcceptancePack) only checked
  lscpu/sensors/stress-ng exit codes — stress-ng exits 0 whether or
  not the CPU throttled while running it, so an 89°C/throttled CPU
  right after a "successful" run still showed cpu:all as OK in
  component-status.json.
- The more thorough platform-stress test already detected throttling
  and fan-spindown correctly, but wrote its verdict as "Overall: FAIL
  — ..." with no "=", which parseSATKV can't parse — so even a real
  detected throttle event never reached the component-status DB.

Fixes:
- cpu_telemetry.go: escalate a CPU's status to Warning (only-escalate,
  same severity ranking already used elsewhere) when Throttled is set.
- sat.go: add a before/after thermal-throttle-counter check job around
  the "cpu" pack's stress-ng run, so a throttle event during the run
  fails that job and (via the existing FAILED->Warning DB mapping)
  flips cpu:all to Warning.
- platform_stress.go: emit a machine-readable overall_status= line
  alongside the human-readable verdict so platform-stress results
  actually reach ApplySATResultToDB.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-08-24 17:41:40 +03:00
co-authored by Claude Sonnet 5
parent 56d12b1f3c
commit 198567dffe
6 changed files with 264 additions and 6 deletions
+91
View File
@@ -684,3 +684,94 @@ func TestRunAcceptancePackCtxRetriesJob(t *testing.T) {
t.Fatalf("summary=%q want flaky_status=OK", got)
}
}
func TestCPUSATJobsIncludeThrottleCheck(t *testing.T) {
t.Parallel()
jobs := cpuSATJobs(60)
if len(jobs) != 6 {
t.Fatalf("jobs=%d want 6", len(jobs))
}
if jobs[2].name != "02-thermal-throttle-before.log" || !jobs[2].informational {
t.Fatalf("throttle-before job=%+v want informational before-snapshot", jobs[2])
}
if jobs[5].name != "05-thermal-throttle-check.log" || jobs[5].informational {
t.Fatalf("throttle-check job=%+v want non-informational so it can fail overall_status", jobs[5])
}
}
// TestCPUThrottleCheckDetectsIncreaseDuringRun runs the real before/check
// scripts (the same ones RunCPUAcceptancePack shells out to) against a fake
// sysfs tree, guarding both that a genuine thermal-throttle event during the
// stress-ng run is caught, and that a host with no new throttling (or none
// at all — e.g. AMD/ARM, which lack this sysfs interface) stays OK.
func TestCPUThrottleCheckDetectsIncreaseDuringRun(t *testing.T) {
fakeSys := t.TempDir()
oldDir := cpuThermalThrottleSysDir
cpuThermalThrottleSysDir = fakeSys
t.Cleanup(func() { cpuThermalThrottleSysDir = oldDir })
writeCounter := func(cpu, file, value string) {
path := filepath.Join(fakeSys, cpu, "thermal_throttle", file)
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
t.Fatalf("mkdir: %v", err)
}
if err := os.WriteFile(path, []byte(value+"\n"), 0644); err != nil {
t.Fatalf("write: %v", err)
}
}
run := func(t *testing.T, dir string) (string, error) {
t.Helper()
runDir, err := runAcceptancePackCtx(context.Background(), dir, "cpu-throttle-test", []satJob{
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
}, nil)
if err != nil {
return "", err
}
summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
if err != nil {
t.Fatalf("reading summary.txt: %v", err)
}
return string(summary), nil
}
t.Run("no throttling stays OK", func(t *testing.T) {
writeCounter("cpu0", "core_throttle_count", "0")
writeCounter("cpu1", "core_throttle_count", "0")
summary, err := run(t, t.TempDir())
if err != nil {
t.Fatalf("runAcceptancePackCtx error: %v", err)
}
if !strings.Contains(summary, "overall_status=OK") {
t.Fatalf("summary=%q want overall_status=OK", summary)
}
})
t.Run("throttle count increasing during run fails overall_status", func(t *testing.T) {
writeCounter("cpu0", "core_throttle_count", "3")
writeCounter("cpu1", "core_throttle_count", "0")
dir := t.TempDir()
runDir, err := runAcceptancePackCtx(context.Background(), dir, "cpu-throttle-test", []satJob{
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
}, nil)
if err != nil {
t.Fatalf("runAcceptancePackCtx (before) error: %v", err)
}
// Simulate the stress-ng run itself tripping the throttle.
writeCounter("cpu0", "core_throttle_count", "5")
out, err := exec.Command("sh", "-c", strings.ReplaceAll(cpuThrottleCheckScript(), "{{run_dir}}", runDir)).CombinedOutput()
if err == nil {
t.Fatalf("check script succeeded, want failure; output=%s", out)
}
if !strings.Contains(string(out), "THROTTLE DETECTED") {
t.Fatalf("output=%s want THROTTLE DETECTED", out)
}
})
}