fix: surface CPU thermal throttling in status and SAT results
A CPU that had thermally throttled (sysfs thermal_throttle counter > 0) still reported status "OK" everywhere: dmidecode-derived CPU status only distinguishes populated/enabled/disabled and never looked at the throttle flag the collector already recorded next to it, and neither SAT path meant to catch this actually could: - The routine "cpu" SAT pack (RunCPUAcceptancePack) only checked lscpu/sensors/stress-ng exit codes — stress-ng exits 0 whether or not the CPU throttled while running it, so an 89°C/throttled CPU right after a "successful" run still showed cpu:all as OK in component-status.json. - The more thorough platform-stress test already detected throttling and fan-spindown correctly, but wrote its verdict as "Overall: FAIL — ..." with no "=", which parseSATKV can't parse — so even a real detected throttle event never reached the component-status DB. Fixes: - cpu_telemetry.go: escalate a CPU's status to Warning (only-escalate, same severity ranking already used elsewhere) when Throttled is set. - sat.go: add a before/after thermal-throttle-counter check job around the "cpu" pack's stress-ng run, so a throttle event during the run fails that job and (via the existing FAILED->Warning DB mapping) flips cpu:all to Warning. - platform_stress.go: emit a machine-readable overall_status= line alongside the human-readable verdict so platform-stress results actually reach ApplySATResultToDB. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
56d12b1f3c
commit
198567dffe
@@ -684,3 +684,94 @@ func TestRunAcceptancePackCtxRetriesJob(t *testing.T) {
|
||||
t.Fatalf("summary=%q want flaky_status=OK", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCPUSATJobsIncludeThrottleCheck(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
jobs := cpuSATJobs(60)
|
||||
|
||||
if len(jobs) != 6 {
|
||||
t.Fatalf("jobs=%d want 6", len(jobs))
|
||||
}
|
||||
if jobs[2].name != "02-thermal-throttle-before.log" || !jobs[2].informational {
|
||||
t.Fatalf("throttle-before job=%+v want informational before-snapshot", jobs[2])
|
||||
}
|
||||
if jobs[5].name != "05-thermal-throttle-check.log" || jobs[5].informational {
|
||||
t.Fatalf("throttle-check job=%+v want non-informational so it can fail overall_status", jobs[5])
|
||||
}
|
||||
}
|
||||
|
||||
// TestCPUThrottleCheckDetectsIncreaseDuringRun runs the real before/check
|
||||
// scripts (the same ones RunCPUAcceptancePack shells out to) against a fake
|
||||
// sysfs tree, guarding both that a genuine thermal-throttle event during the
|
||||
// stress-ng run is caught, and that a host with no new throttling (or none
|
||||
// at all — e.g. AMD/ARM, which lack this sysfs interface) stays OK.
|
||||
func TestCPUThrottleCheckDetectsIncreaseDuringRun(t *testing.T) {
|
||||
fakeSys := t.TempDir()
|
||||
oldDir := cpuThermalThrottleSysDir
|
||||
cpuThermalThrottleSysDir = fakeSys
|
||||
t.Cleanup(func() { cpuThermalThrottleSysDir = oldDir })
|
||||
|
||||
writeCounter := func(cpu, file, value string) {
|
||||
path := filepath.Join(fakeSys, cpu, "thermal_throttle", file)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(value+"\n"), 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
run := func(t *testing.T, dir string) (string, error) {
|
||||
t.Helper()
|
||||
runDir, err := runAcceptancePackCtx(context.Background(), dir, "cpu-throttle-test", []satJob{
|
||||
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
|
||||
{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("reading summary.txt: %v", err)
|
||||
}
|
||||
return string(summary), nil
|
||||
}
|
||||
|
||||
t.Run("no throttling stays OK", func(t *testing.T) {
|
||||
writeCounter("cpu0", "core_throttle_count", "0")
|
||||
writeCounter("cpu1", "core_throttle_count", "0")
|
||||
|
||||
summary, err := run(t, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("runAcceptancePackCtx error: %v", err)
|
||||
}
|
||||
if !strings.Contains(summary, "overall_status=OK") {
|
||||
t.Fatalf("summary=%q want overall_status=OK", summary)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("throttle count increasing during run fails overall_status", func(t *testing.T) {
|
||||
writeCounter("cpu0", "core_throttle_count", "3")
|
||||
writeCounter("cpu1", "core_throttle_count", "0")
|
||||
|
||||
dir := t.TempDir()
|
||||
runDir, err := runAcceptancePackCtx(context.Background(), dir, "cpu-throttle-test", []satJob{
|
||||
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("runAcceptancePackCtx (before) error: %v", err)
|
||||
}
|
||||
|
||||
// Simulate the stress-ng run itself tripping the throttle.
|
||||
writeCounter("cpu0", "core_throttle_count", "5")
|
||||
|
||||
out, err := exec.Command("sh", "-c", strings.ReplaceAll(cpuThrottleCheckScript(), "{{run_dir}}", runDir)).CombinedOutput()
|
||||
if err == nil {
|
||||
t.Fatalf("check script succeeded, want failure; output=%s", out)
|
||||
}
|
||||
if !strings.Contains(string(out), "THROTTLE DETECTED") {
|
||||
t.Fatalf("output=%s want THROTTLE DETECTED", out)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user