fix: surface CPU thermal throttling in status and SAT results

A CPU that had thermally throttled (sysfs thermal_throttle counter >
0) still reported status "OK" everywhere: dmidecode-derived CPU status
only distinguishes populated/enabled/disabled and never looked at the
throttle flag the collector already recorded next to it, and neither
SAT path meant to catch this actually could:

- The routine "cpu" SAT pack (RunCPUAcceptancePack) only checked
  lscpu/sensors/stress-ng exit codes — stress-ng exits 0 whether or
  not the CPU throttled while running it, so an 89°C/throttled CPU
  right after a "successful" run still showed cpu:all as OK in
  component-status.json.
- The more thorough platform-stress test already detected throttling
  and fan-spindown correctly, but wrote its verdict as "Overall: FAIL
  — ..." with no "=", which parseSATKV can't parse — so even a real
  detected throttle event never reached the component-status DB.

Fixes:
- cpu_telemetry.go: escalate a CPU's status to Warning (only-escalate,
  same severity ranking already used elsewhere) when Throttled is set.
- sat.go: add a before/after thermal-throttle-counter check job around
  the "cpu" pack's stress-ng run, so a throttle event during the run
  fails that job and (via the existing FAILED->Warning DB mapping)
  flips cpu:all to Warning.
- platform_stress.go: emit a machine-readable overall_status= line
  alongside the human-readable verdict so platform-stress results
  actually reach ApplySATResultToDB.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-08-24 17:41:40 +03:00
co-authored by Claude Sonnet 5
parent 56d12b1f3c
commit 198567dffe
6 changed files with 264 additions and 6 deletions
+24
View File
@@ -37,12 +37,36 @@ func enrichCPUsWithTelemetry(cpus []schema.HardwareCPU, doc sensorsDoc) []schema
}
if value, ok := throttleBySocket[socket]; ok {
cpus[i].Throttled = &value
if value {
escalateCPUThrottleStatus(&cpus[i])
}
}
}
return cpus
}
// escalateCPUThrottleStatus raises a CPU's status to at least Warning when it
// has hit thermal throttling (cpuPackageThrottled: a cumulative since-boot
// sysfs counter, not "is throttling right now"). dmidecode-derived status
// (parseCPUStatus) only distinguishes populated/enabled/disabled and has no
// way to know about a live thermal event, so without this a CPU that
// throttled during e.g. a stress test still reports status "OK" — see
// enrichCPUsWithTelemetry.
func escalateCPUThrottleStatus(cpu *schema.HardwareCPU) {
current := ""
if cpu.Status != nil {
current = strings.TrimSpace(*cpu.Status)
}
if current != "" && current != statusUnknown && StatusSeverity(statusWarning) <= StatusSeverity(current) {
return
}
status := statusWarning
cpu.Status = &status
desc := "CPU hit thermal throttling (package/core throttle count > 0 since boot)"
cpu.ErrorDescription = &desc
}
func cpuTempsFromSensors(doc sensorsDoc, cpuCount int) map[int]float64 {
out := map[int]float64{}
if len(doc) == 0 {
@@ -49,6 +49,12 @@ func TestEnrichCPUsWithTelemetry(t *testing.T) {
if got[0].Throttled == nil || !*got[0].Throttled {
t.Fatalf("cpu0 throttled mismatch: %#v", got[0].Throttled)
}
if got[0].Status == nil || *got[0].Status != statusWarning {
t.Fatalf("cpu0 status not escalated to Warning on throttle: %#v", got[0].Status)
}
if got[0].ErrorDescription == nil || *got[0].ErrorDescription == "" {
t.Fatalf("cpu0 error description not set on throttle")
}
if got[1].TemperatureC == nil || *got[1].TemperatureC != 58.0 {
t.Fatalf("cpu1 temperature mismatch: %#v", got[1].TemperatureC)
}
@@ -58,6 +64,26 @@ func TestEnrichCPUsWithTelemetry(t *testing.T) {
if got[1].Throttled != nil && *got[1].Throttled {
t.Fatalf("cpu1 throttled mismatch: %#v", got[1].Throttled)
}
if got[1].Status == nil || *got[1].Status != statusOK {
t.Fatalf("cpu1 status should remain OK when not throttled: %#v", got[1].Status)
}
}
func TestEscalateCPUThrottleStatusDoesNotDowngradeCritical(t *testing.T) {
status := statusCritical
desc := "pre-existing critical finding"
cpu := schema.HardwareCPU{
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status, ErrorDescription: &desc},
}
escalateCPUThrottleStatus(&cpu)
if *cpu.Status != statusCritical {
t.Fatalf("status downgraded from Critical: %#v", cpu.Status)
}
if *cpu.ErrorDescription != desc {
t.Fatalf("error description overwritten: %#v", cpu.ErrorDescription)
}
}
func mustWriteFile(t *testing.T, path, content string) {