fix: surface CPU thermal throttling in status and SAT results
A CPU that had thermally throttled (sysfs thermal_throttle counter > 0) still reported status "OK" everywhere: dmidecode-derived CPU status only distinguishes populated/enabled/disabled and never looked at the throttle flag the collector already recorded next to it, and neither SAT path meant to catch this actually could: - The routine "cpu" SAT pack (RunCPUAcceptancePack) only checked lscpu/sensors/stress-ng exit codes — stress-ng exits 0 whether or not the CPU throttled while running it, so an 89°C/throttled CPU right after a "successful" run still showed cpu:all as OK in component-status.json. - The more thorough platform-stress test already detected throttling and fan-spindown correctly, but wrote its verdict as "Overall: FAIL — ..." with no "=", which parseSATKV can't parse — so even a real detected throttle event never reached the component-status DB. Fixes: - cpu_telemetry.go: escalate a CPU's status to Warning (only-escalate, same severity ranking already used elsewhere) when Throttled is set. - sat.go: add a before/after thermal-throttle-counter check job around the "cpu" pack's stress-ng run, so a throttle event during the run fails that job and (via the existing FAILED->Warning DB mapping) flips cpu:all to Warning. - platform_stress.go: emit a machine-readable overall_status= line alongside the human-readable verdict so platform-stress results actually reach ApplySATResultToDB. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
56d12b1f3c
commit
198567dffe
@@ -793,16 +793,71 @@ func (s *System) RunSATStressPack(ctx context.Context, baseDir string, durationS
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
// cpuThermalThrottleSysDir is the sysfs root the throttle-check scripts glob
|
||||
// under. Overridden in tests so they can point at a fake directory tree
|
||||
// instead of the real /sys.
|
||||
var cpuThermalThrottleSysDir = "/sys/devices/system/cpu"
|
||||
|
||||
// cpuThrottleSumScript is the shell fragment both before/after scripts use to
|
||||
// sum the kernel's cumulative-since-boot thermal throttle counters across
|
||||
// every CPU.
|
||||
func cpuThrottleSumScript() string {
|
||||
return fmt.Sprintf(`
|
||||
sum=0
|
||||
for f in %[1]s/cpu*/thermal_throttle/core_throttle_count %[1]s/cpu*/thermal_throttle/package_throttle_count; do
|
||||
[ -f "$f" ] || continue
|
||||
v=$(cat "$f" 2>/dev/null)
|
||||
case "$v" in ''|*[!0-9]*) continue ;; esac
|
||||
sum=$((sum + v))
|
||||
done
|
||||
`, cpuThermalThrottleSysDir)
|
||||
}
|
||||
|
||||
// cpuThrottleBeforeScript snapshots the throttle counter sum into a file in
|
||||
// {{run_dir}} so cpuThrottleCheckScript can later diff before/after despite
|
||||
// each satJob running as an independent process.
|
||||
func cpuThrottleBeforeScript() string {
|
||||
return cpuThrottleSumScript() + `echo "$sum" | tee {{run_dir}}/.cpu-throttle-before` + "\n"
|
||||
}
|
||||
|
||||
// cpuThrottleCheckScript compares the after-run throttle counter sum against
|
||||
// the snapshot cpuThrottleBeforeScript took, and fails (non-zero exit) if it
|
||||
// increased — i.e. the CPU actually hit thermal throttling during this
|
||||
// specific run, not just at some earlier point this boot. classifySATResult
|
||||
// maps a failed job here to SAT status FAILED, which ApplySATResultToDB
|
||||
// records as component status "Warning" for cpu:all — without this, the
|
||||
// "cpu" SAT pack only checks stress-ng's exit code, which is 0 whether or
|
||||
// not the CPU throttled while running it.
|
||||
func cpuThrottleCheckScript() string {
|
||||
return `before=$(cat {{run_dir}}/.cpu-throttle-before 2>/dev/null)
|
||||
case "$before" in ''|*[!0-9]*) before=0 ;; esac
|
||||
` + cpuThrottleSumScript() + `after=$sum
|
||||
echo "throttle_count_before=$before"
|
||||
echo "throttle_count_after=$after"
|
||||
if [ "$after" -gt "$before" ]; then
|
||||
echo "THROTTLE DETECTED: CPU package/core hit thermal throttling during this stress-ng run ($before -> $after)"
|
||||
exit 1
|
||||
fi
|
||||
echo "no new thermal throttling detected during this run"
|
||||
`
|
||||
}
|
||||
|
||||
func cpuSATJobs(durationSec int) []satJob {
|
||||
return []satJob{
|
||||
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
|
||||
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
|
||||
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
|
||||
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
|
||||
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
|
||||
{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
if durationSec <= 0 {
|
||||
durationSec = 60
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "cpu", []satJob{
|
||||
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
|
||||
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
|
||||
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
|
||||
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
|
||||
}, logFunc)
|
||||
return runAcceptancePackCtx(ctx, baseDir, "cpu", cpuSATJobs(durationSec), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) {
|
||||
|
||||
Reference in New Issue
Block a user