fix: tolerate unavailable CPU sensor telemetry

This commit is contained in:
2026-09-08 09:49:40 +03:00
parent b8c45d54c1
commit 5190d50358
5 changed files with 78 additions and 4 deletions
+23 -2
View File
@@ -139,13 +139,34 @@ echo "no new thermal throttling detected during this run"
` `
} }
// cpuSensorsProbeScript preserves lm-sensors output in the SAT evidence while
// distinguishing an unsupported local hwmon interface from a broken command.
//
// Some otherwise healthy servers expose thermal telemetry only through BMC/IPMI
// (or need a platform-specific hwmon module that the current kernel does not
// provide). lm-sensors returns 1 and prints "No sensors found!" in that case.
// That must not turn a successful CPU stress test into a false failure. Other
// non-zero exits remain failures: they can indicate a missing/broken sensors
// binary or a real runtime problem.
func cpuSensorsProbeScript() string {
return `output=$(sensors 2>&1)
rc=$?
printf '%s\n' "$output"
if [ "$rc" -ne 0 ] && printf '%s\n' "$output" | grep -Fq 'No sensors found!'; then
echo 'CPU temperature telemetry is unavailable through lm-sensors; continuing without local hwmon readings.'
exit 0
fi
exit "$rc"
`
}
func cpuSATJobs(durationSec int) []satJob { func cpuSATJobs(durationSec int) []satJob {
return []satJob{ return []satJob{
{name: "01-lscpu.log", cmd: []string{"lscpu"}}, {name: "01-lscpu.log", cmd: []string{"lscpu"}},
{name: "02-sensors-before.log", cmd: []string{"sensors"}}, {name: "02-sensors-before.log", cmd: []string{"sh", "-c", cpuSensorsProbeScript()}},
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true}, {name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true}, {name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
{name: "04-sensors-after.log", cmd: []string{"sensors"}}, {name: "04-sensors-after.log", cmd: []string{"sh", "-c", cpuSensorsProbeScript()}},
{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}}, {name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
} }
} }
+37
View File
@@ -6,6 +6,7 @@ import (
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"strconv"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -719,6 +720,42 @@ func TestCPUSATJobsIncludeThrottleCheck(t *testing.T) {
if jobs[5].name != "05-thermal-throttle-check.log" || jobs[5].informational { if jobs[5].name != "05-thermal-throttle-check.log" || jobs[5].informational {
t.Fatalf("throttle-check job=%+v want non-informational so it can fail overall_status", jobs[5]) t.Fatalf("throttle-check job=%+v want non-informational so it can fail overall_status", jobs[5])
} }
for _, index := range []int{1, 4} {
if jobs[index].cmd[0] != "sh" || jobs[index].cmd[1] != "-c" || !strings.Contains(jobs[index].cmd[2], "No sensors found!") {
t.Fatalf("sensors job=%+v want probe that tolerates unavailable local hwmon", jobs[index])
}
}
}
func TestCPUSensorsProbeScript(t *testing.T) {
run := func(t *testing.T, output string, rc int) ([]byte, error) {
t.Helper()
binDir := t.TempDir()
mockSensors := filepath.Join(binDir, "sensors")
script := "#!/bin/sh\nprintf '%s\\n' '" + output + "'\nexit " + strconv.Itoa(rc) + "\n"
if err := os.WriteFile(mockSensors, []byte(script), 0755); err != nil {
t.Fatalf("write mock sensors: %v", err)
}
t.Setenv("PATH", binDir+":"+os.Getenv("PATH"))
return exec.Command("sh", "-c", cpuSensorsProbeScript()).CombinedOutput()
}
t.Run("no hardware-monitoring device is non-fatal", func(t *testing.T) {
out, err := run(t, "No sensors found!", 1)
if err != nil {
t.Fatalf("probe failed: %v; output=%s", err, out)
}
if !strings.Contains(string(out), "telemetry is unavailable") {
t.Fatalf("output=%s want unavailable telemetry annotation", out)
}
})
t.Run("unexpected sensors failure remains fatal", func(t *testing.T) {
out, err := run(t, "i2c read failed", 1)
if err == nil {
t.Fatalf("probe unexpectedly succeeded; output=%s", out)
}
})
} }
// TestCPUThrottleCheckDetectsIncreaseDuringRun runs the real before/check // TestCPUThrottleCheckDetectsIncreaseDuringRun runs the real before/check
+1 -1
Submodule bible updated: d2600f1279...1977730d93
@@ -0,0 +1,16 @@
# Safe, in-kernel hardware-monitoring drivers used across common server and
# workstation platforms. Each driver simply declines to bind on unsupported
# hardware, so one image can cover both Intel and AMD hosts.
#
# Do not add Super-I/O or arbitrary I2C chip drivers here: forcing probes can
# interfere with platform BMC devices. Those drivers should be enabled only
# for a known platform.
# Intel Core/Xeon and AMD EPYC/Ryzen on-die temperature sensors.
coretemp
k10temp
# Generic hwmon interfaces used by DIMM temperature modules and drive temps.
i2c-dev
jc42
drivetemp