fix: tolerate unavailable CPU sensor telemetry
This commit is contained in:
@@ -139,13 +139,34 @@ echo "no new thermal throttling detected during this run"
|
||||
`
|
||||
}
|
||||
|
||||
// cpuSensorsProbeScript preserves lm-sensors output in the SAT evidence while
|
||||
// distinguishing an unsupported local hwmon interface from a broken command.
|
||||
//
|
||||
// Some otherwise healthy servers expose thermal telemetry only through BMC/IPMI
|
||||
// (or need a platform-specific hwmon module that the current kernel does not
|
||||
// provide). lm-sensors returns 1 and prints "No sensors found!" in that case.
|
||||
// That must not turn a successful CPU stress test into a false failure. Other
|
||||
// non-zero exits remain failures: they can indicate a missing/broken sensors
|
||||
// binary or a real runtime problem.
|
||||
func cpuSensorsProbeScript() string {
|
||||
return `output=$(sensors 2>&1)
|
||||
rc=$?
|
||||
printf '%s\n' "$output"
|
||||
if [ "$rc" -ne 0 ] && printf '%s\n' "$output" | grep -Fq 'No sensors found!'; then
|
||||
echo 'CPU temperature telemetry is unavailable through lm-sensors; continuing without local hwmon readings.'
|
||||
exit 0
|
||||
fi
|
||||
exit "$rc"
|
||||
`
|
||||
}
|
||||
|
||||
func cpuSATJobs(durationSec int) []satJob {
|
||||
return []satJob{
|
||||
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
|
||||
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
|
||||
{name: "02-sensors-before.log", cmd: []string{"sh", "-c", cpuSensorsProbeScript()}},
|
||||
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
|
||||
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
|
||||
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
|
||||
{name: "04-sensors-after.log", cmd: []string{"sh", "-c", cpuSensorsProbeScript()}},
|
||||
{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -719,6 +720,42 @@ func TestCPUSATJobsIncludeThrottleCheck(t *testing.T) {
|
||||
if jobs[5].name != "05-thermal-throttle-check.log" || jobs[5].informational {
|
||||
t.Fatalf("throttle-check job=%+v want non-informational so it can fail overall_status", jobs[5])
|
||||
}
|
||||
for _, index := range []int{1, 4} {
|
||||
if jobs[index].cmd[0] != "sh" || jobs[index].cmd[1] != "-c" || !strings.Contains(jobs[index].cmd[2], "No sensors found!") {
|
||||
t.Fatalf("sensors job=%+v want probe that tolerates unavailable local hwmon", jobs[index])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCPUSensorsProbeScript(t *testing.T) {
|
||||
run := func(t *testing.T, output string, rc int) ([]byte, error) {
|
||||
t.Helper()
|
||||
binDir := t.TempDir()
|
||||
mockSensors := filepath.Join(binDir, "sensors")
|
||||
script := "#!/bin/sh\nprintf '%s\\n' '" + output + "'\nexit " + strconv.Itoa(rc) + "\n"
|
||||
if err := os.WriteFile(mockSensors, []byte(script), 0755); err != nil {
|
||||
t.Fatalf("write mock sensors: %v", err)
|
||||
}
|
||||
t.Setenv("PATH", binDir+":"+os.Getenv("PATH"))
|
||||
return exec.Command("sh", "-c", cpuSensorsProbeScript()).CombinedOutput()
|
||||
}
|
||||
|
||||
t.Run("no hardware-monitoring device is non-fatal", func(t *testing.T) {
|
||||
out, err := run(t, "No sensors found!", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("probe failed: %v; output=%s", err, out)
|
||||
}
|
||||
if !strings.Contains(string(out), "telemetry is unavailable") {
|
||||
t.Fatalf("output=%s want unavailable telemetry annotation", out)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unexpected sensors failure remains fatal", func(t *testing.T) {
|
||||
out, err := run(t, "i2c read failed", 1)
|
||||
if err == nil {
|
||||
t.Fatalf("probe unexpectedly succeeded; output=%s", out)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestCPUThrottleCheckDetectsIncreaseDuringRun runs the real before/check
|
||||
|
||||
Reference in New Issue
Block a user