fix: tolerate unavailable CPU sensor telemetry
This commit is contained in:
@@ -139,13 +139,34 @@ echo "no new thermal throttling detected during this run"
|
|||||||
`
|
`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// cpuSensorsProbeScript preserves lm-sensors output in the SAT evidence while
|
||||||
|
// distinguishing an unsupported local hwmon interface from a broken command.
|
||||||
|
//
|
||||||
|
// Some otherwise healthy servers expose thermal telemetry only through BMC/IPMI
|
||||||
|
// (or need a platform-specific hwmon module that the current kernel does not
|
||||||
|
// provide). lm-sensors returns 1 and prints "No sensors found!" in that case.
|
||||||
|
// That must not turn a successful CPU stress test into a false failure. Other
|
||||||
|
// non-zero exits remain failures: they can indicate a missing/broken sensors
|
||||||
|
// binary or a real runtime problem.
|
||||||
|
func cpuSensorsProbeScript() string {
|
||||||
|
return `output=$(sensors 2>&1)
|
||||||
|
rc=$?
|
||||||
|
printf '%s\n' "$output"
|
||||||
|
if [ "$rc" -ne 0 ] && printf '%s\n' "$output" | grep -Fq 'No sensors found!'; then
|
||||||
|
echo 'CPU temperature telemetry is unavailable through lm-sensors; continuing without local hwmon readings.'
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
exit "$rc"
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
func cpuSATJobs(durationSec int) []satJob {
|
func cpuSATJobs(durationSec int) []satJob {
|
||||||
return []satJob{
|
return []satJob{
|
||||||
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
|
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
|
||||||
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
|
{name: "02-sensors-before.log", cmd: []string{"sh", "-c", cpuSensorsProbeScript()}},
|
||||||
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
|
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
|
||||||
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
|
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
|
||||||
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
|
{name: "04-sensors-after.log", cmd: []string{"sh", "-c", cpuSensorsProbeScript()}},
|
||||||
{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
|
{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -719,6 +720,42 @@ func TestCPUSATJobsIncludeThrottleCheck(t *testing.T) {
|
|||||||
if jobs[5].name != "05-thermal-throttle-check.log" || jobs[5].informational {
|
if jobs[5].name != "05-thermal-throttle-check.log" || jobs[5].informational {
|
||||||
t.Fatalf("throttle-check job=%+v want non-informational so it can fail overall_status", jobs[5])
|
t.Fatalf("throttle-check job=%+v want non-informational so it can fail overall_status", jobs[5])
|
||||||
}
|
}
|
||||||
|
for _, index := range []int{1, 4} {
|
||||||
|
if jobs[index].cmd[0] != "sh" || jobs[index].cmd[1] != "-c" || !strings.Contains(jobs[index].cmd[2], "No sensors found!") {
|
||||||
|
t.Fatalf("sensors job=%+v want probe that tolerates unavailable local hwmon", jobs[index])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPUSensorsProbeScript(t *testing.T) {
|
||||||
|
run := func(t *testing.T, output string, rc int) ([]byte, error) {
|
||||||
|
t.Helper()
|
||||||
|
binDir := t.TempDir()
|
||||||
|
mockSensors := filepath.Join(binDir, "sensors")
|
||||||
|
script := "#!/bin/sh\nprintf '%s\\n' '" + output + "'\nexit " + strconv.Itoa(rc) + "\n"
|
||||||
|
if err := os.WriteFile(mockSensors, []byte(script), 0755); err != nil {
|
||||||
|
t.Fatalf("write mock sensors: %v", err)
|
||||||
|
}
|
||||||
|
t.Setenv("PATH", binDir+":"+os.Getenv("PATH"))
|
||||||
|
return exec.Command("sh", "-c", cpuSensorsProbeScript()).CombinedOutput()
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("no hardware-monitoring device is non-fatal", func(t *testing.T) {
|
||||||
|
out, err := run(t, "No sensors found!", 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("probe failed: %v; output=%s", err, out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(out), "telemetry is unavailable") {
|
||||||
|
t.Fatalf("output=%s want unavailable telemetry annotation", out)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("unexpected sensors failure remains fatal", func(t *testing.T) {
|
||||||
|
out, err := run(t, "i2c read failed", 1)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("probe unexpectedly succeeded; output=%s", out)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCPUThrottleCheckDetectsIncreaseDuringRun runs the real before/check
|
// TestCPUThrottleCheckDetectsIncreaseDuringRun runs the real before/check
|
||||||
|
|||||||
+1
-1
Submodule bible updated: d2600f1279...1977730d93
+1
-1
Submodule internal/chart updated: be68069a51...8105c7ec08
@@ -0,0 +1,16 @@
|
|||||||
|
# Safe, in-kernel hardware-monitoring drivers used across common server and
|
||||||
|
# workstation platforms. Each driver simply declines to bind on unsupported
|
||||||
|
# hardware, so one image can cover both Intel and AMD hosts.
|
||||||
|
#
|
||||||
|
# Do not add Super-I/O or arbitrary I2C chip drivers here: forcing probes can
|
||||||
|
# interfere with platform BMC devices. Those drivers should be enabled only
|
||||||
|
# for a known platform.
|
||||||
|
|
||||||
|
# Intel Core/Xeon and AMD EPYC/Ryzen on-die temperature sensors.
|
||||||
|
coretemp
|
||||||
|
k10temp
|
||||||
|
|
||||||
|
# Generic hwmon interfaces used by DIMM temperature modules and drive temps.
|
||||||
|
i2c-dev
|
||||||
|
jc42
|
||||||
|
drivetemp
|
||||||
Reference in New Issue
Block a user