Files
bee/audit/internal/collector/cpu_telemetry.go
Mikhail ChusavitinandClaude Sonnet 5 198567dffe fix: surface CPU thermal throttling in status and SAT results
A CPU that had thermally throttled (sysfs thermal_throttle counter >
0) still reported status "OK" everywhere: dmidecode-derived CPU status
only distinguishes populated/enabled/disabled and never looked at the
throttle flag the collector already recorded next to it, and neither
SAT path meant to catch this actually could:

- The routine "cpu" SAT pack (RunCPUAcceptancePack) only checked
  lscpu/sensors/stress-ng exit codes — stress-ng exits 0 whether or
  not the CPU throttled while running it, so an 89°C/throttled CPU
  right after a "successful" run still showed cpu:all as OK in
  component-status.json.
- The more thorough platform-stress test already detected throttling
  and fan-spindown correctly, but wrote its verdict as "Overall: FAIL
  — ..." with no "=", which parseSATKV can't parse — so even a real
  detected throttle event never reached the component-status DB.

Fixes:
- cpu_telemetry.go: escalate a CPU's status to Warning (only-escalate,
  same severity ranking already used elsewhere) when Throttled is set.
- sat.go: add a before/after thermal-throttle-counter check job around
  the "cpu" pack's stress-ng run, so a throttle event during the run
  fails that job and (via the existing FAILED->Warning DB mapping)
  flips cpu:all to Warning.
- platform_stress.go: emit a machine-readable overall_status= line
  alongside the human-readable verdict so platform-stress results
  actually reach ApplySATResultToDB.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-08-24 17:41:40 +03:00

221 lines
5.4 KiB
Go

package collector
import (
"bee/audit/internal/schema"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
)
var (
cpuSysBaseDir = "/sys/devices/system/cpu"
socketIndexRe = regexp.MustCompile(`(?i)(?:package id|socket|cpu)\s*([0-9]+)`)
)
func enrichCPUsWithTelemetry(cpus []schema.HardwareCPU, doc sensorsDoc) []schema.HardwareCPU {
if len(cpus) == 0 {
return cpus
}
tempBySocket := cpuTempsFromSensors(doc, len(cpus))
powerBySocket := cpuPowerFromSensors(doc, len(cpus))
throttleBySocket := cpuThrottleBySocket()
for i := range cpus {
socket := 0
if cpus[i].Socket != nil {
socket = *cpus[i].Socket
}
if value, ok := tempBySocket[socket]; ok {
cpus[i].TemperatureC = &value
}
if value, ok := powerBySocket[socket]; ok {
cpus[i].PowerW = &value
}
if value, ok := throttleBySocket[socket]; ok {
cpus[i].Throttled = &value
if value {
escalateCPUThrottleStatus(&cpus[i])
}
}
}
return cpus
}
// escalateCPUThrottleStatus raises a CPU's status to at least Warning when it
// has hit thermal throttling (cpuPackageThrottled: a cumulative since-boot
// sysfs counter, not "is throttling right now"). dmidecode-derived status
// (parseCPUStatus) only distinguishes populated/enabled/disabled and has no
// way to know about a live thermal event, so without this a CPU that
// throttled during e.g. a stress test still reports status "OK" — see
// enrichCPUsWithTelemetry.
func escalateCPUThrottleStatus(cpu *schema.HardwareCPU) {
current := ""
if cpu.Status != nil {
current = strings.TrimSpace(*cpu.Status)
}
if current != "" && current != statusUnknown && StatusSeverity(statusWarning) <= StatusSeverity(current) {
return
}
status := statusWarning
cpu.Status = &status
desc := "CPU hit thermal throttling (package/core throttle count > 0 since boot)"
cpu.ErrorDescription = &desc
}
func cpuTempsFromSensors(doc sensorsDoc, cpuCount int) map[int]float64 {
out := map[int]float64{}
if len(doc) == 0 {
return out
}
var fallback []float64
for chip, features := range doc {
for featureName, raw := range features {
feature, ok := raw.(map[string]any)
if !ok {
continue
}
if classifySensorFeature(feature) != "temp" {
continue
}
temp, ok := firstFeatureFloat(feature, "_input")
if !ok {
continue
}
if socket, ok := detectCPUSocket(chip, featureName); ok {
if _, exists := out[socket]; !exists {
out[socket] = temp
}
continue
}
if isLikelyCPUTemp(chip, featureName) {
fallback = append(fallback, temp)
}
}
}
if len(out) == 0 && cpuCount == 1 && len(fallback) > 0 {
out[0] = fallback[0]
}
return out
}
func cpuPowerFromSensors(doc sensorsDoc, cpuCount int) map[int]float64 {
out := map[int]float64{}
if len(doc) == 0 {
return out
}
var fallback []float64
for chip, features := range doc {
for featureName, raw := range features {
feature, ok := raw.(map[string]any)
if !ok {
continue
}
if classifySensorFeature(feature) != "power" {
continue
}
power, ok := firstFeatureFloatWithContains(feature, []string{"power"})
if !ok {
continue
}
if socket, ok := detectCPUSocket(chip, featureName); ok {
if _, exists := out[socket]; !exists {
out[socket] = power
}
continue
}
if isLikelyCPUPower(chip, featureName) {
fallback = append(fallback, power)
}
}
}
if len(out) == 0 && cpuCount == 1 && len(fallback) > 0 {
out[0] = fallback[0]
}
return out
}
func detectCPUSocket(parts ...string) (int, bool) {
for _, part := range parts {
matches := socketIndexRe.FindStringSubmatch(strings.ToLower(part))
if len(matches) == 2 {
value, err := strconv.Atoi(matches[1])
if err == nil {
return value, true
}
}
}
return 0, false
}
func isLikelyCPUTemp(chip, feature string) bool {
value := strings.ToLower(chip + " " + feature)
return strings.Contains(value, "coretemp") ||
strings.Contains(value, "k10temp") ||
strings.Contains(value, "package id") ||
strings.Contains(value, "tdie") ||
strings.Contains(value, "tctl") ||
strings.Contains(value, "cpu temp")
}
func isLikelyCPUPower(chip, feature string) bool {
value := strings.ToLower(chip + " " + feature)
return strings.Contains(value, "intel-rapl") ||
strings.Contains(value, "package id") ||
strings.Contains(value, "package-") ||
strings.Contains(value, "cpu power")
}
func cpuThrottleBySocket() map[int]bool {
out := map[int]bool{}
cpuDirs, err := filepath.Glob(filepath.Join(cpuSysBaseDir, "cpu[0-9]*"))
if err != nil {
return out
}
sort.Strings(cpuDirs)
for _, cpuDir := range cpuDirs {
socket, ok := readSocketIndex(cpuDir)
if !ok {
continue
}
if cpuPackageThrottled(cpuDir) {
out[socket] = true
}
}
return out
}
func readSocketIndex(cpuDir string) (int, bool) {
raw, err := os.ReadFile(filepath.Join(cpuDir, "topology", "physical_package_id"))
if err != nil {
return 0, false
}
value, err := strconv.Atoi(strings.TrimSpace(string(raw)))
if err != nil || value < 0 {
return 0, false
}
return value, true
}
func cpuPackageThrottled(cpuDir string) bool {
paths := []string{
filepath.Join(cpuDir, "thermal_throttle", "package_throttle_count"),
filepath.Join(cpuDir, "thermal_throttle", "core_throttle_count"),
}
for _, path := range paths {
raw, err := os.ReadFile(path)
if err != nil {
continue
}
value, err := strconv.ParseInt(strings.TrimSpace(string(raw)), 10, 64)
if err == nil && value > 0 {
return true
}
}
return false
}