package platform import ( "context" "fmt" "os" "path/filepath" "sort" "strings" "time" ) func memoryStressSizeArg() string { if mb := envInt("BEE_VM_STRESS_SIZE_MB", 0); mb > 0 { return fmt.Sprintf("%dM", mb) } availBytes := satFreeMemBytes() if availBytes <= 0 { return "80%" } availMB := availBytes / (1024 * 1024) targetMB := (availMB * 2) / 3 if targetMB >= 256 { targetMB = (targetMB / 256) * 256 } if targetMB <= 0 { return "80%" } return fmt.Sprintf("%dM", targetMB) } func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error) { if sizeMB <= 0 { sizeMB = 256 } if passes <= 0 { passes = 1 } // Keep Validate Memory bounded to a quick diagnostic window. The timeout is // intentionally conservative enough for healthy systems while avoiding the // prior 30-80 minute hangs caused by memtester spinning on a bad subtest. timeoutSec := sizeMB*passes*20/100 + 60 if timeoutSec < 180 { timeoutSec = 180 } if timeoutSec > 900 { timeoutSec = 900 } return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{ {name: "01-free-before.log", cmd: []string{"free", "-h"}}, {name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}, syncBracket: true}, {name: "03-free-after.log", cmd: []string{"free", "-h"}}, }, logFunc) } func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { seconds := durationSec if seconds <= 0 { seconds = envInt("BEE_VM_STRESS_SECONDS", 300) } // Base the default on current MemAvailable and keep headroom for the OS and // concurrent stressors so mixed burn runs do not trip the OOM killer. sizeArg := memoryStressSizeArg() return runAcceptancePackCtx(ctx, baseDir, "memory-stress", []satJob{ {name: "01-free-before.log", cmd: []string{"free", "-h"}}, {name: "02-stress-ng-vm.log", cmd: []string{ "stress-ng", "--vm", "1", "--vm-bytes", sizeArg, "--vm-method", "all", "--timeout", fmt.Sprintf("%d", seconds), "--metrics-brief", }, syncBracket: true}, {name: "03-free-after.log", cmd: []string{"free", "-h"}}, }, logFunc) } func (s *System) RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { seconds := durationSec if seconds <= 0 { seconds = envInt("BEE_SAT_STRESS_SECONDS", 300) } cmd := []string{"stressapptest", "-s", fmt.Sprintf("%d", seconds), "-W", "--cc_test"} if mb := envInt("BEE_SAT_STRESS_MB", 0); mb > 0 { cmd = append(cmd, "-M", fmt.Sprintf("%d", mb)) } return runAcceptancePackCtx(ctx, baseDir, "sat-stress", []satJob{ {name: "01-free-before.log", cmd: []string{"free", "-h"}}, {name: "02-stressapptest.log", cmd: cmd}, {name: "03-free-after.log", cmd: []string{"free", "-h"}}, }, logFunc) } // cpuThermalThrottleSysDir is the sysfs root the throttle-check scripts glob // under. Overridden in tests so they can point at a fake directory tree // instead of the real /sys. var cpuThermalThrottleSysDir = "/sys/devices/system/cpu" // cpuThrottleSumScript is the shell fragment both before/after scripts use to // sum the kernel's cumulative-since-boot thermal throttle counters across // every CPU. func cpuThrottleSumScript() string { return fmt.Sprintf(` sum=0 for f in %[1]s/cpu*/thermal_throttle/core_throttle_count %[1]s/cpu*/thermal_throttle/package_throttle_count; do [ -f "$f" ] || continue v=$(cat "$f" 2>/dev/null) case "$v" in ''|*[!0-9]*) continue ;; esac sum=$((sum + v)) done `, cpuThermalThrottleSysDir) } // cpuThrottleBeforeScript snapshots the throttle counter sum into a file in // {{run_dir}} so cpuThrottleCheckScript can later diff before/after despite // each satJob running as an independent process. func cpuThrottleBeforeScript() string { return cpuThrottleSumScript() + `echo "$sum" | tee {{run_dir}}/.cpu-throttle-before` + "\n" } // cpuThrottleCheckScript compares the after-run throttle counter sum against // the snapshot cpuThrottleBeforeScript took, and fails (non-zero exit) if it // increased — i.e. the CPU actually hit thermal throttling during this // specific run, not just at some earlier point this boot. classifySATResult // maps a failed job here to SAT status FAILED, which ApplySATResultToDB // records as component status "Warning" for cpu:all — without this, the // "cpu" SAT pack only checks stress-ng's exit code, which is 0 whether or // not the CPU throttled while running it. func cpuThrottleCheckScript() string { return `before=$(cat {{run_dir}}/.cpu-throttle-before 2>/dev/null) case "$before" in ''|*[!0-9]*) before=0 ;; esac ` + cpuThrottleSumScript() + `after=$sum echo "throttle_count_before=$before" echo "throttle_count_after=$after" if [ "$after" -gt "$before" ]; then echo "THROTTLE DETECTED: CPU package/core hit thermal throttling during this stress-ng run ($before -> $after)" exit 1 fi echo "no new thermal throttling detected during this run" ` } // cpuSensorsProbeScript preserves lm-sensors output in the SAT evidence while // distinguishing an unsupported local hwmon interface from a broken command. // // Some otherwise healthy servers expose thermal telemetry only through BMC/IPMI // (or need a platform-specific hwmon module that the current kernel does not // provide). lm-sensors returns 1 and prints "No sensors found!" in that case. // That must not turn a successful CPU stress test into a false failure. Other // non-zero exits remain failures: they can indicate a missing/broken sensors // binary or a real runtime problem. func cpuSensorsProbeScript() string { return `output=$(sensors 2>&1) rc=$? printf '%s\n' "$output" if [ "$rc" -ne 0 ] && printf '%s\n' "$output" | grep -Fq 'No sensors found!'; then echo 'CPU temperature telemetry is unavailable through lm-sensors; continuing without local hwmon readings.' exit 0 fi exit "$rc" ` } func cpuSATJobs(durationSec int) []satJob { return []satJob{ {name: "01-lscpu.log", cmd: []string{"lscpu"}}, {name: "02-sensors-before.log", cmd: []string{"sh", "-c", cpuSensorsProbeScript()}}, {name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true}, {name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true}, {name: "04-sensors-after.log", cmd: []string{"sh", "-c", cpuSensorsProbeScript()}}, {name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}}, } } func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { if durationSec <= 0 { durationSec = 60 } return runAcceptancePackCtx(ctx, baseDir, "cpu", cpuSATJobs(durationSec), logFunc) } func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) { if baseDir == "" { baseDir = "/var/log/bee-sat" } ts := time.Now().UTC().Format("20060102-150405") runDir := filepath.Join(baseDir, "storage-"+ts) if err := os.MkdirAll(runDir, 0755); err != nil { return "", err } verboseLog := filepath.Join(runDir, "verbose.log") devices, err := listStorageDevices() if err != nil { return "", err } sort.Strings(devices) var summary strings.Builder stats := satStats{} fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339)) if len(devices) == 0 { fmt.Fprintln(&summary, "devices=0") stats.Unsupported++ } else { fmt.Fprintf(&summary, "devices=%d\n", len(devices)) } for index, devPath := range devices { if ctx.Err() != nil { break } prefix := fmt.Sprintf("%02d-%s", index+1, filepath.Base(devPath)) commands := storageSATCommands(devPath, extended) deviceOutputs := make(map[string][]byte, len(commands)) for cmdIndex, job := range commands { if ctx.Err() != nil { break } name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name) livePath := filepath.Join(runDir, name) runSyncBracketHook(job, "before", logFunc) out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc, livePath) deviceOutputs[job.name] = out if writeErr := os.WriteFile(livePath, out, 0644); writeErr != nil { return "", writeErr } if satJobBoundaryHook != nil { satJobBoundaryHook(name) } // smartctl -t short only launches the self-test on the drive firmware and // returns immediately ("Testing has begun"); unlike `nvme device-self-test // --wait`, smartctl has no blocking mode, so we must poll the drive // ourselves until the self-test actually finishes. Hold the "after" sync // until that poll completes — the self-test itself, not just its launch, // is the load worth having durable evidence of. deferSyncBracketAfter := job.name == "smartctl-self-test-short" && err == nil if !deferSyncBracketAfter { runSyncBracketHook(job, "after", logFunc) } status, rc := classifySATResult(job.name, out, err) // A zero smartctl exit status only proves the command ran. If the // drive did not return its overall-health verdict, it must not turn // the storage SAT green. if job.name == "smartctl-health" && status == "OK" && !hasSMARTOverallHealth(out) { status = "UNSUPPORTED" } stats.Add(status) key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_") fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc) fmt.Fprintf(&summary, "%s_status=%s\n", key, status) if deferSyncBracketAfter { statusName := "smartctl-self-test-status" statusOut := waitForSmartctlSelfTest(ctx, verboseLog, devPath, logFunc) deviceOutputs[statusName] = statusOut statusFile := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+2, statusName) if writeErr := os.WriteFile(filepath.Join(runDir, statusFile), statusOut, 0644); writeErr != nil { return "", writeErr } runSyncBracketHook(job, "after", logFunc) sStatus, sRC := classifySATResult(statusName, statusOut, nil) stats.Add(sStatus) sKey := filepath.Base(devPath) + "_" + strings.ReplaceAll(statusName, "-", "_") fmt.Fprintf(&summary, "%s_rc=%d\n", sKey, sRC) fmt.Fprintf(&summary, "%s_status=%s\n", sKey, sStatus) } } reportText := GenerateDiskReportText(index+1, devPath, deviceOutputs, time.Now().UTC()) reportName := "disk-" + prefix + "-report.txt" _ = os.WriteFile(filepath.Join(runDir, reportName), []byte(reportText), 0644) } writeSATStats(&summary, stats) if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil { return "", err } return runDir, nil }