258 lines
9.6 KiB
Go
258 lines
9.6 KiB
Go
package platform
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
func memoryStressSizeArg() string {
|
|
if mb := envInt("BEE_VM_STRESS_SIZE_MB", 0); mb > 0 {
|
|
return fmt.Sprintf("%dM", mb)
|
|
}
|
|
availBytes := satFreeMemBytes()
|
|
if availBytes <= 0 {
|
|
return "80%"
|
|
}
|
|
availMB := availBytes / (1024 * 1024)
|
|
targetMB := (availMB * 2) / 3
|
|
if targetMB >= 256 {
|
|
targetMB = (targetMB / 256) * 256
|
|
}
|
|
if targetMB <= 0 {
|
|
return "80%"
|
|
}
|
|
return fmt.Sprintf("%dM", targetMB)
|
|
}
|
|
|
|
func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error) {
|
|
if sizeMB <= 0 {
|
|
sizeMB = 256
|
|
}
|
|
if passes <= 0 {
|
|
passes = 1
|
|
}
|
|
// Keep Validate Memory bounded to a quick diagnostic window. The timeout is
|
|
// intentionally conservative enough for healthy systems while avoiding the
|
|
// prior 30-80 minute hangs caused by memtester spinning on a bad subtest.
|
|
timeoutSec := sizeMB*passes*20/100 + 60
|
|
if timeoutSec < 180 {
|
|
timeoutSec = 180
|
|
}
|
|
if timeoutSec > 900 {
|
|
timeoutSec = 900
|
|
}
|
|
return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{
|
|
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
|
{name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}, syncBracket: true},
|
|
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
|
}, logFunc)
|
|
}
|
|
|
|
func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
|
seconds := durationSec
|
|
if seconds <= 0 {
|
|
seconds = envInt("BEE_VM_STRESS_SECONDS", 300)
|
|
}
|
|
// Base the default on current MemAvailable and keep headroom for the OS and
|
|
// concurrent stressors so mixed burn runs do not trip the OOM killer.
|
|
sizeArg := memoryStressSizeArg()
|
|
return runAcceptancePackCtx(ctx, baseDir, "memory-stress", []satJob{
|
|
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
|
{name: "02-stress-ng-vm.log", cmd: []string{
|
|
"stress-ng", "--vm", "1",
|
|
"--vm-bytes", sizeArg,
|
|
"--vm-method", "all",
|
|
"--timeout", fmt.Sprintf("%d", seconds),
|
|
"--metrics-brief",
|
|
}, syncBracket: true},
|
|
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
|
}, logFunc)
|
|
}
|
|
|
|
func (s *System) RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
|
seconds := durationSec
|
|
if seconds <= 0 {
|
|
seconds = envInt("BEE_SAT_STRESS_SECONDS", 300)
|
|
}
|
|
cmd := []string{"stressapptest", "-s", fmt.Sprintf("%d", seconds), "-W", "--cc_test"}
|
|
if mb := envInt("BEE_SAT_STRESS_MB", 0); mb > 0 {
|
|
cmd = append(cmd, "-M", fmt.Sprintf("%d", mb))
|
|
}
|
|
return runAcceptancePackCtx(ctx, baseDir, "sat-stress", []satJob{
|
|
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
|
{name: "02-stressapptest.log", cmd: cmd},
|
|
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
|
}, logFunc)
|
|
}
|
|
|
|
// cpuThermalThrottleSysDir is the sysfs root the throttle-check scripts glob
|
|
// under. Overridden in tests so they can point at a fake directory tree
|
|
// instead of the real /sys.
|
|
var cpuThermalThrottleSysDir = "/sys/devices/system/cpu"
|
|
|
|
// cpuThrottleSumScript is the shell fragment both before/after scripts use to
|
|
// sum the kernel's cumulative-since-boot thermal throttle counters across
|
|
// every CPU.
|
|
func cpuThrottleSumScript() string {
|
|
return fmt.Sprintf(`
|
|
sum=0
|
|
for f in %[1]s/cpu*/thermal_throttle/core_throttle_count %[1]s/cpu*/thermal_throttle/package_throttle_count; do
|
|
[ -f "$f" ] || continue
|
|
v=$(cat "$f" 2>/dev/null)
|
|
case "$v" in ''|*[!0-9]*) continue ;; esac
|
|
sum=$((sum + v))
|
|
done
|
|
`, cpuThermalThrottleSysDir)
|
|
}
|
|
|
|
// cpuThrottleBeforeScript snapshots the throttle counter sum into a file in
|
|
// {{run_dir}} so cpuThrottleCheckScript can later diff before/after despite
|
|
// each satJob running as an independent process.
|
|
func cpuThrottleBeforeScript() string {
|
|
return cpuThrottleSumScript() + `echo "$sum" | tee {{run_dir}}/.cpu-throttle-before` + "\n"
|
|
}
|
|
|
|
// cpuThrottleCheckScript compares the after-run throttle counter sum against
|
|
// the snapshot cpuThrottleBeforeScript took, and fails (non-zero exit) if it
|
|
// increased — i.e. the CPU actually hit thermal throttling during this
|
|
// specific run, not just at some earlier point this boot. classifySATResult
|
|
// maps a failed job here to SAT status FAILED, which ApplySATResultToDB
|
|
// records as component status "Warning" for cpu:all — without this, the
|
|
// "cpu" SAT pack only checks stress-ng's exit code, which is 0 whether or
|
|
// not the CPU throttled while running it.
|
|
func cpuThrottleCheckScript() string {
|
|
return `before=$(cat {{run_dir}}/.cpu-throttle-before 2>/dev/null)
|
|
case "$before" in ''|*[!0-9]*) before=0 ;; esac
|
|
` + cpuThrottleSumScript() + `after=$sum
|
|
echo "throttle_count_before=$before"
|
|
echo "throttle_count_after=$after"
|
|
if [ "$after" -gt "$before" ]; then
|
|
echo "THROTTLE DETECTED: CPU package/core hit thermal throttling during this stress-ng run ($before -> $after)"
|
|
exit 1
|
|
fi
|
|
echo "no new thermal throttling detected during this run"
|
|
`
|
|
}
|
|
|
|
func cpuSATJobs(durationSec int) []satJob {
|
|
return []satJob{
|
|
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
|
|
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
|
|
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
|
|
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
|
|
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
|
|
{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
|
|
}
|
|
}
|
|
|
|
func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
|
if durationSec <= 0 {
|
|
durationSec = 60
|
|
}
|
|
return runAcceptancePackCtx(ctx, baseDir, "cpu", cpuSATJobs(durationSec), logFunc)
|
|
}
|
|
|
|
func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) {
|
|
if baseDir == "" {
|
|
baseDir = "/var/log/bee-sat"
|
|
}
|
|
ts := time.Now().UTC().Format("20060102-150405")
|
|
runDir := filepath.Join(baseDir, "storage-"+ts)
|
|
if err := os.MkdirAll(runDir, 0755); err != nil {
|
|
return "", err
|
|
}
|
|
verboseLog := filepath.Join(runDir, "verbose.log")
|
|
|
|
devices, err := listStorageDevices()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
sort.Strings(devices)
|
|
|
|
var summary strings.Builder
|
|
stats := satStats{}
|
|
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
|
if len(devices) == 0 {
|
|
fmt.Fprintln(&summary, "devices=0")
|
|
stats.Unsupported++
|
|
} else {
|
|
fmt.Fprintf(&summary, "devices=%d\n", len(devices))
|
|
}
|
|
|
|
for index, devPath := range devices {
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
prefix := fmt.Sprintf("%02d-%s", index+1, filepath.Base(devPath))
|
|
commands := storageSATCommands(devPath, extended)
|
|
deviceOutputs := make(map[string][]byte, len(commands))
|
|
for cmdIndex, job := range commands {
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
|
|
livePath := filepath.Join(runDir, name)
|
|
runSyncBracketHook(job, "before", logFunc)
|
|
out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc, livePath)
|
|
deviceOutputs[job.name] = out
|
|
if writeErr := os.WriteFile(livePath, out, 0644); writeErr != nil {
|
|
return "", writeErr
|
|
}
|
|
if satJobBoundaryHook != nil {
|
|
satJobBoundaryHook(name)
|
|
}
|
|
// smartctl -t short only launches the self-test on the drive firmware and
|
|
// returns immediately ("Testing has begun"); unlike `nvme device-self-test
|
|
// --wait`, smartctl has no blocking mode, so we must poll the drive
|
|
// ourselves until the self-test actually finishes. Hold the "after" sync
|
|
// until that poll completes — the self-test itself, not just its launch,
|
|
// is the load worth having durable evidence of.
|
|
deferSyncBracketAfter := job.name == "smartctl-self-test-short" && err == nil
|
|
if !deferSyncBracketAfter {
|
|
runSyncBracketHook(job, "after", logFunc)
|
|
}
|
|
status, rc := classifySATResult(job.name, out, err)
|
|
// A zero smartctl exit status only proves the command ran. If the
|
|
// drive did not return its overall-health verdict, it must not turn
|
|
// the storage SAT green.
|
|
if job.name == "smartctl-health" && status == "OK" && !hasSMARTOverallHealth(out) {
|
|
status = "UNSUPPORTED"
|
|
}
|
|
stats.Add(status)
|
|
key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_")
|
|
fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
|
|
fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
|
|
|
|
if deferSyncBracketAfter {
|
|
statusName := "smartctl-self-test-status"
|
|
statusOut := waitForSmartctlSelfTest(ctx, verboseLog, devPath, logFunc)
|
|
deviceOutputs[statusName] = statusOut
|
|
statusFile := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+2, statusName)
|
|
if writeErr := os.WriteFile(filepath.Join(runDir, statusFile), statusOut, 0644); writeErr != nil {
|
|
return "", writeErr
|
|
}
|
|
runSyncBracketHook(job, "after", logFunc)
|
|
sStatus, sRC := classifySATResult(statusName, statusOut, nil)
|
|
stats.Add(sStatus)
|
|
sKey := filepath.Base(devPath) + "_" + strings.ReplaceAll(statusName, "-", "_")
|
|
fmt.Fprintf(&summary, "%s_rc=%d\n", sKey, sRC)
|
|
fmt.Fprintf(&summary, "%s_status=%s\n", sKey, sStatus)
|
|
}
|
|
}
|
|
reportText := GenerateDiskReportText(index+1, devPath, deviceOutputs, time.Now().UTC())
|
|
reportName := "disk-" + prefix + "-report.txt"
|
|
_ = os.WriteFile(filepath.Join(runDir, reportName), []byte(reportText), 0644)
|
|
}
|
|
|
|
writeSATStats(&summary, stats)
|
|
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
|
|
return "", err
|
|
}
|
|
return runDir, nil
|
|
}
|