refactor: modularize audit and harden build validation
This commit is contained in:
Mikhail Chusavitin
@@ -0,0 +1,257 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func memoryStressSizeArg() string {
|
||||
if mb := envInt("BEE_VM_STRESS_SIZE_MB", 0); mb > 0 {
|
||||
return fmt.Sprintf("%dM", mb)
|
||||
}
|
||||
availBytes := satFreeMemBytes()
|
||||
if availBytes <= 0 {
|
||||
return "80%"
|
||||
}
|
||||
availMB := availBytes / (1024 * 1024)
|
||||
targetMB := (availMB * 2) / 3
|
||||
if targetMB >= 256 {
|
||||
targetMB = (targetMB / 256) * 256
|
||||
}
|
||||
if targetMB <= 0 {
|
||||
return "80%"
|
||||
}
|
||||
return fmt.Sprintf("%dM", targetMB)
|
||||
}
|
||||
|
||||
func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error) {
|
||||
if sizeMB <= 0 {
|
||||
sizeMB = 256
|
||||
}
|
||||
if passes <= 0 {
|
||||
passes = 1
|
||||
}
|
||||
// Keep Validate Memory bounded to a quick diagnostic window. The timeout is
|
||||
// intentionally conservative enough for healthy systems while avoiding the
|
||||
// prior 30-80 minute hangs caused by memtester spinning on a bad subtest.
|
||||
timeoutSec := sizeMB*passes*20/100 + 60
|
||||
if timeoutSec < 180 {
|
||||
timeoutSec = 180
|
||||
}
|
||||
if timeoutSec > 900 {
|
||||
timeoutSec = 900
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}, syncBracket: true},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
seconds := durationSec
|
||||
if seconds <= 0 {
|
||||
seconds = envInt("BEE_VM_STRESS_SECONDS", 300)
|
||||
}
|
||||
// Base the default on current MemAvailable and keep headroom for the OS and
|
||||
// concurrent stressors so mixed burn runs do not trip the OOM killer.
|
||||
sizeArg := memoryStressSizeArg()
|
||||
return runAcceptancePackCtx(ctx, baseDir, "memory-stress", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-stress-ng-vm.log", cmd: []string{
|
||||
"stress-ng", "--vm", "1",
|
||||
"--vm-bytes", sizeArg,
|
||||
"--vm-method", "all",
|
||||
"--timeout", fmt.Sprintf("%d", seconds),
|
||||
"--metrics-brief",
|
||||
}, syncBracket: true},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
seconds := durationSec
|
||||
if seconds <= 0 {
|
||||
seconds = envInt("BEE_SAT_STRESS_SECONDS", 300)
|
||||
}
|
||||
cmd := []string{"stressapptest", "-s", fmt.Sprintf("%d", seconds), "-W", "--cc_test"}
|
||||
if mb := envInt("BEE_SAT_STRESS_MB", 0); mb > 0 {
|
||||
cmd = append(cmd, "-M", fmt.Sprintf("%d", mb))
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "sat-stress", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-stressapptest.log", cmd: cmd},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
// cpuThermalThrottleSysDir is the sysfs root the throttle-check scripts glob
|
||||
// under. Overridden in tests so they can point at a fake directory tree
|
||||
// instead of the real /sys.
|
||||
var cpuThermalThrottleSysDir = "/sys/devices/system/cpu"
|
||||
|
||||
// cpuThrottleSumScript is the shell fragment both before/after scripts use to
|
||||
// sum the kernel's cumulative-since-boot thermal throttle counters across
|
||||
// every CPU.
|
||||
func cpuThrottleSumScript() string {
|
||||
return fmt.Sprintf(`
|
||||
sum=0
|
||||
for f in %[1]s/cpu*/thermal_throttle/core_throttle_count %[1]s/cpu*/thermal_throttle/package_throttle_count; do
|
||||
[ -f "$f" ] || continue
|
||||
v=$(cat "$f" 2>/dev/null)
|
||||
case "$v" in ''|*[!0-9]*) continue ;; esac
|
||||
sum=$((sum + v))
|
||||
done
|
||||
`, cpuThermalThrottleSysDir)
|
||||
}
|
||||
|
||||
// cpuThrottleBeforeScript snapshots the throttle counter sum into a file in
|
||||
// {{run_dir}} so cpuThrottleCheckScript can later diff before/after despite
|
||||
// each satJob running as an independent process.
|
||||
func cpuThrottleBeforeScript() string {
|
||||
return cpuThrottleSumScript() + `echo "$sum" | tee {{run_dir}}/.cpu-throttle-before` + "\n"
|
||||
}
|
||||
|
||||
// cpuThrottleCheckScript compares the after-run throttle counter sum against
|
||||
// the snapshot cpuThrottleBeforeScript took, and fails (non-zero exit) if it
|
||||
// increased — i.e. the CPU actually hit thermal throttling during this
|
||||
// specific run, not just at some earlier point this boot. classifySATResult
|
||||
// maps a failed job here to SAT status FAILED, which ApplySATResultToDB
|
||||
// records as component status "Warning" for cpu:all — without this, the
|
||||
// "cpu" SAT pack only checks stress-ng's exit code, which is 0 whether or
|
||||
// not the CPU throttled while running it.
|
||||
func cpuThrottleCheckScript() string {
|
||||
return `before=$(cat {{run_dir}}/.cpu-throttle-before 2>/dev/null)
|
||||
case "$before" in ''|*[!0-9]*) before=0 ;; esac
|
||||
` + cpuThrottleSumScript() + `after=$sum
|
||||
echo "throttle_count_before=$before"
|
||||
echo "throttle_count_after=$after"
|
||||
if [ "$after" -gt "$before" ]; then
|
||||
echo "THROTTLE DETECTED: CPU package/core hit thermal throttling during this stress-ng run ($before -> $after)"
|
||||
exit 1
|
||||
fi
|
||||
echo "no new thermal throttling detected during this run"
|
||||
`
|
||||
}
|
||||
|
||||
func cpuSATJobs(durationSec int) []satJob {
|
||||
return []satJob{
|
||||
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
|
||||
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
|
||||
{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
|
||||
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
|
||||
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
|
||||
{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
if durationSec <= 0 {
|
||||
durationSec = 60
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "cpu", cpuSATJobs(durationSec), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) {
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
runDir := filepath.Join(baseDir, "storage-"+ts)
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||
|
||||
devices, err := listStorageDevices()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sort.Strings(devices)
|
||||
|
||||
var summary strings.Builder
|
||||
stats := satStats{}
|
||||
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
||||
if len(devices) == 0 {
|
||||
fmt.Fprintln(&summary, "devices=0")
|
||||
stats.Unsupported++
|
||||
} else {
|
||||
fmt.Fprintf(&summary, "devices=%d\n", len(devices))
|
||||
}
|
||||
|
||||
for index, devPath := range devices {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
prefix := fmt.Sprintf("%02d-%s", index+1, filepath.Base(devPath))
|
||||
commands := storageSATCommands(devPath, extended)
|
||||
deviceOutputs := make(map[string][]byte, len(commands))
|
||||
for cmdIndex, job := range commands {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
|
||||
livePath := filepath.Join(runDir, name)
|
||||
runSyncBracketHook(job, "before", logFunc)
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc, livePath)
|
||||
deviceOutputs[job.name] = out
|
||||
if writeErr := os.WriteFile(livePath, out, 0644); writeErr != nil {
|
||||
return "", writeErr
|
||||
}
|
||||
if satJobBoundaryHook != nil {
|
||||
satJobBoundaryHook(name)
|
||||
}
|
||||
// smartctl -t short only launches the self-test on the drive firmware and
|
||||
// returns immediately ("Testing has begun"); unlike `nvme device-self-test
|
||||
// --wait`, smartctl has no blocking mode, so we must poll the drive
|
||||
// ourselves until the self-test actually finishes. Hold the "after" sync
|
||||
// until that poll completes — the self-test itself, not just its launch,
|
||||
// is the load worth having durable evidence of.
|
||||
deferSyncBracketAfter := job.name == "smartctl-self-test-short" && err == nil
|
||||
if !deferSyncBracketAfter {
|
||||
runSyncBracketHook(job, "after", logFunc)
|
||||
}
|
||||
status, rc := classifySATResult(job.name, out, err)
|
||||
// A zero smartctl exit status only proves the command ran. If the
|
||||
// drive did not return its overall-health verdict, it must not turn
|
||||
// the storage SAT green.
|
||||
if job.name == "smartctl-health" && status == "OK" && !hasSMARTOverallHealth(out) {
|
||||
status = "UNSUPPORTED"
|
||||
}
|
||||
stats.Add(status)
|
||||
key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_")
|
||||
fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
|
||||
fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
|
||||
|
||||
if deferSyncBracketAfter {
|
||||
statusName := "smartctl-self-test-status"
|
||||
statusOut := waitForSmartctlSelfTest(ctx, verboseLog, devPath, logFunc)
|
||||
deviceOutputs[statusName] = statusOut
|
||||
statusFile := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+2, statusName)
|
||||
if writeErr := os.WriteFile(filepath.Join(runDir, statusFile), statusOut, 0644); writeErr != nil {
|
||||
return "", writeErr
|
||||
}
|
||||
runSyncBracketHook(job, "after", logFunc)
|
||||
sStatus, sRC := classifySATResult(statusName, statusOut, nil)
|
||||
stats.Add(sStatus)
|
||||
sKey := filepath.Base(devPath) + "_" + strings.ReplaceAll(statusName, "-", "_")
|
||||
fmt.Fprintf(&summary, "%s_rc=%d\n", sKey, sRC)
|
||||
fmt.Fprintf(&summary, "%s_status=%s\n", sKey, sStatus)
|
||||
}
|
||||
}
|
||||
reportText := GenerateDiskReportText(index+1, devPath, deviceOutputs, time.Now().UTC())
|
||||
reportName := "disk-" + prefix + "-report.txt"
|
||||
_ = os.WriteFile(filepath.Join(runDir, reportName), []byte(reportText), 0644)
|
||||
}
|
||||
|
||||
writeSATStats(&summary, stats)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
Reference in New Issue
Block a user