Files
bee/audit/internal/app/component_status_db_test.go
T
Mikhail ChusavitinandClaude Sonnet 5 cc3997f7b1 app/webui: fix component-status.json multi-process clobber and unbounded growth
The long-lived bee-web process (writing PSU/kmsg watchdog records ~every
60s) and each short-lived "bee bee-worker" SAT-task subprocess each held
an independent in-memory copy of component-status.json. Whichever saved
last won outright, silently erasing whatever the other had just written —
e.g. a GPU SAT task's pcie:gpu:nvidia result vanishing the next time the
PSU watchdog ticked. ComponentStatusDB.Record now reloads on-disk state
(keyed by newer LastCheckedAt) before merging its own update.

Also stops re-logging identical repeat observations to History: the
ingest contract defines status_history as a transition log ("История
переходов статусов"), not a per-poll journal, but Record appended one
entry per call regardless — a continuously-polled PSU grew an unbounded
run of identical "still OK" entries. Now only appends when a source's
last recorded status for a key actually changes.

The PSU watchdog itself now backs off (60s -> doubling, capped at 30min)
while steady and resets to 60s the moment any PSU's status changes, cutting
ipmitool shellouts and file writes for a fleet that's been stable for a
while.

Also adds the missing "raid" case to the component-detail API (was
returning 404 for any RAID card's detail click on /topo).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-09 11:24:18 +03:00

180 lines
6.8 KiB
Go

package app
import (
"os"
"path/filepath"
"testing"
"bee/audit/internal/schema"
)
func TestExtractArchivePath(t *testing.T) {
cases := map[string]string{
"/appdata/bee/export/bee-sat/gpu-nvidia-20260706-174722": "/appdata/bee/export/bee-sat/gpu-nvidia-20260706-174722",
"Archive written to /appdata/bee/export/bee-sat/gpu-nvidia-20260706-174722": "/appdata/bee/export/bee-sat/gpu-nvidia-20260706-174722",
"Archive written to /path/with spaces/foo.tar.gz": "/path/with spaces/foo.tar.gz",
" Archive written to /path/foo.tar.gz ": "/path/foo.tar.gz",
}
for in, want := range cases {
if got := ExtractArchivePath(in); got != want {
t.Errorf("ExtractArchivePath(%q) = %q, want %q", in, got, want)
}
}
}
func TestReadSATOverallStatus_HandlesActionResultPrefix(t *testing.T) {
runDir := t.TempDir()
summary := "run_at_utc=2026-07-06T17:47:22Z\noverall_status=FAILED\n"
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
t.Fatal(err)
}
// Regression: RunNvidiaAcceptancePackWithOptions wraps the bare run dir as
// "Archive written to <dir>" before it reaches ReadSATOverallStatus. If the
// prefix isn't stripped, the summary.txt lookup silently fails and a FAILED
// sub-job never surfaces as a task failure.
wrapped := "Archive written to " + runDir
if got := ReadSATOverallStatus(ExtractArchivePath(wrapped)); got != "FAILED" {
t.Errorf("ReadSATOverallStatus(wrapped) = %q, want FAILED", got)
}
if got := ReadSATOverallStatus(ExtractArchivePath(runDir)); got != "FAILED" {
t.Errorf("ReadSATOverallStatus(bare) = %q, want FAILED", got)
}
}
func writeSATSummary(t *testing.T, overall string) string {
t.Helper()
runDir := t.TempDir()
summary := "run_at_utc=2026-07-06T17:47:22Z\noverall_status=" + overall + "\n"
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
t.Fatal(err)
}
return runDir
}
func TestApplySATResultToDBNormalizesGPUKeyByVendor(t *testing.T) {
// "nvidia" (Check tier) and "nvidia-stress" (Load/Burn tier) exercise the
// same physical GPUs and must collapse onto one component key so a later
// clean Check run can't erase an earlier Load-tier failure.
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
if err != nil {
t.Fatal(err)
}
ApplySATResultToDB(db, "nvidia-stress", writeSATSummary(t, "FAILED"))
ApplySATResultToDB(db, "nvidia", writeSATSummary(t, "OK"))
rec, ok := db.Get("pcie:gpu:nvidia")
if !ok {
t.Fatalf("expected pcie:gpu:nvidia record to exist")
}
if rec.Status != "Warning" {
t.Fatalf("status=%q, want Warning (FAILED) to survive the later OK Check run", rec.Status)
}
}
func TestApplyComponentStatusDBMatchesGPUByVendor(t *testing.T) {
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
if err != nil {
t.Fatal(err)
}
db.Record("pcie:gpu:nvidia", "sat:nvidia-stress", "Critical", "nvidia-stress SAT: FAILED")
class := "VideoController"
vendor := 0x10de // collector.NvidiaVendorID
snap := &schema.HardwareSnapshot{
PCIeDevices: []schema.HardwarePCIeDevice{
{DeviceClass: &class, VendorID: &vendor, BDF: strPtr("0000:c8:00.0")},
},
}
applyComponentStatusDB(snap, db)
if snap.PCIeDevices[0].Status == nil || *snap.PCIeDevices[0].Status != "Critical" {
t.Fatalf("expected GPU device status Critical, got %v", snap.PCIeDevices[0].Status)
}
}
func strPtr(s string) *string { return &s }
// TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the
// hardware-ingest-contract.md rule that status_history is a transition log
// ("История переходов статусов"), not a per-poll journal. A component
// checked continuously (the PSU watchdog, every 60s indefinitely) must not
// grow one History entry per poll while its status stays unchanged — that
// is exactly what made component-status.json grow without bound in
// practice.
func TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource(t *testing.T) {
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
if err != nil {
t.Fatal(err)
}
for i := 0; i < 50; i++ {
db.Record("psu:0", "watchdog:psu", "OK", "")
}
rec, ok := db.Get("psu:0")
if !ok {
t.Fatalf("expected psu:0 record")
}
if len(rec.History) != 1 {
t.Fatalf("history len=%d want 1 after 50 identical polls (dedup by transition)", len(rec.History))
}
// A real transition must still be recorded, and dedup resumes at the
// new status.
db.Record("psu:0", "watchdog:psu", "Critical", "PSU sensor reported non-OK state")
db.Record("psu:0", "watchdog:psu", "Critical", "PSU sensor reported non-OK state")
rec, _ = db.Get("psu:0")
if len(rec.History) != 2 {
t.Fatalf("history len=%d want 2 (OK, then Critical, second Critical deduped)", len(rec.History))
}
if rec.Status != "Critical" {
t.Fatalf("status=%q want Critical", rec.Status)
}
}
// TestRecordDoesNotClobberConcurrentWriterFromAnotherProcess reproduces the
// bug behind a real support bundle where many GPU/CPU/memory SAT tasks had
// completed successfully but component-status.json only ever held PSU
// records: bee-web keeps one ComponentStatusDB open for its whole process
// lifetime (writing PSU/kmsg watchdog records ~every 60s via a long-running
// health poller), while each SAT task runs as a short-lived "bee bee-worker"
// subprocess that opens its own separate ComponentStatusDB instance backed
// by the same file. Without a reload-before-write, the long-lived process's
// stale in-memory snapshot (which never learned about the subprocess's
// write) overwrites the whole file on its next save and erases it.
func TestRecordDoesNotClobberConcurrentWriterFromAnotherProcess(t *testing.T) {
path := filepath.Join(t.TempDir(), "component-status.json")
// bee-web's long-lived DB instance, opened once at process start.
webDB, err := OpenComponentStatusDB(path)
if err != nil {
t.Fatal(err)
}
// A "bee bee-worker" subprocess for a completed GPU SAT task opens its
// own instance backed by the same file and records the result.
workerDB, err := OpenComponentStatusDB(path)
if err != nil {
t.Fatal(err)
}
workerDB.Record("pcie:gpu:nvidia", "sat:nvidia", "OK", "nvidia SAT: OK")
// bee-web's health poller ticks next, using its own (older) in-memory
// view, and records a PSU status it polled independently.
webDB.Record("psu:0", "watchdog:psu", "OK", "")
// The GPU record written by the worker subprocess must survive.
onDisk, err := OpenComponentStatusDB(path)
if err != nil {
t.Fatal(err)
}
if rec, ok := onDisk.Get("pcie:gpu:nvidia"); !ok || rec.Status != "OK" {
t.Fatalf("pcie:gpu:nvidia record lost after concurrent PSU write: ok=%v rec=%+v", ok, rec)
}
if _, ok := onDisk.Get("psu:0"); !ok {
t.Fatalf("psu:0 record missing")
}
}