gpu: collect reset-required/row-remap status and classify Xid codes by severity

nvidia-smi exposes reset_status.reset_required and remapped_rows.* only on newer drivers for Ampere+ GPUs; queried via a separate exec call since an unrecognized field name fails the whole --query-gpu command and would have wiped out unrelated telemetry (temp/ECC/power) on older drivers otherwise. Also refines Xid severity in both the ingest dmesg collector and the always-on kmsg watcher: Xid 64 (row-remap InfoROM write failure) now escalates to Critical instead of the generic warning every other Xid got, and fixes the SAT-window flush path which previously hardcoded "Warning" and ignored pattern severity entirely. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-07 19:28:59 +03:00
parent d850d4fc3a
commit 1175e6ccd8
10 changed files with 477 additions and 69 deletions
@@ -0,0 +1,62 @@
+package webui
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"bee/audit/internal/app"
+)
+
+func TestParseKmsgLine_xid64EscalatesToCritical(t *testing.T) {
+	line := "6,1234,555555,-;NVRM: Xid (PCI:0000:65:00.0): 64, pid=1234, name=python, Row Remapper: New row remapping failed"
+	evt, ok := parseKmsgLine(line)
+	if !ok {
+		t.Fatalf("expected line to match a pattern")
+	}
+	if evt.severity != "critical" {
+		t.Fatalf("severity: got %q, want %q", evt.severity, "critical")
+	}
+	if len(evt.ids) != 1 || evt.ids[0] != "0000:65:00.0" {
+		t.Fatalf("ids: got %v", evt.ids)
+	}
+}
+
+func TestParseKmsgLine_xid63StaysWarning(t *testing.T) {
+	line := "6,1234,555555,-;NVRM: Xid (PCI:0000:65:00.0): 63, pid=1234, Class 0x90a0"
+	evt, ok := parseKmsgLine(line)
+	if !ok {
+		t.Fatalf("expected line to match a pattern")
+	}
+	if evt.severity != "warning" {
+		t.Fatalf("severity: got %q, want %q", evt.severity, "warning")
+	}
+}
+
+func TestFlushWindow_escalatesToCriticalOnXid64(t *testing.T) {
+	dbPath := filepath.Join(t.TempDir(), "status.json")
+	db, err := app.OpenComponentStatusDB(dbPath)
+	if err != nil {
+		t.Fatalf("open status db: %v", err)
+	}
+	w := newKmsgWatcher(db)
+
+	window := &kmsgWindow{
+		targets:   []string{"gpu-burn"},
+		startedAt: time.Now(),
+		events: []kmsgEvent{
+			{raw: "NVRM: GPU 0000:65:00.0: RmInitAdapter failed", ids: []string{"0000:65:00.0"}, category: "gpu", severity: "warning"},
+			{raw: "NVRM: Xid (PCI:0000:65:00): 64, pid=1234, name=python", ids: []string{"0000:65:00.0"}, category: "gpu", severity: "critical"},
+		},
+	}
+
+	w.flushWindow(window)
+
+	rec, ok := db.Get("pcie:gpu:0000:65:00.0")
+	if !ok {
+		t.Fatalf("expected a status record for the gpu")
+	}
+	if rec.Status != "Critical" {
+		t.Fatalf("status: got %q, want %q", rec.Status, "Critical")
+	}
+}