sat: add tests for informational discovery jobs and retry logic

Covers the sat.go fix from 2599d9c: a failing "dcgmi discovery -l"
preflight job must not flip the pack's overall status, and jobs marked
with retries should recover from a transient first-attempt failure.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-07 15:31:31 +03:00
co-authored by Claude Sonnet 5
parent 53c46465d2
commit f46fc98110
+62
View File
@@ -622,3 +622,65 @@ func TestRunROCmSMIReportsMissingCommand(t *testing.T) {
t.Fatal("expected missing rocm-smi error")
}
}
// TestRunAcceptancePackCtxInformationalJobDoesNotFailOverallStatus guards
// against a real production bug: a preflight/metadata job like "dcgmi
// discovery -l" racing nv-hostengine startup must not flip the whole pack's
// overall status to FAILED when the actual diagnostic that follows it passes.
func TestRunAcceptancePackCtxInformationalJobDoesNotFailOverallStatus(t *testing.T) {
dir := t.TempDir()
runDir, err := runAcceptancePackCtx(context.Background(), dir, "gpu-nvidia-informational-test", []satJob{
{name: "01-preflight.log", cmd: []string{"sh", "-c", "echo boom >&2; exit 1"}, informational: true},
{name: "02-real-test.log", cmd: []string{"sh", "-c", "echo ok"}},
}, nil)
if err != nil {
t.Fatalf("runAcceptancePackCtx error: %v", err)
}
summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
if err != nil {
t.Fatalf("reading summary.txt: %v", err)
}
got := string(summary)
if !strings.Contains(got, "overall_status=OK") {
t.Fatalf("summary=%q want overall_status=OK", got)
}
if !strings.Contains(got, "job_informational_failed=1") {
t.Fatalf("summary=%q want job_informational_failed=1", got)
}
if !strings.Contains(got, "job_failed=0") {
t.Fatalf("summary=%q want job_failed=0", got)
}
}
// TestRunAcceptancePackCtxRetriesJob guards the retry path added for jobs
// racing a service's startup (e.g. dcgmi discovery immediately after nv-hostengine
// restarts): a job that fails once but succeeds on retry should end up OK.
func TestRunAcceptancePackCtxRetriesJob(t *testing.T) {
dir := t.TempDir()
marker := filepath.Join(dir, "attempted")
runDir, err := runAcceptancePackCtx(context.Background(), dir, "gpu-nvidia-retry-test", []satJob{
{
name: "01-flaky.log",
cmd: []string{"sh", "-c", "if [ -f " + marker + " ]; then echo ok; else touch " + marker + "; exit 1; fi"},
retries: 2,
},
}, nil)
if err != nil {
t.Fatalf("runAcceptancePackCtx error: %v", err)
}
summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
if err != nil {
t.Fatalf("reading summary.txt: %v", err)
}
got := string(summary)
if !strings.Contains(got, "overall_status=OK") {
t.Fatalf("summary=%q want overall_status=OK", got)
}
if !strings.Contains(got, "flaky_status=OK") {
t.Fatalf("summary=%q want flaky_status=OK", got)
}
}