sat: add tests for informational discovery jobs and retry logic
Covers the sat.go fix from 2599d9c: a failing "dcgmi discovery -l"
preflight job must not flip the pack's overall status, and jobs marked
with retries should recover from a transient first-attempt failure.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
53c46465d2
commit
f46fc98110
@@ -622,3 +622,65 @@ func TestRunROCmSMIReportsMissingCommand(t *testing.T) {
|
|||||||
t.Fatal("expected missing rocm-smi error")
|
t.Fatal("expected missing rocm-smi error")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRunAcceptancePackCtxInformationalJobDoesNotFailOverallStatus guards
|
||||||
|
// against a real production bug: a preflight/metadata job like "dcgmi
|
||||||
|
// discovery -l" racing nv-hostengine startup must not flip the whole pack's
|
||||||
|
// overall status to FAILED when the actual diagnostic that follows it passes.
|
||||||
|
func TestRunAcceptancePackCtxInformationalJobDoesNotFailOverallStatus(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
runDir, err := runAcceptancePackCtx(context.Background(), dir, "gpu-nvidia-informational-test", []satJob{
|
||||||
|
{name: "01-preflight.log", cmd: []string{"sh", "-c", "echo boom >&2; exit 1"}, informational: true},
|
||||||
|
{name: "02-real-test.log", cmd: []string{"sh", "-c", "echo ok"}},
|
||||||
|
}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runAcceptancePackCtx error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reading summary.txt: %v", err)
|
||||||
|
}
|
||||||
|
got := string(summary)
|
||||||
|
if !strings.Contains(got, "overall_status=OK") {
|
||||||
|
t.Fatalf("summary=%q want overall_status=OK", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, "job_informational_failed=1") {
|
||||||
|
t.Fatalf("summary=%q want job_informational_failed=1", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, "job_failed=0") {
|
||||||
|
t.Fatalf("summary=%q want job_failed=0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunAcceptancePackCtxRetriesJob guards the retry path added for jobs
|
||||||
|
// racing a service's startup (e.g. dcgmi discovery immediately after nv-hostengine
|
||||||
|
// restarts): a job that fails once but succeeds on retry should end up OK.
|
||||||
|
func TestRunAcceptancePackCtxRetriesJob(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
marker := filepath.Join(dir, "attempted")
|
||||||
|
|
||||||
|
runDir, err := runAcceptancePackCtx(context.Background(), dir, "gpu-nvidia-retry-test", []satJob{
|
||||||
|
{
|
||||||
|
name: "01-flaky.log",
|
||||||
|
cmd: []string{"sh", "-c", "if [ -f " + marker + " ]; then echo ok; else touch " + marker + "; exit 1; fi"},
|
||||||
|
retries: 2,
|
||||||
|
},
|
||||||
|
}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runAcceptancePackCtx error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reading summary.txt: %v", err)
|
||||||
|
}
|
||||||
|
got := string(summary)
|
||||||
|
if !strings.Contains(got, "overall_status=OK") {
|
||||||
|
t.Fatalf("summary=%q want overall_status=OK", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, "flaky_status=OK") {
|
||||||
|
t.Fatalf("summary=%q want flaky_status=OK", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user