refactor: harden diagnostics and consolidate runtime code

This commit is contained in:
Mikhail Chusavitin
2026-09-01 13:01:28 +03:00
parent ac4bc0b2b7
commit 0a6ca8ba0f
49 changed files with 1441 additions and 837 deletions
+26
View File
@@ -273,6 +273,32 @@ func TestCheckNvidiaJobHealthReturnsErrorForSelectedResetRequiredGPU(t *testing.
}
}
func TestCheckNvidiaJobHealthReturnsErrorForSelectedNVLinkDegradedGPU(t *testing.T) {
oldExecCommand := satExecCommand
satExecCommand = func(name string, args ...string) *exec.Cmd {
switch name {
case "nvidia-smi":
return exec.Command("sh", "-c", "printf '0, NVIDIA H100 PCIe, 38, 46.89, 0, 0, 81559\n1, NVIDIA H100 PCIe, 39, 47.00, 0, 0, 81559\n'")
case "dmesg":
return exec.Command("sh", "-c", "printf 'NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n'")
default:
return exec.Command(name, args...)
}
}
t.Cleanup(func() { satExecCommand = oldExecCommand })
msg, err := checkNvidiaJobHealth([]int{1})
if err == nil {
t.Fatal("expected NVLink degraded health check error")
}
if !strings.Contains(msg, "GPU1 NVLink degraded mode (linkId 1)") {
t.Fatalf("unexpected message: %q", msg)
}
if msg, err := checkNvidiaJobHealth([]int{0}); err != nil || msg != "" {
t.Fatalf("unaffected selected GPU0 got msg=%q err=%v", msg, err)
}
}
func TestWriteNvidiaGPUStatusFilesCreatesPerGPUFiles(t *testing.T) {
dir := t.TempDir()
oldExecCommand := satExecCommand