refactor: harden diagnostics and consolidate runtime code
This commit is contained in:
@@ -273,6 +273,32 @@ func TestCheckNvidiaJobHealthReturnsErrorForSelectedResetRequiredGPU(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckNvidiaJobHealthReturnsErrorForSelectedNVLinkDegradedGPU(t *testing.T) {
|
||||
oldExecCommand := satExecCommand
|
||||
satExecCommand = func(name string, args ...string) *exec.Cmd {
|
||||
switch name {
|
||||
case "nvidia-smi":
|
||||
return exec.Command("sh", "-c", "printf '0, NVIDIA H100 PCIe, 38, 46.89, 0, 0, 81559\n1, NVIDIA H100 PCIe, 39, 47.00, 0, 0, 81559\n'")
|
||||
case "dmesg":
|
||||
return exec.Command("sh", "-c", "printf 'NVRM: knvlinkSetDegradedMode_IMPL: GPU1 marked Degraded. Error originated on linkId 1!\n'")
|
||||
default:
|
||||
return exec.Command(name, args...)
|
||||
}
|
||||
}
|
||||
t.Cleanup(func() { satExecCommand = oldExecCommand })
|
||||
|
||||
msg, err := checkNvidiaJobHealth([]int{1})
|
||||
if err == nil {
|
||||
t.Fatal("expected NVLink degraded health check error")
|
||||
}
|
||||
if !strings.Contains(msg, "GPU1 NVLink degraded mode (linkId 1)") {
|
||||
t.Fatalf("unexpected message: %q", msg)
|
||||
}
|
||||
if msg, err := checkNvidiaJobHealth([]int{0}); err != nil || msg != "" {
|
||||
t.Fatalf("unaffected selected GPU0 got msg=%q err=%v", msg, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteNvidiaGPUStatusFilesCreatesPerGPUFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
oldExecCommand := satExecCommand
|
||||
|
||||
Reference in New Issue
Block a user