webui: add persistent GPU settings management (ECC/MIG/CC/power limit)
Adds a GPU Settings card to /tools for the settings that actually persist on NVIDIA data-center GPUs: ECC mode, MIG mode, and Confidential Computing mode (all stored in the GPU's inforom/firmware, take effect after a GPU reset or reboot) plus power limit (does not persist — reapplied on demand). Includes a one-click "Reset All to Defaults" that restores factory settings across every visible GPU, touching only whatever has actually drifted. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
2599d9c5e3
commit
53c46465d2
@@ -0,0 +1,183 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// NvidiaGPUSetting reports the persistent GPU settings that nvidia-smi
|
||||
// exposes: ECC mode, MIG mode, and Confidential Computing mode all persist
|
||||
// via the GPU's inforom/firmware and survive a reboot, but only take effect
|
||||
// after a GPU reset (nvidia-smi -r) or host reboot (hence separate
|
||||
// current/pending fields for ECC and MIG). Power limit does NOT persist
|
||||
// across reboot/driver reload — it is re-applied here on demand, same as
|
||||
// nvidia-smi's own behavior.
|
||||
type NvidiaGPUSetting struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
ECCCurrent string `json:"ecc_current"` // "Enabled" / "Disabled" / "N/A"
|
||||
ECCPending string `json:"ecc_pending"` // differs from current until a GPU reset/reboot
|
||||
MIGCurrent string `json:"mig_current"` // "Enabled" / "Disabled" / "N/A" (N/A = not MIG-capable)
|
||||
MIGPending string `json:"mig_pending"`
|
||||
CCState string `json:"cc_state"` // "ON" / "OFF" / "" (unsupported or query failed)
|
||||
PowerLimitW float64 `json:"power_limit_w"`
|
||||
PowerMinLimitW float64 `json:"power_min_limit_w"`
|
||||
PowerMaxLimitW float64 `json:"power_max_limit_w"`
|
||||
PowerDefaultLimitW float64 `json:"power_default_limit_w"`
|
||||
}
|
||||
|
||||
// ListNvidiaGPUSettings reports current ECC, MIG, Confidential Computing and
|
||||
// power-limit configuration per GPU.
|
||||
func (s *System) ListNvidiaGPUSettings() ([]NvidiaGPUSetting, error) {
|
||||
out, err := satExecCommand("nvidia-smi",
|
||||
"--query-gpu=index,name,ecc.mode.current,ecc.mode.pending,mig.mode.current,mig.mode.pending,power.limit,power.min_limit,power.max_limit,power.default_limit",
|
||||
"--format=csv,noheader,nounits").Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nvidia-smi: %w", err)
|
||||
}
|
||||
var settings []NvidiaGPUSetting
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(line, ",")
|
||||
if len(parts) < 10 {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
g := NvidiaGPUSetting{
|
||||
Index: idx,
|
||||
Name: strings.TrimSpace(parts[1]),
|
||||
ECCCurrent: strings.TrimSpace(parts[2]),
|
||||
ECCPending: strings.TrimSpace(parts[3]),
|
||||
MIGCurrent: strings.TrimSpace(parts[4]),
|
||||
MIGPending: strings.TrimSpace(parts[5]),
|
||||
PowerLimitW: parseNvidiaFloat(parts[6]),
|
||||
PowerMinLimitW: parseNvidiaFloat(parts[7]),
|
||||
PowerMaxLimitW: parseNvidiaFloat(parts[8]),
|
||||
PowerDefaultLimitW: parseNvidiaFloat(parts[9]),
|
||||
}
|
||||
g.CCState = s.queryNvidiaGPUCCState(idx)
|
||||
settings = append(settings, g)
|
||||
}
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
// queryNvidiaGPUCCState reads the per-GPU Confidential Computing state via
|
||||
// `nvidia-smi conf-compute -i <index> -q`, reusing the same field-name
|
||||
// convention as RunConfidentialComputingCheckPack's parseConfComputeFields.
|
||||
// Returns "" if the driver/GPU doesn't support conf-compute at all.
|
||||
func (s *System) queryNvidiaGPUCCState(index int) string {
|
||||
out, err := satExecCommand("nvidia-smi", "conf-compute", "-i", strconv.Itoa(index), "-q").Output()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return parseConfComputeFields(out)["CC State"]
|
||||
}
|
||||
|
||||
func parseNvidiaFloat(v string) float64 {
|
||||
f, err := strconv.ParseFloat(strings.TrimSpace(v), 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// SetNvidiaGPUECC sets pending ECC mode for one GPU. Takes effect after the
|
||||
// GPU is reset (nvidia-smi -r) or the host is rebooted.
|
||||
func (s *System) SetNvidiaGPUECC(index int, enabled bool) (string, error) {
|
||||
val := "0"
|
||||
if enabled {
|
||||
val = "1"
|
||||
}
|
||||
out, err := satExecCommand("nvidia-smi", "-i", strconv.Itoa(index), "-e", val).CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
// SetNvidiaGPUMIG sets pending MIG (Multi-Instance GPU) mode for one GPU.
|
||||
// Persists via inforom but takes effect only after a GPU reset (nvidia-smi
|
||||
// -r) or host reboot. Only supported on MIG-capable GPUs (Ampere/Hopper
|
||||
// data-center parts); fails harmlessly on others.
|
||||
func (s *System) SetNvidiaGPUMIG(index int, enabled bool) (string, error) {
|
||||
val := "0"
|
||||
if enabled {
|
||||
val = "1"
|
||||
}
|
||||
out, err := satExecCommand("nvidia-smi", "-i", strconv.Itoa(index), "-mig", val).CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
// SetNvidiaGPUCCMode sets Confidential Computing mode for one GPU
|
||||
// (nvidia-smi conf-compute -scc). Persists via inforom, reversible, but
|
||||
// takes effect only after a GPU reset (nvidia-smi -r) or host reboot.
|
||||
func (s *System) SetNvidiaGPUCCMode(index int, enabled bool) (string, error) {
|
||||
val := "0"
|
||||
if enabled {
|
||||
val = "1"
|
||||
}
|
||||
out, err := satExecCommand("nvidia-smi", "conf-compute", "-i", strconv.Itoa(index), "-scc", val).CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
// SetNvidiaGPUPowerLimit sets the power limit (watts) for one GPU. Does not
|
||||
// persist across reboot/driver reload — callers that want a durable change
|
||||
// must re-apply it (e.g. at boot), same as nvidia-smi's own semantics.
|
||||
func (s *System) SetNvidiaGPUPowerLimit(index int, watts float64) (string, error) {
|
||||
out, err := satExecCommand("nvidia-smi", "-i", strconv.Itoa(index), "-pl", strconv.FormatFloat(watts, 'f', 0, 64)).CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
// ResetNvidiaGPUDefaults restores factory-default settings on every visible
|
||||
// NVIDIA GPU that has drifted from them: power limit back to
|
||||
// power.default_limit (as reported by nvidia-smi), ECC re-enabled (NVIDIA
|
||||
// data-center GPUs ship with ECC on; nvidia-smi has no queryable "ECC
|
||||
// default" field, so this is the documented factory default, not a value
|
||||
// read back from hardware), and MIG/Confidential-Computing mode disabled
|
||||
// (both ship off from the factory). GPUs already at defaults are left
|
||||
// alone.
|
||||
func (s *System) ResetNvidiaGPUDefaults() (string, error) {
|
||||
settings, err := s.ListNvidiaGPUSettings()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var out strings.Builder
|
||||
for _, g := range settings {
|
||||
if g.PowerDefaultLimitW > 0 && g.PowerLimitW != g.PowerDefaultLimitW {
|
||||
res, err := s.SetNvidiaGPUPowerLimit(g.Index, g.PowerDefaultLimitW)
|
||||
out.WriteString(res)
|
||||
if err != nil {
|
||||
fmt.Fprintf(&out, "GPU %d: power limit reset failed: %v\n", g.Index, err)
|
||||
}
|
||||
}
|
||||
if strings.EqualFold(g.ECCCurrent, "disabled") {
|
||||
res, err := s.SetNvidiaGPUECC(g.Index, true)
|
||||
out.WriteString(res)
|
||||
if err != nil {
|
||||
fmt.Fprintf(&out, "GPU %d: ECC re-enable failed: %v\n", g.Index, err)
|
||||
}
|
||||
}
|
||||
if strings.EqualFold(g.MIGCurrent, "enabled") {
|
||||
res, err := s.SetNvidiaGPUMIG(g.Index, false)
|
||||
out.WriteString(res)
|
||||
if err != nil {
|
||||
fmt.Fprintf(&out, "GPU %d: MIG disable failed: %v\n", g.Index, err)
|
||||
}
|
||||
}
|
||||
if strings.EqualFold(g.CCState, "on") {
|
||||
res, err := s.SetNvidiaGPUCCMode(g.Index, false)
|
||||
out.WriteString(res)
|
||||
if err != nil {
|
||||
fmt.Fprintf(&out, "GPU %d: CC mode disable failed: %v\n", g.Index, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if out.Len() == 0 {
|
||||
out.WriteString("All GPUs already at factory-default settings.\n")
|
||||
}
|
||||
return out.String(), nil
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeNvidiaSMI stubs satExecCommand for nvidia-smi calls made by the
|
||||
// GPU-settings code: the CSV query (index,...,power.default_limit), the
|
||||
// per-GPU "conf-compute -q" read, and write commands (-e/-mig/-pl/-scc).
|
||||
// ccStateByIndex maps GPU index -> CC State text ("ON"/"OFF"); a missing
|
||||
// entry simulates conf-compute being unsupported (query fails).
|
||||
func fakeNvidiaSMI(t *testing.T, queryOut string, ccStateByIndex map[int]string, writeCalls *[]string) {
|
||||
t.Helper()
|
||||
old := satExecCommand
|
||||
satExecCommand = func(name string, args ...string) *exec.Cmd {
|
||||
if name != "nvidia-smi" {
|
||||
return exec.Command(name, args...)
|
||||
}
|
||||
if len(args) > 0 && strings.HasPrefix(args[0], "--query-gpu=") {
|
||||
return exec.Command("printf", "%s", queryOut)
|
||||
}
|
||||
if len(args) > 0 && args[0] == "conf-compute" {
|
||||
// args: conf-compute -i <index> -q
|
||||
idx := -1
|
||||
for i, a := range args {
|
||||
if a == "-i" && i+1 < len(args) {
|
||||
var n int
|
||||
fscanInt(args[i+1], &n)
|
||||
idx = n
|
||||
}
|
||||
}
|
||||
if len(args) > 0 && args[len(args)-1] == "-q" {
|
||||
state, ok := ccStateByIndex[idx]
|
||||
if !ok {
|
||||
return exec.Command("false")
|
||||
}
|
||||
return exec.Command("printf", "CC State : %s\\n", state)
|
||||
}
|
||||
if writeCalls != nil {
|
||||
*writeCalls = append(*writeCalls, strings.Join(args, " "))
|
||||
}
|
||||
return exec.Command("true")
|
||||
}
|
||||
if writeCalls != nil {
|
||||
*writeCalls = append(*writeCalls, strings.Join(args, " "))
|
||||
}
|
||||
return exec.Command("true")
|
||||
}
|
||||
t.Cleanup(func() { satExecCommand = old })
|
||||
}
|
||||
|
||||
func fscanInt(s string, out *int) {
|
||||
n := 0
|
||||
neg := false
|
||||
for i, c := range s {
|
||||
if i == 0 && c == '-' {
|
||||
neg = true
|
||||
continue
|
||||
}
|
||||
if c < '0' || c > '9' {
|
||||
return
|
||||
}
|
||||
n = n*10 + int(c-'0')
|
||||
}
|
||||
if neg {
|
||||
n = -n
|
||||
}
|
||||
*out = n
|
||||
}
|
||||
|
||||
func TestListNvidiaGPUSettingsParsesCSV(t *testing.T) {
|
||||
fakeNvidiaSMI(t,
|
||||
"0, H100 80GB HBM3, Enabled, Enabled, Disabled, Disabled, 350.00, 100.00, 700.00, 700.00\n"+
|
||||
"1, H100 80GB HBM3, Disabled, Enabled, N/A, N/A, 300.00, 100.00, 700.00, 700.00\n",
|
||||
map[int]string{0: "OFF", 1: "OFF"}, nil)
|
||||
|
||||
s := &System{}
|
||||
got, err := s.ListNvidiaGPUSettings()
|
||||
if err != nil {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("len=%d want 2 (%#v)", len(got), got)
|
||||
}
|
||||
if got[0].ECCCurrent != "Enabled" || got[0].PowerLimitW != 350 || got[0].PowerDefaultLimitW != 700 {
|
||||
t.Fatalf("gpu0=%#v", got[0])
|
||||
}
|
||||
if got[0].MIGCurrent != "Disabled" || got[0].CCState != "OFF" {
|
||||
t.Fatalf("gpu0 mig/cc=%#v", got[0])
|
||||
}
|
||||
if got[1].ECCCurrent != "Disabled" || got[1].ECCPending != "Enabled" || got[1].MIGCurrent != "N/A" {
|
||||
t.Fatalf("gpu1=%#v", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestListNvidiaGPUSettingsCCUnsupportedLeavesEmptyState(t *testing.T) {
|
||||
fakeNvidiaSMI(t,
|
||||
"0, H100, Enabled, Enabled, Disabled, Disabled, 350.00, 100.00, 700.00, 700.00\n",
|
||||
map[int]string{}, nil)
|
||||
|
||||
s := &System{}
|
||||
got, err := s.ListNvidiaGPUSettings()
|
||||
if err != nil {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].CCState != "" {
|
||||
t.Fatalf("got=%#v want empty CCState when conf-compute query fails", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetNvidiaGPUECCPassesCorrectFlag(t *testing.T) {
|
||||
var calls []string
|
||||
fakeNvidiaSMI(t, "", nil, &calls)
|
||||
|
||||
s := &System{}
|
||||
if _, err := s.SetNvidiaGPUECC(2, true); err != nil {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if len(calls) != 1 || calls[0] != "-i 2 -e 1" {
|
||||
t.Fatalf("calls=%v want [-i 2 -e 1]", calls)
|
||||
}
|
||||
|
||||
if _, err := s.SetNvidiaGPUECC(0, false); err != nil {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if calls[1] != "-i 0 -e 0" {
|
||||
t.Fatalf("calls[1]=%q want -i 0 -e 0", calls[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetNvidiaGPUMIGPassesCorrectFlag(t *testing.T) {
|
||||
var calls []string
|
||||
fakeNvidiaSMI(t, "", nil, &calls)
|
||||
|
||||
s := &System{}
|
||||
if _, err := s.SetNvidiaGPUMIG(3, true); err != nil {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if len(calls) != 1 || calls[0] != "-i 3 -mig 1" {
|
||||
t.Fatalf("calls=%v want [-i 3 -mig 1]", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetNvidiaGPUCCModePassesCorrectFlag(t *testing.T) {
|
||||
var calls []string
|
||||
fakeNvidiaSMI(t, "", nil, &calls)
|
||||
|
||||
s := &System{}
|
||||
if _, err := s.SetNvidiaGPUCCMode(1, false); err != nil {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if len(calls) != 1 || calls[0] != "conf-compute -i 1 -scc 0" {
|
||||
t.Fatalf("calls=%v want [conf-compute -i 1 -scc 0]", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetNvidiaGPUPowerLimitFormatsWatts(t *testing.T) {
|
||||
var calls []string
|
||||
fakeNvidiaSMI(t, "", nil, &calls)
|
||||
|
||||
s := &System{}
|
||||
if _, err := s.SetNvidiaGPUPowerLimit(1, 450); err != nil {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if len(calls) != 1 || calls[0] != "-i 1 -pl 450" {
|
||||
t.Fatalf("calls=%v want [-i 1 -pl 450]", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetNvidiaGPUDefaultsOnlyTouchesDriftedGPUs(t *testing.T) {
|
||||
var calls []string
|
||||
fakeNvidiaSMI(t,
|
||||
"0, H100, Enabled, Enabled, Enabled, Enabled, 350.00, 100.00, 700.00, 700.00\n"+
|
||||
"1, H100, Disabled, Disabled, Disabled, Disabled, 700.00, 100.00, 700.00, 700.00\n",
|
||||
map[int]string{0: "ON", 1: "OFF"}, &calls)
|
||||
|
||||
s := &System{}
|
||||
out, err := s.ResetNvidiaGPUDefaults()
|
||||
if err != nil {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if out == "" {
|
||||
t.Fatalf("expected non-empty output")
|
||||
}
|
||||
// GPU0: power limit drifted (350 vs 700) -> reset; MIG enabled -> disable; CC ON -> disable.
|
||||
// GPU1: power limit already default, MIG already disabled, CC already off;
|
||||
// only ECC disabled -> re-enable.
|
||||
want := []string{
|
||||
"-i 0 -pl 700",
|
||||
"-i 0 -mig 0",
|
||||
"conf-compute -i 0 -scc 0",
|
||||
"-i 1 -e 1",
|
||||
}
|
||||
if len(calls) != len(want) {
|
||||
t.Fatalf("calls=%v want %v", calls, want)
|
||||
}
|
||||
for i := range want {
|
||||
if calls[i] != want[i] {
|
||||
t.Fatalf("calls[%d]=%q want %q (all calls: %v)", i, calls[i], want[i], calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetNvidiaGPUDefaultsNoOpWhenAlreadyDefault(t *testing.T) {
|
||||
var calls []string
|
||||
fakeNvidiaSMI(t,
|
||||
"0, H100, Enabled, Enabled, Disabled, Disabled, 700.00, 100.00, 700.00, 700.00\n",
|
||||
map[int]string{0: "OFF"}, &calls)
|
||||
|
||||
s := &System{}
|
||||
out, err := s.ResetNvidiaGPUDefaults()
|
||||
if err != nil {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if len(calls) != 0 {
|
||||
t.Fatalf("calls=%v want no calls (already at factory defaults)", calls)
|
||||
}
|
||||
if !strings.Contains(out, "already at factory-default") {
|
||||
t.Fatalf("out=%q want no-op message", out)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user