platform/webui: fold confidential-computing into a GPU config + NVLink check
The standalone "confidential-computing" SAT target only ever checked CC readiness, which most fleets never opt into (a NOT_READY verdict there isn't a fault). Meanwhile DCGM diag never asserts GPU config compliance (ECC/MIG/power-limit vs factory default) or NVLink topology (per NVIDIA's own DGX BasePOD deployment guide, this needs a separate validation step) — gaps confirmed against public DCGM docs and a real NV17-vs-expected-NV18 bonded pair found on a live bundle. Repurposes the routine into "nvidia-config": reuses the existing ListNvidiaGPUSettings() (already backing the GPU-settings page) to flag ECC disabled, a MIG mode change stuck pending a reset/reboot, and a power limit capped >5% below default; parses "nvidia-smi topo -m" bonded pairs against "nvlink -s/-e" to flag any inactive lane or nonzero replay/ recovery/CRC counter on an otherwise-active bond. CC readiness is folded in as one informational field (does not gate overall_status) rather than a dedicated test. Reports under the same pcie:gpu:nvidia severity key as every other nvidia-* SAT target instead of an isolated key, so a config/NVLink FAILED result isn't invisible next to stress-test results. Also fixes ApplySATResultToDB silently dropping any target with no matching switch case (exactly what the old confidential-computing target did) with a new coverage test enumerating every real SAT target. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
cc3997f7b1
commit
cfaa15ec7c
@@ -134,7 +134,7 @@ type satRunner interface {
|
|||||||
ResetNvidiaGPU(index int) (string, error)
|
ResetNvidiaGPU(index int) (string, error)
|
||||||
RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error)
|
RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error)
|
||||||
RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error)
|
RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error)
|
||||||
RunConfidentialComputingCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
|
RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
|
||||||
RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
|
RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
|
||||||
ListNvidiaGPUs() ([]platform.NvidiaGPU, error)
|
ListNvidiaGPUs() ([]platform.NvidiaGPU, error)
|
||||||
ListNvidiaGPUSettings() ([]platform.NvidiaGPUSetting, error)
|
ListNvidiaGPUSettings() ([]platform.NvidiaGPUSetting, error)
|
||||||
|
|||||||
@@ -247,20 +247,20 @@ func (a *App) RunStorageAcceptancePackResult(baseDir string) (ActionResult, erro
|
|||||||
return ActionResult{Title: "Storage SAT", Body: satResultBody(path)}, err
|
return ActionResult{Title: "Storage SAT", Body: satResultBody(path)}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) RunConfidentialComputingCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
func (a *App) RunNvidiaConfigCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||||
if strings.TrimSpace(baseDir) == "" {
|
if strings.TrimSpace(baseDir) == "" {
|
||||||
baseDir = DefaultSATBaseDir
|
baseDir = DefaultSATBaseDir
|
||||||
}
|
}
|
||||||
return a.sat.RunConfidentialComputingCheckPack(ctx, baseDir, logFunc)
|
return a.sat.RunNvidiaConfigCheckPack(ctx, baseDir, logFunc)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) RunConfidentialComputingCheckPack(baseDir string, logFunc func(string)) (string, error) {
|
func (a *App) RunNvidiaConfigCheckPack(baseDir string, logFunc func(string)) (string, error) {
|
||||||
return a.RunConfidentialComputingCheckPackCtx(context.Background(), baseDir, logFunc)
|
return a.RunNvidiaConfigCheckPackCtx(context.Background(), baseDir, logFunc)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) RunConfidentialComputingCheckPackResult(baseDir string) (ActionResult, error) {
|
func (a *App) RunNvidiaConfigCheckPackResult(baseDir string) (ActionResult, error) {
|
||||||
path, err := a.RunConfidentialComputingCheckPack(baseDir, nil)
|
path, err := a.RunNvidiaConfigCheckPack(baseDir, nil)
|
||||||
return ActionResult{Title: "Confidential Computing Check", Body: satResultBody(path)}, err
|
return ActionResult{Title: "GPU Config & NVLink Check", Body: satResultBody(path)}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) DetectGPUVendor() string {
|
func (a *App) DetectGPUVendor() string {
|
||||||
|
|||||||
@@ -267,7 +267,7 @@ func (f fakeSAT) RunStorageAcceptancePack(_ context.Context, baseDir string, _ b
|
|||||||
return f.runStorageFn(baseDir)
|
return f.runStorageFn(baseDir)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f fakeSAT) RunConfidentialComputingCheckPack(_ context.Context, baseDir string, _ func(string)) (string, error) {
|
func (f fakeSAT) RunNvidiaConfigCheckPack(_ context.Context, baseDir string, _ func(string)) (string, error) {
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -243,7 +243,7 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
|
|||||||
// otherwise fails to match any real BDF.
|
// otherwise fails to match any real BDF.
|
||||||
switch target {
|
switch target {
|
||||||
case "nvidia", "nvidia-targeted-stress", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
|
case "nvidia", "nvidia-targeted-stress", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
|
||||||
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress":
|
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "nvidia-config":
|
||||||
db.Record("pcie:gpu:nvidia", source, dbStatus, target+" SAT: "+overall)
|
db.Record("pcie:gpu:nvidia", source, dbStatus, target+" SAT: "+overall)
|
||||||
case "amd", "amd-stress", "amd-mem", "amd-bandwidth":
|
case "amd", "amd-stress", "amd-mem", "amd-bandwidth":
|
||||||
db.Record("pcie:gpu:amd", source, dbStatus, target+" SAT: "+overall)
|
db.Record("pcie:gpu:amd", source, dbStatus, target+" SAT: "+overall)
|
||||||
|
|||||||
@@ -98,6 +98,84 @@ func TestApplyComponentStatusDBMatchesGPUByVendor(t *testing.T) {
|
|||||||
|
|
||||||
func strPtr(s string) *string { return &s }
|
func strPtr(s string) *string { return &s }
|
||||||
|
|
||||||
|
// TestApplySATResultToDBCoversAllHealthCheckTargets guards against a target
|
||||||
|
// silently falling through ApplySATResultToDB's switch with no matching
|
||||||
|
// case — exactly what happened to the old "confidential-computing" target
|
||||||
|
// before it was folded into "nvidia-config" (task ran, produced a valid
|
||||||
|
// summary.txt, but no component-status.json record was ever written for it,
|
||||||
|
// so a /topo card or any other consumer had no way to know the check had
|
||||||
|
// even run). Every target below is a real health/acceptance check target
|
||||||
|
// dispatched by task_runner.go's executeTaskWithOptions that produces a
|
||||||
|
// summary.txt with overall_status; each must land somewhere in the DB.
|
||||||
|
func TestApplySATResultToDBCoversAllHealthCheckTargets(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
target string
|
||||||
|
wantKey string
|
||||||
|
}{
|
||||||
|
{"nvidia", "pcie:gpu:nvidia"},
|
||||||
|
{"nvidia-stress", "pcie:gpu:nvidia"},
|
||||||
|
{"nvidia-targeted-stress", "pcie:gpu:nvidia"},
|
||||||
|
{"nvidia-compute", "pcie:gpu:nvidia"},
|
||||||
|
{"nvidia-targeted-power", "pcie:gpu:nvidia"},
|
||||||
|
{"nvidia-pulse", "pcie:gpu:nvidia"},
|
||||||
|
{"nvidia-interconnect", "pcie:gpu:nvidia"},
|
||||||
|
{"nvidia-bandwidth", "pcie:gpu:nvidia"},
|
||||||
|
{"nvidia-config", "pcie:gpu:nvidia"},
|
||||||
|
{"amd", "pcie:gpu:amd"},
|
||||||
|
{"amd-stress", "pcie:gpu:amd"},
|
||||||
|
{"amd-mem", "pcie:gpu:amd"},
|
||||||
|
{"amd-bandwidth", "pcie:gpu:amd"},
|
||||||
|
{"memory", "memory:all"},
|
||||||
|
{"memory-stress", "memory:all"},
|
||||||
|
{"sat-stress", "memory:all"},
|
||||||
|
{"cpu", "cpu:all"},
|
||||||
|
{"platform-stress", "cpu:all"},
|
||||||
|
{"storage", "storage:all"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.target, func(t *testing.T) {
|
||||||
|
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ApplySATResultToDB(db, tc.target, writeSATSummary(t, "OK"))
|
||||||
|
rec, ok := db.Get(tc.wantKey)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("target %q wrote no record under key %q — falls through the switch silently", tc.target, tc.wantKey)
|
||||||
|
}
|
||||||
|
if rec.Status != "OK" {
|
||||||
|
t.Fatalf("target %q key %q status=%q want OK", tc.target, tc.wantKey, rec.Status)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets
|
||||||
|
// confirms "nvidia-config" (GPU config/NVLink/CC check — confidential
|
||||||
|
// computing readiness folded in here rather than run as its own standalone
|
||||||
|
// SAT target) reports under the same pcie:gpu:nvidia key as every other
|
||||||
|
// nvidia-* target, so a config/NVLink FAILED result isn't invisible next to
|
||||||
|
// the plain stress-test results, and a clean run afterward doesn't silently
|
||||||
|
// erase an earlier failure (severity merge, same as
|
||||||
|
// TestApplySATResultToDBNormalizesGPUKeyByVendor above).
|
||||||
|
func TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets(t *testing.T) {
|
||||||
|
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ApplySATResultToDB(db, "nvidia-config", writeSATSummary(t, "FAILED"))
|
||||||
|
ApplySATResultToDB(db, "nvidia", writeSATSummary(t, "OK"))
|
||||||
|
|
||||||
|
rec, ok := db.Get("pcie:gpu:nvidia")
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected pcie:gpu:nvidia record to exist")
|
||||||
|
}
|
||||||
|
if rec.Status != "Warning" {
|
||||||
|
t.Fatalf("status=%q, want Warning (nvidia-config FAILED must survive the later OK nvidia run)", rec.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
// TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the
|
// TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the
|
||||||
// hardware-ingest-contract.md rule that status_history is a transition log
|
// hardware-ingest-contract.md rule that status_history is a transition log
|
||||||
// ("История переходов статусов"), not a per-poll journal. A component
|
// ("История переходов статусов"), not a per-poll journal. A component
|
||||||
|
|||||||
@@ -1,248 +0,0 @@
|
|||||||
package platform
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ConfidentialComputingStatus summarizes whether this server can run NVIDIA
|
|
||||||
// Confidential Computing: CPU-side TEE support (Intel TDX / AMD SEV-SNP) and
|
|
||||||
// GPU firmware CC capability, as reported by `nvidia-smi conf-compute -q`.
|
|
||||||
type ConfidentialComputingStatus struct {
|
|
||||||
CollectedAt time.Time `json:"collected_at"`
|
|
||||||
|
|
||||||
// GPU-reported fields, parsed from `nvidia-smi conf-compute -q`.
|
|
||||||
NvidiaSMIAvailable bool `json:"nvidia_smi_available"`
|
|
||||||
CCState string `json:"cc_state,omitempty"` // ON / OFF
|
|
||||||
MultiGPUMode string `json:"multi_gpu_mode,omitempty"` // Protected PCIe / ...
|
|
||||||
CPUCCCapability string `json:"cpu_cc_capability,omitempty"` // e.g. "INTEL TDX", "AMD SEV-SNP", "NONE"
|
|
||||||
GPUCCCapability string `json:"gpu_cc_capability,omitempty"` // e.g. "CC Capable", "Not Capable"
|
|
||||||
CCGPUsReadyState string `json:"cc_gpus_ready_state,omitempty"` // Ready / Not Ready
|
|
||||||
|
|
||||||
// Host-side evidence that the CPU's TEE is actually active in the running
|
|
||||||
// kernel (BIOS + kernel cmdline + firmware), independent of what the GPU
|
|
||||||
// driver reports. Used as a fallback when the NVIDIA driver isn't loaded.
|
|
||||||
HostAMDSEVSupported bool `json:"host_amd_sev_supported"`
|
|
||||||
HostAMDSEVESSupported bool `json:"host_amd_sev_es_supported"`
|
|
||||||
HostAMDSEVSNPActive bool `json:"host_amd_sev_snp_active"`
|
|
||||||
HostIntelTDXActive bool `json:"host_intel_tdx_active"`
|
|
||||||
|
|
||||||
// GPUCanRunCC is true when the GPU firmware reports CC-capable.
|
|
||||||
GPUCanRunCC bool `json:"gpu_can_run_cc"`
|
|
||||||
// CPUCanRunCC is true when either the GPU driver or the host kernel
|
|
||||||
// reports an active/available CPU TEE (SEV-SNP or TDX).
|
|
||||||
CPUCanRunCC bool `json:"cpu_can_run_cc"`
|
|
||||||
// Ready is true when both the CPU and the GPU support Confidential
|
|
||||||
// Computing, regardless of whether CC mode is currently enabled.
|
|
||||||
Ready bool `json:"ready"`
|
|
||||||
|
|
||||||
Notes []string `json:"notes,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunConfidentialComputingCheckPack runs a read-only check of whether this
|
|
||||||
// server can run NVIDIA Confidential Computing: it queries the GPU driver
|
|
||||||
// (`nvidia-smi conf-compute -q`) and inspects host kernel/dmesg evidence of
|
|
||||||
// AMD SEV-SNP / Intel TDX support. It changes nothing on the system.
|
|
||||||
func (s *System) RunConfidentialComputingCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
|
||||||
if ctx == nil {
|
|
||||||
ctx = context.Background()
|
|
||||||
}
|
|
||||||
if baseDir == "" {
|
|
||||||
baseDir = "/var/log/bee-sat"
|
|
||||||
}
|
|
||||||
ts := time.Now().UTC().Format("20060102-150405")
|
|
||||||
runDir := filepath.Join(baseDir, "confidential-computing-"+ts)
|
|
||||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
|
||||||
|
|
||||||
status := ConfidentialComputingStatus{CollectedAt: time.Now().UTC()}
|
|
||||||
|
|
||||||
// GPU firmware / driver state.
|
|
||||||
ccOut, ccErr := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-conf-compute-q", []string{"nvidia-smi", "conf-compute", "-q"}, nil, logFunc)
|
|
||||||
_ = os.WriteFile(filepath.Join(runDir, "01-nvidia-smi-conf-compute-q.log"), ccOut, 0644)
|
|
||||||
if ccErr == nil {
|
|
||||||
status.NvidiaSMIAvailable = true
|
|
||||||
fields := parseConfComputeFields(ccOut)
|
|
||||||
status.CCState = fields["CC State"]
|
|
||||||
status.MultiGPUMode = fields["Multi-GPU Mode"]
|
|
||||||
status.CPUCCCapability = fields["CPU CC Capabilities"]
|
|
||||||
status.GPUCCCapability = fields["GPU CC Capabilities"]
|
|
||||||
status.CCGPUsReadyState = fields["CC GPUs Ready State"]
|
|
||||||
} else {
|
|
||||||
status.Notes = append(status.Notes, "nvidia-smi conf-compute -q unavailable (no NVIDIA driver, or GPU not present): "+firstLine(string(ccOut)))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Host kernel evidence, independent of the GPU driver.
|
|
||||||
dmesgOut, _ := runSATCommandCtx(ctx, verboseLog, "dmesg", []string{"dmesg"}, nil, nil)
|
|
||||||
ccDmesgLines := filterConfComputeDmesgLines(dmesgOut)
|
|
||||||
_ = os.WriteFile(filepath.Join(runDir, "02-dmesg-cc-relevant.log"), []byte(strings.Join(ccDmesgLines, "\n")+"\n"), 0644)
|
|
||||||
|
|
||||||
lowerDmesg := strings.ToLower(strings.Join(ccDmesgLines, "\n"))
|
|
||||||
status.HostAMDSEVSNPActive = strings.Contains(lowerDmesg, "sev-snp enabled")
|
|
||||||
status.HostIntelTDXActive = strings.Contains(lowerDmesg, "tdx module") && strings.Contains(lowerDmesg, "module initialized") ||
|
|
||||||
strings.Contains(lowerDmesg, "virt/tdx: module initialized")
|
|
||||||
|
|
||||||
for i, path := range []string{
|
|
||||||
"/sys/module/kvm_amd/parameters/sev",
|
|
||||||
"/sys/module/kvm_amd/parameters/sev_es",
|
|
||||||
"/sys/module/kvm_amd/parameters/sev_snp",
|
|
||||||
} {
|
|
||||||
name := fmt.Sprintf("sysfs-%s", filepath.Base(path))
|
|
||||||
out, err := runSATCommandCtx(ctx, verboseLog, name, []string{"cat", path}, nil, nil)
|
|
||||||
_ = os.WriteFile(filepath.Join(runDir, fmt.Sprintf("03-%02d-%s.log", i+1, name)), out, 0644)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
val := strings.TrimSpace(string(out))
|
|
||||||
switch filepath.Base(path) {
|
|
||||||
case "sev":
|
|
||||||
status.HostAMDSEVSupported = strings.EqualFold(val, "Y")
|
|
||||||
case "sev_es":
|
|
||||||
status.HostAMDSEVESSupported = strings.EqualFold(val, "Y")
|
|
||||||
case "sev_snp":
|
|
||||||
if strings.EqualFold(val, "Y") {
|
|
||||||
status.HostAMDSEVSNPActive = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
status.GPUCanRunCC = strings.EqualFold(strings.TrimSpace(status.GPUCCCapability), "CC Capable")
|
|
||||||
cpuCapReported := strings.TrimSpace(status.CPUCCCapability)
|
|
||||||
status.CPUCanRunCC = status.HostAMDSEVSNPActive || status.HostIntelTDXActive ||
|
|
||||||
(cpuCapReported != "" && !strings.EqualFold(cpuCapReported, "NONE"))
|
|
||||||
status.Ready = status.CPUCanRunCC && status.GPUCanRunCC
|
|
||||||
|
|
||||||
if !status.NvidiaSMIAvailable {
|
|
||||||
status.Notes = append(status.Notes, "GPU CC capability unknown — install the NVIDIA driver to query it with `nvidia-smi conf-compute -q`.")
|
|
||||||
}
|
|
||||||
|
|
||||||
summary := renderConfidentialComputingSummary(status)
|
|
||||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
report := renderConfidentialComputingReport(status)
|
|
||||||
if err := os.WriteFile(filepath.Join(runDir, "confidential-computing-report.txt"), []byte(report), 0644); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
return runDir, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseConfComputeFields parses the indented "Key : Value" block emitted by
|
|
||||||
// `nvidia-smi conf-compute -q`, e.g.:
|
|
||||||
//
|
|
||||||
// CC State : OFF
|
|
||||||
// Multi-GPU Mode : Protected PCIe
|
|
||||||
// CPU CC Capabilities : INTEL TDX
|
|
||||||
// GPU CC Capabilities : CC Capable
|
|
||||||
// CC GPUs Ready State : Not Ready
|
|
||||||
func parseConfComputeFields(out []byte) map[string]string {
|
|
||||||
fields := map[string]string{}
|
|
||||||
for _, line := range strings.Split(string(out), "\n") {
|
|
||||||
idx := strings.Index(line, ":")
|
|
||||||
if idx < 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
key := strings.TrimSpace(line[:idx])
|
|
||||||
val := strings.TrimSpace(line[idx+1:])
|
|
||||||
if key == "" || val == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fields[key] = val
|
|
||||||
}
|
|
||||||
return fields
|
|
||||||
}
|
|
||||||
|
|
||||||
// filterConfComputeDmesgLines returns the dmesg lines relevant to CPU
|
|
||||||
// Confidential Computing support (AMD SEV/SEV-ES/SEV-SNP, Intel TDX).
|
|
||||||
func filterConfComputeDmesgLines(dmesgOut []byte) []string {
|
|
||||||
var lines []string
|
|
||||||
scanner := bytes.Split(dmesgOut, []byte("\n"))
|
|
||||||
for _, raw := range scanner {
|
|
||||||
lower := strings.ToLower(string(raw))
|
|
||||||
if strings.Contains(lower, "sev") || strings.Contains(lower, "tdx") {
|
|
||||||
lines = append(lines, string(raw))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return lines
|
|
||||||
}
|
|
||||||
|
|
||||||
func renderConfidentialComputingSummary(status ConfidentialComputingStatus) string {
|
|
||||||
var b strings.Builder
|
|
||||||
fmt.Fprintf(&b, "run_at_utc=%s\n", status.CollectedAt.Format(time.RFC3339))
|
|
||||||
fmt.Fprintf(&b, "nvidia_smi_available=%t\n", status.NvidiaSMIAvailable)
|
|
||||||
fmt.Fprintf(&b, "cc_state=%s\n", status.CCState)
|
|
||||||
fmt.Fprintf(&b, "multi_gpu_mode=%s\n", status.MultiGPUMode)
|
|
||||||
fmt.Fprintf(&b, "cpu_cc_capability=%s\n", status.CPUCCCapability)
|
|
||||||
fmt.Fprintf(&b, "gpu_cc_capability=%s\n", status.GPUCCCapability)
|
|
||||||
fmt.Fprintf(&b, "cc_gpus_ready_state=%s\n", status.CCGPUsReadyState)
|
|
||||||
fmt.Fprintf(&b, "host_amd_sev_supported=%t\n", status.HostAMDSEVSupported)
|
|
||||||
fmt.Fprintf(&b, "host_amd_sev_es_supported=%t\n", status.HostAMDSEVESSupported)
|
|
||||||
fmt.Fprintf(&b, "host_amd_sev_snp_active=%t\n", status.HostAMDSEVSNPActive)
|
|
||||||
fmt.Fprintf(&b, "host_intel_tdx_active=%t\n", status.HostIntelTDXActive)
|
|
||||||
fmt.Fprintf(&b, "cpu_can_run_cc=%t\n", status.CPUCanRunCC)
|
|
||||||
fmt.Fprintf(&b, "gpu_can_run_cc=%t\n", status.GPUCanRunCC)
|
|
||||||
fmt.Fprintf(&b, "ready=%t\n", status.Ready)
|
|
||||||
if status.Ready {
|
|
||||||
fmt.Fprintln(&b, "overall_status=OK")
|
|
||||||
} else {
|
|
||||||
fmt.Fprintln(&b, "overall_status=NOT_READY")
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func renderConfidentialComputingReport(status ConfidentialComputingStatus) string {
|
|
||||||
var b strings.Builder
|
|
||||||
line := strings.Repeat("=", 80)
|
|
||||||
b.WriteString(line + "\n")
|
|
||||||
b.WriteString("Confidential Computing Readiness\n")
|
|
||||||
b.WriteString(line + "\n\n")
|
|
||||||
|
|
||||||
verdict := "NOT READY"
|
|
||||||
if status.Ready {
|
|
||||||
verdict = "READY"
|
|
||||||
}
|
|
||||||
fmt.Fprintf(&b, "Verdict: %s\n\n", verdict)
|
|
||||||
|
|
||||||
b.WriteString("-- CPU ----------------------------------------------------------------------\n")
|
|
||||||
fmt.Fprintf(&b, " Reported by GPU driver : %s\n", nonEmptyOr(status.CPUCCCapability, "unknown"))
|
|
||||||
fmt.Fprintf(&b, " AMD SEV supported : %t\n", status.HostAMDSEVSupported)
|
|
||||||
fmt.Fprintf(&b, " AMD SEV-ES supported : %t\n", status.HostAMDSEVESSupported)
|
|
||||||
fmt.Fprintf(&b, " AMD SEV-SNP active : %t\n", status.HostAMDSEVSNPActive)
|
|
||||||
fmt.Fprintf(&b, " Intel TDX active : %t\n", status.HostIntelTDXActive)
|
|
||||||
fmt.Fprintf(&b, " Can run CC : %t\n\n", status.CPUCanRunCC)
|
|
||||||
|
|
||||||
b.WriteString("-- GPU ----------------------------------------------------------------------\n")
|
|
||||||
fmt.Fprintf(&b, " nvidia-smi available : %t\n", status.NvidiaSMIAvailable)
|
|
||||||
fmt.Fprintf(&b, " GPU CC Capabilities : %s\n", nonEmptyOr(status.GPUCCCapability, "unknown"))
|
|
||||||
fmt.Fprintf(&b, " CC State (current) : %s\n", nonEmptyOr(status.CCState, "unknown"))
|
|
||||||
fmt.Fprintf(&b, " Multi-GPU Mode : %s\n", nonEmptyOr(status.MultiGPUMode, "unknown"))
|
|
||||||
fmt.Fprintf(&b, " CC GPUs Ready State : %s\n", nonEmptyOr(status.CCGPUsReadyState, "unknown"))
|
|
||||||
fmt.Fprintf(&b, " Can run CC : %t\n\n", status.GPUCanRunCC)
|
|
||||||
|
|
||||||
if len(status.Notes) > 0 {
|
|
||||||
b.WriteString("-- Notes ----------------------------------------------------------------------\n")
|
|
||||||
for _, n := range status.Notes {
|
|
||||||
fmt.Fprintf(&b, " - %s\n", n)
|
|
||||||
}
|
|
||||||
b.WriteString("\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Fprintf(&b, "Collected : %s\n", status.CollectedAt.Format("2006-01-02 15:04:05 UTC"))
|
|
||||||
b.WriteString(line + "\n")
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func nonEmptyOr(v, fallback string) string {
|
|
||||||
if strings.TrimSpace(v) == "" {
|
|
||||||
return fallback
|
|
||||||
}
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,540 @@
|
|||||||
|
package platform
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RunNvidiaConfigCheckPack runs a read-only check of NVIDIA GPU
|
||||||
|
// configuration and NVLink topology that DCGM diag does not cover: DCGM's
|
||||||
|
// diagnostic levels stress-test compute/memory/power/bandwidth, but never
|
||||||
|
// assert that persistent config (ECC, MIG, power limit) matches the factory
|
||||||
|
// default, and NVIDIA's own DGX BasePOD deployment guide calls out
|
||||||
|
// topology/NVLink validation as a distinct manual step outside DCGM diag.
|
||||||
|
// Confidential Computing readiness — previously its own standalone SAT
|
||||||
|
// target — is folded in here as one more read-only fact about GPU state
|
||||||
|
// rather than a dedicated test, since none of these are stress tests; they
|
||||||
|
// are one-shot state/config reads that all answer "is this GPU configured
|
||||||
|
// the way we expect", not "does it survive load".
|
||||||
|
func (s *System) RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||||
|
if ctx == nil {
|
||||||
|
ctx = context.Background()
|
||||||
|
}
|
||||||
|
if baseDir == "" {
|
||||||
|
baseDir = "/var/log/bee-sat"
|
||||||
|
}
|
||||||
|
ts := time.Now().UTC().Format("20060102-150405")
|
||||||
|
runDir := filepath.Join(baseDir, "nvidia-config-"+ts)
|
||||||
|
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||||
|
|
||||||
|
status := NvidiaConfigCheckStatus{CollectedAt: time.Now().UTC()}
|
||||||
|
|
||||||
|
// -- GPU config compliance (ECC/MIG/power limit) -----------------------
|
||||||
|
if settings, err := s.ListNvidiaGPUSettings(); err != nil {
|
||||||
|
status.Notes = append(status.Notes, "nvidia-smi GPU settings unavailable (no driver, or no GPU present): "+err.Error())
|
||||||
|
} else {
|
||||||
|
for _, g := range settings {
|
||||||
|
f := NvidiaGPUConfigFinding{
|
||||||
|
Index: g.Index, Name: g.Name,
|
||||||
|
ECCCurrent: g.ECCCurrent, MIGCurrent: g.MIGCurrent, MIGPending: g.MIGPending,
|
||||||
|
PowerLimitW: g.PowerLimitW, PowerDefaultLimitW: g.PowerDefaultLimitW,
|
||||||
|
Issues: evaluateNvidiaGPUConfig(g),
|
||||||
|
}
|
||||||
|
status.GPUs = append(status.GPUs, f)
|
||||||
|
for _, issue := range f.Issues {
|
||||||
|
status.Warnings = append(status.Warnings, fmt.Sprintf("GPU %d (%s): %s", g.Index, g.Name, issue))
|
||||||
|
}
|
||||||
|
if strings.EqualFold(g.CCState, "on") {
|
||||||
|
status.CCState = "ON"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- NVLink topology (bonded pairs, active links, error counters) ------
|
||||||
|
topoOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-topo-m", []string{"nvidia-smi", "topo", "-m"}, nil, logFunc)
|
||||||
|
_ = os.WriteFile(filepath.Join(runDir, "01-nvidia-smi-topo-m.log"), topoOut, 0644)
|
||||||
|
statusOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-s", []string{"nvidia-smi", "nvlink", "-s"}, nil, logFunc)
|
||||||
|
_ = os.WriteFile(filepath.Join(runDir, "02-nvidia-smi-nvlink-s.log"), statusOut, 0644)
|
||||||
|
errOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-e", []string{"nvidia-smi", "nvlink", "-e"}, nil, logFunc)
|
||||||
|
_ = os.WriteFile(filepath.Join(runDir, "03-nvidia-smi-nvlink-e.log"), errOut, 0644)
|
||||||
|
|
||||||
|
pairs := parseNvidiaNVLinkBondedPairs(string(topoOut))
|
||||||
|
linkStatus := parseNvidiaNVLinkStatus(string(statusOut))
|
||||||
|
linkErrors := parseNvidiaNVLinkErrors(string(errOut))
|
||||||
|
for _, pair := range pairs {
|
||||||
|
f := evaluateNvidiaNVLinkPair(pair, linkStatus, linkErrors)
|
||||||
|
status.NVLinkPairs = append(status.NVLinkPairs, f)
|
||||||
|
for _, issue := range f.Issues {
|
||||||
|
status.Warnings = append(status.Warnings, fmt.Sprintf("NVLink GPU%d<->GPU%d: %s", f.GPUA, f.GPUB, issue))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- Confidential Computing readiness (informational only) -------------
|
||||||
|
ccOut, ccErr := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-conf-compute-q", []string{"nvidia-smi", "conf-compute", "-q"}, nil, logFunc)
|
||||||
|
_ = os.WriteFile(filepath.Join(runDir, "04-nvidia-smi-conf-compute-q.log"), ccOut, 0644)
|
||||||
|
if ccErr == nil {
|
||||||
|
fields := parseConfComputeFields(ccOut)
|
||||||
|
if status.CCState == "" {
|
||||||
|
status.CCState = fields["CC State"]
|
||||||
|
}
|
||||||
|
status.CPUCCCapability = fields["CPU CC Capabilities"]
|
||||||
|
status.GPUCCCapability = fields["GPU CC Capabilities"]
|
||||||
|
}
|
||||||
|
dmesgOut, _ := runSATCommandCtx(ctx, verboseLog, "dmesg", []string{"dmesg"}, nil, nil)
|
||||||
|
ccDmesgLines := filterConfComputeDmesgLines(dmesgOut)
|
||||||
|
_ = os.WriteFile(filepath.Join(runDir, "05-dmesg-cc-relevant.log"), []byte(strings.Join(ccDmesgLines, "\n")+"\n"), 0644)
|
||||||
|
lowerDmesg := strings.ToLower(strings.Join(ccDmesgLines, "\n"))
|
||||||
|
status.HostAMDSEVSNPActive = strings.Contains(lowerDmesg, "sev-snp enabled")
|
||||||
|
status.HostIntelTDXActive = strings.Contains(lowerDmesg, "tdx module") && strings.Contains(lowerDmesg, "module initialized") ||
|
||||||
|
strings.Contains(lowerDmesg, "virt/tdx: module initialized")
|
||||||
|
for _, path := range []string{"/sys/module/kvm_amd/parameters/sev_snp"} {
|
||||||
|
if out, err := runSATCommandCtx(ctx, verboseLog, "sysfs-sev-snp", []string{"cat", path}, nil, nil); err == nil &&
|
||||||
|
strings.EqualFold(strings.TrimSpace(string(out)), "Y") {
|
||||||
|
status.HostAMDSEVSNPActive = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
gpuCanRunCC := strings.EqualFold(strings.TrimSpace(status.GPUCCCapability), "CC Capable")
|
||||||
|
cpuCapReported := strings.TrimSpace(status.CPUCCCapability)
|
||||||
|
cpuCanRunCC := status.HostAMDSEVSNPActive || status.HostIntelTDXActive ||
|
||||||
|
(cpuCapReported != "" && !strings.EqualFold(cpuCapReported, "NONE"))
|
||||||
|
status.CCReady = cpuCanRunCC && gpuCanRunCC
|
||||||
|
|
||||||
|
summary := renderNvidiaConfigCheckSummary(status)
|
||||||
|
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
report := renderNvidiaConfigCheckReport(status)
|
||||||
|
if err := os.WriteFile(filepath.Join(runDir, "nvidia-config-report.txt"), []byte(report), 0644); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return runDir, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NvidiaConfigCheckStatus is the result of RunNvidiaConfigCheckPack.
|
||||||
|
type NvidiaConfigCheckStatus struct {
|
||||||
|
CollectedAt time.Time `json:"collected_at"`
|
||||||
|
|
||||||
|
GPUs []NvidiaGPUConfigFinding `json:"gpus,omitempty"`
|
||||||
|
NVLinkPairs []NvidiaNVLinkPairFinding `json:"nvlink_pairs,omitempty"`
|
||||||
|
|
||||||
|
// Confidential Computing readiness — informational only, does not gate
|
||||||
|
// overall_status: an unconfigured/NOT_READY CC state is a
|
||||||
|
// feature-readiness fact on most fleets (CC is opt-in), not a fault.
|
||||||
|
CCReady bool `json:"cc_ready"`
|
||||||
|
CCState string `json:"cc_state,omitempty"`
|
||||||
|
CPUCCCapability string `json:"cpu_cc_capability,omitempty"`
|
||||||
|
GPUCCCapability string `json:"gpu_cc_capability,omitempty"`
|
||||||
|
|
||||||
|
HostAMDSEVSNPActive bool `json:"host_amd_sev_snp_active"`
|
||||||
|
HostIntelTDXActive bool `json:"host_intel_tdx_active"`
|
||||||
|
|
||||||
|
// Warnings drive overall_status: any entry here fails the check.
|
||||||
|
Warnings []string `json:"warnings,omitempty"`
|
||||||
|
Notes []string `json:"notes,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NvidiaGPUConfigFinding is one GPU's config-compliance result.
|
||||||
|
type NvidiaGPUConfigFinding struct {
|
||||||
|
Index int `json:"index"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
ECCCurrent string `json:"ecc_current"`
|
||||||
|
MIGCurrent string `json:"mig_current"`
|
||||||
|
MIGPending string `json:"mig_pending"`
|
||||||
|
PowerLimitW float64 `json:"power_limit_w"`
|
||||||
|
PowerDefaultLimitW float64 `json:"power_default_limit_w"`
|
||||||
|
Issues []string `json:"issues,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NvidiaNVLinkPairFinding is one NVLink-bonded GPU pair's link-count/error
|
||||||
|
// validation result.
|
||||||
|
type NvidiaNVLinkPairFinding struct {
|
||||||
|
GPUA int `json:"gpu_a"`
|
||||||
|
GPUB int `json:"gpu_b"`
|
||||||
|
ExpectedLinks int `json:"expected_links"` // from "nvidia-smi topo -m"'s NVx cell
|
||||||
|
ActiveLinks int `json:"active_links"` // from "nvidia-smi nvlink -s"
|
||||||
|
TotalLinks int `json:"total_links"`
|
||||||
|
ErrorLinks int `json:"error_links"` // links with a nonzero replay/recovery/CRC counter
|
||||||
|
Issues []string `json:"issues,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// evaluateNvidiaGPUConfig flags config drift that DCGM diag does not check:
|
||||||
|
// ECC disabled (NVIDIA data-center GPUs ship with ECC on), a MIG mode
|
||||||
|
// change stuck pending a reset/reboot, and a power limit capped meaningfully
|
||||||
|
// below the card's own factory default (a silent throttle that a stress
|
||||||
|
// test would only surface as "lower than expected performance", with no
|
||||||
|
// clear cause).
|
||||||
|
func evaluateNvidiaGPUConfig(g NvidiaGPUSetting) []string {
|
||||||
|
var issues []string
|
||||||
|
if strings.EqualFold(g.ECCCurrent, "disabled") {
|
||||||
|
issues = append(issues, "ECC is disabled")
|
||||||
|
}
|
||||||
|
if g.MIGCurrent != "" && g.MIGPending != "" && !strings.EqualFold(g.MIGCurrent, g.MIGPending) {
|
||||||
|
issues = append(issues, fmt.Sprintf("MIG mode change pending (current=%s, pending=%s) — apply with a GPU reset or reboot", g.MIGCurrent, g.MIGPending))
|
||||||
|
}
|
||||||
|
if g.PowerDefaultLimitW > 0 && g.PowerLimitW > 0 {
|
||||||
|
deficit := (g.PowerDefaultLimitW - g.PowerLimitW) / g.PowerDefaultLimitW
|
||||||
|
if deficit > 0.05 {
|
||||||
|
issues = append(issues, fmt.Sprintf("power limit %.0fW is %.0f%% below factory default %.0fW", g.PowerLimitW, deficit*100, g.PowerDefaultLimitW))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return issues
|
||||||
|
}
|
||||||
|
|
||||||
|
// evaluateNvidiaNVLinkPair compares a bonded pair's actual link state
|
||||||
|
// against what the topology matrix says should be there.
|
||||||
|
func evaluateNvidiaNVLinkPair(pair nvidiaNVLinkBondedPair, status map[int][]nvidiaNVLinkPort, errors map[int]map[int][3]int64) NvidiaNVLinkPairFinding {
|
||||||
|
f := NvidiaNVLinkPairFinding{GPUA: pair.gpuA, GPUB: pair.gpuB, ExpectedLinks: pair.links}
|
||||||
|
|
||||||
|
active, total, errLinks := 0, 0, 0
|
||||||
|
for _, gpu := range []int{pair.gpuA, pair.gpuB} {
|
||||||
|
for _, port := range status[gpu] {
|
||||||
|
total++
|
||||||
|
if port.active {
|
||||||
|
active++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, counters := range errors[gpu] {
|
||||||
|
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
|
||||||
|
errLinks++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.ActiveLinks, f.TotalLinks, f.ErrorLinks = active, total, errLinks
|
||||||
|
|
||||||
|
if total > 0 && active < total {
|
||||||
|
f.Issues = append(f.Issues, fmt.Sprintf("%d/%d NVLinks inactive on a bonded pair", total-active, total))
|
||||||
|
}
|
||||||
|
if errLinks > 0 {
|
||||||
|
f.Issues = append(f.Issues, fmt.Sprintf("%d link(s) reporting replay/recovery/CRC errors", errLinks))
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// NVLink parsing — deliberately self-contained rather than importing
|
||||||
|
// internal/collector or internal/webui's equivalent parsers (isNVLinkBridgeCandidate
|
||||||
|
// et al. and parseGPUPairAdjacency/parseTopoNVLinkStatus/parseTopoNVLinkErrors,
|
||||||
|
// respectively), matching this codebase's existing precedent of small,
|
||||||
|
// package-local duplication over cross-package coupling for narrow parsing
|
||||||
|
// helpers (see webui/page_topo.go's isNICDeviceClassDev/isRAIDControllerClass).
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
type nvidiaNVLinkBondedPair struct {
|
||||||
|
gpuA, gpuB int
|
||||||
|
links int
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
nvidiaNVLinkANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]")
|
||||||
|
nvidiaNVLinkNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// parseNvidiaNVLinkBondedPairs returns every GPU pair with a nonzero NVLink
|
||||||
|
// bond count from a "nvidia-smi topo -m" matrix, deduplicated (A,B) == (B,A).
|
||||||
|
func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair {
|
||||||
|
lines := strings.Split(nvidiaNVLinkANSIRe.ReplaceAllString(raw, ""), "\n")
|
||||||
|
headerIdx := -1
|
||||||
|
var gpuColIndices []int
|
||||||
|
for i, line := range lines {
|
||||||
|
trimmed := strings.TrimSpace(line)
|
||||||
|
if strings.HasPrefix(trimmed, "GPU0") {
|
||||||
|
parts := strings.Fields(trimmed)
|
||||||
|
for j, col := range parts {
|
||||||
|
if strings.HasPrefix(col, "GPU") {
|
||||||
|
gpuColIndices = append(gpuColIndices, j)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(gpuColIndices) >= 2 {
|
||||||
|
headerIdx = i
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if headerIdx < 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
colIdxToGPU := make(map[int]int, len(gpuColIndices))
|
||||||
|
for gpuIdx, colIdx := range gpuColIndices {
|
||||||
|
colIdxToGPU[colIdx] = gpuIdx
|
||||||
|
}
|
||||||
|
|
||||||
|
seen := map[[2]int]bool{}
|
||||||
|
var pairs []nvidiaNVLinkBondedPair
|
||||||
|
for _, line := range lines[headerIdx+1:] {
|
||||||
|
trimmed := strings.TrimSpace(line)
|
||||||
|
if !strings.HasPrefix(trimmed, "GPU") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cells := strings.Fields(trimmed)
|
||||||
|
if len(cells) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rowGPU, err := strconv.Atoi(strings.TrimPrefix(cells[0], "GPU"))
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for colIdx, colGPU := range colIdxToGPU {
|
||||||
|
if colGPU == rowGPU {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
dataIdx := colIdx + 1
|
||||||
|
if dataIdx >= len(cells) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m := nvidiaNVLinkNVRe.FindStringSubmatch(cells[dataIdx])
|
||||||
|
if len(m) != 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
nv, err := strconv.Atoi(m[1])
|
||||||
|
if err != nil || nv <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
a, b := rowGPU, colGPU
|
||||||
|
if a > b {
|
||||||
|
a, b = b, a
|
||||||
|
}
|
||||||
|
key := [2]int{a, b}
|
||||||
|
if seen[key] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[key] = true
|
||||||
|
pairs = append(pairs, nvidiaNVLinkBondedPair{gpuA: a, gpuB: b, links: nv})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return pairs
|
||||||
|
}
|
||||||
|
|
||||||
|
type nvidiaNVLinkPort struct {
|
||||||
|
active bool
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
nvidiaNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
|
||||||
|
nvidiaNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*[\d.]+\s*GB/s`)
|
||||||
|
nvidiaNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
|
||||||
|
nvidiaNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// parseNvidiaNVLinkStatus parses "nvidia-smi nvlink -s" output into per-GPU
|
||||||
|
// per-link active/inactive state.
|
||||||
|
func parseNvidiaNVLinkStatus(raw string) map[int][]nvidiaNVLinkPort {
|
||||||
|
result := map[int][]nvidiaNVLinkPort{}
|
||||||
|
currentGPU := -1
|
||||||
|
for _, line := range strings.Split(raw, "\n") {
|
||||||
|
trimmed := strings.TrimSpace(line)
|
||||||
|
if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||||||
|
currentGPU, _ = strconv.Atoi(m[1])
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if currentGPU < 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if nvidiaNVLinkInactiveRe.MatchString(trimmed) {
|
||||||
|
result[currentGPU] = append(result[currentGPU], nvidiaNVLinkPort{active: false})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if nvidiaNVLinkSpeedLineRe.MatchString(trimmed) {
|
||||||
|
result[currentGPU] = append(result[currentGPU], nvidiaNVLinkPort{active: true})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseNvidiaNVLinkErrors parses "nvidia-smi nvlink -e" output into, per GPU
|
||||||
|
// then link index, [replay, recovery, crc] error counts.
|
||||||
|
func parseNvidiaNVLinkErrors(raw string) map[int]map[int][3]int64 {
|
||||||
|
result := map[int]map[int][3]int64{}
|
||||||
|
currentGPU := -1
|
||||||
|
for _, line := range strings.Split(raw, "\n") {
|
||||||
|
trimmed := strings.TrimSpace(line)
|
||||||
|
if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||||||
|
currentGPU, _ = strconv.Atoi(m[1])
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if currentGPU < 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m := nvidiaNVLinkErrorLineRe.FindStringSubmatch(trimmed)
|
||||||
|
if m == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
linkIdx, _ := strconv.Atoi(m[1])
|
||||||
|
count, _ := strconv.ParseInt(m[3], 10, 64)
|
||||||
|
if result[currentGPU] == nil {
|
||||||
|
result[currentGPU] = map[int][3]int64{}
|
||||||
|
}
|
||||||
|
c := result[currentGPU][linkIdx]
|
||||||
|
switch m[2] {
|
||||||
|
case "Replay":
|
||||||
|
c[0] = count
|
||||||
|
case "Recovery":
|
||||||
|
c[1] = count
|
||||||
|
case "CRC":
|
||||||
|
c[2] = count
|
||||||
|
}
|
||||||
|
result[currentGPU][linkIdx] = c
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Confidential Computing (folded in — see RunNvidiaConfigCheckPack doc)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// parseConfComputeFields parses the indented "Key : Value" block emitted by
|
||||||
|
// `nvidia-smi conf-compute -q`, e.g.:
|
||||||
|
//
|
||||||
|
// CC State : OFF
|
||||||
|
// Multi-GPU Mode : Protected PCIe
|
||||||
|
// CPU CC Capabilities : INTEL TDX
|
||||||
|
// GPU CC Capabilities : CC Capable
|
||||||
|
// CC GPUs Ready State : Not Ready
|
||||||
|
func parseConfComputeFields(out []byte) map[string]string {
|
||||||
|
fields := map[string]string{}
|
||||||
|
for _, line := range strings.Split(string(out), "\n") {
|
||||||
|
idx := strings.Index(line, ":")
|
||||||
|
if idx < 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := strings.TrimSpace(line[:idx])
|
||||||
|
val := strings.TrimSpace(line[idx+1:])
|
||||||
|
if key == "" || val == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fields[key] = val
|
||||||
|
}
|
||||||
|
return fields
|
||||||
|
}
|
||||||
|
|
||||||
|
// filterConfComputeDmesgLines returns the dmesg lines relevant to CPU
|
||||||
|
// Confidential Computing support (AMD SEV/SEV-ES/SEV-SNP, Intel TDX).
|
||||||
|
func filterConfComputeDmesgLines(dmesgOut []byte) []string {
|
||||||
|
var lines []string
|
||||||
|
for _, raw := range bytes.Split(dmesgOut, []byte("\n")) {
|
||||||
|
lower := strings.ToLower(string(raw))
|
||||||
|
if strings.Contains(lower, "sev") || strings.Contains(lower, "tdx") {
|
||||||
|
lines = append(lines, string(raw))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return lines
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Report rendering
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func renderNvidiaConfigCheckSummary(status NvidiaConfigCheckStatus) string {
|
||||||
|
var b strings.Builder
|
||||||
|
fmt.Fprintf(&b, "run_at_utc=%s\n", status.CollectedAt.Format(time.RFC3339))
|
||||||
|
fmt.Fprintf(&b, "gpu_count=%d\n", len(status.GPUs))
|
||||||
|
eccIssues, migIssues, powerIssues := 0, 0, 0
|
||||||
|
for _, g := range status.GPUs {
|
||||||
|
for _, issue := range g.Issues {
|
||||||
|
switch {
|
||||||
|
case strings.Contains(issue, "ECC"):
|
||||||
|
eccIssues++
|
||||||
|
case strings.Contains(issue, "MIG"):
|
||||||
|
migIssues++
|
||||||
|
case strings.Contains(issue, "power limit"):
|
||||||
|
powerIssues++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "gpu_ecc_issues=%d\n", eccIssues)
|
||||||
|
fmt.Fprintf(&b, "gpu_mig_pending_issues=%d\n", migIssues)
|
||||||
|
fmt.Fprintf(&b, "gpu_power_limit_issues=%d\n", powerIssues)
|
||||||
|
fmt.Fprintf(&b, "nvlink_pairs_checked=%d\n", len(status.NVLinkPairs))
|
||||||
|
nvlinkIssues := 0
|
||||||
|
for _, p := range status.NVLinkPairs {
|
||||||
|
if len(p.Issues) > 0 {
|
||||||
|
nvlinkIssues++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "nvlink_pairs_with_issues=%d\n", nvlinkIssues)
|
||||||
|
fmt.Fprintf(&b, "cc_ready=%t\n", status.CCReady)
|
||||||
|
fmt.Fprintf(&b, "cc_state=%s\n", status.CCState)
|
||||||
|
fmt.Fprintf(&b, "cpu_cc_capability=%s\n", status.CPUCCCapability)
|
||||||
|
fmt.Fprintf(&b, "gpu_cc_capability=%s\n", status.GPUCCCapability)
|
||||||
|
if len(status.Warnings) == 0 {
|
||||||
|
fmt.Fprintln(&b, "overall_status=OK")
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(&b, "overall_status=FAILED")
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderNvidiaConfigCheckReport(status NvidiaConfigCheckStatus) string {
|
||||||
|
var b strings.Builder
|
||||||
|
line := strings.Repeat("=", 80)
|
||||||
|
b.WriteString(line + "\n")
|
||||||
|
b.WriteString("NVIDIA GPU Configuration & NVLink Topology Check\n")
|
||||||
|
b.WriteString(line + "\n\n")
|
||||||
|
|
||||||
|
b.WriteString("-- GPU Configuration ----------------------------------------------------------\n")
|
||||||
|
for _, g := range status.GPUs {
|
||||||
|
verdict := "OK"
|
||||||
|
if len(g.Issues) > 0 {
|
||||||
|
verdict = "ISSUES FOUND"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, " GPU %d (%s): %s\n", g.Index, g.Name, verdict)
|
||||||
|
fmt.Fprintf(&b, " ECC: %s | MIG: %s (pending %s) | Power: %.0fW / %.0fW default\n",
|
||||||
|
nonEmptyOr(g.ECCCurrent, "unknown"), nonEmptyOr(g.MIGCurrent, "unknown"), nonEmptyOr(g.MIGPending, "unknown"),
|
||||||
|
g.PowerLimitW, g.PowerDefaultLimitW)
|
||||||
|
for _, issue := range g.Issues {
|
||||||
|
fmt.Fprintf(&b, " - %s\n", issue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.WriteString("\n-- NVLink Topology --------------------------------------------------------------\n")
|
||||||
|
if len(status.NVLinkPairs) == 0 {
|
||||||
|
b.WriteString(" No NVLink-bonded GPU pairs found.\n")
|
||||||
|
}
|
||||||
|
for _, p := range status.NVLinkPairs {
|
||||||
|
verdict := "OK"
|
||||||
|
if len(p.Issues) > 0 {
|
||||||
|
verdict = "ISSUES FOUND"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, " GPU%d <-> GPU%d: %d/%d links active (topology expects %d): %s\n",
|
||||||
|
p.GPUA, p.GPUB, p.ActiveLinks, p.TotalLinks, p.ExpectedLinks, verdict)
|
||||||
|
for _, issue := range p.Issues {
|
||||||
|
fmt.Fprintf(&b, " - %s\n", issue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
b.WriteString("\n-- Confidential Computing (informational) ---------------------------------------\n")
|
||||||
|
verdict := "NOT READY"
|
||||||
|
if status.CCReady {
|
||||||
|
verdict = "READY"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, " Verdict: %s\n", verdict)
|
||||||
|
fmt.Fprintf(&b, " CC State: %s | CPU CC: %s | GPU CC: %s\n",
|
||||||
|
nonEmptyOr(status.CCState, "unknown"), nonEmptyOr(status.CPUCCCapability, "unknown"), nonEmptyOr(status.GPUCCCapability, "unknown"))
|
||||||
|
|
||||||
|
if len(status.Notes) > 0 {
|
||||||
|
b.WriteString("\n-- Notes -------------------------------------------------------------------------\n")
|
||||||
|
for _, n := range status.Notes {
|
||||||
|
fmt.Fprintf(&b, " - %s\n", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(&b, "\nCollected : %s\n", status.CollectedAt.Format("2006-01-02 15:04:05 UTC"))
|
||||||
|
b.WriteString(line + "\n")
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func nonEmptyOr(v, fallback string) string {
|
||||||
|
if strings.TrimSpace(v) == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
@@ -0,0 +1,168 @@
|
|||||||
|
package platform
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseNvidiaNVLinkBondedPairsRealTwoGPUDump(t *testing.T) {
|
||||||
|
// Real system/nvidia-smi-topo.txt shape: two H100s directly bridged
|
||||||
|
// (NV17).
|
||||||
|
input := "\tGPU0\tGPU1\tNIC0\tNIC1\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\n" +
|
||||||
|
"GPU0\t X \tNV17\tSYS\tSYS\t0-23,48-71\t0\t\tN/A\n" +
|
||||||
|
"GPU1\tNV17\t X \tNODE\tNODE\t24-47,72-95\t1\t\tN/A\n" +
|
||||||
|
"NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" +
|
||||||
|
"NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n"
|
||||||
|
|
||||||
|
pairs := parseNvidiaNVLinkBondedPairs(input)
|
||||||
|
if len(pairs) != 1 {
|
||||||
|
t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
|
||||||
|
}
|
||||||
|
if pairs[0].gpuA != 0 || pairs[0].gpuB != 1 || pairs[0].links != 17 {
|
||||||
|
t.Fatalf("pair=%#v want {0,1,17}", pairs[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseNvidiaNVLinkBondedPairsANSIUnderlinedHeader(t *testing.T) {
|
||||||
|
// nvidia-smi underlines the topo -m header with ANSI CSI codes even when
|
||||||
|
// writing to a file — the same real-world quirk fixed for the /topo page
|
||||||
|
// parser must be handled here too, since this uses a fresh live query
|
||||||
|
// rather than reading the persisted techdump.
|
||||||
|
input := "\x1b[4m\tGPU0\tGPU1\tGPU2\tGPU3\tCPU Affinity\x1b[0m\n" +
|
||||||
|
"GPU0\t X \tNV18\tPIX\tPIX\t0-31,64-95\n" +
|
||||||
|
"GPU1\tNV18\t X \tPIX\tPIX\t0-31,64-95\n" +
|
||||||
|
"GPU2\tPIX\tPIX\t X \tNV18\t0-31,64-95\n" +
|
||||||
|
"GPU3\tPIX\tPIX\tNV18\t X \t0-31,64-95\n"
|
||||||
|
|
||||||
|
pairs := parseNvidiaNVLinkBondedPairs(input)
|
||||||
|
if len(pairs) != 2 {
|
||||||
|
t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs)
|
||||||
|
}
|
||||||
|
want := map[[2]int]int{{0, 1}: 18, {2, 3}: 18}
|
||||||
|
for _, p := range pairs {
|
||||||
|
if want[[2]int{p.gpuA, p.gpuB}] != p.links {
|
||||||
|
t.Fatalf("unexpected pair %#v", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseNvidiaNVLinkStatusMarksInactiveLinks(t *testing.T) {
|
||||||
|
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
||||||
|
Link 0: 26.562 GB/s
|
||||||
|
Link 1: <inactive>
|
||||||
|
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
|
||||||
|
Link 0: 26.562 GB/s
|
||||||
|
`
|
||||||
|
got := parseNvidiaNVLinkStatus(input)
|
||||||
|
if len(got[0]) != 2 || got[0][0].active != true || got[0][1].active != false {
|
||||||
|
t.Fatalf("gpu0=%#v want [active, inactive]", got[0])
|
||||||
|
}
|
||||||
|
if len(got[1]) != 1 || !got[1][0].active {
|
||||||
|
t.Fatalf("gpu1=%#v want [active]", got[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseNvidiaNVLinkErrors(t *testing.T) {
|
||||||
|
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
||||||
|
Link 0: Replay Errors: 0
|
||||||
|
Link 0: Recovery Errors: 0
|
||||||
|
Link 0: CRC Errors: 0
|
||||||
|
Link 1: Replay Errors: 3
|
||||||
|
Link 1: Recovery Errors: 1
|
||||||
|
Link 1: CRC Errors: 2
|
||||||
|
`
|
||||||
|
got := parseNvidiaNVLinkErrors(input)
|
||||||
|
c := got[0][1]
|
||||||
|
if c[0] != 3 || c[1] != 1 || c[2] != 2 {
|
||||||
|
t.Fatalf("link1 counters=%#v want {3,1,2}", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluateNvidiaNVLinkPairFlagsInactiveLinksAndErrors(t *testing.T) {
|
||||||
|
pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18}
|
||||||
|
status := map[int][]nvidiaNVLinkPort{
|
||||||
|
0: {{active: true}, {active: false}},
|
||||||
|
1: {{active: true}, {active: true}},
|
||||||
|
}
|
||||||
|
errors := map[int]map[int][3]int64{
|
||||||
|
0: {1: {3, 0, 0}},
|
||||||
|
}
|
||||||
|
|
||||||
|
f := evaluateNvidiaNVLinkPair(pair, status, errors)
|
||||||
|
if f.ActiveLinks != 3 || f.TotalLinks != 4 {
|
||||||
|
t.Fatalf("active=%d total=%d want 3/4", f.ActiveLinks, f.TotalLinks)
|
||||||
|
}
|
||||||
|
if len(f.Issues) != 2 {
|
||||||
|
t.Fatalf("issues=%#v want 2 (inactive link + error)", f.Issues)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluateNvidiaNVLinkPairHealthyBondHasNoIssues(t *testing.T) {
|
||||||
|
pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18}
|
||||||
|
status := map[int][]nvidiaNVLinkPort{
|
||||||
|
0: {{active: true}},
|
||||||
|
1: {{active: true}},
|
||||||
|
}
|
||||||
|
f := evaluateNvidiaNVLinkPair(pair, status, nil)
|
||||||
|
if len(f.Issues) != 0 {
|
||||||
|
t.Fatalf("issues=%#v want none for a fully active, error-free bond", f.Issues)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluateNvidiaGPUConfigFlagsECCDisabled(t *testing.T) {
|
||||||
|
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Disabled"}
|
||||||
|
issues := evaluateNvidiaGPUConfig(g)
|
||||||
|
if len(issues) != 1 {
|
||||||
|
t.Fatalf("issues=%#v want 1 (ECC disabled)", issues)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluateNvidiaGPUConfigFlagsStuckMIGPending(t *testing.T) {
|
||||||
|
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", MIGCurrent: "Disabled", MIGPending: "Enabled"}
|
||||||
|
issues := evaluateNvidiaGPUConfig(g)
|
||||||
|
if len(issues) != 1 {
|
||||||
|
t.Fatalf("issues=%#v want 1 (MIG pending != current)", issues)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluateNvidiaGPUConfigFlagsPowerLimitBelowDefault(t *testing.T) {
|
||||||
|
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", PowerLimitW: 300, PowerDefaultLimitW: 700}
|
||||||
|
issues := evaluateNvidiaGPUConfig(g)
|
||||||
|
if len(issues) != 1 {
|
||||||
|
t.Fatalf("issues=%#v want 1 (power limit far below default)", issues)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluateNvidiaGPUConfigNominalHasNoIssues(t *testing.T) {
|
||||||
|
g := NvidiaGPUSetting{
|
||||||
|
Index: 0, Name: "H100", ECCCurrent: "Enabled",
|
||||||
|
MIGCurrent: "Disabled", MIGPending: "Disabled",
|
||||||
|
PowerLimitW: 700, PowerDefaultLimitW: 700,
|
||||||
|
}
|
||||||
|
issues := evaluateNvidiaGPUConfig(g)
|
||||||
|
if len(issues) != 0 {
|
||||||
|
t.Fatalf("issues=%#v want none for nominal config", issues)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluateNvidiaGPUConfigSmallPowerLimitDeviationIsNotFlagged(t *testing.T) {
|
||||||
|
// Power limits set slightly below default (e.g. rounding, minor
|
||||||
|
// operator tuning) should not be treated as an anomaly — only a
|
||||||
|
// meaningful cap (>5%) is.
|
||||||
|
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", PowerLimitW: 690, PowerDefaultLimitW: 700}
|
||||||
|
issues := evaluateNvidiaGPUConfig(g)
|
||||||
|
if len(issues) != 0 {
|
||||||
|
t.Fatalf("issues=%#v want none for a <2%% power-limit deviation", issues)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderNvidiaConfigCheckSummaryOverallStatus(t *testing.T) {
|
||||||
|
clean := NvidiaConfigCheckStatus{}
|
||||||
|
if got := renderNvidiaConfigCheckSummary(clean); !strings.Contains(got, "overall_status=OK") {
|
||||||
|
t.Fatalf("clean status summary missing overall_status=OK:\n%s", got)
|
||||||
|
}
|
||||||
|
withWarning := NvidiaConfigCheckStatus{Warnings: []string{"GPU 0: ECC is disabled"}}
|
||||||
|
if got := renderNvidiaConfigCheckSummary(withWarning); !strings.Contains(got, "overall_status=FAILED") {
|
||||||
|
t.Fatalf("status with warnings missing overall_status=FAILED:\n%s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -70,7 +70,7 @@ func (s *System) ListNvidiaGPUSettings() ([]NvidiaGPUSetting, error) {
|
|||||||
|
|
||||||
// queryNvidiaGPUCCState reads the per-GPU Confidential Computing state via
|
// queryNvidiaGPUCCState reads the per-GPU Confidential Computing state via
|
||||||
// `nvidia-smi conf-compute -i <index> -q`, reusing the same field-name
|
// `nvidia-smi conf-compute -i <index> -q`, reusing the same field-name
|
||||||
// convention as RunConfidentialComputingCheckPack's parseConfComputeFields.
|
// convention as RunNvidiaConfigCheckPack's parseConfComputeFields.
|
||||||
// Returns "" if the driver/GPU doesn't support conf-compute at all.
|
// Returns "" if the driver/GPU doesn't support conf-compute at all.
|
||||||
func (s *System) queryNvidiaGPUCCState(index int) string {
|
func (s *System) queryNvidiaGPUCCState(index int) string {
|
||||||
out, err := satExecCommand("nvidia-smi", "conf-compute", "-i", strconv.Itoa(index), "-q").Output()
|
out, err := satExecCommand("nvidia-smi", "conf-compute", "-i", strconv.Itoa(index), "-q").Output()
|
||||||
|
|||||||
@@ -135,7 +135,7 @@ func defaultTaskPriority(target string, params taskParams) int {
|
|||||||
return taskPriorityBurn
|
return taskPriorityBurn
|
||||||
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
|
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
|
||||||
"nvidia-interconnect", "nvidia-bandwidth", "memory", "storage", "cpu",
|
"nvidia-interconnect", "nvidia-bandwidth", "memory", "storage", "cpu",
|
||||||
"amd", "amd-mem", "amd-bandwidth", "confidential-computing":
|
"amd", "amd-mem", "amd-bandwidth", "nvidia-config":
|
||||||
if params.StressMode {
|
if params.StressMode {
|
||||||
return taskPriorityValidateStress
|
return taskPriorityValidateStress
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -676,10 +676,10 @@ func renderCheck(opts HandlerOptions) string {
|
|||||||
`<code>lsblk</code>; NVMe: <code>nvme id-ctrl</code>, <code>nvme smart-log</code>; SATA/SAS: <code>smartctl -H -A</code>`,
|
`<code>lsblk</code>; NVMe: <code>nvme id-ctrl</code>, <code>nvme smart-log</code>; SATA/SAS: <code>smartctl -H -A</code>`,
|
||||||
`Seconds — instantaneous device query, no wear counters incremented.`,
|
`Seconds — instantaneous device query, no wear counters incremented.`,
|
||||||
)) +
|
)) +
|
||||||
renderSATCard("confidential-computing", "Confidential Computing", "runSAT('confidential-computing')", "", renderValidateCardBody(
|
renderSATCard("nvidia-config", "GPU Config & NVLink", "runSAT('nvidia-config')", "", renderValidateCardBody(
|
||||||
inv.NVIDIA,
|
inv.NVIDIA,
|
||||||
`Checks whether this server can run NVIDIA Confidential Computing: CPU TEE support (Intel TDX / AMD SEV-SNP) and GPU firmware CC capability. Read-only — changes nothing.`,
|
`Checks GPU configuration and NVLink topology that DCGM diag does not cover: ECC/MIG/power-limit drift from factory default, NVLink-bonded pair link count and error counters, and (informational) NVIDIA Confidential Computing readiness (CPU TEE support + GPU firmware CC capability). Read-only — changes nothing.`,
|
||||||
`<code>nvidia-smi conf-compute -q</code>, <code>dmesg</code>, <code>/sys/module/kvm_amd/parameters/*</code>`,
|
`<code>nvidia-smi --query-gpu=...</code>, <code>nvidia-smi topo -m</code>, <code>nvidia-smi nvlink -s/-e</code>, <code>nvidia-smi conf-compute -q</code>, <code>dmesg</code>`,
|
||||||
`Seconds — read-only query only.`,
|
`Seconds — read-only query only.`,
|
||||||
)) +
|
)) +
|
||||||
`</div>
|
`</div>
|
||||||
@@ -743,7 +743,7 @@ func renderCheck(opts HandlerOptions) string {
|
|||||||
<script>
|
<script>
|
||||||
let satES = null;
|
let satES = null;
|
||||||
function satLabels() {
|
function satLabels() {
|
||||||
return {nvidia:'Check GPU (DCGM L2)', 'nvidia-interconnect':'NVIDIA Interconnect (NCCL)', 'nvidia-bandwidth':'NVIDIA Bandwidth (NVBandwidth)', memory:'Check Memory', storage:'Check Storage', cpu:'Check CPU', amd:'Check AMD GPU', 'amd-mem':'AMD GPU MEM Integrity', 'amd-bandwidth':'AMD GPU MEM Bandwidth', 'confidential-computing':'Check Confidential Computing'};
|
return {nvidia:'Check GPU (DCGM L2)', 'nvidia-interconnect':'NVIDIA Interconnect (NCCL)', 'nvidia-bandwidth':'NVIDIA Bandwidth (NVBandwidth)', memory:'Check Memory', storage:'Check Storage', cpu:'Check CPU', amd:'Check AMD GPU', 'amd-mem':'AMD GPU MEM Integrity', 'amd-bandwidth':'AMD GPU MEM Bandwidth', 'nvidia-config':'Check GPU Config & NVLink'};
|
||||||
}
|
}
|
||||||
let satNvidiaGPUsPromise = null;
|
let satNvidiaGPUsPromise = null;
|
||||||
function loadSatNvidiaGPUs() {
|
function loadSatNvidiaGPUs() {
|
||||||
@@ -879,7 +879,7 @@ function runAllCheckSAT() {
|
|||||||
status.textContent = 'Enqueuing...';
|
status.textContent = 'Enqueuing...';
|
||||||
const nvidiaIndices = satSelectedGPUIndices();
|
const nvidiaIndices = satSelectedGPUIndices();
|
||||||
const nvidiaAllTargets = ['nvidia', 'nvidia-interconnect', 'nvidia-bandwidth'];
|
const nvidiaAllTargets = ['nvidia', 'nvidia-interconnect', 'nvidia-bandwidth'];
|
||||||
const baseTargets = ['cpu', 'memory', 'storage', 'confidential-computing'];
|
const baseTargets = ['cpu', 'memory', 'storage', 'nvidia-config'];
|
||||||
const amdTargets = selectedAMDValidateTargets();
|
const amdTargets = selectedAMDValidateTargets();
|
||||||
const expanded = [];
|
const expanded = [];
|
||||||
baseTargets.forEach(t => expanded.push({target: t}));
|
baseTargets.forEach(t => expanded.push({target: t}));
|
||||||
|
|||||||
@@ -264,7 +264,7 @@ func NewHandler(opts HandlerOptions) http.Handler {
|
|||||||
mux.HandleFunc("POST /api/sat/nvidia-stress/run", h.handleAPISATRun("nvidia-stress"))
|
mux.HandleFunc("POST /api/sat/nvidia-stress/run", h.handleAPISATRun("nvidia-stress"))
|
||||||
mux.HandleFunc("POST /api/sat/memory/run", h.handleAPISATRun("memory"))
|
mux.HandleFunc("POST /api/sat/memory/run", h.handleAPISATRun("memory"))
|
||||||
mux.HandleFunc("POST /api/sat/storage/run", h.handleAPISATRun("storage"))
|
mux.HandleFunc("POST /api/sat/storage/run", h.handleAPISATRun("storage"))
|
||||||
mux.HandleFunc("POST /api/sat/confidential-computing/run", h.handleAPISATRun("confidential-computing"))
|
mux.HandleFunc("POST /api/sat/nvidia-config/run", h.handleAPISATRun("nvidia-config"))
|
||||||
mux.HandleFunc("POST /api/sat/cpu/run", h.handleAPISATRun("cpu"))
|
mux.HandleFunc("POST /api/sat/cpu/run", h.handleAPISATRun("cpu"))
|
||||||
mux.HandleFunc("POST /api/sat/amd/run", h.handleAPISATRun("amd"))
|
mux.HandleFunc("POST /api/sat/amd/run", h.handleAPISATRun("amd"))
|
||||||
mux.HandleFunc("POST /api/sat/amd-mem/run", h.handleAPISATRun("amd-mem"))
|
mux.HandleFunc("POST /api/sat/amd-mem/run", h.handleAPISATRun("amd-mem"))
|
||||||
|
|||||||
@@ -272,12 +272,12 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
archive, err = runStorageAcceptancePackCtx(a, ctx, "", t.params.StressMode, j.append)
|
archive, err = runStorageAcceptancePackCtx(a, ctx, "", t.params.StressMode, j.append)
|
||||||
case "confidential-computing":
|
case "nvidia-config":
|
||||||
if a == nil {
|
if a == nil {
|
||||||
err = fmt.Errorf("app not configured")
|
err = fmt.Errorf("app not configured")
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
archive, err = runConfidentialComputingCheckPackCtx(a, ctx, "", j.append)
|
archive, err = runNvidiaConfigCheckPackCtx(a, ctx, "", j.append)
|
||||||
case "cpu":
|
case "cpu":
|
||||||
if a == nil {
|
if a == nil {
|
||||||
err = fmt.Errorf("app not configured")
|
err = fmt.Errorf("app not configured")
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ var taskNames = map[string]string{
|
|||||||
"nvidia-stress": "NVIDIA GPU Stress",
|
"nvidia-stress": "NVIDIA GPU Stress",
|
||||||
"memory": "Memory SAT",
|
"memory": "Memory SAT",
|
||||||
"storage": "Storage SAT",
|
"storage": "Storage SAT",
|
||||||
"confidential-computing": "Confidential Computing Check",
|
"nvidia-config": "GPU Config & NVLink Check",
|
||||||
"cpu": "CPU SAT",
|
"cpu": "CPU SAT",
|
||||||
"amd": "AMD GPU SAT",
|
"amd": "AMD GPU SAT",
|
||||||
"amd-mem": "AMD GPU MEM Integrity",
|
"amd-mem": "AMD GPU MEM Integrity",
|
||||||
@@ -313,8 +313,8 @@ var (
|
|||||||
runStorageAcceptancePackCtx = func(a *app.App, ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) {
|
runStorageAcceptancePackCtx = func(a *app.App, ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) {
|
||||||
return a.RunStorageAcceptancePackCtx(ctx, baseDir, extended, logFunc)
|
return a.RunStorageAcceptancePackCtx(ctx, baseDir, extended, logFunc)
|
||||||
}
|
}
|
||||||
runConfidentialComputingCheckPackCtx = func(a *app.App, ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
runNvidiaConfigCheckPackCtx = func(a *app.App, ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||||
return a.RunConfidentialComputingCheckPackCtx(ctx, baseDir, logFunc)
|
return a.RunNvidiaConfigCheckPackCtx(ctx, baseDir, logFunc)
|
||||||
}
|
}
|
||||||
runCPUAcceptancePackCtx = func(a *app.App, ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
runCPUAcceptancePackCtx = func(a *app.App, ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||||
return a.RunCPUAcceptancePackCtx(ctx, baseDir, durationSec, logFunc)
|
return a.RunCPUAcceptancePackCtx(ctx, baseDir, durationSec, logFunc)
|
||||||
@@ -1029,12 +1029,12 @@ func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) {
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
archive, err = runStorageAcceptancePackCtx(a, ctx, "", t.params.StressMode, j.append)
|
archive, err = runStorageAcceptancePackCtx(a, ctx, "", t.params.StressMode, j.append)
|
||||||
case "confidential-computing":
|
case "nvidia-config":
|
||||||
if a == nil {
|
if a == nil {
|
||||||
err = fmt.Errorf("app not configured")
|
err = fmt.Errorf("app not configured")
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
archive, err = runConfidentialComputingCheckPackCtx(a, ctx, "", j.append)
|
archive, err = runNvidiaConfigCheckPackCtx(a, ctx, "", j.append)
|
||||||
case "cpu":
|
case "cpu":
|
||||||
if a == nil {
|
if a == nil {
|
||||||
err = fmt.Errorf("app not configured")
|
err = fmt.Errorf("app not configured")
|
||||||
|
|||||||
Reference in New Issue
Block a user