fix(collector): stop pinning GPU PCIe status on an unverified idle reading
NVIDIA GPUs deliberately downclock PCIe to Gen1 at idle for power saving, and applyPCIeLinkSpeedWarning fired on every idle collector pass regardless - since component-status DB records never downgrade (Record() only ever raises severity), one boot-time idle sample permanently pinned pcie:gpu:nvidia to Warning for the rest of the session even after every load-bearing GPU SAT test passed clean. Two prior fixes (nvidia-smi-sourced link speed, pcie_aspm=off boot flag) didn't hold up against this hardware/driver combination - see bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md for the full history. Rather than add a downgrade path, stop writing an unverified status in the first place: parseLspciDevice no longer calls applyPCIeLinkSpeedWarning on the idle path. LinkSpeed/MaxLinkSpeed stay populated as plain descriptive fields; only a verified-under-load caller may now turn them into a status verdict. Two new SAT targets provide that verified signal: - pcie-link (platform/pcie_link_check.go): forces every enabled PCIe device - not just GPUs - to retrain via the PCIe spec's Link Control "Retrain Link" bit, then compares the negotiated speed against the device's max. Covers NICs/HBAs/switches that have no bee-gpu-burn equivalent load tool. Classifies by PCI class code + vendor ID, not name substrings. Routes gpu_nvidia/gpu_amd/other sub-verdicts into their own component-status keys so a degraded NIC never reads as a GPU fault. - nvidia-pcie-bandwidth (platform/nvidia_pcie_bandwidth.go): drives real host<->device traffic via dcgmi diag -r nvbandwidth and resamples link speed immediately after, independent of nvbandwidth's own pass/fail. Both wired into the task queue/webui the same way as nvidia-config (routes, dispatch, priority, Validate page cards, Run All Check SAT). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
7aa276320b
commit
b11018ac5e
@@ -139,6 +139,8 @@ type satRunner interface {
|
||||
RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error)
|
||||
RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error)
|
||||
RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
|
||||
RunPCIeLinkCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
|
||||
RunNvidiaPCIeBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error)
|
||||
RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
|
||||
ListNvidiaGPUs() ([]platform.NvidiaGPU, error)
|
||||
ListNvidiaGPUSettings() ([]platform.NvidiaGPUSetting, error)
|
||||
|
||||
@@ -274,6 +274,38 @@ func (a *App) RunNvidiaConfigCheckPackResult(baseDir string) (ActionResult, erro
|
||||
return ActionResult{Title: "GPU Config & NVLink Check", Body: satResultBody(path)}, err
|
||||
}
|
||||
|
||||
func (a *App) RunPCIeLinkCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = DefaultSATBaseDir
|
||||
}
|
||||
return a.sat.RunPCIeLinkCheckPack(ctx, baseDir, logFunc)
|
||||
}
|
||||
|
||||
func (a *App) RunPCIeLinkCheckPack(baseDir string, logFunc func(string)) (string, error) {
|
||||
return a.RunPCIeLinkCheckPackCtx(context.Background(), baseDir, logFunc)
|
||||
}
|
||||
|
||||
func (a *App) RunPCIeLinkCheckPackResult(baseDir string) (ActionResult, error) {
|
||||
path, err := a.RunPCIeLinkCheckPack(baseDir, nil)
|
||||
return ActionResult{Title: "PCIe Link Check", Body: satResultBody(path)}, err
|
||||
}
|
||||
|
||||
func (a *App) RunNvidiaPCIeBandwidthPackCtx(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = DefaultSATBaseDir
|
||||
}
|
||||
return a.sat.RunNvidiaPCIeBandwidthPack(ctx, baseDir, gpuIndices, logFunc)
|
||||
}
|
||||
|
||||
func (a *App) RunNvidiaPCIeBandwidthPack(baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
return a.RunNvidiaPCIeBandwidthPackCtx(context.Background(), baseDir, gpuIndices, logFunc)
|
||||
}
|
||||
|
||||
func (a *App) RunNvidiaPCIeBandwidthPackResult(baseDir string) (ActionResult, error) {
|
||||
path, err := a.RunNvidiaPCIeBandwidthPack(baseDir, nil, nil)
|
||||
return ActionResult{Title: "NVIDIA GPU PCIe Bandwidth Check", Body: satResultBody(path)}, err
|
||||
}
|
||||
|
||||
func (a *App) DetectGPUVendor() string {
|
||||
return a.sat.DetectGPUVendor()
|
||||
}
|
||||
|
||||
@@ -287,6 +287,14 @@ func (f fakeSAT) RunNvidiaConfigCheckPack(_ context.Context, baseDir string, _ f
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (f fakeSAT) RunPCIeLinkCheckPack(_ context.Context, baseDir string, _ func(string)) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (f fakeSAT) RunNvidiaPCIeBandwidthPack(_ context.Context, baseDir string, _ []int, _ func(string)) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (f fakeSAT) RunCPUAcceptancePack(_ context.Context, baseDir string, durationSec int, _ func(string)) (string, error) {
|
||||
if f.runCPUFn != nil {
|
||||
return f.runCPUFn(baseDir, durationSec)
|
||||
|
||||
@@ -253,10 +253,32 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
|
||||
// otherwise fails to match any real BDF.
|
||||
switch target {
|
||||
case "nvidia", "nvidia-targeted-stress", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
|
||||
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "nvidia-config":
|
||||
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "nvidia-config", "nvidia-pcie-bandwidth":
|
||||
db.Record("pcie:gpu:nvidia", source, dbStatus, detail)
|
||||
case "amd", "amd-stress", "amd-mem", "amd-bandwidth":
|
||||
db.Record("pcie:gpu:amd", source, dbStatus, detail)
|
||||
case "pcie-link":
|
||||
// Forced-retrain PCIe link check (audit/internal/platform/pcie_link_check.go):
|
||||
// the only verified (non-idle-sampled) source for PCIe link-speed
|
||||
// status. summary.txt carries up to three independent sub-verdicts
|
||||
// — record each into its own component key rather than collapsing
|
||||
// them into one, since a degraded NIC/HBA shouldn't be reported as
|
||||
// a GPU fault or vice versa.
|
||||
recordPCIeLinkSubStatus := func(key, kvKey string) {
|
||||
v, ok := kv[kvKey]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
st := strings.ToUpper(strings.TrimSpace(v))
|
||||
d := "pcie-link SAT: " + st
|
||||
if st != "OK" && kv["warnings"] != "" {
|
||||
d += " — " + kv["warnings"]
|
||||
}
|
||||
db.Record(key, source, satStatusToDBStatus(st), d)
|
||||
}
|
||||
recordPCIeLinkSubStatus("pcie:gpu:nvidia", "gpu_nvidia_status")
|
||||
recordPCIeLinkSubStatus("pcie:gpu:amd", "gpu_amd_status")
|
||||
recordPCIeLinkSubStatus("pcie:link:other", "other_status")
|
||||
case "memory", "memory-stress", "sat-stress":
|
||||
db.Record("memory:all", source, dbStatus, detail)
|
||||
case "cpu", "platform-stress":
|
||||
|
||||
@@ -187,8 +187,20 @@ func parseLspciDevice(fields map[string]string) schema.HardwarePCIeDevice {
|
||||
markNVLinkBridge(&dev)
|
||||
}
|
||||
|
||||
// Warn (or Critical for NVLink bridges) if PCIe link is running below max.
|
||||
applyPCIeLinkSpeedWarning(&dev)
|
||||
// Deliberately not calling applyPCIeLinkSpeedWarning here: this function
|
||||
// runs on every idle-time collector pass, and NVIDIA GPUs legitimately
|
||||
// downclock their PCIe link at idle to save power, making "current <
|
||||
// max" a false positive most of the time it fires. There's no reliable
|
||||
// way to downgrade a status once recorded (component-status DB history
|
||||
// is a one-way transition log, by design — see component_status_db.go),
|
||||
// so writing an unverified idle reading as Warning meant it stuck for
|
||||
// the rest of the session even after every load-bearing SAT test passed
|
||||
// clean. See bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning-unresolved.md.
|
||||
// applyPCIeLinkSpeedWarning is reserved for a future caller that samples
|
||||
// link speed while the device is under verified load (a dedicated SAT,
|
||||
// or a forced link retrain) — only that caller should turn this data
|
||||
// into a status verdict. LinkSpeed/MaxLinkSpeed above stay populated as
|
||||
// plain descriptive fields regardless.
|
||||
|
||||
return dev
|
||||
}
|
||||
@@ -283,6 +295,12 @@ func readPCIStringAttribute(bdf, attribute string) (string, bool) {
|
||||
// their link state has no operational impact. This covers management endpoints
|
||||
// (e.g. PCIe switch fabric controllers on HGX baseboards) that the kernel never
|
||||
// activates but that lspci still reports with link stats.
|
||||
//
|
||||
// Not called from the idle collector path (parseLspciDevice) — see the
|
||||
// comment there. Call this only from a context that just put the device
|
||||
// under real traffic (a load-bearing SAT, a forced link retrain), so a
|
||||
// genuine Warning/Critical is never based on an ASPM/power-managed idle
|
||||
// downclock that a human or SAT would immediately clear anyway.
|
||||
func applyPCIeLinkSpeedWarning(dev *schema.HardwarePCIeDevice) {
|
||||
if dev.LinkSpeed == nil || dev.MaxLinkSpeed == nil {
|
||||
return
|
||||
@@ -330,6 +348,14 @@ func pcieLinkSpeedRank(gen string) int {
|
||||
}
|
||||
|
||||
func normalizePCILinkSpeed(raw string) string {
|
||||
return NormalizePCILinkSpeed(raw)
|
||||
}
|
||||
|
||||
// NormalizePCILinkSpeed maps a raw sysfs current_link_speed/max_link_speed
|
||||
// string (e.g. "16.0 GT/s PCIe") to a "GenN" label. Exported for reuse by
|
||||
// the platform package's PCIe link-retrain SAT, which needs the same
|
||||
// mapping outside the idle collector path.
|
||||
func NormalizePCILinkSpeed(raw string) string {
|
||||
raw = strings.TrimSpace(strings.ToLower(raw))
|
||||
switch {
|
||||
case strings.Contains(raw, "2.5"):
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// nvidiaPCIeBandwidthFinding is one GPU's post-load PCIe link-speed result.
|
||||
type nvidiaPCIeBandwidthFinding struct {
|
||||
Index int
|
||||
BDF string
|
||||
BeforeSpeed string
|
||||
AfterSpeed string
|
||||
MaxSpeed string
|
||||
Width int
|
||||
MaxWidth int
|
||||
Degraded bool
|
||||
}
|
||||
|
||||
// RunNvidiaPCIeBandwidthPack drives real host<->device traffic across the
|
||||
// GPUs' PCIe links (via `dcgmi diag -r nvbandwidth`, the same tool the
|
||||
// existing nvidia-bandwidth SAT uses for GPU-to-GPU throughput) and then
|
||||
// resamples each GPU's PCIe link speed from sysfs immediately afterward.
|
||||
//
|
||||
// This is deliberately a separate, narrower check from "nvidia-bandwidth":
|
||||
// that SAT's overall_status reflects nvbandwidth's own pass/fail (did the
|
||||
// measured GB/s clear its internal threshold), never PCIe link speed
|
||||
// itself. This pack exists purely to answer "did sustained real traffic
|
||||
// make the link train up to its negotiated maximum" — the load-bearing
|
||||
// counterpart to the idle-time collector reading that used to (falsely)
|
||||
// drive pcie:gpu:nvidia's status. See
|
||||
// bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning-unresolved.md.
|
||||
func (s *System) RunNvidiaPCIeBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
runDir := filepath.Join(baseDir, "nvidia-pcie-bandwidth-"+ts)
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||
|
||||
bdfByIndex, err := gpuIndexToBDF(gpuIndices)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve GPU BDFs: %w", err)
|
||||
}
|
||||
indices := make([]int, 0, len(bdfByIndex))
|
||||
for idx := range bdfByIndex {
|
||||
indices = append(indices, idx)
|
||||
}
|
||||
sort.Ints(indices)
|
||||
|
||||
findings := make([]nvidiaPCIeBandwidthFinding, 0, len(indices))
|
||||
for _, idx := range indices {
|
||||
bdf := bdfByIndex[idx]
|
||||
before, _ := readPCIeSysfsString(bdf, "current_link_speed")
|
||||
max, _ := readPCIeSysfsString(bdf, "max_link_speed")
|
||||
maxWidth, _ := readPCIeSysfsInt(bdf, "max_link_width")
|
||||
findings = append(findings, nvidiaPCIeBandwidthFinding{
|
||||
Index: idx, BDF: bdf, BeforeSpeed: before, MaxSpeed: max, MaxWidth: maxWidth,
|
||||
})
|
||||
}
|
||||
|
||||
cmd := []string{"dcgmi", "diag", "-r", "nvbandwidth"}
|
||||
if len(indices) > 0 {
|
||||
cmd = append(cmd, "-i", joinIndexList(indices))
|
||||
}
|
||||
out, runErr := runSATCommandCtx(ctx, verboseLog, "dcgmi-nvbandwidth", cmd, nil, logFunc)
|
||||
_ = os.WriteFile(filepath.Join(runDir, "01-dcgmi-nvbandwidth.log"), out, 0644)
|
||||
|
||||
for i := range findings {
|
||||
bdf := findings[i].BDF
|
||||
after, _ := readPCIeSysfsString(bdf, "current_link_speed")
|
||||
width, _ := readPCIeSysfsInt(bdf, "current_link_width")
|
||||
findings[i].AfterSpeed = after
|
||||
findings[i].Width = width
|
||||
findings[i].Degraded = after != findings[i].MaxSpeed
|
||||
}
|
||||
|
||||
summary := renderNvidiaPCIeBandwidthSummary(findings, runErr)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
report := renderNvidiaPCIeBandwidthReport(findings, runErr)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "nvidia-pcie-bandwidth-report.txt"), []byte(report), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
|
||||
// gpuIndexToBDF resolves each of gpuIndices to its PCI BDF via nvidia-smi.
|
||||
// An empty gpuIndices resolves every GPU nvidia-smi reports.
|
||||
func gpuIndexToBDF(gpuIndices []int) (map[int]string, error) {
|
||||
out, err := satExecCommand("nvidia-smi", "--query-gpu=index,pci.bus_id", "--format=csv,noheader,nounits").Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nvidia-smi: %w", err)
|
||||
}
|
||||
want := make(map[int]struct{}, len(gpuIndices))
|
||||
for _, idx := range gpuIndices {
|
||||
want[idx] = struct{}{}
|
||||
}
|
||||
filterByIndex := len(want) > 0
|
||||
|
||||
result := make(map[int]string)
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
fields := strings.SplitN(line, ",", 2)
|
||||
if len(fields) != 2 {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(strings.TrimSpace(fields[0]))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if filterByIndex {
|
||||
if _, ok := want[idx]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
result[idx] = normalizeNvidiaBDF(strings.TrimSpace(fields[1]))
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func renderNvidiaPCIeBandwidthSummary(findings []nvidiaPCIeBandwidthFinding, runErr error) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
||||
fmt.Fprintf(&b, "gpu_count=%d\n", len(findings))
|
||||
degraded := 0
|
||||
for _, f := range findings {
|
||||
fmt.Fprintf(&b, "gpu%d_status=%s\n", f.Index, statusLabel(!f.Degraded))
|
||||
if f.Degraded {
|
||||
degraded++
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "degraded=%d\n", degraded)
|
||||
if runErr != nil {
|
||||
fmt.Fprintf(&b, "dcgmi_nvbandwidth_error=%s\n", runErr.Error())
|
||||
}
|
||||
if degraded > 0 {
|
||||
fmt.Fprintln(&b, "overall_status=FAILED")
|
||||
var reasons []string
|
||||
for _, f := range findings {
|
||||
if f.Degraded {
|
||||
reasons = append(reasons, fmt.Sprintf("GPU%d (%s): still at %s under load, capable of %s",
|
||||
f.Index, f.BDF, f.AfterSpeed, f.MaxSpeed))
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "warnings=%s\n", strings.Join(reasons, "; "))
|
||||
} else {
|
||||
fmt.Fprintln(&b, "overall_status=OK")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func statusLabel(ok bool) string {
|
||||
if ok {
|
||||
return "OK"
|
||||
}
|
||||
return "FAILED"
|
||||
}
|
||||
|
||||
func renderNvidiaPCIeBandwidthReport(findings []nvidiaPCIeBandwidthFinding, runErr error) string {
|
||||
var b strings.Builder
|
||||
line := strings.Repeat("=", 80)
|
||||
b.WriteString(line + "\n")
|
||||
b.WriteString("NVIDIA GPU PCIe Bandwidth / Link-Under-Load Check\n")
|
||||
b.WriteString(line + "\n\n")
|
||||
if runErr != nil {
|
||||
fmt.Fprintf(&b, "dcgmi diag -r nvbandwidth: %s\n\n", runErr.Error())
|
||||
}
|
||||
for _, f := range findings {
|
||||
verdict := "OK"
|
||||
if f.Degraded {
|
||||
verdict = "DEGRADED"
|
||||
}
|
||||
fmt.Fprintf(&b, "GPU%d (%s): %s\n", f.Index, f.BDF, verdict)
|
||||
fmt.Fprintf(&b, " before=%s after=%s max=%s width=%d/%d\n",
|
||||
f.BeforeSpeed, f.AfterSpeed, f.MaxSpeed, f.Width, f.MaxWidth)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bee/audit/internal/collector"
|
||||
)
|
||||
|
||||
// pcieLinkRetrainTimeout bounds how long we wait for a device to finish
|
||||
// retraining (clear the Link Status "Link Training" bit) before giving up
|
||||
// and reading whatever speed it settled on anyway. The PCIe spec allows up
|
||||
// to 100ms for Gen1-3 and longer for higher generations with equalization;
|
||||
// this is generous headroom above that.
|
||||
const pcieLinkRetrainTimeout = 2 * time.Second
|
||||
|
||||
// pcieLinkFinding is one device's before/after link-speed retrain result.
|
||||
type pcieLinkFinding struct {
|
||||
BDF string
|
||||
Description string
|
||||
VendorID string
|
||||
ClassCode string
|
||||
IsGPU bool
|
||||
GPUVendor string // "nvidia" or "amd", only set when IsGPU
|
||||
Skipped string // non-empty reason this device wasn't retrained
|
||||
BeforeSpeed string
|
||||
AfterSpeed string
|
||||
MaxSpeed string
|
||||
Width int
|
||||
MaxWidth int
|
||||
Degraded bool
|
||||
}
|
||||
|
||||
// RunPCIeLinkCheckPack forces every enabled PCIe device to retrain its link
|
||||
// (via the PCIe Link Control register's spec-defined Retrain Link bit — see
|
||||
// PCIe base spec, Link Control Register, bit 5) and compares the
|
||||
// post-retrain negotiated speed against the device's own reported maximum.
|
||||
//
|
||||
// This exists because a plain idle-time sysfs read of current_link_speed is
|
||||
// not a reliable fault signal: NVIDIA GPUs (and other devices with runtime
|
||||
// power management) deliberately downclock their PCIe link to save power
|
||||
// while idle, which is indistinguishable from a real degraded slot/riser/
|
||||
// cable without either sustained traffic or a forced retrain. Forcing a
|
||||
// retrain sidesteps needing a device-specific load generator (bee-gpu-burn
|
||||
// exists for GPUs; nothing plays that role for NICs, HBAs, or PCIe
|
||||
// switches) — retraining is a PCIe-spec mechanism every endpoint supports,
|
||||
// so this one check covers every PCIe device in the machine, not just
|
||||
// GPUs. See bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning-unresolved.md
|
||||
// for the history of narrower attempts that didn't generalize.
|
||||
//
|
||||
// Disabled devices (sysfs enable==0 — e.g. PCIe fabric-management endpoints
|
||||
// the kernel never activates, per the 2026-06-12 decision) are left alone:
|
||||
// they carry no data traffic, so there is nothing to verify and no reason
|
||||
// to poke them.
|
||||
func (s *System) RunPCIeLinkCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
runDir := filepath.Join(baseDir, "pcie-link-"+ts)
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||
|
||||
bdfs, err := listPCIDeviceBDFs()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("list PCI devices: %w", err)
|
||||
}
|
||||
|
||||
var findings []pcieLinkFinding
|
||||
for _, bdf := range bdfs {
|
||||
if logFunc != nil {
|
||||
logFunc(fmt.Sprintf("=== %s ===", bdf))
|
||||
}
|
||||
f := retrainAndSamplePCIeDevice(ctx, verboseLog, bdf, logFunc)
|
||||
findings = append(findings, f)
|
||||
}
|
||||
|
||||
summary := renderPCIeLinkCheckSummary(findings)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
report := renderPCIeLinkCheckReport(findings)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "pcie-link-report.txt"), []byte(report), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
|
||||
// listPCIDeviceBDFs returns every BDF under /sys/bus/pci/devices, sorted for
|
||||
// deterministic report ordering.
|
||||
func listPCIDeviceBDFs() ([]string, error) {
|
||||
entries, err := os.ReadDir("/sys/bus/pci/devices")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bdfs := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
bdfs = append(bdfs, e.Name())
|
||||
}
|
||||
sort.Strings(bdfs)
|
||||
return bdfs, nil
|
||||
}
|
||||
|
||||
func retrainAndSamplePCIeDevice(ctx context.Context, verboseLog, bdf string, logFunc func(string)) pcieLinkFinding {
|
||||
f := pcieLinkFinding{BDF: bdf}
|
||||
|
||||
vendor, _ := readPCIeSysfsHex(bdf, "vendor")
|
||||
class, _ := readPCIeSysfsHex(bdf, "class")
|
||||
f.VendorID = vendor
|
||||
f.ClassCode = class
|
||||
f.IsGPU, f.GPUVendor = classifyGPUFromVendorClass(vendor, class)
|
||||
f.Description = pcieDeviceDescription(ctx, verboseLog, bdf, logFunc)
|
||||
|
||||
if enabled, ok := readPCIeSysfsInt(bdf, "enable"); ok && enabled == 0 {
|
||||
f.Skipped = "device disabled (no data traffic; link state has no operational impact)"
|
||||
return f
|
||||
}
|
||||
|
||||
before, beforeOK := readPCIeSysfsString(bdf, "current_link_speed")
|
||||
maxSpeed, maxOK := readPCIeSysfsString(bdf, "max_link_speed")
|
||||
width, _ := readPCIeSysfsInt(bdf, "current_link_width")
|
||||
maxWidth, _ := readPCIeSysfsInt(bdf, "max_link_width")
|
||||
f.BeforeSpeed = before
|
||||
f.MaxSpeed = maxSpeed
|
||||
f.MaxWidth = maxWidth
|
||||
if !beforeOK || !maxOK {
|
||||
f.Skipped = "no PCIe link-speed attributes in sysfs (not a link-trained endpoint)"
|
||||
return f
|
||||
}
|
||||
|
||||
if err := retrainPCIeLink(ctx, verboseLog, bdf, logFunc); err != nil {
|
||||
f.Skipped = "retrain failed: " + err.Error()
|
||||
f.AfterSpeed = before
|
||||
f.Width = width
|
||||
// before/maxSpeed are already normalized "GenN" labels (see
|
||||
// readPCIeSysfsString) — compare directly, don't re-normalize.
|
||||
f.Degraded = before != maxSpeed
|
||||
return f
|
||||
}
|
||||
|
||||
after, _ := readPCIeSysfsString(bdf, "current_link_speed")
|
||||
widthAfter, _ := readPCIeSysfsInt(bdf, "current_link_width")
|
||||
f.AfterSpeed = after
|
||||
f.Width = widthAfter
|
||||
f.Degraded = after != maxSpeed
|
||||
return f
|
||||
}
|
||||
|
||||
// retrainPCIeLink sets the Retrain Link bit (bit 5) of the PCI Express
|
||||
// Capability's Link Control register via setpci, then polls the Link
|
||||
// Status register's Link Training bit (bit 11) until it clears or
|
||||
// pcieLinkRetrainTimeout elapses.
|
||||
func retrainPCIeLink(ctx context.Context, verboseLog, bdf string, logFunc func(string)) error {
|
||||
linkCtrlOut, err := runSATCommandCtx(ctx, verboseLog, "setpci-read-"+bdf,
|
||||
[]string{"setpci", "-s", bdf, "CAP_EXP+0x10.w"}, nil, logFunc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read Link Control: %w", err)
|
||||
}
|
||||
cur, err := strconv.ParseUint(strings.TrimSpace(string(linkCtrlOut)), 16, 16)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse Link Control %q: %w", linkCtrlOut, err)
|
||||
}
|
||||
const retrainLinkBit = 0x0020
|
||||
newVal := uint16(cur) | retrainLinkBit
|
||||
|
||||
if _, err := runSATCommandCtx(ctx, verboseLog, "setpci-retrain-"+bdf,
|
||||
[]string{"setpci", "-s", bdf, fmt.Sprintf("CAP_EXP+0x10.w=%04x", newVal)}, nil, logFunc); err != nil {
|
||||
return fmt.Errorf("write Retrain Link bit: %w", err)
|
||||
}
|
||||
|
||||
const linkTrainingBit = 0x0800
|
||||
deadline := time.Now().Add(pcieLinkRetrainTimeout)
|
||||
for time.Now().Before(deadline) {
|
||||
statusOut, err := runSATCommandCtx(ctx, verboseLog, "setpci-status-"+bdf,
|
||||
[]string{"setpci", "-s", bdf, "CAP_EXP+0x12.w"}, nil, nil)
|
||||
if err == nil {
|
||||
if status, perr := strconv.ParseUint(strings.TrimSpace(string(statusOut)), 16, 16); perr == nil {
|
||||
if status&linkTrainingBit == 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
// Timed out waiting for training to clear; the caller still samples
|
||||
// whatever speed sysfs reports, which is the honest answer either way.
|
||||
return nil
|
||||
}
|
||||
|
||||
func pcieDeviceDescription(ctx context.Context, verboseLog, bdf string, logFunc func(string)) string {
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, "lspci-"+bdf, []string{"lspci", "-s", bdf}, nil, logFunc)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
line := strings.TrimSpace(string(out))
|
||||
if idx := strings.Index(line, "\n"); idx >= 0 {
|
||||
line = line[:idx]
|
||||
}
|
||||
if idx := strings.Index(line, " "); idx >= 0 {
|
||||
return strings.TrimSpace(line[idx+1:])
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func readPCIeSysfsString(bdf, attr string) (string, bool) {
|
||||
raw, err := os.ReadFile(filepath.Join("/sys/bus/pci/devices", bdf, attr))
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
v := strings.TrimSpace(string(raw))
|
||||
if v == "" {
|
||||
return "", false
|
||||
}
|
||||
return collector.NormalizePCILinkSpeed(v), true
|
||||
}
|
||||
|
||||
func readPCIeSysfsInt(bdf, attr string) (int, bool) {
|
||||
raw, err := os.ReadFile(filepath.Join("/sys/bus/pci/devices", bdf, attr))
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
v, err := strconv.Atoi(strings.TrimSpace(string(raw)))
|
||||
if err != nil || v < 0 {
|
||||
return 0, false
|
||||
}
|
||||
return v, true
|
||||
}
|
||||
|
||||
func readPCIeSysfsHex(bdf, attr string) (string, bool) {
|
||||
raw, err := os.ReadFile(filepath.Join("/sys/bus/pci/devices", bdf, attr))
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimSpace(string(raw)), true
|
||||
}
|
||||
|
||||
// classifyGPUFromVendorClass reports whether a device is a GPU die itself
|
||||
// (PCI base class 0x03 — Display Controller — under NVIDIA/AMD's vendor
|
||||
// ID), as opposed to a same-vendor companion device (NIC, storage
|
||||
// controller, NVLink bridge) that shares the GPU's PCI vendor ID. Class-code
|
||||
// based, not name-substring based, per the same reasoning as
|
||||
// collector.IsGPUClass.
|
||||
func classifyGPUFromVendorClass(vendorHex, classHex string) (isGPU bool, vendor string) {
|
||||
v := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(vendorHex)), "0x")
|
||||
c := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(classHex)), "0x")
|
||||
if len(c) < 2 || c[:2] != "03" {
|
||||
return false, ""
|
||||
}
|
||||
switch v {
|
||||
case "10de":
|
||||
return true, "nvidia"
|
||||
case "1002":
|
||||
return true, "amd"
|
||||
default:
|
||||
return false, ""
|
||||
}
|
||||
}
|
||||
|
||||
func renderPCIeLinkCheckSummary(findings []pcieLinkFinding) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
||||
fmt.Fprintf(&b, "devices_tested=%d\n", len(findings))
|
||||
|
||||
gpuStatus := map[string]string{} // vendor -> OK/FAILED
|
||||
otherDegraded := 0
|
||||
otherTested := 0
|
||||
anyDegraded := false
|
||||
for _, f := range findings {
|
||||
if f.Skipped != "" && f.AfterSpeed == "" {
|
||||
continue
|
||||
}
|
||||
if f.IsGPU {
|
||||
if _, ok := gpuStatus[f.GPUVendor]; !ok {
|
||||
gpuStatus[f.GPUVendor] = "OK"
|
||||
}
|
||||
if f.Degraded {
|
||||
gpuStatus[f.GPUVendor] = "FAILED"
|
||||
anyDegraded = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
otherTested++
|
||||
if f.Degraded {
|
||||
otherDegraded++
|
||||
anyDegraded = true
|
||||
}
|
||||
}
|
||||
for _, vendor := range []string{"nvidia", "amd"} {
|
||||
if status, ok := gpuStatus[vendor]; ok {
|
||||
fmt.Fprintf(&b, "gpu_%s_status=%s\n", vendor, status)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "other_devices_tested=%d\n", otherTested)
|
||||
fmt.Fprintf(&b, "other_devices_degraded=%d\n", otherDegraded)
|
||||
if otherTested > 0 {
|
||||
if otherDegraded > 0 {
|
||||
fmt.Fprintln(&b, "other_status=FAILED")
|
||||
} else {
|
||||
fmt.Fprintln(&b, "other_status=OK")
|
||||
}
|
||||
}
|
||||
|
||||
if anyDegraded {
|
||||
fmt.Fprintln(&b, "overall_status=FAILED")
|
||||
var reasons []string
|
||||
for _, f := range findings {
|
||||
if f.Degraded {
|
||||
reasons = append(reasons, fmt.Sprintf("%s (%s): retrained to %s, capable of %s",
|
||||
f.BDF, nonEmptyOr(f.Description, "unknown device"), f.AfterSpeed, f.MaxSpeed))
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "warnings=%s\n", strings.Join(reasons, "; "))
|
||||
} else {
|
||||
fmt.Fprintln(&b, "overall_status=OK")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func renderPCIeLinkCheckReport(findings []pcieLinkFinding) string {
|
||||
var b strings.Builder
|
||||
line := strings.Repeat("=", 80)
|
||||
b.WriteString(line + "\n")
|
||||
b.WriteString("PCIe Link Retrain Check\n")
|
||||
b.WriteString(line + "\n\n")
|
||||
for _, f := range findings {
|
||||
fmt.Fprintf(&b, "%s %s\n", f.BDF, nonEmptyOr(f.Description, "(unknown device)"))
|
||||
if f.Skipped != "" && f.AfterSpeed == "" {
|
||||
fmt.Fprintf(&b, " skipped: %s\n", f.Skipped)
|
||||
continue
|
||||
}
|
||||
verdict := "OK"
|
||||
if f.Degraded {
|
||||
verdict = "DEGRADED"
|
||||
}
|
||||
fmt.Fprintf(&b, " %s: before=%s after=%s max=%s width=%d/%d\n",
|
||||
verdict, f.BeforeSpeed, f.AfterSpeed, f.MaxSpeed, f.Width, f.MaxWidth)
|
||||
if f.Skipped != "" {
|
||||
fmt.Fprintf(&b, " note: %s\n", f.Skipped)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestClassifyGPUFromVendorClass(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
vendor string
|
||||
class string
|
||||
wantIsGPU bool
|
||||
wantVendor string
|
||||
}{
|
||||
{"nvidia GPU", "0x10de", "0x030200", true, "nvidia"},
|
||||
{"amd GPU", "0x1002", "0x030000", true, "amd"},
|
||||
{"nvidia NIC (same vendor, not display class)", "0x10de", "0x020000", false, ""},
|
||||
{"intel NIC", "0x8086", "0x020000", false, ""},
|
||||
{"unrelated display-class vendor", "0x1234", "0x030000", false, ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
gotIsGPU, gotVendor := classifyGPUFromVendorClass(tc.vendor, tc.class)
|
||||
if gotIsGPU != tc.wantIsGPU || gotVendor != tc.wantVendor {
|
||||
t.Fatalf("classifyGPUFromVendorClass(%q,%q) = (%v,%q), want (%v,%q)",
|
||||
tc.vendor, tc.class, gotIsGPU, gotVendor, tc.wantIsGPU, tc.wantVendor)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderPCIeLinkCheckSummaryDegradedGPU(t *testing.T) {
|
||||
findings := []pcieLinkFinding{
|
||||
{BDF: "0000:0d:00.0", Description: "NVIDIA GPU", IsGPU: true, GPUVendor: "nvidia",
|
||||
BeforeSpeed: "Gen1", AfterSpeed: "Gen1", MaxSpeed: "Gen5", Degraded: true},
|
||||
{BDF: "0000:37:00.0", Description: "NVIDIA GPU", IsGPU: true, GPUVendor: "nvidia",
|
||||
BeforeSpeed: "Gen1", AfterSpeed: "Gen5", MaxSpeed: "Gen5", Degraded: false},
|
||||
{BDF: "0000:01:00.0", Description: "Mellanox NIC", IsGPU: false,
|
||||
BeforeSpeed: "Gen3", AfterSpeed: "Gen4", MaxSpeed: "Gen4", Degraded: false},
|
||||
}
|
||||
summary := renderPCIeLinkCheckSummary(findings)
|
||||
if !strings.Contains(summary, "gpu_nvidia_status=FAILED") {
|
||||
t.Fatalf("expected gpu_nvidia_status=FAILED (one degraded GPU should fail the vendor group), got:\n%s", summary)
|
||||
}
|
||||
if !strings.Contains(summary, "other_status=OK") {
|
||||
t.Fatalf("expected other_status=OK, got:\n%s", summary)
|
||||
}
|
||||
if !strings.Contains(summary, "overall_status=FAILED") {
|
||||
t.Fatalf("expected overall_status=FAILED, got:\n%s", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderPCIeLinkCheckSummaryAllClean(t *testing.T) {
|
||||
findings := []pcieLinkFinding{
|
||||
{BDF: "0000:0d:00.0", IsGPU: true, GPUVendor: "nvidia", BeforeSpeed: "Gen1", AfterSpeed: "Gen5", MaxSpeed: "Gen5"},
|
||||
{BDF: "0000:2c:00.0", Skipped: "device disabled (no data traffic; link state has no operational impact)"},
|
||||
}
|
||||
summary := renderPCIeLinkCheckSummary(findings)
|
||||
if !strings.Contains(summary, "overall_status=OK") {
|
||||
t.Fatalf("expected overall_status=OK, got:\n%s", summary)
|
||||
}
|
||||
if strings.Contains(summary, "other_status=") {
|
||||
t.Fatalf("disabled/skipped-only device should not produce an other_status line, got:\n%s", summary)
|
||||
}
|
||||
}
|
||||
@@ -135,7 +135,7 @@ func defaultTaskPriority(target string, params taskParams) int {
|
||||
return taskPriorityBurn
|
||||
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
|
||||
"nvidia-interconnect", "nvidia-bandwidth", "memory", "storage", "cpu",
|
||||
"amd", "amd-mem", "amd-bandwidth", "nvidia-config":
|
||||
"amd", "amd-mem", "amd-bandwidth", "nvidia-config", "pcie-link", "nvidia-pcie-bandwidth":
|
||||
if params.StressMode {
|
||||
return taskPriorityValidateStress
|
||||
}
|
||||
|
||||
@@ -671,6 +671,18 @@ func renderCheck(opts HandlerOptions) string {
|
||||
`<code>nvidia-smi --query-gpu=...</code>, <code>nvidia-smi topo -m</code>, <code>nvidia-smi nvlink -s/-e</code>, <code>nvidia-smi conf-compute -q</code>, <code>dmesg</code>`,
|
||||
`Seconds — read-only query only.`,
|
||||
)) +
|
||||
renderSATCard("pcie-link", "PCIe Link Check", "runSAT('pcie-link')", "", renderValidateCardBody(
|
||||
`Every enabled PCIe device in the machine (GPUs, NICs, RAID/HBA controllers, PCIe switches).`,
|
||||
`Forces every enabled PCIe device to retrain its link (PCIe spec Link Control "Retrain Link" bit) and compares the negotiated speed against the device's own maximum. An idle sysfs reading alone can't tell a real degraded slot/riser from a device that's simply power-managed down at idle (GPUs do this routinely); a forced retrain settles that without needing a device-specific load generator, so this one check covers non-GPU PCIe hardware too, not just GPUs.`,
|
||||
`<code>setpci</code> (Link Control/Link Status registers), <code>lspci</code>`,
|
||||
`Seconds per device — brief link retrain, no sustained traffic.`,
|
||||
)) +
|
||||
renderSATCard("nvidia-pcie-bandwidth", "GPU PCIe Bandwidth", "runSAT('nvidia-pcie-bandwidth')", "", renderValidateCardBody(
|
||||
inv.NVIDIA,
|
||||
`Drives real host<->device traffic across each GPU's PCIe link and resamples link speed immediately after, to confirm the link actually trains to its negotiated maximum under real load — the load-bearing counterpart to PCIe Link Check for GPUs specifically.`,
|
||||
`<code>dcgmi diag -r nvbandwidth</code>, sysfs link-speed resample`,
|
||||
`Depends on nvbandwidth's built-in test duration.`,
|
||||
)) +
|
||||
`</div>
|
||||
<div style="height:1px;background:var(--border);margin:16px 0"></div>
|
||||
<div class="card" style="margin-bottom:16px">
|
||||
@@ -732,7 +744,7 @@ func renderCheck(opts HandlerOptions) string {
|
||||
<script>
|
||||
let satES = null;
|
||||
function satLabels() {
|
||||
return {nvidia:'Check GPU (DCGM L2)', 'nvidia-interconnect':'NVIDIA Interconnect (NCCL)', 'nvidia-bandwidth':'NVIDIA Bandwidth (NVBandwidth)', memory:'Check Memory', storage:'Check Storage', cpu:'Check CPU', amd:'Check AMD GPU', 'amd-mem':'AMD GPU MEM Integrity', 'amd-bandwidth':'AMD GPU MEM Bandwidth', 'nvidia-config':'Check GPU Config & NVLink'};
|
||||
return {nvidia:'Check GPU (DCGM L2)', 'nvidia-interconnect':'NVIDIA Interconnect (NCCL)', 'nvidia-bandwidth':'NVIDIA Bandwidth (NVBandwidth)', memory:'Check Memory', storage:'Check Storage', cpu:'Check CPU', amd:'Check AMD GPU', 'amd-mem':'AMD GPU MEM Integrity', 'amd-bandwidth':'AMD GPU MEM Bandwidth', 'nvidia-config':'Check GPU Config & NVLink', 'pcie-link':'PCIe Link Check', 'nvidia-pcie-bandwidth':'GPU PCIe Bandwidth Check'};
|
||||
}
|
||||
let satNvidiaGPUsPromise = null;
|
||||
function loadSatNvidiaGPUs() {
|
||||
@@ -867,8 +879,8 @@ function runAllCheckSAT() {
|
||||
const status = document.getElementById('sat-all-status');
|
||||
status.textContent = 'Enqueuing...';
|
||||
const nvidiaIndices = satSelectedGPUIndices();
|
||||
const nvidiaAllTargets = ['nvidia', 'nvidia-interconnect', 'nvidia-bandwidth'];
|
||||
const baseTargets = ['cpu', 'memory', 'storage', 'nvidia-config'];
|
||||
const nvidiaAllTargets = ['nvidia', 'nvidia-interconnect', 'nvidia-bandwidth', 'nvidia-pcie-bandwidth'];
|
||||
const baseTargets = ['cpu', 'memory', 'storage', 'nvidia-config', 'pcie-link'];
|
||||
const amdTargets = selectedAMDValidateTargets();
|
||||
const expanded = [];
|
||||
baseTargets.forEach(t => expanded.push({target: t}));
|
||||
|
||||
@@ -265,6 +265,8 @@ func NewHandler(opts HandlerOptions) http.Handler {
|
||||
mux.HandleFunc("POST /api/sat/memory/run", h.handleAPISATRun("memory"))
|
||||
mux.HandleFunc("POST /api/sat/storage/run", h.handleAPISATRun("storage"))
|
||||
mux.HandleFunc("POST /api/sat/nvidia-config/run", h.handleAPISATRun("nvidia-config"))
|
||||
mux.HandleFunc("POST /api/sat/pcie-link/run", h.handleAPISATRun("pcie-link"))
|
||||
mux.HandleFunc("POST /api/sat/nvidia-pcie-bandwidth/run", h.handleAPISATRun("nvidia-pcie-bandwidth"))
|
||||
mux.HandleFunc("POST /api/sat/cpu/run", h.handleAPISATRun("cpu"))
|
||||
mux.HandleFunc("POST /api/sat/amd/run", h.handleAPISATRun("amd"))
|
||||
mux.HandleFunc("POST /api/sat/amd-mem/run", h.handleAPISATRun("amd-mem"))
|
||||
|
||||
@@ -294,6 +294,18 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
|
||||
break
|
||||
}
|
||||
archive, err = runNvidiaConfigCheckPackCtx(a, ctx, "", j.append)
|
||||
case "pcie-link":
|
||||
if a == nil {
|
||||
err = fmt.Errorf("app not configured")
|
||||
break
|
||||
}
|
||||
archive, err = runPCIeLinkCheckPackCtx(a, ctx, "", j.append)
|
||||
case "nvidia-pcie-bandwidth":
|
||||
if a == nil {
|
||||
err = fmt.Errorf("app not configured")
|
||||
break
|
||||
}
|
||||
archive, err = runNvidiaPCIeBandwidthPackCtx(a, ctx, "", t.params.GPUIndices, j.append)
|
||||
case "cpu":
|
||||
if a == nil {
|
||||
err = fmt.Errorf("app not configured")
|
||||
|
||||
@@ -46,6 +46,8 @@ var taskNames = map[string]string{
|
||||
"memory": "Memory SAT",
|
||||
"storage": "Storage SAT",
|
||||
"nvidia-config": "GPU Config & NVLink Check",
|
||||
"pcie-link": "PCIe Link Check (forced retrain)",
|
||||
"nvidia-pcie-bandwidth": "NVIDIA GPU PCIe Bandwidth Check",
|
||||
"cpu": "CPU SAT",
|
||||
"amd": "AMD GPU SAT",
|
||||
"amd-mem": "AMD GPU MEM Integrity",
|
||||
@@ -317,6 +319,12 @@ var (
|
||||
runNvidiaConfigCheckPackCtx = func(a *app.App, ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
return a.RunNvidiaConfigCheckPackCtx(ctx, baseDir, logFunc)
|
||||
}
|
||||
runPCIeLinkCheckPackCtx = func(a *app.App, ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
return a.RunPCIeLinkCheckPackCtx(ctx, baseDir, logFunc)
|
||||
}
|
||||
runNvidiaPCIeBandwidthPackCtx = func(a *app.App, ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
return a.RunNvidiaPCIeBandwidthPackCtx(ctx, baseDir, gpuIndices, logFunc)
|
||||
}
|
||||
runCPUAcceptancePackCtx = func(a *app.App, ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
return a.RunCPUAcceptancePackCtx(ctx, baseDir, durationSec, logFunc)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user