bee/audit/internal/platform/nvidia_pcie_bandwidth.go

package platform

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"
)

// nvidiaPCIeBandwidthFinding is one GPU's post-load PCIe link-speed result.
type nvidiaPCIeBandwidthFinding struct {
	Index       int
	BDF         string
	BeforeSpeed string
	AfterSpeed  string
	MaxSpeed    string
	Width       int
	MaxWidth    int
	Degraded    bool
}

// RunNvidiaPCIeBandwidthPack drives real host<->device traffic across the
// GPUs' PCIe links (via `dcgmi diag -r nvbandwidth`, the same tool the
// existing nvidia-bandwidth SAT uses for GPU-to-GPU throughput) and then
// resamples each GPU's PCIe link speed from sysfs immediately afterward.
//
// This is deliberately a separate, narrower check from "nvidia-bandwidth":
// that SAT's overall_status reflects nvbandwidth's own pass/fail (did the
// measured GB/s clear its internal threshold), never PCIe link speed
// itself. This pack exists purely to answer "did sustained real traffic
// make the link train up to its negotiated maximum" — the load-bearing
// counterpart to the idle-time collector reading that used to (falsely)
// drive pcie:gpu:nvidia's status. See
// bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning-unresolved.md.
func (s *System) RunNvidiaPCIeBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
	if ctx == nil {
		ctx = context.Background()
	}
	if baseDir == "" {
		baseDir = "/var/log/bee-sat"
	}
	ts := time.Now().UTC().Format("20060102-150405")
	runDir := filepath.Join(baseDir, "nvidia-pcie-bandwidth-"+ts)
	if err := os.MkdirAll(runDir, 0755); err != nil {
		return "", err
	}
	verboseLog := filepath.Join(runDir, "verbose.log")

	bdfByIndex, err := gpuIndexToBDF(gpuIndices)
	if err != nil {
		return "", fmt.Errorf("resolve GPU BDFs: %w", err)
	}
	indices := make([]int, 0, len(bdfByIndex))
	for idx := range bdfByIndex {
		indices = append(indices, idx)
	}
	sort.Ints(indices)

	findings := make([]nvidiaPCIeBandwidthFinding, 0, len(indices))
	for _, idx := range indices {
		bdf := bdfByIndex[idx]
		before, _ := readPCIeSysfsString(bdf, "current_link_speed")
		max, _ := readPCIeSysfsString(bdf, "max_link_speed")
		maxWidth, _ := readPCIeSysfsInt(bdf, "max_link_width")
		findings = append(findings, nvidiaPCIeBandwidthFinding{
			Index: idx, BDF: bdf, BeforeSpeed: before, MaxSpeed: max, MaxWidth: maxWidth,
		})
	}

	cmd := []string{"dcgmi", "diag", "-r", "nvbandwidth"}
	if len(indices) > 0 {
		cmd = append(cmd, "-i", joinIndexList(indices))
	}
	out, runErr := runSATCommandCtx(ctx, verboseLog, "dcgmi-nvbandwidth", cmd, nil, logFunc)
	_ = os.WriteFile(filepath.Join(runDir, "01-dcgmi-nvbandwidth.log"), out, 0644)

	for i := range findings {
		bdf := findings[i].BDF
		after, _ := readPCIeSysfsString(bdf, "current_link_speed")
		width, _ := readPCIeSysfsInt(bdf, "current_link_width")
		findings[i].AfterSpeed = after
		findings[i].Width = width
		findings[i].Degraded = after != findings[i].MaxSpeed
	}

	summary := renderNvidiaPCIeBandwidthSummary(findings, runErr)
	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
		return "", err
	}
	report := renderNvidiaPCIeBandwidthReport(findings, runErr)
	if err := os.WriteFile(filepath.Join(runDir, "nvidia-pcie-bandwidth-report.txt"), []byte(report), 0644); err != nil {
		return "", err
	}
	return runDir, nil
}

// gpuIndexToBDF resolves each of gpuIndices to its PCI BDF via nvidia-smi.
// An empty gpuIndices resolves every GPU nvidia-smi reports.
func gpuIndexToBDF(gpuIndices []int) (map[int]string, error) {
	out, err := satExecCommand("nvidia-smi", "--query-gpu=index,pci.bus_id", "--format=csv,noheader,nounits").Output()
	if err != nil {
		return nil, fmt.Errorf("nvidia-smi: %w", err)
	}
	want := make(map[int]struct{}, len(gpuIndices))
	for _, idx := range gpuIndices {
		want[idx] = struct{}{}
	}
	filterByIndex := len(want) > 0

	result := make(map[int]string)
	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
		fields := strings.SplitN(line, ",", 2)
		if len(fields) != 2 {
			continue
		}
		idx, err := strconv.Atoi(strings.TrimSpace(fields[0]))
		if err != nil {
			continue
		}
		if filterByIndex {
			if _, ok := want[idx]; !ok {
				continue
			}
		}
		result[idx] = normalizeNvidiaBDF(strings.TrimSpace(fields[1]))
	}
	return result, nil
}

func renderNvidiaPCIeBandwidthSummary(findings []nvidiaPCIeBandwidthFinding, runErr error) string {
	var b strings.Builder
	fmt.Fprintf(&b, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
	fmt.Fprintf(&b, "gpu_count=%d\n", len(findings))
	degraded := 0
	for _, f := range findings {
		fmt.Fprintf(&b, "gpu%d_status=%s\n", f.Index, statusLabel(!f.Degraded))
		if f.Degraded {
			degraded++
		}
	}
	fmt.Fprintf(&b, "degraded=%d\n", degraded)
	if runErr != nil {
		fmt.Fprintf(&b, "dcgmi_nvbandwidth_error=%s\n", runErr.Error())
	}
	if degraded > 0 {
		fmt.Fprintln(&b, "overall_status=FAILED")
		var reasons []string
		for _, f := range findings {
			if f.Degraded {
				reasons = append(reasons, fmt.Sprintf("GPU%d (%s): still at %s under load, capable of %s",
					f.Index, f.BDF, f.AfterSpeed, f.MaxSpeed))
			}
		}
		fmt.Fprintf(&b, "warnings=%s\n", strings.Join(reasons, "; "))
	} else {
		fmt.Fprintln(&b, "overall_status=OK")
	}
	return b.String()
}

func statusLabel(ok bool) string {
	if ok {
		return "OK"
	}
	return "FAILED"
}

func renderNvidiaPCIeBandwidthReport(findings []nvidiaPCIeBandwidthFinding, runErr error) string {
	var b strings.Builder
	line := strings.Repeat("=", 80)
	b.WriteString(line + "\n")
	b.WriteString("NVIDIA GPU PCIe Bandwidth / Link-Under-Load Check\n")
	b.WriteString(line + "\n\n")
	if runErr != nil {
		fmt.Fprintf(&b, "dcgmi diag -r nvbandwidth: %s\n\n", runErr.Error())
	}
	for _, f := range findings {
		verdict := "OK"
		if f.Degraded {
			verdict = "DEGRADED"
		}
		fmt.Fprintf(&b, "GPU%d (%s): %s\n", f.Index, f.BDF, verdict)
		fmt.Fprintf(&b, "    before=%s after=%s max=%s width=%d/%d\n",
			f.BeforeSpeed, f.AfterSpeed, f.MaxSpeed, f.Width, f.MaxWidth)
	}
	return b.String()
}