bee/audit/internal/platform/pcie_link_check.go

package platform

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"bee/audit/internal/collector"
)

// pcieLinkRetrainTimeout bounds how long we wait for a device to finish
// retraining (clear the Link Status "Link Training" bit) before giving up
// and reading whatever speed it settled on anyway. The PCIe spec allows up
// to 100ms for Gen1-3 and longer for higher generations with equalization;
// this is generous headroom above that.
const pcieLinkRetrainTimeout = 2 * time.Second

// pcieLinkFinding is one device's before/after link-speed retrain result.
type pcieLinkFinding struct {
	BDF         string
	Description string
	VendorID    string
	ClassCode   string
	IsGPU       bool
	GPUVendor   string // "nvidia" or "amd", only set when IsGPU
	Skipped     string // non-empty reason this device wasn't retrained
	BeforeSpeed string
	AfterSpeed  string
	MaxSpeed    string
	Width       int
	MaxWidth    int
	Degraded    bool
	NotPresent  bool // true when the slot trained to zero lanes: nothing is plugged in (or it fell off the bus), not a speed regression
}

// RunPCIeLinkCheckPack forces every enabled PCIe device to retrain its link
// (via the PCIe Link Control register's spec-defined Retrain Link bit — see
// PCIe base spec, Link Control Register, bit 5) and compares the
// post-retrain negotiated speed against the device's own reported maximum.
//
// This exists because a plain idle-time sysfs read of current_link_speed is
// not a reliable fault signal: NVIDIA GPUs (and other devices with runtime
// power management) deliberately downclock their PCIe link to save power
// while idle, which is indistinguishable from a real degraded slot/riser/
// cable without either sustained traffic or a forced retrain. Forcing a
// retrain sidesteps needing a device-specific load generator (bee-gpu-burn
// exists for GPUs; nothing plays that role for NICs, HBAs, or PCIe
// switches) — retraining is a PCIe-spec mechanism every endpoint supports,
// so this one check covers every PCIe device in the machine, not just
// GPUs. See bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning-unresolved.md
// for the history of narrower attempts that didn't generalize.
//
// Disabled devices (sysfs enable==0 — e.g. PCIe fabric-management endpoints
// the kernel never activates, per the 2026-06-12 decision) are left alone:
// they carry no data traffic, so there is nothing to verify and no reason
// to poke them.
func (s *System) RunPCIeLinkCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
	if ctx == nil {
		ctx = context.Background()
	}
	if baseDir == "" {
		baseDir = "/var/log/bee-sat"
	}
	ts := time.Now().UTC().Format("20060102-150405")
	runDir := filepath.Join(baseDir, "pcie-link-"+ts)
	if err := os.MkdirAll(runDir, 0755); err != nil {
		return "", err
	}
	verboseLog := filepath.Join(runDir, "verbose.log")

	bdfs, err := listPCIDeviceBDFs()
	if err != nil {
		return "", fmt.Errorf("list PCI devices: %w", err)
	}

	var findings []pcieLinkFinding
	for _, bdf := range bdfs {
		if logFunc != nil {
			logFunc(fmt.Sprintf("=== %s ===", bdf))
		}
		f := retrainAndSamplePCIeDevice(ctx, verboseLog, bdf, logFunc)
		findings = append(findings, f)
	}

	summary := renderPCIeLinkCheckSummary(findings)
	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
		return "", err
	}
	report := renderPCIeLinkCheckReport(findings)
	if err := os.WriteFile(filepath.Join(runDir, "pcie-link-report.txt"), []byte(report), 0644); err != nil {
		return "", err
	}
	return runDir, nil
}

// listPCIDeviceBDFs returns every BDF under /sys/bus/pci/devices, sorted for
// deterministic report ordering.
func listPCIDeviceBDFs() ([]string, error) {
	entries, err := os.ReadDir("/sys/bus/pci/devices")
	if err != nil {
		return nil, err
	}
	bdfs := make([]string, 0, len(entries))
	for _, e := range entries {
		bdfs = append(bdfs, e.Name())
	}
	sort.Strings(bdfs)
	return bdfs, nil
}

func retrainAndSamplePCIeDevice(ctx context.Context, verboseLog, bdf string, logFunc func(string)) pcieLinkFinding {
	f := pcieLinkFinding{BDF: bdf}

	vendor, _ := readPCIeSysfsHex(bdf, "vendor")
	class, _ := readPCIeSysfsHex(bdf, "class")
	f.VendorID = vendor
	f.ClassCode = class
	f.IsGPU, f.GPUVendor = classifyGPUFromVendorClass(vendor, class)
	f.Description = pcieDeviceDescription(ctx, verboseLog, bdf, logFunc)

	if enabled, ok := readPCIeSysfsInt(bdf, "enable"); ok && enabled == 0 {
		f.Skipped = "device disabled (no data traffic; link state has no operational impact)"
		return f
	}

	before, beforeOK := readPCIeSysfsString(bdf, "current_link_speed")
	maxSpeed, maxOK := readPCIeSysfsString(bdf, "max_link_speed")
	width, _ := readPCIeSysfsInt(bdf, "current_link_width")
	maxWidth, _ := readPCIeSysfsInt(bdf, "max_link_width")
	f.BeforeSpeed = before
	f.MaxSpeed = maxSpeed
	f.MaxWidth = maxWidth
	if !beforeOK || !maxOK {
		f.Skipped = "no PCIe link-speed attributes in sysfs (not a link-trained endpoint)"
		return f
	}

	if width == 0 {
		// A downstream switch/root port with nothing seated reads zero
		// trained lanes even before we touch it. Plenty of legitimate
		// configs leave slots like this unpopulated (not every server ships
		// every NIC/riser slot filled), so this is not by itself evidence
		// of anything wrong — retraining an empty slot can't produce a
		// meaningful speed reading, and there's no baseline here to say
		// "this used to have a card." Skip it exactly like a disabled
		// device: nothing to verify, no reason to fail the run over it.
		f.NotPresent = true
		f.Skipped = "no device present downstream (empty slot/riser — nothing to retrain)"
		return f
	}

	if err := retrainPCIeLink(ctx, verboseLog, bdf, logFunc); err != nil {
		f.Skipped = "retrain failed: " + err.Error()
		f.AfterSpeed = before
		f.Width = width
		// before/maxSpeed are already normalized "GenN" labels (see
		// readPCIeSysfsString) — compare directly, don't re-normalize.
		f.Degraded = before != maxSpeed
		return f
	}

	after, _ := readPCIeSysfsString(bdf, "current_link_speed")
	widthAfter, _ := readPCIeSysfsInt(bdf, "current_link_width")
	if widthAfter == 0 {
		// The device answered before the retrain but is gone immediately
		// after it (fell off the bus mid-check) — unlike the pre-retrain
		// case above, this had a live link a moment ago, so it's worth
		// surfacing rather than silently skipping.
		f.AfterSpeed = after
		f.Width = widthAfter
		f.NotPresent = true
		f.Degraded = true
		return f
	}
	f.AfterSpeed = after
	f.Width = widthAfter
	f.Degraded = after != maxSpeed
	return f
}

// retrainPCIeLink sets the Retrain Link bit (bit 5) of the PCI Express
// Capability's Link Control register via setpci, then polls the Link
// Status register's Link Training bit (bit 11) until it clears or
// pcieLinkRetrainTimeout elapses.
func retrainPCIeLink(ctx context.Context, verboseLog, bdf string, logFunc func(string)) error {
	linkCtrlOut, err := runSATCommandCtx(ctx, verboseLog, "setpci-read-"+bdf,
		[]string{"setpci", "-s", bdf, "CAP_EXP+0x10.w"}, nil, logFunc)
	if err != nil {
		return fmt.Errorf("read Link Control: %w", err)
	}
	cur, err := strconv.ParseUint(strings.TrimSpace(string(linkCtrlOut)), 16, 16)
	if err != nil {
		return fmt.Errorf("parse Link Control %q: %w", linkCtrlOut, err)
	}
	const retrainLinkBit = 0x0020
	newVal := uint16(cur) | retrainLinkBit

	if _, err := runSATCommandCtx(ctx, verboseLog, "setpci-retrain-"+bdf,
		[]string{"setpci", "-s", bdf, fmt.Sprintf("CAP_EXP+0x10.w=%04x", newVal)}, nil, logFunc); err != nil {
		return fmt.Errorf("write Retrain Link bit: %w", err)
	}

	const linkTrainingBit = 0x0800
	deadline := time.Now().Add(pcieLinkRetrainTimeout)
	for time.Now().Before(deadline) {
		statusOut, err := runSATCommandCtx(ctx, verboseLog, "setpci-status-"+bdf,
			[]string{"setpci", "-s", bdf, "CAP_EXP+0x12.w"}, nil, nil)
		if err == nil {
			if status, perr := strconv.ParseUint(strings.TrimSpace(string(statusOut)), 16, 16); perr == nil {
				if status&linkTrainingBit == 0 {
					return nil
				}
			}
		}
		time.Sleep(50 * time.Millisecond)
	}
	// Timed out waiting for training to clear; the caller still samples
	// whatever speed sysfs reports, which is the honest answer either way.
	return nil
}

func pcieDeviceDescription(ctx context.Context, verboseLog, bdf string, logFunc func(string)) string {
	out, err := runSATCommandCtx(ctx, verboseLog, "lspci-"+bdf, []string{"lspci", "-s", bdf}, nil, logFunc)
	if err != nil {
		return ""
	}
	line := strings.TrimSpace(string(out))
	if idx := strings.Index(line, "\n"); idx >= 0 {
		line = line[:idx]
	}
	if idx := strings.Index(line, " "); idx >= 0 {
		return strings.TrimSpace(line[idx+1:])
	}
	return line
}

func readPCIeSysfsString(bdf, attr string) (string, bool) {
	raw, err := os.ReadFile(filepath.Join("/sys/bus/pci/devices", bdf, attr))
	if err != nil {
		return "", false
	}
	v := strings.TrimSpace(string(raw))
	if v == "" {
		return "", false
	}
	return collector.NormalizePCILinkSpeed(v), true
}

func readPCIeSysfsInt(bdf, attr string) (int, bool) {
	raw, err := os.ReadFile(filepath.Join("/sys/bus/pci/devices", bdf, attr))
	if err != nil {
		return 0, false
	}
	v, err := strconv.Atoi(strings.TrimSpace(string(raw)))
	if err != nil || v < 0 {
		return 0, false
	}
	return v, true
}

func readPCIeSysfsHex(bdf, attr string) (string, bool) {
	raw, err := os.ReadFile(filepath.Join("/sys/bus/pci/devices", bdf, attr))
	if err != nil {
		return "", false
	}
	return strings.TrimSpace(string(raw)), true
}

// classifyGPUFromVendorClass reports whether a device is a GPU die itself
// (PCI base class 0x03 — Display Controller — under NVIDIA/AMD's vendor
// ID), as opposed to a same-vendor companion device (NIC, storage
// controller, NVLink bridge) that shares the GPU's PCI vendor ID. Class-code
// based, not name-substring based, per the same reasoning as
// collector.IsGPUClass.
func classifyGPUFromVendorClass(vendorHex, classHex string) (isGPU bool, vendor string) {
	v := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(vendorHex)), "0x")
	c := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(classHex)), "0x")
	if len(c) < 2 || c[:2] != "03" {
		return false, ""
	}
	switch v {
	case "10de":
		return true, "nvidia"
	case "1002":
		return true, "amd"
	default:
		return false, ""
	}
}

func renderPCIeLinkCheckSummary(findings []pcieLinkFinding) string {
	var b strings.Builder
	fmt.Fprintf(&b, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
	fmt.Fprintf(&b, "devices_tested=%d\n", len(findings))

	gpuStatus := map[string]string{} // vendor -> OK/FAILED
	otherDegraded := 0
	otherTested := 0
	anyDegraded := false
	for _, f := range findings {
		if f.Skipped != "" && f.AfterSpeed == "" {
			continue
		}
		if f.IsGPU {
			if _, ok := gpuStatus[f.GPUVendor]; !ok {
				gpuStatus[f.GPUVendor] = "OK"
			}
			if f.Degraded {
				gpuStatus[f.GPUVendor] = "FAILED"
				anyDegraded = true
			}
			continue
		}
		otherTested++
		if f.Degraded {
			otherDegraded++
			anyDegraded = true
		}
	}
	for _, vendor := range []string{"nvidia", "amd"} {
		if status, ok := gpuStatus[vendor]; ok {
			fmt.Fprintf(&b, "gpu_%s_status=%s\n", vendor, status)
		}
	}
	fmt.Fprintf(&b, "other_devices_tested=%d\n", otherTested)
	fmt.Fprintf(&b, "other_devices_degraded=%d\n", otherDegraded)
	if otherTested > 0 {
		if otherDegraded > 0 {
			fmt.Fprintln(&b, "other_status=FAILED")
		} else {
			fmt.Fprintln(&b, "other_status=OK")
		}
	}

	if anyDegraded {
		fmt.Fprintln(&b, "overall_status=FAILED")
		var reasons []string
		for _, f := range findings {
			if !f.Degraded {
				continue
			}
			if f.NotPresent {
				reasons = append(reasons, fmt.Sprintf("%s (%s): no device detected downstream (link down / empty slot or riser, capable of %s)",
					f.BDF, nonEmptyOr(f.Description, "unknown device"), f.MaxSpeed))
				continue
			}
			reasons = append(reasons, fmt.Sprintf("%s (%s): retrained to %s, capable of %s",
				f.BDF, nonEmptyOr(f.Description, "unknown device"), f.AfterSpeed, f.MaxSpeed))
		}
		fmt.Fprintf(&b, "warnings=%s\n", strings.Join(reasons, "; "))
	} else {
		fmt.Fprintln(&b, "overall_status=OK")
	}
	return b.String()
}

func renderPCIeLinkCheckReport(findings []pcieLinkFinding) string {
	var b strings.Builder
	line := strings.Repeat("=", 80)
	b.WriteString(line + "\n")
	b.WriteString("PCIe Link Retrain Check\n")
	b.WriteString(line + "\n\n")
	for _, f := range findings {
		fmt.Fprintf(&b, "%s  %s\n", f.BDF, nonEmptyOr(f.Description, "(unknown device)"))
		if f.Skipped != "" && f.AfterSpeed == "" {
			fmt.Fprintf(&b, "    skipped: %s\n", f.Skipped)
			continue
		}
		verdict := "OK"
		switch {
		case f.NotPresent:
			verdict = "FELL OFF BUS"
		case f.Degraded:
			verdict = "DEGRADED"
		}
		fmt.Fprintf(&b, "    %s: before=%s after=%s max=%s width=%d/%d\n",
			verdict, f.BeforeSpeed, f.AfterSpeed, f.MaxSpeed, f.Width, f.MaxWidth)
		if f.Skipped != "" {
			fmt.Fprintf(&b, "    note: %s\n", f.Skipped)
		}
	}
	return b.String()
}