fix(pcie): stop misreporting empty/vanished slots as speed degradation
pcie-link's retrain check compared post-retrain speed against a port's max capability without checking whether anything was actually seated downstream. A switch downstream port with no card plugged in retrains to zero lanes but still reports a reset-state speed, so it was misreported as "retrained to Gen1, capable of Gen5" — indistinguishable from a real degraded link. Now a port that was already empty before the retrain is skipped (same treatment as a disabled device: legitimately unpopulated slots are common and shouldn't fail the check), while a device that answered before the retrain and disappeared right after it still fails, with an honest "no device detected downstream" message instead of a fabricated speed reading. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
ccc781856e
commit
9e466b8a70
@@ -35,6 +35,7 @@ type pcieLinkFinding struct {
|
||||
Width int
|
||||
MaxWidth int
|
||||
Degraded bool
|
||||
NotPresent bool // true when the slot trained to zero lanes: nothing is plugged in (or it fell off the bus), not a speed regression
|
||||
}
|
||||
|
||||
// RunPCIeLinkCheckPack forces every enabled PCIe device to retrain its link
|
||||
@@ -139,6 +140,20 @@ func retrainAndSamplePCIeDevice(ctx context.Context, verboseLog, bdf string, log
|
||||
return f
|
||||
}
|
||||
|
||||
if width == 0 {
|
||||
// A downstream switch/root port with nothing seated reads zero
|
||||
// trained lanes even before we touch it. Plenty of legitimate
|
||||
// configs leave slots like this unpopulated (not every server ships
|
||||
// every NIC/riser slot filled), so this is not by itself evidence
|
||||
// of anything wrong — retraining an empty slot can't produce a
|
||||
// meaningful speed reading, and there's no baseline here to say
|
||||
// "this used to have a card." Skip it exactly like a disabled
|
||||
// device: nothing to verify, no reason to fail the run over it.
|
||||
f.NotPresent = true
|
||||
f.Skipped = "no device present downstream (empty slot/riser — nothing to retrain)"
|
||||
return f
|
||||
}
|
||||
|
||||
if err := retrainPCIeLink(ctx, verboseLog, bdf, logFunc); err != nil {
|
||||
f.Skipped = "retrain failed: " + err.Error()
|
||||
f.AfterSpeed = before
|
||||
@@ -151,6 +166,17 @@ func retrainAndSamplePCIeDevice(ctx context.Context, verboseLog, bdf string, log
|
||||
|
||||
after, _ := readPCIeSysfsString(bdf, "current_link_speed")
|
||||
widthAfter, _ := readPCIeSysfsInt(bdf, "current_link_width")
|
||||
if widthAfter == 0 {
|
||||
// The device answered before the retrain but is gone immediately
|
||||
// after it (fell off the bus mid-check) — unlike the pre-retrain
|
||||
// case above, this had a live link a moment ago, so it's worth
|
||||
// surfacing rather than silently skipping.
|
||||
f.AfterSpeed = after
|
||||
f.Width = widthAfter
|
||||
f.NotPresent = true
|
||||
f.Degraded = true
|
||||
return f
|
||||
}
|
||||
f.AfterSpeed = after
|
||||
f.Width = widthAfter
|
||||
f.Degraded = after != maxSpeed
|
||||
@@ -315,10 +341,16 @@ func renderPCIeLinkCheckSummary(findings []pcieLinkFinding) string {
|
||||
fmt.Fprintln(&b, "overall_status=FAILED")
|
||||
var reasons []string
|
||||
for _, f := range findings {
|
||||
if f.Degraded {
|
||||
reasons = append(reasons, fmt.Sprintf("%s (%s): retrained to %s, capable of %s",
|
||||
f.BDF, nonEmptyOr(f.Description, "unknown device"), f.AfterSpeed, f.MaxSpeed))
|
||||
if !f.Degraded {
|
||||
continue
|
||||
}
|
||||
if f.NotPresent {
|
||||
reasons = append(reasons, fmt.Sprintf("%s (%s): no device detected downstream (link down / empty slot or riser, capable of %s)",
|
||||
f.BDF, nonEmptyOr(f.Description, "unknown device"), f.MaxSpeed))
|
||||
continue
|
||||
}
|
||||
reasons = append(reasons, fmt.Sprintf("%s (%s): retrained to %s, capable of %s",
|
||||
f.BDF, nonEmptyOr(f.Description, "unknown device"), f.AfterSpeed, f.MaxSpeed))
|
||||
}
|
||||
fmt.Fprintf(&b, "warnings=%s\n", strings.Join(reasons, "; "))
|
||||
} else {
|
||||
@@ -340,7 +372,10 @@ func renderPCIeLinkCheckReport(findings []pcieLinkFinding) string {
|
||||
continue
|
||||
}
|
||||
verdict := "OK"
|
||||
if f.Degraded {
|
||||
switch {
|
||||
case f.NotPresent:
|
||||
verdict = "FELL OFF BUS"
|
||||
case f.Degraded:
|
||||
verdict = "DEGRADED"
|
||||
}
|
||||
fmt.Fprintf(&b, " %s: before=%s after=%s max=%s width=%d/%d\n",
|
||||
|
||||
Reference in New Issue
Block a user