fix(pcie): verify GPU links under real bandwidth load

This commit is contained in:
Mikhail Chusavitin
2026-09-03 10:23:39 +03:00
parent 642e68631d
commit 347bc8310a
27 changed files with 384 additions and 902 deletions
+53 -23
View File
@@ -125,6 +125,55 @@ func lastEntryFromSource(history []ComponentStatusEntry, source string) *Compone
return nil
}
// recomputeIgnoringSources retires obsolete producers from the current
// component verdict without deleting their audit history. It selects the
// highest-severity latest observation from every remaining source.
func (db *ComponentStatusDB) recomputeIgnoringSources(key string, sources ...string) {
if db == nil {
return
}
ignored := make(map[string]struct{}, len(sources))
for _, source := range sources {
ignored[source] = struct{}{}
}
db.mu.Lock()
defer db.mu.Unlock()
db.reloadLocked()
rec := db.records[key]
if rec == nil {
return
}
latest := make(map[string]ComponentStatusEntry)
for i := len(rec.History) - 1; i >= 0; i-- {
entry := rec.History[i]
if _, skip := ignored[entry.Source]; skip {
continue
}
if _, seen := latest[entry.Source]; !seen {
latest[entry.Source] = entry
}
}
var winner *ComponentStatusEntry
for _, entry := range latest {
candidate := entry
if winner == nil || componentSeverity(candidate.Status) > componentSeverity(winner.Status) ||
(componentSeverity(candidate.Status) == componentSeverity(winner.Status) && candidate.At.After(winner.At)) {
winner = &candidate
}
}
if winner == nil {
return
}
if rec.Status != winner.Status || rec.ErrorSummary != winner.Detail {
rec.LastChangedAt = time.Now().UTC()
}
rec.Status = winner.Status
rec.ErrorSummary = winner.Detail
_ = db.saveLocked()
}
// Get returns the current record for a component key.
func (db *ComponentStatusDB) Get(key string) (ComponentStatusRecord, bool) {
if db == nil {
@@ -258,32 +307,13 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
// otherwise fails to match any real BDF.
switch target {
case "nvidia", "nvidia-targeted-stress", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "nvidia-config", "nvidia-pcie-bandwidth":
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "nvidia-config":
db.Record("pcie:gpu:nvidia", source, dbStatus, detail)
if target == "nvidia-bandwidth" {
db.recomputeIgnoringSources("pcie:gpu:nvidia", "sat:pcie-link", "sat:nvidia-pcie-bandwidth")
}
case "amd", "amd-stress", "amd-mem", "amd-bandwidth":
db.Record("pcie:gpu:amd", source, dbStatus, detail)
case "pcie-link":
// Forced-retrain PCIe link check (audit/internal/platform/pcie_link_check.go):
// the only verified (non-idle-sampled) source for PCIe link-speed
// status. summary.txt carries up to three independent sub-verdicts
// — record each into its own component key rather than collapsing
// them into one, since a degraded NIC/HBA shouldn't be reported as
// a GPU fault or vice versa.
recordPCIeLinkSubStatus := func(key, kvKey string) {
v, ok := kv[kvKey]
if !ok {
return
}
st := strings.ToUpper(strings.TrimSpace(v))
d := "pcie-link SAT: " + st
if st != "OK" && kv["warnings"] != "" {
d += " — " + kv["warnings"]
}
db.Record(key, source, satStatusToDBStatus(st), d)
}
recordPCIeLinkSubStatus("pcie:gpu:nvidia", "gpu_nvidia_status")
recordPCIeLinkSubStatus("pcie:gpu:amd", "gpu_amd_status")
recordPCIeLinkSubStatus("pcie:link:other", "other_status")
case "memory", "memory-stress", "sat-stress":
db.Record("memory:all", source, dbStatus, detail)
case "cpu", "platform-stress":