fix(pcie): verify GPU links under real bandwidth load
This commit is contained in:
@@ -125,6 +125,55 @@ func lastEntryFromSource(history []ComponentStatusEntry, source string) *Compone
|
||||
return nil
|
||||
}
|
||||
|
||||
// recomputeIgnoringSources retires obsolete producers from the current
|
||||
// component verdict without deleting their audit history. It selects the
|
||||
// highest-severity latest observation from every remaining source.
|
||||
func (db *ComponentStatusDB) recomputeIgnoringSources(key string, sources ...string) {
|
||||
if db == nil {
|
||||
return
|
||||
}
|
||||
ignored := make(map[string]struct{}, len(sources))
|
||||
for _, source := range sources {
|
||||
ignored[source] = struct{}{}
|
||||
}
|
||||
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
db.reloadLocked()
|
||||
rec := db.records[key]
|
||||
if rec == nil {
|
||||
return
|
||||
}
|
||||
|
||||
latest := make(map[string]ComponentStatusEntry)
|
||||
for i := len(rec.History) - 1; i >= 0; i-- {
|
||||
entry := rec.History[i]
|
||||
if _, skip := ignored[entry.Source]; skip {
|
||||
continue
|
||||
}
|
||||
if _, seen := latest[entry.Source]; !seen {
|
||||
latest[entry.Source] = entry
|
||||
}
|
||||
}
|
||||
var winner *ComponentStatusEntry
|
||||
for _, entry := range latest {
|
||||
candidate := entry
|
||||
if winner == nil || componentSeverity(candidate.Status) > componentSeverity(winner.Status) ||
|
||||
(componentSeverity(candidate.Status) == componentSeverity(winner.Status) && candidate.At.After(winner.At)) {
|
||||
winner = &candidate
|
||||
}
|
||||
}
|
||||
if winner == nil {
|
||||
return
|
||||
}
|
||||
if rec.Status != winner.Status || rec.ErrorSummary != winner.Detail {
|
||||
rec.LastChangedAt = time.Now().UTC()
|
||||
}
|
||||
rec.Status = winner.Status
|
||||
rec.ErrorSummary = winner.Detail
|
||||
_ = db.saveLocked()
|
||||
}
|
||||
|
||||
// Get returns the current record for a component key.
|
||||
func (db *ComponentStatusDB) Get(key string) (ComponentStatusRecord, bool) {
|
||||
if db == nil {
|
||||
@@ -258,32 +307,13 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
|
||||
// otherwise fails to match any real BDF.
|
||||
switch target {
|
||||
case "nvidia", "nvidia-targeted-stress", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
|
||||
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "nvidia-config", "nvidia-pcie-bandwidth":
|
||||
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "nvidia-config":
|
||||
db.Record("pcie:gpu:nvidia", source, dbStatus, detail)
|
||||
if target == "nvidia-bandwidth" {
|
||||
db.recomputeIgnoringSources("pcie:gpu:nvidia", "sat:pcie-link", "sat:nvidia-pcie-bandwidth")
|
||||
}
|
||||
case "amd", "amd-stress", "amd-mem", "amd-bandwidth":
|
||||
db.Record("pcie:gpu:amd", source, dbStatus, detail)
|
||||
case "pcie-link":
|
||||
// Forced-retrain PCIe link check (audit/internal/platform/pcie_link_check.go):
|
||||
// the only verified (non-idle-sampled) source for PCIe link-speed
|
||||
// status. summary.txt carries up to three independent sub-verdicts
|
||||
// — record each into its own component key rather than collapsing
|
||||
// them into one, since a degraded NIC/HBA shouldn't be reported as
|
||||
// a GPU fault or vice versa.
|
||||
recordPCIeLinkSubStatus := func(key, kvKey string) {
|
||||
v, ok := kv[kvKey]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
st := strings.ToUpper(strings.TrimSpace(v))
|
||||
d := "pcie-link SAT: " + st
|
||||
if st != "OK" && kv["warnings"] != "" {
|
||||
d += " — " + kv["warnings"]
|
||||
}
|
||||
db.Record(key, source, satStatusToDBStatus(st), d)
|
||||
}
|
||||
recordPCIeLinkSubStatus("pcie:gpu:nvidia", "gpu_nvidia_status")
|
||||
recordPCIeLinkSubStatus("pcie:gpu:amd", "gpu_amd_status")
|
||||
recordPCIeLinkSubStatus("pcie:link:other", "other_status")
|
||||
case "memory", "memory-stress", "sat-stress":
|
||||
db.Record("memory:all", source, dbStatus, detail)
|
||||
case "cpu", "platform-stress":
|
||||
|
||||
Reference in New Issue
Block a user