Fixes #20. This onekeylog variant has no devicefrusdr.log at all: FRU/sensors come from raw ipmitool text output, PCIe/GPU presence has a dedicated structural snapshot, SEL lives at a different path, and BMC component failures are logged separately from SEL/IDL. - Fall back to component/fru.txt (same FRU block format as devicefrusdr.log) and component/sensor.txt / sdr.txt (ipmitool sensor list / sdr elist) when devicefrusdr.log is absent. - Parse log/bmc/diagnose/OtrdDiagnoseComponent.json's PCIe Device Info array for GPU/PCIe presence and link state, independent of SEL/IDL alarm history; flag devices running below their negotiated max link speed/width as degraded with a Warning event. - Fall back to log/sel.csv (same format as selelist.csv) when selelist.csv is absent. - Parse log/bmc/commer-comp/{commerslot,commerhmc,commerswvr,commerswcpld} logs (including rotated *.tar.gz.N parts) into failure events, filtering known-noisy lines. - Collapse SEL events duplicated across sources by (timestamp, event_type, description). - Surface a CollectionError when FRU/sensors are still empty after all fallbacks, instead of silently returning an empty inventory. - Fix ParseFRU: a later placeholder "Product Serial : 0" / "Product Part Number : NULL" line in the same FRU block (e.g. SCM_FRU) was overwriting an already-parsed real Board Serial/Part Number. Verified against dump_23DB01633_20260727-1359.tar.gz (HGX B200, KR9288-X3): fru 0→21, sensors 0→303, 8 GPUs present at Gen5 x16 in slots 100-107. Deferred (not covered by this change): BIOS-change-settings context and BIOS POST codes from the same layout — see bible-local/10-decisions.md ADL-048. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
122 lines
4.0 KiB
Go
122 lines
4.0 KiB
Go
// Parses log/bmc/diagnose/OtrdDiagnoseComponent.json, an HGX BMC structural
|
|
// snapshot of PCIe topology (present GPUs/NICs/NVMe, negotiated link state).
|
|
// This is a direct point-in-time source for GPU presence and PCIe link
|
|
// degradation, independent of SEL/IDL alarm history.
|
|
package inspur
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"git.mchus.pro/mchus/logpile/internal/models"
|
|
"git.mchus.pro/mchus/logpile/internal/parser/vendors/pciids"
|
|
)
|
|
|
|
// otrdDiagnosePCIeEntry mirrors the relevant fields of "Pcie Device Info"
|
|
// entries in OtrdDiagnoseComponent.json.
|
|
type otrdDiagnosePCIeEntry struct {
|
|
LocString string `json:"LocString"`
|
|
PcieSlot int `json:"PcieSlot"`
|
|
PresentStatus int `json:"PresentStatus"`
|
|
VendorId int `json:"VendorId"`
|
|
DeviceId int `json:"DeviceId"`
|
|
BusNumber int `json:"BusNumber"`
|
|
DeviceNumber int `json:"DeviceNumber"`
|
|
FunctionNumber int `json:"FunctionNumber"`
|
|
CurrentLinkSpeed int `json:"CurrentLinkSpeed"`
|
|
MaxLinkSpeed int `json:"MaxLinkSpeed"`
|
|
NegotiatedLinkWidth int `json:"NegotiatedLinkWidth"`
|
|
MaxLinkWidth int `json:"MaxLinkWidth"`
|
|
SerialNumber *string `json:"SerialNumber"`
|
|
PartNumber *string `json:"PartNumber"`
|
|
}
|
|
|
|
type otrdDiagnoseComponent struct {
|
|
PcieDeviceInfo []otrdDiagnosePCIeEntry `json:"Pcie Device Info"`
|
|
}
|
|
|
|
// ParseOtrdDiagnosePCIe parses the "Pcie Device Info" array from
|
|
// OtrdDiagnoseComponent.json into PCIe device inventory entries.
|
|
func ParseOtrdDiagnosePCIe(content []byte) []models.PCIeDevice {
|
|
var doc otrdDiagnoseComponent
|
|
if err := json.Unmarshal(content, &doc); err != nil {
|
|
return nil
|
|
}
|
|
|
|
devices := make([]models.PCIeDevice, 0, len(doc.PcieDeviceInfo))
|
|
for _, e := range doc.PcieDeviceInfo {
|
|
present := e.PresentStatus == 1
|
|
_, deviceName := pciids.DeviceInfo(e.VendorId, e.DeviceId)
|
|
deviceClass := normalizeModelLabel(deviceName)
|
|
if strings.Contains(strings.ToUpper(deviceName), "NVIDIA") {
|
|
deviceClass = "GPU (" + deviceName + ")"
|
|
}
|
|
|
|
serial := ""
|
|
if e.SerialNumber != nil {
|
|
serial = strings.TrimSpace(*e.SerialNumber)
|
|
}
|
|
partNum := ""
|
|
if e.PartNumber != nil {
|
|
partNum = strings.TrimSpace(*e.PartNumber)
|
|
}
|
|
|
|
status := ""
|
|
if present && isLinkDegraded(e) {
|
|
status = "Link Degraded"
|
|
}
|
|
|
|
devices = append(devices, models.PCIeDevice{
|
|
Slot: e.LocString,
|
|
VendorID: e.VendorId,
|
|
DeviceID: e.DeviceId,
|
|
BDF: formatBDF(e.BusNumber, e.DeviceNumber, e.FunctionNumber),
|
|
DeviceClass: deviceClass,
|
|
Manufacturer: normalizeModelLabel(pciids.VendorName(e.VendorId)),
|
|
LinkWidth: e.NegotiatedLinkWidth,
|
|
LinkSpeed: fmt.Sprintf("GEN%d", e.CurrentLinkSpeed),
|
|
MaxLinkWidth: e.MaxLinkWidth,
|
|
MaxLinkSpeed: fmt.Sprintf("GEN%d", e.MaxLinkSpeed),
|
|
PartNumber: partNum,
|
|
SerialNumber: serial,
|
|
Present: &present,
|
|
Status: status,
|
|
})
|
|
}
|
|
|
|
return devices
|
|
}
|
|
|
|
func isLinkDegraded(e otrdDiagnosePCIeEntry) bool {
|
|
if e.MaxLinkSpeed > 0 && e.CurrentLinkSpeed < e.MaxLinkSpeed {
|
|
return true
|
|
}
|
|
if e.MaxLinkWidth > 0 && e.NegotiatedLinkWidth < e.MaxLinkWidth {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// BuildPCIeLinkDegradationEvents emits a Warning event for every PCIe device
|
|
// present in OtrdDiagnoseComponent.json but running below its negotiated max
|
|
// link speed/width (e.g. a GPU baseboard degrading from Gen5 x16 to a lower
|
|
// state), so the log viewer surfaces the same signal the IDL alarms would.
|
|
func BuildPCIeLinkDegradationEvents(devices []models.PCIeDevice) []models.Event {
|
|
var events []models.Event
|
|
for _, d := range devices {
|
|
if d.Present == nil || !*d.Present || d.Status != "Link Degraded" {
|
|
continue
|
|
}
|
|
events = append(events, models.Event{
|
|
ID: fmt.Sprintf("pcie_link_degraded_%s", strings.TrimPrefix(d.Slot, "#")),
|
|
Source: "PCIe Diagnose",
|
|
SensorType: "pcie_link",
|
|
EventType: "Link Degraded",
|
|
Severity: models.SeverityWarning,
|
|
Description: fmt.Sprintf("%s: link degraded to %s x%d (max %s x%d)", d.Slot, d.LinkSpeed, d.LinkWidth, d.MaxLinkSpeed, d.MaxLinkWidth),
|
|
})
|
|
}
|
|
return events
|
|
}
|