Merge remote-tracking branch 'origin/main'
# Conflicts: # bible-local/10-decisions.md # internal/chart # internal/models/models.go
This commit is contained in:
Mikhail Chusavitin
@@ -107,6 +107,26 @@ type HardwareConfig struct {
|
||||
NetworkAdapters []NetworkAdapter `json:"network_adapters,omitempty"`
|
||||
PowerSupply []PSU `json:"power_supplies,omitempty"`
|
||||
Licenses []License `json:"licenses,omitempty"`
|
||||
HGX *HGXIdentity `json:"hgx,omitempty"`
|
||||
}
|
||||
|
||||
// HGXAssemblyIdentity is the Model/PartNumber/SerialNumber triple for one
|
||||
// NVIDIA HGX Redfish assembly (tray or baseboard), read from HWInfo/FWVersion.
|
||||
type HGXAssemblyIdentity struct {
|
||||
Model string `json:"model,omitempty"`
|
||||
PartNumber string `json:"part_number,omitempty"`
|
||||
SerialNumber string `json:"serial_number,omitempty"`
|
||||
}
|
||||
|
||||
// HGXIdentity holds the NVIDIA HGX hardware identity as reported by the BMC's
|
||||
// HGX_HWInfo_FWVersion.log Redfish snapshot. This is distinct from BoardInfo:
|
||||
// BoardInfo/vendor FRU describe the mechanical carrier/tray shipped by the
|
||||
// server OEM (e.g. Inspur's YZCA-* part), which does not change when the
|
||||
// NVIDIA HGX baseboard ("delta board") itself is replaced. Tray/Baseboard
|
||||
// here are the actual NVIDIA-assigned identities that do change on a swap.
|
||||
type HGXIdentity struct {
|
||||
Tray HGXAssemblyIdentity `json:"tray,omitempty"`
|
||||
Baseboard HGXAssemblyIdentity `json:"baseboard,omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
// Parses log/bmc/commer-comp/*, per-component BMC transcripts (slot presence,
|
||||
// HGX management controller, switch VR/power, switch CPLD). Each rotated log
|
||||
// (component.log plus component.tar.gz.0..9) uses the same
|
||||
// "[timestamp][file, line][level] message" line format.
|
||||
package inspur
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
"git.mchus.pro/mchus/logpile/internal/parser"
|
||||
)
|
||||
|
||||
var commerCompComponents = []string{"commerslot", "commerhmc", "commerswvr", "commerswcpld"}
|
||||
|
||||
var commerCompLineRegex = regexp.MustCompile(`^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]\[[^\]]*\]\[(\w+)\]\s*(.+)$`)
|
||||
|
||||
var commerCompNoisePatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)invalid comp data`),
|
||||
regexp.MustCompile(`(?i)error in update redis`),
|
||||
regexp.MustCompile(`(?i)mutextimeout\s*=\s*0`),
|
||||
}
|
||||
|
||||
var commerCompFailureRegex = regexp.MustCompile(`(?i)\b(fail|failed|failure|error)\b`)
|
||||
|
||||
// ParseCommerCompEvents extracts BMC component failure events from
|
||||
// log/bmc/commer-comp/<component>/ logs, including rotated *.tar.gz.N parts.
|
||||
func ParseCommerCompEvents(files []parser.ExtractedFile, location *time.Location) []models.Event {
|
||||
var events []models.Event
|
||||
for _, f := range files {
|
||||
path := strings.ToLower(f.Path)
|
||||
if !strings.Contains(path, "/commer-comp/") {
|
||||
continue
|
||||
}
|
||||
component := commerCompComponentFromPath(path)
|
||||
if component == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.Contains(path, ".tar.gz") {
|
||||
for _, part := range decodeNestedTarGz(f.Content) {
|
||||
events = append(events, parseCommerCompLog(part, component, location)...)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !strings.HasSuffix(path, ".log") {
|
||||
continue
|
||||
}
|
||||
events = append(events, parseCommerCompLog(f.Content, component, location)...)
|
||||
}
|
||||
return events
|
||||
}
|
||||
|
||||
func commerCompComponentFromPath(path string) string {
|
||||
for _, c := range commerCompComponents {
|
||||
if strings.Contains(path, "/"+c+"/") {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// decodeNestedTarGz unpacks a rotated log part (a full tar.gz archive stored
|
||||
// as e.g. commerhmc.tar.gz.0) and returns the contents of every regular file
|
||||
// inside it.
|
||||
func decodeNestedTarGz(content []byte) [][]byte {
|
||||
gz, err := gzip.NewReader(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer gz.Close()
|
||||
|
||||
tr := tar.NewReader(gz)
|
||||
var out [][]byte
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if hdr.Typeflag != tar.TypeReg {
|
||||
continue
|
||||
}
|
||||
buf, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, buf)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseCommerCompLog(content []byte, component string, location *time.Location) []models.Event {
|
||||
var events []models.Event
|
||||
scanner := bufio.NewScanner(bytes.NewReader(content))
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
m := commerCompLineRegex.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
msg := strings.TrimSpace(m[3])
|
||||
if isCommerCompNoise(msg) || !commerCompFailureRegex.MatchString(msg) {
|
||||
continue
|
||||
}
|
||||
|
||||
ts, err := time.ParseInLocation("2006-01-02 15:04:05", m[1], location)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
events = append(events, models.Event{
|
||||
ID: fmt.Sprintf("commer_comp_%s_%d_%s", component, ts.Unix(), commerCompIDSuffix(msg)),
|
||||
Timestamp: ts,
|
||||
Source: fmt.Sprintf("BMC/%s", component),
|
||||
SensorType: "bmc_component",
|
||||
EventType: m[2],
|
||||
Severity: commerCompSeverity(m[2]),
|
||||
Description: msg,
|
||||
RawData: line,
|
||||
})
|
||||
}
|
||||
return events
|
||||
}
|
||||
|
||||
func isCommerCompNoise(msg string) bool {
|
||||
for _, re := range commerCompNoisePatterns {
|
||||
if re.MatchString(msg) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var commerCompNonAlnumRegex = regexp.MustCompile(`[^a-zA-Z0-9]+`)
|
||||
|
||||
// commerCompIDSuffix derives a short, stable suffix from the message so that
|
||||
// two different failures logged in the same second get distinct event IDs.
|
||||
func commerCompIDSuffix(msg string) string {
|
||||
s := commerCompNonAlnumRegex.ReplaceAllString(strings.ToLower(msg), "_")
|
||||
s = strings.Trim(s, "_")
|
||||
if len(s) > 40 {
|
||||
s = s[:40]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func commerCompSeverity(level string) models.Severity {
|
||||
switch strings.ToLower(level) {
|
||||
case "critial", "critical", "error":
|
||||
return models.SeverityCritical
|
||||
case "warning", "warn":
|
||||
return models.SeverityWarning
|
||||
default:
|
||||
return models.SeverityInfo
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package inspur
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/parser"
|
||||
)
|
||||
|
||||
func TestParseCommerCompEvents(t *testing.T) {
|
||||
content := []byte(`[2026-07-08 16:46:19][commer_comp_anal.c, 635][Critial] [COMP]Invalid Comp Data, Continue... ParamList: Index-3, Valid-0x1, MediaType-0(1-PCIE,2-I2C)
|
||||
[2026-07-27 04:52:55][commer_comp_slot.c, 319][Critial] Data Update fail for dev serial-6!
|
||||
[2026-07-27 04:52:55][commer_comp_slot.c, 319][Critial] Data Update fail for dev serial-7!
|
||||
[2026-07-27 05:00:00][commer_comp_slot.c, 100][Info] Error in update redis, retry
|
||||
`)
|
||||
|
||||
files := []parser.ExtractedFile{
|
||||
{Path: "log/bmc/commer-comp/commerslot/commerslot.log", Content: content},
|
||||
}
|
||||
|
||||
events := ParseCommerCompEvents(files, time.UTC)
|
||||
if len(events) != 2 {
|
||||
t.Fatalf("expected 2 events (noise filtered), got %d: %+v", len(events), events)
|
||||
}
|
||||
for _, e := range events {
|
||||
if e.Source != "BMC/commerslot" {
|
||||
t.Fatalf("unexpected source %q", e.Source)
|
||||
}
|
||||
}
|
||||
if events[0].Timestamp.Format("2006-01-02 15:04:05") != "2026-07-27 04:52:55" {
|
||||
t.Fatalf("unexpected timestamp: %v", events[0].Timestamp)
|
||||
}
|
||||
if events[0].ID == events[1].ID {
|
||||
t.Fatalf("expected distinct event IDs for serial-6/serial-7, got same: %q", events[0].ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
// Fallback inventory parsing for the per-file dump_<serial>_<timestamp>/ onekeylog
|
||||
// layout, which has no combined devicefrusdr.log. Instead FRU and sensors live in
|
||||
// separate ipmitool-output text files under component/.
|
||||
package inspur
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
)
|
||||
|
||||
// ParseSensorList parses component/sensor.txt ("ipmitool sensor list" output).
|
||||
// Line format: Name | Value | Unit | Status | lnr | lcr | lnc | unc | ucr | unr
|
||||
func ParseSensorList(content []byte) []models.SensorReading {
|
||||
var readings []models.SensorReading
|
||||
|
||||
scanner := bufio.NewScanner(strings.NewReader(string(content)))
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || !strings.Contains(line, "|") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Split(line, "|")
|
||||
if len(fields) < 4 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(fields[0])
|
||||
valueStr := strings.TrimSpace(fields[1])
|
||||
unit := strings.TrimSpace(fields[2])
|
||||
status := strings.TrimSpace(fields[3])
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
reading := models.SensorReading{
|
||||
Name: name,
|
||||
Status: status,
|
||||
Unit: unit,
|
||||
Type: determineSensorType(name),
|
||||
}
|
||||
|
||||
if valueStr != "" && valueStr != "na" {
|
||||
if v, err := strconv.ParseFloat(valueStr, 64); err == nil {
|
||||
reading.Value = v
|
||||
reading.RawValue = valueStr + " " + unit
|
||||
} else {
|
||||
reading.RawValue = valueStr
|
||||
}
|
||||
}
|
||||
|
||||
readings = append(readings, reading)
|
||||
}
|
||||
|
||||
return readings
|
||||
}
|
||||
|
||||
// ParseSDRElist parses component/sdr.txt ("ipmitool sdr elist" output), used as a
|
||||
// fallback when component/sensor.txt is unavailable.
|
||||
// Line format: Name | ID | Status | Entity | Reading (e.g. "21 degrees C" or "no reading")
|
||||
func ParseSDRElist(content []byte) []models.SensorReading {
|
||||
var readings []models.SensorReading
|
||||
|
||||
scanner := bufio.NewScanner(strings.NewReader(string(content)))
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || !strings.Contains(line, "|") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Split(line, "|")
|
||||
if len(fields) < 3 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(fields[0])
|
||||
status := strings.TrimSpace(fields[2])
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
reading := models.SensorReading{
|
||||
Name: name,
|
||||
Status: status,
|
||||
Type: determineSensorType(name),
|
||||
}
|
||||
|
||||
if len(fields) >= 5 {
|
||||
readingStr := strings.TrimSpace(fields[4])
|
||||
if readingStr != "" && readingStr != "no reading" {
|
||||
reading.RawValue = readingStr
|
||||
if m := valueRegex.FindStringSubmatch(readingStr); m != nil {
|
||||
if v, err := strconv.ParseFloat(m[1], 64); err == nil {
|
||||
reading.Value = v
|
||||
reading.Unit = strings.TrimSpace(m[2])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
readings = append(readings, reading)
|
||||
}
|
||||
|
||||
return readings
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package inspur
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseSensorList(t *testing.T) {
|
||||
content := []byte(`============
|
||||
Description:sensor list
|
||||
Command: ipmitool sensor list
|
||||
Response:
|
||||
Inlet_Temp | 21.000 | degrees C | ok | na | na | na | na | 40.000 | na
|
||||
GPU0_Temp | 29.000 | degrees C | ok | na | na | na | na | 86.000 | na
|
||||
PSU0_Status | 0x0 | discrete | 0x0180| na | na | na | na | na | na
|
||||
`)
|
||||
|
||||
readings := ParseSensorList(content)
|
||||
if len(readings) != 3 {
|
||||
t.Fatalf("expected 3 readings, got %d", len(readings))
|
||||
}
|
||||
if readings[0].Name != "Inlet_Temp" || readings[0].Value != 21.0 || readings[0].Unit != "degrees C" {
|
||||
t.Fatalf("unexpected Inlet_Temp reading: %+v", readings[0])
|
||||
}
|
||||
if readings[1].Type != "temperature" {
|
||||
t.Fatalf("expected temperature type, got %q", readings[1].Type)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSDRElist(t *testing.T) {
|
||||
content := []byte(`============
|
||||
Description:sdr elist
|
||||
Command: ipmitool sdr elist
|
||||
Response:
|
||||
Inlet_Temp | 00h | ok | 3.0 | 21 degrees C
|
||||
GPU0_Temp | 2Ah | ok | 11.0 | 29 degrees C
|
||||
Event Logging Disabled SEL_Status | 15h | ok | 7.0 | no reading
|
||||
`)
|
||||
|
||||
readings := ParseSDRElist(content)
|
||||
if len(readings) != 3 {
|
||||
t.Fatalf("expected 3 readings, got %d", len(readings))
|
||||
}
|
||||
if readings[0].Name != "Inlet_Temp" || readings[0].Value != 21 || readings[0].Unit != "degrees C" {
|
||||
t.Fatalf("unexpected Inlet_Temp reading: %+v", readings[0])
|
||||
}
|
||||
if readings[2].RawValue != "" {
|
||||
t.Fatalf("expected no reading to leave RawValue empty, got %q", readings[2].RawValue)
|
||||
}
|
||||
}
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
// Parses log/bmc/diagnose/OtrdDiagnoseComponent.json, an HGX BMC structural
|
||||
// snapshot of PCIe topology (present GPUs/NICs/NVMe, negotiated link state).
|
||||
// This is a direct point-in-time source for GPU presence and PCIe link
|
||||
// degradation, independent of SEL/IDL alarm history.
|
||||
package inspur
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
"git.mchus.pro/mchus/logpile/internal/parser/vendors/pciids"
|
||||
)
|
||||
|
||||
// otrdDiagnosePCIeEntry mirrors the relevant fields of "Pcie Device Info"
|
||||
// entries in OtrdDiagnoseComponent.json.
|
||||
type otrdDiagnosePCIeEntry struct {
|
||||
LocString string `json:"LocString"`
|
||||
PcieSlot int `json:"PcieSlot"`
|
||||
PresentStatus int `json:"PresentStatus"`
|
||||
VendorId int `json:"VendorId"`
|
||||
DeviceId int `json:"DeviceId"`
|
||||
BusNumber int `json:"BusNumber"`
|
||||
DeviceNumber int `json:"DeviceNumber"`
|
||||
FunctionNumber int `json:"FunctionNumber"`
|
||||
CurrentLinkSpeed int `json:"CurrentLinkSpeed"`
|
||||
MaxLinkSpeed int `json:"MaxLinkSpeed"`
|
||||
NegotiatedLinkWidth int `json:"NegotiatedLinkWidth"`
|
||||
MaxLinkWidth int `json:"MaxLinkWidth"`
|
||||
SerialNumber *string `json:"SerialNumber"`
|
||||
PartNumber *string `json:"PartNumber"`
|
||||
}
|
||||
|
||||
type otrdDiagnoseComponent struct {
|
||||
PcieDeviceInfo []otrdDiagnosePCIeEntry `json:"Pcie Device Info"`
|
||||
}
|
||||
|
||||
// ParseOtrdDiagnosePCIe parses the "Pcie Device Info" array from
|
||||
// OtrdDiagnoseComponent.json into PCIe device inventory entries.
|
||||
func ParseOtrdDiagnosePCIe(content []byte) []models.PCIeDevice {
|
||||
var doc otrdDiagnoseComponent
|
||||
if err := json.Unmarshal(content, &doc); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
devices := make([]models.PCIeDevice, 0, len(doc.PcieDeviceInfo))
|
||||
for _, e := range doc.PcieDeviceInfo {
|
||||
present := e.PresentStatus == 1
|
||||
_, deviceName := pciids.DeviceInfo(e.VendorId, e.DeviceId)
|
||||
deviceClass := normalizeModelLabel(deviceName)
|
||||
if strings.Contains(strings.ToUpper(deviceName), "NVIDIA") {
|
||||
deviceClass = "GPU (" + deviceName + ")"
|
||||
}
|
||||
|
||||
serial := ""
|
||||
if e.SerialNumber != nil {
|
||||
serial = strings.TrimSpace(*e.SerialNumber)
|
||||
}
|
||||
partNum := ""
|
||||
if e.PartNumber != nil {
|
||||
partNum = strings.TrimSpace(*e.PartNumber)
|
||||
}
|
||||
|
||||
status := ""
|
||||
if present && isLinkDegraded(e) {
|
||||
status = "Link Degraded"
|
||||
}
|
||||
|
||||
devices = append(devices, models.PCIeDevice{
|
||||
Slot: e.LocString,
|
||||
VendorID: e.VendorId,
|
||||
DeviceID: e.DeviceId,
|
||||
BDF: formatBDF(e.BusNumber, e.DeviceNumber, e.FunctionNumber),
|
||||
DeviceClass: deviceClass,
|
||||
Manufacturer: normalizeModelLabel(pciids.VendorName(e.VendorId)),
|
||||
LinkWidth: e.NegotiatedLinkWidth,
|
||||
LinkSpeed: fmt.Sprintf("GEN%d", e.CurrentLinkSpeed),
|
||||
MaxLinkWidth: e.MaxLinkWidth,
|
||||
MaxLinkSpeed: fmt.Sprintf("GEN%d", e.MaxLinkSpeed),
|
||||
PartNumber: partNum,
|
||||
SerialNumber: serial,
|
||||
Present: &present,
|
||||
Status: status,
|
||||
})
|
||||
}
|
||||
|
||||
return devices
|
||||
}
|
||||
|
||||
func isLinkDegraded(e otrdDiagnosePCIeEntry) bool {
|
||||
if e.MaxLinkSpeed > 0 && e.CurrentLinkSpeed < e.MaxLinkSpeed {
|
||||
return true
|
||||
}
|
||||
if e.MaxLinkWidth > 0 && e.NegotiatedLinkWidth < e.MaxLinkWidth {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// BuildPCIeLinkDegradationEvents emits a Warning event for every PCIe device
|
||||
// present in OtrdDiagnoseComponent.json but running below its negotiated max
|
||||
// link speed/width (e.g. a GPU baseboard degrading from Gen5 x16 to a lower
|
||||
// state), so the log viewer surfaces the same signal the IDL alarms would.
|
||||
func BuildPCIeLinkDegradationEvents(devices []models.PCIeDevice) []models.Event {
|
||||
var events []models.Event
|
||||
for _, d := range devices {
|
||||
if d.Present == nil || !*d.Present || d.Status != "Link Degraded" {
|
||||
continue
|
||||
}
|
||||
events = append(events, models.Event{
|
||||
ID: fmt.Sprintf("pcie_link_degraded_%s", strings.TrimPrefix(d.Slot, "#")),
|
||||
Source: "PCIe Diagnose",
|
||||
SensorType: "pcie_link",
|
||||
EventType: "Link Degraded",
|
||||
Severity: models.SeverityWarning,
|
||||
Description: fmt.Sprintf("%s: link degraded to %s x%d (max %s x%d)", d.Slot, d.LinkSpeed, d.LinkWidth, d.MaxLinkSpeed, d.MaxLinkWidth),
|
||||
})
|
||||
}
|
||||
return events
|
||||
}
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
package inspur
|
||||
|
||||
import "testing"
|
||||
|
||||
const diagnoseJSONFixture = `{
|
||||
"Pcie Device Info": [
|
||||
{
|
||||
"LocString": "#GPU0",
|
||||
"PcieSlot": 100,
|
||||
"PresentStatus": 1,
|
||||
"VendorId": 4318,
|
||||
"DeviceId": 10497,
|
||||
"BusNumber": 23,
|
||||
"DeviceNumber": 0,
|
||||
"FunctionNumber": 0,
|
||||
"CurrentLinkSpeed": 5,
|
||||
"MaxLinkSpeed": 5,
|
||||
"NegotiatedLinkWidth": 16,
|
||||
"MaxLinkWidth": 16
|
||||
},
|
||||
{
|
||||
"LocString": "#GPU6",
|
||||
"PcieSlot": 106,
|
||||
"PresentStatus": 1,
|
||||
"VendorId": 4318,
|
||||
"DeviceId": 10497,
|
||||
"BusNumber": 220,
|
||||
"DeviceNumber": 0,
|
||||
"FunctionNumber": 0,
|
||||
"CurrentLinkSpeed": 2,
|
||||
"MaxLinkSpeed": 5,
|
||||
"NegotiatedLinkWidth": 8,
|
||||
"MaxLinkWidth": 16
|
||||
}
|
||||
]
|
||||
}`
|
||||
|
||||
func TestParseOtrdDiagnosePCIe(t *testing.T) {
|
||||
devices := ParseOtrdDiagnosePCIe([]byte(diagnoseJSONFixture))
|
||||
if len(devices) != 2 {
|
||||
t.Fatalf("expected 2 devices, got %d", len(devices))
|
||||
}
|
||||
|
||||
gpu0 := devices[0]
|
||||
if gpu0.Slot != "#GPU0" || gpu0.Present == nil || !*gpu0.Present {
|
||||
t.Fatalf("expected GPU0 present, got %+v", gpu0)
|
||||
}
|
||||
if gpu0.Status == "Link Degraded" {
|
||||
t.Fatalf("GPU0 should not be flagged as degraded: %+v", gpu0)
|
||||
}
|
||||
if gpu0.LinkSpeed != "GEN5" || gpu0.MaxLinkSpeed != "GEN5" || gpu0.LinkWidth != 16 {
|
||||
t.Fatalf("unexpected GPU0 link info: %+v", gpu0)
|
||||
}
|
||||
|
||||
gpu6 := devices[1]
|
||||
if gpu6.Status != "Link Degraded" {
|
||||
t.Fatalf("expected GPU6 to be flagged as degraded, got %+v", gpu6)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPCIeLinkDegradationEvents(t *testing.T) {
|
||||
devices := ParseOtrdDiagnosePCIe([]byte(diagnoseJSONFixture))
|
||||
events := BuildPCIeLinkDegradationEvents(devices)
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("expected 1 degradation event, got %d", len(events))
|
||||
}
|
||||
if events[0].Severity != "warning" {
|
||||
t.Fatalf("expected warning severity, got %q", events[0].Severity)
|
||||
}
|
||||
}
|
||||
+9
-2
@@ -72,9 +72,16 @@ func ParseFRU(content []byte) []models.FRUInfo {
|
||||
current.ProductName = fieldValue
|
||||
}
|
||||
case "Board Serial", "Product Serial":
|
||||
current.SerialNumber = fieldValue
|
||||
// Modules with no product-level FRU (e.g. SCM_FRU) report
|
||||
// "Product Serial : 0" after a real "Board Serial" value;
|
||||
// don't let the placeholder overwrite the real serial.
|
||||
if fieldValue != "0" && fieldValue != "NULL" {
|
||||
current.SerialNumber = fieldValue
|
||||
}
|
||||
case "Board Part Number", "Product Part Number":
|
||||
if fieldValue != "0" {
|
||||
// See "Board Serial" above: placeholder "NULL"/"0" product-level
|
||||
// fields must not overwrite a real board-level part number.
|
||||
if fieldValue != "0" && fieldValue != "NULL" {
|
||||
current.PartNumber = fieldValue
|
||||
}
|
||||
case "Product Version":
|
||||
|
||||
+101
-3
@@ -48,8 +48,106 @@ var (
|
||||
reIDLine = regexp.MustCompile(`"Id":\s*"([^"]+)"`)
|
||||
reVersion = regexp.MustCompile(`"Version":\s*"([^"]*)"`)
|
||||
reSlotGPU = regexp.MustCompile(`(?i)gpu\s*#?\s*(\d+)`)
|
||||
|
||||
reCurlLine = regexp.MustCompile(`(?m)^#.*curl.*$`)
|
||||
reRedfishPath = regexp.MustCompile(`(/redfish/v1/\S+)`)
|
||||
reFieldModel = regexp.MustCompile(`"Model"\s*:\s*"([^"]*)"`)
|
||||
reFieldPart = regexp.MustCompile(`"PartNumber"\s*:\s*"([^"]*)"`)
|
||||
reFieldSerial = regexp.MustCompile(`"SerialNumber"\s*:\s*"([^"]*)"`)
|
||||
)
|
||||
|
||||
// hgxValue normalizes a raw HWInfo field value, treating Redfish's "not
|
||||
// applicable" placeholders (e.g. an unpowered/absent GPU) as empty so they
|
||||
// never overwrite or masquerade as real identity data.
|
||||
func hgxValue(raw string) string {
|
||||
v := strings.TrimSpace(raw)
|
||||
switch strings.ToUpper(v) {
|
||||
case "", "NA", "N/A":
|
||||
return ""
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
// splitCurlBlocks splits a HGX_HWInfo_FWVersion.log-style dump (a sequence of
|
||||
// "# curl ... <url>" comment lines each followed by that request's JSON
|
||||
// response) into per-request chunks, so fields can be attributed to the
|
||||
// Redfish path that produced them.
|
||||
func splitCurlBlocks(content []byte) []string {
|
||||
text := string(content)
|
||||
locs := reCurlLine.FindAllStringIndex(text, -1)
|
||||
if len(locs) == 0 {
|
||||
return []string{text}
|
||||
}
|
||||
|
||||
blocks := make([]string, 0, len(locs))
|
||||
for i, loc := range locs {
|
||||
end := len(text)
|
||||
if i+1 < len(locs) {
|
||||
end = locs[i+1][0]
|
||||
}
|
||||
blocks = append(blocks, text[loc[0]:end])
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
// parseHGXIdentity extracts the NVIDIA HGX tray and baseboard hardware
|
||||
// identity (Model/PartNumber/SerialNumber) from a HGX_HWInfo_FWVersion.log
|
||||
// dump. Unlike the vendor mechanical-carrier FRU, these serials change when
|
||||
// the actual HGX tray/baseboard ("delta board") is swapped.
|
||||
func parseHGXIdentity(content []byte) *models.HGXIdentity {
|
||||
if len(content) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var identity models.HGXIdentity
|
||||
found := false
|
||||
|
||||
for _, block := range splitCurlBlocks(content) {
|
||||
path := reRedfishPath.FindString(block)
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
lowerPath := strings.ToLower(path)
|
||||
|
||||
// GPU-specific and GPU-scoped paths are handled by
|
||||
// parseHGXGPUAssembly/parseHGXGPUFirmware; skip them here so a
|
||||
// per-GPU Model/PartNumber/SerialNumber triple never gets
|
||||
// misattributed to the tray or baseboard.
|
||||
if strings.Contains(lowerPath, "gpu_sxm") || strings.Contains(lowerPath, "/processors/") {
|
||||
continue
|
||||
}
|
||||
|
||||
info := models.HGXAssemblyIdentity{}
|
||||
if m := reFieldModel.FindStringSubmatch(block); m != nil {
|
||||
info.Model = hgxValue(m[1])
|
||||
}
|
||||
if m := reFieldPart.FindStringSubmatch(block); m != nil {
|
||||
info.PartNumber = hgxValue(m[1])
|
||||
}
|
||||
if m := reFieldSerial.FindStringSubmatch(block); m != nil {
|
||||
info.SerialNumber = hgxValue(m[1])
|
||||
}
|
||||
if info.Model == "" && info.PartNumber == "" && info.SerialNumber == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.Contains(lowerPath, "tray"):
|
||||
identity.Tray = info
|
||||
found = true
|
||||
case strings.Contains(lowerPath, "baseboard"):
|
||||
identity.Baseboard = info
|
||||
found = true
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
return &identity
|
||||
}
|
||||
|
||||
func enrichGPUsFromHGXHWInfo(content []byte, hw *models.HardwareConfig) {
|
||||
if hw == nil || len(hw.GPUs) == 0 || len(content) == 0 {
|
||||
return
|
||||
@@ -157,9 +255,9 @@ func parseHGXGPUAssembly(content []byte) map[int]hgxGPUAssemblyInfo {
|
||||
}
|
||||
|
||||
result[sxmIdx] = hgxGPUAssemblyInfo{
|
||||
Model: strings.TrimSpace(string(m[2])),
|
||||
Part: strings.TrimSpace(string(m[3])),
|
||||
Serial: strings.TrimSpace(string(m[4])),
|
||||
Model: hgxValue(string(m[2])),
|
||||
Part: hgxValue(string(m[3])),
|
||||
Serial: hgxValue(string(m[4])),
|
||||
}
|
||||
}
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package inspur
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseHGXIdentity_ExtractsTrayAndBaseboard(t *testing.T) {
|
||||
content := []byte(`
|
||||
# curl -X GET http://127.0.0.1/redfish/v1/Chassis/HGX_Tray_0/Assembly
|
||||
{"Name":"HGX Tray Assembly","Model":"P6612-A04","PartNumber":"699-26612-0000-P00","SerialNumber":"TRAYSN1"}
|
||||
# curl -X GET http://127.0.0.1/redfish/v1/Systems/HGX_Baseboard_0
|
||||
{"Model":"NVIDIA HGX B200 8 GPU","PartNumber":"935-26287-00A0-000","SerialNumber":"BBSN1"}
|
||||
# curl -X GET http://127.0.0.1/redfish/v1/Systems/HGX_Baseboard_0/Processors/GPU_SXM_1
|
||||
{"Model":"B200 180GB HBM3e","PartNumber":"692-2G525-0220-501","SerialNumber":"GPUSN1"}
|
||||
`)
|
||||
|
||||
identity := parseHGXIdentity(content)
|
||||
if identity == nil {
|
||||
t.Fatal("expected non-nil identity")
|
||||
}
|
||||
if identity.Tray.Model != "P6612-A04" || identity.Tray.PartNumber != "699-26612-0000-P00" || identity.Tray.SerialNumber != "TRAYSN1" {
|
||||
t.Fatalf("unexpected tray identity: %+v", identity.Tray)
|
||||
}
|
||||
if identity.Baseboard.Model != "NVIDIA HGX B200 8 GPU" || identity.Baseboard.PartNumber != "935-26287-00A0-000" || identity.Baseboard.SerialNumber != "BBSN1" {
|
||||
t.Fatalf("unexpected baseboard identity: %+v", identity.Baseboard)
|
||||
}
|
||||
// The per-GPU Processors path must never leak into baseboard/tray identity.
|
||||
if identity.Baseboard.SerialNumber == "GPUSN1" || identity.Tray.SerialNumber == "GPUSN1" {
|
||||
t.Fatalf("GPU serial leaked into baseboard/tray identity: %+v", identity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHGXIdentity_DetectsBaseboardSwapAcrossDumps(t *testing.T) {
|
||||
dumpA := []byte(`
|
||||
# curl -X GET http://127.0.0.1/redfish/v1/Chassis/HGX_Tray_0/Assembly
|
||||
{"Name":"HGX Tray Assembly","Model":"P6612-A04","PartNumber":"699-26612-0000-P00","SerialNumber":"TRAY-1"}
|
||||
# curl -X GET http://127.0.0.1/redfish/v1/Systems/HGX_Baseboard_0
|
||||
{"Model":"NVIDIA HGX B200 8 GPU","PartNumber":"935-26287-00A0-000","SerialNumber":"BB-1"}
|
||||
`)
|
||||
dumpC := []byte(`
|
||||
# curl -X GET http://127.0.0.1/redfish/v1/Chassis/HGX_Tray_0/Assembly
|
||||
{"Name":"HGX Tray Assembly","Model":"P6612-A04","PartNumber":"699-26612-0000-P00","SerialNumber":"TRAY-2"}
|
||||
# curl -X GET http://127.0.0.1/redfish/v1/Systems/HGX_Baseboard_0
|
||||
{"Model":"NVIDIA HGX B200 8 GPU","PartNumber":"935-26287-00A0-000","SerialNumber":"BB-2"}
|
||||
`)
|
||||
|
||||
a := parseHGXIdentity(dumpA)
|
||||
c := parseHGXIdentity(dumpC)
|
||||
if a == nil || c == nil {
|
||||
t.Fatal("expected identity from both dumps")
|
||||
}
|
||||
if a.Baseboard.SerialNumber == c.Baseboard.SerialNumber {
|
||||
t.Fatal("expected baseboard serial to differ across dumps that recorded a swap")
|
||||
}
|
||||
if a.Tray.SerialNumber == c.Tray.SerialNumber {
|
||||
t.Fatal("expected tray serial to differ across dumps that recorded a swap")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHGXIdentity_TreatsNAAsAbsent(t *testing.T) {
|
||||
content := []byte(`
|
||||
# curl -X GET http://127.0.0.1/redfish/v1/Systems/HGX_Baseboard_0
|
||||
{"Model":"NA","PartNumber":"NA","SerialNumber":"NA"}
|
||||
`)
|
||||
|
||||
if identity := parseHGXIdentity(content); identity != nil {
|
||||
t.Fatalf("expected nil identity when all fields are NA, got %+v", identity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHGXIdentity_NoHGXContentReturnsNil(t *testing.T) {
|
||||
if identity := parseHGXIdentity(nil); identity != nil {
|
||||
t.Fatalf("expected nil identity for empty content, got %+v", identity)
|
||||
}
|
||||
if identity := parseHGXIdentity([]byte("no redfish paths here")); identity != nil {
|
||||
t.Fatalf("expected nil identity when no HGX paths present, got %+v", identity)
|
||||
}
|
||||
}
|
||||
+49
-2
@@ -16,7 +16,7 @@ import (
|
||||
|
||||
// parserVersion - version of this parser module
|
||||
// IMPORTANT: Increment this version when making changes to parser logic!
|
||||
const parserVersion = "2.2"
|
||||
const parserVersion = "2.3"
|
||||
|
||||
func init() {
|
||||
parser.Register(&Parser{})
|
||||
@@ -203,6 +203,26 @@ func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, er
|
||||
}
|
||||
}
|
||||
|
||||
// New dump_<serial>_<timestamp>/ layout has no devicefrusdr.log: FRU and
|
||||
// sensors live in separate ipmitool-output text files under component/.
|
||||
if parser.FindFileByName(files, "devicefrusdr.log") == nil {
|
||||
if f := parser.FindFileByName(files, "fru.txt"); f != nil && len(result.FRU) == 0 {
|
||||
result.FRU = ParseFRU(f.Content)
|
||||
extractBoardInfo(result.FRU, result.Hardware)
|
||||
}
|
||||
if f := parser.FindFileByName(files, "sensor.txt"); f != nil && len(result.Sensors) == 0 {
|
||||
result.Sensors = mergeSensorReadings(result.Sensors, ParseSensorList(f.Content))
|
||||
} else if f := parser.FindFileByName(files, "sdr.txt"); f != nil && len(result.Sensors) == 0 {
|
||||
result.Sensors = mergeSensorReadings(result.Sensors, ParseSDRElist(f.Content))
|
||||
}
|
||||
// Structural PCIe snapshot (GPU presence/link state) from the HGX diagnose dump.
|
||||
if f := parser.FindFileByName(files, "OtrdDiagnoseComponent.json"); f != nil && result.Hardware != nil {
|
||||
diagPCIe := ParseOtrdDiagnosePCIe(f.Content)
|
||||
result.Hardware.PCIeDevices = MergePCIeDevices(result.Hardware.PCIeDevices, diagPCIe)
|
||||
result.Events = append(result.Events, BuildPCIeLinkDegradationEvents(diagPCIe)...)
|
||||
}
|
||||
}
|
||||
|
||||
// Enrich runtime component data from Redis snapshot (serials, FW, telemetry),
|
||||
// when text logs miss these fields.
|
||||
if f := parser.FindFileByName(files, "redis-dump.rdb"); f != nil && result.Hardware != nil {
|
||||
@@ -216,10 +236,14 @@ func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, er
|
||||
result.Events = append(result.Events, idlEvents...)
|
||||
}
|
||||
|
||||
// Parse SEL list (selelist.csv)
|
||||
// Parse SEL list (selelist.csv), falling back to log/sel.csv on the
|
||||
// dump_<serial>_<timestamp>/ layout that has no selelist.csv.
|
||||
if f := parser.FindFileByName(files, "selelist.csv"); f != nil {
|
||||
selEvents := ParseSELListWithLocation(f.Content, selLocation)
|
||||
result.Events = append(result.Events, selEvents...)
|
||||
} else if f := parser.FindFileByName(files, "sel.csv"); f != nil {
|
||||
selEvents := ParseSELListWithLocation(f.Content, selLocation)
|
||||
result.Events = append(result.Events, selEvents...)
|
||||
}
|
||||
|
||||
// Parse syslog files
|
||||
@@ -229,6 +253,14 @@ func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, er
|
||||
result.Events = append(result.Events, events...)
|
||||
}
|
||||
|
||||
// Parse BMC component failure logs (log/bmc/commer-comp/*): HGX management
|
||||
// controller, switch VR/power, switch CPLD and slot presence failures.
|
||||
result.Events = append(result.Events, ParseCommerCompEvents(files, selLocation)...)
|
||||
|
||||
// Same SEL event can be reported twice by the BMC (e.g. once via sel.csv,
|
||||
// once via idl.log); collapse exact duplicates so they don't double-count.
|
||||
result.Events = dedupSELEvents(result.Events)
|
||||
|
||||
// Fallback for archives where board serial is missing in parsed FRU/asset data:
|
||||
// recover it from log content, never from archive filename.
|
||||
if strings.TrimSpace(result.Hardware.BoardInfo.SerialNumber) == "" {
|
||||
@@ -246,6 +278,7 @@ func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, er
|
||||
if f := parser.FindFileByName(files, "HGX_HWInfo_FWVersion.log"); f != nil && result.Hardware != nil {
|
||||
enrichGPUsFromHGXHWInfo(f.Content, result.Hardware)
|
||||
appendHGXFirmwareFromHWInfo(f.Content, result.Hardware)
|
||||
result.Hardware.HGX = parseHGXIdentity(f.Content)
|
||||
}
|
||||
|
||||
// Mark problematic GPUs from IDL errors like "BIOS miss F_GPU6".
|
||||
@@ -261,6 +294,20 @@ func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, er
|
||||
parser.ApplyManufacturedYearWeekFromFRU(result.FRU, result.Hardware)
|
||||
}
|
||||
|
||||
if len(result.FRU) == 0 || len(result.Sensors) == 0 {
|
||||
var missing []string
|
||||
if len(result.FRU) == 0 {
|
||||
missing = append(missing, "FRU")
|
||||
}
|
||||
if len(result.Sensors) == 0 {
|
||||
missing = append(missing, "sensors")
|
||||
}
|
||||
result.CollectionErrors = append(result.CollectionErrors, models.CollectionError{
|
||||
Section: "inventory",
|
||||
Message: fmt.Sprintf("inventory sources not found: devicefrusdr.log absent, fell back to component/fru.txt+sensor.txt/sdr.txt, still missing: %s", strings.Join(missing, ", ")),
|
||||
})
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
|
||||
+6
@@ -314,6 +314,12 @@ func enrichPCIeDevice(dst *models.PCIeDevice, src models.PCIeDevice) {
|
||||
if isGenericPCIeClass(dst.DeviceClass) && !isGenericPCIeClass(src.DeviceClass) {
|
||||
dst.DeviceClass = src.DeviceClass
|
||||
}
|
||||
if dst.Present == nil {
|
||||
dst.Present = src.Present
|
||||
}
|
||||
if strings.TrimSpace(dst.Status) == "" {
|
||||
dst.Status = src.Status
|
||||
}
|
||||
}
|
||||
|
||||
func normalizePCIeBDF(bdf string) string {
|
||||
|
||||
+26
@@ -176,6 +176,32 @@ func determineSELSeverity(sensorStr, eventDesc, status string) models.Severity {
|
||||
return models.SeverityInfo
|
||||
}
|
||||
|
||||
// dedupSELEvents collapses events reported more than once with the same
|
||||
// (timestamp, event_type, description) triple. Unlike ParseIDLLog's
|
||||
// dedup (which must keep recurring alarms with distinct timestamps), this
|
||||
// only removes true duplicates: the same SEL entry surfacing through more
|
||||
// than one source file for the exact same moment.
|
||||
func dedupSELEvents(events []models.Event) []models.Event {
|
||||
if len(events) == 0 {
|
||||
return events
|
||||
}
|
||||
seen := make(map[string]struct{}, len(events))
|
||||
out := make([]models.Event, 0, len(events))
|
||||
for _, e := range events {
|
||||
if e.Source != "SEL" {
|
||||
out = append(out, e)
|
||||
continue
|
||||
}
|
||||
key := e.Timestamp.String() + "|" + e.EventType + "|" + e.Description
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildSELDescription builds human-readable description
|
||||
func buildSELDescription(eventDesc, status string) string {
|
||||
if status == "Asserted" || status == "Deasserted" {
|
||||
|
||||
+17
@@ -3,8 +3,25 @@ package inspur
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
)
|
||||
|
||||
func TestDedupSELEvents(t *testing.T) {
|
||||
ts := time.Date(2026, 7, 27, 4, 52, 55, 0, time.UTC)
|
||||
events := []models.Event{
|
||||
{Source: "SEL", Timestamp: ts, EventType: "Asserted", Description: "PSU0 fail"},
|
||||
{Source: "SEL", Timestamp: ts, EventType: "Asserted", Description: "PSU0 fail"},
|
||||
{Source: "BMC", Timestamp: ts, EventType: "Assert", Description: "recurring alarm"},
|
||||
{Source: "BMC", Timestamp: ts, EventType: "Assert", Description: "recurring alarm"},
|
||||
}
|
||||
|
||||
out := dedupSELEvents(events)
|
||||
if len(out) != 3 {
|
||||
t.Fatalf("expected 3 events (1 SEL dup removed, BMC untouched), got %d: %+v", len(out), out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSELListWithLocation_UsesProvidedTimezone(t *testing.T) {
|
||||
content := []byte("sel elist:\n1,02/28/2026,04:18:18,Sensor X,Event,Asserted\n")
|
||||
shanghai, err := time.LoadLocation("Asia/Shanghai")
|
||||
|
||||
+757
-2030
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user