diff --git a/bible-local/10-decisions.md b/bible-local/10-decisions.md index 4a618e3..50ff0bd 100644 --- a/bible-local/10-decisions.md +++ b/bible-local/10-decisions.md @@ -2071,3 +2071,33 @@ collection may use the current time, because it is the collection time. - `time.Now()` stays legitimate only for job bookkeeping, export metadata (`exported_at`, `generated_at`), live-collection timing and the Redfish live log-entry window. + +--- + +## ADL-071 — Snapshot component state lives on the component, not in the event log + +**Date:** 2026-09-28 +**Context:** Inspur `component.log` RESTful memory status produced synthetic +"Memory module … not detected (capacity 0GB)" events. On 24C319575 all three +duplicated IDL/SEL `Memory Device Disabled … POR Mapout` Asserts already in +the journal, and the text was wrong (modules present, 64 GB each). The same +dump showed the three DIMMs were installed that day: the only record of it is +`Component Replace … SN changed from NULL to …` in `maintenance.log`, which was +not parsed. + +**Decision:** Component state read from a snapshot is not journal data and is +not duplicated into the event log. DIMM state stays on the DIMM: the snapshot +status is authoritative, and the IDL/SEL Assert/Deassert journal supplies +`status_history`, the fault description and `status_changed_at` (latest +active Assert). A component.log fan state event is kept only when the journal +has no active Assert for that fan. Inspur `maintenance.log` (and rotated +`maintenance.log.N.gz`) is parsed for `Component Replace SN changed from +X to Y` records only, emitted as Info events of type `Component Replace` with +`DIMM` mapped to the inventory slot (`DIMM021` → `CPU0_C2D1`). + +**Consequences:** +- Inspur no longer emits "Memory module … not detected" events; a mapped-out + DIMM shows its journal fault on the DIMM record instead. +- Installation is not written to `status_history`: that field carries health + statuses only, and the BMC logs the SN change after POST, i.e. after a POR + mapout of the same module. diff --git a/internal/parser/vendors/inspur/component.go b/internal/parser/vendors/inspur/component.go index 8b699dc..1e06b9e 100644 --- a/internal/parser/vendors/inspur/component.go +++ b/internal/parser/vendors/inspur/component.go @@ -46,17 +46,12 @@ func ParseComponentLog(content []byte, hw *models.HardwareConfig) { extractComponentFirmware(text, hw) } -// ParseComponentLogEvents extracts events from component.log (memory errors, etc.) +// ParseComponentLogEvents extracts fan state reported in component.log. +// It is a snapshot at log collection, not a journal entry: events carry no +// timestamp. DIMM state is not emitted here: it lives on the DIMM itself +// (see applyMemoryStatusFromEvents). func ParseComponentLogEvents(content []byte) []models.Event { - var events []models.Event - text := string(content) - - // Parse RESTful Memory info for Warning/Error status - memEvents := parseMemoryEvents(text) - events = append(events, memEvents...) - events = append(events, parseFanEvents(text)...) - - return events + return parseFanEvents(string(content)) } // ParseComponentLogCollectionErrors detects BMC-reported collection failures in component.log. @@ -1093,6 +1088,38 @@ func parseFanEvents(text string) []models.Event { return events } +// dropFanSnapshotEventsCoveredByJournal removes component.log fan state +// events when the BMC journal already holds an asserted fault for that fan: +// the journal entry is the real, dated record of the same condition. +func dropFanSnapshotEventsCoveredByJournal(events []models.Event) []models.Event { + latest := make(map[string]models.Event) + for _, event := range events { + if !strings.EqualFold(strings.TrimSpace(event.SensorType), "fan") || event.Timestamp.IsZero() { + continue + } + etype := strings.ToLower(strings.TrimSpace(event.EventType)) + if etype != "assert" && etype != "deassert" { + continue + } + name := strings.ToUpper(strings.TrimSpace(event.SensorName)) + if prev, ok := latest[name]; !ok || !event.Timestamp.Before(prev.Timestamp) { + latest[name] = event + } + } + if len(latest) == 0 { + return events + } + out := events[:0] + for _, event := range events { + last, ok := latest[strings.ToUpper(strings.TrimSpace(event.SensorName))] + if event.EventType == "Fan Status" && ok && !eventIsDeassert(last) { + continue + } + out = append(out, event) + } + return out +} + func parseDiskBackplaneSensors(text string) []models.SensorReading { re := regexp.MustCompile(`RESTful diskbackplane info:\s*(\[[\s\S]*?\])\s*BMC`) match := re.FindStringSubmatch(text) @@ -1218,53 +1245,6 @@ func looksLikeRawDeviceID(v string) bool { return rawDeviceIDLikeRegex.MatchString(v) } -func parseMemoryEvents(text string) []models.Event { - var events []models.Event - - // Find RESTful Memory info section - re := regexp.MustCompile(`RESTful Memory info:\s*(\{[\s\S]*?\})\s*RESTful HDD`) - match := re.FindStringSubmatch(text) - if match == nil { - return events - } - - jsonStr := match[1] - jsonStr = strings.ReplaceAll(jsonStr, "\n", "") - - var memInfo MemoryRESTInfo - if err := json.Unmarshal([]byte(jsonStr), &memInfo); err != nil { - return events - } - - // Generate events for memory modules with Warning or Error status - for _, mem := range memInfo.MemModules { - if mem.Status == "Warning" || mem.Status == "Error" || mem.Status == "Critical" { - severity := models.SeverityWarning - if mem.Status == "Error" || mem.Status == "Critical" { - severity = models.SeverityCritical - } - - description := fmt.Sprintf("Memory module %s: %s", mem.MemModSlot, mem.Status) - if mem.MemModSize == 0 { - description = fmt.Sprintf("Memory module %s not detected (capacity 0GB)", mem.MemModSlot) - } - - events = append(events, models.Event{ - ID: fmt.Sprintf("mem_%d", mem.MemModID), - Source: "Memory", - SensorType: "memory", - SensorName: mem.MemModSlot, - EventType: "Memory Status", - Severity: severity, - Description: description, - RawData: fmt.Sprintf("Slot: %s, Vendor: %s, P/N: %s, S/N: %s", mem.MemModSlot, mem.MemModVendor, mem.MemModPartNum, mem.MemModSerial), - }) - } - } - - return events -} - // extractComponentFirmware extracts firmware versions from all component data func extractComponentFirmware(text string, hw *models.HardwareConfig) { // Create a map to track existing firmware entries (avoid duplicates) diff --git a/internal/parser/vendors/inspur/maintenance.go b/internal/parser/vendors/inspur/maintenance.go new file mode 100644 index 0000000..c101ee7 --- /dev/null +++ b/internal/parser/vendors/inspur/maintenance.go @@ -0,0 +1,83 @@ +package inspur + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "regexp" + "strings" + + "git.mchus.pro/mchus/logpile/internal/models" +) + +const componentReplaceEventType = "Component Replace" + +var ( + reComponentReplace = regexp.MustCompile(`(?i)^Component Replace\s+(\S+)\s+(?:SN\s+)?changed from\s+(\S+)\s+to\s+(\S+?)!?(?:\s+-\s+(?:Assert|Deassert))?\s*$`) + reInspurDIMMCode = regexp.MustCompile(`(?i)^DIMM(\d)(\d)(\d)$`) +) + +// ParseMaintenanceLog extracts BMC component replacement records +// ("Component Replace DIMM021 SN changed from NULL to …") from +// maintenance.log. The BMC writes them only there, not to IDL/SEL, and they +// are the only trace of when a DIMM, CPU or PSU was installed or swapped. +// Other maintenance.log lines are daemon chatter and are ignored. +func ParseMaintenanceLog(content []byte) []models.Event { + var events []models.Event + for _, event := range ParseIDLLog(content) { + match := reComponentReplace.FindStringSubmatch(strings.TrimSpace(event.Description)) + if match == nil { + continue + } + device, from, to := match[1], match[2], match[3] + sensorName := device + if slot, ok := inspurDIMMCodeToSlot(device); ok { + sensorName = slot + } + + action := "replaced" + switch { + case strings.EqualFold(from, "NULL"): + action = "installed" + case strings.EqualFold(to, "NULL"): + action = "removed" + } + description := fmt.Sprintf("%s %s: SN %s → %s", sensorName, action, from, to) + if sensorName != device { + description = fmt.Sprintf("%s (%s) %s: SN %s → %s", sensorName, device, action, from, to) + } + + event.EventType = componentReplaceEventType + event.SensorName = sensorName + event.Severity = models.SeverityInfo + event.Description = description + event.RawData = strings.TrimSpace(match[0]) + events = append(events, event) + } + return events +} + +// inspurDIMMCodeToSlot maps the BMC asset code DIMM +// (e.g. DIMM021) to the inventory slot name CPU0_C2D1. +func inspurDIMMCodeToSlot(code string) (string, bool) { + match := reInspurDIMMCode.FindStringSubmatch(strings.TrimSpace(code)) + if match == nil { + return "", false + } + return fmt.Sprintf("CPU%s_C%sD%s", match[1], match[2], match[3]), true +} + +// decodeGzipLog returns the plain text of a rotated "*.log.N.gz" member. +func decodeGzipLog(content []byte) []byte { + gz, err := gzip.NewReader(bytes.NewReader(content)) + if err != nil { + return nil + } + defer gz.Close() + out, err := io.ReadAll(io.LimitReader(gz, 64<<20)) + if err != nil { + return nil + } + return out +} diff --git a/internal/parser/vendors/inspur/maintenance_test.go b/internal/parser/vendors/inspur/maintenance_test.go new file mode 100644 index 0000000..03f3e8e --- /dev/null +++ b/internal/parser/vendors/inspur/maintenance_test.go @@ -0,0 +1,63 @@ +package inspur + +import ( + "bytes" + "compress/gzip" + "testing" + + "git.mchus.pro/mchus/logpile/internal/models" +) + +const maintenanceSample = `<182> 2026-08-12T15:35:22.950000+08:00 24C319575 commerDiagnose: [1846 : 1897 INFO]|2026-08-12T15:35:22+08:00|MEMORY|Assert|Info|0CFFB300|Component Replace DIMM021 SN changed from NULL to H0KN000308444EA65D! - Assert| +<182> 2024-03-22T17:09:44.000000+08:00 24C319575 commerDiagnose: [1846 : 1897 INFO]|2024-03-22T17:09:44+08:00|CPU|Assert|Info|07FFB000|Component Replace CPU0 changed from NULL to 23300E51079AF0AB! - Assert| +<182> 2026-08-12T15:35:30.000000+08:00 24C319575 commerDiagnose: [1846 : 1897 INFO]|2026-08-12T15:35:30+08:00|PSU|Assert|Info|08FFB100|Component Replace PSU1 SN changed from 2O01C154667 to 2O01C999999! - Assert| +<182> 2024-03-22T16:58:34.580000+08:00 24C319575 commerDiagnose: [2212 : 2261 INFO]IDLProgressTask:Init local eeprom info...... +` + +func TestParseMaintenanceLog_ComponentReplace(t *testing.T) { + events := ParseMaintenanceLog([]byte(maintenanceSample)) + if len(events) != 3 { + t.Fatalf("expected 3 component replace events, got %d: %+v", len(events), events) + } + + dimm := events[0] + if dimm.SensorType != "memory" || dimm.SensorName != "CPU0_C2D1" { + t.Fatalf("DIMM021 must map to memory/CPU0_C2D1, got %s/%s", dimm.SensorType, dimm.SensorName) + } + if dimm.EventType != componentReplaceEventType || dimm.Severity != models.SeverityInfo { + t.Fatalf("unexpected type/severity %q/%q", dimm.EventType, dimm.Severity) + } + if dimm.Description != "CPU0_C2D1 (DIMM021) installed: SN NULL → H0KN000308444EA65D" { + t.Fatalf("unexpected description %q", dimm.Description) + } + if dimm.Timestamp.IsZero() { + t.Fatalf("expected journal timestamp") + } + if events[1].Description != "CPU0 installed: SN NULL → 23300E51079AF0AB" { + t.Fatalf("unexpected CPU description %q", events[1].Description) + } + if events[2].Description != "PSU1 replaced: SN 2O01C154667 → 2O01C999999" { + t.Fatalf("unexpected PSU description %q", events[2].Description) + } +} + +func TestParseMaintenanceLog_DoesNotAffectDIMMFaultState(t *testing.T) { + hw := &models.HardwareConfig{Memory: []models.MemoryDIMM{{Slot: "CPU0_C2D1", Status: "Warning"}}} + + applyMemoryStatusFromEvents(hw, ParseMaintenanceLog([]byte(maintenanceSample))) + + if len(hw.Memory[0].StatusHistory) != 0 || hw.Memory[0].StatusChangedAt != nil { + t.Fatalf("installation record must not be read as a fault transition: %+v", hw.Memory[0]) + } +} + +func TestDecodeGzipLog(t *testing.T) { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, _ = gz.Write([]byte(maintenanceSample)) + _ = gz.Close() + + if got := ParseMaintenanceLog(decodeGzipLog(buf.Bytes())); len(got) != 3 { + t.Fatalf("expected rotated maintenance log to decode, got %d events", len(got)) + } +} diff --git a/internal/parser/vendors/inspur/memory_status.go b/internal/parser/vendors/inspur/memory_status.go new file mode 100644 index 0000000..86f48ba --- /dev/null +++ b/internal/parser/vendors/inspur/memory_status.go @@ -0,0 +1,94 @@ +package inspur + +import ( + "fmt" + "sort" + "strings" + + "git.mchus.pro/mchus/logpile/internal/models" +) + +// applyMemoryStatusFromEvents explains DIMM state reported by the +// component.log snapshot using the BMC journal (IDL/SEL Assert/Deassert). +// The snapshot stays authoritative for the current status; the journal +// supplies transition history, the fault description and when the +// currently asserted fault started. Snapshot state is never duplicated +// into the event log. +func applyMemoryStatusFromEvents(hw *models.HardwareConfig, events []models.Event) { + if hw == nil || len(hw.Memory) == 0 { + return + } + + bySlot := make(map[string][]models.Event) + for _, event := range events { + if !strings.EqualFold(strings.TrimSpace(event.SensorType), "memory") || event.Timestamp.IsZero() { + continue + } + etype := strings.ToLower(strings.TrimSpace(event.EventType)) + if etype != "assert" && etype != "deassert" { + continue + } + slot := strings.ToUpper(strings.TrimSpace(event.SensorName)) + if slot == "" { + continue + } + bySlot[slot] = append(bySlot[slot], event) + } + + for i := range hw.Memory { + dimm := &hw.Memory[i] + journal := bySlot[strings.ToUpper(strings.TrimSpace(dimm.Slot))] + sort.SliceStable(journal, func(a, b int) bool { + return journal[a].Timestamp.Before(journal[b].Timestamp) + }) + + var active *models.Event + lastStatus := "" + for j := range journal { + event := journal[j] + status := "OK" + if eventIsDeassert(event) { + active = nil + } else { + status = "Warning" + if event.Severity == models.SeverityCritical { + status = "Critical" + } + active = &journal[j] + } + if status == lastStatus { + continue + } + lastStatus = status + dimm.StatusHistory = append(dimm.StatusHistory, models.StatusHistoryEntry{ + Status: status, + ChangedAt: event.Timestamp, + Details: strings.TrimSpace(event.Description), + }) + } + + if !memoryStatusIsFault(dimm.Status) { + continue + } + if active != nil { + ts := active.Timestamp + dimm.StatusChangedAt = &ts + if strings.TrimSpace(dimm.ErrorDescription) == "" { + dimm.ErrorDescription = strings.Join(strings.Fields(active.Description), " ") + } + continue + } + if strings.TrimSpace(dimm.ErrorDescription) == "" { + dimm.ErrorDescription = fmt.Sprintf("BMC reported %s at log collection; no matching journal entry", dimm.Status) + } + } +} + +func memoryStatusIsFault(status string) bool { + switch strings.ToLower(strings.TrimSpace(status)) { + case "warning", "error", "critical": + return true + default: + return false + } +} diff --git a/internal/parser/vendors/inspur/memory_status_test.go b/internal/parser/vendors/inspur/memory_status_test.go new file mode 100644 index 0000000..36025c3 --- /dev/null +++ b/internal/parser/vendors/inspur/memory_status_test.go @@ -0,0 +1,94 @@ +package inspur + +import ( + "testing" + "time" + + "git.mchus.pro/mchus/logpile/internal/models" +) + +func TestApplyMemoryStatusFromEvents_UsesLatestJournalAssert(t *testing.T) { + loc := time.FixedZone("+08", 8*3600) + assert1 := time.Date(2026, 8, 12, 18, 52, 12, 0, loc) + deassert := time.Date(2026, 8, 12, 19, 23, 39, 0, loc) + assert2 := time.Date(2026, 8, 12, 19, 23, 50, 0, loc) + hw := &models.HardwareConfig{Memory: []models.MemoryDIMM{ + {Slot: "CPU0_C2D1", Status: "Warning"}, + {Slot: "CPU0_C3D1", Status: "OK"}, + }} + events := []models.Event{ + {Timestamp: assert2, SensorType: "memory", SensorName: "CPU0_C2D1", EventType: "Assert", Severity: models.SeverityWarning, Description: "CPU0_C2D1 Memory Device Disabled POR Mapout"}, + {Timestamp: assert1, SensorType: "memory", SensorName: "CPU0_C2D1", EventType: "Assert", Severity: models.SeverityWarning, Description: "CPU0_C2D1 Memory Device Disabled"}, + {Timestamp: deassert, SensorType: "memory", SensorName: "CPU0_C2D1", EventType: "Deassert", Severity: models.SeverityInfo, Description: "CPU0_C2D1 Memory Device Disabled"}, + } + + applyMemoryStatusFromEvents(hw, events) + + dimm := hw.Memory[0] + if dimm.Status != "Warning" { + t.Fatalf("snapshot status must stay authoritative, got %q", dimm.Status) + } + if dimm.StatusChangedAt == nil || !dimm.StatusChangedAt.Equal(assert2) { + t.Fatalf("expected status_changed_at from latest assert %s, got %v", assert2, dimm.StatusChangedAt) + } + if dimm.ErrorDescription != "CPU0_C2D1 Memory Device Disabled POR Mapout" { + t.Fatalf("unexpected error description %q", dimm.ErrorDescription) + } + if len(dimm.StatusHistory) != 3 { + t.Fatalf("expected Warning/OK/Warning history, got %+v", dimm.StatusHistory) + } + if hw.Memory[1].ErrorDescription != "" || len(hw.Memory[1].StatusHistory) != 0 { + t.Fatalf("healthy DIMM without journal entries must stay untouched: %+v", hw.Memory[1]) + } +} + +func TestApplyMemoryStatusFromEvents_SnapshotFaultWithoutJournal(t *testing.T) { + hw := &models.HardwareConfig{Memory: []models.MemoryDIMM{{Slot: "CPU1_C0D0", Status: "Critical"}}} + + applyMemoryStatusFromEvents(hw, nil) + + if hw.Memory[0].StatusChangedAt != nil { + t.Fatalf("no journal entry: change time must stay unknown, got %v", hw.Memory[0].StatusChangedAt) + } + if hw.Memory[0].ErrorDescription == "" { + t.Fatalf("expected note that the fault was seen at log collection") + } +} + +func TestParseComponentLogEvents_NoSyntheticMemoryEvents(t *testing.T) { + text := `RESTful Memory info: +{ "mem_modules": [ { "mem_mod_id": 5, "mem_mod_slot": "CPU0_C2D1", "mem_mod_status": 1, "mem_mod_size": 0, "status": "Warning" } ] } +RESTful HDD info: +{}` + if events := ParseComponentLogEvents([]byte(text)); len(events) != 0 { + t.Fatalf("DIMM snapshot state must not be written to the event log, got %+v", events) + } +} + +func TestDropFanSnapshotEventsCoveredByJournal(t *testing.T) { + ts := time.Date(2026, 8, 12, 10, 0, 0, 0, time.UTC) + events := []models.Event{ + {EventType: "Fan Status", SensorName: "FAN3_R_Speed", Description: "FAN3_R_Speed reports Critical"}, + {EventType: "Fan Status", SensorName: "FAN4_R_Speed", Description: "FAN4_R_Speed reports Critical"}, + {Timestamp: ts, SensorType: "fan", SensorName: "FAN3_R_Speed", EventType: "Assert"}, + {Timestamp: ts.Add(-time.Hour), SensorType: "fan", SensorName: "FAN4_R_Speed", EventType: "Assert"}, + {Timestamp: ts, SensorType: "fan", SensorName: "FAN4_R_Speed", EventType: "Deassert"}, + } + + out := dropFanSnapshotEventsCoveredByJournal(events) + + for _, e := range out { + if e.EventType == "Fan Status" && e.SensorName == "FAN3_R_Speed" { + t.Fatalf("fan state already asserted in journal must not be duplicated") + } + } + found := false + for _, e := range out { + if e.EventType == "Fan Status" && e.SensorName == "FAN4_R_Speed" { + found = true + } + } + if !found { + t.Fatalf("fan state not asserted in journal must be kept") + } +} diff --git a/internal/parser/vendors/inspur/parser.go b/internal/parser/vendors/inspur/parser.go index 5255b6a..22300c5 100644 --- a/internal/parser/vendors/inspur/parser.go +++ b/internal/parser/vendors/inspur/parser.go @@ -246,6 +246,24 @@ func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, er result.Events = append(result.Events, idlEvents...) } + // Component install/replace records live only in maintenance.log + // (current and rotated *.gz parts). + seenReplace := make(map[string]bool) + for _, f := range parser.FindFileByPattern(files, "/maintenance.log") { + content := f.Content + if strings.HasSuffix(strings.ToLower(f.Path), ".gz") { + content = decodeGzipLog(f.Content) + } + for _, event := range ParseMaintenanceLog(content) { + key := event.Timestamp.String() + "|" + event.Description + if seenReplace[key] { + continue + } + seenReplace[key] = true + result.Events = append(result.Events, event) + } + } + // Parse SEL list (selelist.csv), falling back to log/sel.csv on the // dump__/ layout that has no selelist.csv. if f := parser.FindFileByName(files, "selelist.csv"); f != nil { @@ -299,6 +317,8 @@ func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, er // Mark problematic GPUs from IDL errors like "BIOS miss F_GPU6". if result.Hardware != nil { applyGPUStatusFromEvents(result.Hardware, result.Events) + applyMemoryStatusFromEvents(result.Hardware, result.Events) + result.Events = dropFanSnapshotEventsCoveredByJournal(result.Events) var currentDeviceEvents []models.Event currentDeviceSnapshotAvailable := false if f := parser.FindFileByName(files, "dev_status.log"); f != nil {