feat(inspur): source-PRI timezone timeline, PPIN serial, NVMe fault storage status

Event ingestion now uses the source syslog PRI and an explicit-offset
timezone timeline instead of assuming host-local time. CPU PPIN is
exported as the source-backed CPU serial, and an active NVMe fault SEL
event promotes the matching drive's storage status.

See ADL-056, ADL-057.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-08-27 17:05:45 +03:00
co-authored by Claude Sonnet 5
parent 07c270cda2
commit e7b8a8badc
13 changed files with 671 additions and 48 deletions
+2 -1
View File
@@ -129,7 +129,8 @@ func ParseAssetJSON(content []byte, pcieSlotDeviceNames map[int]string, pcieSlot
L2CacheKB: cpu.L2Cache,
L3CacheKB: cpu.L3Cache,
TDP: cpu.CpuTdp,
PPIN: cpu.PPIN,
PPIN: strings.TrimSpace(cpu.PPIN),
SerialNumber: models.ResolveCPUSerialNumber("", cpu.PPIN),
})
if cpu.MicroCodeVer != "" {
+9 -2
View File
@@ -164,6 +164,12 @@ func parseCPUInfo(text string, hw *models.HardwareConfig) {
if strings.TrimSpace(hw.CPUs[idx].Status) == "" {
hw.CPUs[idx].Status = status
}
if strings.TrimSpace(hw.CPUs[idx].PPIN) == "" {
hw.CPUs[idx].PPIN = strings.TrimSpace(proc.PPIN)
}
if strings.TrimSpace(hw.CPUs[idx].SerialNumber) == "" {
hw.CPUs[idx].SerialNumber = models.ResolveCPUSerialNumber("", proc.PPIN)
}
} else {
hw.CPUs = append(hw.CPUs, models.CPU{
Socket: proc.ProcID,
@@ -176,7 +182,8 @@ func parseCPUInfo(text string, hw *models.HardwareConfig) {
L2CacheKB: proc.L2Cache,
L3CacheKB: proc.L3Cache,
TDP: proc.TDP,
PPIN: proc.PPIN,
PPIN: strings.TrimSpace(proc.PPIN),
SerialNumber: models.ResolveCPUSerialNumber("", proc.PPIN),
Status: status,
})
}
@@ -243,7 +250,7 @@ func parseMemoryInfo(text string, hw *models.HardwareConfig) {
Slot: mem.MemModSlot,
Location: mem.MemModSlot,
// status=1 with a known serial/part is definitely present even if BMC reports size=0
Present: mem.MemModStatus == 1 && (mem.MemModSize > 0 || strings.TrimSpace(mem.MemModSerial) != "" || strings.TrimSpace(mem.MemModPartNum) != ""),
Present: mem.MemModStatus == 1 && (mem.MemModSize > 0 || strings.TrimSpace(mem.MemModSerial) != "" || strings.TrimSpace(mem.MemModPartNum) != ""),
SizeMB: mem.MemModSize * 1024, // Convert GB to MB
Type: mem.MemModType,
Technology: strings.TrimSpace(mem.MemModTechnology),
+3
View File
@@ -45,6 +45,9 @@ func TestParseCPUInfo_FromComponentLog(t *testing.T) {
if hw.CPUs[1].PPIN != "475AC1221D41F557" {
t.Errorf("unexpected CPU1 PPIN: %s", hw.CPUs[1].PPIN)
}
if hw.CPUs[0].SerialNumber != "47149E2253E81688" || hw.CPUs[1].SerialNumber != "475AC1221D41F557" {
t.Fatalf("expected source-backed PPIN values as CPU serials, got %+v", hw.CPUs)
}
}
func TestParseMemoryInfo_PresentWithZeroSize(t *testing.T) {
+22
View File
@@ -0,0 +1,22 @@
package inspur
import "testing"
func TestParseAssetJSONUsesPPINAsSourceBackedCPUSerial(t *testing.T) {
content := []byte(`{
"CpuInfo": [
{"ProcessorName":"Intel Xeon", "PPIN":"D46E5D6B1D3E40E1"},
{"ProcessorName":"Intel Xeon", "PPIN":"D44F8D6B9155EE0E"}
]
}`)
hw, err := ParseAssetJSON(content, nil, nil)
if err != nil {
t.Fatalf("parse asset: %v", err)
}
if len(hw.CPUs) != 2 {
t.Fatalf("expected 2 CPUs, got %d", len(hw.CPUs))
}
if hw.CPUs[0].SerialNumber != "D46E5D6B1D3E40E1" || hw.CPUs[1].SerialNumber != "D44F8D6B9155EE0E" {
t.Fatalf("unexpected CPU serial mapping: %+v", hw.CPUs)
}
}
+72 -1
View File
@@ -1,6 +1,12 @@
package inspur
import "testing"
import (
"testing"
"time"
"git.mchus.pro/mchus/logpile/internal/models"
baseparser "git.mchus.pro/mchus/logpile/internal/parser"
)
func TestParseIDLLog_UsesBMCSourceForEventLogs(t *testing.T) {
content := []byte(`|2025-12-02T17:54:27+08:00|MEMORY|Assert|Warning|0C180401|CPU1_C4D0 Memory Device Disabled - Assert|`)
@@ -17,6 +23,14 @@ func TestParseIDLLog_UsesBMCSourceForEventLogs(t *testing.T) {
}
}
func TestParseIDLLog_DeassertedFaultIsInfo(t *testing.T) {
content := []byte(`|2026-05-07T23:57:08+08:00|BMC|Deassert|Critical|1800B002|Sys_Health Transition to Critical from less severe - Deassert|`)
events := ParseIDLLog(content)
if len(events) != 1 || events[0].Severity != models.SeverityInfo {
t.Fatalf("expected deasserted IDL fault to be informational, got %+v", events)
}
}
func TestParseSyslog_UsesHostSourceAndProcessAsSensorName(t *testing.T) {
content := []byte(`<13>2026-03-15T14:03:11+00:00 host123 systemd[1]: Started Example Service`)
@@ -31,3 +45,60 @@ func TestParseSyslog_UsesHostSourceAndProcessAsSensorName(t *testing.T) {
t.Fatalf("expected process name in sensor/component slot, got %#v", events[0])
}
}
func TestParseSyslog_UsesPriorityAndDowngradesKnownBenignMessages(t *testing.T) {
content := []byte("<1> 1970-01-01T08:01:38.795020+08:00 host kernel: [ 17.763621] Helper Module Driver Version 1.2\n" +
"<1> 2026-01-29T16:24:12.888636+08:00 host kernel: [ 16.191801] Helper Module Driver Version 1.2\n" +
"<132> 2026-07-25T08:42:47.001654+03:00 host kernel: [100009.801150] Color Depth is 15 bpp or higher\n" +
"<130> 2026-07-25T08:43:48.001964+03:00 host MCTPMain: MCTP_ERROR: No Response for Application. Timing-out...TimeoutMsec:4000\n")
events := ParseSyslog(content, "onekeylog/log/syslog/alert.log")
if len(events) != 3 {
t.Fatalf("expected 3 events, got %d", len(events))
}
if events[0].Severity != models.SeverityInfo {
t.Fatalf("expected benign driver alert to be info, got %#v", events[0])
}
if events[1].Severity != models.SeverityInfo {
t.Fatalf("expected KVM status line to be info, got %#v", events[1])
}
if events[2].Severity != models.SeverityCritical {
t.Fatalf("expected PRI=2 MCTP fault to be critical, got %#v", events[2])
}
}
func TestParserIncludesCriticalSyslogMCTPAndMESelfTest(t *testing.T) {
files := []baseparser.ExtractedFile{{
Path: "onekeylog/log/syslog/crit.log",
Content: []byte("<130> 2026-07-25T08:42:47.001654+03:00 host MCTPMain: MCTP_ERROR: No Response for Application. Timing-out...TimeoutMsec:4000\n" +
"<130> 2026-07-25T08:43:28.072198+03:00 host IPMIMain: Me self-tests is not normal, return code is 195!\n"),
}}
result, err := (&Parser{}).Parse(files)
if err != nil {
t.Fatalf("parse: %v", err)
}
if len(result.Events) != 2 {
t.Fatalf("expected both critical syslog events, got %d: %+v", len(result.Events), result.Events)
}
for _, event := range result.Events {
if event.Severity != models.SeverityCritical {
t.Fatalf("expected critical event, got %#v", event)
}
}
if result.Events[0].Timestamp.After(result.Events[1].Timestamp) {
t.Fatalf("events are not sorted: %+v", result.Events)
}
}
func TestSortInspurEventsPlacesUnknownTimestampsLast(t *testing.T) {
events := []models.Event{
{ID: "late", Timestamp: time.Date(2026, 7, 25, 8, 0, 0, 0, time.UTC)},
{ID: "unknown"},
{ID: "early", Timestamp: time.Date(2026, 7, 24, 8, 0, 0, 0, time.UTC)},
}
sortInspurEvents(events)
if events[0].ID != "early" || events[1].ID != "late" || events[2].ID != "unknown" {
t.Fatalf("unexpected event order: %+v", events)
}
}
+3
View File
@@ -43,6 +43,9 @@ func ParseIDLLog(content []byte) []models.Event {
// Map severity
severity := mapIDLSeverity(severityStr)
if strings.Contains(strings.ToLower(eventType), "deassert") {
severity = models.SeverityInfo
}
// Clean up description
description = cleanDescription(description)
+19 -6
View File
@@ -16,7 +16,7 @@ import (
// parserVersion - version of this parser module
// IMPORTANT: Increment this version when making changes to parser logic!
const parserVersion = "2.3"
const parserVersion = "2.5"
func init() {
parser.Register(&Parser{})
@@ -96,6 +96,7 @@ func containsInspurMarkers(content []byte) bool {
// Parse parses Inspur/Kaytus archive
func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, error) {
selLocation := inferInspurArchiveLocation(files)
selTimezone := inferInspurTimezoneResolver(files, selLocation)
result := &models.AnalysisResult{
Events: make([]models.Event, 0),
@@ -239,15 +240,18 @@ func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, er
// Parse SEL list (selelist.csv), falling back to log/sel.csv on the
// dump_<serial>_<timestamp>/ layout that has no selelist.csv.
if f := parser.FindFileByName(files, "selelist.csv"); f != nil {
selEvents := ParseSELListWithLocation(f.Content, selLocation)
selEvents := parseSELListWithResolver(f.Content, selTimezone)
result.Events = append(result.Events, selEvents...)
} else if f := parser.FindFileByName(files, "sel.csv"); f != nil {
selEvents := ParseSELListWithLocation(f.Content, selLocation)
selEvents := parseSELListWithResolver(f.Content, selTimezone)
result.Events = append(result.Events, selEvents...)
}
// Parse syslog files
syslogFiles := parser.FindFileByPattern(files, "syslog/alert", "syslog/warning", "syslog/notice", "syslog/info")
syslogFiles := parser.FindFileByPattern(files,
"syslog/emerg", "syslog/alert", "syslog/crit", "syslog/error",
"syslog/warning", "syslog/notice", "syslog/info",
)
for _, f := range syslogFiles {
events := ParseSyslog(f.Content, f.Path)
result.Events = append(result.Events, events...)
@@ -257,9 +261,11 @@ func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, er
// controller, switch VR/power, switch CPLD and slot presence failures.
result.Events = append(result.Events, ParseCommerCompEvents(files, selLocation)...)
// Same SEL event can be reported twice by the BMC (e.g. once via sel.csv,
// once via idl.log); collapse exact duplicates so they don't double-count.
// Same event can be reported by both IDL and SEL. After timezone
// normalization, retain the richer offset-bearing IDL event and remove the
// equivalent SEL copy, then provide a deterministic chronological stream.
result.Events = dedupSELEvents(result.Events)
sortInspurEvents(result.Events)
// Fallback for archives where board serial is missing in parsed FRU/asset data:
// recover it from log content, never from archive filename.
@@ -284,6 +290,13 @@ func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, er
// Mark problematic GPUs from IDL errors like "BIOS miss F_GPU6".
if result.Hardware != nil {
applyGPUStatusFromEvents(result.Hardware, result.Events)
var currentDeviceEvents []models.Event
currentDeviceSnapshotAvailable := false
if f := parser.FindFileByName(files, "dev_status.log"); f != nil {
currentDeviceSnapshotAvailable = true
currentDeviceEvents = ParseIDLLog(f.Content)
}
applyStorageStatusFromEvents(result.Hardware, result.Events, currentDeviceEvents, currentDeviceSnapshotAvailable)
enrichStorageFromSerialFallbackFiles(files, result.Hardware)
// Enrich storage serial numbers from smartd output in SOLHostCapture.log.
// Fills in serial, model, firmware for backplane slots that the BMC HDD API left empty.
+108 -26
View File
@@ -2,8 +2,10 @@ package inspur
import (
"encoding/csv"
"sort"
"strings"
"time"
"unicode"
"git.mchus.pro/mchus/logpile/internal/models"
"git.mchus.pro/mchus/logpile/internal/parser"
@@ -19,6 +21,14 @@ func ParseSELList(content []byte) []models.Event {
// ParseSELListWithLocation parses selelist.csv using provided source timezone
// for timestamps that don't contain an explicit offset.
func ParseSELListWithLocation(content []byte, location *time.Location) []models.Event {
return parseSELList(content, func(local time.Time) *time.Location { return location })
}
func parseSELListWithResolver(content []byte, resolver *inspurTimezoneResolver) []models.Event {
return parseSELList(content, resolver.locationFor)
}
func parseSELList(content []byte, locationFor func(time.Time) *time.Location) []models.Event {
var events []models.Event
text := string(content)
@@ -55,7 +65,7 @@ func ParseSELListWithLocation(content []byte, location *time.Location) []models.
status := strings.TrimSpace(records[5])
// Parse timestamp: MM/DD/YYYY HH:MM:SS
timestamp := parseSELTimestamp(dateStr, timeStr, location)
timestamp := parseSELTimestampWithResolver(dateStr, timeStr, locationFor)
// Extract sensor type and name
sensorType, sensorName := parseSensorInfo(sensorStr)
@@ -82,23 +92,24 @@ func ParseSELListWithLocation(content []byte, location *time.Location) []models.
return events
}
func parseSELTimestampWithResolver(dateStr, timeStr string, locationFor func(time.Time) *time.Location) time.Time {
timestampStr := dateStr + " " + timeStr
local, err := time.ParseInLocation("01/02/2006 15:04:05", timestampStr, time.UTC)
if err != nil {
return time.Time{}
}
location := parser.DefaultArchiveLocation()
if locationFor != nil {
if resolved := locationFor(local); resolved != nil {
location = resolved
}
}
return time.Date(local.Year(), local.Month(), local.Day(), local.Hour(), local.Minute(), local.Second(), 0, location)
}
// parseSELTimestamp parses MM/DD/YYYY and HH:MM:SS into time.Time
func parseSELTimestamp(dateStr, timeStr string, location *time.Location) time.Time {
// Combine date and time: MM/DD/YYYY HH:MM:SS
timestampStr := dateStr + " " + timeStr
if location == nil {
location = parser.DefaultArchiveLocation()
}
// Try parsing with MM/DD/YYYY format
t, err := time.ParseInLocation("01/02/2006 15:04:05", timestampStr, location)
if err != nil {
// Fallback to current time
return time.Now()
}
return t
return parseSELTimestampWithResolver(dateStr, timeStr, func(time.Time) *time.Location { return location })
}
// parseSensorInfo extracts sensor type and name from sensor string
@@ -128,6 +139,9 @@ func determineSELSeverity(sensorStr, eventDesc, status string) models.Severity {
lowerSensor := strings.ToLower(sensorStr)
lowerEvent := strings.ToLower(eventDesc)
lowerStatus := strings.ToLower(status)
if strings.Contains(lowerStatus, "deassert") {
return models.SeverityInfo
}
// Critical indicators
criticalKeywords := []string{
@@ -176,32 +190,100 @@ func determineSELSeverity(sensorStr, eventDesc, status string) models.Severity {
return models.SeverityInfo
}
// dedupSELEvents collapses events reported more than once with the same
// (timestamp, event_type, description) triple. Unlike ParseIDLLog's
// dedup (which must keep recurring alarms with distinct timestamps), this
// only removes true duplicates: the same SEL entry surfacing through more
// than one source file for the exact same moment.
// dedupSELEvents removes exact SEL repeats and equivalent IDL/SEL copies at
// the same normalized instant. IDL is preferred because it carries an
// explicit timezone offset and usually a richer component-prefixed message.
func dedupSELEvents(events []models.Event) []models.Event {
if len(events) == 0 {
return events
}
seen := make(map[string]struct{}, len(events))
out := make([]models.Event, 0, len(events))
bySecond := make(map[int64][]int)
for _, e := range events {
if e.Source != "SEL" {
if e.Timestamp.IsZero() {
out = append(out, e)
continue
}
key := e.Timestamp.String() + "|" + e.EventType + "|" + e.Description
if _, ok := seen[key]; ok {
second := e.Timestamp.Unix()
duplicate := false
for _, idx := range bySecond[second] {
previous := out[idx]
if !equivalentInspurEvent(previous, e) {
continue
}
if previous.Source == "SEL" && e.Source != "SEL" {
out[idx] = e
}
duplicate = true
break
}
if duplicate {
continue
}
seen[key] = struct{}{}
bySecond[second] = append(bySecond[second], len(out))
out = append(out, e)
}
return out
}
func equivalentInspurEvent(a, b models.Event) bool {
if a.Source != "SEL" && b.Source != "SEL" {
return false
}
if a.Severity != b.Severity || eventDirection(a) != eventDirection(b) {
return false
}
left := normalizeEventDescription(a.Description)
right := normalizeEventDescription(b.Description)
if left == "" || right == "" {
return false
}
if left == right {
return true
}
shorter, longer := left, right
if len(shorter) > len(longer) {
shorter, longer = longer, shorter
}
return len(shorter) >= 12 && strings.Contains(longer, shorter)
}
func eventDirection(e models.Event) string {
value := strings.ToLower(e.EventType + " " + e.RawData)
if strings.Contains(value, "deassert") {
return "deassert"
}
if strings.Contains(value, "assert") {
return "assert"
}
return ""
}
func normalizeEventDescription(value string) string {
value = strings.ToLower(value)
value = strings.Map(func(r rune) rune {
if unicode.IsLetter(r) || unicode.IsDigit(r) {
return r
}
return ' '
}, value)
return strings.Join(strings.Fields(value), " ")
}
func sortInspurEvents(events []models.Event) {
sort.SliceStable(events, func(i, j int) bool {
left, right := events[i].Timestamp, events[j].Timestamp
if left.IsZero() {
return false
}
if right.IsZero() {
return true
}
return left.Before(right)
})
}
// buildSELDescription builds human-readable description
func buildSELDescription(eventDesc, status string) string {
if status == "Asserted" || status == "Deasserted" {
+41 -6
View File
@@ -5,20 +5,26 @@ import (
"time"
"git.mchus.pro/mchus/logpile/internal/models"
"git.mchus.pro/mchus/logpile/internal/parser"
)
func TestDedupSELEvents(t *testing.T) {
ts := time.Date(2026, 7, 27, 4, 52, 55, 0, time.UTC)
events := []models.Event{
{Source: "SEL", Timestamp: ts, EventType: "Asserted", Description: "PSU0 fail"},
{Source: "SEL", Timestamp: ts, EventType: "Asserted", Description: "PSU0 fail"},
{Source: "BMC", Timestamp: ts, EventType: "Assert", Description: "recurring alarm"},
{Source: "BMC", Timestamp: ts, EventType: "Assert", Description: "recurring alarm"},
{Source: "BMC", Timestamp: ts, EventType: "Assert", Severity: models.SeverityCritical, Description: "Sys_Health Transition to Critical from less severe"},
{Source: "SEL", Timestamp: ts, EventType: "Transition to Critical from less severe", Severity: models.SeverityCritical, Description: "Transition to Critical from less severe", RawData: "Asserted"},
{Source: "SEL", Timestamp: ts.Add(time.Second), EventType: "Asserted", Description: "PSU0 fail"},
{Source: "SEL", Timestamp: ts.Add(time.Second), EventType: "Asserted", Description: "PSU0 fail"},
{Source: "BMC", Timestamp: ts.Add(2 * time.Second), EventType: "Assert", Description: "recurring alarm"},
{Source: "BMC", Timestamp: ts.Add(2 * time.Second), EventType: "Assert", Description: "recurring alarm"},
}
out := dedupSELEvents(events)
if len(out) != 3 {
t.Fatalf("expected 3 events (1 SEL dup removed, BMC untouched), got %d: %+v", len(out), out)
if len(out) != 4 {
t.Fatalf("expected IDL/SEL copy and exact SEL copy removed, recurring BMC alarms retained; got %d: %+v", len(out), out)
}
if out[0].Source != "BMC" {
t.Fatalf("expected richer IDL/BMC event to be retained, got %+v", out[0])
}
}
@@ -48,3 +54,32 @@ func TestParseTimezoneConfigLocation(t *testing.T) {
t.Fatalf("unexpected timezone: %q", got)
}
}
func TestSELTimezoneResolverUsesExplicitOffsetTimeline(t *testing.T) {
files := []parser.ExtractedFile{
{Path: "onekeylog/configuration/conf/timezone.conf", Content: []byte("[TimeZoneConfig]\ntimezone=Asia/Shanghai\n")},
{Path: "onekeylog/log/syslog/idl.log", Content: []byte(
"|2026-01-30T14:25:14+08:00|BMC|Assert|Info|1|old timezone|\n" +
"|2026-07-24T02:20:38+03:00|BMC|Assert|Critical|2|new timezone|\n")},
}
fallback := inferInspurArchiveLocation(files)
resolver := inferInspurTimezoneResolver(files, fallback)
events := parseSELListWithResolver([]byte("sel elist:\n"+
"1,01/30/2026,14:25:14,Chassis Sys_Health,old timezone,Asserted\n"+
"2,07/24/2026,02:20:38,Chassis Sys_Health,new timezone,Asserted\n"), resolver)
if len(events) != 2 {
t.Fatalf("expected 2 events, got %d", len(events))
}
wantOld := time.Date(2026, 1, 30, 6, 25, 14, 0, time.UTC)
wantNew := time.Date(2026, 7, 23, 23, 20, 38, 0, time.UTC)
if !events[0].Timestamp.UTC().Equal(wantOld) || !events[1].Timestamp.UTC().Equal(wantNew) {
t.Fatalf("unexpected resolved timestamps: got %s and %s", events[0].Timestamp.UTC(), events[1].Timestamp.UTC())
}
}
func TestDetermineSELSeverityDeassertedFaultIsInfo(t *testing.T) {
got := determineSELSeverity("Power Supply PSU0", "Failure detected", "Deasserted")
if got != models.SeverityInfo {
t.Fatalf("expected cleared fault to be info, got %q", got)
}
}
+144
View File
@@ -0,0 +1,144 @@
package inspur
import (
"regexp"
"sort"
"strconv"
"strings"
"git.mchus.pro/mchus/logpile/internal/models"
)
var (
reNVMeEventIndex = regexp.MustCompile(`(?i)\bNvmeIndex\s*:\s*(\d+)\b`)
reNVMeLabelIndex = regexp.MustCompile(`(?i)\bNVME(\d+)\b`)
reOBStorageSlot = regexp.MustCompile(`(?i)^OB(\d+)$`)
)
// applyStorageStatusFromEvents projects source-backed NVMe fault state onto
// the physical storage inventory. Healthy-looking presence data is not enough
// to claim OK, so unaffected drives keep their existing (usually Unknown)
// status. dev_status.log, when available, is the authoritative active-fault
// snapshot; IDL supplies transition history.
func applyStorageStatusFromEvents(hw *models.HardwareConfig, events, currentDeviceEvents []models.Event, currentSnapshotAvailable bool) {
if hw == nil || len(hw.Storage) == 0 {
return
}
storageByIndex := make(map[int]*models.Storage)
for i := range hw.Storage {
if idx, ok := storageIndexFromSlot(hw.Storage[i].Slot); ok {
storageByIndex[idx] = &hw.Storage[i]
}
}
relevant := relevantStorageFaultEvents(events)
sort.SliceStable(relevant, func(i, j int) bool {
return relevant[i].Timestamp.Before(relevant[j].Timestamp)
})
touched := make(map[int]bool)
for _, event := range relevant {
idx, ok := storageFaultIndex(event)
storage := storageByIndex[idx]
if !ok || storage == nil {
continue
}
touched[idx] = true
applyStorageFaultTransition(storage, event)
}
if !currentSnapshotAvailable {
return
}
active := make(map[int]models.Event)
for _, event := range relevantStorageFaultEvents(currentDeviceEvents) {
idx, ok := storageFaultIndex(event)
if !ok || eventIsDeassert(event) {
continue
}
active[idx] = event
}
for idx := range touched {
if _, ok := active[idx]; ok {
continue
}
storage := storageByIndex[idx]
storage.Status = "Unknown"
storage.ErrorDescription = ""
}
for idx, event := range active {
if storage := storageByIndex[idx]; storage != nil {
applyStorageFaultTransition(storage, event)
}
}
}
func relevantStorageFaultEvents(events []models.Event) []models.Event {
out := make([]models.Event, 0)
for _, event := range events {
if _, ok := storageFaultIndex(event); ok {
out = append(out, event)
}
}
return out
}
func storageFaultIndex(event models.Event) (int, bool) {
description := strings.TrimSpace(event.Description)
lower := strings.ToLower(description)
if !strings.Contains(lower, "drive fault") && !strings.EqualFold(strings.TrimSpace(event.ID), "0DFF0702") {
return 0, false
}
for _, re := range []*regexp.Regexp{reNVMeEventIndex, reNVMeLabelIndex} {
match := re.FindStringSubmatch(description)
if len(match) != 2 {
continue
}
idx, err := strconv.Atoi(match[1])
if err == nil && idx >= 0 {
return idx, true
}
}
return 0, false
}
func storageIndexFromSlot(slot string) (int, bool) {
match := reOBStorageSlot.FindStringSubmatch(strings.TrimSpace(slot))
if len(match) != 2 {
return 0, false
}
oneBased, err := strconv.Atoi(match[1])
if err != nil || oneBased < 1 {
return 0, false
}
return oneBased - 1, true
}
func applyStorageFaultTransition(storage *models.Storage, event models.Event) {
status := "Critical"
description := strings.TrimSpace(event.Description)
if eventIsDeassert(event) {
status = "Unknown"
description = ""
}
if storage.Status != status && !event.Timestamp.IsZero() {
storage.StatusHistory = append(storage.StatusHistory, models.StatusHistoryEntry{
Status: status,
ChangedAt: event.Timestamp,
Details: strings.TrimSpace(event.Description),
})
ts := event.Timestamp
storage.StatusChangedAt = &ts
}
storage.Status = status
storage.ErrorDescription = description
if !event.Timestamp.IsZero() {
ts := event.Timestamp
storage.StatusCheckedAt = &ts
}
}
func eventIsDeassert(event models.Event) bool {
return strings.Contains(strings.ToLower(event.EventType+" "+event.RawData), "deassert")
}
+71
View File
@@ -0,0 +1,71 @@
package inspur
import (
"fmt"
"testing"
"time"
"git.mchus.pro/mchus/logpile/internal/models"
)
func TestApplyStorageStatusFromEventsMarksOnlyFaultyNVMeCritical(t *testing.T) {
hw := &models.HardwareConfig{}
for i := 1; i <= 7; i++ {
hw.Storage = append(hw.Storage, models.Storage{Slot: fmt.Sprintf("OB%02d", i), Type: "NVMe", Present: true})
}
ts := time.Date(2026, 7, 23, 23, 20, 32, 0, time.UTC)
fault := models.Event{
ID: "0DFF0702",
Timestamp: ts,
Source: "BMC",
EventType: "Assert",
Severity: models.SeverityCritical,
Description: "NVME BP:Front0 NvmeIndex:6 drive fault: Status_Flags error.",
}
applyStorageStatusFromEvents(hw, []models.Event{fault}, []models.Event{fault}, true)
for i := range hw.Storage {
if hw.Storage[i].Slot == "OB07" {
if hw.Storage[i].Status != "Critical" || hw.Storage[i].ErrorDescription == "" {
t.Fatalf("expected OB07 critical with details, got %+v", hw.Storage[i])
}
if len(hw.Storage[i].StatusHistory) != 1 || hw.Storage[i].StatusChangedAt == nil {
t.Fatalf("expected one source-backed status transition, got %+v", hw.Storage[i])
}
continue
}
if hw.Storage[i].Status != "" {
t.Fatalf("unaffected %s must remain unknown/unset, got %q", hw.Storage[i].Slot, hw.Storage[i].Status)
}
}
}
func TestApplyStorageStatusFromEventsDeassertClearsCriticalToUnknown(t *testing.T) {
hw := &models.HardwareConfig{Storage: []models.Storage{{Slot: "OB07", Type: "NVMe", Present: true}}}
asserted := models.Event{
ID: "0DFF0702", Timestamp: time.Date(2026, 7, 23, 23, 20, 32, 0, time.UTC),
EventType: "Assert", Severity: models.SeverityCritical,
Description: "NVME BP:Front0 NvmeIndex:6 drive fault: Status_Flags error.",
}
deasserted := asserted
deasserted.Timestamp = asserted.Timestamp.Add(time.Hour)
deasserted.EventType = "Deassert"
deasserted.Severity = models.SeverityInfo
applyStorageStatusFromEvents(hw, []models.Event{asserted, deasserted}, nil, true)
if hw.Storage[0].Status != "Unknown" || hw.Storage[0].ErrorDescription != "" {
t.Fatalf("expected cleared drive fault to become Unknown, got %+v", hw.Storage[0])
}
if len(hw.Storage[0].StatusHistory) != 2 {
t.Fatalf("expected assert/deassert history, got %+v", hw.Storage[0].StatusHistory)
}
}
func TestStorageFaultIndexIgnoresNVMeInventoryChange(t *testing.T) {
event := models.Event{Description: "PCIE Device #NVME6 Device ID changed; Vendor ID changed"}
if _, ok := storageFaultIndex(event); ok {
t.Fatalf("inventory change must not be treated as an active drive fault")
}
}
+50 -6
View File
@@ -3,6 +3,7 @@ package inspur
import (
"bufio"
"regexp"
"strconv"
"strings"
"time"
@@ -11,16 +12,14 @@ import (
var (
// Syslog format: <priority> timestamp hostname process: message
syslogRegex = regexp.MustCompile(`^<(\d+)>\s*(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[^\s]*)\s+(\S+)\s+(\S+):\s*(.*)$`)
syslogRegex = regexp.MustCompile(`^<(\d+)>\s*(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[^\s]*)\s+(\S+)\s+(\S+):\s*(.*)$`)
bootRelativeMessageRegex = regexp.MustCompile(`^\[\s*\d+(?:\.\d+)?\]`)
)
// ParseSyslog parses syslog format logs
func ParseSyslog(content []byte, sourcePath string) []models.Event {
var events []models.Event
// Determine severity from file path
severity := determineSeverityFromPath(sourcePath)
scanner := bufio.NewScanner(strings.NewReader(string(content)))
lineNum := 0
@@ -35,6 +34,10 @@ func ParseSyslog(content []byte, sourcePath string) []models.Event {
if matches == nil {
continue
}
priority, err := strconv.Atoi(matches[1])
if err != nil {
continue
}
timestamp, err := time.Parse(time.RFC3339, matches[2])
if err != nil {
@@ -44,6 +47,14 @@ func ParseSyslog(content []byte, sourcePath string) []models.Event {
continue
}
}
message := strings.TrimSpace(matches[5])
// Some AMI BMC boots emit a fabricated 1970 wall-clock timestamp while
// the payload only carries seconds since boot. Without a trustworthy
// boot epoch this cannot become a wall-clock event, so omit it instead
// of exporting either 1970 or the archive collection time.
if timestamp.Year() <= 1971 && bootRelativeMessageRegex.MatchString(message) {
continue
}
event := models.Event{
ID: generateEventID(sourcePath, lineNum),
@@ -51,8 +62,8 @@ func ParseSyslog(content []byte, sourcePath string) []models.Event {
Source: "syslog",
SensorType: "syslog",
SensorName: matches[4],
Description: matches[5],
Severity: severity,
Description: message,
Severity: determineSyslogSeverity(priority, message, sourcePath),
RawData: line,
}
@@ -62,6 +73,39 @@ func ParseSyslog(content []byte, sourcePath string) []models.Event {
return events
}
func determineSyslogSeverity(priority int, message, sourcePath string) models.Severity {
// These AMI driver start-up/status strings are routed to alert.log and
// warning.log despite not describing a fault. The PRI value is therefore
// not trustworthy for this small, observed set of benign messages.
lowerMessage := strings.ToLower(message)
benignMessages := []string{
"helper module driver version",
"copyright (c)",
"color depth is 15 bpp or higher",
"new driver 0 directmode 1",
}
for _, benign := range benignMessages {
if strings.Contains(lowerMessage, benign) {
return models.SeverityInfo
}
}
// RFC 5424 severity is stored in the low three bits of PRI:
// 0..2 emergency/alert/critical, 3..4 error/warning, 5..7 notice/info/debug.
if priority >= 0 {
switch priority & 7 {
case 0, 1, 2:
return models.SeverityCritical
case 3, 4:
return models.SeverityWarning
default:
return models.SeverityInfo
}
}
return determineSeverityFromPath(sourcePath)
}
func determineSeverityFromPath(path string) models.Severity {
pathLower := strings.ToLower(path)
+127
View File
@@ -0,0 +1,127 @@
package inspur
import (
"bufio"
"bytes"
"regexp"
"sort"
"strings"
"time"
"git.mchus.pro/mchus/logpile/internal/parser"
)
var explicitInspurTimestampRegex = regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?[+-]\d{2}:\d{2}`)
type inspurOffsetSample struct {
local time.Time
offset int
}
// inspurTimezoneResolver handles dumps where timezone.conf is stale or the BMC
// timezone changed during the retained SEL history. Offset-bearing IDL/syslog
// timestamps are authoritative; timezone.conf remains the fallback.
type inspurTimezoneResolver struct {
fallback *time.Location
samples []inspurOffsetSample
}
func inferInspurTimezoneResolver(files []parser.ExtractedFile, fallback *time.Location) *inspurTimezoneResolver {
if fallback == nil {
fallback = parser.DefaultArchiveLocation()
}
r := &inspurTimezoneResolver{fallback: fallback}
seen := make(map[string]struct{})
preferIDL := false
for _, f := range files {
if strings.Contains(strings.ToLower(f.Path), "idl") {
preferIDL = true
break
}
}
for pass := 0; pass < 2; pass++ {
idlOnly := preferIDL && pass == 0
if pass == 1 && (!preferIDL || len(r.samples) > 0) {
break
}
for _, f := range files {
path := strings.ToLower(f.Path)
if idlOnly && !strings.Contains(path, "idl") {
continue
}
if !strings.Contains(path, "/log/") && !strings.Contains(path, "idl") {
continue
}
scanner := bufio.NewScanner(bytes.NewReader(f.Content))
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
for scanner.Scan() {
matches := explicitInspurTimestampRegex.FindAllString(scanner.Text(), -1)
if len(matches) == 0 {
continue
}
// idl.log is itself a syslog stream. Its outer timestamp can retain
// an obsolete BMC offset while the embedded IDL record has the offset
// that was active for the hardware event. Prefer the embedded value.
raw := matches[len(matches)-1]
ts, err := time.Parse(time.RFC3339Nano, raw)
if err != nil || ts.Year() < 2000 {
continue
}
_, offset := ts.Zone()
local := time.Date(ts.Year(), ts.Month(), ts.Day(), ts.Hour(), ts.Minute(), ts.Second(), 0, time.UTC)
// One sample per local day and offset is sufficient to identify BMC
// timezone eras while bounding memory on very large rotated logs.
key := local.Format("2006-01-02") + "|" + strings.TrimSpace(raw[len(raw)-6:])
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
r.samples = append(r.samples, inspurOffsetSample{local: local, offset: offset})
}
}
}
sort.Slice(r.samples, func(i, j int) bool { return r.samples[i].local.Before(r.samples[j].local) })
return r
}
func (r *inspurTimezoneResolver) locationFor(local time.Time) *time.Location {
if r == nil || len(r.samples) == 0 {
if r != nil && r.fallback != nil {
return r.fallback
}
return parser.DefaultArchiveLocation()
}
best := r.samples[0]
bestDistance := absDuration(local.Sub(best.local))
for _, sample := range r.samples[1:] {
distance := absDuration(local.Sub(sample.local))
if distance < bestDistance {
best = sample
bestDistance = distance
}
}
return time.FixedZone(formatUTCOffset(best.offset), best.offset)
}
func absDuration(v time.Duration) time.Duration {
if v < 0 {
return -v
}
return v
}
func formatUTCOffset(offset int) string {
sign := "+"
if offset < 0 {
sign = "-"
offset = -offset
}
return "UTC" + sign + twoDigits(offset/3600) + ":" + twoDigits((offset%3600)/60)
}
func twoDigits(v int) string {
return string([]byte{'0' + byte(v/10), '0' + byte(v%10)})
}