Event ingestion now uses the source syslog PRI and an explicit-offset timezone timeline instead of assuming host-local time. CPU PPIN is exported as the source-backed CPU serial, and an active NVMe fault SEL event promotes the matching drive's storage status. See ADL-056, ADL-057. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
145 lines
3.9 KiB
Go
145 lines
3.9 KiB
Go
package inspur
|
|
|
|
import (
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"git.mchus.pro/mchus/logpile/internal/models"
|
|
)
|
|
|
|
var (
|
|
reNVMeEventIndex = regexp.MustCompile(`(?i)\bNvmeIndex\s*:\s*(\d+)\b`)
|
|
reNVMeLabelIndex = regexp.MustCompile(`(?i)\bNVME(\d+)\b`)
|
|
reOBStorageSlot = regexp.MustCompile(`(?i)^OB(\d+)$`)
|
|
)
|
|
|
|
// applyStorageStatusFromEvents projects source-backed NVMe fault state onto
|
|
// the physical storage inventory. Healthy-looking presence data is not enough
|
|
// to claim OK, so unaffected drives keep their existing (usually Unknown)
|
|
// status. dev_status.log, when available, is the authoritative active-fault
|
|
// snapshot; IDL supplies transition history.
|
|
func applyStorageStatusFromEvents(hw *models.HardwareConfig, events, currentDeviceEvents []models.Event, currentSnapshotAvailable bool) {
|
|
if hw == nil || len(hw.Storage) == 0 {
|
|
return
|
|
}
|
|
|
|
storageByIndex := make(map[int]*models.Storage)
|
|
for i := range hw.Storage {
|
|
if idx, ok := storageIndexFromSlot(hw.Storage[i].Slot); ok {
|
|
storageByIndex[idx] = &hw.Storage[i]
|
|
}
|
|
}
|
|
|
|
relevant := relevantStorageFaultEvents(events)
|
|
sort.SliceStable(relevant, func(i, j int) bool {
|
|
return relevant[i].Timestamp.Before(relevant[j].Timestamp)
|
|
})
|
|
touched := make(map[int]bool)
|
|
for _, event := range relevant {
|
|
idx, ok := storageFaultIndex(event)
|
|
storage := storageByIndex[idx]
|
|
if !ok || storage == nil {
|
|
continue
|
|
}
|
|
touched[idx] = true
|
|
applyStorageFaultTransition(storage, event)
|
|
}
|
|
|
|
if !currentSnapshotAvailable {
|
|
return
|
|
}
|
|
|
|
active := make(map[int]models.Event)
|
|
for _, event := range relevantStorageFaultEvents(currentDeviceEvents) {
|
|
idx, ok := storageFaultIndex(event)
|
|
if !ok || eventIsDeassert(event) {
|
|
continue
|
|
}
|
|
active[idx] = event
|
|
}
|
|
for idx := range touched {
|
|
if _, ok := active[idx]; ok {
|
|
continue
|
|
}
|
|
storage := storageByIndex[idx]
|
|
storage.Status = "Unknown"
|
|
storage.ErrorDescription = ""
|
|
}
|
|
for idx, event := range active {
|
|
if storage := storageByIndex[idx]; storage != nil {
|
|
applyStorageFaultTransition(storage, event)
|
|
}
|
|
}
|
|
}
|
|
|
|
func relevantStorageFaultEvents(events []models.Event) []models.Event {
|
|
out := make([]models.Event, 0)
|
|
for _, event := range events {
|
|
if _, ok := storageFaultIndex(event); ok {
|
|
out = append(out, event)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func storageFaultIndex(event models.Event) (int, bool) {
|
|
description := strings.TrimSpace(event.Description)
|
|
lower := strings.ToLower(description)
|
|
if !strings.Contains(lower, "drive fault") && !strings.EqualFold(strings.TrimSpace(event.ID), "0DFF0702") {
|
|
return 0, false
|
|
}
|
|
for _, re := range []*regexp.Regexp{reNVMeEventIndex, reNVMeLabelIndex} {
|
|
match := re.FindStringSubmatch(description)
|
|
if len(match) != 2 {
|
|
continue
|
|
}
|
|
idx, err := strconv.Atoi(match[1])
|
|
if err == nil && idx >= 0 {
|
|
return idx, true
|
|
}
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
func storageIndexFromSlot(slot string) (int, bool) {
|
|
match := reOBStorageSlot.FindStringSubmatch(strings.TrimSpace(slot))
|
|
if len(match) != 2 {
|
|
return 0, false
|
|
}
|
|
oneBased, err := strconv.Atoi(match[1])
|
|
if err != nil || oneBased < 1 {
|
|
return 0, false
|
|
}
|
|
return oneBased - 1, true
|
|
}
|
|
|
|
func applyStorageFaultTransition(storage *models.Storage, event models.Event) {
|
|
status := "Critical"
|
|
description := strings.TrimSpace(event.Description)
|
|
if eventIsDeassert(event) {
|
|
status = "Unknown"
|
|
description = ""
|
|
}
|
|
if storage.Status != status && !event.Timestamp.IsZero() {
|
|
storage.StatusHistory = append(storage.StatusHistory, models.StatusHistoryEntry{
|
|
Status: status,
|
|
ChangedAt: event.Timestamp,
|
|
Details: strings.TrimSpace(event.Description),
|
|
})
|
|
ts := event.Timestamp
|
|
storage.StatusChangedAt = &ts
|
|
}
|
|
storage.Status = status
|
|
storage.ErrorDescription = description
|
|
if !event.Timestamp.IsZero() {
|
|
ts := event.Timestamp
|
|
storage.StatusCheckedAt = &ts
|
|
}
|
|
}
|
|
|
|
func eventIsDeassert(event models.Event) bool {
|
|
return strings.Contains(strings.ToLower(event.EventType+" "+event.RawData), "deassert")
|
|
}
|