logpile no longer carries its own archive/vendor parsers or Redfish raw-payload replay/analysis engine. Both now live in git.mchus.pro/reanimator/parser/v13 (already used by the sibling `core` project), reached through parser.NewService().AnalyzeArchivePayload / AnalyzeRedfishRawPayloads. logpile keeps live BMC/Redfish collection, the DB, exporter, server and UI. - Deleted internal/parser/ (interface, registry, archive extraction, all vendor packages), internal/collector/redfish_replay*.go (replay engine, confirmed near-identical to the module's redfish/ package), internal/ingest/ (pure duplicate of the module's Service), internal/models/ (repointed every import to the module's models package instead), and internal/rar5 (only used by the deleted archive extractor). Removed the now-unused third_party/pciids submodule. - internal/collector/redfish.go: Collect() now calls parser.NewService().AnalyzeRedfishRawPayloads(...) at the one place it used to call the local ReplayRedfishFromRawPayloads (an existing "unified tunnel" between live collection and raw import). A deadcode-driven pass then removed ~150 doc-to-model conversion helpers in redfish.go that duplicated the module's redfish/ package and were already unreachable before this change; kept the small generic dedup/scoring helpers the live crawler still calls. - internal/collector/redfishprofile/ stays whole: Collect() depends on it directly for acquisition-time tuning. Its analysis-only methods (ApplyAnalysisProfiles/PostAnalyze) are now dead but left in place rather than splitting the package's single Profile interface. - pciids moved with the module (now a public package there); logpile's live collector and exporter import it from there instead of keeping a local copy. - go.mod: new require on git.mchus.pro/reanimator/parser/v13 v13.0.0 (plain module dependency, not a submodule like reanimator/chart, since this module is independently versioned and shared with core). - bible-local docs rewritten for the new boundary; see ADL-080. See also the companion change in git.mchus.pro/reanimator/parser (tagged v13.0.0): new top-level exports (IsSupportedArchiveFilename/SupportedArchiveExtensions/ ParseInDefaultArchiveLocation/PrivacyScanEnabled) and pciids moved to a public package, both needed to complete this migration. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
259 lines
7.9 KiB
Go
259 lines
7.9 KiB
Go
package collector
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
redfishLogEntriesWindow = 7 * 24 * time.Hour
|
|
redfishLogEntriesMaxTotal = 500
|
|
redfishLogEntriesMaxPerSvc = 200
|
|
)
|
|
|
|
// collectRedfishLogEntries fetches hardware event log entries from Systems and Managers LogServices.
|
|
// Only hardware-relevant entries from the last 7 days are returned.
|
|
// For Systems: all log services except audit/journal/security/debug.
|
|
// For Managers: only the IPMI SEL service (Id="SEL") — audit and event logs are excluded.
|
|
func (c *RedfishConnector) collectRedfishLogEntries(ctx context.Context, client *http.Client, req Request, baseURL string, systemPaths, managerPaths []string) []map[string]interface{} {
|
|
cutoff := time.Now().UTC().Add(-redfishLogEntriesWindow)
|
|
seen := make(map[string]struct{})
|
|
var out []map[string]interface{}
|
|
|
|
collectFrom := func(logServicesPath string, filter func(map[string]interface{}) bool) {
|
|
if len(out) >= redfishLogEntriesMaxTotal {
|
|
return
|
|
}
|
|
services, err := c.getCollectionMembers(ctx, client, req, baseURL, logServicesPath)
|
|
if err != nil || len(services) == 0 {
|
|
return
|
|
}
|
|
for _, svc := range services {
|
|
if len(out) >= redfishLogEntriesMaxTotal {
|
|
break
|
|
}
|
|
if !filter(svc) {
|
|
continue
|
|
}
|
|
entriesPath := redfishLogServiceEntriesPath(svc)
|
|
if entriesPath == "" {
|
|
continue
|
|
}
|
|
entries := c.fetchRedfishLogEntriesWithPaging(ctx, client, req, baseURL, entriesPath, cutoff, seen, redfishLogEntriesMaxPerSvc)
|
|
out = append(out, entries...)
|
|
}
|
|
}
|
|
|
|
for _, systemPath := range systemPaths {
|
|
for _, logServicesPath := range c.redfishLinkedCollectionPaths(ctx, client, req, baseURL, systemPath, "LogServices") {
|
|
collectFrom(logServicesPath, isHardwareLogService)
|
|
}
|
|
}
|
|
// Managers hold the IPMI SEL on AMI/MSI BMCs — include only the "SEL" service.
|
|
for _, managerPath := range managerPaths {
|
|
for _, logServicesPath := range c.redfishLinkedCollectionPaths(ctx, client, req, baseURL, managerPath, "LogServices") {
|
|
collectFrom(logServicesPath, isManagerSELService)
|
|
}
|
|
}
|
|
|
|
if len(out) > 0 {
|
|
slog.Info("redfish: collected hardware log entries", "count", len(out), "window", "7d")
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (c *RedfishConnector) redfishLinkedCollectionPaths(
|
|
ctx context.Context,
|
|
client *http.Client,
|
|
req Request,
|
|
baseURL, resourcePath, linkKey string,
|
|
) []string {
|
|
resourcePath = normalizeRedfishPath(resourcePath)
|
|
if resourcePath == "" || strings.TrimSpace(linkKey) == "" {
|
|
return nil
|
|
}
|
|
|
|
seen := make(map[string]struct{}, 2)
|
|
var out []string
|
|
add := func(path string) {
|
|
path = normalizeRedfishPath(path)
|
|
if path == "" {
|
|
return
|
|
}
|
|
if _, ok := seen[path]; ok {
|
|
return
|
|
}
|
|
seen[path] = struct{}{}
|
|
out = append(out, path)
|
|
}
|
|
|
|
add(joinPath(resourcePath, "/"+strings.TrimSpace(linkKey)))
|
|
|
|
resourceDoc, err := c.getJSON(ctx, client, req, baseURL, resourcePath)
|
|
if err == nil {
|
|
if linked := redfishLinkedPath(resourceDoc, linkKey); linked != "" {
|
|
add(linked)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// fetchRedfishLogEntriesWithPaging fetches entries from a LogEntry collection,
|
|
// following nextLink pages. Stops early when entries older than cutoff are encountered
|
|
// (assumes BMC returns entries newest-first, which is typical).
|
|
func (c *RedfishConnector) fetchRedfishLogEntriesWithPaging(ctx context.Context, client *http.Client, req Request, baseURL, entriesPath string, cutoff time.Time, seen map[string]struct{}, limit int) []map[string]interface{} {
|
|
var out []map[string]interface{}
|
|
nextPath := entriesPath
|
|
|
|
for nextPath != "" && len(out) < limit {
|
|
collection, err := c.getJSON(ctx, client, req, baseURL, nextPath)
|
|
if err != nil {
|
|
break
|
|
}
|
|
|
|
// Handle both linked members (@odata.id only) and inline members (full objects).
|
|
rawMembers, _ := collection["Members"].([]interface{})
|
|
hitOldEntry := false
|
|
|
|
for _, rawMember := range rawMembers {
|
|
if len(out) >= limit {
|
|
break
|
|
}
|
|
memberMap, ok := rawMember.(map[string]interface{})
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
var entry map[string]interface{}
|
|
if _, hasCreated := memberMap["Created"]; hasCreated {
|
|
// Inline entry — use directly.
|
|
entry = memberMap
|
|
} else {
|
|
// Linked entry — fetch by path.
|
|
memberPath := normalizeRedfishPath(asString(memberMap["@odata.id"]))
|
|
if memberPath == "" {
|
|
continue
|
|
}
|
|
entry, err = c.getJSON(ctx, client, req, baseURL, memberPath)
|
|
if err != nil || len(entry) == 0 {
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Dedup by entry Id or path.
|
|
entryKey := asString(entry["Id"])
|
|
if entryKey == "" {
|
|
entryKey = asString(entry["@odata.id"])
|
|
}
|
|
if entryKey != "" {
|
|
if _, dup := seen[entryKey]; dup {
|
|
continue
|
|
}
|
|
seen[entryKey] = struct{}{}
|
|
}
|
|
|
|
// Time filter.
|
|
created := parseRedfishEntryTime(asString(entry["Created"]))
|
|
if !created.IsZero() && created.Before(cutoff) {
|
|
hitOldEntry = true
|
|
continue
|
|
}
|
|
|
|
// Hardware relevance filter.
|
|
if !isHardwareLogEntry(entry) {
|
|
continue
|
|
}
|
|
|
|
out = append(out, entry)
|
|
}
|
|
|
|
// Stop paging once we've seen entries older than the window.
|
|
if hitOldEntry {
|
|
break
|
|
}
|
|
nextPath = firstNonEmpty(
|
|
normalizeRedfishPath(asString(collection["Members@odata.nextLink"])),
|
|
normalizeRedfishPath(asString(collection["@odata.nextLink"])),
|
|
)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// isManagerSELService returns true only for the IPMI SEL exposed under Managers.
|
|
// On AMI/MSI BMCs the hardware SEL lives at Managers/{mgr}/LogServices/SEL.
|
|
// All other Manager log services (AuditLog, EventLog, Journal) are excluded.
|
|
func isManagerSELService(svc map[string]interface{}) bool {
|
|
id := strings.ToLower(strings.TrimSpace(asString(svc["Id"])))
|
|
return id == "sel"
|
|
}
|
|
|
|
// isHardwareLogService returns true if the log service looks like a hardware event log
|
|
// (SEL, System Event Log) rather than a BMC audit/journal log.
|
|
func isHardwareLogService(svc map[string]interface{}) bool {
|
|
id := strings.ToLower(strings.TrimSpace(asString(svc["Id"])))
|
|
name := strings.ToLower(strings.TrimSpace(asString(svc["Name"])))
|
|
for _, skip := range []string{"audit", "journal", "bmc", "security", "manager", "debug"} {
|
|
if strings.Contains(id, skip) || strings.Contains(name, skip) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// redfishLogServiceEntriesPath returns the Entries collection path for a LogService document.
|
|
func redfishLogServiceEntriesPath(svc map[string]interface{}) string {
|
|
if entriesLink, ok := svc["Entries"].(map[string]interface{}); ok {
|
|
if p := normalizeRedfishPath(asString(entriesLink["@odata.id"])); p != "" {
|
|
return p
|
|
}
|
|
}
|
|
if id := normalizeRedfishPath(asString(svc["@odata.id"])); id != "" {
|
|
return joinPath(id, "/Entries")
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// isHardwareLogEntry returns true if the log entry is hardware-related.
|
|
// Audit, authentication, and session events are excluded.
|
|
func isHardwareLogEntry(entry map[string]interface{}) bool {
|
|
entryType := strings.TrimSpace(asString(entry["EntryType"]))
|
|
if strings.EqualFold(entryType, "Oem") && !strings.EqualFold(strings.TrimSpace(asString(entry["OemRecordFormat"])), "Lenovo") {
|
|
return false
|
|
}
|
|
|
|
msgID := strings.ToLower(strings.TrimSpace(asString(entry["MessageId"])))
|
|
for _, skip := range []string{
|
|
"user", "account", "password", "login", "logon", "session",
|
|
"auth", "certificate", "security", "credential", "privilege",
|
|
} {
|
|
if strings.Contains(msgID, skip) {
|
|
return false
|
|
}
|
|
}
|
|
// Also check the human-readable message for obvious audit patterns.
|
|
msg := strings.ToLower(strings.TrimSpace(asString(entry["Message"])))
|
|
for _, skip := range []string{"logged in", "logged out", "log in", "log out", "sign in", "signed in"} {
|
|
if strings.Contains(msg, skip) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// parseRedfishEntryTime parses a Redfish LogEntry Created timestamp (ISO 8601 / RFC 3339).
|
|
func parseRedfishEntryTime(raw string) time.Time {
|
|
raw = strings.TrimSpace(raw)
|
|
if raw == "" {
|
|
return time.Time{}
|
|
}
|
|
for _, layout := range []string{time.RFC3339, time.RFC3339Nano, "2006-01-02T15:04:05Z07:00"} {
|
|
if t, err := time.Parse(layout, raw); err == nil {
|
|
return t.UTC()
|
|
}
|
|
}
|
|
return time.Time{}
|
|
}
|