feat: Redfish hardware event log collection + MSI ghost GPU filter + inventory improvements
- Collect hardware event logs (last 7 days) from Systems and Managers/SEL LogServices - Parse AMI raw IPMI dump messages into readable descriptions (Sensor_Type: Event_Type) - Filter out audit/journal/non-hardware log services; only SEL from Managers - MSI ghost GPU filter: exclude processor GPU entries with temperature=0 when host is powered on - Reanimator collected_at uses InventoryData/Status.LastModifiedTime (30-day fallback) - Invalidate Redfish inventory CRC groups before host power-on - Log inventory LastModifiedTime age in collection logs - Drop SecureBoot collection (SecureBootMode, SecureBootDatabases) — not hardware inventory - Add build version to UI footer via template - Add MSI Redfish API reference doc to bible-local/docs/ ADL-032–ADL-035 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
30409eef67
commit
96e65d8f65
@@ -311,6 +311,8 @@ func (c *RedfishConnector) Collect(ctx context.Context, req Request, emit Progre
|
||||
if emit != nil {
|
||||
emit(Progress{Status: "running", Progress: 99, Message: "Redfish: анализ raw snapshot..."})
|
||||
}
|
||||
// Collect hardware event logs separately (not part of tree-walk to avoid bloat).
|
||||
rawLogEntries := c.collectRedfishLogEntries(withRedfishTelemetryPhase(ctx, "log_entries"), snapshotClient, req, baseURL, systemPaths, managerPaths)
|
||||
rawPayloads := map[string]any{
|
||||
"redfish_tree": rawTree,
|
||||
"redfish_profiles": map[string]any{
|
||||
@@ -413,12 +415,21 @@ func (c *RedfishConnector) Collect(ctx context.Context, req Request, emit Progre
|
||||
if len(fetchErrMap) > 0 {
|
||||
rawPayloads["redfish_fetch_errors"] = redfishFetchErrorMapToList(fetchErrMap)
|
||||
}
|
||||
if len(rawLogEntries) > 0 {
|
||||
rawPayloads["redfish_log_entries"] = rawLogEntries
|
||||
}
|
||||
// Unified tunnel: live collection and raw import go through the same analyzer over redfish_tree.
|
||||
result, err := ReplayRedfishFromRawPayloads(rawPayloads, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
totalElapsed := time.Since(collectStart).Round(time.Second)
|
||||
if !result.InventoryLastModifiedAt.IsZero() {
|
||||
log.Printf("redfish-collect: inventory last modified at %s (age: %s)",
|
||||
result.InventoryLastModifiedAt.Format(time.RFC3339),
|
||||
time.Since(result.InventoryLastModifiedAt).Round(time.Minute),
|
||||
)
|
||||
}
|
||||
log.Printf(
|
||||
"redfish-postprobe-metrics: nvme_candidates=%d nvme_selected=%d nvme_added=%d candidates=%d selected=%d skipped_explicit=%d added=%d dur=%s",
|
||||
postProbeMetrics.NVMECandidates,
|
||||
@@ -495,6 +506,11 @@ func (c *RedfishConnector) ensureHostPowerForCollection(ctx context.Context, cli
|
||||
return false, false
|
||||
}
|
||||
|
||||
// Invalidate all inventory CRC groups before powering on so the BMC accepts
|
||||
// fresh inventory from the host after boot. Best-effort: failure is logged but
|
||||
// does not block power-on.
|
||||
c.invalidateRedfishInventory(ctx, client, req, baseURL, systemPath, emit)
|
||||
|
||||
resetTarget := redfishResetActionTarget(systemDoc)
|
||||
resetType := redfishPickResetType(systemDoc, "On", "ForceOn")
|
||||
if resetTarget == "" || resetType == "" {
|
||||
@@ -602,6 +618,32 @@ func (c *RedfishConnector) restoreHostPowerAfterCollection(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
// invalidateRedfishInventory POSTs to the AMI/MSI InventoryCrc endpoint to zero out
|
||||
// all known CRC groups before a host power-on. This causes the BMC to accept fresh
|
||||
// inventory from the host after boot, preventing stale inventory (ghost GPUs, wrong
|
||||
// BIOS version, etc.) from persisting across hardware changes.
|
||||
// Best-effort: any error is logged and the call silently returns.
|
||||
func (c *RedfishConnector) invalidateRedfishInventory(ctx context.Context, client *http.Client, req Request, baseURL, systemPath string, emit ProgressFn) {
|
||||
crcPath := joinPath(systemPath, "/Oem/Ami/Inventory/Crc")
|
||||
body := map[string]any{
|
||||
"GroupCrcList": []map[string]any{
|
||||
{"CPU": 0},
|
||||
{"DIMM": 0},
|
||||
{"PCIE": 0},
|
||||
{"CERTIFICATES": 0},
|
||||
{"SECUREBOOT": 0},
|
||||
},
|
||||
}
|
||||
if err := c.postJSON(ctx, client, req, baseURL, crcPath, body); err != nil {
|
||||
log.Printf("redfish: inventory invalidation skipped (not AMI/MSI or endpoint unavailable): %v", err)
|
||||
return
|
||||
}
|
||||
log.Printf("redfish: inventory CRC groups invalidated at %s before host power-on", crcPath)
|
||||
if emit != nil {
|
||||
emit(Progress{Status: "running", Progress: 19, Message: "Redfish: инвентарь BMC инвалидирован перед включением host (все CRC группы сброшены)"})
|
||||
}
|
||||
}
|
||||
|
||||
func (c *RedfishConnector) waitForHostPowerState(ctx context.Context, client *http.Client, req Request, baseURL, systemPath string, wantOn bool, timeout time.Duration) bool {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for {
|
||||
@@ -2627,6 +2669,7 @@ func shouldCrawlPath(path string) bool {
|
||||
"/Bios/Settings",
|
||||
"/GetServerAllUSBStatus",
|
||||
"/Oem/Public/KVM",
|
||||
"/SecureBoot/SecureBootDatabases",
|
||||
} {
|
||||
if strings.Contains(normalized, part) {
|
||||
return false
|
||||
@@ -5548,7 +5591,7 @@ func storageControllerFromPath(path string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func parseFirmware(system, bios, manager, secureBoot, networkProtocol map[string]interface{}) []models.FirmwareInfo {
|
||||
func parseFirmware(system, bios, manager, networkProtocol map[string]interface{}) []models.FirmwareInfo {
|
||||
var out []models.FirmwareInfo
|
||||
|
||||
appendFW := func(name, version string) {
|
||||
@@ -5562,7 +5605,6 @@ func parseFirmware(system, bios, manager, secureBoot, networkProtocol map[string
|
||||
appendFW("BIOS", asString(system["BiosVersion"]))
|
||||
appendFW("BIOS", asString(bios["Version"]))
|
||||
appendFW("BMC", asString(manager["FirmwareVersion"]))
|
||||
appendFW("SecureBoot", asString(secureBoot["SecureBootMode"]))
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,392 @@
|
||||
package collector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
)
|
||||
|
||||
const (
|
||||
redfishLogEntriesWindow = 7 * 24 * time.Hour
|
||||
redfishLogEntriesMaxTotal = 500
|
||||
redfishLogEntriesMaxPerSvc = 200
|
||||
)
|
||||
|
||||
// collectRedfishLogEntries fetches hardware event log entries from Systems and Managers LogServices.
|
||||
// Only hardware-relevant entries from the last 7 days are returned.
|
||||
// For Systems: all log services except audit/journal/security/debug.
|
||||
// For Managers: only the IPMI SEL service (Id="SEL") — audit and event logs are excluded.
|
||||
func (c *RedfishConnector) collectRedfishLogEntries(ctx context.Context, client *http.Client, req Request, baseURL string, systemPaths, managerPaths []string) []map[string]interface{} {
|
||||
cutoff := time.Now().UTC().Add(-redfishLogEntriesWindow)
|
||||
seen := make(map[string]struct{})
|
||||
var out []map[string]interface{}
|
||||
|
||||
collectFrom := func(logServicesPath string, filter func(map[string]interface{}) bool) {
|
||||
if len(out) >= redfishLogEntriesMaxTotal {
|
||||
return
|
||||
}
|
||||
services, err := c.getCollectionMembers(ctx, client, req, baseURL, logServicesPath)
|
||||
if err != nil || len(services) == 0 {
|
||||
return
|
||||
}
|
||||
for _, svc := range services {
|
||||
if len(out) >= redfishLogEntriesMaxTotal {
|
||||
break
|
||||
}
|
||||
if !filter(svc) {
|
||||
continue
|
||||
}
|
||||
entriesPath := redfishLogServiceEntriesPath(svc)
|
||||
if entriesPath == "" {
|
||||
continue
|
||||
}
|
||||
entries := c.fetchRedfishLogEntriesWithPaging(ctx, client, req, baseURL, entriesPath, cutoff, seen, redfishLogEntriesMaxPerSvc)
|
||||
out = append(out, entries...)
|
||||
}
|
||||
}
|
||||
|
||||
for _, systemPath := range systemPaths {
|
||||
collectFrom(joinPath(systemPath, "/LogServices"), isHardwareLogService)
|
||||
}
|
||||
// Managers hold the IPMI SEL on AMI/MSI BMCs — include only the "SEL" service.
|
||||
for _, managerPath := range managerPaths {
|
||||
collectFrom(joinPath(managerPath, "/LogServices"), isManagerSELService)
|
||||
}
|
||||
|
||||
if len(out) > 0 {
|
||||
log.Printf("redfish: collected %d hardware log entries (Systems+Managers SEL, window=7d)", len(out))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fetchRedfishLogEntriesWithPaging fetches entries from a LogEntry collection,
|
||||
// following nextLink pages. Stops early when entries older than cutoff are encountered
|
||||
// (assumes BMC returns entries newest-first, which is typical).
|
||||
func (c *RedfishConnector) fetchRedfishLogEntriesWithPaging(ctx context.Context, client *http.Client, req Request, baseURL, entriesPath string, cutoff time.Time, seen map[string]struct{}, limit int) []map[string]interface{} {
|
||||
var out []map[string]interface{}
|
||||
nextPath := entriesPath
|
||||
|
||||
for nextPath != "" && len(out) < limit {
|
||||
collection, err := c.getJSON(ctx, client, req, baseURL, nextPath)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
||||
// Handle both linked members (@odata.id only) and inline members (full objects).
|
||||
rawMembers, _ := collection["Members"].([]interface{})
|
||||
hitOldEntry := false
|
||||
|
||||
for _, rawMember := range rawMembers {
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
memberMap, ok := rawMember.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
var entry map[string]interface{}
|
||||
if _, hasCreated := memberMap["Created"]; hasCreated {
|
||||
// Inline entry — use directly.
|
||||
entry = memberMap
|
||||
} else {
|
||||
// Linked entry — fetch by path.
|
||||
memberPath := normalizeRedfishPath(asString(memberMap["@odata.id"]))
|
||||
if memberPath == "" {
|
||||
continue
|
||||
}
|
||||
entry, err = c.getJSON(ctx, client, req, baseURL, memberPath)
|
||||
if err != nil || len(entry) == 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Dedup by entry Id or path.
|
||||
entryKey := asString(entry["Id"])
|
||||
if entryKey == "" {
|
||||
entryKey = asString(entry["@odata.id"])
|
||||
}
|
||||
if entryKey != "" {
|
||||
if _, dup := seen[entryKey]; dup {
|
||||
continue
|
||||
}
|
||||
seen[entryKey] = struct{}{}
|
||||
}
|
||||
|
||||
// Time filter.
|
||||
created := parseRedfishEntryTime(asString(entry["Created"]))
|
||||
if !created.IsZero() && created.Before(cutoff) {
|
||||
hitOldEntry = true
|
||||
continue
|
||||
}
|
||||
|
||||
// Hardware relevance filter.
|
||||
if !isHardwareLogEntry(entry) {
|
||||
continue
|
||||
}
|
||||
|
||||
out = append(out, entry)
|
||||
}
|
||||
|
||||
// Stop paging once we've seen entries older than the window.
|
||||
if hitOldEntry {
|
||||
break
|
||||
}
|
||||
nextPath = firstNonEmpty(
|
||||
normalizeRedfishPath(asString(collection["Members@odata.nextLink"])),
|
||||
normalizeRedfishPath(asString(collection["@odata.nextLink"])),
|
||||
)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// isManagerSELService returns true only for the IPMI SEL exposed under Managers.
|
||||
// On AMI/MSI BMCs the hardware SEL lives at Managers/{mgr}/LogServices/SEL.
|
||||
// All other Manager log services (AuditLog, EventLog, Journal) are excluded.
|
||||
func isManagerSELService(svc map[string]interface{}) bool {
|
||||
id := strings.ToLower(strings.TrimSpace(asString(svc["Id"])))
|
||||
return id == "sel"
|
||||
}
|
||||
|
||||
// isHardwareLogService returns true if the log service looks like a hardware event log
|
||||
// (SEL, System Event Log) rather than a BMC audit/journal log.
|
||||
func isHardwareLogService(svc map[string]interface{}) bool {
|
||||
id := strings.ToLower(strings.TrimSpace(asString(svc["Id"])))
|
||||
name := strings.ToLower(strings.TrimSpace(asString(svc["Name"])))
|
||||
for _, skip := range []string{"audit", "journal", "bmc", "security", "manager", "debug"} {
|
||||
if strings.Contains(id, skip) || strings.Contains(name, skip) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// redfishLogServiceEntriesPath returns the Entries collection path for a LogService document.
|
||||
func redfishLogServiceEntriesPath(svc map[string]interface{}) string {
|
||||
if entriesLink, ok := svc["Entries"].(map[string]interface{}); ok {
|
||||
if p := normalizeRedfishPath(asString(entriesLink["@odata.id"])); p != "" {
|
||||
return p
|
||||
}
|
||||
}
|
||||
if id := normalizeRedfishPath(asString(svc["@odata.id"])); id != "" {
|
||||
return joinPath(id, "/Entries")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// isHardwareLogEntry returns true if the log entry is hardware-related.
|
||||
// Audit, authentication, and session events are excluded.
|
||||
func isHardwareLogEntry(entry map[string]interface{}) bool {
|
||||
entryType := strings.TrimSpace(asString(entry["EntryType"]))
|
||||
if strings.EqualFold(entryType, "Oem") {
|
||||
return false
|
||||
}
|
||||
|
||||
msgID := strings.ToLower(strings.TrimSpace(asString(entry["MessageId"])))
|
||||
for _, skip := range []string{
|
||||
"user", "account", "password", "login", "logon", "session",
|
||||
"auth", "certificate", "security", "credential", "privilege",
|
||||
} {
|
||||
if strings.Contains(msgID, skip) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// Also check the human-readable message for obvious audit patterns.
|
||||
msg := strings.ToLower(strings.TrimSpace(asString(entry["Message"])))
|
||||
for _, skip := range []string{"logged in", "logged out", "log in", "log out", "sign in", "signed in"} {
|
||||
if strings.Contains(msg, skip) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// parseRedfishEntryTime parses a Redfish LogEntry Created timestamp (ISO 8601 / RFC 3339).
|
||||
func parseRedfishEntryTime(raw string) time.Time {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, layout := range []string{time.RFC3339, time.RFC3339Nano, "2006-01-02T15:04:05Z07:00"} {
|
||||
if t, err := time.Parse(layout, raw); err == nil {
|
||||
return t.UTC()
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// parseRedfishLogEntries converts raw log entries stored in RawPayloads into models.Event slice.
|
||||
// Called during Redfish replay for both live and offline (archive) collections.
|
||||
func parseRedfishLogEntries(rawPayloads map[string]any, collectedAt time.Time) []models.Event {
|
||||
raw, ok := rawPayloads["redfish_log_entries"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
var entries []map[string]interface{}
|
||||
switch v := raw.(type) {
|
||||
case []map[string]interface{}:
|
||||
entries = v
|
||||
case []interface{}:
|
||||
for _, item := range v {
|
||||
if m, ok := item.(map[string]interface{}); ok {
|
||||
entries = append(entries, m)
|
||||
}
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
out := make([]models.Event, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
ev := redfishLogEntryToEvent(entry, collectedAt)
|
||||
if ev == nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, *ev)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// redfishLogEntryToEvent converts a single Redfish LogEntry document to models.Event.
|
||||
func redfishLogEntryToEvent(entry map[string]interface{}, collectedAt time.Time) *models.Event {
|
||||
// Prefer EventTimestamp (actual hardware event time) over Created (Redfish record creation time).
|
||||
ts := parseRedfishEntryTime(asString(entry["EventTimestamp"]))
|
||||
if ts.IsZero() {
|
||||
ts = parseRedfishEntryTime(asString(entry["Created"]))
|
||||
}
|
||||
if ts.IsZero() {
|
||||
ts = collectedAt
|
||||
}
|
||||
|
||||
severity := redfishLogEntrySeverity(entry)
|
||||
sensorType := strings.TrimSpace(asString(entry["SensorType"]))
|
||||
messageID := strings.TrimSpace(asString(entry["MessageId"]))
|
||||
entryType := strings.TrimSpace(asString(entry["EntryType"]))
|
||||
entryCode := strings.TrimSpace(asString(entry["EntryCode"]))
|
||||
|
||||
// SensorName: prefer "Name", fall back to "SensorNumber" + SensorType.
|
||||
sensorName := strings.TrimSpace(asString(entry["Name"]))
|
||||
if sensorName == "" {
|
||||
num := strings.TrimSpace(asString(entry["SensorNumber"]))
|
||||
if num != "" && sensorType != "" {
|
||||
sensorName = sensorType + " " + num
|
||||
}
|
||||
}
|
||||
|
||||
rawMessage := strings.TrimSpace(asString(entry["Message"]))
|
||||
|
||||
// AMI/MSI BMCs dump raw IPMI record fields into Message instead of human-readable text.
|
||||
// Detect this and build a readable description from structured fields instead.
|
||||
description, rawData := redfishDecodeMessage(rawMessage, sensorType, entryCode, entry)
|
||||
if description == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &models.Event{
|
||||
ID: messageID,
|
||||
Timestamp: ts,
|
||||
Source: "redfish",
|
||||
SensorType: sensorType,
|
||||
SensorName: sensorName,
|
||||
EventType: entryType,
|
||||
Severity: severity,
|
||||
Description: description,
|
||||
RawData: rawData,
|
||||
}
|
||||
}
|
||||
|
||||
// redfishDecodeMessage returns a human-readable description and optional raw data.
|
||||
// AMI/MSI BMCs dump raw IPMI record fields into Message as "Key : Value, Key : Value, ..."
|
||||
// instead of a plain human-readable string. We extract the useful decoded fields from it.
|
||||
func redfishDecodeMessage(message, sensorType, entryCode string, entry map[string]interface{}) (description, rawData string) {
|
||||
if !isRawIPMIDump(message) {
|
||||
description = message
|
||||
return
|
||||
}
|
||||
|
||||
rawData = message
|
||||
kv := parseIPMIDumpKV(message)
|
||||
|
||||
// Sensor_Type inside the dump is more specific than the top-level SensorType field.
|
||||
if v := kv["Sensor_Type"]; v != "" {
|
||||
sensorType = v
|
||||
}
|
||||
eventType := kv["Event_Type"] // human-readable IPMI event type, e.g. "Legacy OFF State"
|
||||
|
||||
var parts []string
|
||||
if sensorType != "" {
|
||||
parts = append(parts, sensorType)
|
||||
}
|
||||
if eventType != "" {
|
||||
parts = append(parts, eventType)
|
||||
} else if entryCode != "" {
|
||||
parts = append(parts, entryCode)
|
||||
}
|
||||
description = strings.Join(parts, ": ")
|
||||
return
|
||||
}
|
||||
|
||||
// isRawIPMIDump returns true if the message is an AMI raw IPMI record dump.
|
||||
func isRawIPMIDump(message string) bool {
|
||||
return strings.Contains(message, "Event_Data_1 :") && strings.Contains(message, "Record_Type :")
|
||||
}
|
||||
|
||||
// parseIPMIDumpKV parses the AMI "Key : Value, Key : Value, " format into a map.
|
||||
func parseIPMIDumpKV(message string) map[string]string {
|
||||
out := make(map[string]string)
|
||||
for _, part := range strings.Split(message, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
idx := strings.Index(part, " : ")
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
k := strings.TrimSpace(part[:idx])
|
||||
v := strings.TrimSpace(part[idx+3:])
|
||||
if k != "" && v != "" {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// redfishLogEntrySeverity maps a Redfish LogEntry to models.Severity.
|
||||
// AMI/MSI BMCs often set Severity="OK" on all SEL records regardless of content,
|
||||
// so we fall back to inferring severity from SensorType when the explicit field is unhelpful.
|
||||
func redfishLogEntrySeverity(entry map[string]interface{}) models.Severity {
|
||||
// Newer Redfish uses MessageSeverity; older uses Severity.
|
||||
raw := strings.ToLower(firstNonEmpty(
|
||||
strings.TrimSpace(asString(entry["MessageSeverity"])),
|
||||
strings.TrimSpace(asString(entry["Severity"])),
|
||||
))
|
||||
switch raw {
|
||||
case "critical":
|
||||
return models.SeverityCritical
|
||||
case "warning":
|
||||
return models.SeverityWarning
|
||||
case "ok", "informational", "":
|
||||
// BMC didn't set a meaningful severity — infer from SensorType.
|
||||
return redfishSeverityFromSensorType(strings.TrimSpace(asString(entry["SensorType"])))
|
||||
default:
|
||||
return models.SeverityInfo
|
||||
}
|
||||
}
|
||||
|
||||
// redfishSeverityFromSensorType infers event severity from the IPMI/Redfish SensorType string.
|
||||
func redfishSeverityFromSensorType(sensorType string) models.Severity {
|
||||
switch strings.ToLower(sensorType) {
|
||||
case "critical interrupt", "processor", "memory", "power unit",
|
||||
"power supply", "drive slot", "system firmware progress":
|
||||
return models.SeverityWarning
|
||||
default:
|
||||
return models.SeverityInfo
|
||||
}
|
||||
}
|
||||
@@ -53,7 +53,7 @@ func ReplayRedfishFromRawPayloads(rawPayloads map[string]any, emit ProgressFn) (
|
||||
chassisDoc, _ := r.getJSON(primaryChassis)
|
||||
managerDoc, _ := r.getJSON(primaryManager)
|
||||
biosDoc, _ := r.getJSON(joinPath(primarySystem, "/Bios"))
|
||||
secureBootDoc, _ := r.getJSON(joinPath(primarySystem, "/SecureBoot"))
|
||||
|
||||
systemFRUDoc, _ := r.getJSON(joinPath(primarySystem, "/Oem/Public/FRU"))
|
||||
chassisFRUDoc, _ := r.getJSON(joinPath(primaryChassis, "/Oem/Public/FRU"))
|
||||
fruDoc := systemFRUDoc
|
||||
@@ -96,16 +96,19 @@ func ReplayRedfishFromRawPayloads(rawPayloads map[string]any, emit ProgressFn) (
|
||||
healthEvents := r.collectHealthSummaryEvents(chassisPaths)
|
||||
driveFetchWarningEvents := buildDriveFetchWarningEvents(rawPayloads)
|
||||
networkProtocolDoc, _ := r.getJSON(joinPath(primaryManager, "/NetworkProtocol"))
|
||||
firmware := parseFirmware(systemDoc, biosDoc, managerDoc, secureBootDoc, networkProtocolDoc)
|
||||
firmware := parseFirmware(systemDoc, biosDoc, managerDoc, networkProtocolDoc)
|
||||
firmware = dedupeFirmwareInfo(append(firmware, r.collectFirmwareInventory()...))
|
||||
boardInfo.BMCMACAddress = r.collectBMCMAC(managerPaths)
|
||||
assemblyFRU := r.collectAssemblyFRU(chassisPaths)
|
||||
collectedAt, sourceTimezone := inferRedfishCollectionTime(managerDoc, rawPayloads)
|
||||
inventoryLastModifiedAt := inferInventoryLastModifiedTime(r.tree)
|
||||
logEntryEvents := parseRedfishLogEntries(rawPayloads, collectedAt)
|
||||
|
||||
result := &models.AnalysisResult{
|
||||
CollectedAt: collectedAt,
|
||||
CollectedAt: collectedAt,
|
||||
InventoryLastModifiedAt: inventoryLastModifiedAt,
|
||||
SourceTimezone: sourceTimezone,
|
||||
Events: append(append(append(make([]models.Event, 0, len(discreteEvents)+len(healthEvents)+len(driveFetchWarningEvents)+1), healthEvents...), discreteEvents...), driveFetchWarningEvents...),
|
||||
Events: append(append(append(append(make([]models.Event, 0, len(discreteEvents)+len(healthEvents)+len(driveFetchWarningEvents)+len(logEntryEvents)+1), healthEvents...), discreteEvents...), driveFetchWarningEvents...), logEntryEvents...),
|
||||
FRU: assemblyFRU,
|
||||
Sensors: dedupeSensorReadings(append(append(thresholdSensors, thermalSensors...), powerSensors...)),
|
||||
RawPayloads: cloneRawPayloads(rawPayloads),
|
||||
@@ -183,6 +186,35 @@ func inferRedfishCollectionTime(managerDoc map[string]interface{}, rawPayloads m
|
||||
return time.Time{}, offset
|
||||
}
|
||||
|
||||
// inferInventoryLastModifiedTime reads InventoryData/Status.InventoryData.LastModifiedTime
|
||||
// from the Redfish snapshot. Returns zero time if not present or unparseable.
|
||||
func inferInventoryLastModifiedTime(snapshot map[string]interface{}) time.Time {
|
||||
docAny, ok := snapshot["/redfish/v1/Oem/Ami/InventoryData/Status"]
|
||||
if !ok {
|
||||
return time.Time{}
|
||||
}
|
||||
doc, ok := docAny.(map[string]interface{})
|
||||
if !ok {
|
||||
return time.Time{}
|
||||
}
|
||||
invData, ok := doc["InventoryData"].(map[string]interface{})
|
||||
if !ok {
|
||||
return time.Time{}
|
||||
}
|
||||
raw := strings.TrimSpace(asString(invData["LastModifiedTime"]))
|
||||
if raw == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, layout := range []string{time.RFC3339, time.RFC3339Nano} {
|
||||
if ts, err := time.Parse(layout, raw); err == nil {
|
||||
t := ts.UTC()
|
||||
log.Printf("redfish replay: inventory last modified at %s (InventoryData/Status.LastModifiedTime)", t.Format(time.RFC3339))
|
||||
return t
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
func appendMissingServerModelWarning(result *models.AnalysisResult, systemDoc map[string]interface{}, systemFRUPath, chassisFRUPath string) {
|
||||
if result == nil || result.Hardware == nil {
|
||||
return
|
||||
|
||||
@@ -58,6 +58,44 @@ func (r redfishSnapshotReader) collectGPUs(systemPaths, chassisPaths []string, p
|
||||
return out
|
||||
}
|
||||
|
||||
// msiGhostGPUFilter returns true when the GPU chassis for gpuID shows a temperature
|
||||
// of 0 on a powered-on host, which is the reliable MSI/AMI signal that the GPU is
|
||||
// no longer physically installed (stale BMC inventory cache).
|
||||
// It only filters when the system PowerState is "On" — when the host is off, all
|
||||
// temperature readings are 0 and we cannot distinguish absent from idle.
|
||||
func (r redfishSnapshotReader) msiGhostGPUFilter(systemPaths []string, gpuID, chassisPath string) bool {
|
||||
// Require host powered on.
|
||||
for _, sp := range systemPaths {
|
||||
doc, err := r.getJSON(sp)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(asString(doc["PowerState"])), "on") {
|
||||
return false
|
||||
}
|
||||
break
|
||||
}
|
||||
// Read the temperature sensor for this GPU chassis.
|
||||
sensorPath := joinPath(chassisPath, "/Sensors/"+gpuID+"_Temperature")
|
||||
sensorDoc, err := r.getJSON(sensorPath)
|
||||
if err != nil || len(sensorDoc) == 0 {
|
||||
return false
|
||||
}
|
||||
reading, ok := sensorDoc["Reading"]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
switch v := reading.(type) {
|
||||
case float64:
|
||||
return v == 0
|
||||
case int:
|
||||
return v == 0
|
||||
case int64:
|
||||
return v == 0
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// collectGPUsFromProcessors finds GPUs that some BMCs (e.g. MSI) expose as
|
||||
// Processor entries with ProcessorType=GPU rather than as PCIe devices.
|
||||
// It supplements the existing gpus slice (already found via PCIe path),
|
||||
@@ -68,6 +106,7 @@ func (r redfishSnapshotReader) collectGPUsFromProcessors(systemPaths, chassisPat
|
||||
return append([]models.GPU{}, existing...)
|
||||
}
|
||||
chassisByID := make(map[string]map[string]interface{})
|
||||
chassisPathByID := make(map[string]string)
|
||||
for _, cp := range chassisPaths {
|
||||
doc, err := r.getJSON(cp)
|
||||
if err != nil || len(doc) == 0 {
|
||||
@@ -76,6 +115,7 @@ func (r redfishSnapshotReader) collectGPUsFromProcessors(systemPaths, chassisPat
|
||||
id := strings.TrimSpace(asString(doc["Id"]))
|
||||
if id != "" {
|
||||
chassisByID[strings.ToUpper(id)] = doc
|
||||
chassisPathByID[strings.ToUpper(id)] = cp
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,6 +148,13 @@ func (r redfishSnapshotReader) collectGPUsFromProcessors(systemPaths, chassisPat
|
||||
serial = resolveProcessorGPUChassisSerial(chassisByID, gpuID, plan)
|
||||
}
|
||||
|
||||
if plan.Directives.EnableMSIGhostGPUFilter {
|
||||
chassisPath := resolveProcessorGPUChassisPath(chassisPathByID, gpuID, plan)
|
||||
if chassisPath != "" && r.msiGhostGPUFilter(systemPaths, gpuID, chassisPath) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
uuid := strings.TrimSpace(asString(doc["UUID"]))
|
||||
uuidKey := strings.ToUpper(uuid)
|
||||
serialKey := strings.ToUpper(serial)
|
||||
|
||||
@@ -45,6 +45,15 @@ func resolveProcessorGPUChassisSerial(chassisByID map[string]map[string]interfac
|
||||
return ""
|
||||
}
|
||||
|
||||
func resolveProcessorGPUChassisPath(chassisPathByID map[string]string, gpuID string, plan redfishprofile.ResolvedAnalysisPlan) string {
|
||||
for _, candidateID := range processorGPUChassisCandidateIDs(gpuID, plan) {
|
||||
if p, ok := chassisPathByID[strings.ToUpper(candidateID)]; ok {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func processorGPUChassisCandidateIDs(gpuID string, plan redfishprofile.ResolvedAnalysisPlan) []string {
|
||||
gpuID = strings.TrimSpace(gpuID)
|
||||
if gpuID == "" {
|
||||
|
||||
@@ -52,7 +52,6 @@ func baselineSeedPaths(discovered DiscoveredResources) []string {
|
||||
for _, p := range discovered.SystemPaths {
|
||||
add(p)
|
||||
add(joinPath(p, "/Bios"))
|
||||
add(joinPath(p, "/SecureBoot"))
|
||||
add(joinPath(p, "/Oem/Public"))
|
||||
add(joinPath(p, "/Oem/Public/FRU"))
|
||||
add(joinPath(p, "/Processors"))
|
||||
|
||||
@@ -10,7 +10,6 @@ func genericProfile() Profile {
|
||||
ensurePrefetchPolicy(plan, AcquisitionPrefetchPolicy{
|
||||
IncludeSuffixes: []string{
|
||||
"/Bios",
|
||||
"/SecureBoot",
|
||||
"/Processors",
|
||||
"/Memory",
|
||||
"/Storage",
|
||||
@@ -47,7 +46,6 @@ func genericProfile() Profile {
|
||||
ensureScopedPathPolicy(plan, AcquisitionScopedPathPolicy{
|
||||
SystemCriticalSuffixes: []string{
|
||||
"/Bios",
|
||||
"/SecureBoot",
|
||||
"/Oem/Public",
|
||||
"/Oem/Public/FRU",
|
||||
"/Processors",
|
||||
|
||||
@@ -64,8 +64,10 @@ func msiProfile() Profile {
|
||||
if snapshotHasGPUProcessor(snapshot, discovered.SystemPaths) && snapshotHasPathPrefix(snapshot, "/redfish/v1/Chassis/GPU") {
|
||||
plan.Directives.EnableProcessorGPUFallback = true
|
||||
plan.Directives.EnableMSIProcessorGPUChassisLookup = true
|
||||
plan.Directives.EnableMSIGhostGPUFilter = true
|
||||
addAnalysisLookupMode(plan, "msi-index")
|
||||
addAnalysisNote(plan, "msi analysis enables processor-gpu fallback from discovered GPU chassis")
|
||||
addAnalysisNote(plan, "msi ghost-gpu filter enabled: GPUs with temperature=0 on powered-on host are excluded")
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
@@ -103,6 +103,7 @@ type AnalysisDirectives struct {
|
||||
EnableProcessorGPUChassisAlias bool
|
||||
EnableGenericGraphicsControllerDedup bool
|
||||
EnableMSIProcessorGPUChassisLookup bool
|
||||
EnableMSIGhostGPUFilter bool
|
||||
EnableStorageEnclosureRecovery bool
|
||||
EnableKnownStorageControllerRecovery bool
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user