Improve Redfish recovery flow and raw export timing diagnostics

2026-02-28 16:55:58 +03:00
parent 9a30705c9a
commit e0146adfff
10 changed files with 1437 additions and 58 deletions
@@ -75,8 +75,11 @@ func ReplayRedfishFromRawPayloads(rawPayloads map[string]any, emit ProgressFn) (
 	nics := r.collectNICs(chassisPaths)
 	r.enrichNICsFromNetworkInterfaces(&nics, systemPaths)
 	thresholdSensors := r.collectThresholdSensors(chassisPaths)
+	thermalSensors := r.collectThermalSensors(chassisPaths)
+	powerSensors := r.collectPowerSensors(chassisPaths)
 	discreteEvents := r.collectDiscreteSensorEvents(chassisPaths)
 	healthEvents := r.collectHealthSummaryEvents(chassisPaths)
+	driveFetchWarningEvents := buildDriveFetchWarningEvents(rawPayloads)
 	managerDoc, _ := r.getJSON(primaryManager)
 	networkProtocolDoc, _ := r.getJSON(joinPath(primaryManager, "/NetworkProtocol"))
 	firmware := parseFirmware(systemDoc, biosDoc, managerDoc, secureBootDoc, networkProtocolDoc)
@@ -85,9 +88,9 @@ func ReplayRedfishFromRawPayloads(rawPayloads map[string]any, emit ProgressFn) (
 	applyBoardInfoFallbackFromDocs(&boardInfo, boardFallbackDocs)

 	result := &models.AnalysisResult{
-		Events:      append(append(make([]models.Event, 0, len(discreteEvents)+len(healthEvents)+1), healthEvents...), discreteEvents...),
+		Events:      append(append(append(make([]models.Event, 0, len(discreteEvents)+len(healthEvents)+len(driveFetchWarningEvents)+1), healthEvents...), discreteEvents...), driveFetchWarningEvents...),
 		FRU:         make([]models.FRUInfo, 0),
-		Sensors:     thresholdSensors,
+		Sensors:     dedupeSensorReadings(append(append(thresholdSensors, thermalSensors...), powerSensors...)),
 		RawPayloads: cloneRawPayloads(rawPayloads),
 		Hardware: &models.HardwareConfig{
 			BoardInfo:       boardInfo,
@@ -168,6 +171,55 @@ func redfishFetchErrorsFromRawPayloads(rawPayloads map[string]any) map[string]st
 	}
 }

+func buildDriveFetchWarningEvents(rawPayloads map[string]any) []models.Event {
+	errs := redfishFetchErrorsFromRawPayloads(rawPayloads)
+	if len(errs) == 0 {
+		return nil
+	}
+
+	paths := make([]string, 0, len(errs))
+	timeoutCount := 0
+	for path, msg := range errs {
+		normalizedPath := normalizeRedfishPath(path)
+		if !strings.Contains(strings.ToLower(normalizedPath), "/drives/") {
+			continue
+		}
+		paths = append(paths, normalizedPath)
+		low := strings.ToLower(msg)
+		if strings.Contains(low, "timeout") || strings.Contains(low, "deadline exceeded") {
+			timeoutCount++
+		}
+	}
+	if len(paths) == 0 {
+		return nil
+	}
+
+	sort.Strings(paths)
+	preview := paths
+	const maxPreview = 8
+	if len(preview) > maxPreview {
+		preview = preview[:maxPreview]
+	}
+	rawData := strings.Join(preview, ", ")
+	if len(paths) > len(preview) {
+		rawData = fmt.Sprintf("%s (+%d more)", rawData, len(paths)-len(preview))
+	}
+	if timeoutCount > 0 {
+		rawData = fmt.Sprintf("timeouts=%d; paths=%s", timeoutCount, rawData)
+	}
+
+	return []models.Event{
+		{
+			Timestamp:   time.Now(),
+			Source:      "Redfish",
+			EventType:   "Collection Warning",
+			Severity:    models.SeverityWarning,
+			Description: fmt.Sprintf("%d drive documents were unavailable; storage details may be incomplete", len(paths)),
+			RawData:     rawData,
+		},
+	}
+}
+
 func (r redfishSnapshotReader) collectFirmwareInventory() []models.FirmwareInfo {
 	docs, err := r.getCollectionMembers("/redfish/v1/UpdateService/FirmwareInventory")
 	if err != nil || len(docs) == 0 {
@@ -219,9 +271,12 @@ func (r redfishSnapshotReader) collectThresholdSensors(chassisPaths []string) []
 	out := make([]models.SensorReading, 0)
 	seen := make(map[string]struct{})
 	for _, chassisPath := range chassisPaths {
-		docs, err := r.getCollectionMembers(joinPath(chassisPath, "/ThresholdSensors"))
-		if err != nil || len(docs) == 0 {
-			continue
+		thresholdPath := joinPath(chassisPath, "/ThresholdSensors")
+		docs, _ := r.getCollectionMembers(thresholdPath)
+		if len(docs) == 0 {
+			if thresholdDoc, err := r.getJSON(thresholdPath); err == nil {
+				docs = append(docs, redfishInlineSensors(thresholdDoc)...)
+			}
 		}
 		for _, doc := range docs {
 			sensor, ok := parseThresholdSensor(doc)
@@ -293,37 +348,235 @@ func parseThresholdSensor(doc map[string]interface{}) (models.SensorReading, boo
 func (r redfishSnapshotReader) collectDiscreteSensorEvents(chassisPaths []string) []models.Event {
 	out := make([]models.Event, 0)
 	for _, chassisPath := range chassisPaths {
-		docs, err := r.getCollectionMembers(joinPath(chassisPath, "/DiscreteSensors"))
-		if err != nil || len(docs) == 0 {
-			continue
+		discretePath := joinPath(chassisPath, "/DiscreteSensors")
+		docs, _ := r.getCollectionMembers(discretePath)
+		if len(docs) == 0 {
+			if discreteDoc, err := r.getJSON(discretePath); err == nil {
+				docs = append(docs, redfishInlineSensors(discreteDoc)...)
+			}
 		}
 		for _, doc := range docs {
-			name := firstNonEmpty(asString(doc["Name"]), asString(doc["Id"]))
-			status := mapStatus(doc["Status"])
-			if status == "" {
-				status = firstNonEmpty(asString(doc["Health"]), asString(doc["State"]))
-			}
-			if name == "" || status == "" {
+			ev, ok := parseDiscreteSensorEvent(doc)
+			if !ok {
 				continue
 			}
-			normalized := strings.ToLower(strings.TrimSpace(status))
-			if normalized == "ok" || normalized == "enabled" || normalized == "normal" || normalized == "present" {
-				continue
-			}
-			out = append(out, models.Event{
-				Timestamp:   time.Now(),
-				Source:      "Redfish",
-				SensorName:  name,
-				EventType:   "Discrete Sensor Status",
-				Severity:    models.SeverityWarning,
-				Description: fmt.Sprintf("%s reports %s", name, status),
-				RawData:     firstNonEmpty(asString(doc["Description"]), status),
-			})
+			out = append(out, ev)
 		}
 	}
 	return out
 }

+func parseDiscreteSensorEvent(doc map[string]interface{}) (models.Event, bool) {
+	name := firstNonEmpty(asString(doc["Name"]), asString(doc["Id"]))
+	status := mapStatus(doc["Status"])
+	if status == "" {
+		status = firstNonEmpty(asString(doc["Health"]), asString(doc["State"]))
+	}
+	if name == "" || status == "" {
+		return models.Event{}, false
+	}
+	normalized := strings.ToLower(strings.TrimSpace(status))
+	if normalized == "ok" || normalized == "enabled" || normalized == "normal" || normalized == "present" {
+		return models.Event{}, false
+	}
+	return models.Event{
+		Timestamp:   time.Now(),
+		Source:      "Redfish",
+		SensorName:  name,
+		EventType:   "Discrete Sensor Status",
+		Severity:    models.SeverityWarning,
+		Description: fmt.Sprintf("%s reports %s", name, status),
+		RawData:     firstNonEmpty(asString(doc["Description"]), status),
+	}, true
+}
+
+func (r redfishSnapshotReader) collectThermalSensors(chassisPaths []string) []models.SensorReading {
+	out := make([]models.SensorReading, 0)
+	for _, chassisPath := range chassisPaths {
+		doc, err := r.getJSON(joinPath(chassisPath, "/Thermal"))
+		if err != nil || len(doc) == 0 {
+			continue
+		}
+		for _, fanDoc := range redfishArrayObjects(doc["Fans"]) {
+			out = append(out, parseThermalFanSensor(fanDoc))
+		}
+		for _, tempDoc := range redfishArrayObjects(doc["Temperatures"]) {
+			out = append(out, parseThermalTemperatureSensor(tempDoc))
+		}
+	}
+	return out
+}
+
+func (r redfishSnapshotReader) collectPowerSensors(chassisPaths []string) []models.SensorReading {
+	out := make([]models.SensorReading, 0)
+	for _, chassisPath := range chassisPaths {
+		doc, err := r.getJSON(joinPath(chassisPath, "/Power"))
+		if err != nil || len(doc) == 0 {
+			continue
+		}
+		out = append(out, parsePowerOemPublicSensors(doc)...)
+		for _, controlDoc := range redfishArrayObjects(doc["PowerControl"]) {
+			if sensor, ok := parsePowerControlSensor(controlDoc); ok {
+				out = append(out, sensor)
+			}
+		}
+		for _, psuDoc := range redfishArrayObjects(doc["PowerSupplies"]) {
+			out = append(out, parsePowerSupplySensors(psuDoc)...)
+		}
+	}
+	return out
+}
+
+func parseThermalFanSensor(doc map[string]interface{}) models.SensorReading {
+	name := firstNonEmpty(asString(doc["Name"]), asString(doc["MemberId"]), "Fan")
+	unit := firstNonEmpty(asString(doc["ReadingUnits"]), "RPM")
+	value := asFloat(doc["Reading"])
+	raw := firstNonEmpty(asString(doc["Reading"]), asString(doc["Name"]))
+	status := firstNonEmpty(mapStatus(doc["Status"]), asString(doc["State"]), asString(doc["Health"]), "unknown")
+	return models.SensorReading{
+		Name:     name,
+		Type:     "fan_speed",
+		Value:    value,
+		Unit:     unit,
+		RawValue: raw,
+		Status:   status,
+	}
+}
+
+func parseThermalTemperatureSensor(doc map[string]interface{}) models.SensorReading {
+	name := firstNonEmpty(asString(doc["Name"]), asString(doc["MemberId"]), "Temperature")
+	reading := asFloat(doc["ReadingCelsius"])
+	raw := asString(doc["ReadingCelsius"])
+	if raw == "" {
+		reading = asFloat(doc["Reading"])
+		raw = asString(doc["Reading"])
+	}
+	status := firstNonEmpty(mapStatus(doc["Status"]), asString(doc["State"]), asString(doc["Health"]), "unknown")
+	return models.SensorReading{
+		Name:     name,
+		Type:     "temperature",
+		Value:    reading,
+		Unit:     "C",
+		RawValue: raw,
+		Status:   status,
+	}
+}
+
+func parsePowerOemPublicSensors(doc map[string]interface{}) []models.SensorReading {
+	oem, ok := doc["Oem"].(map[string]interface{})
+	if !ok {
+		return nil
+	}
+	public, ok := oem["Public"].(map[string]interface{})
+	if !ok {
+		return nil
+	}
+	var out []models.SensorReading
+	add := func(name, key string) {
+		raw := asString(public[key])
+		if strings.TrimSpace(raw) == "" {
+			return
+		}
+		out = append(out, models.SensorReading{
+			Name:     name,
+			Type:     "power",
+			Value:    asFloat(public[key]),
+			Unit:     "W",
+			RawValue: raw,
+			Status:   "OK",
+		})
+	}
+	add("Total_Power", "TotalPower")
+	add("CPU_Power", "CurrentCPUPowerWatts")
+	add("Memory_Power", "CurrentMemoryPowerWatts")
+	add("Fan_Power", "CurrentFANPowerWatts")
+	return out
+}
+
+func parsePowerControlSensor(doc map[string]interface{}) (models.SensorReading, bool) {
+	raw := asString(doc["PowerConsumedWatts"])
+	if strings.TrimSpace(raw) == "" {
+		return models.SensorReading{}, false
+	}
+	name := firstNonEmpty(asString(doc["Name"]), asString(doc["MemberId"]), "PowerControl")
+	status := firstNonEmpty(mapStatus(doc["Status"]), asString(doc["State"]), asString(doc["Health"]), "unknown")
+	return models.SensorReading{
+		Name:     name + "_Consumed",
+		Type:     "power",
+		Value:    asFloat(doc["PowerConsumedWatts"]),
+		Unit:     "W",
+		RawValue: raw,
+		Status:   status,
+	}, true
+}
+
+func parsePowerSupplySensors(doc map[string]interface{}) []models.SensorReading {
+	name := firstNonEmpty(asString(doc["Name"]), "PSU")
+	status := firstNonEmpty(mapStatus(doc["Status"]), asString(doc["State"]), asString(doc["Health"]), "unknown")
+	var out []models.SensorReading
+	add := func(suffix, key, unit string) {
+		raw := asString(doc[key])
+		if strings.TrimSpace(raw) == "" {
+			return
+		}
+		out = append(out, models.SensorReading{
+			Name:     fmt.Sprintf("%s_%s", name, suffix),
+			Type:     strings.ToLower(suffix),
+			Value:    asFloat(doc[key]),
+			Unit:     unit,
+			RawValue: raw,
+			Status:   status,
+		})
+	}
+	add("InputPower", "PowerInputWatts", "W")
+	add("OutputPower", "LastPowerOutputWatts", "W")
+	add("InputVoltage", "LineInputVoltage", "V")
+	return out
+}
+
+func redfishArrayObjects(v any) []map[string]interface{} {
+	list, ok := v.([]interface{})
+	if !ok || len(list) == 0 {
+		return nil
+	}
+	out := make([]map[string]interface{}, 0, len(list))
+	for _, item := range list {
+		m, ok := item.(map[string]interface{})
+		if !ok {
+			continue
+		}
+		out = append(out, m)
+	}
+	return out
+}
+
+func redfishInlineSensors(doc map[string]interface{}) []map[string]interface{} {
+	return redfishArrayObjects(doc["Sensors"])
+}
+
+func dedupeSensorReadings(items []models.SensorReading) []models.SensorReading {
+	if len(items) <= 1 {
+		return items
+	}
+	out := make([]models.SensorReading, 0, len(items))
+	seen := make(map[string]struct{}, len(items))
+	for _, s := range items {
+		key := strings.ToLower(strings.TrimSpace(s.Name) + "|" + strings.TrimSpace(s.Type))
+		if strings.TrimSpace(key) == "|" {
+			key = strings.ToLower(strings.TrimSpace(s.RawValue))
+		}
+		if strings.TrimSpace(key) == "" {
+			continue
+		}
+		if _, ok := seen[key]; ok {
+			continue
+		}
+		seen[key] = struct{}{}
+		out = append(out, s)
+	}
+	return out
+}
+
 func (r redfishSnapshotReader) collectHealthSummaryEvents(chassisPaths []string) []models.Event {
 	out := make([]models.Event, 0)
 	for _, chassisPath := range chassisPaths {