mchus/logpile
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases 20 Wiki Activity

export: align reanimator and enrich redfish metrics

Browse Source
This commit is contained in:
Mikhail Chusavitin
2026-03-15 21:38:28 +03:00
parent 0acdc2b202
commit 9007f1b360
17 changed files with 3756 additions and 650 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/collector/redfish_test.go
+410 -11
View File
@@ -584,18 +584,32 @@ func TestEnrichNICFromPCIeFunctions(t *testing.T) {
}
functionDocs := []map[string]interface{}{
{
"VendorId": "0x15b3",
"DeviceId": "0x1021",
"FunctionId": "0000:17:00.0",
"VendorId": "0x15b3",
"DeviceId": "0x1021",
"CurrentLinkWidth": 16,
"CurrentLinkSpeedGTs": "32 GT/s",
"MaxLinkWidth": 16,
"MaxLinkSpeedGTs": "32 GT/s",
},
}
enrichNICFromPCIe(&nic, pcieDoc, functionDocs)
enrichNICFromPCIe(&nic, pcieDoc, functionDocs, nil)
if nic.VendorID != 0x15b3 || nic.DeviceID != 0x1021 {
t.Fatalf("unexpected NIC IDs: vendor=%#x device=%#x", nic.VendorID, nic.DeviceID)
}
if nic.Location != "PCIe Slot 1 (1)" {
t.Fatalf("unexpected NIC location: %q", nic.Location)
}
if nic.BDF != "0000:17:00.0" {
t.Fatalf("unexpected NIC BDF: %q", nic.BDF)
}
if nic.LinkWidth != 16 || nic.MaxLinkWidth != 16 {
t.Fatalf("unexpected NIC link width state: current=%d max=%d", nic.LinkWidth, nic.MaxLinkWidth)
}
if nic.LinkSpeed != "32 GT/s" || nic.MaxLinkSpeed != "32 GT/s" {
t.Fatalf("unexpected NIC link speed state: current=%q max=%q", nic.LinkSpeed, nic.MaxLinkSpeed)
}
}
func TestParseNIC_PortCountFromControllerCapabilities(t *testing.T) {
@@ -704,6 +718,208 @@ func TestParseComponents_UseNestedSerialNumberFallback(t *testing.T) {
}
}
func TestParseCPUAndMemory_CollectOemDetails(t *testing.T) {
cpus := parseCPUs([]map[string]interface{}{
{
"Id": "CPU0",
"Model": "Intel Xeon",
"CorrectableErrors": 7,
"TemperatureCelsius": 63,
"Oem": map[string]interface{}{
"VendorX": map[string]interface{}{
"MicrocodeVersion": "0x2b000643",
"UncorrectableErrors": 1,
"ThermalThrottled": true,
},
},
},
})
if len(cpus) != 1 || cpus[0].Details == nil {
t.Fatalf("expected CPU details, got %+v", cpus)
}
if cpus[0].Details["microcode"] != "0x2b000643" {
t.Fatalf("expected CPU microcode detail, got %#v", cpus[0].Details)
}
if cpus[0].Details["correctable_error_count"] != int64(7) || cpus[0].Details["uncorrectable_error_count"] != int64(1) {
t.Fatalf("expected CPU error counters, got %#v", cpus[0].Details)
}
if cpus[0].Details["throttled"] != true || cpus[0].Details["temperature_c"] != 63.0 {
t.Fatalf("expected CPU thermal details, got %#v", cpus[0].Details)
}
dimms := parseMemory([]map[string]interface{}{
{
"Id": "DIMM0",
"DeviceLocator": "CPU0_C0D0",
"CapacityMiB": 32768,
"SerialNumber": "DIMM-001",
"Oem": map[string]interface{}{
"VendorX": map[string]interface{}{
"CorrectableECCErrorCount": 12,
"UncorrectableECCErrorCount": 2,
"TemperatureC": 41.5,
"SpareBlocksRemainingPercent": 88,
"PerformanceDegraded": true,
"DataLossDetected": false,
},
},
},
})
if len(dimms) != 1 || dimms[0].Details == nil {
t.Fatalf("expected DIMM details, got %+v", dimms)
}
if dimms[0].Details["correctable_ecc_error_count"] != int64(12) || dimms[0].Details["uncorrectable_ecc_error_count"] != int64(2) {
t.Fatalf("expected DIMM ECC counters, got %#v", dimms[0].Details)
}
if dimms[0].Details["temperature_c"] != 41.5 || dimms[0].Details["spare_blocks_remaining_pct"] != 88.0 {
t.Fatalf("expected DIMM telemetry details, got %#v", dimms[0].Details)
}
if dimms[0].Details["performance_degraded"] != true || dimms[0].Details["data_loss_detected"] != false {
t.Fatalf("expected DIMM boolean health details, got %#v", dimms[0].Details)
}
}
func TestReplayRedfishFromRawPayloads_UsesProcessorAndMemoryMetrics(t *testing.T) {
rawPayloads := map[string]any{
"redfish_tree": map[string]interface{}{
"/redfish/v1": map[string]interface{}{},
"/redfish/v1/Systems": map[string]interface{}{
"Members": []interface{}{
map[string]interface{}{"@odata.id": "/redfish/v1/Systems/1"},
},
},
"/redfish/v1/Systems/1": map[string]interface{}{
"Id": "1",
"Processors": map[string]interface{}{
"@odata.id": "/redfish/v1/Systems/1/Processors",
},
"Memory": map[string]interface{}{
"@odata.id": "/redfish/v1/Systems/1/Memory",
},
},
"/redfish/v1/Systems/1/Processors": map[string]interface{}{
"Members": []interface{}{
map[string]interface{}{"@odata.id": "/redfish/v1/Systems/1/Processors/CPU0"},
},
},
"/redfish/v1/Systems/1/Processors/CPU0": map[string]interface{}{
"Id": "CPU0",
"ProcessorType": "CPU",
"Model": "Intel Xeon",
"ProcessorMetrics": map[string]interface{}{
"@odata.id": "/redfish/v1/Systems/1/Processors/CPU0/ProcessorMetrics",
},
},
"/redfish/v1/Systems/1/Processors/CPU0/ProcessorMetrics": map[string]interface{}{
"CorrectableErrors": 10,
"ThermalThrottled": true,
"MicrocodeVersion": "0x2b000643",
"TemperatureCelsius": 66,
},
"/redfish/v1/Systems/1/Memory": map[string]interface{}{
"Members": []interface{}{
map[string]interface{}{"@odata.id": "/redfish/v1/Systems/1/Memory/DIMM0"},
},
},
"/redfish/v1/Systems/1/Memory/DIMM0": map[string]interface{}{
"Id": "DIMM0",
"DeviceLocator": "CPU0_C0D0",
"CapacityMiB": 32768,
"SerialNumber": "DIMM-001",
"MemoryMetrics": map[string]interface{}{
"@odata.id": "/redfish/v1/Systems/1/Memory/DIMM0/MemoryMetrics",
},
},
"/redfish/v1/Systems/1/Memory/DIMM0/MemoryMetrics": map[string]interface{}{
"CorrectableECCErrorCount": 14,
"TemperatureCelsius": 42,
"PerformanceDegraded": true,
"SpareBlocksRemainingPercent": 91,
},
},
}
result, err := ReplayRedfishFromRawPayloads(rawPayloads, nil)
if err != nil {
t.Fatalf("ReplayRedfishFromRawPayloads() failed: %v", err)
}
if len(result.Hardware.CPUs) != 1 || result.Hardware.CPUs[0].Details == nil {
t.Fatalf("expected CPU details from replay metrics, got %+v", result.Hardware.CPUs)
}
if result.Hardware.CPUs[0].Details["correctable_error_count"] != int64(10) || result.Hardware.CPUs[0].Details["microcode"] != "0x2b000643" {
t.Fatalf("expected CPU replay metrics details, got %#v", result.Hardware.CPUs[0].Details)
}
if len(result.Hardware.Memory) != 1 || result.Hardware.Memory[0].Details == nil {
t.Fatalf("expected memory details from replay metrics, got %+v", result.Hardware.Memory)
}
if result.Hardware.Memory[0].Details["correctable_ecc_error_count"] != int64(14) || result.Hardware.Memory[0].Details["performance_degraded"] != true {
t.Fatalf("expected DIMM replay metrics details, got %#v", result.Hardware.Memory[0].Details)
}
}
func TestReplayRedfishFromRawPayloads_UsesDriveMetrics(t *testing.T) {
rawPayloads := map[string]any{
"redfish_tree": map[string]interface{}{
"/redfish/v1": map[string]interface{}{},
"/redfish/v1/Systems": map[string]interface{}{
"Members": []interface{}{
map[string]interface{}{"@odata.id": "/redfish/v1/Systems/1"},
},
},
"/redfish/v1/Systems/1": map[string]interface{}{
"Id": "1",
"Storage": map[string]interface{}{
"@odata.id": "/redfish/v1/Systems/1/Storage",
},
},
"/redfish/v1/Systems/1/Storage": map[string]interface{}{
"Members": []interface{}{
map[string]interface{}{"@odata.id": "/redfish/v1/Systems/1/Storage/RAID1"},
},
},
"/redfish/v1/Systems/1/Storage/RAID1": map[string]interface{}{
"Id": "RAID1",
"Drives": map[string]interface{}{
"@odata.id": "/redfish/v1/Systems/1/Storage/RAID1/Drives",
},
},
"/redfish/v1/Systems/1/Storage/RAID1/Drives": map[string]interface{}{
"Members": []interface{}{
map[string]interface{}{"@odata.id": "/redfish/v1/Systems/1/Storage/RAID1/Drives/Drive0"},
},
},
"/redfish/v1/Systems/1/Storage/RAID1/Drives/Drive0": map[string]interface{}{
"Id": "Drive0",
"Model": "NVMe SSD",
"SerialNumber": "SSD-001",
"DriveMetrics": map[string]interface{}{
"@odata.id": "/redfish/v1/Systems/1/Storage/RAID1/Drives/Drive0/DriveMetrics",
},
},
"/redfish/v1/Systems/1/Storage/RAID1/Drives/Drive0/DriveMetrics": map[string]interface{}{
"PowerOnHours": 1001,
"MediaErrors": 3,
"AvailableSparePercent": 92,
"TemperatureCelsius": 37,
},
},
}
result, err := ReplayRedfishFromRawPayloads(rawPayloads, nil)
if err != nil {
t.Fatalf("ReplayRedfishFromRawPayloads() failed: %v", err)
}
if len(result.Hardware.Storage) != 1 || result.Hardware.Storage[0].Details == nil {
t.Fatalf("expected storage details from replay drive metrics, got %+v", result.Hardware.Storage)
}
if result.Hardware.Storage[0].Details["power_on_hours"] != int64(1001) || result.Hardware.Storage[0].Details["media_errors"] != int64(3) {
t.Fatalf("expected drive metrics counters, got %#v", result.Hardware.Storage[0].Details)
}
if result.Hardware.Storage[0].Details["available_spare_pct"] != 92.0 || result.Hardware.Storage[0].Details["temperature_c"] != 37.0 {
t.Fatalf("expected drive metrics telemetry, got %#v", result.Hardware.Storage[0].Details)
}
}
func TestRedfishCollectionMemberRefs_IncludesOemPublicMembers(t *testing.T) {
collection := map[string]interface{}{
"Members": []interface{}{
@@ -725,6 +941,186 @@ func TestRedfishCollectionMemberRefs_IncludesOemPublicMembers(t *testing.T) {
}
}
func TestParseDriveAndPSU_CollectComponentMetricsIntoDetails(t *testing.T) {
drive := parseDrive(map[string]interface{}{
"Id": "Drive0",
"Model": "NVMe SSD",
"SerialNumber": "SSD-001",
"TemperatureCelsius": 38.5,
"PowerOnHours": 12450,
"UnsafeShutdowns": 3,
"PredictedMediaLifeLeftPercent": 91,
"Oem": map[string]interface{}{
"Public": map[string]interface{}{
"AvailableSparePercent": 87,
},
},
})
if drive.Details == nil {
t.Fatalf("expected drive details to be populated")
}
if got := drive.Details["temperature_c"]; got != 38.5 {
t.Fatalf("expected drive temperature detail 38.5, got %#v", got)
}
if got := drive.Details["power_on_hours"]; got != int64(12450) {
t.Fatalf("expected drive power_on_hours detail, got %#v", got)
}
if got := drive.Details["life_remaining_pct"]; got != 91.0 {
t.Fatalf("expected drive life_remaining_pct detail, got %#v", got)
}
if got := drive.Details["available_spare_pct"]; got != 87.0 {
t.Fatalf("expected drive available_spare_pct detail from Oem/Public, got %#v", got)
}
driveOEM := parseDrive(map[string]interface{}{
"Id": "Drive1",
"Model": "NVMe SSD 2",
"SerialNumber": "SSD-002",
"Oem": map[string]interface{}{
"Public": map[string]interface{}{
"temperature": 19,
"PercentAvailableSpare": 93,
"PercentageUsed": 7,
},
},
})
if driveOEM.Details == nil {
t.Fatalf("expected oem drive details to be populated")
}
if got := driveOEM.Details["temperature_c"]; got != 19.0 {
t.Fatalf("expected lowercase OEM drive temperature 19, got %#v", got)
}
if got := driveOEM.Details["available_spare_pct"]; got != 93.0 {
t.Fatalf("expected OEM available_spare_pct 93, got %#v", got)
}
if got := driveOEM.Details["life_used_pct"]; got != 7.0 {
t.Fatalf("expected OEM life_used_pct 7, got %#v", got)
}
psu := parsePSU(map[string]interface{}{
"MemberId": "PSU0",
"SerialNumber": "PSU-001",
"TemperatureCelsius": 41,
"Oem": map[string]interface{}{
"Public": map[string]interface{}{
"LifeRemainingPercent": 96,
},
},
}, 1)
if psu.Details == nil {
t.Fatalf("expected psu details to be populated")
}
if got := psu.Details["temperature_c"]; got != 41.0 {
t.Fatalf("expected psu temperature detail 41, got %#v", got)
}
if got := psu.Details["life_remaining_pct"]; got != 96.0 {
t.Fatalf("expected psu life_remaining_pct detail, got %#v", got)
}
}
func TestParseGPUPCIeAndNIC_CollectComponentMetricsIntoDetails(t *testing.T) {
functionDocs := []map[string]interface{}{
{
"FunctionId": "0000:17:00.0",
"VendorId": "0x10de",
"DeviceId": "0x2331",
"TemperatureCelsius": 48.5,
"PowerConsumedWatts": 315.0,
"ECCCorrectedTotal": 12,
"BatteryHealthPercent": 87,
"SFPTemperatureCelsius": 36.2,
},
}
gpu := parseGPU(map[string]interface{}{
"Id": "GPU0",
"Model": "NVIDIA H100",
"Manufacturer": "NVIDIA",
}, functionDocs, 1)
if gpu.Details == nil || gpu.Details["temperature_c"] != 48.5 || gpu.Details["power_w"] != 315.0 {
t.Fatalf("expected gpu details from function docs, got %#v", gpu.Details)
}
pcie := parsePCIeDevice(map[string]interface{}{
"Id": "NIC1",
}, []map[string]interface{}{
{
"FunctionId": "0000:18:00.0",
"VendorId": "0x15b3",
"DeviceId": "0x1021",
"SFPTXPowerDBm": -1.8,
"SFPRXPowerDBm": -2.1,
"SFPBiasMA": 5.5,
"BatteryReplaceRequired": true,
},
})
if pcie.Details == nil || pcie.Details["sfp_tx_power_dbm"] != -1.8 || pcie.Details["battery_replace_required"] != true {
t.Fatalf("expected pcie details from function docs, got %#v", pcie.Details)
}
nic := parseNIC(map[string]interface{}{"Id": "1"})
enrichNICFromPCIe(&nic, map[string]interface{}{}, []map[string]interface{}{
{
"FunctionId": "0000:19:00.0",
"SFPTemperatureCelsius": 34.0,
},
}, nil)
if nic.Details == nil || nic.Details["sfp_temperature_c"] != 34.0 {
t.Fatalf("expected nic details from linked pcie function, got %#v", nic.Details)
}
}
func TestParseComponentDetails_UseLinkedSupplementalMetrics(t *testing.T) {
drive := parseDriveWithSupplementalDocs(
map[string]interface{}{
"Id": "Drive0",
"SerialNumber": "SSD-001",
},
map[string]interface{}{
"PowerOnHours": 5001,
"MediaErrors": 2,
"TemperatureC": 39.5,
"LifeUsedPercent": 9,
},
)
if drive.Details == nil || drive.Details["power_on_hours"] != int64(5001) || drive.Details["temperature_c"] != 39.5 {
t.Fatalf("expected drive details from supplemental metrics, got %#v", drive.Details)
}
psu := parsePSUWithSupplementalDocs(
map[string]interface{}{
"MemberId": "PSU0",
"SerialNumber": "PSU-001",
},
1,
map[string]interface{}{
"Temperature": 44,
"LifeRemainingPercent": 97,
},
)
if psu.Details == nil || psu.Details["temperature_c"] != 44.0 || psu.Details["life_remaining_pct"] != 97.0 {
t.Fatalf("expected psu details from supplemental metrics, got %#v", psu.Details)
}
gpu := parseGPUWithSupplementalDocs(
map[string]interface{}{
"Id": "GPU0",
"Model": "NVIDIA H100",
"Manufacturer": "NVIDIA",
},
nil,
[]map[string]interface{}{
{
"PowerConsumptionWatts": 305.0,
"HWSlowdown": true,
},
},
1,
)
if gpu.Details == nil || gpu.Details["power_w"] != 305.0 || gpu.Details["hw_slowdown"] != true {
t.Fatalf("expected gpu details from supplemental metrics, got %#v", gpu.Details)
}
}
func TestRecoverCriticalRedfishDocsPlanB_RetriesMembersFromExistingCollection(t *testing.T) {
t.Setenv("LOGPILE_REDFISH_CRITICAL_COOLDOWN", "0s")
t.Setenv("LOGPILE_REDFISH_CRITICAL_SLOW_GAP", "0s")
@@ -1621,11 +2017,11 @@ func TestCollectGPUsFromProcessors_SupermicroHGX(t *testing.T) {
},
},
"/redfish/v1/Chassis/1/PCIeDevices/GPU1": map[string]interface{}{
"Id": "GPU1",
"Name": "GPU1",
"Model": "NVIDIA H200",
"Manufacturer": "NVIDIA",
"SerialNumber": "SN001",
"Id": "GPU1",
"Name": "GPU1",
"Model": "NVIDIA H200",
"Manufacturer": "NVIDIA",
"SerialNumber": "SN001",
"FirmwareVersion": "96.00.D9.00.02",
"PCIeFunctions": map[string]interface{}{
"@odata.id": "/redfish/v1/Chassis/1/PCIeDevices/GPU1/PCIeFunctions",
@@ -1810,15 +2206,18 @@ func TestLooksLikeGPU_NVSwitchExcluded(t *testing.T) {
}
}
func TestShouldCrawlPath_MemorySubresourcesAreSkipped(t *testing.T) {
func TestShouldCrawlPath_MemoryAndProcessorMetricsAreAllowed(t *testing.T) {
if !shouldCrawlPath("/redfish/v1/Systems/1/Memory/CPU0_C0D0") {
t.Fatalf("expected direct DIMM resource to be crawlable")
}
if shouldCrawlPath("/redfish/v1/Systems/1/Memory/CPU0_C0D0/Assembly") {
t.Fatalf("expected DIMM assembly subresource to be skipped")
}
if shouldCrawlPath("/redfish/v1/Systems/1/Memory/CPU0_C0D0/MemoryMetrics") {
t.Fatalf("expected DIMM metrics subresource to be skipped")
if !shouldCrawlPath("/redfish/v1/Systems/1/Memory/CPU0_C0D0/MemoryMetrics") {
t.Fatalf("expected DIMM metrics subresource to be crawlable")
}
if !shouldCrawlPath("/redfish/v1/Systems/1/Processors/CPU0/ProcessorMetrics") {
t.Fatalf("expected CPU metrics subresource to be crawlable")
}
if shouldCrawlPath("/redfish/v1/Chassis/1/PCIeDevices/0/PCIeFunctions/1") {
t.Fatalf("expected noisy chassis pciefunctions branch to be skipped")