reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: complete hardware collection diagnostics

Browse Source
This commit is contained in:
mchus
2026-08-25 17:04:33 +03:00
parent 33ace33de5
commit ccc781856e
19 changed files with 311 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/collector/dmesg_events.go
+17 -2
View File
@@ -34,6 +34,20 @@ var dmesgErrorPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)\bdisabled\b`),
}
// Boot-time messages below are noisy configuration/driver diagnostics, not
// hardware incidents. Raw dmesg remains in the bundle, but presenting each
// of them as Critical makes the event log unusable.
var dmesgIgnorePatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)bridge window .* failed to assign`),
regexp.MustCompile(`(?i)^NVRM: loading NVIDIA .* Kernel Module`),
regexp.MustCompile(`(?i)^NVRM: Persistence mode is deprecated`),
regexp.MustCompile(`(?i)^nvidia: module verification failed:.*tainting kernel`),
regexp.MustCompile(`(?i)^Yama: disabled by default`),
regexp.MustCompile(`(?i)^ERST: .*initialized`),
regexp.MustCompile(`(?i)iommu sva bind failed: -95`),
regexp.MustCompile(`(?i)gpuClearFbhubPoisonIntrForBug`),
}
// collectDmesgErrors runs `dmesg -T` (or `dmesg` without -T on failure) and
// returns only lines that match known error/warning patterns.
func collectDmesgErrors() []schema.HardwareEventLog {
@@ -77,6 +91,9 @@ func parseDmesgErrors(output string) []schema.HardwareEventLog {
if !matchesAny(message, dmesgErrorPatterns) {
continue
}
if matchesAny(message, dmesgIgnorePatterns) {
continue
}
severity := dmesgSeverity(message)
source := "dmesg"
@@ -114,10 +131,8 @@ func matchesAny(s string, patterns []*regexp.Regexp) bool {
// "fault" (de-fault), and "undead" would contain "dead".
var dmesgSeverityCriticalPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)\bpanic\b`),
regexp.MustCompile(`(?i)\baer\b`),
regexp.MustCompile(`(?i)\buncorrect`),
regexp.MustCompile(`(?i)\bxid\b`),
regexp.MustCompile(`(?i)\bnvrm\b`),
regexp.MustCompile(`(?i)\berror\b`),
regexp.MustCompile(`(?i)\bfault\b`),
regexp.MustCompile(`(?i)\bfail(ed|ure)?\b`),
audit/internal/collector/dmesg_events_test.go
+9 -5
View File
@@ -38,11 +38,6 @@ func TestDmesgSeverity_xidCodes(t *testing.T) {
msg: "Yama: disabled by default; enable with sysctl kernel.yama.*",
want: statusWarning,
},
{
name: "benign NVRM driver load message still escalates via NVRM keyword",
msg: "NVRM: loading NVIDIA UNIX Open Kernel Module for x86_64 580.159.03",
want: statusCritical,
},
}
for _, tt := range tests {
@@ -54,3 +49,12 @@ func TestDmesgSeverity_xidCodes(t *testing.T) {
})
}
}
func TestParseDmesgErrorsSkipsKnownBootNoise(t *testing.T) {
entries := parseDmesgErrors("[Thu Aug 25 10:00:00 2026] pci 0000:00:02.0: bridge window [mem 0x0000-0x0000] to [bus 01-ff] failed to assign\n" +
"[Thu Aug 25 10:00:01 2026] NVRM: loading NVIDIA UNIX Open Kernel Module for x86_64\n" +
"[Thu Aug 25 10:00:02 2026] blk_update_request: I/O error, dev sda\n")
if len(entries) != 1 || entries[0].Severity == nil || *entries[0].Severity != statusCritical {
t.Fatalf("entries=%#v, want one critical real error", entries)
}
}
audit/internal/collector/nic_telemetry.go
+47 -3
View File
@@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
)
@@ -42,6 +43,7 @@ func enrichPCIeWithNICTelemetry(devs []schema.HardwarePCIeDevice) []schema.Hardw
if len(ifaces) == 0 {
continue
}
sort.Strings(ifaces)
iface := ifaces[0]
devs[i].MacAddresses = collectInterfaceMACs(ifaces)
if devs[i].SerialNumber == nil {
@@ -58,11 +60,24 @@ func enrichPCIeWithNICTelemetry(devs []schema.HardwarePCIeDevice) []schema.Hardw
}
}
if out, err := ethtoolModuleQuery(iface); err == nil {
if injectSFPDOMTelemetry(&devs[i], out) {
enriched++
for port, portIface := range ifaces {
out, err := ethtoolModuleQuery(portIface)
if err != nil {
continue
}
legacy := schema.HardwarePCIeDevice{}
if !injectSFPDOMTelemetry(&legacy, out) || legacy.SFPIdentifier == nil {
continue
}
devs[i].SFPModules = append(devs[i].SFPModules, sfpModuleFromLegacy(port, legacy))
// Keep the v2.10 scalar representation for old Reanimator installs.
if len(devs[i].SFPModules) == 1 {
copyLegacySFPFields(&devs[i], legacy)
}
}
if len(devs[i].SFPModules) > 0 {
enriched++
continue
}
if len(devs[i].MacAddresses) > 0 || devs[i].Firmware != nil {
enriched++
@@ -72,6 +87,27 @@ func enrichPCIeWithNICTelemetry(devs []schema.HardwarePCIeDevice) []schema.Hardw
return devs
}
func sfpModuleFromLegacy(port int, dev schema.HardwarePCIeDevice) schema.HardwareSFPModule {
m := schema.HardwareSFPModule{Port: port, Identifier: dev.SFPIdentifier, Connector: dev.SFPConnector,
Vendor: dev.SFPVendor, PartNumber: dev.SFPPartNumber, SerialNumber: dev.SFPSerialNumber,
Revision: dev.SFPRevision, TransceiverType: dev.SFPTransceiverType,
TemperatureC: dev.SFPTemperatureC, TXPowerDBM: dev.SFPTXPowerDBM, RXPowerDBM: dev.SFPRXPowerDBM,
VoltageV: dev.SFPVoltageV, BiasMA: dev.SFPBiasMA}
if dev.SFPWavelengthNM != nil {
v := int(*dev.SFPWavelengthNM + 0.5)
m.WavelengthNM = &v
}
return m
}
func copyLegacySFPFields(dst *schema.HardwarePCIeDevice, src schema.HardwarePCIeDevice) {
dst.SFPPresent, dst.SFPIdentifier, dst.SFPConnector = src.SFPPresent, src.SFPIdentifier, src.SFPConnector
dst.SFPVendor, dst.SFPPartNumber, dst.SFPSerialNumber = src.SFPVendor, src.SFPPartNumber, src.SFPSerialNumber
dst.SFPRevision, dst.SFPTransceiverType = src.SFPRevision, src.SFPTransceiverType
dst.SFPWavelengthNM, dst.SFPTemperatureC, dst.SFPTXPowerDBM, dst.SFPRXPowerDBM = src.SFPWavelengthNM, src.SFPTemperatureC, src.SFPTXPowerDBM, src.SFPRXPowerDBM
dst.SFPVoltageV, dst.SFPBiasMA = src.SFPVoltageV, src.SFPBiasMA
}
func isNICDevice(dev schema.HardwarePCIeDevice) bool {
if dev.DeviceClass == nil {
return false
@@ -140,6 +176,14 @@ func injectSFPDOMTelemetry(dev *schema.HardwarePCIeDevice, raw string) bool {
s := strings.TrimSpace(val)
dev.SFPSerialNumber = &s
changed = true
case key == "vendor rev":
s := strings.TrimSpace(val)
dev.SFPRevision = &s
changed = true
case key == "transceiver type":
s := strings.TrimSpace(val)
dev.SFPTransceiverType = &s
changed = true
case strings.Contains(key, "laser wavelength"):
if f, ok := firstFloat(val); ok {
dev.SFPWavelengthNM = &f
audit/internal/collector/nic_telemetry_test.go
+8
View File
@@ -33,6 +33,14 @@ func TestParseSFPDOM(t *testing.T) {
}
}
func TestSFPModuleFromLegacyUsesContractShape(t *testing.T) {
identifier, wavelength := "QSFP-DD", 1310.4
m := sfpModuleFromLegacy(1, schema.HardwarePCIeDevice{SFPIdentifier: &identifier, SFPWavelengthNM: &wavelength})
if m.Port != 1 || m.Identifier == nil || *m.Identifier != "QSFP-DD" || m.WavelengthNM == nil || *m.WavelengthNM != 1310 {
t.Fatalf("module=%#v", m)
}
}
func TestParseLSPCIDetailSerial(t *testing.T) {
raw := `
05:00.0 Ethernet controller: Mellanox Technologies MT28908 Family [ConnectX-6]
audit/internal/collector/psu.go
+26 -13
View File
@@ -39,7 +39,7 @@ func collectPSUs(manufacturer string) []schema.HardwarePowerSupply {
if len(psus) == 0 {
psus = synthesizePSUsFromSDR(sdrData)
} else {
mergePSUSDR(psus, sdrData)
psus = mergePSUSDR(psus, sdrData)
}
} else if len(psus) == 0 {
slog.Info("psu: ipmitool unavailable, skipping", "err", err)
@@ -237,21 +237,22 @@ var psuSlotPatterns = []*regexp.Regexp{
// MSI/underscore style: PSU1_POWER_IN, PSU2_POWER_OUT — underscore is \w so \b
// does not fire after the digit; match explicitly with underscore terminator.
regexp.MustCompile(`(?i)\bpsu([0-9]+)_`),
regexp.MustCompile(`(?i)\bpsu?\s*([0-9]+)\b`), // PSU1, PS1, ps 2
regexp.MustCompile(`(?i)\bps\s*([0-9]+)\b`), // PS 6, PS6
regexp.MustCompile(`(?i)\bpws\s*([0-9]+)\b`), // PWS1
regexp.MustCompile(`(?i)\bpsu?\s*([0-9]+)\b`), // PSU1, PS1, ps 2
regexp.MustCompile(`(?i)\bps\s*([0-9]+)\b`), // PS 6, PS6
regexp.MustCompile(`(?i)\bpws\s*([0-9]+)\b`), // PWS1
regexp.MustCompile(`(?i)\bpower\s*supply(?:\s*bay)?\s*([0-9]+)\b`), // Power Supply 1, Power Supply Bay 3
regexp.MustCompile(`(?i)\bbay\s*([0-9]+)\b`), // Bay 1
regexp.MustCompile(`(?i)\bbay\s*([0-9]+)\b`), // Bay 1
// Fallback for xFusion-style generic numbered PSU sensors (Power1, Power2, …).
// Must be last: "power supply N" is already caught by the pattern above.
regexp.MustCompile(`(?i)\bpower([0-9]+)\b`),
}
// psuInputPowerKeywords matches AC-input power sensor names across vendors:
// MSI: PSU1_POWER_IN, PSU1_PIN
// MLT: PSU1_PIN
// xFusion: (matched via default fallback — no explicit keyword)
// HPE: PS1 Input Power, PS1 Input Watts
//
// MSI: PSU1_POWER_IN, PSU1_PIN
// MLT: PSU1_PIN
// xFusion: (matched via default fallback — no explicit keyword)
// HPE: PS1 Input Power, PS1 Input Watts
func isPSUInputPower(name string) bool {
return strings.Contains(name, "input power") ||
strings.Contains(name, "input watts") ||
@@ -262,9 +263,10 @@ func isPSUInputPower(name string) bool {
}
// isPSUOutputPower matches DC-output power sensor names across vendors:
// MSI: PSU1_POWER_OUT
// MLT: PSU1_POUT
// xFusion: PS1 POut
//
// MSI: PSU1_POWER_OUT
// MLT: PSU1_POUT
// xFusion: PS1 POut
func isPSUOutputPower(name string) bool {
return strings.Contains(name, "output power") ||
strings.Contains(name, "output watts") ||
@@ -410,12 +412,14 @@ func synthesizePSUsFromSDR(sdr map[int]psuSDR) []schema.HardwarePowerSupply {
return out
}
func mergePSUSDR(psus []schema.HardwarePowerSupply, sdr map[int]psuSDR) {
func mergePSUSDR(psus []schema.HardwarePowerSupply, sdr map[int]psuSDR) []schema.HardwarePowerSupply {
matched := map[int]bool{}
for i := range psus {
slotIdx, err := strconv.Atoi(derefPSUSlot(psus[i].Slot))
if err != nil {
continue
}
matched[slotIdx+1] = true
entry, ok := sdr[slotIdx+1]
if !ok {
continue
@@ -450,6 +454,15 @@ func mergePSUSDR(psus []schema.HardwarePowerSupply, sdr map[int]psuSDR) {
}
}
}
// FRU can be incomplete (notably on CX270); do not discard a PSU that is
// visible in SDR merely because another PSU was present in FRU.
missing := make(map[int]psuSDR)
for slot, entry := range sdr {
if !matched[slot] {
missing[slot] = entry
}
}
return append(psus, synthesizePSUsFromSDR(missing)...)
}
func splitSDRFields(line string) []string {
audit/internal/collector/psu_sdr_test.go
+16 -1
View File
@@ -1,6 +1,9 @@
package collector
import "testing"
import (
"bee/audit/internal/schema"
"testing"
)
func TestParsePSUSDR(t *testing.T) {
raw := `
@@ -118,3 +121,15 @@ func TestSynthesizePSUsFromSDR(t *testing.T) {
t.Fatalf("life used=%v", got[0].LifeUsedPct)
}
}
func TestMergePSUSDRAppendsPSUMissingFromFRU(t *testing.T) {
model := "PSU0"
slot := "0"
got := mergePSUSDR([]schema.HardwarePowerSupply{{Slot: &slot, Model: &model}}, map[int]psuSDR{
1: {slot: 1, status: statusOK},
2: {slot: 2, status: statusOK},
})
if len(got) != 2 || got[1].Slot == nil || *got[1].Slot != "1" {
t.Fatalf("PSUs=%#v, want FRU PSU0 plus synthesized PSU1", got)
}
}
audit/internal/collector/raid.go
+40 -5
View File
@@ -652,21 +652,56 @@ func appendUniqueStorage(base, extra []schema.HardwareStorage) []schema.Hardware
if len(extra) == 0 {
return base
}
seen := map[string]bool{}
for _, d := range base {
seen[storageIdentityKey(d)] = true
seen := map[string]int{}
for i, d := range base {
if key := storageIdentityKey(d); key != "" {
seen[key] = i
}
}
for _, d := range extra {
key := storageIdentityKey(d)
if key == "" || seen[key] {
if key == "" {
continue
}
if idx, ok := seen[key]; ok {
mergeStorageRecord(&base[idx], d)
continue
}
base = append(base, d)
seen[key] = true
seen[key] = len(base) - 1
}
return base
}
func mergeStorageRecord(dst *schema.HardwareStorage, src schema.HardwareStorage) {
if dst.Model == nil {
dst.Model = src.Model
}
if dst.SizeGB == nil {
dst.SizeGB = src.SizeGB
}
if dst.Interface == nil {
dst.Interface = src.Interface
}
if dst.Firmware == nil {
dst.Firmware = src.Firmware
}
if dst.Slot == nil {
dst.Slot = src.Slot
}
if dst.Manufacturer == nil {
dst.Manufacturer = src.Manufacturer
}
if dst.Present == nil {
dst.Present = src.Present
}
// A RAID controller often reports JBOD as Unknown; never replace a direct
// device health reading with that weaker verdict.
if dst.Status == nil {
dst.Status = src.Status
}
}
func storageIdentityKey(d schema.HardwareStorage) string {
if d.SerialNumber != nil && strings.TrimSpace(*d.SerialNumber) != "" {
return "sn:" + strings.ToLower(strings.TrimSpace(*d.SerialNumber))
audit/internal/collector/raid_parsers_test.go
+10
View File
@@ -6,6 +6,16 @@ import (
"testing"
)
func TestAppendUniqueStorageMergesRAIDMetadataForSameDisk(t *testing.T) {
serial, status, model, iface := "SN-1", "OK", "Disk Model", "SAS"
base := []schema.HardwareStorage{{SerialNumber: &serial, HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status}}}
extra := []schema.HardwareStorage{{SerialNumber: &serial, Model: &model, Interface: &iface}}
got := appendUniqueStorage(base, extra)
if len(got) != 1 || got[0].Model == nil || *got[0].Model != model || got[0].Interface == nil || *got[0].Interface != iface {
t.Fatalf("merged storage=%#v", got)
}
}
func TestParseSASIrcuControllerIDs(t *testing.T) {
raw := `LSI Corporation SAS2 IR Configuration Utility.
Adapter List