diff --git a/audit/internal/collector/collector.go b/audit/internal/collector/collector.go index f9990fc..84213c8 100644 --- a/audit/internal/collector/collector.go +++ b/audit/internal/collector/collector.go @@ -34,6 +34,7 @@ func Run(_ runtimeenv.Mode) schema.HardwareIngestRequest { } snap.CPUs = enrichCPUsWithTelemetry(snap.CPUs, sensorDoc) snap.Memory = enrichMemoryWithTelemetry(snap.Memory, sensorDoc) + bestEffortRescanHotplugStorage() snap.Storage = collectStorage() snap.PCIeDevices = collectPCIe() snap.PCIeDevices = enrichPCIeWithAMD(snap.PCIeDevices) diff --git a/audit/internal/collector/storage.go b/audit/internal/collector/storage.go index c280f52..202c3ff 100644 --- a/audit/internal/collector/storage.go +++ b/audit/internal/collector/storage.go @@ -4,12 +4,52 @@ import ( "bee/audit/internal/schema" "encoding/json" "log/slog" + "os" "os/exec" "path/filepath" + "regexp" "strconv" "strings" ) +var ( + pciRescanPath = "/sys/bus/pci/rescan" + scsiHostScanGlob = "/sys/class/scsi_host/host*/scan" + hotplugWriteFile = os.WriteFile + hotplugExecCommand = exec.Command + hotplugGlob = filepath.Glob + nvmeLBAFCompactRE = regexp.MustCompile(`(?im)^\s*lbaf\s+\d+\s*:\s*ms:(\d+)\s+lbads:(\d+).*?\(in use\)\s*$`) + nvmeLBAFVerboseRE = regexp.MustCompile(`(?im)^\s*LBA Format\s+\d+\s*:\s*Metadata Size:\s*(\d+)\s+bytes\s*-\s*Data Size:\s*(\d+)\s+bytes.*?\(in use\)\s*$`) + sgReadcapBlockRE = regexp.MustCompile(`(?im)logical block length\s*=\s*(\d+)\s+bytes`) + sgReadcapProtRE = regexp.MustCompile(`(?im)prot_en\s*=\s*1`) +) + +func bestEffortRescanHotplugStorage() { + if err := hotplugWriteFile(pciRescanPath, []byte("1\n"), 0644); err != nil { + slog.Info("storage: pci rescan skipped", "path", pciRescanPath, "err", err) + } else { + slog.Info("storage: triggered pci rescan for hotplug discovery") + } + + hostPaths, err := hotplugGlob(scsiHostScanGlob) + if err != nil { + slog.Info("storage: scsi host scan skipped", "pattern", scsiHostScanGlob, "err", err) + } else { + for _, path := range hostPaths { + if err := hotplugWriteFile(path, []byte("- - -\n"), 0644); err != nil { + slog.Info("storage: scsi host scan write failed", "path", path, "err", err) + continue + } + slog.Info("storage: triggered scsi host scan", "path", path) + } + } + + out, err := hotplugExecCommand("udevadm", "settle", "--timeout=10").CombinedOutput() + if err != nil { + slog.Info("storage: udev settle after hotplug rescan failed", "err", err, "output", strings.TrimSpace(string(out))) + } +} + func collectStorage() []schema.HardwareStorage { devs := discoverStorageDevices() result := make([]schema.HardwareStorage, 0, len(devs)) @@ -35,6 +75,8 @@ type lsblkDevice struct { Model string `json:"model"` Tran string `json:"tran"` Hctl string `json:"hctl"` + LogSec string `json:"log-sec"` + PhySec string `json:"phy-sec"` } type lsblkRoot struct { @@ -101,7 +143,7 @@ func isVirtualHDiskModel(model string) bool { func lsblkDevices() []lsblkDevice { out, err := exec.Command("lsblk", "-J", "-d", - "-o", "NAME,TYPE,SIZE,SERIAL,MODEL,TRAN,HCTL").Output() + "-o", "NAME,TYPE,SIZE,SERIAL,MODEL,TRAN,HCTL,LOG-SEC,PHY-SEC").Output() if err != nil { slog.Warn("storage: lsblk failed", "err", err) return nil @@ -208,6 +250,7 @@ func enrichWithSmartctl(dev lsblkDevice) schema.HardwareStorage { present := true s := schema.HardwareStorage{Present: &present} s.Telemetry = map[string]any{"linux_device": "/dev/" + dev.Name} + applyStorageBlockGeometry(&s, dev) tran := strings.ToLower(dev.Tran) devPath := "/dev/" + dev.Name @@ -327,6 +370,7 @@ func enrichWithSmartctl(dev lsblkDevice) schema.HardwareStorage { lifeRemainingPct: lifeRemaining, } applySCSISmartctlTelemetry(&s, raw, &status) + applySCSIProtectionBlockGeometry(&s, devPath) setStorageHealthStatus(&s, status) return s } @@ -374,6 +418,7 @@ func enrichWithNVMe(dev lsblkDevice) schema.HardwareStorage { Interface: &iface, Telemetry: map[string]any{"linux_device": "/dev/" + dev.Name}, } + applyStorageBlockGeometry(&s, dev) devPath := "/dev/" + dev.Name if v := cleanDMIValue(strings.TrimSpace(dev.Model)); v != "" { @@ -408,6 +453,7 @@ func enrichWithNVMe(dev lsblkDevice) schema.HardwareStorage { } } } + applyNVMeBlockGeometry(&s, devPath) // smart-log: wear telemetry if out, err := exec.Command("nvme", "smart-log", devPath, "-o", "json").Output(); err == nil { @@ -540,6 +586,19 @@ func applySCSISmartctlTelemetry(s *schema.HardwareStorage, raw map[string]any, s "path:user_capacity.block_size", ) if hasBlockSize && blockSize > 0 { + if s.LogicalBlockSizeBytes == nil { + s.LogicalBlockSizeBytes = &blockSize + } + if s.MetadataBytesPerBlock == nil { + zero := int64(0) + s.MetadataBytesPerBlock = &zero + } + if s.Telemetry == nil { + s.Telemetry = map[string]any{} + } + s.Telemetry["logical_block_size_bytes"] = *s.LogicalBlockSizeBytes + s.Telemetry["metadata_bytes_per_block"] = *s.MetadataBytesPerBlock + s.Telemetry["block_format"] = formatBlockFormat(*s.LogicalBlockSizeBytes, *s.MetadataBytesPerBlock) if v, ok := firstInt64(raw, "path:logical_blocks_written", "path:total_lbas_written", @@ -557,6 +616,117 @@ func applySCSISmartctlTelemetry(s *schema.HardwareStorage, raw map[string]any, s } } +func applyStorageBlockGeometry(s *schema.HardwareStorage, dev lsblkDevice) { + if s == nil { + return + } + logical := parseStorageBytes(dev.LogSec) + physical := parseStorageBytes(dev.PhySec) + if logical <= 0 && physical <= 0 { + return + } + if s.Telemetry == nil { + s.Telemetry = map[string]any{} + } + if logical > 0 { + s.LogicalBlockSizeBytes = &logical + s.Telemetry["logical_block_size_bytes"] = logical + if s.MetadataBytesPerBlock == nil { + zero := int64(0) + s.MetadataBytesPerBlock = &zero + s.Telemetry["metadata_bytes_per_block"] = zero + } + } + if physical > 0 { + s.PhysicalBlockSizeBytes = &physical + s.Telemetry["physical_block_size_bytes"] = physical + } + if s.LogicalBlockSizeBytes != nil && s.MetadataBytesPerBlock != nil { + s.Telemetry["block_format"] = formatBlockFormat(*s.LogicalBlockSizeBytes, *s.MetadataBytesPerBlock) + } +} + +func applyNVMeBlockGeometry(s *schema.HardwareStorage, devPath string) { + if s == nil || strings.TrimSpace(devPath) == "" { + return + } + out, err := exec.Command("nvme", "id-ns", devPath, "-H").CombinedOutput() + if err != nil { + return + } + dataBytes, metadataBytes, ok := parseNVMeBlockFormat(string(out)) + if !ok { + return + } + setStorageBlockGeometry(s, dataBytes, metadataBytes) +} + +func applySCSIProtectionBlockGeometry(s *schema.HardwareStorage, devPath string) { + if s == nil || strings.TrimSpace(devPath) == "" { + return + } + out, err := exec.Command("sg_readcap", "-l", devPath).CombinedOutput() + if err != nil { + return + } + dataBytes, metadataBytes, ok := parseSCSIBlockFormat(string(out)) + if !ok { + return + } + setStorageBlockGeometry(s, dataBytes, metadataBytes) +} + +func setStorageBlockGeometry(s *schema.HardwareStorage, dataBytes, metadataBytes int64) { + if s == nil || dataBytes <= 0 || metadataBytes < 0 { + return + } + if s.Telemetry == nil { + s.Telemetry = map[string]any{} + } + s.LogicalBlockSizeBytes = &dataBytes + s.MetadataBytesPerBlock = &metadataBytes + s.Telemetry["logical_block_size_bytes"] = dataBytes + s.Telemetry["metadata_bytes_per_block"] = metadataBytes + s.Telemetry["block_format"] = formatBlockFormat(dataBytes, metadataBytes) +} + +func formatBlockFormat(dataBytes, metadataBytes int64) string { + return strconv.FormatInt(dataBytes, 10) + "+" + strconv.FormatInt(metadataBytes, 10) +} + +func parseNVMeBlockFormat(raw string) (dataBytes, metadataBytes int64, ok bool) { + if m := nvmeLBAFCompactRE.FindStringSubmatch(raw); len(m) == 3 { + ms, errMS := strconv.ParseInt(m[1], 10, 64) + lbads, errLBADS := strconv.ParseInt(m[2], 10, 64) + if errMS == nil && errLBADS == nil && lbads >= 0 && lbads < 63 { + return 1 << lbads, ms, true + } + } + if m := nvmeLBAFVerboseRE.FindStringSubmatch(raw); len(m) == 3 { + ms, errMS := strconv.ParseInt(m[1], 10, 64) + ds, errDS := strconv.ParseInt(m[2], 10, 64) + if errMS == nil && errDS == nil && ds > 0 { + return ds, ms, true + } + } + return 0, 0, false +} + +func parseSCSIBlockFormat(raw string) (dataBytes, metadataBytes int64, ok bool) { + m := sgReadcapBlockRE.FindStringSubmatch(raw) + if len(m) != 2 { + return 0, 0, false + } + blockBytes, err := strconv.ParseInt(m[1], 10, 64) + if err != nil || blockBytes <= 0 { + return 0, 0, false + } + if sgReadcapProtRE.MatchString(raw) { + return blockBytes, 8, true + } + return blockBytes, 0, true +} + func firstInt64(root map[string]any, candidates ...string) (int64, bool) { for _, candidate := range candidates { if !strings.HasPrefix(candidate, "path:") { diff --git a/audit/internal/collector/storage_block_format_test.go b/audit/internal/collector/storage_block_format_test.go new file mode 100644 index 0000000..284fe8a --- /dev/null +++ b/audit/internal/collector/storage_block_format_test.go @@ -0,0 +1,69 @@ +package collector + +import "testing" + +func TestParseNVMeBlockFormatCompact(t *testing.T) { + t.Parallel() + + raw := ` +lbaf 0 : ms:0 lbads:9 rp:0x2 (in use) +lbaf 1 : ms:8 lbads:9 rp:0x1 +` + dataBytes, metadataBytes, ok := parseNVMeBlockFormat(raw) + if !ok { + t.Fatal("parseNVMeBlockFormat returned ok=false") + } + if dataBytes != 512 || metadataBytes != 0 { + t.Fatalf("got %d+%d want 512+0", dataBytes, metadataBytes) + } +} + +func TestParseNVMeBlockFormatVerbose(t *testing.T) { + t.Parallel() + + raw := ` +LBA Format 0 : Metadata Size: 8 bytes - Data Size: 512 bytes - Relative Performance: 0 Better (in use) +LBA Format 1 : Metadata Size: 0 bytes - Data Size: 4096 bytes - Relative Performance: 1 Best +` + dataBytes, metadataBytes, ok := parseNVMeBlockFormat(raw) + if !ok { + t.Fatal("parseNVMeBlockFormat returned ok=false") + } + if dataBytes != 512 || metadataBytes != 8 { + t.Fatalf("got %d+%d want 512+8", dataBytes, metadataBytes) + } +} + +func TestParseSCSIBlockFormatWithProtection(t *testing.T) { + t.Parallel() + + raw := ` +Read Capacity results: + Protection: prot_en=1, p_type=1, p_i_exponent=0 + Logical block length=512 bytes +` + dataBytes, metadataBytes, ok := parseSCSIBlockFormat(raw) + if !ok { + t.Fatal("parseSCSIBlockFormat returned ok=false") + } + if dataBytes != 512 || metadataBytes != 8 { + t.Fatalf("got %d+%d want 512+8", dataBytes, metadataBytes) + } +} + +func TestParseSCSIBlockFormatWithoutProtection(t *testing.T) { + t.Parallel() + + raw := ` +Read Capacity results: + Protection: prot_en=0, p_type=0, p_i_exponent=0 + Logical block length=4096 bytes +` + dataBytes, metadataBytes, ok := parseSCSIBlockFormat(raw) + if !ok { + t.Fatal("parseSCSIBlockFormat returned ok=false") + } + if dataBytes != 4096 || metadataBytes != 0 { + t.Fatalf("got %d+%d want 4096+0", dataBytes, metadataBytes) + } +} diff --git a/audit/internal/collector/storage_discovery_test.go b/audit/internal/collector/storage_discovery_test.go index e80c1fe..46d78d2 100644 --- a/audit/internal/collector/storage_discovery_test.go +++ b/audit/internal/collector/storage_discovery_test.go @@ -1,6 +1,12 @@ package collector -import "testing" +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) func TestMergeStorageDevicePrefersNonEmptyFields(t *testing.T) { t.Parallel() @@ -31,3 +37,82 @@ func TestParseStorageBytes(t *testing.T) { t.Fatalf("parseStorageBytes invalid=%d want 0", got) } } + +func TestBestEffortRescanHotplugStorage(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + rescanPath := filepath.Join(tmp, "pci-rescan") + scanDir := filepath.Join(tmp, "scsi_host") + host0Path := filepath.Join(scanDir, "host0", "scan") + host1Path := filepath.Join(scanDir, "host1", "scan") + argsPath := filepath.Join(tmp, "udevadm-args") + toolPath := filepath.Join(tmp, "udevadm") + if err := os.MkdirAll(filepath.Dir(host0Path), 0755); err != nil { + t.Fatalf("mkdir host0: %v", err) + } + if err := os.MkdirAll(filepath.Dir(host1Path), 0755); err != nil { + t.Fatalf("mkdir host1: %v", err) + } + if err := os.WriteFile(host0Path, nil, 0644); err != nil { + t.Fatalf("touch host0 scan: %v", err) + } + if err := os.WriteFile(host1Path, nil, 0644); err != nil { + t.Fatalf("touch host1 scan: %v", err) + } + script := "#!/bin/sh\nprintf '%s' \"$*\" > \"" + argsPath + "\"\n" + if err := os.WriteFile(toolPath, []byte(script), 0755); err != nil { + t.Fatalf("write udevadm stub: %v", err) + } + + oldPath := os.Getenv("PATH") + if err := os.Setenv("PATH", tmp+string(os.PathListSeparator)+oldPath); err != nil { + t.Fatalf("set PATH: %v", err) + } + defer func() { _ = os.Setenv("PATH", oldPath) }() + + oldRescanPath := pciRescanPath + oldSCSIGlob := scsiHostScanGlob + oldWriteFile := hotplugWriteFile + oldExecCommand := hotplugExecCommand + oldGlob := hotplugGlob + pciRescanPath = rescanPath + scsiHostScanGlob = filepath.Join(scanDir, "host*", "scan") + hotplugWriteFile = os.WriteFile + hotplugExecCommand = exec.Command + hotplugGlob = filepath.Glob + defer func() { + pciRescanPath = oldRescanPath + scsiHostScanGlob = oldSCSIGlob + hotplugWriteFile = oldWriteFile + hotplugExecCommand = oldExecCommand + hotplugGlob = oldGlob + }() + + bestEffortRescanHotplugStorage() + + raw, err := os.ReadFile(rescanPath) + if err != nil { + t.Fatalf("read rescan file: %v", err) + } + if string(raw) != "1\n" { + t.Fatalf("rescan payload=%q want %q", string(raw), "1\n") + } + for _, path := range []string{host0Path, host1Path} { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read scsi scan file %s: %v", path, err) + } + if string(raw) != "- - -\n" { + t.Fatalf("scsi scan payload at %s =%q want %q", path, string(raw), "- - -\n") + } + } + + args, err := os.ReadFile(argsPath) + if err != nil { + t.Fatalf("read udevadm args: %v", err) + } + if got := strings.TrimSpace(string(args)); got != "settle --timeout=10" { + t.Fatalf("udevadm args=%q want %q", got, "settle --timeout=10") + } +} diff --git a/audit/internal/collector/storage_scsi_test.go b/audit/internal/collector/storage_scsi_test.go index f07f4cf..2bca785 100644 --- a/audit/internal/collector/storage_scsi_test.go +++ b/audit/internal/collector/storage_scsi_test.go @@ -40,6 +40,12 @@ func TestApplySCSISmartctlTelemetry(t *testing.T) { if disk.ReadBytes == nil || *disk.ReadBytes != 8192000 { t.Fatalf("read_bytes=%v want 8192000", disk.ReadBytes) } + if disk.LogicalBlockSizeBytes == nil || *disk.LogicalBlockSizeBytes != 4096 { + t.Fatalf("logical_block_size_bytes=%v want 4096", disk.LogicalBlockSizeBytes) + } + if disk.MetadataBytesPerBlock == nil || *disk.MetadataBytesPerBlock != 0 { + t.Fatalf("metadata_bytes_per_block=%v want 0", disk.MetadataBytesPerBlock) + } if disk.LifeUsedPct == nil || *disk.LifeUsedPct != 12 { t.Fatalf("life_used_pct=%v want 12", disk.LifeUsedPct) } @@ -80,6 +86,12 @@ func TestApplySCSISmartctlTelemetryDoesNotOverwriteExistingValues(t *testing.T) if *disk.WrittenBytes != 20 { t.Fatalf("written_bytes overwritten: got %d want 20", *disk.WrittenBytes) } + if disk.LogicalBlockSizeBytes == nil || *disk.LogicalBlockSizeBytes != 512 { + t.Fatalf("logical_block_size_bytes=%v want 512", disk.LogicalBlockSizeBytes) + } + if disk.MetadataBytesPerBlock == nil || *disk.MetadataBytesPerBlock != 0 { + t.Fatalf("metadata_bytes_per_block=%v want 0", disk.MetadataBytesPerBlock) + } if *disk.LifeRemainingPct != 30 { t.Fatalf("life_remaining_pct overwritten: got %v want 30", *disk.LifeRemainingPct) } diff --git a/audit/internal/schema/hardware.go b/audit/internal/schema/hardware.go index 18b80f7..e446368 100644 --- a/audit/internal/schema/hardware.go +++ b/audit/internal/schema/hardware.go @@ -143,30 +143,33 @@ type HardwareMemory struct { type HardwareStorage struct { HardwareComponentStatus - Slot *string `json:"slot,omitempty"` - Type *string `json:"type,omitempty"` - Model *string `json:"model,omitempty"` - SizeGB *int `json:"size_gb,omitempty"` - SerialNumber *string `json:"serial_number,omitempty"` - Manufacturer *string `json:"manufacturer,omitempty"` - Firmware *string `json:"firmware,omitempty"` - Interface *string `json:"interface,omitempty"` - Present *bool `json:"present,omitempty"` - TemperatureC *float64 `json:"temperature_c,omitempty"` - PowerOnHours *int64 `json:"power_on_hours,omitempty"` - PowerCycles *int64 `json:"power_cycles,omitempty"` - UnsafeShutdowns *int64 `json:"unsafe_shutdowns,omitempty"` - MediaErrors *int64 `json:"media_errors,omitempty"` - ErrorLogEntries *int64 `json:"error_log_entries,omitempty"` - WrittenBytes *int64 `json:"written_bytes,omitempty"` - ReadBytes *int64 `json:"read_bytes,omitempty"` - LifeUsedPct *float64 `json:"life_used_pct,omitempty"` - LifeRemainingPct *float64 `json:"life_remaining_pct,omitempty"` - AvailableSparePct *float64 `json:"available_spare_pct,omitempty"` - ReallocatedSectors *int64 `json:"reallocated_sectors,omitempty"` - CurrentPendingSectors *int64 `json:"current_pending_sectors,omitempty"` - OfflineUncorrectable *int64 `json:"offline_uncorrectable,omitempty"` - Telemetry map[string]any `json:"-"` + Slot *string `json:"slot,omitempty"` + Type *string `json:"type,omitempty"` + Model *string `json:"model,omitempty"` + SizeGB *int `json:"size_gb,omitempty"` + LogicalBlockSizeBytes *int64 `json:"logical_block_size_bytes,omitempty"` + PhysicalBlockSizeBytes *int64 `json:"physical_block_size_bytes,omitempty"` + MetadataBytesPerBlock *int64 `json:"metadata_bytes_per_block,omitempty"` + SerialNumber *string `json:"serial_number,omitempty"` + Manufacturer *string `json:"manufacturer,omitempty"` + Firmware *string `json:"firmware,omitempty"` + Interface *string `json:"interface,omitempty"` + Present *bool `json:"present,omitempty"` + TemperatureC *float64 `json:"temperature_c,omitempty"` + PowerOnHours *int64 `json:"power_on_hours,omitempty"` + PowerCycles *int64 `json:"power_cycles,omitempty"` + UnsafeShutdowns *int64 `json:"unsafe_shutdowns,omitempty"` + MediaErrors *int64 `json:"media_errors,omitempty"` + ErrorLogEntries *int64 `json:"error_log_entries,omitempty"` + WrittenBytes *int64 `json:"written_bytes,omitempty"` + ReadBytes *int64 `json:"read_bytes,omitempty"` + LifeUsedPct *float64 `json:"life_used_pct,omitempty"` + LifeRemainingPct *float64 `json:"life_remaining_pct,omitempty"` + AvailableSparePct *float64 `json:"available_spare_pct,omitempty"` + ReallocatedSectors *int64 `json:"reallocated_sectors,omitempty"` + CurrentPendingSectors *int64 `json:"current_pending_sectors,omitempty"` + OfflineUncorrectable *int64 `json:"offline_uncorrectable,omitempty"` + Telemetry map[string]any `json:"-"` } type HardwarePCIeDevice struct { diff --git a/audit/internal/schema/hardware_test.go b/audit/internal/schema/hardware_test.go index d92b20a..83c6347 100644 --- a/audit/internal/schema/hardware_test.go +++ b/audit/internal/schema/hardware_test.go @@ -50,6 +50,9 @@ func TestHardwareSnapshotMarshalsStorageTelemetryFields(t *testing.T) { writtenBytes := int64(9876543210) readBytes := int64(1234567890) lifeRemainingPct := 91.0 + logicalBlockSizeBytes := int64(512) + physicalBlockSizeBytes := int64(4096) + metadataBytesPerBlock := int64(8) payload := HardwareIngestRequest{ CollectedAt: "2026-03-15T15:00:00Z", @@ -57,12 +60,15 @@ func TestHardwareSnapshotMarshalsStorageTelemetryFields(t *testing.T) { Board: HardwareBoard{SerialNumber: "SRV-001"}, Storage: []HardwareStorage{ { - SerialNumber: stringPtr("DISK-001"), - Model: stringPtr("TestDisk"), - PowerOnHours: &powerOnHours, - WrittenBytes: &writtenBytes, - ReadBytes: &readBytes, - LifeRemainingPct: &lifeRemainingPct, + SerialNumber: stringPtr("DISK-001"), + Model: stringPtr("TestDisk"), + LogicalBlockSizeBytes: &logicalBlockSizeBytes, + PhysicalBlockSizeBytes: &physicalBlockSizeBytes, + MetadataBytesPerBlock: &metadataBytesPerBlock, + PowerOnHours: &powerOnHours, + WrittenBytes: &writtenBytes, + ReadBytes: &readBytes, + LifeRemainingPct: &lifeRemainingPct, }, }, }, @@ -75,6 +81,9 @@ func TestHardwareSnapshotMarshalsStorageTelemetryFields(t *testing.T) { text := string(data) for _, needle := range []string{ `"storage":[{`, + `"logical_block_size_bytes":512`, + `"physical_block_size_bytes":4096`, + `"metadata_bytes_per_block":8`, `"power_on_hours":12450`, `"written_bytes":9876543210`, `"read_bytes":1234567890`, diff --git a/audit/internal/webui/server.go b/audit/internal/webui/server.go index a7ba4c7..dc986fb 100644 --- a/audit/internal/webui/server.go +++ b/audit/internal/webui/server.go @@ -572,6 +572,7 @@ func (h *handler) handleExportIndex(w http.ResponseWriter, r *http.Request) { func (h *handler) handleViewer(w http.ResponseWriter, r *http.Request) { snapshot, _ := loadSnapshot(h.opts.AuditPath) + snapshot = enrichSnapshotForViewer(snapshot) body, err := viewer.RenderHTML(snapshot, h.opts.Title) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) diff --git a/audit/internal/webui/server_test.go b/audit/internal/webui/server_test.go index 37ee239..9c983a0 100644 --- a/audit/internal/webui/server_test.go +++ b/audit/internal/webui/server_test.go @@ -1016,6 +1016,39 @@ func TestViewerRendersLatestSnapshot(t *testing.T) { } } +func TestViewerRendersDerivedStorageBlockFormat(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "audit.json") + body := `{ + "collected_at":"2026-04-29T00:05:00Z", + "hardware":{ + "board":{"serial_number":"SERIAL-NEW"}, + "storage":[ + { + "serial_number":"DISK-1", + "model":"Test NVMe", + "logical_block_size_bytes":512, + "physical_block_size_bytes":4096, + "metadata_bytes_per_block":8 + } + ] + } + }` + if err := os.WriteFile(path, []byte(body), 0644); err != nil { + t.Fatal(err) + } + + handler := NewHandler(HandlerOptions{AuditPath: path}) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/viewer", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status=%d", rec.Code) + } + if !strings.Contains(rec.Body.String(), "512+8") { + t.Fatalf("viewer body missing derived block format: %s", rec.Body.String()) + } +} + func TestAuditJSONServesLatestSnapshot(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "audit.json") @@ -1038,6 +1071,36 @@ func TestAuditJSONServesLatestSnapshot(t *testing.T) { } } +func TestAuditJSONDoesNotInjectDerivedStorageBlockFormat(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "audit.json") + body := `{ + "hardware":{ + "board":{"serial_number":"SERIAL-API"}, + "storage":[ + { + "serial_number":"DISK-1", + "logical_block_size_bytes":512, + "metadata_bytes_per_block":8 + } + ] + } + }` + if err := os.WriteFile(path, []byte(body), 0644); err != nil { + t.Fatal(err) + } + + handler := NewHandler(HandlerOptions{AuditPath: path}) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/audit.json", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status=%d", rec.Code) + } + if strings.Contains(rec.Body.String(), "block_format") { + t.Fatalf("audit.json should remain contract-only: %s", rec.Body.String()) + } +} + func TestMissingAuditJSONReturnsNotFound(t *testing.T) { handler := NewHandler(HandlerOptions{AuditPath: "/missing/audit.json"}) rec := httptest.NewRecorder() diff --git a/audit/internal/webui/viewer_snapshot.go b/audit/internal/webui/viewer_snapshot.go new file mode 100644 index 0000000..e7fd818 --- /dev/null +++ b/audit/internal/webui/viewer_snapshot.go @@ -0,0 +1,62 @@ +package webui + +import ( + "encoding/json" + "strconv" +) + +func enrichSnapshotForViewer(snapshot []byte) []byte { + if len(snapshot) == 0 { + return snapshot + } + var root map[string]any + if err := json.Unmarshal(snapshot, &root); err != nil { + return snapshot + } + hardware, _ := root["hardware"].(map[string]any) + if len(hardware) == 0 { + return snapshot + } + storage, _ := hardware["storage"].([]any) + if len(storage) == 0 { + return snapshot + } + changed := false + for _, item := range storage { + row, _ := item.(map[string]any) + if len(row) == 0 { + continue + } + if _, exists := row["block_format"]; exists { + continue + } + logical, okLogical := jsonNumberToInt64(row["logical_block_size_bytes"]) + metadata, okMetadata := jsonNumberToInt64(row["metadata_bytes_per_block"]) + if !okLogical || !okMetadata || logical <= 0 || metadata < 0 { + continue + } + row["block_format"] = strconv.FormatInt(logical, 10) + "+" + strconv.FormatInt(metadata, 10) + changed = true + } + if !changed { + return snapshot + } + out, err := json.Marshal(root) + if err != nil { + return snapshot + } + return out +} + +func jsonNumberToInt64(v any) (int64, bool) { + switch x := v.(type) { + case float64: + return int64(x), true + case int64: + return x, true + case int: + return int64(x), true + default: + return 0, false + } +} diff --git a/bible-local/docs/hardware-ingest-contract.md b/bible-local/docs/hardware-ingest-contract.md index cf6b5b5..1e6391f 100644 --- a/bible-local/docs/hardware-ingest-contract.md +++ b/bible-local/docs/hardware-ingest-contract.md @@ -1,7 +1,7 @@ --- title: Hardware Ingest JSON Contract -version: "2.7" -updated: "2026-03-15" +version: "2.10" +updated: "2026-04-29" maintainer: Reanimator Core audience: external-integrators, ai-agents language: ru @@ -9,7 +9,7 @@ language: ru # Интеграция с Reanimator: контракт JSON-импорта аппаратного обеспечения -Версия: **2.7** · Дата: **2026-03-15** +Версия: **2.10** · Дата: **2026-04-29** Документ описывает формат JSON для передачи данных об аппаратном обеспечении серверов в систему **Reanimator** (управление жизненным циклом аппаратного обеспечения). Предназначен для разработчиков смежных систем (Redfish-коллекторов, агентов мониторинга, CMDB-экспортёров) и может быть включён в документацию интегрируемых проектов. @@ -22,6 +22,9 @@ language: ru | Версия | Дата | Изменения | |--------|------|-----------| +| 2.10 | 2026-04-29 | Для `hardware.storage[]` добавлены необязательные числовые поля `logical_block_size_bytes`, `physical_block_size_bytes`, `metadata_bytes_per_block` для нормализованного описания формата блока накопителя | +| 2.9 | 2026-03-19 | Добавлена необязательная секция `hardware.platform_config` — произвольный объект с настройками платформы (BIOS/Redfish); хранится как latest-snapshot per machine | +| 2.8 | 2026-03-15 | Поле `location` удалено из всех `sensors.*`; сенсоры передаются только по `name` и измеренным значениям | | 2.7 | 2026-03-15 | Явно запрещён синтез данных в `event_logs`; интеграторы не должны придумывать серийные номера компонентов, если источник их не отдал | | 2.6 | 2026-03-15 | Добавлена необязательная секция `event_logs` для dedup/upsert логов `host` / `bmc` / `redfish` вне history timeline | | 2.5 | 2026-03-15 | Добавлено общее необязательное поле `manufactured_year_week` для компонентных секций (`YYYY-Www`) | @@ -131,8 +134,9 @@ GET /ingest/hardware/jobs/{job_id} "storage": [ ... ], "pcie_devices": [ ... ], "power_supplies": [ ... ], - "sensors": { ... }, - "event_logs": [ ... ] + "sensors": { ... }, + "event_logs": [ ... ], + "platform_config": { ... } } } ``` @@ -343,6 +347,9 @@ GET /ingest/hardware/jobs/{job_id} | `type` | string | нет | Тип: `NVMe`, `SSD`, `HDD` | | `interface` | string | нет | Интерфейс: `NVMe`, `SATA`, `SAS` | | `size_gb` | int | нет | Размер в ГБ | +| `logical_block_size_bytes` | int64 | нет | Логический размер пользовательского блока данных, например `512` или `4096` | +| `physical_block_size_bytes` | int64 | нет | Физический размер блока, если известен, например `4096` | +| `metadata_bytes_per_block` | int64 | нет | Metadata / protection bytes на логический блок, например `0` или `8` | | `temperature_c` | float | нет | Температура накопителя, °C (telemetry) | | `power_on_hours` | int64 | нет | Время работы, часы | | `power_cycles` | int64 | нет | Количество циклов питания | @@ -363,6 +370,11 @@ GET /ingest/hardware/jobs/{job_id} Диск без `serial_number` игнорируется. Изменение `firmware` создаёт событие `FIRMWARE_CHANGED`. +Формат вида `512+8` в контракт не добавляется отдельным строковым полем. Если источник знает такую форму, он должен передавать её как: +- `logical_block_size_bytes = 512` +- `metadata_bytes_per_block = 8` +- `physical_block_size_bytes = 512` или `4096`, если известен физический размер блока + ```json "storage": [ { @@ -370,6 +382,9 @@ GET /ingest/hardware/jobs/{job_id} "type": "NVMe", "model": "INTEL SSDPF2KX076T1", "size_gb": 7680, + "logical_block_size_bytes": 512, + "physical_block_size_bytes": 4096, + "metadata_bytes_per_block": 8, "temperature_c": 38.5, "power_on_hours": 12450, "unsafe_shutdowns": 3, @@ -592,7 +607,6 @@ PSU без `serial_number` игнорируется. | Поле | Тип | Обязательно | Описание | |------|-----|-------------|----------| | `name` | string | **да** | Уникальное имя сенсора в рамках секции | -| `location` | string | нет | Физическое расположение | | `rpm` | int | нет | Обороты, RPM | | `status` | string | нет | Статус: `OK`, `Warning`, `Critical`, `Unknown` | @@ -601,7 +615,6 @@ PSU без `serial_number` игнорируется. | Поле | Тип | Обязательно | Описание | |------|-----|-------------|----------| | `name` | string | **да** | Уникальное имя сенсора | -| `location` | string | нет | Физическое расположение | | `voltage_v` | float | нет | Напряжение, В | | `current_a` | float | нет | Ток, А | | `power_w` | float | нет | Мощность, Вт | @@ -612,7 +625,6 @@ PSU без `serial_number` игнорируется. | Поле | Тип | Обязательно | Описание | |------|-----|-------------|----------| | `name` | string | **да** | Уникальное имя сенсора | -| `location` | string | нет | Физическое расположение | | `celsius` | float | нет | Температура, °C | | `threshold_warning_celsius` | float | нет | Порог Warning, °C | | `threshold_critical_celsius` | float | нет | Порог Critical, °C | @@ -623,29 +635,29 @@ PSU без `serial_number` игнорируется. | Поле | Тип | Обязательно | Описание | |------|-----|-------------|----------| | `name` | string | **да** | Уникальное имя сенсора | -| `location` | string | нет | Физическое расположение | | `value` | float | нет | Значение | | `unit` | string | нет | Единица измерения | | `status` | string | нет | Статус | **Правила sensors:** - Идентификатор сенсора: пара `(sensor_type, name)`. Дубли в одном payload — берётся первое вхождение. +- `location` для сенсоров передавать не нужно и не следует: в Reanimator location/slot используется только для проверки перемещения и установки компонентов, а не для last-known-value sensor ingest. - Сенсоры без `name` игнорируются. - При каждом импорте значения перезаписываются (upsert по ключу). ```json "sensors": { "fans": [ - { "name": "FAN1", "location": "Front", "rpm": 4200, "status": "OK" }, - { "name": "FAN_CPU0", "location": "CPU0", "rpm": 5600, "status": "OK" } + { "name": "FAN1", "rpm": 4200, "status": "OK" }, + { "name": "FAN_CPU0", "rpm": 5600, "status": "OK" } ], "power": [ - { "name": "12V Rail", "location": "Mainboard", "voltage_v": 12.06, "status": "OK" }, - { "name": "PSU0 Input", "location": "PSU0", "voltage_v": 215.25, "current_a": 0.64, "power_w": 137.0, "status": "OK" } + { "name": "12V Rail", "voltage_v": 12.06, "status": "OK" }, + { "name": "PSU0 Input", "voltage_v": 215.25, "current_a": 0.64, "power_w": 137.0, "status": "OK" } ], "temperatures": [ - { "name": "CPU0 Temp", "location": "CPU0", "celsius": 46.0, "threshold_warning_celsius": 80.0, "threshold_critical_celsius": 95.0, "status": "OK" }, - { "name": "Inlet Temp", "location": "Front", "celsius": 22.0, "threshold_warning_celsius": 40.0, "threshold_critical_celsius": 50.0, "status": "OK" } + { "name": "CPU0 Temp", "celsius": 46.0, "threshold_warning_celsius": 80.0, "threshold_critical_celsius": 95.0, "status": "OK" }, + { "name": "Inlet Temp", "celsius": 22.0, "threshold_warning_celsius": 40.0, "threshold_critical_celsius": 50.0, "status": "OK" } ], "other": [ { "name": "System Humidity", "value": 38.5, "unit": "%", "status": "OK" } @@ -655,6 +667,31 @@ PSU без `serial_number` игнорируется. --- +## Секция platform_config + +Необязательный объект с произвольными ключами — настройки платформы как есть из источника (BIOS, Redfish, IPMI). + +| Поле | Тип | Обязательно | Описание | +|------|-----|-------------|----------| +| `platform_config` | object | нет | Произвольный объект: ключи — строки, значения — строки, числа, булевы | + +**Правила platform_config:** +- Содержимое объекта не валидируется: передавайте параметры как есть. +- При каждом импорте хранится latest-snapshot per machine; история изменений по каждому ключу накапливается отдельно. +- Если секция отсутствует или равна `null` — данные платформы не обновляются. + +```json +"platform_config": { + "SecureBoot": "Enabled", + "BiosVersion": "06.08.05", + "TpmEnabled": true, + "NumaEnabled": false, + "HyperThreading": "Enabled" +} +``` + +--- + ## Обработка статусов компонентов | Статус | Поведение | @@ -787,6 +824,12 @@ PSU без `serial_number` игнорируется. "other": [ { "name": "System Humidity", "value": 38.5, "unit": "%" } ] + }, + "platform_config": { + "SecureBoot": "Enabled", + "BiosVersion": "06.08.05", + "TpmEnabled": true, + "HyperThreading": "Enabled" } } }