Mikhail Chusavitin
9db651c75a
Flags GPUs that reach one or more peers only via a SYS-class PCIe hop (crossing the CPU/NUMA-node boundary) in nvidia-smi topo -m. On servers where GPUs are only bridged pairwise via NVLink bridge (no switched NVLink fabric), this is the exact path that traffic between different bridge pairs has to cross, and can cut multi-GPU throughput by 2x+ for workloads spanning more than one pair. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
73 lines
2.7 KiB
Go
73 lines
2.7 KiB
Go
// Package collector runs all hardware collectors and merges results
|
||
// into a single HardwareSnapshot. Each sub-collector is independent:
|
||
// a failure in one does not abort the others.
|
||
package collector
|
||
|
||
import (
|
||
"bee/audit/internal/runtimeenv"
|
||
"bee/audit/internal/schema"
|
||
"log/slog"
|
||
"os"
|
||
"time"
|
||
)
|
||
|
||
// Run executes all collectors and returns the combined snapshot.
|
||
// Partial failures are logged as warnings; collection always completes.
|
||
func Run(_ runtimeenv.Mode) schema.HardwareIngestRequest {
|
||
start := time.Now()
|
||
collectedAt := time.Now().UTC().Format(time.RFC3339)
|
||
slog.Info("audit started")
|
||
|
||
snap := schema.HardwareSnapshot{}
|
||
|
||
board, biosFW := collectBoard()
|
||
snap.Board = board
|
||
snap.Firmware = append(snap.Firmware, biosFW...)
|
||
snap.Firmware = append(snap.Firmware, collectBMCFirmware(derefString(snap.Board.Manufacturer))...)
|
||
|
||
snap.CPUs = collectCPUs()
|
||
|
||
snap.Memory = collectMemory()
|
||
sensorDoc, err := readSensorsJSONDoc()
|
||
if err != nil {
|
||
slog.Info("sensors: unavailable for enrichment", "err", err)
|
||
}
|
||
snap.CPUs = enrichCPUsWithTelemetry(snap.CPUs, sensorDoc)
|
||
snap.Memory = enrichMemoryWithTelemetry(snap.Memory, sensorDoc)
|
||
bestEffortRescanHotplugStorage()
|
||
snap.Storage = collectStorage()
|
||
snap.PCIeDevices = collectPCIe()
|
||
snap.PCIeDevices = enrichPCIeWithAMD(snap.PCIeDevices)
|
||
snap.PCIeDevices = enrichPCIeWithPCISerials(snap.PCIeDevices)
|
||
snap.PCIeDevices = enrichPCIeWithNVIDIA(snap.PCIeDevices)
|
||
snap.PCIeDevices = enrichNVLinkBridgesWithGPUTopo(snap.PCIeDevices)
|
||
snap.PCIeDevices = enrichGPUCrossNUMATopology(snap.PCIeDevices)
|
||
snap.PCIeDevices = enrichPCIeWithMellanox(snap.PCIeDevices)
|
||
snap.PCIeDevices = enrichPCIeWithNICTelemetry(snap.PCIeDevices)
|
||
snap.PCIeDevices = enrichPCIeWithRAIDTelemetry(snap.PCIeDevices)
|
||
snap.Storage = enrichStorageWithVROC(snap.Storage, snap.PCIeDevices)
|
||
snap.Storage = appendUniqueStorage(snap.Storage, collectRAIDStorage(snap.PCIeDevices))
|
||
snap.VROCLicense = collectVROCLicense(snap.PCIeDevices)
|
||
snap.PowerSupplies = collectPSUs(derefString(snap.Board.Manufacturer))
|
||
snap.PowerSupplies = enrichPSUsWithTelemetry(snap.PowerSupplies, sensorDoc)
|
||
snap.Sensors = mergeIPMISensors(buildSensorsFromDoc(sensorDoc), collectIPMISensors())
|
||
snap.EventLogs = append(collectIPMISEL(), collectDmesgErrors()...)
|
||
finalizeSnapshot(&snap, collectedAt)
|
||
|
||
// remaining collectors added in steps 1.8 – 1.10
|
||
|
||
slog.Info("audit completed", "duration", time.Since(start).Round(time.Millisecond))
|
||
|
||
sourceType := "manual"
|
||
var targetHost *string
|
||
if hostname, err := os.Hostname(); err == nil && hostname != "" {
|
||
targetHost = &hostname
|
||
}
|
||
return schema.HardwareIngestRequest{
|
||
SourceType: &sourceType,
|
||
TargetHost: targetHost,
|
||
CollectedAt: collectedAt,
|
||
Hardware: snap,
|
||
}
|
||
}
|