Compare commits
34 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0a98ed8ae9 | |||
| 911745e4da | |||
| acfd2010d7 | |||
| e904c13790 | |||
| 24c5c72cee | |||
| 6ff0bcad56 | |||
| 4fef26000c | |||
| a393dcb731 | |||
| 9e55728053 | |||
| 4b8023c1cb | |||
| 4c8417d20a | |||
| 0755374dd2 | |||
| c70ae274fa | |||
| 23ad7ff534 | |||
| de130966f7 | |||
| c6fbfc8306 | |||
| 35ad1c74d9 | |||
| 4a02e74b17 | |||
| cd2853ad99 | |||
| 6caf771d6e | |||
| 14fa87b7d7 | |||
| 600ece911b | |||
| 2d424c63cb | |||
| 50f28d1ee6 | |||
| 3579747ae3 | |||
| 09dc7d2613 | |||
| ec0b7f7ff9 | |||
| e7a7ff54b9 | |||
| b4371e291e | |||
| c22b53a406 | |||
| ff0acc3698 | |||
| d50760e7c6 | |||
| ed4f8be019 | |||
| 883592d029 |
18
audit/Makefile
Normal file
18
audit/Makefile
Normal file
@@ -0,0 +1,18 @@
|
||||
LISTEN ?= :8080
|
||||
AUDIT_PATH ?=
|
||||
|
||||
RUN_ARGS := web --listen $(LISTEN)
|
||||
ifneq ($(AUDIT_PATH),)
|
||||
RUN_ARGS += --audit-path $(AUDIT_PATH)
|
||||
endif
|
||||
|
||||
.PHONY: run build test
|
||||
|
||||
run:
|
||||
go run ./cmd/bee $(RUN_ARGS)
|
||||
|
||||
build:
|
||||
go build -o bee ./cmd/bee
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
@@ -346,19 +346,20 @@ func runSAT(args []string, stdout, stderr io.Writer) int {
|
||||
archive string
|
||||
err error
|
||||
)
|
||||
logLine := func(s string) { fmt.Fprintln(os.Stderr, s) }
|
||||
switch target {
|
||||
case "nvidia":
|
||||
archive, err = application.RunNvidiaAcceptancePack("")
|
||||
archive, err = application.RunNvidiaAcceptancePack("", logLine)
|
||||
case "memory":
|
||||
archive, err = application.RunMemoryAcceptancePack("")
|
||||
archive, err = application.RunMemoryAcceptancePack("", logLine)
|
||||
case "storage":
|
||||
archive, err = application.RunStorageAcceptancePack("")
|
||||
archive, err = application.RunStorageAcceptancePack("", logLine)
|
||||
case "cpu":
|
||||
dur := *duration
|
||||
if dur <= 0 {
|
||||
dur = 60
|
||||
}
|
||||
archive, err = application.RunCPUAcceptancePack("", dur)
|
||||
archive, err = application.RunCPUAcceptancePack("", dur, logLine)
|
||||
}
|
||||
if err != nil {
|
||||
slog.Error("run sat", "target", target, "err", err)
|
||||
|
||||
14
audit/go.mod
14
audit/go.mod
@@ -1,3 +1,17 @@
|
||||
module bee/audit
|
||||
|
||||
go 1.24.0
|
||||
|
||||
replace reanimator/chart => ../internal/chart
|
||||
|
||||
require (
|
||||
github.com/go-analyze/charts v0.5.26
|
||||
reanimator/chart v0.0.0-00010101000000-000000000000
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-analyze/bulk v0.1.3 // indirect
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
|
||||
golang.org/x/image v0.24.0 // indirect
|
||||
)
|
||||
|
||||
18
audit/go.sum
Normal file
18
audit/go.sum
Normal file
@@ -0,0 +1,18 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-analyze/bulk v0.1.3 h1:pzRdBqzHDAT9PyROt0SlWE0YqPtdmTcEpIJY0C3vF0c=
|
||||
github.com/go-analyze/bulk v0.1.3/go.mod h1:afon/KtFJYnekIyN20H/+XUvcLFjE8sKR1CfpqfClgM=
|
||||
github.com/go-analyze/charts v0.5.26 h1:rSwZikLQuFX6cJzwI8OAgaWZneG1kDYxD857ms00ZxY=
|
||||
github.com/go-analyze/charts v0.5.26/go.mod h1:s1YvQhjiSwtLx1f2dOKfiV9x2TT49nVSL6v2rlRpTbY=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
golang.org/x/image v0.24.0 h1:AN7zRgVsbvmTfNyqIbbOraYL8mSwcKncEj8ofjgzcMQ=
|
||||
golang.org/x/image v0.24.0/go.mod h1:4b/ITuLfqYq1hqZcjofwctIhi7sZh2WaCjvsBNjjya8=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -53,10 +53,13 @@ type networkManager interface {
|
||||
DHCPOne(iface string) (string, error)
|
||||
DHCPAll() (string, error)
|
||||
SetStaticIPv4(cfg platform.StaticIPv4Config) (string, error)
|
||||
SetInterfaceState(iface string, up bool) error
|
||||
GetInterfaceState(iface string) (bool, error)
|
||||
}
|
||||
|
||||
type serviceManager interface {
|
||||
ListBeeServices() ([]string, error)
|
||||
ServiceState(name string) string
|
||||
ServiceStatus(name string) (string, error)
|
||||
ServiceDo(name string, action platform.ServiceAction) (string, error)
|
||||
}
|
||||
@@ -74,20 +77,46 @@ type toolManager interface {
|
||||
type installer interface {
|
||||
ListInstallDisks() ([]platform.InstallDisk, error)
|
||||
InstallToDisk(ctx context.Context, device string, logFile string) error
|
||||
IsLiveMediaInRAM() bool
|
||||
RunInstallToRAM(logFunc func(string)) error
|
||||
}
|
||||
|
||||
type GPUPresenceResult struct {
|
||||
Nvidia bool
|
||||
AMD bool
|
||||
}
|
||||
|
||||
func (a *App) DetectGPUPresence() GPUPresenceResult {
|
||||
vendor := a.sat.DetectGPUVendor()
|
||||
return GPUPresenceResult{
|
||||
Nvidia: vendor == "nvidia",
|
||||
AMD: vendor == "amd",
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) IsLiveMediaInRAM() bool {
|
||||
return a.installer.IsLiveMediaInRAM()
|
||||
}
|
||||
|
||||
func (a *App) RunInstallToRAM(logFunc func(string)) error {
|
||||
return a.installer.RunInstallToRAM(logFunc)
|
||||
}
|
||||
|
||||
type satRunner interface {
|
||||
RunNvidiaAcceptancePack(baseDir string) (string, error)
|
||||
RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int) (string, error)
|
||||
RunMemoryAcceptancePack(baseDir string) (string, error)
|
||||
RunStorageAcceptancePack(baseDir string) (string, error)
|
||||
RunCPUAcceptancePack(baseDir string, durationSec int) (string, error)
|
||||
RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error)
|
||||
RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int, logFunc func(string)) (string, error)
|
||||
RunMemoryAcceptancePack(baseDir string, logFunc func(string)) (string, error)
|
||||
RunStorageAcceptancePack(baseDir string, logFunc func(string)) (string, error)
|
||||
RunCPUAcceptancePack(baseDir string, durationSec int, logFunc func(string)) (string, error)
|
||||
ListNvidiaGPUs() ([]platform.NvidiaGPU, error)
|
||||
DetectGPUVendor() string
|
||||
ListAMDGPUs() ([]platform.AMDGPUInfo, error)
|
||||
RunAMDAcceptancePack(baseDir string) (string, error)
|
||||
RunAMDAcceptancePack(baseDir string, logFunc func(string)) (string, error)
|
||||
RunAMDStressPack(baseDir string, logFunc func(string)) (string, error)
|
||||
RunMemoryStressPack(baseDir string, logFunc func(string)) (string, error)
|
||||
RunSATStressPack(baseDir string, logFunc func(string)) (string, error)
|
||||
RunFanStressTest(ctx context.Context, baseDir string, opts platform.FanStressOptions) (string, error)
|
||||
RunNCCLTests(ctx context.Context, baseDir string) (string, error)
|
||||
RunNCCLTests(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
|
||||
}
|
||||
|
||||
type runtimeChecker interface {
|
||||
@@ -107,6 +136,17 @@ func New(platform *platform.System) *App {
|
||||
}
|
||||
}
|
||||
|
||||
// ApplySATOverlay parses a raw audit JSON, overlays the latest SAT results,
|
||||
// and returns the updated JSON. Used by the web UI to serve always-fresh status.
|
||||
func ApplySATOverlay(auditJSON []byte) ([]byte, error) {
|
||||
var snap schema.HardwareIngestRequest
|
||||
if err := json.Unmarshal(auditJSON, &snap); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
applyLatestSATStatuses(&snap.Hardware, DefaultSATBaseDir)
|
||||
return json.MarshalIndent(snap, "", " ")
|
||||
}
|
||||
|
||||
func (a *App) RunAudit(runtimeMode runtimeenv.Mode, output string) (string, error) {
|
||||
if runtimeMode == runtimeenv.ModeLiveCD {
|
||||
if err := a.runtime.CaptureTechnicalDump(DefaultTechDumpDir); err != nil {
|
||||
@@ -300,6 +340,14 @@ func (a *App) SetStaticIPv4(cfg platform.StaticIPv4Config) (string, error) {
|
||||
return a.network.SetStaticIPv4(cfg)
|
||||
}
|
||||
|
||||
func (a *App) SetInterfaceState(iface string, up bool) error {
|
||||
return a.network.SetInterfaceState(iface, up)
|
||||
}
|
||||
|
||||
func (a *App) GetInterfaceState(iface string) (bool, error) {
|
||||
return a.network.GetInterfaceState(iface)
|
||||
}
|
||||
|
||||
func (a *App) SetStaticIPv4Result(cfg platform.StaticIPv4Config) (ActionResult, error) {
|
||||
body, err := a.network.SetStaticIPv4(cfg)
|
||||
return ActionResult{Title: "Static IPv4: " + cfg.Interface, Body: bodyOr(body, "Static IPv4 updated.")}, err
|
||||
@@ -356,6 +404,10 @@ func (a *App) ListBeeServices() ([]string, error) {
|
||||
return a.services.ListBeeServices()
|
||||
}
|
||||
|
||||
func (a *App) ServiceState(name string) string {
|
||||
return a.services.ServiceState(name)
|
||||
}
|
||||
|
||||
func (a *App) ServiceStatus(name string) (string, error) {
|
||||
return a.services.ServiceStatus(name)
|
||||
}
|
||||
@@ -411,15 +463,15 @@ func (a *App) AuditLogTailResult() ActionResult {
|
||||
return ActionResult{Title: "Audit log tail", Body: body}
|
||||
}
|
||||
|
||||
func (a *App) RunNvidiaAcceptancePack(baseDir string) (string, error) {
|
||||
func (a *App) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = DefaultSATBaseDir
|
||||
}
|
||||
return a.sat.RunNvidiaAcceptancePack(baseDir)
|
||||
return a.sat.RunNvidiaAcceptancePack(baseDir, logFunc)
|
||||
}
|
||||
|
||||
func (a *App) RunNvidiaAcceptancePackResult(baseDir string) (ActionResult, error) {
|
||||
path, err := a.RunNvidiaAcceptancePack(baseDir)
|
||||
path, err := a.RunNvidiaAcceptancePack(baseDir, nil)
|
||||
body := "Archive written."
|
||||
if path != "" {
|
||||
body = "Archive written to " + path
|
||||
@@ -431,11 +483,11 @@ func (a *App) ListNvidiaGPUs() ([]platform.NvidiaGPU, error) {
|
||||
return a.sat.ListNvidiaGPUs()
|
||||
}
|
||||
|
||||
func (a *App) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int) (ActionResult, error) {
|
||||
func (a *App) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int, logFunc func(string)) (ActionResult, error) {
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = DefaultSATBaseDir
|
||||
}
|
||||
path, err := a.sat.RunNvidiaAcceptancePackWithOptions(ctx, baseDir, diagLevel, gpuIndices)
|
||||
path, err := a.sat.RunNvidiaAcceptancePackWithOptions(ctx, baseDir, diagLevel, gpuIndices, logFunc)
|
||||
body := "Archive written."
|
||||
if path != "" {
|
||||
body = "Archive written to " + path
|
||||
@@ -443,39 +495,39 @@ func (a *App) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir st
|
||||
return ActionResult{Title: "NVIDIA DCGM", Body: body}, err
|
||||
}
|
||||
|
||||
func (a *App) RunMemoryAcceptancePack(baseDir string) (string, error) {
|
||||
func (a *App) RunMemoryAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = DefaultSATBaseDir
|
||||
}
|
||||
return a.sat.RunMemoryAcceptancePack(baseDir)
|
||||
return a.sat.RunMemoryAcceptancePack(baseDir, logFunc)
|
||||
}
|
||||
|
||||
func (a *App) RunMemoryAcceptancePackResult(baseDir string) (ActionResult, error) {
|
||||
path, err := a.RunMemoryAcceptancePack(baseDir)
|
||||
path, err := a.RunMemoryAcceptancePack(baseDir, nil)
|
||||
return ActionResult{Title: "Memory SAT", Body: satResultBody(path)}, err
|
||||
}
|
||||
|
||||
func (a *App) RunCPUAcceptancePack(baseDir string, durationSec int) (string, error) {
|
||||
func (a *App) RunCPUAcceptancePack(baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = DefaultSATBaseDir
|
||||
}
|
||||
return a.sat.RunCPUAcceptancePack(baseDir, durationSec)
|
||||
return a.sat.RunCPUAcceptancePack(baseDir, durationSec, logFunc)
|
||||
}
|
||||
|
||||
func (a *App) RunCPUAcceptancePackResult(baseDir string, durationSec int) (ActionResult, error) {
|
||||
path, err := a.RunCPUAcceptancePack(baseDir, durationSec)
|
||||
path, err := a.RunCPUAcceptancePack(baseDir, durationSec, nil)
|
||||
return ActionResult{Title: "CPU SAT", Body: satResultBody(path)}, err
|
||||
}
|
||||
|
||||
func (a *App) RunStorageAcceptancePack(baseDir string) (string, error) {
|
||||
func (a *App) RunStorageAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = DefaultSATBaseDir
|
||||
}
|
||||
return a.sat.RunStorageAcceptancePack(baseDir)
|
||||
return a.sat.RunStorageAcceptancePack(baseDir, logFunc)
|
||||
}
|
||||
|
||||
func (a *App) RunStorageAcceptancePackResult(baseDir string) (ActionResult, error) {
|
||||
path, err := a.RunStorageAcceptancePack(baseDir)
|
||||
path, err := a.RunStorageAcceptancePack(baseDir, nil)
|
||||
return ActionResult{Title: "Storage SAT", Body: satResultBody(path)}, err
|
||||
}
|
||||
|
||||
@@ -487,18 +539,33 @@ func (a *App) ListAMDGPUs() ([]platform.AMDGPUInfo, error) {
|
||||
return a.sat.ListAMDGPUs()
|
||||
}
|
||||
|
||||
func (a *App) RunAMDAcceptancePack(baseDir string) (string, error) {
|
||||
func (a *App) RunAMDAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = DefaultSATBaseDir
|
||||
}
|
||||
return a.sat.RunAMDAcceptancePack(baseDir)
|
||||
return a.sat.RunAMDAcceptancePack(baseDir, logFunc)
|
||||
}
|
||||
|
||||
func (a *App) RunAMDAcceptancePackResult(baseDir string) (ActionResult, error) {
|
||||
path, err := a.RunAMDAcceptancePack(baseDir)
|
||||
path, err := a.RunAMDAcceptancePack(baseDir, nil)
|
||||
return ActionResult{Title: "AMD GPU SAT", Body: satResultBody(path)}, err
|
||||
}
|
||||
|
||||
func (a *App) RunMemoryStressPack(baseDir string, logFunc func(string)) (string, error) {
|
||||
return a.sat.RunMemoryStressPack(baseDir, logFunc)
|
||||
}
|
||||
|
||||
func (a *App) RunSATStressPack(baseDir string, logFunc func(string)) (string, error) {
|
||||
return a.sat.RunSATStressPack(baseDir, logFunc)
|
||||
}
|
||||
|
||||
func (a *App) RunAMDStressPack(baseDir string, logFunc func(string)) (string, error) {
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = DefaultSATBaseDir
|
||||
}
|
||||
return a.sat.RunAMDStressPack(baseDir, logFunc)
|
||||
}
|
||||
|
||||
func (a *App) RunFanStressTest(ctx context.Context, baseDir string, opts platform.FanStressOptions) (string, error) {
|
||||
if strings.TrimSpace(baseDir) == "" {
|
||||
baseDir = DefaultSATBaseDir
|
||||
@@ -507,7 +574,7 @@ func (a *App) RunFanStressTest(ctx context.Context, baseDir string, opts platfor
|
||||
}
|
||||
|
||||
func (a *App) RunNCCLTestsResult(ctx context.Context) (ActionResult, error) {
|
||||
path, err := a.sat.RunNCCLTests(ctx, DefaultSATBaseDir)
|
||||
path, err := a.sat.RunNCCLTests(ctx, DefaultSATBaseDir, nil)
|
||||
body := "Results: " + path
|
||||
if err != nil && err != context.Canceled {
|
||||
body += "\nERROR: " + err.Error()
|
||||
@@ -1018,3 +1085,62 @@ func (a *App) ListInstallDisks() ([]platform.InstallDisk, error) {
|
||||
func (a *App) InstallToDisk(ctx context.Context, device string, logFile string) error {
|
||||
return a.installer.InstallToDisk(ctx, device, logFile)
|
||||
}
|
||||
|
||||
func formatSATDetail(raw string) string {
|
||||
var b strings.Builder
|
||||
kv := parseKeyValueSummary(raw)
|
||||
|
||||
if t, ok := kv["run_at_utc"]; ok {
|
||||
fmt.Fprintf(&b, "Run: %s\n\n", t)
|
||||
}
|
||||
|
||||
lines := strings.Split(raw, "\n")
|
||||
var stepKeys []string
|
||||
seenStep := map[string]bool{}
|
||||
for _, line := range lines {
|
||||
if idx := strings.Index(line, "_status="); idx >= 0 {
|
||||
key := line[:idx]
|
||||
if !seenStep[key] && key != "overall" {
|
||||
seenStep[key] = true
|
||||
stepKeys = append(stepKeys, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, key := range stepKeys {
|
||||
status := kv[key+"_status"]
|
||||
display := cleanSummaryKey(key)
|
||||
switch status {
|
||||
case "OK":
|
||||
fmt.Fprintf(&b, "PASS %s\n", display)
|
||||
case "FAILED":
|
||||
fmt.Fprintf(&b, "FAIL %s\n", display)
|
||||
case "UNSUPPORTED":
|
||||
fmt.Fprintf(&b, "SKIP %s\n", display)
|
||||
default:
|
||||
fmt.Fprintf(&b, "? %s\n", display)
|
||||
}
|
||||
}
|
||||
|
||||
if overall, ok := kv["overall_status"]; ok {
|
||||
ok2 := kv["job_ok"]
|
||||
failed := kv["job_failed"]
|
||||
fmt.Fprintf(&b, "\nOverall: %s (ok=%s failed=%s)", overall, ok2, failed)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
|
||||
func cleanSummaryKey(key string) string {
|
||||
idx := strings.Index(key, "-")
|
||||
if idx <= 0 {
|
||||
return key
|
||||
}
|
||||
prefix := key[:idx]
|
||||
for _, c := range prefix {
|
||||
if c < '0' || c > '9' {
|
||||
return key
|
||||
}
|
||||
}
|
||||
return key[idx+1:]
|
||||
}
|
||||
|
||||
@@ -43,6 +43,9 @@ func (f fakeNetwork) SetStaticIPv4(cfg platform.StaticIPv4Config) (string, error
|
||||
return f.setStaticIPv4Fn(cfg)
|
||||
}
|
||||
|
||||
func (f fakeNetwork) SetInterfaceState(_ string, _ bool) error { return nil }
|
||||
func (f fakeNetwork) GetInterfaceState(_ string) (bool, error) { return true, nil }
|
||||
|
||||
type fakeServices struct {
|
||||
serviceStatusFn func(string) (string, error)
|
||||
serviceDoFn func(string, platform.ServiceAction) (string, error)
|
||||
@@ -52,6 +55,10 @@ func (f fakeServices) ListBeeServices() ([]string, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f fakeServices) ServiceState(name string) string {
|
||||
return "active"
|
||||
}
|
||||
|
||||
func (f fakeServices) ServiceStatus(name string) (string, error) {
|
||||
return f.serviceStatusFn(name)
|
||||
}
|
||||
@@ -119,11 +126,11 @@ type fakeSAT struct {
|
||||
listNvidiaGPUsFn func() ([]platform.NvidiaGPU, error)
|
||||
}
|
||||
|
||||
func (f fakeSAT) RunNvidiaAcceptancePack(baseDir string) (string, error) {
|
||||
func (f fakeSAT) RunNvidiaAcceptancePack(baseDir string, _ func(string)) (string, error) {
|
||||
return f.runNvidiaFn(baseDir)
|
||||
}
|
||||
|
||||
func (f fakeSAT) RunNvidiaAcceptancePackWithOptions(_ context.Context, baseDir string, _ int, _ []int) (string, error) {
|
||||
func (f fakeSAT) RunNvidiaAcceptancePackWithOptions(_ context.Context, baseDir string, _ int, _ []int, _ func(string)) (string, error) {
|
||||
return f.runNvidiaFn(baseDir)
|
||||
}
|
||||
|
||||
@@ -134,15 +141,15 @@ func (f fakeSAT) ListNvidiaGPUs() ([]platform.NvidiaGPU, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f fakeSAT) RunMemoryAcceptancePack(baseDir string) (string, error) {
|
||||
func (f fakeSAT) RunMemoryAcceptancePack(baseDir string, _ func(string)) (string, error) {
|
||||
return f.runMemoryFn(baseDir)
|
||||
}
|
||||
|
||||
func (f fakeSAT) RunStorageAcceptancePack(baseDir string) (string, error) {
|
||||
func (f fakeSAT) RunStorageAcceptancePack(baseDir string, _ func(string)) (string, error) {
|
||||
return f.runStorageFn(baseDir)
|
||||
}
|
||||
|
||||
func (f fakeSAT) RunCPUAcceptancePack(baseDir string, durationSec int) (string, error) {
|
||||
func (f fakeSAT) RunCPUAcceptancePack(baseDir string, durationSec int, _ func(string)) (string, error) {
|
||||
if f.runCPUFn != nil {
|
||||
return f.runCPUFn(baseDir, durationSec)
|
||||
}
|
||||
@@ -163,18 +170,22 @@ func (f fakeSAT) ListAMDGPUs() ([]platform.AMDGPUInfo, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f fakeSAT) RunAMDAcceptancePack(baseDir string) (string, error) {
|
||||
func (f fakeSAT) RunAMDAcceptancePack(baseDir string, _ func(string)) (string, error) {
|
||||
if f.runAMDPackFn != nil {
|
||||
return f.runAMDPackFn(baseDir)
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (f fakeSAT) RunAMDStressPack(_ string, _ func(string)) (string, error) { return "", nil }
|
||||
func (f fakeSAT) RunMemoryStressPack(_ string, _ func(string)) (string, error) { return "", nil }
|
||||
func (f fakeSAT) RunSATStressPack(_ string, _ func(string)) (string, error) { return "", nil }
|
||||
|
||||
func (f fakeSAT) RunFanStressTest(_ context.Context, _ string, _ platform.FanStressOptions) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (f fakeSAT) RunNCCLTests(_ context.Context, _ string) (string, error) {
|
||||
func (f fakeSAT) RunNCCLTests(_ context.Context, _ string, _ func(string)) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
@@ -570,13 +581,13 @@ func TestRunSATDefaultsToExportDir(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if _, err := a.RunNvidiaAcceptancePack(""); err != nil {
|
||||
if _, err := a.RunNvidiaAcceptancePack("", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := a.RunMemoryAcceptancePack(""); err != nil {
|
||||
if _, err := a.RunMemoryAcceptancePack("", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := a.RunStorageAcceptancePack(""); err != nil {
|
||||
if _, err := a.RunStorageAcceptancePack("", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,387 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"bee/audit/internal/schema"
|
||||
)
|
||||
|
||||
// ComponentRow is one line in the hardware panel.
|
||||
type ComponentRow struct {
|
||||
Key string // "CPU", "MEM", "GPU", "DISK", "PSU"
|
||||
Status string // "PASS", "FAIL", "CANCEL", "N/A"
|
||||
Detail string // compact one-liner
|
||||
}
|
||||
|
||||
// HardwarePanelData holds everything the TUI right panel needs.
|
||||
type HardwarePanelData struct {
|
||||
Header []string
|
||||
Rows []ComponentRow
|
||||
}
|
||||
|
||||
// LoadHardwarePanel reads the latest audit JSON and SAT summaries.
|
||||
// Returns empty panel if no audit data exists yet.
|
||||
func (a *App) LoadHardwarePanel() HardwarePanelData {
|
||||
raw, err := os.ReadFile(DefaultAuditJSONPath)
|
||||
if err != nil {
|
||||
return HardwarePanelData{Header: []string{"No audit data — run audit first."}}
|
||||
}
|
||||
var snap schema.HardwareIngestRequest
|
||||
if err := json.Unmarshal(raw, &snap); err != nil {
|
||||
return HardwarePanelData{Header: []string{"Audit data unreadable."}}
|
||||
}
|
||||
|
||||
statuses := satStatuses()
|
||||
|
||||
var header []string
|
||||
if sys := formatSystemLine(snap.Hardware.Board); sys != "" {
|
||||
header = append(header, sys)
|
||||
}
|
||||
for _, fw := range snap.Hardware.Firmware {
|
||||
if fw.DeviceName == "BIOS" && fw.Version != "" {
|
||||
header = append(header, "BIOS: "+fw.Version)
|
||||
}
|
||||
if fw.DeviceName == "BMC" && fw.Version != "" {
|
||||
header = append(header, "BMC: "+fw.Version)
|
||||
}
|
||||
}
|
||||
if ip := formatIPLine(a.network.ListInterfaces); ip != "" {
|
||||
header = append(header, ip)
|
||||
}
|
||||
|
||||
var rows []ComponentRow
|
||||
|
||||
if cpu := formatCPULine(snap.Hardware.CPUs); cpu != "" {
|
||||
rows = append(rows, ComponentRow{
|
||||
Key: "CPU",
|
||||
Status: statuses["cpu"],
|
||||
Detail: strings.TrimPrefix(cpu, "CPU: "),
|
||||
})
|
||||
}
|
||||
if mem := formatMemoryLine(snap.Hardware.Memory); mem != "" {
|
||||
rows = append(rows, ComponentRow{
|
||||
Key: "MEM",
|
||||
Status: statuses["memory"],
|
||||
Detail: strings.TrimPrefix(mem, "Memory: "),
|
||||
})
|
||||
}
|
||||
if gpu := formatGPULine(snap.Hardware.PCIeDevices); gpu != "" {
|
||||
rows = append(rows, ComponentRow{
|
||||
Key: "GPU",
|
||||
Status: statuses["gpu"],
|
||||
Detail: strings.TrimPrefix(gpu, "GPU: "),
|
||||
})
|
||||
}
|
||||
if disk := formatStorageLine(snap.Hardware.Storage); disk != "" {
|
||||
rows = append(rows, ComponentRow{
|
||||
Key: "DISK",
|
||||
Status: statuses["storage"],
|
||||
Detail: strings.TrimPrefix(disk, "Storage: "),
|
||||
})
|
||||
}
|
||||
if psu := formatPSULine(snap.Hardware.PowerSupplies); psu != "" {
|
||||
rows = append(rows, ComponentRow{
|
||||
Key: "PSU",
|
||||
Status: "N/A",
|
||||
Detail: psu,
|
||||
})
|
||||
}
|
||||
|
||||
return HardwarePanelData{Header: header, Rows: rows}
|
||||
}
|
||||
|
||||
// ComponentDetailResult returns detail text for a component shown in the panel.
|
||||
func (a *App) ComponentDetailResult(key string) ActionResult {
|
||||
switch key {
|
||||
case "CPU":
|
||||
return a.cpuDetailResult(false)
|
||||
case "MEM":
|
||||
return a.satDetailResult("memory", "memory-", "MEM detail")
|
||||
case "GPU":
|
||||
// Prefer whichever GPU SAT was run most recently.
|
||||
nv, _ := filepath.Glob(filepath.Join(DefaultSATBaseDir, "gpu-nvidia-*/summary.txt"))
|
||||
am, _ := filepath.Glob(filepath.Join(DefaultSATBaseDir, "gpu-amd-*/summary.txt"))
|
||||
sort.Strings(nv)
|
||||
sort.Strings(am)
|
||||
latestNV := ""
|
||||
if len(nv) > 0 {
|
||||
latestNV = nv[len(nv)-1]
|
||||
}
|
||||
latestAM := ""
|
||||
if len(am) > 0 {
|
||||
latestAM = am[len(am)-1]
|
||||
}
|
||||
if latestAM > latestNV {
|
||||
return a.satDetailResult("gpu", "gpu-amd-", "GPU detail")
|
||||
}
|
||||
return a.satDetailResult("gpu", "gpu-nvidia-", "GPU detail")
|
||||
case "DISK":
|
||||
return a.satDetailResult("storage", "storage-", "DISK detail")
|
||||
case "PSU":
|
||||
return a.psuDetailResult()
|
||||
default:
|
||||
return ActionResult{Title: key, Body: "No detail available."}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) cpuDetailResult(satOnly bool) ActionResult {
|
||||
var b strings.Builder
|
||||
|
||||
// Show latest SAT summary if available.
|
||||
satResult := a.satDetailResult("cpu", "cpu-", "CPU SAT")
|
||||
if satResult.Body != "No test results found. Run a test first." {
|
||||
fmt.Fprintln(&b, "=== Last SAT ===")
|
||||
fmt.Fprintln(&b, satResult.Body)
|
||||
fmt.Fprintln(&b)
|
||||
}
|
||||
|
||||
if satOnly {
|
||||
body := strings.TrimSpace(b.String())
|
||||
if body == "" {
|
||||
body = "No CPU SAT results found. Run a test first."
|
||||
}
|
||||
return ActionResult{Title: "CPU SAT", Body: body}
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(DefaultAuditJSONPath)
|
||||
if err != nil {
|
||||
return ActionResult{Title: "CPU", Body: strings.TrimSpace(b.String())}
|
||||
}
|
||||
var snap schema.HardwareIngestRequest
|
||||
if err := json.Unmarshal(raw, &snap); err != nil {
|
||||
return ActionResult{Title: "CPU", Body: strings.TrimSpace(b.String())}
|
||||
}
|
||||
if len(snap.Hardware.CPUs) == 0 {
|
||||
return ActionResult{Title: "CPU", Body: strings.TrimSpace(b.String())}
|
||||
}
|
||||
fmt.Fprintln(&b, "=== Audit ===")
|
||||
for i, cpu := range snap.Hardware.CPUs {
|
||||
fmt.Fprintf(&b, "CPU %d\n", i)
|
||||
if cpu.Model != nil {
|
||||
fmt.Fprintf(&b, " Model: %s\n", *cpu.Model)
|
||||
}
|
||||
if cpu.Manufacturer != nil {
|
||||
fmt.Fprintf(&b, " Vendor: %s\n", *cpu.Manufacturer)
|
||||
}
|
||||
if cpu.Cores != nil {
|
||||
fmt.Fprintf(&b, " Cores: %d\n", *cpu.Cores)
|
||||
}
|
||||
if cpu.Threads != nil {
|
||||
fmt.Fprintf(&b, " Threads: %d\n", *cpu.Threads)
|
||||
}
|
||||
if cpu.MaxFrequencyMHz != nil {
|
||||
fmt.Fprintf(&b, " Max freq: %d MHz\n", *cpu.MaxFrequencyMHz)
|
||||
}
|
||||
if cpu.TemperatureC != nil {
|
||||
fmt.Fprintf(&b, " Temp: %.1f°C\n", *cpu.TemperatureC)
|
||||
}
|
||||
if cpu.Throttled != nil {
|
||||
fmt.Fprintf(&b, " Throttled: %v\n", *cpu.Throttled)
|
||||
}
|
||||
if cpu.CorrectableErrorCount != nil && *cpu.CorrectableErrorCount > 0 {
|
||||
fmt.Fprintf(&b, " ECC correctable: %d\n", *cpu.CorrectableErrorCount)
|
||||
}
|
||||
if cpu.UncorrectableErrorCount != nil && *cpu.UncorrectableErrorCount > 0 {
|
||||
fmt.Fprintf(&b, " ECC uncorrectable: %d\n", *cpu.UncorrectableErrorCount)
|
||||
}
|
||||
if i < len(snap.Hardware.CPUs)-1 {
|
||||
fmt.Fprintln(&b)
|
||||
}
|
||||
}
|
||||
return ActionResult{Title: "CPU", Body: strings.TrimSpace(b.String())}
|
||||
}
|
||||
|
||||
func (a *App) satDetailResult(statusKey, prefix, title string) ActionResult {
|
||||
matches, err := filepath.Glob(filepath.Join(DefaultSATBaseDir, prefix+"*/summary.txt"))
|
||||
if err != nil || len(matches) == 0 {
|
||||
return ActionResult{Title: title, Body: "No test results found. Run a test first."}
|
||||
}
|
||||
sort.Strings(matches)
|
||||
raw, err := os.ReadFile(matches[len(matches)-1])
|
||||
if err != nil {
|
||||
return ActionResult{Title: title, Body: "Could not read test results."}
|
||||
}
|
||||
return ActionResult{Title: title, Body: formatSATDetail(strings.TrimSpace(string(raw)))}
|
||||
}
|
||||
|
||||
// formatSATDetail converts raw summary.txt key=value content to a human-readable per-step display.
|
||||
func formatSATDetail(raw string) string {
|
||||
var b strings.Builder
|
||||
kv := parseKeyValueSummary(raw)
|
||||
|
||||
if t, ok := kv["run_at_utc"]; ok {
|
||||
fmt.Fprintf(&b, "Run: %s\n\n", t)
|
||||
}
|
||||
|
||||
// Collect step names in order they appear in the file
|
||||
lines := strings.Split(raw, "\n")
|
||||
var stepKeys []string
|
||||
seenStep := map[string]bool{}
|
||||
for _, line := range lines {
|
||||
if idx := strings.Index(line, "_status="); idx >= 0 {
|
||||
key := line[:idx]
|
||||
if !seenStep[key] && key != "overall" {
|
||||
seenStep[key] = true
|
||||
stepKeys = append(stepKeys, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, key := range stepKeys {
|
||||
status := kv[key+"_status"]
|
||||
display := cleanSummaryKey(key)
|
||||
switch status {
|
||||
case "OK":
|
||||
fmt.Fprintf(&b, "PASS %s\n", display)
|
||||
case "FAILED":
|
||||
fmt.Fprintf(&b, "FAIL %s\n", display)
|
||||
case "UNSUPPORTED":
|
||||
fmt.Fprintf(&b, "SKIP %s\n", display)
|
||||
default:
|
||||
fmt.Fprintf(&b, "? %s\n", display)
|
||||
}
|
||||
}
|
||||
|
||||
if overall, ok := kv["overall_status"]; ok {
|
||||
ok2 := kv["job_ok"]
|
||||
failed := kv["job_failed"]
|
||||
fmt.Fprintf(&b, "\nOverall: %s (ok=%s failed=%s)", overall, ok2, failed)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
|
||||
// cleanSummaryKey strips the leading numeric prefix from a SAT step key.
|
||||
// "1-lscpu" → "lscpu", "3-stress-ng" → "stress-ng"
|
||||
func cleanSummaryKey(key string) string {
|
||||
idx := strings.Index(key, "-")
|
||||
if idx <= 0 {
|
||||
return key
|
||||
}
|
||||
prefix := key[:idx]
|
||||
for _, c := range prefix {
|
||||
if c < '0' || c > '9' {
|
||||
return key
|
||||
}
|
||||
}
|
||||
return key[idx+1:]
|
||||
}
|
||||
|
||||
func (a *App) psuDetailResult() ActionResult {
|
||||
raw, err := os.ReadFile(DefaultAuditJSONPath)
|
||||
if err != nil {
|
||||
return ActionResult{Title: "PSU", Body: "No audit data."}
|
||||
}
|
||||
var snap schema.HardwareIngestRequest
|
||||
if err := json.Unmarshal(raw, &snap); err != nil {
|
||||
return ActionResult{Title: "PSU", Body: "Audit data unreadable."}
|
||||
}
|
||||
if len(snap.Hardware.PowerSupplies) == 0 {
|
||||
return ActionResult{Title: "PSU", Body: "No PSU data in last audit."}
|
||||
}
|
||||
var b strings.Builder
|
||||
for i, psu := range snap.Hardware.PowerSupplies {
|
||||
fmt.Fprintf(&b, "PSU %d\n", i)
|
||||
if psu.Model != nil {
|
||||
fmt.Fprintf(&b, " Model: %s\n", *psu.Model)
|
||||
}
|
||||
if psu.Vendor != nil {
|
||||
fmt.Fprintf(&b, " Vendor: %s\n", *psu.Vendor)
|
||||
}
|
||||
if psu.WattageW != nil {
|
||||
fmt.Fprintf(&b, " Rated: %d W\n", *psu.WattageW)
|
||||
}
|
||||
if psu.InputPowerW != nil {
|
||||
fmt.Fprintf(&b, " Input: %.1f W\n", *psu.InputPowerW)
|
||||
}
|
||||
if psu.OutputPowerW != nil {
|
||||
fmt.Fprintf(&b, " Output: %.1f W\n", *psu.OutputPowerW)
|
||||
}
|
||||
if psu.TemperatureC != nil {
|
||||
fmt.Fprintf(&b, " Temp: %.1f°C\n", *psu.TemperatureC)
|
||||
}
|
||||
if i < len(snap.Hardware.PowerSupplies)-1 {
|
||||
fmt.Fprintln(&b)
|
||||
}
|
||||
}
|
||||
return ActionResult{Title: "PSU", Body: strings.TrimSpace(b.String())}
|
||||
}
|
||||
|
||||
// satStatuses reads the latest summary.txt for each SAT type and returns
|
||||
// a map of component key ("gpu","memory","storage") → status ("PASS","FAIL","CANCEL","N/A").
|
||||
func satStatuses() map[string]string {
|
||||
result := map[string]string{
|
||||
"gpu": "N/A",
|
||||
"memory": "N/A",
|
||||
"storage": "N/A",
|
||||
"cpu": "N/A",
|
||||
}
|
||||
patterns := []struct {
|
||||
key string
|
||||
prefix string
|
||||
}{
|
||||
{"gpu", "gpu-nvidia-"},
|
||||
{"gpu", "gpu-amd-"},
|
||||
{"memory", "memory-"},
|
||||
{"storage", "storage-"},
|
||||
{"cpu", "cpu-"},
|
||||
}
|
||||
for _, item := range patterns {
|
||||
matches, err := filepath.Glob(filepath.Join(DefaultSATBaseDir, item.prefix+"*/summary.txt"))
|
||||
if err != nil || len(matches) == 0 {
|
||||
continue
|
||||
}
|
||||
sort.Strings(matches)
|
||||
raw, err := os.ReadFile(matches[len(matches)-1])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
values := parseKeyValueSummary(string(raw))
|
||||
switch strings.ToUpper(strings.TrimSpace(values["overall_status"])) {
|
||||
case "OK":
|
||||
result[item.key] = "PASS"
|
||||
case "FAILED":
|
||||
result[item.key] = "FAIL"
|
||||
case "CANCELED", "CANCELLED":
|
||||
result[item.key] = "CANCEL"
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func formatPSULine(psus []schema.HardwarePowerSupply) string {
|
||||
var present []schema.HardwarePowerSupply
|
||||
for _, psu := range psus {
|
||||
if psu.Present != nil && !*psu.Present {
|
||||
continue
|
||||
}
|
||||
present = append(present, psu)
|
||||
}
|
||||
if len(present) == 0 {
|
||||
return ""
|
||||
}
|
||||
firstW := 0
|
||||
if present[0].WattageW != nil {
|
||||
firstW = *present[0].WattageW
|
||||
}
|
||||
allSame := firstW > 0
|
||||
for _, p := range present[1:] {
|
||||
w := 0
|
||||
if p.WattageW != nil {
|
||||
w = *p.WattageW
|
||||
}
|
||||
if w != firstW {
|
||||
allSame = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allSame && firstW > 0 {
|
||||
return fmt.Sprintf("%dx %dW", len(present), firstW)
|
||||
}
|
||||
return fmt.Sprintf("%d PSU", len(present))
|
||||
}
|
||||
@@ -141,9 +141,11 @@ func satSummaryStatus(summary satSummary, label string) (string, string, bool) {
|
||||
func satKeyStatus(rawStatus, label string) (string, string, bool) {
|
||||
switch strings.ToUpper(strings.TrimSpace(rawStatus)) {
|
||||
case "OK":
|
||||
return "OK", label + " passed", true
|
||||
// No error description on success — error_description is for problems only.
|
||||
return "OK", "", true
|
||||
case "PARTIAL", "UNSUPPORTED", "CANCELED", "CANCELLED":
|
||||
return "Warning", label + " incomplete", true
|
||||
// Tool couldn't run or test was incomplete — we can't assert hardware health.
|
||||
return "Unknown", "", true
|
||||
case "FAILED":
|
||||
return "Critical", label + " failed", true
|
||||
default:
|
||||
@@ -180,6 +182,8 @@ func statusSeverity(status string) int {
|
||||
return 2
|
||||
case "OK":
|
||||
return 1
|
||||
case "Unknown":
|
||||
return 1 // same as OK — does not override OK from another source
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -13,18 +13,19 @@ import (
|
||||
|
||||
// GPUMetricRow is one telemetry sample from nvidia-smi during a stress test.
|
||||
type GPUMetricRow struct {
|
||||
ElapsedSec float64
|
||||
GPUIndex int
|
||||
TempC float64
|
||||
UsagePct float64
|
||||
PowerW float64
|
||||
ClockMHz float64
|
||||
ElapsedSec float64 `json:"elapsed_sec"`
|
||||
GPUIndex int `json:"index"`
|
||||
TempC float64 `json:"temp_c"`
|
||||
UsagePct float64 `json:"usage_pct"`
|
||||
MemUsagePct float64 `json:"mem_usage_pct"`
|
||||
PowerW float64 `json:"power_w"`
|
||||
ClockMHz float64 `json:"clock_mhz"`
|
||||
}
|
||||
|
||||
// sampleGPUMetrics runs nvidia-smi once and returns current metrics for each GPU.
|
||||
func sampleGPUMetrics(gpuIndices []int) ([]GPUMetricRow, error) {
|
||||
args := []string{
|
||||
"--query-gpu=index,temperature.gpu,utilization.gpu,power.draw,clocks.current.graphics",
|
||||
"--query-gpu=index,temperature.gpu,utilization.gpu,utilization.memory,power.draw,clocks.current.graphics",
|
||||
"--format=csv,noheader,nounits",
|
||||
}
|
||||
if len(gpuIndices) > 0 {
|
||||
@@ -45,16 +46,17 @@ func sampleGPUMetrics(gpuIndices []int) ([]GPUMetricRow, error) {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(line, ", ")
|
||||
if len(parts) < 5 {
|
||||
if len(parts) < 6 {
|
||||
continue
|
||||
}
|
||||
idx, _ := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||||
rows = append(rows, GPUMetricRow{
|
||||
GPUIndex: idx,
|
||||
TempC: parseGPUFloat(parts[1]),
|
||||
UsagePct: parseGPUFloat(parts[2]),
|
||||
PowerW: parseGPUFloat(parts[3]),
|
||||
ClockMHz: parseGPUFloat(parts[4]),
|
||||
GPUIndex: idx,
|
||||
TempC: parseGPUFloat(parts[1]),
|
||||
UsagePct: parseGPUFloat(parts[2]),
|
||||
MemUsagePct: parseGPUFloat(parts[3]),
|
||||
PowerW: parseGPUFloat(parts[4]),
|
||||
ClockMHz: parseGPUFloat(parts[5]),
|
||||
})
|
||||
}
|
||||
return rows, nil
|
||||
@@ -74,6 +76,58 @@ func SampleGPUMetrics(gpuIndices []int) ([]GPUMetricRow, error) {
|
||||
return sampleGPUMetrics(gpuIndices)
|
||||
}
|
||||
|
||||
// sampleAMDGPUMetrics queries rocm-smi for live GPU metrics.
|
||||
func sampleAMDGPUMetrics() ([]GPUMetricRow, error) {
|
||||
// --showtemp --showuse --showpower --csv — one row per GPU
|
||||
out, err := runROCmSMI("--showtemp", "--showuse", "--showpower", "--showmemuse", "--csv")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var rows []GPUMetricRow
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(strings.ToLower(line), "device") {
|
||||
continue
|
||||
}
|
||||
// CSV format: device,temp_c,gpu_use%,mem_use%,power_w (order may vary by rocm-smi version)
|
||||
// We parse by column header from the first line.
|
||||
parts := strings.Split(line, ",")
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
idx := len(rows)
|
||||
row := GPUMetricRow{GPUIndex: idx}
|
||||
// rocm-smi CSV columns vary; extract what we can
|
||||
for i, p := range parts {
|
||||
p = strings.TrimSpace(p)
|
||||
switch {
|
||||
case i == 0:
|
||||
// device index like "card0" or "0"
|
||||
case strings.Contains(strings.ToLower(p), "n/a"):
|
||||
// skip N/A
|
||||
default:
|
||||
// Try to match by position heuristic: temp, use%, memuse%, power
|
||||
v := parseGPUFloat(p)
|
||||
switch {
|
||||
case i == 1 && row.TempC == 0:
|
||||
row.TempC = v
|
||||
case i == 2 && row.UsagePct == 0:
|
||||
row.UsagePct = v
|
||||
case i == 3 && row.MemUsagePct == 0:
|
||||
row.MemUsagePct = v
|
||||
case i == 4 && row.PowerW == 0:
|
||||
row.PowerW = v
|
||||
}
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, fmt.Errorf("rocm-smi: no GPU rows parsed")
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// WriteGPUMetricsCSV writes collected rows as a CSV file.
|
||||
func WriteGPUMetricsCSV(path string, rows []GPUMetricRow) error {
|
||||
var b bytes.Buffer
|
||||
@@ -332,7 +386,7 @@ const (
|
||||
)
|
||||
|
||||
// RenderGPUTerminalChart returns ANSI line charts (asciigraph-style) per GPU.
|
||||
// Suitable for display in the TUI screenOutput.
|
||||
// Used in SAT stress-test logs.
|
||||
func RenderGPUTerminalChart(rows []GPUMetricRow) string {
|
||||
seen := make(map[int]bool)
|
||||
var order []int
|
||||
@@ -375,162 +429,6 @@ func RenderGPUTerminalChart(rows []GPUMetricRow) string {
|
||||
return strings.TrimRight(b.String(), "\n")
|
||||
}
|
||||
|
||||
// RenderGPULiveChart renders all GPU metrics on a single combined chart per GPU.
|
||||
// Each series is normalised to its own min–max and drawn in a different colour.
|
||||
// chartWidth controls the width of the plot area (Y-axis label uses 5 extra chars).
|
||||
func RenderGPULiveChart(rows []GPUMetricRow, chartWidth int) string {
|
||||
if chartWidth < 20 {
|
||||
chartWidth = 70
|
||||
}
|
||||
const chartHeight = 14
|
||||
|
||||
seen := make(map[int]bool)
|
||||
var order []int
|
||||
gpuMap := make(map[int][]GPUMetricRow)
|
||||
for _, r := range rows {
|
||||
if !seen[r.GPUIndex] {
|
||||
seen[r.GPUIndex] = true
|
||||
order = append(order, r.GPUIndex)
|
||||
}
|
||||
gpuMap[r.GPUIndex] = append(gpuMap[r.GPUIndex], r)
|
||||
}
|
||||
|
||||
type seriesDef struct {
|
||||
label string
|
||||
color string
|
||||
unit string
|
||||
fn func(GPUMetricRow) float64
|
||||
}
|
||||
defs := []seriesDef{
|
||||
{"Usage", ansiBlue, "%", func(r GPUMetricRow) float64 { return r.UsagePct }},
|
||||
{"Temp", ansiRed, "°C", func(r GPUMetricRow) float64 { return r.TempC }},
|
||||
{"Power", ansiGreen, "W", func(r GPUMetricRow) float64 { return r.PowerW }},
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
for _, gpuIdx := range order {
|
||||
gr := gpuMap[gpuIdx]
|
||||
if len(gr) == 0 {
|
||||
continue
|
||||
}
|
||||
elapsed := gr[len(gr)-1].ElapsedSec
|
||||
|
||||
// Build value slices for each series.
|
||||
type seriesData struct {
|
||||
seriesDef
|
||||
vals []float64
|
||||
mn float64
|
||||
mx float64
|
||||
}
|
||||
var series []seriesData
|
||||
for _, d := range defs {
|
||||
vals := extractGPUField(gr, d.fn)
|
||||
mn, mx := gpuMinMax(vals)
|
||||
if mn == mx {
|
||||
mx = mn + 1
|
||||
}
|
||||
series = append(series, seriesData{d, vals, mn, mx})
|
||||
}
|
||||
|
||||
// Shared character grid: row 0 = top (max), row chartHeight = bottom (min).
|
||||
type cell struct {
|
||||
ch rune
|
||||
color string
|
||||
}
|
||||
grid := make([][]cell, chartHeight+1)
|
||||
for r := range grid {
|
||||
grid[r] = make([]cell, chartWidth)
|
||||
for c := range grid[r] {
|
||||
grid[r][c] = cell{' ', ""}
|
||||
}
|
||||
}
|
||||
|
||||
// Plot each series onto the shared grid.
|
||||
for _, s := range series {
|
||||
w := chartWidth
|
||||
if len(s.vals) < w {
|
||||
w = len(s.vals)
|
||||
}
|
||||
data := gpuDownsample(s.vals, w)
|
||||
prevRow := -1
|
||||
for x, v := range data {
|
||||
row := chartHeight - int(math.Round((v-s.mn)/(s.mx-s.mn)*float64(chartHeight)))
|
||||
if row < 0 {
|
||||
row = 0
|
||||
}
|
||||
if row > chartHeight {
|
||||
row = chartHeight
|
||||
}
|
||||
if prevRow < 0 || prevRow == row {
|
||||
grid[row][x] = cell{'─', s.color}
|
||||
} else {
|
||||
lo, hi := prevRow, row
|
||||
if lo > hi {
|
||||
lo, hi = hi, lo
|
||||
}
|
||||
for y := lo + 1; y < hi; y++ {
|
||||
grid[y][x] = cell{'│', s.color}
|
||||
}
|
||||
if prevRow < row {
|
||||
grid[prevRow][x] = cell{'╮', s.color}
|
||||
grid[row][x] = cell{'╰', s.color}
|
||||
} else {
|
||||
grid[prevRow][x] = cell{'╯', s.color}
|
||||
grid[row][x] = cell{'╭', s.color}
|
||||
}
|
||||
}
|
||||
prevRow = row
|
||||
}
|
||||
}
|
||||
|
||||
// Render: Y axis + data rows.
|
||||
fmt.Fprintf(&b, "GPU %d (%.0fs) each series normalised to its range\n", gpuIdx, elapsed)
|
||||
for r := 0; r <= chartHeight; r++ {
|
||||
// Y axis label: 100% at top, 50% in middle, 0% at bottom.
|
||||
switch r {
|
||||
case 0:
|
||||
fmt.Fprintf(&b, "%4s┤", "100%")
|
||||
case chartHeight / 2:
|
||||
fmt.Fprintf(&b, "%4s┤", "50%")
|
||||
case chartHeight:
|
||||
fmt.Fprintf(&b, "%4s┤", "0%")
|
||||
default:
|
||||
fmt.Fprintf(&b, "%4s│", "")
|
||||
}
|
||||
for c := 0; c < chartWidth; c++ {
|
||||
cl := grid[r][c]
|
||||
if cl.color != "" {
|
||||
b.WriteString(cl.color)
|
||||
b.WriteRune(cl.ch)
|
||||
b.WriteString(ansiReset)
|
||||
} else {
|
||||
b.WriteRune(' ')
|
||||
}
|
||||
}
|
||||
b.WriteRune('\n')
|
||||
}
|
||||
// Bottom axis.
|
||||
b.WriteString(" └")
|
||||
b.WriteString(strings.Repeat("─", chartWidth))
|
||||
b.WriteRune('\n')
|
||||
|
||||
// Legend with current (last) values.
|
||||
b.WriteString(" ")
|
||||
for i, s := range series {
|
||||
last := s.vals[len(s.vals)-1]
|
||||
b.WriteString(s.color)
|
||||
fmt.Fprintf(&b, "▐ %s: %.0f%s", s.label, last, s.unit)
|
||||
b.WriteString(ansiReset)
|
||||
if i < len(series)-1 {
|
||||
b.WriteString(" ")
|
||||
}
|
||||
}
|
||||
b.WriteRune('\n')
|
||||
}
|
||||
|
||||
return strings.TrimRight(b.String(), "\n")
|
||||
}
|
||||
|
||||
// renderLineChart draws a single time-series line chart using box-drawing characters.
|
||||
// Produces output in the style of asciigraph: ╭─╮ │ ╰─╯ with a Y axis and caption.
|
||||
func renderLineChart(vals []float64, color, caption string, height, width int) string {
|
||||
|
||||
@@ -3,6 +3,7 @@ package platform
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -10,13 +11,17 @@ import (
|
||||
|
||||
// InstallDisk describes a candidate disk for installation.
|
||||
type InstallDisk struct {
|
||||
Device string // e.g. /dev/sda
|
||||
Model string
|
||||
Size string // human-readable, e.g. "500G"
|
||||
Device string // e.g. /dev/sda
|
||||
Model string
|
||||
Size string // human-readable, e.g. "500G"
|
||||
SizeBytes int64 // raw byte count from lsblk
|
||||
MountedParts []string // partition mount points currently active
|
||||
}
|
||||
|
||||
const squashfsPath = "/run/live/medium/live/filesystem.squashfs"
|
||||
|
||||
// ListInstallDisks returns block devices suitable for installation.
|
||||
// Excludes USB drives and the current live boot medium.
|
||||
// Excludes the current live boot medium but includes USB drives.
|
||||
func (s *System) ListInstallDisks() ([]InstallDisk, error) {
|
||||
out, err := exec.Command("lsblk", "-dn", "-o", "NAME,MODEL,SIZE,TYPE,TRAN").Output()
|
||||
if err != nil {
|
||||
@@ -33,7 +38,6 @@ func (s *System) ListInstallDisks() ([]InstallDisk, error) {
|
||||
continue
|
||||
}
|
||||
// Last field: TRAN, second-to-last: TYPE, third-to-last: SIZE
|
||||
tran := fields[len(fields)-1]
|
||||
typ := fields[len(fields)-2]
|
||||
size := fields[len(fields)-3]
|
||||
name := fields[0]
|
||||
@@ -42,24 +46,58 @@ func (s *System) ListInstallDisks() ([]InstallDisk, error) {
|
||||
if typ != "disk" {
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(tran, "usb") {
|
||||
continue
|
||||
}
|
||||
|
||||
device := "/dev/" + name
|
||||
if device == bootDev {
|
||||
continue
|
||||
}
|
||||
|
||||
sizeBytes := diskSizeBytes(device)
|
||||
mounted := mountedParts(device)
|
||||
|
||||
disks = append(disks, InstallDisk{
|
||||
Device: device,
|
||||
Model: strings.TrimSpace(model),
|
||||
Size: size,
|
||||
Device: device,
|
||||
Model: strings.TrimSpace(model),
|
||||
Size: size,
|
||||
SizeBytes: sizeBytes,
|
||||
MountedParts: mounted,
|
||||
})
|
||||
}
|
||||
return disks, nil
|
||||
}
|
||||
|
||||
// diskSizeBytes returns the byte size of a block device using lsblk.
|
||||
func diskSizeBytes(device string) int64 {
|
||||
out, err := exec.Command("lsblk", "-bdn", "-o", "SIZE", device).Output()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
n, _ := strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64)
|
||||
return n
|
||||
}
|
||||
|
||||
// mountedParts returns a list of "<part> at <mountpoint>" strings for any
|
||||
// mounted partitions on the given device.
|
||||
func mountedParts(device string) []string {
|
||||
out, err := exec.Command("lsblk", "-n", "-o", "NAME,MOUNTPOINT", device).Output()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var result []string
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 {
|
||||
continue
|
||||
}
|
||||
mp := fields[1]
|
||||
if mp == "" || mp == "[SWAP]" {
|
||||
continue
|
||||
}
|
||||
result = append(result, "/dev/"+strings.TrimLeft(fields[0], "└─├─")+" at "+mp)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// findLiveBootDevice returns the block device backing /run/live/medium (if any).
|
||||
func findLiveBootDevice() string {
|
||||
out, err := exec.Command("findmnt", "-n", "-o", "SOURCE", "/run/live/medium").Output()
|
||||
@@ -79,6 +117,80 @@ func findLiveBootDevice() string {
|
||||
return "/dev/" + strings.TrimSpace(string(out2))
|
||||
}
|
||||
|
||||
// MinInstallBytes returns the minimum recommended disk size for installation:
|
||||
// squashfs size × 1.5 to allow for extracted filesystem and bootloader.
|
||||
// Returns 0 if the squashfs is not available (non-live environment).
|
||||
func MinInstallBytes() int64 {
|
||||
fi, err := os.Stat(squashfsPath)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return fi.Size() * 3 / 2
|
||||
}
|
||||
|
||||
// toramActive returns true when the live system was booted with toram.
|
||||
func toramActive() bool {
|
||||
data, err := os.ReadFile("/proc/cmdline")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(string(data), "toram")
|
||||
}
|
||||
|
||||
// freeMemBytes returns MemAvailable from /proc/meminfo.
|
||||
func freeMemBytes() int64 {
|
||||
data, err := os.ReadFile("/proc/meminfo")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
if strings.HasPrefix(line, "MemAvailable:") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 2 {
|
||||
n, _ := strconv.ParseInt(fields[1], 10, 64)
|
||||
return n * 1024 // kB → bytes
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// DiskWarnings returns advisory warning strings for a disk candidate.
|
||||
func DiskWarnings(d InstallDisk) []string {
|
||||
var w []string
|
||||
if len(d.MountedParts) > 0 {
|
||||
w = append(w, "has mounted partitions: "+strings.Join(d.MountedParts, ", "))
|
||||
}
|
||||
min := MinInstallBytes()
|
||||
if min > 0 && d.SizeBytes > 0 && d.SizeBytes < min {
|
||||
w = append(w, fmt.Sprintf("disk may be too small (need ≥ %s, have %s)",
|
||||
humanBytes(min), humanBytes(d.SizeBytes)))
|
||||
}
|
||||
if toramActive() {
|
||||
sqFi, err := os.Stat(squashfsPath)
|
||||
if err == nil {
|
||||
free := freeMemBytes()
|
||||
if free > 0 && free < sqFi.Size()*2 {
|
||||
w = append(w, "toram mode — low RAM, extraction may be slow or fail")
|
||||
}
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
func humanBytes(b int64) string {
|
||||
const unit = 1024
|
||||
if b < unit {
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := b / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
// InstallToDisk runs bee-install <device> <logfile> and streams output to logFile.
|
||||
// The context can be used to cancel.
|
||||
func (s *System) InstallToDisk(ctx context.Context, device string, logFile string) error {
|
||||
@@ -92,14 +204,11 @@ func InstallLogPath(device string) string {
|
||||
return "/tmp/bee-install" + safe + ".log"
|
||||
}
|
||||
|
||||
// DiskLabel returns a display label for a disk.
|
||||
// Label returns a display label for a disk.
|
||||
func (d InstallDisk) Label() string {
|
||||
model := d.Model
|
||||
if model == "" {
|
||||
model = "Unknown"
|
||||
}
|
||||
sizeBytes, err := strconv.ParseInt(strings.TrimSuffix(d.Size, "B"), 10, 64)
|
||||
_ = sizeBytes
|
||||
_ = err
|
||||
return fmt.Sprintf("%s %s %s", d.Device, d.Size, model)
|
||||
}
|
||||
|
||||
178
audit/internal/platform/install_to_ram.go
Normal file
178
audit/internal/platform/install_to_ram.go
Normal file
@@ -0,0 +1,178 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (s *System) IsLiveMediaInRAM() bool {
|
||||
out, err := exec.Command("findmnt", "-n", "-o", "FSTYPE", "/run/live/medium").Output()
|
||||
if err != nil {
|
||||
return toramActive()
|
||||
}
|
||||
return strings.TrimSpace(string(out)) == "tmpfs"
|
||||
}
|
||||
|
||||
func (s *System) RunInstallToRAM(logFunc func(string)) error {
|
||||
log := func(msg string) {
|
||||
if logFunc != nil {
|
||||
logFunc(msg)
|
||||
}
|
||||
}
|
||||
|
||||
if s.IsLiveMediaInRAM() {
|
||||
log("Already running from RAM — installation media can be safely disconnected.")
|
||||
return nil
|
||||
}
|
||||
|
||||
squashfsFiles, err := filepath.Glob("/run/live/medium/live/*.squashfs")
|
||||
if err != nil || len(squashfsFiles) == 0 {
|
||||
return fmt.Errorf("no squashfs files found in /run/live/medium/live/")
|
||||
}
|
||||
|
||||
free := freeMemBytes()
|
||||
var needed int64
|
||||
for _, sf := range squashfsFiles {
|
||||
fi, err2 := os.Stat(sf)
|
||||
if err2 != nil {
|
||||
return fmt.Errorf("stat %s: %v", sf, err2)
|
||||
}
|
||||
needed += fi.Size()
|
||||
}
|
||||
const headroom = 256 * 1024 * 1024
|
||||
if free > 0 && needed+headroom > free {
|
||||
return fmt.Errorf("insufficient RAM: need %s, available %s",
|
||||
humanBytes(needed+headroom), humanBytes(free))
|
||||
}
|
||||
|
||||
dstDir := "/dev/shm/bee-live"
|
||||
if err := os.MkdirAll(dstDir, 0755); err != nil {
|
||||
return fmt.Errorf("create tmpfs dir: %v", err)
|
||||
}
|
||||
|
||||
for _, sf := range squashfsFiles {
|
||||
base := filepath.Base(sf)
|
||||
dst := filepath.Join(dstDir, base)
|
||||
log(fmt.Sprintf("Copying %s to RAM...", base))
|
||||
if err := copyFileLarge(sf, dst, log); err != nil {
|
||||
return fmt.Errorf("copy %s: %v", base, err)
|
||||
}
|
||||
log(fmt.Sprintf("Copied %s.", base))
|
||||
|
||||
loopDev, err := findLoopForFile(sf)
|
||||
if err != nil {
|
||||
log(fmt.Sprintf("Loop device for %s not found (%v) — skipping re-association.", base, err))
|
||||
continue
|
||||
}
|
||||
if err := reassociateLoopDevice(loopDev, dst); err != nil {
|
||||
log(fmt.Sprintf("Warning: could not re-associate %s → %s: %v", loopDev, dst, err))
|
||||
} else {
|
||||
log(fmt.Sprintf("Loop device %s now backed by RAM copy.", loopDev))
|
||||
}
|
||||
}
|
||||
|
||||
log("Copying remaining medium files...")
|
||||
if err := cpDir("/run/live/medium", dstDir, log); err != nil {
|
||||
log(fmt.Sprintf("Warning: partial copy: %v", err))
|
||||
}
|
||||
if err := exec.Command("mount", "--bind", dstDir, "/run/live/medium").Run(); err != nil {
|
||||
log(fmt.Sprintf("Warning: rebind /run/live/medium failed: %v", err))
|
||||
}
|
||||
|
||||
log("Done. Installation media can be safely disconnected.")
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyFileLarge(src, dst string, logFunc func(string)) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
fi, err := in.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
total := fi.Size()
|
||||
var copied int64
|
||||
buf := make([]byte, 4*1024*1024)
|
||||
for {
|
||||
n, err := in.Read(buf)
|
||||
if n > 0 {
|
||||
if _, werr := out.Write(buf[:n]); werr != nil {
|
||||
return werr
|
||||
}
|
||||
copied += int64(n)
|
||||
if logFunc != nil && total > 0 {
|
||||
pct := int(float64(copied) / float64(total) * 100)
|
||||
logFunc(fmt.Sprintf(" %s / %s (%d%%)", humanBytes(copied), humanBytes(total), pct))
|
||||
}
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return out.Sync()
|
||||
}
|
||||
|
||||
func cpDir(src, dst string, logFunc func(string)) error {
|
||||
return filepath.Walk(src, func(path string, fi os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(src, path)
|
||||
target := filepath.Join(dst, rel)
|
||||
if fi.IsDir() {
|
||||
return os.MkdirAll(target, fi.Mode())
|
||||
}
|
||||
if strings.HasSuffix(path, ".squashfs") {
|
||||
return nil
|
||||
}
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
return nil
|
||||
}
|
||||
return copyFileLarge(path, target, nil)
|
||||
})
|
||||
}
|
||||
|
||||
func findLoopForFile(backingFile string) (string, error) {
|
||||
out, err := exec.Command("losetup", "--list", "--json").Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var result struct {
|
||||
Loopdevices []struct {
|
||||
Name string `json:"name"`
|
||||
BackFile string `json:"back-file"`
|
||||
} `json:"loopdevices"`
|
||||
}
|
||||
if err := json.Unmarshal(out, &result); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, dev := range result.Loopdevices {
|
||||
if dev.BackFile == backingFile {
|
||||
return dev.Name, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("no loop device found for %s", backingFile)
|
||||
}
|
||||
|
||||
func reassociateLoopDevice(loopDev, newFile string) error {
|
||||
if err := exec.Command("losetup", "--replace", loopDev, newFile).Run(); err == nil {
|
||||
return nil
|
||||
}
|
||||
return loopChangeFD(loopDev, newFile)
|
||||
}
|
||||
28
audit/internal/platform/install_to_ram_linux.go
Normal file
28
audit/internal/platform/install_to_ram_linux.go
Normal file
@@ -0,0 +1,28 @@
|
||||
//go:build linux
|
||||
|
||||
package platform
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
const ioctlLoopChangeFD = 0x4C08
|
||||
|
||||
func loopChangeFD(loopDev, newFile string) error {
|
||||
lf, err := os.OpenFile(loopDev, os.O_RDWR, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer lf.Close()
|
||||
nf, err := os.OpenFile(newFile, os.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer nf.Close()
|
||||
_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, lf.Fd(), ioctlLoopChangeFD, nf.Fd())
|
||||
if errno != 0 {
|
||||
return errno
|
||||
}
|
||||
return nil
|
||||
}
|
||||
9
audit/internal/platform/install_to_ram_other.go
Normal file
9
audit/internal/platform/install_to_ram_other.go
Normal file
@@ -0,0 +1,9 @@
|
||||
//go:build !linux
|
||||
|
||||
package platform
|
||||
|
||||
import "errors"
|
||||
|
||||
func loopChangeFD(loopDev, newFile string) error {
|
||||
return errors.New("LOOP_CHANGE_FD not available on this platform")
|
||||
}
|
||||
@@ -1,15 +1,23 @@
|
||||
package platform
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LiveMetricSample is a single point-in-time snapshot of server metrics
|
||||
// collected for the web UI metrics page.
|
||||
type LiveMetricSample struct {
|
||||
Timestamp time.Time `json:"ts"`
|
||||
Fans []FanReading `json:"fans"`
|
||||
Temps []TempReading `json:"temps"`
|
||||
PowerW float64 `json:"power_w"`
|
||||
GPUs []GPUMetricRow `json:"gpus"`
|
||||
Timestamp time.Time `json:"ts"`
|
||||
Fans []FanReading `json:"fans"`
|
||||
Temps []TempReading `json:"temps"`
|
||||
PowerW float64 `json:"power_w"`
|
||||
CPULoadPct float64 `json:"cpu_load_pct"`
|
||||
MemLoadPct float64 `json:"mem_load_pct"`
|
||||
GPUs []GPUMetricRow `json:"gpus"`
|
||||
}
|
||||
|
||||
// TempReading is a named temperature sensor value.
|
||||
@@ -24,9 +32,12 @@ type TempReading struct {
|
||||
func SampleLiveMetrics() LiveMetricSample {
|
||||
s := LiveMetricSample{Timestamp: time.Now().UTC()}
|
||||
|
||||
// GPU metrics — skipped silently if nvidia-smi unavailable
|
||||
gpus, _ := SampleGPUMetrics(nil)
|
||||
s.GPUs = gpus
|
||||
// GPU metrics — try NVIDIA first, fall back to AMD
|
||||
if gpus, err := SampleGPUMetrics(nil); err == nil && len(gpus) > 0 {
|
||||
s.GPUs = gpus
|
||||
} else if amdGPUs, err := sampleAMDGPUMetrics(); err == nil && len(amdGPUs) > 0 {
|
||||
s.GPUs = amdGPUs
|
||||
}
|
||||
|
||||
// Fan speeds — skipped silently if ipmitool unavailable
|
||||
fans, _ := sampleFanSpeeds()
|
||||
@@ -41,5 +52,91 @@ func SampleLiveMetrics() LiveMetricSample {
|
||||
// System power — returns 0 if unavailable
|
||||
s.PowerW = sampleSystemPower()
|
||||
|
||||
// CPU load — from /proc/stat
|
||||
s.CPULoadPct = sampleCPULoadPct()
|
||||
|
||||
// Memory load — from /proc/meminfo
|
||||
s.MemLoadPct = sampleMemLoadPct()
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// sampleCPULoadPct reads two /proc/stat snapshots 200ms apart and returns
|
||||
// the overall CPU utilisation percentage.
|
||||
var cpuStatPrev [2]uint64 // [total, idle]
|
||||
|
||||
func sampleCPULoadPct() float64 {
|
||||
total, idle := readCPUStat()
|
||||
if total == 0 {
|
||||
return 0
|
||||
}
|
||||
prevTotal, prevIdle := cpuStatPrev[0], cpuStatPrev[1]
|
||||
cpuStatPrev = [2]uint64{total, idle}
|
||||
if prevTotal == 0 {
|
||||
return 0
|
||||
}
|
||||
dt := float64(total - prevTotal)
|
||||
di := float64(idle - prevIdle)
|
||||
if dt <= 0 {
|
||||
return 0
|
||||
}
|
||||
pct := (1 - di/dt) * 100
|
||||
if pct < 0 {
|
||||
return 0
|
||||
}
|
||||
if pct > 100 {
|
||||
return 100
|
||||
}
|
||||
return pct
|
||||
}
|
||||
|
||||
func readCPUStat() (total, idle uint64) {
|
||||
f, err := os.Open("/proc/stat")
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
defer f.Close()
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
if !strings.HasPrefix(line, "cpu ") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)[1:] // skip "cpu"
|
||||
var vals [10]uint64
|
||||
for i := 0; i < len(fields) && i < 10; i++ {
|
||||
vals[i], _ = strconv.ParseUint(fields[i], 10, 64)
|
||||
}
|
||||
// idle = idle + iowait
|
||||
idle = vals[3] + vals[4]
|
||||
for _, v := range vals {
|
||||
total += v
|
||||
}
|
||||
return total, idle
|
||||
}
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
func sampleMemLoadPct() float64 {
|
||||
f, err := os.Open("/proc/meminfo")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
defer f.Close()
|
||||
vals := map[string]uint64{}
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
fields := strings.Fields(sc.Text())
|
||||
if len(fields) >= 2 {
|
||||
v, _ := strconv.ParseUint(fields[1], 10, 64)
|
||||
vals[strings.TrimSuffix(fields[0], ":")] = v
|
||||
}
|
||||
}
|
||||
total := vals["MemTotal"]
|
||||
avail := vals["MemAvailable"]
|
||||
if total == 0 {
|
||||
return 0
|
||||
}
|
||||
used := total - avail
|
||||
return float64(used) / float64(total) * 100
|
||||
}
|
||||
|
||||
@@ -131,6 +131,25 @@ func (s *System) SetStaticIPv4(cfg StaticIPv4Config) (string, error) {
|
||||
return out.String(), nil
|
||||
}
|
||||
|
||||
// SetInterfaceState brings a network interface up or down.
|
||||
func (s *System) SetInterfaceState(iface string, up bool) error {
|
||||
state := "down"
|
||||
if up {
|
||||
state = "up"
|
||||
}
|
||||
return exec.Command("ip", "link", "set", "dev", iface, state).Run()
|
||||
}
|
||||
|
||||
// GetInterfaceState returns true if the interface is UP.
|
||||
func (s *System) GetInterfaceState(iface string) (bool, error) {
|
||||
raw, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/operstate", iface))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
state := strings.TrimSpace(string(raw))
|
||||
return state == "up", nil
|
||||
}
|
||||
|
||||
func listInterfaceNames() ([]string, error) {
|
||||
raw, err := exec.Command("ip", "-o", "link", "show").Output()
|
||||
if err != nil {
|
||||
|
||||
@@ -2,6 +2,8 @@ package platform
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
@@ -13,6 +15,7 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -32,6 +35,40 @@ var (
|
||||
}
|
||||
)
|
||||
|
||||
// streamExecOutput runs cmd and streams each output line to logFunc (if non-nil).
|
||||
// Returns combined stdout+stderr as a byte slice.
|
||||
func streamExecOutput(cmd *exec.Cmd, logFunc func(string)) ([]byte, error) {
|
||||
pr, pw := io.Pipe()
|
||||
cmd.Stdout = pw
|
||||
cmd.Stderr = pw
|
||||
|
||||
var buf bytes.Buffer
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
scanner := bufio.NewScanner(pr)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
buf.WriteString(line + "\n")
|
||||
if logFunc != nil {
|
||||
logFunc(line)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
err := cmd.Start()
|
||||
if err != nil {
|
||||
_ = pw.Close()
|
||||
wg.Wait()
|
||||
return nil, err
|
||||
}
|
||||
waitErr := cmd.Wait()
|
||||
_ = pw.Close()
|
||||
wg.Wait()
|
||||
return buf.Bytes(), waitErr
|
||||
}
|
||||
|
||||
// NvidiaGPU holds basic GPU info from nvidia-smi.
|
||||
type NvidiaGPU struct {
|
||||
Index int
|
||||
@@ -80,13 +117,27 @@ func (s *System) ListAMDGPUs() ([]AMDGPUInfo, error) {
|
||||
}
|
||||
|
||||
// RunAMDAcceptancePack runs an AMD GPU diagnostic pack using rocm-smi.
|
||||
func (s *System) RunAMDAcceptancePack(baseDir string) (string, error) {
|
||||
func (s *System) RunAMDAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
||||
return runAcceptancePack(baseDir, "gpu-amd", []satJob{
|
||||
{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
|
||||
{name: "02-rocm-smi-showallinfo.log", cmd: []string{"rocm-smi", "--showallinfo"}},
|
||||
{name: "03-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
|
||||
{name: "04-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
|
||||
})
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
// RunAMDStressPack runs an AMD GPU burn-in pack.
|
||||
// Missing tools are reported as UNSUPPORTED, consistent with the existing SAT pattern.
|
||||
func (s *System) RunAMDStressPack(baseDir string, logFunc func(string)) (string, error) {
|
||||
seconds := envInt("BEE_AMD_STRESS_SECONDS", 300)
|
||||
return runAcceptancePack(baseDir, "gpu-amd-stress", []satJob{
|
||||
{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
|
||||
{name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}},
|
||||
{name: fmt.Sprintf("03-rocm-smi-monitor-%ds.log", seconds), cmd: []string{
|
||||
"rocm-smi", "--showtemp", "--showpower",
|
||||
fmt.Sprintf("--duration=%d", seconds),
|
||||
}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
// ListNvidiaGPUs returns GPUs visible to nvidia-smi.
|
||||
@@ -123,7 +174,7 @@ func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) {
|
||||
|
||||
// RunNCCLTests runs nccl-tests all_reduce_perf across all NVIDIA GPUs.
|
||||
// Measures collective communication bandwidth over NVLink/PCIe.
|
||||
func (s *System) RunNCCLTests(ctx context.Context, baseDir string) (string, error) {
|
||||
func (s *System) RunNCCLTests(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
// detect GPU count
|
||||
out, _ := exec.Command("nvidia-smi", "--query-gpu=index", "--format=csv,noheader").Output()
|
||||
gpuCount := len(strings.Split(strings.TrimSpace(string(out)), "\n"))
|
||||
@@ -136,32 +187,65 @@ func (s *System) RunNCCLTests(ctx context.Context, baseDir string) (string, erro
|
||||
"all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2",
|
||||
"-g", strconv.Itoa(gpuCount), "--iters", "20",
|
||||
}},
|
||||
})
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunNvidiaAcceptancePack(baseDir string) (string, error) {
|
||||
return runAcceptancePack(baseDir, "gpu-nvidia", nvidiaSATJobs())
|
||||
func (s *System) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
||||
return runAcceptancePack(baseDir, "gpu-nvidia", nvidiaSATJobs(), logFunc)
|
||||
}
|
||||
|
||||
// RunNvidiaAcceptancePackWithOptions runs the NVIDIA diagnostics via DCGM.
|
||||
// diagLevel: 1=quick, 2=medium, 3=targeted stress, 4=extended stress.
|
||||
// gpuIndices: specific GPU indices to test (empty = all GPUs).
|
||||
// ctx cancellation kills the running job.
|
||||
func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int) (string, error) {
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia", nvidiaDCGMJobs(diagLevel, gpuIndices))
|
||||
func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia", nvidiaDCGMJobs(diagLevel, gpuIndices), logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunMemoryAcceptancePack(baseDir string) (string, error) {
|
||||
func (s *System) RunMemoryAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
||||
sizeMB := envInt("BEE_MEMTESTER_SIZE_MB", 128)
|
||||
passes := envInt("BEE_MEMTESTER_PASSES", 1)
|
||||
return runAcceptancePack(baseDir, "memory", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-memtester.log", cmd: []string{"memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
})
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunCPUAcceptancePack(baseDir string, durationSec int) (string, error) {
|
||||
func (s *System) RunMemoryStressPack(baseDir string, logFunc func(string)) (string, error) {
|
||||
seconds := envInt("BEE_VM_STRESS_SECONDS", 300)
|
||||
// Use 80% of RAM by default; override with BEE_VM_STRESS_SIZE_MB.
|
||||
sizeArg := "80%"
|
||||
if mb := envInt("BEE_VM_STRESS_SIZE_MB", 0); mb > 0 {
|
||||
sizeArg = fmt.Sprintf("%dM", mb)
|
||||
}
|
||||
return runAcceptancePack(baseDir, "memory-stress", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-stress-ng-vm.log", cmd: []string{
|
||||
"stress-ng", "--vm", "1",
|
||||
"--vm-bytes", sizeArg,
|
||||
"--vm-method", "all",
|
||||
"--timeout", fmt.Sprintf("%d", seconds),
|
||||
"--metrics-brief",
|
||||
}},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunSATStressPack(baseDir string, logFunc func(string)) (string, error) {
|
||||
seconds := envInt("BEE_SAT_STRESS_SECONDS", 300)
|
||||
cmd := []string{"stressapptest", "-s", fmt.Sprintf("%d", seconds), "-W", "--cc_test"}
|
||||
if mb := envInt("BEE_SAT_STRESS_MB", 0); mb > 0 {
|
||||
cmd = append(cmd, "-M", fmt.Sprintf("%d", mb))
|
||||
}
|
||||
return runAcceptancePack(baseDir, "sat-stress", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-stressapptest.log", cmd: cmd},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunCPUAcceptancePack(baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
if durationSec <= 0 {
|
||||
durationSec = 60
|
||||
}
|
||||
@@ -170,10 +254,10 @@ func (s *System) RunCPUAcceptancePack(baseDir string, durationSec int) (string,
|
||||
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
|
||||
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}},
|
||||
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
|
||||
})
|
||||
}, logFunc)
|
||||
}
|
||||
|
||||
func (s *System) RunStorageAcceptancePack(baseDir string) (string, error) {
|
||||
func (s *System) RunStorageAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
@@ -205,7 +289,7 @@ func (s *System) RunStorageAcceptancePack(baseDir string) (string, error) {
|
||||
commands := storageSATCommands(devPath)
|
||||
for cmdIndex, job := range commands {
|
||||
name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
|
||||
out, err := runSATCommand(verboseLog, job.name, job.cmd)
|
||||
out, err := runSATCommand(verboseLog, job.name, job.cmd, logFunc)
|
||||
if writeErr := os.WriteFile(filepath.Join(runDir, name), out, 0644); writeErr != nil {
|
||||
return "", writeErr
|
||||
}
|
||||
@@ -254,7 +338,7 @@ func nvidiaSATJobs() []satJob {
|
||||
}
|
||||
}
|
||||
|
||||
func runAcceptancePack(baseDir, prefix string, jobs []satJob) (string, error) {
|
||||
func runAcceptancePack(baseDir, prefix string, jobs []satJob, logFunc func(string)) (string, error) {
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
@@ -269,11 +353,13 @@ func runAcceptancePack(baseDir, prefix string, jobs []satJob) (string, error) {
|
||||
stats := satStats{}
|
||||
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
||||
for _, job := range jobs {
|
||||
var out []byte
|
||||
var err error
|
||||
cmd := make([]string, 0, len(job.cmd))
|
||||
for _, arg := range job.cmd {
|
||||
cmd = append(cmd, strings.ReplaceAll(arg, "{{run_dir}}", runDir))
|
||||
}
|
||||
out, err := runSATCommand(verboseLog, job.name, cmd)
|
||||
out, err = runSATCommand(verboseLog, job.name, cmd, logFunc)
|
||||
if writeErr := os.WriteFile(filepath.Join(runDir, job.name), out, 0644); writeErr != nil {
|
||||
return "", writeErr
|
||||
}
|
||||
@@ -315,7 +401,7 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob {
|
||||
}
|
||||
}
|
||||
|
||||
func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []satJob) (string, error) {
|
||||
func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []satJob, logFunc func(string)) (string, error) {
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
@@ -342,9 +428,9 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
|
||||
var err error
|
||||
|
||||
if job.collectGPU {
|
||||
out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir)
|
||||
out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir, logFunc)
|
||||
} else {
|
||||
out, err = runSATCommandCtx(ctx, verboseLog, job.name, cmd, job.env)
|
||||
out, err = runSATCommandCtx(ctx, verboseLog, job.name, cmd, job.env, logFunc)
|
||||
}
|
||||
|
||||
if writeErr := os.WriteFile(filepath.Join(runDir, job.name), out, 0644); writeErr != nil {
|
||||
@@ -368,13 +454,16 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
|
||||
return archive, nil
|
||||
}
|
||||
|
||||
func runSATCommandCtx(ctx context.Context, verboseLog, name string, cmd []string, env []string) ([]byte, error) {
|
||||
func runSATCommandCtx(ctx context.Context, verboseLog, name string, cmd []string, env []string, logFunc func(string)) ([]byte, error) {
|
||||
start := time.Now().UTC()
|
||||
resolvedCmd, err := resolveSATCommand(cmd)
|
||||
appendSATVerboseLog(verboseLog,
|
||||
fmt.Sprintf("[%s] start %s", start.Format(time.RFC3339), name),
|
||||
"cmd: "+strings.Join(resolvedCmd, " "),
|
||||
)
|
||||
if logFunc != nil {
|
||||
logFunc(fmt.Sprintf("=== %s ===", name))
|
||||
}
|
||||
if err != nil {
|
||||
appendSATVerboseLog(verboseLog,
|
||||
fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
|
||||
@@ -389,7 +478,7 @@ func runSATCommandCtx(ctx context.Context, verboseLog, name string, cmd []string
|
||||
if len(env) > 0 {
|
||||
c.Env = append(os.Environ(), env...)
|
||||
}
|
||||
out, err := c.CombinedOutput()
|
||||
out, err := streamExecOutput(c, logFunc)
|
||||
|
||||
rc := 0
|
||||
if err != nil {
|
||||
@@ -464,6 +553,11 @@ func classifySATResult(name string, out []byte, err error) (string, int) {
|
||||
}
|
||||
|
||||
text := strings.ToLower(string(out))
|
||||
// No output at all means the tool failed to start (mlock limit, binary missing,
|
||||
// etc.) — we cannot say anything about hardware health → UNSUPPORTED.
|
||||
if len(strings.TrimSpace(text)) == 0 {
|
||||
return "UNSUPPORTED", rc
|
||||
}
|
||||
if strings.Contains(text, "unsupported") ||
|
||||
strings.Contains(text, "not supported") ||
|
||||
strings.Contains(text, "invalid opcode") ||
|
||||
@@ -472,19 +566,25 @@ func classifySATResult(name string, out []byte, err error) (string, int) {
|
||||
strings.Contains(text, "not available") ||
|
||||
strings.Contains(text, "cuda_error_system_not_ready") ||
|
||||
strings.Contains(text, "no such device") ||
|
||||
// nvidia-smi on a machine with no NVIDIA GPU
|
||||
strings.Contains(text, "couldn't communicate with the nvidia driver") ||
|
||||
strings.Contains(text, "no nvidia gpu") ||
|
||||
(strings.Contains(name, "self-test") && strings.Contains(text, "aborted")) {
|
||||
return "UNSUPPORTED", rc
|
||||
}
|
||||
return "FAILED", rc
|
||||
}
|
||||
|
||||
func runSATCommand(verboseLog, name string, cmd []string) ([]byte, error) {
|
||||
func runSATCommand(verboseLog, name string, cmd []string, logFunc func(string)) ([]byte, error) {
|
||||
start := time.Now().UTC()
|
||||
resolvedCmd, err := resolveSATCommand(cmd)
|
||||
appendSATVerboseLog(verboseLog,
|
||||
fmt.Sprintf("[%s] start %s", start.Format(time.RFC3339), name),
|
||||
"cmd: "+strings.Join(resolvedCmd, " "),
|
||||
)
|
||||
if logFunc != nil {
|
||||
logFunc(fmt.Sprintf("=== %s ===", name))
|
||||
}
|
||||
if err != nil {
|
||||
appendSATVerboseLog(verboseLog,
|
||||
fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
|
||||
@@ -495,7 +595,7 @@ func runSATCommand(verboseLog, name string, cmd []string) ([]byte, error) {
|
||||
return []byte(err.Error() + "\n"), err
|
||||
}
|
||||
|
||||
out, err := satExecCommand(resolvedCmd[0], resolvedCmd[1:]...).CombinedOutput()
|
||||
out, err := streamExecOutput(satExecCommand(resolvedCmd[0], resolvedCmd[1:]...), logFunc)
|
||||
|
||||
rc := 0
|
||||
if err != nil {
|
||||
@@ -597,7 +697,7 @@ func parseStorageDevices(raw string) []string {
|
||||
|
||||
// runSATCommandWithMetrics runs a command while collecting GPU metrics in the background.
|
||||
// On completion it writes gpu-metrics.csv and gpu-metrics.html into runDir.
|
||||
func runSATCommandWithMetrics(ctx context.Context, verboseLog, name string, cmd []string, env []string, gpuIndices []int, runDir string) ([]byte, error) {
|
||||
func runSATCommandWithMetrics(ctx context.Context, verboseLog, name string, cmd []string, env []string, gpuIndices []int, runDir string, logFunc func(string)) ([]byte, error) {
|
||||
stopCh := make(chan struct{})
|
||||
doneCh := make(chan struct{})
|
||||
var metricRows []GPUMetricRow
|
||||
@@ -625,7 +725,7 @@ func runSATCommandWithMetrics(ctx context.Context, verboseLog, name string, cmd
|
||||
}
|
||||
}()
|
||||
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, name, cmd, env)
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, name, cmd, env, logFunc)
|
||||
|
||||
close(stopCh)
|
||||
<-doneCh
|
||||
|
||||
@@ -147,7 +147,7 @@ func (s *System) RunFanStressTest(ctx context.Context, baseDir string, opts FanS
|
||||
"--seconds", strconv.Itoa(durSec),
|
||||
"--size-mb", strconv.Itoa(opts.SizeMB),
|
||||
}
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, stepName, cmd, env)
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, stepName, cmd, env, nil)
|
||||
_ = os.WriteFile(filepath.Join(runDir, stepName+".log"), out, 0644)
|
||||
if err != nil && err != context.Canceled && err.Error() != "signal: killed" {
|
||||
fmt.Fprintf(&summary, "%s_status=FAILED\n", stepName)
|
||||
|
||||
@@ -17,6 +17,10 @@ func (s *System) ListBeeServices() ([]string, error) {
|
||||
}
|
||||
for _, match := range matches {
|
||||
name := strings.TrimSuffix(filepath.Base(match), ".service")
|
||||
// Skip template units (e.g. bee-journal-mirror@) — they have no instances to query.
|
||||
if strings.HasSuffix(name, "@") {
|
||||
continue
|
||||
}
|
||||
if !seen[name] {
|
||||
seen[name] = true
|
||||
out = append(out, name)
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -110,39 +109,37 @@ func runCmdJob(j *jobState, cmd *exec.Cmd) {
|
||||
|
||||
// ── Audit ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *handler) handleAPIAuditRun(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *handler) handleAPIAuditRun(w http.ResponseWriter, _ *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
id := newJobID("audit")
|
||||
j := globalJobs.create(id)
|
||||
go func() {
|
||||
j.append("Running audit...")
|
||||
result, err := h.opts.App.RunAuditNow(h.opts.RuntimeMode)
|
||||
if err != nil {
|
||||
j.append("ERROR: " + err.Error())
|
||||
j.finish(err.Error())
|
||||
return
|
||||
}
|
||||
for _, line := range strings.Split(result.Body, "\n") {
|
||||
if line != "" {
|
||||
j.append(line)
|
||||
}
|
||||
}
|
||||
j.finish("")
|
||||
}()
|
||||
writeJSON(w, map[string]string{"job_id": id})
|
||||
t := &Task{
|
||||
ID: newJobID("audit"),
|
||||
Name: "Audit",
|
||||
Target: "audit",
|
||||
Status: TaskPending,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
globalQueue.enqueue(t)
|
||||
writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPIAuditStream(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.URL.Query().Get("job_id")
|
||||
j, ok := globalJobs.get(id)
|
||||
if !ok {
|
||||
http.Error(w, "job not found", http.StatusNotFound)
|
||||
if id == "" {
|
||||
id = r.URL.Query().Get("task_id")
|
||||
}
|
||||
// Try task queue first, then legacy job manager
|
||||
if j, ok := globalQueue.findJob(id); ok {
|
||||
streamJob(w, r, j)
|
||||
return
|
||||
}
|
||||
streamJob(w, r, j)
|
||||
if j, ok := globalJobs.get(id); ok {
|
||||
streamJob(w, r, j)
|
||||
return
|
||||
}
|
||||
http.Error(w, "job not found", http.StatusNotFound)
|
||||
}
|
||||
|
||||
// ── SAT ───────────────────────────────────────────────────────────────────────
|
||||
@@ -153,74 +150,87 @@ func (h *handler) handleAPISATRun(target string) http.HandlerFunc {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
id := newJobID("sat-" + target)
|
||||
j := globalJobs.create(id)
|
||||
|
||||
go func() {
|
||||
j.append(fmt.Sprintf("Starting %s acceptance test...", target))
|
||||
var (
|
||||
archive string
|
||||
err error
|
||||
)
|
||||
var body struct {
|
||||
Duration int `json:"duration"`
|
||||
DiagLevel int `json:"diag_level"`
|
||||
GPUIndices []int `json:"gpu_indices"`
|
||||
}
|
||||
body.DiagLevel = 1
|
||||
if r.ContentLength > 0 {
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
}
|
||||
|
||||
// Parse optional parameters
|
||||
var body struct {
|
||||
Duration int `json:"duration"`
|
||||
DiagLevel int `json:"diag_level"`
|
||||
GPUIndices []int `json:"gpu_indices"`
|
||||
}
|
||||
body.DiagLevel = 1
|
||||
if r.ContentLength > 0 {
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
}
|
||||
|
||||
switch target {
|
||||
case "nvidia":
|
||||
if len(body.GPUIndices) > 0 || body.DiagLevel > 0 {
|
||||
result, e := h.opts.App.RunNvidiaAcceptancePackWithOptions(
|
||||
context.Background(), "", body.DiagLevel, body.GPUIndices,
|
||||
)
|
||||
if e != nil {
|
||||
err = e
|
||||
} else {
|
||||
archive = result.Body
|
||||
}
|
||||
} else {
|
||||
archive, err = h.opts.App.RunNvidiaAcceptancePack("")
|
||||
}
|
||||
case "memory":
|
||||
archive, err = h.opts.App.RunMemoryAcceptancePack("")
|
||||
case "storage":
|
||||
archive, err = h.opts.App.RunStorageAcceptancePack("")
|
||||
case "cpu":
|
||||
dur := body.Duration
|
||||
if dur <= 0 {
|
||||
dur = 60
|
||||
}
|
||||
archive, err = h.opts.App.RunCPUAcceptancePack("", dur)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
j.append("ERROR: " + err.Error())
|
||||
j.finish(err.Error())
|
||||
return
|
||||
}
|
||||
j.append(fmt.Sprintf("Archive written: %s", archive))
|
||||
j.finish("")
|
||||
}()
|
||||
|
||||
writeJSON(w, map[string]string{"job_id": id})
|
||||
name := taskNames[target]
|
||||
if name == "" {
|
||||
name = target
|
||||
}
|
||||
t := &Task{
|
||||
ID: newJobID("sat-" + target),
|
||||
Name: name,
|
||||
Target: target,
|
||||
Status: TaskPending,
|
||||
CreatedAt: time.Now(),
|
||||
params: taskParams{
|
||||
Duration: body.Duration,
|
||||
DiagLevel: body.DiagLevel,
|
||||
GPUIndices: body.GPUIndices,
|
||||
},
|
||||
}
|
||||
globalQueue.enqueue(t)
|
||||
writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID})
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) handleAPISATStream(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.URL.Query().Get("job_id")
|
||||
j, ok := globalJobs.get(id)
|
||||
if !ok {
|
||||
http.Error(w, "job not found", http.StatusNotFound)
|
||||
if id == "" {
|
||||
id = r.URL.Query().Get("task_id")
|
||||
}
|
||||
if j, ok := globalQueue.findJob(id); ok {
|
||||
streamJob(w, r, j)
|
||||
return
|
||||
}
|
||||
streamJob(w, r, j)
|
||||
if j, ok := globalJobs.get(id); ok {
|
||||
streamJob(w, r, j)
|
||||
return
|
||||
}
|
||||
http.Error(w, "job not found", http.StatusNotFound)
|
||||
}
|
||||
|
||||
func (h *handler) handleAPISATAbort(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.URL.Query().Get("job_id")
|
||||
if id == "" {
|
||||
id = r.URL.Query().Get("task_id")
|
||||
}
|
||||
if t, ok := globalQueue.findByID(id); ok {
|
||||
globalQueue.mu.Lock()
|
||||
switch t.Status {
|
||||
case TaskPending:
|
||||
t.Status = TaskCancelled
|
||||
now := time.Now()
|
||||
t.DoneAt = &now
|
||||
case TaskRunning:
|
||||
if t.job != nil {
|
||||
t.job.abort()
|
||||
}
|
||||
t.Status = TaskCancelled
|
||||
now := time.Now()
|
||||
t.DoneAt = &now
|
||||
}
|
||||
globalQueue.mu.Unlock()
|
||||
writeJSON(w, map[string]string{"status": "aborted"})
|
||||
return
|
||||
}
|
||||
if j, ok := globalJobs.get(id); ok {
|
||||
if j.abort() {
|
||||
writeJSON(w, map[string]string{"status": "aborted"})
|
||||
} else {
|
||||
writeJSON(w, map[string]string{"status": "not_running"})
|
||||
}
|
||||
return
|
||||
}
|
||||
http.Error(w, "job not found", http.StatusNotFound)
|
||||
}
|
||||
|
||||
// ── Services ──────────────────────────────────────────────────────────────────
|
||||
@@ -236,13 +246,15 @@ func (h *handler) handleAPIServicesList(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
type serviceInfo struct {
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
result := make([]serviceInfo, 0, len(names))
|
||||
for _, name := range names {
|
||||
status, _ := h.opts.App.ServiceStatus(name)
|
||||
result = append(result, serviceInfo{Name: name, Status: status})
|
||||
state := h.opts.App.ServiceState(name)
|
||||
body, _ := h.opts.App.ServiceStatus(name)
|
||||
result = append(result, serviceInfo{Name: name, State: state, Body: body})
|
||||
}
|
||||
writeJSON(w, result)
|
||||
}
|
||||
@@ -377,6 +389,51 @@ func (h *handler) handleAPIExportBundle(w http.ResponseWriter, r *http.Request)
|
||||
})
|
||||
}
|
||||
|
||||
// ── GPU presence ──────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *handler) handleAPIGPUPresence(w http.ResponseWriter, r *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
gp := h.opts.App.DetectGPUPresence()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]bool{
|
||||
"nvidia": gp.Nvidia,
|
||||
"amd": gp.AMD,
|
||||
})
|
||||
}
|
||||
|
||||
// ── System ────────────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *handler) handleAPIRAMStatus(w http.ResponseWriter, r *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
inRAM := h.opts.App.IsLiveMediaInRAM()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]bool{"in_ram": inRAM})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPIInstallToRAM(w http.ResponseWriter, r *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
t := &Task{
|
||||
ID: newJobID("install-to-ram"),
|
||||
Name: "Install to RAM",
|
||||
Target: "install-to-ram",
|
||||
Priority: 10,
|
||||
Status: TaskPending,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
globalQueue.enqueue(t)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"task_id": t.ID})
|
||||
}
|
||||
|
||||
// ── Tools ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
var standardTools = []string{
|
||||
@@ -407,6 +464,101 @@ func (h *handler) handleAPIPreflight(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write(data)
|
||||
}
|
||||
|
||||
// ── Install ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *handler) handleAPIInstallDisks(w http.ResponseWriter, r *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
disks, err := h.opts.App.ListInstallDisks()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
type diskJSON struct {
|
||||
Device string `json:"device"`
|
||||
Model string `json:"model"`
|
||||
Size string `json:"size"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
MountedParts []string `json:"mounted_parts"`
|
||||
Warnings []string `json:"warnings"`
|
||||
}
|
||||
result := make([]diskJSON, 0, len(disks))
|
||||
for _, d := range disks {
|
||||
result = append(result, diskJSON{
|
||||
Device: d.Device,
|
||||
Model: d.Model,
|
||||
Size: d.Size,
|
||||
SizeBytes: d.SizeBytes,
|
||||
MountedParts: d.MountedParts,
|
||||
Warnings: platform.DiskWarnings(d),
|
||||
})
|
||||
}
|
||||
writeJSON(w, result)
|
||||
}
|
||||
|
||||
func (h *handler) handleAPIInstallRun(w http.ResponseWriter, r *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Device string `json:"device"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Device == "" {
|
||||
writeError(w, http.StatusBadRequest, "device is required")
|
||||
return
|
||||
}
|
||||
|
||||
// Whitelist: only allow devices that ListInstallDisks() returns.
|
||||
disks, err := h.opts.App.ListInstallDisks()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
allowed := false
|
||||
for _, d := range disks {
|
||||
if d.Device == req.Device {
|
||||
allowed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allowed {
|
||||
writeError(w, http.StatusBadRequest, "device not in install candidate list")
|
||||
return
|
||||
}
|
||||
|
||||
h.installMu.Lock()
|
||||
if h.installJob != nil && !h.installJob.isDone() {
|
||||
h.installMu.Unlock()
|
||||
writeError(w, http.StatusConflict, "install already running")
|
||||
return
|
||||
}
|
||||
j := &jobState{}
|
||||
h.installJob = j
|
||||
h.installMu.Unlock()
|
||||
|
||||
logFile := platform.InstallLogPath(req.Device)
|
||||
go runCmdJob(j, exec.CommandContext(context.Background(), "bee-install", req.Device, logFile))
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *handler) handleAPIInstallStream(w http.ResponseWriter, r *http.Request) {
|
||||
h.installMu.Lock()
|
||||
j := h.installJob
|
||||
h.installMu.Unlock()
|
||||
if j == nil {
|
||||
if !sseStart(w) {
|
||||
return
|
||||
}
|
||||
sseWrite(w, "done", "")
|
||||
return
|
||||
}
|
||||
streamJob(w, r, j)
|
||||
}
|
||||
|
||||
// ── Metrics SSE ───────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *handler) handleAPIMetricsStream(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -421,6 +573,45 @@ func (h *handler) handleAPIMetricsStream(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
case <-ticker.C:
|
||||
sample := platform.SampleLiveMetrics()
|
||||
|
||||
// Feed server ring buffers
|
||||
for _, t := range sample.Temps {
|
||||
if t.Name == "CPU" {
|
||||
h.ringCPUTemp.push(t.Celsius)
|
||||
break
|
||||
}
|
||||
}
|
||||
h.ringPower.push(sample.PowerW)
|
||||
h.ringCPULoad.push(sample.CPULoadPct)
|
||||
h.ringMemLoad.push(sample.MemLoadPct)
|
||||
|
||||
// Feed fan ring buffers (grow on first sight)
|
||||
h.ringsMu.Lock()
|
||||
for i, fan := range sample.Fans {
|
||||
for len(h.ringFans) <= i {
|
||||
h.ringFans = append(h.ringFans, newMetricsRing(120))
|
||||
h.fanNames = append(h.fanNames, fan.Name)
|
||||
}
|
||||
h.ringFans[i].push(float64(fan.RPM))
|
||||
}
|
||||
// Feed per-GPU ring buffers (grow on first sight)
|
||||
for _, gpu := range sample.GPUs {
|
||||
idx := gpu.GPUIndex
|
||||
for len(h.gpuRings) <= idx {
|
||||
h.gpuRings = append(h.gpuRings, &gpuRings{
|
||||
Temp: newMetricsRing(120),
|
||||
Util: newMetricsRing(120),
|
||||
MemUtil: newMetricsRing(120),
|
||||
Power: newMetricsRing(120),
|
||||
})
|
||||
}
|
||||
h.gpuRings[idx].Temp.push(gpu.TempC)
|
||||
h.gpuRings[idx].Util.push(gpu.UsagePct)
|
||||
h.gpuRings[idx].MemUtil.push(gpu.MemUsagePct)
|
||||
h.gpuRings[idx].Power.push(gpu.PowerW)
|
||||
}
|
||||
h.ringsMu.Unlock()
|
||||
|
||||
b, err := json.Marshal(sample)
|
||||
if err != nil {
|
||||
continue
|
||||
@@ -431,3 +622,95 @@ func (h *handler) handleAPIMetricsStream(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Network toggle ────────────────────────────────────────────────────────────
|
||||
|
||||
const netRollbackTimeout = 60 * time.Second
|
||||
|
||||
func (h *handler) handleAPINetworkToggle(w http.ResponseWriter, r *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Iface string `json:"iface"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Iface == "" {
|
||||
writeError(w, http.StatusBadRequest, "iface is required")
|
||||
return
|
||||
}
|
||||
|
||||
wasUp, err := h.opts.App.GetInterfaceState(req.Iface)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.opts.App.SetInterfaceState(req.Iface, !wasUp); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Cancel any existing pending change (rollback it first).
|
||||
h.pendingNetMu.Lock()
|
||||
if h.pendingNet != nil {
|
||||
prev := h.pendingNet
|
||||
prev.mu.Lock()
|
||||
prev.timer.Stop()
|
||||
_ = h.opts.App.SetInterfaceState(prev.iface, prev.wasUp)
|
||||
prev.mu.Unlock()
|
||||
}
|
||||
|
||||
pnc := &pendingNetChange{iface: req.Iface, wasUp: wasUp}
|
||||
pnc.timer = time.AfterFunc(netRollbackTimeout, func() {
|
||||
_ = h.opts.App.SetInterfaceState(req.Iface, wasUp)
|
||||
h.pendingNetMu.Lock()
|
||||
if h.pendingNet == pnc {
|
||||
h.pendingNet = nil
|
||||
}
|
||||
h.pendingNetMu.Unlock()
|
||||
})
|
||||
h.pendingNet = pnc
|
||||
h.pendingNetMu.Unlock()
|
||||
|
||||
newState := "up"
|
||||
if wasUp {
|
||||
newState = "down"
|
||||
}
|
||||
writeJSON(w, map[string]any{
|
||||
"iface": req.Iface,
|
||||
"new_state": newState,
|
||||
"rollback_in": int(netRollbackTimeout.Seconds()),
|
||||
})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPINetworkConfirm(w http.ResponseWriter, _ *http.Request) {
|
||||
h.pendingNetMu.Lock()
|
||||
pnc := h.pendingNet
|
||||
h.pendingNet = nil
|
||||
h.pendingNetMu.Unlock()
|
||||
if pnc != nil {
|
||||
pnc.mu.Lock()
|
||||
pnc.timer.Stop()
|
||||
pnc.mu.Unlock()
|
||||
}
|
||||
writeJSON(w, map[string]string{"status": "confirmed"})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPINetworkRollback(w http.ResponseWriter, _ *http.Request) {
|
||||
h.pendingNetMu.Lock()
|
||||
pnc := h.pendingNet
|
||||
h.pendingNet = nil
|
||||
h.pendingNetMu.Unlock()
|
||||
if pnc == nil {
|
||||
writeError(w, http.StatusConflict, "no pending network change")
|
||||
return
|
||||
}
|
||||
pnc.mu.Lock()
|
||||
pnc.timer.Stop()
|
||||
pnc.mu.Unlock()
|
||||
if h.opts.App != nil {
|
||||
_ = h.opts.App.SetInterfaceState(pnc.iface, pnc.wasUp)
|
||||
}
|
||||
writeJSON(w, map[string]string{"status": "rolled back"})
|
||||
}
|
||||
|
||||
@@ -7,12 +7,23 @@ import (
|
||||
|
||||
// jobState holds the output lines and completion status of an async job.
|
||||
type jobState struct {
|
||||
lines []string
|
||||
done bool
|
||||
err string
|
||||
mu sync.Mutex
|
||||
// subs is a list of channels that receive new lines as they arrive.
|
||||
subs []chan string
|
||||
lines []string
|
||||
done bool
|
||||
err string
|
||||
mu sync.Mutex
|
||||
subs []chan string
|
||||
cancel func() // optional cancel function; nil if job is not cancellable
|
||||
}
|
||||
|
||||
// abort cancels the job if it has a cancel function and is not yet done.
|
||||
func (j *jobState) abort() bool {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
if j.done || j.cancel == nil {
|
||||
return false
|
||||
}
|
||||
j.cancel()
|
||||
return true
|
||||
}
|
||||
|
||||
func (j *jobState) append(line string) {
|
||||
@@ -76,6 +87,13 @@ func (m *jobManager) create(id string) *jobState {
|
||||
return j
|
||||
}
|
||||
|
||||
// isDone returns true if the job has finished (either successfully or with error).
|
||||
func (j *jobState) isDone() bool {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return j.done
|
||||
}
|
||||
|
||||
func (m *jobManager) get(id string) (*jobState, bool) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,17 +4,34 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"bee/audit/internal/app"
|
||||
"bee/audit/internal/runtimeenv"
|
||||
gocharts "github.com/go-analyze/charts"
|
||||
"reanimator/chart/viewer"
|
||||
"reanimator/chart/web"
|
||||
)
|
||||
|
||||
const defaultTitle = "Bee Hardware Audit"
|
||||
|
||||
func init() {
|
||||
// On some LiveCD ramdisk environments, /usr/share/mime/globs2 exists but
|
||||
// causes an I/O error mid-read. Go's mime package panics (not errors) in
|
||||
// that case, crashing the first HTTP goroutine that serves a static file.
|
||||
// Pre-trigger initialization here with recover so subsequent calls are safe.
|
||||
func() {
|
||||
defer func() { recover() }() //nolint:errcheck
|
||||
mime.TypeByExtension(".gz")
|
||||
}()
|
||||
}
|
||||
|
||||
// HandlerOptions configures the web UI handler.
|
||||
type HandlerOptions struct {
|
||||
Title string
|
||||
@@ -24,10 +41,95 @@ type HandlerOptions struct {
|
||||
RuntimeMode runtimeenv.Mode
|
||||
}
|
||||
|
||||
// metricsRing holds a rolling window of live metric samples.
|
||||
type metricsRing struct {
|
||||
mu sync.Mutex
|
||||
vals []float64
|
||||
times []time.Time
|
||||
size int
|
||||
}
|
||||
|
||||
func newMetricsRing(size int) *metricsRing {
|
||||
return &metricsRing{size: size, vals: make([]float64, 0, size), times: make([]time.Time, 0, size)}
|
||||
}
|
||||
|
||||
func (r *metricsRing) push(v float64) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if len(r.vals) >= r.size {
|
||||
r.vals = r.vals[1:]
|
||||
r.times = r.times[1:]
|
||||
}
|
||||
r.vals = append(r.vals, v)
|
||||
r.times = append(r.times, time.Now())
|
||||
}
|
||||
|
||||
func (r *metricsRing) snapshot() ([]float64, []string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
v := make([]float64, len(r.vals))
|
||||
copy(v, r.vals)
|
||||
now := time.Now()
|
||||
labels := make([]string, len(r.times))
|
||||
for i, t := range r.times {
|
||||
labels[i] = relAgeLabel(now.Sub(t))
|
||||
}
|
||||
return v, labels
|
||||
}
|
||||
|
||||
func relAgeLabel(age time.Duration) string {
|
||||
if age <= 0 {
|
||||
return "0"
|
||||
}
|
||||
if age < time.Hour {
|
||||
m := int(age.Minutes())
|
||||
if m == 0 {
|
||||
return "-<1m"
|
||||
}
|
||||
return fmt.Sprintf("-%dm", m)
|
||||
}
|
||||
if age < 24*time.Hour {
|
||||
return fmt.Sprintf("-%dh", int(age.Hours()))
|
||||
}
|
||||
return fmt.Sprintf("-%dd", int(age.Hours()/24))
|
||||
}
|
||||
|
||||
// gpuRings holds per-GPU ring buffers.
|
||||
type gpuRings struct {
|
||||
Temp *metricsRing
|
||||
Util *metricsRing
|
||||
MemUtil *metricsRing
|
||||
Power *metricsRing
|
||||
}
|
||||
|
||||
// pendingNetChange tracks a network state change awaiting confirmation.
|
||||
type pendingNetChange struct {
|
||||
iface string
|
||||
wasUp bool
|
||||
timer *time.Timer
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// handler is the HTTP handler for the web UI.
|
||||
type handler struct {
|
||||
opts HandlerOptions
|
||||
mux *http.ServeMux
|
||||
opts HandlerOptions
|
||||
mux *http.ServeMux
|
||||
// server rings
|
||||
ringCPUTemp *metricsRing
|
||||
ringCPULoad *metricsRing
|
||||
ringMemLoad *metricsRing
|
||||
ringPower *metricsRing
|
||||
ringFans []*metricsRing
|
||||
fanNames []string
|
||||
// per-GPU rings (index = GPU index)
|
||||
gpuRings []*gpuRings
|
||||
ringsMu sync.Mutex
|
||||
// install job (at most one at a time)
|
||||
installJob *jobState
|
||||
installMu sync.Mutex
|
||||
// pending network change (rollback on timeout)
|
||||
pendingNet *pendingNetChange
|
||||
pendingNetMu sync.Mutex
|
||||
}
|
||||
|
||||
// NewHandler creates the HTTP mux with all routes.
|
||||
@@ -42,7 +144,14 @@ func NewHandler(opts HandlerOptions) http.Handler {
|
||||
opts.RuntimeMode = runtimeenv.ModeAuto
|
||||
}
|
||||
|
||||
h := &handler{opts: opts}
|
||||
h := &handler{
|
||||
opts: opts,
|
||||
ringCPUTemp: newMetricsRing(120),
|
||||
ringCPULoad: newMetricsRing(120),
|
||||
ringMemLoad: newMetricsRing(120),
|
||||
ringPower: newMetricsRing(120),
|
||||
}
|
||||
globalQueue.startWorker(&opts)
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// ── Infrastructure ──────────────────────────────────────────────────────
|
||||
@@ -66,7 +175,19 @@ func NewHandler(opts HandlerOptions) http.Handler {
|
||||
mux.HandleFunc("POST /api/sat/memory/run", h.handleAPISATRun("memory"))
|
||||
mux.HandleFunc("POST /api/sat/storage/run", h.handleAPISATRun("storage"))
|
||||
mux.HandleFunc("POST /api/sat/cpu/run", h.handleAPISATRun("cpu"))
|
||||
mux.HandleFunc("POST /api/sat/amd/run", h.handleAPISATRun("amd"))
|
||||
mux.HandleFunc("POST /api/sat/amd-stress/run", h.handleAPISATRun("amd-stress"))
|
||||
mux.HandleFunc("POST /api/sat/memory-stress/run", h.handleAPISATRun("memory-stress"))
|
||||
mux.HandleFunc("POST /api/sat/sat-stress/run", h.handleAPISATRun("sat-stress"))
|
||||
mux.HandleFunc("GET /api/sat/stream", h.handleAPISATStream)
|
||||
mux.HandleFunc("POST /api/sat/abort", h.handleAPISATAbort)
|
||||
|
||||
// Tasks
|
||||
mux.HandleFunc("GET /api/tasks", h.handleAPITasksList)
|
||||
mux.HandleFunc("POST /api/tasks/cancel-all", h.handleAPITasksCancelAll)
|
||||
mux.HandleFunc("POST /api/tasks/{id}/cancel", h.handleAPITasksCancel)
|
||||
mux.HandleFunc("POST /api/tasks/{id}/priority", h.handleAPITasksPriority)
|
||||
mux.HandleFunc("GET /api/tasks/{id}/stream", h.handleAPITasksStream)
|
||||
|
||||
// Services
|
||||
mux.HandleFunc("GET /api/services", h.handleAPIServicesList)
|
||||
@@ -76,6 +197,9 @@ func NewHandler(opts HandlerOptions) http.Handler {
|
||||
mux.HandleFunc("GET /api/network", h.handleAPINetworkStatus)
|
||||
mux.HandleFunc("POST /api/network/dhcp", h.handleAPINetworkDHCP)
|
||||
mux.HandleFunc("POST /api/network/static", h.handleAPINetworkStatic)
|
||||
mux.HandleFunc("POST /api/network/toggle", h.handleAPINetworkToggle)
|
||||
mux.HandleFunc("POST /api/network/confirm", h.handleAPINetworkConfirm)
|
||||
mux.HandleFunc("POST /api/network/rollback", h.handleAPINetworkRollback)
|
||||
|
||||
// Export
|
||||
mux.HandleFunc("GET /api/export/list", h.handleAPIExportList)
|
||||
@@ -84,11 +208,27 @@ func NewHandler(opts HandlerOptions) http.Handler {
|
||||
// Tools
|
||||
mux.HandleFunc("GET /api/tools/check", h.handleAPIToolsCheck)
|
||||
|
||||
// GPU presence
|
||||
mux.HandleFunc("GET /api/gpu/presence", h.handleAPIGPUPresence)
|
||||
|
||||
// System
|
||||
mux.HandleFunc("GET /api/system/ram-status", h.handleAPIRAMStatus)
|
||||
mux.HandleFunc("POST /api/system/install-to-ram", h.handleAPIInstallToRAM)
|
||||
|
||||
// Preflight
|
||||
mux.HandleFunc("GET /api/preflight", h.handleAPIPreflight)
|
||||
|
||||
// Metrics — SSE stream of live sensor data
|
||||
// Install
|
||||
mux.HandleFunc("GET /api/install/disks", h.handleAPIInstallDisks)
|
||||
mux.HandleFunc("POST /api/install/run", h.handleAPIInstallRun)
|
||||
mux.HandleFunc("GET /api/install/stream", h.handleAPIInstallStream)
|
||||
|
||||
// Metrics — SSE stream of live sensor data + server-side SVG charts
|
||||
mux.HandleFunc("GET /api/metrics/stream", h.handleAPIMetricsStream)
|
||||
mux.HandleFunc("GET /api/metrics/chart/", h.handleMetricsChartSVG)
|
||||
|
||||
// Reanimator chart static assets (viewer template expects /static/*)
|
||||
mux.Handle("GET /static/", http.StripPrefix("/static/", web.Static()))
|
||||
|
||||
// ── Pages ────────────────────────────────────────────────────────────────
|
||||
mux.HandleFunc("GET /", h.handlePage)
|
||||
@@ -122,6 +262,11 @@ func (h *handler) handleAuditJSON(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, fmt.Sprintf("read audit snapshot: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// Re-apply SAT overlay on every request so that SAT results run after the
|
||||
// last audit always appear in the downloaded JSON without needing a re-audit.
|
||||
if overlaid, err := app.ApplySATOverlay(data); err == nil {
|
||||
data = overlaid
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
_, _ = w.Write(data)
|
||||
@@ -165,9 +310,33 @@ func (h *handler) handleExportFile(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "invalid path", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// Set Content-Type explicitly to avoid mime.TypeByExtension which panics on
|
||||
// LiveCD environments where /usr/share/mime/globs2 has an I/O read error.
|
||||
w.Header().Set("Content-Type", mimeByExt(filepath.Ext(clean)))
|
||||
http.ServeFile(w, r, filepath.Join(h.opts.ExportDir, clean))
|
||||
}
|
||||
|
||||
// mimeByExt returns a Content-Type for known extensions, falling back to
|
||||
// application/octet-stream. Used to avoid calling mime.TypeByExtension.
|
||||
func mimeByExt(ext string) string {
|
||||
switch strings.ToLower(ext) {
|
||||
case ".json":
|
||||
return "application/json"
|
||||
case ".gz":
|
||||
return "application/gzip"
|
||||
case ".tar":
|
||||
return "application/x-tar"
|
||||
case ".log", ".txt":
|
||||
return "text/plain; charset=utf-8"
|
||||
case ".html":
|
||||
return "text/html; charset=utf-8"
|
||||
case ".svg":
|
||||
return "image/svg+xml"
|
||||
default:
|
||||
return "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) handleExportIndex(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := renderExportIndex(h.opts.ExportDir)
|
||||
if err != nil {
|
||||
@@ -181,10 +350,203 @@ func (h *handler) handleExportIndex(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
func (h *handler) handleViewer(w http.ResponseWriter, r *http.Request) {
|
||||
snapshot, _ := loadSnapshot(h.opts.AuditPath)
|
||||
body := renderViewerPage(h.opts.Title, snapshot)
|
||||
body, err := viewer.RenderHTML(snapshot, h.opts.Title)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write([]byte(body))
|
||||
_, _ = w.Write(body)
|
||||
}
|
||||
|
||||
func (h *handler) handleMetricsChartSVG(w http.ResponseWriter, r *http.Request) {
|
||||
path := strings.TrimPrefix(r.URL.Path, "/api/metrics/chart/")
|
||||
path = strings.TrimSuffix(path, ".svg")
|
||||
|
||||
var datasets [][]float64
|
||||
var names []string
|
||||
var labels []string
|
||||
var title string
|
||||
var yMin, yMax *float64 // nil = auto; for load charts fixed 0-100
|
||||
|
||||
switch {
|
||||
// ── Server sub-charts ─────────────────────────────────────────────────
|
||||
case path == "server-load":
|
||||
title = "CPU / Memory Load"
|
||||
vCPULoad, l := h.ringCPULoad.snapshot()
|
||||
vMemLoad, _ := h.ringMemLoad.snapshot()
|
||||
labels = l
|
||||
datasets = [][]float64{vCPULoad, vMemLoad}
|
||||
names = []string{"CPU Load %", "Mem Load %"}
|
||||
yMin = floatPtr(0)
|
||||
yMax = floatPtr(100)
|
||||
|
||||
case path == "server-temp":
|
||||
title = "CPU Temperature"
|
||||
vCPUTemp, l := h.ringCPUTemp.snapshot()
|
||||
labels = l
|
||||
datasets = [][]float64{vCPUTemp}
|
||||
names = []string{"CPU Temp °C"}
|
||||
yMin = floatPtr(0)
|
||||
yMax = autoMax120(vCPUTemp)
|
||||
|
||||
case path == "server-power":
|
||||
title = "Power & Fans"
|
||||
vPower, l := h.ringPower.snapshot()
|
||||
labels = l
|
||||
datasets = [][]float64{vPower}
|
||||
names = []string{"Power W"}
|
||||
h.ringsMu.Lock()
|
||||
for i, fr := range h.ringFans {
|
||||
fv, _ := fr.snapshot()
|
||||
datasets = append(datasets, fv)
|
||||
name := "Fan"
|
||||
if i < len(h.fanNames) {
|
||||
name = h.fanNames[i]
|
||||
}
|
||||
names = append(names, name+" RPM")
|
||||
}
|
||||
h.ringsMu.Unlock()
|
||||
yMin = floatPtr(0)
|
||||
yMax = autoMax120(datasets...)
|
||||
|
||||
// ── GPU sub-charts ────────────────────────────────────────────────────
|
||||
case strings.HasPrefix(path, "gpu/"):
|
||||
rest := strings.TrimPrefix(path, "gpu/")
|
||||
// rest is either "{idx}-load", "{idx}-temp", "{idx}-power", or legacy "{idx}"
|
||||
sub := ""
|
||||
if i := strings.LastIndex(rest, "-"); i > 0 {
|
||||
sub = rest[i+1:]
|
||||
rest = rest[:i]
|
||||
}
|
||||
idx := 0
|
||||
fmt.Sscanf(rest, "%d", &idx)
|
||||
h.ringsMu.Lock()
|
||||
var gr *gpuRings
|
||||
if idx < len(h.gpuRings) {
|
||||
gr = h.gpuRings[idx]
|
||||
}
|
||||
h.ringsMu.Unlock()
|
||||
if gr == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
switch sub {
|
||||
case "load":
|
||||
vUtil, l := gr.Util.snapshot()
|
||||
vMemUtil, _ := gr.MemUtil.snapshot()
|
||||
labels = l
|
||||
title = fmt.Sprintf("GPU %d Load", idx)
|
||||
datasets = [][]float64{vUtil, vMemUtil}
|
||||
names = []string{"Load %", "Mem %"}
|
||||
yMin = floatPtr(0)
|
||||
yMax = floatPtr(100)
|
||||
case "temp":
|
||||
vTemp, l := gr.Temp.snapshot()
|
||||
labels = l
|
||||
title = fmt.Sprintf("GPU %d Temperature", idx)
|
||||
datasets = [][]float64{vTemp}
|
||||
names = []string{"Temp °C"}
|
||||
yMin = floatPtr(0)
|
||||
yMax = autoMax120(vTemp)
|
||||
default: // "power" or legacy (no sub)
|
||||
vPower, l := gr.Power.snapshot()
|
||||
labels = l
|
||||
title = fmt.Sprintf("GPU %d Power", idx)
|
||||
datasets = [][]float64{vPower}
|
||||
names = []string{"Power W"}
|
||||
yMin = floatPtr(0)
|
||||
yMax = autoMax120(vPower)
|
||||
}
|
||||
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
buf, err := renderChartSVG(title, datasets, names, labels, yMin, yMax)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/svg+xml")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
_, _ = w.Write(buf)
|
||||
}
|
||||
|
||||
// floatPtr returns a pointer to a float64 value.
|
||||
func floatPtr(v float64) *float64 { return &v }
|
||||
|
||||
// autoMax120 returns 0→max+20% Y-axis max across all datasets.
|
||||
func autoMax120(datasets ...[]float64) *float64 {
|
||||
max := 0.0
|
||||
for _, ds := range datasets {
|
||||
for _, v := range ds {
|
||||
if v > max {
|
||||
max = v
|
||||
}
|
||||
}
|
||||
}
|
||||
if max == 0 {
|
||||
return nil // let library auto-scale
|
||||
}
|
||||
v := max * 1.2
|
||||
return &v
|
||||
}
|
||||
|
||||
// renderChartSVG renders a line chart SVG with a fixed Y-axis range.
|
||||
func renderChartSVG(title string, datasets [][]float64, names []string, labels []string, yMin, yMax *float64) ([]byte, error) {
|
||||
n := len(labels)
|
||||
if n == 0 {
|
||||
n = 1
|
||||
labels = []string{""}
|
||||
}
|
||||
for i := range datasets {
|
||||
if len(datasets[i]) == 0 {
|
||||
datasets[i] = make([]float64, n)
|
||||
}
|
||||
}
|
||||
sparse := sparseLabels(labels, 6)
|
||||
|
||||
opt := gocharts.NewLineChartOptionWithData(datasets)
|
||||
opt.Title = gocharts.TitleOption{Text: title}
|
||||
opt.XAxis.Labels = sparse
|
||||
opt.Legend = gocharts.LegendOption{SeriesNames: names}
|
||||
if yMin != nil || yMax != nil {
|
||||
opt.YAxis = []gocharts.YAxisOption{{Min: yMin, Max: yMax}}
|
||||
}
|
||||
|
||||
p := gocharts.NewPainter(gocharts.PainterOptions{
|
||||
OutputFormat: gocharts.ChartOutputSVG,
|
||||
Width: 1400,
|
||||
Height: 240,
|
||||
}, gocharts.PainterThemeOption(gocharts.GetTheme("grafana")))
|
||||
if err := p.LineChart(opt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return p.Bytes()
|
||||
}
|
||||
|
||||
func safeIdx(s []float64, i int) float64 {
|
||||
if i < len(s) {
|
||||
return s[i]
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func sparseLabels(labels []string, n int) []string {
|
||||
out := make([]string, len(labels))
|
||||
step := len(labels) / n
|
||||
if step < 1 {
|
||||
step = 1
|
||||
}
|
||||
for i, l := range labels {
|
||||
if i%step == 0 {
|
||||
out[i] = l
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ── Page handler ─────────────────────────────────────────────────────────────
|
||||
@@ -194,6 +556,15 @@ func (h *handler) handlePage(w http.ResponseWriter, r *http.Request) {
|
||||
if page == "" {
|
||||
page = "dashboard"
|
||||
}
|
||||
// Redirect old routes to new names
|
||||
switch page {
|
||||
case "tests":
|
||||
http.Redirect(w, r, "/validate", http.StatusMovedPermanently)
|
||||
return
|
||||
case "burn-in":
|
||||
http.Redirect(w, r, "/burn", http.StatusMovedPermanently)
|
||||
return
|
||||
}
|
||||
body := renderPage(page, h.opts)
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRootRendersShellWithIframe(t *testing.T) {
|
||||
func TestRootRendersDashboard(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "audit.json")
|
||||
exportDir := filepath.Join(dir, "export")
|
||||
@@ -31,11 +31,12 @@ func TestRootRendersShellWithIframe(t *testing.T) {
|
||||
if first.Code != http.StatusOK {
|
||||
t.Fatalf("first status=%d", first.Code)
|
||||
}
|
||||
if !strings.Contains(first.Body.String(), `iframe`) || !strings.Contains(first.Body.String(), `src="/viewer"`) {
|
||||
t.Fatalf("first body missing iframe viewer: %s", first.Body.String())
|
||||
// Dashboard should contain the audit modal (with viewer link) and hardware summary
|
||||
if !strings.Contains(first.Body.String(), `openAuditModal`) {
|
||||
t.Fatalf("first body missing audit modal trigger: %s", first.Body.String())
|
||||
}
|
||||
if !strings.Contains(first.Body.String(), "/export/support.tar.gz") {
|
||||
t.Fatalf("first body missing support bundle link: %s", first.Body.String())
|
||||
if !strings.Contains(first.Body.String(), `/viewer`) {
|
||||
t.Fatalf("first body missing viewer link: %s", first.Body.String())
|
||||
}
|
||||
if got := first.Header().Get("Cache-Control"); got != "no-store" {
|
||||
t.Fatalf("first cache-control=%q", got)
|
||||
@@ -50,8 +51,8 @@ func TestRootRendersShellWithIframe(t *testing.T) {
|
||||
if second.Code != http.StatusOK {
|
||||
t.Fatalf("second status=%d", second.Code)
|
||||
}
|
||||
if !strings.Contains(second.Body.String(), `src="/viewer"`) {
|
||||
t.Fatalf("second body missing iframe viewer: %s", second.Body.String())
|
||||
if !strings.Contains(second.Body.String(), `Hardware Summary`) {
|
||||
t.Fatalf("second body missing hardware summary: %s", second.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,8 +104,8 @@ func TestAuditJSONServesLatestSnapshot(t *testing.T) {
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d", rec.Code)
|
||||
}
|
||||
if got := strings.TrimSpace(rec.Body.String()); got != body {
|
||||
t.Fatalf("body=%q want %q", got, body)
|
||||
if !strings.Contains(rec.Body.String(), "SERIAL-API") {
|
||||
t.Fatalf("body missing expected serial: %s", rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get("Content-Type"); !strings.Contains(got, "application/json") {
|
||||
t.Fatalf("content-type=%q", got)
|
||||
|
||||
420
audit/internal/webui/tasks.go
Normal file
420
audit/internal/webui/tasks.go
Normal file
@@ -0,0 +1,420 @@
|
||||
package webui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Task statuses.
|
||||
const (
|
||||
TaskPending = "pending"
|
||||
TaskRunning = "running"
|
||||
TaskDone = "done"
|
||||
TaskFailed = "failed"
|
||||
TaskCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// taskNames maps target → human-readable name.
|
||||
var taskNames = map[string]string{
|
||||
"nvidia": "NVIDIA SAT",
|
||||
"memory": "Memory SAT",
|
||||
"storage": "Storage SAT",
|
||||
"cpu": "CPU SAT",
|
||||
"amd": "AMD GPU SAT",
|
||||
"amd-stress": "AMD GPU Burn-in",
|
||||
"memory-stress": "Memory Burn-in",
|
||||
"sat-stress": "SAT Stress (stressapptest)",
|
||||
"audit": "Audit",
|
||||
"install": "Install to Disk",
|
||||
"install-to-ram": "Install to RAM",
|
||||
}
|
||||
|
||||
// Task represents one unit of work in the queue.
|
||||
type Task struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Target string `json:"target"`
|
||||
Priority int `json:"priority"`
|
||||
Status string `json:"status"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
DoneAt *time.Time `json:"done_at,omitempty"`
|
||||
ErrMsg string `json:"error,omitempty"`
|
||||
|
||||
// runtime fields (not serialised)
|
||||
job *jobState
|
||||
params taskParams
|
||||
}
|
||||
|
||||
// taskParams holds optional parameters parsed from the run request.
|
||||
type taskParams struct {
|
||||
Duration int
|
||||
DiagLevel int
|
||||
GPUIndices []int
|
||||
Device string // for install
|
||||
}
|
||||
|
||||
// taskQueue manages a priority-ordered list of tasks and runs them one at a time.
|
||||
type taskQueue struct {
|
||||
mu sync.Mutex
|
||||
tasks []*Task
|
||||
trigger chan struct{}
|
||||
opts *HandlerOptions // set by startWorker
|
||||
}
|
||||
|
||||
var globalQueue = &taskQueue{trigger: make(chan struct{}, 1)}
|
||||
|
||||
const maxTaskHistory = 50
|
||||
|
||||
// enqueue adds a task to the queue and notifies the worker.
|
||||
func (q *taskQueue) enqueue(t *Task) {
|
||||
q.mu.Lock()
|
||||
q.tasks = append(q.tasks, t)
|
||||
q.prune()
|
||||
q.mu.Unlock()
|
||||
select {
|
||||
case q.trigger <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// prune removes oldest completed tasks beyond maxTaskHistory.
|
||||
func (q *taskQueue) prune() {
|
||||
var done []*Task
|
||||
var active []*Task
|
||||
for _, t := range q.tasks {
|
||||
switch t.Status {
|
||||
case TaskDone, TaskFailed, TaskCancelled:
|
||||
done = append(done, t)
|
||||
default:
|
||||
active = append(active, t)
|
||||
}
|
||||
}
|
||||
if len(done) > maxTaskHistory {
|
||||
done = done[len(done)-maxTaskHistory:]
|
||||
}
|
||||
q.tasks = append(active, done...)
|
||||
}
|
||||
|
||||
// nextPending returns the highest-priority pending task (nil if none).
|
||||
func (q *taskQueue) nextPending() *Task {
|
||||
var best *Task
|
||||
for _, t := range q.tasks {
|
||||
if t.Status != TaskPending {
|
||||
continue
|
||||
}
|
||||
if best == nil || t.Priority > best.Priority ||
|
||||
(t.Priority == best.Priority && t.CreatedAt.Before(best.CreatedAt)) {
|
||||
best = t
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// findByID looks up a task by ID.
|
||||
func (q *taskQueue) findByID(id string) (*Task, bool) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
for _, t := range q.tasks {
|
||||
if t.ID == id {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// findJob returns the jobState for a task ID (for SSE streaming compatibility).
|
||||
func (q *taskQueue) findJob(id string) (*jobState, bool) {
|
||||
t, ok := q.findByID(id)
|
||||
if !ok || t.job == nil {
|
||||
return nil, false
|
||||
}
|
||||
return t.job, true
|
||||
}
|
||||
|
||||
// snapshot returns a copy of all tasks sorted for display (running first, then pending by priority, then done by doneAt desc).
|
||||
func (q *taskQueue) snapshot() []Task {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
out := make([]Task, len(q.tasks))
|
||||
for i, t := range q.tasks {
|
||||
out[i] = *t
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
si := statusOrder(out[i].Status)
|
||||
sj := statusOrder(out[j].Status)
|
||||
if si != sj {
|
||||
return si < sj
|
||||
}
|
||||
if out[i].Priority != out[j].Priority {
|
||||
return out[i].Priority > out[j].Priority
|
||||
}
|
||||
return out[i].CreatedAt.Before(out[j].CreatedAt)
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func statusOrder(s string) int {
|
||||
switch s {
|
||||
case TaskRunning:
|
||||
return 0
|
||||
case TaskPending:
|
||||
return 1
|
||||
default:
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
// startWorker launches the queue runner goroutine.
|
||||
func (q *taskQueue) startWorker(opts *HandlerOptions) {
|
||||
q.opts = opts
|
||||
go q.worker()
|
||||
}
|
||||
|
||||
func (q *taskQueue) worker() {
|
||||
for {
|
||||
<-q.trigger
|
||||
for {
|
||||
q.mu.Lock()
|
||||
t := q.nextPending()
|
||||
if t == nil {
|
||||
q.mu.Unlock()
|
||||
break
|
||||
}
|
||||
now := time.Now()
|
||||
t.Status = TaskRunning
|
||||
t.StartedAt = &now
|
||||
j := &jobState{}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
j.cancel = cancel
|
||||
t.job = j
|
||||
q.mu.Unlock()
|
||||
|
||||
q.runTask(t, j, ctx)
|
||||
|
||||
q.mu.Lock()
|
||||
now2 := time.Now()
|
||||
t.DoneAt = &now2
|
||||
if t.Status == TaskRunning { // not cancelled externally
|
||||
if j.err != "" {
|
||||
t.Status = TaskFailed
|
||||
t.ErrMsg = j.err
|
||||
} else {
|
||||
t.Status = TaskDone
|
||||
}
|
||||
}
|
||||
q.prune()
|
||||
q.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runTask executes the work for a task, writing output to j.
|
||||
func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) {
|
||||
if q.opts == nil || q.opts.App == nil {
|
||||
j.append("ERROR: app not configured")
|
||||
j.finish("app not configured")
|
||||
return
|
||||
}
|
||||
a := q.opts.App
|
||||
|
||||
j.append(fmt.Sprintf("Starting %s...", t.Name))
|
||||
|
||||
var (
|
||||
archive string
|
||||
err error
|
||||
)
|
||||
|
||||
switch t.Target {
|
||||
case "nvidia":
|
||||
if len(t.params.GPUIndices) > 0 || t.params.DiagLevel > 0 {
|
||||
result, e := a.RunNvidiaAcceptancePackWithOptions(
|
||||
ctx, "", t.params.DiagLevel, t.params.GPUIndices, j.append,
|
||||
)
|
||||
if e != nil {
|
||||
err = e
|
||||
} else {
|
||||
archive = result.Body
|
||||
}
|
||||
} else {
|
||||
archive, err = a.RunNvidiaAcceptancePack("", j.append)
|
||||
}
|
||||
case "memory":
|
||||
archive, err = a.RunMemoryAcceptancePack("", j.append)
|
||||
case "storage":
|
||||
archive, err = a.RunStorageAcceptancePack("", j.append)
|
||||
case "cpu":
|
||||
dur := t.params.Duration
|
||||
if dur <= 0 {
|
||||
dur = 60
|
||||
}
|
||||
archive, err = a.RunCPUAcceptancePack("", dur, j.append)
|
||||
case "amd":
|
||||
archive, err = a.RunAMDAcceptancePack("", j.append)
|
||||
case "amd-stress":
|
||||
archive, err = a.RunAMDStressPack("", j.append)
|
||||
case "memory-stress":
|
||||
archive, err = a.RunMemoryStressPack("", j.append)
|
||||
case "sat-stress":
|
||||
archive, err = a.RunSATStressPack("", j.append)
|
||||
case "audit":
|
||||
result, e := a.RunAuditNow(q.opts.RuntimeMode)
|
||||
if e != nil {
|
||||
err = e
|
||||
} else {
|
||||
for _, line := range splitLines(result.Body) {
|
||||
j.append(line)
|
||||
}
|
||||
}
|
||||
case "install-to-ram":
|
||||
err = a.RunInstallToRAM(j.append)
|
||||
default:
|
||||
j.append("ERROR: unknown target: " + t.Target)
|
||||
j.finish("unknown target")
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
j.append("Aborted.")
|
||||
j.finish("aborted")
|
||||
} else {
|
||||
j.append("ERROR: " + err.Error())
|
||||
j.finish(err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
if archive != "" {
|
||||
j.append("Archive: " + archive)
|
||||
}
|
||||
j.finish("")
|
||||
}
|
||||
|
||||
func splitLines(s string) []string {
|
||||
var out []string
|
||||
for _, l := range splitNL(s) {
|
||||
if l != "" {
|
||||
out = append(out, l)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func splitNL(s string) []string {
|
||||
var out []string
|
||||
start := 0
|
||||
for i, c := range s {
|
||||
if c == '\n' {
|
||||
out = append(out, s[start:i])
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
out = append(out, s[start:])
|
||||
return out
|
||||
}
|
||||
|
||||
// ── HTTP handlers ─────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *handler) handleAPITasksList(w http.ResponseWriter, _ *http.Request) {
|
||||
tasks := globalQueue.snapshot()
|
||||
writeJSON(w, tasks)
|
||||
}
|
||||
|
||||
func (h *handler) handleAPITasksCancel(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
t, ok := globalQueue.findByID(id)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotFound, "task not found")
|
||||
return
|
||||
}
|
||||
globalQueue.mu.Lock()
|
||||
defer globalQueue.mu.Unlock()
|
||||
switch t.Status {
|
||||
case TaskPending:
|
||||
t.Status = TaskCancelled
|
||||
now := time.Now()
|
||||
t.DoneAt = &now
|
||||
writeJSON(w, map[string]string{"status": "cancelled"})
|
||||
case TaskRunning:
|
||||
if t.job != nil {
|
||||
t.job.abort()
|
||||
}
|
||||
t.Status = TaskCancelled
|
||||
now := time.Now()
|
||||
t.DoneAt = &now
|
||||
writeJSON(w, map[string]string{"status": "cancelled"})
|
||||
default:
|
||||
writeError(w, http.StatusConflict, "task is not running or pending")
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) handleAPITasksPriority(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
t, ok := globalQueue.findByID(id)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotFound, "task not found")
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Delta int `json:"delta"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid body")
|
||||
return
|
||||
}
|
||||
globalQueue.mu.Lock()
|
||||
defer globalQueue.mu.Unlock()
|
||||
if t.Status != TaskPending {
|
||||
writeError(w, http.StatusConflict, "only pending tasks can be reprioritised")
|
||||
return
|
||||
}
|
||||
t.Priority += req.Delta
|
||||
writeJSON(w, map[string]int{"priority": t.Priority})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPITasksCancelAll(w http.ResponseWriter, _ *http.Request) {
|
||||
globalQueue.mu.Lock()
|
||||
now := time.Now()
|
||||
n := 0
|
||||
for _, t := range globalQueue.tasks {
|
||||
switch t.Status {
|
||||
case TaskPending:
|
||||
t.Status = TaskCancelled
|
||||
t.DoneAt = &now
|
||||
n++
|
||||
case TaskRunning:
|
||||
if t.job != nil {
|
||||
t.job.abort()
|
||||
}
|
||||
t.Status = TaskCancelled
|
||||
t.DoneAt = &now
|
||||
n++
|
||||
}
|
||||
}
|
||||
globalQueue.mu.Unlock()
|
||||
writeJSON(w, map[string]int{"cancelled": n})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPITasksStream(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
// Wait up to 5s for the task to get a job (it may be pending)
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
var j *jobState
|
||||
for time.Now().Before(deadline) {
|
||||
if jj, ok := globalQueue.findJob(id); ok {
|
||||
j = jj
|
||||
break
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
if j == nil {
|
||||
http.Error(w, "task not found or not yet started", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
streamJob(w, r, j)
|
||||
}
|
||||
38
bible-local/architecture/charting.md
Normal file
38
bible-local/architecture/charting.md
Normal file
@@ -0,0 +1,38 @@
|
||||
# Charting architecture
|
||||
|
||||
## Decision: one chart engine for all live metrics
|
||||
|
||||
**Engine:** `github.com/go-analyze/charts` (pure Go, no CGO, SVG output)
|
||||
**Theme:** `grafana` (dark background, coloured lines)
|
||||
|
||||
All live metrics charts in the web UI are server-side SVG images served by Go
|
||||
and polled by the browser every 2 seconds via `<img src="...?t=now">`.
|
||||
There is no client-side canvas or JS chart library.
|
||||
|
||||
### Why go-analyze/charts
|
||||
|
||||
- Pure Go, no CGO — builds cleanly inside the live-build container
|
||||
- SVG output — crisp at any display resolution, full-width without pixelation
|
||||
- Grafana theme matches the dark web UI colour scheme
|
||||
- Active fork of the archived wcharczuk/go-chart
|
||||
|
||||
### SAT stress-test charts
|
||||
|
||||
The `drawGPUChartSVG` function in `platform/gpu_metrics.go` is a separate
|
||||
self-contained SVG renderer used **only** for completed SAT run reports
|
||||
(HTML export, burn-in summaries). It is not used for live metrics.
|
||||
|
||||
### Live metrics chart endpoints
|
||||
|
||||
| Path | Content |
|
||||
|------|---------|
|
||||
| `GET /api/metrics/chart/server.svg` | CPU temp, CPU load %, mem load %, power W, fan RPMs |
|
||||
| `GET /api/metrics/chart/gpu/{idx}.svg` | GPU temp °C, load %, mem %, power W |
|
||||
|
||||
Charts are 1400 × 280 px SVG. The page renders them at `width: 100%` in a
|
||||
single-column layout so they always fill the viewport width.
|
||||
|
||||
### Ring buffers
|
||||
|
||||
Each metric is stored in a 120-sample ring buffer (2 minutes of history at 1 Hz).
|
||||
Buffers are per-server or per-GPU and grow dynamically as new GPUs appear.
|
||||
@@ -126,7 +126,7 @@ Key checks: NVIDIA modules loaded, `nvidia-smi` sees all GPUs, lib symlinks pres
|
||||
systemd services running, audit completed with NVIDIA enrichment, LAN reachability.
|
||||
|
||||
Current validation state:
|
||||
- local/libvirt VM boot path is validated for `systemd`, SSH, `bee audit`, `bee-network`, and TUI startup
|
||||
- local/libvirt VM boot path is validated for `systemd`, SSH, `bee audit`, `bee-network`, and Web UI startup
|
||||
- real hardware validation is still required before treating the ISO as release-ready
|
||||
|
||||
## Overlay mechanism
|
||||
@@ -168,33 +168,17 @@ Acceptance flows:
|
||||
- `BEE_MEMTESTER_SIZE_MB`
|
||||
- `BEE_MEMTESTER_PASSES`
|
||||
|
||||
## NVIDIA SAT TUI flow (v1.0.0+)
|
||||
## NVIDIA SAT Web UI flow
|
||||
|
||||
```
|
||||
TUI: Acceptance tests → NVIDIA command pack
|
||||
1. screenNvidiaSATSetup
|
||||
a. enumerate GPUs via `nvidia-smi --query-gpu=index,name,memory.total`
|
||||
b. user selects duration preset: 10 min / 1 h / 8 h / 24 h
|
||||
c. user selects GPUs via checkboxes (all selected by default)
|
||||
d. memory size = max(selected GPU memory) — auto-detected, not exposed to user
|
||||
2. Start → screenNvidiaSATRunning
|
||||
a. CUDA_VISIBLE_DEVICES set to selected GPU indices
|
||||
b. tea.Batch: SAT goroutine + tea.ExecProcess(nvtop) launched concurrently
|
||||
c. nvtop occupies full terminal; SAT result queues in background
|
||||
d. [o] reopen nvtop at any time; [a] abort (cancels context → kills bee-gpu-stress)
|
||||
3. GPU metrics collection (during bee-gpu-stress)
|
||||
- background goroutine polls `nvidia-smi` every second
|
||||
- per-second rows: elapsed, GPU index, temp°C, usage%, power W, clock MHz
|
||||
- outputs: gpu-metrics.csv, gpu-metrics.html (offline SVG chart), gpu-metrics-term.txt
|
||||
4. After SAT completes
|
||||
- result shown in screenOutput with terminal line-chart (gpu-metrics-term.txt)
|
||||
- chart is asciigraph-style: box-drawing chars (╭╮╰╯─│), 4 series per GPU,
|
||||
Y axis with ticks, ANSI colours (red=temp, blue=usage, green=power, yellow=clock)
|
||||
Web UI: Acceptance Tests page → Run Test button
|
||||
1. POST /api/sat/nvidia/run → returns job_id
|
||||
2. GET /api/sat/stream?job_id=... (SSE) — streams stdout/stderr lines live
|
||||
3. After completion — archive written to /appdata/bee/export/bee-sat/
|
||||
summary.txt contains overall_status (OK / FAILED) and per-job status values
|
||||
```
|
||||
|
||||
**Critical invariants:**
|
||||
- `nvtop` must be in `iso/builder/config/package-lists/bee.list.chroot` (baked into ISO).
|
||||
- `bee-gpu-stress` uses `exec.CommandContext` — aborted on cancel.
|
||||
- `bee-gpu-stress` uses `exec.CommandContext` — killed on job context cancel.
|
||||
- Metric goroutine uses stopCh/doneCh pattern; main goroutine waits `<-doneCh` before reading rows (no mutex needed).
|
||||
- If `nvtop` is not found on PATH, SAT still runs without it (graceful degradation).
|
||||
- SVG chart is fully offline: no JS, no external CSS, pure inline SVG.
|
||||
|
||||
Submodule internal/chart updated: 05db6994d4...ac8120c8ab
@@ -8,5 +8,8 @@ NCCL_TESTS_VERSION=2.13.10
|
||||
NVCC_VERSION=12.8
|
||||
CUBLAS_VERSION=13.0.2.14-1
|
||||
CUDA_USERSPACE_VERSION=13.0.96-1
|
||||
DCGM_VERSION=3.3.9
|
||||
ROCM_VERSION=6.3.4
|
||||
ROCM_SMI_VERSION=7.4.0.60304-76~22.04
|
||||
GO_VERSION=1.24.0
|
||||
AUDIT_VERSION=1.0.0
|
||||
|
||||
@@ -32,6 +32,7 @@ lb config noauto \
|
||||
--memtest none \
|
||||
--iso-volume "EASY-BEE" \
|
||||
--iso-application "EASY-BEE" \
|
||||
--bootappend-live "boot=live components quiet nomodeset console=tty0 console=ttyS0,115200n8 loglevel=3 username=bee user-fullname=Bee modprobe.blacklist=nouveau" \
|
||||
--bootappend-live "boot=live components nomodeset video=1920x1080 console=tty0 console=ttyS0,115200n8 loglevel=7 username=bee user-fullname=Bee modprobe.blacklist=nouveau" \
|
||||
--apt-recommends false \
|
||||
--chroot-squashfs-compression-type zstd \
|
||||
"${@}"
|
||||
|
||||
@@ -11,6 +11,7 @@ BUILDER_PLATFORM="${BEE_BUILDER_PLATFORM:-linux/amd64}"
|
||||
CACHE_DIR="${BEE_BUILDER_CACHE_DIR:-${REPO_ROOT}/dist/container-cache}"
|
||||
AUTH_KEYS=""
|
||||
REBUILD_IMAGE=0
|
||||
CLEAN_CACHE=0
|
||||
|
||||
. "${BUILDER_DIR}/VERSIONS"
|
||||
|
||||
@@ -28,14 +29,31 @@ while [ $# -gt 0 ]; do
|
||||
AUTH_KEYS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--clean-build)
|
||||
CLEAN_CACHE=1
|
||||
REBUILD_IMAGE=1
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "unknown arg: $1" >&2
|
||||
echo "usage: $0 [--cache-dir /path] [--rebuild-image] [--authorized-keys /path/to/authorized_keys]" >&2
|
||||
echo "usage: $0 [--cache-dir /path] [--rebuild-image] [--clean-build] [--authorized-keys /path/to/authorized_keys]" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$CLEAN_CACHE" = "1" ]; then
|
||||
echo "=== cleaning build cache: ${CACHE_DIR} ==="
|
||||
rm -rf "${CACHE_DIR:?}/go-build" \
|
||||
"${CACHE_DIR:?}/go-mod" \
|
||||
"${CACHE_DIR:?}/tmp" \
|
||||
"${CACHE_DIR:?}/bee" \
|
||||
"${CACHE_DIR:?}/lb-packages"
|
||||
echo "=== cleaning live-build work dir: ${REPO_ROOT}/dist/live-build-work ==="
|
||||
rm -rf "${REPO_ROOT}/dist/live-build-work"
|
||||
echo "=== caches cleared, proceeding with build ==="
|
||||
fi
|
||||
|
||||
if ! command -v "$CONTAINER_TOOL" >/dev/null 2>&1; then
|
||||
echo "container tool not found: $CONTAINER_TOOL" >&2
|
||||
exit 1
|
||||
|
||||
@@ -16,11 +16,13 @@ NCCL_TESTS_VERSION="$1"
|
||||
NCCL_VERSION="$2"
|
||||
NCCL_CUDA_VERSION="$3"
|
||||
DIST_DIR="$4"
|
||||
NVCC_VERSION="${5:-}"
|
||||
DEBIAN_VERSION="${6:-12}"
|
||||
|
||||
[ -n "$NCCL_TESTS_VERSION" ] || { echo "usage: $0 <nccl-tests-version> <nccl-version> <cuda-version> <dist-dir>"; exit 1; }
|
||||
[ -n "$NCCL_VERSION" ] || { echo "usage: $0 <nccl-tests-version> <nccl-version> <cuda-version> <dist-dir>"; exit 1; }
|
||||
[ -n "$NCCL_CUDA_VERSION" ] || { echo "usage: $0 <nccl-tests-version> <nccl-version> <cuda-version> <dist-dir>"; exit 1; }
|
||||
[ -n "$DIST_DIR" ] || { echo "usage: $0 <nccl-tests-version> <nccl-version> <cuda-version> <dist-dir>"; exit 1; }
|
||||
[ -n "$NCCL_TESTS_VERSION" ] || { echo "usage: $0 <nccl-tests-version> <nccl-version> <cuda-version> <dist-dir> [nvcc-version] [debian-version]"; exit 1; }
|
||||
[ -n "$NCCL_VERSION" ] || { echo "usage: $0 <nccl-tests-version> <nccl-version> <cuda-version> <dist-dir> [nvcc-version] [debian-version]"; exit 1; }
|
||||
[ -n "$NCCL_CUDA_VERSION" ] || { echo "usage: $0 <nccl-tests-version> <nccl-version> <cuda-version> <dist-dir> [nvcc-version] [debian-version]"; exit 1; }
|
||||
[ -n "$DIST_DIR" ] || { echo "usage: $0 <nccl-tests-version> <nccl-version> <cuda-version> <dist-dir> [nvcc-version] [debian-version]"; exit 1; }
|
||||
|
||||
echo "=== nccl-tests ${NCCL_TESTS_VERSION} ==="
|
||||
|
||||
@@ -34,15 +36,16 @@ if [ -f "${CACHE_DIR}/bin/all_reduce_perf" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Resolve nvcc path (cuda-nvcc-12-8 installs to /usr/local/cuda-12.8/bin/nvcc)
|
||||
# Resolve nvcc path (cuda-nvcc-X-Y installs to /usr/local/cuda-X.Y/bin/nvcc)
|
||||
NVCC_VERSION_PATH="$(echo "${NVCC_VERSION}" | tr '.' '.')"
|
||||
NVCC=""
|
||||
for candidate in nvcc /usr/local/cuda-12.8/bin/nvcc /usr/local/cuda-12/bin/nvcc /usr/local/cuda/bin/nvcc; do
|
||||
for candidate in nvcc "/usr/local/cuda-${NVCC_VERSION_PATH}/bin/nvcc" /usr/local/cuda-12/bin/nvcc /usr/local/cuda/bin/nvcc; do
|
||||
if command -v "$candidate" >/dev/null 2>&1 || [ -x "$candidate" ]; then
|
||||
NVCC="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
[ -n "$NVCC" ] || { echo "ERROR: nvcc not found — install cuda-nvcc-13-0"; exit 1; }
|
||||
[ -n "$NVCC" ] || { echo "ERROR: nvcc not found — install cuda-nvcc-$(echo "${NVCC_VERSION}" | tr '.' '-')"; exit 1; }
|
||||
echo "nvcc: $NVCC"
|
||||
|
||||
# Determine CUDA_HOME from nvcc location
|
||||
@@ -50,7 +53,7 @@ CUDA_HOME="$(dirname "$(dirname "$NVCC")")"
|
||||
echo "CUDA_HOME: $CUDA_HOME"
|
||||
|
||||
# Download libnccl-dev for nccl.h
|
||||
REPO_BASE="https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64"
|
||||
REPO_BASE="https://developer.download.nvidia.com/compute/cuda/repos/debian${DEBIAN_VERSION}/x86_64"
|
||||
DEV_PKG="libnccl-dev_${NCCL_VERSION}+cuda${NCCL_CUDA_VERSION}_amd64.deb"
|
||||
DEV_URL="${REPO_BASE}/${DEV_PKG}"
|
||||
|
||||
|
||||
@@ -28,6 +28,9 @@ done
|
||||
|
||||
. "${BUILDER_DIR}/VERSIONS"
|
||||
export PATH="$PATH:/usr/local/go/bin"
|
||||
|
||||
# Allow git to read the bind-mounted repo (different UID inside container).
|
||||
git config --global safe.directory "${REPO_ROOT}"
|
||||
mkdir -p "${DIST_DIR}"
|
||||
mkdir -p "${CACHE_ROOT}"
|
||||
: "${GOCACHE:=${CACHE_ROOT}/go-build}"
|
||||
@@ -42,7 +45,7 @@ resolve_audit_version() {
|
||||
|
||||
tag="$(git -C "${REPO_ROOT}" describe --tags --match 'audit/v*' --abbrev=7 --dirty 2>/dev/null || true)"
|
||||
if [ -z "${tag}" ]; then
|
||||
tag="$(git -C "${REPO_ROOT}" describe --tags --match 'v*' --abbrev=7 --dirty 2>/dev/null || true)"
|
||||
tag="$(git -C "${REPO_ROOT}" describe --tags --match 'v[0-9]*' --abbrev=7 --dirty 2>/dev/null || true)"
|
||||
fi
|
||||
case "${tag}" in
|
||||
audit/v*)
|
||||
@@ -76,6 +79,16 @@ resolve_iso_version() {
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Plain v* tags (e.g. v2.7) take priority — this is the current tagging scheme
|
||||
tag="$(git -C "${REPO_ROOT}" describe --tags --match 'v[0-9]*' --abbrev=7 --dirty 2>/dev/null || true)"
|
||||
case "${tag}" in
|
||||
v*)
|
||||
echo "${tag#v}"
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
|
||||
# Legacy iso/v* tags fallback
|
||||
tag="$(git -C "${REPO_ROOT}" describe --tags --match 'iso/v*' --abbrev=7 --dirty 2>/dev/null || true)"
|
||||
case "${tag}" in
|
||||
iso/v*)
|
||||
@@ -187,9 +200,27 @@ else
|
||||
fi
|
||||
|
||||
echo "=== preparing staged overlay ==="
|
||||
rm -rf "${BUILD_WORK_DIR}" "${OVERLAY_STAGE_DIR}"
|
||||
# Sync builder config into work dir, preserving lb cache (chroot + packages).
|
||||
# We do NOT rm -rf BUILD_WORK_DIR so lb can reuse its chroot on repeat builds.
|
||||
mkdir -p "${BUILD_WORK_DIR}" "${OVERLAY_STAGE_DIR}"
|
||||
rsync -a "${BUILDER_DIR}/" "${BUILD_WORK_DIR}/"
|
||||
rsync -a --delete \
|
||||
--exclude='cache/' \
|
||||
--exclude='chroot/' \
|
||||
--exclude='.build/' \
|
||||
--exclude='*.iso' \
|
||||
--exclude='*.packages' \
|
||||
--exclude='*.contents' \
|
||||
--exclude='*.files' \
|
||||
"${BUILDER_DIR}/" "${BUILD_WORK_DIR}/"
|
||||
# Also persist package cache to CACHE_ROOT so it survives a manual wipe of BUILD_WORK_DIR.
|
||||
LB_PKG_CACHE="${CACHE_ROOT}/lb-packages"
|
||||
mkdir -p "${LB_PKG_CACHE}"
|
||||
if [ -d "${BUILD_WORK_DIR}/cache/packages.chroot" ]; then
|
||||
rsync -a --delete "${BUILD_WORK_DIR}/cache/packages.chroot/" "${LB_PKG_CACHE}/"
|
||||
elif [ -d "${LB_PKG_CACHE}" ] && [ "$(ls -A "${LB_PKG_CACHE}" 2>/dev/null)" ]; then
|
||||
mkdir -p "${BUILD_WORK_DIR}/cache/packages.chroot"
|
||||
rsync -a "${LB_PKG_CACHE}/" "${BUILD_WORK_DIR}/cache/packages.chroot/"
|
||||
fi
|
||||
rsync -a "${OVERLAY_DIR}/" "${OVERLAY_STAGE_DIR}/"
|
||||
rm -f \
|
||||
"${OVERLAY_STAGE_DIR}/etc/bee-ssh-password-fallback" \
|
||||
@@ -306,7 +337,9 @@ sh "${BUILDER_DIR}/build-nccl-tests.sh" \
|
||||
"${NCCL_TESTS_VERSION}" \
|
||||
"${NCCL_VERSION}" \
|
||||
"${NCCL_CUDA_VERSION}" \
|
||||
"${DIST_DIR}"
|
||||
"${DIST_DIR}" \
|
||||
"${NVCC_VERSION}" \
|
||||
"${DEBIAN_VERSION}"
|
||||
|
||||
NCCL_TESTS_CACHE="${DIST_DIR}/nccl-tests-${NCCL_TESTS_VERSION}"
|
||||
cp "${NCCL_TESTS_CACHE}/bin/all_reduce_perf" "${OVERLAY_STAGE_DIR}/usr/local/bin/all_reduce_perf"
|
||||
@@ -340,6 +373,14 @@ if [ -f "${OVERLAY_STAGE_DIR}/etc/motd" ]; then
|
||||
mv "${OVERLAY_STAGE_DIR}/etc/motd.patched" "${OVERLAY_STAGE_DIR}/etc/motd"
|
||||
fi
|
||||
|
||||
# --- substitute version placeholders in package list ---
|
||||
sed -i \
|
||||
-e "s/%%DCGM_VERSION%%/${DCGM_VERSION}/g" \
|
||||
-e "s/%%ROCM_VERSION%%/${ROCM_VERSION}/g" \
|
||||
-e "s/%%ROCM_SMI_VERSION%%/${ROCM_SMI_VERSION}/g" \
|
||||
"${BUILD_WORK_DIR}/config/package-lists/bee.list.chroot" \
|
||||
"${BUILD_WORK_DIR}/config/archives/rocm.list.chroot"
|
||||
|
||||
# --- sync overlay into live-build includes.chroot ---
|
||||
LB_DIR="${BUILD_WORK_DIR}"
|
||||
LB_INCLUDES="${LB_DIR}/config/includes.chroot"
|
||||
|
||||
29
iso/builder/config/archives/nvidia-cuda.key.chroot
Normal file
29
iso/builder/config/archives/nvidia-cuda.key.chroot
Normal file
@@ -0,0 +1,29 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
Version: GnuPG v2.0.22 (GNU/Linux)
|
||||
|
||||
mQINBGJYmlEBEAC6nJmeqByeReM+MSy4palACCnfOg4pOxffrrkldxz4jrDOZNK4
|
||||
q8KG+ZbXrkdP0e9qTFRvZzN+A6Jw3ySfoiKXRBw5l2Zp81AYkghV641OpWNjZOyL
|
||||
syKEtST9LR1ttHv1ZI71pj8NVG/EnpimZPOblEJ1OpibJJCXLrbn+qcJ8JNuGTSK
|
||||
6v2aLBmhR8VR/aSJpmkg7fFjcGklweTI8+Ibj72HuY9JRD/+dtUoSh7z037mWo56
|
||||
ee02lPFRD0pHOEAlLSXxFO/SDqRVMhcgHk0a8roCF+9h5Ni7ZUyxlGK/uHkqN7ED
|
||||
/U/ATpGKgvk4t23eTpdRC8FXAlBZQyf/xnhQXsyF/z7+RV5CL0o1zk1LKgo+5K32
|
||||
5ka5uZb6JSIrEPUaCPEMXu6EEY8zSFnCrRS/Vjkfvc9ViYZWzJ387WTjAhMdS7wd
|
||||
PmdDWw2ASGUP4FrfCireSZiFX+ZAOspKpZdh0P5iR5XSx14XDt3jNK2EQQboaJAD
|
||||
uqksItatOEYNu4JsCbc24roJvJtGhpjTnq1/dyoy6K433afU0DS2ZPLthLpGqeyK
|
||||
MKNY7a2WjxhRmCSu5Zok/fGKcO62XF8a3eSj4NzCRv8LM6mG1Oekz6Zz+tdxHg19
|
||||
ufHO0et7AKE5q+5VjE438Xpl4UWbM/Voj6VPJ9uzywDcnZXpeOqeTQh2pQARAQAB
|
||||
tCBjdWRhdG9vbHMgPGN1ZGF0b29sc0BudmlkaWEuY29tPokCOQQTAQIAIwUCYlia
|
||||
UQIbAwcLCQgHAwIBBhUIAgkKCwQWAgMBAh4BAheAAAoJEKS0aZY7+GPM1y4QALKh
|
||||
BqSozrYbe341Qu7SyxHQgjRCGi4YhI3bHCMj5F6vEOHnwiFH6YmFkxCYtqcGjca6
|
||||
iw7cCYMow/hgKLAPwkwSJ84EYpGLWx62+20rMM4OuZwauSUcY/kE2WgnQ74zbh3+
|
||||
MHs56zntJFfJ9G+NYidvwDWeZn5HIzR4CtxaxRgpiykg0s3ps6X0U+vuVcLnutBF
|
||||
7r81astvlVQERFbce/6KqHK+yj843Qrhb3JEolUoOETK06nD25bVtnAxe0QEyA90
|
||||
9MpRNLfR6BdjPpxqhphDcMOhJfyubAroQUxG/7S+Yw+mtEqHrL/dz9iEYqodYiSo
|
||||
zfi0b+HFI59sRkTfOBDBwb3kcARExwnvLJmqijiVqWkoJ3H67oA0XJN2nelucw+A
|
||||
Hb+Jt9BWjyzKWlLFDnVHdGicyRJ0I8yqi32w8hGeXmu3tU58VWJrkXEXadBftmci
|
||||
pemb6oZ/r5SCkW6kxr2PsNWcJoebUdynyOQGbVwpMtJAnjOYp0ObKOANbcIg+tsi
|
||||
kyCIO5TiY3ADbBDPCeZK8xdcugXoW5WFwACGC0z+Cn0mtw8z3VGIPAMSCYmLusgW
|
||||
t2+EpikwrP2inNp5Pc+YdczRAsa4s30Jpyv/UHEG5P9GKnvofaxJgnU56lJIRPzF
|
||||
iCUGy6cVI0Fq777X/ME1K6A/bzZ4vRYNx8rUmVE5
|
||||
=DO7z
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
1
iso/builder/config/archives/nvidia-cuda.list.chroot
Normal file
1
iso/builder/config/archives/nvidia-cuda.list.chroot
Normal file
@@ -0,0 +1 @@
|
||||
deb https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/ /
|
||||
BIN
iso/builder/config/archives/rocm.key.chroot
Normal file
BIN
iso/builder/config/archives/rocm.key.chroot
Normal file
Binary file not shown.
1
iso/builder/config/archives/rocm.list.chroot
Normal file
1
iso/builder/config/archives/rocm.list.chroot
Normal file
@@ -0,0 +1 @@
|
||||
deb https://repo.radeon.com/rocm/apt/%%ROCM_VERSION%% jammy main
|
||||
@@ -8,7 +8,7 @@ else
|
||||
fi
|
||||
|
||||
if loadfont $font ; then
|
||||
set gfxmode=800x600
|
||||
set gfxmode=1920x1080,1280x1024,auto
|
||||
set gfxpayload=keep
|
||||
insmod efi_gop
|
||||
insmod efi_uga
|
||||
|
||||
@@ -29,6 +29,10 @@ menuentry "EASY-BEE (fail-safe)" {
|
||||
initrd @INITRD_LIVE@
|
||||
}
|
||||
|
||||
menuentry "Memory Test (memtest86+)" {
|
||||
linux16 /boot/memtest86+.bin
|
||||
}
|
||||
|
||||
if [ "${grub_platform}" = "efi" ]; then
|
||||
menuentry "UEFI Firmware Settings" {
|
||||
fwsetup
|
||||
|
||||
@@ -29,6 +29,7 @@ systemctl enable bee-audit.service
|
||||
systemctl enable bee-web.service
|
||||
systemctl enable bee-sshsetup.service
|
||||
systemctl enable ssh.service
|
||||
systemctl enable lightdm.service 2>/dev/null || true
|
||||
systemctl enable qemu-guest-agent.service 2>/dev/null || true
|
||||
systemctl enable serial-getty@ttyS0.service 2>/dev/null || true
|
||||
systemctl enable serial-getty@ttyS1.service 2>/dev/null || true
|
||||
@@ -45,6 +46,12 @@ chmod +x /usr/local/bin/bee-log-run 2>/dev/null || true
|
||||
# Reload udev rules
|
||||
udevadm control --reload-rules 2>/dev/null || true
|
||||
|
||||
# rocm-smi symlink (package installs to /opt/rocm-*/bin/rocm-smi)
|
||||
if [ ! -e /usr/local/bin/rocm-smi ]; then
|
||||
smi_path="$(find /opt -path '*/bin/rocm-smi' -type f 2>/dev/null | sort | tail -1)"
|
||||
[ -n "${smi_path}" ] && ln -sf "${smi_path}" /usr/local/bin/rocm-smi
|
||||
fi
|
||||
|
||||
# Create export directory
|
||||
mkdir -p /appdata/bee/export
|
||||
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
#!/bin/sh
|
||||
# 9001-amd-rocm.hook.chroot — install AMD ROCm SMI tool for Instinct GPU monitoring.
|
||||
# Runs inside the live-build chroot. Adds AMD's apt repository and installs
|
||||
# rocm-smi-lib which provides the `rocm-smi` CLI (analogous to nvidia-smi).
|
||||
#
|
||||
# AMD does NOT publish Debian Bookworm packages. The repo uses Ubuntu codenames
|
||||
# (jammy/noble). We use jammy (Ubuntu 22.04) — its packages install cleanly on
|
||||
# Debian 12 (Bookworm) due to compatible glibc/libstdc++.
|
||||
# Tried versions newest-first; falls back if a point release is missing.
|
||||
|
||||
set -e
|
||||
|
||||
# Ubuntu codename to use for the AMD repo (Debian has no AMD packages).
|
||||
ROCM_UBUNTU_DIST="jammy"
|
||||
|
||||
# ROCm point-releases to try newest-first. AMD drops old point releases
|
||||
# from the repo, so we walk backwards until one responds 200.
|
||||
ROCM_CANDIDATES="6.3.4 6.3.3 6.3.2 6.3.1 6.3 6.2.4 6.2.3 6.2.2 6.2.1 6.2"
|
||||
|
||||
ROCM_KEYRING="/etc/apt/keyrings/rocm.gpg"
|
||||
ROCM_LIST="/etc/apt/sources.list.d/rocm.list"
|
||||
APT_UPDATED=0
|
||||
|
||||
mkdir -p /etc/apt/keyrings
|
||||
|
||||
ensure_tool() {
|
||||
tool="$1"
|
||||
pkg="$2"
|
||||
if command -v "${tool}" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if [ "${APT_UPDATED}" -eq 0 ]; then
|
||||
apt-get update -qq
|
||||
APT_UPDATED=1
|
||||
fi
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${pkg}"
|
||||
}
|
||||
|
||||
ensure_cert_bundle() {
|
||||
if [ -s /etc/ssl/certs/ca-certificates.crt ]; then
|
||||
return 0
|
||||
fi
|
||||
if [ "${APT_UPDATED}" -eq 0 ]; then
|
||||
apt-get update -qq
|
||||
APT_UPDATED=1
|
||||
fi
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates
|
||||
}
|
||||
|
||||
# live-build chroot may not include fetch/signing tools yet
|
||||
if ! ensure_cert_bundle || ! ensure_tool wget wget || ! ensure_tool gpg gpg; then
|
||||
echo "WARN: failed to install wget/gpg/ca-certificates prerequisites — skipping ROCm install"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Download and import AMD GPG key
|
||||
if ! wget -qO- "https://repo.radeon.com/rocm/rocm.gpg.key" \
|
||||
| gpg --dearmor --yes --output "${ROCM_KEYRING}"; then
|
||||
echo "WARN: failed to fetch AMD ROCm GPG key — skipping ROCm install"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Try each ROCm version until apt-get update succeeds.
|
||||
# AMD repo uses Ubuntu codenames; bookworm is not published — use jammy.
|
||||
ROCM_VERSION=""
|
||||
for candidate in ${ROCM_CANDIDATES}; do
|
||||
cat > "${ROCM_LIST}" <<EOF
|
||||
deb [arch=amd64 signed-by=${ROCM_KEYRING}] https://repo.radeon.com/rocm/apt/${candidate} ${ROCM_UBUNTU_DIST} main
|
||||
EOF
|
||||
if apt-get update -qq 2>/dev/null; then
|
||||
ROCM_VERSION="${candidate}"
|
||||
echo "=== AMD ROCm ${ROCM_VERSION} (${ROCM_UBUNTU_DIST}): repository available ==="
|
||||
break
|
||||
fi
|
||||
echo "WARN: ROCm ${candidate} not available, trying next..."
|
||||
rm -f "${ROCM_LIST}"
|
||||
done
|
||||
|
||||
if [ -z "${ROCM_VERSION}" ]; then
|
||||
echo "WARN: no ROCm apt repository available — skipping ROCm install"
|
||||
rm -f "${ROCM_KEYRING}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# rocm-smi-lib provides the rocm-smi CLI tool for GPU monitoring
|
||||
if DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends rocm-smi-lib; then
|
||||
echo "=== AMD ROCm: rocm-smi-lib installed ==="
|
||||
if [ -x /opt/rocm/bin/rocm-smi ]; then
|
||||
ln -sf /opt/rocm/bin/rocm-smi /usr/local/bin/rocm-smi
|
||||
else
|
||||
smi_path="$(find /opt -path '*/bin/rocm-smi' -type f 2>/dev/null | sort | tail -1)"
|
||||
if [ -n "${smi_path}" ]; then
|
||||
ln -sf "${smi_path}" /usr/local/bin/rocm-smi
|
||||
fi
|
||||
fi
|
||||
rocm-smi --version 2>/dev/null || true
|
||||
else
|
||||
echo "WARN: rocm-smi-lib install failed — AMD GPU monitoring unavailable"
|
||||
fi
|
||||
|
||||
# Clean up apt lists to keep ISO size down
|
||||
rm -f "${ROCM_LIST}"
|
||||
apt-get clean
|
||||
@@ -1,66 +0,0 @@
|
||||
#!/bin/sh
|
||||
# 9002-nvidia-dcgm.hook.chroot — install NVIDIA DCGM inside the live-build chroot.
|
||||
# DCGM (Data Center GPU Manager) provides dcgmi diag for acceptance testing.
|
||||
# Adds NVIDIA's CUDA apt repository (debian12/x86_64) and installs datacenter-gpu-manager.
|
||||
|
||||
set -e
|
||||
|
||||
NVIDIA_KEYRING="/usr/share/keyrings/nvidia-cuda.gpg"
|
||||
NVIDIA_LIST="/etc/apt/sources.list.d/nvidia-cuda.list"
|
||||
NVIDIA_KEY_URL="https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/3bf863cc.pub"
|
||||
NVIDIA_REPO="https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/"
|
||||
APT_UPDATED=0
|
||||
|
||||
mkdir -p /usr/share/keyrings /etc/apt/sources.list.d
|
||||
|
||||
ensure_tool() {
|
||||
tool="$1"
|
||||
pkg="$2"
|
||||
if command -v "${tool}" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if [ "${APT_UPDATED}" -eq 0 ]; then
|
||||
apt-get update -qq
|
||||
APT_UPDATED=1
|
||||
fi
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${pkg}"
|
||||
}
|
||||
|
||||
ensure_cert_bundle() {
|
||||
if [ -s /etc/ssl/certs/ca-certificates.crt ]; then
|
||||
return 0
|
||||
fi
|
||||
if [ "${APT_UPDATED}" -eq 0 ]; then
|
||||
apt-get update -qq
|
||||
APT_UPDATED=1
|
||||
fi
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates
|
||||
}
|
||||
|
||||
if ! ensure_cert_bundle || ! ensure_tool wget wget || ! ensure_tool gpg gpg; then
|
||||
echo "WARN: prerequisites missing — skipping DCGM install"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Download and import NVIDIA GPG key
|
||||
if ! wget -qO- "${NVIDIA_KEY_URL}" | gpg --dearmor --yes --output "${NVIDIA_KEYRING}"; then
|
||||
echo "WARN: failed to fetch NVIDIA GPG key — skipping DCGM install"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cat > "${NVIDIA_LIST}" <<EOF
|
||||
deb [signed-by=${NVIDIA_KEYRING}] ${NVIDIA_REPO} /
|
||||
EOF
|
||||
|
||||
apt-get update -qq
|
||||
|
||||
if DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends datacenter-gpu-manager; then
|
||||
echo "=== DCGM: datacenter-gpu-manager installed ==="
|
||||
dcgmi --version 2>/dev/null || true
|
||||
else
|
||||
echo "WARN: datacenter-gpu-manager install failed — DCGM unavailable"
|
||||
fi
|
||||
|
||||
# Clean up apt lists to keep ISO size down
|
||||
rm -f "${NVIDIA_LIST}"
|
||||
apt-get clean
|
||||
32
iso/builder/config/hooks/normal/9999-slim.hook.chroot
Executable file
32
iso/builder/config/hooks/normal/9999-slim.hook.chroot
Executable file
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
# 9999-slim.hook.chroot — strip non-essential files to reduce squashfs size.
|
||||
set -e
|
||||
|
||||
# ── Man pages and documentation ───────────────────────────────────────────────
|
||||
find /usr/share/man -mindepth 1 -delete 2>/dev/null || true
|
||||
find /usr/share/doc -mindepth 1 ! -name 'copyright' -delete 2>/dev/null || true
|
||||
find /usr/share/info -mindepth 1 -delete 2>/dev/null || true
|
||||
find /usr/share/groff -mindepth 1 -delete 2>/dev/null || true
|
||||
find /usr/share/lintian -mindepth 1 -delete 2>/dev/null || true
|
||||
|
||||
# ── Locales — keep only C and en_US ──────────────────────────────────────────
|
||||
find /usr/share/locale -mindepth 1 -maxdepth 1 \
|
||||
! -name 'en' ! -name 'en_US' ! -name 'locale.alias' \
|
||||
-exec rm -rf {} + 2>/dev/null || true
|
||||
find /usr/share/i18n/locales -mindepth 1 \
|
||||
! -name 'en_US' ! -name 'i18n' ! -name 'iso14651_t1' ! -name 'iso14651_t1_common' \
|
||||
-delete 2>/dev/null || true
|
||||
|
||||
# ── Python cache ──────────────────────────────────────────────────────────────
|
||||
find /usr /opt -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true
|
||||
find /usr /opt -name '*.pyc' -delete 2>/dev/null || true
|
||||
|
||||
# ── APT cache and lists ───────────────────────────────────────────────────────
|
||||
apt-get clean
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# ── Misc ──────────────────────────────────────────────────────────────────────
|
||||
rm -rf /tmp/* /var/tmp/* 2>/dev/null || true
|
||||
find /var/log -type f -delete 2>/dev/null || true
|
||||
|
||||
echo "=== slim: done ==="
|
||||
@@ -43,7 +43,9 @@ sudo
|
||||
zstd
|
||||
mstflint
|
||||
memtester
|
||||
memtest86+
|
||||
stress-ng
|
||||
stressapptest
|
||||
|
||||
# QR codes (for displaying audit results)
|
||||
qrencode
|
||||
@@ -52,15 +54,29 @@ qrencode
|
||||
openbox
|
||||
tint2
|
||||
xorg
|
||||
xinit
|
||||
xterm
|
||||
chromium
|
||||
xserver-xorg-video-fbdev
|
||||
xserver-xorg-video-vesa
|
||||
lightdm
|
||||
|
||||
# Firmware
|
||||
firmware-linux-free
|
||||
firmware-linux-nonfree
|
||||
firmware-misc-nonfree
|
||||
firmware-amd-graphics
|
||||
firmware-realtek
|
||||
firmware-intel-sound
|
||||
firmware-bnx2
|
||||
firmware-bnx2x
|
||||
firmware-cavium
|
||||
firmware-qlogic
|
||||
|
||||
# NVIDIA DCGM (Data Center GPU Manager) — dcgmi diag for acceptance testing
|
||||
datacenter-gpu-manager=1:%%DCGM_VERSION%%
|
||||
|
||||
# AMD ROCm SMI — GPU monitoring for Instinct cards (repo: rocm/apt/6.3.4 jammy)
|
||||
rocm-smi-lib=%%ROCM_SMI_VERSION%%
|
||||
|
||||
# glibc compat helpers (for any external binaries that need it)
|
||||
libc6
|
||||
|
||||
@@ -7,4 +7,5 @@ EndSection
|
||||
Section "Screen"
|
||||
Identifier "screen0"
|
||||
Device "fbdev"
|
||||
DefaultDepth 24
|
||||
EndSection
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
[Seat:*]
|
||||
autologin-user=bee
|
||||
autologin-user-timeout=0
|
||||
autologin-session=openbox
|
||||
user-session=openbox
|
||||
@@ -12,6 +12,6 @@
|
||||
Export dir: /appdata/bee/export
|
||||
Self-check: /appdata/bee/export/runtime-health.json
|
||||
|
||||
Open TUI: bee-tui
|
||||
Web UI: http://<ip>/
|
||||
|
||||
SSH access: key auth (developers) or bee/eeb (password fallback)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
[Journal]
|
||||
# Do not forward service logs to the console — bee-tui runs on tty1
|
||||
# and log spam makes the screen unusable on physical monitors.
|
||||
# Do not forward service logs to the console — prevents log spam on
|
||||
# physical monitors and the local openbox desktop.
|
||||
ForwardToConsole=no
|
||||
|
||||
@@ -10,6 +10,7 @@ Restart=always
|
||||
RestartSec=2
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
LimitMEMLOCK=infinity
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
[Service]
|
||||
# On server hardware without a usable framebuffer X may fail to start.
|
||||
# Limit restarts so the console is not flooded on headless deployments.
|
||||
RestartSec=10
|
||||
StartLimitIntervalSec=60
|
||||
StartLimitBurst=3
|
||||
@@ -1,10 +1 @@
|
||||
export PATH="/usr/local/bin:$PATH"
|
||||
|
||||
# On tty1 (IPMI KVM console): start X desktop directly on VT1
|
||||
# so the KVM shows the graphical UI without switching VTs.
|
||||
if [ "$(tty 2>/dev/null)" = "/dev/tty1" ] \
|
||||
&& [ -z "${DISPLAY:-}" ]; then
|
||||
startx /usr/local/bin/bee-openbox-session -- :0 vt1 -nolisten tcp
|
||||
echo ""
|
||||
echo "X session ended. Type 'startx' to retry or use the shell."
|
||||
fi
|
||||
|
||||
@@ -12,14 +12,19 @@ while [ $i -lt 15 ]; do
|
||||
i=$((i+1))
|
||||
done
|
||||
|
||||
# Disable screensaver and DPMS
|
||||
xset s off
|
||||
xset -dpms
|
||||
xset s noblank
|
||||
|
||||
tint2 &
|
||||
chromium \
|
||||
--no-sandbox \
|
||||
--disable-infobars \
|
||||
--disable-translate \
|
||||
--no-first-run \
|
||||
--disable-session-crashed-bubble \
|
||||
--disable-features=TranslateUI \
|
||||
--start-fullscreen \
|
||||
http://localhost/ &
|
||||
|
||||
exec openbox
|
||||
|
||||
@@ -1,50 +1,163 @@
|
||||
#!/bin/sh
|
||||
# Quick network configurator for the local console.
|
||||
# Type 'a' at any prompt to abort, 'b' to go back.
|
||||
set -e
|
||||
|
||||
# List interfaces (exclude lo)
|
||||
IFACES=$(ip -o link show | awk -F': ' '$2 != "lo" {print $2}' | cut -d@ -f1)
|
||||
abort() { echo "Aborted."; exit 0; }
|
||||
|
||||
echo "Interfaces:"
|
||||
i=1
|
||||
for iface in $IFACES; do
|
||||
ip=$(ip -4 addr show "$iface" 2>/dev/null | awk '/inet /{print $2}' | head -1)
|
||||
echo " $i) $iface ${ip:-no IP}"
|
||||
i=$((i+1))
|
||||
done
|
||||
echo ""
|
||||
printf "Interface name [or Enter to pick first]: "
|
||||
read IFACE
|
||||
if [ -z "$IFACE" ]; then
|
||||
IFACE=$(echo "$IFACES" | head -1)
|
||||
fi
|
||||
echo "Selected: $IFACE"
|
||||
echo ""
|
||||
echo " 1) DHCP"
|
||||
echo " 2) Static"
|
||||
printf "Mode [1]: "
|
||||
read MODE
|
||||
MODE=${MODE:-1}
|
||||
ask() {
|
||||
# ask VARNAME "prompt" [default]
|
||||
# Sets VARNAME. Returns 1 on 'b' (back), calls abort on 'a'.
|
||||
_var="$1"; _prompt="$2"; _default="$3"
|
||||
while true; do
|
||||
if [ -n "$_default" ]; then
|
||||
printf "%s [%s] (b=back a=abort): " "$_prompt" "$_default"
|
||||
else
|
||||
printf "%s (b=back a=abort): " "$_prompt"
|
||||
fi
|
||||
read _input
|
||||
case "$_input" in
|
||||
a|A) abort ;;
|
||||
b|B) return 1 ;;
|
||||
"")
|
||||
if [ -n "$_default" ]; then
|
||||
eval "$_var=\"\$_default\""
|
||||
return 0
|
||||
else
|
||||
echo " Required — please enter a value."
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
eval "$_var=\"\$_input\""
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
if [ "$MODE" = "1" ]; then
|
||||
# ── Step 1: choose interface ───────────────────────────────────────────────────
|
||||
|
||||
choose_iface() {
|
||||
IFACES=$(ip -o link show | awk -F': ' '$2 != "lo" {print $2}' | cut -d@ -f1)
|
||||
if [ -z "$IFACES" ]; then
|
||||
echo "No network interfaces found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Interfaces:"
|
||||
i=1
|
||||
for iface in $IFACES; do
|
||||
ip=$(ip -4 addr show "$iface" 2>/dev/null | awk '/inet /{print $2}' | head -1)
|
||||
echo " $i) $iface ${ip:-no IP}"
|
||||
i=$((i+1))
|
||||
done
|
||||
echo ""
|
||||
|
||||
FIRST=$(echo "$IFACES" | head -1)
|
||||
while true; do
|
||||
printf "Interface number or name [%s] (a=abort): " "$FIRST"
|
||||
read INPUT
|
||||
case "$INPUT" in
|
||||
a|A) abort ;;
|
||||
"")
|
||||
IFACE="$FIRST"
|
||||
break
|
||||
;;
|
||||
*)
|
||||
if echo "$INPUT" | grep -qE '^[0-9]+$'; then
|
||||
IFACE=$(echo "$IFACES" | awk "NR==$INPUT")
|
||||
if [ -z "$IFACE" ]; then
|
||||
echo " No interface #$INPUT — try again."
|
||||
continue
|
||||
fi
|
||||
else
|
||||
# Validate name exists
|
||||
if ! echo "$IFACES" | grep -qx "$INPUT"; then
|
||||
echo " Unknown interface '$INPUT' — try again."
|
||||
continue
|
||||
fi
|
||||
IFACE="$INPUT"
|
||||
fi
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
echo "Selected: $IFACE"
|
||||
}
|
||||
|
||||
# ── Step 2: choose mode ────────────────────────────────────────────────────────
|
||||
|
||||
choose_mode() {
|
||||
echo ""
|
||||
echo " 1) DHCP"
|
||||
echo " 2) Static IP"
|
||||
echo ""
|
||||
while true; do
|
||||
printf "Mode [1] (b=back a=abort): "
|
||||
read INPUT
|
||||
case "$INPUT" in
|
||||
a|A) abort ;;
|
||||
b|B) return 1 ;;
|
||||
""|1) MODE=dhcp; break ;;
|
||||
2) MODE=static; break ;;
|
||||
*) echo " Enter 1 or 2." ;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# ── Step 3a: DHCP ─────────────────────────────────────────────────────────────
|
||||
|
||||
run_dhcp() {
|
||||
echo "Running DHCP on $IFACE..."
|
||||
dhclient -v "$IFACE"
|
||||
else
|
||||
printf "IP address (e.g. 192.168.1.100/24): "
|
||||
read ADDR
|
||||
printf "Gateway (e.g. 192.168.1.1): "
|
||||
read GW
|
||||
printf "DNS [8.8.8.8]: "
|
||||
read DNS
|
||||
DNS=${DNS:-8.8.8.8}
|
||||
}
|
||||
|
||||
# ── Step 3b: Static ───────────────────────────────────────────────────────────
|
||||
|
||||
run_static() {
|
||||
while true; do
|
||||
ask ADDR "IP address (e.g. 192.168.1.100/24)" || return 1
|
||||
# Basic format check: must contain a dot and a /
|
||||
if ! echo "$ADDR" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[0-9]+$'; then
|
||||
echo " Invalid format — use x.x.x.x/prefix (e.g. 192.168.1.10/24)."
|
||||
continue
|
||||
fi
|
||||
break
|
||||
done
|
||||
|
||||
while true; do
|
||||
ask GW "Gateway (e.g. 192.168.1.1)" || return 1
|
||||
if ! echo "$GW" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'; then
|
||||
echo " Invalid IP address."
|
||||
continue
|
||||
fi
|
||||
break
|
||||
done
|
||||
|
||||
ask DNS "DNS server" "8.8.8.8" || return 1
|
||||
|
||||
ip addr flush dev "$IFACE"
|
||||
ip addr add "$ADDR" dev "$IFACE"
|
||||
ip link set "$IFACE" up
|
||||
ip route add default via "$GW"
|
||||
ip route add default via "$GW" 2>/dev/null || true
|
||||
echo "nameserver $DNS" > /etc/resolv.conf
|
||||
echo "Done."
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Main loop ─────────────────────────────────────────────────────────────────
|
||||
|
||||
choose_iface
|
||||
|
||||
while true; do
|
||||
choose_mode || { choose_iface; continue; }
|
||||
|
||||
if [ "$MODE" = "dhcp" ]; then
|
||||
run_dhcp && break
|
||||
else
|
||||
run_static && break || continue
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
ip -4 addr show "$IFACE"
|
||||
|
||||
6
iso/overlay/usr/share/xsessions/openbox.desktop
Normal file
6
iso/overlay/usr/share/xsessions/openbox.desktop
Normal file
@@ -0,0 +1,6 @@
|
||||
[Desktop Entry]
|
||||
Name=Openbox
|
||||
Comment=Bee Hardware Audit Desktop
|
||||
Exec=/usr/local/bin/bee-openbox-session
|
||||
TryExec=openbox
|
||||
Type=Application
|
||||
Reference in New Issue
Block a user