reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

refactor: harden diagnostics and consolidate runtime code

Browse Source
This commit is contained in:
Mikhail Chusavitin
2026-09-01 13:01:28 +03:00
parent ac4bc0b2b7
commit 0a6ca8ba0f
49 changed files with 1441 additions and 837 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/app/app.go
-8
View File
@@ -129,14 +129,6 @@ func (a *App) TPMPresent() bool {
return a.sat.TPMPresent()
}
func (a *App) IsLiveMediaInRAM() bool {
return a.installer.IsLiveMediaInRAM()
}
func (a *App) LiveBootSource() platform.LiveBootSource {
return a.installer.LiveBootSource()
}
func (a *App) LiveMediaRAMState() platform.LiveMediaRAMState {
return a.installer.LiveMediaRAMState()
}
audit/internal/app/app_install.go
-19
View File
@@ -1,7 +1,6 @@
package app
import (
"context"
"fmt"
"os"
"path/filepath"
@@ -14,20 +13,6 @@ func (a *App) ListRemovableTargets() ([]platform.RemovableTarget, error) {
return a.exports.ListRemovableTargets()
}
// ListScenarioFilesOnRemovableMedia lists scenarios/*.json found on any
// mounted removable target (e.g. the blackbox USB stick) — see
// platform.System.ListScenarioFilesOnRemovableMedia.
func (a *App) ListScenarioFilesOnRemovableMedia() ([]platform.ScenarioFileOnRemovableMedia, error) {
return a.exports.ListScenarioFilesOnRemovableMedia()
}
// ReadScenarioFromRemovableMedia reads scenarios/<name>.json from whichever
// mounted removable target has it — see
// platform.System.ReadScenarioFromRemovableMedia.
func (a *App) ReadScenarioFromRemovableMedia(name string) ([]byte, error) {
return a.exports.ReadScenarioFromRemovableMedia(name)
}
// ListAvailableScenarios lists every scenario runnable via ReadScenario:
// shipped with the image plus anything found on removable media — see
// platform.System.ListAvailableScenarios.
@@ -86,7 +71,3 @@ func (a *App) ExportSupportBundleResult(target platform.RemovableTarget) (Action
func (a *App) ListInstallDisks() ([]platform.InstallDisk, error) {
return a.installer.ListInstallDisks()
}
func (a *App) InstallToDisk(ctx context.Context, device string, logFile string) error {
return a.installer.InstallToDisk(ctx, device, logFile)
}
audit/internal/app/app_network.go
-12
View File
@@ -15,28 +15,16 @@ func (a *App) DefaultRoute() string {
return a.network.DefaultRoute()
}
func (a *App) DHCPOne(iface string) (string, error) {
return a.network.DHCPOne(iface)
}
func (a *App) DHCPOneResult(iface string) (ActionResult, error) {
body, err := a.network.DHCPOne(iface)
return ActionResult{Title: "DHCP: " + iface, Body: bodyOr(body, "DHCP completed.")}, err
}
func (a *App) DHCPAll() (string, error) {
return a.network.DHCPAll()
}
func (a *App) DHCPAllResult() (ActionResult, error) {
body, err := a.network.DHCPAll()
return ActionResult{Title: "DHCP: all interfaces", Body: bodyOr(body, "DHCP completed.")}, err
}
func (a *App) SetStaticIPv4(cfg platform.StaticIPv4Config) (string, error) {
return a.network.SetStaticIPv4(cfg)
}
func (a *App) SetInterfaceState(iface string, up bool) error {
return a.network.SetInterfaceState(iface, up)
}
audit/internal/app/app_packs.go
+8 -63
View File
@@ -110,36 +110,28 @@ func (a *App) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir s
return a.sat.RunNvidiaTargetedStressValidatePack(ctx, baseDir, durationSec, gpuIndices, logFunc)
}
func (a *App) RunNvidiaStressPack(baseDir string, opts platform.NvidiaStressOptions, logFunc func(string)) (string, error) {
return a.RunNvidiaStressPackCtx(context.Background(), baseDir, opts, logFunc)
}
func (a *App) RunNvidiaBenchmark(baseDir string, opts platform.NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
return a.RunNvidiaBenchmarkCtx(context.Background(), baseDir, opts, logFunc)
}
func (a *App) RunNvidiaBenchmarkCtx(ctx context.Context, baseDir string, opts platform.NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultBeeBenchPerfDir
}
resolved, err := a.ensureBenchmarkPowerAutotune(ctx, baseDir, opts, "performance", logFunc)
if err != nil {
return "", err
}
opts.ServerPowerSource = resolved.SelectedSource
return a.sat.RunNvidiaBenchmark(ctx, baseDir, opts, logFunc)
return a.runNvidiaBenchmarkKind(ctx, baseDir, DefaultBeeBenchPerfDir, opts, "performance", logFunc, a.sat.RunNvidiaBenchmark)
}
func (a *App) RunNvidiaPowerBenchCtx(ctx context.Context, baseDir string, opts platform.NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
return a.runNvidiaBenchmarkKind(ctx, baseDir, DefaultBeeBenchPowerDir, opts, "power-fit", logFunc, a.sat.RunNvidiaPowerBench)
}
func (a *App) runNvidiaBenchmarkKind(ctx context.Context, baseDir, defaultBaseDir string, opts platform.NvidiaBenchmarkOptions, benchmarkKind string, logFunc func(string), run func(context.Context, string, platform.NvidiaBenchmarkOptions, func(string)) (string, error)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultBeeBenchPowerDir
baseDir = defaultBaseDir
}
resolved, err := a.ensureBenchmarkPowerAutotune(ctx, baseDir, opts, "power-fit", logFunc)
resolved, err := a.ensureBenchmarkPowerAutotune(ctx, baseDir, opts, benchmarkKind, logFunc)
if err != nil {
return "", err
}
opts.ServerPowerSource = resolved.SelectedSource
return a.sat.RunNvidiaPowerBench(ctx, baseDir, opts, logFunc)
return run(ctx, baseDir, opts, logFunc)
}
func (a *App) RunNvidiaPowerSourceAutotuneCtx(ctx context.Context, baseDir string, opts platform.NvidiaBenchmarkOptions, benchmarkKind string, logFunc func(string)) (string, error) {
@@ -226,10 +218,6 @@ func (a *App) RunMemoryAcceptancePackResult(baseDir string) (ActionResult, error
return ActionResult{Title: "Memory SAT", Body: satResultBody(path)}, err
}
func (a *App) RunCPUAcceptancePack(baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.RunCPUAcceptancePackCtx(context.Background(), baseDir, durationSec, logFunc)
}
func (a *App) RunCPUAcceptancePackCtx(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
@@ -267,10 +255,6 @@ func (a *App) RunNvidiaConfigCheckPackCtx(ctx context.Context, baseDir string, l
return a.sat.RunNvidiaConfigCheckPack(ctx, baseDir, logFunc)
}
func (a *App) RunNvidiaConfigCheckPack(baseDir string, logFunc func(string)) (string, error) {
return a.RunNvidiaConfigCheckPackCtx(context.Background(), baseDir, logFunc)
}
func (a *App) RunPCIeLinkCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
@@ -278,10 +262,6 @@ func (a *App) RunPCIeLinkCheckPackCtx(ctx context.Context, baseDir string, logFu
return a.sat.RunPCIeLinkCheckPack(ctx, baseDir, logFunc)
}
func (a *App) RunPCIeLinkCheckPack(baseDir string, logFunc func(string)) (string, error) {
return a.RunPCIeLinkCheckPackCtx(context.Background(), baseDir, logFunc)
}
func (a *App) RunNvidiaPCIeBandwidthPackCtx(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
@@ -289,22 +269,6 @@ func (a *App) RunNvidiaPCIeBandwidthPackCtx(ctx context.Context, baseDir string,
return a.sat.RunNvidiaPCIeBandwidthPack(ctx, baseDir, gpuIndices, logFunc)
}
func (a *App) RunNvidiaPCIeBandwidthPack(baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
return a.RunNvidiaPCIeBandwidthPackCtx(context.Background(), baseDir, gpuIndices, logFunc)
}
func (a *App) DetectGPUVendor() string {
return a.sat.DetectGPUVendor()
}
func (a *App) ListAMDGPUs() ([]platform.AMDGPUInfo, error) {
return a.sat.ListAMDGPUs()
}
func (a *App) RunAMDAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
return a.RunAMDAcceptancePackCtx(context.Background(), baseDir, logFunc)
}
func (a *App) RunAMDAcceptancePackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
@@ -326,18 +290,6 @@ func (a *App) RunAMDMemBandwidthPackCtx(ctx context.Context, baseDir string, log
return a.sat.RunAMDMemBandwidthPack(ctx, baseDir, logFunc)
}
func (a *App) RunMemoryStressPack(baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.RunMemoryStressPackCtx(context.Background(), baseDir, durationSec, logFunc)
}
func (a *App) RunSATStressPack(baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.RunSATStressPackCtx(context.Background(), baseDir, durationSec, logFunc)
}
func (a *App) RunAMDStressPack(baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.RunAMDStressPackCtx(context.Background(), baseDir, durationSec, logFunc)
}
func (a *App) RunMemoryStressPackCtx(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.sat.RunMemoryStressPack(ctx, baseDir, durationSec, logFunc)
}
@@ -353,13 +305,6 @@ func (a *App) RunAMDStressPackCtx(ctx context.Context, baseDir string, durationS
return a.sat.RunAMDStressPack(ctx, baseDir, durationSec, logFunc)
}
func (a *App) RunFanStressTest(ctx context.Context, baseDir string, opts platform.FanStressOptions) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
}
return a.sat.RunFanStressTest(ctx, baseDir, opts)
}
func (a *App) RunPlatformStress(ctx context.Context, baseDir string, opts platform.PlatformStressOptions, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
audit/internal/app/app_services.go
-8
View File
@@ -24,19 +24,11 @@ func (a *App) ServiceStatusResult(name string) (ActionResult, error) {
return ActionResult{Title: "service status: " + name, Body: bodyOr(body, "No status output.")}, err
}
func (a *App) ServiceDo(name string, action platform.ServiceAction) (string, error) {
return a.services.ServiceDo(name, action)
}
func (a *App) ServiceActionResult(name string, action platform.ServiceAction) (ActionResult, error) {
body, err := a.services.ServiceDo(name, action)
return ActionResult{Title: "service " + string(action) + ": " + name, Body: bodyOr(body, "Action completed.")}, err
}
func (a *App) TailFile(path string, lines int) string {
return a.tools.TailFile(path, lines)
}
func (a *App) CheckTools(names []string) []platform.ToolStatus {
return a.tools.CheckTools(names)
}
audit/internal/app/blackbox_archive.go
+6 -6
View File
@@ -3,6 +3,7 @@ package app
import (
"archive/zip"
"bufio"
"errors"
"io"
"io/fs"
"os"
@@ -61,9 +62,8 @@ func buildZipArchive(root, destPath string) error {
if err != nil {
return err
}
defer src.Close()
_, err = io.Copy(w, src)
return err
_, copyErr := io.Copy(w, src)
return errors.Join(copyErr, src.Close())
})
if walkErr != nil {
_ = zw.Close()
@@ -136,7 +136,7 @@ func isEOFLike(err error) bool {
// (first run, external tampering, a previous crash mid-write), it falls back
// to writing the whole archive rather than risk corrupting it with a wrong
// truncate point.
func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) error {
func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) (retErr error) {
newInfo, err := os.Stat(newLocalZipPath)
if err != nil {
return err
@@ -159,7 +159,7 @@ func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) error
if err != nil {
return err
}
defer target.Close()
defer func() { retErr = errors.Join(retErr, target.Close()) }()
if err := target.Truncate(prefixLen); err != nil {
return err
@@ -172,7 +172,7 @@ func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) error
if err != nil {
return err
}
defer src.Close()
defer func() { retErr = errors.Join(retErr, src.Close()) }()
if _, err := src.Seek(prefixLen, io.SeekStart); err != nil {
return err
}
audit/internal/app/component_status_db.go
+11 -2
View File
@@ -454,10 +454,19 @@ func satFailureDetailFromKV(kv map[string]string) string {
continue
}
job := strings.TrimSuffix(k, "_status")
detail := strings.TrimSpace(kv[job+"_detail"])
if rc, ok := kv[job+"_rc"]; ok && strings.TrimSpace(rc) != "" {
failed = append(failed, fmt.Sprintf("%s=%s (rc=%s)", job, v, rc))
entry := fmt.Sprintf("%s=%s (rc=%s)", job, v, rc)
if detail != "" {
entry += ": " + detail
}
failed = append(failed, entry)
} else {
failed = append(failed, fmt.Sprintf("%s=%s", job, v))
entry := fmt.Sprintf("%s=%s", job, v)
if detail != "" {
entry += ": " + detail
}
failed = append(failed, entry)
}
}
if len(failed) == 0 {
audit/internal/app/component_status_db_test.go
+31
View File
@@ -3,6 +3,7 @@ package app
import (
"os"
"path/filepath"
"strings"
"testing"
"bee/audit/internal/schema"
@@ -145,6 +146,21 @@ func TestSATFailureDetailFallsBackToFailedSubJobs(t *testing.T) {
}
}
func TestSATFailureDetailIncludesValidatorDetail(t *testing.T) {
runDir := t.TempDir()
summary := "2-all-reduce-perf_rc=0\n" +
"2-all-reduce-perf_status=FAILED\n" +
"2-all-reduce-perf_detail=NVLink state does not match selected topology: GPU0<->GPU1\n" +
"overall_status=FAILED\n"
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
t.Fatal(err)
}
got := SATFailureDetail(runDir)
if !strings.Contains(got, "NVLink state does not match selected topology") {
t.Fatalf("SATFailureDetail() = %q, want validator detail", got)
}
}
// TestSATFailureDetailEmptyWhenNoReasonFound guards the ultimate fallback:
// when summary.txt carries no identifiable per-job or warnings detail (e.g.
// unreadable or from an older binary version), callers must get "" so they
@@ -233,6 +249,21 @@ func TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets(t *tes
}
}
func TestApplySATResultToDBNvidiaInterconnectFailureReachesGPUComponent(t *testing.T) {
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
if err != nil {
t.Fatal(err)
}
ApplySATResultToDB(db, "nvidia-interconnect", writeSATSummary(t, "FAILED"))
rec, ok := db.Get("pcie:gpu:nvidia")
if !ok {
t.Fatal("nvidia-interconnect failure wrote no pcie:gpu:nvidia record")
}
if rec.Status != "Warning" {
t.Fatalf("status=%q want Warning", rec.Status)
}
}
// TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the
// hardware-ingest-contract.md rule that status_history is a transition log
// ("История переходов статусов"), not a per-poll journal. A component
audit/internal/app/support_bundle.go
+50 -34
View File
@@ -6,6 +6,7 @@ import (
"compress/gzip"
"context"
_ "embed"
"errors"
"fmt"
"io"
"os"
@@ -505,21 +506,16 @@ func BuildSupportBundle(exportDir string) (string, error) {
return "", err
}
archiveName := SupportBundleBaseName(now) + ".tar.gz"
archivePath := filepath.Join(os.TempDir(), archiveName)
if err := createSupportTarGz(archivePath, stageRoot); err != nil {
return "", err
}
return archivePath, nil
return createSupportTarGz(os.TempDir(), SupportBundleBaseName(now), stageRoot)
}
func SupportBundleBaseName(at time.Time) string {
at = at.UTC()
date := at.Format("2006-01-02")
tod := at.Format("150405")
ver := bundleVersion()
model := serverModelForBundle()
sn := serverSerialForBundle()
ver := sanitizeFilename(bundleVersion())
model := sanitizeFilename(serverModelForBundle())
sn := sanitizeFilename(serverSerialForBundle())
return fmt.Sprintf("%s (BEE-SP v%s) %s %s %s", date, ver, model, sn, tod)
}
@@ -617,17 +613,15 @@ func copyOptionalFile(src, dst string) error {
if err != nil {
return err
}
defer in.Close()
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return err
return errors.Join(err, in.Close())
}
out, err := os.Create(dst)
if err != nil {
return err
return errors.Join(err, in.Close())
}
defer out.Close()
_, err = io.Copy(out, in)
return err
_, copyErr := io.Copy(out, in)
return errors.Join(copyErr, in.Close(), out.Close())
}
func writeManifest(dst, exportDir, stageRoot string) error {
@@ -823,16 +817,12 @@ func copyPath(src, dst string) error {
}
return err
}
defer in.Close()
out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode().Perm())
if err != nil {
return err
return errors.Join(err, in.Close())
}
defer out.Close()
_, err = io.Copy(out, in)
return err
_, copyErr := io.Copy(out, in)
return errors.Join(copyErr, in.Close(), out.Close())
}
func copyPathFiltered(rootSrc, src, dst string, keep func(rel string, info os.FileInfo) bool) error {
@@ -874,21 +864,35 @@ func copyPathFiltered(rootSrc, src, dst string, keep func(rel string, info os.Fi
return copyPath(src, dst)
}
func createSupportTarGz(dst, srcDir string) error {
file, err := os.Create(dst)
func createSupportTarGz(dir, baseName, srcDir string) (string, error) {
archiveFile, err := os.CreateTemp(dir, baseName+"-*.partial")
if err != nil {
return err
return "", err
}
defer file.Close()
partialPath := archiveFile.Name()
if err := writeSupportTarGz(archiveFile, srcDir); err != nil {
_ = archiveFile.Close()
_ = os.Remove(partialPath)
return "", err
}
if err := archiveFile.Close(); err != nil {
_ = os.Remove(partialPath)
return "", err
}
archivePath := strings.TrimSuffix(partialPath, ".partial") + ".tar.gz"
if err := os.Rename(partialPath, archivePath); err != nil {
_ = os.Remove(partialPath)
return "", err
}
return archivePath, nil
}
func writeSupportTarGz(file *os.File, srcDir string) error {
gz := gzip.NewWriter(file)
defer gz.Close()
tw := tar.NewWriter(gz)
defer tw.Close()
base := filepath.Dir(srcDir)
return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
walkErr := filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
@@ -912,9 +916,21 @@ func createSupportTarGz(dst, srcDir string) error {
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(tw, f)
return err
_, copyErr := io.Copy(tw, f)
closeErr := f.Close()
if copyErr != nil {
return copyErr
}
return closeErr
})
if walkErr != nil {
_ = tw.Close()
_ = gz.Close()
return walkErr
}
if err := tw.Close(); err != nil {
_ = gz.Close()
return err
}
return gz.Close()
}
audit/internal/app/support_bundle_test.go
+97
View File
@@ -1,9 +1,14 @@
package app
import (
"archive/tar"
"compress/gzip"
"errors"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"
)
@@ -29,3 +34,95 @@ func TestWriteBundleDocs(t *testing.T) {
t.Fatalf("README.md should explain how to check SAT pass/fail:\n%s", readme)
}
}
func TestCreateSupportTarGzUsesIndependentFilesConcurrently(t *testing.T) {
tempDir := t.TempDir()
srcDir := filepath.Join(tempDir, "bee-support-stage-test")
if err := os.Mkdir(srcDir, 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(srcDir, "evidence.txt"), []byte("complete evidence\n"), 0644); err != nil {
t.Fatal(err)
}
const builds = 4
paths := make(chan string, builds)
errs := make(chan error, builds)
var wg sync.WaitGroup
for range builds {
wg.Add(1)
go func() {
defer wg.Done()
path, err := createSupportTarGz(tempDir, "bundle", srcDir)
if err != nil {
errs <- err
return
}
paths <- path
}()
}
wg.Wait()
close(paths)
close(errs)
for err := range errs {
t.Fatalf("create archive: %v", err)
}
seen := make(map[string]struct{}, builds)
for path := range paths {
if !strings.HasSuffix(path, ".tar.gz") {
t.Fatalf("archive path %q does not end in .tar.gz", path)
}
if _, exists := seen[path]; exists {
t.Fatalf("duplicate archive path %q", path)
}
seen[path] = struct{}{}
assertSupportArchiveEntry(t, path, "bee-support-stage-test/evidence.txt", "complete evidence\n")
}
if len(seen) != builds {
t.Fatalf("archive count = %d, want %d", len(seen), builds)
}
partials, err := filepath.Glob(filepath.Join(tempDir, "*.partial"))
if err != nil {
t.Fatal(err)
}
if len(partials) != 0 {
t.Fatalf("unpublished partial archives remain: %v", partials)
}
}
func assertSupportArchiveEntry(t *testing.T, path, wantName, wantBody string) {
t.Helper()
file, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer file.Close()
gz, err := gzip.NewReader(file)
if err != nil {
t.Fatal(err)
}
defer gz.Close()
tr := tar.NewReader(gz)
for {
header, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
t.Fatal(err)
}
if header.Name != wantName {
continue
}
body, err := io.ReadAll(tr)
if err != nil {
t.Fatal(err)
}
if string(body) != wantBody {
t.Fatalf("entry body = %q, want %q", body, wantBody)
}
return
}
t.Fatalf("archive %q is missing %q", path, wantName)
}