From 5aee1469035dcc07babfec02aa01e8861ce47d57 Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Thu, 9 Jul 2026 11:37:44 +0300 Subject: [PATCH] app: ship a README.md in every support bundle and blackbox capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An agent handed a bundle path had no way to know its layout without grepping through random logs first — confirmed by watching a separate session read bee-nvidia.log before anything else on a real bundle. Embeds a single README.md (bee-embed, internal/app/assets/) explaining what bee is and giving direct answers to the questions someone analyzing a bundle is most likely to ask (did the tests pass, what hardware is this, is a service healthy, RAID/GPU/NVLink state, etc), written at the bundle root by both BuildSupportBundle (support-bundle archive root, sibling of manifest.txt) and blackboxWorker.syncCycle (removable-media boot-folder root, which otherwise has no manifest.txt-equivalent pointing anywhere). Co-Authored-By: Claude Sonnet 5 --- audit/internal/app/app_test.go | 60 ++++++++++++ audit/internal/app/assets/README.md | 111 ++++++++++++++++++++++ audit/internal/app/blackbox.go | 7 ++ audit/internal/app/support_bundle.go | 17 ++++ audit/internal/app/support_bundle_test.go | 31 ++++++ 5 files changed, 226 insertions(+) create mode 100644 audit/internal/app/assets/README.md create mode 100644 audit/internal/app/support_bundle_test.go diff --git a/audit/internal/app/app_test.go b/audit/internal/app/app_test.go index d2cb1d2..6465f21 100644 --- a/audit/internal/app/app_test.go +++ b/audit/internal/app/app_test.go @@ -954,6 +954,66 @@ func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) { } } +// TestBuildSupportBundleIncludesOrientationDocs guards that README.md — +// written so an agent unfamiliar with bee can orient itself in the bundle +// without reverse-engineering the layout — always ships at the bundle root +// (sibling of manifest.txt, not buried under export/), and that +// manifest.txt's file listing (generated after it's written) picks it up +// too. +func TestBuildSupportBundleIncludesOrientationDocs(t *testing.T) { + tmp := t.TempDir() + exportDir := filepath.Join(tmp, "export") + if err := os.MkdirAll(exportDir, 0755); err != nil { + t.Fatal(err) + } + + archive, err := BuildSupportBundle(exportDir) + if err != nil { + t.Fatalf("BuildSupportBundle error: %v", err) + } + file, err := os.Open(archive) + if err != nil { + t.Fatalf("open archive: %v", err) + } + defer file.Close() + gzr, err := gzip.NewReader(file) + if err != nil { + t.Fatalf("gzip reader: %v", err) + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + var readmeMD, manifest string + var readmeAtRoot bool + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("read tar entry: %v", err) + } + // Root-level: exactly one path separator ("/README.md"), + // not nested under export/ or anywhere else. + switch { + case strings.HasSuffix(hdr.Name, "/README.md"): + readmeAtRoot = strings.Count(hdr.Name, "/") == 1 + body, _ := io.ReadAll(tr) + readmeMD = string(body) + case strings.HasSuffix(hdr.Name, "/manifest.txt"): + body, _ := io.ReadAll(tr) + manifest = string(body) + } + } + + if readmeMD == "" || !readmeAtRoot { + t.Fatalf("expected README.md at bundle root, got content=%q atRoot=%v", readmeMD, readmeAtRoot) + } + if !strings.Contains(manifest, "README.md") { + t.Fatalf("manifest.txt file listing should include README.md:\n%s", manifest) + } +} + func TestMainBanner(t *testing.T) { tmp := t.TempDir() oldAuditPath := DefaultAuditJSONPath diff --git a/audit/internal/app/assets/README.md b/audit/internal/app/assets/README.md new file mode 100644 index 0000000..a3d307e --- /dev/null +++ b/audit/internal/app/assets/README.md @@ -0,0 +1,111 @@ +# README — how to read this bundle + +**If you are an AI agent that just listed or was pointed at this directory: +read this file before opening any other file in it.** It tells you what's +here and exactly where to look, so you don't burn time grepping through +random logs to answer something this file already answers directly. + +You are looking at a diagnostic export from **bee** (BEE-SP), a hardware +audit / acceptance-testing / burn-in tool for GPU servers. It runs on the +server itself (as a live-CD or an installed service) and produces this +export by: collecting a full hardware inventory, running SAT (System +Acceptance Test) checks against CPU/memory/storage/GPU/RAID, capturing raw +diagnostic command output, and logging its own systemd services. + +## Fastest answers to common questions + +- **"What hardware is in this server?"** → `export/bee-audit.json`. The + master inventory: board, firmware, CPUs, memory, storage, PCIe devices + (GPUs/NICs/RAID controllers), power supplies, sensors, event logs. Every + component has a `status` field (`OK`/`Warning`/`Critical`/`Unknown`). +- **"Did the acceptance tests pass?"** → grep `overall_status=` across every + `export/bee-sat/*/summary.txt`. That key is always present and is the + fastest single-line verdict for that run (`OK`, `FAILED`, `PARTIAL`, or + `UNSUPPORTED`). +- **"What's the *current* rolled-up health per component?"** → + `export/component-status.json`. One record per component key (e.g. + `pcie:gpu:nvidia`, `cpu:all`, `psu:0`), each with a `status` and a + `history` array. **The history is a transition log, not a per-check + journal** — a component whose status never changed only has one entry, + even if it was checked hundreds of times. A gap in timestamps does not + mean it stopped being monitored. +- **"Is a specific service healthy, or did it crash/restart-loop?"** → + `systemd/.status.txt` (current `systemctl status`) and + `systemd/.journal.log` (`journalctl -u ` for that + service's window). `systemd/combined.journal.log` has everything, + chronological, if you need cross-service correlation. +- **"What's the RAID/drive state?"** → + `export/techdump/storcli64-drives.json` and `storcli2-show-all.json` (LSI + controllers) — per-slot state like `JBOD`, `UGood`, `Onln`, `UBad`. A + drive in `JBOD`/`UBad` state cannot join a new virtual disk without first + being converted (`set good force`). +- **"GPU topology / NVLink health?"** → + `export/techdump/nvidia-smi-topo.txt` (which GPUs are NVLink-bonded to + which, and how many links), `nvidia-smi-nvlink-status.txt` (per-link + active/inactive — only present in bundles built after this capture was + added; older bundles only have the topo -m aggregate), `nvidia-smi-nvlink-errors.txt` + (replay/recovery/CRC error counters, should be zero). All lanes of a + bonded pair are expected to show active; even one `` lane next + to otherwise-active ones is a real fault signature, not benign — "no + NVLink present" instead shows *all* lanes inactive. +- **"What tasks were run from the web UI, in what order, with what + result?"** → `export/tasks-state.json` is the index (id, target, status, + timestamps, paths). Each task also has its own directory + `export/tasks/__/` with `task.log` (live output), + `report.json`/`report.html` (rendered result + charts). +- **"What build/version is this, and when was it captured?"** → + `manifest.txt` (`bee_version=`, `generated_at_utc=`) at the top level. If + the bundle's own folder/file name embeds `(BEE-SP vXX.YY)`, that's the + same version, useful for cross-referencing against the bee git repo's + release tags if you're checking whether a specific fix shipped in this + build. + +## Top-level layout + +``` +manifest.txt bee_version, host, generated_at_utc, export_dir, + then a flat "pathsize" listing of every file + in this bundle — a quick inventory/sanity check. +README.md this file — read this first. +export/ mirror of the live /appdata/bee/export directory — + see "export/" below. +systemd/ .status.txt + .journal.log per + monitored systemd unit, plus combined.journal.log. +techdump/, system/ raw command output not tied to a specific bee-* + service — see below. +``` + +## `export/` in detail + +This is a straight mirror of the live server's export directory, so +everything below also applies when reading a raw `bee export` output +directly (not wrapped in a support-bundle archive). + +| Path | What it is | +|---|---| +| `bee-audit.json` | The master hardware snapshot (see above). | +| `bee-audit.log` | Log of the hardware-audit collector itself: what it queried, what it skipped and why (e.g. a field unsupported by the current driver — not necessarily an error). | +| `bee-web.log` | Web UI service log. | +| `bee-network.log`, `bee-nvidia.log`, `bee-sshsetup.log`, `bee-selfheal.log`, `bee-blackbox.log`, `bee-hpc-tuning.log` | Per-service startup/runtime logs for the correspondingly-named systemd unit. | +| `runtime-health.json` / `.log` | A lighter, more frequent health snapshot than the full audit — good for "was it fine 5 minutes ago" without the cost of a full re-audit. | +| `component-status.json` | Current rolled-up component health — see "Fastest answers" above. | +| `blackbox-state.json` | Present if continuous blackbox capture (mirroring this export dir to removable media on a schedule) is or was active; tracks the sync target and last successful sync. | +| `techdump/` | Raw diagnostic command output, captured once per audit cycle. This is ground truth: `nvidia-smi-*.txt/.csv` (GPU state/topology/NVLink), `storcli64-drives.json` / `storcli2-show-all.json` (RAID), `lspci-*.txt`, `lscpu.txt`, `lsblk.json`, `dmidecode-*.txt`, `ipmitool-*.txt` (BMC sensors/SEL/FRU), `smartctl-*.json`, `nvme-list.json`, `sensors.json`. Higher-level views (the audit JSON, the web UI's topology page) are built from these. | +| `bee-sat/-/` | One directory per acceptance-test run — `target` is what was tested (`gpu-nvidia`, `cpu`, `memory`, `storage`, `nccl-tests`, `gpu-nvidia-bandwidth`, `nvidia-config` covering GPU config/NVLink/Confidential-Computing readiness, etc). Each contains `summary.txt` (key=value, always has `overall_status`), a full human-readable report, numbered per-job logs, and `verbose.log` (every subprocess invocation + exit code — the place to look when a summary doesn't explain *why* something failed). | +| `tasks/__/` | One directory per task launched from the web UI's task queue — see "Fastest answers" above. | +| `tasks-state.json` | Index of every task (id, target, status, timestamps, artifact paths). | +| `systemd/` (nested) | Historical per-service snapshots captured as part of an audit cycle, same shape as the top-level `systemd/` described below. | + +## `systemd/` and `system/` (top level, sibling of `export/`) + +- `systemd/.status.txt`, `systemd/.journal.log`, + `systemd/combined.journal.log` — captured fresh at bundle-build time (not + mirrored from `export/`), so this is the most current service state. +- `system/` — general OS-level diagnostics not specific to bee: + `dmesg.txt`, X server / display-manager logs. Mostly relevant to + physical/console access issues, not hardware health. + +## Timestamps + +Everything is UTC unless a filename or field name says otherwise +(`*_local`, etc. — rare). diff --git a/audit/internal/app/blackbox.go b/audit/internal/app/blackbox.go index 3bd82be..200820b 100644 --- a/audit/internal/app/blackbox.go +++ b/audit/internal/app/blackbox.go @@ -433,6 +433,13 @@ func (w *blackboxWorker) syncCycle() error { if err := syncDirectoryTree(w.runtime.exportDir, filepath.Join(root, "export")); err != nil { return err } + // Same doc pair the support bundle ships at its root — a blackbox + // capture on removable media has no manifest.txt/support-bundle + // equivalent to explain its layout, so without this an agent handed + // only the media would have nothing pointing it at README.md. + if err := writeBundleDocs(root); err != nil { + return err + } if err := w.captureSnapshots(root); err != nil { return err } diff --git a/audit/internal/app/support_bundle.go b/audit/internal/app/support_bundle.go index 0b7549e..e823aec 100644 --- a/audit/internal/app/support_bundle.go +++ b/audit/internal/app/support_bundle.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bee/audit/internal/platform" "compress/gzip" + _ "embed" "fmt" "io" "os" @@ -14,6 +15,19 @@ import ( "time" ) +//go:embed assets/README.md +var supportBundleReadmeMD []byte + +// writeBundleDocs writes README.md at root, so an AI agent (or a human) +// analyzing this export (support bundle or blackbox capture) can orient +// itself without reverse-engineering the directory layout first. +func writeBundleDocs(root string) error { + if err := os.MkdirAll(root, 0755); err != nil { + return err + } + return os.WriteFile(filepath.Join(root, "README.md"), supportBundleReadmeMD, 0644) +} + var supportBundleServices = []string{ "bee-blackbox.service", "bee-audit.service", @@ -439,6 +453,9 @@ func BuildSupportBundle(exportDir string) (string, error) { for _, item := range supportBundleOptionalFiles { _ = copyOptionalFile(item.src, filepath.Join(stageRoot, item.name)) } + if err := writeBundleDocs(stageRoot); err != nil { + return "", err + } if err := writeManifest(filepath.Join(stageRoot, "manifest.txt"), exportDir, stageRoot); err != nil { return "", err } diff --git a/audit/internal/app/support_bundle_test.go b/audit/internal/app/support_bundle_test.go new file mode 100644 index 0000000..9d2773b --- /dev/null +++ b/audit/internal/app/support_bundle_test.go @@ -0,0 +1,31 @@ +package app + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestWriteBundleDocs covers the primitive shared by both consumers — +// BuildSupportBundle (support-bundle archive root) and blackboxWorker.syncCycle +// (removable-media boot-folder root) — since a blackbox capture has no +// manifest.txt/tar-archive test harness to drive syncCycle end-to-end, but +// both call this same function at their respective root. +func TestWriteBundleDocs(t *testing.T) { + root := t.TempDir() + if err := writeBundleDocs(root); err != nil { + t.Fatalf("writeBundleDocs: %v", err) + } + + readme, err := os.ReadFile(filepath.Join(root, "README.md")) + if err != nil { + t.Fatalf("read README.md: %v", err) + } + if !strings.Contains(string(readme), "bee-audit.json") { + t.Fatalf("README.md should explain where the hardware inventory lives:\n%s", readme) + } + if !strings.Contains(string(readme), "overall_status") { + t.Fatalf("README.md should explain how to check SAT pass/fail:\n%s", readme) + } +}