From fa618d5abb402f6aa14f47448d77dc2ad8f696c5 Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Tue, 28 Jul 2026 14:33:26 +0300 Subject: [PATCH] platform/app: restructure support bundle/blackbox mirror into device-type export/, status/, tasks/, livecd/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the flat export/ + techdump/ + system/ + systemd/ layout (two same-named-but-different techdump/ dirs, disk reports duplicated between a SAT run dir and techdump/) with one shared categorizeExportTree used by both BuildSupportBundle and the blackbox USB mirror: - export/{cpu,memory,storage,gpu,network,platform}/ — raw vendor-tool output grouped by device type, plus export/reanimator.json ready to POST to Reanimator's /ingest/hardware endpoint. - status/ — computed diagnosis (component-status.json, runtime-health.*, and metrics.db, previously missing from every bundle entirely). - tasks/ — bee's own task-run bookkeeping (bee-sat/, bee-bench/, task reports, service logs) where duplication with export/ is expected. - livecd/{gui,host}/ — live-boot/kiosk-session-only diagnostics, kept out of the hardware-facing tree. Drops the redundant disk-report mirror write in RunStorageAcceptancePack and the now-dead syncDirectoryTree/removeMissingPaths; adds a size+mtime skip in copyPath so the blackbox mirror doesn't rewrite unchanged files every cycle. README.md rewritten to match. Co-Authored-By: Claude Sonnet 5 --- audit/internal/app/app_test.go | 24 ++-- audit/internal/app/assets/README.md | 184 ++++++++++++++++--------- audit/internal/app/blackbox.go | 66 +-------- audit/internal/app/bundle_layout.go | 193 +++++++++++++++++++++++++++ audit/internal/app/support_bundle.go | 168 +++++++++++------------ audit/internal/platform/sat.go | 10 -- 6 files changed, 412 insertions(+), 233 deletions(-) create mode 100644 audit/internal/app/bundle_layout.go diff --git a/audit/internal/app/app_test.go b/audit/internal/app/app_test.go index 6465f21..57c10b2 100644 --- a/audit/internal/app/app_test.go +++ b/audit/internal/app/app_test.go @@ -889,7 +889,7 @@ func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) { t.Fatalf("read tar entry: %v", err) } names = append(names, hdr.Name) - if contains(hdr.Name, "/export/bee-audit.json") { + if contains(hdr.Name, "/export/reanimator.json") { body, err := io.ReadAll(tr) if err != nil { t.Fatalf("read audit entry: %v", err) @@ -906,15 +906,15 @@ func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) { } for _, want := range []string{ - "/system/ip-link.txt", - "/system/ip-link-stats.txt", - "/techdump/kernel-aer-nvidia.txt", - "/techdump/lspci-nvidia-bridges-vv.txt", - "/techdump/pcie-aer-sysfs.txt", - "/techdump/ethtool-info.txt", - "/techdump/ethtool-link.txt", - "/techdump/ethtool-module.txt", - "/techdump/mstflint-query.txt", + "/livecd/host/ip-link.txt", + "/livecd/host/ip-link-stats.txt", + "/export/gpu/kernel-aer-nvidia.txt", + "/export/gpu/lspci-nvidia-bridges-vv.txt", + "/export/gpu/pcie-aer-sysfs.txt", + "/export/network/ethtool-info.txt", + "/export/network/ethtool-link.txt", + "/export/network/ethtool-module.txt", + "/export/network/mstflint-query.txt", } { var found bool for _, name := range names { @@ -930,10 +930,10 @@ func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) { var foundRaw bool for _, name := range names { - if contains(name, "/export/bee-sat/memory-run/verbose.log") { + if contains(name, "/tasks/bee-sat/memory-run/verbose.log") { foundRaw = true } - if contains(name, "/export/bee-sat/memory-run.tar.gz") { + if contains(name, "/tasks/bee-sat/memory-run.tar.gz") { t.Fatalf("support bundle should not contain nested SAT archive: %s", name) } } diff --git a/audit/internal/app/assets/README.md b/audit/internal/app/assets/README.md index 9ce63d4..6712f73 100644 --- a/audit/internal/app/assets/README.md +++ b/audit/internal/app/assets/README.md @@ -12,50 +12,79 @@ export by: collecting a full hardware inventory, running SAT (System Acceptance Test) checks against CPU/memory/storage/GPU/RAID, capturing raw diagnostic command output, and logging its own systemd services. +The bundle is organized around **what kind of thing each file is**, not +which bee component happened to produce it: + +- **`export/`** — raw, vendor-native tool output, grouped by physical device + type. Nothing in here needs bee context to read; a NIC vendor can be + handed `export/network/` and it means the same thing it would from any + other collector. Also holds `reanimator.json`, a ready-to-ingest hardware + snapshot. +- **`status/`** — *computed* diagnosis (component health verdicts, runtime + status, metrics history), not raw tool output and not a task log. +- **`tasks/`** — bee's own bookkeeping: full per-run logs, orchestration + state, service logs. Expect duplication with `export/` here (e.g. a + storage SAT run's narrative disk report repeats data already in + `export/storage/`) — that's intentional, this is the "show your work" tier. +- **`livecd/`** — only relevant to *this particular live-boot/kiosk session* + (GUI stack, DHCP-assigned IPs, kernel ring buffer for this boot). Not + hardware facts — ignore this entirely when troubleshooting the physical + machine. + ## Fastest answers to common questions -- **"What hardware is in this server?"** → `export/bee-audit.json`. The +- **"What hardware is in this server?"** → `export/reanimator.json`. The master inventory: board, firmware, CPUs, memory, storage, PCIe devices (GPUs/NICs/RAID controllers), power supplies, sensors, event logs. Every - component has a `status` field (`OK`/`Warning`/`Critical`/`Unknown`). + component has a `status` field (`OK`/`Warning`/`Critical`/`Unknown`). Same + document a `POST /ingest/hardware` to Reanimator expects (see + `bible-local/docs/hardware-ingest-contract.md` in the bee repo). - **"Did the acceptance tests pass?"** → grep `overall_status=` across every - `export/bee-sat/*/summary.txt`. That key is always present and is the + `tasks/bee-sat/*/summary.txt`. That key is always present and is the fastest single-line verdict for that run (`OK`, `FAILED`, `PARTIAL`, or `UNSUPPORTED`). - **"What's the *current* rolled-up health per component?"** → - `export/component-status.json`. One record per component key (e.g. + `status/component-status.json`. One record per component key (e.g. `pcie:gpu:nvidia`, `cpu:all`, `psu:0`), each with a `status` and a - `history` array. **The history is a transition log, not a per-check - journal** — a component whose status never changed only has one entry, - even if it was checked hundreds of times. A gap in timestamps does not - mean it stopped being monitored. -- **"Is a specific service healthy, or did it crash/restart-loop?"** → - `systemd/.status.txt` (current `systemctl status`) and - `systemd/.journal.log` (`journalctl -u ` for that - service's window). `systemd/combined.journal.log` has everything, - chronological, if you need cross-service correlation. + `history` array. This is what feeds `reanimator.json`'s `status` fields — + it's the source, not a duplicate. **The history is a transition log, not a + per-check journal** — a component whose status never changed only has one + entry, even if it was checked hundreds of times. A gap in timestamps does + not mean it stopped being monitored. +- **"Is a specific bee service healthy, or did it crash/restart-loop?"** → + `tasks/_services/.status.txt` (current `systemctl status`) and + `tasks/_services/.journal.log` (`journalctl -u ` for + that service's window). `tasks/_services/combined.journal.log` has + everything, chronological, if you need cross-service correlation. NVIDIA + DCGM/fabric-manager daemon status lives in `export/gpu/` instead (see + below); the display stack's lives in `livecd/gui/`. - **"What's the RAID/drive state?"** → - `export/techdump/storcli64-drives.json` and `storcli2-show-all.json` (LSI + `export/storage/storcli64-drives.json` and `storcli2-show-all.json` (LSI controllers) — per-slot state like `JBOD`, `UGood`, `Onln`, `UBad`. A drive in `JBOD`/`UBad` state cannot join a new virtual disk without first being converted (`set good force`). - **"GPU topology / NVLink health?"** → - `export/techdump/nvidia-smi-topo.txt` (`NV#` = bonded NVLink pair, `PIX`/ + `export/gpu/nvidia-smi-topo.txt` (`NV#` = bonded NVLink pair, `PIX`/ `NODE`/`SYS` = PCIe-only, no NVLink), `nvidia-smi-nvlink-status.txt` (per-link active/inactive), `nvidia-smi-nvlink-errors.txt` (replay/ - recovery/CRC counters, should be zero), and `bee-sat/nvidia-config-*/summary.txt`'s - `nvlink_pairs_checked` (how many pairs it found to check, not how many - passed). **Not every GPU config has NVLink bridges — check whether this - SKU/order is supposed to have them before calling their absence a fault** - (all-`PIX` topology, empty nvlink-status/errors files, and - `nvlink_pairs_checked=0` together mean "none detected," which is only a - problem if the config calls for NVLink). If NVLink is expected, one - `` lane next to active ones on an otherwise-bonded pair is the - real fault signature — a degraded link, not an absent one. + recovery/CRC counters, should be zero), and + `tasks/bee-sat/nvidia-config-*/summary.txt`'s `nvlink_pairs_checked` (how + many pairs it found to check, not how many passed). If `export/gpu/` + also has `nvidia-smi-topo-fresh.txt` / `-nvlink-status-fresh.txt` / + `-nvlink-errors-fresh.txt`, those were recaptured live at bundle-build + time — prefer them over the non-`-fresh` ones if the two disagree, since + the non-`-fresh` files are only as current as the last audit cycle. + **Not every GPU config has NVLink bridges — check whether this SKU/order + is supposed to have them before calling their absence a fault** (all-`PIX` + topology, empty nvlink-status/errors files, and `nvlink_pairs_checked=0` + together mean "none detected," which is only a problem if the config calls + for NVLink). If NVLink is expected, one `` lane next to active + ones on an otherwise-bonded pair is the real fault signature — a degraded + link, not an absent one. - **"What tasks were run from the web UI, in what order, with what - result?"** → `export/tasks-state.json` is the index (id, target, status, - timestamps, paths). Each task also has its own directory - `export/tasks/__/` with `task.log` (live output), + result?"** → `tasks/_state/tasks-state.json` is the index (id, target, + status, timestamps, paths). Each task also has its own directory + `tasks/__/` with `task.log` (live output), `report.json`/`report.html` (rendered result + charts). - **"What build/version is this, and when was it captured?"** → `manifest.txt` (`bee_version=`, `generated_at_utc=`) at the top level. If @@ -63,21 +92,27 @@ diagnostic command output, and logging its own systemd services. same version, useful for cross-referencing against the bee git repo's release tags if you're checking whether a specific fix shipped in this build. +- **"Was there a CPU/memory/power utilization spike around time T?"** → + `status/metrics.db`, a SQLite database (`sys_metrics` table: `ts`, + `cpu_load_pct`, `mem_load_pct`, `power_w`, `power_source`, `power_mode`, + `power_reason`). Open with any SQLite client. ## Signatures -- `ipmitool-sel-time.txt` disagrees with `manifest.txt`'s `generated_at_utc` - by more than minutes → BMC RTC drifted → all `ipmitool-sel.txt` timestamps - unreliable; use record-ID order (hex counter, column 1), not printed date. +- `export/platform/ipmitool-sel-time.txt` disagrees with `manifest.txt`'s + `generated_at_utc` by more than minutes → BMC RTC drifted → all + `ipmitool-sel.txt` timestamps unreliable; use record-ID order (hex + counter, column 1), not printed date. - `storcli64` → `"Status": "Failure", "Description": "No Controller found"` → no legacy MegaRAID controller present, not a drive/RAID fault. Cross-ref `storcli2-show-all.json`'s `"Number of Controllers"` and `lspci`/ `nvme-list.json` for actual storage hardware. Same pattern as `nvlink_pairs_checked=0` above. -- `systemctl restart ... timed out` in a `bee-*.log` → implicates a SAT run - only if its timestamp is inside that run's `run_at_utc`/`started_at`– - `done_at` window in `tasks-state.json`; outside that window (e.g. boot - bring-up) it's unrelated. +- `systemctl restart ... timed out` in a `tasks/_services/bee-*.log` → + implicates a SAT run only if its timestamp is inside that run's + `run_at_utc`/`started_at`–`done_at` window in + `tasks/_state/tasks-state.json`; outside that window (e.g. boot bring-up) + it's unrelated. - SEL sequence `Power Supply Failure detected`/`AC lost` (all PSUs) → `ACPI Legacy OFF` → `Chassis intrusion` → `Power Button pressed` → `Legacy ON` → full power-cycle, not a failing PSU. Isolated failure @@ -89,11 +124,12 @@ diagnostic command output, and logging its own systemd services. ## Diagnosing "it rebooted/crashed during test X" -1. `tasks-state.json` → task stuck `pending`/`running`, or last - `bee-sat/-*/` with no `summary.txt` = check running at crash time. +1. `tasks/_state/tasks-state.json` → task stuck `pending`/`running`, or last + `tasks/bee-sat/-*/` with no `summary.txt` = check running at + crash time. 2. That directory's `verbose.log` → last subprocess with no matching `finish`/exit-code line = the trigger. -3. `system/dmesg.txt` starting at uptime 0, ending after a few hundred +3. `livecd/host/dmesg.txt` starting at uptime 0, ending after a few hundred seconds = fresh-boot log captured after an unclean reset (confirms unclean reboot; does not contain the crash itself). 4. Diff against a step that passed cleanly → isolates what's specific to @@ -108,43 +144,65 @@ manifest.txt bee_version, host, generated_at_utc, export_dir, then a flat "pathsize" listing of every file in this bundle — a quick inventory/sanity check. README.md this file — read this first. -export/ mirror of the live /appdata/bee/export directory — +export/ raw, vendor-native hardware data, by device type — see "export/" below. -systemd/ .status.txt + .journal.log per - monitored systemd unit, plus combined.journal.log. -techdump/, system/ raw command output not tied to a specific bee-* - service — see below. +status/ computed diagnosis, not raw data — see below. +tasks/ bee's own task-run bookkeeping — see below. +livecd/ this boot/kiosk session only, not hardware — see below. ``` ## `export/` in detail -This is a straight mirror of the live server's export directory, so -everything below also applies when reading a raw `bee export` output -directly (not wrapped in a support-bundle archive). +Raw tool/command output, grouped by the physical device it describes. +Vendor-neutral: nothing here assumes familiarity with bee. | Path | What it is | |---|---| -| `bee-audit.json` | The master hardware snapshot (see above). | -| `bee-audit.log` | Log of the hardware-audit collector itself: what it queried, what it skipped and why (e.g. a field unsupported by the current driver — not necessarily an error). | -| `bee-web.log` | Web UI service log. | -| `bee-network.log`, `bee-nvidia.log`, `bee-sshsetup.log`, `bee-selfheal.log`, `bee-blackbox.log`, `bee-hpc-tuning.log` | Per-service startup/runtime logs for the correspondingly-named systemd unit. | +| `reanimator.json` | The master hardware snapshot (see "Fastest answers" above) — ready to `POST /ingest/hardware`. | +| `cpu/` | `lscpu.txt`, `dmidecode-type4.txt` (Processor Information). | +| `memory/` | `dmidecode-type17.txt` (Memory Device, one entry per DIMM). | +| `storage/` | `smartctl-.json`, `nvme-id-ctrl-.json`, `nvme-smart-log-.json`, `nvme-list.json`, `storcli64-drives.json`, `storcli2-show-all.json`, `lsblk.json`. | +| `gpu/` | `nvidia-smi-*.txt/.csv` (state/topology/NVLink; `-fresh` variants are recaptured live at bundle-build time — see "GPU topology" above), `dcgmi-nvlink-status.txt`, `nvidia-bug-report.txt`, `nvidia-dcgm.service`/`nvidia-fabricmanager.service` status+journal, `fabric-manager-paths.txt`, `fabricmanager.log`/`nvlsm.log`, `lspci-nvidia-bridges-vv.txt`, `pcie-nvidia-link.txt`, `pcie-aer-sysfs.txt`, `kernel-aer-nvidia.txt` (AER/NVRM/Xid-filtered dmesg), `lspci-video-vv.txt`, `systemctl-nvidia-units.txt`. AMD systems get `rocm-smi*.txt` here instead. | +| `network/` | `ethtool-{info,link,module}.txt` (per-NIC), `mstflint-query.txt` (Mellanox/NVIDIA NICs). | +| `platform/` | `dmidecode-type{0,1,2}.txt` (BIOS/System/Baseboard), `ipmitool-{fru,sdr,sensor,sel,sel-time}.txt` (BMC), `sensors.json`, `lspci-{nn,vmm,vvv}.txt`. | + +## `status/` in detail + +Computed diagnosis, not raw data — see "Fastest answers" above for +`component-status.json` and `metrics.db`. + +| Path | What it is | +|---|---| +| `component-status.json` | Current rolled-up component health, written by bee's live `/dev/kmsg` watcher and by SAT run results. Source of `reanimator.json`'s `status`/`status_history` fields. | | `runtime-health.json` / `.log` | A lighter, more frequent health snapshot than the full audit — good for "was it fine 5 minutes ago" without the cost of a full re-audit. | -| `component-status.json` | Current rolled-up component health — see "Fastest answers" above. | -| `blackbox-state.json` | Present if continuous blackbox capture (mirroring this export dir to removable media on a schedule) is or was active; tracks the sync target and last successful sync. | -| `techdump/` | Raw diagnostic command output, captured once per audit cycle. This is ground truth: `nvidia-smi-*.txt/.csv` (GPU state/topology/NVLink), `storcli64-drives.json` / `storcli2-show-all.json` (RAID), `lspci-*.txt`, `lscpu.txt`, `lsblk.json`, `dmidecode-*.txt`, `ipmitool-*.txt` (BMC sensors/SEL/FRU), `smartctl-*.json`, `nvme-list.json`, `sensors.json`. Higher-level views (the audit JSON, the web UI's topology page) are built from these. | -| `bee-sat/-/` | One directory per acceptance-test run — `target` is what was tested (`gpu-nvidia`, `cpu`, `memory`, `storage`, `nccl-tests`, `gpu-nvidia-bandwidth`, `nvidia-config` covering GPU config/NVLink/Confidential-Computing readiness, etc). Each contains `summary.txt` (key=value, always has `overall_status`), a full human-readable report, numbered per-job logs, and `verbose.log` (every subprocess invocation + exit code — the place to look when a summary doesn't explain *why* something failed). | -| `tasks/__/` | One directory per task launched from the web UI's task queue — see "Fastest answers" above. | -| `tasks-state.json` | Index of every task (id, target, status, timestamps, artifact paths). | -| `systemd/` (nested) | Historical per-service snapshots captured as part of an audit cycle, same shape as the top-level `systemd/` described below. | +| `metrics.db` | SQLite time-series (CPU/mem load, power draw/source/mode) — see "Fastest answers" above. | -## `systemd/` and `system/` (top level, sibling of `export/`) +## `tasks/` in detail -- `systemd/.status.txt`, `systemd/.journal.log`, - `systemd/combined.journal.log` — captured fresh at bundle-build time (not - mirrored from `export/`), so this is the most current service state. -- `system/` — general OS-level diagnostics not specific to bee: - `dmesg.txt`, X server / display-manager logs. Mostly relevant to - physical/console access issues, not hardware health. +bee's own bookkeeping. Duplication with `export/` is expected here — this +tier keeps the full story of a specific run, `export/` keeps the clean +current-state answer. + +| Path | What it is | +|---|---| +| `bee-sat/-/` | One directory per acceptance-test run — `target` is what was tested (`gpu-nvidia`, `cpu`, `memory`, `storage`, `nccl-tests`, `gpu-nvidia-bandwidth`, `nvidia-config` covering GPU config/NVLink/Confidential-Computing readiness, etc). Each contains `summary.txt` (key=value, always has `overall_status`), a full human-readable report (e.g. `storage`'s per-drive `disk-NN--report.txt` — a narrative verdict, not raw tool output; the raw data behind it lives in `export/storage/`), numbered per-job logs, and `verbose.log` (every subprocess invocation + exit code — the place to look when a summary doesn't explain *why* something failed). | +| `bee-bench/` | Benchmark run artifacts, if any have been run. | +| `__/` | One directory per task launched from the web UI's task queue — see "Fastest answers" above. | +| `_state/bee-audit.json` | The full hardware-audit document as bee last generated it — same content as `export/reanimator.json`, kept here as bee's own internal record. | +| `_state/tasks-state.json` | Index of every task (id, target, status, timestamps, artifact paths). | +| `_state/blackbox-state.json` | Present if continuous blackbox capture (mirroring the export dir to removable media on a schedule) is or was active; tracks the sync target and last successful sync. | +| `_services/bee-*.log` | Per-service startup/runtime logs for each bee-owned systemd unit. | +| `_services/.status.txt`, `.journal.log`, `combined.journal.log` | Captured fresh at bundle-build time, for every bee-owned service (`bee-audit`, `bee-web`, `bee-network`, `bee-nvidia`, `bee-preflight`, `bee-selfheal`, `bee-sshsetup`, `bee-blackbox`). | + +## `livecd/` in detail + +Only relevant to this particular live-boot/kiosk session — not the physical +hardware. Safe to ignore entirely when troubleshooting hardware. + +| Path | What it is | +|---|---| +| `gui/` | `Xorg.0.log`, `lightdm/`, `xinput-list.txt`, `libinput-list-devices.txt`, `drm-cards.txt`, `proc-fb.txt`, `input-devices.txt`, `udevadm-input.txt`, `loginctl-{sessions,seats}.txt`, `ps-gui.txt`, `systemctl-gui-units.txt`, `home-bee-xsession-errors.log`, `home-bee-chromium-debug.log`, `dmesg-gui-video-input.txt` (GUI/video/input-filtered dmesg), `display-manager.service`/`lightdm.service` status+journal. | +| `host/` | `uname.txt`, `cmdline.txt`, `lsmod.txt`, `mount.txt`, `df-h.txt`, `dmesg.txt` (full kernel ring buffer for this boot), `ip-{addr,link,link-stats,route}.txt` (this session's network config, e.g. DHCP-assigned addresses — not a hardware fact about the NIC itself; NIC hardware/firmware state is in `export/network/`). | ## Timestamps diff --git a/audit/internal/app/blackbox.go b/audit/internal/app/blackbox.go index ce8c2da..79492a7 100644 --- a/audit/internal/app/blackbox.go +++ b/audit/internal/app/blackbox.go @@ -8,7 +8,6 @@ import ( "encoding/json" "errors" "fmt" - "io/fs" "os" "os/exec" "path/filepath" @@ -582,10 +581,10 @@ func (w *blackboxWorker) syncCycle() error { w.recordMountpoint(mountpoint, mountedByBee) root := filepath.Join(mountpoint, w.runtime.bootFolder) - if err := os.MkdirAll(filepath.Join(root, "export"), 0755); err != nil { + if err := os.MkdirAll(root, 0755); err != nil { return err } - if err := syncDirectoryTree(w.runtime.exportDir, filepath.Join(root, "export")); err != nil { + if err := categorizeExportTree(w.runtime.exportDir, root); err != nil { return err } // Same doc pair the support bundle ships at its root — a blackbox @@ -625,18 +624,19 @@ func (w *blackboxWorker) recordMountpoint(mountpoint string, mountedByBee bool) } func (w *blackboxWorker) captureSnapshots(root string) error { - if err := captureCommandAtomic(filepath.Join(root, "systemd", "combined.journal.log"), "journalctl", "--no-pager", "--since", w.runtime.bootStarted.Format(time.RFC3339)); err != nil { + if err := captureCommandAtomic(filepath.Join(root, "tasks", "_services", "combined.journal.log"), "journalctl", "--no-pager", "--since", w.runtime.bootStarted.Format(time.RFC3339)); err != nil { return err } for _, svc := range supportBundleServices { - if err := captureCommandAtomic(filepath.Join(root, "systemd", svc+".journal.log"), "journalctl", "--no-pager", "-u", svc, "--since", w.runtime.bootStarted.Format(time.RFC3339)); err != nil { + dir := filepath.Join(root, serviceBundleDir(svc)) + if err := captureCommandAtomic(filepath.Join(dir, svc+".journal.log"), "journalctl", "--no-pager", "-u", svc, "--since", w.runtime.bootStarted.Format(time.RFC3339)); err != nil { return err } - if err := captureCommandAtomic(filepath.Join(root, "systemd", svc+".status.txt"), "systemctl", "status", svc, "--no-pager"); err != nil { + if err := captureCommandAtomic(filepath.Join(dir, svc+".status.txt"), "systemctl", "status", svc, "--no-pager"); err != nil { return err } } - if err := captureCommandAtomic(filepath.Join(root, "system", "dmesg.txt"), "dmesg"); err != nil { + if err := captureCommandAtomic(filepath.Join(root, "livecd", "host", "dmesg.txt"), "dmesg"); err != nil { return err } for _, item := range supportBundleOptionalFiles { @@ -803,58 +803,6 @@ func writeBlackboxMarker(mountpoint string, marker BlackboxMarker) error { return writeJSONAtomic(filepath.Join(mountpoint, blackboxMarkerName), marker) } -func syncDirectoryTree(srcDir, dstDir string) error { - seen := make(map[string]struct{}) - err := filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - rel, err := filepath.Rel(srcDir, path) - if err != nil { - return err - } - rel = filepath.Clean(rel) - if rel == "." { - seen["."] = struct{}{} - return os.MkdirAll(dstDir, 0755) - } - seen[rel] = struct{}{} - dstPath := filepath.Join(dstDir, rel) - if d.IsDir() { - info, err := d.Info() - if err != nil { - return err - } - return os.MkdirAll(dstPath, info.Mode().Perm()) - } - return copyFileIfChanged(path, dstPath) - }) - if err != nil { - return err - } - return removeMissingPaths(dstDir, seen) -} - -func removeMissingPaths(dstDir string, seen map[string]struct{}) error { - return filepath.WalkDir(dstDir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - rel, err := filepath.Rel(dstDir, path) - if err != nil { - return err - } - rel = filepath.Clean(rel) - if rel == "." { - return nil - } - if _, ok := seen[rel]; ok { - return nil - } - return os.RemoveAll(path) - }) -} - func copyFileIfChanged(src, dst string) error { info, err := os.Stat(src) if err != nil { diff --git a/audit/internal/app/bundle_layout.go b/audit/internal/app/bundle_layout.go new file mode 100644 index 0000000..c5bdd7d --- /dev/null +++ b/audit/internal/app/bundle_layout.go @@ -0,0 +1,193 @@ +package app + +import ( + "os" + "path/filepath" + "strings" +) + +// DefaultMetricsDBPath is the SQLite time-series metrics store, kept outside +// DefaultExportDir (unlike everything else this package copies into a +// support bundle / blackbox mirror) because it's a live, actively-written +// database rather than a point-in-time capture. +const DefaultMetricsDBPath = "/appdata/bee/metrics.db" + +// techdumpBucketFor maps a filename produced by platform.CaptureTechnicalDump +// into export/techdump/ to its device-type bucket under export/ in the +// categorized bundle layout, so a vendor engineer can find "the GPU stuff" +// without knowing bee's internal file names. +func techdumpBucketFor(name string) string { + switch { + case name == "lscpu.txt", name == "dmidecode-type4.txt": + return "cpu" + case name == "dmidecode-type17.txt": + return "memory" + case name == "lsblk.json", name == "storcli64-drives.json", name == "storcli2-show-all.json", + strings.HasPrefix(name, "smartctl-"), strings.HasPrefix(name, "nvme-"): + return "storage" + case name == "nvidia-smi-q.txt", name == "nvidia-smi-query.csv", name == "nvidia-smi-conf-compute-q.txt", + name == "nvidia-smi-topo.txt", name == "nvidia-smi-nvlink-status.txt", name == "nvidia-smi-nvlink-errors.txt", + name == "rocm-smi.txt", name == "rocm-smi-showallinfo.txt": + return "gpu" + default: + // dmidecode-type0/1/2, ipmitool-*, sensors.json, lspci-vmm/vvv, and + // anything CaptureTechnicalDump adds later that isn't claimed above. + return "platform" + } +} + +// keepForBundleCopy excludes previously-built bundle archives that may be +// sitting in the export dir (defensive — BuildSupportBundle itself stages +// and writes its .tar.gz under os.TempDir(), not exportDir). +func keepForBundleCopy(rel string, _ os.FileInfo) bool { + cleanRel := filepath.ToSlash(strings.TrimPrefix(filepath.Clean(rel), "./")) + if cleanRel == "" { + return true + } + if strings.HasPrefix(cleanRel, "bee-sat/") && strings.HasSuffix(cleanRel, ".tar.gz") { + return false + } + if strings.HasPrefix(filepath.Base(cleanRel), "bee-support-") && strings.HasSuffix(cleanRel, ".tar.gz") { + return false + } + return true +} + +// categorizeExportTree copies the live bee export directory into destRoot, +// reshaped into the bundle's canonical layout: +// +// - export// — raw vendor-tool output (persisted techdump), +// vendor-neutral, grouped by what the hardware is rather than which bee +// tool produced it. +// - export/reanimator.json — the hardware snapshot, ready to POST to +// Reanimator's /ingest/hardware endpoint. +// - status/ — computed diagnosis, not raw data: component +// health verdicts, runtime status, metrics time series. +// - tasks/ — bee's own task-run bookkeeping (SAT/bench +// run logs, orchestration state, service logs). Duplication with +// export/ here is expected (e.g. a SAT run's narrative disk report). +// +// Used by both BuildSupportBundle (on-demand tar.gz) and the blackbox USB +// mirror, so both artifacts share one shape. +func categorizeExportTree(srcExportDir, destRoot string) error { + exportDir := filepath.Join(destRoot, "export") + statusDir := filepath.Join(destRoot, "status") + tasksDir := filepath.Join(destRoot, "tasks") + stateDir := filepath.Join(tasksDir, "_state") + servicesDir := filepath.Join(tasksDir, "_services") + + for _, dir := range []string{exportDir, statusDir, tasksDir, stateDir, servicesDir} { + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + } + + // Persisted hardware captures -> export// + techdumpSrc := filepath.Join(srcExportDir, "techdump") + if entries, err := os.ReadDir(techdumpSrc); err == nil { + for _, entry := range entries { + if entry.IsDir() { + continue + } + // Stale leftover from before RunStorageAcceptancePack stopped + // mirroring its narrative disk report here (it's bee's own + // verdict, not raw tool output — belongs in tasks/bee-sat/ only). + // Units captured with an older bee build may still have one on + // disk; don't propagate it into the categorized layout. + if strings.HasPrefix(entry.Name(), "disk-") && strings.HasSuffix(entry.Name(), "-report.txt") { + continue + } + dst := filepath.Join(exportDir, techdumpBucketFor(entry.Name()), entry.Name()) + if err := copyPath(filepath.Join(techdumpSrc, entry.Name()), dst); err != nil { + return err + } + } + } else if !os.IsNotExist(err) { + return err + } + + // bee-audit.json -> tasks/_state/ (internal record) and + // export/reanimator.json (vendor-facing copy), both carrying the latest + // SAT-overlay-normalized status verdicts. + if data, err := os.ReadFile(filepath.Join(srcExportDir, "bee-audit.json")); err == nil { + normalized, err := ApplySATOverlay(data) + if err != nil { + normalized = data + } + if err := os.WriteFile(filepath.Join(stateDir, "bee-audit.json"), normalized, 0644); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(exportDir, "reanimator.json"), normalized, 0644); err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err + } + + // Computed diagnosis data -> status/ + for _, name := range []string{"component-status.json", "runtime-health.json", "runtime-health.log"} { + src := filepath.Join(srcExportDir, name) + if _, err := os.Stat(src); err != nil { + continue + } + if err := copyPath(src, filepath.Join(statusDir, name)); err != nil { + return err + } + } + if _, err := os.Stat(DefaultMetricsDBPath); err == nil { + if err := copyPath(DefaultMetricsDBPath, filepath.Join(statusDir, "metrics.db")); err != nil { + return err + } + } + + // Remaining orchestration state -> tasks/_state/ + for _, name := range []string{"blackbox-state.json", "tasks-state.json"} { + src := filepath.Join(srcExportDir, name) + if _, err := os.Stat(src); err != nil { + continue + } + if err := copyPath(src, filepath.Join(stateDir, name)); err != nil { + return err + } + } + + // bee-*.log service logs -> tasks/_services/ + if entries, err := os.ReadDir(srcExportDir); err == nil { + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasPrefix(name, "bee-") || !strings.HasSuffix(name, ".log") { + continue + } + if err := copyPath(filepath.Join(srcExportDir, name), filepath.Join(servicesDir, name)); err != nil { + return err + } + } + } else { + return err + } + + // SAT/bench run dirs, unchanged internally -> tasks/bee-sat/, tasks/bee-bench/ + for _, name := range []string{"bee-sat", "bee-bench"} { + src := filepath.Join(srcExportDir, name) + if _, err := os.Stat(src); err != nil { + continue + } + if err := copyPathFiltered(srcExportDir, src, filepath.Join(tasksDir, name), keepForBundleCopy); err != nil { + return err + } + } + + // Orchestration task-run reports, unchanged internally -> tasks/_..._done/ + if src := filepath.Join(srcExportDir, "tasks"); dirExists(src) { + if err := copyDirContentsFiltered(src, tasksDir, keepForBundleCopy); err != nil { + return err + } + } + + return nil +} + +func dirExists(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} diff --git a/audit/internal/app/support_bundle.go b/audit/internal/app/support_bundle.go index e823aec..6fa49d4 100644 --- a/audit/internal/app/support_bundle.go +++ b/audit/internal/app/support_bundle.go @@ -44,43 +44,58 @@ var supportBundleServices = []string{ "nvidia-fabricmanager.service", } +// serviceBundleDir returns the bundle-relative directory a service's +// status/journal capture belongs in: bee's own daemons are internal +// bookkeeping, NVIDIA/DCGM/fabric-manager daemons are vendor-facing hardware +// diagnostics, and the display stack is LiveCD/GUI-session-only. +func serviceBundleDir(svc string) string { + switch svc { + case "nvidia-dcgm.service", "nvidia-fabricmanager.service": + return "export/gpu" + case "display-manager.service", "lightdm.service": + return "livecd/gui" + default: + return "tasks/_services" + } +} + var supportBundleCommands = []struct { name string cmd []string }{ - {name: "system/uname.txt", cmd: []string{"uname", "-a"}}, - {name: "system/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}}, - {name: "system/lsmod.txt", cmd: []string{"lsmod"}}, - {name: "techdump/lspci-nn.txt", cmd: []string{"lspci", "-nn"}}, - {name: "system/ip-addr.txt", cmd: []string{"ip", "addr"}}, - {name: "system/ip-link.txt", cmd: []string{"ip", "-details", "link", "show"}}, - {name: "system/ip-link-stats.txt", cmd: []string{"ip", "-s", "link", "show"}}, - {name: "system/ip-route.txt", cmd: []string{"ip", "route"}}, - {name: "system/mount.txt", cmd: []string{"mount"}}, - {name: "system/df-h.txt", cmd: []string{"df", "-h"}}, - {name: "system/dmesg.txt", cmd: []string{"dmesg"}}, - {name: "system/dmesg-gui-video-input.txt", cmd: []string{"sh", "-c", ` + {name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}}, + {name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}}, + {name: "livecd/host/lsmod.txt", cmd: []string{"lsmod"}}, + {name: "export/platform/lspci-nn.txt", cmd: []string{"lspci", "-nn"}}, + {name: "livecd/host/ip-addr.txt", cmd: []string{"ip", "addr"}}, + {name: "livecd/host/ip-link.txt", cmd: []string{"ip", "-details", "link", "show"}}, + {name: "livecd/host/ip-link-stats.txt", cmd: []string{"ip", "-s", "link", "show"}}, + {name: "livecd/host/ip-route.txt", cmd: []string{"ip", "route"}}, + {name: "livecd/host/mount.txt", cmd: []string{"mount"}}, + {name: "livecd/host/df-h.txt", cmd: []string{"df", "-h"}}, + {name: "livecd/host/dmesg.txt", cmd: []string{"dmesg"}}, + {name: "livecd/gui/dmesg-gui-video-input.txt", cmd: []string{"sh", "-c", ` if command -v dmesg >/dev/null 2>&1; then dmesg | grep -iE 'nvidia|drm|fb|framebuffer|vesa|efi|lightdm|Xorg|input|hid|usb|keyboard|mouse|virtual keyboard|virtual mouse|ami|aspeed|ast' || echo "no GUI/video/input kernel messages found" else echo "dmesg not found" fi `}}, - {name: "techdump/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", ` if command -v dmesg >/dev/null 2>&1; then dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found" else echo "dmesg not found" fi `}}, - {name: "system/loginctl-sessions.txt", cmd: []string{"sh", "-c", ` + {name: "livecd/gui/loginctl-sessions.txt", cmd: []string{"sh", "-c", ` if command -v loginctl >/dev/null 2>&1; then loginctl list-sessions 2>&1 || true else echo "loginctl not found" fi `}}, - {name: "system/loginctl-seats.txt", cmd: []string{"sh", "-c", ` + {name: "livecd/gui/loginctl-seats.txt", cmd: []string{"sh", "-c", ` if command -v loginctl >/dev/null 2>&1; then loginctl list-seats 2>&1 || true echo @@ -93,10 +108,10 @@ else echo "loginctl not found" fi `}}, - {name: "system/ps-gui.txt", cmd: []string{"sh", "-c", ` + {name: "livecd/gui/ps-gui.txt", cmd: []string{"sh", "-c", ` ps -ef | grep -iE 'lightdm|Xorg|X$|openbox|chromium|chrome|xinit|xsession' | grep -v grep || echo "no GUI processes found" `}}, - {name: "techdump/lspci-video-vv.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/lspci-video-vv.txt", cmd: []string{"sh", "-c", ` if ! command -v lspci >/dev/null 2>&1; then echo "lspci not found" exit 0 @@ -112,8 +127,8 @@ if [ "$found" -eq 0 ]; then echo "no display-class PCI devices found" fi `}}, - {name: "system/proc-fb.txt", cmd: []string{"cat", "/proc/fb"}}, - {name: "system/drm-cards.txt", cmd: []string{"sh", "-c", ` + {name: "livecd/gui/proc-fb.txt", cmd: []string{"cat", "/proc/fb"}}, + {name: "livecd/gui/drm-cards.txt", cmd: []string{"sh", "-c", ` if [ -d /sys/class/drm ]; then for path in /sys/class/drm/card*; do [ -e "$path" ] || continue @@ -130,14 +145,14 @@ else echo "/sys/class/drm not present" fi `}}, - {name: "system/input-devices.txt", cmd: []string{"sh", "-c", ` + {name: "livecd/gui/input-devices.txt", cmd: []string{"sh", "-c", ` if [ -r /proc/bus/input/devices ]; then cat /proc/bus/input/devices else echo "/proc/bus/input/devices not readable" fi `}}, - {name: "system/udevadm-input.txt", cmd: []string{"sh", "-c", ` + {name: "livecd/gui/udevadm-input.txt", cmd: []string{"sh", "-c", ` if ! command -v udevadm >/dev/null 2>&1; then echo "udevadm not found" exit 0 @@ -154,21 +169,21 @@ if [ "$found" -eq 0 ]; then echo "no /dev/input/event* devices found" fi `}}, - {name: "system/xinput-list.txt", cmd: []string{"sh", "-c", ` + {name: "livecd/gui/xinput-list.txt", cmd: []string{"sh", "-c", ` if command -v xinput >/dev/null 2>&1; then DISPLAY=:0 xinput --list 2>&1 || true else echo "xinput not found" fi `}}, - {name: "system/libinput-list-devices.txt", cmd: []string{"sh", "-c", ` + {name: "livecd/gui/libinput-list-devices.txt", cmd: []string{"sh", "-c", ` if command -v libinput >/dev/null 2>&1; then libinput list-devices 2>&1 || true else echo "libinput not found" fi `}}, - {name: "system/systemctl-gui-units.txt", cmd: []string{"sh", "-c", ` + {name: "livecd/gui/systemctl-gui-units.txt", cmd: []string{"sh", "-c", ` if ! command -v systemctl >/dev/null 2>&1; then echo "systemctl not found" exit 0 @@ -182,35 +197,35 @@ echo echo "=== failed units ===" systemctl --failed --no-pager 2>&1 | grep -iE 'lightdm|display-manager|Xorg' || echo "no failed GUI units" `}}, - {name: "techdump/nvidia-smi-topo.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/nvidia-smi-topo-fresh.txt", cmd: []string{"sh", "-c", ` if command -v nvidia-smi >/dev/null 2>&1; then nvidia-smi topo -m 2>&1 || true else echo "nvidia-smi not found" fi `}}, - {name: "techdump/nvidia-smi-nvlink-status.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/nvidia-smi-nvlink-status-fresh.txt", cmd: []string{"sh", "-c", ` if command -v nvidia-smi >/dev/null 2>&1; then nvidia-smi nvlink -s 2>&1 || true else echo "nvidia-smi not found" fi `}}, - {name: "techdump/nvidia-smi-nvlink-errors.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/nvidia-smi-nvlink-errors-fresh.txt", cmd: []string{"sh", "-c", ` if command -v nvidia-smi >/dev/null 2>&1; then nvidia-smi nvlink -e 2>&1 || true else echo "nvidia-smi not found" fi `}}, - {name: "techdump/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", ` if command -v dcgmi >/dev/null 2>&1; then dcgmi nvlink --link-status 2>&1 || true else echo "dcgmi not found" fi `}}, - {name: "techdump/nvidia-bug-report.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", ` if command -v nvidia-bug-report.sh >/dev/null 2>&1; then nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 \ && cat /tmp/bee-nvidia-bug-report.log \ @@ -219,7 +234,7 @@ else echo "nvidia-bug-report.sh not found" fi `}}, - {name: "techdump/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", ` if ! command -v systemctl >/dev/null 2>&1; then echo "systemctl not found" exit 0 @@ -233,7 +248,7 @@ echo echo "=== failed units ===" systemctl --failed --no-pager 2>&1 | grep -iE 'nvidia|fabric' || echo "no failed nvidia/fabric units" `}}, - {name: "techdump/fabric-manager-paths.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/fabric-manager-paths.txt", cmd: []string{"sh", "-c", ` for candidate in \ /usr/bin/nvidia-fabricmanager \ /usr/bin/nv-fabricmanager \ @@ -249,7 +264,7 @@ if ! ls /usr/bin/nvidia-fabricmanager /usr/bin/nv-fabricmanager /usr/bin/nvidia- echo "no fabric manager binaries found" fi `}}, - {name: "techdump/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", ` if ! command -v lspci >/dev/null 2>&1; then echo "lspci not found" exit 0 @@ -271,7 +286,7 @@ if [ "$found" -eq 0 ]; then echo "no NVIDIA PCI devices found" fi `}}, - {name: "techdump/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", ` for d in /sys/bus/pci/devices/*/; do vendor=$(cat "$d/vendor" 2>/dev/null) [ "$vendor" = "0x10de" ] || continue @@ -287,7 +302,7 @@ for d in /sys/bus/pci/devices/*/; do done done `}}, - {name: "techdump/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", ` found=0 for dev in /sys/bus/pci/devices/*; do [ -e "$dev" ] || continue @@ -311,7 +326,7 @@ if [ "$found" -eq 0 ]; then echo "no PCIe AER sysfs counters found" fi `}}, - {name: "techdump/ethtool-info.txt", cmd: []string{"sh", "-c", ` + {name: "export/network/ethtool-info.txt", cmd: []string{"sh", "-c", ` if ! command -v ethtool >/dev/null 2>&1; then echo "ethtool not found" exit 0 @@ -330,7 +345,7 @@ if [ "$found" -eq 0 ]; then echo "no interfaces found" fi `}}, - {name: "techdump/ethtool-link.txt", cmd: []string{"sh", "-c", ` + {name: "export/network/ethtool-link.txt", cmd: []string{"sh", "-c", ` if ! command -v ethtool >/dev/null 2>&1; then echo "ethtool not found" exit 0 @@ -349,7 +364,7 @@ if [ "$found" -eq 0 ]; then echo "no interfaces found" fi `}}, - {name: "techdump/ethtool-module.txt", cmd: []string{"sh", "-c", ` + {name: "export/network/ethtool-module.txt", cmd: []string{"sh", "-c", ` if ! command -v ethtool >/dev/null 2>&1; then echo "ethtool not found" exit 0 @@ -368,7 +383,7 @@ if [ "$found" -eq 0 ]; then echo "no interfaces found" fi `}}, - {name: "techdump/mstflint-query.txt", cmd: []string{"sh", "-c", ` + {name: "export/network/mstflint-query.txt", cmd: []string{"sh", "-c", ` if ! command -v mstflint >/dev/null 2>&1; then echo "mstflint not found" exit 0 @@ -394,19 +409,19 @@ var supportBundleOptionalFiles = []struct { name string src string }{ - {name: "system/kern.log", src: "/var/log/kern.log"}, - {name: "system/syslog.txt", src: "/var/log/syslog"}, - {name: "system/Xorg.0.log", src: "/var/log/Xorg.0.log"}, - {name: "system/Xorg.0.log.old", src: "/var/log/Xorg.0.log.old"}, - {name: "system/lightdm/lightdm.log", src: "/var/log/lightdm/lightdm.log"}, - {name: "system/lightdm/x-0.log", src: "/var/log/lightdm/x-0.log"}, - {name: "system/lightdm/x-0-greeter.log", src: "/var/log/lightdm/x-0-greeter.log"}, - {name: "system/home-bee-xsession-errors.log", src: "/home/bee/.xsession-errors"}, - {name: "system/home-bee-chromium-debug.log", src: "/tmp/bee-chrome/chrome_debug.log"}, - {name: "techdump/fabricmanager.log", src: "/var/log/fabricmanager.log"}, - {name: "techdump/nvlsm.log", src: "/var/log/nvlsm.log"}, - {name: "techdump/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"}, - {name: "techdump/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"}, + {name: "livecd/host/kern.log", src: "/var/log/kern.log"}, + {name: "livecd/host/syslog.txt", src: "/var/log/syslog"}, + {name: "livecd/gui/Xorg.0.log", src: "/var/log/Xorg.0.log"}, + {name: "livecd/gui/Xorg.0.log.old", src: "/var/log/Xorg.0.log.old"}, + {name: "livecd/gui/lightdm/lightdm.log", src: "/var/log/lightdm/lightdm.log"}, + {name: "livecd/gui/lightdm/x-0.log", src: "/var/log/lightdm/x-0.log"}, + {name: "livecd/gui/lightdm/x-0-greeter.log", src: "/var/log/lightdm/x-0-greeter.log"}, + {name: "livecd/gui/home-bee-xsession-errors.log", src: "/home/bee/.xsession-errors"}, + {name: "livecd/gui/home-bee-chromium-debug.log", src: "/tmp/bee-chrome/chrome_debug.log"}, + {name: "export/gpu/fabricmanager.log", src: "/var/log/fabricmanager.log"}, + {name: "export/gpu/nvlsm.log", src: "/var/log/nvlsm.log"}, + {name: "export/gpu/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"}, + {name: "export/gpu/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"}, } const supportBundleGlob = "????-??-?? (BEE-SP*)*.tar.gz" @@ -431,17 +446,18 @@ func BuildSupportBundle(exportDir string) (string, error) { } defer os.RemoveAll(stageRoot) - if err := copyExportDirForSupportBundle(exportDir, filepath.Join(stageRoot, "export")); err != nil { + if err := categorizeExportTree(exportDir, stageRoot); err != nil { return "", err } - if err := writeJournalDump(filepath.Join(stageRoot, "systemd", "combined.journal.log")); err != nil { + if err := writeJournalDump(filepath.Join(stageRoot, "tasks", "_services", "combined.journal.log")); err != nil { return "", err } for _, svc := range supportBundleServices { - if err := writeCommandOutput(filepath.Join(stageRoot, "systemd", svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil { + dir := filepath.Join(stageRoot, serviceBundleDir(svc)) + if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil { return "", err } - if err := writeCommandOutput(filepath.Join(stageRoot, "systemd", svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil { + if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil { return "", err } } @@ -715,40 +731,6 @@ func copyDirContents(srcDir, dstDir string) error { return nil } -func copyExportDirForSupportBundle(srcDir, dstDir string) error { - if err := copyDirContentsFiltered(srcDir, dstDir, func(rel string, info os.FileInfo) bool { - cleanRel := filepath.ToSlash(strings.TrimPrefix(filepath.Clean(rel), "./")) - if cleanRel == "" { - return true - } - if strings.HasPrefix(cleanRel, "bee-sat/") && strings.HasSuffix(cleanRel, ".tar.gz") { - return false - } - if strings.HasPrefix(filepath.Base(cleanRel), "bee-support-") && strings.HasSuffix(cleanRel, ".tar.gz") { - return false - } - return true - }); err != nil { - return err - } - return normalizeSupportBundleAuditJSON(filepath.Join(dstDir, "bee-audit.json")) -} - -func normalizeSupportBundleAuditJSON(path string) error { - data, err := os.ReadFile(path) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - normalized, err := ApplySATOverlay(data) - if err != nil { - return nil - } - return os.WriteFile(path, normalized, 0644) -} - func copyDirContentsFiltered(srcDir, dstDir string, keep func(rel string, info os.FileInfo) bool) error { entries, err := os.ReadDir(srcDir) if err != nil { @@ -791,6 +773,14 @@ func copyPath(src, dst string) error { if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { return err } + // Skip rewriting files the blackbox worker already copied unchanged on a + // prior cycle — avoids needless flash wear on the removable target every + // sync period. Size+mtime, not a full byte comparison, so large files + // (e.g. status/metrics.db) can still stream-copy instead of loading + // whole into memory. + if dstInfo, err := os.Stat(dst); err == nil && dstInfo.Size() == info.Size() && !dstInfo.ModTime().Before(info.ModTime()) { + return nil + } in, err := os.Open(src) if err != nil { return err diff --git a/audit/internal/platform/sat.go b/audit/internal/platform/sat.go index 207e746..0f38dfc 100644 --- a/audit/internal/platform/sat.go +++ b/audit/internal/platform/sat.go @@ -822,15 +822,6 @@ func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, e } sort.Strings(devices) - // Disk reports (per-drive health + pseudographic resource bars) are also - // mirrored into techdump, a sibling of baseDir under the same export dir, - // so support bundles surface them alongside the rest of the diagnostics - // without requiring anyone to dig into a timestamped bee-sat run dir. - techDumpDir := filepath.Join(filepath.Dir(baseDir), "techdump") - if err := os.MkdirAll(techDumpDir, 0755); err != nil { - return "", err - } - var summary strings.Builder stats := satStats{} fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339)) @@ -898,7 +889,6 @@ func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, e reportText := GenerateDiskReportText(index+1, devPath, deviceOutputs, time.Now().UTC()) reportName := "disk-" + prefix + "-report.txt" _ = os.WriteFile(filepath.Join(runDir, reportName), []byte(reportText), 0644) - _ = os.WriteFile(filepath.Join(techDumpDir, reportName), []byte(reportText), 0644) } writeSATStats(&summary, stats)