platform/app: restructure support bundle/blackbox mirror into device-type export/, status/, tasks/, livecd/
Replaces the flat export/ + techdump/ + system/ + systemd/ layout (two
same-named-but-different techdump/ dirs, disk reports duplicated between a
SAT run dir and techdump/) with one shared categorizeExportTree used by both
BuildSupportBundle and the blackbox USB mirror:
- export/{cpu,memory,storage,gpu,network,platform}/ — raw vendor-tool
output grouped by device type, plus export/reanimator.json ready to POST
to Reanimator's /ingest/hardware endpoint.
- status/ — computed diagnosis (component-status.json, runtime-health.*,
and metrics.db, previously missing from every bundle entirely).
- tasks/ — bee's own task-run bookkeeping (bee-sat/, bee-bench/, task
reports, service logs) where duplication with export/ is expected.
- livecd/{gui,host}/ — live-boot/kiosk-session-only diagnostics, kept out
of the hardware-facing tree.
Drops the redundant disk-report mirror write in RunStorageAcceptancePack
and the now-dead syncDirectoryTree/removeMissingPaths; adds a size+mtime
skip in copyPath so the blackbox mirror doesn't rewrite unchanged files
every cycle. README.md rewritten to match.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e03267a72f
commit
fa618d5abb
@@ -889,7 +889,7 @@ func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) {
|
||||
t.Fatalf("read tar entry: %v", err)
|
||||
}
|
||||
names = append(names, hdr.Name)
|
||||
if contains(hdr.Name, "/export/bee-audit.json") {
|
||||
if contains(hdr.Name, "/export/reanimator.json") {
|
||||
body, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
t.Fatalf("read audit entry: %v", err)
|
||||
@@ -906,15 +906,15 @@ func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, want := range []string{
|
||||
"/system/ip-link.txt",
|
||||
"/system/ip-link-stats.txt",
|
||||
"/techdump/kernel-aer-nvidia.txt",
|
||||
"/techdump/lspci-nvidia-bridges-vv.txt",
|
||||
"/techdump/pcie-aer-sysfs.txt",
|
||||
"/techdump/ethtool-info.txt",
|
||||
"/techdump/ethtool-link.txt",
|
||||
"/techdump/ethtool-module.txt",
|
||||
"/techdump/mstflint-query.txt",
|
||||
"/livecd/host/ip-link.txt",
|
||||
"/livecd/host/ip-link-stats.txt",
|
||||
"/export/gpu/kernel-aer-nvidia.txt",
|
||||
"/export/gpu/lspci-nvidia-bridges-vv.txt",
|
||||
"/export/gpu/pcie-aer-sysfs.txt",
|
||||
"/export/network/ethtool-info.txt",
|
||||
"/export/network/ethtool-link.txt",
|
||||
"/export/network/ethtool-module.txt",
|
||||
"/export/network/mstflint-query.txt",
|
||||
} {
|
||||
var found bool
|
||||
for _, name := range names {
|
||||
@@ -930,10 +930,10 @@ func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) {
|
||||
|
||||
var foundRaw bool
|
||||
for _, name := range names {
|
||||
if contains(name, "/export/bee-sat/memory-run/verbose.log") {
|
||||
if contains(name, "/tasks/bee-sat/memory-run/verbose.log") {
|
||||
foundRaw = true
|
||||
}
|
||||
if contains(name, "/export/bee-sat/memory-run.tar.gz") {
|
||||
if contains(name, "/tasks/bee-sat/memory-run.tar.gz") {
|
||||
t.Fatalf("support bundle should not contain nested SAT archive: %s", name)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,50 +12,79 @@ export by: collecting a full hardware inventory, running SAT (System
|
||||
Acceptance Test) checks against CPU/memory/storage/GPU/RAID, capturing raw
|
||||
diagnostic command output, and logging its own systemd services.
|
||||
|
||||
The bundle is organized around **what kind of thing each file is**, not
|
||||
which bee component happened to produce it:
|
||||
|
||||
- **`export/`** — raw, vendor-native tool output, grouped by physical device
|
||||
type. Nothing in here needs bee context to read; a NIC vendor can be
|
||||
handed `export/network/` and it means the same thing it would from any
|
||||
other collector. Also holds `reanimator.json`, a ready-to-ingest hardware
|
||||
snapshot.
|
||||
- **`status/`** — *computed* diagnosis (component health verdicts, runtime
|
||||
status, metrics history), not raw tool output and not a task log.
|
||||
- **`tasks/`** — bee's own bookkeeping: full per-run logs, orchestration
|
||||
state, service logs. Expect duplication with `export/` here (e.g. a
|
||||
storage SAT run's narrative disk report repeats data already in
|
||||
`export/storage/`) — that's intentional, this is the "show your work" tier.
|
||||
- **`livecd/`** — only relevant to *this particular live-boot/kiosk session*
|
||||
(GUI stack, DHCP-assigned IPs, kernel ring buffer for this boot). Not
|
||||
hardware facts — ignore this entirely when troubleshooting the physical
|
||||
machine.
|
||||
|
||||
## Fastest answers to common questions
|
||||
|
||||
- **"What hardware is in this server?"** → `export/bee-audit.json`. The
|
||||
- **"What hardware is in this server?"** → `export/reanimator.json`. The
|
||||
master inventory: board, firmware, CPUs, memory, storage, PCIe devices
|
||||
(GPUs/NICs/RAID controllers), power supplies, sensors, event logs. Every
|
||||
component has a `status` field (`OK`/`Warning`/`Critical`/`Unknown`).
|
||||
component has a `status` field (`OK`/`Warning`/`Critical`/`Unknown`). Same
|
||||
document a `POST /ingest/hardware` to Reanimator expects (see
|
||||
`bible-local/docs/hardware-ingest-contract.md` in the bee repo).
|
||||
- **"Did the acceptance tests pass?"** → grep `overall_status=` across every
|
||||
`export/bee-sat/*/summary.txt`. That key is always present and is the
|
||||
`tasks/bee-sat/*/summary.txt`. That key is always present and is the
|
||||
fastest single-line verdict for that run (`OK`, `FAILED`, `PARTIAL`, or
|
||||
`UNSUPPORTED`).
|
||||
- **"What's the *current* rolled-up health per component?"** →
|
||||
`export/component-status.json`. One record per component key (e.g.
|
||||
`status/component-status.json`. One record per component key (e.g.
|
||||
`pcie:gpu:nvidia`, `cpu:all`, `psu:0`), each with a `status` and a
|
||||
`history` array. **The history is a transition log, not a per-check
|
||||
journal** — a component whose status never changed only has one entry,
|
||||
even if it was checked hundreds of times. A gap in timestamps does not
|
||||
mean it stopped being monitored.
|
||||
- **"Is a specific service healthy, or did it crash/restart-loop?"** →
|
||||
`systemd/<service>.status.txt` (current `systemctl status`) and
|
||||
`systemd/<service>.journal.log` (`journalctl -u <service>` for that
|
||||
service's window). `systemd/combined.journal.log` has everything,
|
||||
chronological, if you need cross-service correlation.
|
||||
`history` array. This is what feeds `reanimator.json`'s `status` fields —
|
||||
it's the source, not a duplicate. **The history is a transition log, not a
|
||||
per-check journal** — a component whose status never changed only has one
|
||||
entry, even if it was checked hundreds of times. A gap in timestamps does
|
||||
not mean it stopped being monitored.
|
||||
- **"Is a specific bee service healthy, or did it crash/restart-loop?"** →
|
||||
`tasks/_services/<service>.status.txt` (current `systemctl status`) and
|
||||
`tasks/_services/<service>.journal.log` (`journalctl -u <service>` for
|
||||
that service's window). `tasks/_services/combined.journal.log` has
|
||||
everything, chronological, if you need cross-service correlation. NVIDIA
|
||||
DCGM/fabric-manager daemon status lives in `export/gpu/` instead (see
|
||||
below); the display stack's lives in `livecd/gui/`.
|
||||
- **"What's the RAID/drive state?"** →
|
||||
`export/techdump/storcli64-drives.json` and `storcli2-show-all.json` (LSI
|
||||
`export/storage/storcli64-drives.json` and `storcli2-show-all.json` (LSI
|
||||
controllers) — per-slot state like `JBOD`, `UGood`, `Onln`, `UBad`. A
|
||||
drive in `JBOD`/`UBad` state cannot join a new virtual disk without first
|
||||
being converted (`set good force`).
|
||||
- **"GPU topology / NVLink health?"** →
|
||||
`export/techdump/nvidia-smi-topo.txt` (`NV#` = bonded NVLink pair, `PIX`/
|
||||
`export/gpu/nvidia-smi-topo.txt` (`NV#` = bonded NVLink pair, `PIX`/
|
||||
`NODE`/`SYS` = PCIe-only, no NVLink), `nvidia-smi-nvlink-status.txt`
|
||||
(per-link active/inactive), `nvidia-smi-nvlink-errors.txt` (replay/
|
||||
recovery/CRC counters, should be zero), and `bee-sat/nvidia-config-*/summary.txt`'s
|
||||
`nvlink_pairs_checked` (how many pairs it found to check, not how many
|
||||
passed). **Not every GPU config has NVLink bridges — check whether this
|
||||
SKU/order is supposed to have them before calling their absence a fault**
|
||||
(all-`PIX` topology, empty nvlink-status/errors files, and
|
||||
`nvlink_pairs_checked=0` together mean "none detected," which is only a
|
||||
problem if the config calls for NVLink). If NVLink is expected, one
|
||||
`<inactive>` lane next to active ones on an otherwise-bonded pair is the
|
||||
real fault signature — a degraded link, not an absent one.
|
||||
recovery/CRC counters, should be zero), and
|
||||
`tasks/bee-sat/nvidia-config-*/summary.txt`'s `nvlink_pairs_checked` (how
|
||||
many pairs it found to check, not how many passed). If `export/gpu/`
|
||||
also has `nvidia-smi-topo-fresh.txt` / `-nvlink-status-fresh.txt` /
|
||||
`-nvlink-errors-fresh.txt`, those were recaptured live at bundle-build
|
||||
time — prefer them over the non-`-fresh` ones if the two disagree, since
|
||||
the non-`-fresh` files are only as current as the last audit cycle.
|
||||
**Not every GPU config has NVLink bridges — check whether this SKU/order
|
||||
is supposed to have them before calling their absence a fault** (all-`PIX`
|
||||
topology, empty nvlink-status/errors files, and `nvlink_pairs_checked=0`
|
||||
together mean "none detected," which is only a problem if the config calls
|
||||
for NVLink). If NVLink is expected, one `<inactive>` lane next to active
|
||||
ones on an otherwise-bonded pair is the real fault signature — a degraded
|
||||
link, not an absent one.
|
||||
- **"What tasks were run from the web UI, in what order, with what
|
||||
result?"** → `export/tasks-state.json` is the index (id, target, status,
|
||||
timestamps, paths). Each task also has its own directory
|
||||
`export/tasks/<NNN>_<slug>_<done|failed>/` with `task.log` (live output),
|
||||
result?"** → `tasks/_state/tasks-state.json` is the index (id, target,
|
||||
status, timestamps, paths). Each task also has its own directory
|
||||
`tasks/<NNN>_<slug>_<done|failed>/` with `task.log` (live output),
|
||||
`report.json`/`report.html` (rendered result + charts).
|
||||
- **"What build/version is this, and when was it captured?"** →
|
||||
`manifest.txt` (`bee_version=`, `generated_at_utc=`) at the top level. If
|
||||
@@ -63,21 +92,27 @@ diagnostic command output, and logging its own systemd services.
|
||||
same version, useful for cross-referencing against the bee git repo's
|
||||
release tags if you're checking whether a specific fix shipped in this
|
||||
build.
|
||||
- **"Was there a CPU/memory/power utilization spike around time T?"** →
|
||||
`status/metrics.db`, a SQLite database (`sys_metrics` table: `ts`,
|
||||
`cpu_load_pct`, `mem_load_pct`, `power_w`, `power_source`, `power_mode`,
|
||||
`power_reason`). Open with any SQLite client.
|
||||
|
||||
## Signatures
|
||||
|
||||
- `ipmitool-sel-time.txt` disagrees with `manifest.txt`'s `generated_at_utc`
|
||||
by more than minutes → BMC RTC drifted → all `ipmitool-sel.txt` timestamps
|
||||
unreliable; use record-ID order (hex counter, column 1), not printed date.
|
||||
- `export/platform/ipmitool-sel-time.txt` disagrees with `manifest.txt`'s
|
||||
`generated_at_utc` by more than minutes → BMC RTC drifted → all
|
||||
`ipmitool-sel.txt` timestamps unreliable; use record-ID order (hex
|
||||
counter, column 1), not printed date.
|
||||
- `storcli64` → `"Status": "Failure", "Description": "No Controller found"`
|
||||
→ no legacy MegaRAID controller present, not a drive/RAID fault. Cross-ref
|
||||
`storcli2-show-all.json`'s `"Number of Controllers"` and `lspci`/
|
||||
`nvme-list.json` for actual storage hardware. Same pattern as
|
||||
`nvlink_pairs_checked=0` above.
|
||||
- `systemctl restart ... timed out` in a `bee-*.log` → implicates a SAT run
|
||||
only if its timestamp is inside that run's `run_at_utc`/`started_at`–
|
||||
`done_at` window in `tasks-state.json`; outside that window (e.g. boot
|
||||
bring-up) it's unrelated.
|
||||
- `systemctl restart ... timed out` in a `tasks/_services/bee-*.log` →
|
||||
implicates a SAT run only if its timestamp is inside that run's
|
||||
`run_at_utc`/`started_at`–`done_at` window in
|
||||
`tasks/_state/tasks-state.json`; outside that window (e.g. boot bring-up)
|
||||
it's unrelated.
|
||||
- SEL sequence `Power Supply Failure detected`/`AC lost` (all PSUs) →
|
||||
`ACPI Legacy OFF` → `Chassis intrusion` → `Power Button pressed` →
|
||||
`Legacy ON` → full power-cycle, not a failing PSU. Isolated failure
|
||||
@@ -89,11 +124,12 @@ diagnostic command output, and logging its own systemd services.
|
||||
|
||||
## Diagnosing "it rebooted/crashed during test X"
|
||||
|
||||
1. `tasks-state.json` → task stuck `pending`/`running`, or last
|
||||
`bee-sat/<target>-*/` with no `summary.txt` = check running at crash time.
|
||||
1. `tasks/_state/tasks-state.json` → task stuck `pending`/`running`, or last
|
||||
`tasks/bee-sat/<target>-*/` with no `summary.txt` = check running at
|
||||
crash time.
|
||||
2. That directory's `verbose.log` → last subprocess with no matching
|
||||
`finish`/exit-code line = the trigger.
|
||||
3. `system/dmesg.txt` starting at uptime 0, ending after a few hundred
|
||||
3. `livecd/host/dmesg.txt` starting at uptime 0, ending after a few hundred
|
||||
seconds = fresh-boot log captured after an unclean reset (confirms
|
||||
unclean reboot; does not contain the crash itself).
|
||||
4. Diff against a step that passed cleanly → isolates what's specific to
|
||||
@@ -108,43 +144,65 @@ manifest.txt bee_version, host, generated_at_utc, export_dir,
|
||||
then a flat "path<TAB>size" listing of every file
|
||||
in this bundle — a quick inventory/sanity check.
|
||||
README.md this file — read this first.
|
||||
export/ mirror of the live /appdata/bee/export directory —
|
||||
export/ raw, vendor-native hardware data, by device type —
|
||||
see "export/" below.
|
||||
systemd/ <service>.status.txt + <service>.journal.log per
|
||||
monitored systemd unit, plus combined.journal.log.
|
||||
techdump/, system/ raw command output not tied to a specific bee-*
|
||||
service — see below.
|
||||
status/ computed diagnosis, not raw data — see below.
|
||||
tasks/ bee's own task-run bookkeeping — see below.
|
||||
livecd/ this boot/kiosk session only, not hardware — see below.
|
||||
```
|
||||
|
||||
## `export/` in detail
|
||||
|
||||
This is a straight mirror of the live server's export directory, so
|
||||
everything below also applies when reading a raw `bee export` output
|
||||
directly (not wrapped in a support-bundle archive).
|
||||
Raw tool/command output, grouped by the physical device it describes.
|
||||
Vendor-neutral: nothing here assumes familiarity with bee.
|
||||
|
||||
| Path | What it is |
|
||||
|---|---|
|
||||
| `bee-audit.json` | The master hardware snapshot (see above). |
|
||||
| `bee-audit.log` | Log of the hardware-audit collector itself: what it queried, what it skipped and why (e.g. a field unsupported by the current driver — not necessarily an error). |
|
||||
| `bee-web.log` | Web UI service log. |
|
||||
| `bee-network.log`, `bee-nvidia.log`, `bee-sshsetup.log`, `bee-selfheal.log`, `bee-blackbox.log`, `bee-hpc-tuning.log` | Per-service startup/runtime logs for the correspondingly-named systemd unit. |
|
||||
| `reanimator.json` | The master hardware snapshot (see "Fastest answers" above) — ready to `POST /ingest/hardware`. |
|
||||
| `cpu/` | `lscpu.txt`, `dmidecode-type4.txt` (Processor Information). |
|
||||
| `memory/` | `dmidecode-type17.txt` (Memory Device, one entry per DIMM). |
|
||||
| `storage/` | `smartctl-<dev>.json`, `nvme-id-ctrl-<dev>.json`, `nvme-smart-log-<dev>.json`, `nvme-list.json`, `storcli64-drives.json`, `storcli2-show-all.json`, `lsblk.json`. |
|
||||
| `gpu/` | `nvidia-smi-*.txt/.csv` (state/topology/NVLink; `-fresh` variants are recaptured live at bundle-build time — see "GPU topology" above), `dcgmi-nvlink-status.txt`, `nvidia-bug-report.txt`, `nvidia-dcgm.service`/`nvidia-fabricmanager.service` status+journal, `fabric-manager-paths.txt`, `fabricmanager.log`/`nvlsm.log`, `lspci-nvidia-bridges-vv.txt`, `pcie-nvidia-link.txt`, `pcie-aer-sysfs.txt`, `kernel-aer-nvidia.txt` (AER/NVRM/Xid-filtered dmesg), `lspci-video-vv.txt`, `systemctl-nvidia-units.txt`. AMD systems get `rocm-smi*.txt` here instead. |
|
||||
| `network/` | `ethtool-{info,link,module}.txt` (per-NIC), `mstflint-query.txt` (Mellanox/NVIDIA NICs). |
|
||||
| `platform/` | `dmidecode-type{0,1,2}.txt` (BIOS/System/Baseboard), `ipmitool-{fru,sdr,sensor,sel,sel-time}.txt` (BMC), `sensors.json`, `lspci-{nn,vmm,vvv}.txt`. |
|
||||
|
||||
## `status/` in detail
|
||||
|
||||
Computed diagnosis, not raw data — see "Fastest answers" above for
|
||||
`component-status.json` and `metrics.db`.
|
||||
|
||||
| Path | What it is |
|
||||
|---|---|
|
||||
| `component-status.json` | Current rolled-up component health, written by bee's live `/dev/kmsg` watcher and by SAT run results. Source of `reanimator.json`'s `status`/`status_history` fields. |
|
||||
| `runtime-health.json` / `.log` | A lighter, more frequent health snapshot than the full audit — good for "was it fine 5 minutes ago" without the cost of a full re-audit. |
|
||||
| `component-status.json` | Current rolled-up component health — see "Fastest answers" above. |
|
||||
| `blackbox-state.json` | Present if continuous blackbox capture (mirroring this export dir to removable media on a schedule) is or was active; tracks the sync target and last successful sync. |
|
||||
| `techdump/` | Raw diagnostic command output, captured once per audit cycle. This is ground truth: `nvidia-smi-*.txt/.csv` (GPU state/topology/NVLink), `storcli64-drives.json` / `storcli2-show-all.json` (RAID), `lspci-*.txt`, `lscpu.txt`, `lsblk.json`, `dmidecode-*.txt`, `ipmitool-*.txt` (BMC sensors/SEL/FRU), `smartctl-*.json`, `nvme-list.json`, `sensors.json`. Higher-level views (the audit JSON, the web UI's topology page) are built from these. |
|
||||
| `bee-sat/<target>-<timestamp>/` | One directory per acceptance-test run — `target` is what was tested (`gpu-nvidia`, `cpu`, `memory`, `storage`, `nccl-tests`, `gpu-nvidia-bandwidth`, `nvidia-config` covering GPU config/NVLink/Confidential-Computing readiness, etc). Each contains `summary.txt` (key=value, always has `overall_status`), a full human-readable report, numbered per-job logs, and `verbose.log` (every subprocess invocation + exit code — the place to look when a summary doesn't explain *why* something failed). |
|
||||
| `tasks/<NNN>_<slug>_<done\|failed>/` | One directory per task launched from the web UI's task queue — see "Fastest answers" above. |
|
||||
| `tasks-state.json` | Index of every task (id, target, status, timestamps, artifact paths). |
|
||||
| `systemd/` (nested) | Historical per-service snapshots captured as part of an audit cycle, same shape as the top-level `systemd/` described below. |
|
||||
| `metrics.db` | SQLite time-series (CPU/mem load, power draw/source/mode) — see "Fastest answers" above. |
|
||||
|
||||
## `systemd/` and `system/` (top level, sibling of `export/`)
|
||||
## `tasks/` in detail
|
||||
|
||||
- `systemd/<service>.status.txt`, `systemd/<service>.journal.log`,
|
||||
`systemd/combined.journal.log` — captured fresh at bundle-build time (not
|
||||
mirrored from `export/`), so this is the most current service state.
|
||||
- `system/` — general OS-level diagnostics not specific to bee:
|
||||
`dmesg.txt`, X server / display-manager logs. Mostly relevant to
|
||||
physical/console access issues, not hardware health.
|
||||
bee's own bookkeeping. Duplication with `export/` is expected here — this
|
||||
tier keeps the full story of a specific run, `export/` keeps the clean
|
||||
current-state answer.
|
||||
|
||||
| Path | What it is |
|
||||
|---|---|
|
||||
| `bee-sat/<target>-<timestamp>/` | One directory per acceptance-test run — `target` is what was tested (`gpu-nvidia`, `cpu`, `memory`, `storage`, `nccl-tests`, `gpu-nvidia-bandwidth`, `nvidia-config` covering GPU config/NVLink/Confidential-Computing readiness, etc). Each contains `summary.txt` (key=value, always has `overall_status`), a full human-readable report (e.g. `storage`'s per-drive `disk-NN-<dev>-report.txt` — a narrative verdict, not raw tool output; the raw data behind it lives in `export/storage/`), numbered per-job logs, and `verbose.log` (every subprocess invocation + exit code — the place to look when a summary doesn't explain *why* something failed). |
|
||||
| `bee-bench/` | Benchmark run artifacts, if any have been run. |
|
||||
| `<NNN>_<slug>_<done\|failed>/` | One directory per task launched from the web UI's task queue — see "Fastest answers" above. |
|
||||
| `_state/bee-audit.json` | The full hardware-audit document as bee last generated it — same content as `export/reanimator.json`, kept here as bee's own internal record. |
|
||||
| `_state/tasks-state.json` | Index of every task (id, target, status, timestamps, artifact paths). |
|
||||
| `_state/blackbox-state.json` | Present if continuous blackbox capture (mirroring the export dir to removable media on a schedule) is or was active; tracks the sync target and last successful sync. |
|
||||
| `_services/bee-*.log` | Per-service startup/runtime logs for each bee-owned systemd unit. |
|
||||
| `_services/<service>.status.txt`, `.journal.log`, `combined.journal.log` | Captured fresh at bundle-build time, for every bee-owned service (`bee-audit`, `bee-web`, `bee-network`, `bee-nvidia`, `bee-preflight`, `bee-selfheal`, `bee-sshsetup`, `bee-blackbox`). |
|
||||
|
||||
## `livecd/` in detail
|
||||
|
||||
Only relevant to this particular live-boot/kiosk session — not the physical
|
||||
hardware. Safe to ignore entirely when troubleshooting hardware.
|
||||
|
||||
| Path | What it is |
|
||||
|---|---|
|
||||
| `gui/` | `Xorg.0.log`, `lightdm/`, `xinput-list.txt`, `libinput-list-devices.txt`, `drm-cards.txt`, `proc-fb.txt`, `input-devices.txt`, `udevadm-input.txt`, `loginctl-{sessions,seats}.txt`, `ps-gui.txt`, `systemctl-gui-units.txt`, `home-bee-xsession-errors.log`, `home-bee-chromium-debug.log`, `dmesg-gui-video-input.txt` (GUI/video/input-filtered dmesg), `display-manager.service`/`lightdm.service` status+journal. |
|
||||
| `host/` | `uname.txt`, `cmdline.txt`, `lsmod.txt`, `mount.txt`, `df-h.txt`, `dmesg.txt` (full kernel ring buffer for this boot), `ip-{addr,link,link-stats,route}.txt` (this session's network config, e.g. DHCP-assigned addresses — not a hardware fact about the NIC itself; NIC hardware/firmware state is in `export/network/`). |
|
||||
|
||||
## Timestamps
|
||||
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -582,10 +581,10 @@ func (w *blackboxWorker) syncCycle() error {
|
||||
w.recordMountpoint(mountpoint, mountedByBee)
|
||||
|
||||
root := filepath.Join(mountpoint, w.runtime.bootFolder)
|
||||
if err := os.MkdirAll(filepath.Join(root, "export"), 0755); err != nil {
|
||||
if err := os.MkdirAll(root, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := syncDirectoryTree(w.runtime.exportDir, filepath.Join(root, "export")); err != nil {
|
||||
if err := categorizeExportTree(w.runtime.exportDir, root); err != nil {
|
||||
return err
|
||||
}
|
||||
// Same doc pair the support bundle ships at its root — a blackbox
|
||||
@@ -625,18 +624,19 @@ func (w *blackboxWorker) recordMountpoint(mountpoint string, mountedByBee bool)
|
||||
}
|
||||
|
||||
func (w *blackboxWorker) captureSnapshots(root string) error {
|
||||
if err := captureCommandAtomic(filepath.Join(root, "systemd", "combined.journal.log"), "journalctl", "--no-pager", "--since", w.runtime.bootStarted.Format(time.RFC3339)); err != nil {
|
||||
if err := captureCommandAtomic(filepath.Join(root, "tasks", "_services", "combined.journal.log"), "journalctl", "--no-pager", "--since", w.runtime.bootStarted.Format(time.RFC3339)); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, svc := range supportBundleServices {
|
||||
if err := captureCommandAtomic(filepath.Join(root, "systemd", svc+".journal.log"), "journalctl", "--no-pager", "-u", svc, "--since", w.runtime.bootStarted.Format(time.RFC3339)); err != nil {
|
||||
dir := filepath.Join(root, serviceBundleDir(svc))
|
||||
if err := captureCommandAtomic(filepath.Join(dir, svc+".journal.log"), "journalctl", "--no-pager", "-u", svc, "--since", w.runtime.bootStarted.Format(time.RFC3339)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := captureCommandAtomic(filepath.Join(root, "systemd", svc+".status.txt"), "systemctl", "status", svc, "--no-pager"); err != nil {
|
||||
if err := captureCommandAtomic(filepath.Join(dir, svc+".status.txt"), "systemctl", "status", svc, "--no-pager"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := captureCommandAtomic(filepath.Join(root, "system", "dmesg.txt"), "dmesg"); err != nil {
|
||||
if err := captureCommandAtomic(filepath.Join(root, "livecd", "host", "dmesg.txt"), "dmesg"); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range supportBundleOptionalFiles {
|
||||
@@ -803,58 +803,6 @@ func writeBlackboxMarker(mountpoint string, marker BlackboxMarker) error {
|
||||
return writeJSONAtomic(filepath.Join(mountpoint, blackboxMarkerName), marker)
|
||||
}
|
||||
|
||||
func syncDirectoryTree(srcDir, dstDir string) error {
|
||||
seen := make(map[string]struct{})
|
||||
err := filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(srcDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.Clean(rel)
|
||||
if rel == "." {
|
||||
seen["."] = struct{}{}
|
||||
return os.MkdirAll(dstDir, 0755)
|
||||
}
|
||||
seen[rel] = struct{}{}
|
||||
dstPath := filepath.Join(dstDir, rel)
|
||||
if d.IsDir() {
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.MkdirAll(dstPath, info.Mode().Perm())
|
||||
}
|
||||
return copyFileIfChanged(path, dstPath)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return removeMissingPaths(dstDir, seen)
|
||||
}
|
||||
|
||||
func removeMissingPaths(dstDir string, seen map[string]struct{}) error {
|
||||
return filepath.WalkDir(dstDir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(dstDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.Clean(rel)
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
if _, ok := seen[rel]; ok {
|
||||
return nil
|
||||
}
|
||||
return os.RemoveAll(path)
|
||||
})
|
||||
}
|
||||
|
||||
func copyFileIfChanged(src, dst string) error {
|
||||
info, err := os.Stat(src)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DefaultMetricsDBPath is the SQLite time-series metrics store, kept outside
|
||||
// DefaultExportDir (unlike everything else this package copies into a
|
||||
// support bundle / blackbox mirror) because it's a live, actively-written
|
||||
// database rather than a point-in-time capture.
|
||||
const DefaultMetricsDBPath = "/appdata/bee/metrics.db"
|
||||
|
||||
// techdumpBucketFor maps a filename produced by platform.CaptureTechnicalDump
|
||||
// into export/techdump/ to its device-type bucket under export/ in the
|
||||
// categorized bundle layout, so a vendor engineer can find "the GPU stuff"
|
||||
// without knowing bee's internal file names.
|
||||
func techdumpBucketFor(name string) string {
|
||||
switch {
|
||||
case name == "lscpu.txt", name == "dmidecode-type4.txt":
|
||||
return "cpu"
|
||||
case name == "dmidecode-type17.txt":
|
||||
return "memory"
|
||||
case name == "lsblk.json", name == "storcli64-drives.json", name == "storcli2-show-all.json",
|
||||
strings.HasPrefix(name, "smartctl-"), strings.HasPrefix(name, "nvme-"):
|
||||
return "storage"
|
||||
case name == "nvidia-smi-q.txt", name == "nvidia-smi-query.csv", name == "nvidia-smi-conf-compute-q.txt",
|
||||
name == "nvidia-smi-topo.txt", name == "nvidia-smi-nvlink-status.txt", name == "nvidia-smi-nvlink-errors.txt",
|
||||
name == "rocm-smi.txt", name == "rocm-smi-showallinfo.txt":
|
||||
return "gpu"
|
||||
default:
|
||||
// dmidecode-type0/1/2, ipmitool-*, sensors.json, lspci-vmm/vvv, and
|
||||
// anything CaptureTechnicalDump adds later that isn't claimed above.
|
||||
return "platform"
|
||||
}
|
||||
}
|
||||
|
||||
// keepForBundleCopy excludes previously-built bundle archives that may be
|
||||
// sitting in the export dir (defensive — BuildSupportBundle itself stages
|
||||
// and writes its .tar.gz under os.TempDir(), not exportDir).
|
||||
func keepForBundleCopy(rel string, _ os.FileInfo) bool {
|
||||
cleanRel := filepath.ToSlash(strings.TrimPrefix(filepath.Clean(rel), "./"))
|
||||
if cleanRel == "" {
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(cleanRel, "bee-sat/") && strings.HasSuffix(cleanRel, ".tar.gz") {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(filepath.Base(cleanRel), "bee-support-") && strings.HasSuffix(cleanRel, ".tar.gz") {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// categorizeExportTree copies the live bee export directory into destRoot,
|
||||
// reshaped into the bundle's canonical layout:
|
||||
//
|
||||
// - export/<device-bucket>/ — raw vendor-tool output (persisted techdump),
|
||||
// vendor-neutral, grouped by what the hardware is rather than which bee
|
||||
// tool produced it.
|
||||
// - export/reanimator.json — the hardware snapshot, ready to POST to
|
||||
// Reanimator's /ingest/hardware endpoint.
|
||||
// - status/ — computed diagnosis, not raw data: component
|
||||
// health verdicts, runtime status, metrics time series.
|
||||
// - tasks/ — bee's own task-run bookkeeping (SAT/bench
|
||||
// run logs, orchestration state, service logs). Duplication with
|
||||
// export/ here is expected (e.g. a SAT run's narrative disk report).
|
||||
//
|
||||
// Used by both BuildSupportBundle (on-demand tar.gz) and the blackbox USB
|
||||
// mirror, so both artifacts share one shape.
|
||||
func categorizeExportTree(srcExportDir, destRoot string) error {
|
||||
exportDir := filepath.Join(destRoot, "export")
|
||||
statusDir := filepath.Join(destRoot, "status")
|
||||
tasksDir := filepath.Join(destRoot, "tasks")
|
||||
stateDir := filepath.Join(tasksDir, "_state")
|
||||
servicesDir := filepath.Join(tasksDir, "_services")
|
||||
|
||||
for _, dir := range []string{exportDir, statusDir, tasksDir, stateDir, servicesDir} {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Persisted hardware captures -> export/<bucket>/
|
||||
techdumpSrc := filepath.Join(srcExportDir, "techdump")
|
||||
if entries, err := os.ReadDir(techdumpSrc); err == nil {
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
// Stale leftover from before RunStorageAcceptancePack stopped
|
||||
// mirroring its narrative disk report here (it's bee's own
|
||||
// verdict, not raw tool output — belongs in tasks/bee-sat/ only).
|
||||
// Units captured with an older bee build may still have one on
|
||||
// disk; don't propagate it into the categorized layout.
|
||||
if strings.HasPrefix(entry.Name(), "disk-") && strings.HasSuffix(entry.Name(), "-report.txt") {
|
||||
continue
|
||||
}
|
||||
dst := filepath.Join(exportDir, techdumpBucketFor(entry.Name()), entry.Name())
|
||||
if err := copyPath(filepath.Join(techdumpSrc, entry.Name()), dst); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
// bee-audit.json -> tasks/_state/ (internal record) and
|
||||
// export/reanimator.json (vendor-facing copy), both carrying the latest
|
||||
// SAT-overlay-normalized status verdicts.
|
||||
if data, err := os.ReadFile(filepath.Join(srcExportDir, "bee-audit.json")); err == nil {
|
||||
normalized, err := ApplySATOverlay(data)
|
||||
if err != nil {
|
||||
normalized = data
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(stateDir, "bee-audit.json"), normalized, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(exportDir, "reanimator.json"), normalized, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
// Computed diagnosis data -> status/
|
||||
for _, name := range []string{"component-status.json", "runtime-health.json", "runtime-health.log"} {
|
||||
src := filepath.Join(srcExportDir, name)
|
||||
if _, err := os.Stat(src); err != nil {
|
||||
continue
|
||||
}
|
||||
if err := copyPath(src, filepath.Join(statusDir, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(DefaultMetricsDBPath); err == nil {
|
||||
if err := copyPath(DefaultMetricsDBPath, filepath.Join(statusDir, "metrics.db")); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Remaining orchestration state -> tasks/_state/
|
||||
for _, name := range []string{"blackbox-state.json", "tasks-state.json"} {
|
||||
src := filepath.Join(srcExportDir, name)
|
||||
if _, err := os.Stat(src); err != nil {
|
||||
continue
|
||||
}
|
||||
if err := copyPath(src, filepath.Join(stateDir, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// bee-*.log service logs -> tasks/_services/
|
||||
if entries, err := os.ReadDir(srcExportDir); err == nil {
|
||||
for _, entry := range entries {
|
||||
name := entry.Name()
|
||||
if entry.IsDir() || !strings.HasPrefix(name, "bee-") || !strings.HasSuffix(name, ".log") {
|
||||
continue
|
||||
}
|
||||
if err := copyPath(filepath.Join(srcExportDir, name), filepath.Join(servicesDir, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
|
||||
// SAT/bench run dirs, unchanged internally -> tasks/bee-sat/, tasks/bee-bench/
|
||||
for _, name := range []string{"bee-sat", "bee-bench"} {
|
||||
src := filepath.Join(srcExportDir, name)
|
||||
if _, err := os.Stat(src); err != nil {
|
||||
continue
|
||||
}
|
||||
if err := copyPathFiltered(srcExportDir, src, filepath.Join(tasksDir, name), keepForBundleCopy); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Orchestration task-run reports, unchanged internally -> tasks/<NNN>_..._done/
|
||||
if src := filepath.Join(srcExportDir, "tasks"); dirExists(src) {
|
||||
if err := copyDirContentsFiltered(src, tasksDir, keepForBundleCopy); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func dirExists(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
return err == nil && info.IsDir()
|
||||
}
|
||||
@@ -44,43 +44,58 @@ var supportBundleServices = []string{
|
||||
"nvidia-fabricmanager.service",
|
||||
}
|
||||
|
||||
// serviceBundleDir returns the bundle-relative directory a service's
|
||||
// status/journal capture belongs in: bee's own daemons are internal
|
||||
// bookkeeping, NVIDIA/DCGM/fabric-manager daemons are vendor-facing hardware
|
||||
// diagnostics, and the display stack is LiveCD/GUI-session-only.
|
||||
func serviceBundleDir(svc string) string {
|
||||
switch svc {
|
||||
case "nvidia-dcgm.service", "nvidia-fabricmanager.service":
|
||||
return "export/gpu"
|
||||
case "display-manager.service", "lightdm.service":
|
||||
return "livecd/gui"
|
||||
default:
|
||||
return "tasks/_services"
|
||||
}
|
||||
}
|
||||
|
||||
var supportBundleCommands = []struct {
|
||||
name string
|
||||
cmd []string
|
||||
}{
|
||||
{name: "system/uname.txt", cmd: []string{"uname", "-a"}},
|
||||
{name: "system/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
|
||||
{name: "system/lsmod.txt", cmd: []string{"lsmod"}},
|
||||
{name: "techdump/lspci-nn.txt", cmd: []string{"lspci", "-nn"}},
|
||||
{name: "system/ip-addr.txt", cmd: []string{"ip", "addr"}},
|
||||
{name: "system/ip-link.txt", cmd: []string{"ip", "-details", "link", "show"}},
|
||||
{name: "system/ip-link-stats.txt", cmd: []string{"ip", "-s", "link", "show"}},
|
||||
{name: "system/ip-route.txt", cmd: []string{"ip", "route"}},
|
||||
{name: "system/mount.txt", cmd: []string{"mount"}},
|
||||
{name: "system/df-h.txt", cmd: []string{"df", "-h"}},
|
||||
{name: "system/dmesg.txt", cmd: []string{"dmesg"}},
|
||||
{name: "system/dmesg-gui-video-input.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}},
|
||||
{name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
|
||||
{name: "livecd/host/lsmod.txt", cmd: []string{"lsmod"}},
|
||||
{name: "export/platform/lspci-nn.txt", cmd: []string{"lspci", "-nn"}},
|
||||
{name: "livecd/host/ip-addr.txt", cmd: []string{"ip", "addr"}},
|
||||
{name: "livecd/host/ip-link.txt", cmd: []string{"ip", "-details", "link", "show"}},
|
||||
{name: "livecd/host/ip-link-stats.txt", cmd: []string{"ip", "-s", "link", "show"}},
|
||||
{name: "livecd/host/ip-route.txt", cmd: []string{"ip", "route"}},
|
||||
{name: "livecd/host/mount.txt", cmd: []string{"mount"}},
|
||||
{name: "livecd/host/df-h.txt", cmd: []string{"df", "-h"}},
|
||||
{name: "livecd/host/dmesg.txt", cmd: []string{"dmesg"}},
|
||||
{name: "livecd/gui/dmesg-gui-video-input.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v dmesg >/dev/null 2>&1; then
|
||||
dmesg | grep -iE 'nvidia|drm|fb|framebuffer|vesa|efi|lightdm|Xorg|input|hid|usb|keyboard|mouse|virtual keyboard|virtual mouse|ami|aspeed|ast' || echo "no GUI/video/input kernel messages found"
|
||||
else
|
||||
echo "dmesg not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "techdump/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/gpu/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v dmesg >/dev/null 2>&1; then
|
||||
dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
|
||||
else
|
||||
echo "dmesg not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/loginctl-sessions.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "livecd/gui/loginctl-sessions.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v loginctl >/dev/null 2>&1; then
|
||||
loginctl list-sessions 2>&1 || true
|
||||
else
|
||||
echo "loginctl not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/loginctl-seats.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "livecd/gui/loginctl-seats.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v loginctl >/dev/null 2>&1; then
|
||||
loginctl list-seats 2>&1 || true
|
||||
echo
|
||||
@@ -93,10 +108,10 @@ else
|
||||
echo "loginctl not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/ps-gui.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "livecd/gui/ps-gui.txt", cmd: []string{"sh", "-c", `
|
||||
ps -ef | grep -iE 'lightdm|Xorg|X$|openbox|chromium|chrome|xinit|xsession' | grep -v grep || echo "no GUI processes found"
|
||||
`}},
|
||||
{name: "techdump/lspci-video-vv.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/gpu/lspci-video-vv.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v lspci >/dev/null 2>&1; then
|
||||
echo "lspci not found"
|
||||
exit 0
|
||||
@@ -112,8 +127,8 @@ if [ "$found" -eq 0 ]; then
|
||||
echo "no display-class PCI devices found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/proc-fb.txt", cmd: []string{"cat", "/proc/fb"}},
|
||||
{name: "system/drm-cards.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "livecd/gui/proc-fb.txt", cmd: []string{"cat", "/proc/fb"}},
|
||||
{name: "livecd/gui/drm-cards.txt", cmd: []string{"sh", "-c", `
|
||||
if [ -d /sys/class/drm ]; then
|
||||
for path in /sys/class/drm/card*; do
|
||||
[ -e "$path" ] || continue
|
||||
@@ -130,14 +145,14 @@ else
|
||||
echo "/sys/class/drm not present"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/input-devices.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "livecd/gui/input-devices.txt", cmd: []string{"sh", "-c", `
|
||||
if [ -r /proc/bus/input/devices ]; then
|
||||
cat /proc/bus/input/devices
|
||||
else
|
||||
echo "/proc/bus/input/devices not readable"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/udevadm-input.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "livecd/gui/udevadm-input.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v udevadm >/dev/null 2>&1; then
|
||||
echo "udevadm not found"
|
||||
exit 0
|
||||
@@ -154,21 +169,21 @@ if [ "$found" -eq 0 ]; then
|
||||
echo "no /dev/input/event* devices found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/xinput-list.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "livecd/gui/xinput-list.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v xinput >/dev/null 2>&1; then
|
||||
DISPLAY=:0 xinput --list 2>&1 || true
|
||||
else
|
||||
echo "xinput not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/libinput-list-devices.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "livecd/gui/libinput-list-devices.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v libinput >/dev/null 2>&1; then
|
||||
libinput list-devices 2>&1 || true
|
||||
else
|
||||
echo "libinput not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/systemctl-gui-units.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "livecd/gui/systemctl-gui-units.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v systemctl >/dev/null 2>&1; then
|
||||
echo "systemctl not found"
|
||||
exit 0
|
||||
@@ -182,35 +197,35 @@ echo
|
||||
echo "=== failed units ==="
|
||||
systemctl --failed --no-pager 2>&1 | grep -iE 'lightdm|display-manager|Xorg' || echo "no failed GUI units"
|
||||
`}},
|
||||
{name: "techdump/nvidia-smi-topo.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/gpu/nvidia-smi-topo-fresh.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v nvidia-smi >/dev/null 2>&1; then
|
||||
nvidia-smi topo -m 2>&1 || true
|
||||
else
|
||||
echo "nvidia-smi not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "techdump/nvidia-smi-nvlink-status.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/gpu/nvidia-smi-nvlink-status-fresh.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v nvidia-smi >/dev/null 2>&1; then
|
||||
nvidia-smi nvlink -s 2>&1 || true
|
||||
else
|
||||
echo "nvidia-smi not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "techdump/nvidia-smi-nvlink-errors.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/gpu/nvidia-smi-nvlink-errors-fresh.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v nvidia-smi >/dev/null 2>&1; then
|
||||
nvidia-smi nvlink -e 2>&1 || true
|
||||
else
|
||||
echo "nvidia-smi not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "techdump/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/gpu/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v dcgmi >/dev/null 2>&1; then
|
||||
dcgmi nvlink --link-status 2>&1 || true
|
||||
else
|
||||
echo "dcgmi not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "techdump/nvidia-bug-report.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
|
||||
nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 \
|
||||
&& cat /tmp/bee-nvidia-bug-report.log \
|
||||
@@ -219,7 +234,7 @@ else
|
||||
echo "nvidia-bug-report.sh not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "techdump/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/gpu/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v systemctl >/dev/null 2>&1; then
|
||||
echo "systemctl not found"
|
||||
exit 0
|
||||
@@ -233,7 +248,7 @@ echo
|
||||
echo "=== failed units ==="
|
||||
systemctl --failed --no-pager 2>&1 | grep -iE 'nvidia|fabric' || echo "no failed nvidia/fabric units"
|
||||
`}},
|
||||
{name: "techdump/fabric-manager-paths.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/gpu/fabric-manager-paths.txt", cmd: []string{"sh", "-c", `
|
||||
for candidate in \
|
||||
/usr/bin/nvidia-fabricmanager \
|
||||
/usr/bin/nv-fabricmanager \
|
||||
@@ -249,7 +264,7 @@ if ! ls /usr/bin/nvidia-fabricmanager /usr/bin/nv-fabricmanager /usr/bin/nvidia-
|
||||
echo "no fabric manager binaries found"
|
||||
fi
|
||||
`}},
|
||||
{name: "techdump/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/gpu/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v lspci >/dev/null 2>&1; then
|
||||
echo "lspci not found"
|
||||
exit 0
|
||||
@@ -271,7 +286,7 @@ if [ "$found" -eq 0 ]; then
|
||||
echo "no NVIDIA PCI devices found"
|
||||
fi
|
||||
`}},
|
||||
{name: "techdump/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/gpu/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", `
|
||||
for d in /sys/bus/pci/devices/*/; do
|
||||
vendor=$(cat "$d/vendor" 2>/dev/null)
|
||||
[ "$vendor" = "0x10de" ] || continue
|
||||
@@ -287,7 +302,7 @@ for d in /sys/bus/pci/devices/*/; do
|
||||
done
|
||||
done
|
||||
`}},
|
||||
{name: "techdump/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
|
||||
found=0
|
||||
for dev in /sys/bus/pci/devices/*; do
|
||||
[ -e "$dev" ] || continue
|
||||
@@ -311,7 +326,7 @@ if [ "$found" -eq 0 ]; then
|
||||
echo "no PCIe AER sysfs counters found"
|
||||
fi
|
||||
`}},
|
||||
{name: "techdump/ethtool-info.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/network/ethtool-info.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v ethtool >/dev/null 2>&1; then
|
||||
echo "ethtool not found"
|
||||
exit 0
|
||||
@@ -330,7 +345,7 @@ if [ "$found" -eq 0 ]; then
|
||||
echo "no interfaces found"
|
||||
fi
|
||||
`}},
|
||||
{name: "techdump/ethtool-link.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/network/ethtool-link.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v ethtool >/dev/null 2>&1; then
|
||||
echo "ethtool not found"
|
||||
exit 0
|
||||
@@ -349,7 +364,7 @@ if [ "$found" -eq 0 ]; then
|
||||
echo "no interfaces found"
|
||||
fi
|
||||
`}},
|
||||
{name: "techdump/ethtool-module.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/network/ethtool-module.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v ethtool >/dev/null 2>&1; then
|
||||
echo "ethtool not found"
|
||||
exit 0
|
||||
@@ -368,7 +383,7 @@ if [ "$found" -eq 0 ]; then
|
||||
echo "no interfaces found"
|
||||
fi
|
||||
`}},
|
||||
{name: "techdump/mstflint-query.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/network/mstflint-query.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v mstflint >/dev/null 2>&1; then
|
||||
echo "mstflint not found"
|
||||
exit 0
|
||||
@@ -394,19 +409,19 @@ var supportBundleOptionalFiles = []struct {
|
||||
name string
|
||||
src string
|
||||
}{
|
||||
{name: "system/kern.log", src: "/var/log/kern.log"},
|
||||
{name: "system/syslog.txt", src: "/var/log/syslog"},
|
||||
{name: "system/Xorg.0.log", src: "/var/log/Xorg.0.log"},
|
||||
{name: "system/Xorg.0.log.old", src: "/var/log/Xorg.0.log.old"},
|
||||
{name: "system/lightdm/lightdm.log", src: "/var/log/lightdm/lightdm.log"},
|
||||
{name: "system/lightdm/x-0.log", src: "/var/log/lightdm/x-0.log"},
|
||||
{name: "system/lightdm/x-0-greeter.log", src: "/var/log/lightdm/x-0-greeter.log"},
|
||||
{name: "system/home-bee-xsession-errors.log", src: "/home/bee/.xsession-errors"},
|
||||
{name: "system/home-bee-chromium-debug.log", src: "/tmp/bee-chrome/chrome_debug.log"},
|
||||
{name: "techdump/fabricmanager.log", src: "/var/log/fabricmanager.log"},
|
||||
{name: "techdump/nvlsm.log", src: "/var/log/nvlsm.log"},
|
||||
{name: "techdump/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"},
|
||||
{name: "techdump/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"},
|
||||
{name: "livecd/host/kern.log", src: "/var/log/kern.log"},
|
||||
{name: "livecd/host/syslog.txt", src: "/var/log/syslog"},
|
||||
{name: "livecd/gui/Xorg.0.log", src: "/var/log/Xorg.0.log"},
|
||||
{name: "livecd/gui/Xorg.0.log.old", src: "/var/log/Xorg.0.log.old"},
|
||||
{name: "livecd/gui/lightdm/lightdm.log", src: "/var/log/lightdm/lightdm.log"},
|
||||
{name: "livecd/gui/lightdm/x-0.log", src: "/var/log/lightdm/x-0.log"},
|
||||
{name: "livecd/gui/lightdm/x-0-greeter.log", src: "/var/log/lightdm/x-0-greeter.log"},
|
||||
{name: "livecd/gui/home-bee-xsession-errors.log", src: "/home/bee/.xsession-errors"},
|
||||
{name: "livecd/gui/home-bee-chromium-debug.log", src: "/tmp/bee-chrome/chrome_debug.log"},
|
||||
{name: "export/gpu/fabricmanager.log", src: "/var/log/fabricmanager.log"},
|
||||
{name: "export/gpu/nvlsm.log", src: "/var/log/nvlsm.log"},
|
||||
{name: "export/gpu/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"},
|
||||
{name: "export/gpu/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"},
|
||||
}
|
||||
|
||||
const supportBundleGlob = "????-??-?? (BEE-SP*)*.tar.gz"
|
||||
@@ -431,17 +446,18 @@ func BuildSupportBundle(exportDir string) (string, error) {
|
||||
}
|
||||
defer os.RemoveAll(stageRoot)
|
||||
|
||||
if err := copyExportDirForSupportBundle(exportDir, filepath.Join(stageRoot, "export")); err != nil {
|
||||
if err := categorizeExportTree(exportDir, stageRoot); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := writeJournalDump(filepath.Join(stageRoot, "systemd", "combined.journal.log")); err != nil {
|
||||
if err := writeJournalDump(filepath.Join(stageRoot, "tasks", "_services", "combined.journal.log")); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, svc := range supportBundleServices {
|
||||
if err := writeCommandOutput(filepath.Join(stageRoot, "systemd", svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil {
|
||||
dir := filepath.Join(stageRoot, serviceBundleDir(svc))
|
||||
if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := writeCommandOutput(filepath.Join(stageRoot, "systemd", svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil {
|
||||
if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
@@ -715,40 +731,6 @@ func copyDirContents(srcDir, dstDir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyExportDirForSupportBundle(srcDir, dstDir string) error {
|
||||
if err := copyDirContentsFiltered(srcDir, dstDir, func(rel string, info os.FileInfo) bool {
|
||||
cleanRel := filepath.ToSlash(strings.TrimPrefix(filepath.Clean(rel), "./"))
|
||||
if cleanRel == "" {
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(cleanRel, "bee-sat/") && strings.HasSuffix(cleanRel, ".tar.gz") {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(filepath.Base(cleanRel), "bee-support-") && strings.HasSuffix(cleanRel, ".tar.gz") {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return normalizeSupportBundleAuditJSON(filepath.Join(dstDir, "bee-audit.json"))
|
||||
}
|
||||
|
||||
func normalizeSupportBundleAuditJSON(path string) error {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
normalized, err := ApplySATOverlay(data)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return os.WriteFile(path, normalized, 0644)
|
||||
}
|
||||
|
||||
func copyDirContentsFiltered(srcDir, dstDir string, keep func(rel string, info os.FileInfo) bool) error {
|
||||
entries, err := os.ReadDir(srcDir)
|
||||
if err != nil {
|
||||
@@ -791,6 +773,14 @@ func copyPath(src, dst string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
// Skip rewriting files the blackbox worker already copied unchanged on a
|
||||
// prior cycle — avoids needless flash wear on the removable target every
|
||||
// sync period. Size+mtime, not a full byte comparison, so large files
|
||||
// (e.g. status/metrics.db) can still stream-copy instead of loading
|
||||
// whole into memory.
|
||||
if dstInfo, err := os.Stat(dst); err == nil && dstInfo.Size() == info.Size() && !dstInfo.ModTime().Before(info.ModTime()) {
|
||||
return nil
|
||||
}
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
Reference in New Issue
Block a user