platform/app: restructure support bundle/blackbox mirror into device-type export/, status/, tasks/, livecd/

Replaces the flat export/ + techdump/ + system/ + systemd/ layout (two
same-named-but-different techdump/ dirs, disk reports duplicated between a
SAT run dir and techdump/) with one shared categorizeExportTree used by both
BuildSupportBundle and the blackbox USB mirror:

- export/{cpu,memory,storage,gpu,network,platform}/ — raw vendor-tool
  output grouped by device type, plus export/reanimator.json ready to POST
  to Reanimator's /ingest/hardware endpoint.
- status/ — computed diagnosis (component-status.json, runtime-health.*,
  and metrics.db, previously missing from every bundle entirely).
- tasks/ — bee's own task-run bookkeeping (bee-sat/, bee-bench/, task
  reports, service logs) where duplication with export/ is expected.
- livecd/{gui,host}/ — live-boot/kiosk-session-only diagnostics, kept out
  of the hardware-facing tree.

Drops the redundant disk-report mirror write in RunStorageAcceptancePack
and the now-dead syncDirectoryTree/removeMissingPaths; adds a size+mtime
skip in copyPath so the blackbox mirror doesn't rewrite unchanged files
every cycle. README.md rewritten to match.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-28 14:33:26 +03:00
co-authored by Claude Sonnet 5
parent e03267a72f
commit fa618d5abb
6 changed files with 412 additions and 233 deletions
+79 -89
View File
@@ -44,43 +44,58 @@ var supportBundleServices = []string{
"nvidia-fabricmanager.service",
}
// serviceBundleDir returns the bundle-relative directory a service's
// status/journal capture belongs in: bee's own daemons are internal
// bookkeeping, NVIDIA/DCGM/fabric-manager daemons are vendor-facing hardware
// diagnostics, and the display stack is LiveCD/GUI-session-only.
func serviceBundleDir(svc string) string {
switch svc {
case "nvidia-dcgm.service", "nvidia-fabricmanager.service":
return "export/gpu"
case "display-manager.service", "lightdm.service":
return "livecd/gui"
default:
return "tasks/_services"
}
}
var supportBundleCommands = []struct {
name string
cmd []string
}{
{name: "system/uname.txt", cmd: []string{"uname", "-a"}},
{name: "system/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
{name: "system/lsmod.txt", cmd: []string{"lsmod"}},
{name: "techdump/lspci-nn.txt", cmd: []string{"lspci", "-nn"}},
{name: "system/ip-addr.txt", cmd: []string{"ip", "addr"}},
{name: "system/ip-link.txt", cmd: []string{"ip", "-details", "link", "show"}},
{name: "system/ip-link-stats.txt", cmd: []string{"ip", "-s", "link", "show"}},
{name: "system/ip-route.txt", cmd: []string{"ip", "route"}},
{name: "system/mount.txt", cmd: []string{"mount"}},
{name: "system/df-h.txt", cmd: []string{"df", "-h"}},
{name: "system/dmesg.txt", cmd: []string{"dmesg"}},
{name: "system/dmesg-gui-video-input.txt", cmd: []string{"sh", "-c", `
{name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}},
{name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
{name: "livecd/host/lsmod.txt", cmd: []string{"lsmod"}},
{name: "export/platform/lspci-nn.txt", cmd: []string{"lspci", "-nn"}},
{name: "livecd/host/ip-addr.txt", cmd: []string{"ip", "addr"}},
{name: "livecd/host/ip-link.txt", cmd: []string{"ip", "-details", "link", "show"}},
{name: "livecd/host/ip-link-stats.txt", cmd: []string{"ip", "-s", "link", "show"}},
{name: "livecd/host/ip-route.txt", cmd: []string{"ip", "route"}},
{name: "livecd/host/mount.txt", cmd: []string{"mount"}},
{name: "livecd/host/df-h.txt", cmd: []string{"df", "-h"}},
{name: "livecd/host/dmesg.txt", cmd: []string{"dmesg"}},
{name: "livecd/gui/dmesg-gui-video-input.txt", cmd: []string{"sh", "-c", `
if command -v dmesg >/dev/null 2>&1; then
dmesg | grep -iE 'nvidia|drm|fb|framebuffer|vesa|efi|lightdm|Xorg|input|hid|usb|keyboard|mouse|virtual keyboard|virtual mouse|ami|aspeed|ast' || echo "no GUI/video/input kernel messages found"
else
echo "dmesg not found"
fi
`}},
{name: "techdump/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", `
{name: "export/gpu/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", `
if command -v dmesg >/dev/null 2>&1; then
dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
else
echo "dmesg not found"
fi
`}},
{name: "system/loginctl-sessions.txt", cmd: []string{"sh", "-c", `
{name: "livecd/gui/loginctl-sessions.txt", cmd: []string{"sh", "-c", `
if command -v loginctl >/dev/null 2>&1; then
loginctl list-sessions 2>&1 || true
else
echo "loginctl not found"
fi
`}},
{name: "system/loginctl-seats.txt", cmd: []string{"sh", "-c", `
{name: "livecd/gui/loginctl-seats.txt", cmd: []string{"sh", "-c", `
if command -v loginctl >/dev/null 2>&1; then
loginctl list-seats 2>&1 || true
echo
@@ -93,10 +108,10 @@ else
echo "loginctl not found"
fi
`}},
{name: "system/ps-gui.txt", cmd: []string{"sh", "-c", `
{name: "livecd/gui/ps-gui.txt", cmd: []string{"sh", "-c", `
ps -ef | grep -iE 'lightdm|Xorg|X$|openbox|chromium|chrome|xinit|xsession' | grep -v grep || echo "no GUI processes found"
`}},
{name: "techdump/lspci-video-vv.txt", cmd: []string{"sh", "-c", `
{name: "export/gpu/lspci-video-vv.txt", cmd: []string{"sh", "-c", `
if ! command -v lspci >/dev/null 2>&1; then
echo "lspci not found"
exit 0
@@ -112,8 +127,8 @@ if [ "$found" -eq 0 ]; then
echo "no display-class PCI devices found"
fi
`}},
{name: "system/proc-fb.txt", cmd: []string{"cat", "/proc/fb"}},
{name: "system/drm-cards.txt", cmd: []string{"sh", "-c", `
{name: "livecd/gui/proc-fb.txt", cmd: []string{"cat", "/proc/fb"}},
{name: "livecd/gui/drm-cards.txt", cmd: []string{"sh", "-c", `
if [ -d /sys/class/drm ]; then
for path in /sys/class/drm/card*; do
[ -e "$path" ] || continue
@@ -130,14 +145,14 @@ else
echo "/sys/class/drm not present"
fi
`}},
{name: "system/input-devices.txt", cmd: []string{"sh", "-c", `
{name: "livecd/gui/input-devices.txt", cmd: []string{"sh", "-c", `
if [ -r /proc/bus/input/devices ]; then
cat /proc/bus/input/devices
else
echo "/proc/bus/input/devices not readable"
fi
`}},
{name: "system/udevadm-input.txt", cmd: []string{"sh", "-c", `
{name: "livecd/gui/udevadm-input.txt", cmd: []string{"sh", "-c", `
if ! command -v udevadm >/dev/null 2>&1; then
echo "udevadm not found"
exit 0
@@ -154,21 +169,21 @@ if [ "$found" -eq 0 ]; then
echo "no /dev/input/event* devices found"
fi
`}},
{name: "system/xinput-list.txt", cmd: []string{"sh", "-c", `
{name: "livecd/gui/xinput-list.txt", cmd: []string{"sh", "-c", `
if command -v xinput >/dev/null 2>&1; then
DISPLAY=:0 xinput --list 2>&1 || true
else
echo "xinput not found"
fi
`}},
{name: "system/libinput-list-devices.txt", cmd: []string{"sh", "-c", `
{name: "livecd/gui/libinput-list-devices.txt", cmd: []string{"sh", "-c", `
if command -v libinput >/dev/null 2>&1; then
libinput list-devices 2>&1 || true
else
echo "libinput not found"
fi
`}},
{name: "system/systemctl-gui-units.txt", cmd: []string{"sh", "-c", `
{name: "livecd/gui/systemctl-gui-units.txt", cmd: []string{"sh", "-c", `
if ! command -v systemctl >/dev/null 2>&1; then
echo "systemctl not found"
exit 0
@@ -182,35 +197,35 @@ echo
echo "=== failed units ==="
systemctl --failed --no-pager 2>&1 | grep -iE 'lightdm|display-manager|Xorg' || echo "no failed GUI units"
`}},
{name: "techdump/nvidia-smi-topo.txt", cmd: []string{"sh", "-c", `
{name: "export/gpu/nvidia-smi-topo-fresh.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-smi >/dev/null 2>&1; then
nvidia-smi topo -m 2>&1 || true
else
echo "nvidia-smi not found"
fi
`}},
{name: "techdump/nvidia-smi-nvlink-status.txt", cmd: []string{"sh", "-c", `
{name: "export/gpu/nvidia-smi-nvlink-status-fresh.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-smi >/dev/null 2>&1; then
nvidia-smi nvlink -s 2>&1 || true
else
echo "nvidia-smi not found"
fi
`}},
{name: "techdump/nvidia-smi-nvlink-errors.txt", cmd: []string{"sh", "-c", `
{name: "export/gpu/nvidia-smi-nvlink-errors-fresh.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-smi >/dev/null 2>&1; then
nvidia-smi nvlink -e 2>&1 || true
else
echo "nvidia-smi not found"
fi
`}},
{name: "techdump/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", `
{name: "export/gpu/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", `
if command -v dcgmi >/dev/null 2>&1; then
dcgmi nvlink --link-status 2>&1 || true
else
echo "dcgmi not found"
fi
`}},
{name: "techdump/nvidia-bug-report.txt", cmd: []string{"sh", "-c", `
{name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 \
&& cat /tmp/bee-nvidia-bug-report.log \
@@ -219,7 +234,7 @@ else
echo "nvidia-bug-report.sh not found"
fi
`}},
{name: "techdump/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", `
{name: "export/gpu/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", `
if ! command -v systemctl >/dev/null 2>&1; then
echo "systemctl not found"
exit 0
@@ -233,7 +248,7 @@ echo
echo "=== failed units ==="
systemctl --failed --no-pager 2>&1 | grep -iE 'nvidia|fabric' || echo "no failed nvidia/fabric units"
`}},
{name: "techdump/fabric-manager-paths.txt", cmd: []string{"sh", "-c", `
{name: "export/gpu/fabric-manager-paths.txt", cmd: []string{"sh", "-c", `
for candidate in \
/usr/bin/nvidia-fabricmanager \
/usr/bin/nv-fabricmanager \
@@ -249,7 +264,7 @@ if ! ls /usr/bin/nvidia-fabricmanager /usr/bin/nv-fabricmanager /usr/bin/nvidia-
echo "no fabric manager binaries found"
fi
`}},
{name: "techdump/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", `
{name: "export/gpu/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", `
if ! command -v lspci >/dev/null 2>&1; then
echo "lspci not found"
exit 0
@@ -271,7 +286,7 @@ if [ "$found" -eq 0 ]; then
echo "no NVIDIA PCI devices found"
fi
`}},
{name: "techdump/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", `
{name: "export/gpu/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", `
for d in /sys/bus/pci/devices/*/; do
vendor=$(cat "$d/vendor" 2>/dev/null)
[ "$vendor" = "0x10de" ] || continue
@@ -287,7 +302,7 @@ for d in /sys/bus/pci/devices/*/; do
done
done
`}},
{name: "techdump/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
{name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
found=0
for dev in /sys/bus/pci/devices/*; do
[ -e "$dev" ] || continue
@@ -311,7 +326,7 @@ if [ "$found" -eq 0 ]; then
echo "no PCIe AER sysfs counters found"
fi
`}},
{name: "techdump/ethtool-info.txt", cmd: []string{"sh", "-c", `
{name: "export/network/ethtool-info.txt", cmd: []string{"sh", "-c", `
if ! command -v ethtool >/dev/null 2>&1; then
echo "ethtool not found"
exit 0
@@ -330,7 +345,7 @@ if [ "$found" -eq 0 ]; then
echo "no interfaces found"
fi
`}},
{name: "techdump/ethtool-link.txt", cmd: []string{"sh", "-c", `
{name: "export/network/ethtool-link.txt", cmd: []string{"sh", "-c", `
if ! command -v ethtool >/dev/null 2>&1; then
echo "ethtool not found"
exit 0
@@ -349,7 +364,7 @@ if [ "$found" -eq 0 ]; then
echo "no interfaces found"
fi
`}},
{name: "techdump/ethtool-module.txt", cmd: []string{"sh", "-c", `
{name: "export/network/ethtool-module.txt", cmd: []string{"sh", "-c", `
if ! command -v ethtool >/dev/null 2>&1; then
echo "ethtool not found"
exit 0
@@ -368,7 +383,7 @@ if [ "$found" -eq 0 ]; then
echo "no interfaces found"
fi
`}},
{name: "techdump/mstflint-query.txt", cmd: []string{"sh", "-c", `
{name: "export/network/mstflint-query.txt", cmd: []string{"sh", "-c", `
if ! command -v mstflint >/dev/null 2>&1; then
echo "mstflint not found"
exit 0
@@ -394,19 +409,19 @@ var supportBundleOptionalFiles = []struct {
name string
src string
}{
{name: "system/kern.log", src: "/var/log/kern.log"},
{name: "system/syslog.txt", src: "/var/log/syslog"},
{name: "system/Xorg.0.log", src: "/var/log/Xorg.0.log"},
{name: "system/Xorg.0.log.old", src: "/var/log/Xorg.0.log.old"},
{name: "system/lightdm/lightdm.log", src: "/var/log/lightdm/lightdm.log"},
{name: "system/lightdm/x-0.log", src: "/var/log/lightdm/x-0.log"},
{name: "system/lightdm/x-0-greeter.log", src: "/var/log/lightdm/x-0-greeter.log"},
{name: "system/home-bee-xsession-errors.log", src: "/home/bee/.xsession-errors"},
{name: "system/home-bee-chromium-debug.log", src: "/tmp/bee-chrome/chrome_debug.log"},
{name: "techdump/fabricmanager.log", src: "/var/log/fabricmanager.log"},
{name: "techdump/nvlsm.log", src: "/var/log/nvlsm.log"},
{name: "techdump/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"},
{name: "techdump/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"},
{name: "livecd/host/kern.log", src: "/var/log/kern.log"},
{name: "livecd/host/syslog.txt", src: "/var/log/syslog"},
{name: "livecd/gui/Xorg.0.log", src: "/var/log/Xorg.0.log"},
{name: "livecd/gui/Xorg.0.log.old", src: "/var/log/Xorg.0.log.old"},
{name: "livecd/gui/lightdm/lightdm.log", src: "/var/log/lightdm/lightdm.log"},
{name: "livecd/gui/lightdm/x-0.log", src: "/var/log/lightdm/x-0.log"},
{name: "livecd/gui/lightdm/x-0-greeter.log", src: "/var/log/lightdm/x-0-greeter.log"},
{name: "livecd/gui/home-bee-xsession-errors.log", src: "/home/bee/.xsession-errors"},
{name: "livecd/gui/home-bee-chromium-debug.log", src: "/tmp/bee-chrome/chrome_debug.log"},
{name: "export/gpu/fabricmanager.log", src: "/var/log/fabricmanager.log"},
{name: "export/gpu/nvlsm.log", src: "/var/log/nvlsm.log"},
{name: "export/gpu/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"},
{name: "export/gpu/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"},
}
const supportBundleGlob = "????-??-?? (BEE-SP*)*.tar.gz"
@@ -431,17 +446,18 @@ func BuildSupportBundle(exportDir string) (string, error) {
}
defer os.RemoveAll(stageRoot)
if err := copyExportDirForSupportBundle(exportDir, filepath.Join(stageRoot, "export")); err != nil {
if err := categorizeExportTree(exportDir, stageRoot); err != nil {
return "", err
}
if err := writeJournalDump(filepath.Join(stageRoot, "systemd", "combined.journal.log")); err != nil {
if err := writeJournalDump(filepath.Join(stageRoot, "tasks", "_services", "combined.journal.log")); err != nil {
return "", err
}
for _, svc := range supportBundleServices {
if err := writeCommandOutput(filepath.Join(stageRoot, "systemd", svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil {
dir := filepath.Join(stageRoot, serviceBundleDir(svc))
if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil {
return "", err
}
if err := writeCommandOutput(filepath.Join(stageRoot, "systemd", svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil {
if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil {
return "", err
}
}
@@ -715,40 +731,6 @@ func copyDirContents(srcDir, dstDir string) error {
return nil
}
func copyExportDirForSupportBundle(srcDir, dstDir string) error {
if err := copyDirContentsFiltered(srcDir, dstDir, func(rel string, info os.FileInfo) bool {
cleanRel := filepath.ToSlash(strings.TrimPrefix(filepath.Clean(rel), "./"))
if cleanRel == "" {
return true
}
if strings.HasPrefix(cleanRel, "bee-sat/") && strings.HasSuffix(cleanRel, ".tar.gz") {
return false
}
if strings.HasPrefix(filepath.Base(cleanRel), "bee-support-") && strings.HasSuffix(cleanRel, ".tar.gz") {
return false
}
return true
}); err != nil {
return err
}
return normalizeSupportBundleAuditJSON(filepath.Join(dstDir, "bee-audit.json"))
}
func normalizeSupportBundleAuditJSON(path string) error {
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
normalized, err := ApplySATOverlay(data)
if err != nil {
return nil
}
return os.WriteFile(path, normalized, 0644)
}
func copyDirContentsFiltered(srcDir, dstDir string, keep func(rel string, info os.FileInfo) bool) error {
entries, err := os.ReadDir(srcDir)
if err != nil {
@@ -791,6 +773,14 @@ func copyPath(src, dst string) error {
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return err
}
// Skip rewriting files the blackbox worker already copied unchanged on a
// prior cycle — avoids needless flash wear on the removable target every
// sync period. Size+mtime, not a full byte comparison, so large files
// (e.g. status/metrics.db) can still stream-copy instead of loading
// whole into memory.
if dstInfo, err := os.Stat(dst); err == nil && dstInfo.Size() == info.Size() && !dstInfo.ModTime().Before(info.ModTime()) {
return nil
}
in, err := os.Open(src)
if err != nil {
return err