bee/audit/internal/app/support_bundle.go

package app

import (
	"archive/tar"
	"bee/audit/internal/platform"
	"compress/gzip"
	"context"
	_ "embed"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

// defaultCommandTimeout bounds every subprocess this file shells out to. GPU
// diagnostics (nvidia-smi, nvidia-bug-report.sh, dcgmi) are the main risk: a
// GPU that has fallen off the bus or needs a physical power-cycle (see
// gpuNeedsPhysicalReboot in webui/pages.go) makes those tools hang
// indefinitely rather than error out. Without a timeout, one wedged GPU
// blocks the whole bundle build forever and the "Download Support Bundle"
// button spins on "Building..." with no way to recover short of restarting
// the service.
const defaultCommandTimeout = 30 * time.Second

// nvidiaBugReportTimeout is longer than defaultCommandTimeout because
// nvidia-bug-report.sh legitimately takes tens of seconds to run on
// multi-GPU boxes even when healthy.
const nvidiaBugReportTimeout = 120 * time.Second

//go:embed assets/README.md
var supportBundleReadmeMD []byte

// writeBundleDocs writes README.md at root, so an AI agent (or a human)
// analyzing this export (support bundle or blackbox capture) can orient
// itself without reverse-engineering the directory layout first.
func writeBundleDocs(root string) error {
	if err := os.MkdirAll(root, 0755); err != nil {
		return err
	}
	return os.WriteFile(filepath.Join(root, "README.md"), supportBundleReadmeMD, 0644)
}

var supportBundleServices = []string{
	"bee-blackbox.service",
	"bee-audit.service",
	"bee-web.service",
	"bee-network.service",
	"bee-nvidia.service",
	"bee-preflight.service",
	"bee-selfheal.service",
	"bee-selfheal.timer",
	"bee-sshsetup.service",
	"display-manager.service",
	"lightdm.service",
	"nvidia-dcgm.service",
	"nvidia-fabricmanager.service",
}

// serviceBundleDir returns the bundle-relative directory a service's
// status/journal capture belongs in: bee's own daemons are internal
// bookkeeping, NVIDIA/DCGM/fabric-manager daemons are vendor-facing hardware
// diagnostics, and the display stack is LiveCD/GUI-session-only.
func serviceBundleDir(svc string) string {
	switch svc {
	case "nvidia-dcgm.service", "nvidia-fabricmanager.service":
		return "export/gpu"
	case "display-manager.service", "lightdm.service":
		return "livecd/gui"
	default:
		return "tasks/_services"
	}
}

var supportBundleCommands = []struct {
	name    string
	cmd     []string
	timeout time.Duration // zero means defaultCommandTimeout
}{
	{name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}},
	{name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
	{name: "livecd/host/lsmod.txt", cmd: []string{"lsmod"}},
	{name: "export/platform/lspci-nn.txt", cmd: []string{"lspci", "-nn"}},
	{name: "livecd/host/ip-addr.txt", cmd: []string{"ip", "addr"}},
	{name: "livecd/host/ip-link.txt", cmd: []string{"ip", "-details", "link", "show"}},
	{name: "livecd/host/ip-link-stats.txt", cmd: []string{"ip", "-s", "link", "show"}},
	{name: "livecd/host/ip-route.txt", cmd: []string{"ip", "route"}},
	{name: "livecd/host/mount.txt", cmd: []string{"mount"}},
	{name: "livecd/host/df-h.txt", cmd: []string{"df", "-h"}},
	{name: "livecd/host/dmesg.txt", cmd: []string{"dmesg"}},
	{name: "livecd/gui/dmesg-gui-video-input.txt", cmd: []string{"sh", "-c", `
if command -v dmesg >/dev/null 2>&1; then
  dmesg | grep -iE 'nvidia|drm|fb|framebuffer|vesa|efi|lightdm|Xorg|input|hid|usb|keyboard|mouse|virtual keyboard|virtual mouse|ami|aspeed|ast' || echo "no GUI/video/input kernel messages found"
else
  echo "dmesg not found"
fi
`}},
	{name: "export/gpu/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", platform.KernelAERNvidiaScript}},
	{name: "livecd/gui/loginctl-sessions.txt", cmd: []string{"sh", "-c", `
if command -v loginctl >/dev/null 2>&1; then
  loginctl list-sessions 2>&1 || true
else
  echo "loginctl not found"
fi
`}},
	{name: "livecd/gui/loginctl-seats.txt", cmd: []string{"sh", "-c", `
if command -v loginctl >/dev/null 2>&1; then
  loginctl list-seats 2>&1 || true
  echo
  for seat in $(loginctl list-seats --no-legend 2>/dev/null | awk '{print $1}'); do
    echo "=== $seat ==="
    loginctl seat-status "$seat" 2>&1 || true
    echo
  done
else
  echo "loginctl not found"
fi
`}},
	{name: "livecd/gui/ps-gui.txt", cmd: []string{"sh", "-c", `
ps -ef | grep -iE 'lightdm|Xorg|X$|openbox|chromium|chrome|xinit|xsession' | grep -v grep || echo "no GUI processes found"
`}},
	{name: "export/gpu/lspci-video-vv.txt", cmd: []string{"sh", "-c", `
if ! command -v lspci >/dev/null 2>&1; then
  echo "lspci not found"
  exit 0
fi
found=0
for dev in $(lspci -Dn | awk '$2 ~ /^03(00|02):$/ {print $1}'); do
  found=1
  echo "=== $dev ==="
  lspci -s "$dev" -vv 2>&1 || true
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no display-class PCI devices found"
fi
`}},
	{name: "livecd/gui/proc-fb.txt", cmd: []string{"cat", "/proc/fb"}},
	{name: "livecd/gui/drm-cards.txt", cmd: []string{"sh", "-c", `
if [ -d /sys/class/drm ]; then
  for path in /sys/class/drm/card*; do
    [ -e "$path" ] || continue
    card=$(basename "$path")
    echo "=== $card ==="
    for f in status enabled dpms modes; do
      [ -r "$path/$f" ] && printf "  %-8s %s\n" "$f" "$(cat "$path/$f" 2>/dev/null)"
    done
    device=$(readlink -f "$path/device" 2>/dev/null || true)
    [ -n "$device" ] && echo "  device   ${device##*/}"
    echo
  done
else
  echo "/sys/class/drm not present"
fi
`}},
	{name: "livecd/gui/input-devices.txt", cmd: []string{"sh", "-c", `
if [ -r /proc/bus/input/devices ]; then
  cat /proc/bus/input/devices
else
  echo "/proc/bus/input/devices not readable"
fi
`}},
	{name: "livecd/gui/udevadm-input.txt", cmd: []string{"sh", "-c", `
if ! command -v udevadm >/dev/null 2>&1; then
  echo "udevadm not found"
  exit 0
fi
found=0
for dev in /dev/input/event*; do
  [ -e "$dev" ] || continue
  found=1
  echo "=== $dev ==="
  udevadm info --query=all --name="$dev" 2>&1 || true
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no /dev/input/event* devices found"
fi
`}},
	{name: "livecd/gui/xinput-list.txt", cmd: []string{"sh", "-c", `
if command -v xinput >/dev/null 2>&1; then
  DISPLAY=:0 xinput --list 2>&1 || true
else
  echo "xinput not found"
fi
`}},
	{name: "livecd/gui/libinput-list-devices.txt", cmd: []string{"sh", "-c", `
if command -v libinput >/dev/null 2>&1; then
  libinput list-devices 2>&1 || true
else
  echo "libinput not found"
fi
`}},
	{name: "livecd/gui/systemctl-gui-units.txt", cmd: []string{"sh", "-c", `
if ! command -v systemctl >/dev/null 2>&1; then
  echo "systemctl not found"
  exit 0
fi
echo "=== unit files ==="
systemctl list-unit-files --no-pager --all 'lightdm*' 'display-manager*' 2>&1 || true
echo
echo "=== active units ==="
systemctl list-units --no-pager --all 'lightdm*' 'display-manager*' 2>&1 || true
echo
echo "=== failed units ==="
systemctl --failed --no-pager 2>&1 | grep -iE 'lightdm|display-manager|Xorg' || echo "no failed GUI units"
`}},
	{name: "export/gpu/nvidia-smi-topo-fresh.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-smi >/dev/null 2>&1; then
  nvidia-smi topo -m 2>&1 || true
else
  echo "nvidia-smi not found"
fi
`}},
	{name: "export/gpu/nvidia-smi-nvlink-status-fresh.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-smi >/dev/null 2>&1; then
  nvidia-smi nvlink -s 2>&1 || true
else
  echo "nvidia-smi not found"
fi
`}},
	{name: "export/gpu/nvidia-smi-nvlink-errors-fresh.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-smi >/dev/null 2>&1; then
  nvidia-smi nvlink -e 2>&1 || true
else
  echo "nvidia-smi not found"
fi
`}},
	{name: "export/gpu/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", `
if command -v dcgmi >/dev/null 2>&1; then
  dcgmi nvlink --link-status 2>&1 || true
else
  echo "dcgmi not found"
fi
`}},
	{name: "export/gpu/nvidia-bug-report.txt", timeout: nvidiaBugReportTimeout, cmd: []string{"sh", "-c", `
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
  rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
  nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1
  if [ -f /tmp/bee-nvidia-bug-report.log.gz ]; then
    gzip -dc /tmp/bee-nvidia-bug-report.log.gz
  elif [ -f /tmp/bee-nvidia-bug-report.log ]; then
    cat /tmp/bee-nvidia-bug-report.log
  else
    echo "nvidia-bug-report.sh produced no output file"
  fi
  rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
else
  echo "nvidia-bug-report.sh not found"
fi
`}},
	{name: "export/gpu/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", `
if ! command -v systemctl >/dev/null 2>&1; then
  echo "systemctl not found"
  exit 0
fi
echo "=== unit files ==="
systemctl list-unit-files --no-pager --all 'nvidia*' 'fabric*' 2>&1 || true
echo
echo "=== active units ==="
systemctl list-units --no-pager --all 'nvidia*' 'fabric*' 2>&1 || true
echo
echo "=== failed units ==="
systemctl --failed --no-pager 2>&1 | grep -iE 'nvidia|fabric' || echo "no failed nvidia/fabric units"
`}},
	{name: "export/gpu/fabric-manager-paths.txt", cmd: []string{"sh", "-c", `
for candidate in \
  /usr/bin/nvidia-fabricmanager \
  /usr/bin/nv-fabricmanager \
  /usr/bin/nvidia-fabricmanagerd \
  /usr/bin/nvlsm; do
  if [ -e "$candidate" ]; then
    echo "=== $candidate ==="
    ls -l "$candidate" 2>&1 || true
    echo
  fi
done
if ! ls /usr/bin/nvidia-fabricmanager /usr/bin/nv-fabricmanager /usr/bin/nvidia-fabricmanagerd /usr/bin/nvlsm >/dev/null 2>&1; then
  echo "no fabric manager binaries found"
fi
`}},
	{name: "export/gpu/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", `
if ! command -v lspci >/dev/null 2>&1; then
  echo "lspci not found"
  exit 0
fi
found=0
	for gpu in $(lspci -Dn | awk '$2 ~ /^03(00|02):$/ && $3 ~ /^10de:/ {print $1}'); do
  found=1
  echo "=== GPU $gpu ==="
  lspci -s "$gpu" -vv 2>&1 || true
  bridge=$(basename "$(readlink -f "/sys/bus/pci/devices/$gpu/.." 2>/dev/null)" 2>/dev/null)
  if [ -n "$bridge" ] && [ "$bridge" != "$gpu" ]; then
    echo
    echo "=== UPSTREAM $bridge for $gpu ==="
    lspci -s "$bridge" -vv 2>&1 || true
  fi
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no NVIDIA PCI devices found"
fi
`}},
	// Also captured once at boot by platform.CaptureTechnicalDump (which the
	// blackbox mirror picks up automatically); re-run here so an on-demand
	// support bundle gets a fresh sample instead of a possibly-stale boot
	// one. See bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md.
	{name: "export/gpu/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", platform.PCIeNvidiaLinkScript}},
	// A Gen1-vs-Gen4 link speed reading at idle is ambiguous: NVIDIA drivers
	// deliberately downclock PCIe in low power states and re-train to full
	// speed under load, so pcie-nvidia-link.txt alone can't tell a real
	// hardware/riser fault from normal power management. Re-sample the same
	// sysfs attributes while bee-gpu-burn is actively loading the GPUs — if
	// the link comes up here, the idle Gen1 reading above was power saving,
	// not a fault.
	{name: "export/gpu/pcie-nvidia-link-under-load.txt", cmd: []string{"sh", "-c", platform.PCIeNvidiaLinkUnderLoadScript}},
	{name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
found=0
for dev in /sys/bus/pci/devices/*; do
  [ -e "$dev" ] || continue
  bdf=$(basename "$dev")
  block=""
  for f in aer_dev_correctable aer_dev_fatal aer_dev_nonfatal aer_rootport_total_err_cor aer_rootport_total_err_fatal aer_rootport_total_err_nonfatal; do
    if [ -r "$dev/$f" ]; then
      if [ -z "$block" ]; then
        block=1
        found=1
        echo "=== $bdf ==="
      fi
      printf "  %-30s %s\n" "$f" "$(cat "$dev/$f" 2>/dev/null)"
    fi
  done
  if [ -n "$block" ]; then
    echo
  fi
done
if [ "$found" -eq 0 ]; then
  echo "no PCIe AER sysfs counters found"
fi
`}},
	{name: "export/network/ethtool-info.txt", cmd: []string{"sh", "-c", `
if ! command -v ethtool >/dev/null 2>&1; then
  echo "ethtool not found"
  exit 0
fi
found=0
for path in /sys/class/net/*; do
  [ -e "$path" ] || continue
  iface=$(basename "$path")
  [ "$iface" = "lo" ] && continue
  found=1
  echo "=== $iface ==="
  ethtool -i "$iface" 2>&1 || true
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no interfaces found"
fi
`}},
	{name: "export/network/ethtool-link.txt", cmd: []string{"sh", "-c", `
if ! command -v ethtool >/dev/null 2>&1; then
  echo "ethtool not found"
  exit 0
fi
found=0
for path in /sys/class/net/*; do
  [ -e "$path" ] || continue
  iface=$(basename "$path")
  [ "$iface" = "lo" ] && continue
  found=1
  echo "=== $iface ==="
  ethtool "$iface" 2>&1 || true
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no interfaces found"
fi
`}},
	{name: "export/network/ethtool-module.txt", cmd: []string{"sh", "-c", `
if ! command -v ethtool >/dev/null 2>&1; then
  echo "ethtool not found"
  exit 0
fi
found=0
for path in /sys/class/net/*; do
  [ -e "$path" ] || continue
  iface=$(basename "$path")
  [ "$iface" = "lo" ] && continue
  found=1
  echo "=== $iface ==="
  ethtool -m "$iface" 2>&1 || true
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no interfaces found"
fi
`}},
	{name: "export/network/mstflint-query.txt", cmd: []string{"sh", "-c", `
if ! command -v mstflint >/dev/null 2>&1; then
  echo "mstflint not found"
  exit 0
fi
found=0
for path in /sys/bus/pci/devices/*; do
  [ -e "$path/vendor" ] || continue
  vendor=$(cat "$path/vendor" 2>/dev/null)
  [ "$vendor" = "0x15b3" ] || continue
  bdf=$(basename "$path")
  found=1
  echo "=== $bdf ==="
  mstflint -d "$bdf" q 2>&1 || true
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no Mellanox/NVIDIA networking devices found"
fi
`}},
}

var supportBundleOptionalFiles = []struct {
	name string
	src  string
}{
	{name: "livecd/host/kern.log", src: "/var/log/kern.log"},
	{name: "livecd/host/syslog.txt", src: "/var/log/syslog"},
	{name: "livecd/gui/Xorg.0.log", src: "/var/log/Xorg.0.log"},
	{name: "livecd/gui/Xorg.0.log.old", src: "/var/log/Xorg.0.log.old"},
	{name: "livecd/gui/lightdm/lightdm.log", src: "/var/log/lightdm/lightdm.log"},
	{name: "livecd/gui/lightdm/x-0.log", src: "/var/log/lightdm/x-0.log"},
	{name: "livecd/gui/lightdm/x-0-greeter.log", src: "/var/log/lightdm/x-0-greeter.log"},
	{name: "livecd/gui/home-bee-xsession-errors.log", src: "/home/bee/.xsession-errors"},
	{name: "livecd/gui/home-bee-chromium-debug.log", src: "/tmp/bee-chrome/chrome_debug.log"},
	{name: "export/gpu/fabricmanager.log", src: "/var/log/fabricmanager.log"},
	{name: "export/gpu/nvlsm.log", src: "/var/log/nvlsm.log"},
	{name: "export/gpu/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"},
	{name: "export/gpu/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"},
}

const supportBundleGlob = "????-??-?? (BEE-SP*)*.tar.gz"

func BuildSupportBundle(exportDir string) (string, error) {
	exportDir = strings.TrimSpace(exportDir)
	if exportDir == "" {
		exportDir = DefaultExportDir
	}
	if err := os.MkdirAll(exportDir, 0755); err != nil {
		return "", err
	}
	if err := cleanupOldSupportBundles(os.TempDir()); err != nil {
		return "", err
	}

	now := time.Now().UTC()

	// Stage under a private parent dir. Two bundle builds started in the same
	// wall-clock second (two operators, or an on-demand build racing the
	// blackbox worker) must not share a staging tree: one's deferred
	// os.RemoveAll would then wipe the other's half-populated tree and yield a
	// truncated archive. The leaf name stays meaningful because it becomes the
	// archive's top-level directory.
	buildParent, err := os.MkdirTemp(os.TempDir(), "bee-support-build-")
	if err != nil {
		return "", err
	}
	defer os.RemoveAll(buildParent)
	stageRoot := filepath.Join(buildParent, fmt.Sprintf("bee-support-stage-%s-%s", sanitizeFilename(hostnameOr("unknown")), now.Format("20060102-150405")))
	if err := os.MkdirAll(stageRoot, 0755); err != nil {
		return "", err
	}

	if err := categorizeExportTree(exportDir, stageRoot, true); err != nil {
		return "", err
	}
	if err := writeJournalDump(filepath.Join(stageRoot, "tasks", "_services", "combined.journal.log")); err != nil {
		return "", err
	}
	for _, svc := range supportBundleServices {
		dir := filepath.Join(stageRoot, serviceBundleDir(svc))
		if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}, defaultCommandTimeout); err != nil {
			return "", err
		}
		if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}, defaultCommandTimeout); err != nil {
			return "", err
		}
	}
	for _, item := range supportBundleCommands {
		timeout := item.timeout
		if timeout <= 0 {
			timeout = defaultCommandTimeout
		}
		if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd, timeout); err != nil {
			return "", err
		}
	}
	for _, item := range supportBundleOptionalFiles {
		_ = copyOptionalFile(item.src, filepath.Join(stageRoot, item.name))
	}
	if err := writeBundleDocs(stageRoot); err != nil {
		return "", err
	}
	if err := writeManifest(filepath.Join(stageRoot, "manifest.txt"), exportDir, stageRoot); err != nil {
		return "", err
	}

	return createSupportTarGz(os.TempDir(), SupportBundleBaseName(now), stageRoot)
}

func SupportBundleBaseName(at time.Time) string {
	at = at.UTC()
	date := at.Format("2006-01-02")
	tod := at.Format("150405")
	ver := sanitizeFilename(bundleVersion())
	model := sanitizeFilename(serverModelForBundle())
	sn := sanitizeFilename(serverSerialForBundle())
	return fmt.Sprintf("%s (BEE-SP v%s) %s %s %s", date, ver, model, sn, tod)
}

func cleanupOldSupportBundles(dir string) error {
	matches, err := filepath.Glob(filepath.Join(dir, supportBundleGlob))
	if err != nil {
		return err
	}
	entries := supportBundleEntries(matches)
	for path, mod := range entries {
		if time.Since(mod) > 24*time.Hour {
			_ = os.Remove(path)
			delete(entries, path)
		}
	}
	ordered := orderSupportBundles(entries)
	if len(ordered) > 3 {
		for _, old := range ordered[3:] {
			_ = os.Remove(old)
		}
	}
	return nil
}

func supportBundleEntries(matches []string) map[string]time.Time {
	entries := make(map[string]time.Time, len(matches))
	for _, match := range matches {
		info, err := os.Stat(match)
		if err != nil {
			continue
		}
		entries[match] = info.ModTime()
	}
	return entries
}

func orderSupportBundles(entries map[string]time.Time) []string {
	ordered := make([]string, 0, len(entries))
	for path := range entries {
		ordered = append(ordered, path)
	}
	sort.Slice(ordered, func(i, j int) bool {
		return entries[ordered[i]].After(entries[ordered[j]])
	})
	return ordered
}

func writeJournalDump(dst string) error {
	args := []string{"--no-pager"}
	for _, svc := range supportBundleServices {
		args = append(args, "-u", svc)
	}
	ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
	defer cancel()
	raw, err := exec.CommandContext(ctx, "journalctl", args...).CombinedOutput()
	if len(raw) == 0 && err != nil {
		raw = []byte(err.Error() + "\n")
	}
	if len(raw) == 0 {
		raw = []byte("no journal output\n")
	}
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return err
	}
	return os.WriteFile(dst, raw, 0644)
}

func writeCommandOutput(dst string, cmd []string, timeout time.Duration) error {
	if len(cmd) == 0 {
		return nil
	}
	if timeout <= 0 {
		timeout = defaultCommandTimeout
	}
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()
	raw, err := exec.CommandContext(ctx, cmd[0], cmd[1:]...).CombinedOutput()
	if len(raw) == 0 {
		if ctx.Err() == context.DeadlineExceeded {
			raw = []byte(fmt.Sprintf("timed out after %s\n", timeout))
		} else if err != nil {
			raw = []byte(err.Error() + "\n")
		} else {
			raw = []byte("no output\n")
		}
	}
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return err
	}
	return os.WriteFile(dst, raw, 0644)
}

func copyOptionalFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return errors.Join(err, in.Close())
	}
	out, err := os.Create(dst)
	if err != nil {
		return errors.Join(err, in.Close())
	}
	_, copyErr := io.Copy(out, in)
	return errors.Join(copyErr, in.Close(), out.Close())
}

func writeManifest(dst, exportDir, stageRoot string) error {
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return err
	}
	var body strings.Builder
	fmt.Fprintf(&body, "bee_version=%s\n", buildVersion())
	fmt.Fprintf(&body, "bee_git_commit=%s\n", buildCommit())
	fmt.Fprintf(&body, "host=%s\n", hostnameOr("unknown"))
	fmt.Fprintf(&body, "generated_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
	fmt.Fprintf(&body, "export_dir=%s\n", exportDir)
	if cfg, err := platform.LoadBenchmarkPowerAutotuneConfig(filepath.Join(exportDir, "bee-bench", "power-source-autotune.json")); err == nil && cfg != nil {
		fmt.Fprintf(&body, "power_autotune_selected_source=%s\n", cfg.SelectedSource)
		fmt.Fprintf(&body, "power_autotune_updated_at=%s\n", cfg.UpdatedAt.UTC().Format(time.RFC3339))
		if strings.TrimSpace(cfg.Reason) != "" {
			fmt.Fprintf(&body, "power_autotune_reason=%s\n", cfg.Reason)
		}
	}
	fmt.Fprintf(&body, "\nfiles:\n")

	var files []string
	if err := filepath.Walk(stageRoot, func(path string, info os.FileInfo, err error) error {
		if err != nil || info.IsDir() {
			return err
		}
		if filepath.Clean(path) == filepath.Clean(dst) {
			return nil
		}
		rel, err := filepath.Rel(stageRoot, path)
		if err != nil {
			return err
		}
		files = append(files, fmt.Sprintf("%s\t%d", rel, info.Size()))
		return nil
	}); err != nil {
		return err
	}
	sort.Strings(files)
	for _, line := range files {
		body.WriteString(line)
		body.WriteByte('\n')
	}
	return os.WriteFile(dst, []byte(body.String()), 0644)
}

func bundleVersion() string {
	v := buildVersion()
	v = strings.TrimPrefix(v, "v")
	v = strings.TrimPrefix(v, "V")
	if v == "" || v == "unknown" {
		return "0.0"
	}
	return v
}

func serverModelForBundle() string {
	ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
	defer cancel()
	raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output()
	if err != nil {
		return "unknown"
	}
	for _, line := range strings.Split(string(raw), "\n") {
		line = strings.TrimSpace(line)
		key, val, ok := strings.Cut(line, ": ")
		if !ok {
			continue
		}
		if strings.TrimSpace(key) == "Product Name" {
			val = strings.TrimSpace(val)
			if val == "" {
				return "unknown"
			}
			return strings.ReplaceAll(val, " ", "_")
		}
	}
	return "unknown"
}

func serverSerialForBundle() string {
	ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
	defer cancel()
	raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output()
	if err != nil {
		return "unknown"
	}
	for _, line := range strings.Split(string(raw), "\n") {
		line = strings.TrimSpace(line)
		key, val, ok := strings.Cut(line, ": ")
		if !ok {
			continue
		}
		if strings.TrimSpace(key) == "Serial Number" {
			val = strings.TrimSpace(val)
			if val == "" {
				return "unknown"
			}
			return val
		}
	}
	return "unknown"
}

func buildVersion() string {
	ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
	defer cancel()
	raw, err := exec.CommandContext(ctx, "bee", "version").CombinedOutput()
	if err != nil {
		return "unknown"
	}
	return strings.TrimSpace(string(raw))
}

func buildCommit() string {
	ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
	defer cancel()
	raw, err := exec.CommandContext(ctx, "bee", "version", "--commit").CombinedOutput()
	if err != nil || strings.TrimSpace(string(raw)) == "" {
		return "unknown"
	}
	return strings.TrimSpace(string(raw))
}

func copyDirContentsFiltered(srcDir, dstDir string, keep func(rel string, info os.FileInfo) bool) error {
	entries, err := os.ReadDir(srcDir)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	for _, entry := range entries {
		src := filepath.Join(srcDir, entry.Name())
		dst := filepath.Join(dstDir, entry.Name())
		if err := copyPathFiltered(srcDir, src, dst, keep); err != nil {
			return err
		}
	}
	return nil
}

func copyPath(src, dst string) error {
	info, err := os.Stat(src)
	if err != nil {
		// src is read from a live export dir that bee's own SAT/task runner
		// keeps writing to and renaming (e.g. a task dir's "_pending" ->
		// "_done" transition) concurrently with this copy — a path present
		// in the parent's os.ReadDir listing a moment ago disappearing by
		// the time we get here is an expected race, not a real failure.
		// Skip it; the next cycle will pick up wherever it landed under its
		// new name. Aborting the whole copy over one renamed-away entry is
		// what previously left blackbox stuck "degraded" indefinitely.
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	if info.IsDir() {
		if err := os.MkdirAll(dst, info.Mode().Perm()); err != nil {
			return err
		}
		entries, err := os.ReadDir(src)
		if err != nil {
			if os.IsNotExist(err) {
				return nil
			}
			return err
		}
		for _, entry := range entries {
			if err := copyPath(filepath.Join(src, entry.Name()), filepath.Join(dst, entry.Name())); err != nil {
				return err
			}
		}
		return nil
	}

	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return err
	}
	// Skip rewriting files the blackbox worker already copied unchanged on a
	// prior cycle — avoids needless flash wear on the removable target every
	// sync period. Size+mtime, not a full byte comparison, so large files
	// (e.g. status/metrics.db) can still stream-copy instead of loading
	// whole into memory.
	if dstInfo, err := os.Stat(dst); err == nil && dstInfo.Size() == info.Size() && !dstInfo.ModTime().Before(info.ModTime()) {
		return nil
	}
	in, err := os.Open(src)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode().Perm())
	if err != nil {
		return errors.Join(err, in.Close())
	}
	_, copyErr := io.Copy(out, in)
	return errors.Join(copyErr, in.Close(), out.Close())
}

func copyPathFiltered(rootSrc, src, dst string, keep func(rel string, info os.FileInfo) bool) error {
	info, err := os.Stat(src)
	if err != nil {
		// See the matching comment in copyPath: src disappearing between the
		// parent's os.ReadDir and this Stat is an expected race against
		// bee's own live task-dir renames, not a real error.
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	rel, err := filepath.Rel(rootSrc, src)
	if err != nil {
		return err
	}
	if keep != nil && !keep(rel, info) {
		return nil
	}
	if info.IsDir() {
		if err := os.MkdirAll(dst, info.Mode().Perm()); err != nil {
			return err
		}
		entries, err := os.ReadDir(src)
		if err != nil {
			if os.IsNotExist(err) {
				return nil
			}
			return err
		}
		for _, entry := range entries {
			if err := copyPathFiltered(rootSrc, filepath.Join(src, entry.Name()), filepath.Join(dst, entry.Name()), keep); err != nil {
				return err
			}
		}
		return nil
	}
	return copyPath(src, dst)
}

func createSupportTarGz(dir, baseName, srcDir string) (string, error) {
	archiveFile, err := os.CreateTemp(dir, baseName+"-*.partial")
	if err != nil {
		return "", err
	}
	partialPath := archiveFile.Name()
	if err := writeSupportTarGz(archiveFile, srcDir); err != nil {
		_ = archiveFile.Close()
		_ = os.Remove(partialPath)
		return "", err
	}
	if err := archiveFile.Close(); err != nil {
		_ = os.Remove(partialPath)
		return "", err
	}
	archivePath := strings.TrimSuffix(partialPath, ".partial") + ".tar.gz"
	if err := os.Rename(partialPath, archivePath); err != nil {
		_ = os.Remove(partialPath)
		return "", err
	}
	return archivePath, nil
}

func writeSupportTarGz(file *os.File, srcDir string) error {
	gz := gzip.NewWriter(file)
	tw := tar.NewWriter(gz)

	base := filepath.Dir(srcDir)
	walkErr := filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}

		header, err := tar.FileInfoHeader(info, "")
		if err != nil {
			return err
		}
		header.Name, err = filepath.Rel(base, path)
		if err != nil {
			return err
		}
		if err := tw.WriteHeader(header); err != nil {
			return err
		}

		f, err := os.Open(path)
		if err != nil {
			return err
		}
		_, copyErr := io.Copy(tw, f)
		closeErr := f.Close()
		if copyErr != nil {
			return copyErr
		}
		return closeErr
	})
	if walkErr != nil {
		_ = tw.Close()
		_ = gz.Close()
		return walkErr
	}
	if err := tw.Close(); err != nil {
		_ = gz.Close()
		return err
	}
	return gz.Close()
}