bee/audit/internal/app/support_bundle.go

package app

import (
	"archive/tar"
	"bee/audit/internal/platform"
	"compress/gzip"
	_ "embed"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

//go:embed assets/README.md
var supportBundleReadmeMD []byte

// writeBundleDocs writes README.md at root, so an AI agent (or a human)
// analyzing this export (support bundle or blackbox capture) can orient
// itself without reverse-engineering the directory layout first.
func writeBundleDocs(root string) error {
	if err := os.MkdirAll(root, 0755); err != nil {
		return err
	}
	return os.WriteFile(filepath.Join(root, "README.md"), supportBundleReadmeMD, 0644)
}

var supportBundleServices = []string{
	"bee-blackbox.service",
	"bee-audit.service",
	"bee-web.service",
	"bee-network.service",
	"bee-nvidia.service",
	"bee-preflight.service",
	"bee-selfheal.service",
	"bee-selfheal.timer",
	"bee-sshsetup.service",
	"display-manager.service",
	"lightdm.service",
	"nvidia-dcgm.service",
	"nvidia-fabricmanager.service",
}

// serviceBundleDir returns the bundle-relative directory a service's
// status/journal capture belongs in: bee's own daemons are internal
// bookkeeping, NVIDIA/DCGM/fabric-manager daemons are vendor-facing hardware
// diagnostics, and the display stack is LiveCD/GUI-session-only.
func serviceBundleDir(svc string) string {
	switch svc {
	case "nvidia-dcgm.service", "nvidia-fabricmanager.service":
		return "export/gpu"
	case "display-manager.service", "lightdm.service":
		return "livecd/gui"
	default:
		return "tasks/_services"
	}
}

var supportBundleCommands = []struct {
	name string
	cmd  []string
}{
	{name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}},
	{name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
	{name: "livecd/host/lsmod.txt", cmd: []string{"lsmod"}},
	{name: "export/platform/lspci-nn.txt", cmd: []string{"lspci", "-nn"}},
	{name: "livecd/host/ip-addr.txt", cmd: []string{"ip", "addr"}},
	{name: "livecd/host/ip-link.txt", cmd: []string{"ip", "-details", "link", "show"}},
	{name: "livecd/host/ip-link-stats.txt", cmd: []string{"ip", "-s", "link", "show"}},
	{name: "livecd/host/ip-route.txt", cmd: []string{"ip", "route"}},
	{name: "livecd/host/mount.txt", cmd: []string{"mount"}},
	{name: "livecd/host/df-h.txt", cmd: []string{"df", "-h"}},
	{name: "livecd/host/dmesg.txt", cmd: []string{"dmesg"}},
	{name: "livecd/gui/dmesg-gui-video-input.txt", cmd: []string{"sh", "-c", `
if command -v dmesg >/dev/null 2>&1; then
  dmesg | grep -iE 'nvidia|drm|fb|framebuffer|vesa|efi|lightdm|Xorg|input|hid|usb|keyboard|mouse|virtual keyboard|virtual mouse|ami|aspeed|ast' || echo "no GUI/video/input kernel messages found"
else
  echo "dmesg not found"
fi
`}},
	{name: "export/gpu/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", `
if command -v dmesg >/dev/null 2>&1; then
  dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
else
  echo "dmesg not found"
fi
`}},
	{name: "livecd/gui/loginctl-sessions.txt", cmd: []string{"sh", "-c", `
if command -v loginctl >/dev/null 2>&1; then
  loginctl list-sessions 2>&1 || true
else
  echo "loginctl not found"
fi
`}},
	{name: "livecd/gui/loginctl-seats.txt", cmd: []string{"sh", "-c", `
if command -v loginctl >/dev/null 2>&1; then
  loginctl list-seats 2>&1 || true
  echo
  for seat in $(loginctl list-seats --no-legend 2>/dev/null | awk '{print $1}'); do
    echo "=== $seat ==="
    loginctl seat-status "$seat" 2>&1 || true
    echo
  done
else
  echo "loginctl not found"
fi
`}},
	{name: "livecd/gui/ps-gui.txt", cmd: []string{"sh", "-c", `
ps -ef | grep -iE 'lightdm|Xorg|X$|openbox|chromium|chrome|xinit|xsession' | grep -v grep || echo "no GUI processes found"
`}},
	{name: "export/gpu/lspci-video-vv.txt", cmd: []string{"sh", "-c", `
if ! command -v lspci >/dev/null 2>&1; then
  echo "lspci not found"
  exit 0
fi
found=0
for dev in $(lspci -Dn | awk '$2 ~ /^03(00|02):$/ {print $1}'); do
  found=1
  echo "=== $dev ==="
  lspci -s "$dev" -vv 2>&1 || true
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no display-class PCI devices found"
fi
`}},
	{name: "livecd/gui/proc-fb.txt", cmd: []string{"cat", "/proc/fb"}},
	{name: "livecd/gui/drm-cards.txt", cmd: []string{"sh", "-c", `
if [ -d /sys/class/drm ]; then
  for path in /sys/class/drm/card*; do
    [ -e "$path" ] || continue
    card=$(basename "$path")
    echo "=== $card ==="
    for f in status enabled dpms modes; do
      [ -r "$path/$f" ] && printf "  %-8s %s\n" "$f" "$(cat "$path/$f" 2>/dev/null)"
    done
    device=$(readlink -f "$path/device" 2>/dev/null || true)
    [ -n "$device" ] && echo "  device   ${device##*/}"
    echo
  done
else
  echo "/sys/class/drm not present"
fi
`}},
	{name: "livecd/gui/input-devices.txt", cmd: []string{"sh", "-c", `
if [ -r /proc/bus/input/devices ]; then
  cat /proc/bus/input/devices
else
  echo "/proc/bus/input/devices not readable"
fi
`}},
	{name: "livecd/gui/udevadm-input.txt", cmd: []string{"sh", "-c", `
if ! command -v udevadm >/dev/null 2>&1; then
  echo "udevadm not found"
  exit 0
fi
found=0
for dev in /dev/input/event*; do
  [ -e "$dev" ] || continue
  found=1
  echo "=== $dev ==="
  udevadm info --query=all --name="$dev" 2>&1 || true
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no /dev/input/event* devices found"
fi
`}},
	{name: "livecd/gui/xinput-list.txt", cmd: []string{"sh", "-c", `
if command -v xinput >/dev/null 2>&1; then
  DISPLAY=:0 xinput --list 2>&1 || true
else
  echo "xinput not found"
fi
`}},
	{name: "livecd/gui/libinput-list-devices.txt", cmd: []string{"sh", "-c", `
if command -v libinput >/dev/null 2>&1; then
  libinput list-devices 2>&1 || true
else
  echo "libinput not found"
fi
`}},
	{name: "livecd/gui/systemctl-gui-units.txt", cmd: []string{"sh", "-c", `
if ! command -v systemctl >/dev/null 2>&1; then
  echo "systemctl not found"
  exit 0
fi
echo "=== unit files ==="
systemctl list-unit-files --no-pager --all 'lightdm*' 'display-manager*' 2>&1 || true
echo
echo "=== active units ==="
systemctl list-units --no-pager --all 'lightdm*' 'display-manager*' 2>&1 || true
echo
echo "=== failed units ==="
systemctl --failed --no-pager 2>&1 | grep -iE 'lightdm|display-manager|Xorg' || echo "no failed GUI units"
`}},
	{name: "export/gpu/nvidia-smi-topo-fresh.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-smi >/dev/null 2>&1; then
  nvidia-smi topo -m 2>&1 || true
else
  echo "nvidia-smi not found"
fi
`}},
	{name: "export/gpu/nvidia-smi-nvlink-status-fresh.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-smi >/dev/null 2>&1; then
  nvidia-smi nvlink -s 2>&1 || true
else
  echo "nvidia-smi not found"
fi
`}},
	{name: "export/gpu/nvidia-smi-nvlink-errors-fresh.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-smi >/dev/null 2>&1; then
  nvidia-smi nvlink -e 2>&1 || true
else
  echo "nvidia-smi not found"
fi
`}},
	{name: "export/gpu/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", `
if command -v dcgmi >/dev/null 2>&1; then
  dcgmi nvlink --link-status 2>&1 || true
else
  echo "dcgmi not found"
fi
`}},
	{name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
  nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 \
    && cat /tmp/bee-nvidia-bug-report.log \
    && rm -f /tmp/bee-nvidia-bug-report.log
else
  echo "nvidia-bug-report.sh not found"
fi
`}},
	{name: "export/gpu/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", `
if ! command -v systemctl >/dev/null 2>&1; then
  echo "systemctl not found"
  exit 0
fi
echo "=== unit files ==="
systemctl list-unit-files --no-pager --all 'nvidia*' 'fabric*' 2>&1 || true
echo
echo "=== active units ==="
systemctl list-units --no-pager --all 'nvidia*' 'fabric*' 2>&1 || true
echo
echo "=== failed units ==="
systemctl --failed --no-pager 2>&1 | grep -iE 'nvidia|fabric' || echo "no failed nvidia/fabric units"
`}},
	{name: "export/gpu/fabric-manager-paths.txt", cmd: []string{"sh", "-c", `
for candidate in \
  /usr/bin/nvidia-fabricmanager \
  /usr/bin/nv-fabricmanager \
  /usr/bin/nvidia-fabricmanagerd \
  /usr/bin/nvlsm; do
  if [ -e "$candidate" ]; then
    echo "=== $candidate ==="
    ls -l "$candidate" 2>&1 || true
    echo
  fi
done
if ! ls /usr/bin/nvidia-fabricmanager /usr/bin/nv-fabricmanager /usr/bin/nvidia-fabricmanagerd /usr/bin/nvlsm >/dev/null 2>&1; then
  echo "no fabric manager binaries found"
fi
`}},
	{name: "export/gpu/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", `
if ! command -v lspci >/dev/null 2>&1; then
  echo "lspci not found"
  exit 0
fi
found=0
	for gpu in $(lspci -Dn | awk '$2 ~ /^03(00|02):$/ && $3 ~ /^10de:/ {print $1}'); do
  found=1
  echo "=== GPU $gpu ==="
  lspci -s "$gpu" -vv 2>&1 || true
  bridge=$(basename "$(readlink -f "/sys/bus/pci/devices/$gpu/.." 2>/dev/null)" 2>/dev/null)
  if [ -n "$bridge" ] && [ "$bridge" != "$gpu" ]; then
    echo
    echo "=== UPSTREAM $bridge for $gpu ==="
    lspci -s "$bridge" -vv 2>&1 || true
  fi
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no NVIDIA PCI devices found"
fi
`}},
	{name: "export/gpu/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", `
for d in /sys/bus/pci/devices/*/; do
  vendor=$(cat "$d/vendor" 2>/dev/null)
	  [ "$vendor" = "0x10de" ] || continue
	  class=$(cat "$d/class" 2>/dev/null)
	  case "$class" in
	    0x030000|0x030200) ;;
	    *) continue ;;
	  esac
	  dev=$(basename "$d")
  echo "=== $dev ==="
  for f in current_link_speed current_link_width max_link_speed max_link_width; do
    printf "  %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
  done
done
`}},
	{name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
found=0
for dev in /sys/bus/pci/devices/*; do
  [ -e "$dev" ] || continue
  bdf=$(basename "$dev")
  block=""
  for f in aer_dev_correctable aer_dev_fatal aer_dev_nonfatal aer_rootport_total_err_cor aer_rootport_total_err_fatal aer_rootport_total_err_nonfatal; do
    if [ -r "$dev/$f" ]; then
      if [ -z "$block" ]; then
        block=1
        found=1
        echo "=== $bdf ==="
      fi
      printf "  %-30s %s\n" "$f" "$(cat "$dev/$f" 2>/dev/null)"
    fi
  done
  if [ -n "$block" ]; then
    echo
  fi
done
if [ "$found" -eq 0 ]; then
  echo "no PCIe AER sysfs counters found"
fi
`}},
	{name: "export/network/ethtool-info.txt", cmd: []string{"sh", "-c", `
if ! command -v ethtool >/dev/null 2>&1; then
  echo "ethtool not found"
  exit 0
fi
found=0
for path in /sys/class/net/*; do
  [ -e "$path" ] || continue
  iface=$(basename "$path")
  [ "$iface" = "lo" ] && continue
  found=1
  echo "=== $iface ==="
  ethtool -i "$iface" 2>&1 || true
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no interfaces found"
fi
`}},
	{name: "export/network/ethtool-link.txt", cmd: []string{"sh", "-c", `
if ! command -v ethtool >/dev/null 2>&1; then
  echo "ethtool not found"
  exit 0
fi
found=0
for path in /sys/class/net/*; do
  [ -e "$path" ] || continue
  iface=$(basename "$path")
  [ "$iface" = "lo" ] && continue
  found=1
  echo "=== $iface ==="
  ethtool "$iface" 2>&1 || true
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no interfaces found"
fi
`}},
	{name: "export/network/ethtool-module.txt", cmd: []string{"sh", "-c", `
if ! command -v ethtool >/dev/null 2>&1; then
  echo "ethtool not found"
  exit 0
fi
found=0
for path in /sys/class/net/*; do
  [ -e "$path" ] || continue
  iface=$(basename "$path")
  [ "$iface" = "lo" ] && continue
  found=1
  echo "=== $iface ==="
  ethtool -m "$iface" 2>&1 || true
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no interfaces found"
fi
`}},
	{name: "export/network/mstflint-query.txt", cmd: []string{"sh", "-c", `
if ! command -v mstflint >/dev/null 2>&1; then
  echo "mstflint not found"
  exit 0
fi
found=0
for path in /sys/bus/pci/devices/*; do
  [ -e "$path/vendor" ] || continue
  vendor=$(cat "$path/vendor" 2>/dev/null)
  [ "$vendor" = "0x15b3" ] || continue
  bdf=$(basename "$path")
  found=1
  echo "=== $bdf ==="
  mstflint -d "$bdf" q 2>&1 || true
  echo
done
if [ "$found" -eq 0 ]; then
  echo "no Mellanox/NVIDIA networking devices found"
fi
`}},
}

var supportBundleOptionalFiles = []struct {
	name string
	src  string
}{
	{name: "livecd/host/kern.log", src: "/var/log/kern.log"},
	{name: "livecd/host/syslog.txt", src: "/var/log/syslog"},
	{name: "livecd/gui/Xorg.0.log", src: "/var/log/Xorg.0.log"},
	{name: "livecd/gui/Xorg.0.log.old", src: "/var/log/Xorg.0.log.old"},
	{name: "livecd/gui/lightdm/lightdm.log", src: "/var/log/lightdm/lightdm.log"},
	{name: "livecd/gui/lightdm/x-0.log", src: "/var/log/lightdm/x-0.log"},
	{name: "livecd/gui/lightdm/x-0-greeter.log", src: "/var/log/lightdm/x-0-greeter.log"},
	{name: "livecd/gui/home-bee-xsession-errors.log", src: "/home/bee/.xsession-errors"},
	{name: "livecd/gui/home-bee-chromium-debug.log", src: "/tmp/bee-chrome/chrome_debug.log"},
	{name: "export/gpu/fabricmanager.log", src: "/var/log/fabricmanager.log"},
	{name: "export/gpu/nvlsm.log", src: "/var/log/nvlsm.log"},
	{name: "export/gpu/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"},
	{name: "export/gpu/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"},
}

const supportBundleGlob = "????-??-?? (BEE-SP*)*.tar.gz"

func BuildSupportBundle(exportDir string) (string, error) {
	exportDir = strings.TrimSpace(exportDir)
	if exportDir == "" {
		exportDir = DefaultExportDir
	}
	if err := os.MkdirAll(exportDir, 0755); err != nil {
		return "", err
	}
	if err := cleanupOldSupportBundles(os.TempDir()); err != nil {
		return "", err
	}

	now := time.Now().UTC()

	stageRoot := filepath.Join(os.TempDir(), fmt.Sprintf("bee-support-stage-%s-%s", sanitizeFilename(hostnameOr("unknown")), now.Format("20060102-150405")))
	if err := os.MkdirAll(stageRoot, 0755); err != nil {
		return "", err
	}
	defer os.RemoveAll(stageRoot)

	if err := categorizeExportTree(exportDir, stageRoot); err != nil {
		return "", err
	}
	if err := writeJournalDump(filepath.Join(stageRoot, "tasks", "_services", "combined.journal.log")); err != nil {
		return "", err
	}
	for _, svc := range supportBundleServices {
		dir := filepath.Join(stageRoot, serviceBundleDir(svc))
		if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil {
			return "", err
		}
		if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil {
			return "", err
		}
	}
	for _, item := range supportBundleCommands {
		if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd); err != nil {
			return "", err
		}
	}
	for _, item := range supportBundleOptionalFiles {
		_ = copyOptionalFile(item.src, filepath.Join(stageRoot, item.name))
	}
	if err := writeBundleDocs(stageRoot); err != nil {
		return "", err
	}
	if err := writeManifest(filepath.Join(stageRoot, "manifest.txt"), exportDir, stageRoot); err != nil {
		return "", err
	}

	archiveName := SupportBundleBaseName(now) + ".tar.gz"
	archivePath := filepath.Join(os.TempDir(), archiveName)
	if err := createSupportTarGz(archivePath, stageRoot); err != nil {
		return "", err
	}
	return archivePath, nil
}

func SupportBundleBaseName(at time.Time) string {
	at = at.UTC()
	date := at.Format("2006-01-02")
	tod := at.Format("150405")
	ver := bundleVersion()
	model := serverModelForBundle()
	sn := serverSerialForBundle()
	return fmt.Sprintf("%s (BEE-SP v%s) %s %s %s", date, ver, model, sn, tod)
}

func LatestSupportBundlePath() (string, error) {
	return latestSupportBundlePath(os.TempDir())
}

func cleanupOldSupportBundles(dir string) error {
	matches, err := filepath.Glob(filepath.Join(dir, supportBundleGlob))
	if err != nil {
		return err
	}
	entries := supportBundleEntries(matches)
	for path, mod := range entries {
		if time.Since(mod) > 24*time.Hour {
			_ = os.Remove(path)
			delete(entries, path)
		}
	}
	ordered := orderSupportBundles(entries)
	if len(ordered) > 3 {
		for _, old := range ordered[3:] {
			_ = os.Remove(old)
		}
	}
	return nil
}

func latestSupportBundlePath(dir string) (string, error) {
	matches, err := filepath.Glob(filepath.Join(dir, supportBundleGlob))
	if err != nil {
		return "", err
	}
	ordered := orderSupportBundles(supportBundleEntries(matches))
	if len(ordered) == 0 {
		return "", os.ErrNotExist
	}
	return ordered[0], nil
}

func supportBundleEntries(matches []string) map[string]time.Time {
	entries := make(map[string]time.Time, len(matches))
	for _, match := range matches {
		info, err := os.Stat(match)
		if err != nil {
			continue
		}
		entries[match] = info.ModTime()
	}
	return entries
}

func orderSupportBundles(entries map[string]time.Time) []string {
	ordered := make([]string, 0, len(entries))
	for path := range entries {
		ordered = append(ordered, path)
	}
	sort.Slice(ordered, func(i, j int) bool {
		return entries[ordered[i]].After(entries[ordered[j]])
	})
	return ordered
}

func writeJournalDump(dst string) error {
	args := []string{"--no-pager"}
	for _, svc := range supportBundleServices {
		args = append(args, "-u", svc)
	}
	raw, err := exec.Command("journalctl", args...).CombinedOutput()
	if len(raw) == 0 && err != nil {
		raw = []byte(err.Error() + "\n")
	}
	if len(raw) == 0 {
		raw = []byte("no journal output\n")
	}
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return err
	}
	return os.WriteFile(dst, raw, 0644)
}

func writeCommandOutput(dst string, cmd []string) error {
	if len(cmd) == 0 {
		return nil
	}
	raw, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
	if len(raw) == 0 {
		if err != nil {
			raw = []byte(err.Error() + "\n")
		} else {
			raw = []byte("no output\n")
		}
	}
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return err
	}
	return os.WriteFile(dst, raw, 0644)
}

func copyOptionalFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return err
	}
	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer out.Close()
	_, err = io.Copy(out, in)
	return err
}

func writeManifest(dst, exportDir, stageRoot string) error {
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return err
	}
	var body strings.Builder
	fmt.Fprintf(&body, "bee_version=%s\n", buildVersion())
	fmt.Fprintf(&body, "host=%s\n", hostnameOr("unknown"))
	fmt.Fprintf(&body, "generated_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
	fmt.Fprintf(&body, "export_dir=%s\n", exportDir)
	if cfg, err := platform.LoadBenchmarkPowerAutotuneConfig(filepath.Join(exportDir, "bee-bench", "power-source-autotune.json")); err == nil && cfg != nil {
		fmt.Fprintf(&body, "power_autotune_selected_source=%s\n", cfg.SelectedSource)
		fmt.Fprintf(&body, "power_autotune_updated_at=%s\n", cfg.UpdatedAt.UTC().Format(time.RFC3339))
		if strings.TrimSpace(cfg.Reason) != "" {
			fmt.Fprintf(&body, "power_autotune_reason=%s\n", cfg.Reason)
		}
	}
	fmt.Fprintf(&body, "\nfiles:\n")

	var files []string
	if err := filepath.Walk(stageRoot, func(path string, info os.FileInfo, err error) error {
		if err != nil || info.IsDir() {
			return err
		}
		if filepath.Clean(path) == filepath.Clean(dst) {
			return nil
		}
		rel, err := filepath.Rel(stageRoot, path)
		if err != nil {
			return err
		}
		files = append(files, fmt.Sprintf("%s\t%d", rel, info.Size()))
		return nil
	}); err != nil {
		return err
	}
	sort.Strings(files)
	for _, line := range files {
		body.WriteString(line)
		body.WriteByte('\n')
	}
	return os.WriteFile(dst, []byte(body.String()), 0644)
}

func bundleVersion() string {
	v := buildVersion()
	v = strings.TrimPrefix(v, "v")
	v = strings.TrimPrefix(v, "V")
	if v == "" || v == "unknown" {
		return "0.0"
	}
	return v
}

func serverModelForBundle() string {
	raw, err := exec.Command("dmidecode", "-t", "1").Output()
	if err != nil {
		return "unknown"
	}
	for _, line := range strings.Split(string(raw), "\n") {
		line = strings.TrimSpace(line)
		key, val, ok := strings.Cut(line, ": ")
		if !ok {
			continue
		}
		if strings.TrimSpace(key) == "Product Name" {
			val = strings.TrimSpace(val)
			if val == "" {
				return "unknown"
			}
			return strings.ReplaceAll(val, " ", "_")
		}
	}
	return "unknown"
}

func serverSerialForBundle() string {
	raw, err := exec.Command("dmidecode", "-t", "1").Output()
	if err != nil {
		return "unknown"
	}
	for _, line := range strings.Split(string(raw), "\n") {
		line = strings.TrimSpace(line)
		key, val, ok := strings.Cut(line, ": ")
		if !ok {
			continue
		}
		if strings.TrimSpace(key) == "Serial Number" {
			val = strings.TrimSpace(val)
			if val == "" {
				return "unknown"
			}
			return val
		}
	}
	return "unknown"
}

func buildVersion() string {
	raw, err := exec.Command("bee", "version").CombinedOutput()
	if err != nil {
		return "unknown"
	}
	return strings.TrimSpace(string(raw))
}

func copyDirContents(srcDir, dstDir string) error {
	entries, err := os.ReadDir(srcDir)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	for _, entry := range entries {
		src := filepath.Join(srcDir, entry.Name())
		dst := filepath.Join(dstDir, entry.Name())
		if err := copyPath(src, dst); err != nil {
			return err
		}
	}
	return nil
}

func copyDirContentsFiltered(srcDir, dstDir string, keep func(rel string, info os.FileInfo) bool) error {
	entries, err := os.ReadDir(srcDir)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	for _, entry := range entries {
		src := filepath.Join(srcDir, entry.Name())
		dst := filepath.Join(dstDir, entry.Name())
		if err := copyPathFiltered(srcDir, src, dst, keep); err != nil {
			return err
		}
	}
	return nil
}

func copyPath(src, dst string) error {
	info, err := os.Stat(src)
	if err != nil {
		// src is read from a live export dir that bee's own SAT/task runner
		// keeps writing to and renaming (e.g. a task dir's "_pending" ->
		// "_done" transition) concurrently with this copy — a path present
		// in the parent's os.ReadDir listing a moment ago disappearing by
		// the time we get here is an expected race, not a real failure.
		// Skip it; the next cycle will pick up wherever it landed under its
		// new name. Aborting the whole copy over one renamed-away entry is
		// what previously left blackbox stuck "degraded" indefinitely.
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	if info.IsDir() {
		if err := os.MkdirAll(dst, info.Mode().Perm()); err != nil {
			return err
		}
		entries, err := os.ReadDir(src)
		if err != nil {
			if os.IsNotExist(err) {
				return nil
			}
			return err
		}
		for _, entry := range entries {
			if err := copyPath(filepath.Join(src, entry.Name()), filepath.Join(dst, entry.Name())); err != nil {
				return err
			}
		}
		return nil
	}

	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return err
	}
	// Skip rewriting files the blackbox worker already copied unchanged on a
	// prior cycle — avoids needless flash wear on the removable target every
	// sync period. Size+mtime, not a full byte comparison, so large files
	// (e.g. status/metrics.db) can still stream-copy instead of loading
	// whole into memory.
	if dstInfo, err := os.Stat(dst); err == nil && dstInfo.Size() == info.Size() && !dstInfo.ModTime().Before(info.ModTime()) {
		return nil
	}
	in, err := os.Open(src)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	defer in.Close()

	out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode().Perm())
	if err != nil {
		return err
	}
	defer out.Close()

	_, err = io.Copy(out, in)
	return err
}

func copyPathFiltered(rootSrc, src, dst string, keep func(rel string, info os.FileInfo) bool) error {
	info, err := os.Stat(src)
	if err != nil {
		// See the matching comment in copyPath: src disappearing between the
		// parent's os.ReadDir and this Stat is an expected race against
		// bee's own live task-dir renames, not a real error.
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	rel, err := filepath.Rel(rootSrc, src)
	if err != nil {
		return err
	}
	if keep != nil && !keep(rel, info) {
		return nil
	}
	if info.IsDir() {
		if err := os.MkdirAll(dst, info.Mode().Perm()); err != nil {
			return err
		}
		entries, err := os.ReadDir(src)
		if err != nil {
			if os.IsNotExist(err) {
				return nil
			}
			return err
		}
		for _, entry := range entries {
			if err := copyPathFiltered(rootSrc, filepath.Join(src, entry.Name()), filepath.Join(dst, entry.Name()), keep); err != nil {
				return err
			}
		}
		return nil
	}
	return copyPath(src, dst)
}

func createSupportTarGz(dst, srcDir string) error {
	file, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer file.Close()

	gz := gzip.NewWriter(file)
	defer gz.Close()

	tw := tar.NewWriter(gz)
	defer tw.Close()

	base := filepath.Dir(srcDir)
	return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}

		header, err := tar.FileInfoHeader(info, "")
		if err != nil {
			return err
		}
		header.Name, err = filepath.Rel(base, path)
		if err != nil {
			return err
		}
		if err := tw.WriteHeader(header); err != nil {
			return err
		}

		f, err := os.Open(path)
		if err != nil {
			return err
		}
		defer f.Close()

		_, err = io.Copy(tw, f)
		return err
	})
}