Mikhail ChusavitinandClaude Sonnet 5
9add5616dd
A wedged GPU (fallen off the bus, needs a physical power-cycle) makes nvidia-smi/nvidia-bug-report.sh/dcgmi hang indefinitely, so building a support bundle on such a host never finished and the "Download Support Bundle" button spun on "Building..." forever with no way to recover short of restarting the service. Give every subprocess this file shells out to a bounded context. nvidia-bug-report.sh gets a longer timeout since it legitimately takes tens of seconds on multi-GPU boxes even when healthy. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
975 lines
28 KiB
Go
975 lines
28 KiB
Go
package app
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bee/audit/internal/platform"
|
|
"compress/gzip"
|
|
"context"
|
|
_ "embed"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// defaultCommandTimeout bounds every subprocess this file shells out to. GPU
|
|
// diagnostics (nvidia-smi, nvidia-bug-report.sh, dcgmi) are the main risk: a
|
|
// GPU that has fallen off the bus or needs a physical power-cycle (see
|
|
// gpuNeedsPhysicalReboot in webui/pages.go) makes those tools hang
|
|
// indefinitely rather than error out. Without a timeout, one wedged GPU
|
|
// blocks the whole bundle build forever and the "Download Support Bundle"
|
|
// button spins on "Building..." with no way to recover short of restarting
|
|
// the service.
|
|
const defaultCommandTimeout = 30 * time.Second
|
|
|
|
// nvidiaBugReportTimeout is longer than defaultCommandTimeout because
|
|
// nvidia-bug-report.sh legitimately takes tens of seconds to run on
|
|
// multi-GPU boxes even when healthy.
|
|
const nvidiaBugReportTimeout = 120 * time.Second
|
|
|
|
//go:embed assets/README.md
|
|
var supportBundleReadmeMD []byte
|
|
|
|
// writeBundleDocs writes README.md at root, so an AI agent (or a human)
|
|
// analyzing this export (support bundle or blackbox capture) can orient
|
|
// itself without reverse-engineering the directory layout first.
|
|
func writeBundleDocs(root string) error {
|
|
if err := os.MkdirAll(root, 0755); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(filepath.Join(root, "README.md"), supportBundleReadmeMD, 0644)
|
|
}
|
|
|
|
var supportBundleServices = []string{
|
|
"bee-blackbox.service",
|
|
"bee-audit.service",
|
|
"bee-web.service",
|
|
"bee-network.service",
|
|
"bee-nvidia.service",
|
|
"bee-preflight.service",
|
|
"bee-selfheal.service",
|
|
"bee-selfheal.timer",
|
|
"bee-sshsetup.service",
|
|
"display-manager.service",
|
|
"lightdm.service",
|
|
"nvidia-dcgm.service",
|
|
"nvidia-fabricmanager.service",
|
|
}
|
|
|
|
// serviceBundleDir returns the bundle-relative directory a service's
|
|
// status/journal capture belongs in: bee's own daemons are internal
|
|
// bookkeeping, NVIDIA/DCGM/fabric-manager daemons are vendor-facing hardware
|
|
// diagnostics, and the display stack is LiveCD/GUI-session-only.
|
|
func serviceBundleDir(svc string) string {
|
|
switch svc {
|
|
case "nvidia-dcgm.service", "nvidia-fabricmanager.service":
|
|
return "export/gpu"
|
|
case "display-manager.service", "lightdm.service":
|
|
return "livecd/gui"
|
|
default:
|
|
return "tasks/_services"
|
|
}
|
|
}
|
|
|
|
var supportBundleCommands = []struct {
|
|
name string
|
|
cmd []string
|
|
timeout time.Duration // zero means defaultCommandTimeout
|
|
}{
|
|
{name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}},
|
|
{name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
|
|
{name: "livecd/host/lsmod.txt", cmd: []string{"lsmod"}},
|
|
{name: "export/platform/lspci-nn.txt", cmd: []string{"lspci", "-nn"}},
|
|
{name: "livecd/host/ip-addr.txt", cmd: []string{"ip", "addr"}},
|
|
{name: "livecd/host/ip-link.txt", cmd: []string{"ip", "-details", "link", "show"}},
|
|
{name: "livecd/host/ip-link-stats.txt", cmd: []string{"ip", "-s", "link", "show"}},
|
|
{name: "livecd/host/ip-route.txt", cmd: []string{"ip", "route"}},
|
|
{name: "livecd/host/mount.txt", cmd: []string{"mount"}},
|
|
{name: "livecd/host/df-h.txt", cmd: []string{"df", "-h"}},
|
|
{name: "livecd/host/dmesg.txt", cmd: []string{"dmesg"}},
|
|
{name: "livecd/gui/dmesg-gui-video-input.txt", cmd: []string{"sh", "-c", `
|
|
if command -v dmesg >/dev/null 2>&1; then
|
|
dmesg | grep -iE 'nvidia|drm|fb|framebuffer|vesa|efi|lightdm|Xorg|input|hid|usb|keyboard|mouse|virtual keyboard|virtual mouse|ami|aspeed|ast' || echo "no GUI/video/input kernel messages found"
|
|
else
|
|
echo "dmesg not found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", `
|
|
if command -v dmesg >/dev/null 2>&1; then
|
|
dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
|
|
else
|
|
echo "dmesg not found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/loginctl-sessions.txt", cmd: []string{"sh", "-c", `
|
|
if command -v loginctl >/dev/null 2>&1; then
|
|
loginctl list-sessions 2>&1 || true
|
|
else
|
|
echo "loginctl not found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/loginctl-seats.txt", cmd: []string{"sh", "-c", `
|
|
if command -v loginctl >/dev/null 2>&1; then
|
|
loginctl list-seats 2>&1 || true
|
|
echo
|
|
for seat in $(loginctl list-seats --no-legend 2>/dev/null | awk '{print $1}'); do
|
|
echo "=== $seat ==="
|
|
loginctl seat-status "$seat" 2>&1 || true
|
|
echo
|
|
done
|
|
else
|
|
echo "loginctl not found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/ps-gui.txt", cmd: []string{"sh", "-c", `
|
|
ps -ef | grep -iE 'lightdm|Xorg|X$|openbox|chromium|chrome|xinit|xsession' | grep -v grep || echo "no GUI processes found"
|
|
`}},
|
|
{name: "export/gpu/lspci-video-vv.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v lspci >/dev/null 2>&1; then
|
|
echo "lspci not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for dev in $(lspci -Dn | awk '$2 ~ /^03(00|02):$/ {print $1}'); do
|
|
found=1
|
|
echo "=== $dev ==="
|
|
lspci -s "$dev" -vv 2>&1 || true
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no display-class PCI devices found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/proc-fb.txt", cmd: []string{"cat", "/proc/fb"}},
|
|
{name: "livecd/gui/drm-cards.txt", cmd: []string{"sh", "-c", `
|
|
if [ -d /sys/class/drm ]; then
|
|
for path in /sys/class/drm/card*; do
|
|
[ -e "$path" ] || continue
|
|
card=$(basename "$path")
|
|
echo "=== $card ==="
|
|
for f in status enabled dpms modes; do
|
|
[ -r "$path/$f" ] && printf " %-8s %s\n" "$f" "$(cat "$path/$f" 2>/dev/null)"
|
|
done
|
|
device=$(readlink -f "$path/device" 2>/dev/null || true)
|
|
[ -n "$device" ] && echo " device ${device##*/}"
|
|
echo
|
|
done
|
|
else
|
|
echo "/sys/class/drm not present"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/input-devices.txt", cmd: []string{"sh", "-c", `
|
|
if [ -r /proc/bus/input/devices ]; then
|
|
cat /proc/bus/input/devices
|
|
else
|
|
echo "/proc/bus/input/devices not readable"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/udevadm-input.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v udevadm >/dev/null 2>&1; then
|
|
echo "udevadm not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for dev in /dev/input/event*; do
|
|
[ -e "$dev" ] || continue
|
|
found=1
|
|
echo "=== $dev ==="
|
|
udevadm info --query=all --name="$dev" 2>&1 || true
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no /dev/input/event* devices found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/xinput-list.txt", cmd: []string{"sh", "-c", `
|
|
if command -v xinput >/dev/null 2>&1; then
|
|
DISPLAY=:0 xinput --list 2>&1 || true
|
|
else
|
|
echo "xinput not found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/libinput-list-devices.txt", cmd: []string{"sh", "-c", `
|
|
if command -v libinput >/dev/null 2>&1; then
|
|
libinput list-devices 2>&1 || true
|
|
else
|
|
echo "libinput not found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/systemctl-gui-units.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v systemctl >/dev/null 2>&1; then
|
|
echo "systemctl not found"
|
|
exit 0
|
|
fi
|
|
echo "=== unit files ==="
|
|
systemctl list-unit-files --no-pager --all 'lightdm*' 'display-manager*' 2>&1 || true
|
|
echo
|
|
echo "=== active units ==="
|
|
systemctl list-units --no-pager --all 'lightdm*' 'display-manager*' 2>&1 || true
|
|
echo
|
|
echo "=== failed units ==="
|
|
systemctl --failed --no-pager 2>&1 | grep -iE 'lightdm|display-manager|Xorg' || echo "no failed GUI units"
|
|
`}},
|
|
{name: "export/gpu/nvidia-smi-topo-fresh.txt", cmd: []string{"sh", "-c", `
|
|
if command -v nvidia-smi >/dev/null 2>&1; then
|
|
nvidia-smi topo -m 2>&1 || true
|
|
else
|
|
echo "nvidia-smi not found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/nvidia-smi-nvlink-status-fresh.txt", cmd: []string{"sh", "-c", `
|
|
if command -v nvidia-smi >/dev/null 2>&1; then
|
|
nvidia-smi nvlink -s 2>&1 || true
|
|
else
|
|
echo "nvidia-smi not found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/nvidia-smi-nvlink-errors-fresh.txt", cmd: []string{"sh", "-c", `
|
|
if command -v nvidia-smi >/dev/null 2>&1; then
|
|
nvidia-smi nvlink -e 2>&1 || true
|
|
else
|
|
echo "nvidia-smi not found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", `
|
|
if command -v dcgmi >/dev/null 2>&1; then
|
|
dcgmi nvlink --link-status 2>&1 || true
|
|
else
|
|
echo "dcgmi not found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/nvidia-bug-report.txt", timeout: nvidiaBugReportTimeout, cmd: []string{"sh", "-c", `
|
|
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
|
|
rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
|
|
nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1
|
|
if [ -f /tmp/bee-nvidia-bug-report.log.gz ]; then
|
|
gzip -dc /tmp/bee-nvidia-bug-report.log.gz
|
|
elif [ -f /tmp/bee-nvidia-bug-report.log ]; then
|
|
cat /tmp/bee-nvidia-bug-report.log
|
|
else
|
|
echo "nvidia-bug-report.sh produced no output file"
|
|
fi
|
|
rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
|
|
else
|
|
echo "nvidia-bug-report.sh not found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v systemctl >/dev/null 2>&1; then
|
|
echo "systemctl not found"
|
|
exit 0
|
|
fi
|
|
echo "=== unit files ==="
|
|
systemctl list-unit-files --no-pager --all 'nvidia*' 'fabric*' 2>&1 || true
|
|
echo
|
|
echo "=== active units ==="
|
|
systemctl list-units --no-pager --all 'nvidia*' 'fabric*' 2>&1 || true
|
|
echo
|
|
echo "=== failed units ==="
|
|
systemctl --failed --no-pager 2>&1 | grep -iE 'nvidia|fabric' || echo "no failed nvidia/fabric units"
|
|
`}},
|
|
{name: "export/gpu/fabric-manager-paths.txt", cmd: []string{"sh", "-c", `
|
|
for candidate in \
|
|
/usr/bin/nvidia-fabricmanager \
|
|
/usr/bin/nv-fabricmanager \
|
|
/usr/bin/nvidia-fabricmanagerd \
|
|
/usr/bin/nvlsm; do
|
|
if [ -e "$candidate" ]; then
|
|
echo "=== $candidate ==="
|
|
ls -l "$candidate" 2>&1 || true
|
|
echo
|
|
fi
|
|
done
|
|
if ! ls /usr/bin/nvidia-fabricmanager /usr/bin/nv-fabricmanager /usr/bin/nvidia-fabricmanagerd /usr/bin/nvlsm >/dev/null 2>&1; then
|
|
echo "no fabric manager binaries found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v lspci >/dev/null 2>&1; then
|
|
echo "lspci not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for gpu in $(lspci -Dn | awk '$2 ~ /^03(00|02):$/ && $3 ~ /^10de:/ {print $1}'); do
|
|
found=1
|
|
echo "=== GPU $gpu ==="
|
|
lspci -s "$gpu" -vv 2>&1 || true
|
|
bridge=$(basename "$(readlink -f "/sys/bus/pci/devices/$gpu/.." 2>/dev/null)" 2>/dev/null)
|
|
if [ -n "$bridge" ] && [ "$bridge" != "$gpu" ]; then
|
|
echo
|
|
echo "=== UPSTREAM $bridge for $gpu ==="
|
|
lspci -s "$bridge" -vv 2>&1 || true
|
|
fi
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no NVIDIA PCI devices found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", `
|
|
for d in /sys/bus/pci/devices/*/; do
|
|
vendor=$(cat "$d/vendor" 2>/dev/null)
|
|
[ "$vendor" = "0x10de" ] || continue
|
|
class=$(cat "$d/class" 2>/dev/null)
|
|
case "$class" in
|
|
0x030000|0x030200) ;;
|
|
*) continue ;;
|
|
esac
|
|
dev=$(basename "$d")
|
|
echo "=== $dev ==="
|
|
for f in current_link_speed current_link_width max_link_speed max_link_width; do
|
|
printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
|
|
done
|
|
done
|
|
`}},
|
|
// A Gen1-vs-Gen4 link speed reading at idle is ambiguous: NVIDIA drivers
|
|
// deliberately downclock PCIe in low power states and re-train to full
|
|
// speed under load, so pcie-nvidia-link.txt alone can't tell a real
|
|
// hardware/riser fault from normal power management. Re-sample the same
|
|
// sysfs attributes while bee-gpu-burn is actively loading the GPUs — if
|
|
// the link comes up here, the idle Gen1 reading above was power saving,
|
|
// not a fault.
|
|
{name: "export/gpu/pcie-nvidia-link-under-load.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v bee-gpu-burn >/dev/null 2>&1; then
|
|
echo "bee-gpu-burn not found; cannot sample PCIe link speed under load"
|
|
exit 0
|
|
fi
|
|
bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 &
|
|
burn_pid=$!
|
|
sleep 3
|
|
for d in /sys/bus/pci/devices/*/; do
|
|
vendor=$(cat "$d/vendor" 2>/dev/null)
|
|
[ "$vendor" = "0x10de" ] || continue
|
|
class=$(cat "$d/class" 2>/dev/null)
|
|
case "$class" in
|
|
0x030000|0x030200) ;;
|
|
*) continue ;;
|
|
esac
|
|
dev=$(basename "$d")
|
|
echo "=== $dev ==="
|
|
for f in current_link_speed current_link_width max_link_speed max_link_width; do
|
|
printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
|
|
done
|
|
done
|
|
wait "$burn_pid" 2>/dev/null || true
|
|
rm -f /tmp/bee-pcie-load-burn.log
|
|
`}},
|
|
{name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
|
|
found=0
|
|
for dev in /sys/bus/pci/devices/*; do
|
|
[ -e "$dev" ] || continue
|
|
bdf=$(basename "$dev")
|
|
block=""
|
|
for f in aer_dev_correctable aer_dev_fatal aer_dev_nonfatal aer_rootport_total_err_cor aer_rootport_total_err_fatal aer_rootport_total_err_nonfatal; do
|
|
if [ -r "$dev/$f" ]; then
|
|
if [ -z "$block" ]; then
|
|
block=1
|
|
found=1
|
|
echo "=== $bdf ==="
|
|
fi
|
|
printf " %-30s %s\n" "$f" "$(cat "$dev/$f" 2>/dev/null)"
|
|
fi
|
|
done
|
|
if [ -n "$block" ]; then
|
|
echo
|
|
fi
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no PCIe AER sysfs counters found"
|
|
fi
|
|
`}},
|
|
{name: "export/network/ethtool-info.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v ethtool >/dev/null 2>&1; then
|
|
echo "ethtool not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for path in /sys/class/net/*; do
|
|
[ -e "$path" ] || continue
|
|
iface=$(basename "$path")
|
|
[ "$iface" = "lo" ] && continue
|
|
found=1
|
|
echo "=== $iface ==="
|
|
ethtool -i "$iface" 2>&1 || true
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no interfaces found"
|
|
fi
|
|
`}},
|
|
{name: "export/network/ethtool-link.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v ethtool >/dev/null 2>&1; then
|
|
echo "ethtool not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for path in /sys/class/net/*; do
|
|
[ -e "$path" ] || continue
|
|
iface=$(basename "$path")
|
|
[ "$iface" = "lo" ] && continue
|
|
found=1
|
|
echo "=== $iface ==="
|
|
ethtool "$iface" 2>&1 || true
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no interfaces found"
|
|
fi
|
|
`}},
|
|
{name: "export/network/ethtool-module.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v ethtool >/dev/null 2>&1; then
|
|
echo "ethtool not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for path in /sys/class/net/*; do
|
|
[ -e "$path" ] || continue
|
|
iface=$(basename "$path")
|
|
[ "$iface" = "lo" ] && continue
|
|
found=1
|
|
echo "=== $iface ==="
|
|
ethtool -m "$iface" 2>&1 || true
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no interfaces found"
|
|
fi
|
|
`}},
|
|
{name: "export/network/mstflint-query.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v mstflint >/dev/null 2>&1; then
|
|
echo "mstflint not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for path in /sys/bus/pci/devices/*; do
|
|
[ -e "$path/vendor" ] || continue
|
|
vendor=$(cat "$path/vendor" 2>/dev/null)
|
|
[ "$vendor" = "0x15b3" ] || continue
|
|
bdf=$(basename "$path")
|
|
found=1
|
|
echo "=== $bdf ==="
|
|
mstflint -d "$bdf" q 2>&1 || true
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no Mellanox/NVIDIA networking devices found"
|
|
fi
|
|
`}},
|
|
}
|
|
|
|
var supportBundleOptionalFiles = []struct {
|
|
name string
|
|
src string
|
|
}{
|
|
{name: "livecd/host/kern.log", src: "/var/log/kern.log"},
|
|
{name: "livecd/host/syslog.txt", src: "/var/log/syslog"},
|
|
{name: "livecd/gui/Xorg.0.log", src: "/var/log/Xorg.0.log"},
|
|
{name: "livecd/gui/Xorg.0.log.old", src: "/var/log/Xorg.0.log.old"},
|
|
{name: "livecd/gui/lightdm/lightdm.log", src: "/var/log/lightdm/lightdm.log"},
|
|
{name: "livecd/gui/lightdm/x-0.log", src: "/var/log/lightdm/x-0.log"},
|
|
{name: "livecd/gui/lightdm/x-0-greeter.log", src: "/var/log/lightdm/x-0-greeter.log"},
|
|
{name: "livecd/gui/home-bee-xsession-errors.log", src: "/home/bee/.xsession-errors"},
|
|
{name: "livecd/gui/home-bee-chromium-debug.log", src: "/tmp/bee-chrome/chrome_debug.log"},
|
|
{name: "export/gpu/fabricmanager.log", src: "/var/log/fabricmanager.log"},
|
|
{name: "export/gpu/nvlsm.log", src: "/var/log/nvlsm.log"},
|
|
{name: "export/gpu/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"},
|
|
{name: "export/gpu/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"},
|
|
}
|
|
|
|
const supportBundleGlob = "????-??-?? (BEE-SP*)*.tar.gz"
|
|
|
|
func BuildSupportBundle(exportDir string) (string, error) {
|
|
exportDir = strings.TrimSpace(exportDir)
|
|
if exportDir == "" {
|
|
exportDir = DefaultExportDir
|
|
}
|
|
if err := os.MkdirAll(exportDir, 0755); err != nil {
|
|
return "", err
|
|
}
|
|
if err := cleanupOldSupportBundles(os.TempDir()); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
now := time.Now().UTC()
|
|
|
|
stageRoot := filepath.Join(os.TempDir(), fmt.Sprintf("bee-support-stage-%s-%s", sanitizeFilename(hostnameOr("unknown")), now.Format("20060102-150405")))
|
|
if err := os.MkdirAll(stageRoot, 0755); err != nil {
|
|
return "", err
|
|
}
|
|
defer os.RemoveAll(stageRoot)
|
|
|
|
if err := categorizeExportTree(exportDir, stageRoot, true); err != nil {
|
|
return "", err
|
|
}
|
|
if err := writeJournalDump(filepath.Join(stageRoot, "tasks", "_services", "combined.journal.log")); err != nil {
|
|
return "", err
|
|
}
|
|
for _, svc := range supportBundleServices {
|
|
dir := filepath.Join(stageRoot, serviceBundleDir(svc))
|
|
if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}, defaultCommandTimeout); err != nil {
|
|
return "", err
|
|
}
|
|
if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}, defaultCommandTimeout); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
for _, item := range supportBundleCommands {
|
|
timeout := item.timeout
|
|
if timeout <= 0 {
|
|
timeout = defaultCommandTimeout
|
|
}
|
|
if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd, timeout); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
for _, item := range supportBundleOptionalFiles {
|
|
_ = copyOptionalFile(item.src, filepath.Join(stageRoot, item.name))
|
|
}
|
|
if err := writeBundleDocs(stageRoot); err != nil {
|
|
return "", err
|
|
}
|
|
if err := writeManifest(filepath.Join(stageRoot, "manifest.txt"), exportDir, stageRoot); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
archiveName := SupportBundleBaseName(now) + ".tar.gz"
|
|
archivePath := filepath.Join(os.TempDir(), archiveName)
|
|
if err := createSupportTarGz(archivePath, stageRoot); err != nil {
|
|
return "", err
|
|
}
|
|
return archivePath, nil
|
|
}
|
|
|
|
func SupportBundleBaseName(at time.Time) string {
|
|
at = at.UTC()
|
|
date := at.Format("2006-01-02")
|
|
tod := at.Format("150405")
|
|
ver := bundleVersion()
|
|
model := serverModelForBundle()
|
|
sn := serverSerialForBundle()
|
|
return fmt.Sprintf("%s (BEE-SP v%s) %s %s %s", date, ver, model, sn, tod)
|
|
}
|
|
|
|
func LatestSupportBundlePath() (string, error) {
|
|
return latestSupportBundlePath(os.TempDir())
|
|
}
|
|
|
|
func cleanupOldSupportBundles(dir string) error {
|
|
matches, err := filepath.Glob(filepath.Join(dir, supportBundleGlob))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
entries := supportBundleEntries(matches)
|
|
for path, mod := range entries {
|
|
if time.Since(mod) > 24*time.Hour {
|
|
_ = os.Remove(path)
|
|
delete(entries, path)
|
|
}
|
|
}
|
|
ordered := orderSupportBundles(entries)
|
|
if len(ordered) > 3 {
|
|
for _, old := range ordered[3:] {
|
|
_ = os.Remove(old)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func latestSupportBundlePath(dir string) (string, error) {
|
|
matches, err := filepath.Glob(filepath.Join(dir, supportBundleGlob))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
ordered := orderSupportBundles(supportBundleEntries(matches))
|
|
if len(ordered) == 0 {
|
|
return "", os.ErrNotExist
|
|
}
|
|
return ordered[0], nil
|
|
}
|
|
|
|
func supportBundleEntries(matches []string) map[string]time.Time {
|
|
entries := make(map[string]time.Time, len(matches))
|
|
for _, match := range matches {
|
|
info, err := os.Stat(match)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
entries[match] = info.ModTime()
|
|
}
|
|
return entries
|
|
}
|
|
|
|
func orderSupportBundles(entries map[string]time.Time) []string {
|
|
ordered := make([]string, 0, len(entries))
|
|
for path := range entries {
|
|
ordered = append(ordered, path)
|
|
}
|
|
sort.Slice(ordered, func(i, j int) bool {
|
|
return entries[ordered[i]].After(entries[ordered[j]])
|
|
})
|
|
return ordered
|
|
}
|
|
|
|
func writeJournalDump(dst string) error {
|
|
args := []string{"--no-pager"}
|
|
for _, svc := range supportBundleServices {
|
|
args = append(args, "-u", svc)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
|
|
defer cancel()
|
|
raw, err := exec.CommandContext(ctx, "journalctl", args...).CombinedOutput()
|
|
if len(raw) == 0 && err != nil {
|
|
raw = []byte(err.Error() + "\n")
|
|
}
|
|
if len(raw) == 0 {
|
|
raw = []byte("no journal output\n")
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(dst, raw, 0644)
|
|
}
|
|
|
|
func writeCommandOutput(dst string, cmd []string, timeout time.Duration) error {
|
|
if len(cmd) == 0 {
|
|
return nil
|
|
}
|
|
if timeout <= 0 {
|
|
timeout = defaultCommandTimeout
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
defer cancel()
|
|
raw, err := exec.CommandContext(ctx, cmd[0], cmd[1:]...).CombinedOutput()
|
|
if len(raw) == 0 {
|
|
if ctx.Err() == context.DeadlineExceeded {
|
|
raw = []byte(fmt.Sprintf("timed out after %s\n", timeout))
|
|
} else if err != nil {
|
|
raw = []byte(err.Error() + "\n")
|
|
} else {
|
|
raw = []byte("no output\n")
|
|
}
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(dst, raw, 0644)
|
|
}
|
|
|
|
func copyOptionalFile(src, dst string) error {
|
|
in, err := os.Open(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer in.Close()
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
out, err := os.Create(dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer out.Close()
|
|
_, err = io.Copy(out, in)
|
|
return err
|
|
}
|
|
|
|
func writeManifest(dst, exportDir, stageRoot string) error {
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
var body strings.Builder
|
|
fmt.Fprintf(&body, "bee_version=%s\n", buildVersion())
|
|
fmt.Fprintf(&body, "host=%s\n", hostnameOr("unknown"))
|
|
fmt.Fprintf(&body, "generated_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
|
fmt.Fprintf(&body, "export_dir=%s\n", exportDir)
|
|
if cfg, err := platform.LoadBenchmarkPowerAutotuneConfig(filepath.Join(exportDir, "bee-bench", "power-source-autotune.json")); err == nil && cfg != nil {
|
|
fmt.Fprintf(&body, "power_autotune_selected_source=%s\n", cfg.SelectedSource)
|
|
fmt.Fprintf(&body, "power_autotune_updated_at=%s\n", cfg.UpdatedAt.UTC().Format(time.RFC3339))
|
|
if strings.TrimSpace(cfg.Reason) != "" {
|
|
fmt.Fprintf(&body, "power_autotune_reason=%s\n", cfg.Reason)
|
|
}
|
|
}
|
|
fmt.Fprintf(&body, "\nfiles:\n")
|
|
|
|
var files []string
|
|
if err := filepath.Walk(stageRoot, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil || info.IsDir() {
|
|
return err
|
|
}
|
|
if filepath.Clean(path) == filepath.Clean(dst) {
|
|
return nil
|
|
}
|
|
rel, err := filepath.Rel(stageRoot, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
files = append(files, fmt.Sprintf("%s\t%d", rel, info.Size()))
|
|
return nil
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
sort.Strings(files)
|
|
for _, line := range files {
|
|
body.WriteString(line)
|
|
body.WriteByte('\n')
|
|
}
|
|
return os.WriteFile(dst, []byte(body.String()), 0644)
|
|
}
|
|
|
|
func bundleVersion() string {
|
|
v := buildVersion()
|
|
v = strings.TrimPrefix(v, "v")
|
|
v = strings.TrimPrefix(v, "V")
|
|
if v == "" || v == "unknown" {
|
|
return "0.0"
|
|
}
|
|
return v
|
|
}
|
|
|
|
func serverModelForBundle() string {
|
|
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
|
|
defer cancel()
|
|
raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output()
|
|
if err != nil {
|
|
return "unknown"
|
|
}
|
|
for _, line := range strings.Split(string(raw), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
key, val, ok := strings.Cut(line, ": ")
|
|
if !ok {
|
|
continue
|
|
}
|
|
if strings.TrimSpace(key) == "Product Name" {
|
|
val = strings.TrimSpace(val)
|
|
if val == "" {
|
|
return "unknown"
|
|
}
|
|
return strings.ReplaceAll(val, " ", "_")
|
|
}
|
|
}
|
|
return "unknown"
|
|
}
|
|
|
|
func serverSerialForBundle() string {
|
|
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
|
|
defer cancel()
|
|
raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output()
|
|
if err != nil {
|
|
return "unknown"
|
|
}
|
|
for _, line := range strings.Split(string(raw), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
key, val, ok := strings.Cut(line, ": ")
|
|
if !ok {
|
|
continue
|
|
}
|
|
if strings.TrimSpace(key) == "Serial Number" {
|
|
val = strings.TrimSpace(val)
|
|
if val == "" {
|
|
return "unknown"
|
|
}
|
|
return val
|
|
}
|
|
}
|
|
return "unknown"
|
|
}
|
|
|
|
func buildVersion() string {
|
|
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
|
|
defer cancel()
|
|
raw, err := exec.CommandContext(ctx, "bee", "version").CombinedOutput()
|
|
if err != nil {
|
|
return "unknown"
|
|
}
|
|
return strings.TrimSpace(string(raw))
|
|
}
|
|
|
|
func copyDirContents(srcDir, dstDir string) error {
|
|
entries, err := os.ReadDir(srcDir)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
for _, entry := range entries {
|
|
src := filepath.Join(srcDir, entry.Name())
|
|
dst := filepath.Join(dstDir, entry.Name())
|
|
if err := copyPath(src, dst); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func copyDirContentsFiltered(srcDir, dstDir string, keep func(rel string, info os.FileInfo) bool) error {
|
|
entries, err := os.ReadDir(srcDir)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
for _, entry := range entries {
|
|
src := filepath.Join(srcDir, entry.Name())
|
|
dst := filepath.Join(dstDir, entry.Name())
|
|
if err := copyPathFiltered(srcDir, src, dst, keep); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func copyPath(src, dst string) error {
|
|
info, err := os.Stat(src)
|
|
if err != nil {
|
|
// src is read from a live export dir that bee's own SAT/task runner
|
|
// keeps writing to and renaming (e.g. a task dir's "_pending" ->
|
|
// "_done" transition) concurrently with this copy — a path present
|
|
// in the parent's os.ReadDir listing a moment ago disappearing by
|
|
// the time we get here is an expected race, not a real failure.
|
|
// Skip it; the next cycle will pick up wherever it landed under its
|
|
// new name. Aborting the whole copy over one renamed-away entry is
|
|
// what previously left blackbox stuck "degraded" indefinitely.
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if info.IsDir() {
|
|
if err := os.MkdirAll(dst, info.Mode().Perm()); err != nil {
|
|
return err
|
|
}
|
|
entries, err := os.ReadDir(src)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
for _, entry := range entries {
|
|
if err := copyPath(filepath.Join(src, entry.Name()), filepath.Join(dst, entry.Name())); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
// Skip rewriting files the blackbox worker already copied unchanged on a
|
|
// prior cycle — avoids needless flash wear on the removable target every
|
|
// sync period. Size+mtime, not a full byte comparison, so large files
|
|
// (e.g. status/metrics.db) can still stream-copy instead of loading
|
|
// whole into memory.
|
|
if dstInfo, err := os.Stat(dst); err == nil && dstInfo.Size() == info.Size() && !dstInfo.ModTime().Before(info.ModTime()) {
|
|
return nil
|
|
}
|
|
in, err := os.Open(src)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
defer in.Close()
|
|
|
|
out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode().Perm())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer out.Close()
|
|
|
|
_, err = io.Copy(out, in)
|
|
return err
|
|
}
|
|
|
|
func copyPathFiltered(rootSrc, src, dst string, keep func(rel string, info os.FileInfo) bool) error {
|
|
info, err := os.Stat(src)
|
|
if err != nil {
|
|
// See the matching comment in copyPath: src disappearing between the
|
|
// parent's os.ReadDir and this Stat is an expected race against
|
|
// bee's own live task-dir renames, not a real error.
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
rel, err := filepath.Rel(rootSrc, src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if keep != nil && !keep(rel, info) {
|
|
return nil
|
|
}
|
|
if info.IsDir() {
|
|
if err := os.MkdirAll(dst, info.Mode().Perm()); err != nil {
|
|
return err
|
|
}
|
|
entries, err := os.ReadDir(src)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
for _, entry := range entries {
|
|
if err := copyPathFiltered(rootSrc, filepath.Join(src, entry.Name()), filepath.Join(dst, entry.Name()), keep); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
return copyPath(src, dst)
|
|
}
|
|
|
|
func createSupportTarGz(dst, srcDir string) error {
|
|
file, err := os.Create(dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
|
|
gz := gzip.NewWriter(file)
|
|
defer gz.Close()
|
|
|
|
tw := tar.NewWriter(gz)
|
|
defer tw.Close()
|
|
|
|
base := filepath.Dir(srcDir)
|
|
return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.IsDir() {
|
|
return nil
|
|
}
|
|
|
|
header, err := tar.FileInfoHeader(info, "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
header.Name, err = filepath.Rel(base, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := tw.WriteHeader(header); err != nil {
|
|
return err
|
|
}
|
|
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
|
|
_, err = io.Copy(tw, f)
|
|
return err
|
|
})
|
|
}
|