Mikhail ChusavitinandClaude Sonnet 5
b1f165edb3
pcie:gpu:<vendor> in the component-status DB (what the Hardware Summary/webui "check passed" status reads) was only ever written by SAT diag results, and none of the nvidia/nvidia-config/nvidia-interconnect/ nvidia-bandwidth SAT jobs check PCIe link speed. So a real Gen1/Gen4 degradation the collector already flagged as Warning in the hardware snapshot never reached the DB-backed status, and the audit kept reporting "OK" despite GPUs training at Gen1 with BMC showing x16. Add writePCIeGPUStatusesToDB, mirroring the existing PSU write-through, so RunAudit pushes the collector's PCIe status into the DB alongside SAT results. Also add export/gpu/pcie-nvidia-link-under-load.txt to the support bundle: NVIDIA drivers deliberately downclock PCIe at idle to save power and re-train to full speed under load, so an idle Gen1 reading alone can't distinguish real hardware/riser degradation from normal power management. Resample the same sysfs link attributes while bee-gpu-burn is actively loading the GPUs so both bundles ship together. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
939 lines
26 KiB
Go
939 lines
26 KiB
Go
package app
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bee/audit/internal/platform"
|
|
"compress/gzip"
|
|
_ "embed"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
//go:embed assets/README.md
|
|
var supportBundleReadmeMD []byte
|
|
|
|
// writeBundleDocs writes README.md at root, so an AI agent (or a human)
|
|
// analyzing this export (support bundle or blackbox capture) can orient
|
|
// itself without reverse-engineering the directory layout first.
|
|
func writeBundleDocs(root string) error {
|
|
if err := os.MkdirAll(root, 0755); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(filepath.Join(root, "README.md"), supportBundleReadmeMD, 0644)
|
|
}
|
|
|
|
var supportBundleServices = []string{
|
|
"bee-blackbox.service",
|
|
"bee-audit.service",
|
|
"bee-web.service",
|
|
"bee-network.service",
|
|
"bee-nvidia.service",
|
|
"bee-preflight.service",
|
|
"bee-selfheal.service",
|
|
"bee-selfheal.timer",
|
|
"bee-sshsetup.service",
|
|
"display-manager.service",
|
|
"lightdm.service",
|
|
"nvidia-dcgm.service",
|
|
"nvidia-fabricmanager.service",
|
|
}
|
|
|
|
// serviceBundleDir returns the bundle-relative directory a service's
|
|
// status/journal capture belongs in: bee's own daemons are internal
|
|
// bookkeeping, NVIDIA/DCGM/fabric-manager daemons are vendor-facing hardware
|
|
// diagnostics, and the display stack is LiveCD/GUI-session-only.
|
|
func serviceBundleDir(svc string) string {
|
|
switch svc {
|
|
case "nvidia-dcgm.service", "nvidia-fabricmanager.service":
|
|
return "export/gpu"
|
|
case "display-manager.service", "lightdm.service":
|
|
return "livecd/gui"
|
|
default:
|
|
return "tasks/_services"
|
|
}
|
|
}
|
|
|
|
var supportBundleCommands = []struct {
|
|
name string
|
|
cmd []string
|
|
}{
|
|
{name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}},
|
|
{name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
|
|
{name: "livecd/host/lsmod.txt", cmd: []string{"lsmod"}},
|
|
{name: "export/platform/lspci-nn.txt", cmd: []string{"lspci", "-nn"}},
|
|
{name: "livecd/host/ip-addr.txt", cmd: []string{"ip", "addr"}},
|
|
{name: "livecd/host/ip-link.txt", cmd: []string{"ip", "-details", "link", "show"}},
|
|
{name: "livecd/host/ip-link-stats.txt", cmd: []string{"ip", "-s", "link", "show"}},
|
|
{name: "livecd/host/ip-route.txt", cmd: []string{"ip", "route"}},
|
|
{name: "livecd/host/mount.txt", cmd: []string{"mount"}},
|
|
{name: "livecd/host/df-h.txt", cmd: []string{"df", "-h"}},
|
|
{name: "livecd/host/dmesg.txt", cmd: []string{"dmesg"}},
|
|
{name: "livecd/gui/dmesg-gui-video-input.txt", cmd: []string{"sh", "-c", `
|
|
if command -v dmesg >/dev/null 2>&1; then
|
|
dmesg | grep -iE 'nvidia|drm|fb|framebuffer|vesa|efi|lightdm|Xorg|input|hid|usb|keyboard|mouse|virtual keyboard|virtual mouse|ami|aspeed|ast' || echo "no GUI/video/input kernel messages found"
|
|
else
|
|
echo "dmesg not found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", `
|
|
if command -v dmesg >/dev/null 2>&1; then
|
|
dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
|
|
else
|
|
echo "dmesg not found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/loginctl-sessions.txt", cmd: []string{"sh", "-c", `
|
|
if command -v loginctl >/dev/null 2>&1; then
|
|
loginctl list-sessions 2>&1 || true
|
|
else
|
|
echo "loginctl not found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/loginctl-seats.txt", cmd: []string{"sh", "-c", `
|
|
if command -v loginctl >/dev/null 2>&1; then
|
|
loginctl list-seats 2>&1 || true
|
|
echo
|
|
for seat in $(loginctl list-seats --no-legend 2>/dev/null | awk '{print $1}'); do
|
|
echo "=== $seat ==="
|
|
loginctl seat-status "$seat" 2>&1 || true
|
|
echo
|
|
done
|
|
else
|
|
echo "loginctl not found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/ps-gui.txt", cmd: []string{"sh", "-c", `
|
|
ps -ef | grep -iE 'lightdm|Xorg|X$|openbox|chromium|chrome|xinit|xsession' | grep -v grep || echo "no GUI processes found"
|
|
`}},
|
|
{name: "export/gpu/lspci-video-vv.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v lspci >/dev/null 2>&1; then
|
|
echo "lspci not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for dev in $(lspci -Dn | awk '$2 ~ /^03(00|02):$/ {print $1}'); do
|
|
found=1
|
|
echo "=== $dev ==="
|
|
lspci -s "$dev" -vv 2>&1 || true
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no display-class PCI devices found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/proc-fb.txt", cmd: []string{"cat", "/proc/fb"}},
|
|
{name: "livecd/gui/drm-cards.txt", cmd: []string{"sh", "-c", `
|
|
if [ -d /sys/class/drm ]; then
|
|
for path in /sys/class/drm/card*; do
|
|
[ -e "$path" ] || continue
|
|
card=$(basename "$path")
|
|
echo "=== $card ==="
|
|
for f in status enabled dpms modes; do
|
|
[ -r "$path/$f" ] && printf " %-8s %s\n" "$f" "$(cat "$path/$f" 2>/dev/null)"
|
|
done
|
|
device=$(readlink -f "$path/device" 2>/dev/null || true)
|
|
[ -n "$device" ] && echo " device ${device##*/}"
|
|
echo
|
|
done
|
|
else
|
|
echo "/sys/class/drm not present"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/input-devices.txt", cmd: []string{"sh", "-c", `
|
|
if [ -r /proc/bus/input/devices ]; then
|
|
cat /proc/bus/input/devices
|
|
else
|
|
echo "/proc/bus/input/devices not readable"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/udevadm-input.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v udevadm >/dev/null 2>&1; then
|
|
echo "udevadm not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for dev in /dev/input/event*; do
|
|
[ -e "$dev" ] || continue
|
|
found=1
|
|
echo "=== $dev ==="
|
|
udevadm info --query=all --name="$dev" 2>&1 || true
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no /dev/input/event* devices found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/xinput-list.txt", cmd: []string{"sh", "-c", `
|
|
if command -v xinput >/dev/null 2>&1; then
|
|
DISPLAY=:0 xinput --list 2>&1 || true
|
|
else
|
|
echo "xinput not found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/libinput-list-devices.txt", cmd: []string{"sh", "-c", `
|
|
if command -v libinput >/dev/null 2>&1; then
|
|
libinput list-devices 2>&1 || true
|
|
else
|
|
echo "libinput not found"
|
|
fi
|
|
`}},
|
|
{name: "livecd/gui/systemctl-gui-units.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v systemctl >/dev/null 2>&1; then
|
|
echo "systemctl not found"
|
|
exit 0
|
|
fi
|
|
echo "=== unit files ==="
|
|
systemctl list-unit-files --no-pager --all 'lightdm*' 'display-manager*' 2>&1 || true
|
|
echo
|
|
echo "=== active units ==="
|
|
systemctl list-units --no-pager --all 'lightdm*' 'display-manager*' 2>&1 || true
|
|
echo
|
|
echo "=== failed units ==="
|
|
systemctl --failed --no-pager 2>&1 | grep -iE 'lightdm|display-manager|Xorg' || echo "no failed GUI units"
|
|
`}},
|
|
{name: "export/gpu/nvidia-smi-topo-fresh.txt", cmd: []string{"sh", "-c", `
|
|
if command -v nvidia-smi >/dev/null 2>&1; then
|
|
nvidia-smi topo -m 2>&1 || true
|
|
else
|
|
echo "nvidia-smi not found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/nvidia-smi-nvlink-status-fresh.txt", cmd: []string{"sh", "-c", `
|
|
if command -v nvidia-smi >/dev/null 2>&1; then
|
|
nvidia-smi nvlink -s 2>&1 || true
|
|
else
|
|
echo "nvidia-smi not found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/nvidia-smi-nvlink-errors-fresh.txt", cmd: []string{"sh", "-c", `
|
|
if command -v nvidia-smi >/dev/null 2>&1; then
|
|
nvidia-smi nvlink -e 2>&1 || true
|
|
else
|
|
echo "nvidia-smi not found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", `
|
|
if command -v dcgmi >/dev/null 2>&1; then
|
|
dcgmi nvlink --link-status 2>&1 || true
|
|
else
|
|
echo "dcgmi not found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", `
|
|
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
|
|
rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
|
|
nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1
|
|
if [ -f /tmp/bee-nvidia-bug-report.log.gz ]; then
|
|
gzip -dc /tmp/bee-nvidia-bug-report.log.gz
|
|
elif [ -f /tmp/bee-nvidia-bug-report.log ]; then
|
|
cat /tmp/bee-nvidia-bug-report.log
|
|
else
|
|
echo "nvidia-bug-report.sh produced no output file"
|
|
fi
|
|
rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
|
|
else
|
|
echo "nvidia-bug-report.sh not found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v systemctl >/dev/null 2>&1; then
|
|
echo "systemctl not found"
|
|
exit 0
|
|
fi
|
|
echo "=== unit files ==="
|
|
systemctl list-unit-files --no-pager --all 'nvidia*' 'fabric*' 2>&1 || true
|
|
echo
|
|
echo "=== active units ==="
|
|
systemctl list-units --no-pager --all 'nvidia*' 'fabric*' 2>&1 || true
|
|
echo
|
|
echo "=== failed units ==="
|
|
systemctl --failed --no-pager 2>&1 | grep -iE 'nvidia|fabric' || echo "no failed nvidia/fabric units"
|
|
`}},
|
|
{name: "export/gpu/fabric-manager-paths.txt", cmd: []string{"sh", "-c", `
|
|
for candidate in \
|
|
/usr/bin/nvidia-fabricmanager \
|
|
/usr/bin/nv-fabricmanager \
|
|
/usr/bin/nvidia-fabricmanagerd \
|
|
/usr/bin/nvlsm; do
|
|
if [ -e "$candidate" ]; then
|
|
echo "=== $candidate ==="
|
|
ls -l "$candidate" 2>&1 || true
|
|
echo
|
|
fi
|
|
done
|
|
if ! ls /usr/bin/nvidia-fabricmanager /usr/bin/nv-fabricmanager /usr/bin/nvidia-fabricmanagerd /usr/bin/nvlsm >/dev/null 2>&1; then
|
|
echo "no fabric manager binaries found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v lspci >/dev/null 2>&1; then
|
|
echo "lspci not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for gpu in $(lspci -Dn | awk '$2 ~ /^03(00|02):$/ && $3 ~ /^10de:/ {print $1}'); do
|
|
found=1
|
|
echo "=== GPU $gpu ==="
|
|
lspci -s "$gpu" -vv 2>&1 || true
|
|
bridge=$(basename "$(readlink -f "/sys/bus/pci/devices/$gpu/.." 2>/dev/null)" 2>/dev/null)
|
|
if [ -n "$bridge" ] && [ "$bridge" != "$gpu" ]; then
|
|
echo
|
|
echo "=== UPSTREAM $bridge for $gpu ==="
|
|
lspci -s "$bridge" -vv 2>&1 || true
|
|
fi
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no NVIDIA PCI devices found"
|
|
fi
|
|
`}},
|
|
{name: "export/gpu/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", `
|
|
for d in /sys/bus/pci/devices/*/; do
|
|
vendor=$(cat "$d/vendor" 2>/dev/null)
|
|
[ "$vendor" = "0x10de" ] || continue
|
|
class=$(cat "$d/class" 2>/dev/null)
|
|
case "$class" in
|
|
0x030000|0x030200) ;;
|
|
*) continue ;;
|
|
esac
|
|
dev=$(basename "$d")
|
|
echo "=== $dev ==="
|
|
for f in current_link_speed current_link_width max_link_speed max_link_width; do
|
|
printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
|
|
done
|
|
done
|
|
`}},
|
|
// A Gen1-vs-Gen4 link speed reading at idle is ambiguous: NVIDIA drivers
|
|
// deliberately downclock PCIe in low power states and re-train to full
|
|
// speed under load, so pcie-nvidia-link.txt alone can't tell a real
|
|
// hardware/riser fault from normal power management. Re-sample the same
|
|
// sysfs attributes while bee-gpu-burn is actively loading the GPUs — if
|
|
// the link comes up here, the idle Gen1 reading above was power saving,
|
|
// not a fault.
|
|
{name: "export/gpu/pcie-nvidia-link-under-load.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v bee-gpu-burn >/dev/null 2>&1; then
|
|
echo "bee-gpu-burn not found; cannot sample PCIe link speed under load"
|
|
exit 0
|
|
fi
|
|
bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 &
|
|
burn_pid=$!
|
|
sleep 3
|
|
for d in /sys/bus/pci/devices/*/; do
|
|
vendor=$(cat "$d/vendor" 2>/dev/null)
|
|
[ "$vendor" = "0x10de" ] || continue
|
|
class=$(cat "$d/class" 2>/dev/null)
|
|
case "$class" in
|
|
0x030000|0x030200) ;;
|
|
*) continue ;;
|
|
esac
|
|
dev=$(basename "$d")
|
|
echo "=== $dev ==="
|
|
for f in current_link_speed current_link_width max_link_speed max_link_width; do
|
|
printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
|
|
done
|
|
done
|
|
wait "$burn_pid" 2>/dev/null || true
|
|
rm -f /tmp/bee-pcie-load-burn.log
|
|
`}},
|
|
{name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
|
|
found=0
|
|
for dev in /sys/bus/pci/devices/*; do
|
|
[ -e "$dev" ] || continue
|
|
bdf=$(basename "$dev")
|
|
block=""
|
|
for f in aer_dev_correctable aer_dev_fatal aer_dev_nonfatal aer_rootport_total_err_cor aer_rootport_total_err_fatal aer_rootport_total_err_nonfatal; do
|
|
if [ -r "$dev/$f" ]; then
|
|
if [ -z "$block" ]; then
|
|
block=1
|
|
found=1
|
|
echo "=== $bdf ==="
|
|
fi
|
|
printf " %-30s %s\n" "$f" "$(cat "$dev/$f" 2>/dev/null)"
|
|
fi
|
|
done
|
|
if [ -n "$block" ]; then
|
|
echo
|
|
fi
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no PCIe AER sysfs counters found"
|
|
fi
|
|
`}},
|
|
{name: "export/network/ethtool-info.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v ethtool >/dev/null 2>&1; then
|
|
echo "ethtool not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for path in /sys/class/net/*; do
|
|
[ -e "$path" ] || continue
|
|
iface=$(basename "$path")
|
|
[ "$iface" = "lo" ] && continue
|
|
found=1
|
|
echo "=== $iface ==="
|
|
ethtool -i "$iface" 2>&1 || true
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no interfaces found"
|
|
fi
|
|
`}},
|
|
{name: "export/network/ethtool-link.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v ethtool >/dev/null 2>&1; then
|
|
echo "ethtool not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for path in /sys/class/net/*; do
|
|
[ -e "$path" ] || continue
|
|
iface=$(basename "$path")
|
|
[ "$iface" = "lo" ] && continue
|
|
found=1
|
|
echo "=== $iface ==="
|
|
ethtool "$iface" 2>&1 || true
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no interfaces found"
|
|
fi
|
|
`}},
|
|
{name: "export/network/ethtool-module.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v ethtool >/dev/null 2>&1; then
|
|
echo "ethtool not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for path in /sys/class/net/*; do
|
|
[ -e "$path" ] || continue
|
|
iface=$(basename "$path")
|
|
[ "$iface" = "lo" ] && continue
|
|
found=1
|
|
echo "=== $iface ==="
|
|
ethtool -m "$iface" 2>&1 || true
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no interfaces found"
|
|
fi
|
|
`}},
|
|
{name: "export/network/mstflint-query.txt", cmd: []string{"sh", "-c", `
|
|
if ! command -v mstflint >/dev/null 2>&1; then
|
|
echo "mstflint not found"
|
|
exit 0
|
|
fi
|
|
found=0
|
|
for path in /sys/bus/pci/devices/*; do
|
|
[ -e "$path/vendor" ] || continue
|
|
vendor=$(cat "$path/vendor" 2>/dev/null)
|
|
[ "$vendor" = "0x15b3" ] || continue
|
|
bdf=$(basename "$path")
|
|
found=1
|
|
echo "=== $bdf ==="
|
|
mstflint -d "$bdf" q 2>&1 || true
|
|
echo
|
|
done
|
|
if [ "$found" -eq 0 ]; then
|
|
echo "no Mellanox/NVIDIA networking devices found"
|
|
fi
|
|
`}},
|
|
}
|
|
|
|
var supportBundleOptionalFiles = []struct {
|
|
name string
|
|
src string
|
|
}{
|
|
{name: "livecd/host/kern.log", src: "/var/log/kern.log"},
|
|
{name: "livecd/host/syslog.txt", src: "/var/log/syslog"},
|
|
{name: "livecd/gui/Xorg.0.log", src: "/var/log/Xorg.0.log"},
|
|
{name: "livecd/gui/Xorg.0.log.old", src: "/var/log/Xorg.0.log.old"},
|
|
{name: "livecd/gui/lightdm/lightdm.log", src: "/var/log/lightdm/lightdm.log"},
|
|
{name: "livecd/gui/lightdm/x-0.log", src: "/var/log/lightdm/x-0.log"},
|
|
{name: "livecd/gui/lightdm/x-0-greeter.log", src: "/var/log/lightdm/x-0-greeter.log"},
|
|
{name: "livecd/gui/home-bee-xsession-errors.log", src: "/home/bee/.xsession-errors"},
|
|
{name: "livecd/gui/home-bee-chromium-debug.log", src: "/tmp/bee-chrome/chrome_debug.log"},
|
|
{name: "export/gpu/fabricmanager.log", src: "/var/log/fabricmanager.log"},
|
|
{name: "export/gpu/nvlsm.log", src: "/var/log/nvlsm.log"},
|
|
{name: "export/gpu/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"},
|
|
{name: "export/gpu/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"},
|
|
}
|
|
|
|
const supportBundleGlob = "????-??-?? (BEE-SP*)*.tar.gz"
|
|
|
|
func BuildSupportBundle(exportDir string) (string, error) {
|
|
exportDir = strings.TrimSpace(exportDir)
|
|
if exportDir == "" {
|
|
exportDir = DefaultExportDir
|
|
}
|
|
if err := os.MkdirAll(exportDir, 0755); err != nil {
|
|
return "", err
|
|
}
|
|
if err := cleanupOldSupportBundles(os.TempDir()); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
now := time.Now().UTC()
|
|
|
|
stageRoot := filepath.Join(os.TempDir(), fmt.Sprintf("bee-support-stage-%s-%s", sanitizeFilename(hostnameOr("unknown")), now.Format("20060102-150405")))
|
|
if err := os.MkdirAll(stageRoot, 0755); err != nil {
|
|
return "", err
|
|
}
|
|
defer os.RemoveAll(stageRoot)
|
|
|
|
if err := categorizeExportTree(exportDir, stageRoot, true); err != nil {
|
|
return "", err
|
|
}
|
|
if err := writeJournalDump(filepath.Join(stageRoot, "tasks", "_services", "combined.journal.log")); err != nil {
|
|
return "", err
|
|
}
|
|
for _, svc := range supportBundleServices {
|
|
dir := filepath.Join(stageRoot, serviceBundleDir(svc))
|
|
if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil {
|
|
return "", err
|
|
}
|
|
if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
for _, item := range supportBundleCommands {
|
|
if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
for _, item := range supportBundleOptionalFiles {
|
|
_ = copyOptionalFile(item.src, filepath.Join(stageRoot, item.name))
|
|
}
|
|
if err := writeBundleDocs(stageRoot); err != nil {
|
|
return "", err
|
|
}
|
|
if err := writeManifest(filepath.Join(stageRoot, "manifest.txt"), exportDir, stageRoot); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
archiveName := SupportBundleBaseName(now) + ".tar.gz"
|
|
archivePath := filepath.Join(os.TempDir(), archiveName)
|
|
if err := createSupportTarGz(archivePath, stageRoot); err != nil {
|
|
return "", err
|
|
}
|
|
return archivePath, nil
|
|
}
|
|
|
|
func SupportBundleBaseName(at time.Time) string {
|
|
at = at.UTC()
|
|
date := at.Format("2006-01-02")
|
|
tod := at.Format("150405")
|
|
ver := bundleVersion()
|
|
model := serverModelForBundle()
|
|
sn := serverSerialForBundle()
|
|
return fmt.Sprintf("%s (BEE-SP v%s) %s %s %s", date, ver, model, sn, tod)
|
|
}
|
|
|
|
func LatestSupportBundlePath() (string, error) {
|
|
return latestSupportBundlePath(os.TempDir())
|
|
}
|
|
|
|
func cleanupOldSupportBundles(dir string) error {
|
|
matches, err := filepath.Glob(filepath.Join(dir, supportBundleGlob))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
entries := supportBundleEntries(matches)
|
|
for path, mod := range entries {
|
|
if time.Since(mod) > 24*time.Hour {
|
|
_ = os.Remove(path)
|
|
delete(entries, path)
|
|
}
|
|
}
|
|
ordered := orderSupportBundles(entries)
|
|
if len(ordered) > 3 {
|
|
for _, old := range ordered[3:] {
|
|
_ = os.Remove(old)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func latestSupportBundlePath(dir string) (string, error) {
|
|
matches, err := filepath.Glob(filepath.Join(dir, supportBundleGlob))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
ordered := orderSupportBundles(supportBundleEntries(matches))
|
|
if len(ordered) == 0 {
|
|
return "", os.ErrNotExist
|
|
}
|
|
return ordered[0], nil
|
|
}
|
|
|
|
func supportBundleEntries(matches []string) map[string]time.Time {
|
|
entries := make(map[string]time.Time, len(matches))
|
|
for _, match := range matches {
|
|
info, err := os.Stat(match)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
entries[match] = info.ModTime()
|
|
}
|
|
return entries
|
|
}
|
|
|
|
func orderSupportBundles(entries map[string]time.Time) []string {
|
|
ordered := make([]string, 0, len(entries))
|
|
for path := range entries {
|
|
ordered = append(ordered, path)
|
|
}
|
|
sort.Slice(ordered, func(i, j int) bool {
|
|
return entries[ordered[i]].After(entries[ordered[j]])
|
|
})
|
|
return ordered
|
|
}
|
|
|
|
func writeJournalDump(dst string) error {
|
|
args := []string{"--no-pager"}
|
|
for _, svc := range supportBundleServices {
|
|
args = append(args, "-u", svc)
|
|
}
|
|
raw, err := exec.Command("journalctl", args...).CombinedOutput()
|
|
if len(raw) == 0 && err != nil {
|
|
raw = []byte(err.Error() + "\n")
|
|
}
|
|
if len(raw) == 0 {
|
|
raw = []byte("no journal output\n")
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(dst, raw, 0644)
|
|
}
|
|
|
|
func writeCommandOutput(dst string, cmd []string) error {
|
|
if len(cmd) == 0 {
|
|
return nil
|
|
}
|
|
raw, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
|
|
if len(raw) == 0 {
|
|
if err != nil {
|
|
raw = []byte(err.Error() + "\n")
|
|
} else {
|
|
raw = []byte("no output\n")
|
|
}
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(dst, raw, 0644)
|
|
}
|
|
|
|
func copyOptionalFile(src, dst string) error {
|
|
in, err := os.Open(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer in.Close()
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
out, err := os.Create(dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer out.Close()
|
|
_, err = io.Copy(out, in)
|
|
return err
|
|
}
|
|
|
|
func writeManifest(dst, exportDir, stageRoot string) error {
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
var body strings.Builder
|
|
fmt.Fprintf(&body, "bee_version=%s\n", buildVersion())
|
|
fmt.Fprintf(&body, "host=%s\n", hostnameOr("unknown"))
|
|
fmt.Fprintf(&body, "generated_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
|
fmt.Fprintf(&body, "export_dir=%s\n", exportDir)
|
|
if cfg, err := platform.LoadBenchmarkPowerAutotuneConfig(filepath.Join(exportDir, "bee-bench", "power-source-autotune.json")); err == nil && cfg != nil {
|
|
fmt.Fprintf(&body, "power_autotune_selected_source=%s\n", cfg.SelectedSource)
|
|
fmt.Fprintf(&body, "power_autotune_updated_at=%s\n", cfg.UpdatedAt.UTC().Format(time.RFC3339))
|
|
if strings.TrimSpace(cfg.Reason) != "" {
|
|
fmt.Fprintf(&body, "power_autotune_reason=%s\n", cfg.Reason)
|
|
}
|
|
}
|
|
fmt.Fprintf(&body, "\nfiles:\n")
|
|
|
|
var files []string
|
|
if err := filepath.Walk(stageRoot, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil || info.IsDir() {
|
|
return err
|
|
}
|
|
if filepath.Clean(path) == filepath.Clean(dst) {
|
|
return nil
|
|
}
|
|
rel, err := filepath.Rel(stageRoot, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
files = append(files, fmt.Sprintf("%s\t%d", rel, info.Size()))
|
|
return nil
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
sort.Strings(files)
|
|
for _, line := range files {
|
|
body.WriteString(line)
|
|
body.WriteByte('\n')
|
|
}
|
|
return os.WriteFile(dst, []byte(body.String()), 0644)
|
|
}
|
|
|
|
func bundleVersion() string {
|
|
v := buildVersion()
|
|
v = strings.TrimPrefix(v, "v")
|
|
v = strings.TrimPrefix(v, "V")
|
|
if v == "" || v == "unknown" {
|
|
return "0.0"
|
|
}
|
|
return v
|
|
}
|
|
|
|
func serverModelForBundle() string {
|
|
raw, err := exec.Command("dmidecode", "-t", "1").Output()
|
|
if err != nil {
|
|
return "unknown"
|
|
}
|
|
for _, line := range strings.Split(string(raw), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
key, val, ok := strings.Cut(line, ": ")
|
|
if !ok {
|
|
continue
|
|
}
|
|
if strings.TrimSpace(key) == "Product Name" {
|
|
val = strings.TrimSpace(val)
|
|
if val == "" {
|
|
return "unknown"
|
|
}
|
|
return strings.ReplaceAll(val, " ", "_")
|
|
}
|
|
}
|
|
return "unknown"
|
|
}
|
|
|
|
func serverSerialForBundle() string {
|
|
raw, err := exec.Command("dmidecode", "-t", "1").Output()
|
|
if err != nil {
|
|
return "unknown"
|
|
}
|
|
for _, line := range strings.Split(string(raw), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
key, val, ok := strings.Cut(line, ": ")
|
|
if !ok {
|
|
continue
|
|
}
|
|
if strings.TrimSpace(key) == "Serial Number" {
|
|
val = strings.TrimSpace(val)
|
|
if val == "" {
|
|
return "unknown"
|
|
}
|
|
return val
|
|
}
|
|
}
|
|
return "unknown"
|
|
}
|
|
|
|
func buildVersion() string {
|
|
raw, err := exec.Command("bee", "version").CombinedOutput()
|
|
if err != nil {
|
|
return "unknown"
|
|
}
|
|
return strings.TrimSpace(string(raw))
|
|
}
|
|
|
|
func copyDirContents(srcDir, dstDir string) error {
|
|
entries, err := os.ReadDir(srcDir)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
for _, entry := range entries {
|
|
src := filepath.Join(srcDir, entry.Name())
|
|
dst := filepath.Join(dstDir, entry.Name())
|
|
if err := copyPath(src, dst); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func copyDirContentsFiltered(srcDir, dstDir string, keep func(rel string, info os.FileInfo) bool) error {
|
|
entries, err := os.ReadDir(srcDir)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
for _, entry := range entries {
|
|
src := filepath.Join(srcDir, entry.Name())
|
|
dst := filepath.Join(dstDir, entry.Name())
|
|
if err := copyPathFiltered(srcDir, src, dst, keep); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func copyPath(src, dst string) error {
|
|
info, err := os.Stat(src)
|
|
if err != nil {
|
|
// src is read from a live export dir that bee's own SAT/task runner
|
|
// keeps writing to and renaming (e.g. a task dir's "_pending" ->
|
|
// "_done" transition) concurrently with this copy — a path present
|
|
// in the parent's os.ReadDir listing a moment ago disappearing by
|
|
// the time we get here is an expected race, not a real failure.
|
|
// Skip it; the next cycle will pick up wherever it landed under its
|
|
// new name. Aborting the whole copy over one renamed-away entry is
|
|
// what previously left blackbox stuck "degraded" indefinitely.
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if info.IsDir() {
|
|
if err := os.MkdirAll(dst, info.Mode().Perm()); err != nil {
|
|
return err
|
|
}
|
|
entries, err := os.ReadDir(src)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
for _, entry := range entries {
|
|
if err := copyPath(filepath.Join(src, entry.Name()), filepath.Join(dst, entry.Name())); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
// Skip rewriting files the blackbox worker already copied unchanged on a
|
|
// prior cycle — avoids needless flash wear on the removable target every
|
|
// sync period. Size+mtime, not a full byte comparison, so large files
|
|
// (e.g. status/metrics.db) can still stream-copy instead of loading
|
|
// whole into memory.
|
|
if dstInfo, err := os.Stat(dst); err == nil && dstInfo.Size() == info.Size() && !dstInfo.ModTime().Before(info.ModTime()) {
|
|
return nil
|
|
}
|
|
in, err := os.Open(src)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
defer in.Close()
|
|
|
|
out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode().Perm())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer out.Close()
|
|
|
|
_, err = io.Copy(out, in)
|
|
return err
|
|
}
|
|
|
|
func copyPathFiltered(rootSrc, src, dst string, keep func(rel string, info os.FileInfo) bool) error {
|
|
info, err := os.Stat(src)
|
|
if err != nil {
|
|
// See the matching comment in copyPath: src disappearing between the
|
|
// parent's os.ReadDir and this Stat is an expected race against
|
|
// bee's own live task-dir renames, not a real error.
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
rel, err := filepath.Rel(rootSrc, src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if keep != nil && !keep(rel, info) {
|
|
return nil
|
|
}
|
|
if info.IsDir() {
|
|
if err := os.MkdirAll(dst, info.Mode().Perm()); err != nil {
|
|
return err
|
|
}
|
|
entries, err := os.ReadDir(src)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
for _, entry := range entries {
|
|
if err := copyPathFiltered(rootSrc, filepath.Join(src, entry.Name()), filepath.Join(dst, entry.Name()), keep); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
return copyPath(src, dst)
|
|
}
|
|
|
|
func createSupportTarGz(dst, srcDir string) error {
|
|
file, err := os.Create(dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
|
|
gz := gzip.NewWriter(file)
|
|
defer gz.Close()
|
|
|
|
tw := tar.NewWriter(gz)
|
|
defer tw.Close()
|
|
|
|
base := filepath.Dir(srcDir)
|
|
return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.IsDir() {
|
|
return nil
|
|
}
|
|
|
|
header, err := tar.FileInfoHeader(info, "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
header.Name, err = filepath.Rel(base, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := tw.WriteHeader(header); err != nil {
|
|
return err
|
|
}
|
|
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
|
|
_, err = io.Copy(tw, f)
|
|
return err
|
|
})
|
|
}
|