fix(app): bound support-bundle subprocess calls with timeouts

A wedged GPU (fallen off the bus, needs a physical power-cycle) makes
nvidia-smi/nvidia-bug-report.sh/dcgmi hang indefinitely, so building a
support bundle on such a host never finished and the "Download Support
Bundle" button spun on "Building..." forever with no way to recover
short of restarting the service.

Give every subprocess this file shells out to a bounded context.
nvidia-bug-report.sh gets a longer timeout since it legitimately takes
tens of seconds on multi-GPU boxes even when healthy.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-08-24 17:41:18 +03:00
co-authored by Claude Sonnet 5
parent 134bed3eae
commit 9add5616dd
+47 -11
View File
@@ -4,6 +4,7 @@ import (
"archive/tar"
"bee/audit/internal/platform"
"compress/gzip"
"context"
_ "embed"
"fmt"
"io"
@@ -15,6 +16,21 @@ import (
"time"
)
// defaultCommandTimeout bounds every subprocess this file shells out to. GPU
// diagnostics (nvidia-smi, nvidia-bug-report.sh, dcgmi) are the main risk: a
// GPU that has fallen off the bus or needs a physical power-cycle (see
// gpuNeedsPhysicalReboot in webui/pages.go) makes those tools hang
// indefinitely rather than error out. Without a timeout, one wedged GPU
// blocks the whole bundle build forever and the "Download Support Bundle"
// button spins on "Building..." with no way to recover short of restarting
// the service.
const defaultCommandTimeout = 30 * time.Second
// nvidiaBugReportTimeout is longer than defaultCommandTimeout because
// nvidia-bug-report.sh legitimately takes tens of seconds to run on
// multi-GPU boxes even when healthy.
const nvidiaBugReportTimeout = 120 * time.Second
//go:embed assets/README.md
var supportBundleReadmeMD []byte
@@ -62,6 +78,7 @@ func serviceBundleDir(svc string) string {
var supportBundleCommands = []struct {
name string
cmd []string
timeout time.Duration // zero means defaultCommandTimeout
}{
{name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}},
{name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
@@ -225,7 +242,7 @@ else
echo "dcgmi not found"
fi
`}},
{name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", `
{name: "export/gpu/nvidia-bug-report.txt", timeout: nvidiaBugReportTimeout, cmd: []string{"sh", "-c", `
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1
@@ -493,15 +510,19 @@ func BuildSupportBundle(exportDir string) (string, error) {
}
for _, svc := range supportBundleServices {
dir := filepath.Join(stageRoot, serviceBundleDir(svc))
if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil {
if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}, defaultCommandTimeout); err != nil {
return "", err
}
if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil {
if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}, defaultCommandTimeout); err != nil {
return "", err
}
}
for _, item := range supportBundleCommands {
if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd); err != nil {
timeout := item.timeout
if timeout <= 0 {
timeout = defaultCommandTimeout
}
if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd, timeout); err != nil {
return "", err
}
}
@@ -598,7 +619,9 @@ func writeJournalDump(dst string) error {
for _, svc := range supportBundleServices {
args = append(args, "-u", svc)
}
raw, err := exec.Command("journalctl", args...).CombinedOutput()
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
defer cancel()
raw, err := exec.CommandContext(ctx, "journalctl", args...).CombinedOutput()
if len(raw) == 0 && err != nil {
raw = []byte(err.Error() + "\n")
}
@@ -611,13 +634,20 @@ func writeJournalDump(dst string) error {
return os.WriteFile(dst, raw, 0644)
}
func writeCommandOutput(dst string, cmd []string) error {
func writeCommandOutput(dst string, cmd []string, timeout time.Duration) error {
if len(cmd) == 0 {
return nil
}
raw, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
if timeout <= 0 {
timeout = defaultCommandTimeout
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
raw, err := exec.CommandContext(ctx, cmd[0], cmd[1:]...).CombinedOutput()
if len(raw) == 0 {
if err != nil {
if ctx.Err() == context.DeadlineExceeded {
raw = []byte(fmt.Sprintf("timed out after %s\n", timeout))
} else if err != nil {
raw = []byte(err.Error() + "\n")
} else {
raw = []byte("no output\n")
@@ -701,7 +731,9 @@ func bundleVersion() string {
}
func serverModelForBundle() string {
raw, err := exec.Command("dmidecode", "-t", "1").Output()
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
defer cancel()
raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output()
if err != nil {
return "unknown"
}
@@ -723,7 +755,9 @@ func serverModelForBundle() string {
}
func serverSerialForBundle() string {
raw, err := exec.Command("dmidecode", "-t", "1").Output()
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
defer cancel()
raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output()
if err != nil {
return "unknown"
}
@@ -745,7 +779,9 @@ func serverSerialForBundle() string {
}
func buildVersion() string {
raw, err := exec.Command("bee", "version").CombinedOutput()
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
defer cancel()
raw, err := exec.CommandContext(ctx, "bee", "version").CombinedOutput()
if err != nil {
return "unknown"
}