fix(app): bound support-bundle subprocess calls with timeouts

A wedged GPU (fallen off the bus, needs a physical power-cycle) makes
nvidia-smi/nvidia-bug-report.sh/dcgmi hang indefinitely, so building a
support bundle on such a host never finished and the "Download Support
Bundle" button spun on "Building..." forever with no way to recover
short of restarting the service.

Give every subprocess this file shells out to a bounded context.
nvidia-bug-report.sh gets a longer timeout since it legitimately takes
tens of seconds on multi-GPU boxes even when healthy.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-08-24 17:41:18 +03:00
co-authored by Claude Sonnet 5
parent 134bed3eae
commit 9add5616dd
+47 -11
View File
@@ -4,6 +4,7 @@ import (
"archive/tar" "archive/tar"
"bee/audit/internal/platform" "bee/audit/internal/platform"
"compress/gzip" "compress/gzip"
"context"
_ "embed" _ "embed"
"fmt" "fmt"
"io" "io"
@@ -15,6 +16,21 @@ import (
"time" "time"
) )
// defaultCommandTimeout bounds every subprocess this file shells out to. GPU
// diagnostics (nvidia-smi, nvidia-bug-report.sh, dcgmi) are the main risk: a
// GPU that has fallen off the bus or needs a physical power-cycle (see
// gpuNeedsPhysicalReboot in webui/pages.go) makes those tools hang
// indefinitely rather than error out. Without a timeout, one wedged GPU
// blocks the whole bundle build forever and the "Download Support Bundle"
// button spins on "Building..." with no way to recover short of restarting
// the service.
const defaultCommandTimeout = 30 * time.Second
// nvidiaBugReportTimeout is longer than defaultCommandTimeout because
// nvidia-bug-report.sh legitimately takes tens of seconds to run on
// multi-GPU boxes even when healthy.
const nvidiaBugReportTimeout = 120 * time.Second
//go:embed assets/README.md //go:embed assets/README.md
var supportBundleReadmeMD []byte var supportBundleReadmeMD []byte
@@ -62,6 +78,7 @@ func serviceBundleDir(svc string) string {
var supportBundleCommands = []struct { var supportBundleCommands = []struct {
name string name string
cmd []string cmd []string
timeout time.Duration // zero means defaultCommandTimeout
}{ }{
{name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}}, {name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}},
{name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}}, {name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
@@ -225,7 +242,7 @@ else
echo "dcgmi not found" echo "dcgmi not found"
fi fi
`}}, `}},
{name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", ` {name: "export/gpu/nvidia-bug-report.txt", timeout: nvidiaBugReportTimeout, cmd: []string{"sh", "-c", `
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1
@@ -493,15 +510,19 @@ func BuildSupportBundle(exportDir string) (string, error) {
} }
for _, svc := range supportBundleServices { for _, svc := range supportBundleServices {
dir := filepath.Join(stageRoot, serviceBundleDir(svc)) dir := filepath.Join(stageRoot, serviceBundleDir(svc))
if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil { if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}, defaultCommandTimeout); err != nil {
return "", err return "", err
} }
if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil { if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}, defaultCommandTimeout); err != nil {
return "", err return "", err
} }
} }
for _, item := range supportBundleCommands { for _, item := range supportBundleCommands {
if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd); err != nil { timeout := item.timeout
if timeout <= 0 {
timeout = defaultCommandTimeout
}
if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd, timeout); err != nil {
return "", err return "", err
} }
} }
@@ -598,7 +619,9 @@ func writeJournalDump(dst string) error {
for _, svc := range supportBundleServices { for _, svc := range supportBundleServices {
args = append(args, "-u", svc) args = append(args, "-u", svc)
} }
raw, err := exec.Command("journalctl", args...).CombinedOutput() ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
defer cancel()
raw, err := exec.CommandContext(ctx, "journalctl", args...).CombinedOutput()
if len(raw) == 0 && err != nil { if len(raw) == 0 && err != nil {
raw = []byte(err.Error() + "\n") raw = []byte(err.Error() + "\n")
} }
@@ -611,13 +634,20 @@ func writeJournalDump(dst string) error {
return os.WriteFile(dst, raw, 0644) return os.WriteFile(dst, raw, 0644)
} }
func writeCommandOutput(dst string, cmd []string) error { func writeCommandOutput(dst string, cmd []string, timeout time.Duration) error {
if len(cmd) == 0 { if len(cmd) == 0 {
return nil return nil
} }
raw, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput() if timeout <= 0 {
timeout = defaultCommandTimeout
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
raw, err := exec.CommandContext(ctx, cmd[0], cmd[1:]...).CombinedOutput()
if len(raw) == 0 { if len(raw) == 0 {
if err != nil { if ctx.Err() == context.DeadlineExceeded {
raw = []byte(fmt.Sprintf("timed out after %s\n", timeout))
} else if err != nil {
raw = []byte(err.Error() + "\n") raw = []byte(err.Error() + "\n")
} else { } else {
raw = []byte("no output\n") raw = []byte("no output\n")
@@ -701,7 +731,9 @@ func bundleVersion() string {
} }
func serverModelForBundle() string { func serverModelForBundle() string {
raw, err := exec.Command("dmidecode", "-t", "1").Output() ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
defer cancel()
raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output()
if err != nil { if err != nil {
return "unknown" return "unknown"
} }
@@ -723,7 +755,9 @@ func serverModelForBundle() string {
} }
func serverSerialForBundle() string { func serverSerialForBundle() string {
raw, err := exec.Command("dmidecode", "-t", "1").Output() ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
defer cancel()
raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output()
if err != nil { if err != nil {
return "unknown" return "unknown"
} }
@@ -745,7 +779,9 @@ func serverSerialForBundle() string {
} }
func buildVersion() string { func buildVersion() string {
raw, err := exec.Command("bee", "version").CombinedOutput() ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
defer cancel()
raw, err := exec.CommandContext(ctx, "bee", "version").CombinedOutput()
if err != nil { if err != nil {
return "unknown" return "unknown"
} }