From 9add5616ddacafdfa9d4e1a2f38ff4a02f759e31 Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Mon, 24 Aug 2026 17:41:18 +0300 Subject: [PATCH] fix(app): bound support-bundle subprocess calls with timeouts A wedged GPU (fallen off the bus, needs a physical power-cycle) makes nvidia-smi/nvidia-bug-report.sh/dcgmi hang indefinitely, so building a support bundle on such a host never finished and the "Download Support Bundle" button spun on "Building..." forever with no way to recover short of restarting the service. Give every subprocess this file shells out to a bounded context. nvidia-bug-report.sh gets a longer timeout since it legitimately takes tens of seconds on multi-GPU boxes even when healthy. Co-Authored-By: Claude Sonnet 5 --- audit/internal/app/support_bundle.go | 62 ++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/audit/internal/app/support_bundle.go b/audit/internal/app/support_bundle.go index ca4402c..e11758f 100644 --- a/audit/internal/app/support_bundle.go +++ b/audit/internal/app/support_bundle.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bee/audit/internal/platform" "compress/gzip" + "context" _ "embed" "fmt" "io" @@ -15,6 +16,21 @@ import ( "time" ) +// defaultCommandTimeout bounds every subprocess this file shells out to. GPU +// diagnostics (nvidia-smi, nvidia-bug-report.sh, dcgmi) are the main risk: a +// GPU that has fallen off the bus or needs a physical power-cycle (see +// gpuNeedsPhysicalReboot in webui/pages.go) makes those tools hang +// indefinitely rather than error out. Without a timeout, one wedged GPU +// blocks the whole bundle build forever and the "Download Support Bundle" +// button spins on "Building..." with no way to recover short of restarting +// the service. +const defaultCommandTimeout = 30 * time.Second + +// nvidiaBugReportTimeout is longer than defaultCommandTimeout because +// nvidia-bug-report.sh legitimately takes tens of seconds to run on +// multi-GPU boxes even when healthy. +const nvidiaBugReportTimeout = 120 * time.Second + //go:embed assets/README.md var supportBundleReadmeMD []byte @@ -60,8 +76,9 @@ func serviceBundleDir(svc string) string { } var supportBundleCommands = []struct { - name string - cmd []string + name string + cmd []string + timeout time.Duration // zero means defaultCommandTimeout }{ {name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}}, {name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}}, @@ -225,7 +242,7 @@ else echo "dcgmi not found" fi `}}, - {name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/nvidia-bug-report.txt", timeout: nvidiaBugReportTimeout, cmd: []string{"sh", "-c", ` if command -v nvidia-bug-report.sh >/dev/null 2>&1; then rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 @@ -493,15 +510,19 @@ func BuildSupportBundle(exportDir string) (string, error) { } for _, svc := range supportBundleServices { dir := filepath.Join(stageRoot, serviceBundleDir(svc)) - if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil { + if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}, defaultCommandTimeout); err != nil { return "", err } - if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil { + if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}, defaultCommandTimeout); err != nil { return "", err } } for _, item := range supportBundleCommands { - if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd); err != nil { + timeout := item.timeout + if timeout <= 0 { + timeout = defaultCommandTimeout + } + if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd, timeout); err != nil { return "", err } } @@ -598,7 +619,9 @@ func writeJournalDump(dst string) error { for _, svc := range supportBundleServices { args = append(args, "-u", svc) } - raw, err := exec.Command("journalctl", args...).CombinedOutput() + ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout) + defer cancel() + raw, err := exec.CommandContext(ctx, "journalctl", args...).CombinedOutput() if len(raw) == 0 && err != nil { raw = []byte(err.Error() + "\n") } @@ -611,13 +634,20 @@ func writeJournalDump(dst string) error { return os.WriteFile(dst, raw, 0644) } -func writeCommandOutput(dst string, cmd []string) error { +func writeCommandOutput(dst string, cmd []string, timeout time.Duration) error { if len(cmd) == 0 { return nil } - raw, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput() + if timeout <= 0 { + timeout = defaultCommandTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + raw, err := exec.CommandContext(ctx, cmd[0], cmd[1:]...).CombinedOutput() if len(raw) == 0 { - if err != nil { + if ctx.Err() == context.DeadlineExceeded { + raw = []byte(fmt.Sprintf("timed out after %s\n", timeout)) + } else if err != nil { raw = []byte(err.Error() + "\n") } else { raw = []byte("no output\n") @@ -701,7 +731,9 @@ func bundleVersion() string { } func serverModelForBundle() string { - raw, err := exec.Command("dmidecode", "-t", "1").Output() + ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout) + defer cancel() + raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output() if err != nil { return "unknown" } @@ -723,7 +755,9 @@ func serverModelForBundle() string { } func serverSerialForBundle() string { - raw, err := exec.Command("dmidecode", "-t", "1").Output() + ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout) + defer cancel() + raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output() if err != nil { return "unknown" } @@ -745,7 +779,9 @@ func serverSerialForBundle() string { } func buildVersion() string { - raw, err := exec.Command("bee", "version").CombinedOutput() + ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout) + defer cancel() + raw, err := exec.CommandContext(ctx, "bee", "version").CombinedOutput() if err != nil { return "unknown" }