diff --git a/audit/internal/app/support_bundle.go b/audit/internal/app/support_bundle.go index ca4402c..e11758f 100644 --- a/audit/internal/app/support_bundle.go +++ b/audit/internal/app/support_bundle.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bee/audit/internal/platform" "compress/gzip" + "context" _ "embed" "fmt" "io" @@ -15,6 +16,21 @@ import ( "time" ) +// defaultCommandTimeout bounds every subprocess this file shells out to. GPU +// diagnostics (nvidia-smi, nvidia-bug-report.sh, dcgmi) are the main risk: a +// GPU that has fallen off the bus or needs a physical power-cycle (see +// gpuNeedsPhysicalReboot in webui/pages.go) makes those tools hang +// indefinitely rather than error out. Without a timeout, one wedged GPU +// blocks the whole bundle build forever and the "Download Support Bundle" +// button spins on "Building..." with no way to recover short of restarting +// the service. +const defaultCommandTimeout = 30 * time.Second + +// nvidiaBugReportTimeout is longer than defaultCommandTimeout because +// nvidia-bug-report.sh legitimately takes tens of seconds to run on +// multi-GPU boxes even when healthy. +const nvidiaBugReportTimeout = 120 * time.Second + //go:embed assets/README.md var supportBundleReadmeMD []byte @@ -60,8 +76,9 @@ func serviceBundleDir(svc string) string { } var supportBundleCommands = []struct { - name string - cmd []string + name string + cmd []string + timeout time.Duration // zero means defaultCommandTimeout }{ {name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}}, {name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}}, @@ -225,7 +242,7 @@ else echo "dcgmi not found" fi `}}, - {name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", ` + {name: "export/gpu/nvidia-bug-report.txt", timeout: nvidiaBugReportTimeout, cmd: []string{"sh", "-c", ` if command -v nvidia-bug-report.sh >/dev/null 2>&1; then rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 @@ -493,15 +510,19 @@ func BuildSupportBundle(exportDir string) (string, error) { } for _, svc := range supportBundleServices { dir := filepath.Join(stageRoot, serviceBundleDir(svc)) - if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil { + if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}, defaultCommandTimeout); err != nil { return "", err } - if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil { + if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}, defaultCommandTimeout); err != nil { return "", err } } for _, item := range supportBundleCommands { - if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd); err != nil { + timeout := item.timeout + if timeout <= 0 { + timeout = defaultCommandTimeout + } + if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd, timeout); err != nil { return "", err } } @@ -598,7 +619,9 @@ func writeJournalDump(dst string) error { for _, svc := range supportBundleServices { args = append(args, "-u", svc) } - raw, err := exec.Command("journalctl", args...).CombinedOutput() + ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout) + defer cancel() + raw, err := exec.CommandContext(ctx, "journalctl", args...).CombinedOutput() if len(raw) == 0 && err != nil { raw = []byte(err.Error() + "\n") } @@ -611,13 +634,20 @@ func writeJournalDump(dst string) error { return os.WriteFile(dst, raw, 0644) } -func writeCommandOutput(dst string, cmd []string) error { +func writeCommandOutput(dst string, cmd []string, timeout time.Duration) error { if len(cmd) == 0 { return nil } - raw, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput() + if timeout <= 0 { + timeout = defaultCommandTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + raw, err := exec.CommandContext(ctx, cmd[0], cmd[1:]...).CombinedOutput() if len(raw) == 0 { - if err != nil { + if ctx.Err() == context.DeadlineExceeded { + raw = []byte(fmt.Sprintf("timed out after %s\n", timeout)) + } else if err != nil { raw = []byte(err.Error() + "\n") } else { raw = []byte("no output\n") @@ -701,7 +731,9 @@ func bundleVersion() string { } func serverModelForBundle() string { - raw, err := exec.Command("dmidecode", "-t", "1").Output() + ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout) + defer cancel() + raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output() if err != nil { return "unknown" } @@ -723,7 +755,9 @@ func serverModelForBundle() string { } func serverSerialForBundle() string { - raw, err := exec.Command("dmidecode", "-t", "1").Output() + ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout) + defer cancel() + raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output() if err != nil { return "unknown" } @@ -745,7 +779,9 @@ func serverSerialForBundle() string { } func buildVersion() string { - raw, err := exec.Command("bee", "version").CombinedOutput() + ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout) + defer cancel() + raw, err := exec.CommandContext(ctx, "bee", "version").CombinedOutput() if err != nil { return "unknown" }