fix(app): bound support-bundle subprocess calls with timeouts
A wedged GPU (fallen off the bus, needs a physical power-cycle) makes nvidia-smi/nvidia-bug-report.sh/dcgmi hang indefinitely, so building a support bundle on such a host never finished and the "Download Support Bundle" button spun on "Building..." forever with no way to recover short of restarting the service. Give every subprocess this file shells out to a bounded context. nvidia-bug-report.sh gets a longer timeout since it legitimately takes tens of seconds on multi-GPU boxes even when healthy. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
134bed3eae
commit
9add5616dd
@@ -4,6 +4,7 @@ import (
|
||||
"archive/tar"
|
||||
"bee/audit/internal/platform"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -15,6 +16,21 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// defaultCommandTimeout bounds every subprocess this file shells out to. GPU
|
||||
// diagnostics (nvidia-smi, nvidia-bug-report.sh, dcgmi) are the main risk: a
|
||||
// GPU that has fallen off the bus or needs a physical power-cycle (see
|
||||
// gpuNeedsPhysicalReboot in webui/pages.go) makes those tools hang
|
||||
// indefinitely rather than error out. Without a timeout, one wedged GPU
|
||||
// blocks the whole bundle build forever and the "Download Support Bundle"
|
||||
// button spins on "Building..." with no way to recover short of restarting
|
||||
// the service.
|
||||
const defaultCommandTimeout = 30 * time.Second
|
||||
|
||||
// nvidiaBugReportTimeout is longer than defaultCommandTimeout because
|
||||
// nvidia-bug-report.sh legitimately takes tens of seconds to run on
|
||||
// multi-GPU boxes even when healthy.
|
||||
const nvidiaBugReportTimeout = 120 * time.Second
|
||||
|
||||
//go:embed assets/README.md
|
||||
var supportBundleReadmeMD []byte
|
||||
|
||||
@@ -60,8 +76,9 @@ func serviceBundleDir(svc string) string {
|
||||
}
|
||||
|
||||
var supportBundleCommands = []struct {
|
||||
name string
|
||||
cmd []string
|
||||
name string
|
||||
cmd []string
|
||||
timeout time.Duration // zero means defaultCommandTimeout
|
||||
}{
|
||||
{name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}},
|
||||
{name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
|
||||
@@ -225,7 +242,7 @@ else
|
||||
echo "dcgmi not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "export/gpu/nvidia-bug-report.txt", timeout: nvidiaBugReportTimeout, cmd: []string{"sh", "-c", `
|
||||
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
|
||||
rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
|
||||
nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1
|
||||
@@ -493,15 +510,19 @@ func BuildSupportBundle(exportDir string) (string, error) {
|
||||
}
|
||||
for _, svc := range supportBundleServices {
|
||||
dir := filepath.Join(stageRoot, serviceBundleDir(svc))
|
||||
if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil {
|
||||
if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}, defaultCommandTimeout); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil {
|
||||
if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}, defaultCommandTimeout); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
for _, item := range supportBundleCommands {
|
||||
if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd); err != nil {
|
||||
timeout := item.timeout
|
||||
if timeout <= 0 {
|
||||
timeout = defaultCommandTimeout
|
||||
}
|
||||
if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd, timeout); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
@@ -598,7 +619,9 @@ func writeJournalDump(dst string) error {
|
||||
for _, svc := range supportBundleServices {
|
||||
args = append(args, "-u", svc)
|
||||
}
|
||||
raw, err := exec.Command("journalctl", args...).CombinedOutput()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
|
||||
defer cancel()
|
||||
raw, err := exec.CommandContext(ctx, "journalctl", args...).CombinedOutput()
|
||||
if len(raw) == 0 && err != nil {
|
||||
raw = []byte(err.Error() + "\n")
|
||||
}
|
||||
@@ -611,13 +634,20 @@ func writeJournalDump(dst string) error {
|
||||
return os.WriteFile(dst, raw, 0644)
|
||||
}
|
||||
|
||||
func writeCommandOutput(dst string, cmd []string) error {
|
||||
func writeCommandOutput(dst string, cmd []string, timeout time.Duration) error {
|
||||
if len(cmd) == 0 {
|
||||
return nil
|
||||
}
|
||||
raw, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
|
||||
if timeout <= 0 {
|
||||
timeout = defaultCommandTimeout
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
raw, err := exec.CommandContext(ctx, cmd[0], cmd[1:]...).CombinedOutput()
|
||||
if len(raw) == 0 {
|
||||
if err != nil {
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
raw = []byte(fmt.Sprintf("timed out after %s\n", timeout))
|
||||
} else if err != nil {
|
||||
raw = []byte(err.Error() + "\n")
|
||||
} else {
|
||||
raw = []byte("no output\n")
|
||||
@@ -701,7 +731,9 @@ func bundleVersion() string {
|
||||
}
|
||||
|
||||
func serverModelForBundle() string {
|
||||
raw, err := exec.Command("dmidecode", "-t", "1").Output()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
|
||||
defer cancel()
|
||||
raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output()
|
||||
if err != nil {
|
||||
return "unknown"
|
||||
}
|
||||
@@ -723,7 +755,9 @@ func serverModelForBundle() string {
|
||||
}
|
||||
|
||||
func serverSerialForBundle() string {
|
||||
raw, err := exec.Command("dmidecode", "-t", "1").Output()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
|
||||
defer cancel()
|
||||
raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output()
|
||||
if err != nil {
|
||||
return "unknown"
|
||||
}
|
||||
@@ -745,7 +779,9 @@ func serverSerialForBundle() string {
|
||||
}
|
||||
|
||||
func buildVersion() string {
|
||||
raw, err := exec.Command("bee", "version").CombinedOutput()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
|
||||
defer cancel()
|
||||
raw, err := exec.CommandContext(ctx, "bee", "version").CombinedOutput()
|
||||
if err != nil {
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user