fix(app): bound support-bundle subprocess calls with timeouts
A wedged GPU (fallen off the bus, needs a physical power-cycle) makes nvidia-smi/nvidia-bug-report.sh/dcgmi hang indefinitely, so building a support bundle on such a host never finished and the "Download Support Bundle" button spun on "Building..." forever with no way to recover short of restarting the service. Give every subprocess this file shells out to a bounded context. nvidia-bug-report.sh gets a longer timeout since it legitimately takes tens of seconds on multi-GPU boxes even when healthy. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
134bed3eae
commit
9add5616dd
@@ -4,6 +4,7 @@ import (
|
|||||||
"archive/tar"
|
"archive/tar"
|
||||||
"bee/audit/internal/platform"
|
"bee/audit/internal/platform"
|
||||||
"compress/gzip"
|
"compress/gzip"
|
||||||
|
"context"
|
||||||
_ "embed"
|
_ "embed"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@@ -15,6 +16,21 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// defaultCommandTimeout bounds every subprocess this file shells out to. GPU
|
||||||
|
// diagnostics (nvidia-smi, nvidia-bug-report.sh, dcgmi) are the main risk: a
|
||||||
|
// GPU that has fallen off the bus or needs a physical power-cycle (see
|
||||||
|
// gpuNeedsPhysicalReboot in webui/pages.go) makes those tools hang
|
||||||
|
// indefinitely rather than error out. Without a timeout, one wedged GPU
|
||||||
|
// blocks the whole bundle build forever and the "Download Support Bundle"
|
||||||
|
// button spins on "Building..." with no way to recover short of restarting
|
||||||
|
// the service.
|
||||||
|
const defaultCommandTimeout = 30 * time.Second
|
||||||
|
|
||||||
|
// nvidiaBugReportTimeout is longer than defaultCommandTimeout because
|
||||||
|
// nvidia-bug-report.sh legitimately takes tens of seconds to run on
|
||||||
|
// multi-GPU boxes even when healthy.
|
||||||
|
const nvidiaBugReportTimeout = 120 * time.Second
|
||||||
|
|
||||||
//go:embed assets/README.md
|
//go:embed assets/README.md
|
||||||
var supportBundleReadmeMD []byte
|
var supportBundleReadmeMD []byte
|
||||||
|
|
||||||
@@ -62,6 +78,7 @@ func serviceBundleDir(svc string) string {
|
|||||||
var supportBundleCommands = []struct {
|
var supportBundleCommands = []struct {
|
||||||
name string
|
name string
|
||||||
cmd []string
|
cmd []string
|
||||||
|
timeout time.Duration // zero means defaultCommandTimeout
|
||||||
}{
|
}{
|
||||||
{name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}},
|
{name: "livecd/host/uname.txt", cmd: []string{"uname", "-a"}},
|
||||||
{name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
|
{name: "livecd/host/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
|
||||||
@@ -225,7 +242,7 @@ else
|
|||||||
echo "dcgmi not found"
|
echo "dcgmi not found"
|
||||||
fi
|
fi
|
||||||
`}},
|
`}},
|
||||||
{name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", `
|
{name: "export/gpu/nvidia-bug-report.txt", timeout: nvidiaBugReportTimeout, cmd: []string{"sh", "-c", `
|
||||||
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
|
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
|
||||||
rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
|
rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
|
||||||
nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1
|
nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1
|
||||||
@@ -493,15 +510,19 @@ func BuildSupportBundle(exportDir string) (string, error) {
|
|||||||
}
|
}
|
||||||
for _, svc := range supportBundleServices {
|
for _, svc := range supportBundleServices {
|
||||||
dir := filepath.Join(stageRoot, serviceBundleDir(svc))
|
dir := filepath.Join(stageRoot, serviceBundleDir(svc))
|
||||||
if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}); err != nil {
|
if err := writeCommandOutput(filepath.Join(dir, svc+".status.txt"), []string{"systemctl", "status", svc, "--no-pager"}, defaultCommandTimeout); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}); err != nil {
|
if err := writeCommandOutput(filepath.Join(dir, svc+".journal.log"), []string{"journalctl", "--no-pager", "-u", svc}, defaultCommandTimeout); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, item := range supportBundleCommands {
|
for _, item := range supportBundleCommands {
|
||||||
if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd); err != nil {
|
timeout := item.timeout
|
||||||
|
if timeout <= 0 {
|
||||||
|
timeout = defaultCommandTimeout
|
||||||
|
}
|
||||||
|
if err := writeCommandOutput(filepath.Join(stageRoot, item.name), item.cmd, timeout); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -598,7 +619,9 @@ func writeJournalDump(dst string) error {
|
|||||||
for _, svc := range supportBundleServices {
|
for _, svc := range supportBundleServices {
|
||||||
args = append(args, "-u", svc)
|
args = append(args, "-u", svc)
|
||||||
}
|
}
|
||||||
raw, err := exec.Command("journalctl", args...).CombinedOutput()
|
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
|
||||||
|
defer cancel()
|
||||||
|
raw, err := exec.CommandContext(ctx, "journalctl", args...).CombinedOutput()
|
||||||
if len(raw) == 0 && err != nil {
|
if len(raw) == 0 && err != nil {
|
||||||
raw = []byte(err.Error() + "\n")
|
raw = []byte(err.Error() + "\n")
|
||||||
}
|
}
|
||||||
@@ -611,13 +634,20 @@ func writeJournalDump(dst string) error {
|
|||||||
return os.WriteFile(dst, raw, 0644)
|
return os.WriteFile(dst, raw, 0644)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeCommandOutput(dst string, cmd []string) error {
|
func writeCommandOutput(dst string, cmd []string, timeout time.Duration) error {
|
||||||
if len(cmd) == 0 {
|
if len(cmd) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
raw, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
|
if timeout <= 0 {
|
||||||
|
timeout = defaultCommandTimeout
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||||
|
defer cancel()
|
||||||
|
raw, err := exec.CommandContext(ctx, cmd[0], cmd[1:]...).CombinedOutput()
|
||||||
if len(raw) == 0 {
|
if len(raw) == 0 {
|
||||||
if err != nil {
|
if ctx.Err() == context.DeadlineExceeded {
|
||||||
|
raw = []byte(fmt.Sprintf("timed out after %s\n", timeout))
|
||||||
|
} else if err != nil {
|
||||||
raw = []byte(err.Error() + "\n")
|
raw = []byte(err.Error() + "\n")
|
||||||
} else {
|
} else {
|
||||||
raw = []byte("no output\n")
|
raw = []byte("no output\n")
|
||||||
@@ -701,7 +731,9 @@ func bundleVersion() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func serverModelForBundle() string {
|
func serverModelForBundle() string {
|
||||||
raw, err := exec.Command("dmidecode", "-t", "1").Output()
|
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
|
||||||
|
defer cancel()
|
||||||
|
raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "unknown"
|
return "unknown"
|
||||||
}
|
}
|
||||||
@@ -723,7 +755,9 @@ func serverModelForBundle() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func serverSerialForBundle() string {
|
func serverSerialForBundle() string {
|
||||||
raw, err := exec.Command("dmidecode", "-t", "1").Output()
|
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
|
||||||
|
defer cancel()
|
||||||
|
raw, err := exec.CommandContext(ctx, "dmidecode", "-t", "1").Output()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "unknown"
|
return "unknown"
|
||||||
}
|
}
|
||||||
@@ -745,7 +779,9 @@ func serverSerialForBundle() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func buildVersion() string {
|
func buildVersion() string {
|
||||||
raw, err := exec.Command("bee", "version").CombinedOutput()
|
ctx, cancel := context.WithTimeout(context.Background(), defaultCommandTimeout)
|
||||||
|
defer cancel()
|
||||||
|
raw, err := exec.CommandContext(ctx, "bee", "version").CombinedOutput()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "unknown"
|
return "unknown"
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user