package webui import ( "fmt" "log/slog" "runtime/debug" "time" ) const ( recoverLoopMaxDelay = 60 * time.Second recoverLoopResetAfter = 30 * time.Second ) // goRecoverLoop starts fn in a goroutine, restarting after panics. // restartDelay is the initial delay; successive panics double it up to // recoverLoopMaxDelay. The delay resets to restartDelay once fn runs // successfully for recoverLoopResetAfter without panicking. func goRecoverLoop(name string, restartDelay time.Duration, fn func()) { go func() { delay := restartDelay consecutive := 0 for { start := time.Now() panicked := runRecoverable(name, fn) if !panicked { return } consecutive++ if time.Since(start) >= recoverLoopResetAfter { delay = restartDelay consecutive = 1 } slog.Warn("goroutine restarting after panic", "component", name, "consecutive_panics", consecutive, "next_delay", delay, ) if delay > 0 { time.Sleep(delay) } if delay < recoverLoopMaxDelay { delay *= 2 if delay > recoverLoopMaxDelay { delay = recoverLoopMaxDelay } } } }() } func goRecoverOnce(name string, fn func()) { go func() { _ = runRecoverable(name, fn) }() } func runRecoverable(name string, fn func()) (panicked bool) { defer func() { if rec := recover(); rec != nil { panicked = true slog.Error("recovered panic", "component", name, "panic", fmt.Sprint(rec), "stack", string(debug.Stack()), ) } }() fn() return false }