bee/audit/internal/webui/stability.go

package webui

import (
	"fmt"
	"log/slog"
	"runtime/debug"
	"time"
)

const (
	recoverLoopMaxDelay   = 60 * time.Second
	recoverLoopResetAfter = 30 * time.Second
)

// goRecoverLoop starts fn in a goroutine, restarting after panics.
// restartDelay is the initial delay; successive panics double it up to
// recoverLoopMaxDelay. The delay resets to restartDelay once fn runs
// successfully for recoverLoopResetAfter without panicking.
func goRecoverLoop(name string, restartDelay time.Duration, fn func()) {
	go func() {
		delay := restartDelay
		consecutive := 0
		for {
			start := time.Now()
			panicked := runRecoverable(name, fn)
			if !panicked {
				return
			}
			consecutive++
			if time.Since(start) >= recoverLoopResetAfter {
				delay = restartDelay
				consecutive = 1
			}
			slog.Warn("goroutine restarting after panic",
				"component", name,
				"consecutive_panics", consecutive,
				"next_delay", delay,
			)
			if delay > 0 {
				time.Sleep(delay)
			}
			if delay < recoverLoopMaxDelay {
				delay *= 2
				if delay > recoverLoopMaxDelay {
					delay = recoverLoopMaxDelay
				}
			}
		}
	}()
}

func goRecoverOnce(name string, fn func()) {
	go func() {
		_ = runRecoverable(name, fn)
	}()
}

func runRecoverable(name string, fn func()) (panicked bool) {
	defer func() {
		if rec := recover(); rec != nil {
			panicked = true
			slog.Error("recovered panic",
				"component", name,
				"panic", fmt.Sprint(rec),
				"stack", string(debug.Stack()),
			)
		}
	}()
	fn()
	return false
}