fix(webui): prevent orphaned workers on restart, reduce metrics polling, add Kill Workers button
- tasks: mark TaskRunning tasks as TaskFailed on bee-web restart instead of re-queueing them — prevents duplicate gpu-burn-worker spawns when bee-web crashes mid-test (each restart was launching a new set of 8 workers on top of still-alive orphans from the previous crash) - server: reduce metrics collector interval 1s→5s, grow ring buffer to 360 samples (30 min); cuts nvidia-smi/ipmitool/sensors subprocess rate by 5× - platform: add KillTestWorkers() — scans /proc and SIGKILLs bee-gpu-burn, stress-ng, stressapptest, memtester without relying on pkill/killall - webui: add "Kill Workers" button next to Cancel All; calls POST /api/tasks/kill-workers which cancels the task queue then kills orphaned OS-level processes; shows toast with killed count - metricsdb: sort GPU indices and fan/temp names after map iteration to fix non-deterministic sample reconstruction order (flaky test) - server: fix chartYAxisNumber to use one decimal place for 1000–9999 (e.g. "1,7к" instead of "2к") so Y-axis ticks are distinguishable Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -128,7 +128,11 @@ type namedMetricsRing struct {
|
||||
Ring *metricsRing
|
||||
}
|
||||
|
||||
const metricsChartWindow = 120
|
||||
// metricsChartWindow is the number of samples kept in the live ring buffer.
|
||||
// At metricsCollectInterval = 5 s this covers 30 minutes of live history.
|
||||
const metricsChartWindow = 360
|
||||
|
||||
var metricsCollectInterval = 5 * time.Second
|
||||
|
||||
// pendingNetChange tracks a network state change awaiting confirmation.
|
||||
type pendingNetChange struct {
|
||||
@@ -238,6 +242,7 @@ func NewHandler(opts HandlerOptions) http.Handler {
|
||||
// Tasks
|
||||
mux.HandleFunc("GET /api/tasks", h.handleAPITasksList)
|
||||
mux.HandleFunc("POST /api/tasks/cancel-all", h.handleAPITasksCancelAll)
|
||||
mux.HandleFunc("POST /api/tasks/kill-workers", h.handleAPITasksKillWorkers)
|
||||
mux.HandleFunc("POST /api/tasks/{id}/cancel", h.handleAPITasksCancel)
|
||||
mux.HandleFunc("POST /api/tasks/{id}/priority", h.handleAPITasksPriority)
|
||||
mux.HandleFunc("GET /api/tasks/{id}/stream", h.handleAPITasksStream)
|
||||
@@ -301,7 +306,7 @@ func NewHandler(opts HandlerOptions) http.Handler {
|
||||
|
||||
func (h *handler) startMetricsCollector() {
|
||||
go func() {
|
||||
ticker := time.NewTicker(1 * time.Second)
|
||||
ticker := time.NewTicker(metricsCollectInterval)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
sample := platform.SampleLiveMetrics()
|
||||
@@ -1059,9 +1064,16 @@ func chartYAxisNumber(v float64) string {
|
||||
v = -v
|
||||
}
|
||||
var out string
|
||||
if v >= 1000 {
|
||||
switch {
|
||||
case v >= 10000:
|
||||
out = fmt.Sprintf("%dк", int((v+500)/1000))
|
||||
} else {
|
||||
case v >= 1000:
|
||||
// Use one decimal place so ticks like 1400, 1600, 1800 read as
|
||||
// "1,4к", "1,6к", "1,8к" instead of the ambiguous "1к"/"2к".
|
||||
s := fmt.Sprintf("%.1f", v/1000)
|
||||
s = strings.TrimRight(strings.TrimRight(s, "0"), ".")
|
||||
out = strings.ReplaceAll(s, ".", ",") + "к"
|
||||
default:
|
||||
out = fmt.Sprintf("%.0f", v)
|
||||
}
|
||||
if neg {
|
||||
|
||||
Reference in New Issue
Block a user