From 86aa11c2d78d05ef811978387cde452b59347ca9 Mon Sep 17 00:00:00 2001 From: Michael Chus Date: Wed, 29 Jul 2026 13:00:41 +0300 Subject: [PATCH] fix(webui): dispatch "scenario" tasks in the real task runner, collapse duplicate target switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Scenario page's Run button enqueued a task with target "scenario" that never launched any load: the queue runs each task in an external bee-worker subprocess (RunPersistedTask -> executeTaskWithOptions in task_runner.go), whose target switch had no "scenario" case, so the task died with "unknown target: scenario". The case had only been added to taskQueue.runTask's switch in tasks.go — a stale copy exercised solely by unit tests, which is why the tests passed while the button did nothing. Add the "scenario" case (ReadScenario -> ParseScenarioJSON -> RunScenario) to executeTaskWithOptions, and collapse runTask into a thin delegate to it so there is a single target dispatch. The old runTask switch had already drifted into a subset (missing nvme-format, *-write, raid-*, nvidia-config, ...); removing it eliminates the divergence that hid this bug. Also drop the now-orphaned taskQueue.statusDB helper. Co-Authored-By: Claude Opus 4.8 --- audit/internal/webui/task_runner.go | 16 ++ audit/internal/webui/tasks.go | 422 +++------------------------- 2 files changed, 52 insertions(+), 386 deletions(-) diff --git a/audit/internal/webui/task_runner.go b/audit/internal/webui/task_runner.go index 355d4ee..04a3a6e 100644 --- a/audit/internal/webui/task_runner.go +++ b/audit/internal/webui/task_runner.go @@ -231,6 +231,22 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont break } archive, err = a.RunNCCLTests(ctx, "", t.params.GPUIndices, j.append) + case "scenario": + if a == nil { + err = fmt.Errorf("app not configured") + break + } + var data []byte + data, err = a.ReadScenario(t.params.ScenarioName) + if err != nil { + break + } + var spec platform.ScenarioSpec + spec, err = platform.ParseScenarioJSON(data) + if err != nil { + break + } + archive, err = a.RunScenario(ctx, "", spec, j.append) case "nvidia-stress": if a == nil { err = fmt.Errorf("app not configured") diff --git a/audit/internal/webui/tasks.go b/audit/internal/webui/tasks.go index bf153f7..b9ecf54 100644 --- a/audit/internal/webui/tasks.go +++ b/audit/internal/webui/tasks.go @@ -45,7 +45,7 @@ var taskNames = map[string]string{ "nvidia-stress": "NVIDIA GPU Stress", "memory": "Memory SAT", "storage": "Storage SAT", - "nvidia-config": "GPU Config & NVLink Check", + "nvidia-config": "GPU Config & NVLink Check", "cpu": "CPU SAT", "amd": "AMD GPU SAT", "amd-mem": "AMD GPU MEM Integrity", @@ -121,34 +121,34 @@ type Task struct { // taskParams holds optional parameters parsed from the run request. type taskParams struct { - Duration int `json:"duration,omitempty"` - StressMode bool `json:"stress_mode,omitempty"` - GPUIndices []int `json:"gpu_indices,omitempty"` - ExcludeGPUIndices []int `json:"exclude_gpu_indices,omitempty"` - StaggerGPUStart bool `json:"stagger_gpu_start,omitempty"` - SizeMB int `json:"size_mb,omitempty"` - Passes int `json:"passes,omitempty"` - Loader string `json:"loader,omitempty"` - BurnProfile string `json:"burn_profile,omitempty"` - BenchmarkProfile string `json:"benchmark_profile,omitempty"` - BenchmarkKind string `json:"benchmark_kind,omitempty"` - RunNCCL bool `json:"run_nccl,omitempty"` - ParallelGPUs bool `json:"parallel_gpus,omitempty"` - RampStep int `json:"ramp_step,omitempty"` - RampTotal int `json:"ramp_total,omitempty"` - RampRunID string `json:"ramp_run_id,omitempty"` - DisplayName string `json:"display_name,omitempty"` - Device string `json:"device,omitempty"` // for install - LBAF int `json:"lbaf,omitempty"` - PlatformComponents []string `json:"platform_components,omitempty"` - SAADmiChanges []saaChange `json:"saa_dmi_changes,omitempty"` - FRUChanges []fruChange `json:"fru_changes,omitempty"` - HuaweiElabelChanges []huaweiChange `json:"huawei_elabel_changes,omitempty"` - RAIDController int `json:"raid_controller,omitempty"` - RAIDDevices []string `json:"raid_devices,omitempty"` - RAIDArrayName string `json:"raid_array_name,omitempty"` - RAIDSlot string `json:"raid_slot,omitempty"` - ScenarioName string `json:"scenario_name,omitempty"` + Duration int `json:"duration,omitempty"` + StressMode bool `json:"stress_mode,omitempty"` + GPUIndices []int `json:"gpu_indices,omitempty"` + ExcludeGPUIndices []int `json:"exclude_gpu_indices,omitempty"` + StaggerGPUStart bool `json:"stagger_gpu_start,omitempty"` + SizeMB int `json:"size_mb,omitempty"` + Passes int `json:"passes,omitempty"` + Loader string `json:"loader,omitempty"` + BurnProfile string `json:"burn_profile,omitempty"` + BenchmarkProfile string `json:"benchmark_profile,omitempty"` + BenchmarkKind string `json:"benchmark_kind,omitempty"` + RunNCCL bool `json:"run_nccl,omitempty"` + ParallelGPUs bool `json:"parallel_gpus,omitempty"` + RampStep int `json:"ramp_step,omitempty"` + RampTotal int `json:"ramp_total,omitempty"` + RampRunID string `json:"ramp_run_id,omitempty"` + DisplayName string `json:"display_name,omitempty"` + Device string `json:"device,omitempty"` // for install + LBAF int `json:"lbaf,omitempty"` + PlatformComponents []string `json:"platform_components,omitempty"` + SAADmiChanges []saaChange `json:"saa_dmi_changes,omitempty"` + FRUChanges []fruChange `json:"fru_changes,omitempty"` + HuaweiElabelChanges []huaweiChange `json:"huawei_elabel_changes,omitempty"` + RAIDController int `json:"raid_controller,omitempty"` + RAIDDevices []string `json:"raid_devices,omitempty"` + RAIDArrayName string `json:"raid_array_name,omitempty"` + RAIDSlot string `json:"raid_slot,omitempty"` + ScenarioName string `json:"scenario_name,omitempty"` } type persistedTask struct { @@ -846,364 +846,14 @@ func setCPUGovernor(governor string) { } } -// runTask executes the work for a task, writing output to j. +// runTask executes the work for a task, writing output to j. It delegates +// to executeTaskWithOptions — the exact same target dispatch the external +// task runner (RunPersistedTask, see task_runner.go) uses in its own +// subprocess — so the in-process path (recovered tasks, unit tests) and the +// subprocess path can never drift to different target coverage. Adding a new +// task target means editing one switch, not two. func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) { - if q.opts == nil { - j.append("ERROR: handler options not configured") - j.finish("handler options not configured") - return - } - a := q.opts.App - - recovered := len(j.lines) > 0 - j.append(fmt.Sprintf("Starting %s...", t.Name)) - if recovered { - j.append(fmt.Sprintf("Recovered after bee-web restart at %s", time.Now().UTC().Format(time.RFC3339))) - } - - var ( - archive string - err error - ) - - switch t.Target { - case "nvidia": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - diagLevel := 2 - if t.params.StressMode { - diagLevel = 3 - } - if len(t.params.GPUIndices) > 0 || diagLevel > 0 { - result, e := a.RunNvidiaAcceptancePackWithOptions( - ctx, "", diagLevel, t.params.GPUIndices, j.append, - ) - if e != nil { - err = e - } else { - archive = result.Body - } - } else { - archive, err = a.RunNvidiaAcceptancePack("", j.append) - } - case "nvidia-targeted-stress": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - dur := t.params.Duration - if dur <= 0 { - dur = 300 - } - archive, err = a.RunNvidiaTargetedStressValidatePack(ctx, "", dur, t.params.GPUIndices, j.append) - case "nvidia-bench-perf": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - archive, err = a.RunNvidiaBenchmarkCtx(ctx, "", platform.NvidiaBenchmarkOptions{ - Profile: t.params.BenchmarkProfile, - SizeMB: t.params.SizeMB, - GPUIndices: t.params.GPUIndices, - ExcludeGPUIndices: t.params.ExcludeGPUIndices, - RunNCCL: t.params.RunNCCL, - ParallelGPUs: t.params.ParallelGPUs, - RampStep: t.params.RampStep, - RampTotal: t.params.RampTotal, - RampRunID: t.params.RampRunID, - }, j.append) - case "nvidia-bench-power": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - archive, err = a.RunNvidiaPowerBenchCtx(ctx, app.DefaultBeeBenchPowerDir, platform.NvidiaBenchmarkOptions{ - Profile: t.params.BenchmarkProfile, - GPUIndices: t.params.GPUIndices, - ExcludeGPUIndices: t.params.ExcludeGPUIndices, - RampStep: t.params.RampStep, - RampTotal: t.params.RampTotal, - RampRunID: t.params.RampRunID, - }, j.append) - case "nvidia-bench-autotune": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - archive, err = a.RunNvidiaPowerSourceAutotuneCtx(ctx, app.DefaultBeeBenchAutotuneDir, platform.NvidiaBenchmarkOptions{ - Profile: t.params.BenchmarkProfile, - SizeMB: t.params.SizeMB, - }, t.params.BenchmarkKind, j.append) - case "nvidia-compute": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - dur := t.params.Duration - if t.params.BurnProfile != "" && dur <= 0 { - dur = resolveBurnPreset(t.params.BurnProfile).DurationSec - } - rampPlan, planErr := resolveNvidiaRampPlan(t.params.BurnProfile, t.params.StaggerGPUStart, t.params.GPUIndices) - if planErr != nil { - err = planErr - break - } - if t.params.BurnProfile != "" && t.params.StaggerGPUStart && dur <= 0 { - dur = rampPlan.DurationSec - } - if rampPlan.StaggerSeconds > 0 { - j.append(fmt.Sprintf("NVIDIA staggered ramp-up enabled: %ds per GPU; post-ramp hold: %ds; total runtime: %ds", rampPlan.StaggerSeconds, dur, rampPlan.TotalDurationSec)) - } - archive, err = a.RunNvidiaOfficialComputePack(ctx, "", dur, t.params.GPUIndices, rampPlan.StaggerSeconds, j.append) - case "nvidia-targeted-power": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - dur := t.params.Duration - if t.params.BurnProfile != "" && dur <= 0 { - dur = resolveBurnPreset(t.params.BurnProfile).DurationSec - } - archive, err = a.RunNvidiaTargetedPowerPack(ctx, "", dur, t.params.GPUIndices, j.append) - case "nvidia-pulse": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - dur := t.params.Duration - if t.params.BurnProfile != "" && dur <= 0 { - dur = resolveBurnPreset(t.params.BurnProfile).DurationSec - } - archive, err = a.RunNvidiaPulseTestPack(ctx, "", dur, t.params.GPUIndices, j.append) - case "nvidia-bandwidth": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - archive, err = a.RunNvidiaBandwidthPack(ctx, "", t.params.GPUIndices, j.append) - case "nvidia-interconnect": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - archive, err = a.RunNCCLTests(ctx, "", t.params.GPUIndices, j.append) - case "scenario": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - var data []byte - data, err = a.ReadScenario(t.params.ScenarioName) - if err != nil { - break - } - var spec platform.ScenarioSpec - spec, err = platform.ParseScenarioJSON(data) - if err != nil { - break - } - archive, err = a.RunScenario(ctx, "", spec, j.append) - case "nvidia-stress": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - dur := t.params.Duration - if t.params.BurnProfile != "" && dur <= 0 { - dur = resolveBurnPreset(t.params.BurnProfile).DurationSec - } - rampPlan, planErr := resolveNvidiaRampPlan(t.params.BurnProfile, t.params.StaggerGPUStart, t.params.GPUIndices) - if planErr != nil { - err = planErr - break - } - if t.params.BurnProfile != "" && t.params.StaggerGPUStart && dur <= 0 { - dur = rampPlan.DurationSec - } - if rampPlan.StaggerSeconds > 0 { - j.append(fmt.Sprintf("NVIDIA staggered ramp-up enabled: %ds per GPU; post-ramp hold: %ds; total runtime: %ds", rampPlan.StaggerSeconds, dur, rampPlan.TotalDurationSec)) - } - archive, err = runNvidiaStressPackCtx(a, ctx, "", platform.NvidiaStressOptions{ - DurationSec: dur, - Loader: t.params.Loader, - GPUIndices: t.params.GPUIndices, - ExcludeGPUIndices: t.params.ExcludeGPUIndices, - StaggerSeconds: rampPlan.StaggerSeconds, - }, j.append) - case "memory": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - sizeMB, passes := resolveMemoryValidatePreset(t.params.BurnProfile, t.params.StressMode) - j.append(fmt.Sprintf("Memory validate preset: %d MB x %d pass(es)", sizeMB, passes)) - archive, err = runMemoryAcceptancePackCtx(a, ctx, "", sizeMB, passes, j.append) - case "storage": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - archive, err = runStorageAcceptancePackCtx(a, ctx, "", t.params.StressMode, j.append) - case "nvidia-config": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - archive, err = runNvidiaConfigCheckPackCtx(a, ctx, "", j.append) - case "cpu": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - dur := t.params.Duration - if t.params.BurnProfile != "" && dur <= 0 { - dur = resolveBurnPreset(t.params.BurnProfile).DurationSec - } - if dur <= 0 { - if t.params.StressMode { - dur = 1800 - } else { - dur = 60 - } - } - j.append(fmt.Sprintf("CPU stress duration: %ds", dur)) - archive, err = runCPUAcceptancePackCtx(a, ctx, "", dur, j.append) - case "amd": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - archive, err = runAMDAcceptancePackCtx(a, ctx, "", j.append) - case "amd-mem": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - archive, err = runAMDMemIntegrityPackCtx(a, ctx, "", j.append) - case "amd-bandwidth": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - archive, err = runAMDMemBandwidthPackCtx(a, ctx, "", j.append) - case "amd-stress": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - dur := t.params.Duration - if t.params.BurnProfile != "" && dur <= 0 { - dur = resolveBurnPreset(t.params.BurnProfile).DurationSec - } - archive, err = runAMDStressPackCtx(a, ctx, "", dur, j.append) - case "memory-stress": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - dur := t.params.Duration - if t.params.BurnProfile != "" && dur <= 0 { - dur = resolveBurnPreset(t.params.BurnProfile).DurationSec - } - archive, err = runMemoryStressPackCtx(a, ctx, "", dur, j.append) - case "sat-stress": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - dur := t.params.Duration - if t.params.BurnProfile != "" && dur <= 0 { - dur = resolveBurnPreset(t.params.BurnProfile).DurationSec - } - archive, err = runSATStressPackCtx(a, ctx, "", dur, j.append) - case "platform-stress": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - opts := resolvePlatformStressPreset(t.params.BurnProfile) - opts.Components = t.params.PlatformComponents - archive, err = a.RunPlatformStress(ctx, "", opts, j.append) - case "audit": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - result, e := a.RunAuditNow(q.opts.RuntimeMode) - if e != nil { - err = e - } else { - for _, line := range splitLines(result.Body) { - j.append(line) - } - } - case "support-bundle": - j.append("Building support bundle...") - archive, err = buildSupportBundle(q.opts.ExportDir) - case "install": - if strings.TrimSpace(t.params.Device) == "" { - err = fmt.Errorf("device is required") - break - } - installLogPath := platform.InstallLogPath(t.params.Device) - j.append("Install log: " + installLogPath) - err = streamCmdJob(j, installCommand(ctx, t.params.Device, installLogPath)) - case "install-to-ram": - if a == nil { - err = fmt.Errorf("app not configured") - break - } - err = a.RunInstallToRAM(ctx, j.append) - default: - j.append("ERROR: unknown target: " + t.Target) - j.finish("unknown target") - return - } - - // If the SAT archive was produced, check overall_status and write to component DB. - if archive != "" { - archivePath := app.ExtractArchivePath(archive) - if err == nil { - if app.ReadSATOverallStatus(archivePath) == "FAILED" { - if reason := app.SATFailureDetail(archivePath); reason != "" { - err = fmt.Errorf("SAT FAILED: %s", reason) - } else { - err = fmt.Errorf("SAT overall_status=FAILED (see summary.txt)") - } - } - } - // A user-aborted run (ctx canceled) may still have produced a partial - // archive/summary.txt — that incomplete result must not overwrite the - // component status DB, which is why this is skipped here rather than - // relying on satKeyStatus's PARTIAL/UNSUPPORTED handling below. - if db := q.statusDB(); db != nil && ctx.Err() == nil { - app.ApplySATResultToDB(db, t.Target, archivePath) - } - } - - if err != nil { - if ctx.Err() != nil { - j.append("Aborted.") - j.finish("aborted") - } else { - j.append("ERROR: " + err.Error()) - j.finish(err.Error()) - } - return - } - if archive != "" { - j.append("Archive: " + archive) - } - j.finish("") -} - -func (q *taskQueue) statusDB() *app.ComponentStatusDB { - if q.opts == nil || q.opts.App == nil { - return nil - } - return q.opts.App.StatusDB + executeTaskWithOptions(q.opts, t, j, ctx) } func splitLines(s string) []string {