Adjust burn GPU ramp timing by profile
This commit is contained in:
@@ -1342,11 +1342,11 @@ func renderValidate(opts HandlerOptions) string {
|
|||||||
|
|
||||||
<div class="grid3">
|
<div class="grid3">
|
||||||
` + renderSATCard("nvidia", "NVIDIA GPU", "runNvidiaValidateSet('nvidia')", "", renderValidateCardBody(
|
` + renderSATCard("nvidia", "NVIDIA GPU", "runNvidiaValidateSet('nvidia')", "", renderValidateCardBody(
|
||||||
inv.NVIDIA,
|
inv.NVIDIA,
|
||||||
`Runs NVIDIA diagnostics and board inventory checks.`,
|
`Runs NVIDIA diagnostics and board inventory checks.`,
|
||||||
`<code>nvidia-smi</code>, <code>dmidecode</code>, <code>dcgmi diag</code>`,
|
`<code>nvidia-smi</code>, <code>dmidecode</code>, <code>dcgmi diag</code>`,
|
||||||
`Level 2 in Validate, Level 3 in Stress. Runs one GPU at a time on the selected NVIDIA GPUs.`,
|
`Level 2 in Validate, Level 3 in Stress. Runs one GPU at a time on the selected NVIDIA GPUs.`,
|
||||||
)) +
|
)) +
|
||||||
`<div id="sat-card-nvidia-targeted-stress">` +
|
`<div id="sat-card-nvidia-targeted-stress">` +
|
||||||
renderSATCard("nvidia-targeted-stress", "NVIDIA GPU Targeted Stress", "runNvidiaValidateSet('nvidia-targeted-stress')", "", renderValidateCardBody(
|
renderSATCard("nvidia-targeted-stress", "NVIDIA GPU Targeted Stress", "runNvidiaValidateSet('nvidia-targeted-stress')", "", renderValidateCardBody(
|
||||||
inv.NVIDIA,
|
inv.NVIDIA,
|
||||||
@@ -2373,7 +2373,7 @@ func renderBurn() string {
|
|||||||
<p id="burn-selection-note" style="font-size:12px;color:var(--muted);margin:10px 0 0">Select at least one NVIDIA GPU to enable NVIDIA burn recipes.</p>
|
<p id="burn-selection-note" style="font-size:12px;color:var(--muted);margin:10px 0 0">Select at least one NVIDIA GPU to enable NVIDIA burn recipes.</p>
|
||||||
<label class="cb-row" style="margin-top:10px">
|
<label class="cb-row" style="margin-top:10px">
|
||||||
<input type="checkbox" id="burn-stagger-nvidia">
|
<input type="checkbox" id="burn-stagger-nvidia">
|
||||||
<span>Ramp selected NVIDIA GPUs one by one before full-load hold. Uses a 3-minute stabilization window per GPU, then keeps all selected GPUs under load for the chosen Burn Profile duration.</span>
|
<span>Ramp selected NVIDIA GPUs one by one before the full-load hold. Smoke: +2 min per GPU, then 5 min with all selected GPUs under load. Acceptance: +10 min per GPU, then at least 1 hour with all selected GPUs under load. Overnight: +1 hour per GPU, then at least 1 hour with all selected GPUs under load, capped at 10 hours total.</span>
|
||||||
</label>
|
</label>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -3108,7 +3108,6 @@ usbRefresh();
|
|||||||
</script>`
|
</script>`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func renderNvidiaSelfHealInline() string {
|
func renderNvidiaSelfHealInline() string {
|
||||||
return `<p style="font-size:13px;color:var(--muted);margin-bottom:12px">Inspect NVIDIA GPU health, restart the bee-nvidia driver service, and issue a per-GPU reset when the driver reports reset required.</p>
|
return `<p style="font-size:13px;color:var(--muted);margin-bottom:12px">Inspect NVIDIA GPU health, restart the bee-nvidia driver service, and issue a per-GPU reset when the driver reports reset required.</p>
|
||||||
<div style="display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px">
|
<div style="display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px">
|
||||||
|
|||||||
@@ -152,6 +152,12 @@ type burnPreset struct {
|
|||||||
DurationSec int
|
DurationSec int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type nvidiaRampSpec struct {
|
||||||
|
DurationSec int
|
||||||
|
StaggerSeconds int
|
||||||
|
TotalDurationSec int
|
||||||
|
}
|
||||||
|
|
||||||
func resolveBurnPreset(profile string) burnPreset {
|
func resolveBurnPreset(profile string) burnPreset {
|
||||||
switch profile {
|
switch profile {
|
||||||
case "overnight":
|
case "overnight":
|
||||||
@@ -163,11 +169,43 @@ func resolveBurnPreset(profile string) burnPreset {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func boolToNvidiaStaggerSeconds(enabled bool, selected []int) int {
|
func resolveNvidiaRampPlan(profile string, enabled bool, selected []int) (nvidiaRampSpec, error) {
|
||||||
if enabled && len(selected) > 1 {
|
base := resolveBurnPreset(profile).DurationSec
|
||||||
return 180
|
plan := nvidiaRampSpec{
|
||||||
|
DurationSec: base,
|
||||||
|
TotalDurationSec: base,
|
||||||
}
|
}
|
||||||
return 0
|
if !enabled {
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
|
count := len(selected)
|
||||||
|
if count == 0 {
|
||||||
|
return nvidiaRampSpec{}, fmt.Errorf("staggered NVIDIA burn requires explicit GPU selection")
|
||||||
|
}
|
||||||
|
if count == 1 {
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
switch profile {
|
||||||
|
case "acceptance":
|
||||||
|
plan.StaggerSeconds = 10 * 60
|
||||||
|
plan.TotalDurationSec = plan.DurationSec + plan.StaggerSeconds*(count-1)
|
||||||
|
case "overnight":
|
||||||
|
plan.StaggerSeconds = 60 * 60
|
||||||
|
plan.TotalDurationSec = 8 * 60 * 60
|
||||||
|
minTotal := count * 60 * 60
|
||||||
|
if plan.TotalDurationSec < minTotal {
|
||||||
|
plan.TotalDurationSec = minTotal
|
||||||
|
}
|
||||||
|
if plan.TotalDurationSec > 10*60*60 {
|
||||||
|
return nvidiaRampSpec{}, fmt.Errorf("overnight staggered NVIDIA burn supports at most 10 GPUs")
|
||||||
|
}
|
||||||
|
plan.DurationSec = plan.TotalDurationSec - plan.StaggerSeconds*(count-1)
|
||||||
|
default:
|
||||||
|
plan.StaggerSeconds = 2 * 60
|
||||||
|
plan.TotalDurationSec = plan.DurationSec + plan.StaggerSeconds*(count-1)
|
||||||
|
}
|
||||||
|
return plan, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolvePlatformStressPreset(profile string) platform.PlatformStressOptions {
|
func resolvePlatformStressPreset(profile string) platform.PlatformStressOptions {
|
||||||
@@ -600,7 +638,7 @@ func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) {
|
|||||||
RunNCCL: t.params.RunNCCL,
|
RunNCCL: t.params.RunNCCL,
|
||||||
ParallelGPUs: t.params.ParallelGPUs,
|
ParallelGPUs: t.params.ParallelGPUs,
|
||||||
}, j.append)
|
}, j.append)
|
||||||
case "nvidia-compute":
|
case "nvidia-compute":
|
||||||
if a == nil {
|
if a == nil {
|
||||||
err = fmt.Errorf("app not configured")
|
err = fmt.Errorf("app not configured")
|
||||||
break
|
break
|
||||||
@@ -609,11 +647,18 @@ func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) {
|
|||||||
if t.params.BurnProfile != "" && dur <= 0 {
|
if t.params.BurnProfile != "" && dur <= 0 {
|
||||||
dur = resolveBurnPreset(t.params.BurnProfile).DurationSec
|
dur = resolveBurnPreset(t.params.BurnProfile).DurationSec
|
||||||
}
|
}
|
||||||
staggerSec := boolToNvidiaStaggerSeconds(t.params.StaggerGPUStart, t.params.GPUIndices)
|
rampPlan, planErr := resolveNvidiaRampPlan(t.params.BurnProfile, t.params.StaggerGPUStart, t.params.GPUIndices)
|
||||||
if staggerSec > 0 {
|
if planErr != nil {
|
||||||
j.append(fmt.Sprintf("NVIDIA staggered ramp-up enabled: %ds per GPU", staggerSec))
|
err = planErr
|
||||||
}
|
break
|
||||||
archive, err = a.RunNvidiaOfficialComputePack(ctx, "", dur, t.params.GPUIndices, staggerSec, j.append)
|
}
|
||||||
|
if t.params.BurnProfile != "" && t.params.StaggerGPUStart && dur <= 0 {
|
||||||
|
dur = rampPlan.DurationSec
|
||||||
|
}
|
||||||
|
if rampPlan.StaggerSeconds > 0 {
|
||||||
|
j.append(fmt.Sprintf("NVIDIA staggered ramp-up enabled: %ds per GPU; post-ramp hold: %ds; total runtime: %ds", rampPlan.StaggerSeconds, dur, rampPlan.TotalDurationSec))
|
||||||
|
}
|
||||||
|
archive, err = a.RunNvidiaOfficialComputePack(ctx, "", dur, t.params.GPUIndices, rampPlan.StaggerSeconds, j.append)
|
||||||
case "nvidia-targeted-power":
|
case "nvidia-targeted-power":
|
||||||
if a == nil {
|
if a == nil {
|
||||||
err = fmt.Errorf("app not configured")
|
err = fmt.Errorf("app not configured")
|
||||||
@@ -663,13 +708,24 @@ func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) {
|
|||||||
if t.params.BurnProfile != "" && dur <= 0 {
|
if t.params.BurnProfile != "" && dur <= 0 {
|
||||||
dur = resolveBurnPreset(t.params.BurnProfile).DurationSec
|
dur = resolveBurnPreset(t.params.BurnProfile).DurationSec
|
||||||
}
|
}
|
||||||
archive, err = runNvidiaStressPackCtx(a, ctx, "", platform.NvidiaStressOptions{
|
rampPlan, planErr := resolveNvidiaRampPlan(t.params.BurnProfile, t.params.StaggerGPUStart, t.params.GPUIndices)
|
||||||
DurationSec: dur,
|
if planErr != nil {
|
||||||
Loader: t.params.Loader,
|
err = planErr
|
||||||
GPUIndices: t.params.GPUIndices,
|
break
|
||||||
ExcludeGPUIndices: t.params.ExcludeGPUIndices,
|
}
|
||||||
StaggerSeconds: boolToNvidiaStaggerSeconds(t.params.StaggerGPUStart, t.params.GPUIndices),
|
if t.params.BurnProfile != "" && t.params.StaggerGPUStart && dur <= 0 {
|
||||||
}, j.append)
|
dur = rampPlan.DurationSec
|
||||||
|
}
|
||||||
|
if rampPlan.StaggerSeconds > 0 {
|
||||||
|
j.append(fmt.Sprintf("NVIDIA staggered ramp-up enabled: %ds per GPU; post-ramp hold: %ds; total runtime: %ds", rampPlan.StaggerSeconds, dur, rampPlan.TotalDurationSec))
|
||||||
|
}
|
||||||
|
archive, err = runNvidiaStressPackCtx(a, ctx, "", platform.NvidiaStressOptions{
|
||||||
|
DurationSec: dur,
|
||||||
|
Loader: t.params.Loader,
|
||||||
|
GPUIndices: t.params.GPUIndices,
|
||||||
|
ExcludeGPUIndices: t.params.ExcludeGPUIndices,
|
||||||
|
StaggerSeconds: rampPlan.StaggerSeconds,
|
||||||
|
}, j.append)
|
||||||
case "memory":
|
case "memory":
|
||||||
if a == nil {
|
if a == nil {
|
||||||
err = fmt.Errorf("app not configured")
|
err = fmt.Errorf("app not configured")
|
||||||
|
|||||||
@@ -491,6 +491,83 @@ func TestResolveBurnPreset(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolveNvidiaRampPlan(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
profile string
|
||||||
|
enabled bool
|
||||||
|
selected []int
|
||||||
|
want nvidiaRampSpec
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "disabled uses base preset",
|
||||||
|
profile: "acceptance",
|
||||||
|
selected: []int{0, 1},
|
||||||
|
want: nvidiaRampSpec{DurationSec: 60 * 60, TotalDurationSec: 60 * 60},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "smoke ramp uses two minute steps",
|
||||||
|
profile: "smoke",
|
||||||
|
enabled: true,
|
||||||
|
selected: []int{0, 1, 2},
|
||||||
|
want: nvidiaRampSpec{DurationSec: 5 * 60, StaggerSeconds: 2 * 60, TotalDurationSec: 9 * 60},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "acceptance ramp uses ten minute steps",
|
||||||
|
profile: "acceptance",
|
||||||
|
enabled: true,
|
||||||
|
selected: []int{0, 1, 2},
|
||||||
|
want: nvidiaRampSpec{DurationSec: 60 * 60, StaggerSeconds: 10 * 60, TotalDurationSec: 80 * 60},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "overnight stays at eight hours when possible",
|
||||||
|
profile: "overnight",
|
||||||
|
enabled: true,
|
||||||
|
selected: []int{0, 1, 2},
|
||||||
|
want: nvidiaRampSpec{DurationSec: 6 * 60 * 60, StaggerSeconds: 60 * 60, TotalDurationSec: 8 * 60 * 60},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "overnight extends to keep one hour after final gpu",
|
||||||
|
profile: "overnight",
|
||||||
|
enabled: true,
|
||||||
|
selected: []int{0, 1, 2, 3, 4, 5, 6, 7, 8},
|
||||||
|
want: nvidiaRampSpec{DurationSec: 60 * 60, StaggerSeconds: 60 * 60, TotalDurationSec: 9 * 60 * 60},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "overnight rejects impossible gpu count",
|
||||||
|
profile: "overnight",
|
||||||
|
enabled: true,
|
||||||
|
selected: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
|
||||||
|
wantErr: "at most 10 GPUs",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "enabled requires explicit selection",
|
||||||
|
profile: "smoke",
|
||||||
|
enabled: true,
|
||||||
|
wantErr: "requires explicit GPU selection",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got, err := resolveNvidiaRampPlan(tc.profile, tc.enabled, tc.selected)
|
||||||
|
if tc.wantErr != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
|
||||||
|
t.Fatalf("err=%v want substring %q", err, tc.wantErr)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveNvidiaRampPlan error: %v", err)
|
||||||
|
}
|
||||||
|
if got != tc.want {
|
||||||
|
t.Fatalf("resolveNvidiaRampPlan(%q, %t, %v)=%+v want %+v", tc.profile, tc.enabled, tc.selected, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestTaskDisplayNameUsesNvidiaStressLoader(t *testing.T) {
|
func TestTaskDisplayNameUsesNvidiaStressLoader(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
loader string
|
loader string
|
||||||
|
|||||||
Reference in New Issue
Block a user