Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e16d0f34b5 | |||
|
|
525ed8b8fc | ||
|
|
4f94ebcb2c |
@@ -121,15 +121,22 @@ func (s *System) RunNvidiaBenchmark(ctx context.Context, baseDir string, opts Nv
|
||||
var serverIdleOK, serverLoadedOK bool
|
||||
var serverLoadedSamples int
|
||||
|
||||
// Run nvidia-smi -q first: used both for the log file and as a fallback
|
||||
// source of max clock values when CSV clock fields are unsupported.
|
||||
var nvsmiQOut []byte
|
||||
if out, err := runSATCommandCtx(ctx, verboseLog, "00-nvidia-smi-q.log", []string{"nvidia-smi", "-q"}, nil, nil); err == nil {
|
||||
nvsmiQOut = out
|
||||
_ = os.WriteFile(filepath.Join(runDir, "00-nvidia-smi-q.log"), out, 0644)
|
||||
}
|
||||
|
||||
infoByIndex, infoErr := queryBenchmarkGPUInfo(selected)
|
||||
if infoErr != nil {
|
||||
result.Warnings = append(result.Warnings, "gpu inventory query failed: "+infoErr.Error())
|
||||
result.Normalization.Status = "partial"
|
||||
}
|
||||
|
||||
if out, err := runSATCommandCtx(ctx, verboseLog, "00-nvidia-smi-q.log", []string{"nvidia-smi", "-q"}, nil, nil); err == nil {
|
||||
_ = os.WriteFile(filepath.Join(runDir, "00-nvidia-smi-q.log"), out, 0644)
|
||||
}
|
||||
// Enrich with max clocks from verbose output — covers GPUs where
|
||||
// clocks.max.* CSV fields are unsupported (e.g. Blackwell / driver 98.x).
|
||||
enrichGPUInfoWithMaxClocks(infoByIndex, nvsmiQOut)
|
||||
|
||||
activeApps, err := queryActiveComputeApps(selected)
|
||||
if err == nil && len(activeApps) > 0 {
|
||||
@@ -370,9 +377,13 @@ func resolveBenchmarkProfile(profile string) benchmarkProfileSpec {
|
||||
// Fields are tried in order; the first successful query wins. Extended fields
|
||||
// (attribute.multiprocessor_count, power.default_limit) are not supported on
|
||||
// all driver versions, so we fall back to the base set if the full query fails.
|
||||
// The minimal fallback omits clock fields entirely — clocks.max.* returns
|
||||
// exit status 2 on some GPU generations (e.g. Blackwell); max clocks are
|
||||
// then recovered from nvidia-smi -q via enrichGPUInfoWithMaxClocks.
|
||||
var benchmarkGPUInfoQueries = []struct {
|
||||
fields string
|
||||
extended bool // whether this query includes optional extended fields
|
||||
minimal bool // clock fields omitted; max clocks must be filled separately
|
||||
}{
|
||||
{
|
||||
fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics,attribute.multiprocessor_count,power.default_limit",
|
||||
@@ -382,6 +393,83 @@ var benchmarkGPUInfoQueries = []struct {
|
||||
fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics",
|
||||
extended: false,
|
||||
},
|
||||
{
|
||||
fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit",
|
||||
minimal: true,
|
||||
},
|
||||
}
|
||||
|
||||
// enrichGPUInfoWithMaxClocks fills MaxGraphicsClockMHz / MaxMemoryClockMHz for
|
||||
// any GPU in infoByIndex where those values are still zero. It parses the
|
||||
// "Max Clocks" section of nvidia-smi -q output (already available as nvsmiQ).
|
||||
// This is the fallback for GPUs (e.g. Blackwell) where clocks.max.* CSV fields
|
||||
// return exit status 2 but the verbose query works fine.
|
||||
func enrichGPUInfoWithMaxClocks(infoByIndex map[int]benchmarkGPUInfo, nvsmiQ []byte) {
|
||||
if len(infoByIndex) == 0 || len(nvsmiQ) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Build bus_id → index map for matching verbose sections to GPU indices.
|
||||
busToBenchIdx := make(map[string]int, len(infoByIndex))
|
||||
for idx, info := range infoByIndex {
|
||||
if info.BusID != "" {
|
||||
// nvidia-smi -q uses "GPU 00000000:4E:00.0" (8-digit domain),
|
||||
// while --query-gpu returns the same format; normalise to lower.
|
||||
busToBenchIdx[strings.ToLower(strings.TrimSpace(info.BusID))] = idx
|
||||
}
|
||||
}
|
||||
|
||||
// Split the verbose output into per-GPU sections on "^GPU " lines.
|
||||
gpuSectionRe := regexp.MustCompile(`(?m)^GPU\s+([\dA-Fa-f:\.]+)`)
|
||||
maxGfxRe := regexp.MustCompile(`(?i)Max Clocks[\s\S]*?Graphics\s*:\s*(\d+)\s*MHz`)
|
||||
maxMemRe := regexp.MustCompile(`(?i)Max Clocks[\s\S]*?Memory\s*:\s*(\d+)\s*MHz`)
|
||||
|
||||
sectionStarts := gpuSectionRe.FindAllSubmatchIndex(nvsmiQ, -1)
|
||||
for i, loc := range sectionStarts {
|
||||
busID := strings.ToLower(string(nvsmiQ[loc[2]:loc[3]]))
|
||||
benchIdx, ok := busToBenchIdx[busID]
|
||||
if !ok {
|
||||
// Bus IDs from verbose output may have a different domain prefix;
|
||||
// try suffix match on the slot portion (XX:XX.X).
|
||||
for k, v := range busToBenchIdx {
|
||||
if strings.HasSuffix(k, busID) || strings.HasSuffix(busID, k) {
|
||||
benchIdx = v
|
||||
ok = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
info := infoByIndex[benchIdx]
|
||||
if info.MaxGraphicsClockMHz > 0 && info.MaxMemoryClockMHz > 0 {
|
||||
continue // already populated
|
||||
}
|
||||
|
||||
end := len(nvsmiQ)
|
||||
if i+1 < len(sectionStarts) {
|
||||
end = sectionStarts[i+1][0]
|
||||
}
|
||||
section := nvsmiQ[loc[0]:end]
|
||||
|
||||
if info.MaxGraphicsClockMHz == 0 {
|
||||
if m := maxGfxRe.FindSubmatch(section); m != nil {
|
||||
if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil {
|
||||
info.MaxGraphicsClockMHz = v
|
||||
}
|
||||
}
|
||||
}
|
||||
if info.MaxMemoryClockMHz == 0 {
|
||||
if m := maxMemRe.FindSubmatch(section); m != nil {
|
||||
if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil {
|
||||
info.MaxMemoryClockMHz = v
|
||||
}
|
||||
}
|
||||
}
|
||||
infoByIndex[benchIdx] = info
|
||||
}
|
||||
}
|
||||
|
||||
func queryBenchmarkGPUInfo(gpuIndices []int) (map[int]benchmarkGPUInfo, error) {
|
||||
@@ -409,9 +497,13 @@ func queryBenchmarkGPUInfo(gpuIndices []int) (map[int]benchmarkGPUInfo, error) {
|
||||
continue
|
||||
}
|
||||
|
||||
minFields := 6
|
||||
if !q.minimal {
|
||||
minFields = 9
|
||||
}
|
||||
infoByIndex := make(map[int]benchmarkGPUInfo, len(rows))
|
||||
for _, row := range rows {
|
||||
if len(row) < 9 {
|
||||
if len(row) < minFields {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(strings.TrimSpace(row[0]))
|
||||
@@ -419,24 +511,26 @@ func queryBenchmarkGPUInfo(gpuIndices []int) (map[int]benchmarkGPUInfo, error) {
|
||||
continue
|
||||
}
|
||||
info := benchmarkGPUInfo{
|
||||
Index: idx,
|
||||
UUID: strings.TrimSpace(row[1]),
|
||||
Name: strings.TrimSpace(row[2]),
|
||||
BusID: strings.TrimSpace(row[3]),
|
||||
VBIOS: strings.TrimSpace(row[4]),
|
||||
PowerLimitW: parseBenchmarkFloat(row[5]),
|
||||
MaxGraphicsClockMHz: parseBenchmarkFloat(row[6]),
|
||||
MaxMemoryClockMHz: parseBenchmarkFloat(row[7]),
|
||||
Index: idx,
|
||||
UUID: strings.TrimSpace(row[1]),
|
||||
Name: strings.TrimSpace(row[2]),
|
||||
BusID: strings.TrimSpace(row[3]),
|
||||
VBIOS: strings.TrimSpace(row[4]),
|
||||
PowerLimitW: parseBenchmarkFloat(row[5]),
|
||||
}
|
||||
if len(row) >= 9 {
|
||||
info.BaseGraphicsClockMHz = parseBenchmarkFloat(row[8])
|
||||
}
|
||||
if q.extended {
|
||||
if len(row) >= 10 {
|
||||
info.MultiprocessorCount = int(parseBenchmarkFloat(row[9]))
|
||||
if !q.minimal {
|
||||
info.MaxGraphicsClockMHz = parseBenchmarkFloat(row[6])
|
||||
info.MaxMemoryClockMHz = parseBenchmarkFloat(row[7])
|
||||
if len(row) >= 9 {
|
||||
info.BaseGraphicsClockMHz = parseBenchmarkFloat(row[8])
|
||||
}
|
||||
if len(row) >= 11 {
|
||||
info.DefaultPowerLimitW = parseBenchmarkFloat(row[10])
|
||||
if q.extended {
|
||||
if len(row) >= 10 {
|
||||
info.MultiprocessorCount = int(parseBenchmarkFloat(row[9]))
|
||||
}
|
||||
if len(row) >= 11 {
|
||||
info.DefaultPowerLimitW = parseBenchmarkFloat(row[10])
|
||||
}
|
||||
}
|
||||
}
|
||||
infoByIndex[idx] = info
|
||||
|
||||
@@ -178,3 +178,67 @@ func TestRenderBenchmarkReportIncludesTerminalChartsWithoutANSI(t *testing.T) {
|
||||
t.Fatalf("report should not contain ANSI escapes\n%s", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichGPUInfoWithMaxClocks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
nvsmiQ := []byte(`
|
||||
GPU 00000000:4E:00.0
|
||||
Product Name : NVIDIA RTX PRO 6000 Blackwell Server Edition
|
||||
Clocks
|
||||
Graphics : 2422 MHz
|
||||
Memory : 12481 MHz
|
||||
Max Clocks
|
||||
Graphics : 2430 MHz
|
||||
SM : 2430 MHz
|
||||
Memory : 12481 MHz
|
||||
Video : 2107 MHz
|
||||
|
||||
GPU 00000000:4F:00.0
|
||||
Product Name : NVIDIA RTX PRO 6000 Blackwell Server Edition
|
||||
Max Clocks
|
||||
Graphics : 2430 MHz
|
||||
Memory : 12481 MHz
|
||||
`)
|
||||
|
||||
infoByIndex := map[int]benchmarkGPUInfo{
|
||||
0: {Index: 0, BusID: "00000000:4E:00.0"},
|
||||
1: {Index: 1, BusID: "00000000:4F:00.0"},
|
||||
}
|
||||
|
||||
enrichGPUInfoWithMaxClocks(infoByIndex, nvsmiQ)
|
||||
|
||||
if infoByIndex[0].MaxGraphicsClockMHz != 2430 {
|
||||
t.Errorf("GPU 0 MaxGraphicsClockMHz = %v, want 2430", infoByIndex[0].MaxGraphicsClockMHz)
|
||||
}
|
||||
if infoByIndex[0].MaxMemoryClockMHz != 12481 {
|
||||
t.Errorf("GPU 0 MaxMemoryClockMHz = %v, want 12481", infoByIndex[0].MaxMemoryClockMHz)
|
||||
}
|
||||
if infoByIndex[1].MaxGraphicsClockMHz != 2430 {
|
||||
t.Errorf("GPU 1 MaxGraphicsClockMHz = %v, want 2430", infoByIndex[1].MaxGraphicsClockMHz)
|
||||
}
|
||||
if infoByIndex[1].MaxMemoryClockMHz != 12481 {
|
||||
t.Errorf("GPU 1 MaxMemoryClockMHz = %v, want 12481", infoByIndex[1].MaxMemoryClockMHz)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichGPUInfoWithMaxClocksSkipsPopulated(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
nvsmiQ := []byte(`
|
||||
GPU 00000000:4E:00.0
|
||||
Max Clocks
|
||||
Graphics : 9999 MHz
|
||||
Memory : 9999 MHz
|
||||
`)
|
||||
// Already populated — must not be overwritten.
|
||||
infoByIndex := map[int]benchmarkGPUInfo{
|
||||
0: {Index: 0, BusID: "00000000:4E:00.0", MaxGraphicsClockMHz: 2430, MaxMemoryClockMHz: 12481},
|
||||
}
|
||||
|
||||
enrichGPUInfoWithMaxClocks(infoByIndex, nvsmiQ)
|
||||
|
||||
if infoByIndex[0].MaxGraphicsClockMHz != 2430 {
|
||||
t.Errorf("expected existing value to be preserved, got %v", infoByIndex[0].MaxGraphicsClockMHz)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1342,11 +1342,11 @@ func renderValidate(opts HandlerOptions) string {
|
||||
|
||||
<div class="grid3">
|
||||
` + renderSATCard("nvidia", "NVIDIA GPU", "runNvidiaValidateSet('nvidia')", "", renderValidateCardBody(
|
||||
inv.NVIDIA,
|
||||
`Runs NVIDIA diagnostics and board inventory checks.`,
|
||||
`<code>nvidia-smi</code>, <code>dmidecode</code>, <code>dcgmi diag</code>`,
|
||||
`Level 2 in Validate, Level 3 in Stress. Runs one GPU at a time on the selected NVIDIA GPUs.`,
|
||||
)) +
|
||||
inv.NVIDIA,
|
||||
`Runs NVIDIA diagnostics and board inventory checks.`,
|
||||
`<code>nvidia-smi</code>, <code>dmidecode</code>, <code>dcgmi diag</code>`,
|
||||
`Level 2 in Validate, Level 3 in Stress. Runs one GPU at a time on the selected NVIDIA GPUs.`,
|
||||
)) +
|
||||
`<div id="sat-card-nvidia-targeted-stress">` +
|
||||
renderSATCard("nvidia-targeted-stress", "NVIDIA GPU Targeted Stress", "runNvidiaValidateSet('nvidia-targeted-stress')", "", renderValidateCardBody(
|
||||
inv.NVIDIA,
|
||||
@@ -2373,7 +2373,7 @@ func renderBurn() string {
|
||||
<p id="burn-selection-note" style="font-size:12px;color:var(--muted);margin:10px 0 0">Select at least one NVIDIA GPU to enable NVIDIA burn recipes.</p>
|
||||
<label class="cb-row" style="margin-top:10px">
|
||||
<input type="checkbox" id="burn-stagger-nvidia">
|
||||
<span>Ramp selected NVIDIA GPUs one by one before full-load hold. Uses a 3-minute stabilization window per GPU, then keeps all selected GPUs under load for the chosen Burn Profile duration.</span>
|
||||
<span>Ramp selected NVIDIA GPUs one by one before the full-load hold. Smoke: +2 min per GPU, then 5 min with all selected GPUs under load. Acceptance: +10 min per GPU, then at least 1 hour with all selected GPUs under load. Overnight: +1 hour per GPU, then at least 1 hour with all selected GPUs under load, capped at 10 hours total.</span>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
@@ -3108,7 +3108,6 @@ usbRefresh();
|
||||
</script>`
|
||||
}
|
||||
|
||||
|
||||
func renderNvidiaSelfHealInline() string {
|
||||
return `<p style="font-size:13px;color:var(--muted);margin-bottom:12px">Inspect NVIDIA GPU health, restart the bee-nvidia driver service, and issue a per-GPU reset when the driver reports reset required.</p>
|
||||
<div style="display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px">
|
||||
|
||||
@@ -152,6 +152,12 @@ type burnPreset struct {
|
||||
DurationSec int
|
||||
}
|
||||
|
||||
type nvidiaRampSpec struct {
|
||||
DurationSec int
|
||||
StaggerSeconds int
|
||||
TotalDurationSec int
|
||||
}
|
||||
|
||||
func resolveBurnPreset(profile string) burnPreset {
|
||||
switch profile {
|
||||
case "overnight":
|
||||
@@ -163,11 +169,43 @@ func resolveBurnPreset(profile string) burnPreset {
|
||||
}
|
||||
}
|
||||
|
||||
func boolToNvidiaStaggerSeconds(enabled bool, selected []int) int {
|
||||
if enabled && len(selected) > 1 {
|
||||
return 180
|
||||
func resolveNvidiaRampPlan(profile string, enabled bool, selected []int) (nvidiaRampSpec, error) {
|
||||
base := resolveBurnPreset(profile).DurationSec
|
||||
plan := nvidiaRampSpec{
|
||||
DurationSec: base,
|
||||
TotalDurationSec: base,
|
||||
}
|
||||
return 0
|
||||
if !enabled {
|
||||
return plan, nil
|
||||
}
|
||||
count := len(selected)
|
||||
if count == 0 {
|
||||
return nvidiaRampSpec{}, fmt.Errorf("staggered NVIDIA burn requires explicit GPU selection")
|
||||
}
|
||||
if count == 1 {
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
switch profile {
|
||||
case "acceptance":
|
||||
plan.StaggerSeconds = 10 * 60
|
||||
plan.TotalDurationSec = plan.DurationSec + plan.StaggerSeconds*(count-1)
|
||||
case "overnight":
|
||||
plan.StaggerSeconds = 60 * 60
|
||||
plan.TotalDurationSec = 8 * 60 * 60
|
||||
minTotal := count * 60 * 60
|
||||
if plan.TotalDurationSec < minTotal {
|
||||
plan.TotalDurationSec = minTotal
|
||||
}
|
||||
if plan.TotalDurationSec > 10*60*60 {
|
||||
return nvidiaRampSpec{}, fmt.Errorf("overnight staggered NVIDIA burn supports at most 10 GPUs")
|
||||
}
|
||||
plan.DurationSec = plan.TotalDurationSec - plan.StaggerSeconds*(count-1)
|
||||
default:
|
||||
plan.StaggerSeconds = 2 * 60
|
||||
plan.TotalDurationSec = plan.DurationSec + plan.StaggerSeconds*(count-1)
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
func resolvePlatformStressPreset(profile string) platform.PlatformStressOptions {
|
||||
@@ -600,7 +638,7 @@ func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) {
|
||||
RunNCCL: t.params.RunNCCL,
|
||||
ParallelGPUs: t.params.ParallelGPUs,
|
||||
}, j.append)
|
||||
case "nvidia-compute":
|
||||
case "nvidia-compute":
|
||||
if a == nil {
|
||||
err = fmt.Errorf("app not configured")
|
||||
break
|
||||
@@ -609,11 +647,18 @@ func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) {
|
||||
if t.params.BurnProfile != "" && dur <= 0 {
|
||||
dur = resolveBurnPreset(t.params.BurnProfile).DurationSec
|
||||
}
|
||||
staggerSec := boolToNvidiaStaggerSeconds(t.params.StaggerGPUStart, t.params.GPUIndices)
|
||||
if staggerSec > 0 {
|
||||
j.append(fmt.Sprintf("NVIDIA staggered ramp-up enabled: %ds per GPU", staggerSec))
|
||||
}
|
||||
archive, err = a.RunNvidiaOfficialComputePack(ctx, "", dur, t.params.GPUIndices, staggerSec, j.append)
|
||||
rampPlan, planErr := resolveNvidiaRampPlan(t.params.BurnProfile, t.params.StaggerGPUStart, t.params.GPUIndices)
|
||||
if planErr != nil {
|
||||
err = planErr
|
||||
break
|
||||
}
|
||||
if t.params.BurnProfile != "" && t.params.StaggerGPUStart && dur <= 0 {
|
||||
dur = rampPlan.DurationSec
|
||||
}
|
||||
if rampPlan.StaggerSeconds > 0 {
|
||||
j.append(fmt.Sprintf("NVIDIA staggered ramp-up enabled: %ds per GPU; post-ramp hold: %ds; total runtime: %ds", rampPlan.StaggerSeconds, dur, rampPlan.TotalDurationSec))
|
||||
}
|
||||
archive, err = a.RunNvidiaOfficialComputePack(ctx, "", dur, t.params.GPUIndices, rampPlan.StaggerSeconds, j.append)
|
||||
case "nvidia-targeted-power":
|
||||
if a == nil {
|
||||
err = fmt.Errorf("app not configured")
|
||||
@@ -663,13 +708,24 @@ func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) {
|
||||
if t.params.BurnProfile != "" && dur <= 0 {
|
||||
dur = resolveBurnPreset(t.params.BurnProfile).DurationSec
|
||||
}
|
||||
archive, err = runNvidiaStressPackCtx(a, ctx, "", platform.NvidiaStressOptions{
|
||||
DurationSec: dur,
|
||||
Loader: t.params.Loader,
|
||||
GPUIndices: t.params.GPUIndices,
|
||||
ExcludeGPUIndices: t.params.ExcludeGPUIndices,
|
||||
StaggerSeconds: boolToNvidiaStaggerSeconds(t.params.StaggerGPUStart, t.params.GPUIndices),
|
||||
}, j.append)
|
||||
rampPlan, planErr := resolveNvidiaRampPlan(t.params.BurnProfile, t.params.StaggerGPUStart, t.params.GPUIndices)
|
||||
if planErr != nil {
|
||||
err = planErr
|
||||
break
|
||||
}
|
||||
if t.params.BurnProfile != "" && t.params.StaggerGPUStart && dur <= 0 {
|
||||
dur = rampPlan.DurationSec
|
||||
}
|
||||
if rampPlan.StaggerSeconds > 0 {
|
||||
j.append(fmt.Sprintf("NVIDIA staggered ramp-up enabled: %ds per GPU; post-ramp hold: %ds; total runtime: %ds", rampPlan.StaggerSeconds, dur, rampPlan.TotalDurationSec))
|
||||
}
|
||||
archive, err = runNvidiaStressPackCtx(a, ctx, "", platform.NvidiaStressOptions{
|
||||
DurationSec: dur,
|
||||
Loader: t.params.Loader,
|
||||
GPUIndices: t.params.GPUIndices,
|
||||
ExcludeGPUIndices: t.params.ExcludeGPUIndices,
|
||||
StaggerSeconds: rampPlan.StaggerSeconds,
|
||||
}, j.append)
|
||||
case "memory":
|
||||
if a == nil {
|
||||
err = fmt.Errorf("app not configured")
|
||||
|
||||
@@ -491,6 +491,83 @@ func TestResolveBurnPreset(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveNvidiaRampPlan(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
profile string
|
||||
enabled bool
|
||||
selected []int
|
||||
want nvidiaRampSpec
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "disabled uses base preset",
|
||||
profile: "acceptance",
|
||||
selected: []int{0, 1},
|
||||
want: nvidiaRampSpec{DurationSec: 60 * 60, TotalDurationSec: 60 * 60},
|
||||
},
|
||||
{
|
||||
name: "smoke ramp uses two minute steps",
|
||||
profile: "smoke",
|
||||
enabled: true,
|
||||
selected: []int{0, 1, 2},
|
||||
want: nvidiaRampSpec{DurationSec: 5 * 60, StaggerSeconds: 2 * 60, TotalDurationSec: 9 * 60},
|
||||
},
|
||||
{
|
||||
name: "acceptance ramp uses ten minute steps",
|
||||
profile: "acceptance",
|
||||
enabled: true,
|
||||
selected: []int{0, 1, 2},
|
||||
want: nvidiaRampSpec{DurationSec: 60 * 60, StaggerSeconds: 10 * 60, TotalDurationSec: 80 * 60},
|
||||
},
|
||||
{
|
||||
name: "overnight stays at eight hours when possible",
|
||||
profile: "overnight",
|
||||
enabled: true,
|
||||
selected: []int{0, 1, 2},
|
||||
want: nvidiaRampSpec{DurationSec: 6 * 60 * 60, StaggerSeconds: 60 * 60, TotalDurationSec: 8 * 60 * 60},
|
||||
},
|
||||
{
|
||||
name: "overnight extends to keep one hour after final gpu",
|
||||
profile: "overnight",
|
||||
enabled: true,
|
||||
selected: []int{0, 1, 2, 3, 4, 5, 6, 7, 8},
|
||||
want: nvidiaRampSpec{DurationSec: 60 * 60, StaggerSeconds: 60 * 60, TotalDurationSec: 9 * 60 * 60},
|
||||
},
|
||||
{
|
||||
name: "overnight rejects impossible gpu count",
|
||||
profile: "overnight",
|
||||
enabled: true,
|
||||
selected: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
|
||||
wantErr: "at most 10 GPUs",
|
||||
},
|
||||
{
|
||||
name: "enabled requires explicit selection",
|
||||
profile: "smoke",
|
||||
enabled: true,
|
||||
wantErr: "requires explicit GPU selection",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := resolveNvidiaRampPlan(tc.profile, tc.enabled, tc.selected)
|
||||
if tc.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
|
||||
t.Fatalf("err=%v want substring %q", err, tc.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("resolveNvidiaRampPlan error: %v", err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("resolveNvidiaRampPlan(%q, %t, %v)=%+v want %+v", tc.profile, tc.enabled, tc.selected, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskDisplayNameUsesNvidiaStressLoader(t *testing.T) {
|
||||
tests := []struct {
|
||||
loader string
|
||||
|
||||
@@ -11,18 +11,18 @@ echo " Hardware Audit LiveCD"
|
||||
echo ""
|
||||
|
||||
menuentry "EASY-BEE" {
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ nomodeset bee.nvidia.mode=normal net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ nomodeset bee.nvidia.mode=normal net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1 nowatchdog nosoftlockup
|
||||
initrd @INITRD_LIVE@
|
||||
}
|
||||
|
||||
submenu "EASY-BEE (advanced options) -->" {
|
||||
menuentry "EASY-BEE — GSP=off" {
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ nomodeset bee.nvidia.mode=gsp-off net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ nomodeset bee.nvidia.mode=gsp-off net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1 nowatchdog nosoftlockup
|
||||
initrd @INITRD_LIVE@
|
||||
}
|
||||
|
||||
menuentry "EASY-BEE — KMS (no nomodeset)" {
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ bee.nvidia.mode=normal net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ bee.nvidia.mode=normal net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1 nowatchdog nosoftlockup
|
||||
initrd @INITRD_LIVE@
|
||||
}
|
||||
|
||||
|
||||
@@ -3,31 +3,31 @@ label live-@FLAVOUR@-normal
|
||||
menu default
|
||||
linux @LINUX@
|
||||
initrd @INITRD@
|
||||
append @APPEND_LIVE@ bee.nvidia.mode=normal
|
||||
append @APPEND_LIVE@ bee.nvidia.mode=normal pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1
|
||||
|
||||
label live-@FLAVOUR@-kms
|
||||
menu label EASY-BEE (^graphics/KMS)
|
||||
linux @LINUX@
|
||||
initrd @INITRD@
|
||||
append @APPEND_LIVE@ bee.display=kms bee.nvidia.mode=normal
|
||||
append @APPEND_LIVE@ bee.display=kms bee.nvidia.mode=normal pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1
|
||||
|
||||
label live-@FLAVOUR@-toram
|
||||
menu label EASY-BEE (^load to RAM)
|
||||
linux @LINUX@
|
||||
initrd @INITRD@
|
||||
append @APPEND_LIVE@ toram bee.nvidia.mode=normal
|
||||
append @APPEND_LIVE@ toram bee.nvidia.mode=normal pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1
|
||||
|
||||
label live-@FLAVOUR@-gsp-off
|
||||
menu label EASY-BEE (^NVIDIA GSP=off)
|
||||
linux @LINUX@
|
||||
initrd @INITRD@
|
||||
append @APPEND_LIVE@ nomodeset bee.nvidia.mode=gsp-off
|
||||
append @APPEND_LIVE@ nomodeset bee.nvidia.mode=gsp-off pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1
|
||||
|
||||
label live-@FLAVOUR@-kms-gsp-off
|
||||
menu label EASY-BEE (g^raphics/KMS, GSP=off)
|
||||
linux @LINUX@
|
||||
initrd @INITRD@
|
||||
append @APPEND_LIVE@ bee.display=kms bee.nvidia.mode=gsp-off
|
||||
append @APPEND_LIVE@ bee.display=kms bee.nvidia.mode=gsp-off pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1
|
||||
|
||||
label live-@FLAVOUR@-failsafe
|
||||
menu label EASY-BEE (^fail-safe)
|
||||
|
||||
@@ -25,6 +25,7 @@ ensure_bee_console_user() {
|
||||
ensure_bee_console_user
|
||||
|
||||
# Enable common bee services
|
||||
systemctl enable bee-hpc-tuning.service
|
||||
systemctl enable bee-network.service
|
||||
systemctl enable bee-preflight.service
|
||||
systemctl enable bee-audit.service
|
||||
@@ -55,6 +56,7 @@ fi
|
||||
# nogpu: no GPU services needed
|
||||
|
||||
# Ensure scripts are executable
|
||||
chmod +x /usr/local/bin/bee-hpc-tuning 2>/dev/null || true
|
||||
chmod +x /usr/local/bin/bee-network.sh 2>/dev/null || true
|
||||
chmod +x /usr/local/bin/bee-sshsetup 2>/dev/null || true
|
||||
chmod +x /usr/local/bin/bee-smoketest 2>/dev/null || true
|
||||
|
||||
14
iso/overlay/etc/systemd/system/bee-hpc-tuning.service
Normal file
14
iso/overlay/etc/systemd/system/bee-hpc-tuning.service
Normal file
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=Bee: HPC tuning (CPU governor, C-states)
|
||||
After=local-fs.target
|
||||
Before=bee-nvidia.service bee-audit.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/bee-log-run /appdata/bee/export/bee-hpc-tuning.log /usr/local/bin/bee-hpc-tuning
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
RemainAfterExit=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
41
iso/overlay/usr/local/bin/bee-hpc-tuning
Normal file
41
iso/overlay/usr/local/bin/bee-hpc-tuning
Normal file
@@ -0,0 +1,41 @@
|
||||
#!/bin/sh
|
||||
# bee-hpc-tuning — apply HPC tuning for deterministic benchmarking
|
||||
# Called by bee-hpc-tuning.service at boot.
|
||||
|
||||
log() { echo "[bee-hpc-tuning] $*"; }
|
||||
|
||||
# ── CPU governor ────────────────────────────────────────────────────────────
|
||||
# Set all CPU cores to performance governor via sysfs.
|
||||
# cpupower is not available; write directly to scaling_governor.
|
||||
governor_ok=0
|
||||
governor_fail=0
|
||||
for gov_path in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
|
||||
[ -f "$gov_path" ] || continue
|
||||
if echo performance > "$gov_path" 2>/dev/null; then
|
||||
governor_ok=$((governor_ok + 1))
|
||||
else
|
||||
governor_fail=$((governor_fail + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$governor_ok" -gt 0 ] && [ "$governor_fail" -eq 0 ]; then
|
||||
log "CPU governor set to performance on ${governor_ok} core(s)"
|
||||
elif [ "$governor_ok" -gt 0 ]; then
|
||||
log "WARN: CPU governor: ${governor_ok} OK, ${governor_fail} failed"
|
||||
elif [ "$governor_fail" -gt 0 ]; then
|
||||
log "WARN: failed to set CPU governor on ${governor_fail} core(s)"
|
||||
else
|
||||
log "WARN: no cpufreq scaling_governor paths found (C-state governor or HW-controlled)"
|
||||
fi
|
||||
|
||||
# ── Transparent Huge Pages ───────────────────────────────────────────────────
|
||||
# Kernel cmdline sets transparent_hugepage=always at boot, but confirm and log.
|
||||
thp_path=/sys/kernel/mm/transparent_hugepage/enabled
|
||||
if [ -f "$thp_path" ]; then
|
||||
current=$(cat "$thp_path" 2>/dev/null)
|
||||
log "transparent_hugepage: ${current}"
|
||||
else
|
||||
log "WARN: transparent_hugepage sysfs path not found"
|
||||
fi
|
||||
|
||||
log "done"
|
||||
Reference in New Issue
Block a user