Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e16d0f34b5 | |||
|
|
525ed8b8fc | ||
|
|
4f94ebcb2c |
@@ -121,15 +121,22 @@ func (s *System) RunNvidiaBenchmark(ctx context.Context, baseDir string, opts Nv
|
|||||||
var serverIdleOK, serverLoadedOK bool
|
var serverIdleOK, serverLoadedOK bool
|
||||||
var serverLoadedSamples int
|
var serverLoadedSamples int
|
||||||
|
|
||||||
|
// Run nvidia-smi -q first: used both for the log file and as a fallback
|
||||||
|
// source of max clock values when CSV clock fields are unsupported.
|
||||||
|
var nvsmiQOut []byte
|
||||||
|
if out, err := runSATCommandCtx(ctx, verboseLog, "00-nvidia-smi-q.log", []string{"nvidia-smi", "-q"}, nil, nil); err == nil {
|
||||||
|
nvsmiQOut = out
|
||||||
|
_ = os.WriteFile(filepath.Join(runDir, "00-nvidia-smi-q.log"), out, 0644)
|
||||||
|
}
|
||||||
|
|
||||||
infoByIndex, infoErr := queryBenchmarkGPUInfo(selected)
|
infoByIndex, infoErr := queryBenchmarkGPUInfo(selected)
|
||||||
if infoErr != nil {
|
if infoErr != nil {
|
||||||
result.Warnings = append(result.Warnings, "gpu inventory query failed: "+infoErr.Error())
|
result.Warnings = append(result.Warnings, "gpu inventory query failed: "+infoErr.Error())
|
||||||
result.Normalization.Status = "partial"
|
result.Normalization.Status = "partial"
|
||||||
}
|
}
|
||||||
|
// Enrich with max clocks from verbose output — covers GPUs where
|
||||||
if out, err := runSATCommandCtx(ctx, verboseLog, "00-nvidia-smi-q.log", []string{"nvidia-smi", "-q"}, nil, nil); err == nil {
|
// clocks.max.* CSV fields are unsupported (e.g. Blackwell / driver 98.x).
|
||||||
_ = os.WriteFile(filepath.Join(runDir, "00-nvidia-smi-q.log"), out, 0644)
|
enrichGPUInfoWithMaxClocks(infoByIndex, nvsmiQOut)
|
||||||
}
|
|
||||||
|
|
||||||
activeApps, err := queryActiveComputeApps(selected)
|
activeApps, err := queryActiveComputeApps(selected)
|
||||||
if err == nil && len(activeApps) > 0 {
|
if err == nil && len(activeApps) > 0 {
|
||||||
@@ -370,9 +377,13 @@ func resolveBenchmarkProfile(profile string) benchmarkProfileSpec {
|
|||||||
// Fields are tried in order; the first successful query wins. Extended fields
|
// Fields are tried in order; the first successful query wins. Extended fields
|
||||||
// (attribute.multiprocessor_count, power.default_limit) are not supported on
|
// (attribute.multiprocessor_count, power.default_limit) are not supported on
|
||||||
// all driver versions, so we fall back to the base set if the full query fails.
|
// all driver versions, so we fall back to the base set if the full query fails.
|
||||||
|
// The minimal fallback omits clock fields entirely — clocks.max.* returns
|
||||||
|
// exit status 2 on some GPU generations (e.g. Blackwell); max clocks are
|
||||||
|
// then recovered from nvidia-smi -q via enrichGPUInfoWithMaxClocks.
|
||||||
var benchmarkGPUInfoQueries = []struct {
|
var benchmarkGPUInfoQueries = []struct {
|
||||||
fields string
|
fields string
|
||||||
extended bool // whether this query includes optional extended fields
|
extended bool // whether this query includes optional extended fields
|
||||||
|
minimal bool // clock fields omitted; max clocks must be filled separately
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics,attribute.multiprocessor_count,power.default_limit",
|
fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics,attribute.multiprocessor_count,power.default_limit",
|
||||||
@@ -382,6 +393,83 @@ var benchmarkGPUInfoQueries = []struct {
|
|||||||
fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics",
|
fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit,clocks.max.graphics,clocks.max.memory,clocks.base.graphics",
|
||||||
extended: false,
|
extended: false,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
fields: "index,uuid,name,pci.bus_id,vbios_version,power.limit",
|
||||||
|
minimal: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// enrichGPUInfoWithMaxClocks fills MaxGraphicsClockMHz / MaxMemoryClockMHz for
|
||||||
|
// any GPU in infoByIndex where those values are still zero. It parses the
|
||||||
|
// "Max Clocks" section of nvidia-smi -q output (already available as nvsmiQ).
|
||||||
|
// This is the fallback for GPUs (e.g. Blackwell) where clocks.max.* CSV fields
|
||||||
|
// return exit status 2 but the verbose query works fine.
|
||||||
|
func enrichGPUInfoWithMaxClocks(infoByIndex map[int]benchmarkGPUInfo, nvsmiQ []byte) {
|
||||||
|
if len(infoByIndex) == 0 || len(nvsmiQ) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build bus_id → index map for matching verbose sections to GPU indices.
|
||||||
|
busToBenchIdx := make(map[string]int, len(infoByIndex))
|
||||||
|
for idx, info := range infoByIndex {
|
||||||
|
if info.BusID != "" {
|
||||||
|
// nvidia-smi -q uses "GPU 00000000:4E:00.0" (8-digit domain),
|
||||||
|
// while --query-gpu returns the same format; normalise to lower.
|
||||||
|
busToBenchIdx[strings.ToLower(strings.TrimSpace(info.BusID))] = idx
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Split the verbose output into per-GPU sections on "^GPU " lines.
|
||||||
|
gpuSectionRe := regexp.MustCompile(`(?m)^GPU\s+([\dA-Fa-f:\.]+)`)
|
||||||
|
maxGfxRe := regexp.MustCompile(`(?i)Max Clocks[\s\S]*?Graphics\s*:\s*(\d+)\s*MHz`)
|
||||||
|
maxMemRe := regexp.MustCompile(`(?i)Max Clocks[\s\S]*?Memory\s*:\s*(\d+)\s*MHz`)
|
||||||
|
|
||||||
|
sectionStarts := gpuSectionRe.FindAllSubmatchIndex(nvsmiQ, -1)
|
||||||
|
for i, loc := range sectionStarts {
|
||||||
|
busID := strings.ToLower(string(nvsmiQ[loc[2]:loc[3]]))
|
||||||
|
benchIdx, ok := busToBenchIdx[busID]
|
||||||
|
if !ok {
|
||||||
|
// Bus IDs from verbose output may have a different domain prefix;
|
||||||
|
// try suffix match on the slot portion (XX:XX.X).
|
||||||
|
for k, v := range busToBenchIdx {
|
||||||
|
if strings.HasSuffix(k, busID) || strings.HasSuffix(busID, k) {
|
||||||
|
benchIdx = v
|
||||||
|
ok = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
info := infoByIndex[benchIdx]
|
||||||
|
if info.MaxGraphicsClockMHz > 0 && info.MaxMemoryClockMHz > 0 {
|
||||||
|
continue // already populated
|
||||||
|
}
|
||||||
|
|
||||||
|
end := len(nvsmiQ)
|
||||||
|
if i+1 < len(sectionStarts) {
|
||||||
|
end = sectionStarts[i+1][0]
|
||||||
|
}
|
||||||
|
section := nvsmiQ[loc[0]:end]
|
||||||
|
|
||||||
|
if info.MaxGraphicsClockMHz == 0 {
|
||||||
|
if m := maxGfxRe.FindSubmatch(section); m != nil {
|
||||||
|
if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil {
|
||||||
|
info.MaxGraphicsClockMHz = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if info.MaxMemoryClockMHz == 0 {
|
||||||
|
if m := maxMemRe.FindSubmatch(section); m != nil {
|
||||||
|
if v, err := strconv.ParseFloat(string(m[1]), 64); err == nil {
|
||||||
|
info.MaxMemoryClockMHz = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
infoByIndex[benchIdx] = info
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func queryBenchmarkGPUInfo(gpuIndices []int) (map[int]benchmarkGPUInfo, error) {
|
func queryBenchmarkGPUInfo(gpuIndices []int) (map[int]benchmarkGPUInfo, error) {
|
||||||
@@ -409,9 +497,13 @@ func queryBenchmarkGPUInfo(gpuIndices []int) (map[int]benchmarkGPUInfo, error) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
minFields := 6
|
||||||
|
if !q.minimal {
|
||||||
|
minFields = 9
|
||||||
|
}
|
||||||
infoByIndex := make(map[int]benchmarkGPUInfo, len(rows))
|
infoByIndex := make(map[int]benchmarkGPUInfo, len(rows))
|
||||||
for _, row := range rows {
|
for _, row := range rows {
|
||||||
if len(row) < 9 {
|
if len(row) < minFields {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
idx, err := strconv.Atoi(strings.TrimSpace(row[0]))
|
idx, err := strconv.Atoi(strings.TrimSpace(row[0]))
|
||||||
@@ -425,9 +517,10 @@ func queryBenchmarkGPUInfo(gpuIndices []int) (map[int]benchmarkGPUInfo, error) {
|
|||||||
BusID: strings.TrimSpace(row[3]),
|
BusID: strings.TrimSpace(row[3]),
|
||||||
VBIOS: strings.TrimSpace(row[4]),
|
VBIOS: strings.TrimSpace(row[4]),
|
||||||
PowerLimitW: parseBenchmarkFloat(row[5]),
|
PowerLimitW: parseBenchmarkFloat(row[5]),
|
||||||
MaxGraphicsClockMHz: parseBenchmarkFloat(row[6]),
|
|
||||||
MaxMemoryClockMHz: parseBenchmarkFloat(row[7]),
|
|
||||||
}
|
}
|
||||||
|
if !q.minimal {
|
||||||
|
info.MaxGraphicsClockMHz = parseBenchmarkFloat(row[6])
|
||||||
|
info.MaxMemoryClockMHz = parseBenchmarkFloat(row[7])
|
||||||
if len(row) >= 9 {
|
if len(row) >= 9 {
|
||||||
info.BaseGraphicsClockMHz = parseBenchmarkFloat(row[8])
|
info.BaseGraphicsClockMHz = parseBenchmarkFloat(row[8])
|
||||||
}
|
}
|
||||||
@@ -439,6 +532,7 @@ func queryBenchmarkGPUInfo(gpuIndices []int) (map[int]benchmarkGPUInfo, error) {
|
|||||||
info.DefaultPowerLimitW = parseBenchmarkFloat(row[10])
|
info.DefaultPowerLimitW = parseBenchmarkFloat(row[10])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
infoByIndex[idx] = info
|
infoByIndex[idx] = info
|
||||||
}
|
}
|
||||||
return infoByIndex, nil
|
return infoByIndex, nil
|
||||||
|
|||||||
@@ -178,3 +178,67 @@ func TestRenderBenchmarkReportIncludesTerminalChartsWithoutANSI(t *testing.T) {
|
|||||||
t.Fatalf("report should not contain ANSI escapes\n%s", report)
|
t.Fatalf("report should not contain ANSI escapes\n%s", report)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestEnrichGPUInfoWithMaxClocks(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
nvsmiQ := []byte(`
|
||||||
|
GPU 00000000:4E:00.0
|
||||||
|
Product Name : NVIDIA RTX PRO 6000 Blackwell Server Edition
|
||||||
|
Clocks
|
||||||
|
Graphics : 2422 MHz
|
||||||
|
Memory : 12481 MHz
|
||||||
|
Max Clocks
|
||||||
|
Graphics : 2430 MHz
|
||||||
|
SM : 2430 MHz
|
||||||
|
Memory : 12481 MHz
|
||||||
|
Video : 2107 MHz
|
||||||
|
|
||||||
|
GPU 00000000:4F:00.0
|
||||||
|
Product Name : NVIDIA RTX PRO 6000 Blackwell Server Edition
|
||||||
|
Max Clocks
|
||||||
|
Graphics : 2430 MHz
|
||||||
|
Memory : 12481 MHz
|
||||||
|
`)
|
||||||
|
|
||||||
|
infoByIndex := map[int]benchmarkGPUInfo{
|
||||||
|
0: {Index: 0, BusID: "00000000:4E:00.0"},
|
||||||
|
1: {Index: 1, BusID: "00000000:4F:00.0"},
|
||||||
|
}
|
||||||
|
|
||||||
|
enrichGPUInfoWithMaxClocks(infoByIndex, nvsmiQ)
|
||||||
|
|
||||||
|
if infoByIndex[0].MaxGraphicsClockMHz != 2430 {
|
||||||
|
t.Errorf("GPU 0 MaxGraphicsClockMHz = %v, want 2430", infoByIndex[0].MaxGraphicsClockMHz)
|
||||||
|
}
|
||||||
|
if infoByIndex[0].MaxMemoryClockMHz != 12481 {
|
||||||
|
t.Errorf("GPU 0 MaxMemoryClockMHz = %v, want 12481", infoByIndex[0].MaxMemoryClockMHz)
|
||||||
|
}
|
||||||
|
if infoByIndex[1].MaxGraphicsClockMHz != 2430 {
|
||||||
|
t.Errorf("GPU 1 MaxGraphicsClockMHz = %v, want 2430", infoByIndex[1].MaxGraphicsClockMHz)
|
||||||
|
}
|
||||||
|
if infoByIndex[1].MaxMemoryClockMHz != 12481 {
|
||||||
|
t.Errorf("GPU 1 MaxMemoryClockMHz = %v, want 12481", infoByIndex[1].MaxMemoryClockMHz)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnrichGPUInfoWithMaxClocksSkipsPopulated(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
nvsmiQ := []byte(`
|
||||||
|
GPU 00000000:4E:00.0
|
||||||
|
Max Clocks
|
||||||
|
Graphics : 9999 MHz
|
||||||
|
Memory : 9999 MHz
|
||||||
|
`)
|
||||||
|
// Already populated — must not be overwritten.
|
||||||
|
infoByIndex := map[int]benchmarkGPUInfo{
|
||||||
|
0: {Index: 0, BusID: "00000000:4E:00.0", MaxGraphicsClockMHz: 2430, MaxMemoryClockMHz: 12481},
|
||||||
|
}
|
||||||
|
|
||||||
|
enrichGPUInfoWithMaxClocks(infoByIndex, nvsmiQ)
|
||||||
|
|
||||||
|
if infoByIndex[0].MaxGraphicsClockMHz != 2430 {
|
||||||
|
t.Errorf("expected existing value to be preserved, got %v", infoByIndex[0].MaxGraphicsClockMHz)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -2373,7 +2373,7 @@ func renderBurn() string {
|
|||||||
<p id="burn-selection-note" style="font-size:12px;color:var(--muted);margin:10px 0 0">Select at least one NVIDIA GPU to enable NVIDIA burn recipes.</p>
|
<p id="burn-selection-note" style="font-size:12px;color:var(--muted);margin:10px 0 0">Select at least one NVIDIA GPU to enable NVIDIA burn recipes.</p>
|
||||||
<label class="cb-row" style="margin-top:10px">
|
<label class="cb-row" style="margin-top:10px">
|
||||||
<input type="checkbox" id="burn-stagger-nvidia">
|
<input type="checkbox" id="burn-stagger-nvidia">
|
||||||
<span>Ramp selected NVIDIA GPUs one by one before full-load hold. Uses a 3-minute stabilization window per GPU, then keeps all selected GPUs under load for the chosen Burn Profile duration.</span>
|
<span>Ramp selected NVIDIA GPUs one by one before the full-load hold. Smoke: +2 min per GPU, then 5 min with all selected GPUs under load. Acceptance: +10 min per GPU, then at least 1 hour with all selected GPUs under load. Overnight: +1 hour per GPU, then at least 1 hour with all selected GPUs under load, capped at 10 hours total.</span>
|
||||||
</label>
|
</label>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -3108,7 +3108,6 @@ usbRefresh();
|
|||||||
</script>`
|
</script>`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func renderNvidiaSelfHealInline() string {
|
func renderNvidiaSelfHealInline() string {
|
||||||
return `<p style="font-size:13px;color:var(--muted);margin-bottom:12px">Inspect NVIDIA GPU health, restart the bee-nvidia driver service, and issue a per-GPU reset when the driver reports reset required.</p>
|
return `<p style="font-size:13px;color:var(--muted);margin-bottom:12px">Inspect NVIDIA GPU health, restart the bee-nvidia driver service, and issue a per-GPU reset when the driver reports reset required.</p>
|
||||||
<div style="display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px">
|
<div style="display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px">
|
||||||
|
|||||||
@@ -152,6 +152,12 @@ type burnPreset struct {
|
|||||||
DurationSec int
|
DurationSec int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type nvidiaRampSpec struct {
|
||||||
|
DurationSec int
|
||||||
|
StaggerSeconds int
|
||||||
|
TotalDurationSec int
|
||||||
|
}
|
||||||
|
|
||||||
func resolveBurnPreset(profile string) burnPreset {
|
func resolveBurnPreset(profile string) burnPreset {
|
||||||
switch profile {
|
switch profile {
|
||||||
case "overnight":
|
case "overnight":
|
||||||
@@ -163,11 +169,43 @@ func resolveBurnPreset(profile string) burnPreset {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func boolToNvidiaStaggerSeconds(enabled bool, selected []int) int {
|
func resolveNvidiaRampPlan(profile string, enabled bool, selected []int) (nvidiaRampSpec, error) {
|
||||||
if enabled && len(selected) > 1 {
|
base := resolveBurnPreset(profile).DurationSec
|
||||||
return 180
|
plan := nvidiaRampSpec{
|
||||||
|
DurationSec: base,
|
||||||
|
TotalDurationSec: base,
|
||||||
}
|
}
|
||||||
return 0
|
if !enabled {
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
|
count := len(selected)
|
||||||
|
if count == 0 {
|
||||||
|
return nvidiaRampSpec{}, fmt.Errorf("staggered NVIDIA burn requires explicit GPU selection")
|
||||||
|
}
|
||||||
|
if count == 1 {
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
switch profile {
|
||||||
|
case "acceptance":
|
||||||
|
plan.StaggerSeconds = 10 * 60
|
||||||
|
plan.TotalDurationSec = plan.DurationSec + plan.StaggerSeconds*(count-1)
|
||||||
|
case "overnight":
|
||||||
|
plan.StaggerSeconds = 60 * 60
|
||||||
|
plan.TotalDurationSec = 8 * 60 * 60
|
||||||
|
minTotal := count * 60 * 60
|
||||||
|
if plan.TotalDurationSec < minTotal {
|
||||||
|
plan.TotalDurationSec = minTotal
|
||||||
|
}
|
||||||
|
if plan.TotalDurationSec > 10*60*60 {
|
||||||
|
return nvidiaRampSpec{}, fmt.Errorf("overnight staggered NVIDIA burn supports at most 10 GPUs")
|
||||||
|
}
|
||||||
|
plan.DurationSec = plan.TotalDurationSec - plan.StaggerSeconds*(count-1)
|
||||||
|
default:
|
||||||
|
plan.StaggerSeconds = 2 * 60
|
||||||
|
plan.TotalDurationSec = plan.DurationSec + plan.StaggerSeconds*(count-1)
|
||||||
|
}
|
||||||
|
return plan, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolvePlatformStressPreset(profile string) platform.PlatformStressOptions {
|
func resolvePlatformStressPreset(profile string) platform.PlatformStressOptions {
|
||||||
@@ -609,11 +647,18 @@ func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) {
|
|||||||
if t.params.BurnProfile != "" && dur <= 0 {
|
if t.params.BurnProfile != "" && dur <= 0 {
|
||||||
dur = resolveBurnPreset(t.params.BurnProfile).DurationSec
|
dur = resolveBurnPreset(t.params.BurnProfile).DurationSec
|
||||||
}
|
}
|
||||||
staggerSec := boolToNvidiaStaggerSeconds(t.params.StaggerGPUStart, t.params.GPUIndices)
|
rampPlan, planErr := resolveNvidiaRampPlan(t.params.BurnProfile, t.params.StaggerGPUStart, t.params.GPUIndices)
|
||||||
if staggerSec > 0 {
|
if planErr != nil {
|
||||||
j.append(fmt.Sprintf("NVIDIA staggered ramp-up enabled: %ds per GPU", staggerSec))
|
err = planErr
|
||||||
|
break
|
||||||
}
|
}
|
||||||
archive, err = a.RunNvidiaOfficialComputePack(ctx, "", dur, t.params.GPUIndices, staggerSec, j.append)
|
if t.params.BurnProfile != "" && t.params.StaggerGPUStart && dur <= 0 {
|
||||||
|
dur = rampPlan.DurationSec
|
||||||
|
}
|
||||||
|
if rampPlan.StaggerSeconds > 0 {
|
||||||
|
j.append(fmt.Sprintf("NVIDIA staggered ramp-up enabled: %ds per GPU; post-ramp hold: %ds; total runtime: %ds", rampPlan.StaggerSeconds, dur, rampPlan.TotalDurationSec))
|
||||||
|
}
|
||||||
|
archive, err = a.RunNvidiaOfficialComputePack(ctx, "", dur, t.params.GPUIndices, rampPlan.StaggerSeconds, j.append)
|
||||||
case "nvidia-targeted-power":
|
case "nvidia-targeted-power":
|
||||||
if a == nil {
|
if a == nil {
|
||||||
err = fmt.Errorf("app not configured")
|
err = fmt.Errorf("app not configured")
|
||||||
@@ -663,12 +708,23 @@ func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) {
|
|||||||
if t.params.BurnProfile != "" && dur <= 0 {
|
if t.params.BurnProfile != "" && dur <= 0 {
|
||||||
dur = resolveBurnPreset(t.params.BurnProfile).DurationSec
|
dur = resolveBurnPreset(t.params.BurnProfile).DurationSec
|
||||||
}
|
}
|
||||||
|
rampPlan, planErr := resolveNvidiaRampPlan(t.params.BurnProfile, t.params.StaggerGPUStart, t.params.GPUIndices)
|
||||||
|
if planErr != nil {
|
||||||
|
err = planErr
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if t.params.BurnProfile != "" && t.params.StaggerGPUStart && dur <= 0 {
|
||||||
|
dur = rampPlan.DurationSec
|
||||||
|
}
|
||||||
|
if rampPlan.StaggerSeconds > 0 {
|
||||||
|
j.append(fmt.Sprintf("NVIDIA staggered ramp-up enabled: %ds per GPU; post-ramp hold: %ds; total runtime: %ds", rampPlan.StaggerSeconds, dur, rampPlan.TotalDurationSec))
|
||||||
|
}
|
||||||
archive, err = runNvidiaStressPackCtx(a, ctx, "", platform.NvidiaStressOptions{
|
archive, err = runNvidiaStressPackCtx(a, ctx, "", platform.NvidiaStressOptions{
|
||||||
DurationSec: dur,
|
DurationSec: dur,
|
||||||
Loader: t.params.Loader,
|
Loader: t.params.Loader,
|
||||||
GPUIndices: t.params.GPUIndices,
|
GPUIndices: t.params.GPUIndices,
|
||||||
ExcludeGPUIndices: t.params.ExcludeGPUIndices,
|
ExcludeGPUIndices: t.params.ExcludeGPUIndices,
|
||||||
StaggerSeconds: boolToNvidiaStaggerSeconds(t.params.StaggerGPUStart, t.params.GPUIndices),
|
StaggerSeconds: rampPlan.StaggerSeconds,
|
||||||
}, j.append)
|
}, j.append)
|
||||||
case "memory":
|
case "memory":
|
||||||
if a == nil {
|
if a == nil {
|
||||||
|
|||||||
@@ -491,6 +491,83 @@ func TestResolveBurnPreset(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolveNvidiaRampPlan(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
profile string
|
||||||
|
enabled bool
|
||||||
|
selected []int
|
||||||
|
want nvidiaRampSpec
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "disabled uses base preset",
|
||||||
|
profile: "acceptance",
|
||||||
|
selected: []int{0, 1},
|
||||||
|
want: nvidiaRampSpec{DurationSec: 60 * 60, TotalDurationSec: 60 * 60},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "smoke ramp uses two minute steps",
|
||||||
|
profile: "smoke",
|
||||||
|
enabled: true,
|
||||||
|
selected: []int{0, 1, 2},
|
||||||
|
want: nvidiaRampSpec{DurationSec: 5 * 60, StaggerSeconds: 2 * 60, TotalDurationSec: 9 * 60},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "acceptance ramp uses ten minute steps",
|
||||||
|
profile: "acceptance",
|
||||||
|
enabled: true,
|
||||||
|
selected: []int{0, 1, 2},
|
||||||
|
want: nvidiaRampSpec{DurationSec: 60 * 60, StaggerSeconds: 10 * 60, TotalDurationSec: 80 * 60},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "overnight stays at eight hours when possible",
|
||||||
|
profile: "overnight",
|
||||||
|
enabled: true,
|
||||||
|
selected: []int{0, 1, 2},
|
||||||
|
want: nvidiaRampSpec{DurationSec: 6 * 60 * 60, StaggerSeconds: 60 * 60, TotalDurationSec: 8 * 60 * 60},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "overnight extends to keep one hour after final gpu",
|
||||||
|
profile: "overnight",
|
||||||
|
enabled: true,
|
||||||
|
selected: []int{0, 1, 2, 3, 4, 5, 6, 7, 8},
|
||||||
|
want: nvidiaRampSpec{DurationSec: 60 * 60, StaggerSeconds: 60 * 60, TotalDurationSec: 9 * 60 * 60},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "overnight rejects impossible gpu count",
|
||||||
|
profile: "overnight",
|
||||||
|
enabled: true,
|
||||||
|
selected: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
|
||||||
|
wantErr: "at most 10 GPUs",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "enabled requires explicit selection",
|
||||||
|
profile: "smoke",
|
||||||
|
enabled: true,
|
||||||
|
wantErr: "requires explicit GPU selection",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got, err := resolveNvidiaRampPlan(tc.profile, tc.enabled, tc.selected)
|
||||||
|
if tc.wantErr != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
|
||||||
|
t.Fatalf("err=%v want substring %q", err, tc.wantErr)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveNvidiaRampPlan error: %v", err)
|
||||||
|
}
|
||||||
|
if got != tc.want {
|
||||||
|
t.Fatalf("resolveNvidiaRampPlan(%q, %t, %v)=%+v want %+v", tc.profile, tc.enabled, tc.selected, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestTaskDisplayNameUsesNvidiaStressLoader(t *testing.T) {
|
func TestTaskDisplayNameUsesNvidiaStressLoader(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
loader string
|
loader string
|
||||||
|
|||||||
@@ -11,18 +11,18 @@ echo " Hardware Audit LiveCD"
|
|||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
menuentry "EASY-BEE" {
|
menuentry "EASY-BEE" {
|
||||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ nomodeset bee.nvidia.mode=normal net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
linux @KERNEL_LIVE@ @APPEND_LIVE@ nomodeset bee.nvidia.mode=normal net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1 nowatchdog nosoftlockup
|
||||||
initrd @INITRD_LIVE@
|
initrd @INITRD_LIVE@
|
||||||
}
|
}
|
||||||
|
|
||||||
submenu "EASY-BEE (advanced options) -->" {
|
submenu "EASY-BEE (advanced options) -->" {
|
||||||
menuentry "EASY-BEE — GSP=off" {
|
menuentry "EASY-BEE — GSP=off" {
|
||||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ nomodeset bee.nvidia.mode=gsp-off net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
linux @KERNEL_LIVE@ @APPEND_LIVE@ nomodeset bee.nvidia.mode=gsp-off net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1 nowatchdog nosoftlockup
|
||||||
initrd @INITRD_LIVE@
|
initrd @INITRD_LIVE@
|
||||||
}
|
}
|
||||||
|
|
||||||
menuentry "EASY-BEE — KMS (no nomodeset)" {
|
menuentry "EASY-BEE — KMS (no nomodeset)" {
|
||||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ bee.nvidia.mode=normal net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
linux @KERNEL_LIVE@ @APPEND_LIVE@ bee.nvidia.mode=normal net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1 nowatchdog nosoftlockup
|
||||||
initrd @INITRD_LIVE@
|
initrd @INITRD_LIVE@
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,31 +3,31 @@ label live-@FLAVOUR@-normal
|
|||||||
menu default
|
menu default
|
||||||
linux @LINUX@
|
linux @LINUX@
|
||||||
initrd @INITRD@
|
initrd @INITRD@
|
||||||
append @APPEND_LIVE@ bee.nvidia.mode=normal
|
append @APPEND_LIVE@ bee.nvidia.mode=normal pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1
|
||||||
|
|
||||||
label live-@FLAVOUR@-kms
|
label live-@FLAVOUR@-kms
|
||||||
menu label EASY-BEE (^graphics/KMS)
|
menu label EASY-BEE (^graphics/KMS)
|
||||||
linux @LINUX@
|
linux @LINUX@
|
||||||
initrd @INITRD@
|
initrd @INITRD@
|
||||||
append @APPEND_LIVE@ bee.display=kms bee.nvidia.mode=normal
|
append @APPEND_LIVE@ bee.display=kms bee.nvidia.mode=normal pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1
|
||||||
|
|
||||||
label live-@FLAVOUR@-toram
|
label live-@FLAVOUR@-toram
|
||||||
menu label EASY-BEE (^load to RAM)
|
menu label EASY-BEE (^load to RAM)
|
||||||
linux @LINUX@
|
linux @LINUX@
|
||||||
initrd @INITRD@
|
initrd @INITRD@
|
||||||
append @APPEND_LIVE@ toram bee.nvidia.mode=normal
|
append @APPEND_LIVE@ toram bee.nvidia.mode=normal pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1
|
||||||
|
|
||||||
label live-@FLAVOUR@-gsp-off
|
label live-@FLAVOUR@-gsp-off
|
||||||
menu label EASY-BEE (^NVIDIA GSP=off)
|
menu label EASY-BEE (^NVIDIA GSP=off)
|
||||||
linux @LINUX@
|
linux @LINUX@
|
||||||
initrd @INITRD@
|
initrd @INITRD@
|
||||||
append @APPEND_LIVE@ nomodeset bee.nvidia.mode=gsp-off
|
append @APPEND_LIVE@ nomodeset bee.nvidia.mode=gsp-off pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1
|
||||||
|
|
||||||
label live-@FLAVOUR@-kms-gsp-off
|
label live-@FLAVOUR@-kms-gsp-off
|
||||||
menu label EASY-BEE (g^raphics/KMS, GSP=off)
|
menu label EASY-BEE (g^raphics/KMS, GSP=off)
|
||||||
linux @LINUX@
|
linux @LINUX@
|
||||||
initrd @INITRD@
|
initrd @INITRD@
|
||||||
append @APPEND_LIVE@ bee.display=kms bee.nvidia.mode=gsp-off
|
append @APPEND_LIVE@ bee.display=kms bee.nvidia.mode=gsp-off pcie_aspm=off intel_idle.max_cstate=1 processor.max_cstate=1
|
||||||
|
|
||||||
label live-@FLAVOUR@-failsafe
|
label live-@FLAVOUR@-failsafe
|
||||||
menu label EASY-BEE (^fail-safe)
|
menu label EASY-BEE (^fail-safe)
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ ensure_bee_console_user() {
|
|||||||
ensure_bee_console_user
|
ensure_bee_console_user
|
||||||
|
|
||||||
# Enable common bee services
|
# Enable common bee services
|
||||||
|
systemctl enable bee-hpc-tuning.service
|
||||||
systemctl enable bee-network.service
|
systemctl enable bee-network.service
|
||||||
systemctl enable bee-preflight.service
|
systemctl enable bee-preflight.service
|
||||||
systemctl enable bee-audit.service
|
systemctl enable bee-audit.service
|
||||||
@@ -55,6 +56,7 @@ fi
|
|||||||
# nogpu: no GPU services needed
|
# nogpu: no GPU services needed
|
||||||
|
|
||||||
# Ensure scripts are executable
|
# Ensure scripts are executable
|
||||||
|
chmod +x /usr/local/bin/bee-hpc-tuning 2>/dev/null || true
|
||||||
chmod +x /usr/local/bin/bee-network.sh 2>/dev/null || true
|
chmod +x /usr/local/bin/bee-network.sh 2>/dev/null || true
|
||||||
chmod +x /usr/local/bin/bee-sshsetup 2>/dev/null || true
|
chmod +x /usr/local/bin/bee-sshsetup 2>/dev/null || true
|
||||||
chmod +x /usr/local/bin/bee-smoketest 2>/dev/null || true
|
chmod +x /usr/local/bin/bee-smoketest 2>/dev/null || true
|
||||||
|
|||||||
14
iso/overlay/etc/systemd/system/bee-hpc-tuning.service
Normal file
14
iso/overlay/etc/systemd/system/bee-hpc-tuning.service
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Bee: HPC tuning (CPU governor, C-states)
|
||||||
|
After=local-fs.target
|
||||||
|
Before=bee-nvidia.service bee-audit.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/usr/local/bin/bee-log-run /appdata/bee/export/bee-hpc-tuning.log /usr/local/bin/bee-hpc-tuning
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
|
RemainAfterExit=yes
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
41
iso/overlay/usr/local/bin/bee-hpc-tuning
Normal file
41
iso/overlay/usr/local/bin/bee-hpc-tuning
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# bee-hpc-tuning — apply HPC tuning for deterministic benchmarking
|
||||||
|
# Called by bee-hpc-tuning.service at boot.
|
||||||
|
|
||||||
|
log() { echo "[bee-hpc-tuning] $*"; }
|
||||||
|
|
||||||
|
# ── CPU governor ────────────────────────────────────────────────────────────
|
||||||
|
# Set all CPU cores to performance governor via sysfs.
|
||||||
|
# cpupower is not available; write directly to scaling_governor.
|
||||||
|
governor_ok=0
|
||||||
|
governor_fail=0
|
||||||
|
for gov_path in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
|
||||||
|
[ -f "$gov_path" ] || continue
|
||||||
|
if echo performance > "$gov_path" 2>/dev/null; then
|
||||||
|
governor_ok=$((governor_ok + 1))
|
||||||
|
else
|
||||||
|
governor_fail=$((governor_fail + 1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$governor_ok" -gt 0 ] && [ "$governor_fail" -eq 0 ]; then
|
||||||
|
log "CPU governor set to performance on ${governor_ok} core(s)"
|
||||||
|
elif [ "$governor_ok" -gt 0 ]; then
|
||||||
|
log "WARN: CPU governor: ${governor_ok} OK, ${governor_fail} failed"
|
||||||
|
elif [ "$governor_fail" -gt 0 ]; then
|
||||||
|
log "WARN: failed to set CPU governor on ${governor_fail} core(s)"
|
||||||
|
else
|
||||||
|
log "WARN: no cpufreq scaling_governor paths found (C-state governor or HW-controlled)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Transparent Huge Pages ───────────────────────────────────────────────────
|
||||||
|
# Kernel cmdline sets transparent_hugepage=always at boot, but confirm and log.
|
||||||
|
thp_path=/sys/kernel/mm/transparent_hugepage/enabled
|
||||||
|
if [ -f "$thp_path" ]; then
|
||||||
|
current=$(cat "$thp_path" 2>/dev/null)
|
||||||
|
log "transparent_hugepage: ${current}"
|
||||||
|
else
|
||||||
|
log "WARN: transparent_hugepage sysfs path not found"
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "done"
|
||||||
Reference in New Issue
Block a user