From 4602f978367604e386eb86bb1259e37d71fd0cf4 Mon Sep 17 00:00:00 2001
From: Michael Chus <mike@mchus.pro>
Date: Sun, 5 Apr 2026 22:10:42 +0300
Subject: [PATCH] Enforce sequential task orchestration

---
 audit/internal/webui/pages.go                 | 243 ++++++++++++++----
 audit/internal/webui/server_test.go           |   6 +-
 audit/internal/webui/tasks.go                 |  30 +--
 .../hooks/normal/9001-wallpaper.hook.chroot   |  24 +-
 4 files changed, 213 insertions(+), 90 deletions(-)

diff --git a/audit/internal/webui/pages.go b/audit/internal/webui/pages.go
index 18ec4d3..b70f05b 100644
--- a/audit/internal/webui/pages.go
+++ b/audit/internal/webui/pages.go
@@ -1068,17 +1068,23 @@ func renderValidate(opts HandlerOptions) string {
 		`</div>
 <div style="height:1px;background:var(--border);margin:16px 0"></div>
 <div class="grid3">
-` + renderSATCard("nvidia", "NVIDIA GPU", "runNvidiaValidateSet('nvidia')", "", renderValidateCardBody(
+` + renderSATCard("nvidia-selection", "NVIDIA GPU Selection", "", "", renderValidateCardBody(
+		inv.NVIDIA,
+		`Select which NVIDIA GPUs to include in Validate. The same selection is used by both NVIDIA GPU cards below and by Validate one by one.`,
+		`<code>nvidia-smi --query-gpu=index,name,memory.total</code>`,
+	`<div id="sat-gpu-list"><p style="color:var(--muted);font-size:13px">Loading NVIDIA GPUs…</p></div><div style="display:flex;gap:8px;flex-wrap:wrap;margin-top:8px"><button type="button" class="btn btn-sm btn-secondary" onclick="satSelectAllGPUs()">Select all</button><button type="button" class="btn btn-sm btn-secondary" onclick="satSelectNoGPUs()">Clear</button></div><div id="sat-gpu-selection-note" style="font-size:12px;color:var(--muted);margin-top:8px"></div>`,
+	)) +
+	renderSATCard("nvidia", "NVIDIA GPU", "runNvidiaValidateSet('nvidia')", "", renderValidateCardBody(
 		inv.NVIDIA,
 		`Runs NVIDIA diagnostics and board inventory checks.`,
 		`<code>nvidia-smi</code>, <code>dmidecode</code>, <code>dcgmi diag</code>`,
-		`Runs one GPU at a time. Diag level is taken from Validate Profile.`,
+		`Runs one GPU at a time on the selected NVIDIA GPUs. Diag level is taken from Validate Profile.`,
 	)) +
 		renderSATCard("nvidia-targeted-stress", "NVIDIA GPU Targeted Stress", "runNvidiaValidateSet('nvidia-targeted-stress')", "", renderValidateCardBody(
 			inv.NVIDIA,
 			`Runs a controlled NVIDIA DCGM load in Validate to check stability under moderate stress.`,
 			`<code>dcgmi diag targeted_stress</code>`,
-			`Runs one GPU at a time with the fixed DCGM targeted stress recipe.`,
+			`Runs one GPU at a time on the selected NVIDIA GPUs with the fixed DCGM targeted stress recipe.`,
 		)) +
 		`</div>
 <div class="grid3" style="margin-top:16px">
@@ -1100,6 +1106,8 @@ func renderValidate(opts HandlerOptions) string {
 .validate-card-body { padding:0; }
 .validate-card-section { padding:12px 16px 0; }
 .validate-card-section:last-child { padding-bottom:16px; }
+.sat-gpu-row { display:flex; align-items:flex-start; gap:8px; padding:6px 0; cursor:pointer; font-size:13px; }
+.sat-gpu-row input[type=checkbox] { width:16px; height:16px; margin-top:2px; flex-shrink:0; }
 @media(max-width:900px){ .validate-profile-body { grid-template-columns:1fr; } }
 </style>
 <script>
@@ -1128,6 +1136,59 @@ function loadSatNvidiaGPUs() {
   }
   return satNvidiaGPUsPromise;
 }
+function satSelectedGPUIndices() {
+  return Array.from(document.querySelectorAll('.sat-nvidia-checkbox'))
+    .filter(function(el) { return el.checked && !el.disabled; })
+    .map(function(el) { return parseInt(el.value, 10); })
+    .filter(function(v) { return !Number.isNaN(v); })
+    .sort(function(a, b) { return a - b; });
+}
+function satUpdateGPUSelectionNote() {
+  const note = document.getElementById('sat-gpu-selection-note');
+  if (!note) return;
+  const selected = satSelectedGPUIndices();
+  if (!selected.length) {
+    note.textContent = 'Select at least one NVIDIA GPU to enable NVIDIA validate tasks.';
+    return;
+  }
+  note.textContent = 'Selected NVIDIA GPUs: ' + selected.join(', ') + '.';
+}
+function satRenderGPUList(gpus) {
+  const root = document.getElementById('sat-gpu-list');
+  if (!root) return;
+  if (!gpus || !gpus.length) {
+    root.innerHTML = '<p style="color:var(--muted);font-size:13px">No NVIDIA GPUs detected.</p>';
+    satUpdateGPUSelectionNote();
+    return;
+  }
+  root.innerHTML = gpus.map(function(gpu) {
+    const mem = gpu.memory_mb > 0 ? ' · ' + gpu.memory_mb + ' MiB' : '';
+    return '<label class="sat-gpu-row">'
+      + '<input class="sat-nvidia-checkbox" type="checkbox" value="' + gpu.index + '" checked onchange="satUpdateGPUSelectionNote()">'
+      + '<span><strong>GPU ' + gpu.index + '</strong> — ' + gpu.name + mem + '</span>'
+      + '</label>';
+  }).join('');
+  satUpdateGPUSelectionNote();
+}
+function satSelectAllGPUs() {
+  document.querySelectorAll('.sat-nvidia-checkbox').forEach(function(el) { el.checked = true; });
+  satUpdateGPUSelectionNote();
+}
+function satSelectNoGPUs() {
+  document.querySelectorAll('.sat-nvidia-checkbox').forEach(function(el) { el.checked = false; });
+  satUpdateGPUSelectionNote();
+}
+function satLoadGPUs() {
+  loadSatNvidiaGPUs().then(function(gpus) {
+    satRenderGPUList(gpus);
+  }).catch(function(err) {
+    const root = document.getElementById('sat-gpu-list');
+    if (root) {
+      root.innerHTML = '<p style="color:var(--crit-fg);font-size:13px">Error: ' + err.message + '</p>';
+    }
+    satUpdateGPUSelectionNote();
+  });
+}
 function satGPUDisplayName(gpu) {
   const idx = (gpu && Number.isFinite(Number(gpu.index))) ? Number(gpu.index) : 0;
   const name = gpu && gpu.name ? gpu.name : ('GPU ' + idx);
@@ -1149,6 +1210,36 @@ function enqueueSATTarget(target, overrides) {
   return fetch('/api/sat/'+target+'/run', {method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(satRequestBody(target, overrides))})
     .then(r => r.json());
 }
+function streamSATTask(taskId, title, resetTerminal) {
+  if (satES) { satES.close(); satES = null; }
+  document.getElementById('sat-output').style.display='block';
+  document.getElementById('sat-title').textContent = '— ' + title;
+  const term = document.getElementById('sat-terminal');
+  if (resetTerminal) {
+    term.textContent = '';
+  }
+  term.textContent += 'Task ' + taskId + ' queued. Streaming log...\n';
+  return new Promise(function(resolve) {
+    satES = new EventSource('/api/tasks/' + taskId + '/stream');
+    satES.onmessage = function(e) { term.textContent += e.data + '\n'; term.scrollTop = term.scrollHeight; };
+    satES.addEventListener('done', function(e) {
+      satES.close();
+      satES = null;
+      term.textContent += (e.data ? '\nERROR: ' + e.data : '\nCompleted.') + '\n';
+      term.scrollTop = term.scrollHeight;
+      resolve({ok: !e.data, error: e.data || ''});
+    });
+    satES.onerror = function() {
+      if (satES) {
+        satES.close();
+        satES = null;
+      }
+      term.textContent += '\nERROR: stream disconnected.\n';
+      term.scrollTop = term.scrollHeight;
+      resolve({ok: false, error: 'stream disconnected'});
+    };
+  });
+}
 function selectedAMDValidateTargets() {
   const targets = [];
   const gpu = document.getElementById('sat-amd-target');
@@ -1163,24 +1254,23 @@ function runSAT(target) {
   return runSATWithOverrides(target, null);
 }
 function runSATWithOverrides(target, overrides) {
-  if (satES) { satES.close(); satES = null; }
-  document.getElementById('sat-output').style.display='block';
-  document.getElementById('sat-title').textContent = '— ' + target;
+  const title = (overrides && overrides.display_name) || target;
   const term = document.getElementById('sat-terminal');
-  term.textContent = 'Enqueuing ' + target + ' test...\n';
+  document.getElementById('sat-output').style.display='block';
+  document.getElementById('sat-title').textContent = '— ' + title;
+  term.textContent = 'Enqueuing ' + title + ' test...\n';
   return enqueueSATTarget(target, overrides)
-    .then(d => {
-      term.textContent += 'Task ' + d.task_id + ' queued. Streaming log...\n';
-      satES = new EventSource('/api/tasks/'+d.task_id+'/stream');
-      satES.onmessage = e => { term.textContent += e.data+'\n'; term.scrollTop=term.scrollHeight; };
-      satES.addEventListener('done', e => { satES.close(); satES=null; term.textContent += (e.data ? '\nERROR: '+e.data : '\nCompleted.')+'\n'; });
-    });
+    .then(d => streamSATTask(d.task_id, title, false));
 }
 function expandSATTarget(target) {
   if (target !== 'nvidia' && target !== 'nvidia-targeted-stress') {
     return Promise.resolve([{target: target}]);
   }
-  return loadSatNvidiaGPUs().then(gpus => gpus.map(gpu => ({
+  const selected = satSelectedGPUIndices();
+  if (!selected.length) {
+    return Promise.reject(new Error('Select at least one NVIDIA GPU.'));
+  }
+  return loadSatNvidiaGPUs().then(gpus => gpus.filter(gpu => selected.indexOf(Number(gpu.index)) >= 0).map(gpu => ({
     target: target,
     overrides: {
       gpu_indices: [Number(gpu.index)],
@@ -1191,65 +1281,61 @@ function expandSATTarget(target) {
 }
 function runNvidiaValidateSet(target) {
   return loadSatNvidiaGPUs().then(gpus => {
-    if (!gpus.length) return;
-    if (gpus.length === 1) {
-      const gpu = gpus[0];
+    const selected = satSelectedGPUIndices();
+    const picked = gpus.filter(gpu => selected.indexOf(Number(gpu.index)) >= 0);
+    if (!picked.length) {
+      throw new Error('Select at least one NVIDIA GPU.');
+    }
+    if (picked.length === 1) {
+      const gpu = picked[0];
       return runSATWithOverrides(target, {
         gpu_indices: [Number(gpu.index)],
         display_name: (satLabels()[target] || ('Validate ' + target)) + ' (' + satGPUDisplayName(gpu) + ')'
       });
     }
-    if (satES) { satES.close(); satES = null; }
     document.getElementById('sat-output').style.display='block';
     document.getElementById('sat-title').textContent = '— ' + target;
     const term = document.getElementById('sat-terminal');
-    term.textContent = 'Enqueuing ' + target + ' tests one GPU at a time...\n';
+    term.textContent = 'Running ' + target + ' one GPU at a time...\n';
     const labelBase = satLabels()[target] || ('Validate ' + target);
-    const enqueueNext = (idx) => {
-      if (idx >= gpus.length) return;
-      const gpu = gpus[idx];
+    const runNext = (idx) => {
+      if (idx >= picked.length) return Promise.resolve();
+      const gpu = picked[idx];
       const gpuLabel = satGPUDisplayName(gpu);
-      enqueueSATTarget(target, {
+      term.textContent += '\n[' + (idx + 1) + '/' + picked.length + '] ' + gpuLabel + '\n';
+      return enqueueSATTarget(target, {
         gpu_indices: [Number(gpu.index)],
         display_name: labelBase + ' (' + gpuLabel + ')'
       }).then(d => {
-        term.textContent += 'Task ' + d.task_id + ' queued for ' + gpuLabel + '.\n';
-        if (idx === gpus.length - 1) {
-          satES = new EventSource('/api/tasks/' + d.task_id + '/stream');
-          satES.onmessage = e => { term.textContent += e.data+'\n'; term.scrollTop=term.scrollHeight; };
-          satES.addEventListener('done', e => { satES.close(); satES=null; term.textContent += (e.data ? '\nERROR: '+e.data : '\nCompleted.')+'\n'; });
-        }
-        enqueueNext(idx + 1);
+        return streamSATTask(d.task_id, labelBase + ' (' + gpuLabel + ')', false);
+      }).then(function() {
+        return runNext(idx + 1);
       });
     };
-    enqueueNext(0);
+    return runNext(0);
   });
 }
 function runAMDValidateSet() {
   const targets = selectedAMDValidateTargets();
   if (!targets.length) return;
   if (targets.length === 1) return runSAT(targets[0]);
-  if (satES) { satES.close(); satES = null; }
   document.getElementById('sat-output').style.display='block';
   document.getElementById('sat-title').textContent = '— amd';
   const term = document.getElementById('sat-terminal');
-  term.textContent = 'Enqueuing AMD validate set...\n';
+  term.textContent = 'Running AMD validate set one by one...\n';
   const labels = satLabels();
-  const enqueueNext = (idx) => {
-    if (idx >= targets.length) return;
+  const runNext = (idx) => {
+    if (idx >= targets.length) return Promise.resolve();
     const target = targets[idx];
-    enqueueSATTarget(target)
+    term.textContent += '\n[' + (idx + 1) + '/' + targets.length + '] ' + labels[target] + '\n';
+    return enqueueSATTarget(target)
       .then(d => {
-        term.textContent += 'Task ' + d.task_id + ' queued for ' + labels[target] + '.\n';
-        if (idx === targets.length - 1) {
-          satES = new EventSource('/api/tasks/'+d.task_id+'/stream');
-          satES.onmessage = e => { term.textContent += e.data+'\n'; term.scrollTop=term.scrollHeight; };
-          satES.addEventListener('done', e => { satES.close(); satES=null; term.textContent += (e.data ? '\nERROR: '+e.data : '\nCompleted.')+'\n'; });
-        }
-        enqueueNext(idx + 1);
+        return streamSATTask(d.task_id, labels[target], false);
+      }).then(function() {
+        return runNext(idx + 1);
       });
   };
-  enqueueNext(0);
+  return runNext(0);
 }
 function runAllSAT() {
   const cycles = Math.max(1, parseInt(document.getElementById('sat-cycles').value)||1);
@@ -1271,17 +1357,17 @@ function runAllSAT() {
       status.textContent = 'No tasks selected.';
       return;
     }
-    const enqueueNext = (idx) => {
-      if (idx >= expanded.length) { status.textContent = 'Enqueued ' + total + ' tasks.'; return; }
+    const runNext = (idx) => {
+      if (idx >= expanded.length) { status.textContent = 'Completed ' + total + ' task(s).'; return Promise.resolve(); }
       const item = expanded[idx];
-      enqueueSATTarget(item.target, item.overrides)
+      status.textContent = 'Running ' + (idx + 1) + '/' + total + '...';
+      return enqueueSATTarget(item.target, item.overrides)
         .then(() => {
           enqueued++;
-          status.textContent = 'Enqueued ' + enqueued + '/' + total + '...';
-          enqueueNext(idx + 1);
+          return runNext(idx + 1);
         });
     };
-    enqueueNext(0);
+    return runNext(0);
   }).catch(err => {
     status.textContent = 'Error: ' + err.message;
   });
@@ -1294,6 +1380,7 @@ fetch('/api/gpu/presence').then(r=>r.json()).then(gp => {
     if (!gp.amd) disableSATCard('amd', 'No AMD GPU detected');
     if (!gp.amd) disableSATAMDOptions('No AMD GPU detected');
 });
+satLoadGPUs();
 function disableSATAMDOptions(reason) {
     ['sat-amd-target','sat-amd-mem-target','sat-amd-bandwidth-target'].forEach(function(id) {
         const cb = document.getElementById(id);
@@ -1886,6 +1973,36 @@ function streamTask(taskId, label) {
     term.scrollTop = term.scrollHeight;
   });
 }
+function streamBurnTask(taskId, label, resetTerminal) {
+  if (biES) { biES.close(); biES = null; }
+  document.getElementById('bi-output').style.display = 'block';
+  document.getElementById('bi-title').textContent = '— ' + label + ' [' + burnProfile() + ']';
+  const term = document.getElementById('bi-terminal');
+  if (resetTerminal) {
+    term.textContent = '';
+  }
+  term.textContent += 'Task ' + taskId + ' queued. Streaming...\n';
+  return new Promise(function(resolve) {
+    biES = new EventSource('/api/tasks/' + taskId + '/stream');
+    biES.onmessage = function(e) { term.textContent += e.data + '\n'; term.scrollTop = term.scrollHeight; };
+    biES.addEventListener('done', function(e) {
+      biES.close();
+      biES = null;
+      term.textContent += (e.data ? '\nERROR: ' + e.data : '\nCompleted.') + '\n';
+      term.scrollTop = term.scrollHeight;
+      resolve({ok: !e.data, error: e.data || ''});
+    });
+    biES.onerror = function() {
+      if (biES) {
+        biES.close();
+        biES = null;
+      }
+      term.textContent += '\nERROR: stream disconnected.\n';
+      term.scrollTop = term.scrollHeight;
+      resolve({ok: false, error: 'stream disconnected'});
+    };
+  });
+}
 
 function runBurnTaskSet(tasks, statusElId) {
   const enabled = tasks.filter(function(t) {
@@ -1898,19 +2015,33 @@ function runBurnTaskSet(tasks, statusElId) {
     if (status) status.textContent = 'No tasks selected.';
     return;
   }
-  enabled.forEach(function(t) {
-    enqueueBurnTask(t.target, t.label, t.extra, !!t.nvidia)
+  const term = document.getElementById('bi-terminal');
+  document.getElementById('bi-output').style.display = 'block';
+  document.getElementById('bi-title').textContent = '— Burn one by one [' + burnProfile() + ']';
+  term.textContent = '';
+  const runNext = function(idx) {
+    if (idx >= enabled.length) {
+      if (status) status.textContent = 'Completed ' + enabled.length + ' task(s).';
+      return Promise.resolve();
+    }
+    const t = enabled[idx];
+    term.textContent += '\n[' + (idx + 1) + '/' + enabled.length + '] ' + t.label + '\n';
+    if (status) status.textContent = 'Running ' + (idx + 1) + '/' + enabled.length + '...';
+    return enqueueBurnTask(t.target, t.label, t.extra, !!t.nvidia)
       .then(function(d) {
-        if (status) status.textContent = enabled.length + ' task(s) queued.';
-        streamTask(d.task_id, t.label);
+        return streamBurnTask(d.task_id, t.label, false);
+      })
+      .then(function() {
+        return runNext(idx + 1);
       })
       .catch(function(err) {
         if (status) status.textContent = 'Error: ' + err.message;
-        const term = document.getElementById('bi-terminal');
         document.getElementById('bi-output').style.display = 'block';
         term.textContent += 'ERROR: ' + err.message + '\n';
+        return Promise.reject(err);
       });
-  });
+  };
+  return runNext(0);
 }
 
 function runPlatformStress() {
diff --git a/audit/internal/webui/server_test.go b/audit/internal/webui/server_test.go
index 7e5a3a5..89d9735 100644
--- a/audit/internal/webui/server_test.go
+++ b/audit/internal/webui/server_test.go
@@ -601,8 +601,8 @@ func TestToolsPageRendersRestartGPUDriversButton(t *testing.T) {
 	if !strings.Contains(body, `Restart GPU Drivers`) {
 		t.Fatalf("tools page missing restart gpu drivers button: %s", body)
 	}
-	if !strings.Contains(body, `svcAction('bee-nvidia', 'restart')`) {
-		t.Fatalf("tools page missing bee-nvidia restart action: %s", body)
+	if !strings.Contains(body, `restartGPUDrivers()`) {
+		t.Fatalf("tools page missing restartGPUDrivers action: %s", body)
 	}
 	if !strings.Contains(body, `id="boot-source-text"`) {
 		t.Fatalf("tools page missing boot source field: %s", body)
@@ -649,6 +649,8 @@ func TestValidatePageRendersNvidiaTargetedStressCard(t *testing.T) {
 		`nvidia-targeted-stress`,
 		`controlled NVIDIA DCGM load`,
 		`<code>dcgmi diag targeted_stress</code>`,
+		`NVIDIA GPU Selection`,
+		`id="sat-gpu-list"`,
 	} {
 		if !strings.Contains(body, needle) {
 			t.Fatalf("validate page missing %q: %s", needle, body)
diff --git a/audit/internal/webui/tasks.go b/audit/internal/webui/tasks.go
index 19056b3..35b2e70 100644
--- a/audit/internal/webui/tasks.go
+++ b/audit/internal/webui/tasks.go
@@ -423,13 +423,14 @@ func (q *taskQueue) worker() {
 			setCPUGovernor("performance")
 			defer setCPUGovernor("powersave")
 
-			// Drain all pending tasks and start them in parallel.
-			q.mu.Lock()
-			var batch []*Task
 			for {
+				q.mu.Lock()
 				t := q.nextPending()
 				if t == nil {
-					break
+					q.prune()
+					q.persistLocked()
+					q.mu.Unlock()
+					return
 				}
 				now := time.Now()
 				t.Status = TaskRunning
@@ -438,29 +439,14 @@ func (q *taskQueue) worker() {
 				t.ErrMsg = ""
 				j := newTaskJobState(t.LogPath, taskSerialPrefix(t))
 				t.job = j
-				batch = append(batch, t)
-			}
-			if len(batch) > 0 {
 				q.persistLocked()
-			}
-			q.mu.Unlock()
+				q.mu.Unlock()
 
-			var wg sync.WaitGroup
-			for _, t := range batch {
-				t := t
-				j := t.job
 				taskCtx, taskCancel := context.WithCancel(context.Background())
 				j.cancel = taskCancel
-				wg.Add(1)
-				goRecoverOnce("task "+t.Target, func() {
-					defer wg.Done()
-					defer taskCancel()
-					q.executeTask(t, j, taskCtx)
-				})
-			}
-			wg.Wait()
+				q.executeTask(t, j, taskCtx)
+				taskCancel()
 
-			if len(batch) > 0 {
 				q.mu.Lock()
 				q.prune()
 				q.persistLocked()
diff --git a/iso/builder/config/hooks/normal/9001-wallpaper.hook.chroot b/iso/builder/config/hooks/normal/9001-wallpaper.hook.chroot
index 88ee77a..b22c25a 100755
--- a/iso/builder/config/hooks/normal/9001-wallpaper.hook.chroot
+++ b/iso/builder/config/hooks/normal/9001-wallpaper.hook.chroot
@@ -44,23 +44,27 @@ else:
 img  = Image.new('RGB', (W, H), (0, 0, 0))
 draw = ImageDraw.Draw(img)
 
-# Measure logo block
+# Measure logo block line by line to avoid font ascender offset
 lines = LOGO.split('\n')
-bbox = draw.textbbox((0, 0), LOGO, font=font_logo)
-text_w = bbox[2] - bbox[0]
-text_h = bbox[3] - bbox[1]
+logo_lines = lines[:6]
+sub_line   = lines[6] if len(lines) > 6 else ''
 
-x = (W - text_w) // 2
-y = (H - text_h) // 2
+line_h = SIZE + 2
+block_h = len(logo_lines) * line_h + 8 + (SIZE if sub_line else 0)
 
-# Draw logo lines: first 6 in amber, last line (subtitle) dimmer
-logo_lines  = lines[:6]
-sub_line    = lines[6] if len(lines) > 6 else ''
+# Width: measure the widest logo line
+max_w = 0
+for line in logo_lines:
+    bb = draw.textbbox((0, 0), line, font=font_logo)
+    max_w = max(max_w, bb[2] - bb[0])
+
+x = (W - max_w) // 2
+y = (H - block_h) // 2
 
 cy = y
 for line in logo_lines:
     draw.text((x, cy), line, font=font_logo, fill=(0xf6, 0xc9, 0x0e))
-    cy += SIZE + 2
+    cy += line_h
 cy += 8
 if sub_line:
     draw.text((x, cy), sub_line, font=font_sub, fill=(0x80, 0x68, 0x18))