bee/audit/internal/webui/page_validate.go

package webui

import (
	"encoding/json"
	"fmt"
	"html"
	"sort"
	"strings"

	"bee/audit/internal/platform"
	"bee/audit/internal/schema"
)

// PCI vendor IDs used for GPU classification (source: pci-ids.ucw.cz).
const (
	pciVendorNvidia = 0x10de
	pciVendorAMD    = 0x1002
	pciVendorAspeed = 0x1a03
)

type validateInventory struct {
	CPU            string
	Memory         string
	Storage        string
	TPM            string
	NVIDIA         string
	AMD            string
	NvidiaGPUCount int
	AMDGPUCount    int
}

func validateFmtDur(secs int) string {
	if secs < 120 {
		return fmt.Sprintf("~%d s", secs)
	}
	mins := (secs + 29) / 60
	return fmt.Sprintf("~%d min", mins)
}

func validateTotalValidateSec(n int) int {
	if n < 0 {
		n = 0
	}
	total := platform.SATEstimatedCPUValidateSec +
		platform.SATEstimatedMemoryValidateSec +
		platform.SATEstimatedTPMValidateSec +
		platform.SATEstimatedNvidiaInterconnectSec +
		platform.SATEstimatedNvidiaBandwidthSec
	if n > 0 {
		total += platform.SATEstimatedNvidiaGPUValidateSec
	}
	return total
}

func validateTotalStressSec(n int) int {
	if n < 0 {
		n = 0
	}
	total := platform.SATEstimatedCPUStressSec +
		platform.SATEstimatedMemoryStressSec +
		platform.SATEstimatedNvidiaPulseTestSec +
		platform.SATEstimatedNvidiaInterconnectSec +
		platform.SATEstimatedNvidiaBandwidthSec
	if n > 0 {
		total += platform.SATEstimatedNvidiaGPUStressSec +
			platform.SATEstimatedNvidiaTargetedStressSec +
			platform.SATEstimatedNvidiaTargetedPowerSec
	}
	return total
}

func renderValidate(opts HandlerOptions) string {
	return renderValidateMode(opts, false)
}

func renderValidateStress(opts HandlerOptions) string {
	return renderValidateMode(opts, true)
}

func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
	inv := loadValidateInventory(opts)
	n := inv.NvidiaGPUCount
	validateTotalStr := validateFmtDur(validateTotalValidateSec(n))
	stressTotalStr := validateFmtDur(validateTotalStressSec(n))
	gpuNote := ""
	if n > 0 {
		gpuNote = fmt.Sprintf(" (%d GPU)", n)
	}
	estStr := validateTotalStr
	if stressDefault {
		estStr = stressTotalStr
	}
	alert := `<div class="alert alert-info" style="margin-bottom:16px"><strong>Non-destructive:</strong> Validate tests collect diagnostics only. They do not write to disks, do not run sustained load, and do not increment hardware wear counters.</div>`
	if stressDefault {
		alert = `<div class="alert alert-warn" style="margin-bottom:16px"><strong>&#9888; Stress mode:</strong> Runs extended load tests — CPU stress-ng, memory passes, DCGM targeted diagnostics. Higher wear than Validate.</div>`
	}

	stressOnlyCards := ""
	if stressDefault {
		stressOnlyCards = renderSATCard("nvidia-targeted-stress", "NVIDIA GPU Targeted Stress", "runNvidiaValidateSet('nvidia-targeted-stress')", "", renderValidateCardBody(
			inv.NVIDIA,
			`Runs a controlled NVIDIA DCGM load to check stability under moderate stress.`,
			`<code>dcgmi diag targeted_stress</code>`,
			validateFmtDur(platform.SATEstimatedNvidiaTargetedStressSec)+` (all GPUs simultaneously).`,
		)) +
			renderSATCard("nvidia-targeted-power", "NVIDIA Targeted Power", "runNvidiaValidateSet('nvidia-targeted-power')", "", renderValidateCardBody(
				inv.NVIDIA,
				`Checks that the GPU can sustain its declared power delivery envelope. Pass/fail determined by DCGM.`,
				`<code>dcgmi diag targeted_power</code>`,
				validateFmtDur(platform.SATEstimatedNvidiaTargetedPowerSec)+` (all GPUs simultaneously).`,
			)) +
			renderSATCard("nvidia-pulse", "NVIDIA PSU Pulse Test", "runNvidiaFabricValidate('nvidia-pulse')", "", renderValidateCardBody(
				inv.NVIDIA,
				`Tests power supply transient response by pulsing all GPUs simultaneously between idle and full load. Synchronous pulses across all GPUs create worst-case PSU load spikes — running per-GPU would miss PSU-level failures.`,
				`<code>dcgmi diag pulse_test</code>`,
				validateFmtDur(platform.SATEstimatedNvidiaPulseTestSec)+` (all GPUs simultaneously; measured on 8-GPU system).`,
			))
	}

	// Fan Ceiling Check is a platform cooling test (CPU/memory always, GPU only
	// as extra heat when present), so it sits with CPU/Memory/Storage, not in
	// the NVIDIA section. Load page only.
	fanCard := renderLoadOnlySATCard(stressDefault, renderSATCard("fan", "Fan Ceiling Check", "runSAT('fan')", "", renderValidateCardBody(
		"All system fans reported over IPMI / lm-sensors.",
		`Drives CPU (+memory) and, when a GPU is present, the hottest sustained GPU load (dcgmproftester targeted-power — the Power/Thermal Fit engine) at the same time, and watches every fan until none has climbed for ~1 min. The peak RPM reached is recorded as each fan's ceiling and is what the Topology view sizes the fan tiles and duty-cycle fill against. Success once every fan plateaus (or the time cap is hit). A fan reading 0 RPM or an IPMI status of cr/nr under full load fails. IPMI polling backs off automatically if the BMC gets slow under load. If the host cannot be loaded at all, or exposes no fan sensors, the task is cancelled as "not applicable" rather than failed — the platform does not support forcing fans, so this is the closest safe equivalent.`,
		`<code>stressapptest</code> / <code>stress-ng</code> + <code>dcgmproftester -t 1004</code> / <code>rvs gst</code>; <code>ipmitool sdr type Fan</code>`,
		`~3–10 min depending on how fast the fan curve settles (hard cap 15 min).`,
	)))

	satStressModeJS := "function satStressMode() { return false; }"
	if stressDefault {
		satStressModeJS = "function satStressMode() { return true; }"
	}

	return alert + `
<p style="color:var(--muted);font-size:13px;margin-bottom:16px">Tasks continue in the background — view progress in <a href="/tasks">Tasks</a>.</p>
<div style="display:flex;align-items:center;gap:12px;margin-bottom:16px">
  <button type="button" class="btn btn-primary" onclick="runAllSAT()">Run All</button>
  <span id="sat-all-status" style="font-size:12px;color:var(--muted)"></span>
  <span style="font-size:12px;color:var(--muted)">est. ` + estStr + gpuNote + `</span>
</div>

<div class="grid3">
` + renderSATCard("cpu", "CPU", "runSAT('cpu')", "", renderValidateCardBody(
		inv.CPU,
		`Collects CPU inventory and temperatures, then runs a bounded CPU stress pass.`,
		`<code>lscpu</code>, <code>sensors</code>, <code>stress-ng</code>`,
		validateFmtDur(platform.SATEstimatedCPUValidateSec)+` in Validate (stress-ng 60 s). `+validateFmtDur(platform.SATEstimatedCPUStressSec)+` in Stress (stress-ng 30 min).`,
	)) +
		renderSATCard("memory", "Memory", "runSAT('memory')", "", renderValidateCardBody(
			inv.Memory,
			`Runs a RAM validation pass and records memory state around the test.`,
			`<code>free</code>, <code>memtester</code>`,
			validateFmtDur(platform.SATEstimatedMemoryValidateSec)+` in Validate (256 MB × 1 pass). `+validateFmtDur(platform.SATEstimatedMemoryStressSec)+` in Stress (512 MB × 1 pass).`,
		)) +
		renderSATCard("storage", "Storage", "runSAT('storage')", "", renderValidateCardBody(
			inv.Storage,
			`Collects SMART data and runs a short self-test on each storage device.`,
			`<code>lsblk</code>; NVMe: <code>nvme id-ctrl</code>, <code>nvme smart-log</code>, <code>nvme device-self-test -s 1</code>; SATA/SAS: <code>smartctl -H -A</code>, <code>smartctl -t short</code>`,
			`~2 min per device (NVMe short self-test; SATA/SAS short self-test — duration device-dependent).`,
		)) +
		renderCheckOnlySATCard(stressDefault, renderSATCard("tpm", "TPM", "runSAT('tpm')", "", renderValidateCardBody(
			inv.TPM,
			`Checks TPM 2.0 communication and reports its fixed properties, allocated PCR banks, current PCR values, and the result of self-tests already performed by the TPM. It does not start a new self-test or change TPM state.`,
			`<code>tpm2_getcap properties-fixed</code>, <code>tpm2_getcap pcrs</code>, <code>tpm2_pcrread</code>, <code>tpm2_gettestresult</code>`,
			`Seconds - read-only queries; no ownership, NV, PCR, or key changes.`,
		))) +
		fanCard +
		`</div>
<div style="height:1px;background:var(--border);margin:16px 0"></div>
<div class="card" style="margin-bottom:16px">
  <div class="card-head">NVIDIA GPU Selection</div>
  <div class="card-body">
    <p style="font-size:12px;color:var(--muted);margin:0 0 8px">` + inv.NVIDIA + `</p>
    <p style="font-size:12px;color:var(--muted);margin:0 0 10px">All NVIDIA validate tasks use only the GPUs selected here. The same selection is used by Run All.</p>
    <div style="display:flex;gap:8px;flex-wrap:wrap;margin-bottom:8px">
      <button class="btn btn-sm btn-secondary" type="button" onclick="satSelectAllGPUs()">Select All</button>
      <button class="btn btn-sm btn-secondary" type="button" onclick="satSelectNoGPUs()">Clear</button>
    </div>
    <div id="sat-gpu-list" style="border:1px solid var(--border);border-radius:4px;padding:12px;min-height:88px">
      <p style="color:var(--muted);font-size:13px">Loading NVIDIA GPUs...</p>
    </div>
    <p id="sat-gpu-selection-note" style="font-size:12px;color:var(--muted);margin:10px 0 0">Select at least one NVIDIA GPU to enable NVIDIA validate tasks.</p>
  </div>
</div>

<div class="grid3">
` + renderSATCard("nvidia", "NVIDIA GPU", "runNvidiaValidateSet('nvidia')", "", renderValidateCardBody(
		inv.NVIDIA,
		`Runs NVIDIA diagnostics and board inventory checks.`,
		`<code>nvidia-smi</code>, <code>dmidecode</code>, <code>dcgmi diag</code>`,
		fmt.Sprintf("Validate: %s (Level 2, all GPUs simultaneously). Stress: %s (Level 3, all GPUs simultaneously).",
			validateFmtDur(platform.SATEstimatedNvidiaGPUValidateSec),
			validateFmtDur(platform.SATEstimatedNvidiaGPUStressSec)),
	)) +
		stressOnlyCards +
		renderCheckOnlySATCard(stressDefault, renderSATCard("nvidia-interconnect", "NVIDIA Interconnect (NCCL)", "runNvidiaFabricValidate('nvidia-interconnect')", "", renderValidateCardBody(
			inv.NVIDIA,
			`Verifies NVLink/NVSwitch fabric bandwidth using NCCL all_reduce_perf across all selected GPUs. Pass/fail based on achieved bandwidth vs. theoretical.`,
			`<code>all_reduce_perf</code> (NCCL tests)`,
			validateFmtDur(platform.SATEstimatedNvidiaInterconnectSec)+` (all GPUs simultaneously, requires ≥2).`,
		))) +
		renderSATCard("nvidia-bandwidth", "NVIDIA Bandwidth + PCIe Link", "runNvidiaFabricValidate('nvidia-bandwidth')", "", renderValidateCardBody(
			inv.NVIDIA,
			`Validates GPU memory copy and peer-to-peer bandwidth paths, then samples each GPU's negotiated PCIe speed and width immediately after real traffic. Idle or forced-retrain Gen1 readings do not fail the test.`,
			`<code>dcgmi diag -r nvbandwidth</code>, sysfs PCIe link attributes`,
			validateFmtDur(platform.SATEstimatedNvidiaBandwidthSec)+` (all GPUs simultaneously; nvbandwidth runs all built-in tests without a time limit - duration set by the tool).`,
		)) +
		`</div>
<div class="grid3" style="margin-top:16px">
` + renderCheckOnlySATCard(stressDefault, renderSATCard("amd", "AMD GPU", "runAMDValidateSet()", "", renderValidateCardBody(
		inv.AMD,
		`Runs the selected AMD checks only. GPU Validate collects inventory; MEM Integrity uses the RVS MEM module; MEM Bandwidth uses rocm-bandwidth-test and the RVS BABEL module.`,
		`GPU Validate: <code>rocm-smi</code>, <code>dmidecode</code>; MEM Integrity: <code>rvs mem</code>; MEM Bandwidth: <code>rocm-bandwidth-test</code>, <code>rvs babel</code>`,
		`<div style="display:flex;flex-direction:column;gap:4px"><label class="cb-row"><input type="checkbox" id="sat-amd-target" checked><span>GPU Validate</span></label><label class="cb-row"><input type="checkbox" id="sat-amd-mem-target" checked><span>MEM Integrity</span></label><label class="cb-row"><input type="checkbox" id="sat-amd-bandwidth-target" checked><span>MEM Bandwidth</span></label></div>`,
	))) +
		`</div>
<div id="sat-output" style="display:none;margin-top:16px" class="card">
  <div class="card-head">Test Output <span id="sat-title"></span></div>
  <div class="card-body"><div id="sat-terminal" class="terminal"></div></div>
</div>
<style>
.validate-card-body { padding:0; }
.validate-card-section { padding:12px 16px 0; }
.validate-card-section:last-child { padding-bottom:16px; }
</style>
<script>
let satES = null;
` + satStressModeJS + `
function satLabels() {
  return {nvidia:'Validate GPU', 'nvidia-targeted-stress':'NVIDIA Targeted Stress (dcgmi diag targeted_stress)', 'nvidia-targeted-power':'NVIDIA Targeted Power (dcgmi diag targeted_power)', 'nvidia-pulse':'NVIDIA PSU Pulse Test (dcgmi diag pulse_test)', 'nvidia-interconnect':'NVIDIA Interconnect (NCCL all_reduce_perf)', 'nvidia-bandwidth':'NVIDIA Bandwidth (NVBandwidth)', memory:'Validate Memory', storage:'Validate Storage', tpm:'Validate TPM (read-only)', cpu:'Validate CPU', amd:'Validate AMD GPU', 'amd-mem':'AMD GPU MEM Integrity', 'amd-bandwidth':'AMD GPU MEM Bandwidth'};
}
let satNvidiaGPUsPromise = null;
function loadSatNvidiaGPUs() {
  if (!satNvidiaGPUsPromise) {
    satNvidiaGPUsPromise = fetch('/api/gpu/nvidia')
      .then(r => {
        if (!r.ok) throw new Error('Failed to load NVIDIA GPUs.');
        return r.json();
      })
      .then(list => Array.isArray(list) ? list : []);
  }
  return satNvidiaGPUsPromise;
}
function satSelectedGPUIndices() {
  return Array.from(document.querySelectorAll('.sat-nvidia-checkbox'))
    .filter(function(el) { return el.checked && !el.disabled; })
    .map(function(el) { return parseInt(el.value, 10); })
    .filter(function(v) { return !Number.isNaN(v); })
    .sort(function(a, b) { return a - b; });
}
function satUpdateGPUSelectionNote() {
  const note = document.getElementById('sat-gpu-selection-note');
  if (!note) return;
  const selected = satSelectedGPUIndices();
  if (!selected.length) {
    note.textContent = 'Select at least one NVIDIA GPU to enable NVIDIA validate tasks.';
    return;
  }
  note.textContent = 'Selected GPUs: ' + selected.join(', ') + '. Multi-GPU tests will use all selected GPUs.';
}
function satRenderGPUList(gpus) {
  beeGpuPicker.render({
    rootId: 'sat-gpu-list', gpus: gpus,
    checkboxClass: 'sat-nvidia-checkbox', onToggle: 'satUpdateGPUSelectionNote()',
    after: function() { satUpdateGPUSelectionNote(); },
  });
}
function satSelectAllGPUs() {
  document.querySelectorAll('.sat-nvidia-checkbox').forEach(function(el) { el.checked = true; });
  satUpdateGPUSelectionNote();
}
function satSelectNoGPUs() {
  document.querySelectorAll('.sat-nvidia-checkbox').forEach(function(el) { el.checked = false; });
  satUpdateGPUSelectionNote();
}
function satLoadGPUs() {
  loadSatNvidiaGPUs().then(function(gpus) {
    satRenderGPUList(gpus);
  }).catch(function(err) {
    const root = document.getElementById('sat-gpu-list');
    if (root) {
      root.innerHTML = '<p style="color:var(--crit-fg);font-size:13px">Error: ' + err.message + '</p>';
    }
    satUpdateGPUSelectionNote();
  });
}
function satRequestBody(target, overrides) {
  const body = {};
  const labels = satLabels();
  body.display_name = labels[target] || ('Validate ' + target);
  body.stress_mode = satStressMode();
  if (target === 'cpu') body.duration = satStressMode() ? 1800 : 60;
  if (overrides) {
    Object.keys(overrides).forEach(key => { body[key] = overrides[key]; });
  }
  return body;
}
function enqueueSATTarget(target, overrides) {
  return fetch('/api/sat/'+target+'/run', {method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(satRequestBody(target, overrides))})
    .then(r => r.json());
}
function streamSATTask(taskId, title, resetTerminal) {
  if (satES) { satES.close(); satES = null; }
  document.getElementById('sat-output').style.display='block';
  document.getElementById('sat-title').textContent = '— ' + title;
  const term = document.getElementById('sat-terminal');
  if (resetTerminal) {
    term.textContent = '';
  }
  term.textContent += 'Task ' + taskId + ' queued. Streaming log...\n';
  return new Promise(function(resolve) {
    satES = new EventSource('/api/tasks/' + taskId + '/stream');
    satES.onmessage = function(e) { term.textContent += e.data + '\n'; term.scrollTop = term.scrollHeight; };
    satES.addEventListener('done', function(e) {
      satES.close();
      satES = null;
      term.textContent += (e.data ? '\nERROR: ' + e.data : '\nCompleted.') + '\n';
      term.scrollTop = term.scrollHeight;
      resolve({ok: !e.data, error: e.data || ''});
    });
    satES.onerror = function() {
      if (satES) {
        satES.close();
        satES = null;
      }
      term.textContent += '\nERROR: stream disconnected.\n';
      term.scrollTop = term.scrollHeight;
      resolve({ok: false, error: 'stream disconnected'});
    };
  });
}
function selectedAMDValidateTargets() {
  const targets = [];
  const gpu = document.getElementById('sat-amd-target');
  const mem = document.getElementById('sat-amd-mem-target');
  const bw = document.getElementById('sat-amd-bandwidth-target');
  if (gpu && gpu.checked && !gpu.disabled) targets.push('amd');
  if (mem && mem.checked && !mem.disabled) targets.push('amd-mem');
  if (bw && bw.checked && !bw.disabled) targets.push('amd-bandwidth');
  return targets;
}
function runSAT(target) {
  return runSATWithOverrides(target, null);
}
function runSATWithOverrides(target, overrides) {
  const title = (overrides && overrides.display_name) || target;
  const term = document.getElementById('sat-terminal');
  document.getElementById('sat-output').style.display='block';
  document.getElementById('sat-title').textContent = '— ' + title;
  term.textContent = 'Enqueuing ' + title + ' test...\n';
  return enqueueSATTarget(target, overrides)
    .then(d => streamSATTask(d.task_id, title, false));
}
function satAllGPUIndicesForMulti() {
  return Promise.resolve(satSelectedGPUIndices());
}
function runNvidiaFabricValidate(target) {
  satAllGPUIndicesForMulti().then(function(indices) {
    if (!indices.length) { alert('No NVIDIA GPUs available.'); return; }
    runSATWithOverrides(target, {gpu_indices: indices, display_name: satLabels()[target] || target});
  });
}
function runNvidiaValidateSet(target) {
  const selected = satSelectedGPUIndices();
  if (!selected.length) { alert('Select at least one NVIDIA GPU.'); return; }
  return runSATWithOverrides(target, {gpu_indices: selected, display_name: satLabels()[target] || target});
}
function runAMDValidateSet() {
  const targets = selectedAMDValidateTargets();
  if (!targets.length) return;
  if (targets.length === 1) return runSAT(targets[0]);
  document.getElementById('sat-output').style.display='block';
  document.getElementById('sat-title').textContent = '— amd';
  const term = document.getElementById('sat-terminal');
  term.textContent = 'Running AMD validate set one by one...\n';
  const labels = satLabels();
  const runNext = (idx) => {
    if (idx >= targets.length) return Promise.resolve();
    const target = targets[idx];
    term.textContent += '\n[' + (idx + 1) + '/' + targets.length + '] ' + labels[target] + '\n';
    return enqueueSATTarget(target)
      .then(d => {
        return streamSATTask(d.task_id, labels[target], false);
      }).then(function() {
        return runNext(idx + 1);
      });
  };
  return runNext(0);
}
// runAllSAT hands the whole decision to the backend: which hardware is
// present and ready, and therefore which tasks to enqueue, is decided by
// POST /api/sat/run-all. The page only sends operator intent.
function runAllSAT() {
  const status = document.getElementById('sat-all-status');
  status.textContent = 'Planning on server...';
  const body = {
    stress_mode: satStressMode(),
    amd_targets: selectedAMDValidateTargets(),
  };
  const gpuSubset = satSelectedGPUIndices();
  if (gpuSubset.length) body.nvidia_gpu_indices = gpuSubset;
  fetch('/api/sat/run-all', {
    method: 'POST',
    headers: {'Content-Type': 'application/json'},
    body: JSON.stringify(body),
  }).then(r => r.json().then(d => { if (!r.ok) throw new Error(d.error || ('HTTP ' + r.status)); return d; }))
    .then(d => {
      let msg = 'Enqueued ' + (d.task_count || 0) + ' task(s).';
      if (d.notes && d.notes.length) msg += ' ' + d.notes.join('; ');
      status.textContent = msg;
    })
    .catch(err => { status.textContent = 'Error: ' + err.message; });
}
</script>
<script>
fetch('/api/gpu/presence').then(r=>r.json()).then(gp => {
    if (!gp.nvidia) {
      const why = gp.nvidia_initializing
        ? 'NVIDIA GPU present, driver still initializing; Run All will wait for it'
        : 'No NVIDIA GPU detected';
      ['nvidia','nvidia-targeted-stress','nvidia-targeted-power','nvidia-pulse','nvidia-interconnect','nvidia-bandwidth']
        .forEach(t => disableSATCard(t, why));
    }
    if (!gp.amd) disableSATCard('amd', 'No AMD GPU detected');
    if (!gp.amd) disableSATAMDOptions('No AMD GPU detected');
});
satLoadGPUs();
function disableSATAMDOptions(reason) {
    ['sat-amd-target','sat-amd-mem-target','sat-amd-bandwidth-target'].forEach(function(id) {
        const cb = document.getElementById(id);
        if (!cb) return;
        cb.disabled = true;
        cb.checked = false;
        cb.title = reason;
    });
}
function disableSATCard(id, reason) {
    const btn = document.getElementById('sat-btn-' + id);
    if (!btn) return;
    btn.disabled = true;
    btn.title = reason;
    btn.style.opacity = '0.4';
    const card = btn.closest('.card');
    if (card) {
        let note = card.querySelector('.sat-unavail');
        if (!note) {
            note = document.createElement('p');
            note.className = 'sat-unavail';
            note.style.cssText = 'color:var(--muted);font-size:12px;margin:0 0 8px';
            const body = card.querySelector('.card-body');
            if (body) body.insertBefore(note, body.firstChild);
        }
        note.textContent = reason;
    }
}
</script>`
}

func loadValidateInventory(opts HandlerOptions) validateInventory {
	unknown := "Audit snapshot not loaded."
	out := validateInventory{
		CPU:     unknown,
		Memory:  unknown,
		Storage: unknown,
		TPM:     unknown,
		NVIDIA:  unknown,
		AMD:     unknown,
	}
	data, err := loadSnapshot(opts.AuditPath)
	if err != nil {
		return out
	}
	var snap schema.HardwareIngestRequest
	if err := json.Unmarshal(data, &snap); err != nil {
		return out
	}

	cpuCounts := map[string]int{}
	cpuTotal := 0
	for _, cpu := range snap.Hardware.CPUs {
		if cpu.Present != nil && !*cpu.Present {
			continue
		}
		cpuTotal++
		addValidateModel(cpuCounts, firstNonEmpty(validateTrimPtr(cpu.Model), validateTrimPtr(cpu.Manufacturer), "unknown"))
	}

	memCounts := map[string]int{}
	memTotal := 0
	for _, dimm := range snap.Hardware.Memory {
		if dimm.Present != nil && !*dimm.Present {
			continue
		}
		memTotal++
		addValidateModel(memCounts, firstNonEmpty(validateTrimPtr(dimm.PartNumber), validateTrimPtr(dimm.Type), validateTrimPtr(dimm.Manufacturer), "unknown"))
	}

	storageCounts := map[string]int{}
	storageTotal := 0
	for _, dev := range snap.Hardware.Storage {
		if dev.Present != nil && !*dev.Present {
			continue
		}
		storageTotal++
		addValidateModel(storageCounts, firstNonEmpty(validateTrimPtr(dev.Model), validateTrimPtr(dev.Manufacturer), "unknown"))
	}

	nvidiaCounts := map[string]int{}
	nvidiaTotal := 0
	amdCounts := map[string]int{}
	amdTotal := 0
	for _, dev := range snap.Hardware.PCIeDevices {
		if dev.Present != nil && !*dev.Present {
			continue
		}
		if validateIsVendorGPU(dev, "nvidia") {
			nvidiaTotal++
			addValidateModel(nvidiaCounts, firstNonEmpty(validateTrimPtr(dev.Model), validateTrimPtr(dev.Manufacturer), "unknown"))
		}
		if validateIsVendorGPU(dev, "amd") {
			amdTotal++
			addValidateModel(amdCounts, firstNonEmpty(validateTrimPtr(dev.Model), validateTrimPtr(dev.Manufacturer), "unknown"))
		}
	}

	out.CPU = formatValidateDeviceSummary(cpuTotal, cpuCounts, "CPU")
	out.Memory = formatValidateDeviceSummary(memTotal, memCounts, "module")
	out.Storage = formatValidateDeviceSummary(storageTotal, storageCounts, "device")
	out.TPM = formatValidateTPMSummary(snap.Hardware.PlatformConfig)
	out.NVIDIA = formatValidateDeviceSummary(nvidiaTotal, nvidiaCounts, "GPU")
	out.AMD = formatValidateDeviceSummary(amdTotal, amdCounts, "GPU")
	out.NvidiaGPUCount = nvidiaTotal
	out.AMDGPUCount = amdTotal
	return out
}

func formatValidateTPMSummary(raw *json.RawMessage) string {
	if raw == nil {
		return "TPM presence was not collected."
	}
	var config map[string]any
	if err := json.Unmarshal(*raw, &config); err != nil {
		return "TPM presence was not collected."
	}
	present, ok := config["TpmPresent"].(bool)
	if !ok {
		return "TPM presence was not collected."
	}
	if !present {
		return "No TPM detected."
	}
	parts := []string{"TPM detected"}
	for _, key := range []string{"TpmVersion", "TpmManufacturer", "TpmFirmwareVersion", "TpmInterface"} {
		if value, ok := config[key].(string); ok && strings.TrimSpace(value) != "" {
			parts = append(parts, html.EscapeString(value))
		}
	}
	return strings.Join(parts, " / ")
}

func renderValidateCardBody(devices, description, commands, settings string) string {
	return `<div class="validate-card-section"><div style="font-size:13px;color:var(--muted)">` + devices + `</div></div>` +
		`<div class="validate-card-section"><div style="font-size:13px">` + description + `</div></div>` +
		`<div class="validate-card-section"><div style="font-size:13px">` + commands + `</div></div>` +
		`<div class="validate-card-section"><div style="font-size:13px;color:var(--muted)">` + settings + `</div></div>`
}

func formatValidateDeviceSummary(total int, models map[string]int, unit string) string {
	if total == 0 {
		return "0 " + unit + "s detected."
	}
	keys := make([]string, 0, len(models))
	for key := range models {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	parts := make([]string, 0, len(keys))
	for _, key := range keys {
		parts = append(parts, fmt.Sprintf("%d x %s", models[key], html.EscapeString(key)))
	}
	label := unit
	if total != 1 {
		label += "s"
	}
	if len(parts) == 1 {
		return parts[0] + " " + label
	}
	return fmt.Sprintf("%d %s: %s", total, label, strings.Join(parts, ", "))
}

func addValidateModel(counts map[string]int, name string) {
	name = strings.TrimSpace(name)
	if name == "" {
		name = "unknown"
	}
	counts[name]++
}

func validateTrimPtr(value *string) string {
	if value == nil {
		return ""
	}
	return strings.TrimSpace(*value)
}

func validateIsVendorGPU(dev schema.HardwarePCIeDevice, vendor string) bool {
	if dev.VendorID != nil && *dev.VendorID == pciVendorAspeed {
		return false
	}
	isGPUClass := isGPUDeviceClass(validateTrimPtr(dev.DeviceClass))
	switch vendor {
	case "nvidia":
		return isGPUClass && dev.VendorID != nil && *dev.VendorID == pciVendorNvidia
	case "amd":
		return isGPUClass && dev.VendorID != nil && *dev.VendorID == pciVendorAMD
	default:
		return false
	}
}

func renderCheckOnlySATCard(stressMode bool, card string) string {
	if stressMode {
		return ""
	}
	return card
}

// renderLoadOnlySATCard is the inverse of renderCheckOnlySATCard: the card
// shows only on the Load page (a sustained-load test, not a Check-tier one).
func renderLoadOnlySATCard(stressMode bool, card string) string {
	if !stressMode {
		return ""
	}
	return card
}

// renderCheck renders the non-destructive Check page (step 2).
// Shows validate-mode tests only: CPU, Memory, Storage, NVIDIA L2, NCCL, NVBandwidth, AMD.
// Stress-mode tests (targeted-stress, targeted-power, pulse) are on the Load page.
func renderCheck(opts HandlerOptions) string {
	inv := loadValidateInventory(opts)
	n := inv.NvidiaGPUCount
	validateTotalStr := validateFmtDur(validateTotalValidateSec(n))
	gpuNote := ""
	if n > 0 {
		gpuNote = fmt.Sprintf(" (%d GPU)", n)
	}
	return `<div class="alert alert-info" style="margin-bottom:16px"><strong>Non-destructive:</strong> Check tests collect diagnostics only — no writes to disks, no sustained load, no hardware wear counters incremented. For stress testing, go to <a href="/burn">4. Burn</a>.</div>
<div style="display:flex;align-items:center;gap:12px;margin-bottom:16px">
  <button type="button" class="btn btn-primary" onclick="runAllCheckSAT()">Run All Checks</button>
  <span id="sat-all-status" style="font-size:12px;color:var(--muted)"></span>
  <span style="font-size:12px;color:var(--muted)">est. ` + validateTotalStr + gpuNote + `</span>
</div>

<div class="grid3">
` + renderSATCard("cpu", "CPU", "runSAT('cpu')", "", renderValidateCardBody(
		inv.CPU,
		`Collects CPU inventory and temperatures, then runs a bounded CPU stress pass.`,
		`<code>lscpu</code>, <code>sensors</code>, <code>stress-ng</code>`,
		validateFmtDur(platform.SATEstimatedCPUValidateSec)+` (stress-ng 60 s).`,
	)) +
		renderSATCard("memory", "Memory", "runSAT('memory')", "", renderValidateCardBody(
			inv.Memory,
			`Runs a RAM validation pass and records memory state around the test.`,
			`<code>free</code>, <code>memtester</code>`,
			validateFmtDur(platform.SATEstimatedMemoryValidateSec)+` (256 MB × 1 pass).`,
		)) +
		renderSATCard("storage", "Storage", "runSAT('storage')", "", renderValidateCardBody(
			inv.Storage,
			`Collects SMART health and attributes for each storage device. No self-test is triggered — read-only query only.`,
			`<code>lsblk</code>; NVMe: <code>nvme id-ctrl</code>, <code>nvme smart-log</code>; SATA/SAS: <code>smartctl -H -A</code>`,
			`Seconds — instantaneous device query, no wear counters incremented.`,
		)) +
		renderSATCard("tpm", "TPM", "runSAT('tpm')", "", renderValidateCardBody(
			inv.TPM,
			`Checks TPM 2.0 communication and reads fixed properties, allocated PCR banks, current PCR values, and the result of self-tests already performed by the TPM. It never starts a new self-test.`,
			`<code>tpm2_getcap properties-fixed</code>, <code>tpm2_getcap pcrs</code>, <code>tpm2_pcrread</code>, <code>tpm2_gettestresult</code>`,
			`Seconds - read-only queries; no ownership, NV, PCR, or key changes.`,
		)) +
		renderSATCard("nvidia-config", "GPU Config & NVLink", "runSAT('nvidia-config')", "", renderValidateCardBody(
			inv.NVIDIA,
			`Checks GPU configuration and NVLink topology that DCGM diag does not cover: ECC/MIG/power-limit drift from factory default, NVLink-bonded pair link count and error counters, and (informational) NVIDIA Confidential Computing readiness (CPU TEE support + GPU firmware CC capability). Read-only — changes nothing.`,
			`<code>nvidia-smi --query-gpu=...</code>, <code>nvidia-smi topo -m</code>, <code>nvidia-smi nvlink -s/-e</code>, <code>nvidia-smi conf-compute -q</code>, <code>dmesg</code>`,
			`Seconds — read-only query only.`,
		)) +
		`</div>
<div style="height:1px;background:var(--border);margin:16px 0"></div>
<div class="card" style="margin-bottom:16px">
  <div class="card-head">NVIDIA GPU Selection</div>
  <div class="card-body">
    <p style="font-size:12px;color:var(--muted);margin:0 0 8px">` + inv.NVIDIA + `</p>
    <div style="display:flex;gap:8px;flex-wrap:wrap;margin-bottom:8px">
      <button class="btn btn-sm btn-secondary" type="button" onclick="satSelectAllGPUs()">Select All</button>
      <button class="btn btn-sm btn-secondary" type="button" onclick="satSelectNoGPUs()">Clear</button>
    </div>
    <div id="sat-gpu-list" style="border:1px solid var(--border);border-radius:4px;padding:12px;min-height:88px">
      <p style="color:var(--muted);font-size:13px">Loading NVIDIA GPUs...</p>
    </div>
    <p id="sat-gpu-selection-note" style="font-size:12px;color:var(--muted);margin:10px 0 0">Select at least one NVIDIA GPU to enable NVIDIA check tasks.</p>
  </div>
</div>

<div class="grid3">
` + renderSATCard("nvidia", "NVIDIA GPU", "runNvidiaValidateSet('nvidia')", "", renderValidateCardBody(
		inv.NVIDIA,
		`Runs NVIDIA diagnostics and board inventory checks (DCGM Level 2).`,
		`<code>nvidia-smi</code>, <code>dmidecode</code>, <code>dcgmi diag</code>`,
		validateFmtDur(platform.SATEstimatedNvidiaGPUValidateSec)+` (Level 2, all GPUs simultaneously).`,
	)) +
		renderSATCard("nvidia-interconnect", "NVIDIA Interconnect (NCCL)", "runNvidiaFabricValidate('nvidia-interconnect')", "", renderValidateCardBody(
			inv.NVIDIA,
			`Verifies NVLink/NVSwitch fabric bandwidth using NCCL all_reduce_perf across all selected GPUs.`,
			`<code>all_reduce_perf</code> (NCCL tests)`,
			validateFmtDur(platform.SATEstimatedNvidiaInterconnectSec)+` (all GPUs simultaneously, requires ≥2).`,
		)) +
		renderSATCard("nvidia-bandwidth", "NVIDIA Bandwidth + PCIe Link", "runNvidiaFabricValidate('nvidia-bandwidth')", "", renderValidateCardBody(
			inv.NVIDIA,
			`Validates GPU memory copy and peer-to-peer bandwidth paths, then samples each GPU's negotiated PCIe speed and width immediately after real traffic.`,
			`<code>dcgmi diag -r nvbandwidth</code>, sysfs PCIe link attributes`,
			validateFmtDur(platform.SATEstimatedNvidiaBandwidthSec)+` (all GPUs simultaneously).`,
		)) +
		`</div>
<div class="grid3" style="margin-top:16px">
` + renderSATCard("amd", "AMD GPU", "runAMDValidateSet()", "", renderValidateCardBody(
		inv.AMD,
		`Runs AMD GPU inventory, MEM integrity, and MEM bandwidth checks.`,
		`GPU Validate: <code>rocm-smi</code>, <code>dmidecode</code>; MEM Integrity: <code>rvs mem</code>; MEM Bandwidth: <code>rocm-bandwidth-test</code>, <code>rvs babel</code>`,
		`<div style="display:flex;flex-direction:column;gap:4px"><label class="cb-row"><input type="checkbox" id="sat-amd-target" checked><span>GPU Validate</span></label><label class="cb-row"><input type="checkbox" id="sat-amd-mem-target" checked><span>MEM Integrity</span></label><label class="cb-row"><input type="checkbox" id="sat-amd-bandwidth-target" checked><span>MEM Bandwidth</span></label></div>`,
	)) +
		`</div>
<div id="sat-output" style="display:none;margin-top:16px" class="card">
  <div class="card-head">Test Output <span id="sat-title"></span></div>
  <div class="card-body"><div id="sat-terminal" class="terminal"></div></div>
</div>
<style>
.validate-card-body { padding:0; }
.validate-card-section { padding:12px 16px 0; }
.validate-card-section:last-child { padding-bottom:16px; }
.cb-row { display:flex; align-items:flex-start; gap:8px; padding:4px 0; cursor:pointer; font-size:13px; }
.cb-row input[type=checkbox] { width:16px; height:16px; margin-top:2px; flex-shrink:0; }
</style>
<script>
let satES = null;
function satLabels() {
  return {nvidia:'Check GPU (DCGM L2)', 'nvidia-interconnect':'NVIDIA Interconnect (NCCL)', 'nvidia-bandwidth':'NVIDIA Bandwidth + PCIe Link Under Load', memory:'Check Memory', storage:'Check Storage', tpm:'Check TPM (read-only)', cpu:'Check CPU', amd:'Check AMD GPU', 'amd-mem':'AMD GPU MEM Integrity', 'amd-bandwidth':'AMD GPU MEM Bandwidth', 'nvidia-config':'Check GPU Config & NVLink'};
}
let satNvidiaGPUsPromise = null;
function loadSatNvidiaGPUs() {
  if (!satNvidiaGPUsPromise) {
    satNvidiaGPUsPromise = fetch('/api/gpu/nvidia').then(r => {
      if (!r.ok) throw new Error('Failed to load NVIDIA GPUs.');
      return r.json();
    }).then(list => Array.isArray(list) ? list : []);
  }
  return satNvidiaGPUsPromise;
}
function satSelectedGPUIndices() {
  return Array.from(document.querySelectorAll('.sat-nvidia-checkbox'))
    .filter(el => el.checked && !el.disabled)
    .map(el => parseInt(el.value, 10))
    .filter(v => !Number.isNaN(v))
    .sort((a, b) => a - b);
}
function satUpdateGPUSelectionNote() {
  const note = document.getElementById('sat-gpu-selection-note');
  if (!note) return;
  const sel = satSelectedGPUIndices();
  note.textContent = sel.length
    ? 'Selected GPUs: ' + sel.join(', ') + '. Multi-GPU tests will use all selected GPUs.'
    : 'Select at least one NVIDIA GPU to enable NVIDIA check tasks.';
}
function satRenderGPUList(gpus) {
  beeGpuPicker.render({
    rootId: 'sat-gpu-list', gpus: gpus,
    checkboxClass: 'sat-nvidia-checkbox', onToggle: 'satUpdateGPUSelectionNote()',
    after: function() { satUpdateGPUSelectionNote(); },
  });
}
function satSelectAllGPUs() { document.querySelectorAll('.sat-nvidia-checkbox').forEach(el => { el.checked = true; }); satUpdateGPUSelectionNote(); }
function satSelectNoGPUs() { document.querySelectorAll('.sat-nvidia-checkbox').forEach(el => { el.checked = false; }); satUpdateGPUSelectionNote(); }
function satGPULoadInit() {
  loadSatNvidiaGPUs().then(satRenderGPUList).catch(err => {
    const root = document.getElementById('sat-gpu-list');
    if (root) root.innerHTML = '<p style="color:var(--crit-fg);font-size:13px">Error: ' + err.message + '</p>';
    satUpdateGPUSelectionNote();
  });
}
function satRequestBody(target, overrides) {
  const body = {};
  const labels = satLabels();
  body.display_name = labels[target] || ('Check ' + target);
  body.stress_mode = false;
  if (target === 'cpu') body.duration = 60;
  if (overrides) Object.keys(overrides).forEach(k => { body[k] = overrides[k]; });
  return body;
}
function enqueueSATTarget(target, overrides) {
  return fetch('/api/sat/' + target + '/run', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(satRequestBody(target, overrides))}).then(r => r.json());
}
function streamSATTask(taskId, title, resetTerminal) {
  if (satES) { satES.close(); satES = null; }
  document.getElementById('sat-output').style.display = 'block';
  document.getElementById('sat-title').textContent = '— ' + title;
  const term = document.getElementById('sat-terminal');
  if (resetTerminal) term.textContent = '';
  term.textContent += 'Task ' + taskId + ' queued. Streaming log...\n';
  return new Promise(resolve => {
    satES = new EventSource('/api/tasks/' + taskId + '/stream');
    satES.onmessage = e => { term.textContent += e.data + '\n'; term.scrollTop = term.scrollHeight; };
    satES.addEventListener('done', e => {
      satES.close(); satES = null;
      term.textContent += (e.data ? '\nERROR: ' + e.data : '\nCompleted.') + '\n';
      term.scrollTop = term.scrollHeight;
      resolve({ok: !e.data, error: e.data || ''});
    });
    satES.onerror = () => {
      if (satES) { satES.close(); satES = null; }
      term.textContent += '\nERROR: stream disconnected.\n';
      term.scrollTop = term.scrollHeight;
      resolve({ok: false, error: 'stream disconnected'});
    };
  });
}
function selectedAMDValidateTargets() {
  const targets = [];
  const gpu = document.getElementById('sat-amd-target');
  const mem = document.getElementById('sat-amd-mem-target');
  const bw = document.getElementById('sat-amd-bandwidth-target');
  if (gpu && gpu.checked && !gpu.disabled) targets.push('amd');
  if (mem && mem.checked && !mem.disabled) targets.push('amd-mem');
  if (bw && bw.checked && !bw.disabled) targets.push('amd-bandwidth');
  return targets;
}
function runSAT(target) { return runSATWithOverrides(target, null); }
function runSATWithOverrides(target, overrides) {
  const title = (overrides && overrides.display_name) || target;
  document.getElementById('sat-output').style.display = 'block';
  document.getElementById('sat-title').textContent = '— ' + title;
  const term = document.getElementById('sat-terminal');
  term.textContent = 'Enqueuing ' + title + ' test...\n';
  return enqueueSATTarget(target, overrides).then(d => streamSATTask(d.task_id, title, false));
}
function runNvidiaFabricValidate(target) {
  const indices = satSelectedGPUIndices();
  if (!indices.length) { alert('No NVIDIA GPUs available.'); return; }
  runSATWithOverrides(target, {gpu_indices: indices, display_name: satLabels()[target] || target});
}
function runNvidiaValidateSet(target) {
  const sel = satSelectedGPUIndices();
  if (!sel.length) { alert('Select at least one NVIDIA GPU.'); return; }
  return runSATWithOverrides(target, {gpu_indices: sel, display_name: satLabels()[target] || target});
}
function runAMDValidateSet() {
  const targets = selectedAMDValidateTargets();
  if (!targets.length) return;
  if (targets.length === 1) return runSAT(targets[0]);
  const term = document.getElementById('sat-terminal');
  document.getElementById('sat-output').style.display = 'block';
  document.getElementById('sat-title').textContent = '— amd';
  term.textContent = 'Running AMD check set...\n';
  const labels = satLabels();
  const runNext = idx => {
    if (idx >= targets.length) return Promise.resolve();
    const t = targets[idx];
    term.textContent += '\n[' + (idx + 1) + '/' + targets.length + '] ' + labels[t] + '\n';
    return enqueueSATTarget(t).then(d => streamSATTask(d.task_id, labels[t], false)).then(() => runNext(idx + 1));
  };
  return runNext(0);
}
// runAllCheckSAT delegates hardware detection and task planning to the
// backend (POST /api/sat/run-all). The browser no longer decides whether a
// GPU is present: a stale or empty GPU list can no longer silently drop the
// GPU checks.
function runAllCheckSAT() {
  const status = document.getElementById('sat-all-status');
  status.textContent = 'Planning on server...';
  const body = {stress_mode: false, amd_targets: selectedAMDValidateTargets()};
  const gpuSubset = satSelectedGPUIndices();
  if (gpuSubset.length) body.nvidia_gpu_indices = gpuSubset;
  fetch('/api/sat/run-all', {
    method: 'POST',
    headers: {'Content-Type': 'application/json'},
    body: JSON.stringify(body),
  }).then(r => r.json().then(d => { if (!r.ok) throw new Error(d.error || ('HTTP ' + r.status)); return d; }))
    .then(d => {
      let msg = 'Enqueued ' + (d.task_count || 0) + ' task(s).';
      if (d.notes && d.notes.length) msg += ' ' + d.notes.join('; ');
      status.textContent = msg;
    })
    .catch(err => { status.textContent = 'Error: ' + err.message; });
}
function disableSATCard(id, reason) {
  const btn = document.getElementById('sat-btn-' + id);
  if (!btn) return;
  btn.disabled = true; btn.title = reason; btn.style.opacity = '0.4';
  const card = btn.closest('.card');
  if (card) {
    let note = card.querySelector('.sat-unavail');
    if (!note) {
      note = document.createElement('p');
      note.className = 'sat-unavail';
      note.style.cssText = 'color:var(--muted);font-size:12px;margin:0 0 8px';
      const body = card.querySelector('.card-body');
      if (body) body.insertBefore(note, body.firstChild);
    }
    note.textContent = reason;
  }
}
fetch('/api/gpu/presence').then(r => r.json()).then(gp => {
  if (!gp.nvidia) {
    const why = gp.nvidia_initializing
      ? 'NVIDIA GPU present, driver still initializing; Run All will wait for it'
      : 'No NVIDIA GPU detected';
    ['nvidia','nvidia-interconnect','nvidia-bandwidth'].forEach(t => disableSATCard(t, why));
  }
  if (!gp.amd) {
    disableSATCard('amd', 'No AMD GPU detected');
    ['sat-amd-target','sat-amd-mem-target','sat-amd-bandwidth-target'].forEach(id => {
      const cb = document.getElementById(id);
      if (cb) { cb.disabled = true; cb.checked = false; }
    });
  }
});
satGPULoadInit();
</script>`
}

func renderSATCard(id, label, runAction, headerActions, body string) string {
	actions := `<button id="sat-btn-` + id + `" class="btn btn-primary btn-sm" onclick="` + runAction + `">Run</button>`
	if strings.TrimSpace(headerActions) != "" {
		actions += headerActions
	}
	return fmt.Sprintf(`<div class="card"><div class="card-head card-head-actions"><span>%s</span><div class="card-head-buttons">%s</div></div><div class="card-body validate-card-body">%s</div></div>`,
		label, actions, body)
}