feat(tui): NVIDIA SAT with nvtop, GPU selection, metrics and chart — v1.0.0

- TUI: duration presets (10m/1h/8h/24h), GPU multi-select checkboxes - nvtop launched concurrently with SAT via tea.ExecProcess; can reopen or abort - GPU metrics collected per-second during bee-gpu-stress (temp/usage/power/clock) - Outputs: gpu-metrics.csv, gpu-metrics.html (offline SVG), gpu-metrics-term.txt - Terminal chart: asciigraph-style line chart with box-drawing chars and ANSI colours - AUDIT_VERSION bumped 0.1.1 → 1.0.0; nvtop added to ISO package list - runtime-flows.md updated with full NVIDIA SAT TUI flow documentation Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-18 15:18:57 +03:00
parent b965184e71
commit 76a17937f3
14 changed files with 1162 additions and 24 deletions
@@ -3,6 +3,7 @@ package platform
 import (
 	"archive/tar"
 	"compress/gzip"
+	"context"
 	"fmt"
 	"io"
 	"os"
@@ -14,10 +15,55 @@ import (
 	"time"
 )

+// NvidiaGPU holds basic GPU info from nvidia-smi.
+type NvidiaGPU struct {
+	Index    int
+	Name     string
+	MemoryMB int
+}
+
+// ListNvidiaGPUs returns GPUs visible to nvidia-smi.
+func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) {
+	out, err := exec.Command("nvidia-smi",
+		"--query-gpu=index,name,memory.total",
+		"--format=csv,noheader,nounits").Output()
+	if err != nil {
+		return nil, fmt.Errorf("nvidia-smi: %w", err)
+	}
+	var gpus []NvidiaGPU
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		parts := strings.SplitN(line, ", ", 3)
+		if len(parts) != 3 {
+			continue
+		}
+		idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
+		if err != nil {
+			continue
+		}
+		memMB, _ := strconv.Atoi(strings.TrimSpace(parts[2]))
+		gpus = append(gpus, NvidiaGPU{
+			Index:    idx,
+			Name:     strings.TrimSpace(parts[1]),
+			MemoryMB: memMB,
+		})
+	}
+	return gpus, nil
+}
+
 func (s *System) RunNvidiaAcceptancePack(baseDir string) (string, error) {
 	return runAcceptancePack(baseDir, "gpu-nvidia", nvidiaSATJobs())
 }

+// RunNvidiaAcceptancePackWithOptions runs the NVIDIA SAT with explicit duration,
+// GPU memory size, and GPU index selection. ctx cancellation kills the running job.
+func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, durationSec int, sizeMB int, gpuIndices []int) (string, error) {
+	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia", nvidiaSATJobsWithOptions(durationSec, sizeMB, gpuIndices))
+}
+
 func (s *System) RunMemoryAcceptancePack(baseDir string) (string, error) {
 	sizeMB := envInt("BEE_MEMTESTER_SIZE_MB", 128)
 	passes := envInt("BEE_MEMTESTER_PASSES", 1)
@@ -84,8 +130,11 @@ func (s *System) RunStorageAcceptancePack(baseDir string) (string, error) {
 }

 type satJob struct {
-	name string
-	cmd  []string
+	name       string
+	cmd        []string
+	env        []string // extra env vars (appended to os.Environ)
+	collectGPU bool     // collect GPU metrics via nvidia-smi while this job runs
+	gpuIndices []int    // GPU indices to collect metrics for (empty = all)
 }

 type satStats struct {
@@ -147,6 +196,109 @@ func runAcceptancePack(baseDir, prefix string, jobs []satJob) (string, error) {
 	return archive, nil
 }

+func nvidiaSATJobsWithOptions(durationSec, sizeMB int, gpuIndices []int) []satJob {
+	var env []string
+	if len(gpuIndices) > 0 {
+		ids := make([]string, len(gpuIndices))
+		for i, idx := range gpuIndices {
+			ids[i] = strconv.Itoa(idx)
+		}
+		env = []string{"CUDA_VISIBLE_DEVICES=" + strings.Join(ids, ",")}
+	}
+	return []satJob{
+		{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
+		{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
+		{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
+		{name: "04-nvidia-bug-report.log", cmd: []string{"nvidia-bug-report.sh", "--output-file", "{{run_dir}}/nvidia-bug-report.log"}},
+		{
+			name:       "05-bee-gpu-stress.log",
+			cmd:        []string{"bee-gpu-stress", "--seconds", strconv.Itoa(durationSec), "--size-mb", strconv.Itoa(sizeMB)},
+			env:        env,
+			collectGPU: true,
+			gpuIndices: gpuIndices,
+		},
+	}
+}
+
+func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []satJob) (string, error) {
+	if baseDir == "" {
+		baseDir = "/var/log/bee-sat"
+	}
+	ts := time.Now().UTC().Format("20060102-150405")
+	runDir := filepath.Join(baseDir, prefix+"-"+ts)
+	if err := os.MkdirAll(runDir, 0755); err != nil {
+		return "", err
+	}
+	verboseLog := filepath.Join(runDir, "verbose.log")
+
+	var summary strings.Builder
+	stats := satStats{}
+	fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
+	for _, job := range jobs {
+		if ctx.Err() != nil {
+			break
+		}
+		cmd := make([]string, 0, len(job.cmd))
+		for _, arg := range job.cmd {
+			cmd = append(cmd, strings.ReplaceAll(arg, "{{run_dir}}", runDir))
+		}
+
+		var out []byte
+		var err error
+
+		if job.collectGPU {
+			out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir)
+		} else {
+			out, err = runSATCommandCtx(ctx, verboseLog, job.name, cmd, job.env)
+		}
+
+		if writeErr := os.WriteFile(filepath.Join(runDir, job.name), out, 0644); writeErr != nil {
+			return "", writeErr
+		}
+		status, rc := classifySATResult(job.name, out, err)
+		stats.Add(status)
+		key := strings.TrimSuffix(strings.TrimPrefix(job.name, "0"), ".log")
+		fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
+		fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
+	}
+	writeSATStats(&summary, stats)
+	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
+		return "", err
+	}
+
+	archive := filepath.Join(baseDir, prefix+"-"+ts+".tar.gz")
+	if err := createTarGz(archive, runDir); err != nil {
+		return "", err
+	}
+	return archive, nil
+}
+
+func runSATCommandCtx(ctx context.Context, verboseLog, name string, cmd []string, env []string) ([]byte, error) {
+	start := time.Now().UTC()
+	appendSATVerboseLog(verboseLog,
+		fmt.Sprintf("[%s] start %s", start.Format(time.RFC3339), name),
+		"cmd: "+strings.Join(cmd, " "),
+	)
+
+	c := exec.CommandContext(ctx, cmd[0], cmd[1:]...)
+	if len(env) > 0 {
+		c.Env = append(os.Environ(), env...)
+	}
+	out, err := c.CombinedOutput()
+
+	rc := 0
+	if err != nil {
+		rc = 1
+	}
+	appendSATVerboseLog(verboseLog,
+		fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
+		fmt.Sprintf("rc: %d", rc),
+		fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()),
+		"",
+	)
+	return out, err
+}
+
 func listStorageDevices() ([]string, error) {
 	out, err := exec.Command("lsblk", "-dn", "-o", "NAME,TYPE").Output()
 	if err != nil {
@@ -251,6 +403,51 @@ func runSATCommand(verboseLog, name string, cmd []string) ([]byte, error) {
 	return out, err
 }

+// runSATCommandWithMetrics runs a command while collecting GPU metrics in the background.
+// On completion it writes gpu-metrics.csv and gpu-metrics.html into runDir.
+func runSATCommandWithMetrics(ctx context.Context, verboseLog, name string, cmd []string, env []string, gpuIndices []int, runDir string) ([]byte, error) {
+	stopCh := make(chan struct{})
+	doneCh := make(chan struct{})
+	var metricRows []GPUMetricRow
+	start := time.Now()
+
+	go func() {
+		defer close(doneCh)
+		ticker := time.NewTicker(time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-stopCh:
+				return
+			case <-ticker.C:
+				samples, err := sampleGPUMetrics(gpuIndices)
+				if err != nil {
+					continue
+				}
+				elapsed := time.Since(start).Seconds()
+				for i := range samples {
+					samples[i].ElapsedSec = elapsed
+				}
+				metricRows = append(metricRows, samples...)
+			}
+		}
+	}()
+
+	out, err := runSATCommandCtx(ctx, verboseLog, name, cmd, env)
+
+	close(stopCh)
+	<-doneCh
+
+	if len(metricRows) > 0 {
+		_ = WriteGPUMetricsCSV(filepath.Join(runDir, "gpu-metrics.csv"), metricRows)
+		_ = WriteGPUMetricsHTML(filepath.Join(runDir, "gpu-metrics.html"), metricRows)
+		chart := RenderGPUTerminalChart(metricRows)
+		_ = os.WriteFile(filepath.Join(runDir, "gpu-metrics-term.txt"), []byte(chart), 0644)
+	}
+
+	return out, err
+}
+
 func appendSATVerboseLog(path string, lines ...string) {
 	if path == "" {
 		return