bee/audit/internal/platform/sat.go

package platform

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"
)

func (s *System) RunNvidiaAcceptancePack(baseDir string) (string, error) {
	return runAcceptancePack(baseDir, "gpu-nvidia", nvidiaSATJobs())
}

func (s *System) RunMemoryAcceptancePack(baseDir string) (string, error) {
	sizeMB := envInt("BEE_MEMTESTER_SIZE_MB", 128)
	passes := envInt("BEE_MEMTESTER_PASSES", 1)
	return runAcceptancePack(baseDir, "memory", []satJob{
		{name: "01-free-before.log", cmd: []string{"free", "-h"}},
		{name: "02-memtester.log", cmd: []string{"memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}},
		{name: "03-free-after.log", cmd: []string{"free", "-h"}},
	})
}

func (s *System) RunStorageAcceptancePack(baseDir string) (string, error) {
	if baseDir == "" {
		baseDir = "/var/log/bee-sat"
	}
	ts := time.Now().UTC().Format("20060102-150405")
	runDir := filepath.Join(baseDir, "storage-"+ts)
	if err := os.MkdirAll(runDir, 0755); err != nil {
		return "", err
	}

	devices, err := listStorageDevices()
	if err != nil {
		return "", err
	}
	sort.Strings(devices)

	var summary strings.Builder
	stats := satStats{}
	fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
	if len(devices) == 0 {
		fmt.Fprintln(&summary, "devices=0")
		stats.Unsupported++
	} else {
		fmt.Fprintf(&summary, "devices=%d\n", len(devices))
	}

	for index, devPath := range devices {
		prefix := fmt.Sprintf("%02d-%s", index+1, filepath.Base(devPath))
		commands := storageSATCommands(devPath)
		for cmdIndex, job := range commands {
			name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
			out, err := exec.Command(job.cmd[0], job.cmd[1:]...).CombinedOutput()
			if writeErr := os.WriteFile(filepath.Join(runDir, name), out, 0644); writeErr != nil {
				return "", writeErr
			}
			status, rc := classifySATResult(job.name, out, err)
			stats.Add(status)
			key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_")
			fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
			fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
		}
	}

	writeSATStats(&summary, stats)
	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
		return "", err
	}
	archive := filepath.Join(baseDir, "storage-"+ts+".tar.gz")
	if err := createTarGz(archive, runDir); err != nil {
		return "", err
	}
	return archive, nil
}

type satJob struct {
	name string
	cmd  []string
}

type satStats struct {
	OK          int
	Failed      int
	Unsupported int
}

func nvidiaSATJobs() []satJob {
	seconds := envInt("BEE_GPU_STRESS_SECONDS", 5)
	sizeMB := envInt("BEE_GPU_STRESS_SIZE_MB", 64)
	return []satJob{
		{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
		{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
		{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
		{name: "04-nvidia-bug-report.log", cmd: []string{"nvidia-bug-report.sh", "--output", "{{run_dir}}/nvidia-bug-report.log"}},
		{name: "05-bee-gpu-stress.log", cmd: []string{"bee-gpu-stress", "--seconds", fmt.Sprintf("%d", seconds), "--size-mb", fmt.Sprintf("%d", sizeMB)}},
	}
}

func runAcceptancePack(baseDir, prefix string, jobs []satJob) (string, error) {
	if baseDir == "" {
		baseDir = "/var/log/bee-sat"
	}
	ts := time.Now().UTC().Format("20060102-150405")
	runDir := filepath.Join(baseDir, prefix+"-"+ts)
	if err := os.MkdirAll(runDir, 0755); err != nil {
		return "", err
	}

	var summary strings.Builder
	stats := satStats{}
	fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
	for _, job := range jobs {
		cmd := make([]string, 0, len(job.cmd))
		for _, arg := range job.cmd {
			cmd = append(cmd, strings.ReplaceAll(arg, "{{run_dir}}", runDir))
		}
		out, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
		if writeErr := os.WriteFile(filepath.Join(runDir, job.name), out, 0644); writeErr != nil {
			return "", writeErr
		}
		status, rc := classifySATResult(job.name, out, err)
		stats.Add(status)
		key := strings.TrimSuffix(strings.TrimPrefix(job.name, "0"), ".log")
		fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
		fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
	}
	writeSATStats(&summary, stats)
	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
		return "", err
	}

	archive := filepath.Join(baseDir, prefix+"-"+ts+".tar.gz")
	if err := createTarGz(archive, runDir); err != nil {
		return "", err
	}
	return archive, nil
}

func listStorageDevices() ([]string, error) {
	out, err := exec.Command("lsblk", "-dn", "-o", "NAME,TYPE").Output()
	if err != nil {
		return nil, err
	}
	var devices []string
	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
		fields := strings.Fields(strings.TrimSpace(line))
		if len(fields) != 2 || fields[1] != "disk" {
			continue
		}
		devices = append(devices, "/dev/"+fields[0])
	}
	return devices, nil
}

func storageSATCommands(devPath string) []satJob {
	if strings.Contains(filepath.Base(devPath), "nvme") {
		return []satJob{
			{name: "nvme-id-ctrl", cmd: []string{"nvme", "id-ctrl", devPath, "-o", "json"}},
			{name: "nvme-smart-log", cmd: []string{"nvme", "smart-log", devPath, "-o", "json"}},
			{name: "nvme-device-self-test", cmd: []string{"nvme", "device-self-test", devPath, "--start", "1"}},
		}
	}
	return []satJob{
		{name: "smartctl-health", cmd: []string{"smartctl", "-H", "-A", devPath}},
		{name: "smartctl-self-test-short", cmd: []string{"smartctl", "-t", "short", devPath}},
	}
}

func (s *satStats) Add(status string) {
	switch status {
	case "OK":
		s.OK++
	case "UNSUPPORTED":
		s.Unsupported++
	default:
		s.Failed++
	}
}

func (s satStats) Overall() string {
	if s.Failed > 0 {
		return "FAILED"
	}
	if s.Unsupported > 0 {
		return "PARTIAL"
	}
	return "OK"
}

func writeSATStats(summary *strings.Builder, stats satStats) {
	fmt.Fprintf(summary, "overall_status=%s\n", stats.Overall())
	fmt.Fprintf(summary, "job_ok=%d\n", stats.OK)
	fmt.Fprintf(summary, "job_failed=%d\n", stats.Failed)
	fmt.Fprintf(summary, "job_unsupported=%d\n", stats.Unsupported)
}

func classifySATResult(name string, out []byte, err error) (string, int) {
	rc := 0
	if err != nil {
		rc = 1
	}
	if err == nil {
		return "OK", rc
	}

	text := strings.ToLower(string(out))
	if strings.Contains(text, "unsupported") ||
		strings.Contains(text, "not supported") ||
		strings.Contains(text, "invalid opcode") ||
		strings.Contains(text, "unknown command") ||
		strings.Contains(text, "not implemented") ||
		strings.Contains(text, "not available") ||
		strings.Contains(text, "no such device") ||
		(strings.Contains(name, "self-test") && strings.Contains(text, "aborted")) {
		return "UNSUPPORTED", rc
	}
	return "FAILED", rc
}

func envInt(name string, fallback int) int {
	raw := strings.TrimSpace(os.Getenv(name))
	if raw == "" {
		return fallback
	}
	value, err := strconv.Atoi(raw)
	if err != nil || value <= 0 {
		return fallback
	}
	return value
}

func createTarGz(dst, srcDir string) error {
	file, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer file.Close()

	gz := gzip.NewWriter(file)
	defer gz.Close()

	tw := tar.NewWriter(gz)
	defer tw.Close()

	base := filepath.Dir(srcDir)
	return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		header, err := tar.FileInfoHeader(info, "")
		if err != nil {
			return err
		}
		rel, err := filepath.Rel(base, path)
		if err != nil {
			return err
		}
		header.Name = rel
		if err := tw.WriteHeader(header); err != nil {
			return err
		}
		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()
		_, err = io.Copy(tw, file)
		return err
	})
}