feat: add support bundle and raw audit export

2026-03-16 18:20:26 +03:00
parent d18cde19c1
commit b25a2f6d30
37 changed files with 1699 additions and 97 deletions
@@ -0,0 +1,164 @@
+package platform
+
+import (
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"bee/audit/internal/schema"
+)
+
+var runtimeRequiredTools = []string{
+	"dmidecode",
+	"lspci",
+	"lsblk",
+	"smartctl",
+	"nvme",
+	"ipmitool",
+	"nvidia-smi",
+	"nvidia-bug-report.sh",
+	"bee-gpu-stress",
+	"dhclient",
+	"mount",
+}
+
+var runtimeTrackedServices = []string{
+	"bee-network",
+	"bee-nvidia",
+	"bee-preflight",
+	"bee-audit",
+	"bee-web",
+	"bee-sshsetup",
+}
+
+func (s *System) CollectRuntimeHealth(exportDir string) (schema.RuntimeHealth, error) {
+	checkedAt := time.Now().UTC().Format(time.RFC3339)
+	health := schema.RuntimeHealth{
+		Status:    "OK",
+		CheckedAt: checkedAt,
+		ExportDir: strings.TrimSpace(exportDir),
+	}
+
+	if health.ExportDir != "" {
+		if err := os.MkdirAll(health.ExportDir, 0755); err != nil {
+			health.Status = "FAILED"
+			health.Issues = append(health.Issues, schema.RuntimeIssue{
+				Code:        "export_dir_unavailable",
+				Severity:    "critical",
+				Description: err.Error(),
+			})
+		}
+	}
+
+	interfaces, err := s.ListInterfaces()
+	if err == nil {
+		health.Interfaces = make([]schema.RuntimeInterface, 0, len(interfaces))
+		hasIPv4 := false
+		missingIPv4 := false
+		for _, iface := range interfaces {
+			outcome := "no_offer"
+			if len(iface.IPv4) > 0 {
+				outcome = "lease_acquired"
+				hasIPv4 = true
+			} else if strings.EqualFold(iface.State, "DOWN") {
+				outcome = "link_down"
+			} else {
+				missingIPv4 = true
+			}
+			health.Interfaces = append(health.Interfaces, schema.RuntimeInterface{
+				Name:    iface.Name,
+				State:   iface.State,
+				IPv4:    iface.IPv4,
+				Outcome: outcome,
+			})
+		}
+		switch {
+		case hasIPv4 && !missingIPv4:
+			health.NetworkStatus = "OK"
+		case hasIPv4:
+			health.NetworkStatus = "PARTIAL"
+			health.Issues = append(health.Issues, schema.RuntimeIssue{
+				Code:        "dhcp_partial",
+				Severity:    "warning",
+				Description: "At least one interface did not obtain IPv4 connectivity.",
+			})
+		default:
+			health.NetworkStatus = "FAILED"
+			health.Issues = append(health.Issues, schema.RuntimeIssue{
+				Code:        "dhcp_failed",
+				Severity:    "warning",
+				Description: "No physical interface obtained IPv4 connectivity.",
+			})
+		}
+	}
+
+	for _, tool := range s.CheckTools(runtimeRequiredTools) {
+		health.Tools = append(health.Tools, schema.RuntimeToolStatus{
+			Name: tool.Name,
+			Path: tool.Path,
+			OK:   tool.OK,
+		})
+		if !tool.OK {
+			health.Issues = append(health.Issues, schema.RuntimeIssue{
+				Code:        "tool_missing",
+				Severity:    "warning",
+				Description: "Required tool missing: " + tool.Name,
+			})
+		}
+	}
+
+	for _, name := range runtimeTrackedServices {
+		health.Services = append(health.Services, schema.RuntimeServiceStatus{
+			Name:   name,
+			Status: s.ServiceState(name),
+		})
+	}
+
+	lsmodText := commandText("lsmod")
+	health.DriverReady = strings.Contains(lsmodText, "nvidia ")
+	if !health.DriverReady {
+		health.Issues = append(health.Issues, schema.RuntimeIssue{
+			Code:        "nvidia_kernel_module_missing",
+			Severity:    "warning",
+			Description: "NVIDIA kernel module is not loaded.",
+		})
+	}
+	if health.DriverReady && !strings.Contains(lsmodText, "nvidia_modeset") {
+		health.Issues = append(health.Issues, schema.RuntimeIssue{
+			Code:        "nvidia_modeset_failed",
+			Severity:    "warning",
+			Description: "nvidia-modeset is not loaded; display/CUDA stack may be partial.",
+		})
+	}
+	if out, err := exec.Command("nvidia-smi", "-L").CombinedOutput(); err == nil && strings.TrimSpace(string(out)) != "" {
+		health.DriverReady = true
+	}
+
+	health.CUDAReady = false
+	if lookErr := exec.Command("sh", "-c", "command -v bee-gpu-stress >/dev/null 2>&1").Run(); lookErr == nil {
+		out, err := exec.Command("bee-gpu-stress", "--seconds", "1", "--size-mb", "1").CombinedOutput()
+		if err == nil {
+			health.CUDAReady = true
+		} else if strings.Contains(strings.ToLower(string(out)), "cuda_error_system_not_ready") {
+			health.Issues = append(health.Issues, schema.RuntimeIssue{
+				Code:        "cuda_runtime_not_ready",
+				Severity:    "warning",
+				Description: "CUDA runtime is not ready for GPU SAT.",
+			})
+		}
+	}
+
+	if health.Status != "FAILED" && len(health.Issues) > 0 {
+		health.Status = "PARTIAL"
+	}
+	return health, nil
+}
+
+func commandText(name string, args ...string) string {
+	raw, err := exec.Command(name, args...).CombinedOutput()
+	if err != nil && len(raw) == 0 {
+		return ""
+	}
+	return string(raw)
+}
@@ -37,6 +37,7 @@ func (s *System) RunStorageAcceptancePack(baseDir string) (string, error) {
 	if err := os.MkdirAll(runDir, 0755); err != nil {
 		return "", err
 	}
+	verboseLog := filepath.Join(runDir, "verbose.log")

 	devices, err := listStorageDevices()
 	if err != nil {
@@ -59,7 +60,7 @@ func (s *System) RunStorageAcceptancePack(baseDir string) (string, error) {
 		commands := storageSATCommands(devPath)
 		for cmdIndex, job := range commands {
 			name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
-			out, err := exec.Command(job.cmd[0], job.cmd[1:]...).CombinedOutput()
+			out, err := runSATCommand(verboseLog, job.name, job.cmd)
 			if writeErr := os.WriteFile(filepath.Join(runDir, name), out, 0644); writeErr != nil {
 				return "", writeErr
 			}
@@ -100,7 +101,7 @@ func nvidiaSATJobs() []satJob {
 		{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
 		{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
 		{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
-		{name: "04-nvidia-bug-report.log", cmd: []string{"nvidia-bug-report.sh", "--output", "{{run_dir}}/nvidia-bug-report.log"}},
+		{name: "04-nvidia-bug-report.log", cmd: []string{"nvidia-bug-report.sh", "--output-file", "{{run_dir}}/nvidia-bug-report.log"}},
 		{name: "05-bee-gpu-stress.log", cmd: []string{"bee-gpu-stress", "--seconds", fmt.Sprintf("%d", seconds), "--size-mb", fmt.Sprintf("%d", sizeMB)}},
 	}
 }
@@ -114,6 +115,7 @@ func runAcceptancePack(baseDir, prefix string, jobs []satJob) (string, error) {
 	if err := os.MkdirAll(runDir, 0755); err != nil {
 		return "", err
 	}
+	verboseLog := filepath.Join(runDir, "verbose.log")

 	var summary strings.Builder
 	stats := satStats{}
@@ -123,7 +125,7 @@ func runAcceptancePack(baseDir, prefix string, jobs []satJob) (string, error) {
 		for _, arg := range job.cmd {
 			cmd = append(cmd, strings.ReplaceAll(arg, "{{run_dir}}", runDir))
 		}
-		out, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
+		out, err := runSATCommand(verboseLog, job.name, cmd)
 		if writeErr := os.WriteFile(filepath.Join(runDir, job.name), out, 0644); writeErr != nil {
 			return "", writeErr
 		}
@@ -219,6 +221,7 @@ func classifySATResult(name string, out []byte, err error) (string, int) {
 		strings.Contains(text, "unknown command") ||
 		strings.Contains(text, "not implemented") ||
 		strings.Contains(text, "not available") ||
+		strings.Contains(text, "cuda_error_system_not_ready") ||
 		strings.Contains(text, "no such device") ||
 		(strings.Contains(name, "self-test") && strings.Contains(text, "aborted")) {
 		return "UNSUPPORTED", rc
@@ -226,6 +229,42 @@ func classifySATResult(name string, out []byte, err error) (string, int) {
 	return "FAILED", rc
 }

+func runSATCommand(verboseLog, name string, cmd []string) ([]byte, error) {
+	start := time.Now().UTC()
+	appendSATVerboseLog(verboseLog,
+		fmt.Sprintf("[%s] start %s", start.Format(time.RFC3339), name),
+		"cmd: "+strings.Join(cmd, " "),
+	)
+
+	out, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
+
+	rc := 0
+	if err != nil {
+		rc = 1
+	}
+	appendSATVerboseLog(verboseLog,
+		fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
+		fmt.Sprintf("rc: %d", rc),
+		fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()),
+		"",
+	)
+	return out, err
+}
+
+func appendSATVerboseLog(path string, lines ...string) {
+	if path == "" {
+		return
+	}
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	for _, line := range lines {
+		_, _ = io.WriteString(f, line+"\n")
+	}
+}
+
 func envInt(name string, fallback int) int {
 	raw := strings.TrimSpace(os.Getenv(name))
 	if raw == "" {
@@ -31,6 +31,9 @@ func TestRunNvidiaAcceptancePackIncludesGPUStress(t *testing.T) {
 	if got := jobs[4].cmd[0]; got != "bee-gpu-stress" {
 		t.Fatalf("gpu stress command=%q want bee-gpu-stress", got)
 	}
+	if got := jobs[3].cmd[1]; got != "--output-file" {
+		t.Fatalf("bug report flag=%q want --output-file", got)
+	}
 }

 func TestNvidiaSATJobsUseEnvOverrides(t *testing.T) {
@@ -76,6 +79,7 @@ func TestClassifySATResult(t *testing.T) {
 		{name: "ok", job: "memtester", out: "done", err: nil, status: "OK"},
 		{name: "unsupported", job: "smartctl-self-test-short", out: "Self-test not supported", err: errors.New("rc 1"), status: "UNSUPPORTED"},
 		{name: "failed", job: "bee-gpu-stress", out: "cuda error", err: errors.New("rc 1"), status: "FAILED"},
+		{name: "cuda not ready", job: "bee-gpu-stress", out: "cuInit failed: CUDA_ERROR_SYSTEM_NOT_READY", err: errors.New("rc 1"), status: "UNSUPPORTED"},
 	}

 	for _, tt := range tests {
@@ -0,0 +1,122 @@
+package platform
+
+import (
+	"encoding/json"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+var techDumpFixedCommands = []struct {
+	Name string
+	Args []string
+	File string
+}{
+	{Name: "dmidecode", Args: []string{"-t", "0"}, File: "dmidecode-type0.txt"},
+	{Name: "dmidecode", Args: []string{"-t", "1"}, File: "dmidecode-type1.txt"},
+	{Name: "dmidecode", Args: []string{"-t", "2"}, File: "dmidecode-type2.txt"},
+	{Name: "dmidecode", Args: []string{"-t", "4"}, File: "dmidecode-type4.txt"},
+	{Name: "dmidecode", Args: []string{"-t", "17"}, File: "dmidecode-type17.txt"},
+	{Name: "lspci", Args: []string{"-vmm", "-D"}, File: "lspci-vmm.txt"},
+	{Name: "lsblk", Args: []string{"-J", "-d", "-o", "NAME,TYPE,SIZE,SERIAL,MODEL,TRAN,HCTL"}, File: "lsblk.json"},
+	{Name: "sensors", Args: []string{"-j"}, File: "sensors.json"},
+	{Name: "ipmitool", Args: []string{"fru", "print"}, File: "ipmitool-fru.txt"},
+	{Name: "ipmitool", Args: []string{"sdr"}, File: "ipmitool-sdr.txt"},
+	{Name: "nvidia-smi", Args: []string{"-q"}, File: "nvidia-smi-q.txt"},
+	{Name: "nvidia-smi", Args: []string{"--query-gpu=index,pci.bus_id,serial,vbios_version,temperature.gpu,power.draw,ecc.errors.uncorrected.aggregate.total,ecc.errors.corrected.aggregate.total,clocks_throttle_reasons.hw_slowdown", "--format=csv,noheader,nounits"}, File: "nvidia-smi-query.csv"},
+	{Name: "nvme", Args: []string{"list", "-o", "json"}, File: "nvme-list.json"},
+}
+
+type lsblkDumpRoot struct {
+	Blockdevices []struct {
+		Name string `json:"name"`
+		Type string `json:"type"`
+	} `json:"blockdevices"`
+}
+
+type nvmeDumpRoot struct {
+	Devices []struct {
+		DevicePath string `json:"DevicePath"`
+	} `json:"Devices"`
+}
+
+func (s *System) CaptureTechnicalDump(baseDir string) error {
+	if err := os.MkdirAll(baseDir, 0755); err != nil {
+		return err
+	}
+
+	for _, cmd := range techDumpFixedCommands {
+		writeCommandDump(filepath.Join(baseDir, cmd.File), cmd.Name, cmd.Args...)
+	}
+
+	for _, dev := range lsblkDumpDevices(filepath.Join(baseDir, "lsblk.json")) {
+		writeCommandDump(filepath.Join(baseDir, "smartctl-"+sanitizeDumpName(dev)+".json"), "smartctl", "-j", "-a", "/dev/"+dev)
+	}
+	for _, dev := range nvmeDumpDevices(filepath.Join(baseDir, "nvme-list.json")) {
+		writeCommandDump(filepath.Join(baseDir, "nvme-id-ctrl-"+sanitizeDumpName(dev)+".json"), "nvme", "id-ctrl", dev, "-o", "json")
+		writeCommandDump(filepath.Join(baseDir, "nvme-smart-log-"+sanitizeDumpName(dev)+".json"), "nvme", "smart-log", dev, "-o", "json")
+	}
+	return nil
+}
+
+func writeCommandDump(path, name string, args ...string) {
+	out, err := exec.Command(name, args...).CombinedOutput()
+	if err != nil && len(out) == 0 {
+		return
+	}
+	_ = os.WriteFile(path, out, 0644)
+}
+
+func lsblkDumpDevices(path string) []string {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil
+	}
+	var root lsblkDumpRoot
+	if err := json.Unmarshal(raw, &root); err != nil {
+		return nil
+	}
+	var devices []string
+	for _, dev := range root.Blockdevices {
+		if dev.Type == "disk" && strings.TrimSpace(dev.Name) != "" {
+			devices = append(devices, strings.TrimSpace(dev.Name))
+		}
+	}
+	sort.Strings(devices)
+	return devices
+}
+
+func nvmeDumpDevices(path string) []string {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil
+	}
+	var root nvmeDumpRoot
+	if err := json.Unmarshal(raw, &root); err != nil {
+		return nil
+	}
+	seen := map[string]bool{}
+	var devices []string
+	for _, dev := range root.Devices {
+		name := strings.TrimSpace(dev.DevicePath)
+		if name == "" || seen[name] {
+			continue
+		}
+		seen[name] = true
+		devices = append(devices, name)
+	}
+	sort.Strings(devices)
+	return devices
+}
+
+func sanitizeDumpName(value string) string {
+	value = strings.TrimSpace(value)
+	value = strings.TrimPrefix(value, "/dev/")
+	value = strings.ReplaceAll(value, "/", "_")
+	if value == "" {
+		return "unknown"
+	}
+	return value
+}
@@ -0,0 +1,48 @@
+package platform
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+)
+
+func TestLSBLKDumpDevices(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "lsblk.json")
+	if err := os.WriteFile(path, []byte(`{"blockdevices":[{"name":"sda","type":"disk"},{"name":"sda1","type":"part"},{"name":"nvme0n1","type":"disk"}]}`), 0644); err != nil {
+		t.Fatalf("write lsblk fixture: %v", err)
+	}
+
+	got := lsblkDumpDevices(path)
+	want := []string{"nvme0n1", "sda"}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("lsblkDumpDevices=%v want %v", got, want)
+	}
+}
+
+func TestNVMEDumpDevices(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "nvme-list.json")
+	if err := os.WriteFile(path, []byte(`{"Devices":[{"DevicePath":"/dev/nvme1n1"},{"DevicePath":"/dev/nvme0n1"},{"DevicePath":"/dev/nvme1n1"}]}`), 0644); err != nil {
+		t.Fatalf("write nvme fixture: %v", err)
+	}
+
+	got := nvmeDumpDevices(path)
+	want := []string{"/dev/nvme0n1", "/dev/nvme1n1"}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("nvmeDumpDevices=%v want %v", got, want)
+	}
+}
+
+func TestSanitizeDumpName(t *testing.T) {
+	t.Parallel()
+
+	if got := sanitizeDumpName("/dev/nvme0n1"); got != "nvme0n1" {
+		t.Fatalf("sanitizeDumpName=%q want nvme0n1", got)
+	}
+}