Support TXT uploads and extend XigmaNAS event parsing

2026-02-04 22:25:43 +03:00
parent ae588ae75a
commit 92134a6cc1
6 changed files with 296 additions and 5 deletions
--- a/internal/parser/archive.go
+++ b/internal/parser/archive.go
@@ -12,6 +12,8 @@ import (
 	"strings"
 )

+const maxSingleFileSize = 10 * 1024 * 1024
+
 // ExtractedFile represents a file extracted from archive
 type ExtractedFile struct {
 	Path    string
@@ -29,6 +31,8 @@ func ExtractArchive(archivePath string) ([]ExtractedFile, error) {
 		return extractTar(archivePath)
 	case ".zip":
 		return extractZip(archivePath)
+	case ".txt", ".log":
+		return extractSingleFile(archivePath)
 	default:
 		return nil, fmt.Errorf("unsupported archive format: %s", ext)
 	}
@@ -43,6 +47,8 @@ func ExtractArchiveFromReader(r io.Reader, filename string) ([]ExtractedFile, er
 		return extractTarGzFromReader(r, filename)
 	case ".tar":
 		return extractTarFromReader(r)
+	case ".txt", ".log":
+		return extractSingleFileFromReader(r, filename)
 	default:
 		return nil, fmt.Errorf("unsupported archive format: %s", ext)
 	}
@@ -213,6 +219,33 @@ func extractZip(archivePath string) ([]ExtractedFile, error) {
 	return files, nil
 }

+func extractSingleFile(path string) ([]ExtractedFile, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("open file: %w", err)
+	}
+	defer f.Close()
+
+	return extractSingleFileFromReader(f, filepath.Base(path))
+}
+
+func extractSingleFileFromReader(r io.Reader, filename string) ([]ExtractedFile, error) {
+	content, err := io.ReadAll(io.LimitReader(r, maxSingleFileSize+1))
+	if err != nil {
+		return nil, fmt.Errorf("read file content: %w", err)
+	}
+	if len(content) > maxSingleFileSize {
+		return nil, fmt.Errorf("file too large: max %d bytes", maxSingleFileSize)
+	}
+
+	return []ExtractedFile{
+		{
+			Path:    filepath.Base(filename),
+			Content: content,
+		},
+	}, nil
+}
+
 // FindFileByPattern finds files matching pattern in extracted files
 func FindFileByPattern(files []ExtractedFile, patterns ...string) []ExtractedFile {
 	var result []ExtractedFile
--- a/internal/parser/archive_test.go
+++ b/internal/parser/archive_test.go
@@ -0,0 +1,48 @@
+package parser
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestExtractArchiveFromReaderTXT(t *testing.T) {
+	content := "loader_brand=\"XigmaNAS\"\nSystem uptime:\n"
+	files, err := ExtractArchiveFromReader(strings.NewReader(content), "xigmanas.txt")
+	if err != nil {
+		t.Fatalf("extract txt from reader: %v", err)
+	}
+	if len(files) != 1 {
+		t.Fatalf("expected 1 file, got %d", len(files))
+	}
+	if files[0].Path != "xigmanas.txt" {
+		t.Fatalf("expected filename xigmanas.txt, got %q", files[0].Path)
+	}
+	if string(files[0].Content) != content {
+		t.Fatalf("content mismatch")
+	}
+}
+
+func TestExtractArchiveTXT(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "sample.txt")
+	want := "plain text log"
+	if err := os.WriteFile(path, []byte(want), 0o600); err != nil {
+		t.Fatalf("write sample txt: %v", err)
+	}
+
+	files, err := ExtractArchive(path)
+	if err != nil {
+		t.Fatalf("extract txt file: %v", err)
+	}
+	if len(files) != 1 {
+		t.Fatalf("expected 1 file, got %d", len(files))
+	}
+	if files[0].Path != "sample.txt" {
+		t.Fatalf("expected sample.txt, got %q", files[0].Path)
+	}
+	if string(files[0].Content) != want {
+		t.Fatalf("content mismatch")
+	}
+}
--- a/internal/parser/vendors/xigmanas/parser.go
+++ b/internal/parser/vendors/xigmanas/parser.go
@@ -12,7 +12,7 @@ import (
 )

 // parserVersion - increment when parsing logic changes.
-const parserVersion = "2.0.0"
+const parserVersion = "2.1.0"

 func init() {
 	parser.Register(&Parser{})
@@ -86,6 +86,7 @@ func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, er
 	parseUptime(content, result)
 	parseZFSState(content, result)
 	parseStorageAndSMART(content, result)
+	parseJournalLogSections(content, result)

 	return result, nil
 }
@@ -337,6 +338,138 @@ func parseStorageAndSMART(content string, result *models.AnalysisResult) {
 	}
 }

+func parseJournalLogSections(content string, result *models.AnalysisResult) {
+	sections := []struct {
+		heading   string
+		eventType string
+		source    string
+	}{
+		{heading: "Last 275 System log entries:", eventType: "System Log", source: "system.log"},
+		{heading: "Last 275 SMARTD log entries:", eventType: "SMARTD Log", source: "smartd.log"},
+		{heading: "Last 275 Daemon log entries:", eventType: "Daemon Log", source: "daemon.log"},
+	}
+
+	for _, sec := range sections {
+		body := extractLogSection(content, sec.heading)
+		if body == "" {
+			continue
+		}
+
+		for _, line := range strings.Split(body, "\n") {
+			line = strings.TrimSpace(line)
+			if line == "" {
+				continue
+			}
+
+			msg := extractSyslogMessage(line)
+			if msg == "" {
+				msg = line
+			}
+
+			result.Events = append(result.Events, models.Event{
+				Timestamp:   parseEventTimestamp(line),
+				Source:      sec.source,
+				EventType:   sec.eventType,
+				Severity:    classifyEventSeverity(line),
+				Description: msg,
+				RawData:     line,
+			})
+		}
+	}
+}
+
+func extractLogSection(content, heading string) string {
+	start := strings.Index(content, heading)
+	if start == -1 {
+		return ""
+	}
+
+	tail := content[start+len(heading):]
+	lines := strings.Split(tail, "\n")
+	i := 0
+	for i < len(lines) && strings.TrimSpace(lines[i]) == "" {
+		i++
+	}
+	if i < len(lines) && isDashLine(lines[i]) {
+		i++
+	}
+
+	out := make([]string, 0, 64)
+	for ; i < len(lines); i++ {
+		line := lines[i]
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "Last 275 ") && strings.HasSuffix(trimmed, " log entries:") {
+			break
+		}
+		out = append(out, line)
+	}
+
+	return strings.TrimSpace(strings.Join(out, "\n"))
+}
+
+func isDashLine(s string) bool {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return false
+	}
+	for _, r := range s {
+		if r != '-' {
+			return false
+		}
+	}
+	return true
+}
+
+func parseEventTimestamp(line string) time.Time {
+	isoRe := regexp.MustCompile(`\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?[+-]\d{2}:\d{2}\b`)
+	if iso := isoRe.FindString(line); iso != "" {
+		if ts, err := time.Parse(time.RFC3339Nano, iso); err == nil {
+			return ts
+		}
+	}
+
+	prefixRe := regexp.MustCompile(`^[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}`)
+	if prefix := prefixRe.FindString(line); prefix != "" {
+		year := time.Now().Year()
+		if ts, err := time.Parse("Jan 2 15:04:05 2006", prefix+" "+strconv.Itoa(year)); err == nil {
+			return ts
+		}
+	}
+
+	return time.Now()
+}
+
+func classifyEventSeverity(line string) models.Severity {
+	lower := strings.ToLower(line)
+	switch {
+	case strings.Contains(lower, "panic"), strings.Contains(lower, "fatal"), strings.Contains(lower, "critical"):
+		return models.SeverityCritical
+	case strings.Contains(lower, "warning"),
+		strings.Contains(lower, "error"),
+		strings.Contains(lower, "failed"),
+		strings.Contains(lower, "failure"),
+		strings.Contains(lower, "login failure"),
+		strings.Contains(lower, "limiting open port"):
+		return models.SeverityWarning
+	default:
+		return models.SeverityInfo
+	}
+}
+
+func extractSyslogMessage(line string) string {
+	if idx := strings.Index(line, ": "); idx != -1 && idx+2 < len(line) {
+		return strings.TrimSpace(line[idx+2:])
+	}
+
+	// RFC5424-like segment in XigmaNAS dumps: "... <host> <proc> <pid> - - <message>"
+	fields := strings.Fields(line)
+	if len(fields) > 10 {
+		return strings.TrimSpace(strings.Join(fields[10:], " "))
+	}
+
+	return strings.TrimSpace(line)
+}
+
 func splitModelAndFirmware(raw string) (string, string) {
 	fields := strings.Fields(raw)
 	if len(fields) < 2 {
--- a/internal/parser/vendors/xigmanas/parser_test.go
+++ b/internal/parser/vendors/xigmanas/parser_test.go
@@ -91,4 +91,26 @@ func TestParserParseExample(t *testing.T) {
 	if len(result.Events) == 0 {
 		t.Fatal("expected events from uptime/zfs sections")
 	}
+
+	var hasSystemLog, hasSmartdLog, hasDaemonLog, hasLoginFailure bool
+	for _, ev := range result.Events {
+		if ev.EventType == "System Log" {
+			hasSystemLog = true
+		}
+		if ev.EventType == "SMARTD Log" {
+			hasSmartdLog = true
+		}
+		if ev.EventType == "Daemon Log" {
+			hasDaemonLog = true
+		}
+		if strings.Contains(strings.ToLower(ev.Description), "login failure") {
+			hasLoginFailure = true
+		}
+	}
+	if !hasSystemLog || !hasSmartdLog || !hasDaemonLog {
+		t.Fatalf("expected events from System/SMARTD/Daemon sections, got system=%v smartd=%v daemon=%v", hasSystemLog, hasSmartdLog, hasDaemonLog)
+	}
+	if !hasLoginFailure {
+		t.Fatal("expected to parse login failure event from system log section")
+	}
 }
--- a/internal/server/upload_live_smoke_test.go
+++ b/internal/server/upload_live_smoke_test.go
@@ -15,7 +15,7 @@ import (

 func newFlowTestServer() (*Server, *httptest.Server) {
 	s := &Server{
-		jobManager:  NewJobManager(),
+		jobManager: NewJobManager(),
 		collectors: testCollectorRegistry(),
 	}
 	mux := http.NewServeMux()
@@ -110,6 +110,61 @@ func TestUploadArchiveRegressionAndSourceMetadata(t *testing.T) {
 	}
 }

+func TestUploadTXTFile(t *testing.T) {
+	_, ts := newFlowTestServer()
+	defer ts.Close()
+
+	txt := `Version:
+--------
+14.3.0.5
+
+loader_brand="XigmaNAS"
+`
+
+	reqBody := &bytes.Buffer{}
+	writer := multipart.NewWriter(reqBody)
+	part, err := writer.CreateFormFile("archive", "xigmanas.txt")
+	if err != nil {
+		t.Fatalf("create form file: %v", err)
+	}
+	if _, err := part.Write([]byte(txt)); err != nil {
+		t.Fatalf("write txt body: %v", err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatalf("close multipart writer: %v", err)
+	}
+
+	uploadReq, err := http.NewRequest(http.MethodPost, ts.URL+"/api/upload", reqBody)
+	if err != nil {
+		t.Fatalf("build upload request: %v", err)
+	}
+	uploadReq.Header.Set("Content-Type", writer.FormDataContentType())
+
+	uploadResp, err := http.DefaultClient.Do(uploadReq)
+	if err != nil {
+		t.Fatalf("upload request failed: %v", err)
+	}
+	defer uploadResp.Body.Close()
+
+	if uploadResp.StatusCode != http.StatusOK {
+		t.Fatalf("expected 200 from /api/upload, got %d", uploadResp.StatusCode)
+	}
+
+	var uploadPayload map[string]interface{}
+	if err := json.NewDecoder(uploadResp.Body).Decode(&uploadPayload); err != nil {
+		t.Fatalf("decode upload response: %v", err)
+	}
+	if uploadPayload["status"] != "ok" {
+		t.Fatalf("expected upload status ok, got %v", uploadPayload["status"])
+	}
+	if uploadPayload["filename"] != "xigmanas.txt" {
+		t.Fatalf("expected filename xigmanas.txt, got %v", uploadPayload["filename"])
+	}
+	if uploadPayload["vendor"] != "XigmaNAS Parser" {
+		t.Fatalf("expected vendor XigmaNAS Parser, got %v", uploadPayload["vendor"])
+	}
+}
+
 func TestCollectSmokeErrorFormat(t *testing.T) {
 	_, ts := newFlowTestServer()
 	defer ts.Close()
--- a/web/templates/index.html
+++ b/web/templates/index.html
@@ -21,10 +21,10 @@

            <div id="archive-source-content">
                <div class="upload-area" id="drop-zone">
-                    <p>Перетащите архив или JSON snapshot сюда</p>
-                    <input type="file" id="file-input" accept="application/gzip,application/x-gzip,application/x-tar,application/zip,application/json,.json,.tar,.tar.gz,.tgz,.zip" hidden>
+                    <p>Перетащите архив, TXT/LOG или JSON snapshot сюда</p>
+                    <input type="file" id="file-input" accept="application/gzip,application/x-gzip,application/x-tar,application/zip,application/json,text/plain,.json,.tar,.tar.gz,.tgz,.zip,.txt,.log" hidden>
                    <button type="button" onclick="document.getElementById('file-input').click()">Выберите файл</button>
-                    <p class="hint">Поддерживаемые форматы: tar.gz, zip, json</p>
+                    <p class="hint">Поддерживаемые форматы: tar.gz, zip, json, txt, log</p>
                </div>
                <div id="upload-status"></div>
                <div id="parsers-info" class="parsers-info"></div>