diff --git a/internal/parser/archive.go b/internal/parser/archive.go index 6fd6aa7..1e3cd37 100644 --- a/internal/parser/archive.go +++ b/internal/parser/archive.go @@ -12,6 +12,8 @@ import ( "strings" ) +const maxSingleFileSize = 10 * 1024 * 1024 + // ExtractedFile represents a file extracted from archive type ExtractedFile struct { Path string @@ -29,6 +31,8 @@ func ExtractArchive(archivePath string) ([]ExtractedFile, error) { return extractTar(archivePath) case ".zip": return extractZip(archivePath) + case ".txt", ".log": + return extractSingleFile(archivePath) default: return nil, fmt.Errorf("unsupported archive format: %s", ext) } @@ -43,6 +47,8 @@ func ExtractArchiveFromReader(r io.Reader, filename string) ([]ExtractedFile, er return extractTarGzFromReader(r, filename) case ".tar": return extractTarFromReader(r) + case ".txt", ".log": + return extractSingleFileFromReader(r, filename) default: return nil, fmt.Errorf("unsupported archive format: %s", ext) } @@ -213,6 +219,33 @@ func extractZip(archivePath string) ([]ExtractedFile, error) { return files, nil } +func extractSingleFile(path string) ([]ExtractedFile, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open file: %w", err) + } + defer f.Close() + + return extractSingleFileFromReader(f, filepath.Base(path)) +} + +func extractSingleFileFromReader(r io.Reader, filename string) ([]ExtractedFile, error) { + content, err := io.ReadAll(io.LimitReader(r, maxSingleFileSize+1)) + if err != nil { + return nil, fmt.Errorf("read file content: %w", err) + } + if len(content) > maxSingleFileSize { + return nil, fmt.Errorf("file too large: max %d bytes", maxSingleFileSize) + } + + return []ExtractedFile{ + { + Path: filepath.Base(filename), + Content: content, + }, + }, nil +} + // FindFileByPattern finds files matching pattern in extracted files func FindFileByPattern(files []ExtractedFile, patterns ...string) []ExtractedFile { var result []ExtractedFile diff --git a/internal/parser/archive_test.go b/internal/parser/archive_test.go new file mode 100644 index 0000000..b1e1e42 --- /dev/null +++ b/internal/parser/archive_test.go @@ -0,0 +1,48 @@ +package parser + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestExtractArchiveFromReaderTXT(t *testing.T) { + content := "loader_brand=\"XigmaNAS\"\nSystem uptime:\n" + files, err := ExtractArchiveFromReader(strings.NewReader(content), "xigmanas.txt") + if err != nil { + t.Fatalf("extract txt from reader: %v", err) + } + if len(files) != 1 { + t.Fatalf("expected 1 file, got %d", len(files)) + } + if files[0].Path != "xigmanas.txt" { + t.Fatalf("expected filename xigmanas.txt, got %q", files[0].Path) + } + if string(files[0].Content) != content { + t.Fatalf("content mismatch") + } +} + +func TestExtractArchiveTXT(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sample.txt") + want := "plain text log" + if err := os.WriteFile(path, []byte(want), 0o600); err != nil { + t.Fatalf("write sample txt: %v", err) + } + + files, err := ExtractArchive(path) + if err != nil { + t.Fatalf("extract txt file: %v", err) + } + if len(files) != 1 { + t.Fatalf("expected 1 file, got %d", len(files)) + } + if files[0].Path != "sample.txt" { + t.Fatalf("expected sample.txt, got %q", files[0].Path) + } + if string(files[0].Content) != want { + t.Fatalf("content mismatch") + } +} diff --git a/internal/parser/vendors/xigmanas/parser.go b/internal/parser/vendors/xigmanas/parser.go index a8b66c6..759215d 100644 --- a/internal/parser/vendors/xigmanas/parser.go +++ b/internal/parser/vendors/xigmanas/parser.go @@ -12,7 +12,7 @@ import ( ) // parserVersion - increment when parsing logic changes. -const parserVersion = "2.0.0" +const parserVersion = "2.1.0" func init() { parser.Register(&Parser{}) @@ -86,6 +86,7 @@ func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, er parseUptime(content, result) parseZFSState(content, result) parseStorageAndSMART(content, result) + parseJournalLogSections(content, result) return result, nil } @@ -337,6 +338,138 @@ func parseStorageAndSMART(content string, result *models.AnalysisResult) { } } +func parseJournalLogSections(content string, result *models.AnalysisResult) { + sections := []struct { + heading string + eventType string + source string + }{ + {heading: "Last 275 System log entries:", eventType: "System Log", source: "system.log"}, + {heading: "Last 275 SMARTD log entries:", eventType: "SMARTD Log", source: "smartd.log"}, + {heading: "Last 275 Daemon log entries:", eventType: "Daemon Log", source: "daemon.log"}, + } + + for _, sec := range sections { + body := extractLogSection(content, sec.heading) + if body == "" { + continue + } + + for _, line := range strings.Split(body, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + msg := extractSyslogMessage(line) + if msg == "" { + msg = line + } + + result.Events = append(result.Events, models.Event{ + Timestamp: parseEventTimestamp(line), + Source: sec.source, + EventType: sec.eventType, + Severity: classifyEventSeverity(line), + Description: msg, + RawData: line, + }) + } + } +} + +func extractLogSection(content, heading string) string { + start := strings.Index(content, heading) + if start == -1 { + return "" + } + + tail := content[start+len(heading):] + lines := strings.Split(tail, "\n") + i := 0 + for i < len(lines) && strings.TrimSpace(lines[i]) == "" { + i++ + } + if i < len(lines) && isDashLine(lines[i]) { + i++ + } + + out := make([]string, 0, 64) + for ; i < len(lines); i++ { + line := lines[i] + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "Last 275 ") && strings.HasSuffix(trimmed, " log entries:") { + break + } + out = append(out, line) + } + + return strings.TrimSpace(strings.Join(out, "\n")) +} + +func isDashLine(s string) bool { + s = strings.TrimSpace(s) + if s == "" { + return false + } + for _, r := range s { + if r != '-' { + return false + } + } + return true +} + +func parseEventTimestamp(line string) time.Time { + isoRe := regexp.MustCompile(`\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?[+-]\d{2}:\d{2}\b`) + if iso := isoRe.FindString(line); iso != "" { + if ts, err := time.Parse(time.RFC3339Nano, iso); err == nil { + return ts + } + } + + prefixRe := regexp.MustCompile(`^[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}`) + if prefix := prefixRe.FindString(line); prefix != "" { + year := time.Now().Year() + if ts, err := time.Parse("Jan 2 15:04:05 2006", prefix+" "+strconv.Itoa(year)); err == nil { + return ts + } + } + + return time.Now() +} + +func classifyEventSeverity(line string) models.Severity { + lower := strings.ToLower(line) + switch { + case strings.Contains(lower, "panic"), strings.Contains(lower, "fatal"), strings.Contains(lower, "critical"): + return models.SeverityCritical + case strings.Contains(lower, "warning"), + strings.Contains(lower, "error"), + strings.Contains(lower, "failed"), + strings.Contains(lower, "failure"), + strings.Contains(lower, "login failure"), + strings.Contains(lower, "limiting open port"): + return models.SeverityWarning + default: + return models.SeverityInfo + } +} + +func extractSyslogMessage(line string) string { + if idx := strings.Index(line, ": "); idx != -1 && idx+2 < len(line) { + return strings.TrimSpace(line[idx+2:]) + } + + // RFC5424-like segment in XigmaNAS dumps: "... - - " + fields := strings.Fields(line) + if len(fields) > 10 { + return strings.TrimSpace(strings.Join(fields[10:], " ")) + } + + return strings.TrimSpace(line) +} + func splitModelAndFirmware(raw string) (string, string) { fields := strings.Fields(raw) if len(fields) < 2 { diff --git a/internal/parser/vendors/xigmanas/parser_test.go b/internal/parser/vendors/xigmanas/parser_test.go index 146f6bb..c49541d 100644 --- a/internal/parser/vendors/xigmanas/parser_test.go +++ b/internal/parser/vendors/xigmanas/parser_test.go @@ -91,4 +91,26 @@ func TestParserParseExample(t *testing.T) { if len(result.Events) == 0 { t.Fatal("expected events from uptime/zfs sections") } + + var hasSystemLog, hasSmartdLog, hasDaemonLog, hasLoginFailure bool + for _, ev := range result.Events { + if ev.EventType == "System Log" { + hasSystemLog = true + } + if ev.EventType == "SMARTD Log" { + hasSmartdLog = true + } + if ev.EventType == "Daemon Log" { + hasDaemonLog = true + } + if strings.Contains(strings.ToLower(ev.Description), "login failure") { + hasLoginFailure = true + } + } + if !hasSystemLog || !hasSmartdLog || !hasDaemonLog { + t.Fatalf("expected events from System/SMARTD/Daemon sections, got system=%v smartd=%v daemon=%v", hasSystemLog, hasSmartdLog, hasDaemonLog) + } + if !hasLoginFailure { + t.Fatal("expected to parse login failure event from system log section") + } } diff --git a/internal/server/upload_live_smoke_test.go b/internal/server/upload_live_smoke_test.go index 0923b2a..36cdf80 100644 --- a/internal/server/upload_live_smoke_test.go +++ b/internal/server/upload_live_smoke_test.go @@ -15,7 +15,7 @@ import ( func newFlowTestServer() (*Server, *httptest.Server) { s := &Server{ - jobManager: NewJobManager(), + jobManager: NewJobManager(), collectors: testCollectorRegistry(), } mux := http.NewServeMux() @@ -110,6 +110,61 @@ func TestUploadArchiveRegressionAndSourceMetadata(t *testing.T) { } } +func TestUploadTXTFile(t *testing.T) { + _, ts := newFlowTestServer() + defer ts.Close() + + txt := `Version: +-------- +14.3.0.5 + +loader_brand="XigmaNAS" +` + + reqBody := &bytes.Buffer{} + writer := multipart.NewWriter(reqBody) + part, err := writer.CreateFormFile("archive", "xigmanas.txt") + if err != nil { + t.Fatalf("create form file: %v", err) + } + if _, err := part.Write([]byte(txt)); err != nil { + t.Fatalf("write txt body: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("close multipart writer: %v", err) + } + + uploadReq, err := http.NewRequest(http.MethodPost, ts.URL+"/api/upload", reqBody) + if err != nil { + t.Fatalf("build upload request: %v", err) + } + uploadReq.Header.Set("Content-Type", writer.FormDataContentType()) + + uploadResp, err := http.DefaultClient.Do(uploadReq) + if err != nil { + t.Fatalf("upload request failed: %v", err) + } + defer uploadResp.Body.Close() + + if uploadResp.StatusCode != http.StatusOK { + t.Fatalf("expected 200 from /api/upload, got %d", uploadResp.StatusCode) + } + + var uploadPayload map[string]interface{} + if err := json.NewDecoder(uploadResp.Body).Decode(&uploadPayload); err != nil { + t.Fatalf("decode upload response: %v", err) + } + if uploadPayload["status"] != "ok" { + t.Fatalf("expected upload status ok, got %v", uploadPayload["status"]) + } + if uploadPayload["filename"] != "xigmanas.txt" { + t.Fatalf("expected filename xigmanas.txt, got %v", uploadPayload["filename"]) + } + if uploadPayload["vendor"] != "XigmaNAS Parser" { + t.Fatalf("expected vendor XigmaNAS Parser, got %v", uploadPayload["vendor"]) + } +} + func TestCollectSmokeErrorFormat(t *testing.T) { _, ts := newFlowTestServer() defer ts.Close() diff --git a/web/templates/index.html b/web/templates/index.html index 5c40e62..9069188 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -21,10 +21,10 @@
-

Перетащите архив или JSON snapshot сюда

- +

Перетащите архив, TXT/LOG или JSON snapshot сюда

+ -

Поддерживаемые форматы: tar.gz, zip, json

+

Поддерживаемые форматы: tar.gz, zip, json, txt, log