Merge branch 'main' of https://git.mchus.pro/mchus/logpile
This commit is contained in:
Mikhail Chusavitin
@@ -12,6 +12,8 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
const maxSingleFileSize = 10 * 1024 * 1024
|
||||
|
||||
// ExtractedFile represents a file extracted from archive
|
||||
type ExtractedFile struct {
|
||||
Path string
|
||||
@@ -29,6 +31,8 @@ func ExtractArchive(archivePath string) ([]ExtractedFile, error) {
|
||||
return extractTar(archivePath)
|
||||
case ".zip":
|
||||
return extractZip(archivePath)
|
||||
case ".txt", ".log":
|
||||
return extractSingleFile(archivePath)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported archive format: %s", ext)
|
||||
}
|
||||
@@ -43,6 +47,8 @@ func ExtractArchiveFromReader(r io.Reader, filename string) ([]ExtractedFile, er
|
||||
return extractTarGzFromReader(r, filename)
|
||||
case ".tar":
|
||||
return extractTarFromReader(r)
|
||||
case ".txt", ".log":
|
||||
return extractSingleFileFromReader(r, filename)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported archive format: %s", ext)
|
||||
}
|
||||
@@ -213,6 +219,33 @@ func extractZip(archivePath string) ([]ExtractedFile, error) {
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func extractSingleFile(path string) ([]ExtractedFile, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
return extractSingleFileFromReader(f, filepath.Base(path))
|
||||
}
|
||||
|
||||
func extractSingleFileFromReader(r io.Reader, filename string) ([]ExtractedFile, error) {
|
||||
content, err := io.ReadAll(io.LimitReader(r, maxSingleFileSize+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read file content: %w", err)
|
||||
}
|
||||
if len(content) > maxSingleFileSize {
|
||||
return nil, fmt.Errorf("file too large: max %d bytes", maxSingleFileSize)
|
||||
}
|
||||
|
||||
return []ExtractedFile{
|
||||
{
|
||||
Path: filepath.Base(filename),
|
||||
Content: content,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// FindFileByPattern finds files matching pattern in extracted files
|
||||
func FindFileByPattern(files []ExtractedFile, patterns ...string) []ExtractedFile {
|
||||
var result []ExtractedFile
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExtractArchiveFromReaderTXT(t *testing.T) {
|
||||
content := "loader_brand=\"XigmaNAS\"\nSystem uptime:\n"
|
||||
files, err := ExtractArchiveFromReader(strings.NewReader(content), "xigmanas.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("extract txt from reader: %v", err)
|
||||
}
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("expected 1 file, got %d", len(files))
|
||||
}
|
||||
if files[0].Path != "xigmanas.txt" {
|
||||
t.Fatalf("expected filename xigmanas.txt, got %q", files[0].Path)
|
||||
}
|
||||
if string(files[0].Content) != content {
|
||||
t.Fatalf("content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractArchiveTXT(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "sample.txt")
|
||||
want := "plain text log"
|
||||
if err := os.WriteFile(path, []byte(want), 0o600); err != nil {
|
||||
t.Fatalf("write sample txt: %v", err)
|
||||
}
|
||||
|
||||
files, err := ExtractArchive(path)
|
||||
if err != nil {
|
||||
t.Fatalf("extract txt file: %v", err)
|
||||
}
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("expected 1 file, got %d", len(files))
|
||||
}
|
||||
if files[0].Path != "sample.txt" {
|
||||
t.Fatalf("expected sample.txt, got %q", files[0].Path)
|
||||
}
|
||||
if string(files[0].Content) != want {
|
||||
t.Fatalf("content mismatch")
|
||||
}
|
||||
}
|
||||
Vendored
+1
@@ -8,6 +8,7 @@ import (
|
||||
_ "git.mchus.pro/mchus/logpile/internal/parser/vendors/nvidia"
|
||||
_ "git.mchus.pro/mchus/logpile/internal/parser/vendors/nvidia_bug_report"
|
||||
_ "git.mchus.pro/mchus/logpile/internal/parser/vendors/supermicro"
|
||||
_ "git.mchus.pro/mchus/logpile/internal/parser/vendors/xigmanas"
|
||||
|
||||
// Generic fallback parser (must be last for lowest priority)
|
||||
_ "git.mchus.pro/mchus/logpile/internal/parser/vendors/generic"
|
||||
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
# Xigmanas Parser
|
||||
|
||||
Parser for Xigmanas (FreeBSD-based NAS) system logs.
|
||||
|
||||
## Supported Files
|
||||
|
||||
- `xigmanas` - Main system log file with configuration and status information
|
||||
- `dmesg` - Kernel messages and hardware initialization information
|
||||
- SMART data from disk monitoring
|
||||
|
||||
## Features
|
||||
|
||||
This parser extracts the following information from Xigmanas logs:
|
||||
|
||||
### System Information
|
||||
- Firmware version
|
||||
- System uptime
|
||||
- CPU model and specifications
|
||||
- Memory configuration
|
||||
- Hardware platform information
|
||||
|
||||
### Storage Information
|
||||
- Disk models and serial numbers
|
||||
- Disk capacity and health status
|
||||
- SMART temperature readings
|
||||
|
||||
### Hardware Configuration
|
||||
- CPU information
|
||||
- Memory modules
|
||||
- Storage devices
|
||||
|
||||
## Detection Logic
|
||||
|
||||
The parser detects Xigmanas format by looking for:
|
||||
- Files with "xigmanas", "system", or "dmesg" in their names
|
||||
- Content containing "XigmaNAS" or "FreeBSD" strings
|
||||
- SMART-related information in log content
|
||||
|
||||
## Example Output
|
||||
|
||||
The parser populates the following fields in AnalysisResult:
|
||||
- `Hardware.Firmware` - Firmware versions
|
||||
- `Hardware.CPUs` - CPU information
|
||||
- `Hardware.Memory` - Memory configuration
|
||||
- `Hardware.Storage` - Storage devices with SMART data
|
||||
- `Sensors` - Temperature readings from SMART data
|
||||
+525
@@ -0,0 +1,525 @@
|
||||
// Package xigmanas provides parser for XigmaNAS diagnostic dumps.
|
||||
package xigmanas
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
"git.mchus.pro/mchus/logpile/internal/parser"
|
||||
)
|
||||
|
||||
// parserVersion - increment when parsing logic changes.
|
||||
const parserVersion = "2.1.0"
|
||||
|
||||
func init() {
|
||||
parser.Register(&Parser{})
|
||||
}
|
||||
|
||||
// Parser implements VendorParser for XigmaNAS logs.
|
||||
type Parser struct{}
|
||||
|
||||
func (p *Parser) Name() string { return "XigmaNAS Parser" }
|
||||
func (p *Parser) Vendor() string { return "xigmanas" }
|
||||
func (p *Parser) Version() string {
|
||||
return parserVersion
|
||||
}
|
||||
|
||||
// Detect checks if files contain typical XigmaNAS markers.
|
||||
func (p *Parser) Detect(files []parser.ExtractedFile) int {
|
||||
confidence := 0
|
||||
|
||||
for _, f := range files {
|
||||
path := strings.ToLower(f.Path)
|
||||
content := strings.ToLower(string(f.Content))
|
||||
|
||||
if strings.Contains(path, "xigmanas") || strings.HasSuffix(path, "dmesg") {
|
||||
confidence += 20
|
||||
}
|
||||
if strings.Contains(content, `loader_brand="xigmanas"`) {
|
||||
confidence += 70
|
||||
}
|
||||
if strings.Contains(content, "xigmanas kernel build") {
|
||||
confidence += 35
|
||||
}
|
||||
if strings.Contains(content, "system uptime:") && strings.Contains(content, "routing tables:") {
|
||||
confidence += 20
|
||||
}
|
||||
if strings.Contains(content, "s.m.a.r.t. [/dev/") {
|
||||
confidence += 10
|
||||
}
|
||||
if confidence >= 100 {
|
||||
return 100
|
||||
}
|
||||
}
|
||||
|
||||
if confidence > 100 {
|
||||
return 100
|
||||
}
|
||||
return confidence
|
||||
}
|
||||
|
||||
// Parse parses XigmaNAS logs and returns normalized data.
|
||||
func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, error) {
|
||||
result := &models.AnalysisResult{
|
||||
Events: make([]models.Event, 0),
|
||||
FRU: make([]models.FRUInfo, 0),
|
||||
Sensors: make([]models.SensorReading, 0),
|
||||
Hardware: &models.HardwareConfig{
|
||||
Firmware: make([]models.FirmwareInfo, 0),
|
||||
CPUs: make([]models.CPU, 0),
|
||||
Memory: make([]models.MemoryDIMM, 0),
|
||||
Storage: make([]models.Storage, 0),
|
||||
},
|
||||
}
|
||||
|
||||
content := joinFileContents(files)
|
||||
if strings.TrimSpace(content) == "" {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
parseSystemInfo(content, result)
|
||||
parseCPU(content, result)
|
||||
parseMemory(content, result)
|
||||
parseUptime(content, result)
|
||||
parseZFSState(content, result)
|
||||
parseStorageAndSMART(content, result)
|
||||
parseJournalLogSections(content, result)
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func joinFileContents(files []parser.ExtractedFile) string {
|
||||
var b strings.Builder
|
||||
for _, f := range files {
|
||||
b.Write(f.Content)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func parseSystemInfo(content string, result *models.AnalysisResult) {
|
||||
if m := regexp.MustCompile(`(?m)^Version:\s*\n-+\s*\n([^\n]+)`).FindStringSubmatch(content); len(m) == 2 {
|
||||
result.Hardware.Firmware = append(result.Hardware.Firmware, models.FirmwareInfo{
|
||||
DeviceName: "XigmaNAS",
|
||||
Version: strings.TrimSpace(m[1]),
|
||||
})
|
||||
}
|
||||
if m := regexp.MustCompile(`(?m)^smbios\.bios\.version="([^"]+)"`).FindStringSubmatch(content); len(m) == 2 {
|
||||
result.Hardware.Firmware = append(result.Hardware.Firmware, models.FirmwareInfo{
|
||||
DeviceName: "System BIOS",
|
||||
Version: strings.TrimSpace(m[1]),
|
||||
})
|
||||
}
|
||||
|
||||
board := models.BoardInfo{}
|
||||
if m := regexp.MustCompile(`(?m)^smbios\.system\.maker="([^"]+)"`).FindStringSubmatch(content); len(m) == 2 {
|
||||
board.Manufacturer = strings.TrimSpace(m[1])
|
||||
}
|
||||
if m := regexp.MustCompile(`(?m)^smbios\.system\.product="([^"]+)"`).FindStringSubmatch(content); len(m) == 2 {
|
||||
board.ProductName = strings.TrimSpace(m[1])
|
||||
}
|
||||
if m := regexp.MustCompile(`(?m)^smbios\.system\.serial="([^"]+)"`).FindStringSubmatch(content); len(m) == 2 {
|
||||
board.SerialNumber = strings.TrimSpace(m[1])
|
||||
}
|
||||
if m := regexp.MustCompile(`(?m)^smbios\.system\.uuid="([^"]+)"`).FindStringSubmatch(content); len(m) == 2 {
|
||||
board.UUID = strings.TrimSpace(m[1])
|
||||
}
|
||||
result.Hardware.BoardInfo = board
|
||||
}
|
||||
|
||||
func parseCPU(content string, result *models.AnalysisResult) {
|
||||
var cores, threads int
|
||||
if m := regexp.MustCompile(`(?m)^FreeBSD/SMP:\s+\d+\s+package\(s\)\s+x\s+(\d+)\s+core\(s\)`).FindStringSubmatch(content); len(m) == 2 {
|
||||
cores = parseInt(m[1])
|
||||
threads = cores
|
||||
}
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
cpuRe := regexp.MustCompile(`(?m)^CPU:\s+(.+?)\s+\(([\d.]+)-MHz`)
|
||||
for _, m := range cpuRe.FindAllStringSubmatch(content, -1) {
|
||||
model := strings.TrimSpace(m[1])
|
||||
if _, ok := seen[model]; ok {
|
||||
continue
|
||||
}
|
||||
seen[model] = struct{}{}
|
||||
|
||||
result.Hardware.CPUs = append(result.Hardware.CPUs, models.CPU{
|
||||
Socket: len(result.Hardware.CPUs),
|
||||
Model: model,
|
||||
Cores: cores,
|
||||
Threads: threads,
|
||||
FrequencyMHz: int(parseFloat(m[2])),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func parseMemory(content string, result *models.AnalysisResult) {
|
||||
if m := regexp.MustCompile(`(?m)^real memory\s*=\s*\d+\s+\((\d+)\s+MB\)`).FindStringSubmatch(content); len(m) == 2 {
|
||||
result.Hardware.Memory = append(result.Hardware.Memory, models.MemoryDIMM{
|
||||
Slot: "system",
|
||||
Present: true,
|
||||
SizeMB: parseInt(m[1]),
|
||||
Type: "DRAM",
|
||||
Status: "ok",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Fallback for logs that only have active/inactive breakdown.
|
||||
if m := regexp.MustCompile(`(?m)^Mem:\s+(.+)$`).FindStringSubmatch(content); len(m) == 2 {
|
||||
totalMB := 0
|
||||
tokenRe := regexp.MustCompile(`(\d+)M`)
|
||||
for _, t := range tokenRe.FindAllStringSubmatch(m[1], -1) {
|
||||
totalMB += parseInt(t[1])
|
||||
}
|
||||
if totalMB > 0 {
|
||||
result.Hardware.Memory = append(result.Hardware.Memory, models.MemoryDIMM{
|
||||
Slot: "system",
|
||||
Present: true,
|
||||
SizeMB: totalMB,
|
||||
Type: "DRAM",
|
||||
Status: "estimated",
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseUptime(content string, result *models.AnalysisResult) {
|
||||
upRe := regexp.MustCompile(`(?m)^(\d+:\d+(?:AM|PM))\s+up\s+(.+?),\s+(\d+)\s+users?,\s+load averages?:\s+([\d.]+),\s+([\d.]+),\s+([\d.]+)$`)
|
||||
m := upRe.FindStringSubmatch(content)
|
||||
if len(m) != 7 {
|
||||
return
|
||||
}
|
||||
|
||||
result.Events = append(result.Events, models.Event{
|
||||
Timestamp: time.Now(),
|
||||
Source: "System",
|
||||
EventType: "Uptime",
|
||||
Severity: models.SeverityInfo,
|
||||
Description: "System uptime and load averages parsed",
|
||||
RawData: "time=" + m[1] + "; uptime=" + m[2] + "; users=" + m[3] + "; load=" + m[4] + "," + m[5] + "," + m[6],
|
||||
})
|
||||
}
|
||||
|
||||
func parseZFSState(content string, result *models.AnalysisResult) {
|
||||
m := regexp.MustCompile(`(?m)^state:\s+([A-Z]+)$`).FindStringSubmatch(content)
|
||||
if len(m) != 2 {
|
||||
return
|
||||
}
|
||||
|
||||
state := m[1]
|
||||
severity := models.SeverityInfo
|
||||
if state != "ONLINE" {
|
||||
severity = models.SeverityWarning
|
||||
}
|
||||
result.Events = append(result.Events, models.Event{
|
||||
Timestamp: time.Now(),
|
||||
Source: "ZFS",
|
||||
EventType: "Pool State",
|
||||
Severity: severity,
|
||||
Description: "ZFS pool state: " + state,
|
||||
RawData: state,
|
||||
})
|
||||
}
|
||||
|
||||
func parseStorageAndSMART(content string, result *models.AnalysisResult) {
|
||||
type smartInfo struct {
|
||||
model string
|
||||
serial string
|
||||
firmware string
|
||||
health string
|
||||
tempC int
|
||||
capacityB int64
|
||||
}
|
||||
|
||||
storageBySlot := make(map[string]*models.Storage)
|
||||
scsiRe := regexp.MustCompile(`(?m)^<([^>]+)>\s+at\s+scbus\d+\s+target\s+\d+\s+lun\s+\d+\s+\(([^,]+),([^)]+)\)$`)
|
||||
for _, m := range scsiRe.FindAllStringSubmatch(content, -1) {
|
||||
slot := strings.TrimSpace(m[3])
|
||||
model, fw := splitModelAndFirmware(strings.TrimSpace(m[1]))
|
||||
entry := &models.Storage{
|
||||
Slot: slot,
|
||||
Type: guessStorageType(slot),
|
||||
Model: model,
|
||||
Firmware: fw,
|
||||
Present: true,
|
||||
Interface: "SCSI/SATA",
|
||||
}
|
||||
storageBySlot[slot] = entry
|
||||
}
|
||||
|
||||
smartBySlot := make(map[string]smartInfo)
|
||||
sectionRe := regexp.MustCompile(`(?m)^S\.M\.A\.R\.T\.\s+\[(/dev/[^\]]+)\]:\s*\n-+\n`)
|
||||
sections := sectionRe.FindAllStringSubmatchIndex(content, -1)
|
||||
for i, sec := range sections {
|
||||
// sec indexes:
|
||||
// [0]=full start, [1]=full end, [2]=capture 1 start, [3]=capture 1 end
|
||||
if len(sec) < 4 {
|
||||
continue
|
||||
}
|
||||
slot := strings.TrimPrefix(strings.TrimSpace(content[sec[2]:sec[3]]), "/dev/")
|
||||
bodyStart := sec[1]
|
||||
bodyEnd := len(content)
|
||||
if i+1 < len(sections) {
|
||||
bodyEnd = sections[i+1][0]
|
||||
}
|
||||
body := content[bodyStart:bodyEnd]
|
||||
|
||||
info := smartInfo{
|
||||
model: findFirst(body, `(?m)^Device Model:\s+(.+)$`),
|
||||
serial: findFirst(body, `(?m)^Serial Number:\s+(.+)$`),
|
||||
firmware: findFirst(body, `(?m)^Firmware Version:\s+(.+)$`),
|
||||
health: findFirst(body, `(?m)^SMART overall-health self-assessment test result:\s+(.+)$`),
|
||||
}
|
||||
info.capacityB = parseCapacityBytes(findFirst(body, `(?m)^User Capacity:\s+([\d,]+)\s+bytes`))
|
||||
if t := findFirst(body, `(?m)^\s*194\s+Temperature_Celsius.*?-\s+(\d+)(?:\s|\()`); t != "" {
|
||||
info.tempC = parseInt(t)
|
||||
}
|
||||
smartBySlot[slot] = info
|
||||
|
||||
if info.tempC > 0 {
|
||||
status := "ok"
|
||||
if info.health != "" && !strings.EqualFold(info.health, "PASSED") {
|
||||
status = "warning"
|
||||
}
|
||||
result.Sensors = append(result.Sensors, models.SensorReading{
|
||||
Name: "disk_temp_" + slot,
|
||||
Type: "temperature",
|
||||
Value: float64(info.tempC),
|
||||
Unit: "C",
|
||||
Status: status,
|
||||
RawValue: strconv.Itoa(info.tempC),
|
||||
})
|
||||
}
|
||||
if info.health != "" && !strings.EqualFold(info.health, "PASSED") {
|
||||
result.Events = append(result.Events, models.Event{
|
||||
Timestamp: time.Now(),
|
||||
Source: "SMART",
|
||||
EventType: "Disk Health",
|
||||
Severity: models.SeverityWarning,
|
||||
Description: "SMART health is not PASSED for " + slot,
|
||||
RawData: info.health,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Merge SMART data into storage entries and add missing entries.
|
||||
for slot, info := range smartBySlot {
|
||||
s := storageBySlot[slot]
|
||||
if s == nil {
|
||||
s = &models.Storage{
|
||||
Slot: slot,
|
||||
Type: guessStorageType(slot),
|
||||
Present: true,
|
||||
Interface: "SATA",
|
||||
}
|
||||
storageBySlot[slot] = s
|
||||
}
|
||||
|
||||
if s.Model == "" && info.model != "" {
|
||||
s.Model = info.model
|
||||
}
|
||||
if info.serial != "" {
|
||||
s.SerialNumber = info.serial
|
||||
}
|
||||
if s.Firmware == "" && info.firmware != "" {
|
||||
s.Firmware = info.firmware
|
||||
}
|
||||
if info.capacityB > 0 {
|
||||
s.SizeGB = int(info.capacityB / 1_000_000_000)
|
||||
}
|
||||
}
|
||||
|
||||
for _, s := range storageBySlot {
|
||||
result.Hardware.Storage = append(result.Hardware.Storage, *s)
|
||||
}
|
||||
}
|
||||
|
||||
func parseJournalLogSections(content string, result *models.AnalysisResult) {
|
||||
sections := []struct {
|
||||
heading string
|
||||
eventType string
|
||||
source string
|
||||
}{
|
||||
{heading: "Last 275 System log entries:", eventType: "System Log", source: "system.log"},
|
||||
{heading: "Last 275 SMARTD log entries:", eventType: "SMARTD Log", source: "smartd.log"},
|
||||
{heading: "Last 275 Daemon log entries:", eventType: "Daemon Log", source: "daemon.log"},
|
||||
}
|
||||
|
||||
for _, sec := range sections {
|
||||
body := extractLogSection(content, sec.heading)
|
||||
if body == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, line := range strings.Split(body, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
msg := extractSyslogMessage(line)
|
||||
if msg == "" {
|
||||
msg = line
|
||||
}
|
||||
|
||||
result.Events = append(result.Events, models.Event{
|
||||
Timestamp: parseEventTimestamp(line),
|
||||
Source: sec.source,
|
||||
EventType: sec.eventType,
|
||||
Severity: classifyEventSeverity(line),
|
||||
Description: msg,
|
||||
RawData: line,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func extractLogSection(content, heading string) string {
|
||||
start := strings.Index(content, heading)
|
||||
if start == -1 {
|
||||
return ""
|
||||
}
|
||||
|
||||
tail := content[start+len(heading):]
|
||||
lines := strings.Split(tail, "\n")
|
||||
i := 0
|
||||
for i < len(lines) && strings.TrimSpace(lines[i]) == "" {
|
||||
i++
|
||||
}
|
||||
if i < len(lines) && isDashLine(lines[i]) {
|
||||
i++
|
||||
}
|
||||
|
||||
out := make([]string, 0, 64)
|
||||
for ; i < len(lines); i++ {
|
||||
line := lines[i]
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "Last 275 ") && strings.HasSuffix(trimmed, " log entries:") {
|
||||
break
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(strings.Join(out, "\n"))
|
||||
}
|
||||
|
||||
func isDashLine(s string) bool {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range s {
|
||||
if r != '-' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func parseEventTimestamp(line string) time.Time {
|
||||
isoRe := regexp.MustCompile(`\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?[+-]\d{2}:\d{2}\b`)
|
||||
if iso := isoRe.FindString(line); iso != "" {
|
||||
if ts, err := time.Parse(time.RFC3339Nano, iso); err == nil {
|
||||
return ts
|
||||
}
|
||||
}
|
||||
|
||||
prefixRe := regexp.MustCompile(`^[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}`)
|
||||
if prefix := prefixRe.FindString(line); prefix != "" {
|
||||
year := time.Now().Year()
|
||||
if ts, err := time.Parse("Jan 2 15:04:05 2006", prefix+" "+strconv.Itoa(year)); err == nil {
|
||||
return ts
|
||||
}
|
||||
}
|
||||
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func classifyEventSeverity(line string) models.Severity {
|
||||
lower := strings.ToLower(line)
|
||||
switch {
|
||||
case strings.Contains(lower, "panic"), strings.Contains(lower, "fatal"), strings.Contains(lower, "critical"):
|
||||
return models.SeverityCritical
|
||||
case strings.Contains(lower, "warning"),
|
||||
strings.Contains(lower, "error"),
|
||||
strings.Contains(lower, "failed"),
|
||||
strings.Contains(lower, "failure"),
|
||||
strings.Contains(lower, "login failure"),
|
||||
strings.Contains(lower, "limiting open port"):
|
||||
return models.SeverityWarning
|
||||
default:
|
||||
return models.SeverityInfo
|
||||
}
|
||||
}
|
||||
|
||||
func extractSyslogMessage(line string) string {
|
||||
if idx := strings.Index(line, ": "); idx != -1 && idx+2 < len(line) {
|
||||
return strings.TrimSpace(line[idx+2:])
|
||||
}
|
||||
|
||||
// RFC5424-like segment in XigmaNAS dumps: "... <host> <proc> <pid> - - <message>"
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) > 10 {
|
||||
return strings.TrimSpace(strings.Join(fields[10:], " "))
|
||||
}
|
||||
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
|
||||
func splitModelAndFirmware(raw string) (string, string) {
|
||||
fields := strings.Fields(raw)
|
||||
if len(fields) < 2 {
|
||||
return raw, ""
|
||||
}
|
||||
last := fields[len(fields)-1]
|
||||
// Firmware token is usually compact (e.g. GKAOAB0A, 1.00).
|
||||
if regexp.MustCompile(`^[A-Za-z0-9._-]{2,12}$`).MatchString(last) {
|
||||
return strings.TrimSpace(strings.Join(fields[:len(fields)-1], " ")), last
|
||||
}
|
||||
return raw, ""
|
||||
}
|
||||
|
||||
func guessStorageType(slot string) string {
|
||||
switch {
|
||||
case strings.HasPrefix(slot, "cd"):
|
||||
return "optical"
|
||||
case strings.HasPrefix(slot, "da"), strings.HasPrefix(slot, "ada"):
|
||||
return "hdd"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
func findFirst(content, expr string) string {
|
||||
m := regexp.MustCompile(expr).FindStringSubmatch(content)
|
||||
if len(m) != 2 {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(m[1])
|
||||
}
|
||||
|
||||
func parseCapacityBytes(s string) int64 {
|
||||
clean := strings.ReplaceAll(strings.TrimSpace(s), ",", "")
|
||||
if clean == "" {
|
||||
return 0
|
||||
}
|
||||
v, err := strconv.ParseInt(clean, 10, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func parseInt(s string) int {
|
||||
v, _ := strconv.Atoi(strings.TrimSpace(s))
|
||||
return v
|
||||
}
|
||||
|
||||
func parseFloat(s string) float64 {
|
||||
v, _ := strconv.ParseFloat(strings.TrimSpace(s), 64)
|
||||
return v
|
||||
}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
package xigmanas
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/parser"
|
||||
)
|
||||
|
||||
func TestParserDetect(t *testing.T) {
|
||||
p := &Parser{}
|
||||
|
||||
files := []parser.ExtractedFile{
|
||||
{
|
||||
Path: "xigmanas",
|
||||
Content: []byte(`Version:
|
||||
--------
|
||||
14.3.0.5
|
||||
loader_brand="XigmaNAS"`),
|
||||
},
|
||||
}
|
||||
|
||||
if got := p.Detect(files); got < 70 {
|
||||
t.Fatalf("expected high confidence, got %d", got)
|
||||
}
|
||||
|
||||
files2 := []parser.ExtractedFile{
|
||||
{
|
||||
Path: "random_file.txt",
|
||||
Content: []byte("Some random content"),
|
||||
},
|
||||
}
|
||||
|
||||
if got := p.Detect(files2); got != 0 {
|
||||
t.Fatalf("expected zero confidence, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParserParseExample(t *testing.T) {
|
||||
p := &Parser{}
|
||||
|
||||
examplePath := filepath.Join("..", "..", "..", "..", "example", "xigmanas.txt")
|
||||
raw, err := os.ReadFile(examplePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read example file: %v", err)
|
||||
}
|
||||
|
||||
files := []parser.ExtractedFile{
|
||||
{Path: "xigmanas", Content: raw},
|
||||
}
|
||||
|
||||
result, err := p.Parse(files)
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if result == nil || result.Hardware == nil {
|
||||
t.Fatal("expected non-nil result with hardware")
|
||||
}
|
||||
|
||||
if len(result.Hardware.Firmware) == 0 {
|
||||
t.Fatal("expected firmware data")
|
||||
}
|
||||
foundXigmaVersion := false
|
||||
for _, fw := range result.Hardware.Firmware {
|
||||
if fw.DeviceName == "XigmaNAS" && fw.Version == "14.3.0.5" {
|
||||
foundXigmaVersion = true
|
||||
}
|
||||
}
|
||||
if !foundXigmaVersion {
|
||||
t.Fatalf("expected XigmaNAS firmware version 14.3.0.5, got %+v", result.Hardware.Firmware)
|
||||
}
|
||||
|
||||
if result.Hardware.BoardInfo.Manufacturer != "HP" {
|
||||
t.Fatalf("expected board manufacturer HP, got %q", result.Hardware.BoardInfo.Manufacturer)
|
||||
}
|
||||
if len(result.Hardware.CPUs) == 0 {
|
||||
t.Fatal("expected at least one CPU")
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(result.Hardware.CPUs[0].Model), "athlon") {
|
||||
t.Fatalf("expected CPU model to contain athlon, got %q", result.Hardware.CPUs[0].Model)
|
||||
}
|
||||
|
||||
if len(result.Hardware.Storage) < 4 {
|
||||
t.Fatalf("expected at least 4 storage devices, got %d", len(result.Hardware.Storage))
|
||||
}
|
||||
if len(result.Sensors) == 0 {
|
||||
t.Fatal("expected SMART temperature sensors")
|
||||
}
|
||||
if len(result.Events) == 0 {
|
||||
t.Fatal("expected events from uptime/zfs sections")
|
||||
}
|
||||
|
||||
var hasSystemLog, hasSmartdLog, hasDaemonLog, hasLoginFailure bool
|
||||
for _, ev := range result.Events {
|
||||
if ev.EventType == "System Log" {
|
||||
hasSystemLog = true
|
||||
}
|
||||
if ev.EventType == "SMARTD Log" {
|
||||
hasSmartdLog = true
|
||||
}
|
||||
if ev.EventType == "Daemon Log" {
|
||||
hasDaemonLog = true
|
||||
}
|
||||
if strings.Contains(strings.ToLower(ev.Description), "login failure") {
|
||||
hasLoginFailure = true
|
||||
}
|
||||
}
|
||||
if !hasSystemLog || !hasSmartdLog || !hasDaemonLog {
|
||||
t.Fatalf("expected events from System/SMARTD/Daemon sections, got system=%v smartd=%v daemon=%v", hasSystemLog, hasSmartdLog, hasDaemonLog)
|
||||
}
|
||||
if !hasLoginFailure {
|
||||
t.Fatal("expected to parse login failure event from system log section")
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
func newFlowTestServer() (*Server, *httptest.Server) {
|
||||
s := &Server{
|
||||
jobManager: NewJobManager(),
|
||||
jobManager: NewJobManager(),
|
||||
collectors: testCollectorRegistry(),
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
@@ -110,6 +110,61 @@ func TestUploadArchiveRegressionAndSourceMetadata(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadTXTFile(t *testing.T) {
|
||||
_, ts := newFlowTestServer()
|
||||
defer ts.Close()
|
||||
|
||||
txt := `Version:
|
||||
--------
|
||||
14.3.0.5
|
||||
|
||||
loader_brand="XigmaNAS"
|
||||
`
|
||||
|
||||
reqBody := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(reqBody)
|
||||
part, err := writer.CreateFormFile("archive", "xigmanas.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("create form file: %v", err)
|
||||
}
|
||||
if _, err := part.Write([]byte(txt)); err != nil {
|
||||
t.Fatalf("write txt body: %v", err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
uploadReq, err := http.NewRequest(http.MethodPost, ts.URL+"/api/upload", reqBody)
|
||||
if err != nil {
|
||||
t.Fatalf("build upload request: %v", err)
|
||||
}
|
||||
uploadReq.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
uploadResp, err := http.DefaultClient.Do(uploadReq)
|
||||
if err != nil {
|
||||
t.Fatalf("upload request failed: %v", err)
|
||||
}
|
||||
defer uploadResp.Body.Close()
|
||||
|
||||
if uploadResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected 200 from /api/upload, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
var uploadPayload map[string]interface{}
|
||||
if err := json.NewDecoder(uploadResp.Body).Decode(&uploadPayload); err != nil {
|
||||
t.Fatalf("decode upload response: %v", err)
|
||||
}
|
||||
if uploadPayload["status"] != "ok" {
|
||||
t.Fatalf("expected upload status ok, got %v", uploadPayload["status"])
|
||||
}
|
||||
if uploadPayload["filename"] != "xigmanas.txt" {
|
||||
t.Fatalf("expected filename xigmanas.txt, got %v", uploadPayload["filename"])
|
||||
}
|
||||
if uploadPayload["vendor"] != "XigmaNAS Parser" {
|
||||
t.Fatalf("expected vendor XigmaNAS Parser, got %v", uploadPayload["vendor"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectSmokeErrorFormat(t *testing.T) {
|
||||
_, ts := newFlowTestServer()
|
||||
defer ts.Close()
|
||||
|
||||
@@ -21,10 +21,10 @@
|
||||
|
||||
<div id="archive-source-content">
|
||||
<div class="upload-area" id="drop-zone">
|
||||
<p>Перетащите архив или JSON snapshot сюда</p>
|
||||
<input type="file" id="file-input" accept="application/gzip,application/x-gzip,application/x-tar,application/zip,application/json,.json,.tar,.tar.gz,.tgz,.zip" hidden>
|
||||
<p>Перетащите архив, TXT/LOG или JSON snapshot сюда</p>
|
||||
<input type="file" id="file-input" accept="application/gzip,application/x-gzip,application/x-tar,application/zip,application/json,text/plain,.json,.tar,.tar.gz,.tgz,.zip,.txt,.log" hidden>
|
||||
<button type="button" onclick="document.getElementById('file-input').click()">Выберите файл</button>
|
||||
<p class="hint">Поддерживаемые форматы: tar.gz, zip, json</p>
|
||||
<p class="hint">Поддерживаемые форматы: tar.gz, zip, json, txt, log</p>
|
||||
</div>
|
||||
<div id="upload-status"></div>
|
||||
<div id="parsers-info" class="parsers-info"></div>
|
||||
|
||||
Reference in New Issue
Block a user