feat(parser): linux-generic content-driven parser and RAR archive support
Field bundles from ad-hoc service scripts (first sample: Giga G894-SD3 .rar with dmidecode/lscpu/lspci/nvme/lsblk output, no fixed layout). ADL-078: - linux-generic: recognizes utility output by content, merges records by stable key, low detect confidence (20-30) below all vendor parsers - pciids.LookupByDescription: resolve a textual lspci description back to vendor/device IDs and manufacturer/model - archive: accept .rar via the in-repo internal/rar5 reader (RAR 5 / RAR 7, single volume, solid; encrypted, multi-volume, SFX and RAR 4.x rejected), same per-file and total limits as tar/zip - UI: .rar in the upload accept list Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ff5b99fa5c
commit
b4b15e58a2
25 files changed
+4901
-5
No files matched your search
+243
@@ -0,0 +1,243 @@
|
||||
// Package linux_generic parses archives that carry raw output of standard
|
||||
// Linux utilities (dmidecode, lscpu, lspci, nvme, smartctl, lsblk) collected
|
||||
// by ad-hoc service scripts. Such bundles have no fixed layout: file names,
|
||||
// directory structure and the set of captured utilities differ from one
|
||||
// script to another, so every file is recognized by its content, never by its
|
||||
// name, and one file may hold the output of several utilities.
|
||||
package linux_generic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.mchus.pro/mchus/logpile/internal/models"
|
||||
"git.mchus.pro/mchus/logpile/internal/parser"
|
||||
)
|
||||
|
||||
// parserVersion - version of this parser module
|
||||
// IMPORTANT: Increment version when modifying parser logic!
|
||||
const parserVersion = "1.0"
|
||||
|
||||
// detectScanLimit bounds how much of each file Detect inspects, so detection
|
||||
// stays cheap on large vendor archives that this parser will not win anyway.
|
||||
const detectScanLimit = 256 * 1024
|
||||
|
||||
func init() {
|
||||
parser.Register(&Parser{})
|
||||
}
|
||||
|
||||
// Parser implements VendorParser for Linux utility output bundles
|
||||
type Parser struct{}
|
||||
|
||||
// Name returns human-readable parser name
|
||||
func (p *Parser) Name() string {
|
||||
return "Linux Generic Utility Output Parser"
|
||||
}
|
||||
|
||||
// Vendor returns vendor identifier
|
||||
func (p *Parser) Vendor() string {
|
||||
return "linux-generic"
|
||||
}
|
||||
|
||||
// Version returns parser version
|
||||
func (p *Parser) Version() string {
|
||||
return parserVersion
|
||||
}
|
||||
|
||||
// sourceFile is one archive member prepared for line-oriented parsing.
|
||||
type sourceFile struct {
|
||||
path string
|
||||
content string
|
||||
}
|
||||
|
||||
// utility describes one recognized Linux utility output format. Adding a new
|
||||
// utility means adding one entry here plus its parse function.
|
||||
type utility struct {
|
||||
name string
|
||||
// inventory marks output that identifies hardware on its own. Detect
|
||||
// requires at least one such utility: lsblk or a SMART log alone is not
|
||||
// enough to claim an archive.
|
||||
inventory bool
|
||||
signature *regexp.Regexp
|
||||
parse func(file sourceFile, inv *inventory)
|
||||
}
|
||||
|
||||
var utilities = []utility{
|
||||
{
|
||||
name: "dmidecode",
|
||||
inventory: true,
|
||||
signature: regexp.MustCompile(`(?m)^(?:System Information|Base Board Information|Chassis Information|BIOS Information|Processor Information|Memory Device|System Power Supply)[ \t]*\r?\n[ \t]+[^:\r\n]+:`),
|
||||
parse: parseDMI,
|
||||
},
|
||||
{
|
||||
name: "lscpu",
|
||||
inventory: true,
|
||||
signature: regexp.MustCompile(`(?m)^[ \t]*(?:Socket\(s\)|Core\(s\) per socket|Thread\(s\) per core):[ \t]+\d`),
|
||||
parse: parseLscpu,
|
||||
},
|
||||
{
|
||||
name: "lspci",
|
||||
inventory: true,
|
||||
signature: regexp.MustCompile(`(?m)^(?:[0-9a-f]{4}:)?[0-9a-f]{2}:[0-9a-f]{2}\.[0-7] [A-Za-z0-9][^:\r\n]{1,60}: \S`),
|
||||
parse: parseLspci,
|
||||
},
|
||||
{
|
||||
name: "nvme list",
|
||||
inventory: true,
|
||||
signature: regexp.MustCompile(`(?m)^Node[ \t]+.*\bSN[ \t]+Model\b`),
|
||||
parse: parseNvmeList,
|
||||
},
|
||||
{
|
||||
name: "smartctl",
|
||||
inventory: true,
|
||||
signature: regexp.MustCompile(`=== START OF INFORMATION SECTION ===`),
|
||||
parse: parseSmartctl,
|
||||
},
|
||||
{
|
||||
name: "nvme smart-log",
|
||||
signature: regexp.MustCompile(`(?m)^Smart Log for NVME device:`),
|
||||
parse: parseNvmeSmartLog,
|
||||
},
|
||||
{
|
||||
name: "lsblk",
|
||||
signature: regexp.MustCompile(`(?m)^NAME[ \t]+.*\bSIZE\b.*\bTYPE\b`),
|
||||
parse: parseLsblk,
|
||||
},
|
||||
}
|
||||
|
||||
// Detect checks if the archive carries recognizable Linux utility output.
|
||||
// Returns confidence 0-100. The score stays below every dedicated vendor
|
||||
// parser (and below the Redfish-walk fallback), because BMC dumps and support
|
||||
// bundles routinely embed the same utilities' output.
|
||||
func (p *Parser) Detect(files []parser.ExtractedFile) int {
|
||||
found := make(map[string]bool, len(utilities))
|
||||
hasInventory := false
|
||||
|
||||
for _, f := range files {
|
||||
if !isText(f.Content) {
|
||||
continue
|
||||
}
|
||||
head := f.Content
|
||||
if len(head) > detectScanLimit {
|
||||
head = head[:detectScanLimit]
|
||||
}
|
||||
for _, u := range utilities {
|
||||
if found[u.name] || !u.signature.Match(head) {
|
||||
continue
|
||||
}
|
||||
found[u.name] = true
|
||||
hasInventory = hasInventory || u.inventory
|
||||
}
|
||||
if hasInventory && len(found) >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !hasInventory {
|
||||
return 0
|
||||
}
|
||||
switch len(found) {
|
||||
case 1:
|
||||
return 20
|
||||
case 2:
|
||||
return 25
|
||||
default:
|
||||
return 30
|
||||
}
|
||||
}
|
||||
|
||||
// Parse runs every matching utility parser over every text file and merges
|
||||
// the results into one inventory.
|
||||
func (p *Parser) Parse(files []parser.ExtractedFile) (*models.AnalysisResult, error) {
|
||||
result := &models.AnalysisResult{
|
||||
Events: make([]models.Event, 0),
|
||||
FRU: make([]models.FRUInfo, 0),
|
||||
Sensors: make([]models.SensorReading, 0),
|
||||
}
|
||||
|
||||
// Archive member order is arbitrary; a stable order keeps merge results
|
||||
// (first non-empty value wins) reproducible for the same file set.
|
||||
ordered := make([]parser.ExtractedFile, len(files))
|
||||
copy(ordered, files)
|
||||
sort.SliceStable(ordered, func(i, j int) bool { return ordered[i].Path < ordered[j].Path })
|
||||
|
||||
inv := newInventory()
|
||||
for _, f := range ordered {
|
||||
if !isText(f.Content) {
|
||||
continue
|
||||
}
|
||||
file := sourceFile{path: f.Path, content: normalizeText(f.Content)}
|
||||
for _, u := range utilities {
|
||||
if u.signature.MatchString(file.content) {
|
||||
u.parse(file, inv)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.Hardware = inv.hardware()
|
||||
result.FRU = append(result.FRU, inv.fru...)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// isText reports whether content looks like text. A NUL byte in the leading
|
||||
// block marks a binary file.
|
||||
func isText(content []byte) bool {
|
||||
if len(content) == 0 {
|
||||
return false
|
||||
}
|
||||
sample := content
|
||||
if len(sample) > 512 {
|
||||
sample = sample[:512]
|
||||
}
|
||||
return bytes.IndexByte(sample, 0) < 0
|
||||
}
|
||||
|
||||
// normalizeText strips a UTF-8 BOM and converts CRLF line endings: bundles are
|
||||
// often repacked on Windows.
|
||||
func normalizeText(content []byte) string {
|
||||
content = bytes.TrimPrefix(content, []byte{0xEF, 0xBB, 0xBF})
|
||||
return strings.ReplaceAll(string(content), "\r\n", "\n")
|
||||
}
|
||||
|
||||
// splitKeyValue splits "key : value" on the first colon and trims both parts.
|
||||
func splitKeyValue(line string) (key, value string, ok bool) {
|
||||
key, value, ok = strings.Cut(line, ":")
|
||||
if !ok {
|
||||
return "", "", false
|
||||
}
|
||||
return strings.TrimSpace(key), strings.TrimSpace(value), true
|
||||
}
|
||||
|
||||
func isIndented(line string) bool {
|
||||
return line != "" && (line[0] == ' ' || line[0] == '\t')
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, v := range values {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// leadingInt parses the leading decimal number of s, ignoring thousands
|
||||
// separators ("1,920,383,410,176 bytes" -> 1920383410176).
|
||||
func leadingInt(s string) (int64, bool) {
|
||||
s = strings.TrimSpace(s)
|
||||
var n int64
|
||||
digits := 0
|
||||
for _, c := range s {
|
||||
if c == ',' && digits > 0 {
|
||||
continue
|
||||
}
|
||||
if c < '0' || c > '9' {
|
||||
break
|
||||
}
|
||||
n = n*10 + int64(c-'0')
|
||||
digits++
|
||||
}
|
||||
return n, digits > 0
|
||||
}
|
||||
Reference in new issue
Block a user