From ce30f943df34de53eecd64b009548ff4e463704e Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Tue, 24 Feb 2026 17:36:44 +0300 Subject: [PATCH] Export raw bundles with collection logs and parser field snapshot --- cmd/logpile/main.go | 2 + internal/server/handlers.go | 34 +++-- internal/server/raw_export.go | 227 ++++++++++++++++++++++++++++++++++ internal/server/server.go | 16 +++ 4 files changed, 270 insertions(+), 9 deletions(-) diff --git a/cmd/logpile/main.go b/cmd/logpile/main.go index cf5e8f8..f3cd4a4 100644 --- a/cmd/logpile/main.go +++ b/cmd/logpile/main.go @@ -40,6 +40,8 @@ func main() { cfg := server.Config{ Port: *port, PreloadFile: *file, + AppVersion: version, + AppCommit: commit, } srv := server.New(cfg) diff --git a/internal/server/handlers.go b/internal/server/handlers.go index 8694211..52e9a72 100644 --- a/internal/server/handlers.go +++ b/internal/server/handlers.go @@ -70,7 +70,22 @@ func (s *Server) handleUpload(w http.ResponseWriter, r *http.Request) { vendor string ) - if looksLikeJSONSnapshot(header.Filename, payload) { + if rawPkg, ok, err := parseRawExportBundle(payload); err != nil { + jsonError(w, "Failed to parse raw export bundle: "+err.Error(), http.StatusBadRequest) + return + } else if ok { + replayed, replayVendor, replayErr := s.reanalyzeRawExportPackage(rawPkg) + if replayErr != nil { + jsonError(w, "Failed to reanalyze raw export package: "+replayErr.Error(), http.StatusBadRequest) + return + } + result = replayed + vendor = replayVendor + if strings.TrimSpace(vendor) == "" { + vendor = "snapshot" + } + s.SetRawExport(rawPkg) + } else if looksLikeJSONSnapshot(header.Filename, payload) { if rawPkg, ok, err := parseRawExportPackage(payload); err != nil { jsonError(w, "Failed to parse raw export package: "+err.Error(), http.StatusBadRequest) return @@ -755,20 +770,20 @@ func (s *Server) handleExportCSV(w http.ResponseWriter, r *http.Request) { func (s *Server) handleExportJSON(w http.ResponseWriter, r *http.Request) { result := s.GetResult() - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", exportFilename(result, "json"))) - if rawPkg := s.GetRawExport(); rawPkg != nil { - rawPkg.ExportedAt = time.Now().UTC() - rawPkg.Analysis = nil - encoder := json.NewEncoder(w) - encoder.SetIndent("", " ") - if err := encoder.Encode(rawPkg); err != nil { + bundle, err := buildRawExportBundle(rawPkg, result, s.ClientVersionString()) + if err != nil { + jsonError(w, "Failed to build raw export bundle: "+err.Error(), http.StatusInternalServerError) return } + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", exportFilename(result, "zip"))) + _, _ = w.Write(bundle) return } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", exportFilename(result, "json"))) exp := exporter.New(result) _ = exp.ExportJSON(w) } @@ -840,6 +855,7 @@ func (s *Server) handleCollectStart(w http.ResponseWriter, r *http.Request) { } job := s.jobManager.CreateJob(req) + s.jobManager.AppendJobLog(job.ID, "Клиент: "+s.ClientVersionString()) s.startCollectionJob(job.ID, req) w.Header().Set("Content-Type", "application/json") diff --git a/internal/server/raw_export.go b/internal/server/raw_export.go index 04077b0..363bd0b 100644 --- a/internal/server/raw_export.go +++ b/internal/server/raw_export.go @@ -1,8 +1,13 @@ package server import ( + "archive/zip" + "bytes" "encoding/base64" "encoding/json" + "fmt" + "io" + "strings" "time" "git.mchus.pro/mchus/logpile/internal/models" @@ -10,6 +15,12 @@ import ( const rawExportFormatV1 = "logpile.raw-export.v1" +const ( + rawExportBundlePackageFile = "raw_export.json" + rawExportBundleLogFile = "collect.log" + rawExportBundleFieldsFile = "parser_fields.json" +) + type RawExportPackage struct { Format string `json:"format"` ExportedAt time.Time `json:"exported_at"` @@ -89,6 +100,222 @@ func parseRawExportPackage(payload []byte) (*RawExportPackage, bool, error) { return &pkg, true, nil } +func buildRawExportBundle(pkg *RawExportPackage, result *models.AnalysisResult, clientVersion string) ([]byte, error) { + if pkg == nil { + return nil, fmt.Errorf("nil raw export package") + } + pkgCopy := *pkg + pkgCopy.Analysis = nil + pkgCopy.ExportedAt = time.Now().UTC() + + jsonBytes, err := json.MarshalIndent(&pkgCopy, "", " ") + if err != nil { + return nil, err + } + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + jf, err := zw.Create(rawExportBundlePackageFile) + if err != nil { + return nil, err + } + if _, err := jf.Write(jsonBytes); err != nil { + return nil, err + } + + lf, err := zw.Create(rawExportBundleLogFile) + if err != nil { + return nil, err + } + if _, err := io.WriteString(lf, buildHumanReadableCollectionLog(&pkgCopy, result, clientVersion)); err != nil { + return nil, err + } + + ff, err := zw.Create(rawExportBundleFieldsFile) + if err != nil { + return nil, err + } + fieldsJSON, err := json.MarshalIndent(buildParserFieldSummary(result), "", " ") + if err != nil { + return nil, err + } + if _, err := ff.Write(fieldsJSON); err != nil { + return nil, err + } + + if err := zw.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func parseRawExportBundle(payload []byte) (*RawExportPackage, bool, error) { + if len(payload) < 4 || payload[0] != 'P' || payload[1] != 'K' { + return nil, false, nil + } + zr, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload))) + if err != nil { + return nil, false, nil + } + for _, f := range zr.File { + if f.Name != rawExportBundlePackageFile { + continue + } + rc, err := f.Open() + if err != nil { + return nil, true, err + } + defer rc.Close() + body, err := io.ReadAll(rc) + if err != nil { + return nil, true, err + } + pkg, ok, err := parseRawExportPackage(body) + return pkg, ok, err + } + return nil, false, nil +} + +func buildHumanReadableCollectionLog(pkg *RawExportPackage, result *models.AnalysisResult, clientVersion string) string { + var b strings.Builder + now := time.Now().UTC().Format(time.RFC3339) + fmt.Fprintf(&b, "LOGPile Raw Export Log\n") + fmt.Fprintf(&b, "Generated: %s\n", now) + if clientVersion != "" { + fmt.Fprintf(&b, "Client: %s\n", clientVersion) + } + if pkg != nil { + fmt.Fprintf(&b, "Format: %s\n", pkg.Format) + fmt.Fprintf(&b, "Source Kind: %s\n", pkg.Source.Kind) + if pkg.Source.Protocol != "" { + fmt.Fprintf(&b, "Protocol: %s\n", pkg.Source.Protocol) + } + if pkg.Source.TargetHost != "" { + fmt.Fprintf(&b, "Target Host: %s\n", pkg.Source.TargetHost) + } + if pkg.Source.Filename != "" { + fmt.Fprintf(&b, "Source Filename: %s\n", pkg.Source.Filename) + } + } + + if pkg != nil && len(pkg.Source.CollectLogs) > 0 { + b.WriteString("\n=== Redfish Collection Log ===\n") + for _, line := range pkg.Source.CollectLogs { + b.WriteString(line) + b.WriteByte('\n') + } + } + + b.WriteString("\n=== Parsed Field Summary ===\n") + if result == nil || result.Hardware == nil { + b.WriteString("No parsed hardware data available\n") + return b.String() + } + hw := result.Hardware + fmt.Fprintf(&b, "Board: manufacturer=%s model=%s serial=%s part=%s\n", + hw.BoardInfo.Manufacturer, hw.BoardInfo.ProductName, hw.BoardInfo.SerialNumber, hw.BoardInfo.PartNumber) + fmt.Fprintf(&b, "Counts: cpus=%d memory=%d storage=%d pcie=%d gpus=%d nics=%d psus=%d firmware=%d\n", + len(hw.CPUs), len(hw.Memory), len(hw.Storage), len(hw.PCIeDevices), len(hw.GPUs), len(hw.NetworkAdapters), len(hw.PowerSupply), len(hw.Firmware)) + + if len(hw.CPUs) > 0 { + b.WriteString("\n[CPUs]\n") + for _, cpu := range hw.CPUs { + fmt.Fprintf(&b, "- socket=%d model=%s cores=%d threads=%d serial=%s\n", cpu.Socket, cpu.Model, cpu.Cores, cpu.Threads, cpu.SerialNumber) + } + } + if len(hw.Memory) > 0 { + b.WriteString("\n[Memory]\n") + for _, m := range hw.Memory { + fmt.Fprintf(&b, "- slot=%s location=%s size_mb=%d part=%s serial=%s status=%s\n", m.Slot, m.Location, m.SizeMB, m.PartNumber, m.SerialNumber, m.Status) + } + } + if len(hw.Storage) > 0 { + b.WriteString("\n[Storage]\n") + for _, s := range hw.Storage { + fmt.Fprintf(&b, "- slot=%s type=%s model=%s size_gb=%d serial=%s\n", s.Slot, s.Type, s.Model, s.SizeGB, s.SerialNumber) + } + } + if len(hw.PCIeDevices) > 0 { + b.WriteString("\n[PCIe Devices]\n") + for _, d := range hw.PCIeDevices { + fmt.Fprintf(&b, "- slot=%s class=%s model=%s bdf=%s vendor=%s serial=%s\n", d.Slot, d.DeviceClass, d.PartNumber, d.BDF, d.Manufacturer, d.SerialNumber) + } + } + if len(hw.GPUs) > 0 { + b.WriteString("\n[GPUs]\n") + for _, g := range hw.GPUs { + fmt.Fprintf(&b, "- slot=%s model=%s bdf=%s serial=%s status=%s\n", g.Slot, g.Model, g.BDF, g.SerialNumber, g.Status) + } + } + if len(hw.NetworkAdapters) > 0 { + b.WriteString("\n[Network Adapters]\n") + for _, n := range hw.NetworkAdapters { + fmt.Fprintf(&b, "- slot=%s location=%s model=%s serial=%s status=%s\n", n.Slot, n.Location, n.Model, n.SerialNumber, n.Status) + } + } + if len(hw.PowerSupply) > 0 { + b.WriteString("\n[Power Supplies]\n") + for _, p := range hw.PowerSupply { + fmt.Fprintf(&b, "- slot=%s model=%s serial=%s status=%s watt=%d\n", p.Slot, p.Model, p.SerialNumber, p.Status, p.WattageW) + } + } + if len(hw.Firmware) > 0 { + b.WriteString("\n[Firmware]\n") + for _, fw := range hw.Firmware { + fmt.Fprintf(&b, "- device=%s version=%s\n", fw.DeviceName, fw.Version) + } + } + return b.String() +} + +func buildParserFieldSummary(result *models.AnalysisResult) map[string]any { + out := map[string]any{ + "generated_at": time.Now().UTC(), + } + if result == nil { + out["available"] = false + return out + } + out["available"] = true + out["filename"] = result.Filename + out["source_type"] = result.SourceType + out["protocol"] = result.Protocol + out["target_host"] = result.TargetHost + out["collected_at"] = result.CollectedAt + + if result.Hardware == nil { + out["hardware"] = map[string]any{} + return out + } + hw := result.Hardware + out["hardware"] = map[string]any{ + "board": hw.BoardInfo, + "counts": map[string]int{ + "cpus": len(hw.CPUs), + "memory": len(hw.Memory), + "storage": len(hw.Storage), + "pcie": len(hw.PCIeDevices), + "gpus": len(hw.GPUs), + "nics": len(hw.NetworkAdapters), + "psus": len(hw.PowerSupply), + "firmware": len(hw.Firmware), + "events": len(result.Events), + "sensors": len(result.Sensors), + "fru": len(result.FRU), + }, + "cpus": hw.CPUs, + "memory": hw.Memory, + "storage": hw.Storage, + "pcie_devices": hw.PCIeDevices, + "gpus": hw.GPUs, + "network_adapters": hw.NetworkAdapters, + "power_supplies": hw.PowerSupply, + "firmware": hw.Firmware, + } + return out +} + func resultProtocol(result *models.AnalysisResult) string { if result == nil { return "" diff --git a/internal/server/server.go b/internal/server/server.go index f74b0e9..5290525 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -19,6 +19,8 @@ var WebFS embed.FS type Config struct { Port int PreloadFile string + AppVersion string + AppCommit string } type Server struct { @@ -124,6 +126,20 @@ func (s *Server) GetRawExport() *RawExportPackage { return &cloned } +func (s *Server) ClientVersionString() string { + s.mu.RLock() + defer s.mu.RUnlock() + v := s.config.AppVersion + c := s.config.AppCommit + if v == "" { + v = "dev" + } + if c == "" { + c = "none" + } + return fmt.Sprintf("LOGPile %s (commit: %s)", v, c) +} + // SetDetectedVendor sets the detected vendor name func (s *Server) SetDetectedVendor(vendor string) { s.mu.Lock()