Files
logpile/internal/server/raw_export.go
T
Mikhail ChusavitinandClaude Sonnet 5 2b0a2cb562 feat(raw-export): accept AUDIT-SP's signed audit.raw-export.v1 format
The AUDIT-SP collector (rebrand of the logpile collection tool) now stamps
raw_export.json with format "audit.raw-export.v1" instead of
"logpile.raw-export.v1" and ships a raw_export.sig alongside it, which broke
import entirely ("no parser found for this archive format").

- accept both format strings when parsing a raw-export bundle
- for the audit format, verify raw_export.sig (Ed25519, 64 bytes) against the
  exact bytes of raw_export.json using a trusted public key embedded in the
  binary (keys/audit-signing-public.pem); raw_export.sha256 is not consulted
- reject bundles with duplicate raw_export.json or raw_export.sig entries
- import proceeds even if the signature is missing or invalid, but the result
  is marked unverified rather than silently trusted
- expose GET /api/signature-status and show a "Signed" badge in the header
  when the signature verifies

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-08 11:55:42 +03:00

544 lines
17 KiB
Go

package server
import (
"archive/zip"
"bytes"
"crypto/ed25519"
"crypto/x509"
"encoding/base64"
"encoding/json"
"encoding/pem"
"fmt"
"io"
"strings"
"time"
"git.mchus.pro/mchus/logpile/internal/models"
"git.mchus.pro/mchus/logpile/keys"
)
const rawExportFormatV1 = "logpile.raw-export.v1"
// rawExportFormatAuditV1 is emitted by the AUDIT-SP collection tool
// (a rebrand of the same collector LOGPile used to call "logpile"). Unlike
// rawExportFormatV1, bundles in this format carry a raw_export.sig
// signature over the exact bytes of raw_export.json that LOGPile verifies
// against the embedded trusted key (see keys.AuditSigningPublicKeyPEM).
const rawExportFormatAuditV1 = "audit.raw-export.v1"
const (
rawExportBundlePackageFile = "raw_export.json"
rawExportBundleSignatureFile = "raw_export.sig"
rawExportBundleLogFile = "collect.log"
rawExportBundleFieldsFile = "parser_fields.json"
rawExportBundlePrivacyFile = "privacy_report.json"
rawExportBundleTopologyFile = "topology.json"
)
const ed25519SignatureSize = ed25519.SignatureSize // 64 bytes
// auditSigningPublicKey lazily parses the embedded trusted public key.
var auditSigningPublicKey = func() ed25519.PublicKey {
block, _ := pem.Decode(keys.AuditSigningPublicKeyPEM)
if block == nil {
panic("raw_export: embedded audit-signing-public.pem has no PEM block")
}
pub, err := x509.ParsePKIXPublicKey(block.Bytes)
if err != nil {
panic("raw_export: embedded audit-signing-public.pem is not a valid public key: " + err.Error())
}
edPub, ok := pub.(ed25519.PublicKey)
if !ok {
panic("raw_export: embedded audit-signing-public.pem is not an Ed25519 key")
}
return edPub
}()
// verifyAuditSignature checks sig (raw_export.sig) against jsonBody (the
// exact, unmodified bytes of raw_export.json) using the embedded AUDIT-SP
// trusted public key. It never consults raw_export.sha256 — only the
// signature is treated as proof of authenticity.
func verifyAuditSignature(jsonBody, sig []byte) bool {
if len(sig) != ed25519SignatureSize {
return false
}
return ed25519.Verify(auditSigningPublicKey, jsonBody, sig)
}
type RawExportPackage struct {
Format string `json:"format"`
ExportedAt time.Time `json:"exported_at"`
Source RawExportSource `json:"source"`
Analysis *models.AnalysisResult `json:"analysis_result,omitempty"`
// CollectedAtHint is extracted from parser_fields.json when importing
// a raw-export bundle and represents original collection time.
CollectedAtHint time.Time `json:"-"`
// SignatureChecked and SignatureVerified describe the outcome of
// verifying an AUDIT-SP raw_export.sig against the embedded trusted
// key. SignatureChecked is false for bundles in a format that doesn't
// carry a signature (e.g. plain rawExportFormatV1); it is true for
// rawExportFormatAuditV1 bundles whether or not the signature
// actually verified. Both are recomputed on every import, never
// persisted across re-export.
SignatureChecked bool `json:"-"`
SignatureVerified bool `json:"-"`
}
type RawExportSource struct {
Kind string `json:"kind"` // file_bytes | live_redfish | snapshot_json
Filename string `json:"filename,omitempty"`
MIMEType string `json:"mime_type,omitempty"`
Encoding string `json:"encoding,omitempty"` // base64
Data string `json:"data,omitempty"`
Protocol string `json:"protocol,omitempty"`
TargetHost string `json:"target_host,omitempty"`
SourceTimezone string `json:"source_timezone,omitempty"`
RawPayloads map[string]any `json:"raw_payloads,omitempty"`
CollectLogs []string `json:"collect_logs,omitempty"`
CollectMeta *CollectRequestMeta `json:"collect_meta,omitempty"`
}
func newRawExportFromUploadedFile(filename, mimeType string, payload []byte, result *models.AnalysisResult) *RawExportPackage {
return &RawExportPackage{
Format: rawExportFormatV1,
ExportedAt: time.Now().UTC(),
Source: RawExportSource{
Kind: "file_bytes",
Filename: filename,
MIMEType: mimeType,
Encoding: "base64",
Data: base64.StdEncoding.EncodeToString(payload),
Protocol: resultProtocol(result),
TargetHost: resultTargetHost(result),
SourceTimezone: resultSourceTimezone(result),
},
}
}
func newRawExportFromLiveCollect(result *models.AnalysisResult, req CollectRequest, logs []string) *RawExportPackage {
rawPayloads := map[string]any{}
if result != nil && result.RawPayloads != nil {
for k, v := range result.RawPayloads {
rawPayloads[k] = v
}
}
meta := CollectRequestMeta{
Host: req.Host,
Protocol: req.Protocol,
Port: req.Port,
Username: req.Username,
AuthType: req.AuthType,
TLSMode: req.TLSMode,
}
return &RawExportPackage{
Format: rawExportFormatV1,
ExportedAt: time.Now().UTC(),
Source: RawExportSource{
Kind: "live_redfish",
Protocol: req.Protocol,
TargetHost: req.Host,
SourceTimezone: resultSourceTimezone(result),
RawPayloads: rawPayloads,
CollectLogs: append([]string(nil), logs...),
CollectMeta: &meta,
},
}
}
// parseRawExportPackage parses a bare raw_export.json payload. sig is the
// accompanying raw_export.sig bytes when one was found alongside payload in
// the same bundle (nil if there wasn't one, e.g. a bare .json upload).
func parseRawExportPackage(payload []byte, sig []byte) (*RawExportPackage, bool, error) {
var pkg RawExportPackage
if err := json.Unmarshal(payload, &pkg); err != nil {
return nil, false, err
}
switch pkg.Format {
case rawExportFormatV1:
// No signature in this format; nothing to check.
case rawExportFormatAuditV1:
pkg.SignatureChecked = true
pkg.SignatureVerified = sig != nil && verifyAuditSignature(payload, sig)
default:
return nil, false, nil
}
if pkg.ExportedAt.IsZero() {
pkg.ExportedAt = time.Now().UTC()
}
return &pkg, true, nil
}
func buildRawExportBundle(pkg *RawExportPackage, result *models.AnalysisResult, clientVersion string) ([]byte, error) {
if pkg == nil {
return nil, fmt.Errorf("nil raw export package")
}
pkgCopy := *pkg
pkgCopy.Analysis = nil
pkgCopy.ExportedAt = time.Now().UTC()
jsonBytes, err := json.MarshalIndent(&pkgCopy, "", " ")
if err != nil {
return nil, err
}
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
jf, err := zw.Create(rawExportBundlePackageFile)
if err != nil {
return nil, err
}
if _, err := jf.Write(jsonBytes); err != nil {
return nil, err
}
lf, err := zw.Create(rawExportBundleLogFile)
if err != nil {
return nil, err
}
if _, err := io.WriteString(lf, buildHumanReadableCollectionLog(&pkgCopy, result, clientVersion)); err != nil {
return nil, err
}
ff, err := zw.Create(rawExportBundleFieldsFile)
if err != nil {
return nil, err
}
fieldsJSON, err := json.MarshalIndent(buildParserFieldSummary(result), "", " ")
if err != nil {
return nil, err
}
if _, err := ff.Write(fieldsJSON); err != nil {
return nil, err
}
if result != nil && result.PrivacyScan != nil {
pf, err := zw.Create(rawExportBundlePrivacyFile)
if err != nil {
return nil, err
}
privacyJSON, err := json.MarshalIndent(result.PrivacyScan, "", " ")
if err != nil {
return nil, err
}
if _, err := pf.Write(privacyJSON); err != nil {
return nil, err
}
}
if topologyDoc, topologyErr := currentTopologyDocument(result); topologyErr == nil && len(topologyDoc.Nodes) > 0 {
tf, err := zw.Create(rawExportBundleTopologyFile)
if err != nil {
return nil, err
}
topologyJSON, err := json.MarshalIndent(topologyDoc, "", " ")
if err != nil {
return nil, err
}
if _, err := tf.Write(topologyJSON); err != nil {
return nil, err
}
}
if err := zw.Close(); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func parseRawExportBundle(payload []byte) (*RawExportPackage, bool, error) {
if len(payload) < 4 || payload[0] != 'P' || payload[1] != 'K' {
return nil, false, nil
}
zr, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
if err != nil {
return nil, false, nil
}
var pkgBody []byte
var sigBody []byte
var parserFieldsBody []byte
var pkgSeen, sigSeen bool
for _, f := range zr.File {
if f.Name != rawExportBundlePackageFile && f.Name != rawExportBundleSignatureFile && f.Name != rawExportBundleFieldsFile {
continue
}
if f.Name == rawExportBundlePackageFile && pkgSeen {
return nil, true, fmt.Errorf("raw export bundle contains duplicate %s entries", rawExportBundlePackageFile)
}
if f.Name == rawExportBundleSignatureFile && sigSeen {
return nil, true, fmt.Errorf("raw export bundle contains duplicate %s entries", rawExportBundleSignatureFile)
}
rc, err := f.Open()
if err != nil {
return nil, true, err
}
body, err := io.ReadAll(rc)
rc.Close()
if err != nil {
return nil, true, err
}
switch f.Name {
case rawExportBundlePackageFile:
pkgBody = body
pkgSeen = true
case rawExportBundleSignatureFile:
sigBody = body
sigSeen = true
case rawExportBundleFieldsFile:
parserFieldsBody = body
}
}
if len(pkgBody) == 0 {
return nil, false, nil
}
pkg, ok, err := parseRawExportPackage(pkgBody, sigBody)
if err != nil || !ok {
return pkg, ok, err
}
if ts, ok := parseCollectedAtHint(parserFieldsBody); ok {
pkg.CollectedAtHint = ts.UTC()
}
return pkg, true, nil
}
func parseCollectedAtHint(parserFieldsBody []byte) (time.Time, bool) {
if len(parserFieldsBody) == 0 {
return time.Time{}, false
}
var payload struct {
CollectedAt string `json:"collected_at"`
}
if err := json.Unmarshal(parserFieldsBody, &payload); err != nil {
return time.Time{}, false
}
collectedAt := strings.TrimSpace(payload.CollectedAt)
if collectedAt == "" {
return time.Time{}, false
}
ts, err := time.Parse(time.RFC3339Nano, collectedAt)
if err != nil {
return time.Time{}, false
}
return ts, true
}
func buildHumanReadableCollectionLog(pkg *RawExportPackage, result *models.AnalysisResult, clientVersion string) string {
var b strings.Builder
now := time.Now().UTC().Format(time.RFC3339)
fmt.Fprintf(&b, "LOGPile Raw Export Log\n")
fmt.Fprintf(&b, "Generated: %s\n", now)
if clientVersion != "" {
fmt.Fprintf(&b, "Client: %s\n", clientVersion)
}
if pkg != nil {
fmt.Fprintf(&b, "Format: %s\n", pkg.Format)
fmt.Fprintf(&b, "Source Kind: %s\n", pkg.Source.Kind)
if pkg.Source.Protocol != "" {
fmt.Fprintf(&b, "Protocol: %s\n", pkg.Source.Protocol)
}
if pkg.Source.TargetHost != "" {
fmt.Fprintf(&b, "Target Host: %s\n", pkg.Source.TargetHost)
}
if pkg.Source.Filename != "" {
fmt.Fprintf(&b, "Source Filename: %s\n", pkg.Source.Filename)
}
if startedAt, finishedAt, ok := collectLogTimeBounds(pkg.Source.CollectLogs); ok {
fmt.Fprintf(&b, "Collection Started: %s\n", startedAt.Format(time.RFC3339Nano))
fmt.Fprintf(&b, "Collection Finished: %s\n", finishedAt.Format(time.RFC3339Nano))
fmt.Fprintf(&b, "Collection Duration: %s\n", formatRawExportDuration(finishedAt.Sub(startedAt)))
}
}
if pkg != nil && len(pkg.Source.CollectLogs) > 0 {
b.WriteString("\n=== Redfish Collection Log ===\n")
for _, line := range pkg.Source.CollectLogs {
b.WriteString(line)
b.WriteByte('\n')
}
}
b.WriteString("\n=== Parsed Field Summary ===\n")
if result == nil || result.Hardware == nil {
b.WriteString("No parsed hardware data available\n")
return b.String()
}
hw := result.Hardware
fmt.Fprintf(&b, "Board: manufacturer=%s model=%s serial=%s part=%s\n",
hw.BoardInfo.Manufacturer, hw.BoardInfo.ProductName, hw.BoardInfo.SerialNumber, hw.BoardInfo.PartNumber)
fmt.Fprintf(&b, "Counts: cpus=%d memory=%d storage=%d pcie=%d gpus=%d nics=%d psus=%d firmware=%d\n",
len(hw.CPUs), len(hw.Memory), len(hw.Storage), len(hw.PCIeDevices), len(hw.GPUs), len(hw.NetworkAdapters), len(hw.PowerSupply), len(hw.Firmware))
if len(hw.CPUs) > 0 {
b.WriteString("\n[CPUs]\n")
for _, cpu := range hw.CPUs {
fmt.Fprintf(&b, "- socket=%d model=%s cores=%d threads=%d serial=%s\n", cpu.Socket, cpu.Model, cpu.Cores, cpu.Threads, cpu.SerialNumber)
}
}
if len(hw.Memory) > 0 {
b.WriteString("\n[Memory]\n")
for _, m := range hw.Memory {
fmt.Fprintf(&b, "- slot=%s location=%s size_mb=%d part=%s serial=%s status=%s\n", m.Slot, m.Location, m.SizeMB, m.PartNumber, m.SerialNumber, m.Status)
}
}
if len(hw.Storage) > 0 {
b.WriteString("\n[Storage]\n")
for _, s := range hw.Storage {
fmt.Fprintf(&b, "- slot=%s type=%s model=%s size_gb=%d serial=%s\n", s.Slot, s.Type, s.Model, s.SizeGB, s.SerialNumber)
}
}
if len(hw.Volumes) > 0 {
b.WriteString("\n[Volumes]\n")
for _, v := range hw.Volumes {
name := v.Name
if name == "" {
name = v.ID
}
fmt.Fprintf(&b, "- controller=%s name=%s raid=%s size_gb=%d status=%s\n", v.Controller, name, v.RAIDLevel, v.SizeGB, v.Status)
}
}
if len(hw.PCIeDevices) > 0 {
b.WriteString("\n[PCIe Devices]\n")
for _, d := range hw.PCIeDevices {
fmt.Fprintf(&b, "- slot=%s class=%s model=%s bdf=%s vendor=%s serial=%s\n", d.Slot, d.DeviceClass, d.PartNumber, d.BDF, d.Manufacturer, d.SerialNumber)
}
}
if len(hw.GPUs) > 0 {
b.WriteString("\n[GPUs]\n")
for _, g := range hw.GPUs {
fmt.Fprintf(&b, "- slot=%s model=%s bdf=%s serial=%s status=%s\n", g.Slot, g.Model, g.BDF, g.SerialNumber, g.Status)
}
}
if len(hw.NetworkAdapters) > 0 {
b.WriteString("\n[Network Adapters]\n")
for _, n := range hw.NetworkAdapters {
fmt.Fprintf(&b, "- slot=%s location=%s model=%s serial=%s status=%s\n", n.Slot, n.Location, n.Model, n.SerialNumber, n.Status)
}
}
if len(hw.PowerSupply) > 0 {
b.WriteString("\n[Power Supplies]\n")
for _, p := range hw.PowerSupply {
fmt.Fprintf(&b, "- slot=%s model=%s serial=%s status=%s watt=%d\n", p.Slot, p.Model, p.SerialNumber, p.Status, p.WattageW)
}
}
if len(hw.Firmware) > 0 {
b.WriteString("\n[Firmware]\n")
for _, fw := range hw.Firmware {
fmt.Fprintf(&b, "- device=%s version=%s\n", fw.DeviceName, fw.Version)
}
}
return b.String()
}
func collectLogTimeBounds(lines []string) (time.Time, time.Time, bool) {
var first time.Time
var last time.Time
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
tsToken := line
if idx := strings.IndexByte(line, ' '); idx > 0 {
tsToken = line[:idx]
}
ts, err := time.Parse(time.RFC3339Nano, tsToken)
if err != nil {
continue
}
if first.IsZero() || ts.Before(first) {
first = ts
}
if last.IsZero() || ts.After(last) {
last = ts
}
}
if first.IsZero() || last.IsZero() || last.Before(first) {
return time.Time{}, time.Time{}, false
}
return first, last, true
}
func formatRawExportDuration(d time.Duration) string {
if d < 0 {
d = 0
}
return d.Round(time.Second).String()
}
func buildParserFieldSummary(result *models.AnalysisResult) map[string]any {
out := map[string]any{
"generated_at": time.Now().UTC(),
}
if result == nil {
out["available"] = false
return out
}
out["available"] = true
out["filename"] = result.Filename
out["source_type"] = result.SourceType
out["protocol"] = result.Protocol
out["target_host"] = result.TargetHost
out["source_timezone"] = result.SourceTimezone
out["collected_at"] = result.CollectedAt
if result.Hardware == nil {
out["hardware"] = map[string]any{}
return out
}
hw := result.Hardware
out["vendor"] = hw.BoardInfo.Manufacturer
out["model"] = hw.BoardInfo.ProductName
out["serial"] = hw.BoardInfo.SerialNumber
out["part_number"] = hw.BoardInfo.PartNumber
out["hardware"] = map[string]any{
"board": hw.BoardInfo,
"counts": map[string]int{
"cpus": len(hw.CPUs),
"memory": len(hw.Memory),
"storage": len(hw.Storage),
"volumes": len(hw.Volumes),
"pcie": len(hw.PCIeDevices),
"gpus": len(hw.GPUs),
"nics": len(hw.NetworkAdapters),
"psus": len(hw.PowerSupply),
"firmware": len(hw.Firmware),
"events": len(result.Events),
"sensors": len(result.Sensors),
"fru": len(result.FRU),
},
"cpus": hw.CPUs,
"memory": hw.Memory,
"storage": hw.Storage,
"volumes": hw.Volumes,
"pcie_devices": hw.PCIeDevices,
"gpus": hw.GPUs,
"network_adapters": hw.NetworkAdapters,
"power_supplies": hw.PowerSupply,
"firmware": hw.Firmware,
}
return out
}
func resultProtocol(result *models.AnalysisResult) string {
if result == nil {
return ""
}
return result.Protocol
}
func resultTargetHost(result *models.AnalysisResult) string {
if result == nil {
return ""
}
return result.TargetHost
}
func resultSourceTimezone(result *models.AnalysisResult) string {
if result == nil {
return ""
}
return strings.TrimSpace(result.SourceTimezone)
}