mchus/logpile
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases 24 Wiki Activity

sync file-type support across upload/convert and fix collected_at timezone handling

Browse Source
This commit is contained in:
mchus
2026-02-28 23:27:49 +03:00
parent 736b77f055
commit 4940cd9645
20 changed files with 931 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/server/handlers.go
+154 -27
View File
@@ -138,6 +138,9 @@ func (s *Server) analyzeUploadedFile(filename, mimeType string, payload []byte)
}
return snapshotResult, vendor, newRawExportFromUploadedFile(filename, mimeType, payload, snapshotResult), nil
}
if !parser.IsSupportedArchiveFilename(filename) {
return nil, "", nil, fmt.Errorf("unsupported archive format: %s", strings.ToLower(filepath.Ext(filename)))
}
p := parser.NewBMCParser()
if err := p.ParseFromReader(bytes.NewReader(payload), filename); err != nil {
@@ -208,9 +211,10 @@ func (s *Server) reanalyzeRawExportPackage(pkg *RawExportPackage) (*models.Analy
if strings.TrimSpace(result.TargetHost) == "" {
result.TargetHost = strings.TrimSpace(pkg.Source.TargetHost)
}
if result.CollectedAt.IsZero() {
result.CollectedAt = time.Now().UTC()
if strings.TrimSpace(result.SourceTimezone) == "" {
result.SourceTimezone = strings.TrimSpace(pkg.Source.SourceTimezone)
}
result.CollectedAt = inferRawExportCollectedAt(result, pkg)
if strings.TrimSpace(result.Filename) == "" {
target := result.TargetHost
if target == "" {
@@ -253,6 +257,39 @@ func (s *Server) handleGetParsers(w http.ResponseWriter, r *http.Request) {
})
}
func (s *Server) handleGetFileTypes(w http.ResponseWriter, r *http.Request) {
archiveExt := parser.SupportedArchiveExtensions()
uploadExt := append([]string{}, archiveExt...)
uploadExt = append(uploadExt, ".json")
jsonResponse(w, map[string]any{
"archive_extensions": archiveExt,
"upload_extensions": uniqueSortedExtensions(uploadExt),
"convert_extensions": uniqueSortedExtensions(uploadExt),
})
}
func uniqueSortedExtensions(exts []string) []string {
if len(exts) == 0 {
return nil
}
seen := make(map[string]struct{}, len(exts))
out := make([]string, 0, len(exts))
for _, e := range exts {
e = strings.ToLower(strings.TrimSpace(e))
if e == "" {
continue
}
if _, ok := seen[e]; ok {
continue
}
seen[e] = struct{}{}
out = append(out, e)
}
sort.Strings(out)
return out
}
func (s *Server) handleGetEvents(w http.ResponseWriter, r *http.Request) {
result := s.GetResult()
if result == nil {
@@ -324,10 +361,11 @@ func (s *Server) handleGetConfig(w http.ResponseWriter, r *http.Request) {
}
response := map[string]interface{}{
"source_type": result.SourceType,
"protocol": result.Protocol,
"target_host": result.TargetHost,
"collected_at": result.CollectedAt,
"source_type": result.SourceType,
"protocol": result.Protocol,
"target_host": result.TargetHost,
"source_timezone": result.SourceTimezone,
"collected_at": result.CollectedAt,
}
if result.RawPayloads != nil {
if fetchErrors, ok := result.RawPayloads["redfish_fetch_errors"]; ok {
@@ -1012,13 +1050,14 @@ func (s *Server) handleGetStatus(w http.ResponseWriter, r *http.Request) {
}
jsonResponse(w, map[string]interface{}{
"loaded": true,
"filename": result.Filename,
"vendor": s.GetDetectedVendor(),
"source_type": result.SourceType,
"protocol": result.Protocol,
"target_host": result.TargetHost,
"collected_at": result.CollectedAt,
"loaded": true,
"filename": result.Filename,
"vendor": s.GetDetectedVendor(),
"source_type": result.SourceType,
"protocol": result.Protocol,
"target_host": result.TargetHost,
"source_timezone": result.SourceTimezone,
"collected_at": result.CollectedAt,
"stats": map[string]int{
"events": len(result.Events),
"sensors": len(result.Sensors),
@@ -1362,17 +1401,14 @@ func (s *Server) handleConvertDownload(w http.ResponseWriter, r *http.Request) {
}
func isSupportedConvertFileName(filename string) bool {
name := strings.ToLower(strings.TrimSpace(filename))
name := strings.TrimSpace(filename)
if name == "" {
return false
}
return strings.HasSuffix(name, ".zip") ||
strings.HasSuffix(name, ".tar") ||
strings.HasSuffix(name, ".tar.gz") ||
strings.HasSuffix(name, ".tgz") ||
strings.HasSuffix(name, ".json") ||
strings.HasSuffix(name, ".txt") ||
strings.HasSuffix(name, ".log")
if strings.HasSuffix(strings.ToLower(name), ".json") {
return true
}
return parser.IsSupportedArchiveFilename(name)
}
func sanitizeZipPath(filename string) string {
@@ -1599,17 +1635,101 @@ func inferArchiveCollectedAt(result *models.AnalysisResult) time.Time {
return time.Now().UTC()
}
var latest time.Time
var latestReliable time.Time
var latestAny time.Time
for _, event := range result.Events {
if event.Timestamp.IsZero() {
continue
}
if latest.IsZero() || event.Timestamp.After(latest) {
latest = event.Timestamp
// Drop obviously bad epochs from broken RTC logs.
if event.Timestamp.Year() < 2000 {
continue
}
if latestAny.IsZero() || event.Timestamp.After(latestAny) {
latestAny = event.Timestamp
}
if !isReliableCollectedAtEvent(event) {
continue
}
if latestReliable.IsZero() || event.Timestamp.After(latestReliable) {
latestReliable = event.Timestamp
}
}
if !latest.IsZero() {
return latest.UTC()
if !latestReliable.IsZero() {
return latestReliable.UTC()
}
if !latestAny.IsZero() {
return latestAny.UTC()
}
if fromFilename, ok := inferCollectedAtFromFilename(result.Filename); ok {
return fromFilename.UTC()
}
return time.Now().UTC()
}
func isReliableCollectedAtEvent(event models.Event) bool {
// component.log-derived synthetic states are created "at parse time"
// and must not override real log timestamps.
src := strings.ToLower(strings.TrimSpace(event.Source))
etype := strings.ToLower(strings.TrimSpace(event.EventType))
stype := strings.ToLower(strings.TrimSpace(event.SensorType))
if etype == "fan status" && (src == "fan" || stype == "fan") {
return false
}
if etype == "memory status" && (src == "memory" || stype == "memory") {
return false
}
return true
}
var (
filenameDateTimeRegex = regexp.MustCompile(`(?i)(\d{8})[-_](\d{4})(\d{2})?`)
filenameDateRegex = regexp.MustCompile(`(?i)(\d{4})-(\d{2})-(\d{2})`)
)
func inferCollectedAtFromFilename(name string) (time.Time, bool) {
base := strings.TrimSpace(filepath.Base(name))
if base == "" {
return time.Time{}, false
}
if m := filenameDateTimeRegex.FindStringSubmatch(base); len(m) == 4 {
datePart := m[1]
timePart := m[2]
if strings.TrimSpace(m[3]) != "" {
timePart += m[3]
} else {
timePart += "00"
}
if ts, err := parser.ParseInDefaultArchiveLocation("20060102 150405", datePart+" "+timePart); err == nil {
return ts, true
}
}
if m := filenameDateRegex.FindStringSubmatch(base); len(m) == 4 {
datePart := m[1] + "-" + m[2] + "-" + m[3]
if ts, err := parser.ParseInDefaultArchiveLocation("2006-01-02 15:04:05", datePart+" 00:00:00"); err == nil {
return ts, true
}
}
return time.Time{}, false
}
func inferRawExportCollectedAt(result *models.AnalysisResult, pkg *RawExportPackage) time.Time {
if result != nil && !result.CollectedAt.IsZero() {
return result.CollectedAt.UTC()
}
if pkg != nil {
if !pkg.CollectedAtHint.IsZero() {
return pkg.CollectedAtHint.UTC()
}
if _, finishedAt, ok := collectLogTimeBounds(pkg.Source.CollectLogs); ok {
return finishedAt.UTC()
}
if !pkg.ExportedAt.IsZero() {
return pkg.ExportedAt.UTC()
}
}
return time.Now().UTC()
}
@@ -1621,7 +1741,14 @@ func applyCollectSourceMetadata(result *models.AnalysisResult, req CollectReques
result.SourceType = models.SourceTypeAPI
result.Protocol = req.Protocol
result.TargetHost = req.Host
result.CollectedAt = time.Now().UTC()
if strings.TrimSpace(result.SourceTimezone) == "" && result.RawPayloads != nil {
if tz, ok := result.RawPayloads["source_timezone"].(string); ok {
result.SourceTimezone = strings.TrimSpace(tz)
}
}
if result.CollectedAt.IsZero() {
result.CollectedAt = time.Now().UTC()
}
if strings.TrimSpace(result.Filename) == "" {
result.Filename = fmt.Sprintf("%s://%s", req.Protocol, req.Host)
}