mchus/logpile
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases 20 Wiki Activity

fix(parser): raise file size limit for .ahs to 1 GB

Browse Source 
AHS files can exceed 100 MB; the previous 10 MB universal cap silently
truncated them and caused incomplete event parsing. Per-extension limits
are now used: .ahs gets 1 GB, all other single-file types keep 10 MB.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-06-19 15:20:57 +03:00
parent 6b52a1876f
commit e2c81758b5
1 changed files with 13 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/parser/archive.go
+13 -11
View File
@@ -15,9 +15,11 @@ import (
) )
const maxSingleFileSize = 10 * 1024 * 1024 const maxSingleFileSize = 10 * 1024 * 1024
const maxSingleFileSizeLarge = 1024 * 1024 * 1024
const maxZipArchiveSize = 50 * 1024 * 1024 const maxZipArchiveSize = 50 * 1024 * 1024
const maxGzipDecompressedSize = 50 * 1024 * 1024 const maxGzipDecompressedSize = 50 * 1024 * 1024
var supportedArchiveExt = map[string]struct{}{ var supportedArchiveExt = map[string]struct{}{
".ahs": {}, ".ahs": {},
".gz": {}, ".gz": {},
@@ -47,7 +49,7 @@ func ExtractArchive(archivePath string) ([]ExtractedFile, error) {
switch ext { switch ext {
case ".ahs": case ".ahs":
return extractSingleFile(archivePath) return extractSingleFileWithLimit(archivePath, maxSingleFileSizeLarge)
case ".gz", ".tgz": case ".gz", ".tgz":
return extractTarGz(archivePath) return extractTarGz(archivePath)
case ".tar", ".sds": case ".tar", ".sds":
@@ -55,7 +57,7 @@ func ExtractArchive(archivePath string) ([]ExtractedFile, error) {
case ".zip": case ".zip":
return extractZip(archivePath) return extractZip(archivePath)
case ".txt", ".log": case ".txt", ".log":
return extractSingleFile(archivePath) return extractSingleFileWithLimit(archivePath, maxSingleFileSize)
default: default:
return nil, fmt.Errorf("unsupported archive format: %s", ext) return nil, fmt.Errorf("unsupported archive format: %s", ext)
} }
@@ -70,7 +72,7 @@ func ExtractArchiveFromReader(r io.Reader, filename string) ([]ExtractedFile, er
switch ext { switch ext {
case ".ahs": case ".ahs":
return extractSingleFileFromReader(r, filename) return extractSingleFileFromReaderWithLimit(r, filename, maxSingleFileSizeLarge)
case ".gz", ".tgz": case ".gz", ".tgz":
return extractTarGzFromReader(r, filename) return extractTarGzFromReader(r, filename)
case ".tar", ".sds": case ".tar", ".sds":
@@ -78,7 +80,7 @@ func ExtractArchiveFromReader(r io.Reader, filename string) ([]ExtractedFile, er
case ".zip": case ".zip":
return extractZipFromReader(r) return extractZipFromReader(r)
case ".txt", ".log": case ".txt", ".log":
return extractSingleFileFromReader(r, filename) return extractSingleFileFromReaderWithLimit(r, filename, maxSingleFileSize)
default: default:
return nil, fmt.Errorf("unsupported archive format: %s", ext) return nil, fmt.Errorf("unsupported archive format: %s", ext)
} }
@@ -337,7 +339,7 @@ func extractZipFromReader(r io.Reader) ([]ExtractedFile, error) {
return files, nil return files, nil
} }
func extractSingleFile(path string) ([]ExtractedFile, error) { func extractSingleFileWithLimit(path string, limit int64) ([]ExtractedFile, error) {
info, err := os.Stat(path) info, err := os.Stat(path)
if err != nil { if err != nil {
return nil, fmt.Errorf("stat file: %w", err) return nil, fmt.Errorf("stat file: %w", err)
@@ -348,7 +350,7 @@ func extractSingleFile(path string) ([]ExtractedFile, error) {
} }
defer f.Close() defer f.Close()
files, err := extractSingleFileFromReader(f, filepath.Base(path)) files, err := extractSingleFileFromReaderWithLimit(f, filepath.Base(path), limit)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -358,14 +360,14 @@ func extractSingleFile(path string) ([]ExtractedFile, error) {
return files, nil return files, nil
} }
func extractSingleFileFromReader(r io.Reader, filename string) ([]ExtractedFile, error) { func extractSingleFileFromReaderWithLimit(r io.Reader, filename string, limit int64) ([]ExtractedFile, error) {
content, err := io.ReadAll(io.LimitReader(r, maxSingleFileSize+1)) content, err := io.ReadAll(io.LimitReader(r, limit+1))
if err != nil { if err != nil {
return nil, fmt.Errorf("read file content: %w", err) return nil, fmt.Errorf("read file content: %w", err)
} }
truncated := len(content) > maxSingleFileSize truncated := int64(len(content)) > limit
if truncated { if truncated {
content = content[:maxSingleFileSize] content = content[:limit]
} }
file := ExtractedFile{ file := ExtractedFile{
@@ -376,7 +378,7 @@ func extractSingleFileFromReader(r io.Reader, filename string) ([]ExtractedFile,
file.Truncated = true file.Truncated = true
file.TruncatedMessage = fmt.Sprintf( file.TruncatedMessage = fmt.Sprintf(
"file exceeded %d bytes and was truncated", "file exceeded %d bytes and was truncated",
maxSingleFileSize, limit,
) )
} }