Files
logpile/internal/parser/archive_zip_recover.go
T
Mikhail ChusavitinandClaude Opus 5.5 1cc2faa567 fix(archive): open zips with a missing or truncated central directory
A BEE-SP v14.23 blackbox (Dell PowerEdge R760) streamed from the live CD was
cut off inside the central directory: all 1920 members intact, no
end-of-central-directory record, so archive/zip rejected the whole file
("zip: not a valid zip file"). ADL-079:
- when archive/zip cannot open a .zip, read it front to back by local file
  headers; intact archives are unaffected
- streamed members (data descriptors) are delimited from the data itself,
  deflate and stored; every member is CRC32/size checked
- recovery stops at the first damaged, truncated or encrypted member and
  keeps the entries before it; the original error is returned when nothing
  could be read
- same 10 MB per-file / 1 GB total limits as the regular zip path

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-02 15:00:39 +03:00

270 lines
8.1 KiB
Go

package parser
import (
"archive/zip"
"bufio"
"compress/flate"
"encoding/binary"
"fmt"
"hash/crc32"
"io"
"strings"
"time"
)
const (
zipLocalHeaderSignature = 0x04034b50
zipDataDescriptorSignature = 0x08074b50
zipLocalHeaderLen = 30
zipFlagEncrypted = 0x1
zipFlagDataDescriptor = 0x8
zipExtraZip64 = 0x0001
zipExtraExtendedTimestamp = 0x5455
zipMaxEntrySize = 10 * 1024 * 1024
)
// recoverZipLocalEntries reads a zip archive front to back by its local file
// headers, ignoring the central directory. It is the fallback for archives
// whose central directory is missing or cut off — typically a zip streamed
// over HTTP whose connection closed before the trailer was fully sent: the
// member data is intact, but archive/zip refuses the file because it only
// reads through the end-of-central-directory record.
//
// Recovery stops at the first thing that is not an intact local entry (the
// central directory, a truncated or CRC-mismatching member, an encrypted or
// otherwise unreadable one) and returns the entries before it. Size limits
// match the regular zip path. An error is returned only when no entry at all
// could be read.
func recoverZipLocalEntries(r io.Reader) ([]ExtractedFile, error) {
br := bufio.NewReaderSize(r, 64*1024)
var files []ExtractedFile
var totalExtracted int64
entries := 0
for {
var fixed [zipLocalHeaderLen]byte
if _, err := io.ReadFull(br, fixed[:]); err != nil {
break
}
if binary.LittleEndian.Uint32(fixed[0:]) != zipLocalHeaderSignature {
break
}
flags := binary.LittleEndian.Uint16(fixed[6:])
method := binary.LittleEndian.Uint16(fixed[8:])
dosTime := binary.LittleEndian.Uint16(fixed[10:])
dosDate := binary.LittleEndian.Uint16(fixed[12:])
headerCRC := binary.LittleEndian.Uint32(fixed[14:])
compressedSize := uint64(binary.LittleEndian.Uint32(fixed[18:]))
uncompressedSize := uint64(binary.LittleEndian.Uint32(fixed[22:]))
nameLen := int(binary.LittleEndian.Uint16(fixed[26:]))
extraLen := int(binary.LittleEndian.Uint16(fixed[28:]))
nameAndExtra := make([]byte, nameLen+extraLen)
if _, err := io.ReadFull(br, nameAndExtra); err != nil {
break
}
name := string(nameAndExtra[:nameLen])
modTime := zipDOSTime(dosDate, dosTime)
parseZipLocalExtra(nameAndExtra[nameLen:], &modTime, &uncompressedSize, &compressedSize)
if flags&zipFlagEncrypted != 0 {
break
}
var content []byte
var size int64
var ok bool
if flags&zipFlagDataDescriptor != 0 {
// Streamed entry: sizes and CRC follow the data, so the end of
// the member has to be found from the data itself.
switch method {
case zip.Deflate:
// bufio.Reader is an io.ByteReader, so flate consumes exactly
// the compressed stream and leaves br at the data descriptor.
var sum uint32
var err error
content, size, sum, err = readZipEntryBody(flate.NewReader(br))
ok = err == nil && consumeZipDataDescriptor(br, sum, size)
case zip.Store:
content, ok = readStoredZipEntryUntilDescriptor(br)
size = int64(len(content))
}
} else {
body := io.LimitReader(br, int64(compressedSize))
switch method {
case zip.Store, zip.Deflate:
src := body
if method == zip.Deflate {
src = flate.NewReader(body)
}
var sum uint32
var err error
content, size, sum, err = readZipEntryBody(src)
if err == nil {
_, err = io.Copy(io.Discard, body)
}
ok = err == nil && sum == headerCRC && uint64(size) == uncompressedSize
default:
// Unknown compression but a known length: skip the member.
if n, err := io.Copy(io.Discard, body); err != nil || uint64(n) != compressedSize {
break
}
entries++
continue
}
}
if !ok {
break
}
entries++
if strings.HasSuffix(name, "/") {
continue
}
// Skip large files (>10MB)
if size > zipMaxEntrySize {
continue
}
if totalExtracted+size > maxSingleFileSizeLarge {
break
}
files = append(files, ExtractedFile{
Path: name,
Content: content,
ModTime: modTime,
})
totalExtracted += size
}
if entries == 0 {
return nil, fmt.Errorf("no intact local file entries")
}
return files, nil
}
// readZipEntryBody reads src to EOF and returns its length and CRC32. The
// content is kept only while it fits the per-file limit; a larger member is
// still read through (its end has to be found) but returned with nil content.
func readZipEntryBody(src io.Reader) ([]byte, int64, uint32, error) {
sum := crc32.NewIEEE()
content, err := io.ReadAll(io.TeeReader(io.LimitReader(src, zipMaxEntrySize+1), sum))
if err != nil {
return nil, 0, 0, err
}
size := int64(len(content))
if size > zipMaxEntrySize {
rest, err := io.Copy(sum, src)
if err != nil {
return nil, 0, 0, err
}
return nil, size + rest, sum.Sum32(), nil
}
return content, size, sum.Sum32(), nil
}
// consumeZipDataDescriptor consumes the data descriptor at the head of br if
// it matches the given CRC32 and uncompressed size. The descriptor signature
// is optional and the sizes are 4 or 8 bytes wide (zip64), with nothing in
// the descriptor saying which; a layout is accepted only when it is followed
// by another record ("PK") or the end of input.
func consumeZipDataDescriptor(br *bufio.Reader, sum uint32, size int64) bool {
head, _ := br.Peek(26)
for _, sigLen := range []int{4, 0} {
if sigLen == 4 && (len(head) < 4 || binary.LittleEndian.Uint32(head) != zipDataDescriptorSignature) {
continue
}
for _, sizeLen := range []int{4, 8} {
n := sigLen + 4 + 2*sizeLen
if len(head) < n || binary.LittleEndian.Uint32(head[sigLen:]) != sum {
continue
}
sizeField := head[sigLen+4+sizeLen : n]
var got uint64
if sizeLen == 4 {
got = uint64(binary.LittleEndian.Uint32(sizeField))
} else {
got = binary.LittleEndian.Uint64(sizeField)
}
if got != uint64(size) {
continue
}
if len(head) >= n+2 && string(head[n:n+2]) != "PK" {
continue
}
_, _ = br.Discard(n)
return true
}
}
return false
}
// readStoredZipEntryUntilDescriptor reads an uncompressed streamed member,
// whose length is not recorded ahead of the data: the member ends at the
// first data descriptor signature whose CRC32 and size match the bytes read
// so far. Members over the per-file limit, or written without the optional
// descriptor signature, are not recovered.
func readStoredZipEntryUntilDescriptor(br *bufio.Reader) ([]byte, bool) {
var content []byte
for {
if head, _ := br.Peek(4); len(head) == 4 && binary.LittleEndian.Uint32(head) == zipDataDescriptorSignature {
if consumeZipDataDescriptor(br, crc32.ChecksumIEEE(content), int64(len(content))) {
return content, true
}
}
b, err := br.ReadByte()
if err != nil {
return nil, false
}
content = append(content, b)
if len(content) > zipMaxEntrySize {
return nil, false
}
}
}
// parseZipLocalExtra applies the local-header extra fields that matter for
// recovery: the extended timestamp (UTC mtime, overrides the DOS time) and
// zip64 sizes (only present for the 32-bit fields set to 0xFFFFFFFF).
func parseZipLocalExtra(extra []byte, modTime *time.Time, uncompressedSize, compressedSize *uint64) {
for len(extra) >= 4 {
id := binary.LittleEndian.Uint16(extra[0:])
n := int(binary.LittleEndian.Uint16(extra[2:]))
extra = extra[4:]
if n > len(extra) {
return
}
field := extra[:n]
extra = extra[n:]
switch id {
case zipExtraExtendedTimestamp:
if len(field) >= 5 && field[0]&1 != 0 {
*modTime = time.Unix(int64(int32(binary.LittleEndian.Uint32(field[1:]))), 0).UTC()
}
case zipExtraZip64:
if *uncompressedSize == 0xFFFFFFFF && len(field) >= 8 {
*uncompressedSize = binary.LittleEndian.Uint64(field)
field = field[8:]
}
if *compressedSize == 0xFFFFFFFF && len(field) >= 8 {
*compressedSize = binary.LittleEndian.Uint64(field)
}
}
}
}
// zipDOSTime converts an MS-DOS date and time to UTC, as archive/zip does.
func zipDOSTime(dosDate, dosTime uint16) time.Time {
return time.Date(
int(dosDate>>9)+1980,
time.Month(dosDate>>5&0xf),
int(dosDate&0x1f),
int(dosTime>>11),
int(dosTime>>5&0x3f),
int(dosTime&0x1f)*2,
0,
time.UTC,
)
}