A BEE-SP v14.23 blackbox (Dell PowerEdge R760) streamed from the live CD was
cut off inside the central directory: all 1920 members intact, no
end-of-central-directory record, so archive/zip rejected the whole file
("zip: not a valid zip file"). ADL-079:
- when archive/zip cannot open a .zip, read it front to back by local file
headers; intact archives are unaffected
- streamed members (data descriptors) are delimited from the data itself,
deflate and stored; every member is CRC32/size checked
- recovery stops at the first damaged, truncated or encrypted member and
keeps the entries before it; the original error is returned when nothing
could be read
- same 10 MB per-file / 1 GB total limits as the regular zip path
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
270 lines
8.1 KiB
Go
270 lines
8.1 KiB
Go
package parser
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bufio"
|
|
"compress/flate"
|
|
"encoding/binary"
|
|
"fmt"
|
|
"hash/crc32"
|
|
"io"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
zipLocalHeaderSignature = 0x04034b50
|
|
zipDataDescriptorSignature = 0x08074b50
|
|
zipLocalHeaderLen = 30
|
|
zipFlagEncrypted = 0x1
|
|
zipFlagDataDescriptor = 0x8
|
|
zipExtraZip64 = 0x0001
|
|
zipExtraExtendedTimestamp = 0x5455
|
|
zipMaxEntrySize = 10 * 1024 * 1024
|
|
)
|
|
|
|
// recoverZipLocalEntries reads a zip archive front to back by its local file
|
|
// headers, ignoring the central directory. It is the fallback for archives
|
|
// whose central directory is missing or cut off — typically a zip streamed
|
|
// over HTTP whose connection closed before the trailer was fully sent: the
|
|
// member data is intact, but archive/zip refuses the file because it only
|
|
// reads through the end-of-central-directory record.
|
|
//
|
|
// Recovery stops at the first thing that is not an intact local entry (the
|
|
// central directory, a truncated or CRC-mismatching member, an encrypted or
|
|
// otherwise unreadable one) and returns the entries before it. Size limits
|
|
// match the regular zip path. An error is returned only when no entry at all
|
|
// could be read.
|
|
func recoverZipLocalEntries(r io.Reader) ([]ExtractedFile, error) {
|
|
br := bufio.NewReaderSize(r, 64*1024)
|
|
var files []ExtractedFile
|
|
var totalExtracted int64
|
|
entries := 0
|
|
|
|
for {
|
|
var fixed [zipLocalHeaderLen]byte
|
|
if _, err := io.ReadFull(br, fixed[:]); err != nil {
|
|
break
|
|
}
|
|
if binary.LittleEndian.Uint32(fixed[0:]) != zipLocalHeaderSignature {
|
|
break
|
|
}
|
|
flags := binary.LittleEndian.Uint16(fixed[6:])
|
|
method := binary.LittleEndian.Uint16(fixed[8:])
|
|
dosTime := binary.LittleEndian.Uint16(fixed[10:])
|
|
dosDate := binary.LittleEndian.Uint16(fixed[12:])
|
|
headerCRC := binary.LittleEndian.Uint32(fixed[14:])
|
|
compressedSize := uint64(binary.LittleEndian.Uint32(fixed[18:]))
|
|
uncompressedSize := uint64(binary.LittleEndian.Uint32(fixed[22:]))
|
|
nameLen := int(binary.LittleEndian.Uint16(fixed[26:]))
|
|
extraLen := int(binary.LittleEndian.Uint16(fixed[28:]))
|
|
|
|
nameAndExtra := make([]byte, nameLen+extraLen)
|
|
if _, err := io.ReadFull(br, nameAndExtra); err != nil {
|
|
break
|
|
}
|
|
name := string(nameAndExtra[:nameLen])
|
|
modTime := zipDOSTime(dosDate, dosTime)
|
|
parseZipLocalExtra(nameAndExtra[nameLen:], &modTime, &uncompressedSize, &compressedSize)
|
|
|
|
if flags&zipFlagEncrypted != 0 {
|
|
break
|
|
}
|
|
|
|
var content []byte
|
|
var size int64
|
|
var ok bool
|
|
if flags&zipFlagDataDescriptor != 0 {
|
|
// Streamed entry: sizes and CRC follow the data, so the end of
|
|
// the member has to be found from the data itself.
|
|
switch method {
|
|
case zip.Deflate:
|
|
// bufio.Reader is an io.ByteReader, so flate consumes exactly
|
|
// the compressed stream and leaves br at the data descriptor.
|
|
var sum uint32
|
|
var err error
|
|
content, size, sum, err = readZipEntryBody(flate.NewReader(br))
|
|
ok = err == nil && consumeZipDataDescriptor(br, sum, size)
|
|
case zip.Store:
|
|
content, ok = readStoredZipEntryUntilDescriptor(br)
|
|
size = int64(len(content))
|
|
}
|
|
} else {
|
|
body := io.LimitReader(br, int64(compressedSize))
|
|
switch method {
|
|
case zip.Store, zip.Deflate:
|
|
src := body
|
|
if method == zip.Deflate {
|
|
src = flate.NewReader(body)
|
|
}
|
|
var sum uint32
|
|
var err error
|
|
content, size, sum, err = readZipEntryBody(src)
|
|
if err == nil {
|
|
_, err = io.Copy(io.Discard, body)
|
|
}
|
|
ok = err == nil && sum == headerCRC && uint64(size) == uncompressedSize
|
|
default:
|
|
// Unknown compression but a known length: skip the member.
|
|
if n, err := io.Copy(io.Discard, body); err != nil || uint64(n) != compressedSize {
|
|
break
|
|
}
|
|
entries++
|
|
continue
|
|
}
|
|
}
|
|
if !ok {
|
|
break
|
|
}
|
|
entries++
|
|
|
|
if strings.HasSuffix(name, "/") {
|
|
continue
|
|
}
|
|
// Skip large files (>10MB)
|
|
if size > zipMaxEntrySize {
|
|
continue
|
|
}
|
|
if totalExtracted+size > maxSingleFileSizeLarge {
|
|
break
|
|
}
|
|
|
|
files = append(files, ExtractedFile{
|
|
Path: name,
|
|
Content: content,
|
|
ModTime: modTime,
|
|
})
|
|
totalExtracted += size
|
|
}
|
|
|
|
if entries == 0 {
|
|
return nil, fmt.Errorf("no intact local file entries")
|
|
}
|
|
return files, nil
|
|
}
|
|
|
|
// readZipEntryBody reads src to EOF and returns its length and CRC32. The
|
|
// content is kept only while it fits the per-file limit; a larger member is
|
|
// still read through (its end has to be found) but returned with nil content.
|
|
func readZipEntryBody(src io.Reader) ([]byte, int64, uint32, error) {
|
|
sum := crc32.NewIEEE()
|
|
content, err := io.ReadAll(io.TeeReader(io.LimitReader(src, zipMaxEntrySize+1), sum))
|
|
if err != nil {
|
|
return nil, 0, 0, err
|
|
}
|
|
size := int64(len(content))
|
|
if size > zipMaxEntrySize {
|
|
rest, err := io.Copy(sum, src)
|
|
if err != nil {
|
|
return nil, 0, 0, err
|
|
}
|
|
return nil, size + rest, sum.Sum32(), nil
|
|
}
|
|
return content, size, sum.Sum32(), nil
|
|
}
|
|
|
|
// consumeZipDataDescriptor consumes the data descriptor at the head of br if
|
|
// it matches the given CRC32 and uncompressed size. The descriptor signature
|
|
// is optional and the sizes are 4 or 8 bytes wide (zip64), with nothing in
|
|
// the descriptor saying which; a layout is accepted only when it is followed
|
|
// by another record ("PK") or the end of input.
|
|
func consumeZipDataDescriptor(br *bufio.Reader, sum uint32, size int64) bool {
|
|
head, _ := br.Peek(26)
|
|
for _, sigLen := range []int{4, 0} {
|
|
if sigLen == 4 && (len(head) < 4 || binary.LittleEndian.Uint32(head) != zipDataDescriptorSignature) {
|
|
continue
|
|
}
|
|
for _, sizeLen := range []int{4, 8} {
|
|
n := sigLen + 4 + 2*sizeLen
|
|
if len(head) < n || binary.LittleEndian.Uint32(head[sigLen:]) != sum {
|
|
continue
|
|
}
|
|
sizeField := head[sigLen+4+sizeLen : n]
|
|
var got uint64
|
|
if sizeLen == 4 {
|
|
got = uint64(binary.LittleEndian.Uint32(sizeField))
|
|
} else {
|
|
got = binary.LittleEndian.Uint64(sizeField)
|
|
}
|
|
if got != uint64(size) {
|
|
continue
|
|
}
|
|
if len(head) >= n+2 && string(head[n:n+2]) != "PK" {
|
|
continue
|
|
}
|
|
_, _ = br.Discard(n)
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// readStoredZipEntryUntilDescriptor reads an uncompressed streamed member,
|
|
// whose length is not recorded ahead of the data: the member ends at the
|
|
// first data descriptor signature whose CRC32 and size match the bytes read
|
|
// so far. Members over the per-file limit, or written without the optional
|
|
// descriptor signature, are not recovered.
|
|
func readStoredZipEntryUntilDescriptor(br *bufio.Reader) ([]byte, bool) {
|
|
var content []byte
|
|
for {
|
|
if head, _ := br.Peek(4); len(head) == 4 && binary.LittleEndian.Uint32(head) == zipDataDescriptorSignature {
|
|
if consumeZipDataDescriptor(br, crc32.ChecksumIEEE(content), int64(len(content))) {
|
|
return content, true
|
|
}
|
|
}
|
|
b, err := br.ReadByte()
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
content = append(content, b)
|
|
if len(content) > zipMaxEntrySize {
|
|
return nil, false
|
|
}
|
|
}
|
|
}
|
|
|
|
// parseZipLocalExtra applies the local-header extra fields that matter for
|
|
// recovery: the extended timestamp (UTC mtime, overrides the DOS time) and
|
|
// zip64 sizes (only present for the 32-bit fields set to 0xFFFFFFFF).
|
|
func parseZipLocalExtra(extra []byte, modTime *time.Time, uncompressedSize, compressedSize *uint64) {
|
|
for len(extra) >= 4 {
|
|
id := binary.LittleEndian.Uint16(extra[0:])
|
|
n := int(binary.LittleEndian.Uint16(extra[2:]))
|
|
extra = extra[4:]
|
|
if n > len(extra) {
|
|
return
|
|
}
|
|
field := extra[:n]
|
|
extra = extra[n:]
|
|
|
|
switch id {
|
|
case zipExtraExtendedTimestamp:
|
|
if len(field) >= 5 && field[0]&1 != 0 {
|
|
*modTime = time.Unix(int64(int32(binary.LittleEndian.Uint32(field[1:]))), 0).UTC()
|
|
}
|
|
case zipExtraZip64:
|
|
if *uncompressedSize == 0xFFFFFFFF && len(field) >= 8 {
|
|
*uncompressedSize = binary.LittleEndian.Uint64(field)
|
|
field = field[8:]
|
|
}
|
|
if *compressedSize == 0xFFFFFFFF && len(field) >= 8 {
|
|
*compressedSize = binary.LittleEndian.Uint64(field)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// zipDOSTime converts an MS-DOS date and time to UTC, as archive/zip does.
|
|
func zipDOSTime(dosDate, dosTime uint16) time.Time {
|
|
return time.Date(
|
|
int(dosDate>>9)+1980,
|
|
time.Month(dosDate>>5&0xf),
|
|
int(dosDate&0x1f),
|
|
int(dosTime>>11),
|
|
int(dosTime>>5&0x3f),
|
|
int(dosTime&0x1f)*2,
|
|
0,
|
|
time.UTC,
|
|
)
|
|
}
|