From 1cc2faa567a4c65cfe29d87a175ea00baeddbe4f Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Fri, 2 Oct 2026 15:00:39 +0300 Subject: [PATCH] fix(archive): open zips with a missing or truncated central directory A BEE-SP v14.23 blackbox (Dell PowerEdge R760) streamed from the live CD was cut off inside the central directory: all 1920 members intact, no end-of-central-directory record, so archive/zip rejected the whole file ("zip: not a valid zip file"). ADL-079: - when archive/zip cannot open a .zip, read it front to back by local file headers; intact archives are unaffected - streamed members (data descriptors) are delimited from the data itself, deflate and stored; every member is CRC32/size checked - recovery stops at the first damaged, truncated or encrypted member and keeps the entries before it; the original error is returned when nothing could be read - same 10 MB per-file / 1 GB total limits as the regular zip path Co-Authored-By: Claude Opus 5.5 --- bible-local/06-parsers.md | 4 + bible-local/10-decisions.md | 43 ++++ internal/parser/archive.go | 13 + internal/parser/archive_zip_recover.go | 269 ++++++++++++++++++++ internal/parser/archive_zip_recover_test.go | 192 ++++++++++++++ 5 files changed, 521 insertions(+) create mode 100644 internal/parser/archive_zip_recover.go create mode 100644 internal/parser/archive_zip_recover_test.go diff --git a/bible-local/06-parsers.md b/bible-local/06-parsers.md index eda326e..d58b929 100644 --- a/bible-local/06-parsers.md +++ b/bible-local/06-parsers.md @@ -20,6 +20,10 @@ extension: `.tar`/`.sds`, `.tar.gz`/`.tgz`/`.gz`, `.zip`, `.rar`, `.ahs`, and pl volume, including solid archives. Password-protected archives, multi-volume sets, self-extracting archives and the legacy RAR 4.x format are rejected with an explicit error. +A `.zip` that `archive/zip` rejects (central directory missing or cut off, e.g. a streamed +blackbox whose download was interrupted in the trailer) is read front to back by its local file +headers (ADL-079); members are CRC-checked and recovery stops at the first damaged one. + `VendorParser` contract: ```go diff --git a/bible-local/10-decisions.md b/bible-local/10-decisions.md index 8bb4d22..16c8ff2 100644 --- a/bible-local/10-decisions.md +++ b/bible-local/10-decisions.md @@ -2328,3 +2328,46 @@ and no parser matched the content. now parses as `linux-generic` with partial inventory instead of failing with "no parser found". - Not parsed yet: `ip`/`ethtool` (MAC addresses), `nvidia-smi` (GPU serials), `ipmitool` (FRU/sensors). Each is one more entry in the utility table. + +## ADL-079 — A zip without a usable central directory is read by its local file headers + +**Date:** 2026-10-02 +**Context:** A BEE-SP v14.23 blackbox of a Dell PowerEdge R760 (1XWT5J4) could not be opened: +`open zip: zip: not a valid zip file`. The live CD streams the zip over HTTP, and the stream was +cut while the trailer was being sent — the file ends 39 bytes into the 1251st of 1920 central +directory records, with no end-of-central-directory record. All 1920 members (local header, +deflate data, data descriptor) are intact. `archive/zip` reads an archive only through the +end-of-central-directory record, so it rejects the whole file; so does Info-ZIP `unzip`. + +**Decision:** +- When `archive/zip` cannot open a `.zip`, `extractZip` / `extractZipFromReader` fall back to + `recoverZipLocalEntries` (`internal/parser/archive_zip_recover.go`), which reads the archive + front to back by local file headers and ignores the central directory. If the fallback + recovers no entry, the original `archive/zip` error is returned unchanged. +- The fallback is used only after `archive/zip` fails. For an intact archive the central + directory stays authoritative. +- Streamed members (general-purpose flag bit 3) have no sizes in the local header, so the end of + a member is found from the data: deflate is self-terminating; a stored member ends at the first + data descriptor signature whose CRC32 and size match the bytes read so far. Every recovered + member is verified against its CRC32 and size (data descriptor or local header). +- Recovery stops at the first thing that is not an intact local entry — the central directory, a + truncated or CRC-mismatching member, an encrypted member, a streamed member in a compression + method other than store/deflate — and keeps the entries before it. +- Limits match the regular zip path: members over 10 MB are skipped, extraction stops at 1 GB + total, the upload path keeps its 50 MB archive cap. +- Member mtime comes from the extended-timestamp extra field (UTC), else the DOS time, the same + as `archive/zip`. + +**Consequences:** +- Truncated streamed blackbox bundles open as long as the member data arrived; the cut-off + trailer costs nothing. +- An archive cut inside the member data opens with the members before the cut and no warning: + the extractor has no channel for "archive incomplete" (`ExtractedFile.Truncated` is per-file and + is reported as "input too large"). Parsers see a partial bundle. +- Local headers can disagree with the central directory in a crafted or appended-to archive + (deleted or superseded members remain as local entries). This is why the fallback never runs + for an archive `archive/zip` accepts. +- Not covered: nested zips opened by parsers themselves (`dell`, `redfishtree`), + `internal/sanitize` and `internal/server/raw_export.go` still require an intact archive. +- The producer-side bug (live CD closing the stream before the zip trailer is flushed) is not + fixed by this and should be fixed in bee. diff --git a/internal/parser/archive.go b/internal/parser/archive.go index ff38c2e..e38e6e4 100644 --- a/internal/parser/archive.go +++ b/internal/parser/archive.go @@ -326,6 +326,14 @@ func extractTarEntriesWithLimit(r io.Reader, totalLimit int64) ([]ExtractedFile, func extractZip(archivePath string) ([]ExtractedFile, error) { r, err := zip.OpenReader(archivePath) if err != nil { + // No usable central directory (e.g. a streamed zip cut off before + // its trailer): fall back to reading the local entries in order. + if f, openErr := os.Open(archivePath); openErr == nil { + defer f.Close() + if files, recoverErr := recoverZipLocalEntries(f); recoverErr == nil { + return files, nil + } + } return nil, fmt.Errorf("open zip: %w", err) } defer r.Close() @@ -379,6 +387,11 @@ func extractZipFromReader(r io.Reader) ([]ExtractedFile, error) { // Open the zip archive zipReader, err := zip.NewReader(readerAt, int64(len(data))) if err != nil { + // No usable central directory (e.g. a streamed zip cut off before + // its trailer): fall back to reading the local entries in order. + if files, recoverErr := recoverZipLocalEntries(bytes.NewReader(data)); recoverErr == nil { + return files, nil + } return nil, fmt.Errorf("open zip: %w", err) } diff --git a/internal/parser/archive_zip_recover.go b/internal/parser/archive_zip_recover.go new file mode 100644 index 0000000..b445adb --- /dev/null +++ b/internal/parser/archive_zip_recover.go @@ -0,0 +1,269 @@ +package parser + +import ( + "archive/zip" + "bufio" + "compress/flate" + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "strings" + "time" +) + +const ( + zipLocalHeaderSignature = 0x04034b50 + zipDataDescriptorSignature = 0x08074b50 + zipLocalHeaderLen = 30 + zipFlagEncrypted = 0x1 + zipFlagDataDescriptor = 0x8 + zipExtraZip64 = 0x0001 + zipExtraExtendedTimestamp = 0x5455 + zipMaxEntrySize = 10 * 1024 * 1024 +) + +// recoverZipLocalEntries reads a zip archive front to back by its local file +// headers, ignoring the central directory. It is the fallback for archives +// whose central directory is missing or cut off — typically a zip streamed +// over HTTP whose connection closed before the trailer was fully sent: the +// member data is intact, but archive/zip refuses the file because it only +// reads through the end-of-central-directory record. +// +// Recovery stops at the first thing that is not an intact local entry (the +// central directory, a truncated or CRC-mismatching member, an encrypted or +// otherwise unreadable one) and returns the entries before it. Size limits +// match the regular zip path. An error is returned only when no entry at all +// could be read. +func recoverZipLocalEntries(r io.Reader) ([]ExtractedFile, error) { + br := bufio.NewReaderSize(r, 64*1024) + var files []ExtractedFile + var totalExtracted int64 + entries := 0 + + for { + var fixed [zipLocalHeaderLen]byte + if _, err := io.ReadFull(br, fixed[:]); err != nil { + break + } + if binary.LittleEndian.Uint32(fixed[0:]) != zipLocalHeaderSignature { + break + } + flags := binary.LittleEndian.Uint16(fixed[6:]) + method := binary.LittleEndian.Uint16(fixed[8:]) + dosTime := binary.LittleEndian.Uint16(fixed[10:]) + dosDate := binary.LittleEndian.Uint16(fixed[12:]) + headerCRC := binary.LittleEndian.Uint32(fixed[14:]) + compressedSize := uint64(binary.LittleEndian.Uint32(fixed[18:])) + uncompressedSize := uint64(binary.LittleEndian.Uint32(fixed[22:])) + nameLen := int(binary.LittleEndian.Uint16(fixed[26:])) + extraLen := int(binary.LittleEndian.Uint16(fixed[28:])) + + nameAndExtra := make([]byte, nameLen+extraLen) + if _, err := io.ReadFull(br, nameAndExtra); err != nil { + break + } + name := string(nameAndExtra[:nameLen]) + modTime := zipDOSTime(dosDate, dosTime) + parseZipLocalExtra(nameAndExtra[nameLen:], &modTime, &uncompressedSize, &compressedSize) + + if flags&zipFlagEncrypted != 0 { + break + } + + var content []byte + var size int64 + var ok bool + if flags&zipFlagDataDescriptor != 0 { + // Streamed entry: sizes and CRC follow the data, so the end of + // the member has to be found from the data itself. + switch method { + case zip.Deflate: + // bufio.Reader is an io.ByteReader, so flate consumes exactly + // the compressed stream and leaves br at the data descriptor. + var sum uint32 + var err error + content, size, sum, err = readZipEntryBody(flate.NewReader(br)) + ok = err == nil && consumeZipDataDescriptor(br, sum, size) + case zip.Store: + content, ok = readStoredZipEntryUntilDescriptor(br) + size = int64(len(content)) + } + } else { + body := io.LimitReader(br, int64(compressedSize)) + switch method { + case zip.Store, zip.Deflate: + src := body + if method == zip.Deflate { + src = flate.NewReader(body) + } + var sum uint32 + var err error + content, size, sum, err = readZipEntryBody(src) + if err == nil { + _, err = io.Copy(io.Discard, body) + } + ok = err == nil && sum == headerCRC && uint64(size) == uncompressedSize + default: + // Unknown compression but a known length: skip the member. + if n, err := io.Copy(io.Discard, body); err != nil || uint64(n) != compressedSize { + break + } + entries++ + continue + } + } + if !ok { + break + } + entries++ + + if strings.HasSuffix(name, "/") { + continue + } + // Skip large files (>10MB) + if size > zipMaxEntrySize { + continue + } + if totalExtracted+size > maxSingleFileSizeLarge { + break + } + + files = append(files, ExtractedFile{ + Path: name, + Content: content, + ModTime: modTime, + }) + totalExtracted += size + } + + if entries == 0 { + return nil, fmt.Errorf("no intact local file entries") + } + return files, nil +} + +// readZipEntryBody reads src to EOF and returns its length and CRC32. The +// content is kept only while it fits the per-file limit; a larger member is +// still read through (its end has to be found) but returned with nil content. +func readZipEntryBody(src io.Reader) ([]byte, int64, uint32, error) { + sum := crc32.NewIEEE() + content, err := io.ReadAll(io.TeeReader(io.LimitReader(src, zipMaxEntrySize+1), sum)) + if err != nil { + return nil, 0, 0, err + } + size := int64(len(content)) + if size > zipMaxEntrySize { + rest, err := io.Copy(sum, src) + if err != nil { + return nil, 0, 0, err + } + return nil, size + rest, sum.Sum32(), nil + } + return content, size, sum.Sum32(), nil +} + +// consumeZipDataDescriptor consumes the data descriptor at the head of br if +// it matches the given CRC32 and uncompressed size. The descriptor signature +// is optional and the sizes are 4 or 8 bytes wide (zip64), with nothing in +// the descriptor saying which; a layout is accepted only when it is followed +// by another record ("PK") or the end of input. +func consumeZipDataDescriptor(br *bufio.Reader, sum uint32, size int64) bool { + head, _ := br.Peek(26) + for _, sigLen := range []int{4, 0} { + if sigLen == 4 && (len(head) < 4 || binary.LittleEndian.Uint32(head) != zipDataDescriptorSignature) { + continue + } + for _, sizeLen := range []int{4, 8} { + n := sigLen + 4 + 2*sizeLen + if len(head) < n || binary.LittleEndian.Uint32(head[sigLen:]) != sum { + continue + } + sizeField := head[sigLen+4+sizeLen : n] + var got uint64 + if sizeLen == 4 { + got = uint64(binary.LittleEndian.Uint32(sizeField)) + } else { + got = binary.LittleEndian.Uint64(sizeField) + } + if got != uint64(size) { + continue + } + if len(head) >= n+2 && string(head[n:n+2]) != "PK" { + continue + } + _, _ = br.Discard(n) + return true + } + } + return false +} + +// readStoredZipEntryUntilDescriptor reads an uncompressed streamed member, +// whose length is not recorded ahead of the data: the member ends at the +// first data descriptor signature whose CRC32 and size match the bytes read +// so far. Members over the per-file limit, or written without the optional +// descriptor signature, are not recovered. +func readStoredZipEntryUntilDescriptor(br *bufio.Reader) ([]byte, bool) { + var content []byte + for { + if head, _ := br.Peek(4); len(head) == 4 && binary.LittleEndian.Uint32(head) == zipDataDescriptorSignature { + if consumeZipDataDescriptor(br, crc32.ChecksumIEEE(content), int64(len(content))) { + return content, true + } + } + b, err := br.ReadByte() + if err != nil { + return nil, false + } + content = append(content, b) + if len(content) > zipMaxEntrySize { + return nil, false + } + } +} + +// parseZipLocalExtra applies the local-header extra fields that matter for +// recovery: the extended timestamp (UTC mtime, overrides the DOS time) and +// zip64 sizes (only present for the 32-bit fields set to 0xFFFFFFFF). +func parseZipLocalExtra(extra []byte, modTime *time.Time, uncompressedSize, compressedSize *uint64) { + for len(extra) >= 4 { + id := binary.LittleEndian.Uint16(extra[0:]) + n := int(binary.LittleEndian.Uint16(extra[2:])) + extra = extra[4:] + if n > len(extra) { + return + } + field := extra[:n] + extra = extra[n:] + + switch id { + case zipExtraExtendedTimestamp: + if len(field) >= 5 && field[0]&1 != 0 { + *modTime = time.Unix(int64(int32(binary.LittleEndian.Uint32(field[1:]))), 0).UTC() + } + case zipExtraZip64: + if *uncompressedSize == 0xFFFFFFFF && len(field) >= 8 { + *uncompressedSize = binary.LittleEndian.Uint64(field) + field = field[8:] + } + if *compressedSize == 0xFFFFFFFF && len(field) >= 8 { + *compressedSize = binary.LittleEndian.Uint64(field) + } + } + } +} + +// zipDOSTime converts an MS-DOS date and time to UTC, as archive/zip does. +func zipDOSTime(dosDate, dosTime uint16) time.Time { + return time.Date( + int(dosDate>>9)+1980, + time.Month(dosDate>>5&0xf), + int(dosDate&0x1f), + int(dosTime>>11), + int(dosTime>>5&0x3f), + int(dosTime&0x1f)*2, + 0, + time.UTC, + ) +} diff --git a/internal/parser/archive_zip_recover_test.go b/internal/parser/archive_zip_recover_test.go new file mode 100644 index 0000000..9378ca3 --- /dev/null +++ b/internal/parser/archive_zip_recover_test.go @@ -0,0 +1,192 @@ +package parser + +import ( + "archive/zip" + "bytes" + "hash/crc32" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +type zipTestFile struct { + name string + content string + method uint16 + raw bool // sizes and CRC in the local header, no data descriptor +} + +// buildTestZip returns the archive bytes and the offset where the central +// directory starts. +func buildTestZip(t *testing.T, modTime time.Time, files []zipTestFile) ([]byte, int) { + t.Helper() + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, f := range files { + hdr := &zip.FileHeader{Name: f.name, Method: f.method, Modified: modTime} + if f.raw { + // CreateRaw writes the header as given: only the DOS time is set. + hdr = &zip.FileHeader{Name: f.name, Method: zip.Store} + hdr.SetModTime(modTime) + hdr.CRC32 = crc32.ChecksumIEEE([]byte(f.content)) + hdr.CompressedSize64 = uint64(len(f.content)) + hdr.UncompressedSize64 = uint64(len(f.content)) + w, err := zw.CreateRaw(hdr) + if err != nil { + t.Fatalf("create raw %s: %v", f.name, err) + } + if _, err := w.Write([]byte(f.content)); err != nil { + t.Fatalf("write %s: %v", f.name, err) + } + continue + } + w, err := zw.CreateHeader(hdr) + if err != nil { + t.Fatalf("create %s: %v", f.name, err) + } + if _, err := w.Write([]byte(f.content)); err != nil { + t.Fatalf("write %s: %v", f.name, err) + } + } + if err := zw.Close(); err != nil { + t.Fatalf("close zip: %v", err) + } + centralDirOffset := bytes.Index(buf.Bytes(), []byte("PK\x01\x02")) + if centralDirOffset < 0 { + t.Fatal("central directory not found") + } + return buf.Bytes(), centralDirOffset +} + +func recoverTestFiles() []zipTestFile { + return []zipTestFile{ + {name: "README.md", content: strings.Repeat("streamed deflate member\n", 200), method: zip.Deflate}, + {name: "export/", method: zip.Store, raw: true}, + // Contains a data descriptor signature that must not end the member. + {name: "export/stored.bin", content: "head PK\x07\x08 middle PK\x03\x04 tail", method: zip.Store}, + {name: "export/empty.txt", content: "", method: zip.Deflate}, + {name: "export/raw.txt", content: "sizes in the local header", raw: true}, + {name: "tasks/last.log", content: strings.Repeat("last member ", 50), method: zip.Deflate}, + } +} + +func assertRecoveredFiles(t *testing.T, got []ExtractedFile, want []zipTestFile, modTime time.Time) { + t.Helper() + + var wantFiles []zipTestFile + for _, f := range want { + if !strings.HasSuffix(f.name, "/") { + wantFiles = append(wantFiles, f) + } + } + if len(got) != len(wantFiles) { + t.Fatalf("expected %d files, got %d", len(wantFiles), len(got)) + } + for i, f := range wantFiles { + if got[i].Path != f.name { + t.Fatalf("file %d: expected path %q, got %q", i, f.name, got[i].Path) + } + if string(got[i].Content) != f.content { + t.Fatalf("file %s: content mismatch", f.name) + } + if !got[i].ModTime.Equal(modTime) { + t.Fatalf("file %s: expected mtime %v, got %v", f.name, modTime, got[i].ModTime) + } + } +} + +func TestExtractArchiveFromReaderZIP_RecoversTruncatedCentralDirectory(t *testing.T) { + modTime := time.Date(2026, 10, 2, 16, 4, 12, 0, time.UTC) + want := recoverTestFiles() + data, centralDirOffset := buildTestZip(t, modTime, want) + + // Sanity: the intact archive goes through archive/zip. + files, err := ExtractArchiveFromReader(bytes.NewReader(data), "blackbox.zip") + if err != nil { + t.Fatalf("extract intact zip: %v", err) + } + assertRecoveredFiles(t, files, want, modTime) + + cuts := map[string]int{ + "mid central directory": centralDirOffset + (len(data)-centralDirOffset)/2, + "central directory missing": centralDirOffset, + "end of central directory lost": len(data) - 10, + } + for name, cut := range cuts { + t.Run(name, func(t *testing.T) { + files, err := ExtractArchiveFromReader(bytes.NewReader(data[:cut]), "blackbox.zip") + if err != nil { + t.Fatalf("extract truncated zip: %v", err) + } + assertRecoveredFiles(t, files, want, modTime) + }) + } +} + +func TestExtractArchiveZIP_RecoversTruncatedCentralDirectoryFromPath(t *testing.T) { + modTime := time.Date(2026, 10, 2, 16, 4, 12, 0, time.UTC) + want := recoverTestFiles() + data, centralDirOffset := buildTestZip(t, modTime, want) + + path := filepath.Join(t.TempDir(), "blackbox.zip") + if err := os.WriteFile(path, data[:centralDirOffset+20], 0o600); err != nil { + t.Fatalf("write zip: %v", err) + } + + files, err := ExtractArchive(path) + if err != nil { + t.Fatalf("extract truncated zip: %v", err) + } + assertRecoveredFiles(t, files, want, modTime) +} + +func TestExtractArchiveFromReaderZIP_TruncatedMemberKeepsEarlierEntries(t *testing.T) { + modTime := time.Date(2026, 10, 2, 16, 4, 12, 0, time.UTC) + want := recoverTestFiles() + data, centralDirOffset := buildTestZip(t, modTime, want) + + // Cut inside the last member's compressed data. + files, err := ExtractArchiveFromReader(bytes.NewReader(data[:centralDirOffset-20]), "blackbox.zip") + if err != nil { + t.Fatalf("extract truncated zip: %v", err) + } + assertRecoveredFiles(t, files, want[:len(want)-1], modTime) +} + +func TestExtractArchiveFromReaderZIP_CorruptMemberStopsRecovery(t *testing.T) { + modTime := time.Date(2026, 10, 2, 16, 4, 12, 0, time.UTC) + want := recoverTestFiles() + data, centralDirOffset := buildTestZip(t, modTime, want) + + damaged := append([]byte(nil), data[:centralDirOffset]...) + at := bytes.Index(damaged, []byte("sizes in the local header")) + if at < 0 { + t.Fatal("raw member content not found") + } + damaged[at] ^= 0xff + + files, err := ExtractArchiveFromReader(bytes.NewReader(damaged), "blackbox.zip") + if err != nil { + t.Fatalf("extract damaged zip: %v", err) + } + // Everything before export/raw.txt; the CRC mismatch ends recovery. + assertRecoveredFiles(t, files, want[:4], modTime) +} + +func TestExtractArchiveFromReaderZIP_NotAZipStillFails(t *testing.T) { + for name, data := range map[string][]byte{ + "garbage": []byte("this is not a zip archive at all"), + "empty": {}, + "truncated first entry": []byte("PK\x03\x04\x14\x00\x08\x00\x08\x00"), + } { + t.Run(name, func(t *testing.T) { + _, err := ExtractArchiveFromReader(bytes.NewReader(data), "blackbox.zip") + if err == nil || !strings.Contains(err.Error(), "open zip") { + t.Fatalf("expected open zip error, got %v", err) + } + }) + } +}