Files
logpile/internal/rar5/decode.go
T
Mikhail ChusavitinandClaude Opus 5.5 b4b15e58a2 feat(parser): linux-generic content-driven parser and RAR archive support
Field bundles from ad-hoc service scripts (first sample: Giga G894-SD3 .rar
with dmidecode/lscpu/lspci/nvme/lsblk output, no fixed layout). ADL-078:
- linux-generic: recognizes utility output by content, merges records by
  stable key, low detect confidence (20-30) below all vendor parsers
- pciids.LookupByDescription: resolve a textual lspci description back to
  vendor/device IDs and manufacturer/model
- archive: accept .rar via the in-repo internal/rar5 reader (RAR 5 / RAR 7,
  single volume, solid; encrypted, multi-volume, SFX and RAR 4.x rejected),
  same per-file and total limits as tar/zip
- UI: .rar in the upload accept list

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-02 15:00:38 +03:00

556 lines
12 KiB
Go

// Derived from github.com/nwaples/rardecode (BSD 2-Clause), see LICENSE.
package rar5
import (
"encoding/binary"
"io"
"math/bits"
)
const (
mainSize = 306
offsetSize5 = 64
offsetSize7 = 80
lowOffsetSize = 16
lengthSize = 44
maxTableSize = mainSize + offsetSize7 + lowOffsetSize + lengthSize
minWindowSize = 0x40000
maxFilterLength = 0x400000
maxQueuedFilters = 8192
// e8FileSize is the address space the x86 filters assume.
e8FileSize = 0x1000000
)
// filterBlock is a span of decoded output that a filter transforms before it
// is handed to the caller.
type filterBlock struct {
offset int // bytes to output before the block starts, relative to the previous filter
length int
filter func(buf []byte, fileOffset int64) []byte
}
// decoder unpacks RAR 5 (and RAR 7) compressed data. Input is a sequence of
// blocks, each with a header carrying the block length and optional Huffman
// code length tables. Output goes through a sliding window, which is kept
// between the files of a solid archive.
type decoder struct {
br bitReader
codeLength [maxTableSize]byte
offsetSize int
lastBlock bool // current block is the last one of the file
mainDecoder huffmanDecoder
offsetDecoder huffmanDecoder
lowOffsetDecoder huffmanDecoder
lengthDecoder huffmanDecoder
offset [4]int // most recent match offsets
length int // most recent match length
win []byte // sliding window
size int // len(win)
r int // index of the first byte not yet output
w int // index of the next byte to decode
pendLen int // rest of a match that did not fit before the window end
pendOff int
filters []*filterBlock
total int64 // bytes of the current file output so far
out []byte // decoded bytes ready for Read
buf []byte // filter scratch buffer
err error
}
// init prepares the decoder for the next file. reset is false for a file that
// continues the previous one in a solid archive.
func (d *decoder) init(r io.ByteReader, reset bool, windowSize int, rar7 bool) {
d.br.reset(r)
d.lastBlock = false
d.offsetSize = offsetSize5
if rar7 {
d.offsetSize = offsetSize7
}
d.out = nil
d.total = 0
d.err = nil
windowSize = max(windowSize, minWindowSize)
switch {
case reset:
clear(d.offset[:])
d.length = 0
clear(d.codeLength[:])
d.filters = nil
d.pendLen = 0
if cap(d.win) >= windowSize {
d.win = d.win[:windowSize]
clear(d.win)
} else {
d.win = make([]byte, windowSize)
}
d.w = 0
case windowSize > len(d.win):
// A solid file with a larger dictionary: keep the history.
grown := make([]byte, windowSize)
n := copy(grown, d.win[d.w:])
n += copy(grown[n:], d.win[:d.w])
d.win = grown
d.w = n
}
d.size = len(d.win)
d.r = d.w
}
func (d *decoder) readBlockHeader() error {
flags, err := d.br.readByte()
if err != nil {
return err
}
byteCount := (flags>>3)&3 + 1
if byteCount == 4 {
return ErrCorrupt
}
headerSum, err := d.br.readByte()
if err != nil {
return err
}
blockBits := int(flags)&0x07 + 1
blockBytes := 0
sum := 0x5a ^ flags
for i := byte(0); i < byteCount; i++ {
n, err := d.br.readByte()
if err != nil {
return err
}
sum ^= n
blockBytes |= int(n) << (i * 8)
}
if sum != headerSum {
return ErrCorrupt
}
blockBits += (blockBytes - 1) * 8
if blockBits < 0 {
return ErrCorrupt
}
d.br.setLimit(blockBits)
d.lastBlock = flags&0x40 > 0
if flags&0x80 > 0 {
// new code length tables
cl := d.codeLength[:mainSize+d.offsetSize+lowOffsetSize+lengthSize]
if err := readCodeLengthTable(&d.br, cl); err != nil {
return err
}
d.mainDecoder.init(cl[:mainSize])
cl = cl[mainSize:]
d.offsetDecoder.init(cl[:d.offsetSize])
cl = cl[d.offsetSize:]
d.lowOffsetDecoder.init(cl[:lowOffsetSize])
cl = cl[lowOffsetSize:]
d.lengthDecoder.init(cl)
}
return nil
}
func slotToLength(br *bitReader, n int) (int, error) {
if n >= 8 {
extra := uint8(n/4 - 1)
n = (4 | (n & 3)) << extra
if extra > 0 {
b, err := br.readBits(extra)
if err != nil {
return 0, err
}
n |= b
}
}
return n + 2, nil
}
// readFilterData reads a variable-length integer of a filter record.
func readFilterData(br *bitReader) (int, error) {
byteCount, err := br.readBits(2)
if err != nil {
return 0, err
}
var data int
for i := 0; i <= byteCount; i++ {
n, err := br.readBits(8)
if err != nil {
return 0, err
}
data |= n << (uint(i) * 8)
}
return data, nil
}
func (d *decoder) readFilter() error {
fb := new(filterBlock)
var err error
if fb.offset, err = readFilterData(&d.br); err != nil {
return err
}
if fb.length, err = readFilterData(&d.br); err != nil {
return err
}
filterType, err := d.br.readBits(3)
if err != nil {
return err
}
switch filterType {
case 0:
channels, err := d.br.readBits(5)
if err != nil {
return err
}
fb.filter = func(buf []byte, _ int64) []byte { return filterDelta(channels+1, buf) }
case 1:
fb.filter = func(buf []byte, offset int64) []byte { return filterE8(0xe8, buf, offset) }
case 2:
fb.filter = func(buf []byte, offset int64) []byte { return filterE8(0xe9, buf, offset) }
case 3:
fb.filter = filterARM
default:
return ErrCorrupt
}
return d.queueFilter(fb)
}
// queueFilter appends fb, converting its offset from "relative to the write
// position" to "relative to the previous queued filter".
func (d *decoder) queueFilter(fb *filterBlock) error {
if len(d.filters) >= maxQueuedFilters || fb.length > maxFilterLength || fb.length > d.size {
return ErrCorrupt
}
fb.offset += d.w - d.r
for _, queued := range d.filters {
if fb.offset < queued.offset {
// a filter block must not start before the previous one
return ErrCorrupt
}
fb.offset -= queued.offset
}
d.filters = append(d.filters, fb)
return nil
}
// copyBytes appends a match of the given length found offset bytes back.
// A match that crosses the window end is finished after the window wraps.
func (d *decoder) copyBytes(length, offset int) {
n := min(length, d.size-d.w)
d.pendLen, d.pendOff = length-n, offset
i := (d.w - offset) % d.size
if i < 0 {
i += d.size
}
for ; n > 0; n-- {
d.win[d.w] = d.win[i]
d.w++
if i++; i == d.size {
i = 0
}
}
}
// decodeLength handles a match that reuses one of the recent offsets.
func (d *decoder) decodeLength(i int) error {
offset := d.offset[i]
copy(d.offset[1:i+1], d.offset[:i])
d.offset[0] = offset
slot, err := d.lengthDecoder.readSym(&d.br)
if err != nil {
return err
}
if d.length, err = slotToLength(&d.br, slot); err != nil {
return err
}
d.copyBytes(d.length, d.offset[0])
return nil
}
// decodeOffset handles a match with an explicit offset.
func (d *decoder) decodeOffset(lengthSlot int) error {
length, err := slotToLength(&d.br, lengthSlot)
if err != nil {
return err
}
offset := 1
slot, err := d.offsetDecoder.readSym(&d.br)
if err != nil {
return err
}
if slot < 4 {
offset += slot
} else {
bitCount := uint8(slot/2 - 1)
offset += (2 | (slot & 1)) << bitCount
if bitCount >= 4 {
bitCount -= 4
if bitCount > 0 {
if bits.UintSize == 32 && bitCount > 24 {
// readBits handles at most intSize-8 bits at once.
n, err := d.br.readBits(24)
if err != nil {
return err
}
bitCount -= 24
offset += n << (4 + bitCount)
}
n, err := d.br.readBits(bitCount)
if err != nil {
return err
}
offset += n << 4
}
n, err := d.lowOffsetDecoder.readSym(&d.br)
if err != nil {
return err
}
offset += n
} else {
n, err := d.br.readBits(bitCount)
if err != nil {
return err
}
offset += n
}
}
if offset > 0x100 {
length++
if offset > 0x2000 {
length++
if offset > 0x40000 {
length++
}
}
}
copy(d.offset[1:], d.offset[:])
d.offset[0] = offset
d.length = length
d.copyBytes(length, offset)
return nil
}
// decodeSymbols decodes until the window is full or the file ends (io.EOF).
func (d *decoder) decodeSymbols() error {
for d.w < d.size {
sym, err := d.mainDecoder.readSym(&d.br)
if err == nil {
switch {
case sym < 256:
d.win[d.w] = byte(sym)
d.w++
continue
case sym >= 262:
err = d.decodeOffset(sym - 262)
case sym >= 258:
err = d.decodeLength(sym - 258)
case sym == 257:
// repeat the previous match
d.copyBytes(d.length, d.offset[0])
continue
default: // 256
err = d.readFilter()
}
} else if err == io.EOF {
// end of the block
if d.lastBlock {
return io.EOF
}
err = d.readBlockHeader()
}
if err != nil {
if err == io.EOF {
return ErrCorrupt
}
return err
}
}
return nil
}
// fill decodes more data into the window. It returns an error only when no
// new bytes are available.
func (d *decoder) fill() error {
if d.err != nil {
return d.err
}
if d.w == d.size {
d.r, d.w = 0, 0
}
if d.pendLen > 0 {
d.copyBytes(d.pendLen, d.pendOff)
}
d.err = d.decodeSymbols()
if d.w == d.r {
return d.err
}
return nil
}
// windowBytes moves n decoded bytes out of the window into the scratch buffer.
func (d *decoder) windowBytes(n int) ([]byte, error) {
if cap(d.buf) < n {
d.buf = make([]byte, n, 2*n)
}
b := d.buf[:n]
for copied := 0; ; {
c := copy(b[copied:], d.win[d.r:d.w])
d.r += c
copied += c
if copied == n {
return b, nil
}
if err := d.fill(); err != nil {
if err == io.EOF {
err = ErrCorrupt
}
return nil, err
}
}
}
// applyFilters runs every filter queued for the current output position.
func (d *decoder) applyFilters() ([]byte, error) {
fb := d.filters[0]
length := fb.length
b, err := d.windowBytes(length)
if err != nil {
return nil, err
}
for {
d.filters = d.filters[1:]
b = fb.filter(b, d.total)
if len(d.filters) == 0 {
d.filters = nil
return b, nil
}
fb = d.filters[0]
if fb.offset != 0 {
// the next filter starts later
fb.offset -= length
return b, nil
}
if fb.length != len(b) {
return nil, ErrCorrupt
}
}
}
// decode sets d.out to the next run of output bytes.
func (d *decoder) decode() error {
if d.w == d.r {
if err := d.fill(); err != nil {
return err
}
}
n := d.w - d.r
if len(d.filters) > 0 {
fb := d.filters[0]
if fb.offset < 0 {
return ErrCorrupt
}
if fb.offset == 0 {
out, err := d.applyFilters()
if err != nil {
return err
}
d.out = out
d.total += int64(len(out))
return nil
}
// output the bytes in front of the filter block
n = min(fb.offset, n)
fb.offset -= n
}
d.out = d.win[d.r : d.r+n]
d.r += n
d.total += int64(n)
return nil
}
func (d *decoder) Read(p []byte) (int, error) {
for len(d.out) == 0 {
if err := d.decode(); err != nil {
return 0, err
}
}
n := copy(p, d.out)
d.out = d.out[n:]
return n, nil
}
// filterE8 undoes the x86 CALL (and, with c == 0xe9, JMP) address transform.
func filterE8(c byte, buf []byte, offset int64) []byte {
off := int32(offset)
for b := buf; len(b) >= 5; {
ch := b[0]
b = b[1:]
off++
if ch != 0xe8 && ch != c {
continue
}
off %= e8FileSize
addr := int32(binary.LittleEndian.Uint32(b))
if addr < 0 {
if addr+off >= 0 {
binary.LittleEndian.PutUint32(b, uint32(addr+e8FileSize))
}
} else if addr < e8FileSize {
binary.LittleEndian.PutUint32(b, uint32(addr-off))
}
off += 4
b = b[4:]
}
return buf
}
// filterDelta undoes per-channel delta coding.
func filterDelta(channels int, buf []byte) []byte {
var res []byte
l := len(buf)
if cap(buf) >= 2*l {
res = buf[l : 2*l] // use unused capacity
} else {
res = make([]byte, l, 2*l)
}
i := 0
for j := 0; j < channels; j++ {
var c byte
for k := j; k < len(res); k += channels {
c -= buf[i]
i++
res[k] = c
}
}
return res
}
// filterARM undoes the ARM BL address transform.
func filterARM(buf []byte, offset int64) []byte {
for i := 0; len(buf)-i > 3; i += 4 {
if buf[i+3] == 0xeb {
n := uint(buf[i])
n += uint(buf[i+1]) * 0x100
n += uint(buf[i+2]) * 0x10000
n -= (uint(offset) + uint(i)) / 4
buf[i] = byte(n)
buf[i+1] = byte(n >> 8)
buf[i+2] = byte(n >> 16)
}
}
return buf
}