Field bundles from ad-hoc service scripts (first sample: Giga G894-SD3 .rar with dmidecode/lscpu/lspci/nvme/lsblk output, no fixed layout). ADL-078: - linux-generic: recognizes utility output by content, merges records by stable key, low detect confidence (20-30) below all vendor parsers - pciids.LookupByDescription: resolve a textual lspci description back to vendor/device IDs and manufacturer/model - archive: accept .rar via the in-repo internal/rar5 reader (RAR 5 / RAR 7, single volume, solid; encrypted, multi-volume, SFX and RAR 4.x rejected), same per-file and total limits as tar/zip - UI: .rar in the upload accept list Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
556 lines
12 KiB
Go
556 lines
12 KiB
Go
// Derived from github.com/nwaples/rardecode (BSD 2-Clause), see LICENSE.
|
|
|
|
package rar5
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"io"
|
|
"math/bits"
|
|
)
|
|
|
|
const (
|
|
mainSize = 306
|
|
offsetSize5 = 64
|
|
offsetSize7 = 80
|
|
lowOffsetSize = 16
|
|
lengthSize = 44
|
|
maxTableSize = mainSize + offsetSize7 + lowOffsetSize + lengthSize
|
|
|
|
minWindowSize = 0x40000
|
|
maxFilterLength = 0x400000
|
|
maxQueuedFilters = 8192
|
|
// e8FileSize is the address space the x86 filters assume.
|
|
e8FileSize = 0x1000000
|
|
)
|
|
|
|
// filterBlock is a span of decoded output that a filter transforms before it
|
|
// is handed to the caller.
|
|
type filterBlock struct {
|
|
offset int // bytes to output before the block starts, relative to the previous filter
|
|
length int
|
|
filter func(buf []byte, fileOffset int64) []byte
|
|
}
|
|
|
|
// decoder unpacks RAR 5 (and RAR 7) compressed data. Input is a sequence of
|
|
// blocks, each with a header carrying the block length and optional Huffman
|
|
// code length tables. Output goes through a sliding window, which is kept
|
|
// between the files of a solid archive.
|
|
type decoder struct {
|
|
br bitReader
|
|
codeLength [maxTableSize]byte
|
|
offsetSize int
|
|
lastBlock bool // current block is the last one of the file
|
|
|
|
mainDecoder huffmanDecoder
|
|
offsetDecoder huffmanDecoder
|
|
lowOffsetDecoder huffmanDecoder
|
|
lengthDecoder huffmanDecoder
|
|
|
|
offset [4]int // most recent match offsets
|
|
length int // most recent match length
|
|
|
|
win []byte // sliding window
|
|
size int // len(win)
|
|
r int // index of the first byte not yet output
|
|
w int // index of the next byte to decode
|
|
pendLen int // rest of a match that did not fit before the window end
|
|
pendOff int
|
|
|
|
filters []*filterBlock
|
|
total int64 // bytes of the current file output so far
|
|
out []byte // decoded bytes ready for Read
|
|
buf []byte // filter scratch buffer
|
|
err error
|
|
}
|
|
|
|
// init prepares the decoder for the next file. reset is false for a file that
|
|
// continues the previous one in a solid archive.
|
|
func (d *decoder) init(r io.ByteReader, reset bool, windowSize int, rar7 bool) {
|
|
d.br.reset(r)
|
|
d.lastBlock = false
|
|
d.offsetSize = offsetSize5
|
|
if rar7 {
|
|
d.offsetSize = offsetSize7
|
|
}
|
|
d.out = nil
|
|
d.total = 0
|
|
d.err = nil
|
|
|
|
windowSize = max(windowSize, minWindowSize)
|
|
switch {
|
|
case reset:
|
|
clear(d.offset[:])
|
|
d.length = 0
|
|
clear(d.codeLength[:])
|
|
d.filters = nil
|
|
d.pendLen = 0
|
|
if cap(d.win) >= windowSize {
|
|
d.win = d.win[:windowSize]
|
|
clear(d.win)
|
|
} else {
|
|
d.win = make([]byte, windowSize)
|
|
}
|
|
d.w = 0
|
|
case windowSize > len(d.win):
|
|
// A solid file with a larger dictionary: keep the history.
|
|
grown := make([]byte, windowSize)
|
|
n := copy(grown, d.win[d.w:])
|
|
n += copy(grown[n:], d.win[:d.w])
|
|
d.win = grown
|
|
d.w = n
|
|
}
|
|
d.size = len(d.win)
|
|
d.r = d.w
|
|
}
|
|
|
|
func (d *decoder) readBlockHeader() error {
|
|
flags, err := d.br.readByte()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
byteCount := (flags>>3)&3 + 1
|
|
if byteCount == 4 {
|
|
return ErrCorrupt
|
|
}
|
|
headerSum, err := d.br.readByte()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
blockBits := int(flags)&0x07 + 1
|
|
blockBytes := 0
|
|
sum := 0x5a ^ flags
|
|
for i := byte(0); i < byteCount; i++ {
|
|
n, err := d.br.readByte()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sum ^= n
|
|
blockBytes |= int(n) << (i * 8)
|
|
}
|
|
if sum != headerSum {
|
|
return ErrCorrupt
|
|
}
|
|
blockBits += (blockBytes - 1) * 8
|
|
if blockBits < 0 {
|
|
return ErrCorrupt
|
|
}
|
|
|
|
d.br.setLimit(blockBits)
|
|
d.lastBlock = flags&0x40 > 0
|
|
|
|
if flags&0x80 > 0 {
|
|
// new code length tables
|
|
cl := d.codeLength[:mainSize+d.offsetSize+lowOffsetSize+lengthSize]
|
|
if err := readCodeLengthTable(&d.br, cl); err != nil {
|
|
return err
|
|
}
|
|
d.mainDecoder.init(cl[:mainSize])
|
|
cl = cl[mainSize:]
|
|
d.offsetDecoder.init(cl[:d.offsetSize])
|
|
cl = cl[d.offsetSize:]
|
|
d.lowOffsetDecoder.init(cl[:lowOffsetSize])
|
|
cl = cl[lowOffsetSize:]
|
|
d.lengthDecoder.init(cl)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func slotToLength(br *bitReader, n int) (int, error) {
|
|
if n >= 8 {
|
|
extra := uint8(n/4 - 1)
|
|
n = (4 | (n & 3)) << extra
|
|
if extra > 0 {
|
|
b, err := br.readBits(extra)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
n |= b
|
|
}
|
|
}
|
|
return n + 2, nil
|
|
}
|
|
|
|
// readFilterData reads a variable-length integer of a filter record.
|
|
func readFilterData(br *bitReader) (int, error) {
|
|
byteCount, err := br.readBits(2)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
var data int
|
|
for i := 0; i <= byteCount; i++ {
|
|
n, err := br.readBits(8)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
data |= n << (uint(i) * 8)
|
|
}
|
|
return data, nil
|
|
}
|
|
|
|
func (d *decoder) readFilter() error {
|
|
fb := new(filterBlock)
|
|
var err error
|
|
if fb.offset, err = readFilterData(&d.br); err != nil {
|
|
return err
|
|
}
|
|
if fb.length, err = readFilterData(&d.br); err != nil {
|
|
return err
|
|
}
|
|
filterType, err := d.br.readBits(3)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch filterType {
|
|
case 0:
|
|
channels, err := d.br.readBits(5)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fb.filter = func(buf []byte, _ int64) []byte { return filterDelta(channels+1, buf) }
|
|
case 1:
|
|
fb.filter = func(buf []byte, offset int64) []byte { return filterE8(0xe8, buf, offset) }
|
|
case 2:
|
|
fb.filter = func(buf []byte, offset int64) []byte { return filterE8(0xe9, buf, offset) }
|
|
case 3:
|
|
fb.filter = filterARM
|
|
default:
|
|
return ErrCorrupt
|
|
}
|
|
return d.queueFilter(fb)
|
|
}
|
|
|
|
// queueFilter appends fb, converting its offset from "relative to the write
|
|
// position" to "relative to the previous queued filter".
|
|
func (d *decoder) queueFilter(fb *filterBlock) error {
|
|
if len(d.filters) >= maxQueuedFilters || fb.length > maxFilterLength || fb.length > d.size {
|
|
return ErrCorrupt
|
|
}
|
|
fb.offset += d.w - d.r
|
|
for _, queued := range d.filters {
|
|
if fb.offset < queued.offset {
|
|
// a filter block must not start before the previous one
|
|
return ErrCorrupt
|
|
}
|
|
fb.offset -= queued.offset
|
|
}
|
|
d.filters = append(d.filters, fb)
|
|
return nil
|
|
}
|
|
|
|
// copyBytes appends a match of the given length found offset bytes back.
|
|
// A match that crosses the window end is finished after the window wraps.
|
|
func (d *decoder) copyBytes(length, offset int) {
|
|
n := min(length, d.size-d.w)
|
|
d.pendLen, d.pendOff = length-n, offset
|
|
|
|
i := (d.w - offset) % d.size
|
|
if i < 0 {
|
|
i += d.size
|
|
}
|
|
for ; n > 0; n-- {
|
|
d.win[d.w] = d.win[i]
|
|
d.w++
|
|
if i++; i == d.size {
|
|
i = 0
|
|
}
|
|
}
|
|
}
|
|
|
|
// decodeLength handles a match that reuses one of the recent offsets.
|
|
func (d *decoder) decodeLength(i int) error {
|
|
offset := d.offset[i]
|
|
copy(d.offset[1:i+1], d.offset[:i])
|
|
d.offset[0] = offset
|
|
|
|
slot, err := d.lengthDecoder.readSym(&d.br)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if d.length, err = slotToLength(&d.br, slot); err != nil {
|
|
return err
|
|
}
|
|
d.copyBytes(d.length, d.offset[0])
|
|
return nil
|
|
}
|
|
|
|
// decodeOffset handles a match with an explicit offset.
|
|
func (d *decoder) decodeOffset(lengthSlot int) error {
|
|
length, err := slotToLength(&d.br, lengthSlot)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
offset := 1
|
|
slot, err := d.offsetDecoder.readSym(&d.br)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if slot < 4 {
|
|
offset += slot
|
|
} else {
|
|
bitCount := uint8(slot/2 - 1)
|
|
offset += (2 | (slot & 1)) << bitCount
|
|
|
|
if bitCount >= 4 {
|
|
bitCount -= 4
|
|
if bitCount > 0 {
|
|
if bits.UintSize == 32 && bitCount > 24 {
|
|
// readBits handles at most intSize-8 bits at once.
|
|
n, err := d.br.readBits(24)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
bitCount -= 24
|
|
offset += n << (4 + bitCount)
|
|
}
|
|
n, err := d.br.readBits(bitCount)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
offset += n << 4
|
|
}
|
|
n, err := d.lowOffsetDecoder.readSym(&d.br)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
offset += n
|
|
} else {
|
|
n, err := d.br.readBits(bitCount)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
offset += n
|
|
}
|
|
}
|
|
if offset > 0x100 {
|
|
length++
|
|
if offset > 0x2000 {
|
|
length++
|
|
if offset > 0x40000 {
|
|
length++
|
|
}
|
|
}
|
|
}
|
|
copy(d.offset[1:], d.offset[:])
|
|
d.offset[0] = offset
|
|
d.length = length
|
|
d.copyBytes(length, offset)
|
|
return nil
|
|
}
|
|
|
|
// decodeSymbols decodes until the window is full or the file ends (io.EOF).
|
|
func (d *decoder) decodeSymbols() error {
|
|
for d.w < d.size {
|
|
sym, err := d.mainDecoder.readSym(&d.br)
|
|
if err == nil {
|
|
switch {
|
|
case sym < 256:
|
|
d.win[d.w] = byte(sym)
|
|
d.w++
|
|
continue
|
|
case sym >= 262:
|
|
err = d.decodeOffset(sym - 262)
|
|
case sym >= 258:
|
|
err = d.decodeLength(sym - 258)
|
|
case sym == 257:
|
|
// repeat the previous match
|
|
d.copyBytes(d.length, d.offset[0])
|
|
continue
|
|
default: // 256
|
|
err = d.readFilter()
|
|
}
|
|
} else if err == io.EOF {
|
|
// end of the block
|
|
if d.lastBlock {
|
|
return io.EOF
|
|
}
|
|
err = d.readBlockHeader()
|
|
}
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
return ErrCorrupt
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// fill decodes more data into the window. It returns an error only when no
|
|
// new bytes are available.
|
|
func (d *decoder) fill() error {
|
|
if d.err != nil {
|
|
return d.err
|
|
}
|
|
if d.w == d.size {
|
|
d.r, d.w = 0, 0
|
|
}
|
|
if d.pendLen > 0 {
|
|
d.copyBytes(d.pendLen, d.pendOff)
|
|
}
|
|
d.err = d.decodeSymbols()
|
|
if d.w == d.r {
|
|
return d.err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// windowBytes moves n decoded bytes out of the window into the scratch buffer.
|
|
func (d *decoder) windowBytes(n int) ([]byte, error) {
|
|
if cap(d.buf) < n {
|
|
d.buf = make([]byte, n, 2*n)
|
|
}
|
|
b := d.buf[:n]
|
|
for copied := 0; ; {
|
|
c := copy(b[copied:], d.win[d.r:d.w])
|
|
d.r += c
|
|
copied += c
|
|
if copied == n {
|
|
return b, nil
|
|
}
|
|
if err := d.fill(); err != nil {
|
|
if err == io.EOF {
|
|
err = ErrCorrupt
|
|
}
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
|
|
// applyFilters runs every filter queued for the current output position.
|
|
func (d *decoder) applyFilters() ([]byte, error) {
|
|
fb := d.filters[0]
|
|
length := fb.length
|
|
|
|
b, err := d.windowBytes(length)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for {
|
|
d.filters = d.filters[1:]
|
|
b = fb.filter(b, d.total)
|
|
if len(d.filters) == 0 {
|
|
d.filters = nil
|
|
return b, nil
|
|
}
|
|
fb = d.filters[0]
|
|
if fb.offset != 0 {
|
|
// the next filter starts later
|
|
fb.offset -= length
|
|
return b, nil
|
|
}
|
|
if fb.length != len(b) {
|
|
return nil, ErrCorrupt
|
|
}
|
|
}
|
|
}
|
|
|
|
// decode sets d.out to the next run of output bytes.
|
|
func (d *decoder) decode() error {
|
|
if d.w == d.r {
|
|
if err := d.fill(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
n := d.w - d.r
|
|
|
|
if len(d.filters) > 0 {
|
|
fb := d.filters[0]
|
|
if fb.offset < 0 {
|
|
return ErrCorrupt
|
|
}
|
|
if fb.offset == 0 {
|
|
out, err := d.applyFilters()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
d.out = out
|
|
d.total += int64(len(out))
|
|
return nil
|
|
}
|
|
// output the bytes in front of the filter block
|
|
n = min(fb.offset, n)
|
|
fb.offset -= n
|
|
}
|
|
|
|
d.out = d.win[d.r : d.r+n]
|
|
d.r += n
|
|
d.total += int64(n)
|
|
return nil
|
|
}
|
|
|
|
func (d *decoder) Read(p []byte) (int, error) {
|
|
for len(d.out) == 0 {
|
|
if err := d.decode(); err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
n := copy(p, d.out)
|
|
d.out = d.out[n:]
|
|
return n, nil
|
|
}
|
|
|
|
// filterE8 undoes the x86 CALL (and, with c == 0xe9, JMP) address transform.
|
|
func filterE8(c byte, buf []byte, offset int64) []byte {
|
|
off := int32(offset)
|
|
for b := buf; len(b) >= 5; {
|
|
ch := b[0]
|
|
b = b[1:]
|
|
off++
|
|
if ch != 0xe8 && ch != c {
|
|
continue
|
|
}
|
|
off %= e8FileSize
|
|
addr := int32(binary.LittleEndian.Uint32(b))
|
|
if addr < 0 {
|
|
if addr+off >= 0 {
|
|
binary.LittleEndian.PutUint32(b, uint32(addr+e8FileSize))
|
|
}
|
|
} else if addr < e8FileSize {
|
|
binary.LittleEndian.PutUint32(b, uint32(addr-off))
|
|
}
|
|
off += 4
|
|
b = b[4:]
|
|
}
|
|
return buf
|
|
}
|
|
|
|
// filterDelta undoes per-channel delta coding.
|
|
func filterDelta(channels int, buf []byte) []byte {
|
|
var res []byte
|
|
l := len(buf)
|
|
if cap(buf) >= 2*l {
|
|
res = buf[l : 2*l] // use unused capacity
|
|
} else {
|
|
res = make([]byte, l, 2*l)
|
|
}
|
|
|
|
i := 0
|
|
for j := 0; j < channels; j++ {
|
|
var c byte
|
|
for k := j; k < len(res); k += channels {
|
|
c -= buf[i]
|
|
i++
|
|
res[k] = c
|
|
}
|
|
}
|
|
return res
|
|
}
|
|
|
|
// filterARM undoes the ARM BL address transform.
|
|
func filterARM(buf []byte, offset int64) []byte {
|
|
for i := 0; len(buf)-i > 3; i += 4 {
|
|
if buf[i+3] == 0xeb {
|
|
n := uint(buf[i])
|
|
n += uint(buf[i+1]) * 0x100
|
|
n += uint(buf[i+2]) * 0x10000
|
|
n -= (uint(offset) + uint(i)) / 4
|
|
buf[i] = byte(n)
|
|
buf[i+1] = byte(n >> 8)
|
|
buf[i+2] = byte(n >> 16)
|
|
}
|
|
}
|
|
return buf
|
|
}
|