Files
logpile/internal/rar5/encode_test.go
T
Mikhail ChusavitinandClaude Opus 5.5 b4b15e58a2 feat(parser): linux-generic content-driven parser and RAR archive support
Field bundles from ad-hoc service scripts (first sample: Giga G894-SD3 .rar
with dmidecode/lscpu/lspci/nvme/lsblk output, no fixed layout). ADL-078:
- linux-generic: recognizes utility output by content, merges records by
  stable key, low detect confidence (20-30) below all vendor parsers
- pciids.LookupByDescription: resolve a textual lspci description back to
  vendor/device IDs and manufacturer/model
- archive: accept .rar via the in-repo internal/rar5 reader (RAR 5 / RAR 7,
  single volume, solid; encrypted, multi-volume, SFX and RAR 4.x rejected),
  same per-file and total limits as tar/zip
- UI: .rar in the upload accept list

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-02 15:00:38 +03:00

344 lines
9.0 KiB
Go

package rar5
import (
"bytes"
"encoding/binary"
"hash/crc32"
"math/bits"
"time"
)
// This file is a minimal RAR 5 writer used only to produce test archives:
// there is no free RAR compressor to generate fixtures with. It emits real
// compressed streams (literals, matches, repeated matches, delta filters)
// with a fixed Huffman code. Its output is accepted by the reference `unrar`
// (see dumpArchive).
// Fixed code lengths: literals 9 bits, the 16 control/length symbols 5 bits,
// offset slots 6 bits, low offset bits and length slots 4 bits.
const (
testLiteralBits = 9
testControlBits = 5
testOffsetBits = 6
testNibbleBits = 4
testMaxBase = 9 // longest match base length a 5-bit length slot encodes
)
type testBitWriter struct {
buf []byte
bits int // bits written
}
// write appends the n low bits of v, most significant first.
func (w *testBitWriter) write(v, n int) {
for i := n - 1; i >= 0; i-- {
if w.bits%8 == 0 {
w.buf = append(w.buf, 0)
}
if v>>i&1 != 0 {
w.buf[len(w.buf)-1] |= 0x80 >> (w.bits % 8)
}
w.bits++
}
}
// testEncoder compresses one file, or several files of a solid stream.
type testEncoder struct {
out bytes.Buffer
block testBitWriter
tables bool // the pending block starts with code length tables
symbols int
lastOff int
lastLen int
blockLen int // symbols per block
// shareTables makes later blocks reuse the tables of the first one.
// Valid, but multi-threaded unrar only accepts it with -mt1.
shareTables bool
}
func (e *testEncoder) writeTables() {
e.tables = true
// Pre-code: symbols 0..15 with length 4, so a code length is its own code.
for i := 0; i < 20; i++ {
if i < 16 {
e.block.write(4, 4)
} else {
e.block.write(0, 4)
}
}
lengths := make([]int, 0, mainSize+offsetSize5+lowOffsetSize+lengthSize)
for i := 0; i < mainSize; i++ {
switch {
case i < 256:
lengths = append(lengths, testLiteralBits)
case i < 272:
lengths = append(lengths, testControlBits)
default:
lengths = append(lengths, 0)
}
}
for i := 0; i < offsetSize5; i++ {
lengths = append(lengths, testOffsetBits)
}
for i := 0; i < lowOffsetSize; i++ {
lengths = append(lengths, testNibbleBits)
}
for i := 0; i < lengthSize; i++ {
if i < 16 {
lengths = append(lengths, testNibbleBits)
} else {
lengths = append(lengths, 0)
}
}
for _, l := range lengths {
e.block.write(l, 4)
}
}
// flushBlock writes the pending block with its header.
func (e *testEncoder) flushBlock(last bool) {
total := e.block.bits
blockBytes := (total + 7) / 8
flags := byte(total-(blockBytes-1)*8-1) | 2<<3 // 3 size bytes
if last {
flags |= 0x40
}
if e.tables {
flags |= 0x80
}
e.tables = false
size := []byte{byte(blockBytes), byte(blockBytes >> 8), byte(blockBytes >> 16)}
sum := 0x5a ^ flags ^ size[0] ^ size[1] ^ size[2]
e.out.Write([]byte{flags, sum})
e.out.Write(size)
e.out.Write(e.block.buf)
e.block = testBitWriter{}
}
func (e *testEncoder) control(sym int) { e.block.write(sym-256, testControlBits) }
func (e *testEncoder) literal(b byte) { e.block.write(256+int(b), testLiteralBits) }
func lengthBonus(offset int) int {
switch {
case offset > 0x40000:
return 3
case offset > 0x2000:
return 2
case offset > 0x100:
return 1
}
return 0
}
// match emits a match, preferring the repeat symbols when they apply.
func (e *testEncoder) match(length, offset int) {
switch {
case offset == e.lastOff && length == e.lastLen:
e.control(257)
case offset == e.lastOff && length <= testMaxBase:
e.control(258) // most recent offset, new length
e.block.write(length-2, testNibbleBits)
default:
e.control(262 + length - lengthBonus(offset) - 2)
x := offset - 1
if x < 4 {
e.block.write(x, testOffsetBits)
} else {
bitCount := bits.Len(uint(x)) - 2
slot := 2*(bitCount+1) + x>>bitCount&1
extra := x - (2|slot&1)<<bitCount
e.block.write(slot, testOffsetBits)
if bitCount >= 4 {
e.block.write(extra>>4, bitCount-4)
e.block.write(extra&15, testNibbleBits)
} else {
e.block.write(extra, bitCount)
}
}
}
e.lastOff, e.lastLen = offset, length
}
// deltaFilter announces a delta filter over the next length bytes.
func (e *testEncoder) deltaFilter(length, channels int) {
e.control(256)
e.block.write(0, 2) // offset: one byte
e.block.write(0, 8)
e.block.write(2, 2) // length: three bytes
e.block.write(length&0xff, 8)
e.block.write(length>>8&0xff, 8)
e.block.write(length>>16&0xff, 8)
e.block.write(0, 3) // delta
e.block.write(channels-1, 5)
}
// compress encodes data[start:] with greedy matching against everything
// before it (data[:start] is the solid history), limited to the dictionary.
func (e *testEncoder) compress(data []byte, start, dictionary int) []byte {
e.block = testBitWriter{}
e.symbols = 0
e.writeTables()
return e.compressBody(data, start, dictionary)
}
// compressBody encodes after the tables (and any filter records) were written.
func (e *testEncoder) compressBody(data []byte, start, dictionary int) []byte {
e.out.Reset()
if e.blockLen == 0 {
e.blockLen = 40000
}
recent := make(map[[4]byte]int)
key := func(i int) [4]byte { return [4]byte{data[i], data[i+1], data[i+2], data[i+3]} }
for i := 0; i+4 <= start; i++ {
recent[key(i)] = i
}
for i := start; i < len(data); {
length, offset := 0, 0
if i+4 <= len(data) {
if prev, ok := recent[key(i)]; ok && i-prev <= dictionary {
offset = i - prev
limit := testMaxBase + lengthBonus(offset)
for length < limit && i+length < len(data) && data[prev+length] == data[i+length] {
length++
}
}
}
// Extend the previous match through its repeat symbols when possible.
if e.lastOff > 0 && e.lastOff <= i && e.lastOff <= dictionary {
rep := 0
for rep < testMaxBase && i+rep < len(data) && data[i-e.lastOff+rep] == data[i+rep] {
rep++
}
if rep >= 2 && rep >= length {
length, offset = rep, e.lastOff
}
}
step := 1
if (length >= 4 && length-lengthBonus(offset) >= 2) || (length >= 2 && offset == e.lastOff) {
e.match(length, offset)
step = length
} else {
e.literal(data[i])
}
for end := i + step; i < end; i++ {
if i+4 <= len(data) {
recent[key(i)] = i
}
}
e.symbols++
if e.symbols%e.blockLen == 0 && i < len(data) {
e.flushBlock(false)
if !e.shareTables {
e.writeTables()
}
}
}
e.flushBlock(true)
return bytes.Clone(e.out.Bytes())
}
type testMember struct {
name string
content []byte // unpacked content
packed []byte // nil: store content
solid bool
dictBits int // dictionary = 128 KiB << dictBits
isDir bool
mtime time.Time
filetime bool // record mtime as a Windows FILETIME extra record
extra []byte
flags uint64 // extra block header flags
badCRC bool
}
func testVint(v uint64) []byte {
var out []byte
for v >= 0x80 {
out = append(out, byte(v)|0x80)
v >>= 7
}
return append(out, byte(v))
}
func testBlock(body []byte) []byte {
sized := append(testVint(uint64(len(body))), body...)
out := binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(sized))
return append(out, sized...)
}
// testArchive assembles a RAR 5 archive.
func testArchive(archiveFlags uint64, members ...testMember) []byte {
archive := []byte(signature)
archive = append(archive, testBlock(bytes.Join([][]byte{testVint(blockMain), testVint(0), testVint(archiveFlags)}, nil))...)
for _, m := range members {
data := m.content
compression := uint64(0)
if m.packed != nil {
data = m.packed
compression = 3<<7 | uint64(m.dictBits)<<10
if m.solid {
compression |= 0x40
}
}
sum := crc32.ChecksumIEEE(m.content)
if m.badCRC {
sum ^= 1
}
extra := m.extra
fileFlags := uint64(fileHasCRC32)
if m.isDir {
fileFlags = fileIsDirectory
}
var mtime []byte
switch {
case m.mtime.IsZero():
case m.filetime:
ticks := uint64(m.mtime.UnixNano()/100) + windowsEpochOffset
record := append(testVint(extraTime), testVint(0x02)...)
record = binary.LittleEndian.AppendUint64(record, ticks)
extra = append(append(testVint(uint64(len(record))), record...), extra...)
default:
fileFlags |= fileHasMTime
mtime = binary.LittleEndian.AppendUint32(nil, uint32(m.mtime.Unix()))
}
headerFlags := uint64(headerHasData) | m.flags
parts := [][]byte{testVint(blockFile)}
if len(extra) > 0 {
headerFlags |= headerHasExtra
}
parts = append(parts, testVint(headerFlags))
if len(extra) > 0 {
parts = append(parts, testVint(uint64(len(extra))))
}
parts = append(parts,
testVint(uint64(len(data))),
testVint(fileFlags),
testVint(uint64(len(m.content))),
testVint(0x20), // attributes
mtime,
)
if fileFlags&fileHasCRC32 != 0 {
parts = append(parts, binary.LittleEndian.AppendUint32(nil, sum))
}
parts = append(parts,
testVint(compression),
testVint(1), // host OS: Unix
testVint(uint64(len(m.name))),
[]byte(m.name),
extra,
)
archive = append(archive, testBlock(bytes.Join(parts, nil))...)
archive = append(archive, data...)
}
return append(archive, testBlock(bytes.Join([][]byte{testVint(blockEnd), testVint(0), testVint(0)}, nil))...)
}