Field bundles from ad-hoc service scripts (first sample: Giga G894-SD3 .rar with dmidecode/lscpu/lspci/nvme/lsblk output, no fixed layout). ADL-078: - linux-generic: recognizes utility output by content, merges records by stable key, low detect confidence (20-30) below all vendor parsers - pciids.LookupByDescription: resolve a textual lspci description back to vendor/device IDs and manufacturer/model - archive: accept .rar via the in-repo internal/rar5 reader (RAR 5 / RAR 7, single volume, solid; encrypted, multi-volume, SFX and RAR 4.x rejected), same per-file and total limits as tar/zip - UI: .rar in the upload accept list Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
344 lines
9.0 KiB
Go
344 lines
9.0 KiB
Go
package rar5
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/binary"
|
|
"hash/crc32"
|
|
"math/bits"
|
|
"time"
|
|
)
|
|
|
|
// This file is a minimal RAR 5 writer used only to produce test archives:
|
|
// there is no free RAR compressor to generate fixtures with. It emits real
|
|
// compressed streams (literals, matches, repeated matches, delta filters)
|
|
// with a fixed Huffman code. Its output is accepted by the reference `unrar`
|
|
// (see dumpArchive).
|
|
|
|
// Fixed code lengths: literals 9 bits, the 16 control/length symbols 5 bits,
|
|
// offset slots 6 bits, low offset bits and length slots 4 bits.
|
|
const (
|
|
testLiteralBits = 9
|
|
testControlBits = 5
|
|
testOffsetBits = 6
|
|
testNibbleBits = 4
|
|
testMaxBase = 9 // longest match base length a 5-bit length slot encodes
|
|
)
|
|
|
|
type testBitWriter struct {
|
|
buf []byte
|
|
bits int // bits written
|
|
}
|
|
|
|
// write appends the n low bits of v, most significant first.
|
|
func (w *testBitWriter) write(v, n int) {
|
|
for i := n - 1; i >= 0; i-- {
|
|
if w.bits%8 == 0 {
|
|
w.buf = append(w.buf, 0)
|
|
}
|
|
if v>>i&1 != 0 {
|
|
w.buf[len(w.buf)-1] |= 0x80 >> (w.bits % 8)
|
|
}
|
|
w.bits++
|
|
}
|
|
}
|
|
|
|
// testEncoder compresses one file, or several files of a solid stream.
|
|
type testEncoder struct {
|
|
out bytes.Buffer
|
|
block testBitWriter
|
|
tables bool // the pending block starts with code length tables
|
|
symbols int
|
|
lastOff int
|
|
lastLen int
|
|
blockLen int // symbols per block
|
|
// shareTables makes later blocks reuse the tables of the first one.
|
|
// Valid, but multi-threaded unrar only accepts it with -mt1.
|
|
shareTables bool
|
|
}
|
|
|
|
func (e *testEncoder) writeTables() {
|
|
e.tables = true
|
|
// Pre-code: symbols 0..15 with length 4, so a code length is its own code.
|
|
for i := 0; i < 20; i++ {
|
|
if i < 16 {
|
|
e.block.write(4, 4)
|
|
} else {
|
|
e.block.write(0, 4)
|
|
}
|
|
}
|
|
lengths := make([]int, 0, mainSize+offsetSize5+lowOffsetSize+lengthSize)
|
|
for i := 0; i < mainSize; i++ {
|
|
switch {
|
|
case i < 256:
|
|
lengths = append(lengths, testLiteralBits)
|
|
case i < 272:
|
|
lengths = append(lengths, testControlBits)
|
|
default:
|
|
lengths = append(lengths, 0)
|
|
}
|
|
}
|
|
for i := 0; i < offsetSize5; i++ {
|
|
lengths = append(lengths, testOffsetBits)
|
|
}
|
|
for i := 0; i < lowOffsetSize; i++ {
|
|
lengths = append(lengths, testNibbleBits)
|
|
}
|
|
for i := 0; i < lengthSize; i++ {
|
|
if i < 16 {
|
|
lengths = append(lengths, testNibbleBits)
|
|
} else {
|
|
lengths = append(lengths, 0)
|
|
}
|
|
}
|
|
for _, l := range lengths {
|
|
e.block.write(l, 4)
|
|
}
|
|
}
|
|
|
|
// flushBlock writes the pending block with its header.
|
|
func (e *testEncoder) flushBlock(last bool) {
|
|
total := e.block.bits
|
|
blockBytes := (total + 7) / 8
|
|
flags := byte(total-(blockBytes-1)*8-1) | 2<<3 // 3 size bytes
|
|
if last {
|
|
flags |= 0x40
|
|
}
|
|
if e.tables {
|
|
flags |= 0x80
|
|
}
|
|
e.tables = false
|
|
size := []byte{byte(blockBytes), byte(blockBytes >> 8), byte(blockBytes >> 16)}
|
|
sum := 0x5a ^ flags ^ size[0] ^ size[1] ^ size[2]
|
|
e.out.Write([]byte{flags, sum})
|
|
e.out.Write(size)
|
|
e.out.Write(e.block.buf)
|
|
e.block = testBitWriter{}
|
|
}
|
|
|
|
func (e *testEncoder) control(sym int) { e.block.write(sym-256, testControlBits) }
|
|
|
|
func (e *testEncoder) literal(b byte) { e.block.write(256+int(b), testLiteralBits) }
|
|
|
|
func lengthBonus(offset int) int {
|
|
switch {
|
|
case offset > 0x40000:
|
|
return 3
|
|
case offset > 0x2000:
|
|
return 2
|
|
case offset > 0x100:
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// match emits a match, preferring the repeat symbols when they apply.
|
|
func (e *testEncoder) match(length, offset int) {
|
|
switch {
|
|
case offset == e.lastOff && length == e.lastLen:
|
|
e.control(257)
|
|
case offset == e.lastOff && length <= testMaxBase:
|
|
e.control(258) // most recent offset, new length
|
|
e.block.write(length-2, testNibbleBits)
|
|
default:
|
|
e.control(262 + length - lengthBonus(offset) - 2)
|
|
x := offset - 1
|
|
if x < 4 {
|
|
e.block.write(x, testOffsetBits)
|
|
} else {
|
|
bitCount := bits.Len(uint(x)) - 2
|
|
slot := 2*(bitCount+1) + x>>bitCount&1
|
|
extra := x - (2|slot&1)<<bitCount
|
|
e.block.write(slot, testOffsetBits)
|
|
if bitCount >= 4 {
|
|
e.block.write(extra>>4, bitCount-4)
|
|
e.block.write(extra&15, testNibbleBits)
|
|
} else {
|
|
e.block.write(extra, bitCount)
|
|
}
|
|
}
|
|
}
|
|
e.lastOff, e.lastLen = offset, length
|
|
}
|
|
|
|
// deltaFilter announces a delta filter over the next length bytes.
|
|
func (e *testEncoder) deltaFilter(length, channels int) {
|
|
e.control(256)
|
|
e.block.write(0, 2) // offset: one byte
|
|
e.block.write(0, 8)
|
|
e.block.write(2, 2) // length: three bytes
|
|
e.block.write(length&0xff, 8)
|
|
e.block.write(length>>8&0xff, 8)
|
|
e.block.write(length>>16&0xff, 8)
|
|
e.block.write(0, 3) // delta
|
|
e.block.write(channels-1, 5)
|
|
}
|
|
|
|
// compress encodes data[start:] with greedy matching against everything
|
|
// before it (data[:start] is the solid history), limited to the dictionary.
|
|
func (e *testEncoder) compress(data []byte, start, dictionary int) []byte {
|
|
e.block = testBitWriter{}
|
|
e.symbols = 0
|
|
e.writeTables()
|
|
return e.compressBody(data, start, dictionary)
|
|
}
|
|
|
|
// compressBody encodes after the tables (and any filter records) were written.
|
|
func (e *testEncoder) compressBody(data []byte, start, dictionary int) []byte {
|
|
e.out.Reset()
|
|
if e.blockLen == 0 {
|
|
e.blockLen = 40000
|
|
}
|
|
|
|
recent := make(map[[4]byte]int)
|
|
key := func(i int) [4]byte { return [4]byte{data[i], data[i+1], data[i+2], data[i+3]} }
|
|
for i := 0; i+4 <= start; i++ {
|
|
recent[key(i)] = i
|
|
}
|
|
|
|
for i := start; i < len(data); {
|
|
length, offset := 0, 0
|
|
if i+4 <= len(data) {
|
|
if prev, ok := recent[key(i)]; ok && i-prev <= dictionary {
|
|
offset = i - prev
|
|
limit := testMaxBase + lengthBonus(offset)
|
|
for length < limit && i+length < len(data) && data[prev+length] == data[i+length] {
|
|
length++
|
|
}
|
|
}
|
|
}
|
|
// Extend the previous match through its repeat symbols when possible.
|
|
if e.lastOff > 0 && e.lastOff <= i && e.lastOff <= dictionary {
|
|
rep := 0
|
|
for rep < testMaxBase && i+rep < len(data) && data[i-e.lastOff+rep] == data[i+rep] {
|
|
rep++
|
|
}
|
|
if rep >= 2 && rep >= length {
|
|
length, offset = rep, e.lastOff
|
|
}
|
|
}
|
|
|
|
step := 1
|
|
if (length >= 4 && length-lengthBonus(offset) >= 2) || (length >= 2 && offset == e.lastOff) {
|
|
e.match(length, offset)
|
|
step = length
|
|
} else {
|
|
e.literal(data[i])
|
|
}
|
|
for end := i + step; i < end; i++ {
|
|
if i+4 <= len(data) {
|
|
recent[key(i)] = i
|
|
}
|
|
}
|
|
|
|
e.symbols++
|
|
if e.symbols%e.blockLen == 0 && i < len(data) {
|
|
e.flushBlock(false)
|
|
if !e.shareTables {
|
|
e.writeTables()
|
|
}
|
|
}
|
|
}
|
|
e.flushBlock(true)
|
|
return bytes.Clone(e.out.Bytes())
|
|
}
|
|
|
|
type testMember struct {
|
|
name string
|
|
content []byte // unpacked content
|
|
packed []byte // nil: store content
|
|
solid bool
|
|
dictBits int // dictionary = 128 KiB << dictBits
|
|
isDir bool
|
|
mtime time.Time
|
|
filetime bool // record mtime as a Windows FILETIME extra record
|
|
extra []byte
|
|
flags uint64 // extra block header flags
|
|
badCRC bool
|
|
}
|
|
|
|
func testVint(v uint64) []byte {
|
|
var out []byte
|
|
for v >= 0x80 {
|
|
out = append(out, byte(v)|0x80)
|
|
v >>= 7
|
|
}
|
|
return append(out, byte(v))
|
|
}
|
|
|
|
func testBlock(body []byte) []byte {
|
|
sized := append(testVint(uint64(len(body))), body...)
|
|
out := binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(sized))
|
|
return append(out, sized...)
|
|
}
|
|
|
|
// testArchive assembles a RAR 5 archive.
|
|
func testArchive(archiveFlags uint64, members ...testMember) []byte {
|
|
archive := []byte(signature)
|
|
archive = append(archive, testBlock(bytes.Join([][]byte{testVint(blockMain), testVint(0), testVint(archiveFlags)}, nil))...)
|
|
|
|
for _, m := range members {
|
|
data := m.content
|
|
compression := uint64(0)
|
|
if m.packed != nil {
|
|
data = m.packed
|
|
compression = 3<<7 | uint64(m.dictBits)<<10
|
|
if m.solid {
|
|
compression |= 0x40
|
|
}
|
|
}
|
|
sum := crc32.ChecksumIEEE(m.content)
|
|
if m.badCRC {
|
|
sum ^= 1
|
|
}
|
|
|
|
extra := m.extra
|
|
fileFlags := uint64(fileHasCRC32)
|
|
if m.isDir {
|
|
fileFlags = fileIsDirectory
|
|
}
|
|
var mtime []byte
|
|
switch {
|
|
case m.mtime.IsZero():
|
|
case m.filetime:
|
|
ticks := uint64(m.mtime.UnixNano()/100) + windowsEpochOffset
|
|
record := append(testVint(extraTime), testVint(0x02)...)
|
|
record = binary.LittleEndian.AppendUint64(record, ticks)
|
|
extra = append(append(testVint(uint64(len(record))), record...), extra...)
|
|
default:
|
|
fileFlags |= fileHasMTime
|
|
mtime = binary.LittleEndian.AppendUint32(nil, uint32(m.mtime.Unix()))
|
|
}
|
|
|
|
headerFlags := uint64(headerHasData) | m.flags
|
|
parts := [][]byte{testVint(blockFile)}
|
|
if len(extra) > 0 {
|
|
headerFlags |= headerHasExtra
|
|
}
|
|
parts = append(parts, testVint(headerFlags))
|
|
if len(extra) > 0 {
|
|
parts = append(parts, testVint(uint64(len(extra))))
|
|
}
|
|
parts = append(parts,
|
|
testVint(uint64(len(data))),
|
|
testVint(fileFlags),
|
|
testVint(uint64(len(m.content))),
|
|
testVint(0x20), // attributes
|
|
mtime,
|
|
)
|
|
if fileFlags&fileHasCRC32 != 0 {
|
|
parts = append(parts, binary.LittleEndian.AppendUint32(nil, sum))
|
|
}
|
|
parts = append(parts,
|
|
testVint(compression),
|
|
testVint(1), // host OS: Unix
|
|
testVint(uint64(len(m.name))),
|
|
[]byte(m.name),
|
|
extra,
|
|
)
|
|
archive = append(archive, testBlock(bytes.Join(parts, nil))...)
|
|
archive = append(archive, data...)
|
|
}
|
|
|
|
return append(archive, testBlock(bytes.Join([][]byte{testVint(blockEnd), testVint(0), testVint(0)}, nil))...)
|
|
}
|