package rar5 import ( "bytes" "encoding/binary" "hash/crc32" "math/bits" "time" ) // This file is a minimal RAR 5 writer used only to produce test archives: // there is no free RAR compressor to generate fixtures with. It emits real // compressed streams (literals, matches, repeated matches, delta filters) // with a fixed Huffman code. Its output is accepted by the reference `unrar` // (see dumpArchive). // Fixed code lengths: literals 9 bits, the 16 control/length symbols 5 bits, // offset slots 6 bits, low offset bits and length slots 4 bits. const ( testLiteralBits = 9 testControlBits = 5 testOffsetBits = 6 testNibbleBits = 4 testMaxBase = 9 // longest match base length a 5-bit length slot encodes ) type testBitWriter struct { buf []byte bits int // bits written } // write appends the n low bits of v, most significant first. func (w *testBitWriter) write(v, n int) { for i := n - 1; i >= 0; i-- { if w.bits%8 == 0 { w.buf = append(w.buf, 0) } if v>>i&1 != 0 { w.buf[len(w.buf)-1] |= 0x80 >> (w.bits % 8) } w.bits++ } } // testEncoder compresses one file, or several files of a solid stream. type testEncoder struct { out bytes.Buffer block testBitWriter tables bool // the pending block starts with code length tables symbols int lastOff int lastLen int blockLen int // symbols per block // shareTables makes later blocks reuse the tables of the first one. // Valid, but multi-threaded unrar only accepts it with -mt1. shareTables bool } func (e *testEncoder) writeTables() { e.tables = true // Pre-code: symbols 0..15 with length 4, so a code length is its own code. for i := 0; i < 20; i++ { if i < 16 { e.block.write(4, 4) } else { e.block.write(0, 4) } } lengths := make([]int, 0, mainSize+offsetSize5+lowOffsetSize+lengthSize) for i := 0; i < mainSize; i++ { switch { case i < 256: lengths = append(lengths, testLiteralBits) case i < 272: lengths = append(lengths, testControlBits) default: lengths = append(lengths, 0) } } for i := 0; i < offsetSize5; i++ { lengths = append(lengths, testOffsetBits) } for i := 0; i < lowOffsetSize; i++ { lengths = append(lengths, testNibbleBits) } for i := 0; i < lengthSize; i++ { if i < 16 { lengths = append(lengths, testNibbleBits) } else { lengths = append(lengths, 0) } } for _, l := range lengths { e.block.write(l, 4) } } // flushBlock writes the pending block with its header. func (e *testEncoder) flushBlock(last bool) { total := e.block.bits blockBytes := (total + 7) / 8 flags := byte(total-(blockBytes-1)*8-1) | 2<<3 // 3 size bytes if last { flags |= 0x40 } if e.tables { flags |= 0x80 } e.tables = false size := []byte{byte(blockBytes), byte(blockBytes >> 8), byte(blockBytes >> 16)} sum := 0x5a ^ flags ^ size[0] ^ size[1] ^ size[2] e.out.Write([]byte{flags, sum}) e.out.Write(size) e.out.Write(e.block.buf) e.block = testBitWriter{} } func (e *testEncoder) control(sym int) { e.block.write(sym-256, testControlBits) } func (e *testEncoder) literal(b byte) { e.block.write(256+int(b), testLiteralBits) } func lengthBonus(offset int) int { switch { case offset > 0x40000: return 3 case offset > 0x2000: return 2 case offset > 0x100: return 1 } return 0 } // match emits a match, preferring the repeat symbols when they apply. func (e *testEncoder) match(length, offset int) { switch { case offset == e.lastOff && length == e.lastLen: e.control(257) case offset == e.lastOff && length <= testMaxBase: e.control(258) // most recent offset, new length e.block.write(length-2, testNibbleBits) default: e.control(262 + length - lengthBonus(offset) - 2) x := offset - 1 if x < 4 { e.block.write(x, testOffsetBits) } else { bitCount := bits.Len(uint(x)) - 2 slot := 2*(bitCount+1) + x>>bitCount&1 extra := x - (2|slot&1)<= 4 { e.block.write(extra>>4, bitCount-4) e.block.write(extra&15, testNibbleBits) } else { e.block.write(extra, bitCount) } } } e.lastOff, e.lastLen = offset, length } // deltaFilter announces a delta filter over the next length bytes. func (e *testEncoder) deltaFilter(length, channels int) { e.control(256) e.block.write(0, 2) // offset: one byte e.block.write(0, 8) e.block.write(2, 2) // length: three bytes e.block.write(length&0xff, 8) e.block.write(length>>8&0xff, 8) e.block.write(length>>16&0xff, 8) e.block.write(0, 3) // delta e.block.write(channels-1, 5) } // compress encodes data[start:] with greedy matching against everything // before it (data[:start] is the solid history), limited to the dictionary. func (e *testEncoder) compress(data []byte, start, dictionary int) []byte { e.block = testBitWriter{} e.symbols = 0 e.writeTables() return e.compressBody(data, start, dictionary) } // compressBody encodes after the tables (and any filter records) were written. func (e *testEncoder) compressBody(data []byte, start, dictionary int) []byte { e.out.Reset() if e.blockLen == 0 { e.blockLen = 40000 } recent := make(map[[4]byte]int) key := func(i int) [4]byte { return [4]byte{data[i], data[i+1], data[i+2], data[i+3]} } for i := 0; i+4 <= start; i++ { recent[key(i)] = i } for i := start; i < len(data); { length, offset := 0, 0 if i+4 <= len(data) { if prev, ok := recent[key(i)]; ok && i-prev <= dictionary { offset = i - prev limit := testMaxBase + lengthBonus(offset) for length < limit && i+length < len(data) && data[prev+length] == data[i+length] { length++ } } } // Extend the previous match through its repeat symbols when possible. if e.lastOff > 0 && e.lastOff <= i && e.lastOff <= dictionary { rep := 0 for rep < testMaxBase && i+rep < len(data) && data[i-e.lastOff+rep] == data[i+rep] { rep++ } if rep >= 2 && rep >= length { length, offset = rep, e.lastOff } } step := 1 if (length >= 4 && length-lengthBonus(offset) >= 2) || (length >= 2 && offset == e.lastOff) { e.match(length, offset) step = length } else { e.literal(data[i]) } for end := i + step; i < end; i++ { if i+4 <= len(data) { recent[key(i)] = i } } e.symbols++ if e.symbols%e.blockLen == 0 && i < len(data) { e.flushBlock(false) if !e.shareTables { e.writeTables() } } } e.flushBlock(true) return bytes.Clone(e.out.Bytes()) } type testMember struct { name string content []byte // unpacked content packed []byte // nil: store content solid bool dictBits int // dictionary = 128 KiB << dictBits isDir bool mtime time.Time filetime bool // record mtime as a Windows FILETIME extra record extra []byte flags uint64 // extra block header flags badCRC bool } func testVint(v uint64) []byte { var out []byte for v >= 0x80 { out = append(out, byte(v)|0x80) v >>= 7 } return append(out, byte(v)) } func testBlock(body []byte) []byte { sized := append(testVint(uint64(len(body))), body...) out := binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(sized)) return append(out, sized...) } // testArchive assembles a RAR 5 archive. func testArchive(archiveFlags uint64, members ...testMember) []byte { archive := []byte(signature) archive = append(archive, testBlock(bytes.Join([][]byte{testVint(blockMain), testVint(0), testVint(archiveFlags)}, nil))...) for _, m := range members { data := m.content compression := uint64(0) if m.packed != nil { data = m.packed compression = 3<<7 | uint64(m.dictBits)<<10 if m.solid { compression |= 0x40 } } sum := crc32.ChecksumIEEE(m.content) if m.badCRC { sum ^= 1 } extra := m.extra fileFlags := uint64(fileHasCRC32) if m.isDir { fileFlags = fileIsDirectory } var mtime []byte switch { case m.mtime.IsZero(): case m.filetime: ticks := uint64(m.mtime.UnixNano()/100) + windowsEpochOffset record := append(testVint(extraTime), testVint(0x02)...) record = binary.LittleEndian.AppendUint64(record, ticks) extra = append(append(testVint(uint64(len(record))), record...), extra...) default: fileFlags |= fileHasMTime mtime = binary.LittleEndian.AppendUint32(nil, uint32(m.mtime.Unix())) } headerFlags := uint64(headerHasData) | m.flags parts := [][]byte{testVint(blockFile)} if len(extra) > 0 { headerFlags |= headerHasExtra } parts = append(parts, testVint(headerFlags)) if len(extra) > 0 { parts = append(parts, testVint(uint64(len(extra)))) } parts = append(parts, testVint(uint64(len(data))), testVint(fileFlags), testVint(uint64(len(m.content))), testVint(0x20), // attributes mtime, ) if fileFlags&fileHasCRC32 != 0 { parts = append(parts, binary.LittleEndian.AppendUint32(nil, sum)) } parts = append(parts, testVint(compression), testVint(1), // host OS: Unix testVint(uint64(len(m.name))), []byte(m.name), extra, ) archive = append(archive, testBlock(bytes.Join(parts, nil))...) archive = append(archive, data...) } return append(archive, testBlock(bytes.Join([][]byte{testVint(blockEnd), testVint(0), testVint(0)}, nil))...) }