diff --git a/.gitattributes b/.gitattributes index 0e0863aaf6de..6cd4c59f1fff 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,6 +4,7 @@ langref.html.in text eol=lf lib/std/compress/testdata/** binary lib/std/compress/deflate/testdata/** binary +lib/std/compress/flate/testdata/** binary lib/include/** linguist-vendored lib/libc/** linguist-vendored diff --git a/build.zig b/build.zig index 5dcf8709e5bd..3ec58d969649 100644 --- a/build.zig +++ b/build.zig @@ -150,7 +150,7 @@ pub fn build(b: *std.Build) !void { "rfc1951.txt", "rfc1952.txt", "rfc8478.txt", - // exclude files from lib/std/compress/deflate/testdata + // exclude files from lib/std/compress/flate/testdata ".expect", ".expect-noinput", ".golden", diff --git a/lib/std/compress.zig b/lib/std/compress.zig index e56008cefe7d..a6d0a40b26ef 100644 --- a/lib/std/compress.zig +++ b/lib/std/compress.zig @@ -1,11 +1,11 @@ const std = @import("std.zig"); -pub const deflate = @import("compress/deflate.zig"); +pub const flate = @import("compress/flate.zig"); pub const gzip = @import("compress/gzip.zig"); +pub const zlib = @import("compress/zlib.zig"); pub const lzma = @import("compress/lzma.zig"); pub const lzma2 = @import("compress/lzma2.zig"); pub const xz = @import("compress/xz.zig"); -pub const zlib = @import("compress/zlib.zig"); pub const zstd = @import("compress/zstandard.zig"); pub fn HashedReader( @@ -69,11 +69,11 @@ pub fn hashedWriter( } test { - _ = deflate; - _ = gzip; _ = lzma; _ = lzma2; _ = xz; - _ = zlib; _ = zstd; + _ = flate; + _ = gzip; + _ = zlib; } diff --git a/lib/std/compress/deflate.zig b/lib/std/compress/deflate.zig deleted file mode 100644 index 2fe596906775..000000000000 --- a/lib/std/compress/deflate.zig +++ /dev/null @@ -1,44 +0,0 @@ -//! The deflate package is a translation of the Go code of the compress/flate package from -//! https://go.googlesource.com/go/+/refs/tags/go1.17/src/compress/flate/ - -const deflate = @import("deflate/compressor.zig"); -const inflate = @import("deflate/decompressor.zig"); - -pub const Compression = deflate.Compression; -pub const CompressorOptions = deflate.CompressorOptions; -pub const Compressor = deflate.Compressor; -pub const Decompressor = inflate.Decompressor; - -pub const compressor = deflate.compressor; -pub const decompressor = inflate.decompressor; - -/// Copies elements from a source `src` slice into a destination `dst` slice. -/// The copy never returns an error but might not be complete if the destination is too small. -/// Returns the number of elements copied, which will be the minimum of `src.len` and `dst.len`. -/// TODO: remove this smelly function -pub fn copy(dst: []u8, src: []const u8) usize { - if (dst.len <= src.len) { - @memcpy(dst, src[0..dst.len]); - return dst.len; - } else { - @memcpy(dst[0..src.len], src); - return src.len; - } -} - -test { - _ = @import("deflate/token.zig"); - _ = @import("deflate/bits_utils.zig"); - _ = @import("deflate/dict_decoder.zig"); - - _ = @import("deflate/huffman_code.zig"); - _ = @import("deflate/huffman_bit_writer.zig"); - - _ = @import("deflate/compressor.zig"); - _ = @import("deflate/compressor_test.zig"); - - _ = @import("deflate/deflate_fast.zig"); - _ = @import("deflate/deflate_fast_test.zig"); - - _ = @import("deflate/decompressor.zig"); -} diff --git a/lib/std/compress/deflate/bits_utils.zig b/lib/std/compress/deflate/bits_utils.zig deleted file mode 100644 index 97a557d0da42..000000000000 --- a/lib/std/compress/deflate/bits_utils.zig +++ /dev/null @@ -1,33 +0,0 @@ -const math = @import("std").math; - -// Reverse bit-by-bit a N-bit code. -pub fn bitReverse(comptime T: type, value: T, N: usize) T { - const r = @bitReverse(value); - return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).Int.bits - N)); -} - -test "bitReverse" { - const std = @import("std"); - - const ReverseBitsTest = struct { - in: u16, - bit_count: u5, - out: u16, - }; - - const reverse_bits_tests = [_]ReverseBitsTest{ - .{ .in = 1, .bit_count = 1, .out = 1 }, - .{ .in = 1, .bit_count = 2, .out = 2 }, - .{ .in = 1, .bit_count = 3, .out = 4 }, - .{ .in = 1, .bit_count = 4, .out = 8 }, - .{ .in = 1, .bit_count = 5, .out = 16 }, - .{ .in = 17, .bit_count = 5, .out = 17 }, - .{ .in = 257, .bit_count = 9, .out = 257 }, - .{ .in = 29, .bit_count = 5, .out = 23 }, - }; - - for (reverse_bits_tests) |h| { - const v = bitReverse(u16, h.in, h.bit_count); - try std.testing.expectEqual(h.out, v); - } -} diff --git a/lib/std/compress/deflate/compressor.zig b/lib/std/compress/deflate/compressor.zig deleted file mode 100644 index 0326668793b0..000000000000 --- a/lib/std/compress/deflate/compressor.zig +++ /dev/null @@ -1,1110 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const fmt = std.fmt; -const io = std.io; -const math = std.math; -const mem = std.mem; - -const Allocator = std.mem.Allocator; - -const deflate_const = @import("deflate_const.zig"); -const fast = @import("deflate_fast.zig"); -const hm_bw = @import("huffman_bit_writer.zig"); -const token = @import("token.zig"); - -pub const Compression = enum(i5) { - /// huffman_only disables Lempel-Ziv match searching and only performs Huffman - /// entropy encoding. This mode is useful in compressing data that has - /// already been compressed with an LZ style algorithm (e.g. Snappy or LZ4) - /// that lacks an entropy encoder. Compression gains are achieved when - /// certain bytes in the input stream occur more frequently than others. - /// - /// Note that huffman_only produces a compressed output that is - /// RFC 1951 compliant. That is, any valid DEFLATE decompressor will - /// continue to be able to decompress this output. - huffman_only = -2, - /// Same as level_6 - default_compression = -1, - /// Does not attempt any compression; only adds the necessary DEFLATE framing. - no_compression = 0, - /// Prioritizes speed over output size, based on Snappy's LZ77-style encoder - best_speed = 1, - level_2 = 2, - level_3 = 3, - level_4 = 4, - level_5 = 5, - level_6 = 6, - level_7 = 7, - level_8 = 8, - /// Prioritizes smaller output size over speed - best_compression = 9, -}; - -const log_window_size = 15; -const window_size = 1 << log_window_size; -const window_mask = window_size - 1; - -// The LZ77 step produces a sequence of literal tokens and -// pair tokens. The offset is also known as distance. The underlying wire -// format limits the range of lengths and offsets. For example, there are -// 256 legitimate lengths: those in the range [3, 258]. This package's -// compressor uses a higher minimum match length, enabling optimizations -// such as finding matches via 32-bit loads and compares. -const base_match_length = deflate_const.base_match_length; // The smallest match length per the RFC section 3.2.5 -const min_match_length = 4; // The smallest match length that the compressor actually emits -const max_match_length = deflate_const.max_match_length; -const base_match_offset = deflate_const.base_match_offset; // The smallest match offset -const max_match_offset = deflate_const.max_match_offset; // The largest match offset - -// The maximum number of tokens we put into a single flate block, just to -// stop things from getting too large. -const max_flate_block_tokens = 1 << 14; -const max_store_block_size = deflate_const.max_store_block_size; -const hash_bits = 17; // After 17 performance degrades -const hash_size = 1 << hash_bits; -const hash_mask = (1 << hash_bits) - 1; -const max_hash_offset = 1 << 24; - -const skip_never = math.maxInt(u32); - -const CompressionLevel = struct { - good: u16, - lazy: u16, - nice: u16, - chain: u16, - fast_skip_hashshing: u32, -}; - -fn levels(compression: Compression) CompressionLevel { - switch (compression) { - .no_compression, - .best_speed, // best_speed uses a custom algorithm; see deflate_fast.zig - .huffman_only, - => return .{ - .good = 0, - .lazy = 0, - .nice = 0, - .chain = 0, - .fast_skip_hashshing = 0, - }, - // For levels 2-3 we don't bother trying with lazy matches. - .level_2 => return .{ - .good = 4, - .lazy = 0, - .nice = 16, - .chain = 8, - .fast_skip_hashshing = 5, - }, - .level_3 => return .{ - .good = 4, - .lazy = 0, - .nice = 32, - .chain = 32, - .fast_skip_hashshing = 6, - }, - - // Levels 4-9 use increasingly more lazy matching and increasingly stringent conditions for - // "good enough". - .level_4 => return .{ - .good = 4, - .lazy = 4, - .nice = 16, - .chain = 16, - .fast_skip_hashshing = skip_never, - }, - .level_5 => return .{ - .good = 8, - .lazy = 16, - .nice = 32, - .chain = 32, - .fast_skip_hashshing = skip_never, - }, - .default_compression, - .level_6, - => return .{ - .good = 8, - .lazy = 16, - .nice = 128, - .chain = 128, - .fast_skip_hashshing = skip_never, - }, - .level_7 => return .{ - .good = 8, - .lazy = 32, - .nice = 128, - .chain = 256, - .fast_skip_hashshing = skip_never, - }, - .level_8 => return .{ - .good = 32, - .lazy = 128, - .nice = 258, - .chain = 1024, - .fast_skip_hashshing = skip_never, - }, - .best_compression => return .{ - .good = 32, - .lazy = 258, - .nice = 258, - .chain = 4096, - .fast_skip_hashshing = skip_never, - }, - } -} - -// matchLen returns the number of matching bytes in a and b -// up to length 'max'. Both slices must be at least 'max' -// bytes in size. -fn matchLen(a: []u8, b: []u8, max: u32) u32 { - const bounded_a = a[0..max]; - const bounded_b = b[0..max]; - for (bounded_a, 0..) |av, i| { - if (bounded_b[i] != av) { - return @as(u32, @intCast(i)); - } - } - return max; -} - -const hash_mul = 0x1e35a7bd; - -// hash4 returns a hash representation of the first 4 bytes -// of the supplied slice. -// The caller must ensure that b.len >= 4. -fn hash4(b: []u8) u32 { - return ((@as(u32, b[3]) | - @as(u32, b[2]) << 8 | - @as(u32, b[1]) << 16 | - @as(u32, b[0]) << 24) *% hash_mul) >> (32 - hash_bits); -} - -// bulkHash4 will compute hashes using the same -// algorithm as hash4 -fn bulkHash4(b: []u8, dst: []u32) u32 { - if (b.len < min_match_length) { - return 0; - } - var hb = - @as(u32, b[3]) | - @as(u32, b[2]) << 8 | - @as(u32, b[1]) << 16 | - @as(u32, b[0]) << 24; - - dst[0] = (hb *% hash_mul) >> (32 - hash_bits); - const end = b.len - min_match_length + 1; - var i: u32 = 1; - while (i < end) : (i += 1) { - hb = (hb << 8) | @as(u32, b[i + 3]); - dst[i] = (hb *% hash_mul) >> (32 - hash_bits); - } - - return hb; -} - -pub const CompressorOptions = struct { - level: Compression = .default_compression, - dictionary: ?[]const u8 = null, -}; - -/// Returns a new Compressor compressing data at the given level. -/// Following zlib, levels range from 1 (best_speed) to 9 (best_compression); -/// higher levels typically run slower but compress more. Level 0 -/// (no_compression) does not attempt any compression; it only adds the -/// necessary DEFLATE framing. -/// Level -1 (default_compression) uses the default compression level. -/// Level -2 (huffman_only) will use Huffman compression only, giving -/// a very fast compression for all types of input, but sacrificing considerable -/// compression efficiency. -/// -/// `dictionary` is optional and initializes the new `Compressor` with a preset dictionary. -/// The returned Compressor behaves as if the dictionary had been written to it without producing -/// any compressed output. The compressed data written to hm_bw can only be decompressed by a -/// Decompressor initialized with the same dictionary. -/// -/// The compressed data will be passed to the provided `writer`, see `writer()` and `write()`. -pub fn compressor( - allocator: Allocator, - writer: anytype, - options: CompressorOptions, -) !Compressor(@TypeOf(writer)) { - return Compressor(@TypeOf(writer)).init(allocator, writer, options); -} - -pub fn Compressor(comptime WriterType: anytype) type { - return struct { - const Self = @This(); - - /// A Writer takes data written to it and writes the compressed - /// form of that data to an underlying writer. - pub const Writer = io.Writer(*Self, Error, write); - - /// Returns a Writer that takes data written to it and writes the compressed - /// form of that data to an underlying writer. - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } - - pub const Error = WriterType.Error; - - allocator: Allocator, - - compression: Compression, - compression_level: CompressionLevel, - - // Inner writer wrapped in a HuffmanBitWriter - hm_bw: hm_bw.HuffmanBitWriter(WriterType) = undefined, - bulk_hasher: *const fn ([]u8, []u32) u32, - - sync: bool, // requesting flush - best_speed_enc: *fast.DeflateFast, // Encoder for best_speed - - // Input hash chains - // hash_head[hashValue] contains the largest inputIndex with the specified hash value - // If hash_head[hashValue] is within the current window, then - // hash_prev[hash_head[hashValue] & window_mask] contains the previous index - // with the same hash value. - chain_head: u32, - hash_head: []u32, // [hash_size]u32, - hash_prev: []u32, // [window_size]u32, - hash_offset: u32, - - // input window: unprocessed data is window[index..window_end] - index: u32, - window: []u8, - window_end: usize, - block_start: usize, // window index where current tokens start - byte_available: bool, // if true, still need to process window[index-1]. - - // queued output tokens - tokens: []token.Token, - tokens_count: u16, - - // deflate state - length: u32, - offset: u32, - hash: u32, - max_insert_index: usize, - err: bool, - - // hash_match must be able to contain hashes for the maximum match length. - hash_match: []u32, // [max_match_length - 1]u32, - - // dictionary - dictionary: ?[]const u8, - - fn fillDeflate(self: *Self, b: []const u8) u32 { - if (self.index >= 2 * window_size - (min_match_length + max_match_length)) { - // shift the window by window_size - mem.copyForwards(u8, self.window, self.window[window_size .. 2 * window_size]); - self.index -= window_size; - self.window_end -= window_size; - if (self.block_start >= window_size) { - self.block_start -= window_size; - } else { - self.block_start = math.maxInt(u32); - } - self.hash_offset += window_size; - if (self.hash_offset > max_hash_offset) { - const delta = self.hash_offset - 1; - self.hash_offset -= delta; - self.chain_head -|= delta; - - // Iterate over slices instead of arrays to avoid copying - // the entire table onto the stack (https://golang.org/issue/18625). - for (self.hash_prev, 0..) |v, i| { - if (v > delta) { - self.hash_prev[i] = @as(u32, @intCast(v - delta)); - } else { - self.hash_prev[i] = 0; - } - } - for (self.hash_head, 0..) |v, i| { - if (v > delta) { - self.hash_head[i] = @as(u32, @intCast(v - delta)); - } else { - self.hash_head[i] = 0; - } - } - } - } - const n = std.compress.deflate.copy(self.window[self.window_end..], b); - self.window_end += n; - return @as(u32, @intCast(n)); - } - - fn writeBlock(self: *Self, tokens: []token.Token, index: usize) !void { - if (index > 0) { - var window: ?[]u8 = null; - if (self.block_start <= index) { - window = self.window[self.block_start..index]; - } - self.block_start = index; - try self.hm_bw.writeBlock(tokens, false, window); - return; - } - return; - } - - // fillWindow will fill the current window with the supplied - // dictionary and calculate all hashes. - // This is much faster than doing a full encode. - // Should only be used after a reset. - fn fillWindow(self: *Self, in_b: []const u8) void { - var b = in_b; - // Do not fill window if we are in store-only mode (look at the fill() function to see - // Compressions which use fillStore() instead of fillDeflate()). - if (self.compression == .no_compression or - self.compression == .huffman_only or - self.compression == .best_speed) - { - return; - } - - // fillWindow() must not be called with stale data - assert(self.index == 0 and self.window_end == 0); - - // If we are given too much, cut it. - if (b.len > window_size) { - b = b[b.len - window_size ..]; - } - // Add all to window. - @memcpy(self.window[0..b.len], b); - const n = b.len; - - // Calculate 256 hashes at the time (more L1 cache hits) - const loops = (n + 256 - min_match_length) / 256; - var j: usize = 0; - while (j < loops) : (j += 1) { - const index = j * 256; - var end = index + 256 + min_match_length - 1; - if (end > n) { - end = n; - } - const to_check = self.window[index..end]; - const dst_size = to_check.len - min_match_length + 1; - - if (dst_size <= 0) { - continue; - } - - const dst = self.hash_match[0..dst_size]; - _ = self.bulk_hasher(to_check, dst); - var new_h: u32 = 0; - for (dst, 0..) |val, i| { - const di = i + index; - new_h = val; - const hh = &self.hash_head[new_h & hash_mask]; - // Get previous value with the same hash. - // Our chain should point to the previous value. - self.hash_prev[di & window_mask] = hh.*; - // Set the head of the hash chain to us. - hh.* = @as(u32, @intCast(di + self.hash_offset)); - } - self.hash = new_h; - } - // Update window information. - self.window_end = n; - self.index = @as(u32, @intCast(n)); - } - - const Match = struct { - length: u32, - offset: u32, - ok: bool, - }; - - // Try to find a match starting at pos whose length is greater than prev_length. - // We only look at self.compression_level.chain possibilities before giving up. - fn findMatch( - self: *Self, - pos: u32, - prev_head: u32, - prev_length: u32, - lookahead: u32, - ) Match { - var length: u32 = 0; - var offset: u32 = 0; - var ok: bool = false; - - var min_match_look: u32 = max_match_length; - if (lookahead < min_match_look) { - min_match_look = lookahead; - } - - var win = self.window[0 .. pos + min_match_look]; - - // We quit when we get a match that's at least nice long - var nice = win.len - pos; - if (self.compression_level.nice < nice) { - nice = self.compression_level.nice; - } - - // If we've got a match that's good enough, only look in 1/4 the chain. - var tries = self.compression_level.chain; - length = prev_length; - if (length >= self.compression_level.good) { - tries >>= 2; - } - - var w_end = win[pos + length]; - const w_pos = win[pos..]; - const min_index = pos -| window_size; - - var i = prev_head; - while (tries > 0) : (tries -= 1) { - if (w_end == win[i + length]) { - const n = matchLen(win[i..], w_pos, min_match_look); - - if (n > length and (n > min_match_length or pos - i <= 4096)) { - length = n; - offset = pos - i; - ok = true; - if (n >= nice) { - // The match is good enough that we don't try to find a better one. - break; - } - w_end = win[pos + n]; - } - } - if (i == min_index) { - // hash_prev[i & window_mask] has already been overwritten, so stop now. - break; - } - - if (@as(u32, @intCast(self.hash_prev[i & window_mask])) < self.hash_offset) { - break; - } - - i = @as(u32, @intCast(self.hash_prev[i & window_mask])) - self.hash_offset; - if (i < min_index) { - break; - } - } - - return Match{ .length = length, .offset = offset, .ok = ok }; - } - - fn writeStoredBlock(self: *Self, buf: []u8) !void { - try self.hm_bw.writeStoredHeader(buf.len, false); - try self.hm_bw.writeBytes(buf); - } - - // encSpeed will compress and store the currently added data, - // if enough has been accumulated or we at the end of the stream. - fn encSpeed(self: *Self) !void { - // We only compress if we have max_store_block_size. - if (self.window_end < max_store_block_size) { - if (!self.sync) { - return; - } - - // Handle small sizes. - if (self.window_end < 128) { - switch (self.window_end) { - 0 => return, - 1...16 => { - try self.writeStoredBlock(self.window[0..self.window_end]); - }, - else => { - try self.hm_bw.writeBlockHuff(false, self.window[0..self.window_end]); - self.err = self.hm_bw.err; - }, - } - self.window_end = 0; - self.best_speed_enc.reset(); - return; - } - } - // Encode the block. - self.tokens_count = 0; - self.best_speed_enc.encode( - self.tokens, - &self.tokens_count, - self.window[0..self.window_end], - ); - - // If we removed less than 1/16th, Huffman compress the block. - if (self.tokens_count > self.window_end - (self.window_end >> 4)) { - try self.hm_bw.writeBlockHuff(false, self.window[0..self.window_end]); - } else { - try self.hm_bw.writeBlockDynamic( - self.tokens[0..self.tokens_count], - false, - self.window[0..self.window_end], - ); - } - self.err = self.hm_bw.err; - self.window_end = 0; - } - - fn initDeflate(self: *Self) !void { - self.window = try self.allocator.alloc(u8, 2 * window_size); - self.hash_offset = 1; - self.tokens = try self.allocator.alloc(token.Token, max_flate_block_tokens); - self.tokens_count = 0; - @memset(self.tokens, 0); - self.length = min_match_length - 1; - self.offset = 0; - self.byte_available = false; - self.index = 0; - self.hash = 0; - self.chain_head = 0; - self.bulk_hasher = bulkHash4; - } - - fn deflate(self: *Self) !void { - if (self.window_end - self.index < min_match_length + max_match_length and !self.sync) { - return; - } - - self.max_insert_index = self.window_end -| (min_match_length - 1); - if (self.index < self.max_insert_index) { - self.hash = hash4(self.window[self.index .. self.index + min_match_length]); - } - - while (true) { - assert(self.index <= self.window_end); - - const lookahead = self.window_end -| self.index; - if (lookahead < min_match_length + max_match_length) { - if (!self.sync) { - break; - } - assert(self.index <= self.window_end); - - if (lookahead == 0) { - // Flush current output block if any. - if (self.byte_available) { - // There is still one pending token that needs to be flushed - self.tokens[self.tokens_count] = token.literalToken(@as(u32, @intCast(self.window[self.index - 1]))); - self.tokens_count += 1; - self.byte_available = false; - } - if (self.tokens.len > 0) { - try self.writeBlock(self.tokens[0..self.tokens_count], self.index); - self.tokens_count = 0; - } - break; - } - } - if (self.index < self.max_insert_index) { - // Update the hash - self.hash = hash4(self.window[self.index .. self.index + min_match_length]); - const hh = &self.hash_head[self.hash & hash_mask]; - self.chain_head = @as(u32, @intCast(hh.*)); - self.hash_prev[self.index & window_mask] = @as(u32, @intCast(self.chain_head)); - hh.* = @as(u32, @intCast(self.index + self.hash_offset)); - } - const prev_length = self.length; - const prev_offset = self.offset; - self.length = min_match_length - 1; - self.offset = 0; - const min_index = self.index -| window_size; - - if (self.hash_offset <= self.chain_head and - self.chain_head - self.hash_offset >= min_index and - (self.compression_level.fast_skip_hashshing != skip_never and - lookahead > min_match_length - 1 or - self.compression_level.fast_skip_hashshing == skip_never and - lookahead > prev_length and - prev_length < self.compression_level.lazy)) - { - { - const fmatch = self.findMatch( - self.index, - self.chain_head -| self.hash_offset, - min_match_length - 1, - @as(u32, @intCast(lookahead)), - ); - if (fmatch.ok) { - self.length = fmatch.length; - self.offset = fmatch.offset; - } - } - } - if (self.compression_level.fast_skip_hashshing != skip_never and - self.length >= min_match_length or - self.compression_level.fast_skip_hashshing == skip_never and - prev_length >= min_match_length and - self.length <= prev_length) - { - // There was a match at the previous step, and the current match is - // not better. Output the previous match. - if (self.compression_level.fast_skip_hashshing != skip_never) { - self.tokens[self.tokens_count] = token.matchToken(@as(u32, @intCast(self.length - base_match_length)), @as(u32, @intCast(self.offset - base_match_offset))); - self.tokens_count += 1; - } else { - self.tokens[self.tokens_count] = token.matchToken( - @as(u32, @intCast(prev_length - base_match_length)), - @as(u32, @intCast(prev_offset -| base_match_offset)), - ); - self.tokens_count += 1; - } - // Insert in the hash table all strings up to the end of the match. - // index and index-1 are already inserted. If there is not enough - // lookahead, the last two strings are not inserted into the hash - // table. - if (self.length <= self.compression_level.fast_skip_hashshing) { - var newIndex: u32 = 0; - if (self.compression_level.fast_skip_hashshing != skip_never) { - newIndex = self.index + self.length; - } else { - newIndex = self.index + prev_length - 1; - } - var index = self.index; - index += 1; - while (index < newIndex) : (index += 1) { - if (index < self.max_insert_index) { - self.hash = hash4(self.window[index .. index + min_match_length]); - // Get previous value with the same hash. - // Our chain should point to the previous value. - const hh = &self.hash_head[self.hash & hash_mask]; - self.hash_prev[index & window_mask] = hh.*; - // Set the head of the hash chain to us. - hh.* = @as(u32, @intCast(index + self.hash_offset)); - } - } - self.index = index; - - if (self.compression_level.fast_skip_hashshing == skip_never) { - self.byte_available = false; - self.length = min_match_length - 1; - } - } else { - // For matches this long, we don't bother inserting each individual - // item into the table. - self.index += self.length; - if (self.index < self.max_insert_index) { - self.hash = hash4(self.window[self.index .. self.index + min_match_length]); - } - } - if (self.tokens_count == max_flate_block_tokens) { - // The block includes the current character - try self.writeBlock(self.tokens[0..self.tokens_count], self.index); - self.tokens_count = 0; - } - } else { - if (self.compression_level.fast_skip_hashshing != skip_never or self.byte_available) { - var i = self.index -| 1; - if (self.compression_level.fast_skip_hashshing != skip_never) { - i = self.index; - } - self.tokens[self.tokens_count] = token.literalToken(@as(u32, @intCast(self.window[i]))); - self.tokens_count += 1; - if (self.tokens_count == max_flate_block_tokens) { - try self.writeBlock(self.tokens[0..self.tokens_count], i + 1); - self.tokens_count = 0; - } - } - self.index += 1; - if (self.compression_level.fast_skip_hashshing == skip_never) { - self.byte_available = true; - } - } - } - } - - fn fillStore(self: *Self, b: []const u8) u32 { - const n = std.compress.deflate.copy(self.window[self.window_end..], b); - self.window_end += n; - return @as(u32, @intCast(n)); - } - - fn store(self: *Self) !void { - if (self.window_end > 0 and (self.window_end == max_store_block_size or self.sync)) { - try self.writeStoredBlock(self.window[0..self.window_end]); - self.window_end = 0; - } - } - - // storeHuff compresses and stores the currently added data - // when the self.window is full or we are at the end of the stream. - fn storeHuff(self: *Self) !void { - if (self.window_end < self.window.len and !self.sync or self.window_end == 0) { - return; - } - try self.hm_bw.writeBlockHuff(false, self.window[0..self.window_end]); - self.err = self.hm_bw.err; - self.window_end = 0; - } - - pub fn bytesWritten(self: *Self) usize { - return self.hm_bw.bytes_written; - } - - /// Writes the compressed form of `input` to the underlying writer. - pub fn write(self: *Self, input: []const u8) Error!usize { - var buf = input; - - // writes data to hm_bw, which will eventually write the - // compressed form of data to its underlying writer. - while (buf.len > 0) { - try self.step(); - const filled = self.fill(buf); - buf = buf[filled..]; - } - - return input.len; - } - - /// Flushes any pending data to the underlying writer. - /// It is useful mainly in compressed network protocols, to ensure that - /// a remote reader has enough data to reconstruct a packet. - /// Flush does not return until the data has been written. - /// Calling `flush()` when there is no pending data still causes the Writer - /// to emit a sync marker of at least 4 bytes. - /// If the underlying writer returns an error, `flush()` returns that error. - /// - /// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. - pub fn flush(self: *Self) Error!void { - self.sync = true; - try self.step(); - try self.hm_bw.writeStoredHeader(0, false); - try self.hm_bw.flush(); - self.sync = false; - return; - } - - fn step(self: *Self) !void { - switch (self.compression) { - .no_compression => return self.store(), - .huffman_only => return self.storeHuff(), - .best_speed => return self.encSpeed(), - .default_compression, - .level_2, - .level_3, - .level_4, - .level_5, - .level_6, - .level_7, - .level_8, - .best_compression, - => return self.deflate(), - } - } - - fn fill(self: *Self, b: []const u8) u32 { - switch (self.compression) { - .no_compression => return self.fillStore(b), - .huffman_only => return self.fillStore(b), - .best_speed => return self.fillStore(b), - .default_compression, - .level_2, - .level_3, - .level_4, - .level_5, - .level_6, - .level_7, - .level_8, - .best_compression, - => return self.fillDeflate(b), - } - } - - fn init( - allocator: Allocator, - in_writer: WriterType, - options: CompressorOptions, - ) !Self { - var s = Self{ - .allocator = undefined, - .compression = undefined, - .compression_level = undefined, - .hm_bw = undefined, // HuffmanBitWriter - .bulk_hasher = undefined, - .sync = false, - .best_speed_enc = undefined, // Best speed encoder - .chain_head = 0, - .hash_head = undefined, - .hash_prev = undefined, // previous hash - .hash_offset = 0, - .index = 0, - .window = undefined, - .window_end = 0, - .block_start = 0, - .byte_available = false, - .tokens = undefined, - .tokens_count = 0, - .length = 0, - .offset = 0, - .hash = 0, - .max_insert_index = 0, - .err = false, // Error - .hash_match = undefined, - .dictionary = options.dictionary, - }; - - s.hm_bw = try hm_bw.huffmanBitWriter(allocator, in_writer); - s.allocator = allocator; - - s.hash_head = try allocator.alloc(u32, hash_size); - s.hash_prev = try allocator.alloc(u32, window_size); - s.hash_match = try allocator.alloc(u32, max_match_length - 1); - @memset(s.hash_head, 0); - @memset(s.hash_prev, 0); - @memset(s.hash_match, 0); - - switch (options.level) { - .no_compression => { - s.compression = options.level; - s.compression_level = levels(options.level); - s.window = try allocator.alloc(u8, max_store_block_size); - s.tokens = try allocator.alloc(token.Token, 0); - }, - .huffman_only => { - s.compression = options.level; - s.compression_level = levels(options.level); - s.window = try allocator.alloc(u8, max_store_block_size); - s.tokens = try allocator.alloc(token.Token, 0); - }, - .best_speed => { - s.compression = options.level; - s.compression_level = levels(options.level); - s.window = try allocator.alloc(u8, max_store_block_size); - s.tokens = try allocator.alloc(token.Token, max_store_block_size); - s.best_speed_enc = try allocator.create(fast.DeflateFast); - s.best_speed_enc.* = fast.deflateFast(); - try s.best_speed_enc.init(allocator); - }, - .default_compression => { - s.compression = .level_6; - s.compression_level = levels(.level_6); - try s.initDeflate(); - if (options.dictionary != null) { - s.fillWindow(options.dictionary.?); - } - }, - .level_2, - .level_3, - .level_4, - .level_5, - .level_6, - .level_7, - .level_8, - .best_compression, - => { - s.compression = options.level; - s.compression_level = levels(options.level); - try s.initDeflate(); - if (options.dictionary != null) { - s.fillWindow(options.dictionary.?); - } - }, - } - return s; - } - - /// Release all allocated memory. - pub fn deinit(self: *Self) void { - self.hm_bw.deinit(); - self.allocator.free(self.window); - self.allocator.free(self.tokens); - self.allocator.free(self.hash_head); - self.allocator.free(self.hash_prev); - self.allocator.free(self.hash_match); - if (self.compression == .best_speed) { - self.best_speed_enc.deinit(); - self.allocator.destroy(self.best_speed_enc); - } - } - - /// Reset discards the inner writer's state and replace the inner writer with new_writer. - /// new_writer must be of the same type as the previous writer. - pub fn reset(self: *Self, new_writer: WriterType) void { - self.hm_bw.reset(new_writer); - self.sync = false; - switch (self.compression) { - // Reset window - .no_compression => self.window_end = 0, - // Reset window, tokens, and encoder - .best_speed => { - self.window_end = 0; - self.tokens_count = 0; - self.best_speed_enc.reset(); - }, - // Reset everything and reinclude the dictionary if there is one - .huffman_only, - .default_compression, - .level_2, - .level_3, - .level_4, - .level_5, - .level_6, - .level_7, - .level_8, - .best_compression, - => { - self.chain_head = 0; - @memset(self.hash_head, 0); - @memset(self.hash_prev, 0); - self.hash_offset = 1; - self.index = 0; - self.window_end = 0; - self.block_start = 0; - self.byte_available = false; - self.tokens_count = 0; - self.length = min_match_length - 1; - self.offset = 0; - self.hash = 0; - self.max_insert_index = 0; - - if (self.dictionary != null) { - self.fillWindow(self.dictionary.?); - } - }, - } - } - - /// Writes any pending data to the underlying writer. - pub fn close(self: *Self) Error!void { - self.sync = true; - try self.step(); - try self.hm_bw.writeStoredHeader(0, true); - try self.hm_bw.flush(); - return; - } - }; -} - -// tests - -const expect = std.testing.expect; -const testing = std.testing; - -const ArrayList = std.ArrayList; - -const DeflateTest = struct { - in: []const u8, - level: Compression, - out: []const u8, -}; - -var deflate_tests = [_]DeflateTest{ - // Level 0 - .{ - .in = &[_]u8{}, - .level = .no_compression, - .out = &[_]u8{ 1, 0, 0, 255, 255 }, - }, - - // Level -1 - .{ - .in = &[_]u8{0x11}, - .level = .default_compression, - .out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 }, - }, - .{ - .in = &[_]u8{0x11}, - .level = .level_6, - .out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 }, - }, - - // Level 4 - .{ - .in = &[_]u8{0x11}, - .level = .level_4, - .out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 }, - }, - - // Level 0 - .{ - .in = &[_]u8{0x11}, - .level = .no_compression, - .out = &[_]u8{ 0, 1, 0, 254, 255, 17, 1, 0, 0, 255, 255 }, - }, - .{ - .in = &[_]u8{ 0x11, 0x12 }, - .level = .no_compression, - .out = &[_]u8{ 0, 2, 0, 253, 255, 17, 18, 1, 0, 0, 255, 255 }, - }, - .{ - .in = &[_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 }, - .level = .no_compression, - .out = &[_]u8{ 0, 8, 0, 247, 255, 17, 17, 17, 17, 17, 17, 17, 17, 1, 0, 0, 255, 255 }, - }, - - // Level 2 - .{ - .in = &[_]u8{}, - .level = .level_2, - .out = &[_]u8{ 1, 0, 0, 255, 255 }, - }, - .{ - .in = &[_]u8{0x11}, - .level = .level_2, - .out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 }, - }, - .{ - .in = &[_]u8{ 0x11, 0x12 }, - .level = .level_2, - .out = &[_]u8{ 18, 20, 2, 4, 0, 0, 255, 255 }, - }, - .{ - .in = &[_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 }, - .level = .level_2, - .out = &[_]u8{ 18, 132, 2, 64, 0, 0, 0, 255, 255 }, - }, - - // Level 9 - .{ - .in = &[_]u8{}, - .level = .best_compression, - .out = &[_]u8{ 1, 0, 0, 255, 255 }, - }, - .{ - .in = &[_]u8{0x11}, - .level = .best_compression, - .out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 }, - }, - .{ - .in = &[_]u8{ 0x11, 0x12 }, - .level = .best_compression, - .out = &[_]u8{ 18, 20, 2, 4, 0, 0, 255, 255 }, - }, - .{ - .in = &[_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 }, - .level = .best_compression, - .out = &[_]u8{ 18, 132, 2, 64, 0, 0, 0, 255, 255 }, - }, -}; - -test "deflate" { - for (deflate_tests) |dt| { - var output = ArrayList(u8).init(testing.allocator); - defer output.deinit(); - - var comp = try compressor(testing.allocator, output.writer(), .{ .level = dt.level }); - _ = try comp.write(dt.in); - try comp.close(); - comp.deinit(); - - try testing.expectEqualSlices(u8, dt.out, output.items); - } -} - -test "bulkHash4" { - for (deflate_tests) |x| { - if (x.out.len < min_match_length) { - continue; - } - // double the test data - var out = try testing.allocator.alloc(u8, x.out.len * 2); - defer testing.allocator.free(out); - @memcpy(out[0..x.out.len], x.out); - @memcpy(out[x.out.len..], x.out); - - var j: usize = 4; - while (j < out.len) : (j += 1) { - var y = out[0..j]; - - const dst = try testing.allocator.alloc(u32, y.len - min_match_length + 1); - defer testing.allocator.free(dst); - - _ = bulkHash4(y, dst); - for (dst, 0..) |got, i| { - const want = hash4(y[i..]); - try testing.expectEqual(want, got); - } - } - } -} diff --git a/lib/std/compress/deflate/compressor_test.zig b/lib/std/compress/deflate/compressor_test.zig deleted file mode 100644 index f7f5b34a9aa3..000000000000 --- a/lib/std/compress/deflate/compressor_test.zig +++ /dev/null @@ -1,531 +0,0 @@ -const std = @import("std"); -const expect = std.testing.expect; -const fifo = std.fifo; -const io = std.io; -const math = std.math; -const mem = std.mem; -const testing = std.testing; - -const ArrayList = std.ArrayList; - -const deflate = @import("compressor.zig"); -const inflate = @import("decompressor.zig"); - -const compressor = deflate.compressor; -const decompressor = inflate.decompressor; -const huffman_only = deflate.huffman_only; - -fn testSync(level: deflate.Compression, input: []const u8) !void { - if (input.len == 0) { - return; - } - - var divided_buf = fifo - .LinearFifo(u8, fifo.LinearFifoBufferType.Dynamic) - .init(testing.allocator); - defer divided_buf.deinit(); - var whole_buf = std.ArrayList(u8).init(testing.allocator); - defer whole_buf.deinit(); - - const multi_writer = io.multiWriter(.{ - divided_buf.writer(), - whole_buf.writer(), - }).writer(); - - var comp = try compressor( - testing.allocator, - multi_writer, - .{ .level = level }, - ); - defer comp.deinit(); - - { - var decomp = try decompressor( - testing.allocator, - divided_buf.reader(), - null, - ); - defer decomp.deinit(); - - // Write first half of the input and flush() - const half: usize = (input.len + 1) / 2; - var half_len: usize = half - 0; - { - _ = try comp.writer().writeAll(input[0..half]); - - // Flush - try comp.flush(); - - // Read back - const decompressed = try testing.allocator.alloc(u8, half_len); - defer testing.allocator.free(decompressed); - - const read = try decomp.reader().readAll(decompressed); // read at least half - try testing.expectEqual(half_len, read); - try testing.expectEqualSlices(u8, input[0..half], decompressed); - } - - // Write last half of the input and close() - half_len = input.len - half; - { - _ = try comp.writer().writeAll(input[half..]); - - // Close - try comp.close(); - - // Read back - const decompressed = try testing.allocator.alloc(u8, half_len); - defer testing.allocator.free(decompressed); - - var read = try decomp.reader().readAll(decompressed); - try testing.expectEqual(half_len, read); - try testing.expectEqualSlices(u8, input[half..], decompressed); - - // Extra read - var final: [10]u8 = undefined; - read = try decomp.reader().readAll(&final); - try testing.expectEqual(@as(usize, 0), read); // expect ended stream to return 0 bytes - - try decomp.close(); - } - } - - _ = try comp.writer().writeAll(input); - try comp.close(); - - // stream should work for ordinary reader too (reading whole_buf in one go) - const whole_buf_reader = io.fixedBufferStream(whole_buf.items).reader(); - var decomp = try decompressor(testing.allocator, whole_buf_reader, null); - defer decomp.deinit(); - - const decompressed = try testing.allocator.alloc(u8, input.len); - defer testing.allocator.free(decompressed); - - _ = try decomp.reader().readAll(decompressed); - try decomp.close(); - - try testing.expectEqualSlices(u8, input, decompressed); -} - -fn testToFromWithLevelAndLimit(level: deflate.Compression, input: []const u8, limit: u32) !void { - var compressed = std.ArrayList(u8).init(testing.allocator); - defer compressed.deinit(); - - var comp = try compressor(testing.allocator, compressed.writer(), .{ .level = level }); - defer comp.deinit(); - - try comp.writer().writeAll(input); - try comp.close(); - - if (limit > 0) { - try expect(compressed.items.len <= limit); - } - - var fib = io.fixedBufferStream(compressed.items); - var decomp = try decompressor(testing.allocator, fib.reader(), null); - defer decomp.deinit(); - - const decompressed = try testing.allocator.alloc(u8, input.len); - defer testing.allocator.free(decompressed); - - const read: usize = try decomp.reader().readAll(decompressed); - try testing.expectEqual(input.len, read); - try testing.expectEqualSlices(u8, input, decompressed); - - if (false) { - // TODO: this test has regressed - try testSync(level, input); - } -} - -fn testToFromWithLimit(input: []const u8, limit: [11]u32) !void { - try testToFromWithLevelAndLimit(.no_compression, input, limit[0]); - try testToFromWithLevelAndLimit(.best_speed, input, limit[1]); - try testToFromWithLevelAndLimit(.level_2, input, limit[2]); - try testToFromWithLevelAndLimit(.level_3, input, limit[3]); - try testToFromWithLevelAndLimit(.level_4, input, limit[4]); - try testToFromWithLevelAndLimit(.level_5, input, limit[5]); - try testToFromWithLevelAndLimit(.level_6, input, limit[6]); - try testToFromWithLevelAndLimit(.level_7, input, limit[7]); - try testToFromWithLevelAndLimit(.level_8, input, limit[8]); - try testToFromWithLevelAndLimit(.best_compression, input, limit[9]); - try testToFromWithLevelAndLimit(.huffman_only, input, limit[10]); -} - -test "deflate/inflate" { - const limits = [_]u32{0} ** 11; - - var test0 = [_]u8{}; - var test1 = [_]u8{0x11}; - var test2 = [_]u8{ 0x11, 0x12 }; - var test3 = [_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 }; - var test4 = [_]u8{ 0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13 }; - - try testToFromWithLimit(&test0, limits); - try testToFromWithLimit(&test1, limits); - try testToFromWithLimit(&test2, limits); - try testToFromWithLimit(&test3, limits); - try testToFromWithLimit(&test4, limits); - - var large_data_chunk = try testing.allocator.alloc(u8, 100_000); - defer testing.allocator.free(large_data_chunk); - // fill with random data - for (large_data_chunk, 0..) |_, i| { - large_data_chunk[i] = @as(u8, @truncate(i)) *% @as(u8, @truncate(i)); - } - try testToFromWithLimit(large_data_chunk, limits); -} - -test "very long sparse chunk" { - // A SparseReader returns a stream consisting of 0s ending with 65,536 (1<<16) 1s. - // This tests missing hash references in a very large input. - const SparseReader = struct { - l: usize, // length - cur: usize, // current position - - const Self = @This(); - const Error = error{}; - - pub const Reader = io.Reader(*Self, Error, read); - - pub fn reader(self: *Self) Reader { - return .{ .context = self }; - } - - fn read(s: *Self, b: []u8) Error!usize { - var n: usize = 0; // amount read - - if (s.cur >= s.l) { - return 0; - } - n = b.len; - var cur = s.cur + n; - if (cur > s.l) { - n -= cur - s.l; - cur = s.l; - } - for (b[0..n], 0..) |_, i| { - if (s.cur + i >= s.l -| (1 << 16)) { - b[i] = 1; - } else { - b[i] = 0; - } - } - s.cur = cur; - return n; - } - }; - - var comp = try compressor( - testing.allocator, - io.null_writer, - .{ .level = .best_speed }, - ); - defer comp.deinit(); - var writer = comp.writer(); - - var sparse = SparseReader{ .l = 0x23e8, .cur = 0 }; - var reader = sparse.reader(); - - var read: usize = 1; - var written: usize = 0; - while (read > 0) { - var buf: [1 << 15]u8 = undefined; // 32,768 bytes buffer - read = try reader.read(&buf); - written += try writer.write(buf[0..read]); - } - try testing.expectEqual(@as(usize, 0x23e8), written); -} - -test "compressor reset" { - for (std.enums.values(deflate.Compression)) |c| { - try testWriterReset(c, null); - try testWriterReset(c, "dict"); - try testWriterReset(c, "hello"); - } -} - -fn testWriterReset(level: deflate.Compression, dict: ?[]const u8) !void { - const filler = struct { - fn writeData(c: anytype) !void { - const msg = "all your base are belong to us"; - try c.writer().writeAll(msg); - try c.flush(); - - const hello = "hello world"; - var i: usize = 0; - while (i < 1024) : (i += 1) { - try c.writer().writeAll(hello); - } - - i = 0; - while (i < 65000) : (i += 1) { - try c.writer().writeAll("x"); - } - } - }; - - var buf1 = ArrayList(u8).init(testing.allocator); - defer buf1.deinit(); - var buf2 = ArrayList(u8).init(testing.allocator); - defer buf2.deinit(); - - var comp = try compressor( - testing.allocator, - buf1.writer(), - .{ .level = level, .dictionary = dict }, - ); - defer comp.deinit(); - - try filler.writeData(&comp); - try comp.close(); - - comp.reset(buf2.writer()); - try filler.writeData(&comp); - try comp.close(); - - try testing.expectEqualSlices(u8, buf1.items, buf2.items); -} - -test "decompressor dictionary" { - const dict = "hello world"; // dictionary - const text = "hello again world"; - - var compressed = fifo - .LinearFifo(u8, fifo.LinearFifoBufferType.Dynamic) - .init(testing.allocator); - defer compressed.deinit(); - - var comp = try compressor( - testing.allocator, - compressed.writer(), - .{ - .level = .level_5, - .dictionary = null, // no dictionary - }, - ); - defer comp.deinit(); - - // imitate a compressor with a dictionary - try comp.writer().writeAll(dict); - try comp.flush(); - compressed.discard(compressed.readableLength()); // empty the output - try comp.writer().writeAll(text); - try comp.close(); - - const decompressed = try testing.allocator.alloc(u8, text.len); - defer testing.allocator.free(decompressed); - - var decomp = try decompressor( - testing.allocator, - compressed.reader(), - dict, - ); - defer decomp.deinit(); - - _ = try decomp.reader().readAll(decompressed); - try testing.expectEqualSlices(u8, "hello again world", decompressed); -} - -test "compressor dictionary" { - const dict = "hello world"; - const text = "hello again world"; - - var compressed_nd = fifo - .LinearFifo(u8, fifo.LinearFifoBufferType.Dynamic) - .init(testing.allocator); // compressed with no dictionary - defer compressed_nd.deinit(); - - var compressed_d = ArrayList(u8).init(testing.allocator); // compressed with a dictionary - defer compressed_d.deinit(); - - // imitate a compressor with a dictionary - var comp_nd = try compressor( - testing.allocator, - compressed_nd.writer(), - .{ - .level = .level_5, - .dictionary = null, // no dictionary - }, - ); - defer comp_nd.deinit(); - try comp_nd.writer().writeAll(dict); - try comp_nd.flush(); - compressed_nd.discard(compressed_nd.readableLength()); // empty the output - try comp_nd.writer().writeAll(text); - try comp_nd.close(); - - // use a compressor with a dictionary - var comp_d = try compressor( - testing.allocator, - compressed_d.writer(), - .{ - .level = .level_5, - .dictionary = dict, // with a dictionary - }, - ); - defer comp_d.deinit(); - try comp_d.writer().writeAll(text); - try comp_d.close(); - - try testing.expectEqualSlices(u8, compressed_d.items, compressed_nd.readableSlice(0)); -} - -// Update the hash for best_speed only if d.index < d.maxInsertIndex -// See https://golang.org/issue/2508 -test "Go non-regression test for 2508" { - var comp = try compressor( - testing.allocator, - io.null_writer, - .{ .level = .best_speed }, - ); - defer comp.deinit(); - - var buf = [_]u8{0} ** 1024; - - var i: usize = 0; - while (i < 131_072) : (i += 1) { - try comp.writer().writeAll(&buf); - try comp.close(); - } -} - -test "deflate/inflate string" { - const StringTest = struct { - filename: []const u8, - limit: [11]u32, - }; - - const deflate_inflate_string_tests = [_]StringTest{ - .{ - .filename = "compress-e.txt", - .limit = [11]u32{ - 100_018, // no_compression - 50_650, // best_speed - 50_960, // 2 - 51_150, // 3 - 50_930, // 4 - 50_790, // 5 - 50_790, // 6 - 50_790, // 7 - 50_790, // 8 - 50_790, // best_compression - 43_683, // huffman_only - }, - }, - .{ - .filename = "rfc1951.txt", - .limit = [11]u32{ - 36_954, // no_compression - 12_952, // best_speed - 12_228, // 2 - 12_016, // 3 - 11_466, // 4 - 11_191, // 5 - 11_129, // 6 - 11_120, // 7 - 11_112, // 8 - 11_109, // best_compression - 20_273, // huffman_only - }, - }, - }; - - inline for (deflate_inflate_string_tests) |t| { - const golden = @embedFile("testdata/" ++ t.filename); - try testToFromWithLimit(golden, t.limit); - } -} - -test "inflate reset" { - const strings = [_][]const u8{ - "lorem ipsum izzle fo rizzle", - "the quick brown fox jumped over", - }; - - var compressed_strings = [_]ArrayList(u8){ - ArrayList(u8).init(testing.allocator), - ArrayList(u8).init(testing.allocator), - }; - defer compressed_strings[0].deinit(); - defer compressed_strings[1].deinit(); - - for (strings, 0..) |s, i| { - var comp = try compressor( - testing.allocator, - compressed_strings[i].writer(), - .{ .level = .level_6 }, - ); - defer comp.deinit(); - - try comp.writer().writeAll(s); - try comp.close(); - } - - var fib = io.fixedBufferStream(compressed_strings[0].items); - var decomp = try decompressor(testing.allocator, fib.reader(), null); - defer decomp.deinit(); - - const decompressed_0: []u8 = try decomp.reader() - .readAllAlloc(testing.allocator, math.maxInt(usize)); - defer testing.allocator.free(decompressed_0); - - fib = io.fixedBufferStream(compressed_strings[1].items); - try decomp.reset(fib.reader(), null); - - const decompressed_1: []u8 = try decomp.reader() - .readAllAlloc(testing.allocator, math.maxInt(usize)); - defer testing.allocator.free(decompressed_1); - - try decomp.close(); - - try testing.expectEqualSlices(u8, strings[0], decompressed_0); - try testing.expectEqualSlices(u8, strings[1], decompressed_1); -} - -test "inflate reset dictionary" { - const dict = "the lorem fox"; - const strings = [_][]const u8{ - "lorem ipsum izzle fo rizzle", - "the quick brown fox jumped over", - }; - - var compressed_strings = [_]ArrayList(u8){ - ArrayList(u8).init(testing.allocator), - ArrayList(u8).init(testing.allocator), - }; - defer compressed_strings[0].deinit(); - defer compressed_strings[1].deinit(); - - for (strings, 0..) |s, i| { - var comp = try compressor( - testing.allocator, - compressed_strings[i].writer(), - .{ .level = .level_6 }, - ); - defer comp.deinit(); - - try comp.writer().writeAll(s); - try comp.close(); - } - - var fib = io.fixedBufferStream(compressed_strings[0].items); - var decomp = try decompressor(testing.allocator, fib.reader(), dict); - defer decomp.deinit(); - - const decompressed_0: []u8 = try decomp.reader() - .readAllAlloc(testing.allocator, math.maxInt(usize)); - defer testing.allocator.free(decompressed_0); - - fib = io.fixedBufferStream(compressed_strings[1].items); - try decomp.reset(fib.reader(), dict); - - const decompressed_1: []u8 = try decomp.reader() - .readAllAlloc(testing.allocator, math.maxInt(usize)); - defer testing.allocator.free(decompressed_1); - - try decomp.close(); - - try testing.expectEqualSlices(u8, strings[0], decompressed_0); - try testing.expectEqualSlices(u8, strings[1], decompressed_1); -} diff --git a/lib/std/compress/deflate/decompressor.zig b/lib/std/compress/deflate/decompressor.zig deleted file mode 100644 index 896f931a663d..000000000000 --- a/lib/std/compress/deflate/decompressor.zig +++ /dev/null @@ -1,1119 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const math = std.math; -const mem = std.mem; - -const Allocator = std.mem.Allocator; -const ArrayList = std.ArrayList; - -const bu = @import("bits_utils.zig"); -const ddec = @import("dict_decoder.zig"); -const deflate_const = @import("deflate_const.zig"); - -const max_match_offset = deflate_const.max_match_offset; -const end_block_marker = deflate_const.end_block_marker; - -const max_code_len = 16; // max length of Huffman code -// The next three numbers come from the RFC section 3.2.7, with the -// additional proviso in section 3.2.5 which implies that distance codes -// 30 and 31 should never occur in compressed data. -const max_num_lit = 286; -const max_num_dist = 30; -const num_codes = 19; // number of codes in Huffman meta-code - -var corrupt_input_error_offset: u64 = undefined; - -const InflateError = error{ - CorruptInput, // A CorruptInput error reports the presence of corrupt input at a given offset. - BadInternalState, // An BadInternalState reports an error in the flate code itself. - BadReaderState, // An error was encountered while accessing the inner reader - UnexpectedEndOfStream, - EndOfStreamWithNoError, -}; - -// The data structure for decoding Huffman tables is based on that of -// zlib. There is a lookup table of a fixed bit width (huffman_chunk_bits), -// For codes smaller than the table width, there are multiple entries -// (each combination of trailing bits has the same value). For codes -// larger than the table width, the table contains a link to an overflow -// table. The width of each entry in the link table is the maximum code -// size minus the chunk width. -// -// Note that you can do a lookup in the table even without all bits -// filled. Since the extra bits are zero, and the DEFLATE Huffman codes -// have the property that shorter codes come before longer ones, the -// bit length estimate in the result is a lower bound on the actual -// number of bits. -// -// See the following: -// https://github.com/madler/zlib/raw/master/doc/algorithm.txt - -// chunk & 15 is number of bits -// chunk >> 4 is value, including table link - -const huffman_chunk_bits = 9; -const huffman_num_chunks = 1 << huffman_chunk_bits; // 512 -const huffman_count_mask = 15; // 0b1111 -const huffman_value_shift = 4; - -const HuffmanDecoder = struct { - const Self = @This(); - - allocator: Allocator = undefined, - - min: u32 = 0, // the minimum code length - chunks: [huffman_num_chunks]u16 = [1]u16{0} ** huffman_num_chunks, // chunks as described above - links: [][]u16 = undefined, // overflow links - link_mask: u32 = 0, // mask the width of the link table - initialized: bool = false, - sub_chunks: ArrayList(u32) = undefined, - - // Initialize Huffman decoding tables from array of code lengths. - // Following this function, self is guaranteed to be initialized into a complete - // tree (i.e., neither over-subscribed nor under-subscribed). The exception is a - // degenerate case where the tree has only a single symbol with length 1. Empty - // trees are permitted. - fn init(self: *Self, allocator: Allocator, lengths: []u32) !bool { - - // Sanity enables additional runtime tests during Huffman - // table construction. It's intended to be used during - // development and debugging - const sanity = false; - - if (self.min != 0) { - self.* = HuffmanDecoder{}; - } - - self.allocator = allocator; - - // Count number of codes of each length, - // compute min and max length. - var count: [max_code_len]u32 = [1]u32{0} ** max_code_len; - var min: u32 = 0; - var max: u32 = 0; - for (lengths) |n| { - if (n == 0) { - continue; - } - if (min == 0) { - min = n; - } - min = @min(n, min); - max = @max(n, max); - count[n] += 1; - } - - // Empty tree. The decompressor.huffSym function will fail later if the tree - // is used. Technically, an empty tree is only valid for the HDIST tree and - // not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree - // is guaranteed to fail since it will attempt to use the tree to decode the - // codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is - // guaranteed to fail later since the compressed data section must be - // composed of at least one symbol (the end-of-block marker). - if (max == 0) { - return true; - } - - var next_code: [max_code_len]u32 = [1]u32{0} ** max_code_len; - var code: u32 = 0; - { - var i = min; - while (i <= max) : (i += 1) { - code <<= 1; - next_code[i] = code; - code += count[i]; - } - } - - // Check that the coding is complete (i.e., that we've - // assigned all 2-to-the-max possible bit sequences). - // Exception: To be compatible with zlib, we also need to - // accept degenerate single-code codings. See also - // TestDegenerateHuffmanCoding. - if (code != @as(u32, 1) << @as(u5, @intCast(max)) and !(code == 1 and max == 1)) { - return false; - } - - self.min = min; - if (max > huffman_chunk_bits) { - const num_links = @as(u32, 1) << @as(u5, @intCast(max - huffman_chunk_bits)); - self.link_mask = @as(u32, @intCast(num_links - 1)); - - // create link tables - const link = next_code[huffman_chunk_bits + 1] >> 1; - self.links = try self.allocator.alloc([]u16, huffman_num_chunks - link); - self.sub_chunks = ArrayList(u32).init(self.allocator); - self.initialized = true; - var j = @as(u32, @intCast(link)); - while (j < huffman_num_chunks) : (j += 1) { - var reverse = @as(u32, @intCast(bu.bitReverse(u16, @as(u16, @intCast(j)), 16))); - reverse >>= @as(u32, @intCast(16 - huffman_chunk_bits)); - const off = j - @as(u32, @intCast(link)); - if (sanity) { - // check we are not overwriting an existing chunk - assert(self.chunks[reverse] == 0); - } - self.chunks[reverse] = @as(u16, @intCast(off << huffman_value_shift | (huffman_chunk_bits + 1))); - self.links[off] = try self.allocator.alloc(u16, num_links); - if (sanity) { - // initialize to a known invalid chunk code (0) to see if we overwrite - // this value later on - @memset(self.links[off], 0); - } - try self.sub_chunks.append(off); - } - } - - for (lengths, 0..) |n, li| { - if (n == 0) { - continue; - } - const ncode = next_code[n]; - next_code[n] += 1; - const chunk = @as(u16, @intCast((li << huffman_value_shift) | n)); - var reverse = @as(u16, @intCast(bu.bitReverse(u16, @as(u16, @intCast(ncode)), 16))); - reverse >>= @as(u4, @intCast(16 - n)); - if (n <= huffman_chunk_bits) { - var off = reverse; - while (off < self.chunks.len) : (off += @as(u16, 1) << @as(u4, @intCast(n))) { - // We should never need to overwrite - // an existing chunk. Also, 0 is - // never a valid chunk, because the - // lower 4 "count" bits should be - // between 1 and 15. - if (sanity) { - assert(self.chunks[off] == 0); - } - self.chunks[off] = chunk; - } - } else { - const j = reverse & (huffman_num_chunks - 1); - if (sanity) { - // Expect an indirect chunk - assert(self.chunks[j] & huffman_count_mask == huffman_chunk_bits + 1); - // Longer codes should have been - // associated with a link table above. - } - const value = self.chunks[j] >> huffman_value_shift; - var link_tab = self.links[value]; - reverse >>= huffman_chunk_bits; - var off = reverse; - while (off < link_tab.len) : (off += @as(u16, 1) << @as(u4, @intCast(n - huffman_chunk_bits))) { - if (sanity) { - // check we are not overwriting an existing chunk - assert(link_tab[off] == 0); - } - link_tab[off] = @as(u16, @intCast(chunk)); - } - } - } - - if (sanity) { - // Above we've sanity checked that we never overwrote - // an existing entry. Here we additionally check that - // we filled the tables completely. - for (self.chunks, 0..) |chunk, i| { - // As an exception, in the degenerate - // single-code case, we allow odd - // chunks to be missing. - if (code == 1 and i % 2 == 1) { - continue; - } - - // Assert we are not missing a chunk. - // All chunks should have been written once - // thus losing their initial value of 0 - assert(chunk != 0); - } - - if (self.initialized) { - for (self.links) |link_tab| { - for (link_tab) |chunk| { - // Assert we are not missing a chunk. - assert(chunk != 0); - } - } - } - } - - return true; - } - - /// Release all allocated memory. - pub fn deinit(self: *Self) void { - if (self.initialized and self.links.len > 0) { - for (self.sub_chunks.items) |off| { - self.allocator.free(self.links[off]); - } - self.allocator.free(self.links); - self.sub_chunks.deinit(); - self.initialized = false; - } - } -}; - -var fixed_huffman_decoder: ?HuffmanDecoder = null; - -fn fixedHuffmanDecoderInit(allocator: Allocator) !HuffmanDecoder { - if (fixed_huffman_decoder != null) { - return fixed_huffman_decoder.?; - } - - // These come from the RFC section 3.2.6. - var bits: [288]u32 = undefined; - var i: u32 = 0; - while (i < 144) : (i += 1) { - bits[i] = 8; - } - while (i < 256) : (i += 1) { - bits[i] = 9; - } - while (i < 280) : (i += 1) { - bits[i] = 7; - } - while (i < 288) : (i += 1) { - bits[i] = 8; - } - - fixed_huffman_decoder = HuffmanDecoder{}; - _ = try fixed_huffman_decoder.?.init(allocator, &bits); - return fixed_huffman_decoder.?; -} - -const DecompressorState = enum { - init, - dict, -}; - -/// Returns a new Decompressor that can be used to read the uncompressed version of `reader`. -/// `dictionary` is optional and initializes the Decompressor with a preset dictionary. -/// The returned Decompressor behaves as if the uncompressed data stream started with the given -/// dictionary, which has already been read. Use the same `dictionary` as the compressor used to -/// compress the data. -/// This decompressor may use at most 300 KiB of heap memory from the provided allocator. -/// The uncompressed data will be written into the provided buffer, see `reader()` and `read()`. -pub fn decompressor(allocator: Allocator, reader: anytype, dictionary: ?[]const u8) !Decompressor(@TypeOf(reader)) { - return Decompressor(@TypeOf(reader)).init(allocator, reader, dictionary); -} - -pub fn Decompressor(comptime ReaderType: type) type { - return struct { - const Self = @This(); - - pub const Error = - ReaderType.Error || - error{EndOfStream} || - InflateError || - Allocator.Error; - pub const Reader = io.Reader(*Self, Error, read); - - allocator: Allocator, - - // Input source. - inner_reader: ReaderType, - roffset: u64, - - // Input bits, in top of b. - b: u32, - nb: u32, - - // Huffman decoders for literal/length, distance. - hd1: HuffmanDecoder, - hd2: HuffmanDecoder, - - // Length arrays used to define Huffman codes. - bits: *[max_num_lit + max_num_dist]u32, - codebits: *[num_codes]u32, - - // Output history, buffer. - dict: ddec.DictDecoder, - - // Temporary buffer (avoids repeated allocation). - buf: [4]u8, - - // Next step in the decompression, - // and decompression state. - step: *const fn (*Self) Error!void, - step_state: DecompressorState, - final: bool, - err: ?Error, - to_read: []u8, - // Huffman states for the lit/length values - hl: ?*HuffmanDecoder, - // Huffman states for the distance values. - hd: ?*HuffmanDecoder, - copy_len: u32, - copy_dist: u32, - - /// Returns a Reader that reads compressed data from an underlying reader and outputs - /// uncompressed data. - pub fn reader(self: *Self) Reader { - return .{ .context = self }; - } - - fn init(allocator: Allocator, in_reader: ReaderType, dict: ?[]const u8) !Self { - fixed_huffman_decoder = try fixedHuffmanDecoderInit(allocator); - - const bits = try allocator.create([max_num_lit + max_num_dist]u32); - const codebits = try allocator.create([num_codes]u32); - - var dd = ddec.DictDecoder{}; - try dd.init(allocator, max_match_offset, dict); - - return Self{ - .allocator = allocator, - - // Input source. - .inner_reader = in_reader, - .roffset = 0, - - // Input bits, in top of b. - .b = 0, - .nb = 0, - - // Huffman decoders for literal/length, distance. - .hd1 = HuffmanDecoder{}, - .hd2 = HuffmanDecoder{}, - - // Length arrays used to define Huffman codes. - .bits = bits, - .codebits = codebits, - - // Output history, buffer. - .dict = dd, - - // Temporary buffer (avoids repeated allocation). - .buf = [_]u8{0} ** 4, - - // Next step in the decompression and decompression state. - .step = nextBlock, - .step_state = .init, - .final = false, - .err = null, - .to_read = &[0]u8{}, - .hl = null, - .hd = null, - .copy_len = 0, - .copy_dist = 0, - }; - } - - /// Release all allocated memory. - pub fn deinit(self: *Self) void { - self.hd2.deinit(); - self.hd1.deinit(); - self.dict.deinit(); - self.allocator.destroy(self.codebits); - self.allocator.destroy(self.bits); - } - - fn nextBlock(self: *Self) Error!void { - while (self.nb < 1 + 2) { - self.moreBits() catch |e| { - self.err = e; - return e; - }; - } - self.final = self.b & 1 == 1; - self.b >>= 1; - const typ = self.b & 3; - self.b >>= 2; - self.nb -= 1 + 2; - switch (typ) { - 0 => try self.dataBlock(), - 1 => { - // compressed, fixed Huffman tables - self.hl = &fixed_huffman_decoder.?; - self.hd = null; - try self.huffmanBlock(); - }, - 2 => { - // compressed, dynamic Huffman tables - self.hd2.deinit(); - self.hd1.deinit(); - try self.readHuffman(); - self.hl = &self.hd1; - self.hd = &self.hd2; - try self.huffmanBlock(); - }, - else => { - // 3 is reserved. - corrupt_input_error_offset = self.roffset; - self.err = InflateError.CorruptInput; - return InflateError.CorruptInput; - }, - } - } - - /// Reads compressed data from the underlying reader and outputs uncompressed data into - /// `output`. - pub fn read(self: *Self, output: []u8) Error!usize { - while (true) { - if (self.to_read.len > 0) { - const n = std.compress.deflate.copy(output, self.to_read); - self.to_read = self.to_read[n..]; - if (self.to_read.len == 0 and - self.err != null) - { - if (self.err.? == InflateError.EndOfStreamWithNoError) { - return n; - } - return self.err.?; - } - return n; - } - if (self.err != null) { - if (self.err.? == InflateError.EndOfStreamWithNoError) { - return 0; - } - return self.err.?; - } - self.step(self) catch |e| { - self.err = e; - if (self.to_read.len == 0) { - self.to_read = self.dict.readFlush(); // Flush what's left in case of error - } - }; - } - } - - pub fn close(self: *Self) Error!void { - if (self.err) |err| { - if (err != error.EndOfStreamWithNoError) return err; - } - } - - // RFC 1951 section 3.2.7. - // Compression with dynamic Huffman codes - - const code_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; - - fn readHuffman(self: *Self) Error!void { - // HLIT[5], HDIST[5], HCLEN[4]. - while (self.nb < 5 + 5 + 4) { - try self.moreBits(); - } - const nlit = @as(u32, @intCast(self.b & 0x1F)) + 257; - if (nlit > max_num_lit) { - corrupt_input_error_offset = self.roffset; - self.err = InflateError.CorruptInput; - return InflateError.CorruptInput; - } - self.b >>= 5; - const ndist = @as(u32, @intCast(self.b & 0x1F)) + 1; - if (ndist > max_num_dist) { - corrupt_input_error_offset = self.roffset; - self.err = InflateError.CorruptInput; - return InflateError.CorruptInput; - } - self.b >>= 5; - const nclen = @as(u32, @intCast(self.b & 0xF)) + 4; - // num_codes is 19, so nclen is always valid. - self.b >>= 4; - self.nb -= 5 + 5 + 4; - - // (HCLEN+4)*3 bits: code lengths in the magic code_order order. - var i: u32 = 0; - while (i < nclen) : (i += 1) { - while (self.nb < 3) { - try self.moreBits(); - } - self.codebits[code_order[i]] = @as(u32, @intCast(self.b & 0x7)); - self.b >>= 3; - self.nb -= 3; - } - i = nclen; - while (i < code_order.len) : (i += 1) { - self.codebits[code_order[i]] = 0; - } - if (!try self.hd1.init(self.allocator, self.codebits[0..])) { - corrupt_input_error_offset = self.roffset; - self.err = InflateError.CorruptInput; - return InflateError.CorruptInput; - } - - // HLIT + 257 code lengths, HDIST + 1 code lengths, - // using the code length Huffman code. - i = 0; - const n = nlit + ndist; - while (i < n) { - const x = try self.huffSym(&self.hd1); - if (x < 16) { - // Actual length. - self.bits[i] = x; - i += 1; - continue; - } - // Repeat previous length or zero. - var rep: u32 = 0; - var nb: u32 = 0; - var b: u32 = 0; - switch (x) { - 16 => { - rep = 3; - nb = 2; - if (i == 0) { - corrupt_input_error_offset = self.roffset; - self.err = InflateError.CorruptInput; - return InflateError.CorruptInput; - } - b = self.bits[i - 1]; - }, - 17 => { - rep = 3; - nb = 3; - b = 0; - }, - 18 => { - rep = 11; - nb = 7; - b = 0; - }, - else => return error.BadInternalState, // unexpected length code - } - while (self.nb < nb) { - try self.moreBits(); - } - rep += @as(u32, @intCast(self.b & (@as(u32, 1) << @as(u5, @intCast(nb))) - 1)); - self.b >>= @as(u5, @intCast(nb)); - self.nb -= nb; - if (i + rep > n) { - corrupt_input_error_offset = self.roffset; - self.err = InflateError.CorruptInput; - return InflateError.CorruptInput; - } - var j: u32 = 0; - while (j < rep) : (j += 1) { - self.bits[i] = b; - i += 1; - } - } - - if (!try self.hd1.init(self.allocator, self.bits[0..nlit]) or - !try self.hd2.init(self.allocator, self.bits[nlit..][0..ndist])) - { - corrupt_input_error_offset = self.roffset; - self.err = InflateError.CorruptInput; - return InflateError.CorruptInput; - } - - // As an optimization, we can initialize the min bits to read at a time - // for the HLIT tree to the length of the EOB marker since we know that - // every block must terminate with one. This preserves the property that - // we never read any extra bytes after the end of the DEFLATE stream. - if (self.hd1.min < self.bits[end_block_marker]) { - self.hd1.min = self.bits[end_block_marker]; - } - - return; - } - - // Decode a single Huffman block. - // hl and hd are the Huffman states for the lit/length values - // and the distance values, respectively. If hd == null, using the - // fixed distance encoding associated with fixed Huffman blocks. - fn huffmanBlock(self: *Self) Error!void { - while (true) { - switch (self.step_state) { - .init => { - // Read literal and/or (length, distance) according to RFC section 3.2.3. - const v = try self.huffSym(self.hl.?); - var n: u32 = 0; // number of bits extra - var length: u32 = 0; - switch (v) { - 0...255 => { - self.dict.writeByte(@as(u8, @intCast(v))); - if (self.dict.availWrite() == 0) { - self.to_read = self.dict.readFlush(); - self.step = huffmanBlock; - self.step_state = .init; - return; - } - self.step_state = .init; - continue; - }, - 256 => { - self.finishBlock(); - return; - }, - // otherwise, reference to older data - 257...264 => { - length = v - (257 - 3); - n = 0; - }, - 265...268 => { - length = v * 2 - (265 * 2 - 11); - n = 1; - }, - 269...272 => { - length = v * 4 - (269 * 4 - 19); - n = 2; - }, - 273...276 => { - length = v * 8 - (273 * 8 - 35); - n = 3; - }, - 277...280 => { - length = v * 16 - (277 * 16 - 67); - n = 4; - }, - 281...284 => { - length = v * 32 - (281 * 32 - 131); - n = 5; - }, - max_num_lit - 1 => { // 285 - length = 258; - n = 0; - }, - else => { - corrupt_input_error_offset = self.roffset; - self.err = InflateError.CorruptInput; - return InflateError.CorruptInput; - }, - } - if (n > 0) { - while (self.nb < n) { - try self.moreBits(); - } - length += @as(u32, @intCast(self.b)) & ((@as(u32, 1) << @as(u5, @intCast(n))) - 1); - self.b >>= @as(u5, @intCast(n)); - self.nb -= n; - } - - var dist: u32 = 0; - if (self.hd == null) { - while (self.nb < 5) { - try self.moreBits(); - } - dist = @as( - u32, - @intCast(bu.bitReverse(u8, @as(u8, @intCast((self.b & 0x1F) << 3)), 8)), - ); - self.b >>= 5; - self.nb -= 5; - } else { - dist = try self.huffSym(self.hd.?); - } - - switch (dist) { - 0...3 => dist += 1, - 4...max_num_dist - 1 => { // 4...29 - const nb = @as(u32, @intCast(dist - 2)) >> 1; - // have 1 bit in bottom of dist, need nb more. - var extra = (dist & 1) << @as(u5, @intCast(nb)); - while (self.nb < nb) { - try self.moreBits(); - } - extra |= @as(u32, @intCast(self.b & (@as(u32, 1) << @as(u5, @intCast(nb))) - 1)); - self.b >>= @as(u5, @intCast(nb)); - self.nb -= nb; - dist = (@as(u32, 1) << @as(u5, @intCast(nb + 1))) + 1 + extra; - }, - else => { - corrupt_input_error_offset = self.roffset; - self.err = InflateError.CorruptInput; - return InflateError.CorruptInput; - }, - } - - // No check on length; encoding can be prescient. - if (dist > self.dict.histSize()) { - corrupt_input_error_offset = self.roffset; - self.err = InflateError.CorruptInput; - return InflateError.CorruptInput; - } - - self.copy_len = length; - self.copy_dist = dist; - self.step_state = .dict; - }, - - .dict => { - // Perform a backwards copy according to RFC section 3.2.3. - var cnt = self.dict.tryWriteCopy(self.copy_dist, self.copy_len); - if (cnt == 0) { - cnt = self.dict.writeCopy(self.copy_dist, self.copy_len); - } - self.copy_len -= cnt; - - if (self.dict.availWrite() == 0 or self.copy_len > 0) { - self.to_read = self.dict.readFlush(); - self.step = huffmanBlock; // We need to continue this work - self.step_state = .dict; - return; - } - self.step_state = .init; - }, - } - } - } - - // Copy a single uncompressed data block from input to output. - fn dataBlock(self: *Self) Error!void { - // Uncompressed. - // Discard current half-byte. - self.nb = 0; - self.b = 0; - - // Length then ones-complement of length. - const nr: u32 = 4; - self.inner_reader.readNoEof(self.buf[0..nr]) catch { - self.err = InflateError.UnexpectedEndOfStream; - return InflateError.UnexpectedEndOfStream; - }; - self.roffset += @as(u64, @intCast(nr)); - const n = @as(u32, @intCast(self.buf[0])) | @as(u32, @intCast(self.buf[1])) << 8; - const nn = @as(u32, @intCast(self.buf[2])) | @as(u32, @intCast(self.buf[3])) << 8; - if (@as(u16, @intCast(nn)) != @as(u16, @truncate(~n))) { - corrupt_input_error_offset = self.roffset; - self.err = InflateError.CorruptInput; - return InflateError.CorruptInput; - } - - if (n == 0) { - self.to_read = self.dict.readFlush(); - self.finishBlock(); - return; - } - - self.copy_len = n; - try self.copyData(); - } - - // copyData copies self.copy_len bytes from the underlying reader into self.hist. - // It pauses for reads when self.hist is full. - fn copyData(self: *Self) Error!void { - var buf = self.dict.writeSlice(); - if (buf.len > self.copy_len) { - buf = buf[0..self.copy_len]; - } - - const cnt = try self.inner_reader.read(buf); - if (cnt < buf.len) { - self.err = InflateError.UnexpectedEndOfStream; - } - self.roffset += @as(u64, @intCast(cnt)); - self.copy_len -= @as(u32, @intCast(cnt)); - self.dict.writeMark(@as(u32, @intCast(cnt))); - if (self.err != null) { - return InflateError.UnexpectedEndOfStream; - } - - if (self.dict.availWrite() == 0 or self.copy_len > 0) { - self.to_read = self.dict.readFlush(); - self.step = copyData; - return; - } - self.finishBlock(); - } - - fn finishBlock(self: *Self) void { - if (self.final) { - if (self.dict.availRead() > 0) { - self.to_read = self.dict.readFlush(); - } - self.err = InflateError.EndOfStreamWithNoError; - } - self.step = nextBlock; - } - - fn moreBits(self: *Self) InflateError!void { - const c = self.inner_reader.readByte() catch |e| { - if (e == error.EndOfStream) { - return InflateError.UnexpectedEndOfStream; - } - return InflateError.BadReaderState; - }; - self.roffset += 1; - self.b |= @as(u32, c) << @as(u5, @intCast(self.nb)); - self.nb += 8; - return; - } - - // Read the next Huffman-encoded symbol according to h. - fn huffSym(self: *Self, h: *HuffmanDecoder) InflateError!u32 { - // Since a HuffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - var n: u32 = h.min; - // Optimization. Go compiler isn't smart enough to keep self.b, self.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b, nb back to self on return. - var nb = self.nb; - var b = self.b; - while (true) { - while (nb < n) { - const c = self.inner_reader.readByte() catch |e| { - self.b = b; - self.nb = nb; - if (e == error.EndOfStream) { - return error.UnexpectedEndOfStream; - } - return InflateError.BadReaderState; - }; - self.roffset += 1; - b |= @as(u32, @intCast(c)) << @as(u5, @intCast(nb & 31)); - nb += 8; - } - var chunk = h.chunks[b & (huffman_num_chunks - 1)]; - n = @as(u32, @intCast(chunk & huffman_count_mask)); - if (n > huffman_chunk_bits) { - chunk = h.links[chunk >> huffman_value_shift][(b >> huffman_chunk_bits) & h.link_mask]; - n = @as(u32, @intCast(chunk & huffman_count_mask)); - } - if (n <= nb) { - if (n == 0) { - self.b = b; - self.nb = nb; - corrupt_input_error_offset = self.roffset; - self.err = InflateError.CorruptInput; - return InflateError.CorruptInput; - } - self.b = b >> @as(u5, @intCast(n & 31)); - self.nb = nb - n; - return @as(u32, @intCast(chunk >> huffman_value_shift)); - } - } - } - - /// Replaces the inner reader and dictionary with new_reader and new_dict. - /// new_reader must be of the same type as the reader being replaced. - pub fn reset(s: *Self, new_reader: ReaderType, new_dict: ?[]const u8) Error!void { - s.inner_reader = new_reader; - s.step = nextBlock; - s.err = null; - s.nb = 0; - - s.dict.deinit(); - try s.dict.init(s.allocator, max_match_offset, new_dict); - - return; - } - }; -} - -// tests -const expectError = std.testing.expectError; -const io = std.io; -const testing = std.testing; - -test "confirm decompressor resets" { - var compressed = std.ArrayList(u8).init(std.testing.allocator); - defer compressed.deinit(); - - inline for (.{ - &[_]u8{ 0x5d, 0xc0, 0x21, 0x01, 0x00, 0x00, 0x00, 0x80, 0x20, 0xff, 0xaf, 0xa6, 0x4b, 0x03 }, - &[_]u8{ 0x55, 0xc1, 0x41, 0x0d, 0x00, 0x00, 0x00, 0x02, 0xa1, 0x94, 0x96, 0x34, 0x25, 0xef, 0x1b, 0x5f, 0x01 }, - }) |data| { - try compressed.writer().writeAll(data); - } - - var stream = std.io.fixedBufferStream(compressed.items); - var decomp = try decompressor(std.testing.allocator, stream.reader(), null); - defer decomp.deinit(); - - while (true) { - if (try stream.getPos() == try stream.getEndPos()) break; - - const buf = try decomp.reader().readAllAlloc(std.testing.allocator, 1024 * 100); - defer std.testing.allocator.free(buf); - - try decomp.close(); - - try decomp.reset(stream.reader(), null); - } -} - -test "truncated input" { - const TruncatedTest = struct { - input: []const u8, - output: []const u8, - }; - - const tests = [_]TruncatedTest{ - .{ .input = "\x00", .output = "" }, - .{ .input = "\x00\x0c", .output = "" }, - .{ .input = "\x00\x0c\x00", .output = "" }, - .{ .input = "\x00\x0c\x00\xf3\xff", .output = "" }, - .{ .input = "\x00\x0c\x00\xf3\xffhello", .output = "hello" }, - .{ .input = "\x00\x0c\x00\xf3\xffhello, world", .output = "hello, world" }, - .{ .input = "\x02", .output = "" }, - .{ .input = "\xf2H\xcd", .output = "He" }, - .{ .input = "\xf2HÍ™0a\u{0084}\t", .output = "Hel\x90\x90\x90\x90\x90" }, - .{ .input = "\xf2HÍ™0a\u{0084}\t\x00", .output = "Hel\x90\x90\x90\x90\x90" }, - }; - - for (tests) |t| { - var fib = io.fixedBufferStream(t.input); - const r = fib.reader(); - var z = try decompressor(testing.allocator, r, null); - defer z.deinit(); - var zr = z.reader(); - - var output = [1]u8{0} ** 12; - try expectError(error.UnexpectedEndOfStream, zr.readAll(&output)); - try testing.expectEqualSlices(u8, t.output, output[0..t.output.len]); - } -} - -test "Go non-regression test for 9842" { - // See https://golang.org/issue/9842 - - const Test = struct { - err: ?anyerror, - input: []const u8, - }; - - const tests = [_]Test{ - .{ .err = error.UnexpectedEndOfStream, .input = ("\x95\x90=o\xc20\x10\x86\xf30") }, - .{ .err = error.CorruptInput, .input = ("\x950\x00\x0000000") }, - - // Huffman.construct errors - - // lencode - .{ .err = error.CorruptInput, .input = ("\x950000") }, - .{ .err = error.CorruptInput, .input = ("\x05000") }, - // hlen - .{ .err = error.CorruptInput, .input = ("\x05\xea\x01\t\x00\x00\x00\x01\x00\\\xbf.\t\x00") }, - // hdist - .{ .err = error.CorruptInput, .input = ("\x05\xe0\x01A\x00\x00\x00\x00\x10\\\xbf.") }, - - // like the "empty distance alphabet" test but for ndist instead of nlen - .{ .err = error.CorruptInput, .input = ("\x05\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf\xce") }, - .{ .err = null, .input = "\x15\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf.0" }, - }; - - for (tests) |t| { - var fib = std.io.fixedBufferStream(t.input); - const reader = fib.reader(); - var decomp = try decompressor(testing.allocator, reader, null); - defer decomp.deinit(); - - var output: [10]u8 = undefined; - if (t.err != null) { - try expectError(t.err.?, decomp.reader().read(&output)); - } else { - _ = try decomp.reader().read(&output); - } - } -} - -test "inflate A Tale of Two Cities (1859) intro" { - const compressed = [_]u8{ - 0x74, 0xeb, 0xcd, 0x0d, 0x80, 0x20, 0x0c, 0x47, 0x71, 0xdc, 0x9d, 0xa2, 0x03, 0xb8, 0x88, - 0x63, 0xf0, 0xf1, 0x47, 0x9a, 0x00, 0x35, 0xb4, 0x86, 0xf5, 0x0d, 0x27, 0x63, 0x82, 0xe7, - 0xdf, 0x7b, 0x87, 0xd1, 0x70, 0x4a, 0x96, 0x41, 0x1e, 0x6a, 0x24, 0x89, 0x8c, 0x2b, 0x74, - 0xdf, 0xf8, 0x95, 0x21, 0xfd, 0x8f, 0xdc, 0x89, 0x09, 0x83, 0x35, 0x4a, 0x5d, 0x49, 0x12, - 0x29, 0xac, 0xb9, 0x41, 0xbf, 0x23, 0x2e, 0x09, 0x79, 0x06, 0x1e, 0x85, 0x91, 0xd6, 0xc6, - 0x2d, 0x74, 0xc4, 0xfb, 0xa1, 0x7b, 0x0f, 0x52, 0x20, 0x84, 0x61, 0x28, 0x0c, 0x63, 0xdf, - 0x53, 0xf4, 0x00, 0x1e, 0xc3, 0xa5, 0x97, 0x88, 0xf4, 0xd9, 0x04, 0xa5, 0x2d, 0x49, 0x54, - 0xbc, 0xfd, 0x90, 0xa5, 0x0c, 0xae, 0xbf, 0x3f, 0x84, 0x77, 0x88, 0x3f, 0xaf, 0xc0, 0x40, - 0xd6, 0x5b, 0x14, 0x8b, 0x54, 0xf6, 0x0f, 0x9b, 0x49, 0xf7, 0xbf, 0xbf, 0x36, 0x54, 0x5a, - 0x0d, 0xe6, 0x3e, 0xf0, 0x9e, 0x29, 0xcd, 0xa1, 0x41, 0x05, 0x36, 0x48, 0x74, 0x4a, 0xe9, - 0x46, 0x66, 0x2a, 0x19, 0x17, 0xf4, 0x71, 0x8e, 0xcb, 0x15, 0x5b, 0x57, 0xe4, 0xf3, 0xc7, - 0xe7, 0x1e, 0x9d, 0x50, 0x08, 0xc3, 0x50, 0x18, 0xc6, 0x2a, 0x19, 0xa0, 0xdd, 0xc3, 0x35, - 0x82, 0x3d, 0x6a, 0xb0, 0x34, 0x92, 0x16, 0x8b, 0xdb, 0x1b, 0xeb, 0x7d, 0xbc, 0xf8, 0x16, - 0xf8, 0xc2, 0xe1, 0xaf, 0x81, 0x7e, 0x58, 0xf4, 0x9f, 0x74, 0xf8, 0xcd, 0x39, 0xd3, 0xaa, - 0x0f, 0x26, 0x31, 0xcc, 0x8d, 0x9a, 0xd2, 0x04, 0x3e, 0x51, 0xbe, 0x7e, 0xbc, 0xc5, 0x27, - 0x3d, 0xa5, 0xf3, 0x15, 0x63, 0x94, 0x42, 0x75, 0x53, 0x6b, 0x61, 0xc8, 0x01, 0x13, 0x4d, - 0x23, 0xba, 0x2a, 0x2d, 0x6c, 0x94, 0x65, 0xc7, 0x4b, 0x86, 0x9b, 0x25, 0x3e, 0xba, 0x01, - 0x10, 0x84, 0x81, 0x28, 0x80, 0x55, 0x1c, 0xc0, 0xa5, 0xaa, 0x36, 0xa6, 0x09, 0xa8, 0xa1, - 0x85, 0xf9, 0x7d, 0x45, 0xbf, 0x80, 0xe4, 0xd1, 0xbb, 0xde, 0xb9, 0x5e, 0xf1, 0x23, 0x89, - 0x4b, 0x00, 0xd5, 0x59, 0x84, 0x85, 0xe3, 0xd4, 0xdc, 0xb2, 0x66, 0xe9, 0xc1, 0x44, 0x0b, - 0x1e, 0x84, 0xec, 0xe6, 0xa1, 0xc7, 0x42, 0x6a, 0x09, 0x6d, 0x9a, 0x5e, 0x70, 0xa2, 0x36, - 0x94, 0x29, 0x2c, 0x85, 0x3f, 0x24, 0x39, 0xf3, 0xae, 0xc3, 0xca, 0xca, 0xaf, 0x2f, 0xce, - 0x8e, 0x58, 0x91, 0x00, 0x25, 0xb5, 0xb3, 0xe9, 0xd4, 0xda, 0xef, 0xfa, 0x48, 0x7b, 0x3b, - 0xe2, 0x63, 0x12, 0x00, 0x00, 0x20, 0x04, 0x80, 0x70, 0x36, 0x8c, 0xbd, 0x04, 0x71, 0xff, - 0xf6, 0x0f, 0x66, 0x38, 0xcf, 0xa1, 0x39, 0x11, 0x0f, - }; - - const expected = - \\It was the best of times, - \\it was the worst of times, - \\it was the age of wisdom, - \\it was the age of foolishness, - \\it was the epoch of belief, - \\it was the epoch of incredulity, - \\it was the season of Light, - \\it was the season of Darkness, - \\it was the spring of hope, - \\it was the winter of despair, - \\ - \\we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way---in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. - \\ - ; - - var fib = std.io.fixedBufferStream(&compressed); - const reader = fib.reader(); - var decomp = try decompressor(testing.allocator, reader, null); - defer decomp.deinit(); - - var got: [700]u8 = undefined; - const got_len = try decomp.reader().read(&got); - try testing.expectEqual(@as(usize, 616), got_len); - try testing.expectEqualSlices(u8, expected, got[0..expected.len]); -} - -test "lengths overflow" { - // malformed final dynamic block, tries to write 321 code lengths (MAXCODES is 316) - // f dy hlit hdist hclen 16 17 18 0 (18) x138 (18) x138 (18) x39 (16) x6 - // 1 10 11101 11101 0000 010 010 010 010 (11) 1111111 (11) 1111111 (11) 0011100 (01) 11 - const stream = [_]u8{ - 0b11101101, 0b00011101, 0b00100100, 0b11101001, 0b11111111, 0b11111111, 0b00111001, - 0b00001110, - }; - try expectError(error.CorruptInput, decompress(stream[0..])); -} - -test "empty distance alphabet" { - // dynamic block with empty distance alphabet is valid if only literals and end of data symbol are used - // f dy hlit hdist hclen 16 17 18 0 8 7 9 6 10 5 11 4 12 3 13 2 14 1 15 (18) x128 (18) x128 (1) ( 0) (256) - // 1 10 00000 00000 1111 000 000 010 010 000 000 000 000 000 000 000 000 000 000 000 000 000 001 000 (11) 1110101 (11) 1110101 (0) (10) (0) - const stream = [_]u8{ - 0b00000101, 0b11100000, 0b00000001, 0b00001001, 0b00000000, 0b00000000, - 0b00000000, 0b00000000, 0b00010000, 0b01011100, 0b10111111, 0b00101110, - }; - try decompress(stream[0..]); -} - -test "distance past beginning of output stream" { - // f fx ('A') ('B') ('C') (end) - // 1 01 (01110001) (01110010) (01110011) (0000010) (00011) (0000000) - const stream = [_]u8{ 0b01110011, 0b01110100, 0b01110010, 0b00000110, 0b01100001, 0b00000000 }; - try std.testing.expectError(error.CorruptInput, decompress(stream[0..])); -} - -test "fuzzing" { - const compressed = [_]u8{ - 0x0a, 0x08, 0x50, 0xeb, 0x25, 0x05, 0xfc, 0x30, 0x0b, 0x0a, 0x08, 0x50, 0xeb, 0x25, 0x05, - } ++ [_]u8{0xe1} ** 15 ++ [_]u8{0x30} ++ [_]u8{0xe1} ** 1481; - try expectError(error.UnexpectedEndOfStream, decompress(&compressed)); - - // see https://github.com/ziglang/zig/issues/9842 - try expectError(error.UnexpectedEndOfStream, decompress("\x95\x90=o\xc20\x10\x86\xf30")); - try expectError(error.CorruptInput, decompress("\x950\x00\x0000000")); - - // Huffman errors - // lencode - try expectError(error.CorruptInput, decompress("\x950000")); - try expectError(error.CorruptInput, decompress("\x05000")); - // hlen - try expectError(error.CorruptInput, decompress("\x05\xea\x01\t\x00\x00\x00\x01\x00\\\xbf.\t\x00")); - // hdist - try expectError(error.CorruptInput, decompress("\x05\xe0\x01A\x00\x00\x00\x00\x10\\\xbf.")); - - // like the "empty distance alphabet" test but for ndist instead of nlen - try expectError(error.CorruptInput, decompress("\x05\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf\xce")); - try decompress("\x15\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf.0"); -} - -fn decompress(input: []const u8) !void { - const allocator = testing.allocator; - var fib = std.io.fixedBufferStream(input); - const reader = fib.reader(); - var decomp = try decompressor(allocator, reader, null); - defer decomp.deinit(); - const output = try decomp.reader().readAllAlloc(allocator, math.maxInt(usize)); - defer std.testing.allocator.free(output); -} diff --git a/lib/std/compress/deflate/deflate_const.zig b/lib/std/compress/deflate/deflate_const.zig deleted file mode 100644 index bf328b562c3d..000000000000 --- a/lib/std/compress/deflate/deflate_const.zig +++ /dev/null @@ -1,28 +0,0 @@ -// Deflate - -// Biggest block size for uncompressed block. -pub const max_store_block_size = 65535; -// The special code used to mark the end of a block. -pub const end_block_marker = 256; - -// LZ77 - -// The smallest match length per the RFC section 3.2.5 -pub const base_match_length = 3; -// The smallest match offset. -pub const base_match_offset = 1; -// The largest match length. -pub const max_match_length = 258; -// The largest match offset. -pub const max_match_offset = 1 << 15; - -// Huffman Codes - -// The largest offset code. -pub const offset_code_count = 30; -// Max number of frequencies used for a Huffman Code -// Possible lengths are codegenCodeCount (19), offset_code_count (30) and max_num_lit (286). -// The largest of these is max_num_lit. -pub const max_num_frequencies = max_num_lit; -// Maximum number of literals. -pub const max_num_lit = 286; diff --git a/lib/std/compress/deflate/deflate_fast.zig b/lib/std/compress/deflate/deflate_fast.zig deleted file mode 100644 index 4fda56bfb770..000000000000 --- a/lib/std/compress/deflate/deflate_fast.zig +++ /dev/null @@ -1,728 +0,0 @@ -// This encoding algorithm, which prioritizes speed over output size, is -// based on Snappy's LZ77-style encoder: github.com/golang/snappy - -const std = @import("std"); -const math = std.math; -const mem = std.mem; - -const Allocator = std.mem.Allocator; - -const deflate_const = @import("deflate_const.zig"); -const deflate = @import("compressor.zig"); -const token = @import("token.zig"); - -const base_match_length = deflate_const.base_match_length; -const base_match_offset = deflate_const.base_match_offset; -const max_match_length = deflate_const.max_match_length; -const max_match_offset = deflate_const.max_match_offset; -const max_store_block_size = deflate_const.max_store_block_size; - -const table_bits = 14; // Bits used in the table. -const table_mask = table_size - 1; // Mask for table indices. Redundant, but can eliminate bounds checks. -const table_shift = 32 - table_bits; // Right-shift to get the table_bits most significant bits of a uint32. -const table_size = 1 << table_bits; // Size of the table. - -// Reset the buffer offset when reaching this. -// Offsets are stored between blocks as i32 values. -// Since the offset we are checking against is at the beginning -// of the buffer, we need to subtract the current and input -// buffer to not risk overflowing the i32. -const buffer_reset = math.maxInt(i32) - max_store_block_size * 2; - -fn load32(b: []u8, i: i32) u32 { - const s = b[@as(usize, @intCast(i)) .. @as(usize, @intCast(i)) + 4]; - return @as(u32, @intCast(s[0])) | - @as(u32, @intCast(s[1])) << 8 | - @as(u32, @intCast(s[2])) << 16 | - @as(u32, @intCast(s[3])) << 24; -} - -fn load64(b: []u8, i: i32) u64 { - const s = b[@as(usize, @intCast(i))..@as(usize, @intCast(i + 8))]; - return @as(u64, @intCast(s[0])) | - @as(u64, @intCast(s[1])) << 8 | - @as(u64, @intCast(s[2])) << 16 | - @as(u64, @intCast(s[3])) << 24 | - @as(u64, @intCast(s[4])) << 32 | - @as(u64, @intCast(s[5])) << 40 | - @as(u64, @intCast(s[6])) << 48 | - @as(u64, @intCast(s[7])) << 56; -} - -fn hash(u: u32) u32 { - return (u *% 0x1e35a7bd) >> table_shift; -} - -// These constants are defined by the Snappy implementation so that its -// assembly implementation can fast-path some 16-bytes-at-a-time copies. -// They aren't necessary in the pure Go implementation, and may not be -// necessary in Zig, but using the same thresholds doesn't really hurt. -const input_margin = 16 - 1; -const min_non_literal_block_size = 1 + 1 + input_margin; - -const TableEntry = struct { - val: u32, // Value at destination - offset: i32, -}; - -pub fn deflateFast() DeflateFast { - return DeflateFast{ - .table = [_]TableEntry{.{ .val = 0, .offset = 0 }} ** table_size, - .prev = undefined, - .prev_len = 0, - .cur = max_store_block_size, - .allocator = undefined, - }; -} - -// DeflateFast maintains the table for matches, -// and the previous byte block for cross block matching. -pub const DeflateFast = struct { - table: [table_size]TableEntry, - prev: []u8, // Previous block, zero length if unknown. - prev_len: u32, // Previous block length - cur: i32, // Current match offset. - allocator: Allocator, - - const Self = @This(); - - pub fn init(self: *Self, allocator: Allocator) !void { - self.allocator = allocator; - self.prev = try allocator.alloc(u8, max_store_block_size); - self.prev_len = 0; - } - - pub fn deinit(self: *Self) void { - self.allocator.free(self.prev); - self.prev_len = 0; - } - - // Encodes a block given in `src` and appends tokens to `dst` and returns the result. - pub fn encode(self: *Self, dst: []token.Token, tokens_count: *u16, src: []u8) void { - - // Ensure that self.cur doesn't wrap. - if (self.cur >= buffer_reset) { - self.shiftOffsets(); - } - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if (src.len < min_non_literal_block_size) { - self.cur += max_store_block_size; - self.prev_len = 0; - emitLiteral(dst, tokens_count, src); - return; - } - - // s_limit is when to stop looking for offset/length copies. The input_margin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - const s_limit = @as(i32, @intCast(src.len - input_margin)); - - // next_emit is where in src the next emitLiteral should start from. - var next_emit: i32 = 0; - var s: i32 = 0; - var cv: u32 = load32(src, s); - var next_hash: u32 = hash(cv); - - outer: while (true) { - // Copied from the C++ snappy implementation: - // - // Heuristic match skipping: If 32 bytes are scanned with no matches - // found, start looking only at every other byte. If 32 more bytes are - // scanned (or skipped), look at every third byte, etc.. When a match - // is found, immediately go back to looking at every byte. This is a - // small loss (~5% performance, ~0.1% density) for compressible data - // due to more bookkeeping, but for non-compressible data (such as - // JPEG) it's a huge win since the compressor quickly "realizes" the - // data is incompressible and doesn't bother looking for matches - // everywhere. - // - // The "skip" variable keeps track of how many bytes there are since - // the last match; dividing it by 32 (ie. right-shifting by five) gives - // the number of bytes to move ahead for each iteration. - var skip: i32 = 32; - - var next_s: i32 = s; - var candidate: TableEntry = undefined; - while (true) { - s = next_s; - const bytes_between_hash_lookups = skip >> 5; - next_s = s + bytes_between_hash_lookups; - skip += bytes_between_hash_lookups; - if (next_s > s_limit) { - break :outer; - } - candidate = self.table[next_hash & table_mask]; - const now = load32(src, next_s); - self.table[next_hash & table_mask] = .{ .offset = s + self.cur, .val = cv }; - next_hash = hash(now); - - const offset = s - (candidate.offset - self.cur); - if (offset > max_match_offset or cv != candidate.val) { - // Out of range or not matched. - cv = now; - continue; - } - break; - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[next_emit..s] are unmatched. Emit - // them as literal bytes. - emitLiteral(dst, tokens_count, src[@as(usize, @intCast(next_emit))..@as(usize, @intCast(s))]); - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - while (true) { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - - // Extend the 4-byte match as long as possible. - // - s += 4; - const t = candidate.offset - self.cur + 4; - const l = self.matchLen(s, t, src); - - // matchToken is flate's equivalent of Snappy's emitCopy. (length,offset) - dst[tokens_count.*] = token.matchToken( - @as(u32, @intCast(l + 4 - base_match_length)), - @as(u32, @intCast(s - t - base_match_offset)), - ); - tokens_count.* += 1; - s += l; - next_emit = s; - if (s >= s_limit) { - break :outer; - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-1 and at s. If - // another emitCopy is not our next move, also calculate next_hash - // at s+1. At least on amd64 architecture, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - var x = load64(src, s - 1); - const prev_hash = hash(@as(u32, @truncate(x))); - self.table[prev_hash & table_mask] = TableEntry{ - .offset = self.cur + s - 1, - .val = @as(u32, @truncate(x)), - }; - x >>= 8; - const curr_hash = hash(@as(u32, @truncate(x))); - candidate = self.table[curr_hash & table_mask]; - self.table[curr_hash & table_mask] = TableEntry{ - .offset = self.cur + s, - .val = @as(u32, @truncate(x)), - }; - - const offset = s - (candidate.offset - self.cur); - if (offset > max_match_offset or @as(u32, @truncate(x)) != candidate.val) { - cv = @as(u32, @truncate(x >> 8)); - next_hash = hash(cv); - s += 1; - break; - } - } - } - - if (@as(u32, @intCast(next_emit)) < src.len) { - emitLiteral(dst, tokens_count, src[@as(usize, @intCast(next_emit))..]); - } - self.cur += @as(i32, @intCast(src.len)); - self.prev_len = @as(u32, @intCast(src.len)); - @memcpy(self.prev[0..self.prev_len], src); - return; - } - - fn emitLiteral(dst: []token.Token, tokens_count: *u16, lit: []u8) void { - for (lit) |v| { - dst[tokens_count.*] = token.literalToken(@as(u32, @intCast(v))); - tokens_count.* += 1; - } - return; - } - - // matchLen returns the match length between src[s..] and src[t..]. - // t can be negative to indicate the match is starting in self.prev. - // We assume that src[s-4 .. s] and src[t-4 .. t] already match. - fn matchLen(self: *Self, s: i32, t: i32, src: []u8) i32 { - var s1 = @as(u32, @intCast(s)) + max_match_length - 4; - if (s1 > src.len) { - s1 = @as(u32, @intCast(src.len)); - } - - // If we are inside the current block - if (t >= 0) { - var b = src[@as(usize, @intCast(t))..]; - const a = src[@as(usize, @intCast(s))..@as(usize, @intCast(s1))]; - b = b[0..a.len]; - // Extend the match to be as long as possible. - for (a, 0..) |_, i| { - if (a[i] != b[i]) { - return @as(i32, @intCast(i)); - } - } - return @as(i32, @intCast(a.len)); - } - - // We found a match in the previous block. - const tp = @as(i32, @intCast(self.prev_len)) + t; - if (tp < 0) { - return 0; - } - - // Extend the match to be as long as possible. - var a = src[@as(usize, @intCast(s))..@as(usize, @intCast(s1))]; - var b = self.prev[@as(usize, @intCast(tp))..@as(usize, @intCast(self.prev_len))]; - if (b.len > a.len) { - b = b[0..a.len]; - } - a = a[0..b.len]; - for (b, 0..) |_, i| { - if (a[i] != b[i]) { - return @as(i32, @intCast(i)); - } - } - - // If we reached our limit, we matched everything we are - // allowed to in the previous block and we return. - const n = @as(i32, @intCast(b.len)); - if (@as(u32, @intCast(s + n)) == s1) { - return n; - } - - // Continue looking for more matches in the current block. - a = src[@as(usize, @intCast(s + n))..@as(usize, @intCast(s1))]; - b = src[0..a.len]; - for (a, 0..) |_, i| { - if (a[i] != b[i]) { - return @as(i32, @intCast(i)) + n; - } - } - return @as(i32, @intCast(a.len)) + n; - } - - // Reset resets the encoding history. - // This ensures that no matches are made to the previous block. - pub fn reset(self: *Self) void { - self.prev_len = 0; - // Bump the offset, so all matches will fail distance check. - // Nothing should be >= self.cur in the table. - self.cur += max_match_offset; - - // Protect against self.cur wraparound. - if (self.cur >= buffer_reset) { - self.shiftOffsets(); - } - } - - // shiftOffsets will shift down all match offset. - // This is only called in rare situations to prevent integer overflow. - // - // See https://golang.org/issue/18636 and https://golang.org/issues/34121. - fn shiftOffsets(self: *Self) void { - if (self.prev_len == 0) { - // We have no history; just clear the table. - for (self.table, 0..) |_, i| { - self.table[i] = TableEntry{ .val = 0, .offset = 0 }; - } - self.cur = max_match_offset + 1; - return; - } - - // Shift down everything in the table that isn't already too far away. - for (self.table, 0..) |_, i| { - var v = self.table[i].offset - self.cur + max_match_offset + 1; - if (v < 0) { - // We want to reset self.cur to max_match_offset + 1, so we need to shift - // all table entries down by (self.cur - (max_match_offset + 1)). - // Because we ignore matches > max_match_offset, we can cap - // any negative offsets at 0. - v = 0; - } - self.table[i].offset = v; - } - self.cur = max_match_offset + 1; - } -}; - -test "best speed match 1/3" { - if (@import("builtin").os.tag == .wasi) { - // https://github.com/ziglang/zig/issues/18885 - return error.SkipZigTest; - } - const expectEqual = std.testing.expectEqual; - - { - var previous = [_]u8{ 0, 0, 0, 1, 2 }; - var e = DeflateFast{ - .prev = &previous, - .prev_len = previous.len, - .table = undefined, - .allocator = undefined, - .cur = 0, - }; - var current = [_]u8{ 3, 4, 5, 0, 1, 2, 3, 4, 5 }; - const got: i32 = e.matchLen(3, -3, ¤t); - try expectEqual(@as(i32, 6), got); - } - { - var previous = [_]u8{ 0, 0, 0, 1, 2 }; - var e = DeflateFast{ - .prev = &previous, - .prev_len = previous.len, - .table = undefined, - .allocator = undefined, - .cur = 0, - }; - var current = [_]u8{ 2, 4, 5, 0, 1, 2, 3, 4, 5 }; - const got: i32 = e.matchLen(3, -3, ¤t); - try expectEqual(@as(i32, 3), got); - } - { - var previous = [_]u8{ 0, 0, 0, 1, 1 }; - var e = DeflateFast{ - .prev = &previous, - .prev_len = previous.len, - .table = undefined, - .allocator = undefined, - .cur = 0, - }; - var current = [_]u8{ 3, 4, 5, 0, 1, 2, 3, 4, 5 }; - const got: i32 = e.matchLen(3, -3, ¤t); - try expectEqual(@as(i32, 2), got); - } - { - var previous = [_]u8{ 0, 0, 0, 1, 2 }; - var e = DeflateFast{ - .prev = &previous, - .prev_len = previous.len, - .table = undefined, - .allocator = undefined, - .cur = 0, - }; - var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 }; - const got: i32 = e.matchLen(0, -1, ¤t); - try expectEqual(@as(i32, 4), got); - } - { - var previous = [_]u8{ 0, 0, 0, 1, 2, 3, 4, 5, 2, 2 }; - var e = DeflateFast{ - .prev = &previous, - .prev_len = previous.len, - .table = undefined, - .allocator = undefined, - .cur = 0, - }; - var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 }; - const got: i32 = e.matchLen(4, -7, ¤t); - try expectEqual(@as(i32, 5), got); - } - { - var previous = [_]u8{ 9, 9, 9, 9, 9 }; - var e = DeflateFast{ - .prev = &previous, - .prev_len = previous.len, - .table = undefined, - .allocator = undefined, - .cur = 0, - }; - var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 }; - const got: i32 = e.matchLen(0, -1, ¤t); - try expectEqual(@as(i32, 0), got); - } - { - var previous = [_]u8{ 9, 9, 9, 9, 9 }; - var e = DeflateFast{ - .prev = &previous, - .prev_len = previous.len, - .table = undefined, - .allocator = undefined, - .cur = 0, - }; - var current = [_]u8{ 9, 2, 2, 2, 1, 2, 3, 4, 5 }; - const got: i32 = e.matchLen(1, 0, ¤t); - try expectEqual(@as(i32, 0), got); - } -} - -test "best speed match 2/3" { - if (@import("builtin").os.tag == .wasi) { - // https://github.com/ziglang/zig/issues/18885 - return error.SkipZigTest; - } - const expectEqual = std.testing.expectEqual; - - { - var previous = [_]u8{}; - var e = DeflateFast{ - .prev = &previous, - .prev_len = previous.len, - .table = undefined, - .allocator = undefined, - .cur = 0, - }; - var current = [_]u8{ 9, 2, 2, 2, 1, 2, 3, 4, 5 }; - const got: i32 = e.matchLen(1, -5, ¤t); - try expectEqual(@as(i32, 0), got); - } - { - var previous = [_]u8{}; - var e = DeflateFast{ - .prev = &previous, - .prev_len = previous.len, - .table = undefined, - .allocator = undefined, - .cur = 0, - }; - var current = [_]u8{ 9, 2, 2, 2, 1, 2, 3, 4, 5 }; - const got: i32 = e.matchLen(1, -1, ¤t); - try expectEqual(@as(i32, 0), got); - } - { - var previous = [_]u8{}; - var e = DeflateFast{ - .prev = &previous, - .prev_len = previous.len, - .table = undefined, - .allocator = undefined, - .cur = 0, - }; - var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 }; - const got: i32 = e.matchLen(1, 0, ¤t); - try expectEqual(@as(i32, 3), got); - } - { - var previous = [_]u8{ 3, 4, 5 }; - var e = DeflateFast{ - .prev = &previous, - .prev_len = previous.len, - .table = undefined, - .allocator = undefined, - .cur = 0, - }; - var current = [_]u8{ 3, 4, 5 }; - const got: i32 = e.matchLen(0, -3, ¤t); - try expectEqual(@as(i32, 3), got); - } -} - -test "best speed match 2/2" { - const testing = std.testing; - const expectEqual = testing.expectEqual; - - const Case = struct { - previous: u32, - current: u32, - s: i32, - t: i32, - expected: i32, - }; - - const cases = [_]Case{ - .{ - .previous = 1000, - .current = 1000, - .s = 0, - .t = -1000, - .expected = max_match_length - 4, - }, - .{ - .previous = 200, - .s = 0, - .t = -200, - .current = 500, - .expected = max_match_length - 4, - }, - .{ - .previous = 200, - .s = 1, - .t = 0, - .current = 500, - .expected = max_match_length - 4, - }, - .{ - .previous = max_match_length - 4, - .s = 0, - .t = -(max_match_length - 4), - .current = 500, - .expected = max_match_length - 4, - }, - .{ - .previous = 200, - .s = 400, - .t = -200, - .current = 500, - .expected = 100, - }, - .{ - .previous = 10, - .s = 400, - .t = 200, - .current = 500, - .expected = 100, - }, - }; - - for (cases) |c| { - const previous = try testing.allocator.alloc(u8, c.previous); - defer testing.allocator.free(previous); - @memset(previous, 0); - - const current = try testing.allocator.alloc(u8, c.current); - defer testing.allocator.free(current); - @memset(current, 0); - - var e = DeflateFast{ - .prev = previous, - .prev_len = @as(u32, @intCast(previous.len)), - .table = undefined, - .allocator = undefined, - .cur = 0, - }; - const got: i32 = e.matchLen(c.s, c.t, current); - try expectEqual(@as(i32, c.expected), got); - } -} - -test "best speed shift offsets" { - const testing = std.testing; - const expect = std.testing.expect; - - // Test if shiftoffsets properly preserves matches and resets out-of-range matches - // seen in https://github.com/golang/go/issues/4142 - var enc = deflateFast(); - try enc.init(testing.allocator); - defer enc.deinit(); - - // test_data may not generate internal matches. - var test_data = [32]u8{ - 0xf5, 0x25, 0xf2, 0x55, 0xf6, 0xc1, 0x1f, 0x0b, 0x10, 0xa1, - 0xd0, 0x77, 0x56, 0x38, 0xf1, 0x9c, 0x7f, 0x85, 0xc5, 0xbd, - 0x16, 0x28, 0xd4, 0xf9, 0x03, 0xd4, 0xc0, 0xa1, 0x1e, 0x58, - 0x5b, 0xc9, - }; - - var tokens = [_]token.Token{0} ** 32; - var tokens_count: u16 = 0; - - // Encode the testdata with clean state. - // Second part should pick up matches from the first block. - tokens_count = 0; - enc.encode(&tokens, &tokens_count, &test_data); - const want_first_tokens = tokens_count; - tokens_count = 0; - enc.encode(&tokens, &tokens_count, &test_data); - const want_second_tokens = tokens_count; - - try expect(want_first_tokens > want_second_tokens); - - // Forward the current indicator to before wraparound. - enc.cur = buffer_reset - @as(i32, @intCast(test_data.len)); - - // Part 1 before wrap, should match clean state. - tokens_count = 0; - enc.encode(&tokens, &tokens_count, &test_data); - var got = tokens_count; - try testing.expectEqual(want_first_tokens, got); - - // Verify we are about to wrap. - try testing.expectEqual(@as(i32, buffer_reset), enc.cur); - - // Part 2 should match clean state as well even if wrapped. - tokens_count = 0; - enc.encode(&tokens, &tokens_count, &test_data); - got = tokens_count; - try testing.expectEqual(want_second_tokens, got); - - // Verify that we wrapped. - try expect(enc.cur < buffer_reset); - - // Forward the current buffer, leaving the matches at the bottom. - enc.cur = buffer_reset; - enc.shiftOffsets(); - - // Ensure that no matches were picked up. - tokens_count = 0; - enc.encode(&tokens, &tokens_count, &test_data); - got = tokens_count; - try testing.expectEqual(want_first_tokens, got); -} - -test "best speed reset" { - // test that encoding is consistent across a warparound of the table offset. - // See https://github.com/golang/go/issues/34121 - const fmt = std.fmt; - const testing = std.testing; - - const ArrayList = std.ArrayList; - - const input_size = 65536; - const input = try testing.allocator.alloc(u8, input_size); - defer testing.allocator.free(input); - - var i: usize = 0; - while (i < input_size) : (i += 1) { - _ = try fmt.bufPrint(input, "asdfasdfasdfasdf{d}{d}fghfgujyut{d}yutyu\n", .{ i, i, i }); - } - // This is specific to level 1 (best_speed). - const level = .best_speed; - const offset: usize = 1; - - // We do an encode with a clean buffer to compare. - var want = ArrayList(u8).init(testing.allocator); - defer want.deinit(); - var clean_comp = try deflate.compressor( - testing.allocator, - want.writer(), - .{ .level = level }, - ); - defer clean_comp.deinit(); - - // Write 3 times, close. - try clean_comp.writer().writeAll(input); - try clean_comp.writer().writeAll(input); - try clean_comp.writer().writeAll(input); - try clean_comp.close(); - - var o = offset; - while (o <= 256) : (o *= 2) { - var discard = ArrayList(u8).init(testing.allocator); - defer discard.deinit(); - - var comp = try deflate.compressor( - testing.allocator, - discard.writer(), - .{ .level = level }, - ); - defer comp.deinit(); - - // Reset until we are right before the wraparound. - // Each reset adds max_match_offset to the offset. - i = 0; - const limit = (buffer_reset - input.len - o - max_match_offset) / max_match_offset; - while (i < limit) : (i += 1) { - // skip ahead to where we are close to wrap around... - comp.reset(discard.writer()); - } - var got = ArrayList(u8).init(testing.allocator); - defer got.deinit(); - comp.reset(got.writer()); - - // Write 3 times, close. - try comp.writer().writeAll(input); - try comp.writer().writeAll(input); - try comp.writer().writeAll(input); - try comp.close(); - - // output must match at wraparound - try testing.expectEqualSlices(u8, want.items, got.items); - } -} diff --git a/lib/std/compress/deflate/deflate_fast_test.zig b/lib/std/compress/deflate/deflate_fast_test.zig deleted file mode 100644 index fdb8e3fd6a9d..000000000000 --- a/lib/std/compress/deflate/deflate_fast_test.zig +++ /dev/null @@ -1,160 +0,0 @@ -const std = @import("std"); -const expect = std.testing.expect; -const io = std.io; -const mem = std.mem; -const testing = std.testing; - -const ArrayList = std.ArrayList; - -const deflate = @import("compressor.zig"); -const inflate = @import("decompressor.zig"); -const deflate_const = @import("deflate_const.zig"); - -test "best speed" { - // Tests that round-tripping through deflate and then inflate recovers the original input. - // The Write sizes are near the thresholds in the compressor.encSpeed method (0, 16, 128), as well - // as near `deflate_const.max_store_block_size` (65535). - - var abcabc = try testing.allocator.alloc(u8, 131_072); - defer testing.allocator.free(abcabc); - - for (abcabc, 0..) |_, i| { - abcabc[i] = @as(u8, @intCast(i % 128)); - } - - var tc_01 = [_]u32{ 65536, 0 }; - var tc_02 = [_]u32{ 65536, 1 }; - var tc_03 = [_]u32{ 65536, 1, 256 }; - var tc_04 = [_]u32{ 65536, 1, 65536 }; - var tc_05 = [_]u32{ 65536, 14 }; - var tc_06 = [_]u32{ 65536, 15 }; - var tc_07 = [_]u32{ 65536, 16 }; - var tc_08 = [_]u32{ 65536, 16, 256 }; - var tc_09 = [_]u32{ 65536, 16, 65536 }; - var tc_10 = [_]u32{ 65536, 127 }; - var tc_11 = [_]u32{ 65536, 127 }; - var tc_12 = [_]u32{ 65536, 128 }; - var tc_13 = [_]u32{ 65536, 128, 256 }; - var tc_14 = [_]u32{ 65536, 128, 65536 }; - var tc_15 = [_]u32{ 65536, 129 }; - var tc_16 = [_]u32{ 65536, 65536, 256 }; - var tc_17 = [_]u32{ 65536, 65536, 65536 }; - const test_cases = [_][]u32{ - &tc_01, &tc_02, &tc_03, &tc_04, &tc_05, &tc_06, &tc_07, &tc_08, &tc_09, &tc_10, - &tc_11, &tc_12, &tc_13, &tc_14, &tc_15, &tc_16, &tc_17, - }; - - for (test_cases) |tc| { - const firsts = [_]u32{ 1, 65534, 65535, 65536, 65537, 131072 }; - - for (firsts) |first_n| { - tc[0] = first_n; - - const to_flush = [_]bool{ false, true }; - for (to_flush) |flush| { - var compressed = ArrayList(u8).init(testing.allocator); - defer compressed.deinit(); - - var want = ArrayList(u8).init(testing.allocator); - defer want.deinit(); - - var comp = try deflate.compressor( - testing.allocator, - compressed.writer(), - .{ .level = .best_speed }, - ); - defer comp.deinit(); - - for (tc) |n| { - try want.appendSlice(abcabc[0..n]); - try comp.writer().writeAll(abcabc[0..n]); - if (flush) { - try comp.flush(); - } - } - - try comp.close(); - - const decompressed = try testing.allocator.alloc(u8, want.items.len); - defer testing.allocator.free(decompressed); - - var fib = io.fixedBufferStream(compressed.items); - var decomp = try inflate.decompressor(testing.allocator, fib.reader(), null); - defer decomp.deinit(); - - const read = try decomp.reader().readAll(decompressed); - try decomp.close(); - - try testing.expectEqual(want.items.len, read); - try testing.expectEqualSlices(u8, want.items, decompressed); - } - } - } -} - -test "best speed max match offset" { - const abc = "abcdefgh"; - const xyz = "stuvwxyz"; - const input_margin = 16 - 1; - - const match_before = [_]bool{ false, true }; - for (match_before) |do_match_before| { - const extras = [_]u32{ - 0, - input_margin - 1, - input_margin, - input_margin + 1, - 2 * input_margin, - }; - for (extras) |extra| { - var offset_adj: i32 = -5; - while (offset_adj <= 5) : (offset_adj += 1) { - const offset = deflate_const.max_match_offset + offset_adj; - - // Make src to be a []u8 of the form - // fmt("{s}{s}{s}{s}{s}", .{abc, zeros0, xyzMaybe, abc, zeros1}) - // where: - // zeros0 is approximately max_match_offset zeros. - // xyzMaybe is either xyz or the empty string. - // zeros1 is between 0 and 30 zeros. - // The difference between the two abc's will be offset, which - // is max_match_offset plus or minus a small adjustment. - const src_len: usize = @as(usize, @intCast(offset + @as(i32, abc.len) + @as(i32, @intCast(extra)))); - var src = try testing.allocator.alloc(u8, src_len); - defer testing.allocator.free(src); - - @memcpy(src[0..abc.len], abc); - if (!do_match_before) { - const src_offset: usize = @as(usize, @intCast(offset - @as(i32, xyz.len))); - @memcpy(src[src_offset..][0..xyz.len], xyz); - } - const src_offset: usize = @as(usize, @intCast(offset)); - @memcpy(src[src_offset..][0..abc.len], abc); - - var compressed = ArrayList(u8).init(testing.allocator); - defer compressed.deinit(); - - var comp = try deflate.compressor( - testing.allocator, - compressed.writer(), - .{ .level = .best_speed }, - ); - defer comp.deinit(); - try comp.writer().writeAll(src); - _ = try comp.close(); - - const decompressed = try testing.allocator.alloc(u8, src.len); - defer testing.allocator.free(decompressed); - - var fib = io.fixedBufferStream(compressed.items); - var decomp = try inflate.decompressor(testing.allocator, fib.reader(), null); - defer decomp.deinit(); - const read = try decomp.reader().readAll(decompressed); - try decomp.close(); - - try testing.expectEqual(src.len, read); - try testing.expectEqualSlices(u8, src, decompressed); - } - } - } -} diff --git a/lib/std/compress/deflate/dict_decoder.zig b/lib/std/compress/deflate/dict_decoder.zig deleted file mode 100644 index 72a3f6310b60..000000000000 --- a/lib/std/compress/deflate/dict_decoder.zig +++ /dev/null @@ -1,423 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const mem = std.mem; - -const Allocator = std.mem.Allocator; - -// Implements the LZ77 sliding dictionary as used in decompression. -// LZ77 decompresses data through sequences of two forms of commands: -// -// * Literal insertions: Runs of one or more symbols are inserted into the data -// stream as is. This is accomplished through the writeByte method for a -// single symbol, or combinations of writeSlice/writeMark for multiple symbols. -// Any valid stream must start with a literal insertion if no preset dictionary -// is used. -// -// * Backward copies: Runs of one or more symbols are copied from previously -// emitted data. Backward copies come as the tuple (dist, length) where dist -// determines how far back in the stream to copy from and length determines how -// many bytes to copy. Note that it is valid for the length to be greater than -// the distance. Since LZ77 uses forward copies, that situation is used to -// perform a form of run-length encoding on repeated runs of symbols. -// The writeCopy and tryWriteCopy are used to implement this command. -// -// For performance reasons, this implementation performs little to no sanity -// checks about the arguments. As such, the invariants documented for each -// method call must be respected. -pub const DictDecoder = struct { - const Self = @This(); - - allocator: Allocator = undefined, - - hist: []u8 = undefined, // Sliding window history - - // Invariant: 0 <= rd_pos <= wr_pos <= hist.len - wr_pos: u32 = 0, // Current output position in buffer - rd_pos: u32 = 0, // Have emitted hist[0..rd_pos] already - full: bool = false, // Has a full window length been written yet? - - // init initializes DictDecoder to have a sliding window dictionary of the given - // size. If a preset dict is provided, it will initialize the dictionary with - // the contents of dict. - pub fn init(self: *Self, allocator: Allocator, size: u32, dict: ?[]const u8) !void { - self.allocator = allocator; - - self.hist = try allocator.alloc(u8, size); - - self.wr_pos = 0; - - if (dict != null) { - const src = dict.?[dict.?.len -| self.hist.len..]; - @memcpy(self.hist[0..src.len], src); - self.wr_pos = @as(u32, @intCast(dict.?.len)); - } - - if (self.wr_pos == self.hist.len) { - self.wr_pos = 0; - self.full = true; - } - self.rd_pos = self.wr_pos; - } - - pub fn deinit(self: *Self) void { - self.allocator.free(self.hist); - } - - // Reports the total amount of historical data in the dictionary. - pub fn histSize(self: *Self) u32 { - if (self.full) { - return @as(u32, @intCast(self.hist.len)); - } - return self.wr_pos; - } - - // Reports the number of bytes that can be flushed by readFlush. - pub fn availRead(self: *Self) u32 { - return self.wr_pos - self.rd_pos; - } - - // Reports the available amount of output buffer space. - pub fn availWrite(self: *Self) u32 { - return @as(u32, @intCast(self.hist.len - self.wr_pos)); - } - - // Returns a slice of the available buffer to write data to. - // - // This invariant will be kept: s.len <= availWrite() - pub fn writeSlice(self: *Self) []u8 { - return self.hist[self.wr_pos..]; - } - - // Advances the writer pointer by `count`. - // - // This invariant must be kept: 0 <= count <= availWrite() - pub fn writeMark(self: *Self, count: u32) void { - assert(0 <= count and count <= self.availWrite()); - self.wr_pos += count; - } - - // Writes a single byte to the dictionary. - // - // This invariant must be kept: 0 < availWrite() - pub fn writeByte(self: *Self, byte: u8) void { - self.hist[self.wr_pos] = byte; - self.wr_pos += 1; - } - - /// TODO: eliminate this function because the callsites should care about whether - /// or not their arguments alias and then they should directly call `@memcpy` or - /// `mem.copyForwards`. - fn copy(dst: []u8, src: []const u8) u32 { - if (src.len > dst.len) { - mem.copyForwards(u8, dst, src[0..dst.len]); - return @as(u32, @intCast(dst.len)); - } - mem.copyForwards(u8, dst[0..src.len], src); - return @as(u32, @intCast(src.len)); - } - - // Copies a string at a given (dist, length) to the output. - // This returns the number of bytes copied and may be less than the requested - // length if the available space in the output buffer is too small. - // - // This invariant must be kept: 0 < dist <= histSize() - pub fn writeCopy(self: *Self, dist: u32, length: u32) u32 { - assert(0 < dist and dist <= self.histSize()); - const dst_base = self.wr_pos; - var dst_pos = dst_base; - var src_pos: i32 = @as(i32, @intCast(dst_pos)) - @as(i32, @intCast(dist)); - var end_pos = dst_pos + length; - if (end_pos > self.hist.len) { - end_pos = @as(u32, @intCast(self.hist.len)); - } - - // Copy non-overlapping section after destination position. - // - // This section is non-overlapping in that the copy length for this section - // is always less than or equal to the backwards distance. This can occur - // if a distance refers to data that wraps-around in the buffer. - // Thus, a backwards copy is performed here; that is, the exact bytes in - // the source prior to the copy is placed in the destination. - if (src_pos < 0) { - src_pos += @as(i32, @intCast(self.hist.len)); - dst_pos += copy(self.hist[dst_pos..end_pos], self.hist[@as(usize, @intCast(src_pos))..]); - src_pos = 0; - } - - // Copy possibly overlapping section before destination position. - // - // This section can overlap if the copy length for this section is larger - // than the backwards distance. This is allowed by LZ77 so that repeated - // strings can be succinctly represented using (dist, length) pairs. - // Thus, a forwards copy is performed here; that is, the bytes copied is - // possibly dependent on the resulting bytes in the destination as the copy - // progresses along. This is functionally equivalent to the following: - // - // var i = 0; - // while(i < end_pos - dst_pos) : (i+=1) { - // self.hist[dst_pos+i] = self.hist[src_pos+i]; - // } - // dst_pos = end_pos; - // - while (dst_pos < end_pos) { - dst_pos += copy(self.hist[dst_pos..end_pos], self.hist[@as(usize, @intCast(src_pos))..dst_pos]); - } - - self.wr_pos = dst_pos; - return dst_pos - dst_base; - } - - // Tries to copy a string at a given (distance, length) to the - // output. This specialized version is optimized for short distances. - // - // This method is designed to be inlined for performance reasons. - // - // This invariant must be kept: 0 < dist <= histSize() - pub fn tryWriteCopy(self: *Self, dist: u32, length: u32) u32 { - var dst_pos = self.wr_pos; - const end_pos = dst_pos + length; - if (dst_pos < dist or end_pos > self.hist.len) { - return 0; - } - const dst_base = dst_pos; - const src_pos = dst_pos - dist; - - // Copy possibly overlapping section before destination position. - while (dst_pos < end_pos) { - dst_pos += copy(self.hist[dst_pos..end_pos], self.hist[src_pos..dst_pos]); - } - - self.wr_pos = dst_pos; - return dst_pos - dst_base; - } - - // Returns a slice of the historical buffer that is ready to be - // emitted to the user. The data returned by readFlush must be fully consumed - // before calling any other DictDecoder methods. - pub fn readFlush(self: *Self) []u8 { - const to_read = self.hist[self.rd_pos..self.wr_pos]; - self.rd_pos = self.wr_pos; - if (self.wr_pos == self.hist.len) { - self.wr_pos = 0; - self.rd_pos = 0; - self.full = true; - } - return to_read; - } -}; - -// tests - -test "dictionary decoder" { - const ArrayList = std.ArrayList; - const testing = std.testing; - - const abc = "ABC\n"; - const fox = "The quick brown fox jumped over the lazy dog!\n"; - const poem: []const u8 = - \\The Road Not Taken - \\Robert Frost - \\ - \\Two roads diverged in a yellow wood, - \\And sorry I could not travel both - \\And be one traveler, long I stood - \\And looked down one as far as I could - \\To where it bent in the undergrowth; - \\ - \\Then took the other, as just as fair, - \\And having perhaps the better claim, - \\Because it was grassy and wanted wear; - \\Though as for that the passing there - \\Had worn them really about the same, - \\ - \\And both that morning equally lay - \\In leaves no step had trodden black. - \\Oh, I kept the first for another day! - \\Yet knowing how way leads on to way, - \\I doubted if I should ever come back. - \\ - \\I shall be telling this with a sigh - \\Somewhere ages and ages hence: - \\Two roads diverged in a wood, and I- - \\I took the one less traveled by, - \\And that has made all the difference. - \\ - ; - - const uppercase: []const u8 = - \\THE ROAD NOT TAKEN - \\ROBERT FROST - \\ - \\TWO ROADS DIVERGED IN A YELLOW WOOD, - \\AND SORRY I COULD NOT TRAVEL BOTH - \\AND BE ONE TRAVELER, LONG I STOOD - \\AND LOOKED DOWN ONE AS FAR AS I COULD - \\TO WHERE IT BENT IN THE UNDERGROWTH; - \\ - \\THEN TOOK THE OTHER, AS JUST AS FAIR, - \\AND HAVING PERHAPS THE BETTER CLAIM, - \\BECAUSE IT WAS GRASSY AND WANTED WEAR; - \\THOUGH AS FOR THAT THE PASSING THERE - \\HAD WORN THEM REALLY ABOUT THE SAME, - \\ - \\AND BOTH THAT MORNING EQUALLY LAY - \\IN LEAVES NO STEP HAD TRODDEN BLACK. - \\OH, I KEPT THE FIRST FOR ANOTHER DAY! - \\YET KNOWING HOW WAY LEADS ON TO WAY, - \\I DOUBTED IF I SHOULD EVER COME BACK. - \\ - \\I SHALL BE TELLING THIS WITH A SIGH - \\SOMEWHERE AGES AND AGES HENCE: - \\TWO ROADS DIVERGED IN A WOOD, AND I- - \\I TOOK THE ONE LESS TRAVELED BY, - \\AND THAT HAS MADE ALL THE DIFFERENCE. - \\ - ; - - const PoemRefs = struct { - dist: u32, // Backward distance (0 if this is an insertion) - length: u32, // Length of copy or insertion - }; - - const poem_refs = [_]PoemRefs{ - .{ .dist = 0, .length = 38 }, .{ .dist = 33, .length = 3 }, .{ .dist = 0, .length = 48 }, - .{ .dist = 79, .length = 3 }, .{ .dist = 0, .length = 11 }, .{ .dist = 34, .length = 5 }, - .{ .dist = 0, .length = 6 }, .{ .dist = 23, .length = 7 }, .{ .dist = 0, .length = 8 }, - .{ .dist = 50, .length = 3 }, .{ .dist = 0, .length = 2 }, .{ .dist = 69, .length = 3 }, - .{ .dist = 34, .length = 5 }, .{ .dist = 0, .length = 4 }, .{ .dist = 97, .length = 3 }, - .{ .dist = 0, .length = 4 }, .{ .dist = 43, .length = 5 }, .{ .dist = 0, .length = 6 }, - .{ .dist = 7, .length = 4 }, .{ .dist = 88, .length = 7 }, .{ .dist = 0, .length = 12 }, - .{ .dist = 80, .length = 3 }, .{ .dist = 0, .length = 2 }, .{ .dist = 141, .length = 4 }, - .{ .dist = 0, .length = 1 }, .{ .dist = 196, .length = 3 }, .{ .dist = 0, .length = 3 }, - .{ .dist = 157, .length = 3 }, .{ .dist = 0, .length = 6 }, .{ .dist = 181, .length = 3 }, - .{ .dist = 0, .length = 2 }, .{ .dist = 23, .length = 3 }, .{ .dist = 77, .length = 3 }, - .{ .dist = 28, .length = 5 }, .{ .dist = 128, .length = 3 }, .{ .dist = 110, .length = 4 }, - .{ .dist = 70, .length = 3 }, .{ .dist = 0, .length = 4 }, .{ .dist = 85, .length = 6 }, - .{ .dist = 0, .length = 2 }, .{ .dist = 182, .length = 6 }, .{ .dist = 0, .length = 4 }, - .{ .dist = 133, .length = 3 }, .{ .dist = 0, .length = 7 }, .{ .dist = 47, .length = 5 }, - .{ .dist = 0, .length = 20 }, .{ .dist = 112, .length = 5 }, .{ .dist = 0, .length = 1 }, - .{ .dist = 58, .length = 3 }, .{ .dist = 0, .length = 8 }, .{ .dist = 59, .length = 3 }, - .{ .dist = 0, .length = 4 }, .{ .dist = 173, .length = 3 }, .{ .dist = 0, .length = 5 }, - .{ .dist = 114, .length = 3 }, .{ .dist = 0, .length = 4 }, .{ .dist = 92, .length = 5 }, - .{ .dist = 0, .length = 2 }, .{ .dist = 71, .length = 3 }, .{ .dist = 0, .length = 2 }, - .{ .dist = 76, .length = 5 }, .{ .dist = 0, .length = 1 }, .{ .dist = 46, .length = 3 }, - .{ .dist = 96, .length = 4 }, .{ .dist = 130, .length = 4 }, .{ .dist = 0, .length = 3 }, - .{ .dist = 360, .length = 3 }, .{ .dist = 0, .length = 3 }, .{ .dist = 178, .length = 5 }, - .{ .dist = 0, .length = 7 }, .{ .dist = 75, .length = 3 }, .{ .dist = 0, .length = 3 }, - .{ .dist = 45, .length = 6 }, .{ .dist = 0, .length = 6 }, .{ .dist = 299, .length = 6 }, - .{ .dist = 180, .length = 3 }, .{ .dist = 70, .length = 6 }, .{ .dist = 0, .length = 1 }, - .{ .dist = 48, .length = 3 }, .{ .dist = 66, .length = 4 }, .{ .dist = 0, .length = 3 }, - .{ .dist = 47, .length = 5 }, .{ .dist = 0, .length = 9 }, .{ .dist = 325, .length = 3 }, - .{ .dist = 0, .length = 1 }, .{ .dist = 359, .length = 3 }, .{ .dist = 318, .length = 3 }, - .{ .dist = 0, .length = 2 }, .{ .dist = 199, .length = 3 }, .{ .dist = 0, .length = 1 }, - .{ .dist = 344, .length = 3 }, .{ .dist = 0, .length = 3 }, .{ .dist = 248, .length = 3 }, - .{ .dist = 0, .length = 10 }, .{ .dist = 310, .length = 3 }, .{ .dist = 0, .length = 3 }, - .{ .dist = 93, .length = 6 }, .{ .dist = 0, .length = 3 }, .{ .dist = 252, .length = 3 }, - .{ .dist = 157, .length = 4 }, .{ .dist = 0, .length = 2 }, .{ .dist = 273, .length = 5 }, - .{ .dist = 0, .length = 14 }, .{ .dist = 99, .length = 4 }, .{ .dist = 0, .length = 1 }, - .{ .dist = 464, .length = 4 }, .{ .dist = 0, .length = 2 }, .{ .dist = 92, .length = 4 }, - .{ .dist = 495, .length = 3 }, .{ .dist = 0, .length = 1 }, .{ .dist = 322, .length = 4 }, - .{ .dist = 16, .length = 4 }, .{ .dist = 0, .length = 3 }, .{ .dist = 402, .length = 3 }, - .{ .dist = 0, .length = 2 }, .{ .dist = 237, .length = 4 }, .{ .dist = 0, .length = 2 }, - .{ .dist = 432, .length = 4 }, .{ .dist = 0, .length = 1 }, .{ .dist = 483, .length = 5 }, - .{ .dist = 0, .length = 2 }, .{ .dist = 294, .length = 4 }, .{ .dist = 0, .length = 2 }, - .{ .dist = 306, .length = 3 }, .{ .dist = 113, .length = 5 }, .{ .dist = 0, .length = 1 }, - .{ .dist = 26, .length = 4 }, .{ .dist = 164, .length = 3 }, .{ .dist = 488, .length = 4 }, - .{ .dist = 0, .length = 1 }, .{ .dist = 542, .length = 3 }, .{ .dist = 248, .length = 6 }, - .{ .dist = 0, .length = 5 }, .{ .dist = 205, .length = 3 }, .{ .dist = 0, .length = 8 }, - .{ .dist = 48, .length = 3 }, .{ .dist = 449, .length = 6 }, .{ .dist = 0, .length = 2 }, - .{ .dist = 192, .length = 3 }, .{ .dist = 328, .length = 4 }, .{ .dist = 9, .length = 5 }, - .{ .dist = 433, .length = 3 }, .{ .dist = 0, .length = 3 }, .{ .dist = 622, .length = 25 }, - .{ .dist = 615, .length = 5 }, .{ .dist = 46, .length = 5 }, .{ .dist = 0, .length = 2 }, - .{ .dist = 104, .length = 3 }, .{ .dist = 475, .length = 10 }, .{ .dist = 549, .length = 3 }, - .{ .dist = 0, .length = 4 }, .{ .dist = 597, .length = 8 }, .{ .dist = 314, .length = 3 }, - .{ .dist = 0, .length = 1 }, .{ .dist = 473, .length = 6 }, .{ .dist = 317, .length = 5 }, - .{ .dist = 0, .length = 1 }, .{ .dist = 400, .length = 3 }, .{ .dist = 0, .length = 3 }, - .{ .dist = 109, .length = 3 }, .{ .dist = 151, .length = 3 }, .{ .dist = 48, .length = 4 }, - .{ .dist = 0, .length = 4 }, .{ .dist = 125, .length = 3 }, .{ .dist = 108, .length = 3 }, - .{ .dist = 0, .length = 2 }, - }; - - var got_list = ArrayList(u8).init(testing.allocator); - defer got_list.deinit(); - var got = got_list.writer(); - - var want_list = ArrayList(u8).init(testing.allocator); - defer want_list.deinit(); - var want = want_list.writer(); - - var dd = DictDecoder{}; - try dd.init(testing.allocator, 1 << 11, null); - defer dd.deinit(); - - const util = struct { - fn writeCopy(dst_dd: *DictDecoder, dst: anytype, dist: u32, length: u32) !void { - var len = length; - while (len > 0) { - var n = dst_dd.tryWriteCopy(dist, len); - if (n == 0) { - n = dst_dd.writeCopy(dist, len); - } - - len -= n; - if (dst_dd.availWrite() == 0) { - _ = try dst.write(dst_dd.readFlush()); - } - } - } - fn writeString(dst_dd: *DictDecoder, dst: anytype, str: []const u8) !void { - var string = str; - while (string.len > 0) { - const cnt = DictDecoder.copy(dst_dd.writeSlice(), string); - dst_dd.writeMark(cnt); - string = string[cnt..]; - if (dst_dd.availWrite() == 0) { - _ = try dst.write(dst_dd.readFlush()); - } - } - } - }; - - try util.writeString(&dd, got, "."); - _ = try want.write("."); - - var str = poem; - for (poem_refs, 0..) |ref, i| { - _ = i; - if (ref.dist == 0) { - try util.writeString(&dd, got, str[0..ref.length]); - } else { - try util.writeCopy(&dd, got, ref.dist, ref.length); - } - str = str[ref.length..]; - } - _ = try want.write(poem); - - try util.writeCopy(&dd, got, dd.histSize(), 33); - _ = try want.write(want_list.items[0..33]); - - try util.writeString(&dd, got, abc); - try util.writeCopy(&dd, got, abc.len, 59 * abc.len); - _ = try want.write(abc ** 60); - - try util.writeString(&dd, got, fox); - try util.writeCopy(&dd, got, fox.len, 9 * fox.len); - _ = try want.write(fox ** 10); - - try util.writeString(&dd, got, "."); - try util.writeCopy(&dd, got, 1, 9); - _ = try want.write("." ** 10); - - try util.writeString(&dd, got, uppercase); - try util.writeCopy(&dd, got, uppercase.len, 7 * uppercase.len); - var i: u8 = 0; - while (i < 8) : (i += 1) { - _ = try want.write(uppercase); - } - - try util.writeCopy(&dd, got, dd.histSize(), 10); - _ = try want.write(want_list.items[want_list.items.len - dd.histSize() ..][0..10]); - - _ = try got.write(dd.readFlush()); - try testing.expectEqualSlices(u8, want_list.items, got_list.items); -} diff --git a/lib/std/compress/deflate/huffman_bit_writer.zig b/lib/std/compress/deflate/huffman_bit_writer.zig deleted file mode 100644 index a79dc91aa890..000000000000 --- a/lib/std/compress/deflate/huffman_bit_writer.zig +++ /dev/null @@ -1,1686 +0,0 @@ -const std = @import("std"); -const io = std.io; - -const Allocator = std.mem.Allocator; - -const deflate_const = @import("deflate_const.zig"); -const hm_code = @import("huffman_code.zig"); -const token = @import("token.zig"); - -// The first length code. -const length_codes_start = 257; - -// The number of codegen codes. -const codegen_code_count = 19; -const bad_code = 255; - -// buffer_flush_size indicates the buffer size -// after which bytes are flushed to the writer. -// Should preferably be a multiple of 6, since -// we accumulate 6 bytes between writes to the buffer. -const buffer_flush_size = 240; - -// buffer_size is the actual output byte buffer size. -// It must have additional headroom for a flush -// which can contain up to 8 bytes. -const buffer_size = buffer_flush_size + 8; - -// The number of extra bits needed by length code X - LENGTH_CODES_START. -var length_extra_bits = [_]u8{ - 0, 0, 0, // 257 - 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, // 260 - 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, // 270 - 4, 5, 5, 5, 5, 0, // 280 -}; - -// The length indicated by length code X - LENGTH_CODES_START. -var length_base = [_]u32{ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, - 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, - 64, 80, 96, 112, 128, 160, 192, 224, 255, -}; - -// offset code word extra bits. -var offset_extra_bits = [_]i8{ - 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, - 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, - 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, -}; - -var offset_base = [_]u32{ - 0x000000, 0x000001, 0x000002, 0x000003, 0x000004, - 0x000006, 0x000008, 0x00000c, 0x000010, 0x000018, - 0x000020, 0x000030, 0x000040, 0x000060, 0x000080, - 0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300, - 0x000400, 0x000600, 0x000800, 0x000c00, 0x001000, - 0x001800, 0x002000, 0x003000, 0x004000, 0x006000, -}; - -// The odd order in which the codegen code sizes are written. -var codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; - -pub fn HuffmanBitWriter(comptime WriterType: type) type { - return struct { - const Self = @This(); - pub const Error = WriterType.Error; - - // writer is the underlying writer. - // Do not use it directly; use the write method, which ensures - // that Write errors are sticky. - inner_writer: WriterType, - bytes_written: usize, - - // Data waiting to be written is bytes[0 .. nbytes] - // and then the low nbits of bits. Data is always written - // sequentially into the bytes array. - bits: u64, - nbits: u32, // number of bits - bytes: [buffer_size]u8, - codegen_freq: [codegen_code_count]u16, - nbytes: u32, // number of bytes - literal_freq: []u16, - offset_freq: []u16, - codegen: []u8, - literal_encoding: hm_code.HuffmanEncoder, - offset_encoding: hm_code.HuffmanEncoder, - codegen_encoding: hm_code.HuffmanEncoder, - err: bool = false, - fixed_literal_encoding: hm_code.HuffmanEncoder, - fixed_offset_encoding: hm_code.HuffmanEncoder, - allocator: Allocator, - huff_offset: hm_code.HuffmanEncoder, - - pub fn reset(self: *Self, new_writer: WriterType) void { - self.inner_writer = new_writer; - self.bytes_written = 0; - self.bits = 0; - self.nbits = 0; - self.nbytes = 0; - self.err = false; - } - - pub fn flush(self: *Self) Error!void { - if (self.err) { - self.nbits = 0; - return; - } - var n = self.nbytes; - while (self.nbits != 0) { - self.bytes[n] = @as(u8, @truncate(self.bits)); - self.bits >>= 8; - if (self.nbits > 8) { // Avoid underflow - self.nbits -= 8; - } else { - self.nbits = 0; - } - n += 1; - } - self.bits = 0; - try self.write(self.bytes[0..n]); - self.nbytes = 0; - } - - fn write(self: *Self, b: []const u8) Error!void { - if (self.err) { - return; - } - try self.inner_writer.writeAll(b); - self.bytes_written += b.len; - } - - fn writeBits(self: *Self, b: u32, nb: u32) Error!void { - if (self.err) { - return; - } - self.bits |= @as(u64, @intCast(b)) << @as(u6, @intCast(self.nbits)); - self.nbits += nb; - if (self.nbits >= 48) { - const bits = self.bits; - self.bits >>= 48; - self.nbits -= 48; - var n = self.nbytes; - var bytes = self.bytes[n..][0..6]; - bytes[0] = @as(u8, @truncate(bits)); - bytes[1] = @as(u8, @truncate(bits >> 8)); - bytes[2] = @as(u8, @truncate(bits >> 16)); - bytes[3] = @as(u8, @truncate(bits >> 24)); - bytes[4] = @as(u8, @truncate(bits >> 32)); - bytes[5] = @as(u8, @truncate(bits >> 40)); - n += 6; - if (n >= buffer_flush_size) { - try self.write(self.bytes[0..n]); - n = 0; - } - self.nbytes = n; - } - } - - pub fn writeBytes(self: *Self, bytes: []const u8) Error!void { - if (self.err) { - return; - } - var n = self.nbytes; - if (self.nbits & 7 != 0) { - self.err = true; // unfinished bits - return; - } - while (self.nbits != 0) { - self.bytes[n] = @as(u8, @truncate(self.bits)); - self.bits >>= 8; - self.nbits -= 8; - n += 1; - } - if (n != 0) { - try self.write(self.bytes[0..n]); - } - self.nbytes = 0; - try self.write(bytes); - } - - // RFC 1951 3.2.7 specifies a special run-length encoding for specifying - // the literal and offset lengths arrays (which are concatenated into a single - // array). This method generates that run-length encoding. - // - // The result is written into the codegen array, and the frequencies - // of each code is written into the codegen_freq array. - // Codes 0-15 are single byte codes. Codes 16-18 are followed by additional - // information. Code bad_code is an end marker - // - // num_literals: The number of literals in literal_encoding - // num_offsets: The number of offsets in offset_encoding - // lit_enc: The literal encoder to use - // off_enc: The offset encoder to use - fn generateCodegen( - self: *Self, - num_literals: u32, - num_offsets: u32, - lit_enc: *hm_code.HuffmanEncoder, - off_enc: *hm_code.HuffmanEncoder, - ) void { - for (self.codegen_freq, 0..) |_, i| { - self.codegen_freq[i] = 0; - } - - // Note that we are using codegen both as a temporary variable for holding - // a copy of the frequencies, and as the place where we put the result. - // This is fine because the output is always shorter than the input used - // so far. - var codegen = self.codegen; // cache - // Copy the concatenated code sizes to codegen. Put a marker at the end. - var cgnl = codegen[0..num_literals]; - for (cgnl, 0..) |_, i| { - cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len)); - } - - cgnl = codegen[num_literals .. num_literals + num_offsets]; - for (cgnl, 0..) |_, i| { - cgnl[i] = @as(u8, @intCast(off_enc.codes[i].len)); - } - codegen[num_literals + num_offsets] = bad_code; - - var size = codegen[0]; - var count: i32 = 1; - var out_index: u32 = 0; - var in_index: u32 = 1; - while (size != bad_code) : (in_index += 1) { - // INVARIANT: We have seen "count" copies of size that have not yet - // had output generated for them. - const next_size = codegen[in_index]; - if (next_size == size) { - count += 1; - continue; - } - // We need to generate codegen indicating "count" of size. - if (size != 0) { - codegen[out_index] = size; - out_index += 1; - self.codegen_freq[size] += 1; - count -= 1; - while (count >= 3) { - var n: i32 = 6; - if (n > count) { - n = count; - } - codegen[out_index] = 16; - out_index += 1; - codegen[out_index] = @as(u8, @intCast(n - 3)); - out_index += 1; - self.codegen_freq[16] += 1; - count -= n; - } - } else { - while (count >= 11) { - var n: i32 = 138; - if (n > count) { - n = count; - } - codegen[out_index] = 18; - out_index += 1; - codegen[out_index] = @as(u8, @intCast(n - 11)); - out_index += 1; - self.codegen_freq[18] += 1; - count -= n; - } - if (count >= 3) { - // 3 <= count <= 10 - codegen[out_index] = 17; - out_index += 1; - codegen[out_index] = @as(u8, @intCast(count - 3)); - out_index += 1; - self.codegen_freq[17] += 1; - count = 0; - } - } - count -= 1; - while (count >= 0) : (count -= 1) { - codegen[out_index] = size; - out_index += 1; - self.codegen_freq[size] += 1; - } - // Set up invariant for next time through the loop. - size = next_size; - count = 1; - } - // Marker indicating the end of the codegen. - codegen[out_index] = bad_code; - } - - // dynamicSize returns the size of dynamically encoded data in bits. - fn dynamicSize( - self: *Self, - lit_enc: *hm_code.HuffmanEncoder, // literal encoder - off_enc: *hm_code.HuffmanEncoder, // offset encoder - extra_bits: u32, - ) DynamicSize { - var num_codegens = self.codegen_freq.len; - while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) { - num_codegens -= 1; - } - const header = 3 + 5 + 5 + 4 + (3 * num_codegens) + - self.codegen_encoding.bitLength(self.codegen_freq[0..]) + - self.codegen_freq[16] * 2 + - self.codegen_freq[17] * 3 + - self.codegen_freq[18] * 7; - const size = header + - lit_enc.bitLength(self.literal_freq) + - off_enc.bitLength(self.offset_freq) + - extra_bits; - - return DynamicSize{ - .size = @as(u32, @intCast(size)), - .num_codegens = @as(u32, @intCast(num_codegens)), - }; - } - - // fixedSize returns the size of dynamically encoded data in bits. - fn fixedSize(self: *Self, extra_bits: u32) u32 { - return 3 + - self.fixed_literal_encoding.bitLength(self.literal_freq) + - self.fixed_offset_encoding.bitLength(self.offset_freq) + - extra_bits; - } - - // storedSizeFits calculates the stored size, including header. - // The function returns the size in bits and whether the block - // fits inside a single block. - fn storedSizeFits(in: ?[]const u8) StoredSize { - if (in == null) { - return .{ .size = 0, .storable = false }; - } - if (in.?.len <= deflate_const.max_store_block_size) { - return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true }; - } - return .{ .size = 0, .storable = false }; - } - - fn writeCode(self: *Self, c: hm_code.HuffCode) Error!void { - if (self.err) { - return; - } - self.bits |= @as(u64, @intCast(c.code)) << @as(u6, @intCast(self.nbits)); - self.nbits += @as(u32, @intCast(c.len)); - if (self.nbits >= 48) { - const bits = self.bits; - self.bits >>= 48; - self.nbits -= 48; - var n = self.nbytes; - var bytes = self.bytes[n..][0..6]; - bytes[0] = @as(u8, @truncate(bits)); - bytes[1] = @as(u8, @truncate(bits >> 8)); - bytes[2] = @as(u8, @truncate(bits >> 16)); - bytes[3] = @as(u8, @truncate(bits >> 24)); - bytes[4] = @as(u8, @truncate(bits >> 32)); - bytes[5] = @as(u8, @truncate(bits >> 40)); - n += 6; - if (n >= buffer_flush_size) { - try self.write(self.bytes[0..n]); - n = 0; - } - self.nbytes = n; - } - } - - // Write the header of a dynamic Huffman block to the output stream. - // - // num_literals: The number of literals specified in codegen - // num_offsets: The number of offsets specified in codegen - // num_codegens: The number of codegens used in codegen - // is_eof: Is it the end-of-file? (end of stream) - fn writeDynamicHeader( - self: *Self, - num_literals: u32, - num_offsets: u32, - num_codegens: u32, - is_eof: bool, - ) Error!void { - if (self.err) { - return; - } - var first_bits: u32 = 4; - if (is_eof) { - first_bits = 5; - } - try self.writeBits(first_bits, 3); - try self.writeBits(@as(u32, @intCast(num_literals - 257)), 5); - try self.writeBits(@as(u32, @intCast(num_offsets - 1)), 5); - try self.writeBits(@as(u32, @intCast(num_codegens - 4)), 4); - - var i: u32 = 0; - while (i < num_codegens) : (i += 1) { - const value = @as(u32, @intCast(self.codegen_encoding.codes[codegen_order[i]].len)); - try self.writeBits(@as(u32, @intCast(value)), 3); - } - - i = 0; - while (true) { - const code_word: u32 = @as(u32, @intCast(self.codegen[i])); - i += 1; - if (code_word == bad_code) { - break; - } - try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]); - - switch (code_word) { - 16 => { - try self.writeBits(@as(u32, @intCast(self.codegen[i])), 2); - i += 1; - }, - 17 => { - try self.writeBits(@as(u32, @intCast(self.codegen[i])), 3); - i += 1; - }, - 18 => { - try self.writeBits(@as(u32, @intCast(self.codegen[i])), 7); - i += 1; - }, - else => {}, - } - } - } - - pub fn writeStoredHeader(self: *Self, length: usize, is_eof: bool) Error!void { - if (self.err) { - return; - } - var flag: u32 = 0; - if (is_eof) { - flag = 1; - } - try self.writeBits(flag, 3); - try self.flush(); - try self.writeBits(@as(u32, @intCast(length)), 16); - try self.writeBits(@as(u32, @intCast(~@as(u16, @intCast(length)))), 16); - } - - fn writeFixedHeader(self: *Self, is_eof: bool) Error!void { - if (self.err) { - return; - } - // Indicate that we are a fixed Huffman block - var value: u32 = 2; - if (is_eof) { - value = 3; - } - try self.writeBits(value, 3); - } - - // Write a block of tokens with the smallest encoding. - // The original input can be supplied, and if the huffman encoded data - // is larger than the original bytes, the data will be written as a - // stored block. - // If the input is null, the tokens will always be Huffman encoded. - pub fn writeBlock( - self: *Self, - tokens: []const token.Token, - eof: bool, - input: ?[]const u8, - ) Error!void { - if (self.err) { - return; - } - - const lit_and_off = self.indexTokens(tokens); - const num_literals = lit_and_off.num_literals; - const num_offsets = lit_and_off.num_offsets; - - var extra_bits: u32 = 0; - const ret = storedSizeFits(input); - const stored_size = ret.size; - const storable = ret.storable; - - if (storable) { - // We only bother calculating the costs of the extra bits required by - // the length of offset fields (which will be the same for both fixed - // and dynamic encoding), if we need to compare those two encodings - // against stored encoding. - var length_code: u32 = length_codes_start + 8; - while (length_code < num_literals) : (length_code += 1) { - // First eight length codes have extra size = 0. - extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) * - @as(u32, @intCast(length_extra_bits[length_code - length_codes_start])); - } - var offset_code: u32 = 4; - while (offset_code < num_offsets) : (offset_code += 1) { - // First four offset codes have extra size = 0. - extra_bits += @as(u32, @intCast(self.offset_freq[offset_code])) * - @as(u32, @intCast(offset_extra_bits[offset_code])); - } - } - - // Figure out smallest code. - // Fixed Huffman baseline. - var literal_encoding = &self.fixed_literal_encoding; - var offset_encoding = &self.fixed_offset_encoding; - var size = self.fixedSize(extra_bits); - - // Dynamic Huffman? - var num_codegens: u32 = 0; - - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literal_encoding and the offset_encoding. - self.generateCodegen( - num_literals, - num_offsets, - &self.literal_encoding, - &self.offset_encoding, - ); - self.codegen_encoding.generate(self.codegen_freq[0..], 7); - const dynamic_size = self.dynamicSize( - &self.literal_encoding, - &self.offset_encoding, - extra_bits, - ); - const dyn_size = dynamic_size.size; - num_codegens = dynamic_size.num_codegens; - - if (dyn_size < size) { - size = dyn_size; - literal_encoding = &self.literal_encoding; - offset_encoding = &self.offset_encoding; - } - - // Stored bytes? - if (storable and stored_size < size) { - try self.writeStoredHeader(input.?.len, eof); - try self.writeBytes(input.?); - return; - } - - // Huffman. - if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) { - try self.writeFixedHeader(eof); - } else { - try self.writeDynamicHeader(num_literals, num_offsets, num_codegens, eof); - } - - // Write the tokens. - try self.writeTokens(tokens, literal_encoding.codes, offset_encoding.codes); - } - - // writeBlockDynamic encodes a block using a dynamic Huffman table. - // This should be used if the symbols used have a disproportionate - // histogram distribution. - // If input is supplied and the compression savings are below 1/16th of the - // input size the block is stored. - pub fn writeBlockDynamic( - self: *Self, - tokens: []const token.Token, - eof: bool, - input: ?[]const u8, - ) Error!void { - if (self.err) { - return; - } - - const total_tokens = self.indexTokens(tokens); - const num_literals = total_tokens.num_literals; - const num_offsets = total_tokens.num_offsets; - - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literal_encoding and the offset_encoding. - self.generateCodegen( - num_literals, - num_offsets, - &self.literal_encoding, - &self.offset_encoding, - ); - self.codegen_encoding.generate(self.codegen_freq[0..], 7); - const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.offset_encoding, 0); - const size = dynamic_size.size; - const num_codegens = dynamic_size.num_codegens; - - // Store bytes, if we don't get a reasonable improvement. - - const stored_size = storedSizeFits(input); - const ssize = stored_size.size; - const storable = stored_size.storable; - if (storable and ssize < (size + (size >> 4))) { - try self.writeStoredHeader(input.?.len, eof); - try self.writeBytes(input.?); - return; - } - - // Write Huffman table. - try self.writeDynamicHeader(num_literals, num_offsets, num_codegens, eof); - - // Write the tokens. - try self.writeTokens(tokens, self.literal_encoding.codes, self.offset_encoding.codes); - } - - const TotalIndexedTokens = struct { - num_literals: u32, - num_offsets: u32, - }; - - // Indexes a slice of tokens followed by an end_block_marker, and updates - // literal_freq and offset_freq, and generates literal_encoding - // and offset_encoding. - // The number of literal and offset tokens is returned. - fn indexTokens(self: *Self, tokens: []const token.Token) TotalIndexedTokens { - var num_literals: u32 = 0; - var num_offsets: u32 = 0; - - for (self.literal_freq, 0..) |_, i| { - self.literal_freq[i] = 0; - } - for (self.offset_freq, 0..) |_, i| { - self.offset_freq[i] = 0; - } - - for (tokens) |t| { - if (t < token.match_type) { - self.literal_freq[token.literal(t)] += 1; - continue; - } - const length = token.length(t); - const offset = token.offset(t); - self.literal_freq[length_codes_start + token.lengthCode(length)] += 1; - self.offset_freq[token.offsetCode(offset)] += 1; - } - // add end_block_marker token at the end - self.literal_freq[token.literal(deflate_const.end_block_marker)] += 1; - - // get the number of literals - num_literals = @as(u32, @intCast(self.literal_freq.len)); - while (self.literal_freq[num_literals - 1] == 0) { - num_literals -= 1; - } - // get the number of offsets - num_offsets = @as(u32, @intCast(self.offset_freq.len)); - while (num_offsets > 0 and self.offset_freq[num_offsets - 1] == 0) { - num_offsets -= 1; - } - if (num_offsets == 0) { - // We haven't found a single match. If we want to go with the dynamic encoding, - // we should count at least one offset to be sure that the offset huffman tree could be encoded. - self.offset_freq[0] = 1; - num_offsets = 1; - } - self.literal_encoding.generate(self.literal_freq, 15); - self.offset_encoding.generate(self.offset_freq, 15); - return TotalIndexedTokens{ - .num_literals = num_literals, - .num_offsets = num_offsets, - }; - } - - // Writes a slice of tokens to the output followed by and end_block_marker. - // codes for literal and offset encoding must be supplied. - fn writeTokens( - self: *Self, - tokens: []const token.Token, - le_codes: []hm_code.HuffCode, - oe_codes: []hm_code.HuffCode, - ) Error!void { - if (self.err) { - return; - } - for (tokens) |t| { - if (t < token.match_type) { - try self.writeCode(le_codes[token.literal(t)]); - continue; - } - // Write the length - const length = token.length(t); - const length_code = token.lengthCode(length); - try self.writeCode(le_codes[length_code + length_codes_start]); - const extra_length_bits = @as(u32, @intCast(length_extra_bits[length_code])); - if (extra_length_bits > 0) { - const extra_length = @as(u32, @intCast(length - length_base[length_code])); - try self.writeBits(extra_length, extra_length_bits); - } - // Write the offset - const offset = token.offset(t); - const offset_code = token.offsetCode(offset); - try self.writeCode(oe_codes[offset_code]); - const extra_offset_bits = @as(u32, @intCast(offset_extra_bits[offset_code])); - if (extra_offset_bits > 0) { - const extra_offset = @as(u32, @intCast(offset - offset_base[offset_code])); - try self.writeBits(extra_offset, extra_offset_bits); - } - } - // add end_block_marker at the end - try self.writeCode(le_codes[token.literal(deflate_const.end_block_marker)]); - } - - // Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes - // if the results only gains very little from compression. - pub fn writeBlockHuff(self: *Self, eof: bool, input: []const u8) Error!void { - if (self.err) { - return; - } - - // Clear histogram - for (self.literal_freq, 0..) |_, i| { - self.literal_freq[i] = 0; - } - - // Add everything as literals - histogram(input, &self.literal_freq); - - self.literal_freq[deflate_const.end_block_marker] = 1; - - const num_literals = deflate_const.end_block_marker + 1; - self.offset_freq[0] = 1; - const num_offsets = 1; - - self.literal_encoding.generate(self.literal_freq, 15); - - // Figure out smallest code. - // Always use dynamic Huffman or Store - var num_codegens: u32 = 0; - - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literal_encoding and the offset_encoding. - self.generateCodegen( - num_literals, - num_offsets, - &self.literal_encoding, - &self.huff_offset, - ); - self.codegen_encoding.generate(self.codegen_freq[0..], 7); - const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_offset, 0); - const size = dynamic_size.size; - num_codegens = dynamic_size.num_codegens; - - // Store bytes, if we don't get a reasonable improvement. - - const stored_size_ret = storedSizeFits(input); - const ssize = stored_size_ret.size; - const storable = stored_size_ret.storable; - - if (storable and ssize < (size + (size >> 4))) { - try self.writeStoredHeader(input.len, eof); - try self.writeBytes(input); - return; - } - - // Huffman. - try self.writeDynamicHeader(num_literals, num_offsets, num_codegens, eof); - const encoding = self.literal_encoding.codes[0..257]; - var n = self.nbytes; - for (input) |t| { - // Bitwriting inlined, ~30% speedup - const c = encoding[t]; - self.bits |= @as(u64, @intCast(c.code)) << @as(u6, @intCast(self.nbits)); - self.nbits += @as(u32, @intCast(c.len)); - if (self.nbits < 48) { - continue; - } - // Store 6 bytes - const bits = self.bits; - self.bits >>= 48; - self.nbits -= 48; - var bytes = self.bytes[n..][0..6]; - bytes[0] = @as(u8, @truncate(bits)); - bytes[1] = @as(u8, @truncate(bits >> 8)); - bytes[2] = @as(u8, @truncate(bits >> 16)); - bytes[3] = @as(u8, @truncate(bits >> 24)); - bytes[4] = @as(u8, @truncate(bits >> 32)); - bytes[5] = @as(u8, @truncate(bits >> 40)); - n += 6; - if (n < buffer_flush_size) { - continue; - } - try self.write(self.bytes[0..n]); - if (self.err) { - return; // Return early in the event of write failures - } - n = 0; - } - self.nbytes = n; - try self.writeCode(encoding[deflate_const.end_block_marker]); - } - - pub fn deinit(self: *Self) void { - self.allocator.free(self.literal_freq); - self.allocator.free(self.offset_freq); - self.allocator.free(self.codegen); - self.literal_encoding.deinit(); - self.codegen_encoding.deinit(); - self.offset_encoding.deinit(); - self.fixed_literal_encoding.deinit(); - self.fixed_offset_encoding.deinit(); - self.huff_offset.deinit(); - } - }; -} - -const DynamicSize = struct { - size: u32, - num_codegens: u32, -}; - -const StoredSize = struct { - size: u32, - storable: bool, -}; - -pub fn huffmanBitWriter(allocator: Allocator, writer: anytype) !HuffmanBitWriter(@TypeOf(writer)) { - var offset_freq = [1]u16{0} ** deflate_const.offset_code_count; - offset_freq[0] = 1; - // huff_offset is a static offset encoder used for huffman only encoding. - // It can be reused since we will not be encoding offset values. - var huff_offset = try hm_code.newHuffmanEncoder(allocator, deflate_const.offset_code_count); - huff_offset.generate(offset_freq[0..], 15); - - return HuffmanBitWriter(@TypeOf(writer)){ - .inner_writer = writer, - .bytes_written = 0, - .bits = 0, - .nbits = 0, - .nbytes = 0, - .bytes = [1]u8{0} ** buffer_size, - .codegen_freq = [1]u16{0} ** codegen_code_count, - .literal_freq = try allocator.alloc(u16, deflate_const.max_num_lit), - .offset_freq = try allocator.alloc(u16, deflate_const.offset_code_count), - .codegen = try allocator.alloc(u8, deflate_const.max_num_lit + deflate_const.offset_code_count + 1), - .literal_encoding = try hm_code.newHuffmanEncoder(allocator, deflate_const.max_num_lit), - .codegen_encoding = try hm_code.newHuffmanEncoder(allocator, codegen_code_count), - .offset_encoding = try hm_code.newHuffmanEncoder(allocator, deflate_const.offset_code_count), - .allocator = allocator, - .fixed_literal_encoding = try hm_code.generateFixedLiteralEncoding(allocator), - .fixed_offset_encoding = try hm_code.generateFixedOffsetEncoding(allocator), - .huff_offset = huff_offset, - }; -} - -// histogram accumulates a histogram of b in h. -// -// h.len must be >= 256, and h's elements must be all zeroes. -fn histogram(b: []const u8, h: *[]u16) void { - var lh = h.*[0..256]; - for (b) |t| { - lh[t] += 1; - } -} - -// tests -const expect = std.testing.expect; -const fmt = std.fmt; -const math = std.math; -const mem = std.mem; -const testing = std.testing; - -const ArrayList = std.ArrayList; - -test "writeBlockHuff" { - // Tests huffman encoding against reference files to detect possible regressions. - // If encoding/bit allocation changes you can regenerate these files - - try testBlockHuff( - "huffman-null-max.input", - "huffman-null-max.golden", - ); - try testBlockHuff( - "huffman-pi.input", - "huffman-pi.golden", - ); - try testBlockHuff( - "huffman-rand-1k.input", - "huffman-rand-1k.golden", - ); - try testBlockHuff( - "huffman-rand-limit.input", - "huffman-rand-limit.golden", - ); - try testBlockHuff( - "huffman-rand-max.input", - "huffman-rand-max.golden", - ); - try testBlockHuff( - "huffman-shifts.input", - "huffman-shifts.golden", - ); - try testBlockHuff( - "huffman-text.input", - "huffman-text.golden", - ); - try testBlockHuff( - "huffman-text-shift.input", - "huffman-text-shift.golden", - ); - try testBlockHuff( - "huffman-zero.input", - "huffman-zero.golden", - ); -} - -fn testBlockHuff(comptime in_name: []const u8, comptime want_name: []const u8) !void { - const in: []const u8 = @embedFile("testdata/" ++ in_name); - const want: []const u8 = @embedFile("testdata/" ++ want_name); - - var buf = ArrayList(u8).init(testing.allocator); - defer buf.deinit(); - var bw = try huffmanBitWriter(testing.allocator, buf.writer()); - defer bw.deinit(); - try bw.writeBlockHuff(false, in); - try bw.flush(); - - try std.testing.expectEqualSlices(u8, want, buf.items); - - // Test if the writer produces the same output after reset. - var buf_after_reset = ArrayList(u8).init(testing.allocator); - defer buf_after_reset.deinit(); - - bw.reset(buf_after_reset.writer()); - - try bw.writeBlockHuff(false, in); - try bw.flush(); - - try std.testing.expectEqualSlices(u8, buf.items, buf_after_reset.items); - try std.testing.expectEqualSlices(u8, want, buf_after_reset.items); - - try testWriterEOF(.write_huffman_block, &[0]token.Token{}, in); -} - -const HuffTest = struct { - tokens: []const token.Token, - input: []const u8 = "", // File name of input data matching the tokens. - want: []const u8 = "", // File name of data with the expected output with input available. - want_no_input: []const u8 = "", // File name of the expected output when no input is available. -}; - -const ml = 0x7fc00000; // Maximum length token. Used to reduce the size of writeBlockTests - -const writeBlockTests = &[_]HuffTest{ - HuffTest{ - .input = "huffman-null-max.input", - .want = "huffman-null-max.{s}.expect", - .want_no_input = "huffman-null-max.{s}.expect-noinput", - .tokens = &[_]token.Token{ - 0x0, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, 0x0, 0x0, - }, - }, - HuffTest{ - .input = "huffman-pi.input", - .want = "huffman-pi.{s}.expect", - .want_no_input = "huffman-pi.{s}.expect-noinput", - .tokens = &[_]token.Token{ - 0x33, 0x2e, 0x31, 0x34, 0x31, 0x35, 0x39, - 0x32, 0x36, 0x35, 0x33, 0x35, 0x38, 0x39, - 0x37, 0x39, 0x33, 0x32, 0x33, 0x38, 0x34, - 0x36, 0x32, 0x36, 0x34, 0x33, 0x33, 0x38, - 0x33, 0x32, 0x37, 0x39, 0x35, 0x30, 0x32, - 0x38, 0x38, 0x34, 0x31, 0x39, 0x37, 0x31, - 0x36, 0x39, 0x33, 0x39, 0x39, 0x33, 0x37, - 0x35, 0x31, 0x30, 0x35, 0x38, 0x32, 0x30, - 0x39, 0x37, 0x34, 0x39, 0x34, 0x34, 0x35, - 0x39, 0x32, 0x33, 0x30, 0x37, 0x38, 0x31, - 0x36, 0x34, 0x30, 0x36, 0x32, 0x38, 0x36, - 0x32, 0x30, 0x38, 0x39, 0x39, 0x38, 0x36, - 0x32, 0x38, 0x30, 0x33, 0x34, 0x38, 0x32, - 0x35, 0x33, 0x34, 0x32, 0x31, 0x31, 0x37, - 0x30, 0x36, 0x37, 0x39, 0x38, 0x32, 0x31, - 0x34, 0x38, 0x30, 0x38, 0x36, 0x35, 0x31, - 0x33, 0x32, 0x38, 0x32, 0x33, 0x30, 0x36, - 0x36, 0x34, 0x37, 0x30, 0x39, 0x33, 0x38, - 0x34, 0x34, 0x36, 0x30, 0x39, 0x35, 0x35, - 0x30, 0x35, 0x38, 0x32, 0x32, 0x33, 0x31, - 0x37, 0x32, 0x35, 0x33, 0x35, 0x39, 0x34, - 0x30, 0x38, 0x31, 0x32, 0x38, 0x34, 0x38, - 0x31, 0x31, 0x31, 0x37, 0x34, 0x4040007e, 0x34, - 0x31, 0x30, 0x32, 0x37, 0x30, 0x31, 0x39, - 0x33, 0x38, 0x35, 0x32, 0x31, 0x31, 0x30, - 0x35, 0x35, 0x35, 0x39, 0x36, 0x34, 0x34, - 0x36, 0x32, 0x32, 0x39, 0x34, 0x38, 0x39, - 0x35, 0x34, 0x39, 0x33, 0x30, 0x33, 0x38, - 0x31, 0x40400012, 0x32, 0x38, 0x38, 0x31, 0x30, - 0x39, 0x37, 0x35, 0x36, 0x36, 0x35, 0x39, - 0x33, 0x33, 0x34, 0x34, 0x36, 0x40400047, 0x37, - 0x35, 0x36, 0x34, 0x38, 0x32, 0x33, 0x33, - 0x37, 0x38, 0x36, 0x37, 0x38, 0x33, 0x31, - 0x36, 0x35, 0x32, 0x37, 0x31, 0x32, 0x30, - 0x31, 0x39, 0x30, 0x39, 0x31, 0x34, 0x4040001a, - 0x35, 0x36, 0x36, 0x39, 0x32, 0x33, 0x34, - 0x36, 0x404000b2, 0x36, 0x31, 0x30, 0x34, 0x35, - 0x34, 0x33, 0x32, 0x36, 0x40400032, 0x31, 0x33, - 0x33, 0x39, 0x33, 0x36, 0x30, 0x37, 0x32, - 0x36, 0x30, 0x32, 0x34, 0x39, 0x31, 0x34, - 0x31, 0x32, 0x37, 0x33, 0x37, 0x32, 0x34, - 0x35, 0x38, 0x37, 0x30, 0x30, 0x36, 0x36, - 0x30, 0x36, 0x33, 0x31, 0x35, 0x35, 0x38, - 0x38, 0x31, 0x37, 0x34, 0x38, 0x38, 0x31, - 0x35, 0x32, 0x30, 0x39, 0x32, 0x30, 0x39, - 0x36, 0x32, 0x38, 0x32, 0x39, 0x32, 0x35, - 0x34, 0x30, 0x39, 0x31, 0x37, 0x31, 0x35, - 0x33, 0x36, 0x34, 0x33, 0x36, 0x37, 0x38, - 0x39, 0x32, 0x35, 0x39, 0x30, 0x33, 0x36, - 0x30, 0x30, 0x31, 0x31, 0x33, 0x33, 0x30, - 0x35, 0x33, 0x30, 0x35, 0x34, 0x38, 0x38, - 0x32, 0x30, 0x34, 0x36, 0x36, 0x35, 0x32, - 0x31, 0x33, 0x38, 0x34, 0x31, 0x34, 0x36, - 0x39, 0x35, 0x31, 0x39, 0x34, 0x31, 0x35, - 0x31, 0x31, 0x36, 0x30, 0x39, 0x34, 0x33, - 0x33, 0x30, 0x35, 0x37, 0x32, 0x37, 0x30, - 0x33, 0x36, 0x35, 0x37, 0x35, 0x39, 0x35, - 0x39, 0x31, 0x39, 0x35, 0x33, 0x30, 0x39, - 0x32, 0x31, 0x38, 0x36, 0x31, 0x31, 0x37, - 0x404000e9, 0x33, 0x32, 0x40400009, 0x39, 0x33, 0x31, - 0x30, 0x35, 0x31, 0x31, 0x38, 0x35, 0x34, - 0x38, 0x30, 0x37, 0x4040010e, 0x33, 0x37, 0x39, - 0x39, 0x36, 0x32, 0x37, 0x34, 0x39, 0x35, - 0x36, 0x37, 0x33, 0x35, 0x31, 0x38, 0x38, - 0x35, 0x37, 0x35, 0x32, 0x37, 0x32, 0x34, - 0x38, 0x39, 0x31, 0x32, 0x32, 0x37, 0x39, - 0x33, 0x38, 0x31, 0x38, 0x33, 0x30, 0x31, - 0x31, 0x39, 0x34, 0x39, 0x31, 0x32, 0x39, - 0x38, 0x33, 0x33, 0x36, 0x37, 0x33, 0x33, - 0x36, 0x32, 0x34, 0x34, 0x30, 0x36, 0x35, - 0x36, 0x36, 0x34, 0x33, 0x30, 0x38, 0x36, - 0x30, 0x32, 0x31, 0x33, 0x39, 0x34, 0x39, - 0x34, 0x36, 0x33, 0x39, 0x35, 0x32, 0x32, - 0x34, 0x37, 0x33, 0x37, 0x31, 0x39, 0x30, - 0x37, 0x30, 0x32, 0x31, 0x37, 0x39, 0x38, - 0x40800099, 0x37, 0x30, 0x32, 0x37, 0x37, 0x30, - 0x35, 0x33, 0x39, 0x32, 0x31, 0x37, 0x31, - 0x37, 0x36, 0x32, 0x39, 0x33, 0x31, 0x37, - 0x36, 0x37, 0x35, 0x40800232, 0x37, 0x34, 0x38, - 0x31, 0x40400006, 0x36, 0x36, 0x39, 0x34, 0x30, - 0x404001e7, 0x30, 0x30, 0x30, 0x35, 0x36, 0x38, - 0x31, 0x32, 0x37, 0x31, 0x34, 0x35, 0x32, - 0x36, 0x33, 0x35, 0x36, 0x30, 0x38, 0x32, - 0x37, 0x37, 0x38, 0x35, 0x37, 0x37, 0x31, - 0x33, 0x34, 0x32, 0x37, 0x35, 0x37, 0x37, - 0x38, 0x39, 0x36, 0x40400129, 0x33, 0x36, 0x33, - 0x37, 0x31, 0x37, 0x38, 0x37, 0x32, 0x31, - 0x34, 0x36, 0x38, 0x34, 0x34, 0x30, 0x39, - 0x30, 0x31, 0x32, 0x32, 0x34, 0x39, 0x35, - 0x33, 0x34, 0x33, 0x30, 0x31, 0x34, 0x36, - 0x35, 0x34, 0x39, 0x35, 0x38, 0x35, 0x33, - 0x37, 0x31, 0x30, 0x35, 0x30, 0x37, 0x39, - 0x404000ca, 0x36, 0x40400153, 0x38, 0x39, 0x32, 0x33, - 0x35, 0x34, 0x404001c9, 0x39, 0x35, 0x36, 0x31, - 0x31, 0x32, 0x31, 0x32, 0x39, 0x30, 0x32, - 0x31, 0x39, 0x36, 0x30, 0x38, 0x36, 0x34, - 0x30, 0x33, 0x34, 0x34, 0x31, 0x38, 0x31, - 0x35, 0x39, 0x38, 0x31, 0x33, 0x36, 0x32, - 0x39, 0x37, 0x37, 0x34, 0x40400074, 0x30, 0x39, - 0x39, 0x36, 0x30, 0x35, 0x31, 0x38, 0x37, - 0x30, 0x37, 0x32, 0x31, 0x31, 0x33, 0x34, - 0x39, 0x40800000, 0x38, 0x33, 0x37, 0x32, 0x39, - 0x37, 0x38, 0x30, 0x34, 0x39, 0x39, 0x404002da, - 0x39, 0x37, 0x33, 0x31, 0x37, 0x33, 0x32, - 0x38, 0x4040018a, 0x36, 0x33, 0x31, 0x38, 0x35, - 0x40400301, 0x404002e8, 0x34, 0x35, 0x35, 0x33, 0x34, - 0x36, 0x39, 0x30, 0x38, 0x33, 0x30, 0x32, - 0x36, 0x34, 0x32, 0x35, 0x32, 0x32, 0x33, - 0x30, 0x404002e3, 0x40400267, 0x38, 0x35, 0x30, 0x33, - 0x35, 0x32, 0x36, 0x31, 0x39, 0x33, 0x31, - 0x31, 0x40400212, 0x31, 0x30, 0x31, 0x30, 0x30, - 0x30, 0x33, 0x31, 0x33, 0x37, 0x38, 0x33, - 0x38, 0x37, 0x35, 0x32, 0x38, 0x38, 0x36, - 0x35, 0x38, 0x37, 0x35, 0x33, 0x33, 0x32, - 0x30, 0x38, 0x33, 0x38, 0x31, 0x34, 0x32, - 0x30, 0x36, 0x40400140, 0x4040012b, 0x31, 0x34, 0x37, - 0x33, 0x30, 0x33, 0x35, 0x39, 0x4080032e, 0x39, - 0x30, 0x34, 0x32, 0x38, 0x37, 0x35, 0x35, - 0x34, 0x36, 0x38, 0x37, 0x33, 0x31, 0x31, - 0x35, 0x39, 0x35, 0x40400355, 0x33, 0x38, 0x38, - 0x32, 0x33, 0x35, 0x33, 0x37, 0x38, 0x37, - 0x35, 0x4080037f, 0x39, 0x4040013a, 0x31, 0x40400148, 0x38, - 0x30, 0x35, 0x33, 0x4040018a, 0x32, 0x32, 0x36, - 0x38, 0x30, 0x36, 0x36, 0x31, 0x33, 0x30, - 0x30, 0x31, 0x39, 0x32, 0x37, 0x38, 0x37, - 0x36, 0x36, 0x31, 0x31, 0x31, 0x39, 0x35, - 0x39, 0x40400237, 0x36, 0x40800124, 0x38, 0x39, 0x33, - 0x38, 0x30, 0x39, 0x35, 0x32, 0x35, 0x37, - 0x32, 0x30, 0x31, 0x30, 0x36, 0x35, 0x34, - 0x38, 0x35, 0x38, 0x36, 0x33, 0x32, 0x37, - 0x4040009a, 0x39, 0x33, 0x36, 0x31, 0x35, 0x33, - 0x40400220, 0x4080015c, 0x32, 0x33, 0x30, 0x33, 0x30, - 0x31, 0x39, 0x35, 0x32, 0x30, 0x33, 0x35, - 0x33, 0x30, 0x31, 0x38, 0x35, 0x32, 0x40400171, - 0x40400075, 0x33, 0x36, 0x32, 0x32, 0x35, 0x39, - 0x39, 0x34, 0x31, 0x33, 0x40400254, 0x34, 0x39, - 0x37, 0x32, 0x31, 0x37, 0x404000de, 0x33, 0x34, - 0x37, 0x39, 0x31, 0x33, 0x31, 0x35, 0x31, - 0x35, 0x35, 0x37, 0x34, 0x38, 0x35, 0x37, - 0x32, 0x34, 0x32, 0x34, 0x35, 0x34, 0x31, - 0x35, 0x30, 0x36, 0x39, 0x4040013f, 0x38, 0x32, - 0x39, 0x35, 0x33, 0x33, 0x31, 0x31, 0x36, - 0x38, 0x36, 0x31, 0x37, 0x32, 0x37, 0x38, - 0x40400337, 0x39, 0x30, 0x37, 0x35, 0x30, 0x39, - 0x4040010d, 0x37, 0x35, 0x34, 0x36, 0x33, 0x37, - 0x34, 0x36, 0x34, 0x39, 0x33, 0x39, 0x33, - 0x31, 0x39, 0x32, 0x35, 0x35, 0x30, 0x36, - 0x30, 0x34, 0x30, 0x30, 0x39, 0x4040026b, 0x31, - 0x36, 0x37, 0x31, 0x31, 0x33, 0x39, 0x30, - 0x30, 0x39, 0x38, 0x40400335, 0x34, 0x30, 0x31, - 0x32, 0x38, 0x35, 0x38, 0x33, 0x36, 0x31, - 0x36, 0x30, 0x33, 0x35, 0x36, 0x33, 0x37, - 0x30, 0x37, 0x36, 0x36, 0x30, 0x31, 0x30, - 0x34, 0x40400172, 0x38, 0x31, 0x39, 0x34, 0x32, - 0x39, 0x4080041e, 0x404000ef, 0x4040028b, 0x37, 0x38, 0x33, - 0x37, 0x34, 0x404004a8, 0x38, 0x32, 0x35, 0x35, - 0x33, 0x37, 0x40800209, 0x32, 0x36, 0x38, 0x4040002e, - 0x34, 0x30, 0x34, 0x37, 0x404001d1, 0x34, 0x404004b5, - 0x4040038d, 0x38, 0x34, 0x404003a8, 0x36, 0x40c0031f, 0x33, - 0x33, 0x31, 0x33, 0x36, 0x37, 0x37, 0x30, - 0x32, 0x38, 0x39, 0x38, 0x39, 0x31, 0x35, - 0x32, 0x40400062, 0x35, 0x32, 0x31, 0x36, 0x32, - 0x30, 0x35, 0x36, 0x39, 0x36, 0x40400411, 0x30, - 0x35, 0x38, 0x40400477, 0x35, 0x40400498, 0x35, 0x31, - 0x31, 0x40400209, 0x38, 0x32, 0x34, 0x33, 0x30, - 0x30, 0x33, 0x35, 0x35, 0x38, 0x37, 0x36, - 0x34, 0x30, 0x32, 0x34, 0x37, 0x34, 0x39, - 0x36, 0x34, 0x37, 0x33, 0x32, 0x36, 0x33, - 0x4040043e, 0x39, 0x39, 0x32, 0x4040044b, 0x34, 0x32, - 0x36, 0x39, 0x40c002c5, 0x37, 0x404001d6, 0x34, 0x4040053d, - 0x4040041d, 0x39, 0x33, 0x34, 0x31, 0x37, 0x404001ad, - 0x31, 0x32, 0x4040002a, 0x34, 0x4040019e, 0x31, 0x35, - 0x30, 0x33, 0x30, 0x32, 0x38, 0x36, 0x31, - 0x38, 0x32, 0x39, 0x37, 0x34, 0x35, 0x35, - 0x35, 0x37, 0x30, 0x36, 0x37, 0x34, 0x40400135, - 0x35, 0x30, 0x35, 0x34, 0x39, 0x34, 0x35, - 0x38, 0x404001c5, 0x39, 0x40400051, 0x35, 0x36, 0x404001ec, - 0x37, 0x32, 0x31, 0x30, 0x37, 0x39, 0x40400159, - 0x33, 0x30, 0x4040010a, 0x33, 0x32, 0x31, 0x31, - 0x36, 0x35, 0x33, 0x34, 0x34, 0x39, 0x38, - 0x37, 0x32, 0x30, 0x32, 0x37, 0x4040011b, 0x30, - 0x32, 0x33, 0x36, 0x34, 0x4040022e, 0x35, 0x34, - 0x39, 0x39, 0x31, 0x31, 0x39, 0x38, 0x40400418, - 0x34, 0x4040011b, 0x35, 0x33, 0x35, 0x36, 0x36, - 0x33, 0x36, 0x39, 0x40400450, 0x32, 0x36, 0x35, - 0x404002e4, 0x37, 0x38, 0x36, 0x32, 0x35, 0x35, - 0x31, 0x404003da, 0x31, 0x37, 0x35, 0x37, 0x34, - 0x36, 0x37, 0x32, 0x38, 0x39, 0x30, 0x39, - 0x37, 0x37, 0x37, 0x37, 0x40800453, 0x30, 0x30, - 0x30, 0x404005fd, 0x37, 0x30, 0x404004df, 0x36, 0x404003e9, - 0x34, 0x39, 0x31, 0x4040041e, 0x40400297, 0x32, 0x31, - 0x34, 0x37, 0x37, 0x32, 0x33, 0x35, 0x30, - 0x31, 0x34, 0x31, 0x34, 0x40400643, 0x33, 0x35, - 0x36, 0x404004af, 0x31, 0x36, 0x31, 0x33, 0x36, - 0x31, 0x31, 0x35, 0x37, 0x33, 0x35, 0x32, - 0x35, 0x40400504, 0x33, 0x34, 0x4040005b, 0x31, 0x38, - 0x4040047b, 0x38, 0x34, 0x404005e7, 0x33, 0x33, 0x32, - 0x33, 0x39, 0x30, 0x37, 0x33, 0x39, 0x34, - 0x31, 0x34, 0x33, 0x33, 0x33, 0x34, 0x35, - 0x34, 0x37, 0x37, 0x36, 0x32, 0x34, 0x40400242, - 0x32, 0x35, 0x31, 0x38, 0x39, 0x38, 0x33, - 0x35, 0x36, 0x39, 0x34, 0x38, 0x35, 0x35, - 0x36, 0x32, 0x30, 0x39, 0x39, 0x32, 0x31, - 0x39, 0x32, 0x32, 0x32, 0x31, 0x38, 0x34, - 0x32, 0x37, 0x4040023e, 0x32, 0x404000ba, 0x36, 0x38, - 0x38, 0x37, 0x36, 0x37, 0x31, 0x37, 0x39, - 0x30, 0x40400055, 0x30, 0x40800106, 0x36, 0x36, 0x404003e7, - 0x38, 0x38, 0x36, 0x32, 0x37, 0x32, 0x404006dc, - 0x31, 0x37, 0x38, 0x36, 0x30, 0x38, 0x35, - 0x37, 0x40400073, 0x33, 0x408002fc, 0x37, 0x39, 0x37, - 0x36, 0x36, 0x38, 0x31, 0x404002bd, 0x30, 0x30, - 0x39, 0x35, 0x33, 0x38, 0x38, 0x40400638, 0x33, - 0x404006a5, 0x30, 0x36, 0x38, 0x30, 0x30, 0x36, - 0x34, 0x32, 0x32, 0x35, 0x31, 0x32, 0x35, - 0x32, 0x4040057b, 0x37, 0x33, 0x39, 0x32, 0x40400297, - 0x40400474, 0x34, 0x408006b3, 0x38, 0x36, 0x32, 0x36, - 0x39, 0x34, 0x35, 0x404001e5, 0x34, 0x31, 0x39, - 0x36, 0x35, 0x32, 0x38, 0x35, 0x30, 0x40400099, - 0x4040039c, 0x31, 0x38, 0x36, 0x33, 0x404001be, 0x34, - 0x40800154, 0x32, 0x30, 0x33, 0x39, 0x4040058b, 0x34, - 0x35, 0x404002bc, 0x32, 0x33, 0x37, 0x4040042c, 0x36, - 0x40400510, 0x35, 0x36, 0x40400638, 0x37, 0x31, 0x39, - 0x31, 0x37, 0x32, 0x38, 0x40400171, 0x37, 0x36, - 0x34, 0x36, 0x35, 0x37, 0x35, 0x37, 0x33, - 0x39, 0x40400101, 0x33, 0x38, 0x39, 0x40400748, 0x38, - 0x33, 0x32, 0x36, 0x34, 0x35, 0x39, 0x39, - 0x35, 0x38, 0x404006a7, 0x30, 0x34, 0x37, 0x38, - 0x404001de, 0x40400328, 0x39, 0x4040002d, 0x36, 0x34, 0x30, - 0x37, 0x38, 0x39, 0x35, 0x31, 0x4040008e, 0x36, - 0x38, 0x33, 0x4040012f, 0x32, 0x35, 0x39, 0x35, - 0x37, 0x30, 0x40400468, 0x38, 0x32, 0x32, 0x404002c8, - 0x32, 0x4040061b, 0x34, 0x30, 0x37, 0x37, 0x32, - 0x36, 0x37, 0x31, 0x39, 0x34, 0x37, 0x38, - 0x40400319, 0x38, 0x32, 0x36, 0x30, 0x31, 0x34, - 0x37, 0x36, 0x39, 0x39, 0x30, 0x39, 0x404004e8, - 0x30, 0x31, 0x33, 0x36, 0x33, 0x39, 0x34, - 0x34, 0x33, 0x4040027f, 0x33, 0x30, 0x40400105, 0x32, - 0x30, 0x33, 0x34, 0x39, 0x36, 0x32, 0x35, - 0x32, 0x34, 0x35, 0x31, 0x37, 0x404003b5, 0x39, - 0x36, 0x35, 0x31, 0x34, 0x33, 0x31, 0x34, - 0x32, 0x39, 0x38, 0x30, 0x39, 0x31, 0x39, - 0x30, 0x36, 0x35, 0x39, 0x32, 0x40400282, 0x37, - 0x32, 0x32, 0x31, 0x36, 0x39, 0x36, 0x34, - 0x36, 0x40400419, 0x4040007a, 0x35, 0x4040050e, 0x34, 0x40800565, - 0x38, 0x40400559, 0x39, 0x37, 0x4040057b, 0x35, 0x34, - 0x4040049d, 0x4040023e, 0x37, 0x4040065a, 0x38, 0x34, 0x36, - 0x38, 0x31, 0x33, 0x4040008c, 0x36, 0x38, 0x33, - 0x38, 0x36, 0x38, 0x39, 0x34, 0x32, 0x37, - 0x37, 0x34, 0x31, 0x35, 0x35, 0x39, 0x39, - 0x31, 0x38, 0x35, 0x4040005a, 0x32, 0x34, 0x35, - 0x39, 0x35, 0x33, 0x39, 0x35, 0x39, 0x34, - 0x33, 0x31, 0x404005b7, 0x37, 0x40400012, 0x36, 0x38, - 0x30, 0x38, 0x34, 0x35, 0x404002e7, 0x37, 0x33, - 0x4040081e, 0x39, 0x35, 0x38, 0x34, 0x38, 0x36, - 0x35, 0x33, 0x38, 0x404006e8, 0x36, 0x32, 0x404000f2, - 0x36, 0x30, 0x39, 0x404004b6, 0x36, 0x30, 0x38, - 0x30, 0x35, 0x31, 0x32, 0x34, 0x33, 0x38, - 0x38, 0x34, 0x4040013a, 0x4040000b, 0x34, 0x31, 0x33, - 0x4040030f, 0x37, 0x36, 0x32, 0x37, 0x38, 0x40400341, - 0x37, 0x31, 0x35, 0x4040059b, 0x33, 0x35, 0x39, - 0x39, 0x37, 0x37, 0x30, 0x30, 0x31, 0x32, - 0x39, 0x40400472, 0x38, 0x39, 0x34, 0x34, 0x31, - 0x40400277, 0x36, 0x38, 0x35, 0x35, 0x4040005f, 0x34, - 0x30, 0x36, 0x33, 0x404008e6, 0x32, 0x30, 0x37, - 0x32, 0x32, 0x40400158, 0x40800203, 0x34, 0x38, 0x31, - 0x35, 0x38, 0x40400205, 0x404001fe, 0x4040027a, 0x40400298, 0x33, - 0x39, 0x34, 0x35, 0x32, 0x32, 0x36, 0x37, - 0x40c00496, 0x38, 0x4040058a, 0x32, 0x31, 0x404002ea, 0x32, - 0x40400387, 0x35, 0x34, 0x36, 0x36, 0x36, 0x4040051b, - 0x32, 0x33, 0x39, 0x38, 0x36, 0x34, 0x35, - 0x36, 0x404004c4, 0x31, 0x36, 0x33, 0x35, 0x40800253, - 0x40400811, 0x37, 0x404008ad, 0x39, 0x38, 0x4040045e, 0x39, - 0x33, 0x36, 0x33, 0x34, 0x4040075b, 0x37, 0x34, - 0x33, 0x32, 0x34, 0x4040047b, 0x31, 0x35, 0x30, - 0x37, 0x36, 0x404004bb, 0x37, 0x39, 0x34, 0x35, - 0x31, 0x30, 0x39, 0x4040003e, 0x30, 0x39, 0x34, - 0x30, 0x404006a6, 0x38, 0x38, 0x37, 0x39, 0x37, - 0x31, 0x30, 0x38, 0x39, 0x33, 0x404008f0, 0x36, - 0x39, 0x31, 0x33, 0x36, 0x38, 0x36, 0x37, - 0x32, 0x4040025b, 0x404001fe, 0x35, 0x4040053f, 0x40400468, 0x40400801, - 0x31, 0x37, 0x39, 0x32, 0x38, 0x36, 0x38, - 0x404008cc, 0x38, 0x37, 0x34, 0x37, 0x4080079e, 0x38, - 0x32, 0x34, 0x4040097a, 0x38, 0x4040025b, 0x37, 0x31, - 0x34, 0x39, 0x30, 0x39, 0x36, 0x37, 0x35, - 0x39, 0x38, 0x404006ef, 0x33, 0x36, 0x35, 0x40400134, - 0x38, 0x31, 0x4040005c, 0x40400745, 0x40400936, 0x36, 0x38, - 0x32, 0x39, 0x4040057e, 0x38, 0x37, 0x32, 0x32, - 0x36, 0x35, 0x38, 0x38, 0x30, 0x40400611, 0x35, - 0x40400249, 0x34, 0x32, 0x37, 0x30, 0x34, 0x37, - 0x37, 0x35, 0x35, 0x4040081e, 0x33, 0x37, 0x39, - 0x36, 0x34, 0x31, 0x34, 0x35, 0x31, 0x35, - 0x32, 0x404005fd, 0x32, 0x33, 0x34, 0x33, 0x36, - 0x34, 0x35, 0x34, 0x404005de, 0x34, 0x34, 0x34, - 0x37, 0x39, 0x35, 0x4040003c, 0x40400523, 0x408008e6, 0x34, - 0x31, 0x4040052a, 0x33, 0x40400304, 0x35, 0x32, 0x33, - 0x31, 0x40800841, 0x31, 0x36, 0x36, 0x31, 0x404008b2, - 0x35, 0x39, 0x36, 0x39, 0x35, 0x33, 0x36, - 0x32, 0x33, 0x31, 0x34, 0x404005ff, 0x32, 0x34, - 0x38, 0x34, 0x39, 0x33, 0x37, 0x31, 0x38, - 0x37, 0x31, 0x31, 0x30, 0x31, 0x34, 0x35, - 0x37, 0x36, 0x35, 0x34, 0x40400761, 0x30, 0x32, - 0x37, 0x39, 0x39, 0x33, 0x34, 0x34, 0x30, - 0x33, 0x37, 0x34, 0x32, 0x30, 0x30, 0x37, - 0x4040093f, 0x37, 0x38, 0x35, 0x33, 0x39, 0x30, - 0x36, 0x32, 0x31, 0x39, 0x40800299, 0x40400345, 0x38, - 0x34, 0x37, 0x408003d2, 0x38, 0x33, 0x33, 0x32, - 0x31, 0x34, 0x34, 0x35, 0x37, 0x31, 0x40400284, - 0x40400776, 0x34, 0x33, 0x35, 0x30, 0x40400928, 0x40400468, - 0x35, 0x33, 0x31, 0x39, 0x31, 0x30, 0x34, - 0x38, 0x34, 0x38, 0x31, 0x30, 0x30, 0x35, - 0x33, 0x37, 0x30, 0x36, 0x404008bc, 0x4080059d, 0x40800781, - 0x31, 0x40400559, 0x37, 0x4040031b, 0x35, 0x404007ec, 0x4040040c, - 0x36, 0x33, 0x408007dc, 0x34, 0x40400971, 0x4080034e, 0x408003f5, - 0x38, 0x4080052d, 0x40800887, 0x39, 0x40400187, 0x39, 0x31, - 0x404008ce, 0x38, 0x31, 0x34, 0x36, 0x37, 0x35, - 0x31, 0x4040062b, 0x31, 0x32, 0x33, 0x39, 0x40c001a9, - 0x39, 0x30, 0x37, 0x31, 0x38, 0x36, 0x34, - 0x39, 0x34, 0x32, 0x33, 0x31, 0x39, 0x36, - 0x31, 0x35, 0x36, 0x404001ec, 0x404006bc, 0x39, 0x35, - 0x40400926, 0x40400469, 0x4040011b, 0x36, 0x30, 0x33, 0x38, - 0x40400a25, 0x4040016f, 0x40400384, 0x36, 0x32, 0x4040045a, 0x35, - 0x4040084c, 0x36, 0x33, 0x38, 0x39, 0x33, 0x37, - 0x37, 0x38, 0x37, 0x404008c5, 0x404000f8, 0x39, 0x37, - 0x39, 0x32, 0x30, 0x37, 0x37, 0x33, 0x404005d7, - 0x32, 0x31, 0x38, 0x32, 0x35, 0x36, 0x404007df, - 0x36, 0x36, 0x404006d6, 0x34, 0x32, 0x4080067e, 0x36, - 0x404006e6, 0x34, 0x34, 0x40400024, 0x35, 0x34, 0x39, - 0x32, 0x30, 0x32, 0x36, 0x30, 0x35, 0x40400ab3, - 0x408003e4, 0x32, 0x30, 0x31, 0x34, 0x39, 0x404004d2, - 0x38, 0x35, 0x30, 0x37, 0x33, 0x40400599, 0x36, - 0x36, 0x36, 0x30, 0x40400194, 0x32, 0x34, 0x33, - 0x34, 0x30, 0x40400087, 0x30, 0x4040076b, 0x38, 0x36, - 0x33, 0x40400956, 0x404007e4, 0x4040042b, 0x40400174, 0x35, 0x37, - 0x39, 0x36, 0x32, 0x36, 0x38, 0x35, 0x36, - 0x40400140, 0x35, 0x30, 0x38, 0x40400523, 0x35, 0x38, - 0x37, 0x39, 0x36, 0x39, 0x39, 0x40400711, 0x35, - 0x37, 0x34, 0x40400a18, 0x38, 0x34, 0x30, 0x404008b3, - 0x31, 0x34, 0x35, 0x39, 0x31, 0x4040078c, 0x37, - 0x30, 0x40400234, 0x30, 0x31, 0x40400be7, 0x31, 0x32, - 0x40400c74, 0x30, 0x404003c3, 0x33, 0x39, 0x40400b2a, 0x40400112, - 0x37, 0x31, 0x35, 0x404003b0, 0x34, 0x32, 0x30, - 0x40800bf2, 0x39, 0x40400bc2, 0x30, 0x37, 0x40400341, 0x40400795, - 0x40400aaf, 0x40400c62, 0x32, 0x31, 0x40400960, 0x32, 0x35, - 0x31, 0x4040057b, 0x40400944, 0x39, 0x32, 0x404001b2, 0x38, - 0x32, 0x36, 0x40400b66, 0x32, 0x40400278, 0x33, 0x32, - 0x31, 0x35, 0x37, 0x39, 0x31, 0x39, 0x38, - 0x34, 0x31, 0x34, 0x4080087b, 0x39, 0x31, 0x36, - 0x34, 0x408006e8, 0x39, 0x40800b58, 0x404008db, 0x37, 0x32, - 0x32, 0x40400321, 0x35, 0x404008a4, 0x40400141, 0x39, 0x31, - 0x30, 0x404000bc, 0x40400c5b, 0x35, 0x32, 0x38, 0x30, - 0x31, 0x37, 0x40400231, 0x37, 0x31, 0x32, 0x40400914, - 0x38, 0x33, 0x32, 0x40400373, 0x31, 0x40400589, 0x30, - 0x39, 0x33, 0x35, 0x33, 0x39, 0x36, 0x35, - 0x37, 0x4040064b, 0x31, 0x30, 0x38, 0x33, 0x40400069, - 0x35, 0x31, 0x4040077a, 0x40400d5a, 0x31, 0x34, 0x34, - 0x34, 0x32, 0x31, 0x30, 0x30, 0x40400202, 0x30, - 0x33, 0x4040019c, 0x31, 0x31, 0x30, 0x33, 0x40400c81, - 0x40400009, 0x40400026, 0x40c00602, 0x35, 0x31, 0x36, 0x404005d9, - 0x40800883, 0x4040092a, 0x35, 0x40800c42, 0x38, 0x35, 0x31, - 0x37, 0x31, 0x34, 0x33, 0x37, 0x40400605, 0x4040006d, - 0x31, 0x35, 0x35, 0x36, 0x35, 0x30, 0x38, - 0x38, 0x404003b9, 0x39, 0x38, 0x39, 0x38, 0x35, - 0x39, 0x39, 0x38, 0x32, 0x33, 0x38, 0x404001cf, - 0x404009ba, 0x33, 0x4040016c, 0x4040043e, 0x404009c3, 0x38, 0x40800e05, - 0x33, 0x32, 0x40400107, 0x35, 0x40400305, 0x33, 0x404001ca, - 0x39, 0x4040041b, 0x39, 0x38, 0x4040087d, 0x34, 0x40400cb8, - 0x37, 0x4040064b, 0x30, 0x37, 0x404000e5, 0x34, 0x38, - 0x31, 0x34, 0x31, 0x40400539, 0x38, 0x35, 0x39, - 0x34, 0x36, 0x31, 0x40400bc9, 0x38, 0x30, - }, - }, - HuffTest{ - .input = "huffman-rand-1k.input", - .want = "huffman-rand-1k.{s}.expect", - .want_no_input = "huffman-rand-1k.{s}.expect-noinput", - .tokens = &[_]token.Token{ - 0xf8, 0x8b, 0x96, 0x76, 0x48, 0xd, 0x85, 0x94, 0x25, 0x80, 0xaf, 0xc2, 0xfe, 0x8d, - 0xe8, 0x20, 0xeb, 0x17, 0x86, 0xc9, 0xb7, 0xc5, 0xde, 0x6, 0xea, 0x7d, 0x18, 0x8b, - 0xe7, 0x3e, 0x7, 0xda, 0xdf, 0xff, 0x6c, 0x73, 0xde, 0xcc, 0xe7, 0x6d, 0x8d, 0x4, - 0x19, 0x49, 0x7f, 0x47, 0x1f, 0x48, 0x15, 0xb0, 0xe8, 0x9e, 0xf2, 0x31, 0x59, 0xde, - 0x34, 0xb4, 0x5b, 0xe5, 0xe0, 0x9, 0x11, 0x30, 0xc2, 0x88, 0x5b, 0x7c, 0x5d, 0x14, - 0x13, 0x6f, 0x23, 0xa9, 0xd, 0xbc, 0x2d, 0x23, 0xbe, 0xd9, 0xed, 0x75, 0x4, 0x6c, - 0x99, 0xdf, 0xfd, 0x70, 0x66, 0xe6, 0xee, 0xd9, 0xb1, 0x9e, 0x6e, 0x83, 0x59, 0xd5, - 0xd4, 0x80, 0x59, 0x98, 0x77, 0x89, 0x43, 0x38, 0xc9, 0xaf, 0x30, 0x32, 0x9a, 0x20, - 0x1b, 0x46, 0x3d, 0x67, 0x6e, 0xd7, 0x72, 0x9e, 0x4e, 0x21, 0x4f, 0xc6, 0xe0, 0xd4, - 0x7b, 0x4, 0x8d, 0xa5, 0x3, 0xf6, 0x5, 0x9b, 0x6b, 0xdc, 0x2a, 0x93, 0x77, 0x28, - 0xfd, 0xb4, 0x62, 0xda, 0x20, 0xe7, 0x1f, 0xab, 0x6b, 0x51, 0x43, 0x39, 0x2f, 0xa0, - 0x92, 0x1, 0x6c, 0x75, 0x3e, 0xf4, 0x35, 0xfd, 0x43, 0x2e, 0xf7, 0xa4, 0x75, 0xda, - 0xea, 0x9b, 0xa, 0x64, 0xb, 0xe0, 0x23, 0x29, 0xbd, 0xf7, 0xe7, 0x83, 0x3c, 0xfb, - 0xdf, 0xb3, 0xae, 0x4f, 0xa4, 0x47, 0x55, 0x99, 0xde, 0x2f, 0x96, 0x6e, 0x1c, 0x43, - 0x4c, 0x87, 0xe2, 0x7c, 0xd9, 0x5f, 0x4c, 0x7c, 0xe8, 0x90, 0x3, 0xdb, 0x30, 0x95, - 0xd6, 0x22, 0xc, 0x47, 0xb8, 0x4d, 0x6b, 0xbd, 0x24, 0x11, 0xab, 0x2c, 0xd7, 0xbe, - 0x6e, 0x7a, 0xd6, 0x8, 0xa3, 0x98, 0xd8, 0xdd, 0x15, 0x6a, 0xfa, 0x93, 0x30, 0x1, - 0x25, 0x1d, 0xa2, 0x74, 0x86, 0x4b, 0x6a, 0x95, 0xe8, 0xe1, 0x4e, 0xe, 0x76, 0xb9, - 0x49, 0xa9, 0x5f, 0xa0, 0xa6, 0x63, 0x3c, 0x7e, 0x7e, 0x20, 0x13, 0x4f, 0xbb, 0x66, - 0x92, 0xb8, 0x2e, 0xa4, 0xfa, 0x48, 0xcb, 0xae, 0xb9, 0x3c, 0xaf, 0xd3, 0x1f, 0xe1, - 0xd5, 0x8d, 0x42, 0x6d, 0xf0, 0xfc, 0x8c, 0xc, 0x0, 0xde, 0x40, 0xab, 0x8b, 0x47, - 0x97, 0x4e, 0xa8, 0xcf, 0x8e, 0xdb, 0xa6, 0x8b, 0x20, 0x9, 0x84, 0x7a, 0x66, 0xe5, - 0x98, 0x29, 0x2, 0x95, 0xe6, 0x38, 0x32, 0x60, 0x3, 0xe3, 0x9a, 0x1e, 0x54, 0xe8, - 0x63, 0x80, 0x48, 0x9c, 0xe7, 0x63, 0x33, 0x6e, 0xa0, 0x65, 0x83, 0xfa, 0xc6, 0xba, - 0x7a, 0x43, 0x71, 0x5, 0xf5, 0x68, 0x69, 0x85, 0x9c, 0xba, 0x45, 0xcd, 0x6b, 0xb, - 0x19, 0xd1, 0xbb, 0x7f, 0x70, 0x85, 0x92, 0xd1, 0xb4, 0x64, 0x82, 0xb1, 0xe4, 0x62, - 0xc5, 0x3c, 0x46, 0x1f, 0x92, 0x31, 0x1c, 0x4e, 0x41, 0x77, 0xf7, 0xe7, 0x87, 0xa2, - 0xf, 0x6e, 0xe8, 0x92, 0x3, 0x6b, 0xa, 0xe7, 0xa9, 0x3b, 0x11, 0xda, 0x66, 0x8a, - 0x29, 0xda, 0x79, 0xe1, 0x64, 0x8d, 0xe3, 0x54, 0xd4, 0xf5, 0xef, 0x64, 0x87, 0x3b, - 0xf4, 0xc2, 0xf4, 0x71, 0x13, 0xa9, 0xe9, 0xe0, 0xa2, 0x6, 0x14, 0xab, 0x5d, 0xa7, - 0x96, 0x0, 0xd6, 0xc3, 0xcc, 0x57, 0xed, 0x39, 0x6a, 0x25, 0xcd, 0x76, 0xea, 0xba, - 0x3a, 0xf2, 0xa1, 0x95, 0x5d, 0xe5, 0x71, 0xcf, 0x9c, 0x62, 0x9e, 0x6a, 0xfa, 0xd5, - 0x31, 0xd1, 0xa8, 0x66, 0x30, 0x33, 0xaa, 0x51, 0x17, 0x13, 0x82, 0x99, 0xc8, 0x14, - 0x60, 0x9f, 0x4d, 0x32, 0x6d, 0xda, 0x19, 0x26, 0x21, 0xdc, 0x7e, 0x2e, 0x25, 0x67, - 0x72, 0xca, 0xf, 0x92, 0xcd, 0xf6, 0xd6, 0xcb, 0x97, 0x8a, 0x33, 0x58, 0x73, 0x70, - 0x91, 0x1d, 0xbf, 0x28, 0x23, 0xa3, 0xc, 0xf1, 0x83, 0xc3, 0xc8, 0x56, 0x77, 0x68, - 0xe3, 0x82, 0xba, 0xb9, 0x57, 0x56, 0x57, 0x9c, 0xc3, 0xd6, 0x14, 0x5, 0x3c, 0xb1, - 0xaf, 0x93, 0xc8, 0x8a, 0x57, 0x7f, 0x53, 0xfa, 0x2f, 0xaa, 0x6e, 0x66, 0x83, 0xfa, - 0x33, 0xd1, 0x21, 0xab, 0x1b, 0x71, 0xb4, 0x7c, 0xda, 0xfd, 0xfb, 0x7f, 0x20, 0xab, - 0x5e, 0xd5, 0xca, 0xfd, 0xdd, 0xe0, 0xee, 0xda, 0xba, 0xa8, 0x27, 0x99, 0x97, 0x69, - 0xc1, 0x3c, 0x82, 0x8c, 0xa, 0x5c, 0x2d, 0x5b, 0x88, 0x3e, 0x34, 0x35, 0x86, 0x37, - 0x46, 0x79, 0xe1, 0xaa, 0x19, 0xfb, 0xaa, 0xde, 0x15, 0x9, 0xd, 0x1a, 0x57, 0xff, - 0xb5, 0xf, 0xf3, 0x2b, 0x5a, 0x6a, 0x4d, 0x19, 0x77, 0x71, 0x45, 0xdf, 0x4f, 0xb3, - 0xec, 0xf1, 0xeb, 0x18, 0x53, 0x3e, 0x3b, 0x47, 0x8, 0x9a, 0x73, 0xa0, 0x5c, 0x8c, - 0x5f, 0xeb, 0xf, 0x3a, 0xc2, 0x43, 0x67, 0xb4, 0x66, 0x67, 0x80, 0x58, 0xe, 0xc1, - 0xec, 0x40, 0xd4, 0x22, 0x94, 0xca, 0xf9, 0xe8, 0x92, 0xe4, 0x69, 0x38, 0xbe, 0x67, - 0x64, 0xca, 0x50, 0xc7, 0x6, 0x67, 0x42, 0x6e, 0xa3, 0xf0, 0xb7, 0x6c, 0xf2, 0xe8, - 0x5f, 0xb1, 0xaf, 0xe7, 0xdb, 0xbb, 0x77, 0xb5, 0xf8, 0xcb, 0x8, 0xc4, 0x75, 0x7e, - 0xc0, 0xf9, 0x1c, 0x7f, 0x3c, 0x89, 0x2f, 0xd2, 0x58, 0x3a, 0xe2, 0xf8, 0x91, 0xb6, - 0x7b, 0x24, 0x27, 0xe9, 0xae, 0x84, 0x8b, 0xde, 0x74, 0xac, 0xfd, 0xd9, 0xb7, 0x69, - 0x2a, 0xec, 0x32, 0x6f, 0xf0, 0x92, 0x84, 0xf1, 0x40, 0xc, 0x8a, 0xbc, 0x39, 0x6e, - 0x2e, 0x73, 0xd4, 0x6e, 0x8a, 0x74, 0x2a, 0xdc, 0x60, 0x1f, 0xa3, 0x7, 0xde, 0x75, - 0x8b, 0x74, 0xc8, 0xfe, 0x63, 0x75, 0xf6, 0x3d, 0x63, 0xac, 0x33, 0x89, 0xc3, 0xf0, - 0xf8, 0x2d, 0x6b, 0xb4, 0x9e, 0x74, 0x8b, 0x5c, 0x33, 0xb4, 0xca, 0xa8, 0xe4, 0x99, - 0xb6, 0x90, 0xa1, 0xef, 0xf, 0xd3, 0x61, 0xb2, 0xc6, 0x1a, 0x94, 0x7c, 0x44, 0x55, - 0xf4, 0x45, 0xff, 0x9e, 0xa5, 0x5a, 0xc6, 0xa0, 0xe8, 0x2a, 0xc1, 0x8d, 0x6f, 0x34, - 0x11, 0xb9, 0xbe, 0x4e, 0xd9, 0x87, 0x97, 0x73, 0xcf, 0x3d, 0x23, 0xae, 0xd5, 0x1a, - 0x5e, 0xae, 0x5d, 0x6a, 0x3, 0xf9, 0x22, 0xd, 0x10, 0xd9, 0x47, 0x69, 0x15, 0x3f, - 0xee, 0x52, 0xa3, 0x8, 0xd2, 0x3c, 0x51, 0xf4, 0xf8, 0x9d, 0xe4, 0x98, 0x89, 0xc8, - 0x67, 0x39, 0xd5, 0x5e, 0x35, 0x78, 0x27, 0xe8, 0x3c, 0x80, 0xae, 0x79, 0x71, 0xd2, - 0x93, 0xf4, 0xaa, 0x51, 0x12, 0x1c, 0x4b, 0x1b, 0xe5, 0x6e, 0x15, 0x6f, 0xe4, 0xbb, - 0x51, 0x9b, 0x45, 0x9f, 0xf9, 0xc4, 0x8c, 0x2a, 0xfb, 0x1a, 0xdf, 0x55, 0xd3, 0x48, - 0x93, 0x27, 0x1, 0x26, 0xc2, 0x6b, 0x55, 0x6d, 0xa2, 0xfb, 0x84, 0x8b, 0xc9, 0x9e, - 0x28, 0xc2, 0xef, 0x1a, 0x24, 0xec, 0x9b, 0xae, 0xbd, 0x60, 0xe9, 0x15, 0x35, 0xee, - 0x42, 0xa4, 0x33, 0x5b, 0xfa, 0xf, 0xb6, 0xf7, 0x1, 0xa6, 0x2, 0x4c, 0xca, 0x90, - 0x58, 0x3a, 0x96, 0x41, 0xe7, 0xcb, 0x9, 0x8c, 0xdb, 0x85, 0x4d, 0xa8, 0x89, 0xf3, - 0xb5, 0x8e, 0xfd, 0x75, 0x5b, 0x4f, 0xed, 0xde, 0x3f, 0xeb, 0x38, 0xa3, 0xbe, 0xb0, - 0x73, 0xfc, 0xb8, 0x54, 0xf7, 0x4c, 0x30, 0x67, 0x2e, 0x38, 0xa2, 0x54, 0x18, 0xba, - 0x8, 0xbf, 0xf2, 0x39, 0xd5, 0xfe, 0xa5, 0x41, 0xc6, 0x66, 0x66, 0xba, 0x81, 0xef, - 0x67, 0xe4, 0xe6, 0x3c, 0xc, 0xca, 0xa4, 0xa, 0x79, 0xb3, 0x57, 0x8b, 0x8a, 0x75, - 0x98, 0x18, 0x42, 0x2f, 0x29, 0xa3, 0x82, 0xef, 0x9f, 0x86, 0x6, 0x23, 0xe1, 0x75, - 0xfa, 0x8, 0xb1, 0xde, 0x17, 0x4a, - }, - }, - HuffTest{ - .input = "huffman-rand-limit.input", - .want = "huffman-rand-limit.{s}.expect", - .want_no_input = "huffman-rand-limit.{s}.expect-noinput", - .tokens = &[_]token.Token{ - 0x61, 0x51c00000, 0xa, 0xf8, 0x8b, 0x96, 0x76, 0x48, 0xa, 0x85, 0x94, 0x25, 0x80, - 0xaf, 0xc2, 0xfe, 0x8d, 0xe8, 0x20, 0xeb, 0x17, 0x86, 0xc9, 0xb7, 0xc5, 0xde, - 0x6, 0xea, 0x7d, 0x18, 0x8b, 0xe7, 0x3e, 0x7, 0xda, 0xdf, 0xff, 0x6c, 0x73, - 0xde, 0xcc, 0xe7, 0x6d, 0x8d, 0x4, 0x19, 0x49, 0x7f, 0x47, 0x1f, 0x48, 0x15, - 0xb0, 0xe8, 0x9e, 0xf2, 0x31, 0x59, 0xde, 0x34, 0xb4, 0x5b, 0xe5, 0xe0, 0x9, - 0x11, 0x30, 0xc2, 0x88, 0x5b, 0x7c, 0x5d, 0x14, 0x13, 0x6f, 0x23, 0xa9, 0xa, - 0xbc, 0x2d, 0x23, 0xbe, 0xd9, 0xed, 0x75, 0x4, 0x6c, 0x99, 0xdf, 0xfd, 0x70, - 0x66, 0xe6, 0xee, 0xd9, 0xb1, 0x9e, 0x6e, 0x83, 0x59, 0xd5, 0xd4, 0x80, 0x59, - 0x98, 0x77, 0x89, 0x43, 0x38, 0xc9, 0xaf, 0x30, 0x32, 0x9a, 0x20, 0x1b, 0x46, - 0x3d, 0x67, 0x6e, 0xd7, 0x72, 0x9e, 0x4e, 0x21, 0x4f, 0xc6, 0xe0, 0xd4, 0x7b, - 0x4, 0x8d, 0xa5, 0x3, 0xf6, 0x5, 0x9b, 0x6b, 0xdc, 0x2a, 0x93, 0x77, 0x28, - 0xfd, 0xb4, 0x62, 0xda, 0x20, 0xe7, 0x1f, 0xab, 0x6b, 0x51, 0x43, 0x39, 0x2f, - 0xa0, 0x92, 0x1, 0x6c, 0x75, 0x3e, 0xf4, 0x35, 0xfd, 0x43, 0x2e, 0xf7, 0xa4, - 0x75, 0xda, 0xea, 0x9b, 0xa, - }, - }, - HuffTest{ - .input = "huffman-shifts.input", - .want = "huffman-shifts.{s}.expect", - .want_no_input = "huffman-shifts.{s}.expect-noinput", - .tokens = &[_]token.Token{ - 0x31, 0x30, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, - 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, - 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x52400001, 0xd, 0xa, 0x32, - 0x33, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, - 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7f400001, - }, - }, - HuffTest{ - .input = "huffman-text-shift.input", - .want = "huffman-text-shift.{s}.expect", - .want_no_input = "huffman-text-shift.{s}.expect-noinput", - .tokens = &[_]token.Token{ - 0x2f, 0x2f, 0x43, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68, - 0x74, 0x32, 0x30, 0x30, 0x39, 0x54, 0x68, 0x47, 0x6f, 0x41, - 0x75, 0x74, 0x68, 0x6f, 0x72, 0x2e, 0x41, 0x6c, 0x6c, 0x40800016, - 0x72, 0x72, 0x76, 0x64, 0x2e, 0xd, 0xa, 0x2f, 0x2f, 0x55, - 0x6f, 0x66, 0x74, 0x68, 0x69, 0x6f, 0x75, 0x72, 0x63, 0x63, - 0x6f, 0x64, 0x69, 0x67, 0x6f, 0x76, 0x72, 0x6e, 0x64, 0x62, - 0x79, 0x42, 0x53, 0x44, 0x2d, 0x74, 0x79, 0x6c, 0x40400020, 0x6c, - 0x69, 0x63, 0x6e, 0x74, 0x68, 0x74, 0x63, 0x6e, 0x62, 0x66, - 0x6f, 0x75, 0x6e, 0x64, 0x69, 0x6e, 0x74, 0x68, 0x4c, 0x49, - 0x43, 0x45, 0x4e, 0x53, 0x45, 0x66, 0x69, 0x6c, 0x2e, 0xd, - 0xa, 0xd, 0xa, 0x70, 0x63, 0x6b, 0x67, 0x6d, 0x69, 0x6e, - 0x4040000a, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x6f, 0x22, - 0x4040000c, 0x66, 0x75, 0x6e, 0x63, 0x6d, 0x69, 0x6e, 0x28, 0x29, - 0x7b, 0xd, 0xa, 0x9, 0x76, 0x72, 0x62, 0x3d, 0x6d, 0x6b, - 0x28, 0x5b, 0x5d, 0x62, 0x79, 0x74, 0x2c, 0x36, 0x35, 0x35, - 0x33, 0x35, 0x29, 0xd, 0xa, 0x9, 0x66, 0x2c, 0x5f, 0x3a, - 0x3d, 0x6f, 0x2e, 0x43, 0x72, 0x74, 0x28, 0x22, 0x68, 0x75, - 0x66, 0x66, 0x6d, 0x6e, 0x2d, 0x6e, 0x75, 0x6c, 0x6c, 0x2d, - 0x6d, 0x78, 0x2e, 0x69, 0x6e, 0x22, 0x40800021, 0x2e, 0x57, 0x72, - 0x69, 0x74, 0x28, 0x62, 0x29, 0xd, 0xa, 0x7d, 0xd, 0xa, - 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, - 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, - 0x55, 0x56, 0x58, 0x78, 0x79, 0x7a, 0x21, 0x22, 0x23, 0xc2, - 0xa4, 0x25, 0x26, 0x2f, 0x3f, 0x22, - }, - }, - HuffTest{ - .input = "huffman-text.input", - .want = "huffman-text.{s}.expect", - .want_no_input = "huffman-text.{s}.expect-noinput", - .tokens = &[_]token.Token{ - 0x2f, 0x2f, 0x20, 0x7a, 0x69, 0x67, 0x20, 0x76, - 0x30, 0x2e, 0x31, 0x30, 0x2e, 0x30, 0x0a, 0x2f, - 0x2f, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x20, 0x61, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x40400004, - 0x6c, 0x65, 0x64, 0x20, 0x77, 0x69, 0x74, 0x68, - 0x20, 0x30, 0x78, 0x30, 0x30, 0x0a, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x20, 0x73, 0x74, 0x64, 0x20, - 0x3d, 0x20, 0x40, 0x69, 0x6d, 0x70, 0x6f, 0x72, - 0x74, 0x28, 0x22, 0x73, 0x74, 0x64, 0x22, 0x29, - 0x3b, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x20, 0x66, - 0x6e, 0x20, 0x6d, 0x61, 0x69, 0x6e, 0x28, 0x29, - 0x20, 0x21, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x76, 0x61, 0x72, - 0x20, 0x62, 0x20, 0x3d, 0x20, 0x5b, 0x31, 0x5d, - 0x75, 0x38, 0x7b, 0x30, 0x7d, 0x20, 0x2a, 0x2a, - 0x20, 0x36, 0x35, 0x35, 0x33, 0x35, 0x3b, 0x4080001e, - 0x40c00055, 0x66, 0x20, 0x3d, 0x20, 0x74, 0x72, 0x79, - 0x4040005d, 0x2e, 0x66, 0x73, 0x2e, 0x63, 0x77, 0x64, - 0x28, 0x29, 0x2e, 0x40c0008f, 0x46, 0x69, 0x6c, 0x65, - 0x28, 0x4080002a, 0x40400000, 0x22, 0x68, 0x75, 0x66, 0x66, - 0x6d, 0x61, 0x6e, 0x2d, 0x6e, 0x75, 0x6c, 0x6c, - 0x2d, 0x6d, 0x61, 0x78, 0x2e, 0x69, 0x6e, 0x22, - 0x2c, 0x4180001e, 0x2e, 0x7b, 0x20, 0x2e, 0x72, 0x65, - 0x61, 0x64, 0x4080004e, 0x75, 0x65, 0x20, 0x7d, 0x40c0001a, - 0x29, 0x40c0006b, 0x64, 0x65, 0x66, 0x65, 0x72, 0x20, - 0x66, 0x2e, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x28, - 0x404000b6, 0x40400015, 0x5f, 0x4100007b, 0x66, 0x2e, 0x77, 0x72, - 0x69, 0x74, 0x65, 0x41, 0x6c, 0x6c, 0x28, 0x62, - 0x5b, 0x30, 0x2e, 0x2e, 0x5d, 0x29, 0x3b, 0x0a, - 0x7d, 0x0a, - }, - }, - HuffTest{ - .input = "huffman-zero.input", - .want = "huffman-zero.{s}.expect", - .want_no_input = "huffman-zero.{s}.expect-noinput", - .tokens = &[_]token.Token{ 0x30, ml, 0x4b800000 }, - }, - HuffTest{ - .input = "", - .want = "", - .want_no_input = "null-long-match.{s}.expect-noinput", - .tokens = &[_]token.Token{ - 0x0, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, - ml, ml, ml, 0x41400000, - }, - }, -}; - -const TestType = enum { - write_block, - write_dyn_block, // write dynamic block - write_huffman_block, - - fn to_s(self: TestType) []const u8 { - return switch (self) { - .write_block => "wb", - .write_dyn_block => "dyn", - .write_huffman_block => "huff", - }; - } -}; - -test "writeBlock" { - // tests if the writeBlock encoding has changed. - - const ttype: TestType = .write_block; - try testBlock(writeBlockTests[0], ttype); - try testBlock(writeBlockTests[1], ttype); - try testBlock(writeBlockTests[2], ttype); - try testBlock(writeBlockTests[3], ttype); - try testBlock(writeBlockTests[4], ttype); - try testBlock(writeBlockTests[5], ttype); - try testBlock(writeBlockTests[6], ttype); - try testBlock(writeBlockTests[7], ttype); - try testBlock(writeBlockTests[8], ttype); -} - -test "writeBlockDynamic" { - // tests if the writeBlockDynamic encoding has changed. - - const ttype: TestType = .write_dyn_block; - try testBlock(writeBlockTests[0], ttype); - try testBlock(writeBlockTests[1], ttype); - try testBlock(writeBlockTests[2], ttype); - try testBlock(writeBlockTests[3], ttype); - try testBlock(writeBlockTests[4], ttype); - try testBlock(writeBlockTests[5], ttype); - try testBlock(writeBlockTests[6], ttype); - try testBlock(writeBlockTests[7], ttype); - try testBlock(writeBlockTests[8], ttype); -} - -// testBlock tests a block against its references, -// or regenerate the references, if "-update" flag is set. -fn testBlock(comptime ht: HuffTest, comptime ttype: TestType) !void { - if (ht.input.len != 0 and ht.want.len != 0) { - const want_name = comptime fmt.comptimePrint(ht.want, .{ttype.to_s()}); - const input = @embedFile("testdata/" ++ ht.input); - const want = @embedFile("testdata/" ++ want_name); - - var buf = ArrayList(u8).init(testing.allocator); - var bw = try huffmanBitWriter(testing.allocator, buf.writer()); - try writeToType(ttype, &bw, ht.tokens, input); - - var got = buf.items; - try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result - - // Test if the writer produces the same output after reset. - buf.deinit(); - buf = ArrayList(u8).init(testing.allocator); - defer buf.deinit(); - - bw.reset(buf.writer()); - defer bw.deinit(); - - try writeToType(ttype, &bw, ht.tokens, input); - try bw.flush(); - got = buf.items; - try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result - try testWriterEOF(.write_block, ht.tokens, input); - } - - const want_name_no_input = comptime fmt.comptimePrint(ht.want_no_input, .{ttype.to_s()}); - const want_ni = @embedFile("testdata/" ++ want_name_no_input); - - var buf = ArrayList(u8).init(testing.allocator); - var bw = try huffmanBitWriter(testing.allocator, buf.writer()); - - try writeToType(ttype, &bw, ht.tokens, null); - - var got = buf.items; - try testing.expectEqualSlices(u8, want_ni, got); // expect writeBlock to yield expected result - try expect(got[0] & 1 != 1); // expect no EOF - - // Test if the writer produces the same output after reset. - buf.deinit(); - buf = ArrayList(u8).init(testing.allocator); - defer buf.deinit(); - - bw.reset(buf.writer()); - defer bw.deinit(); - - try writeToType(ttype, &bw, ht.tokens, null); - try bw.flush(); - got = buf.items; - - try testing.expectEqualSlices(u8, want_ni, got); // expect writeBlock to yield expected result - try testWriterEOF(.write_block, ht.tokens, &[0]u8{}); -} - -fn writeToType(ttype: TestType, bw: anytype, tok: []const token.Token, input: ?[]const u8) !void { - switch (ttype) { - .write_block => try bw.writeBlock(tok, false, input), - .write_dyn_block => try bw.writeBlockDynamic(tok, false, input), - else => unreachable, - } - try bw.flush(); -} - -// Tests if the written block contains an EOF marker. -fn testWriterEOF(ttype: TestType, ht_tokens: []const token.Token, input: []const u8) !void { - var buf = ArrayList(u8).init(testing.allocator); - defer buf.deinit(); - var bw = try huffmanBitWriter(testing.allocator, buf.writer()); - defer bw.deinit(); - - switch (ttype) { - .write_block => try bw.writeBlock(ht_tokens, true, input), - .write_dyn_block => try bw.writeBlockDynamic(ht_tokens, true, input), - .write_huffman_block => try bw.writeBlockHuff(true, input), - } - - try bw.flush(); - - const b = buf.items; - try expect(b.len > 0); - try expect(b[0] & 1 == 1); -} diff --git a/lib/std/compress/deflate/huffman_code.zig b/lib/std/compress/deflate/huffman_code.zig deleted file mode 100644 index c484d71fad3f..000000000000 --- a/lib/std/compress/deflate/huffman_code.zig +++ /dev/null @@ -1,432 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const math = std.math; -const mem = std.mem; -const sort = std.sort; -const testing = std.testing; - -const Allocator = std.mem.Allocator; - -const bu = @import("bits_utils.zig"); -const deflate_const = @import("deflate_const.zig"); - -const max_bits_limit = 16; - -const LiteralNode = struct { - literal: u16, - freq: u16, -}; - -// Describes the state of the constructed tree for a given depth. -const LevelInfo = struct { - // Our level. for better printing - level: u32, - - // The frequency of the last node at this level - last_freq: u32, - - // The frequency of the next character to add to this level - next_char_freq: u32, - - // The frequency of the next pair (from level below) to add to this level. - // Only valid if the "needed" value of the next lower level is 0. - next_pair_freq: u32, - - // The number of chains remaining to generate for this level before moving - // up to the next level - needed: u32, -}; - -// hcode is a huffman code with a bit code and bit length. -pub const HuffCode = struct { - code: u16 = 0, - len: u16 = 0, - - // set sets the code and length of an hcode. - fn set(self: *HuffCode, code: u16, length: u16) void { - self.len = length; - self.code = code; - } -}; - -pub const HuffmanEncoder = struct { - codes: []HuffCode, - freq_cache: []LiteralNode = undefined, - bit_count: [17]u32 = undefined, - lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate - lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate - allocator: Allocator, - - pub fn deinit(self: *HuffmanEncoder) void { - self.allocator.free(self.codes); - self.allocator.free(self.freq_cache); - } - - // Update this Huffman Code object to be the minimum code for the specified frequency count. - // - // freq An array of frequencies, in which frequency[i] gives the frequency of literal i. - // max_bits The maximum number of bits to use for any literal. - pub fn generate(self: *HuffmanEncoder, freq: []u16, max_bits: u32) void { - var list = self.freq_cache[0 .. freq.len + 1]; - // Number of non-zero literals - var count: u32 = 0; - // Set list to be the set of all non-zero literals and their frequencies - for (freq, 0..) |f, i| { - if (f != 0) { - list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f }; - count += 1; - } else { - list[count] = LiteralNode{ .literal = 0x00, .freq = 0 }; - self.codes[i].len = 0; - } - } - list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 }; - - list = list[0..count]; - if (count <= 2) { - // Handle the small cases here, because they are awkward for the general case code. With - // two or fewer literals, everything has bit length 1. - for (list, 0..) |node, i| { - // "list" is in order of increasing literal value. - self.codes[node.literal].set(@as(u16, @intCast(i)), 1); - } - return; - } - self.lfs = list; - mem.sort(LiteralNode, self.lfs, {}, byFreq); - - // Get the number of literals for each bit count - const bit_count = self.bitCounts(list, max_bits); - // And do the assignment - self.assignEncodingAndSize(bit_count, list); - } - - pub fn bitLength(self: *HuffmanEncoder, freq: []u16) u32 { - var total: u32 = 0; - for (freq, 0..) |f, i| { - if (f != 0) { - total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len)); - } - } - return total; - } - - // Return the number of literals assigned to each bit size in the Huffman encoding - // - // This method is only called when list.len >= 3 - // The cases of 0, 1, and 2 literals are handled by special case code. - // - // list: An array of the literals with non-zero frequencies - // and their associated frequencies. The array is in order of increasing - // frequency, and has as its last element a special element with frequency - // std.math.maxInt(i32) - // - // max_bits: The maximum number of bits that should be used to encode any literal. - // Must be less than 16. - // - // Returns an integer array in which array[i] indicates the number of literals - // that should be encoded in i bits. - fn bitCounts(self: *HuffmanEncoder, list: []LiteralNode, max_bits_to_use: usize) []u32 { - var max_bits = max_bits_to_use; - const n = list.len; - - assert(max_bits < max_bits_limit); - - // The tree can't have greater depth than n - 1, no matter what. This - // saves a little bit of work in some small cases - max_bits = @min(max_bits, n - 1); - - // Create information about each of the levels. - // A bogus "Level 0" whose sole purpose is so that - // level1.prev.needed == 0. This makes level1.next_pair_freq - // be a legitimate value that never gets chosen. - var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo); - // leaf_counts[i] counts the number of literals at the left - // of ancestors of the rightmost node at level i. - // leaf_counts[i][j] is the number of literals at the left - // of the level j ancestor. - var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32); - - { - var level = @as(u32, 1); - while (level <= max_bits) : (level += 1) { - // For every level, the first two items are the first two characters. - // We initialize the levels as if we had already figured this out. - levels[level] = LevelInfo{ - .level = level, - .last_freq = list[1].freq, - .next_char_freq = list[2].freq, - .next_pair_freq = list[0].freq + list[1].freq, - .needed = 0, - }; - leaf_counts[level][level] = 2; - if (level == 1) { - levels[level].next_pair_freq = math.maxInt(i32); - } - } - } - - // We need a total of 2*n - 2 items at top level and have already generated 2. - levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4; - - { - var level = max_bits; - while (true) { - var l = &levels[level]; - if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) { - // We've run out of both leafs and pairs. - // End all calculations for this level. - // To make sure we never come back to this level or any lower level, - // set next_pair_freq impossibly large. - l.needed = 0; - levels[level + 1].next_pair_freq = math.maxInt(i32); - level += 1; - continue; - } - - const prev_freq = l.last_freq; - if (l.next_char_freq < l.next_pair_freq) { - // The next item on this row is a leaf node. - const next = leaf_counts[level][level] + 1; - l.last_freq = l.next_char_freq; - // Lower leaf_counts are the same of the previous node. - leaf_counts[level][level] = next; - if (next >= list.len) { - l.next_char_freq = maxNode().freq; - } else { - l.next_char_freq = list[next].freq; - } - } else { - // The next item on this row is a pair from the previous row. - // next_pair_freq isn't valid until we generate two - // more values in the level below - l.last_freq = l.next_pair_freq; - // Take leaf counts from the lower level, except counts[level] remains the same. - @memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]); - levels[l.level - 1].needed = 2; - } - - l.needed -= 1; - if (l.needed == 0) { - // We've done everything we need to do for this level. - // Continue calculating one level up. Fill in next_pair_freq - // of that level with the sum of the two nodes we've just calculated on - // this level. - if (l.level == max_bits) { - // All done! - break; - } - levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq; - level += 1; - } else { - // If we stole from below, move down temporarily to replenish it. - while (levels[level - 1].needed > 0) { - level -= 1; - if (level == 0) { - break; - } - } - } - } - } - - // Somethings is wrong if at the end, the top level is null or hasn't used - // all of the leaves. - assert(leaf_counts[max_bits][max_bits] == n); - - var bit_count = self.bit_count[0 .. max_bits + 1]; - var bits: u32 = 1; - const counts = &leaf_counts[max_bits]; - { - var level = max_bits; - while (level > 0) : (level -= 1) { - // counts[level] gives the number of literals requiring at least "bits" - // bits to encode. - bit_count[bits] = counts[level] - counts[level - 1]; - bits += 1; - if (level == 0) { - break; - } - } - } - return bit_count; - } - - // Look at the leaves and assign them a bit count and an encoding as specified - // in RFC 1951 3.2.2 - fn assignEncodingAndSize(self: *HuffmanEncoder, bit_count: []u32, list_arg: []LiteralNode) void { - var code = @as(u16, 0); - var list = list_arg; - - for (bit_count, 0..) |bits, n| { - code <<= 1; - if (n == 0 or bits == 0) { - continue; - } - // The literals list[list.len-bits] .. list[list.len-bits] - // are encoded using "bits" bits, and get the values - // code, code + 1, .... The code values are - // assigned in literal order (not frequency order). - const chunk = list[list.len - @as(u32, @intCast(bits)) ..]; - - self.lns = chunk; - mem.sort(LiteralNode, self.lns, {}, byLiteral); - - for (chunk) |node| { - self.codes[node.literal] = HuffCode{ - .code = bu.bitReverse(u16, code, @as(u5, @intCast(n))), - .len = @as(u16, @intCast(n)), - }; - code += 1; - } - list = list[0 .. list.len - @as(u32, @intCast(bits))]; - } - } -}; - -fn maxNode() LiteralNode { - return LiteralNode{ - .literal = math.maxInt(u16), - .freq = math.maxInt(u16), - }; -} - -pub fn newHuffmanEncoder(allocator: Allocator, size: u32) !HuffmanEncoder { - return HuffmanEncoder{ - .codes = try allocator.alloc(HuffCode, size), - // Allocate a reusable buffer with the longest possible frequency table. - // (deflate_const.max_num_frequencies). - .freq_cache = try allocator.alloc(LiteralNode, deflate_const.max_num_frequencies + 1), - .allocator = allocator, - }; -} - -// Generates a HuffmanCode corresponding to the fixed literal table -pub fn generateFixedLiteralEncoding(allocator: Allocator) !HuffmanEncoder { - const h = try newHuffmanEncoder(allocator, deflate_const.max_num_frequencies); - var codes = h.codes; - var ch: u16 = 0; - - while (ch < deflate_const.max_num_frequencies) : (ch += 1) { - var bits: u16 = undefined; - var size: u16 = undefined; - switch (ch) { - 0...143 => { - // size 8, 000110000 .. 10111111 - bits = ch + 48; - size = 8; - }, - 144...255 => { - // size 9, 110010000 .. 111111111 - bits = ch + 400 - 144; - size = 9; - }, - 256...279 => { - // size 7, 0000000 .. 0010111 - bits = ch - 256; - size = 7; - }, - else => { - // size 8, 11000000 .. 11000111 - bits = ch + 192 - 280; - size = 8; - }, - } - codes[ch] = HuffCode{ .code = bu.bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size }; - } - return h; -} - -pub fn generateFixedOffsetEncoding(allocator: Allocator) !HuffmanEncoder { - const h = try newHuffmanEncoder(allocator, 30); - var codes = h.codes; - for (codes, 0..) |_, ch| { - codes[ch] = HuffCode{ .code = bu.bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 }; - } - return h; -} - -fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool { - _ = context; - return a.literal < b.literal; -} - -fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool { - _ = context; - if (a.freq == b.freq) { - return a.literal < b.literal; - } - return a.freq < b.freq; -} - -test "generate a Huffman code from an array of frequencies" { - var freqs: [19]u16 = [_]u16{ - 8, // 0 - 1, // 1 - 1, // 2 - 2, // 3 - 5, // 4 - 10, // 5 - 9, // 6 - 1, // 7 - 0, // 8 - 0, // 9 - 0, // 10 - 0, // 11 - 0, // 12 - 0, // 13 - 0, // 14 - 0, // 15 - 1, // 16 - 3, // 17 - 5, // 18 - }; - - var enc = try newHuffmanEncoder(testing.allocator, freqs.len); - defer enc.deinit(); - enc.generate(freqs[0..], 7); - - try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..])); - - try testing.expectEqual(@as(usize, 3), enc.codes[0].len); - try testing.expectEqual(@as(usize, 6), enc.codes[1].len); - try testing.expectEqual(@as(usize, 6), enc.codes[2].len); - try testing.expectEqual(@as(usize, 5), enc.codes[3].len); - try testing.expectEqual(@as(usize, 3), enc.codes[4].len); - try testing.expectEqual(@as(usize, 2), enc.codes[5].len); - try testing.expectEqual(@as(usize, 2), enc.codes[6].len); - try testing.expectEqual(@as(usize, 6), enc.codes[7].len); - try testing.expectEqual(@as(usize, 0), enc.codes[8].len); - try testing.expectEqual(@as(usize, 0), enc.codes[9].len); - try testing.expectEqual(@as(usize, 0), enc.codes[10].len); - try testing.expectEqual(@as(usize, 0), enc.codes[11].len); - try testing.expectEqual(@as(usize, 0), enc.codes[12].len); - try testing.expectEqual(@as(usize, 0), enc.codes[13].len); - try testing.expectEqual(@as(usize, 0), enc.codes[14].len); - try testing.expectEqual(@as(usize, 0), enc.codes[15].len); - try testing.expectEqual(@as(usize, 6), enc.codes[16].len); - try testing.expectEqual(@as(usize, 5), enc.codes[17].len); - try testing.expectEqual(@as(usize, 3), enc.codes[18].len); - - try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code); - try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code); - try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code); - try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code); - try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code); - try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code); - try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code); - try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code); - try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code); - try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code); - try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code); -} - -test "generate a Huffman code for the fixed literal table specific to Deflate" { - var enc = try generateFixedLiteralEncoding(testing.allocator); - defer enc.deinit(); -} - -test "generate a Huffman code for the 30 possible relative offsets (LZ77 distances) of Deflate" { - var enc = try generateFixedOffsetEncoding(testing.allocator); - defer enc.deinit(); -} diff --git a/lib/std/compress/deflate/testdata/compress-e.txt b/lib/std/compress/deflate/testdata/compress-e.txt deleted file mode 100644 index 5ca186f14c15..000000000000 --- a/lib/std/compress/deflate/testdata/compress-e.txt +++ /dev/null @@ -1 +0,0 @@ -2.7182818284590452353602874713526624977572470936999595749669676277240766303535475945713821785251664274274663919320030599218174135966290435729003342952605956307381323286279434907632338298807531952510190115738341879307021540891499348841675092447614606680822648001684774118537423454424371075390777449920695517027618386062613313845830007520449338265602976067371132007093287091274437470472306969772093101416928368190255151086574637721112523897844250569536967707854499699679468644549059879316368892300987931277361782154249992295763514822082698951936680331825288693984964651058209392398294887933203625094431173012381970684161403970198376793206832823764648042953118023287825098194558153017567173613320698112509961818815930416903515988885193458072738667385894228792284998920868058257492796104841984443634632449684875602336248270419786232090021609902353043699418491463140934317381436405462531520961836908887070167683964243781405927145635490613031072085103837505101157477041718986106873969655212671546889570350354021234078498193343210681701210056278802351930332247450158539047304199577770935036604169973297250886876966403555707162268447162560798826517871341951246652010305921236677194325278675398558944896970964097545918569563802363701621120477427228364896134225164450781824423529486363721417402388934412479635743702637552944483379980161254922785092577825620926226483262779333865664816277251640191059004916449982893150566047258027786318641551956532442586982946959308019152987211725563475463964479101459040905862984967912874068705048958586717479854667757573205681288459205413340539220001137863009455606881667400169842055804033637953764520304024322566135278369511778838638744396625322498506549958862342818997077332761717839280349465014345588970719425863987727547109629537415211151368350627526023264847287039207643100595841166120545297030236472549296669381151373227536450988890313602057248176585118063036442812314965507047510254465011727211555194866850800368532281831521960037356252794495158284188294787610852639813955990067376482922443752871846245780361929819713991475644882626039033814418232625150974827987779964373089970388867782271383605772978824125611907176639465070633045279546618550966661856647097113444740160704626215680717481877844371436988218559670959102596862002353718588748569652200050311734392073211390803293634479727355955277349071783793421637012050054513263835440001863239914907054797780566978533580489669062951194324730995876552368128590413832411607226029983305353708761389396391779574540161372236187893652605381558415871869255386061647798340254351284396129460352913325942794904337299085731580290958631382683291477116396337092400316894586360606458459251269946557248391865642097526850823075442545993769170419777800853627309417101634349076964237222943523661255725088147792231519747780605696725380171807763603462459278778465850656050780844211529697521890874019660906651803516501792504619501366585436632712549639908549144200014574760819302212066024330096412704894390397177195180699086998606636583232278709376502260149291011517177635944602023249300280401867723910288097866605651183260043688508817157238669842242201024950551881694803221002515426494639812873677658927688163598312477886520141174110913601164995076629077943646005851941998560162647907615321038727557126992518275687989302761761146162549356495903798045838182323368612016243736569846703785853305275833337939907521660692380533698879565137285593883499894707416181550125397064648171946708348197214488898790676503795903669672494992545279033729636162658976039498576741397359441023744329709355477982629614591442936451428617158587339746791897571211956187385783644758448423555581050025611492391518893099463428413936080383091662818811503715284967059741625628236092168075150177725387402564253470879089137291722828611515915683725241630772254406337875931059826760944203261924285317018781772960235413060672136046000389661093647095141417185777014180606443636815464440053316087783143174440811949422975599314011888683314832802706553833004693290115744147563139997221703804617092894579096271662260740718749975359212756084414737823303270330168237193648002173285734935947564334129943024850235732214597843282641421684878721673367010615094243456984401873312810107945127223737886126058165668053714396127888732527373890392890506865324138062796025930387727697783792868409325365880733988457218746021005311483351323850047827169376218004904795597959290591655470505777514308175112698985188408718564026035305583737832422924185625644255022672155980274012617971928047139600689163828665277009752767069777036439260224372841840883251848770472638440379530166905465937461619323840363893131364327137688841026811219891275223056256756254701725086349765367288605966752740868627407912856576996313789753034660616669804218267724560530660773899624218340859882071864682623215080288286359746839654358856685503773131296587975810501214916207656769950659715344763470320853215603674828608378656803073062657633469774295634643716709397193060876963495328846833613038829431040800296873869117066666146800015121143442256023874474325250769387077775193299942137277211258843608715834835626961661980572526612206797540621062080649882918454395301529982092503005498257043390553570168653120526495614857249257386206917403695213533732531666345466588597286659451136441370331393672118569553952108458407244323835586063106806964924851232632699514603596037297253198368423363904632136710116192821711150282801604488058802382031981493096369596735832742024988245684941273860566491352526706046234450549227581151709314921879592718001940968866986837037302200475314338181092708030017205935530520700706072233999463990571311587099635777359027196285061146514837526209565346713290025994397663114545902685898979115837093419370441155121920117164880566945938131183843765620627846310490346293950029458341164824114969758326011800731699437393506966295712410273239138741754923071862454543222039552735295240245903805744502892246886285336542213815722131163288112052146489805180092024719391710555390113943316681515828843687606961102505171007392762385553386272553538830960671644662370922646809671254061869502143176211668140097595281493907222601112681153108387317617323235263605838173151034595736538223534992935822836851007810884634349983518404451704270189381994243410090575376257767571118090088164183319201962623416288166521374717325477727783488774366518828752156685719506371936565390389449366421764003121527870222366463635755503565576948886549500270853923617105502131147413744106134445544192101336172996285694899193369184729478580729156088510396781959429833186480756083679551496636448965592948187851784038773326247051945050419847742014183947731202815886845707290544057510601285258056594703046836344592652552137008068752009593453607316226118728173928074623094685367823106097921599360019946237993434210687813497346959246469752506246958616909178573976595199392993995567542714654910456860702099012606818704984178079173924071945996323060254707901774527513186809982284730860766536866855516467702911336827563107223346726113705490795365834538637196235856312618387156774118738527722922594743373785695538456246801013905727871016512966636764451872465653730402443684140814488732957847348490003019477888020460324660842875351848364959195082888323206522128104190448047247949291342284951970022601310430062410717971502793433263407995960531446053230488528972917659876016667811937932372453857209607582277178483361613582612896226118129455927462767137794487586753657544861407611931125958512655759734573015333642630767985443385761715333462325270572005303988289499034259566232975782488735029259166825894456894655992658454762694528780516501720674785417887982276806536650641910973434528878338621726156269582654478205672987756426325321594294418039943217000090542650763095588465895171709147607437136893319469090981904501290307099566226620303182649365733698419555776963787624918852865686607600566025605445711337286840205574416030837052312242587223438854123179481388550075689381124935386318635287083799845692619981794523364087429591180747453419551420351726184200845509170845682368200897739455842679214273477560879644279202708312150156406341341617166448069815483764491573900121217041547872591998943825364950514771379399147205219529079396137621107238494290616357604596231253506068537651423115349665683715116604220796394466621163255157729070978473156278277598788136491951257483328793771571459091064841642678309949723674420175862269402159407924480541255360431317992696739157542419296607312393763542139230617876753958711436104089409966089471418340698362993675362621545247298464213752891079884381306095552622720837518629837066787224430195793793786072107254277289071732854874374355781966511716618330881129120245204048682200072344035025448202834254187884653602591506445271657700044521097735585897622655484941621714989532383421600114062950718490427789258552743035221396835679018076406042138307308774460170842688272261177180842664333651780002171903449234264266292261456004337383868335555343453004264818473989215627086095650629340405264943244261445665921291225648893569655009154306426134252668472594914314239398845432486327461842846655985332312210466259890141712103446084271616619001257195870793217569698544013397622096749454185407118446433946990162698351607848924514058940946395267807354579700307051163682519487701189764002827648414160587206184185297189154019688253289309149665345753571427318482016384644832499037886069008072709327673127581966563941148961716832980455139729506687604740915420428429993541025829113502241690769431668574242522509026939034814856451303069925199590436384028429267412573422447765584177886171737265462085498294498946787350929581652632072258992368768457017823038096567883112289305809140572610865884845873101658151167533327674887014829167419701512559782572707406431808601428149024146780472327597684269633935773542930186739439716388611764209004068663398856841681003872389214483176070116684503887212364367043314091155733280182977988736590916659612402021778558854876176161989370794380056663364884365089144805571039765214696027662583599051987042300179465536788567430285974600143785483237068701190078499404930918919181649327259774030074879681484882342932023012128032327460392219687528340516906974194257614673978110715464186273369091584973185011183960482533518748438923177292613543024932562896371361977285456622924461644497284597867711574125670307871885109336344480149675240618536569532074170533486782754827815415561966911055101472799040386897220465550833170782394808785990501947563108984124144672821865459971596639015641941751820935932616316888380132758752601460507676098392625726411120135288591317848299475682472564885533357279772205543568126302535748216585414000805314820697137262149755576051890481622376790414926742600071045922695314835188137463887104273544767623577933993970632396604969145303273887874557905934937772320142954803345000695256980935282887783710670585567749481373858630385762823040694005665340584887527005308832459182183494318049834199639981458773435863115940570443683515285383609442955964360676090221741896883548131643997437764158365242234642619597390455450680695232850751868719449064767791886720306418630751053512149851051207313846648717547518382979990189317751550639981016466414592102406838294603208535554058147159273220677567669213664081505900806952540610628536408293276621931939933861623836069111767785448236129326858199965239275488427435414402884536455595124735546139403154952097397051896240157976832639450633230452192645049651735466775699295718989690470902730288544945416699791992948038254980285946029052763145580316514066229171223429375806143993484914362107993576737317948964252488813720435579287511385856973381976083524423240466778020948399639946684833774706725483618848273000648319163826022110555221246733323184463005504481849916996622087746140216157021029603318588727333298779352570182393861244026868339555870607758169954398469568540671174444932479519572159419645863736126915526457574786985964242176592896862383506370433939811671397544736228625506803682664135541448048997721373174119199970017293907303350869020922519124447393278376156321810842898207706974138707053266117683698647741787180202729412982310888796831880854367327806879771659111654224453806625861711729498038248879986504061563975629936962809358189761491017145343556659542757064194408833816841111166200759787244137082333917886114708228657531078536674695018462140736493917366254937783014074302668422150335117736471853872324040421037907750266020114814935482228916663640782450166815341213505278578539332606110249802273093636740213515386431693015267460536064351732154701091440650878823636764236831187390937464232609021646365627553976834019482932795750624399645272578624400375983422050808935129023122475970644105678361870877172333555465482598906861201410107222465904008553798235253885171623518256518482203125214950700378300411216212126052726059944320443056274522916128891766814160639131235975350390320077529587392412476451850809163911459296071156344204347133544720981178461451077872399140606290228276664309264900592249810291068759434533858330391178747575977065953570979640012224092199031158229259667913153991561438070129260780197022589662923368154312499412259460023399472228171056603931877226800493833148980338548909468685130789292064242819174795866199944411196208730498064385006852620258432842085582338566936649849720817046135376163584015342840674118587581546514598270228676671855309311923340191286170613364873183197560812569460089402953094429119590295968563923037689976327462283900735457144596414108229285922239332836210192822937243590283003884445701383771632056518351970100115722010956997890484964453434612129224964732356126321951155701565824427661599326463155806672053127596948538057364208384918887095176052287817339462747644656858900936266123311152910816041524100214195937349786431661556732702792109593543055579732660554677963552005378304619540636971842916168582734122217145885870814274090248185446421774876925093328785670674677381226752831653559245204578070541352576903253522738963847495646255940378924925007624386893776475310102323746733771474581625530698032499033676455430305274561512961214585944432150749051491453950981001388737926379964873728396416897555132275962011838248650746985492038097691932606437608743209385602815642849756549307909733854185583515789409814007691892389063090542534883896831762904120212949167195811935791203162514344096503132835216728021372415947344095498316138322505486708172221475138425166790445416617303200820330902895488808516797258495813407132180533988828139346049850532340472595097214331492586604248511405819579711564191458842833000525684776874305916390494306871343118796189637475503362820939949343690321031976898112055595369465424704173323895394046035325396758354395350516720261647961347790912327995264929045151148307923369382166010702872651938143844844532639517394110131152502750465749343063766541866128915264446926222884366299462732467958736383501937142786471398054038215513463223702071533134887083174146591492406359493020921122052610312390682941345696785958518393491382340884274312419099152870804332809132993078936867127413922890033069995875921815297612482409116951587789964090352577345938248232053055567238095022266790439614231852991989181065554412477204508510210071522352342792531266930108270633942321762570076323139159349709946933241013908779161651226804414809765618979735043151396066913258379033748620836695475083280318786707751177525663963479259219733577949555498655214193398170268639987388347010255262052312317215254062571636771270010760912281528326508984359568975961038372157726831170734552250194121701541318793651818502020877326906133592182000762327269503283827391243828198170871168108951187896746707073377869592565542713340052326706040004348843432902760360498027862160749469654989210474443927871934536701798673920803845633723311983855862638008516345597194441994344624761123844617615736242015935078520825600604101556889899501732554337298073561699861101908472096600708320280569917042590103876928658336557728758684250492690370934262028022399861803400211320742198642917383679176232826444645756330336556777374808644109969141827774253417010988435853189339175934511574023847292909015468559163792696196841000676598399744972047287881831200233383298030567865480871476464512824264478216644266616732096012564794514827125671326697067367144617795643752391742928503987022583734069852309190464967260243411270345611114149835783901793499713790913696706497637127248466613279908254305449295528594932793818341607827091326680865655921102733746700132583428715240835661522165574998431236278287106649401564670141943713823863454729606978693335973109537126499416282656463708490580151538205338326511289504938566468752921135932220265681856418260827538790002407915892646028490894922299966167437731347776134150965262448332709343898412056926145108857812249139616912534202918139898683901335795857624435194008943955180554746554000051766240202825944828833811886381749594284892013520090951007864941868256009273977667585642598378587497776669563350170748579027248701370264203283965756348010818356182372177082236423186591595883669487322411726504487268392328453010991677518376831599821263237123854357312681202445175401852132663740538802901249728180895021553100673598184430429105288459323064725590442355960551978839325930339572934663055160430923785677229293537208416693134575284011873746854691620648991164726909428982971065606801805807843600461866223562874591385185904416250663222249561448724413813849763797102676020845531824111963927941069619465426480006761727618115630063644321116224837379105623611358836334550102286170517890440570419577859833348463317921904494652923021469259756566389965893747728751393377105569802455757436190501772466214587592374418657530064998056688376964229825501195065837843125232135309371235243969149662310110328243570065781487677299160941153954063362752423712935549926713485031578238899567545287915578420483105749330060197958207739558522807307048950936235550769837881926357141779338750216344391014187576711938914416277109602859415809719913429313295145924373636456473035037374538503489286113141638094752301745088784885645741275003353303416138096560043105860548355773946625033230034341587814634602169235079216111013148948281895391028916816328709309713184139815427678818067628650978085718262117003140003377301581536334149093237034703637513354537634521050370995452942055232078817449370937677056009306353645510913481627378204985657055608784211964039972344556458607689515569686899384896439195225232309703301037277227710870564912966121061494072782442033414057441446459968236966118878411656290355117839944070961772567164919790168195234523807446299877664824873753313018142763910519234685081979001796519907050490865237442841652776611425351538665162781316090964802801234493372427866930894827913465443931965254154829494577875758599482099181824522449312077768250830768282335001597040419199560509705364696473142448453825888112602753909548852639708652339052941829691802357120545328231809270356491743371932080628731303589640570873779967845174740515317401384878082881006046388936711640477755985481263907504747295012609419990373721246201677030517790352952793168766305099837441859803498821239340919805055103821539827677291373138006715339240126954586376422065097810852907639079727841301764553247527073788764069366420012194745702358295481365781809867944020220280822637957006755393575808086318932075864444206644691649334467698180811716568665213389686173592450920801465312529777966137198695916451869432324246404401672381978020728394418264502183131483366019384891972317817154372192103946638473715630226701801343515930442853848941825678870721238520597263859224934763623122188113706307506918260109689069251417142514218153491532129077723748506635489170892850760234351768218355008829647410655814882049239533702270536705630750317499788187009989251020178015601042277836283644323729779929935160925884515772055232896978333126427671291093993103773425910592303277652667641874842441076564447767097790392324958416348527735171981064673837142742974468992320406932506062834468937543016787815320616009057693404906146176607094380110915443261929000745209895959201159412324102274845482605404361871836330268992858623582145643879695210235266673372434423091577183277565800211928270391042391966426911155333594569685782817020325495552528875464466074620294766116004435551604735044292127916358748473501590215522120388281168021413865865168464569964810015633741255098479730138656275460161279246359783661480163871602794405482710196290774543628092612567507181773641749763254436773503632580004042919906963117397787875081560227368824967077635559869284901628768699628053790181848148810833946900016380791075960745504688912686792812391148880036720729730801354431325347713094186717178607522981373539126772812593958220524289991371690685650421575056729991274177149279608831502358697816190894908487717722503860872618384947939757440664912760518878124233683125467278331513186758915668300679210215947336858591201395360301678110413444411030903388761520488296909104689167671555373346622545575975202624771242796225983278405833585897671474205724047439720232895903726148688388003174146490203843590358527993123871042845981608996101945691646983837718267264685264869172948414153004604004299585035164101899027529366867431834955447458124140190754681607770977920579383895378192128847409929537040546962226547278807248685508046571043123854873351653070570784584243335550958221912862797205455466267099131902370311779690892786623112661337671178512943059323281605826535623848164192144732543731002062738466812351691016359252588256806438946389880872735284406462208149513862275239938938734905082625472417781702582044129853760499827899020083498387362992498125742354568439023012261733665820546785671147973065077035475620567428300187473019197310881157516777005071432012726354601912460800451608108641835539669946936947322271670748972850464195392966434725254724357659192969949061670189061433616907056148280980363243454128229968275980226694045642181328624517549652147221620839824594576613342710564957193564431561774500828376935700995419541839029151033187933907614207467028867968594985439789457300768939890070073924697461812855764662265412913204052279071212820653775058280040897163467163709024906774736309136904002615646432159560910851092445162454420141442641660181385990017417408244245378610158433361777292580611159192008414091888191208858207627011483671760749046980914443057262211104583300789331698191603917150622792986282709446275915009683226345073725451366858172483498470080840163868209726371345205439802277866337293290829914010645589761697455978409211409167684020269370229231743334499986901841510888993165125090001163719114994852024821586396216294981753094623047604832399379391002142532996476235163569009445086058091202459904612118623318278614464727795523218635916551883057930657703331498510068357135624341881884405780028844018129031378653794869614630467726914552953690154167025838032477842272417994513653582260971652588356712133519546838335349801503269359798167463231847628306340588324731228951257944267639877946713121042763380872695738609314631539148548792514028885025189788076023838995615684850391995855029256054176767663145354058496296796781349420116003325874431438746248313850214980401681940795687219268462617287403480967931949965604299190281810597603263251746405016454606266765529010639868703668263299050577706266397868453584384057673298268163448646707439990917504018892319267557518354054956017732907127219134577524905771512773358423314008356080926962298894163047287780054743798498545562870729968407382937218623831766524716090967192007237658894226186550487552614557855898773008703234726418384831040394818743616224455286163287628541175946460497027724490799275146445792982549802258601001772437840167723166802004162547244179415547810554178036773553354467030326469619447560812831933095679685582771932031205941616693902049665352189672822671972640029493307384717544753761937017882976382487233361813499414541694736549254840633793674361541081593464960431603544354737728802361047743115330785159902977771499610274627769759612488879448609863349422852847651310277926279743981957617505591300993377368240510902583759345170015340522266144077237050890044496613295859536020556034009492820943862994618834790932894161098856594954213114335608810239423706087108026465913203560121875933791639666437282836752328391688865373751335794859860107569374889645657187292540448508624449947816273842517229343960137212406286783636675845331904743954740664015260871940915743955282773904303868772728262065663129387459875317749973799293043294371763801856280061141619563942414312254397099163565102848315765427037906837175764870230052388197498746636856292655058222887713221781440489538099681072143012394693530931524054081215705402274414521876541901428386744260011889041724570537470755550581632831687247110220353727166112304857340460879272501694701067831178927095527253222125224361673343366384756590949728221809418684074238351567868893421148203905824224324264643630201441787982022116248471657468291146315407563770222740135841109076078464780070182766336227978104546331131294044833570134869585165267459515187680033395522410548181767867772152798270250117195816577603549732923724732067853690257536233971216884390878879262188202305529937132397194333083536231248870386416194361506529551267334207198502259771408638122015980894363561808597010080081622557455039101321981979045520049618583777721048046635533806616517023595097133203631578945644487800945620369784973459902004606886572701865867757842758530645706617127194967371083950603267501532435909029491516973738110897934782297684100117657987098185725131372267749706609250481876835516003714638685918913011736805218743265426063700710595364425062760458252336880552521181566417553430681181548267844169315284408461087588214317641649835663127518728182948655658524206852221830755306118393326934164459415342651778653397980580828158806300749952897558204686612590853678738603318442905510689778698417735603118111677563872589911516803236547002987989628986181014596471307916144369564690909518788574398821730583884980809523077569358851616027719521488998358632323127308909861560777386006984035267826785387215920936255817889813416247486456433211043194821421299793188104636399541496539441501383868748384870224681829391860319598667962363489309283087840712400431022706137591368056518861313458307990705003607588327248867879324093380071864152853317943535073401891193638546730000660453783784472469288830546979000131248952100446949032058838294923613919284305249167833012980192255157050378521810552961623637523647962685751660066539364142273063001648652613891842243501797455993616794063303522111829071597538821839777552812981538570168702202620274678647916644030729018445497956399844836807851997088201407769199261674991148329821854382718946282165387064858588646221611410343570342878862979083418871606214430014533275029715104673156021000043869510583773779766003460887624861640938645252177935289947578496255243925598620521409052346250847830487046492688313289470553891357290706967599556298586669559721686506052072801342104355762779184021797626656484580261591407173477009039475168017709900129391137881248534255949312866653465033728846390649968460644741907524313323903404908195233044389559060547854954620263256676813262435925020249516275607080900436460421497025691488555265022810327762115842282433269528629137662675481993546118143913367579700141255870143319434764035725376914388899683088262844616425575034001428982557620386364384137906519612917777354183694676232982904981261717676191554292570438432239918482261744350470199171258214687683172646078959690569981353264435973965173473319484798758064137926885413552523275720457329477215706850016950046959758389373527538622664943456437071610511521617176237598050900553232154896062817794302268640579555845730600598376482703339859420098582351400179507104569019191359062304102336798080907240196312675268916362136351032648077232914950859151265812143823371072949148088472355286394195993455684156344577951727033374238129903260198160571971183950662758220321837136059718025940870615534713104482272716848395524105913605919812444978458110854511231668173534838253724825347636777581712867205865148285317273569069839935110763432091319780314031658897379628301178409806410175016511072932907832177487566289310650383806093372841399226733384778203302020700517188941706465146238366720632742644336612174011766914919235570905644803016342294301837655263108450172510307540942604409687066288066265900569082451407632599158164499361455172452057020443093722305550217222299706209749268609762787409626448772056043078634808885709143464793241536214303199965695610753570417207285334250171325558818113295504095217830139465216436594262960768570585698507157151317262928960072587601564840556088613165411835958628710665496282599535127193244635791046554389165150954187306071015034430609582302257455974944275067630926322529966338219395202927917973247094559691016402983683080426309910481567503623509654924302589575273521412445149542462972258510120707802110188106722347972579330653187713438466713807546383471635428854957610942841898601794658721444495198801550804042506452191484989920400007310672369944655246020908767882300064337725657385010969899058191290957079866699453765080407917852438222041070599278889267745752084287526377986730360561230710723922581504781379172731261234878334034473833573601973235946604273704635201327182592410906040097638585857716958419563109577748529579836844756803121874818202833941887076311731615289811756429711334181497218078040465077657204457082859417475114926179367379999220181789399433337731146911970737861041963986422166045588965683206701337505745038872111332436739840284188639147633491695114032583475841514170325690161784931455706904169858050217798497637014758914810543205854914100662201721719726878930012101267481270235940855162601689425111458499658315589660460091525797881670384625905383256920520425791378948827579603278877535466861441826827797651258953563761485994485049706638406266121957141911063246061774180577212381659872472432252969098533628440799030007594546281549235506086481557928961969617060715201589825299772803520002610888814176506636216905928021516429198484077446143617891415191517976537848282687018750030264867608433204658525470555882410254654806040437372771834769014720664234434374255514129178503032471263418076525187802925534774001104853996960549926508093910691337614841834884596365621526610332239417467064368340504749943339802285610313083038484571294767389856293937641914407036507544622061186499127249643799875806537850203753189972618014404667793050140301580709266213229273649718653952866567538572115133606114457222800851183757899219543063413692302293139751143702404830227357629039911794499248480915071002444078482866598579406525539141041497342780203520135419925977628178182825372022920108186449448349255421793982723279357095828748597126780783134286180750497175747373730296280477376908932558914598141724852658299510882230055223242218586191394795184220131553319634363922684259164168669438122537135960710031743651959027712571604588486044820674410935215327906816032054215967959066411120187618531256710150212239401285668608469435937408158536481912528004920724042172170913983123118054043277015835629513656274610248827706488865037765175678806872498861657094846665770674577000207144332525555736557083150320019082992096545498737419756608619533492312940263904930982014700371161829485939931199955070455381196711289367735249958182011774799788636393286405807810818657337668157893827656450642917396685579555053188715314552353070355994740186225988149854660737787698781542360397080977412361518245964026869979609564523828584235953564615185448165799966460648261396618720304839119560250381111550938420209894591555760083897989949964566262540514195610780090298667014635238532066032574466820259430618801773091109212741138269148784355679352572808875543164693077235363768226036080174040660997151176880434927489197133087822951123746632635635328517394189466510943745768270782209928468034684157443127739811044186762032954475468077511126663685479944460934809992951875666499902261686019672053749149951226823637895865245462813439289338365156536992413109638102559114643923805213907862893561660998836479175633176725856523591069520326895990054884753424160586689820067483163174286329119633399132709086065074595260357157323069712106423424081597068328707624437165532750228797802598690981111226558888151520837482450034463046505984569690276166958278982913613535306291331427881888249342136442417833519319786543940201465328083410341785272489879050919932369270996567133507711905899945951923990615156165480300145359212550696405345263823452155999210578191371030188979206408883974767667144727314254467923500524618849237455307575734902707342496298879996942094595961008702501329453325358045689285707241207965919809225550560061971283541270202072583994171175520920820151096509526685113897577150810849443508285458749912943857563115668324566827992991861539009255871716840495663991959154034218364537212023678608655364745175654879318925644085274489190918193411667583563439758886046349413111875241038425467937999203546910411935443113219136068129657568583611774564654674861061988591414805799318725367531243470335482637527081353105570818049642498584646147973467599315946514787025065271083508782350656532331797738656666181652390017664988485456054961300215776115255813396184027067814900350252876823607822107397102339146870159735868589015297010347780503292154014359595298683404657471756232196640515401477953167461726208727304820634652469109953327375561090578378455945469160223687689641425960164689647106348074109928546482353083540132332924864037318003195202317476206537726163717445360549726690601711176761047774971666890152163838974311714180622222345718567941507299526201086205084783127474791909996889937275229053674785020500038630036526218800670926674104806027341997756660029427941090400064654281074454007616429525362460261476180471744322889953285828397762184600967669267581270302806519535452053173536808954589902180783145775891280203970053633193821100095443241244197949192916205234421346395653840781209416214835001155883618421164283992454027590719621537570187067083731012246141362048926555668109467076386536083015847614512581588569610030337081197058344452874666198891534664244887911940711423940115986970795745946337170243268484864632018986352827092313047089215684758207753034387689978702323438584381125011714013265769320554911860153519551654627941175593967947958810333935413289702528893533748106257875620364294270257512121137330213811951395756419122685155962476203282038726342066227347868223036522019655729325905068134849292299647248229359787842720945578267329975853818536442370617353517653060396801087899490506654491544577952166038552398013798104340564182403396162494910454712104839439200945914647542424785991096900046541371091630096785951563947332190934511838669964622788855817353221326876634958059123761251203010983867841195725887799206041260049865895027247133146763722204388398558347770112599424691208308595666787531942465131444389971195968105937957532155524204659410081418351120174196853432672343271868099625045432475688702055341969199545300952644398446384346598830418262932239295612610045884644244285011551557765935780379565026806130721758672048541797157896401554276881090475899564605488362989140226580026134158039480357971019004151547655018391755772677897148793477372747525743898158705040701968215101218826088040084551332795162841280679678965570163917067779841529149397403158167896865448841319046368332179115059107813898261026271979696826411179918656038993895418928488851750122504754778999508544083983800725431468842988412616042682248823097788556495765424017114510393927980290997604904428832198976751320535115230545666467143795931915272680278210241540629795828828466355623580986725638200565215519951793551069127710538552661926903526081367717666435071213453983711357500975854405939558661737828297120544693182260401670308530911657973113259516101749193468250063285777004686987177255226525708428745733039859744230639751837209975339055095883623642814493247460522424051972825153787541962759327436278819283740253185668545040893929401040561666867664402868211607294830305236465560955351079987185041352121321534713770667681396211443891632403235741573773787908838267618458756361026435182951815392455211729022985278518025598478407179607904114472041476091765804302984501746867981277584971731733287305281134969591668387877072315968334322509070204019030503595891994666652037530271923764252552910347950343816357721698115464329245608951158732012675424975710520894362639501382962152214033621065422821876739580121286442788547491928976959315766891987305176388698461503354594898541849550251690616888419122873385522699976822609645007504500096116866129171093180282355042553653997166054753907348915189650027442328981181709248273610863801576007240601649547082331349361582435128299050405405333992577071321011503713898695076713447940748097845416328110406350804863393555238405735580863718763530261867971725608155328716436111474875107033512913923595452951407437943144900950809932872153235195999616750297532475931909938012968640379783553559071355708369947311923538531051736669154087312467233440702525006918026747725078958903448856673081487299464807786497709361969389290891718228134002845552513917355978456150353144603409441211512001738697261466786933733154341007587514908295822756919350542184106448264951943804240543255345965248373785310657979037977505031436474651422484768831323479762673689855474944277949916560108528257618964374464656819789319422077536824661110427671936481836360534108748971066866318805026555929568123959680449295166615409802610781691689418764353363449482900125929366840591370059526914934421861891742142561071896846626335874414976973921566392767687720145153302241853125308442727245771161505550519076276250016522166274796257424425420546785767478190959486500575711016264847833741198041625940813327229905891486422127968042984725356237202887830051788539737909455265135144073130049869453403245984236934627060242579432563660640597549471239092372458126154582526667304702319359866523378856244229188278436440434628094888288712101968642736370461639297485616780079779959696843367730352483047478240669928277140069031660709951473154191919911453182543906294573298686613524886500574780251977607442660798300291573030523199052185718628543687577860915726925232573171665625274275808460620177046433101212443409281314659760221360416223031167750085960128475289259463348312408766740128170543067985261868949895004918275008304998926472034986965363326210919830621495095877228260815566702155693484634079776879525038204442326697479264829899016938511552124688935873289878336267819361764023681714606495185508780596635354698788205094762016350757090024201498400967867845405354130050482404996646978558002628931826518708714613909521454987992300431779500489569529280112698632533646737179519363094399609176354568799002814515169743717518330632232942199132137614506411391269837128970829395360832883050256072727563548374205497856659895469089938558918441085605111510354367477810778500572718180809661542709143010161515013086522842238721618109043183163796046431523184434669799904865336375319295967726080853457652274714047941973192220960296582500937408249714373040087376988068797038047223488825819819025644086847749767508999164153502160223967816357097637814023962825054332801828798160046910336602415904504637333597488119998663995617171089911809851197616486499233594328274275983382931099806461605360243604040848379619072542165869409486682092396143083817303621520642297839982533698027039931804024928814430649614747600087654305571672697259114631990688823893005380061568007730984416061355843701277573463708822073792921409548717956947854414951731561828176343929570234710460088230637509877521391223419548471196982303169544468045517922669260631327498272520906329003279972932906827204647650366969765227673645419031639887433042226322021325368176044169612053532174352764937901877252263626883107879345194133825996368795020985033021472307603375442346871647223795507794130304865403488955400210765171630884759704098331306109510294140865574071074640401937347718815339902047036749084359309086354777210564861918603858715882024476138160390378532660185842568914109194464566162667753712365992832481865739251429498555141512136758288423285957759412684479036912662015308418041737698963759002546999454131659341985624780714434977201991702665380714107259910648709897259362243300706760476097690456341576573395549588448948093604077155688747288451838106069038026528318275560395905381507241627615047252487759578650784894547389096573312763852962664517004459626327934637721151028545472312880039058405918498833810711366073657536918428084655898982349219315205257478363855266205400703561310260405145079325925798227406012199249391735122145336707913500607486561657301854049217477162051678486507913573336334257685988361252720250944019430674728667983441293018131344299088234006652915385763779110955708000600143579956351811596764725075668367726052352939773016348235753572874236648294604770429166438403558846422370760111774821079625901180265548868995181239470625954254584491340203400196442965370643088660925268811549596291166168612036195319253262662271108142149856132646467211954801142455133946382385908540917878668826947602781853283155445565265933912487885639504644196022475186011405239187543742526581685003052301877096152411653980646785444273124462179491306502631062903402737260479940181929954454297256377507172705659271779285537195547433852182309492703218343678206382655341157162788603990157495208065443409462446634653253581574814022471260618973060860559065082163068709634119751925774318683671722139063093061019303182326666420628155129647685313861018672921889347039342072245556791239578260248978371473556820782675452142687314252252601795889759116238720807580527221031327444754083319215135934526961397220564699247718289310588394769170851420631557192703636345039529604362885088555160008371973526383838996789184600327073682083234847108471706160879195227388252347506380811606090840124222431476103563328940609282430125462013806032608121942876847907192546246309055749298781661271916548229644317263587524548607563020667656942355342774617635549231817456159185668061686428714964129290560130053913469569829490891003991259088290348791943368696942620662946948514931472688923571615032405542263391673583102728579723061998175868700492227418629077079508809336215346303842967525604369606110193842723883107587771653594778681499030978765900869583480043137176832954871752604714113064847270887246697164585218774442100900090916189819413456305028950484575822161887397443918833085509908566008543102796375247476265353031558684515120283396640547496946343986288291957510384781539068343717740714095628337554413567955424664601335663617305811711646062717854078898495334329100315985673932305693426085376230981047171826940937686754301837015557540822371538037838383342702379535934403549452173960327095407712107332936507766465603712364707109272580867897181182493799540477008369348889220963814281561595610931815183701135104790176383595168144627670903450457460997444500166918675661035889313483800512736411157304599205955471122443903196476642761038164285918037488354360663299436899730090925177601162043761411616688128178292382311221745850238080733727204908880095181889576314103157447684338100457385008523652069340710078955916549813037292944462306371284357984809871964143085146878525033128989319500645722582281175483887671061073178169281242483613796475692482076321356427357261609825142445262515952514875273805633150964052552659776922077806644338105562443538136258941809788015677378951310313157361136026047890761945591820289365770116416881703644242694283057457471567494391573593353763114830246668754727566653059819746822346578699972291792416156043557665183382167059157867799311835820189855730344883681934418305987021880502259192818047775223884407167894780414701414651073580452021499197980812095692195622632313741870979731320870864552236740416185590793816745658234353037283309503729022429802768451559528656923189798000383061378732434546500582722712325031420712488100290697226311129067629080951145758060270806092801504406139446350643069742785469477459876821004441453438033759717384777232052065301037861326418823586036569054773343070911759152582503029410738914441818378779490613137536794654893375260322906277631983337976816641721083140551864133302224787118511817036598365960493964571491686005656771360533192423185262166760222073368844844409234470948568027905894191829969467724456269443308241243846160408284006424867072583661011433404214473683453638496544701067827313169538435919120440283949541956874453676459875488726170687163109591315801609722382049772577307454562979127906177531663252857205858766376754282917933549923678212008601904369428956102301731743150352204665675088491593025926618816581008701658499456495586855628208747248318351516339189292646558880593601275151838235485893426165223086697314511412035659916934103076974774451947043836739600076578628245472064617380804602903639144493859012422380173377038154675297645596518492676039300171943042511794045679862114630138402371099347243455794730048929825402680821621522346560274258486595687074510352794291633405915025075992398611224340312056999780516223878772230396359709132856830486160362127579561601328561866388146004722200580017580282279272167842720649966956840905752590774886105493806116954293569077377792821084159737469613143291808510446953973485067590503662391722108732333169909603363771705474725026941732982890400239372879549386540463828596742216318201530139629734398479588628632934746650690284066719018081265539973675916799759010867483920062877888531102781695087545740384607594616919584610655963327283485609570305572502494416337066573150237126843581984154103154401008430380631442183776750349813408169325201240813452285974626715177152223063741359255747513535160669108359443999692315898156732033027129284241219651936303734407981204656795322986357374589031654007016472204989445629050395873788912680565516464274460174738175296313458739390484560414203426465560422112239134631023161290836446988901247285192778589195228773637440432659264672239982186452797664826673070168802722052338600372842903155828454593854349099449420750911108532138744823216151007808922516285123275724355101999038195993350032641446053470357293073912578481757987468353429629749652545426864234949270336399427519354240001973125098882419600095766257217621860474573769577649582201796258392376391717855799468922496750179251915218219624653575570564228220399546682648329822996167217080156801080799777126517156274295763666959661983507435667132218383358509536665806605597148376773866922551603463644386269977295750658468929599809168949981898588529537874489519527097766262684177088590284321676352132630838812766335363319004134332844347630067982023716933653652880580156390360562722752187272454764258840995216482554453662083811789117725225682611478014242896970967121967502094421226279437073328703410646312100557376727450271638975234111426287828736758358819056742163061523416789476056879277154789714326222041069587947186435439940738639948986836168919377836648327137363654676901173760246643082285362494712605173293777247276797635865806019396287718060679122426813922872134061694882029506831654589707623668302556167559477498715183426989208952182644710514911419441192277010977616645850068963849426165593473112961064282379048216056210094265076173838082479030510998790719611852832556787472942907151041468948104916751035295897242381802288151276582257190705537652455285511598636421244284176256230139538669970308943645907600684938040875210854159851278070333207779865635907968462191534944587677170063778573171211036517486371634098385626541555573292664616402279791195975248525300376741774056125700303625811704838385391207273191845064713669122576415213769896260940351804147432053600369234179035440735703058314741623452840188940808983125191307741823338981880316339159565954543405777784331681162551898060409183018907512170192983622897099598983405484962284289398469847938668614293324543983592637036699355184231661615244505980576745765335552338715678211466689996845227042954589710922163652573965950289645637766038988037941517917867910675199009966139206238732318786758420544279396366759104126821843375015743069045967947046685602358283919759975285865384338189120042853787549302768972168199113340697282255535300044743958830079799736518459131437946494086272149669719100359399974735262764126125995350902609540048669398955899487421379590802893196914845826873123710180229775301190684280440780938156598081694611679374425663244656799606363751546304833112722231812338371779800439731087402647536582575657351059978314264831879619843765495877803685261751835391844920488198629786329743136948511780579298636452193232481339393090754566368038513630619718033957979522539508697432546502659123585049283028832934489284591373621624852528877442891851104093746333590660233239711922814450735588373324057814862662207486215513375036775585494138678352928273109003823116855374520901095101174796663003330352534143230024288248051396631446632656081582045216883922312025671065388459503224002320453633895521539919011035217362720909565500846486605368975498478995875596103167696587161281951919668893326641203784750417081752273735270989343717167642329956935697166213782736138899530515711822960896394055380431939398453970864418654291655853168697537052760701061488025700785387150835779480952313152747735711713643356413242974208137266896149109564214803567792270566625834289773407718710649866150447478726164249976671481383053947984958938064202886667951943482750168192023591633247099185942520392818083953020434979919361853380201407072481627304313418985942503858404365993281651941497377286729589582881907490040331593436076189609669494800067194371424058105327517721952474344983414191979918179909864631583246021516575531754156198940698289315745851842783390581029411600498699307751428513021286202539508732388779357409781288187000829944831476678183644656510024467827445695591845768068704978044824105799710771577579093525803824227377612436908709875189149049904225568041463131309240101049368241449253427992201346380538342369643767428862595140146178201810734100565466708236854312816339049676558789901487477972479202502227218169405159042170892104287552188658308608452708423928652597536146290037780167001654671681605343292907573031466562485809639550080023347676187068086526878722783177420214068980703410506200235273632267291964034093571225623659496432076928058165514428643204955256838543079254299909353199329432966018220787933122323225928276556048763399988478426451731890365879756498207607478270258861409976050788036706732268192473513646356758611212953074644777149423343867876705824452296605797007134458987594126654609414211447540007211790607458330686866231309155780005966522736183536340439991445294960728379007338249976020630448806064574892740547730693971337007962746135534442514745423654662752252624869916077111131569725392943756732215758704952417232428206555322808868670153681482911738542735797154157943689491063759749151524510096986573825654899585216747260540468342338610760823605782941948009334370046866568258579827323875158302566720152604684361412652956519894291184887986819088277339147282063794512260294515707367105637720023427811802621502691790400488001808901847311751199425460594416773315777951735444490965752131026306836047140331442314298077895617051256930051804287472368435536402764392777908638966566390166776625678575354239947427919442544664643315554138265543388487778859972063679660692327601733858843763144148113561693030468420017434061395220072403658812798249143261731617813894970955038369479594617979829257740992171922783223006387384996138434398468502234780438733784470928703890536420557474836284616809363650973790900204118525835525201575239280826462555785658190226958376345342663420946214426672453987171047721482128157607275305173330963455909323664528978019175132987747952929099598069790148515839540444283988381797511245355548426126784217797728268989735007954505834273726937288386902125284843370917479603207479554080911491866208687184899550445210616155437083299502854903659617362726552868081324793106686855857401668022408227992433394360936223390321499357262507480617409173636062365464458476384647869520547719533384203403990244761056010612777546471464177412625548519830144627405538601855708359981544891286863480720710061787059669365218674805943569985859699554089329219507269337550235821561424994538234781138316591662683103065194730233419384164076823699357668723462219641322516076261161976034708844046473083172682611277723613381938490606534404043904909864126903479263503943531836741051762565704797064478004684323069430241749029731181951132935746854550484711078742905499870600373983113761544808189067620753424526993443755719446665453524088287267537759197074526286322840219629557247932987132852479994638938924943286917770190128914220188747760484939855471168524810559991574441551507431214406120333762869533792439547155394213121021954430556748370425907553004950664994802614794524739012802842646689229455664958621308118913500279654910344806150170407268010067948926855360944990373928383520627992820181576427054962997401900837493444950600754365525758905546552402103412862124809003162941975876195941956592556732874237856112669741771367104424821916671499611728903944393665340294226514575682907490402153401026923964977275904729573320027982816062130523130658731513076913832317193626664465502290735017347656293033318520949298475227462534564256702254695786484819977513326393221579478212493307051107367474918016345667888810782101151826314878755138027101379868751299375133303843885631415175908928986956197561123025310875057188962535763225834275763348421016668109884514141469311719314272028007223449941999003964948245457520704922091620614222912795322688239046498239081592961111003756999529251250673688233852648213896986384052437049402152187547825163347082430303521036927849762517317825860862215614519165573478940019558704784741658847364803865995119651409542615026615147651220820245816010801218275982577477652393859159165067449846149161165153821266726927461290533753163055654440793427876550267301214578324885948736899073512166118397877342715872870912311383472485146035661382188014840560716074652441118841800734067898587159273982452147328317214621907330492060817440914125388918087968538960627860118193099489240811702350413554126823863744341209267781729790694714759018264824761112414556423937732224538665992861551475342773370683344173073150805440138894084087253197595538897613986400165639906934600670780501058567196636796167140097031535132386972899001749862948883362389858632127176571330142071330179992326381982094042993377790345261665892577931395405145369730429462079488033141099249907113241694504241391265397274078984953073730364134893688060340009640631540701820289244667315059736321311926231179142794944897281477264038321021720718017561601025111179022163703476297572233435788863537030535008357679180120653016668316780269873860755423748298548246360981608957670421903145684942967286646362305101773132268579232832164818921732941553151386988781837232271364011755881332524294135348699384658137175857614330952147617551708342432434174779579226338663454959438736807839569911987059388085500837507984051126658973018149321061950769007587519836861526164087252594820126991923916722273718430385263107266000047367872474915828601694439920041571102706081507270147619679971490141639274282889578424398001497985658130305740620028554097382687819891158955487586486645709231721825870342960508203415938806006561845735081804032347750084214100574577342802985404049555529215986404933246481040773076611691605586804857302606467764258503301836174306413323887707999698641372275526317649662882467901094531117120243890323410259937511584651917675138077575448307953064925086002835629697045016137935696266759775923436166369375035368699454550392874449940328328128905560530091416446608691247256021455381248285307613556149618444364923014290938289373215312818797541139219415606631622784836152140668972661027123715779503062132916001988806369127647416567067485490795342762338253943990022498972883660263920518704790601584084302914787302246651371144395418253441269003331181914268070735159284180415100555199146564934872796969351992963117195821262627236458009708099166752820365818699111948365866102758375863322993225541477479210421324166848264953111826527351008031659958888814809945737293785681411438021523876706455063233067233939551964260397443829874822322662036352861302543796600943104500158604854027036789711934695579989189112302233381602302236277726084846296189550730850698061500281436425336666311433321645213882557346329366870956708432252564333895997812402164189946978348320376011613913855499933990786652305860332060641949298931012423081105800169745975038516887112037747631577311831360002742502722451570906304496369230938382329175076469684003556425503797106891999812319602533733677437970687713814747552190142928586781724044248049323750330957002929126630316970587409214456472022710796484778657310660832173093768033821742156446602190335203981531618935787083561603302255162155107179460621892674335641960083663483835896703409115513087820138723494714321400450513941428998350576038799343355677628023346565854351219361896876831439866735726040869511136649881229957801618882834124004126142251475184552502502640896823664946401177803776799157180146386554733265278569418005501363433953502870836220605121839418516239153709790768084909674194289061134979961034672077354959593868862427986411437928435620575955500144308051267664432183688321434583708549082240014585748228606859593502657405750939203135881722442164955416889785558265198046245527898343289578416968890756237467281044803018524217706136533236073856228166664597654076844715963930782091017090763377917711485205493367936868430832404126789220929930411890501756484917499452393770674524578019171841679541825554377930299249277892416277257788147974770446005423669346157135208417428211847353652367573702352791459837645712257646122605628127852169580892808988394594406165340521932514843306105322700231133680378433377389724881307874325614952744243584753011150345103737688223837573804282007358586938044331529253129961025096113761670187568525921208929131354473196308440066835155160913925692912175784379179004808848023029304392630921342768601226558630456913133560978156776098711809238440656353136182676923761613389237802972720736243967239854144480757286813436768000573823963610796223140429490728058551444771338682314499547929338131259971996894072233847404542592316639781608209399269744676323921370773991899853301483814622364299493902073285072098040905300059160091641710175605409814301906444379905831277826625762288108104414704097708248077905168225857235732665234414956169007985520848841886027352780861218049418060017941147110410688703738674378147161236141950474056521041002268987858525470689031657094677131822113205505046579701869337769278257145248837213394613987859786320048011792814546859096532616616068403160077901584946840224344163938313618742275417712170336151163782359059685168880561304838542087505126933144171705880517278127917564053282929427357971823360842784676292324980318169828654166132873909074116734612367109059236155113860447246378721244612580406931724769152219217409096880209008801535633471775664392125733993165330324425899852598966724744126503608416484160724482125980550754851232313331300621490042708542735985913041306918279258584509440150719217604794274047740253314305451367710311947544521321732225875550489799267468541529538871443696399406391099267018219539890685186755868574434469213792094590683677929528246795437302263472495359466300235998990248299853826140395410812427393530207575128774273992824866921285637240069184859771126480352376025469714309316636539718514623865421671429236191647402172547787238964043145364190541101514371773797752463632741619269990461595895793940622986041489302535678633503526382069821487003578061101552210224486633247184367035502326672749787730470216165019711937442505629639916559369593557640005236360445141148916155147776301876302136068825296274460238077523189646894043033182148655637014692476427395401909403584437251915352134557610698046469739424511797999048754951422010043090235713636892619493763602673645872492900162675597083797995647487354531686531900176427222751039446099641439322672532108666047912598938351926694497553568096931962642014042788365702610390456105151611792018698900673027082384103280213487456720062839744828713298223957579105420819286308176631987048287388639069922461848323992902685392499812367091421613488781501234093387999776097433615750910992585468475923085725368613605356762146929424264323906626708602846163376051573599050869800314239735368928435294958099434465414316189806451480849292695749412903363373410480943579407321266012450796613789442208485840536446021616517885568969302685188950832476793300404851688934411125834396590422211152736276278672366665845757559585409486248261694480201791748223085835007862255216359325125768382924978090431102048708975715033330963651576804501966025215527080352103848176167004443740572131294252820989545456276344353575741673638980108310579931697917916718271145837435222026387771805250290791645414791173616253155840768495583288190293564201219633684854080865928095131505012602919562576032932512847250469881908146475324342363863860247943921015193235101390117789997483527186469346024554247028375300033725403910085997650987642832802908445662021678362267272292737780213652404028817217012490974899454430826861772239385250883760749742195942655217301733355851389407457348144161511380845358039740277795072051893487170722955427683655826706766313911972211811528466502223383490906676554168336907959409404576472940901354356409277969379842065738891481990225399022315913388145851487225126560927576795873759207013915029216513720851137197522734365458411622066281660256333632074449918511469174455062297146086578736313585389023662557285424516018080487167823688885575325066254262367702604215835160174851981885460860036597606743233346410471991027562358645341748631726556391320606407754779439671383653877377610828300019937359760370467245737880967939894493795829602910746901609451288456550071458091887879542641820145369659962842686882363495879277007025298960996798975941955735253914237782443302746708282008722602053415292735847582937522487377937899136764642153727843553986244015856488692101644781661602962113570056638347990334049623875941092886778920270077504951511405782565295015024484968204744379710872943108541684540513016310902267112951959140520827546866418137305837933236150599142045255880213558474751516267815309465541240524091663857551298894834797423322854504140527354235070335984964593699534959698554244978249586929179182415068053002553370412778703476446244329205906832901886692400222391918714603175399666877477960121790688623311002908668305431787009355066944389131913333586368037447530664502418437136030852288582121720231274167009740351431532131803978033680228154223490183737494117973254478594157962104378787072154814091725163615415163381388912588517924237727229603497305533840942889918919161186249580560073570527227874940321250645426206304469470804277945973817146810395192821550688079136701210109944220737024613687196031491162370967939354636396448139025711768057799751751298979667073292674886430097398814873780767363792886767781170520534367705731566895899181530825761606591843760505051704242093231358724816618683821026679970982966436224723644898648976857100173643547336955619347638598187756855912376232580849341570570863450733443976604780386678461711520325115528237161469200634713570383377229877321365028868868859434051205798386937002783312365427450532283462669786446920780944052138528653384627970748017872477988461146015077617116261800781557915472305214759943058006652042710117125674185860274188801377931279938153727692612114066810156521441903567333926116697140453812010040811760123270513163743154487571768761575554916236601762880220601068655524141619314312671535587154866747899398685510873576261006923021359580838145290642217792987748784161516349497309700794368305080955621264592795333690631936594413261117944256602433064619312002953123619348034504503004315096798588111896950537335671086336886944665564112662287921812114121425167348136472449021275252555647623248505638391391630760976364990288930588053406631352470996993362568102360392264043588787550723319888417590521211390376609272658409023873553418516426444865247805763826160023858280693148922231457758783791564902227590699346481624734399733206013058796068136378152964615963260698744961105368384203105364183675373594176373955988088591188920114871545460924735613515979992999722298041707112256996310945945097765566409972722824015293663094891067963296735505830412258608050740410916678539569261234499102819759563955711753011823480304181029089719655278245770283085321733741593938595853203645590564229716679900322284081259569032886928291260139267587858284765599075828016611120063145411315144108875767081854894287737618991537664505164279985451077400771946398046265077776614053524831090497899859510873112620613018757108643735744708366215377470972660188656210681516328000908086198554303597948479869789466434027029290899143432223920333487108261968698934611177160561910681226015874410833093070377506876977485840324132474643763087889666151972556180371472590029550718424245405129246729039791532535999005557334600111693557020225722442772950263840538309433999383388018839553821540371447394465152512354603526742382254148328248990134023054550811390236768038649723899924257800315803725555410178461863478690646045865826036072306952576113184134225274786464852363324759102670562466350802553058142201552282050989197818420425028259521880098846231828512448393059455162005455907776121981297954040150653985341579053629101777939776957892084510979265382905626736402636703151957650493344879513766262192237185642999150828898080904189181015450813145034385734032579549707819385285699926238835221520814478940626889936085239827537174490903769904145555260249190126341431327373827075950390882531223536876389814182564965563294518709637484074360669912550026080424160562533591856230955376566866124027875883101021495284600804805028045254063691285010599912421270508133194975917146762267305044225075915290251742774636494555052325186322411388406191257012917881384181566918237215400893603475101448554254698937834239606460813666829750019379115061709452680984785152862123171377897417492087541064556959508967969794980679770961683057941674310519254486327358885118436597143583348756027405400165571178309126113117314169066606067613797690123141099672013123730329707678988740099317309687380126740538923612230370779727025191340850390101739924877352408881040807749924412635346413181858792480760553268122881584307471326768283097203149049868884456187976015468233715478415429742230166504759393312132256510189175368566338139736836336126010908419590215582111816677413843969205870515074254852744810154541079359513596653630049188769523677579147319184225806802539818418929888943038224766186405856591859943091324575886587044653095332668532261321209825839180538360814144791320319699276037194760191286674308615217243049852806380129834255379486287824758850820609389214668693729881191560115633701248675404205911464930888219050248857645752083363921499441937170268576222251074166230901665867067714568862793343153513505688216165112807318529333124070912343832502302341169501745502360505475824093175657701604884577017762183184615567978427541088499501610912720817913532406784267161792013428902861583277304794830971705537485109380418091491750245433432217445924133037928381694330975012918544596923388733288616144238100112755828623259628572648121538348900698511503485369544461542161283241700533583180520082915722904696365553178152398468725451306350506984981006205514844020769539324155096762680887603572463913955278222246439122592651921288446961107463586148252820017348957533954255019475442643148903233373926763409115527189768429887783617346613535388507656327107814312435018965109238453660236940276060642119384227665755210663671879603217527184404651560427289869560206997012906367847161654793068868305846508082886614111979138822898112498261434559408961813509226857611474609406147937240008842153535862052780125014270055274468359151840373309373580494342483940467505708347927948338133276237937844629209323999417593374917899786484958148818865149169302451512835579818112344900827168644548306546633975256079615935830821400021951611342337058359111545217293721664061708131602078213341260356852013161345136871600980378712556766143923146458085652084039744217352744813741215277475202259244561520365608268890193913957991844109971588312780020898275935898106482117936157951837937026741451400902833064466209280549839169261068975151083963132117128513257434964510681479694782619701483204392206140109523453209269311762298139422044308117317394338867965739135764377642819353621467837436136161591167926578700137748127848510041447845416464568496606699139509524527949914769441031612575776863713634644477006787131066832417871556281779122339077841275184193161188155887229676749605752053192594847679397486414128879475647133049543555044790277128690095643357913405127375570391806822344718167939329121448449553897728696601037841520390662890781218240141299368590465146519209198605347788576842696538459445700169758422531241268031418456268722581132040056433413524302102739213788415250475704533878002467378571470021087314693254557923134757243640544448132093266582986850659125571745568328831440322798049274104403921761438405750750288608423536966715191668510428001748971774811216784160854454400190449242294333666338347684438072624307319019363571067447363413698467328522605570126450123348367412135721830146848071241856625742852208909104583727386227300781566668914250733456373259567253354316171586533339843321723688126003809020585719930855573100508771533737446465211874481748868710652311198691114058503492239156755462142467550498676710264926176510110766876596258810039163948397811986615585196216487695936398904500383258041054420595482859955239065758108017936807080830518996468540836412752905182813744878769639548306385089756146421874889271294890398025623046812175145502330254086076115859321603465240763923593699949180470780496764486889980902123735780457040380820770357387588525976042434608851075199334470112741787878845674656640471901619633546770714090590826954225196409446319547658653032104723804625249971910690110456227579220926904132753699634145768795242244563973018311291451151322757841320376225862458224784696669785947914981610522628786944136373683125108310682898766123782697506343047263278453719024447970975017396831214493357290791648779915089163278018852504558488782722376705263811803792477835540018117452957747339714012352011459901984753358434861297092928529424139865507522507808919352104173963493428604871342370429572757862549365917805401652536330410692033704691093097588782938291296447890613200063096560747882082122140978472301680600835812336957051454650181292694364578357815608503303392466039553797630836137289498678842851139853615593352782103740733076818433040893624460576706096188294529171362940967592507631348636606011346115980434147450705511490716640635688739020690279453438236930531133440901381392849163507484449076828386687476663619303412376248380175840467851210698290605196112357188811150723607303158506622574566366740720668999061320627793994112805759798332878792144188725498543014546662945079670707688135022230580562225942983096887732856788971494623888272184647618153045844390967248232348259587963698908456664795754200195991919240707615823002328977439748112690476546256873684352229063217889227643289360535947903046811114130586348244566489159211382258867880972564351646404364328416076247766114349880319792230537889671148058968061594279189647401954989466232962162567264739015818692956765601444248501821713300527995551312539849919933907083138030214072556753022600033565715934283182650908979350869698950542635843046765145668997627989606295925119763672907762567862769469947280606094290314917493590511523235698715397127866718077578671910380368991445381484562682604003456798248689847811138328054940490519768008320299631757043011485087384048591850157264392187414592464617404735275250506783992273121600117160338604710710015235631159734711153198198710616109850375758965576728904060387168114313084172893710817412764581206119054145955378853200366615264923610030157044627231777788649806700723598889528747481372190175074700005571108178930354895017924552067329003818814068686247959272205591627902292600592107710510448103392878991286820705448979977319695574374529708195463942431669050083984398993036790655541596099324867822475424361758944371791403787168166189093900243862038610001362193667280872414291108080291896093127526202667881902085595708111853836166128848729527875143202956393295910508349687029060692838441522579419764824996318479414814660898281725690484184326061946254276693688953540732363428302189694947766126078346328490315128061501009539164530614554234923393806214007779256337619373052025699319099789404390847443596972052065999017828537676265683558625452697455260991024576619614037537859594506363227095122489241931813728141668427013096050734578659047904243852086508154491350136491698639048125666610843702294730266721499164849610746803261583352580352858275799038584091667618877199539888680431991650866887781701439663176815592262016991396613153738021294160006906947533431677802632207226265881842757216055461439677336258462997385077307751473833315101468395296411397329672457933540390136107395245686243008096720460995545708974893048753897955544443791303790422346037768729236001386569593952300768091377768847789746299699489949016141866131552200856673695770822720338936659590666350594330040363762591189195691561626122704788696510356062748423100605472091437069471661080277379848576543481249822444235828329813543645124092220896643987201997945619030397327254617823136363375927622656301565813545578319730419339269008282952718252138855126583037630477490625995514925943105307478901043009876580816508144862607975129633326675259272351611791836777128931053144471668835182920514343609292493191180249366051791485330421043899773019267686085347768149502299280938065840007311767895491286098112311307002535600347898600653805084532572431553654422067661352337408211307834360326940015926958459588297845649462271300855594293344520727007718206398887404742186697709349647758173683580193168322111365547392288184271373843690526638607662451284299368435082612881367358536293873792369928837047900484722240370919885912556341130849457067599032002751632513926694249485692320904596897775676762684224768120033279577059394613185252356456291805905295974791266162882381429824622654141067246487216174351317397697122228010100668178786776119825961537643641828573481088089988571570279722274734750248439022607880448075724807701621064670166965100202654371260046641935546165838945950143502160890185703558173661823437491622669077311800121188299737319891006060966841193266075165452741829459541189277264192546108246351931647783837078295218389645376236304858042774417907169146356546201215125418664885396161542055152375000426794253417764590821513675258479774465114750438460596325820468809667795709044645884673847481638045635188183210386594798204376334738389017759714236223057776395541011294523488098341476645559342209402059733452337956309441446698222457026367119493286653989491344225517746402732596722993581333110831711807234044326813737231209669052411856734897392234152750707954137453460386506786693396236535556479102508529284294227710593056660625152290924148057080971159783458351173168204129645967070633303569271821496292272073250126955216172649821895790908865085382490848904421755530946832055636316431893917626269931034289485184392539670922412565933079102365485294162132200251193795272480340133135247014182195618419055761030190199521647459734401211601239235679307823190770288415814605647291481745105388060109787505925537152356112290181284710137917215124667428500061818271276125025241876177485994084521492727902567005925854431027704636911098800554312457229683836980470864041706010966962231877065395275783874454229129966623016408054769705821417128636329650130416501278156397799631957412627634011130135082721772287129164002237230234809031485343677016544959380750634285293053131127965945266651960426350406454862543383772209428482543536823186182982713182489884498260285705690699045790998144649193654563259496570044689011049923939218088155626191834404362264965506449848521612498442375928443642612004256628602157801140467879662339228190804577624109076487087406157070486658398144845855803277997327929143195789110373530019873110486895656281917362036703039179710646309906285483702836118486672219457621775034511770110458001291255925462680537427727378863726783016568351092332280649908459179620305691566806180826586923920561895421631986004793961133953226395999749526798801074576466538377400437463695133685671362553184054638475191646737948743270916620098057717103475575333102702706317395612448413745782734376330101853438497450236265733191742446567787499665000938706441886733491099877926005340862442833450486907338279348425305698737469497333364267191968992849534561045719338665222471536681145666596959735075972188416698767321649331898967182978657974612216573922404856900225324160367805329990925438960169901664189038843548375648056012628830409421321300206164540821986138099462721214327234457806819925823202851398237118926541234460723597174777907172041523181575194793527456442984630888846385381068621715274531612303165705848974316209831401326306699896632888532682145204083110738032052784669279984003137878996525635126885368435559620598057278951754498694219326972133205286374577983487319388899574634252048213337552584571056619586932031563299451502519194559691231437579991138301656117185508816658756751184338145761060365142858427872190232598107834593970738225147111878311540875777560020664124562293239116606733386480367086953749244898068000217666674827426925968686433731916548717750106343608307376281613984107392410037196754833838054369880310983922140260514297591221159148505938770679068701351029862207502287721123345624421024715163941251258954337788492834236361124473822814504596821452253550035968325337489186278678359443979041598043992124889848660795045011701169092519383155609441705397900600291315024253848282782826223304151370929502192196508374714697845805550615914539506437316401173317807741497557116733034632008408954066541694665746735785483133770133628948904397670025863002540635264006601631712883920305576358989492412827022489373848906764385339931878608019223108328847459816417701264089078551777830131616162049792779670521847212730327970738223860581986744668610994383049960437407323195784473254857416239738852016202384784256163512597161783106850156299135559874758848151014815490937380933394074455700842090155903853444962128368313687375166780513082594599771257467939781491953642874321122421579851584491669362551569370916855252644720786527971466476760328471332985501945689772758983450586004316822658631176606237201721007922216410188299330808409384014213759697185976897042759041500946595252763487628135867117352364964121058854934496645898651826545634382851159137631569519895230262881794959971545221250667461174394884433312659432286710965281109501693028351496524082850120190831078678067061851145740970787563117610746428835593915985421673115153096948758378955979586132649569817205284291038172721213138681565524428109871168862743968021885581515367531218374119972919471325465199144188500672036481975944167950887487934416759598361960010994838744709079104099785974656112459851972157558134628546189728615020774374529539536929655449012953097288963767713353842429715394179547179095580120134210175150931491664699052366350233024087218654727629639065723341455005903913890253699317155917179823065162679744711857951506573868504088229934804445549850597823297898617029498418376255258757455303112991914341109413088238114443068843062655305601658801408561023324210300218460588586954418502977463085858496130037238190325162225570729975710727306066072916922978033647048840958711228045188511908718588299514331534128549297173849768523136276076868494780364948299904475715771141080958058141208956059471668626290036145602625334863284986816039463372436667112964460292915746181117789169695839947080954788863503281129626899231110099889317815313946681882028368363373822281414974006917942192888817139116283910295684918233358930813360131488748366464224381776081007739183393749346933644748150564933649323157235306109385796839902153381449126925350768211098738352197507736653475499431740580563099143218212547336281359488317681489194306530426029773885492974570569448783077945878865062970895499843760181694031056909587141386804846359853684034105948341788438963179956468815791937174656705047441528027712541569401365862097760735632832966564135817028088013546326104892768731829917950379944446328158595181380144716817284996793061814177131912099236282922612543236071226270324572637946863533391758737446552006008819975294017572421299723542069630427857950608911113416534893431149175314953530067419744979017235181671568754163484949491289001739377451431928382431183263265079530371177806185851153508809998200482761808307209649636476943066172549186143700971387567940218696710148540307471561091358933165600167252126542502898612259306484105898847129649230941215144563947889999327145875969555737090855150648002321476443037232466147111552578583071024936898814562568786834745518893385181791667579054210421036349316257870476543126790661216644142285017446278477132740595579600648343288827864837043456066966456899746910373987712891593313271266247505582258634928427718355831641593667712218537642376222104779338956378722902509543014182257180331300148113377736941508488867501893156994849838936052666818012783912005801431596441910546663236810148207799356523056490420711364192200177189107935243234322761787712568251126481332974354926568682748715986654943041648468220593921673359485057849622807932422649812705271398407720995707236227009245067665680069149966555737866411877079767754867028786431817941521796178310655030287157272282250812017060713380339641841211253856248920130010782462165136989511064611133562443838185366273563783436921279354709230119655914915800561707258518503167289370411936374780625824298250726464801821523430268081486978164824349353456855843696378384153838051184406043696871666416514036129729992912630842812149152469877429332305214999981829046119471676727503742221367186614654042534463141660649871499001000660041544868437352208483059495953182872280520828676300361091734508632133033647289584176588755345227938480297724485711815574893561311524926772006362198369980664159549388683836411891430443767715498026544959061738265591178545999378510861446014967645550103653971251138583505085112442517772923814396233043724036032603181442991365750246012787514117944901305803452199992701148071712847770301254994886841867572975189214295652512486943983729047410363121899124217339550688778643130750024823361832738729697376598820053895902935486054979802320400472236873557411858132734337978931582039412878989728973298812553514507641535360519462112217000676321611195841029252568536561813138784086477147099724553013170761712163186600291464501378587854802096244703771373587720086738054108140042311418525803293267396324596914044834665722042880679280616029884043400536534009706581694636096660911110968789751801325224478246957913251892122653056085866541115373584912790254654369020869419871125588453729063224423222287139122012248769976837147645598526739225904997885514250047585260297929306159913444898341973583316070107516452301310796620382579278533125161760789984630103493496981494261055367836366022561213767081421091373531780682420175737470287189310207606953355721704357535177461573524838432101571399813798596607129664438314791296359275429627129436142685922138993054980645399144588692472767598544271527788443836760149912897358259961869729756588978741082189422337344547375227693199222635973520722998387368484349176841191020246627479579564349615012657433845758638834735832242535328142047826934473129971189346354502994681747128179298167439644524956655532311649920677163664580318205849626132234652606175413532444702007661807418914040158148560001030119994109595492321434406067634769713089513389171050503856336503545166431774489640061738861761193622676890576955693918707703942304940038440622614449572516631017080642923345170422426679607075404028551182398361531383751432493056398381877995594942545196756559181968690885283434886050828529642437578712929439366177362830136595872723080969468398938676366226456791132977469812675226595621009318322081754694778878755356188335083870248295346078597023609865656376722755704495258739871812593441903785275571333409842450127258596692434317689018966145404453679047136294238156127656824247864736176671770647002431119711090007474065945650315375044177982192306323700872039212085499569681061379189029961178936752146022386905665481382858280449537530160921422195940638787074787991194920898374091788534417523064715030278397979864517336625329511775105559014160459873338186887977858817291976604516353353556047648420520888811722831990044504284486852338334530105533929637308039738230604714104525470094899407601215247602819963846343554852932377161410869591950786873276075400085220065031871239272857835807010762542769655355964789450166013816295177908531139811092831583216931563867459747449584385282701658246192092219529134323496779345585613140207765996142546463288677356891785576835169608392864188830094883324700447958316931533832382377876344426323456301679513671047510469669001217777128065522453689371871451567394733440447280450959433090683667110655953338602938000999949010642769859623260401863733572846679531229683156358145420890540651226419162015504500430562136991850941034609601030543816694795964585804425194905110733387679946734471718615647723811737035654917628707589456035519195603962301157866323750234725054461073979402475184415558178087962822231972692984516683306919505079993357259165675557294585962182052650473353712351623662770479333289322136141858785972771685682725303734836891911847197133753088446777943274857148827821608844765700041403499921376794209627560883081509438030705666022764678117533361028187800710219794428777313146387857817205661409023041499923248268982477222109852189758140879763486146763606368674611966620347304608917277240045953051376938375381543486981101990651706961774052218247422657652138152740612699012706880875386408669901461740890540981877671880076124151967064152117653084325544261017536348281196837493395825742541244634247233586360777980960199745187758845459645895956779558869098404768259253477849930457883128541747079059795909431627722327844578918694214929451540174214623240300841907975296782445969183509474202123617940309048634960534054931299919496087957952586977170236680033862505764938088740994009589948109397983231108838769236490221499111120870639202892490698435333152727991330986335454324971441378059132240814960156485679843966464780280409057580889190254236606774500413415794312112501275232250148067232979652230488493751166084976116412777395311302041566848265531411348993243747890268935173904043294851610659785832253168204202834993641595980197343889883020994152152288611175126686173051956249367180053845637855129171848417841594797435580617856680758491080185805695567990185198397660693358224779136504562705766735170961550493338390452612404395517449136885115987454340932040102218982707539212403241042424451570052968378815749468441508011138612561164102477190903050040240662278945607061512108266146098662040425010583978098192019726759010749924884966139441184159734610382401178556739080566483321039073867083298691078093495828888707110651559651222542929154212923108071159723275797510859911398076844732639426419452063138217862260999160086752446265457028969067192282283045169111363652774517975842147102219099906257373383472726498678244401048998507631630668050267115944636293525120269424810854530602810627264236538250773340575475701704367039596467715959261029438313074897245505729085688496091346323165819468660587092144653716755655531962091865952628448253731353698162517351930115341581171353292035873164168839107994000677266031617527582917398395852606454113318985505747847121053505795649095931672167565624818782002769963734155880000867852567422461511406015760115910256449002264980039498403358091309140197877843650167960167465370287466062584346329708303725980494653589318912163976013193079476972058034710553111117215859219066231028099212084069283091906017370764654655683413207556315315006453462321007133584907633048328153458698497332599801187479664273140279381289961720524540674695271948079930396730194274036466594154400092799908634806622334906695224044652158992864203435098858422692019340575496840904812955522654754650713532842543496616084954788090727649930252702815067862810825243222979985391759845188868387004477101866772159439708514664612871148749531862180941719676843144666435175837688436786081446319641912566574047718699160915550910878919431253671945651261878486910876729910565595155159739659034383628124629118117760949411880105946336671039049777312004243578115790429823045072038322781246413671297959415082918378213212876890545963586369344879749784841123274921331663162812456388238288715648447883142417650147980187858215768793063001153788998014623690135803753306246148576074932567807682651045738059018831237617271889933790487113395588485234240255002352200613574914318259142479829367775490496399350755839668967578364316618369307625603528602940662803255416535431518013714821941772672244005268401996533334184004345525296592918502940131600651124395297874364222806977720437363717873457948420238745151249157913139411148608416429347958793681868609689684640858334131017858142710955416293375915178392341303110543328703526599993904966822112768158316511246866451167351378214345336650598328347443536290312393672084593164394941881138607974670134709640378534907149089842317891739783650654751982883367395714360000003439863363212091718954899055748693397700245632475954504411422582410783866837655467400137324322809113692670682805397549111166171102397437749479335174036135005397581475520834285772800986189401984375446435081498218360112577632447389452051636938585136484259964518361856989088721789764694721246807900330925083496645841656554261294195108847197209106605105540933731954888406444080280579549008076040034154662137669606444293774985897353625591959618552448187940317374508256072895120945456562159540405425814886929842786582357673195799285293120866275922366115137445767916063621675267440451221051052090834707443986137829082352772895849625656881972792768694795806100573787084121444815034797422312103295359297822377134077549545477791813823542607184617108389097825964406170543546968567030745411634244134486308676327949177682923093183221341455482591367202823284396549001805653203960795517074496039006696990334199278212696767771835209083959545341866777944872740383733381985235884202840150981579594685874537989503257362809837592216229258598599123843993575573285028613155970362934249814178056461615863415338635077223269996508860870999964899373049307170967888740149746147542880387421250689212155876692242387434701120990859082164073576380817386959755176083877600277517253037133445654852635661720197563001580049790223419586738061442401502436288957503206533690825756785507020555105572381878574650371086308158185862815883054564662297694803970618265491385181326737485227188267917919091354407852685476254126683398240534022469989966652573155637645862251862823092085424412805997628505488913098331761884983352975136073772030571342739638126588567405013841074788943393996603591853934198416322617654857376671943132840050626295140357877264680649549355746326408186979718630218760025813995719923601345374229758918285167511358171472625828596940798518571870075823122317068134867930884899275181661399609753105295773584618525865211893339375771859916335112163441037910451845019023066893064178977808158101360449495409665363660370075881004450265734935127707426742578608784898185628869980851665713320835842613381142623855420315774246613108873106318111989880289722849790551075148403702290580483052731884959994156606537314021296702220821915862905952604040620011815269664910068587592655660567562963361434230232810747488395040380984981860056164646099819257616235478710913832967563761506732550860683433720438748186791668975746563456020002562889601191100980453350423842063824039434163502977688802779835087481178298349417211674919425601608685332435385951152061809031241698182079314615062073826097180458265687043623935757495737332781578904386011378078508110273049446611821957450170106059384336519458628360682108585130499820420578458577175933849015564447305834515291412561679970569657426139901681932056241927977282026714297258700193234337873153939403115411184101414292741703537542003698760608765500109345299007034032401334806388514095769557147190364152027721127070187421548123931953220997506553022646844227700020589045922742423904937051507367764629844971682121994198274794049092601715727439368569721862936007387077810797440975556627807371228030350048829843919546433753355787895064018998685060281902452191177018634505171087023903398550540704454189088472042376499749035038518949505897971286631644699407490959473411581934618336692169573605081585080837952036335619947691937965065016808710250735070825260046821242820434367245824478859256555487861614478717581068572356895150707602217433511627331709472765932413249132702425519391509083601346239612335001086614623850633127072987745618984384288764099836164964775714638573247333226653894523588365972955159905187411779288608760239306160016168434070611663449248395156319152882728822831375458678269830696691220130954815935450754923554167766876455212545681242936427474153815692219503331560151614492247512488957534835926226263545406704767033866410025277276800886383266629488582740369655329362236090572479794734434077704284318507901973469071141230364111729224929307731939309795452877412451183953480382210373644697046967493042810911797232448615413264031578430955396671061468083815548947146733652483679138566431084747848676243012018489329109615281108087617422779131629345494425395422727309645057976122885347393189600810965202090151104579377602529543130188938184010247010134929317443562883578609861545691161669857388024973756940558138630581099823372565164920155443216861690537054630176154809626620800633059320775897175589925862195462096455464624399535391743228225433267174308492508396461328929584567927365409119947616225155964704061297047759818551878441419948614013153859322060745185909608884280218943358691959604936409651570327527570641500776261323783648149005245481413195989296398441371781402764122087644989688629798910870164270169014007825748311598976330612951195680427485317886333041169767175063822135213839779138443325644288490872919067009802496281560626258636942322658490628628035057282983101266919109637258378149363774960594515216932644945188292639525772348420077356021656909077097264985642831778694777804964343991762549216500608626285329471055602670413384500507827390640287529864161287496473708235188892189612641279553536442286955430551308700009878557534223100547153412810957024870812654319123261956462149376527526356402127388765103883255007364899937167183280028398832319373301564123277185395654932422977953016534830128490677845037490891749347389015649588574802194996722621185874361039774946338633057887487405540005440439344888192044102134790034598411927024921557026873700970995205391930979319495883265922171508324621942300185974396706491149559411733728199869021311629886680267446443489233020607003821262841723679627307191405008084085703978151998148822390059948911946474438682533745889962375133378280532928272016815977970066488394482446332210928320504045983008943565954267256879714918703447338237767914829203283196838105907715727191903042365315650957464549643425328069510396558733549803850995143463506175361480050195045201350200180281506933241918267855737764414097080945745624854867704904368368717590918057269794010465019484853146726642978667687697789291431128505043098192949736165944259471754765135205245072597538577958372797702972231435199958499522344049394502115428867244188717409524554771867484911475031801773304689909317974472957035192387686405544278134169807249382219749124257510162187439772902147704638010731470653154201300583810458905006764557332998149945854655105526374914354195867992595981412218735238407957416123372264063860431988936249867649693592569592128495906254446474331759999685163660305216426770428154681777589339252115538590526823311608302751194384823861552852465010329467297198112105314125898165100120742688143577590825227466863206188376830450921784582526239594189673003640808624233657620979111641766331328852352062487922978959456450333733139422384778582717195412347860434376165241568717943562570215636666680088531006728947033079540804583324192188488870712275670333173939262509073556164513677064199539111948881240659821685787131385056850623094155206877987539740658484250135205615103489821873770245063583314243624807432542464195984647411575625441010389671576677263196442524931941806472423789334668561083789808830313571333157729435664956078125304917594015895146954965223118559669048559467607968190167266634650186182955669893965019614544401768162810604465068448139561667220729261210164692339016793399632833013163850830967942792934551268435760356901970523138364640961311774904600772840862214747547653221505518116489887879087780918009050706040061220010051271575991225725282523378026809030528461581739558198122397010092017202251606352922464781615533532275453264543087093320924631855976580561717446840450048285353396546862678852330044967795580761661801833668792312510460809773895565488962815089519622093675058841609752282328250433712970186608193748968699961301486924694482420723632912367052542145464162968910442981633373266871675946715392611950649224725627254543274193495995569590243279097174392258098103601486364409101491734183079646345064833303404765711827040276868271418084574998493392039317445402616663674646668754385093967129918067471909885312710726724428584870694307099756567949198418996425748884764622030325637751112534060087936904565779272035205921345924272965206683338510673615276261016026647772485083344719891986802656197236420847504962661607797092906844757798251795569758235084371746103310387911789239441630112634077535773520558040066982523191225570519133631407211349723226549151062961739050617857127509403623146700931176133132018631158730886798239298009805089491510788371194099750375473674305745187265414016446924576792185753680363289139664155342066705623272936001177781498886100830877849571709880858667023104043242526785955562077310543072298032125941107957349146684680220501816192150766649106862033378713826058987655210423668198670177861672671972374156917880001690656659046965316154923604061891820982414006103779407166342002735828911994182647812782659666207030384795881442790246669264032799404016800137293477301530941805070587421153284642203006550763966756168318897005152026656649929417382840327305940740147117478464839241225676523593418554066440983706083636457657081801664285044258224551650808864421212113914352453935225522162483791737330329812349528984098613273709957407786789349311975204237925022851375880436791854547836416773151821457226504640800104202100410766027807729152555503218182387221708112766208665317651926458452495269685376314437998340336947124447247796973890514941120010934140073794061859447165516612674930799374705772930521750426383798367668159183589049652163726492960837147204067428996276720315410211504333742057182854090136325721437592054640471894328548696883599785122262130812989581571391597464534806099601555877223193450760315411663112963843719400333736013305526352571490454327925190794007111504785378036370897340146753465517470747096935814912797188187854376797751675927822300312945518595042883902735494672667647506072643698761394806879080593531793001711000214417701504495496412454361656210150919997862972495905809191825255486358703529320142005857057855419217730505342687533799076038746689684283402648733290888881745453047194740939258407362058242849349024756883352446212456101562729065130618520732925434179252299417447855189995098959999877410951464170076989305620163502192692653166599093238118295411937545448509428621839424186218067457128099385258842631930670182098008050900019819621758458932516877698594110522845465835679362969619219080897536813210484518784516230623911878024604050824909336069998094776253792973597037759066145994638578378211017122446355845171941670344732162722443265914858595797823752976323442911242311368603724514438765801271594060878788638511089680883165505046309006148832545452819908256238805872042843941834687865142541377686054291079721004271658 diff --git a/lib/std/compress/deflate/testdata/compress-gettysburg.txt b/lib/std/compress/deflate/testdata/compress-gettysburg.txt deleted file mode 100644 index 2c9bcde36057..000000000000 --- a/lib/std/compress/deflate/testdata/compress-gettysburg.txt +++ /dev/null @@ -1,29 +0,0 @@ - Four score and seven years ago our fathers brought forth on -this continent, a new nation, conceived in Liberty, and dedicated -to the proposition that all men are created equal. - Now we are engaged in a great Civil War, testing whether that -nation, or any nation so conceived and so dedicated, can long -endure. - We are met on a great battle-field of that war. - We have come to dedicate a portion of that field, as a final -resting place for those who here gave their lives that that -nation might live. It is altogether fitting and proper that -we should do this. - But, in a larger sense, we can not dedicate - we can not -consecrate - we can not hallow - this ground. - The brave men, living and dead, who struggled here, have -consecrated it, far above our poor power to add or detract. -The world will little note, nor long remember what we say here, -but it can never forget what they did here. - It is for us the living, rather, to be dedicated here to the -unfinished work which they who fought here have thus far so -nobly advanced. It is rather for us to be here dedicated to -the great task remaining before us - that from these honored -dead we take increased devotion to that cause for which they -gave the last full measure of devotion - - that we here highly resolve that these dead shall not have -died in vain - that this nation, under God, shall have a new -birth of freedom - and that government of the people, by the -people, for the people, shall not perish from this earth. - -Abraham Lincoln, November 19, 1863, Gettysburg, Pennsylvania diff --git a/lib/std/compress/deflate/testdata/compress-pi.txt b/lib/std/compress/deflate/testdata/compress-pi.txt deleted file mode 100644 index ca99bbc2a255..000000000000 --- a/lib/std/compress/deflate/testdata/compress-pi.txt +++ /dev/null @@ -1 +0,0 @@ -3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679821480865132823066470938446095505822317253594081284811174502841027019385211055596446229489549303819644288109756659334461284756482337867831652712019091456485669234603486104543266482133936072602491412737245870066063155881748815209209628292540917153643678925903600113305305488204665213841469519415116094330572703657595919530921861173819326117931051185480744623799627495673518857527248912279381830119491298336733624406566430860213949463952247371907021798609437027705392171762931767523846748184676694051320005681271452635608277857713427577896091736371787214684409012249534301465495853710507922796892589235420199561121290219608640344181598136297747713099605187072113499999983729780499510597317328160963185950244594553469083026425223082533446850352619311881710100031378387528865875332083814206171776691473035982534904287554687311595628638823537875937519577818577805321712268066130019278766111959092164201989380952572010654858632788659361533818279682303019520353018529689957736225994138912497217752834791315155748572424541506959508295331168617278558890750983817546374649393192550604009277016711390098488240128583616035637076601047101819429555961989467678374494482553797747268471040475346462080466842590694912933136770289891521047521620569660240580381501935112533824300355876402474964732639141992726042699227967823547816360093417216412199245863150302861829745557067498385054945885869269956909272107975093029553211653449872027559602364806654991198818347977535663698074265425278625518184175746728909777727938000816470600161452491921732172147723501414419735685481613611573525521334757418494684385233239073941433345477624168625189835694855620992192221842725502542568876717904946016534668049886272327917860857843838279679766814541009538837863609506800642251252051173929848960841284886269456042419652850222106611863067442786220391949450471237137869609563643719172874677646575739624138908658326459958133904780275900994657640789512694683983525957098258226205224894077267194782684826014769909026401363944374553050682034962524517493996514314298091906592509372216964615157098583874105978859597729754989301617539284681382686838689427741559918559252459539594310499725246808459872736446958486538367362226260991246080512438843904512441365497627807977156914359977001296160894416948685558484063534220722258284886481584560285060168427394522674676788952521385225499546667278239864565961163548862305774564980355936345681743241125150760694794510965960940252288797108931456691368672287489405601015033086179286809208747609178249385890097149096759852613655497818931297848216829989487226588048575640142704775551323796414515237462343645428584447952658678210511413547357395231134271661021359695362314429524849371871101457654035902799344037420073105785390621983874478084784896833214457138687519435064302184531910484810053706146806749192781911979399520614196634287544406437451237181921799983910159195618146751426912397489409071864942319615679452080951465502252316038819301420937621378559566389377870830390697920773467221825625996615014215030680384477345492026054146659252014974428507325186660021324340881907104863317346496514539057962685610055081066587969981635747363840525714591028970641401109712062804390397595156771577004203378699360072305587631763594218731251471205329281918261861258673215791984148488291644706095752706957220917567116722910981690915280173506712748583222871835209353965725121083579151369882091444210067510334671103141267111369908658516398315019701651511685171437657618351556508849099898599823873455283316355076479185358932261854896321329330898570642046752590709154814165498594616371802709819943099244889575712828905923233260972997120844335732654893823911932597463667305836041428138830320382490375898524374417029132765618093773444030707469211201913020330380197621101100449293215160842444859637669838952286847831235526582131449576857262433441893039686426243410773226978028073189154411010446823252716201052652272111660396665573092547110557853763466820653109896526918620564769312570586356620185581007293606598764861179104533488503461136576867532494416680396265797877185560845529654126654085306143444318586769751456614068007002378776591344017127494704205622305389945613140711270004078547332699390814546646458807972708266830634328587856983052358089330657574067954571637752542021149557615814002501262285941302164715509792592309907965473761255176567513575178296664547791745011299614890304639947132962107340437518957359614589019389713111790429782856475032031986915140287080859904801094121472213179476477726224142548545403321571853061422881375850430633217518297986622371721591607716692547487389866549494501146540628433663937900397692656721463853067360965712091807638327166416274888800786925602902284721040317211860820419000422966171196377921337575114959501566049631862947265473642523081770367515906735023507283540567040386743513622224771589150495309844489333096340878076932599397805419341447377441842631298608099888687413260472156951623965864573021631598193195167353812974167729478672422924654366800980676928238280689964004824354037014163149658979409243237896907069779422362508221688957383798623001593776471651228935786015881617557829735233446042815126272037343146531977774160319906655418763979293344195215413418994854447345673831624993419131814809277771038638773431772075456545322077709212019051660962804909263601975988281613323166636528619326686336062735676303544776280350450777235547105859548702790814356240145171806246436267945612753181340783303362542327839449753824372058353114771199260638133467768796959703098339130771098704085913374641442822772634659470474587847787201927715280731767907707157213444730605700733492436931138350493163128404251219256517980694113528013147013047816437885185290928545201165839341965621349143415956258658655705526904965209858033850722426482939728584783163057777560688876446248246857926039535277348030480290058760758251047470916439613626760449256274204208320856611906254543372131535958450687724602901618766795240616342522577195429162991930645537799140373404328752628889639958794757291746426357455254079091451357111369410911939325191076020825202618798531887705842972591677813149699009019211697173727847684726860849003377024242916513005005168323364350389517029893922334517220138128069650117844087451960121228599371623130171144484640903890644954440061986907548516026327505298349187407866808818338510228334508504860825039302133219715518430635455007668282949304137765527939751754613953984683393638304746119966538581538420568533862186725233402830871123282789212507712629463229563989898935821167456270102183564622013496715188190973038119800497340723961036854066431939509790190699639552453005450580685501956730229219139339185680344903982059551002263535361920419947455385938102343955449597783779023742161727111723643435439478221818528624085140066604433258885698670543154706965747458550332323342107301545940516553790686627333799585115625784322988273723198987571415957811196358330059408730681216028764962867446047746491599505497374256269010490377819868359381465741268049256487985561453723478673303904688383436346553794986419270563872931748723320837601123029911367938627089438799362016295154133714248928307220126901475466847653576164773794675200490757155527819653621323926406160136358155907422020203187277605277219005561484255518792530343513984425322341576233610642506390497500865627109535919465897514131034822769306247435363256916078154781811528436679570611086153315044521274739245449454236828860613408414863776700961207151249140430272538607648236341433462351897576645216413767969031495019108575984423919862916421939949072362346468441173940326591840443780513338945257423995082965912285085558215725031071257012668302402929525220118726767562204154205161841634847565169998116141010029960783869092916030288400269104140792886215078424516709087000699282120660418371806535567252532567532861291042487761825829765157959847035622262934860034158722980534989650226291748788202734209222245339856264766914905562842503912757710284027998066365825488926488025456610172967026640765590429099456815065265305371829412703369313785178609040708667114965583434347693385781711386455873678123014587687126603489139095620099393610310291616152881384379099042317473363948045759314931405297634757481193567091101377517210080315590248530906692037671922033229094334676851422144773793937517034436619910403375111735471918550464490263655128162288244625759163330391072253837421821408835086573917715096828874782656995995744906617583441375223970968340800535598491754173818839994469748676265516582765848358845314277568790029095170283529716344562129640435231176006651012412006597558512761785838292041974844236080071930457618932349229279650198751872127267507981255470958904556357921221033346697499235630254947802490114195212382815309114079073860251522742995818072471625916685451333123948049470791191532673430282441860414263639548000448002670496248201792896476697583183271314251702969234889627668440323260927524960357996469256504936818360900323809293459588970695365349406034021665443755890045632882250545255640564482465151875471196218443965825337543885690941130315095261793780029741207665147939425902989695946995565761218656196733786236256125216320862869222103274889218654364802296780705765615144632046927906821207388377814233562823608963208068222468012248261177185896381409183903673672220888321513755600372798394004152970028783076670944474560134556417254370906979396122571429894671543578468788614445812314593571984922528471605049221242470141214780573455105008019086996033027634787081081754501193071412233908663938339529425786905076431006383519834389341596131854347546495569781038293097164651438407007073604112373599843452251610507027056235266012764848308407611830130527932054274628654036036745328651057065874882256981579367897669742205750596834408697350201410206723585020072452256326513410559240190274216248439140359989535394590944070469120914093870012645600162374288021092764579310657922955249887275846101264836999892256959688159205600101655256375678566722796619885782794848855834397518744545512965634434803966420557982936804352202770984294232533022576341807039476994159791594530069752148293366555661567873640053666564165473217043903521329543529169414599041608753201868379370234888689479151071637852902345292440773659495630510074210871426134974595615138498713757047101787957310422969066670214498637464595280824369445789772330048764765241339075920434019634039114732023380715095222010682563427471646024335440051521266932493419673977041595683753555166730273900749729736354964533288869844061196496162773449518273695588220757355176651589855190986665393549481068873206859907540792342402300925900701731960362254756478940647548346647760411463233905651343306844953979070903023460461470961696886885014083470405460742958699138296682468185710318879065287036650832431974404771855678934823089431068287027228097362480939962706074726455399253994428081137369433887294063079261595995462624629707062594845569034711972996409089418059534393251236235508134949004364278527138315912568989295196427287573946914272534366941532361004537304881985517065941217352462589548730167600298865925786628561249665523533829428785425340483083307016537228563559152534784459818313411290019992059813522051173365856407826484942764411376393866924803118364453698589175442647399882284621844900877769776312795722672655562596282542765318300134070922334365779160128093179401718598599933849235495640057099558561134980252499066984233017350358044081168552653117099570899427328709258487894436460050410892266917835258707859512983441729535195378855345737426085902908176515578039059464087350612322611200937310804854852635722825768203416050484662775045003126200800799804925485346941469775164932709504934639382432227188515974054702148289711177792376122578873477188196825462981268685817050740272550263329044976277894423621674119186269439650671515779586756482399391760426017633870454990176143641204692182370764887834196896861181558158736062938603810171215855272668300823834046564758804051380801633638874216371406435495561868964112282140753302655100424104896783528588290243670904887118190909494533144218287661810310073547705498159680772009474696134360928614849417850171807793068108546900094458995279424398139213505586422196483491512639012803832001097738680662877923971801461343244572640097374257007359210031541508936793008169980536520276007277496745840028362405346037263416554259027601834840306811381855105979705664007509426087885735796037324514146786703688098806097164258497595138069309449401515422221943291302173912538355915031003330325111749156969174502714943315155885403922164097229101129035521815762823283182342548326111912800928252561902052630163911477247331485739107775874425387611746578671169414776421441111263583553871361011023267987756410246824032264834641766369806637857681349204530224081972785647198396308781543221166912246415911776732253264335686146186545222681268872684459684424161078540167681420808850280054143613146230821025941737562389942075713627516745731891894562835257044133543758575342698699472547031656613991999682628247270641336222178923903176085428943733935618891651250424404008952719837873864805847268954624388234375178852014395600571048119498842390606136957342315590796703461491434478863604103182350736502778590897578272731305048893989009923913503373250855982655867089242612429473670193907727130706869170926462548423240748550366080136046689511840093668609546325002145852930950000907151058236267293264537382104938724996699339424685516483261134146110680267446637334375340764294026682973865220935701626384648528514903629320199199688285171839536691345222444708045923966028171565515656661113598231122506289058549145097157553900243931535190902107119457300243880176615035270862602537881797519478061013715004489917210022201335013106016391541589578037117792775225978742891917915522417189585361680594741234193398420218745649256443462392531953135103311476394911995072858430658361935369329699289837914941939406085724863968836903265564364216644257607914710869984315733749648835292769328220762947282381537409961545598798259891093717126218283025848112389011968221429457667580718653806506487026133892822994972574530332838963818439447707794022843598834100358385423897354243956475556840952248445541392394100016207693636846776413017819659379971557468541946334893748439129742391433659360410035234377706588867781139498616478747140793263858738624732889645643598774667638479466504074111825658378878454858148962961273998413442726086061872455452360643153710112746809778704464094758280348769758948328241239292960582948619196670918958089833201210318430340128495116203534280144127617285830243559830032042024512072872535581195840149180969253395075778400067465526031446167050827682772223534191102634163157147406123850425845988419907611287258059113935689601431668283176323567325417073420817332230462987992804908514094790368878687894930546955703072619009502076433493359106024545086453628935456862958531315337183868265617862273637169757741830239860065914816164049449650117321313895747062088474802365371031150898427992754426853277974311395143574172219759799359685252285745263796289612691572357986620573408375766873884266405990993505000813375432454635967504844235284874701443545419576258473564216198134073468541117668831186544893776979566517279662326714810338643913751865946730024434500544995399742372328712494834706044063471606325830649829795510109541836235030309453097335834462839476304775645015008507578949548931393944899216125525597701436858943585877526379625597081677643800125436502371412783467926101995585224717220177723700417808419423948725406801556035998390548985723546745642390585850216719031395262944554391316631345308939062046784387785054239390524731362012947691874975191011472315289326772533918146607300089027768963114810902209724520759167297007850580717186381054967973100167870850694207092232908070383263453452038027860990556900134137182368370991949516489600755049341267876436746384902063964019766685592335654639138363185745698147196210841080961884605456039038455343729141446513474940784884423772175154334260306698831768331001133108690421939031080143784334151370924353013677631084913516156422698475074303297167469640666531527035325467112667522460551199581831963763707617991919203579582007595605302346267757943936307463056901080114942714100939136913810725813781357894005599500183542511841721360557275221035268037357265279224173736057511278872181908449006178013889710770822931002797665935838758909395688148560263224393726562472776037890814458837855019702843779362407825052704875816470324581290878395232453237896029841669225489649715606981192186584926770403956481278102179913217416305810554598801300484562997651121241536374515005635070127815926714241342103301566165356024733807843028655257222753049998837015348793008062601809623815161366903341111386538510919367393835229345888322550887064507539473952043968079067086806445096986548801682874343786126453815834280753061845485903798217994599681154419742536344399602902510015888272164745006820704193761584547123183460072629339550548239557137256840232268213012476794522644820910235647752723082081063518899152692889108455571126603965034397896278250016110153235160519655904211844949907789992007329476905868577878720982901352956613978884860509786085957017731298155314951681467176959760994210036183559138777817698458758104466283998806006162298486169353373865787735983361613384133853684211978938900185295691967804554482858483701170967212535338758621582310133103877668272115726949518179589754693992642197915523385766231676275475703546994148929041301863861194391962838870543677743224276809132365449485366768000001065262485473055861598999140170769838548318875014293890899506854530765116803337322265175662207526951791442252808165171667766727930354851542040238174608923283917032754257508676551178593950027933895920576682789677644531840404185540104351348389531201326378369283580827193783126549617459970567450718332065034556644034490453627560011250184335607361222765949278393706478426456763388188075656121689605041611390390639601620221536849410926053876887148379895599991120991646464411918568277004574243434021672276445589330127781586869525069499364610175685060167145354315814801054588605645501332037586454858403240298717093480910556211671546848477803944756979804263180991756422809873998766973237695737015808068229045992123661689025962730430679316531149401764737693873514093361833216142802149763399189835484875625298752423873077559555955465196394401821840998412489826236737714672260616336432964063357281070788758164043814850188411431885988276944901193212968271588841338694346828590066640806314077757725705630729400492940302420498416565479736705485580445865720227637840466823379852827105784319753541795011347273625774080213476826045022851579795797647467022840999561601569108903845824502679265942055503958792298185264800706837650418365620945554346135134152570065974881916341359556719649654032187271602648593049039787489589066127250794828276938953521753621850796297785146188432719223223810158744450528665238022532843891375273845892384422535472653098171578447834215822327020690287232330053862163479885094695472004795231120150432932266282727632177908840087861480221475376578105819702226309717495072127248479478169572961423658595782090830733233560348465318730293026659645013718375428897557971449924654038681799213893469244741985097334626793321072686870768062639919361965044099542167627840914669856925715074315740793805323925239477557441591845821562518192155233709607483329234921034514626437449805596103307994145347784574699992128599999399612281615219314888769388022281083001986016549416542616968586788372609587745676182507275992950893180521872924610867639958916145855058397274209809097817293239301067663868240401113040247007350857828724627134946368531815469690466968693925472519413992914652423857762550047485295476814795467007050347999588867695016124972282040303995463278830695976249361510102436555352230690612949388599015734661023712235478911292547696176005047974928060721268039226911027772261025441492215765045081206771735712027180242968106203776578837166909109418074487814049075517820385653909910477594141321543284406250301802757169650820964273484146957263978842560084531214065935809041271135920041975985136254796160632288736181367373244506079244117639975974619383584574915988097667447093006546342423460634237474666080431701260052055928493695941434081468529815053947178900451835755154125223590590687264878635752541911288877371766374860276606349603536794702692322971868327717393236192007774522126247518698334951510198642698878471719396649769070825217423365662725928440620430214113719922785269984698847702323823840055655517889087661360130477098438611687052310553149162517283732728676007248172987637569816335415074608838663640693470437206688651275688266149730788657015685016918647488541679154596507234287730699853713904300266530783987763850323818215535597323530686043010675760838908627049841888595138091030423595782495143988590113185835840667472370297149785084145853085781339156270760356390763947311455495832266945702494139831634332378975955680856836297253867913275055542524491943589128405045226953812179131914513500993846311774017971512283785460116035955402864405902496466930707769055481028850208085800878115773817191741776017330738554758006056014337743299012728677253043182519757916792969965041460706645712588834697979642931622965520168797300035646304579308840327480771811555330909887025505207680463034608658165394876951960044084820659673794731680864156456505300498816164905788311543454850526600698230931577765003780704661264706021457505793270962047825615247145918965223608396645624105195510522357239739512881816405978591427914816542632892004281609136937773722299983327082082969955737727375667615527113922588055201898876201141680054687365580633471603734291703907986396522961312801782679717289822936070288069087768660593252746378405397691848082041021944719713869256084162451123980620113184541244782050110798760717155683154078865439041210873032402010685341947230476666721749869868547076781205124736792479193150856444775379853799732234456122785843296846647513336573692387201464723679427870042503255589926884349592876124007558756946413705625140011797133166207153715436006876477318675587148783989081074295309410605969443158477539700943988394914432353668539209946879645066533985738887866147629443414010498889931600512076781035886116602029611936396821349607501116498327856353161451684576956871090029997698412632665023477167286573785790857466460772283415403114415294188047825438761770790430001566986776795760909966936075594965152736349811896413043311662774712338817406037317439705406703109676765748695358789670031925866259410510533584384656023391796749267844763708474978333655579007384191473198862713525954625181604342253729962863267496824058060296421146386436864224724887283434170441573482481833301640566959668866769563491416328426414974533349999480002669987588815935073578151958899005395120853510357261373640343675347141048360175464883004078464167452167371904831096767113443494819262681110739948250607394950735031690197318521195526356325843390998224986240670310768318446607291248747540316179699411397387765899868554170318847788675929026070043212666179192235209382278788809886335991160819235355570464634911320859189796132791319756490976000139962344455350143464268604644958624769094347048293294140411146540923988344435159133201077394411184074107684981066347241048239358274019449356651610884631256785297769734684303061462418035852933159734583038455410337010916767763742762102137013548544509263071901147318485749233181672072137279355679528443925481560913728128406333039373562420016045664557414588166052166608738748047243391212955877763906969037078828527753894052460758496231574369171131761347838827194168606625721036851321566478001476752310393578606896111259960281839309548709059073861351914591819510297327875571049729011487171897180046961697770017913919613791417162707018958469214343696762927459109940060084983568425201915593703701011049747339493877885989417433031785348707603221982970579751191440510994235883034546353492349826883624043327267415540301619505680654180939409982020609994140216890900708213307230896621197755306659188141191577836272927461561857103721724710095214236964830864102592887457999322374955191221951903424452307535133806856807354464995127203174487195403976107308060269906258076020292731455252078079914184290638844373499681458273372072663917670201183004648190002413083508846584152148991276106513741539435657211390328574918769094413702090517031487773461652879848235338297260136110984514841823808120540996125274580881099486972216128524897425555516076371675054896173016809613803811914361143992106380050832140987604599309324851025168294467260666138151745712559754953580239983146982203613380828499356705575524712902745397762140493182014658008021566536067765508783804304134310591804606800834591136640834887408005741272586704792258319127415739080914383138456424150940849133918096840251163991936853225557338966953749026620923261318855891580832455571948453875628786128859004106006073746501402627824027346962528217174941582331749239683530136178653673760642166778137739951006589528877427662636841830680190804609849809469763667335662282915132352788806157768278159588669180238940333076441912403412022316368577860357276941541778826435238131905028087018575047046312933353757285386605888904583111450773942935201994321971171642235005644042979892081594307167019857469273848653833436145794634175922573898588001698014757420542995801242958105456510831046297282937584161162532562516572498078492099897990620035936509934721582965174135798491047111660791587436986541222348341887722929446335178653856731962559852026072947674072616767145573649812105677716893484917660771705277187601199908144113058645577910525684304811440261938402322470939249802933550731845890355397133088446174107959162511714864874468611247605428673436709046678468670274091881014249711149657817724279347070216688295610877794405048437528443375108828264771978540006509704033021862556147332117771174413350281608840351781452541964320309576018694649088681545285621346988355444560249556668436602922195124830910605377201980218310103270417838665447181260397190688462370857518080035327047185659499476124248110999288679158969049563947624608424065930948621507690314987020673533848349550836366017848771060809804269247132410009464014373603265645184566792456669551001502298330798496079949882497061723674493612262229617908143114146609412341593593095854079139087208322733549572080757165171876599449856937956238755516175754380917805280294642004472153962807463602113294255916002570735628126387331060058910652457080244749375431841494014821199962764531068006631183823761639663180931444671298615527598201451410275600689297502463040173514891945763607893528555053173314164570504996443890936308438744847839616840518452732884032345202470568516465716477139323775517294795126132398229602394548579754586517458787713318138752959809412174227300352296508089177705068259248822322154938048371454781647213976820963320508305647920482085920475499857320388876391601995240918938945576768749730856955958010659526503036266159750662225084067428898265907510637563569968211510949669744580547288693631020367823250182323708459790111548472087618212477813266330412076216587312970811230758159821248639807212407868878114501655825136178903070860870198975889807456643955157415363193191981070575336633738038272152798849350397480015890519420879711308051233933221903466249917169150948541401871060354603794643379005890957721180804465743962806186717861017156740967662080295766577051291209907944304632892947306159510430902221439371849560634056189342513057268291465783293340524635028929175470872564842600349629611654138230077313327298305001602567240141851520418907011542885799208121984493156999059182011819733500126187728036812481995877070207532406361259313438595542547781961142935163561223496661522614735399674051584998603552953329245752388810136202347624669055816438967863097627365504724348643071218494373485300606387644566272186661701238127715621379746149861328744117714552444708997144522885662942440230184791205478498574521634696448973892062401943518310088283480249249085403077863875165911302873958787098100772718271874529013972836614842142871705531796543076504534324600536361472618180969976933486264077435199928686323835088756683595097265574815431940195576850437248001020413749831872259677387154958399718444907279141965845930083942637020875635398216962055324803212267498911402678528599673405242031091797899905718821949391320753431707980023736590985375520238911643467185582906853711897952626234492483392496342449714656846591248918556629589329909035239233333647435203707701010843880032907598342170185542283861617210417603011645918780539367447472059985023582891833692922337323999480437108419659473162654825748099482509991833006976569367159689364493348864744213500840700660883597235039532340179582557036016936990988671132109798897070517280755855191269930673099250704070245568507786790694766126298082251633136399521170984528092630375922426742575599892892783704744452189363203489415521044597261883800300677617931381399162058062701651024458869247649246891924612125310275731390840470007143561362316992371694848132554200914530410371354532966206392105479824392125172540132314902740585892063217589494345489068463993137570910346332714153162232805522972979538018801628590735729554162788676498274186164218789885741071649069191851162815285486794173638906653885764229158342500673612453849160674137340173572779956341043326883569507814931378007362354180070619180267328551191942676091221035987469241172837493126163395001239599240508454375698507957046222664619000103500490183034153545842833764378111988556318777792537201166718539541835984438305203762819440761594106820716970302285152250573126093046898423433152732131361216582808075212631547730604423774753505952287174402666389148817173086436111389069420279088143119448799417154042103412190847094080254023932942945493878640230512927119097513536000921971105412096683111516328705423028470073120658032626417116165957613272351566662536672718998534199895236884830999302757419916463841427077988708874229277053891227172486322028898425125287217826030500994510824783572905691988555467886079462805371227042466543192145281760741482403827835829719301017888345674167811398954750448339314689630763396657226727043393216745421824557062524797219978668542798977992339579057581890622525473582205236424850783407110144980478726691990186438822932305382318559732869780922253529591017341407334884761005564018242392192695062083183814546983923664613639891012102177095976704908305081854704194664371312299692358895384930136356576186106062228705599423371631021278457446463989738188566746260879482018647487672727222062676465338099801966883680994159075776852639865146253336312450536402610569605513183813174261184420189088853196356986962795036738424313011331753305329802016688817481342988681585577810343231753064784983210629718425184385534427620128234570716988530518326179641178579608888150329602290705614476220915094739035946646916235396809201394578175891088931992112260073928149169481615273842736264298098234063200244024495894456129167049508235812487391799648641133480324757775219708932772262349486015046652681439877051615317026696929704928316285504212898146706195331970269507214378230476875280287354126166391708245925170010714180854800636923259462019002278087409859771921805158532147392653251559035410209284665925299914353791825314545290598415817637058927906909896911164381187809435371521332261443625314490127454772695739393481546916311624928873574718824071503995009446731954316193855485207665738825139639163576723151005556037263394867208207808653734942440115799667507360711159351331959197120948964717553024531364770942094635696982226673775209945168450643623824211853534887989395673187806606107885440005508276570305587448541805778891719207881423351138662929667179643468760077047999537883387870348718021842437342112273940255717690819603092018240188427057046092622564178375265263358324240661253311529423457965569502506810018310900411245379015332966156970522379210325706937051090830789479999004999395322153622748476603613677697978567386584670936679588583788795625946464891376652199588286933801836011932368578558558195556042156250883650203322024513762158204618106705195330653060606501054887167245377942831338871631395596905832083416898476065607118347136218123246227258841990286142087284956879639325464285343075301105285713829643709990356948885285190402956047346131138263878897551788560424998748316382804046848618938189590542039889872650697620201995548412650005394428203930127481638158530396439925470201672759328574366661644110962566337305409219519675148328734808957477775278344221091073111351828046036347198185655572957144747682552857863349342858423118749440003229690697758315903858039353521358860079600342097547392296733310649395601812237812854584317605561733861126734780745850676063048229409653041118306671081893031108871728167519579675347188537229309616143204006381322465841111157758358581135018569047815368938137718472814751998350504781297718599084707621974605887423256995828892535041937958260616211842368768511418316068315867994601652057740529423053601780313357263267054790338401257305912339601880137825421927094767337191987287385248057421248921183470876629667207272325650565129333126059505777727542471241648312832982072361750574673870128209575544305968395555686861188397135522084452852640081252027665557677495969626612604565245684086139238265768583384698499778726706555191854468698469478495734622606294219624557085371272776523098955450193037732166649182578154677292005212667143463209637891852323215018976126034373684067194193037746880999296877582441047878123266253181845960453853543839114496775312864260925211537673258866722604042523491087026958099647595805794663973419064010036361904042033113579336542426303561457009011244800890020801478056603710154122328891465722393145076071670643556827437743965789067972687438473076346451677562103098604092717090951280863090297385044527182892749689212106670081648583395537735919136950153162018908887484210798706899114804669270650940762046502772528650728905328548561433160812693005693785417861096969202538865034577183176686885923681488475276498468821949739729707737187188400414323127636504814531122850990020742409255859252926103021067368154347015252348786351643976235860419194129697690405264832347009911154242601273438022089331096686367898694977994001260164227609260823493041180643829138347354679725399262338791582998486459271734059225620749105308531537182911681637219395188700957788181586850464507699343940987433514431626330317247747486897918209239480833143970840673084079589358108966564775859905563769525232653614424780230826811831037735887089240613031336477371011628214614661679404090518615260360092521947218890918107335871964142144478654899528582343947050079830388538860831035719306002771194558021911942899922722353458707566246926177663178855144350218287026685610665003531050216318206017609217984684936863161293727951873078972637353717150256378733579771808184878458866504335824377004147710414934927438457587107159731559439426412570270965125108115548247939403597681188117282472158250109496096625393395380922195591918188552678062149923172763163218339896938075616855911752998450132067129392404144593862398809381240452191484831646210147389182510109096773869066404158973610476436500068077105656718486281496371118832192445663945814491486165500495676982690308911185687986929470513524816091743243015383684707292898982846022237301452655679898627767968091469798378268764311598832109043715611299766521539635464420869197567370005738764978437686287681792497469438427465256316323005551304174227341646455127812784577772457520386543754282825671412885834544435132562054464241011037955464190581168623059644769587054072141985212106734332410756767575818456990693046047522770167005684543969234041711089888993416350585157887353430815520811772071880379104046983069578685473937656433631979786803671873079693924236321448450354776315670255390065423117920153464977929066241508328858395290542637687668968805033317227800185885069736232403894700471897619347344308437443759925034178807972235859134245813144049847701732361694719765715353197754997162785663119046912609182591249890367654176979903623755286526375733763526969344354400473067198868901968147428767790866979688522501636949856730217523132529265375896415171479559538784278499866456302878831962099830494519874396369070682762657485810439112232618794059941554063270131989895703761105323606298674803779153767511583043208498720920280929752649812569163425000522908872646925284666104665392171482080130502298052637836426959733707053922789153510568883938113249757071331029504430346715989448786847116438328050692507766274500122003526203709466023414648998390252588830148678162196775194583167718762757200505439794412459900771152051546199305098386982542846407255540927403132571632640792934183342147090412542533523248021932277075355546795871638358750181593387174236061551171013123525633485820365146141870049205704372018261733194715700867578539336078622739558185797587258744102542077105475361294047460100094095444959662881486915903899071865980563617137692227290764197755177720104276496949611056220592502420217704269622154958726453989227697660310524980855759471631075870133208861463266412591148633881220284440694169488261529577625325019870359870674380469821942056381255833436421949232275937221289056420943082352544084110864545369404969271494003319782861318186188811118408257865928757426384450059944229568586460481033015388911499486935436030221810943466764000022362550573631294626296096198760564259963946138692330837196265954739234624134597795748524647837980795693198650815977675350553918991151335252298736112779182748542008689539658359421963331502869561192012298889887006079992795411188269023078913107603617634779489432032102773359416908650071932804017163840644987871753756781185321328408216571107549528294974936214608215583205687232185574065161096274874375098092230211609982633033915469494644491004515280925089745074896760324090768983652940657920198315265410658136823791984090645712468948470209357761193139980246813405200394781949866202624008902150166163813538381515037735022966074627952910384068685569070157516624192987244482719429331004854824454580718897633003232525821581280327467962002814762431828622171054352898348208273451680186131719593324711074662228508710666117703465352839577625997744672185715816126411143271794347885990892808486694914139097716736900277758502686646540565950394867841110790116104008572744562938425494167594605487117235946429105850909950214958793112196135908315882620682332156153086833730838173279328196983875087083483880463884784418840031847126974543709373298362402875197920802321878744882872843727378017827008058782410749357514889978911739746129320351081432703251409030487462262942344327571260086642508333187688650756429271605525289544921537651751492196367181049435317858383453865255656640657251363575064353236508936790431702597878177190314867963840828810209461490079715137717099061954969640070867667102330048672631475510537231757114322317411411680622864206388906210192355223546711662137499693269321737043105987225039456574924616978260970253359475020913836673772894438696400028110344026084712899000746807764844088711341352503367877316797709372778682166117865344231732264637847697875144332095340001650692130546476890985050203015044880834261845208730530973189492916425322933612431514306578264070283898409841602950309241897120971601649265613413433422298827909921786042679812457285345801338260995877178113102167340256562744007296834066198480676615805021691833723680399027931606420436812079900316264449146190219458229690992122788553948783538305646864881655562294315673128274390826450611628942803501661336697824051770155219626522725455850738640585299830379180350432876703809252167907571204061237596327685674845079151147313440001832570344920909712435809447900462494313455028900680648704293534037436032625820535790118395649089354345101342969617545249573960621490288728932792520696535386396443225388327522499605986974759882329916263545973324445163755334377492928990581175786355555626937426910947117002165411718219750519831787137106051063795558588905568852887989084750915764639074693619881507814685262133252473837651192990156109189777922008705793396463827490680698769168197492365624226087154176100430608904377976678519661891404144925270480881971498801542057787006521594009289777601330756847966992955433656139847738060394368895887646054983871478968482805384701730871117761159663505039979343869339119789887109156541709133082607647406305711411098839388095481437828474528838368079418884342666222070438722887413947801017721392281911992365405516395893474263953824829609036900288359327745855060801317988407162446563997948275783650195514221551339281978226984278638391679715091262410548725700924070045488485692950448110738087996547481568913935380943474556972128919827177020766613602489581468119133614121258783895577357194986317210844398901423948496659251731388171602663261931065366535041473070804414939169363262373767777095850313255990095762731957308648042467701212327020533742667053142448208168130306397378736642483672539837487690980602182785786216512738563513290148903509883270617258932575363993979055729175160097615459044771692265806315111028038436017374742152476085152099016158582312571590733421736576267142390478279587281505095633092802668458937649649770232973641319060982740633531089792464242134583740901169391964250459128813403498810635400887596820054408364386516617880557608956896727531538081942077332597917278437625661184319891025007491829086475149794003160703845549465385946027452447466812314687943441610993338908992638411847425257044572517459325738989565185716575961481266020310797628254165590506042479114016957900338356574869252800743025623419498286467914476322774005529460903940177536335655471931000175430047504719144899841040015867946179241610016454716551337074073950260442769538553834397550548871099785205401175169747581344926079433689543783221172450687344231989878844128542064742809735625807066983106979935260693392135685881391214807354728463227784908087002467776303605551232386656295178853719673034634701222939581606792509153217489030840886516061119011498443412350124646928028805996134283511884715449771278473361766285062169778717743824362565711779450064477718370221999106695021656757644044997940765037999954845002710665987813603802314126836905783190460792765297277694043613023051787080546511542469395265127101052927070306673024447125973939950514628404767431363739978259184541176413327906460636584152927019030276017339474866960348694976541752429306040727005059039503148522921392575594845078867977925253931765156416197168443524369794447355964260633391055126826061595726217036698506473281266724521989060549880280782881429796336696744124805982192146339565745722102298677599746738126069367069134081559412016115960190237753525556300606247983261249881288192937343476862689219239777833910733106588256813777172328315329082525092733047850724977139448333892552081175608452966590553940965568541706001179857293813998258319293679100391844099286575605993598910002969864460974714718470101531283762631146774209145574041815908800064943237855839308530828305476076799524357391631221886057549673832243195650655460852881201902363644712703748634421727257879503428486312944916318475347531435041392096108796057730987201352484075057637199253650470908582513936863463863368042891767107602111159828875539940120076013947033661793715396306139863655492213741597905119083588290097656647300733879314678913181465109316761575821351424860442292445304113160652700974330088499034675405518640677342603583409608605533747362760935658853109760994238347382222087292464497684560579562516765574088410321731345627735856052358236389532038534024842273371639123973215995440828421666636023296545694703577184873442034227706653837387506169212768015766181095420097708363604361110592409117889540338021426523948929686439808926114635414571535194342850721353453018315875628275733898268898523557799295727645229391567477566676051087887648453493636068278050564622813598885879259940946446041705204470046315137975431737187756039815962647501410906658866162180038266989961965580587208639721176995219466789857011798332440601811575658074284182910615193917630059194314434605154047710570054339000182453117733718955857603607182860506356479979004139761808955363669603162193113250223851791672055180659263518036251214575926238369348222665895576994660491938112486609099798128571823494006615552196112207203092277646200999315244273589488710576623894693889446495093960330454340842102462401048723328750081749179875543879387381439894238011762700837196053094383940063756116458560943129517597713935396074322792489221267045808183313764165818269562105872892447740035947009268662659651422050630078592002488291860839743732353849083964326147000532423540647042089499210250404726781059083644007466380020870126664209457181702946752278540074508552377720890581683918446592829417018288233014971554235235911774818628592967605048203864343108779562892925405638946621948268711042828163893975711757786915430165058602965217459581988878680408110328432739867198621306205559855266036405046282152306154594474489908839081999738747452969810776201487134000122535522246695409315213115337915798026979555710508507473874750758068765376445782524432638046143042889235934852961058269382103498000405248407084403561167817170512813378805705643450616119330424440798260377951198548694559152051960093041271007277849301555038895360338261929343797081874320949914159593396368110627557295278004254863060054523839151068998913578820019411786535682149118528207852130125518518493711503422159542244511900207393539627400208110465530207932867254740543652717595893500716336076321614725815407642053020045340183572338292661915308354095120226329165054426123619197051613839357326693760156914429944943744856809775696303129588719161129294681884936338647392747601226964158848900965717086160598147204467428664208765334799858222090619802173211614230419477754990738738567941189824660913091691772274207233367635032678340586301930193242996397204445179288122854478211953530898910125342975524727635730226281382091807439748671453590778633530160821559911314144205091447293535022230817193663509346865858656314855575862447818620108711889760652969899269328178705576435143382060141077329261063431525337182243385263520217735440715281898137698755157574546939727150488469793619500477720970561793913828989845327426227288647108883270173723258818244658436249580592560338105215606206155713299156084892064340303395262263451454283678698288074251422567451806184149564686111635404971897682154227722479474033571527436819409892050113653400123846714296551867344153741615042563256713430247655125219218035780169240326699541746087592409207004669340396510178134857835694440760470232540755557764728450751826890418293966113310160131119077398632462778219023650660374041606724962490137433217246454097412995570529142438208076098364823465973886691349919784013108015581343979194852830436739012482082444814128095443773898320059864909159505322857914576884962578665885999179867520554558099004556461178755249370124553217170194282884617402736649978475508294228020232901221630102309772151569446427909802190826689868834263071609207914085197695235553488657743425277531197247430873043619511396119080030255878387644206085044730631299277888942729189727169890575925244679660189707482960949190648764693702750773866432391919042254290235318923377293166736086996228032557185308919284403805071030064776847863243191000223929785255372375566213644740096760539439838235764606992465260089090624105904215453927904411529580345334500256244101006359530039598864466169595626351878060688513723462707997327233134693971456285542615467650632465676620279245208581347717608521691340946520307673391841147504140168924121319826881568664561485380287539331160232292555618941042995335640095786495340935115266454024418775949316930560448686420862757201172319526405023099774567647838488973464317215980626787671838005247696884084989185086149003432403476742686245952395890358582135006450998178244636087317754378859677672919526111213859194725451400301180503437875277664402762618941017576872680428176623860680477885242887430259145247073950546525135339459598789619778911041890292943818567205070964606263541732944649576612651953495701860015412623962286413897796733329070567376962156498184506842263690367849555970026079867996261019039331263768556968767029295371162528005543100786408728939225714512481135778627664902425161990277471090335933309304948380597856628844787441469841499067123764789582263294904679812089984857163571087831191848630254501620929805829208334813638405421720056121989353669371336733392464416125223196943471206417375491216357008573694397305979709719726666642267431117762176403068681310351899112271339724036887000996862922546465006385288620393800504778276912835603372548255793912985251506829969107754257647488325341412132800626717094009098223529657957997803018282428490221470748111124018607613415150387569830918652780658896682362523937845272634530420418802508442363190383318384550522367992357752929106925043261446950109861088899914658551881873582528164302520939285258077969737620845637482114433988162710031703151334402309526351929588680690821355853680161000213740851154484912685841268695899174149133820578492800698255195740201818105641297250836070356851055331787840829000041552511865779453963317538532092149720526607831260281961164858098684587525129997404092797683176639914655386108937587952214971731728131517932904431121815871023518740757222100123768721944747209349312324107065080618562372526732540733324875754482967573450019321902199119960797989373383673242576103938985349278777473980508080015544764061053522202325409443567718794565430406735896491017610775948364540823486130254718476485189575836674399791508512858020607820554462991723202028222914886959399729974297471155371858924238493855858595407438104882624648788053304271463011941589896328792678327322456103852197011130466587100500083285177311776489735230926661234588873102883515626446023671996644554727608310118788389151149340939344750073025855814756190881398752357812331342279866503522725367171230756861045004548970360079569827626392344107146584895780241408158405229536937499710665594894459246286619963556350652623405339439142111271810691052290024657423604130093691889255865784668461215679554256605416005071276641766056874274200329577160643448606201239821698271723197826816628249938714995449137302051843669076723577400053932662622760323659751718925901801104290384274185507894887438832703063283279963007200698012244365116394086922220745320244624121155804354542064215121585056896157356414313068883443185280853975927734433655384188340303517822946253702015782157373265523185763554098954033236382319219892171177449469403678296185920803403867575834111518824177439145077366384071880489358256868542011645031357633355509440319236720348651010561049872726472131986543435450409131859513145181276437310438972507004981987052176272494065214619959232142314439776546708351714749367986186552791715824080651063799500184295938799158350171580759883784962257398512129810326379376218322456594236685376799113140108043139732335449090824910499143325843298821033984698141715756010829706583065211347076803680695322971990599904451209087275776225351040902392888779424630483280319132710495478599180196967835321464441189260631526618167443193550817081875477050802654025294109218264858213857526688155584113198560022135158887210365696087515063187533002942118682221893775546027227291290504292259787710667873840000616772154638441292371193521828499824350920891801685572798156421858191197490985730570332667646460728757430565372602768982373259745084479649545648030771598153955827779139373601717422996027353102768719449444917939785144631597314435351850491413941557329382048542123508173912549749819308714396615132942045919380106231421774199184060180347949887691051557905554806953878540066453375981862846419905220452803306263695626490910827627115903856995051246529996062855443838330327638599800792922846659503551211245284087516229060262011857775313747949362055496401073001348853150735487353905602908933526400713274732621960311773433943673385759124508149335736911664541281788171454023054750667136518258284898099512139193995633241336556777098003081910272040997148687418134667006094051021462690280449159646545330107754695413088714165312544813061192407821188690056027781824235022696189344352547633573536485619363254417756613981703930632872166905722259745209192917262199844409646158269456380239502837121686446561785235565164127712826918688615572716201474934052276946595712198314943381622114006936307430444173284786101777743837977037231795255434107223445512555589998646183876764903972461167959018100035098928641204195163551108763204267612979826529425882951141275841262732790798807559751851576841264742209479721843309352972665210015662514552994745127631550917636730259462132930190402837954246323258550301096706922720227074863419005438302650681214142135057154175057508639907673946335146209082888934938376439399256900604067311422093312195936202982972351163259386772241477911629572780752395056251581603133359382311500518626890530658368129988108663263271980611271548858798093487912913707498230575929091862939195014721197586067270092547718025750337730799397134539532646195269996596385654917590458333585799102012713204583903200853878881633637685182083727885131175227769609787962142372162545214591281831798216044111311671406914827170981015457781939202311563871950805024679725792497605772625913328559726371211201905720771409148645074094926718035815157571514050397610963846755569298970383547314100223802583468767350129775413279532060971154506484212185936490997917766874774481882870632315515865032898164228288232746866106592732197907162384642153489852476216789050260998045266483929542357287343977680495774091449538391575565485459058976495198513801007958010783759945775299196700547602252552034453988712538780171960718164078124847847257912407824544361682345239570689514272269750431873633263011103053423335821609333191218806608268341428910415173247216053355849993224548730778822905252324234861531520976938461042582849714963475341837562003014915703279685301868631572488401526639835689563634657435321783493199825542117308467745297085839507616458229630324424328237737450517028560698067889521768198156710781633405266759539424926280756968326107495323390536223090807081455919837355377748742029039018142937311529334644468151212945097596534306284215319445727118614900017650558177095302468875263250119705209476159416768727784472000192789137251841622857783792284439084301181121496366424659033634194540657183544771912446621259392656620306888520055599121235363718226922531781458792593750441448933981608657900876165024635197045828895481793756681046474614105142498870252139936870509372305447734112641354892806841059107716677821238332810262185587751312721179344448201440425745083063944738363793906283008973306241380614589414227694747931665717623182472168350678076487573420491557628217583972975134478990696589532548940335615613167403276472469212505759116251529654568544633498114317670257295661844775487469378464233737238981920662048511894378868224807279352022501796545343757274163910791972952950812942922205347717304184477915673991738418311710362524395716152714669005814700002633010452643547865903290733205468338872078735444762647925297690170912007874183736735087713376977683496344252419949951388315074877537433849458259765560996555954318040920178497184685497370696212088524377013853757681416632722412634423982152941645378000492507262765150789085071265997036708726692764308377229685985169122305037462744310852934305273078865283977335246017463527703205938179125396915621063637625882937571373840754406468964783100704580613446731271591194608435935825987782835266531151065041623295329047772174083559349723758552138048305090009646676088301540612824308740645594431853413755220166305812111033453120745086824339432159043594430312431227471385842030390106070940315235556172767994160020393975099897629335325855575624808996691829864222677502360193257974726742578211119734709402357457222271212526852384295874273501563660093188045493338989741571490544182559738080871565281430102670460284316819230392535297795765862414392701549740879273131051636119137577008929564823323648298263024607975875767745377160102490804624301856524161756655600160859121534556267602192689982855377872583145144082654583484409478463178777374794653580169960779405568701192328608041130904629350871827125934668712766694873899824598527786499569165464029458935064964335809824765965165142090986755203808309203230487342703468288751604071546653834619611223013759451579252696743642531927390036038608236450762698827497618723575476762889950752114804852527950845033958570838130476937881321123674281319487950228066320170022460331989671970649163741175854851878484012054844672588851401562725019821719066960812627785485964818369621410721714214986361918774754509650308957099470934337856981674465828267911940611956037845397855839240761276344105766751024307559814552786167815949657062559755074306521085301597908073343736079432866757890533483669555486803913433720156498834220893399971641479746938696905480089193067138057171505857307148815649920714086758259602876056459782423770242469805328056632787041926768467116266879463486950464507420219373945259262668613552940624781361206202636498199999498405143868285258956342264328707663299304891723400725471764188685351372332667877921738347541480022803392997357936152412755829569276837231234798989446274330454566790062032420516396282588443085438307201495672106460533238537203143242112607424485845094580494081820927639140008540422023556260218564348994145439950410980591817948882628052066441086319001688568155169229486203010738897181007709290590480749092427141018933542818429995988169660993836961644381528877214085268088757488293258735809905670755817017949161906114001908553744882726200936685604475596557476485674008177381703307380305476973609786543859382187220583902344443508867499866506040645874346005331827436296177862518081893144363251205107094690813586440519229512932450078833398788429339342435126343365204385812912834345297308652909783300671261798130316794385535726296998740359570458452230856390098913179475948752126397078375944861139451960286751210561638976008880092746115860800207803341591451797073036835196977766076373785333012024120112046988609209339085365773222392412449051532780950955866459477634482269986074813297302630975028812103517723124465095349653693090018637764094094349837313251321862080214809922685502948454661814715557444709669530177690434272031892770604717784527939160472281534379803539679861424370956683221491465438014593829277393396032754048009552231816667380357183932757077142046723838624617803976292377131209580789363841447929802588065522129262093623930637313496640186619510811583471173312025805866727639992763579078063818813069156366274125431259589936119647626101405563503399523140323113819656236327198961837254845333702062563464223952766943568376761368711962921818754576081617053031590728828700712313666308722754918661395773730546065997437810987649802414011242142773668082751390959313404155826266789510846776118665957660165998178089414985754976284387856100263796543178313634025135814161151902096499133548733131115022700681930135929595971640197196053625033558479980963488718039111612813595968565478868325856437896173159762002419621552896297904819822199462269487137462444729093456470028537694958859591606789282491054412515996300781368367490209374915732896270028656829344431342347351239298259166739503425995868970697267332582735903121288746660451461487850346142827765991608090398652575717263081833494441820193533385071292345774375579344062178711330063106003324053991693682603746176638565758877580201229366353270267100681261825172914608202541892885935244491070138206211553827793565296914576502048643282865557934707209634807372692141186895467322767751335690190153723669036865389161291688887876407525493494249733427181178892759931596719354758988097924525262363659036320070854440784544797348291802082044926670634420437555325050527522833778887040804033531923407685630109347772125639088640413101073817853338316038135280828119040832564401842053746792992622037698718018061122624490909242641985820861751177113789051609140381575003366424156095216328197122335023167422600567941281406217219641842705784328959802882335059828208196666249035857789940333152274817776952843681630088531769694783690580671064828083598046698841098135158654906933319522394363287923990534810987830274500172065433699066117784554364687723631844464768069142828004551074686645392805399409108754939166095731619715033166968309929466349142798780842257220697148875580637480308862995118473187124777291910070227588893486939456289515802965372150409603107761289831263589964893410247036036645058687287589051406841238124247386385427908282733827973326885504935874303160274749063129572349742611221517417153133618622410913869500688835898962349276317316478340077460886655598733382113829928776911495492184192087771606068472874673681886167507221017261103830671787856694812948785048943063086169948798703160515884108282351274153538513365895332948629494495061868514779105804696039069372662670386512905201137810858616188886947957607413585534585151768051973334433495230120395770739623771316030242887200537320998253008977618973129817881944671731160647231476248457551928732782825127182446807824215216469567819294098238926284943760248852279003620219386696482215628093605373178040863727268426696421929946819214908701707533361094791381804063287387593848269535583077395761447997270003472880182785281389503217986345216111066608839314053226944905455527867894417579202440021450780192099804461382547805858048442416404775031536054906591430078158372430123137511562284015838644270890718284816757527123846782459534334449622010096071051370608461801187543120725491334994247617115633321408934609156561550600317384218701570226103101916603887064661438897736318780940711527528174689576401581047016965247557740891644568677717158500583269943401677202156767724068128366565264122982439465133197359199709403275938502669557470231813203243716420586141033606524536939160050644953060161267822648942437397166717661231048975031885732165554988342121802846912529086101485527815277625623750456375769497734336846015607727035509629049392487088406281067943622418704747008368842671022558302403599841645951122485272633632645114017395248086194635840783753556885622317115520947223065437092606797351000565549381224575483728545711797393615756167641692895805257297522338558611388322171107362265816218842443178857488798109026653793426664216990914056536432249301334867988154886628665052346997235574738424830590423677143278792316422403877764330192600192284778313837632536121025336935812624086866699738275977365682227907215832478888642369346396164363308730139814211430306008730666164803678984091335926293402304324974926887831643602681011309570716141912830686577323532639653677390317661361315965553584999398600565155921936759977717933019744688148371103206503693192894521402650915465184309936553493337183425298433679915939417466223900389527673813330617747629574943868716978453767219493506590875711917720875477107189937960894774512654757501871194870738736785890200617373321075693302216320628432065671192096950585761173961632326217708945426214609858410237813215817727602222738133495410481003073275107799948991977963883530734443457532975914263768405442264784216063122769646967156473999043715903323906560726644116438605404838847161912109008701019130726071044114143241976796828547885524779476481802959736049439700479596040292746299203572099761950140348315380947714601056333446998820822120587281510729182971211917876424880354672316916541852256729234429187128163232596965413548589577133208339911288775917226115273379010341362085614577992398778325083550730199818459025958355989260553299673770491722454935329683300002230181517226575787524058832249085821280089747909326100762578770428656006996176212176845478996440705066241710213327486796237430229155358200780141165348065647488230615003392068983794766255036549822805329662862117930628430170492402301985719978948836897183043805182174419147660429752437251683435411217038631379411422095295885798060152938752753799030938871683572095760715221900279379292786303637268765822681241993384808166021603722154710143007377537792699069587121289288019052031601285861825494413353820784883465311632650407642428390870121015194231961652268422003711230464300673442064747718021353070124098860353399152667923871101706221865883573781210935179775604425634694999787251125440854522274810914874307259869602040275941178942581281882159952359658979181144077653354321757595255536158128001163846720319346507296807990793963714961774312119402021297573125165253768017359101557338153772001952444543620071848475663415407442328621060997613243487548847434539665981338717466093020535070271952983943271425371155766600025784423031073429551533945060486222764966687624079324353192992639253731076892135352572321080889819339168668278948281170472624501948409700975760920983724090074717973340788141825195842598096241747610138252643955135259311885045636264188300338539652435997416931322894719878308427600401368074703904097238473945834896186539790594118599310356168436869219485382055780395773881360679549900085123259442529724486666766834641402189915944565309423440650667851948417766779470472041958822043295380326310537494883122180391279678446100139726753892195119117836587662528083690053249004597410947068772912328214304635337283519953648274325833119144459017809607782883583730111857543659958982724531925310588115026307542571493943024453931870179923608166611305426253995833897942971602070338767815033010280120095997252222280801423571094760351925544434929986767817891045559063015953809761875920358937341978962358931125983902598310267193304189215109689156225069659119828323455503059081730735195503721665870288053992138576037035377105178021280129566841984140362872725623214428754302210909472721073474134975514190737043318276626177275996888826027225247133683353452816692779591328861381766349857728936900965749562287103024362590772412219094300871755692625758065709912016659622436080242870024547362036394841255954881727272473653467783647201918303998717627037515724649922289467932322693619177641614618795613956699567783068290316589699430767333508234990790624100202506134057344300695745474682175690441651540636584680463692621274211075399042188716127617787014258864825775223889184599523376292377915585744549477361295525952226578636462118377598473700347971408206994145580719080213590732269233100831759510659019121294795408603640757358750205890208704579670007055262505811420663907459215273309406823649441590891009220296680523325266198911311842016291631076894084723564366808182168657219688268358402785500782804043453710183651096951782335743030504852653738073531074185917705610397395062640355442275156101107261779370634723804990666922161971194259120445084641746383589938239946517395509000859479990136026674261494290066467115067175422177038774507673563742154782905911012619157555870238957001405117822646989944917908301795475876760168094100135837613578591356924455647764464178667115391951357696104864922490083446715486383054477914330097680486878348184672733758436892724310447406807685278625585165092088263813233623148733336714764520450876627614950389949504809560460989604329123358348859990294526400284994280878624039811814884767301216754161106629995553668193123287425702063738352020086863691311733469731741219153633246745325630871347302792174956227014687325867891734558379964351358800959350877556356248810493852999007675135513527792412429277488565888566513247302514710210575352516511814850902750476845518252096331899068527614435138213662152368890578786699432288816028377482035506016029894009119713850179871683633744139275973644017007014763706655703504338121113576415018451821413619823495159601064752712575935185304332875537783057509567425442684712219618709178560783936144511383335649103256405733898667178123972237519316430617013859539474367843392670986712452211189690840236327411496601243483098929941738030588417166613073040067588380432111555379440605497721705942821514886165672771240903387727745629097110134885184374118695655449745736845218066982911045058004299887953899027804383596282409421860556287788428802127553884803728640019441614257499904272009595204654170598104989967504511936471172772220436102614079750809686975176600237187748348016120310234680567112644766123747627852190241202569943534716226660893675219833111813511146503854895025120655772636145473604426859498074396932331297127377157347099713952291182653485155587137336629120242714302503763269501350911612952993785864681307226486008270881333538193703682598867893321238327053297625857382790097826460545598555131836688844628265133798491667839409761353766251798258249663458771950124384040359140849209733754642474488176184070023569580177410177696925077814893386672557898564589851056891960924398841569280696983352240225634570497312245269354193837004843183357196516626721575524193401933099018319309196582920969656247667683659647019595754739345514337413708761517323677204227385674279170698204549953095918872434939524094441678998846319845504852393662972079777452814399418256789457795712552426826089940863317371538896262889629402112108884427376568624527612130371017300785135715404533041507959447776143597437803742436646973247138410492124314138903579092416036406314038149831481905251720937103964026808994832572297954564042701757722904173234796073618787889913318305843069394825961318713816423467218730845133877219086975104942843769325024981656673816260615941768252509993741672883951744066932549653403101452225316189009235376486378482881344209870048096227171226407489571939002918573307460104360729190945767994614929290427981687729426487729952858434647775386906950148984133924540394144680263625402118614317031251117577642829914644533408920976961699098372652361768745605894704968170136974909523072082682887890730190018253425805343421705928713931737993142410852647390948284596418093614138475831136130576108462366837237695913492615824516221552134879244145041756848064120636520170386330129532777699023118648020067556905682295016354931992305914246396217025329747573114094220180199368035026495636955866425906762685687372110339156793839895765565193177883000241613539562437777840801748819373095020699900890899328088397430367736595524891300156633294077907139615464534088791510300651321934486673248275907946807879819425019582622320395131252014109960531260696555404248670549986786923021746989009547850725672978794769888831093487464426400718183160331655511534276155622405474473378049246214952133258527698847336269182649174338987824789278468918828054669982303689939783413747587025805716349413568433929396068192061773331791738208562436433635359863494496890781064019674074436583667071586924521182997893804077137501290858646578905771426833582768978554717687184427726120509266486102051535642840632368481807287940717127966820060727559555904040233178749447346454760628189541512139162918444297651066947969354016866010055196077687335396511614930937570968554559381513789569039251014953265628147011998326992200066392875374713135236421589265126204072887716578358405219646054105435443642166562244565042999010256586927279142752931172082793937751326106052881235373451068372939893580871243869385934389175713376300720319760816604464683937725806909237297523486702916910426369262090199605204121024077648190316014085863558427609537086558164273995349346546314504040199528537252004957805254656251154109252437991326262713609099402902262062836752132305065183934057450112099341464918433323646569371725914489324159006242020612885732926133596808726500045628284557574596592120530341310111827501306961509835515632004310784601906565493806542525229161991819959602752327702249855738824899882707465936355768582560518068964285376850772012220347920993936179268206590142165615925306737944568949070853263568196831861772268249911472615732035807646298116244013316737892788689229032593349861797021994981925739617673075834417098559222170171825712777534491508205278430904619460835217402005838672849709411023266953921445461066215006410674740207009189911951376466904481267253691537162290791385403937560077835153374167747942100384002308951850994548779039346122220865060160500351776264831611153325587705073541279249909859373473787081194253055121436979749914951860535920403830235716352727630874693219622190064260886183676103346002255477477813641012691906569686495012688376296907233961276287223041141813610060264044030035996988919945827397624114613744804059697062576764723766065541618574690527229238228275186799156983390747671146103022776606020061246876477728819096791613354019881402757992174167678799231603963569492851513633647219540611171767387372555728522940054361785176502307544693869307873499110352182532929726044553210797887711449898870911511237250604238753734841257086064069052058452122754533848008205302450456517669518576913200042816758054924811780519832646032445792829730129105318385636821206215531288668564956512613892261367064093953334570526986959692350353094224543865278677673027540402702246384483553239914751363441044050092330361271496081355490531539021002299595756583705381261965683144286057956696622154721695620870013727768536960840704833325132793112232507148630206951245395003735723346807094656483089209801534878705633491092366057554050864111521441481434630437273271045027768661953107858323334857840297160925215326092558932655600672124359464255065996771770388445396181632879614460817789272171836908880126778207430106422524634807454300476492885553409062185153654355474125476152769772667769772777058315801412185688011705028365275543214803488004442979998062157904564161957212784508928489806426497427090579129069217807298769477975112447305991406050629946894280931034216416629935614828130998870745292716048433630818404126469637925843094185442216359084576146078558562473814931427078266215185541603870206876980461747400808324343665382354555109449498431093494759944672673665352517662706772194183191977196378015702169933675083760057163454643671776723387588643405644871566964321041282595645349841388412890420682047007615596916843038999348366793542549210328113363184722592305554383058206941675629992013373175489122037230349072681068534454035993561823576312837767640631013125335212141994611869350833176587852047112364331226765129964171325217513553261867681942338790365468908001827135283584888444111761234101179918709236507184857856221021104009776994453121795022479578069506532965940383987369907240797679040826794007618729547835963492793904576973661643405359792219285870574957481696694062334272619733518136626063735982575552496509807260123668283605928341855848026958413772558970883789942910549800331113884603401939166122186696058491571485733568286149500019097591125218800396419762163559375743718011480559442298730418196808085647265713547612831629200449880315402105530597076666362749328308916880932359290081787411985738317192616728834918402429721290434965526942726402559641463525914348400675867690350382320572934132981593533044446496829441367323442158380761694831219333119819061096142952201536170298575105594326461468505452684975764807808009221335811378197749271768545075538328768874474591593731162470601091244609829424841287520224462594477638749491997840446829257360968534549843266536862844489365704111817793806441616531223600214918768769467398407517176307516849856359201486892943105940202457969622924566644881967576294349535326382171613395757790766370764569570259738800438415805894336137106551859987600754924187211714889295221737721146081154344982665479872580056674724051122007383459271575727715218589946948117940644466399432370044291140747218180224825837736017346685300744985564715420036123593397312914458591522887408719508708632218837288262822884631843717261903305777147651564143822306791847386039147683108141358275755853643597721650028277803713422869688787349795096031108899196143386664068450697420787700280509367203387232629637856038653216432348815557557018469089074647879122436375556668678067610544955017260791142930831285761254481944449473244819093795369008206384631678225064809531810406570254327604385703505922818919878065865412184299217273720955103242251079718077833042609086794273428955735559252723805511440438001239041687716445180226491681641927401106451622431101700056691121733189423400547959684669804298017362570406733282129962153684881404102194463424646220745575643960452985313071409084608499653767803793201899140865814662175319337665970114330608625009829566917638846056762972931464911493704624469351984039534449135141193667933301936617663652555149174982307987072280860859626112660504289296966535652516688885572112276802772743708917389639772257564890533401038855931125679991516589025016486961427207005916056166159702451989051832969278935550303934681219761582183980483960562523091462638447386296039848924386187298507775928792722068554807210497817653286210187476766897248841139560349480376727036316921007350834073865261684507482496448597428134936480372426116704266870831925040997615319076855770327421785010006441984124207396400139603601583810565928413684574119102736420274163723488214524101347716529603128408658419787951116511529827814620379139855006399960326591248525308493690313130100799977191362230866011099929142871249388541612038020411340188887219693477904497527454288072803509305828754420755134816660927879353566521255620139988249628478726214432362853676502591450468377635282587652139156480972141929675549384375582600253168536356731379262475878049445944183429172756988376226261846365452743497662411138451305481449836311789784489732076719508784158618879692955819733250699951402601511675529750575437810242238957925786562128432731202200716730574069286869363930186765958251326499145950260917069347519408975357464016830811798846452473618956056479426358070562563281189269663026479535951097127659136233180866921535788607812759910537171402204506186075374866306350591483916467656723205714516886170790984695932236724946737583099607042589220481550799132752088583781117685214269334786921895240622657921043620348852926267984013953216458791151579050460579710838983371864038024417511347226472547010794793996953554669619726763255229914654933499663234185951450360980344092212206712567698723427940708857070474293173329188523896721971353924492426178641188637790962814486917869468177591717150669111480020759432012061969637795103227089029566085562225452602610460736131368869009281721068198618553780982018471154163630326265699283424155023600978046417108525537612728905335045506135684143775854429677977014660294387687225115363801191758154028120818255606485410787933598921064427244898618961629413418001295130683638609294100083136673372153008352696235737175330738653338204842190308186449184093723944033405244909554558016406460761581010301767488475017661908692946098769201691202181688291040870709560951470416921147027413390052253340834812870353031023919699978597413908593605433599697075604460134242453682496098772581311024732798562072126572499003468293886872304895562253204463602639854225258416464324271611419817802482595563544907219226583863662663750835944314877635156145710745528016159677048442714194435183275698407552677926411261765250615965235457187956673170913319358761628255920783080185206890151504713340386100310055914817852110384754542933389188444120517943969970194112695119526564919594189975418393234647424290702718875223534393673633663200307232747037407123982562024662651974090199762452056198557625760008708173083288344381831070054514493545885422678578551915372292379555494333410174420169600090696415612732297770221217951868376359082255128816470021992348864043959153018464004714321186360622527011541122283802778538911098490201342741014121559769965438877197485376431158229838533123071751132961904559007938064276695819014842627991221792947987348901868471676503827328552059082984529806259250352128451925927986593506132961946796252373972565584157853744567558998032405492186962888490332560851455344391660226257775512916200772796852629387937530454181080729285891989715381797343496187232927614747850192611450413274873242970583408471112333746274617274626582415324271059322506255302314738759251724787322881491455915605036334575424233779160374952502493022351481961381162563911415610326844958072508273431765944054098269765269344579863479709743124498271933113863873159636361218623497261409556079920628316999420072054811525353393946076850019909886553861433495781650089961649079678142901148387645682174914075623767618453775144031475411206760160726460556859257799322070337333398916369504346690694828436629980037414527627716547623825546170883189810868806847853705536480469350958818025360529740793538676511195079373282083146268960071075175520614433784114549950136432446328193346389050936545714506900864483440180428363390513578157273973334537284263372174065775771079830517555721036795976901889958494130195999573017901240193908681356585539661941371794487632079868800371607303220547423572266896801882123424391885984168972277652194032493227314793669234004848976059037958094696041754279613782553781223947646147832926976545162290281701100437846038756544151739433960048915318817576650500951697402415644771293656614253949368884230517400129920556854289853897942669956777027089146513736892206104415481662156804219838476730871787590279209175900695273456682026513373111518000181434120962601658629821076663523361774007837783423709152644063054071807843358061072961105550020415131696373046849213356837265400307509829089364612047891114753037049893952833457824082817386441322710002968311940203323456420826473276233830294639378998375836554559919340866235090967961134004867027123176526663710778725111860354037554487418693519733656621772359229396776463251562023487570113795712096237723431370212031004965152111976013176419408203437348512852602913334915125083119802850177855710725373149139215709105130965059885999931560863655477403551898166733535880048214665099741433761182777723351910741217572841592580872591315074606025634903777263373914461377038021318347447301113032670296917335047701632106616227830027269283365584011791419447808748253360714403296252285775009808599609040936312635621328162071453406104224112083010008587264252112262480142647519426184325853386753874054743491072710049754281159466017136122590440158991600229827801796035194080046513534752698777609527839984368086908989197839693532179980139135442552717910225397010810632143048511378291498511381969143043497500189980681644412123273328307192824362406733196554692677851193152775113446468905504248113361434984604849051258345683266441528489713972376040328212660253516693914082049947320486021627759791771234751097502403078935759937715095021751693555827072533911892334070223832077585802137174778378778391015234132098489423459613692340497998279304144463162707214796117456975719681239291913740982925805561955207434243295982898980529233366415419256367380689494201471241340525072204061794355252555225008748790086568314542835167750542294803274783044056438581591952666758282929705226127628711040134801787224801789684052407924360582742467443076721645270313451354167649668901274786801010295133862698649748212118629040337691568576240699296372493097201628707200189835423690364149270236961938547372480329855045112089192879829874467864129159417531675602533435310626745254507114181483239880607297140234725520713490798398982355268723950909365667878992383712578976248755990443228895388377317348941122757071410959790047919301046740750411435381782464630795989555638991884773781341347070246747362112048986226991888517456251732519341352038115863350123913054441910073628447567514161050410973505852762044489190978901984315485280533985777844313933883994310444465669244550885946314081751220331390681596592510546858013133838152176418210433429788826119630443111388796258746090226130900849975430395771243230616906262919403921439740270894777663702488155499322458825979020631257436910946393252806241642476868495455324938017639371615636847859823715902385421265840615367228607131702674740131145261063765383390315921943469817605358380310612887852051546933639241088467632009567089718367490578163085158138161966882222047570437590614338040725853862083565176998426774523195824182683698270160237414938363496629351576854061397342746470899685618170160551104880971554859118617189668025973541705423985135560018720335079060946421271143993196046527424050882225359773481519135438571253258540493946010865793798058620143366078825219717809025817370870916460452727977153509910340736425020386386718220522879694458387652947951048660717390229327455426785669776865939923416834122274663015062155320502655341460995249356050854921756549134830958906536175693817637473644183378974229700703545206663170929607591989627732423090252397443861014263098687733913882518684316501027964911497737582888913450341148865948670215492101084328080783428089417298008983297536940644969903125399863919581601468995220880662285408414864274786281975546629278814621607171381880180840572084715868906836919393381864278454537956719272397972364651667592011057995663962598535512763558768140213409829016296873429850792471846056874828331381259161962476156902875901072733103299140623864608333378638257926302391590003557609032477281338887339178096966601469615031754226751125993315529674213336300222964906480934582008181061802100227664580400278213336758573019011371754672763059044353131319036092489097246427928455549913490005180295707082919052556781889913899625138662319380053611346224294610248954072404857123256628888931722116432947816190554868054943441034090680716088028227959686950133643814268252170472870863010137301155236861416908375675747637239763185757038109443390564564468524183028148107998376918512127201935044041804604721626939445788377090105974693219720558114078775989772072009689382249303236830515862657281114637996983137517937623215111252349734305240622105244234353732905655163406669506165892878218707756794176080712973781335187117931650033155523822487730653444179453415395202424449703410120874072188109388268167512042299404948179449472732894770111574139441228455521828424922240658752689172272780607116754046973008037039618787796694882555614674384392570115829546661358678671897661297311267200072971553613027503556167817765442287442114729881614802705243806817653573275578602505847084013208837932816008769081300492491473682517035382219619039014999523495387105997351143478292339499187936608692301375596368532373806703591144243268561512109404259582639301678017128669239283231057658851714020211196957064799814031505633045141564414623163763809904402816256917576489142569714163598439317433270237812336938043012892626375382667795034169334323607500248175741808750388475094939454896209740485442635637164995949920980884294790363666297526003243856352945844728944547166209297495496616877414120882130477022816116456044007236351581149729739218966737382647204722642221242016560150284971306332795814302516013694825567014780935790889657134926158161346901806965089556310121218491805847922720691871696316330044858020102860657858591269974637661741463934159569539554203314628026518951167938074573315759846086173702687867602943677780500244673391332431669880354073232388281847501051641331189537036488422690270478052742490603492082954755054003457160184072574536938145531175354210726557835615499874447480427323457880061873149341566046352979779455075359304795687209316724536547208381685855606043801977030764246083489876101345709394877002946175792061952549255757109038525171488525265671045349813419803390641529876343695420256080277614421914318921393908834543131769685101840103844472348948869520981943531906506555354617335814045544837884752526253949665869992058417652780125341033896469818642430034146791380619028059607854888010789705516946215228773090104467462497979992627120951684779568482583341402266477210843362437593741610536734041954738964197895425335036301861400951534766961476255651873823292468547356935802896011536791787303553159378363082248615177770541577576561759358512016692943111138863582159667618830326104164651714846979385422621687161400122378213779774131268977266712992025922017408770076956283473932201088159356286281928563571893384958850603853158179760679479840878360975960149733420572704603521790605647603285569276273495182203236144112584182426247712012035776388895974318232827871314608053533574494297621796789034568169889553518504478325616380709476951699086247100019748809205009521943632378719764870339223811540363475488626845956159755193765410115014067001226927474393888589943859730245414801061235908036274585288493563251585384383242493252666087588908318700709100237377106576985056433928854337658342596750653715005333514489908293887737352051459333049626531415141386124437935885070944688045486975358170212908490787347806814366323322819415827345671356443171537967818058195852464840084032909981943781718177302317003989733050495387356116261023999433259780126893432605584710278764901070923443884634011735556865903585244919370181041626208504299258697435817098133894045934471937493877624232409852832762266604942385129709453245586252103600829286649724174919141988966129558076770979594795306013119159011773943104209049079424448868513086844493705909026006120649425744710353547657859242708130410618546219881830090634588187038755856274911587375421064667951346487586771543838018521348281915812462599335160198935595167968932852205824799421034512715877163345222995418839680448835529753361286837225935390079201666941339091168758803988828869216002373257361588207163516271332810518187602104852180675526648673908900907195138058626735124312215691637902277328705410842037841525683288718046987952513073266340278519059417338920358540395677035611329354482585628287610610698229721420961993509331312171187891078766872044548876089410174798647137882462153955933333275562009439580434537919782280590395959927436913793778664940964048777841748336432684026282932406260081908081804390914556351936856063045089142289645219987798849347477729132797266027658401667890136490508741142126861969862044126965282981087045479861559545338021201155646979976785738920186243599326777689454060508218838227909833627167124490026761178498264377033002081844590009717235204331994708242098771514449751017055643029542821819670009202515615844174205933658148134902693111517093872260026458630561325605792560927332265579346280805683443921373688405650434307396574061017779370141424615493070741360805442100295600095663588977899267630517718781943706761498217564186590116160865408635391513039201316805769034172596453692350806417446562351523929050409479953184074862151210561833854566176652606393713658802521666223576132201941701372664966073252010771947931265282763302413805164907174565964853748354669194523580315301969160480994606814904037819829732360930087135760798621425422096419004367905479049930078372421581954535418371129368658430553842717628035279128821129308351575656599944741788438381565148434229858704245592434693295232821803508333726283791830216591836181554217157448465778420134329982594566884558266171979012180849480332448787258183774805522268151011371745368417870280274452442905474518234674919564188551244421337783521423865979925988203287085109338386829906571994614906290257427686038850511032638544540419184958866538545040571323629681069146814847869659166861842756798460041868762298055562963045953227923051616721591968675849523635298935788507746081537321454642984792310511676357749494622952569497660359473962430995343310404994209677883827002714478494069037073249106444151696053256560586778757417472110827435774315194060757983563629143326397812218946287447798119807225646714664054850131009656786314880090303749338875364183165134982546694673316118123364854397649325026179549357204305402182974871251107404011611405899911093062492312813116340549262571356721818628932786138833718028535056503591952741400869510926167541476792668032109237467087213606278332922386413619594121339278036118276324106004740971111048140003623342714514483334641675466354699731494756643423659493496845884551524150756376605086632827424794136062876041290644913828519456402643153225858624043141838669590633245063000392213192647625962691510904457695301444054618037857503036686212462278639752746667870121003392984873375014475600322100622358029343774955032037012738468163061026570300872275462966796880890587127676361066225722352229739206443093524327228100859973095132528630601105497915644791845004618046762408928925680912930592960642357021061524646205023248966593987324933967376952023991760898474571843531936646529125848064480196520162838795189499336759241485626136995945307287254532463291529110128763770605570609531377527751867923292134955245133089867969165129073841302167573238637575820080363575728002754490327953079900799442541108725693188014667935595834676432868876966610097395749967836593397846346959948950610490383647409504695226063858046758073069912290474089879166872117147527644711604401952718169508289733537148530928937046384420893299771125856840846608339934045689026787516008775461267988015465856522061210953490796707365539702576199431376639960606061106406959330828171876426043573425361756943784848495250108266488395159700490598380812105221111091943323951136051446459834210799058082093716464523127704023160072138543723461267260997870385657091998507595634613248460188409850194287687902268734556500519121546544063829253851276317663922050938345204300773017029940362615434001322763910912988327863920412300445551684054889809080779174636092439334912641164240093880746356607262336695842764583698268734815881961058571835767462009650526065929263548291499045768307210893245857073701660717398194485028842603963660746031184786225831056580870870305567595861341700745402965687634774176431051751036732869245558582082372038601781739405175130437994868822320044378043103170921034261674998000073016094814586374488778522273076330495383944345382770608760763542098445008306247630253572781032783461766970544287155315340016497076657195985041748199087201490875686037783591994719343352772947285537925787684832301101859365800717291186967617655053775030293033830706448912811412025506150896411007623824574488655182581058140345320124754723269087547507078577659732542844459353044992070014538748948226556442223696365544194225441338212225477497535494624827680533336983284156138692363443358553868471111430498248398991803165458638289353799130535222833430137953372954016257623228081138499491876144141322933767106563492528814528239506209022357876684650116660097382753660405446941653422239052108314585847035529352219928272760574821266065291385530345549744551470344939486863429459658431024190785923680224560763936784166270518555178702904073557304620639692453307795782245949710420188043000183881429008173039450507342787013124466860092778581811040911511729374873627887874907465285565434748886831064110051023020875107768918781525622735251550379532444857787277617001964853703555167655209119339343762866284619844026295252183678522367475108809781507098978413086245881522660963551401874495836926917799047120726494905737264286005211403581231076006699518536124862746756375896225299116496066876508261734178484789337295056739007878617925351440621045366250640463728815698232317500596261080921955211150859302955654967538862612972339914628358476048627627027309739202001432248707582337354915246085608210328882974183906478869923273691360048837436615223517058437705545210815513361262142911815615301758882573594892507108879262128641392443309383797333867806131795237315266773820858024701433527009243803266951742119507670884326346442749127558907746863582162166042741315170212458586056233631493164646913946562497471741958354218607748711057338458433689939645913740603382159352243594751626239188685307822821763983237306180204246560477527943104796189724299533029792497481684052893791044947004590864991872727345413508101983881864673609392571930511968645601855782450218231065889437986522432050677379966196955472440585922417953006820451795370043472451762893566770508490213107736625751697335527462302943031203596260953423574397249659211010657817826108745318874803187430823573699195156340957162700992444929749105489851519658664740148225106335367949737142510229341882585117371994499115097583746130105505064197721531929354875371191630262030328588658528480193509225875775597425276584011721342323648084027143356367542046375182552524944329657043861387865901965738802868401894087672816714137033661732650120578653915780703088714261519075001492576112927675193096728453971160213606303090542243966320674323582797889332324405779199278484633339777737655901870574806828678347965624146102899508487399692970750432753029972872297327934442988646412725348160603779707298299173029296308695801996312413304939350493325412355071054461182591141116454534710329881047844067780138077131465400099386306481266614330858206811395838319169545558259426895769841428893743467084107946318932539106963955780706021245974898293564613560788983472419979478564362042094613412387613198865352358312996862268948608408456655606876954501274486631405054735351746873009806322780468912246821460806727627708402402266155485024008952891657117617439020337584877842911289623247059191874691042005848326140677333751027195653994697162517248312230633919328707983800748485726516123434933273356664473358556430235280883924348278760886164943289399166399210488307847777048045728491456303353265070029588906265915498509407972767567129795010098229476228961891591441520032283878773485130979081019129267227103778898053964156362364169154985768408398468861684375407065121039062506128107663799047908879674778069738473170475253442156390387201238806323688037017949308954900776331523063548374256816653361606641980030188287123767481898330246836371488309259283375902278942588060087286038859168849730693948020511221766359138251524278670094406942355120201568377778851824670025651708509249623747726813694284350062938814429987905301056217375459182679973217735029368928065210025396268807498092643458011655715886700443503976505323478287327368840863540002740676783821963522226539290939807367391364082898722017776747168118195856133721583119054682936083236976113450281757830202934845982925000895682630271263295866292147653142233351793093387951357095346377183684092444422096319331295620305575517340067973740614162107923633423805646850092037167152642556371853889571416419772387422610596667396997173168169415435095283193556417705668622215217991151355639707143312893657553844648326201206424338016955862698561022460646069330793847858814367407000599769703649019273328826135329363112403650698652160638987250267238087403396744397830258296894256896741864336134979475245526291426522842419243083388103580053787023999542172113686550275341362211693140694669513186928102574795985605145005021715913317751609957865551981886193211282110709442287240442481153406055895958355815232012184605820563592699303478851132068626627588771446035996656108430725696500563064489187599466596772847171539573612108180841547273142661748933134174632662354222072600146012701206934639520564445543291662986660783089068118790090815295063626782075614388815781351134695366303878412092346942868730839320432333872775496805210302821544324723388845215343727250128589747691460808314404125868181540049187772287869801853454537006526655649170915429522756709222217474112062720656622989806032891672068743654948246108697367225547404812889242471854323605753411672850757552057131156697954584887398742228135887985840783135060548290551482785294891121905383195624228719484759407859398047901094194070671764439032730712135887385049993638838205501683402777496070276844880281912220636888636811043569529300652195528261526991271637277388418993287130563464688227398288763198645709836308917786487086676185485680047672552675414742851028145807403152992197814557756843681110185317498167016426647884090262682824448258027532094549915104518517716546311804904567985713257528117913656278158111288816562285876030875974963849435275676612168959261485030785362045274507752950631012480341804584059432926079854435620093708091821523920371790678121992280496069738238743312626730306795943960954957189577217915597300588693646845576676092450906088202212235719254536715191834872587423919410890444115959932760044506556206461164655665487594247369252336955993030355095817626176231849561906494839673002037763874369343999829430209147073618947932692762445186560239559053705128978163455423320114975994896278424327483788032701418676952621180975006405149755889650293004867605208010491537885413909424531691719987628941277221129464568294860281493181560249677887949813777216229359437811004448060797672429276249510784153446429150842764520002042769470698041775832209097020291657347251582904630910359037842977572651720877244740952267166306005469716387943171196873484688738186656751279298575016363411314627530499019135646823804329970695770150789337728658035712790913767420805655493624646 diff --git a/lib/std/compress/deflate/token.zig b/lib/std/compress/deflate/token.zig deleted file mode 100644 index 744fcdeb12ea..000000000000 --- a/lib/std/compress/deflate/token.zig +++ /dev/null @@ -1,103 +0,0 @@ -// 2 bits: type, can be 0 (literal), 1 (EOF), 2 (Match) or 3 (Unused). -// 8 bits: xlength (length - MIN_MATCH_LENGTH). -// 22 bits: xoffset (offset - MIN_OFFSET_SIZE), or literal. -const length_shift = 22; -const offset_mask = (1 << length_shift) - 1; // 4_194_303 -const literal_type = 0 << 30; // 0 -pub const match_type = 1 << 30; // 1_073_741_824 - -// The length code for length X (MIN_MATCH_LENGTH <= X <= MAX_MATCH_LENGTH) -// is length_codes[length - MIN_MATCH_LENGTH] -var length_codes = [_]u32{ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, - 9, 9, 10, 10, 11, 11, 12, 12, 12, 12, - 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, - 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, - 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, - 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, - 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 21, 21, 21, 21, 22, 22, 22, 22, - 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, - 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, - 23, 23, 23, 23, 23, 23, 23, 23, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 27, 27, 27, 27, 27, 27, - 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, - 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, - 27, 27, 27, 27, 27, 28, -}; - -var offset_codes = [_]u32{ - 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, - 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, - 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, - 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, - 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, - 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, - 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, - 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, - 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, - 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, - 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, - 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, - 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, - 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, -}; - -pub const Token = u32; - -// Convert a literal into a literal token. -pub fn literalToken(lit: u32) Token { - return literal_type + lit; -} - -// Convert a < xlength, xoffset > pair into a match token. -pub fn matchToken(xlength: u32, xoffset: u32) Token { - return match_type + (xlength << length_shift) + xoffset; -} - -// Returns the literal of a literal token -pub fn literal(t: Token) u32 { - return @as(u32, @intCast(t - literal_type)); -} - -// Returns the extra offset of a match token -pub fn offset(t: Token) u32 { - return @as(u32, @intCast(t)) & offset_mask; -} - -pub fn length(t: Token) u32 { - return @as(u32, @intCast((t - match_type) >> length_shift)); -} - -pub fn lengthCode(len: u32) u32 { - return length_codes[len]; -} - -// Returns the offset code corresponding to a specific offset -pub fn offsetCode(off: u32) u32 { - if (off < @as(u32, @intCast(offset_codes.len))) { - return offset_codes[off]; - } - if (off >> 7 < @as(u32, @intCast(offset_codes.len))) { - return offset_codes[off >> 7] + 14; - } - return offset_codes[off >> 14] + 28; -} - -test { - const std = @import("std"); - try std.testing.expectEqual(@as(Token, 3_401_581_099), matchToken(555, 555)); -} diff --git a/lib/std/compress/flate.zig b/lib/std/compress/flate.zig new file mode 100644 index 000000000000..1c6964eac12a --- /dev/null +++ b/lib/std/compress/flate.zig @@ -0,0 +1,481 @@ +/// Deflate is a lossless data compression file format that uses a combination +/// of LZ77 and Huffman coding. +pub const deflate = @import("flate/deflate.zig"); + +/// Inflate is the decoding process that takes a Deflate bitstream for +/// decompression and correctly produces the original full-size data or file. +pub const inflate = @import("flate/inflate.zig"); + +/// Decompress compressed data from reader and write plain data to the writer. +pub fn decompress(reader: anytype, writer: anytype) !void { + try inflate.decompress(.raw, reader, writer); +} + +/// Decompressor type +pub fn Decompressor(comptime ReaderType: type) type { + return inflate.Inflate(.raw, ReaderType); +} + +/// Create Decompressor which will read compressed data from reader. +pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) { + return inflate.decompressor(.raw, reader); +} + +/// Compression level, trades between speed and compression size. +pub const Options = deflate.Options; + +/// Compress plain data from reader and write compressed data to the writer. +pub fn compress(reader: anytype, writer: anytype, options: Options) !void { + try deflate.compress(.raw, reader, writer, options); +} + +/// Compressor type +pub fn Compressor(comptime WriterType: type) type { + return deflate.Compressor(.raw, WriterType); +} + +/// Create Compressor which outputs compressed data to the writer. +pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) { + return try deflate.compressor(.raw, writer, options); +} + +/// Huffman only compression. Without Lempel-Ziv match searching. Faster +/// compression, less memory requirements but bigger compressed sizes. +pub const huffman = struct { + pub fn compress(reader: anytype, writer: anytype) !void { + try deflate.huffman.compress(.raw, reader, writer); + } + + pub fn Compressor(comptime WriterType: type) type { + return deflate.huffman.Compressor(.raw, WriterType); + } + + pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) { + return deflate.huffman.compressor(.raw, writer); + } +}; + +// No compression store only. Compressed size is slightly bigger than plain. +pub const store = struct { + pub fn compress(reader: anytype, writer: anytype) !void { + try deflate.store.compress(.raw, reader, writer); + } + + pub fn Compressor(comptime WriterType: type) type { + return deflate.store.Compressor(.raw, WriterType); + } + + pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) { + return deflate.store.compressor(.raw, writer); + } +}; + +/// Container defines header/footer arround deflate bit stream. Gzip and zlib +/// compression algorithms are containers arround deflate bit stream body. +const Container = @import("flate/container.zig").Container; +const std = @import("std"); +const testing = std.testing; +const fixedBufferStream = std.io.fixedBufferStream; +const print = std.debug.print; +const builtin = @import("builtin"); + +test "flate" { + _ = @import("flate/deflate.zig"); + _ = @import("flate/inflate.zig"); +} + +test "flate compress/decompress" { + if (builtin.target.cpu.arch == .wasm32) return error.SkipZigTest; + + var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer + var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer + + const levels = [_]deflate.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 }; + const cases = [_]struct { + data: []const u8, // uncompressed content + // compressed data sizes per level 4-9 + gzip_sizes: [levels.len]usize = [_]usize{0} ** levels.len, + huffman_only_size: usize = 0, + store_size: usize = 0, + }{ + .{ + .data = @embedFile("flate/testdata/rfc1951.txt"), + .gzip_sizes = [_]usize{ 11513, 11217, 11139, 11126, 11122, 11119 }, + .huffman_only_size = 20287, + .store_size = 36967, + }, + .{ + .data = @embedFile("flate/testdata/fuzz/roundtrip1.input"), + .gzip_sizes = [_]usize{ 373, 370, 370, 370, 370, 370 }, + .huffman_only_size = 393, + .store_size = 393, + }, + .{ + .data = @embedFile("flate/testdata/fuzz/roundtrip2.input"), + .gzip_sizes = [_]usize{ 373, 373, 373, 373, 373, 373 }, + .huffman_only_size = 394, + .store_size = 394, + }, + .{ + .data = @embedFile("flate/testdata/fuzz/deflate-stream.expect"), + .gzip_sizes = [_]usize{ 351, 347, 347, 347, 347, 347 }, + .huffman_only_size = 498, + .store_size = 747, + }, + }; + + for (cases, 0..) |case, case_no| { // for each case + const data = case.data; + + for (levels, 0..) |level, i| { // for each compression level + + inline for (Container.list) |container| { // for each wrapping + var compressed_size: usize = if (case.gzip_sizes[i] > 0) + case.gzip_sizes[i] - Container.gzip.size() + container.size() + else + 0; + + // compress original stream to compressed stream + { + var original = fixedBufferStream(data); + var compressed = fixedBufferStream(&cmp_buf); + try deflate.compress(container, original.reader(), compressed.writer(), .{ .level = level }); + if (compressed_size == 0) { + if (container == .gzip) + print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos }); + compressed_size = compressed.pos; + } + try testing.expectEqual(compressed_size, compressed.pos); + } + // decompress compressed stream to decompressed stream + { + var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); + var decompressed = fixedBufferStream(&dcm_buf); + try inflate.decompress(container, compressed.reader(), decompressed.writer()); + try testing.expectEqualSlices(u8, data, decompressed.getWritten()); + } + + // compressor writer interface + { + var compressed = fixedBufferStream(&cmp_buf); + var cmp = try deflate.compressor(container, compressed.writer(), .{ .level = level }); + var cmp_wrt = cmp.writer(); + try cmp_wrt.writeAll(data); + try cmp.finish(); + + try testing.expectEqual(compressed_size, compressed.pos); + } + // decompressor reader interface + { + var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); + var dcm = inflate.decompressor(container, compressed.reader()); + var dcm_rdr = dcm.reader(); + const n = try dcm_rdr.readAll(&dcm_buf); + try testing.expectEqual(data.len, n); + try testing.expectEqualSlices(u8, data, dcm_buf[0..n]); + } + } + } + // huffman only compression + { + inline for (Container.list) |container| { // for each wrapping + var compressed_size: usize = if (case.huffman_only_size > 0) + case.huffman_only_size - Container.gzip.size() + container.size() + else + 0; + + // compress original stream to compressed stream + { + var original = fixedBufferStream(data); + var compressed = fixedBufferStream(&cmp_buf); + var cmp = try deflate.huffman.compressor(container, compressed.writer()); + try cmp.compress(original.reader()); + try cmp.finish(); + if (compressed_size == 0) { + if (container == .gzip) + print("case {d} huffman only compressed size: {d}\n", .{ case_no, compressed.pos }); + compressed_size = compressed.pos; + } + try testing.expectEqual(compressed_size, compressed.pos); + } + // decompress compressed stream to decompressed stream + { + var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); + var decompressed = fixedBufferStream(&dcm_buf); + try inflate.decompress(container, compressed.reader(), decompressed.writer()); + try testing.expectEqualSlices(u8, data, decompressed.getWritten()); + } + } + } + + // store only + { + inline for (Container.list) |container| { // for each wrapping + var compressed_size: usize = if (case.store_size > 0) + case.store_size - Container.gzip.size() + container.size() + else + 0; + + // compress original stream to compressed stream + { + var original = fixedBufferStream(data); + var compressed = fixedBufferStream(&cmp_buf); + var cmp = try deflate.store.compressor(container, compressed.writer()); + try cmp.compress(original.reader()); + try cmp.finish(); + if (compressed_size == 0) { + if (container == .gzip) + print("case {d} store only compressed size: {d}\n", .{ case_no, compressed.pos }); + compressed_size = compressed.pos; + } + + try testing.expectEqual(compressed_size, compressed.pos); + } + // decompress compressed stream to decompressed stream + { + var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); + var decompressed = fixedBufferStream(&dcm_buf); + try inflate.decompress(container, compressed.reader(), decompressed.writer()); + try testing.expectEqualSlices(u8, data, decompressed.getWritten()); + } + } + } + } +} + +fn testDecompress(comptime container: Container, compressed: []const u8, expected_plain: []const u8) !void { + var in = fixedBufferStream(compressed); + var out = std.ArrayList(u8).init(testing.allocator); + defer out.deinit(); + + try inflate.decompress(container, in.reader(), out.writer()); + try testing.expectEqualSlices(u8, expected_plain, out.items); +} + +test "flate don't read past deflate stream's end" { + try testDecompress(.zlib, &[_]u8{ + 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff, + 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01, + 0x83, 0x95, 0x0b, 0xf5, + }, &[_]u8{ + 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, + 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, + }); +} + +test "flate zlib header" { + // Truncated header + try testing.expectError( + error.EndOfStream, + testDecompress(.zlib, &[_]u8{0x78}, ""), + ); + // Wrong CM + try testing.expectError( + error.BadZlibHeader, + testDecompress(.zlib, &[_]u8{ 0x79, 0x94 }, ""), + ); + // Wrong CINFO + try testing.expectError( + error.BadZlibHeader, + testDecompress(.zlib, &[_]u8{ 0x88, 0x98 }, ""), + ); + // Wrong checksum + try testing.expectError( + error.WrongZlibChecksum, + testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""), + ); + // Truncated checksum + try testing.expectError( + error.EndOfStream, + testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""), + ); +} + +test "flate gzip header" { + // Truncated header + try testing.expectError( + error.EndOfStream, + testDecompress(.gzip, &[_]u8{ 0x1f, 0x8B }, undefined), + ); + // Wrong CM + try testing.expectError( + error.BadGzipHeader, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, + }, undefined), + ); + + // Wrong checksum + try testing.expectError( + error.WrongGzipChecksum, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, + }, undefined), + ); + // Truncated checksum + try testing.expectError( + error.EndOfStream, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, + }, undefined), + ); + // Wrong initial size + try testing.expectError( + error.WrongGzipSize, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, + }, undefined), + ); + // Truncated initial size field + try testing.expectError( + error.EndOfStream, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, + }, undefined), + ); + + try testDecompress(.gzip, &[_]u8{ + // GZIP header + 0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00, + // header.FHCRC (should cover entire header) + 0x99, 0xd6, + // GZIP data + 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, ""); +} + +test "flate public interface" { + if (builtin.target.cpu.arch == .wasm32) return error.SkipZigTest; + + const plain_data = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a }; + + // deflate final stored block, header + plain (stored) data + const deflate_block = [_]u8{ + 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen + } ++ plain_data; + + // gzip header/footer + deflate block + const gzip_data = + [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes) + deflate_block ++ + [_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes) + + // zlib header/footer + deflate block + const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)} + deflate_block ++ + [_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum + + const gzip = @import("gzip.zig"); + const zlib = @import("zlib.zig"); + const flate = @This(); + + try testInterface(gzip, &gzip_data, &plain_data); + try testInterface(zlib, &zlib_data, &plain_data); + try testInterface(flate, &deflate_block, &plain_data); +} + +fn testInterface(comptime pkg: type, gzip_data: []const u8, plain_data: []const u8) !void { + var buffer1: [64]u8 = undefined; + var buffer2: [64]u8 = undefined; + + var compressed = fixedBufferStream(&buffer1); + var plain = fixedBufferStream(&buffer2); + + // decompress + { + var in = fixedBufferStream(gzip_data); + try pkg.decompress(in.reader(), plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + plain.reset(); + compressed.reset(); + + // compress/decompress + { + var in = fixedBufferStream(plain_data); + try pkg.compress(in.reader(), compressed.writer(), .{}); + compressed.reset(); + try pkg.decompress(compressed.reader(), plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + plain.reset(); + compressed.reset(); + + // compressor/decompressor + { + var in = fixedBufferStream(plain_data); + var cmp = try pkg.compressor(compressed.writer(), .{}); + try cmp.compress(in.reader()); + try cmp.finish(); + + compressed.reset(); + var dcp = pkg.decompressor(compressed.reader()); + try dcp.decompress(plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + plain.reset(); + compressed.reset(); + + // huffman + { + // huffman compress/decompress + { + var in = fixedBufferStream(plain_data); + try pkg.huffman.compress(in.reader(), compressed.writer()); + compressed.reset(); + try pkg.decompress(compressed.reader(), plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + plain.reset(); + compressed.reset(); + + // huffman compressor/decompressor + { + var in = fixedBufferStream(plain_data); + var cmp = try pkg.huffman.compressor(compressed.writer()); + try cmp.compress(in.reader()); + try cmp.finish(); + + compressed.reset(); + try pkg.decompress(compressed.reader(), plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + } + plain.reset(); + compressed.reset(); + + // store + { + // store compress/decompress + { + var in = fixedBufferStream(plain_data); + try pkg.store.compress(in.reader(), compressed.writer()); + compressed.reset(); + try pkg.decompress(compressed.reader(), plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + plain.reset(); + compressed.reset(); + + // store compressor/decompressor + { + var in = fixedBufferStream(plain_data); + var cmp = try pkg.store.compressor(compressed.writer()); + try cmp.compress(in.reader()); + try cmp.finish(); + + compressed.reset(); + try pkg.decompress(compressed.reader(), plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + } +} diff --git a/lib/std/compress/flate/CircularBuffer.zig b/lib/std/compress/flate/CircularBuffer.zig new file mode 100644 index 000000000000..f25a231ee40d --- /dev/null +++ b/lib/std/compress/flate/CircularBuffer.zig @@ -0,0 +1,234 @@ +//! 64K buffer of uncompressed data created in inflate (decompression). Has enough +//! history to support writing match; copying length of bytes +//! from the position distance backward from current. +//! +//! Reads can return less than available bytes if they are spread across +//! different circles. So reads should repeat until get required number of bytes +//! or until returned slice is zero length. +//! +//! Note on deflate limits: +//! * non-compressible block is limited to 65,535 bytes. +//! * backward pointer is limited in distance to 32K bytes and in length to 258 bytes. +//! +//! Whole non-compressed block can be written without overlap. We always have +//! history of up to 64K, more then 32K needed. +//! +const std = @import("std"); +const assert = std.debug.assert; +const testing = std.testing; + +const consts = @import("consts.zig").match; + +const mask = 0xffff; // 64K - 1 +const buffer_len = mask + 1; // 64K buffer + +const Self = @This(); + +buffer: [buffer_len]u8 = undefined, +wp: usize = 0, // write position +rp: usize = 0, // read position + +fn writeAll(self: *Self, buf: []const u8) void { + for (buf) |c| self.write(c); +} + +/// Write literal. +pub fn write(self: *Self, b: u8) void { + assert(self.wp - self.rp < mask); + self.buffer[self.wp & mask] = b; + self.wp += 1; +} + +/// Write match (back-reference to the same data slice) starting at `distance` +/// back from current write position, and `length` of bytes. +pub fn writeMatch(self: *Self, length: u16, distance: u16) !void { + if (self.wp < distance or + length < consts.base_length or length > consts.max_length or + distance < consts.min_distance or distance > consts.max_distance) + { + return error.InvalidMatch; + } + assert(self.wp - self.rp < mask); + + var from: usize = self.wp - distance; + const from_end: usize = from + length; + var to: usize = self.wp; + const to_end: usize = to + length; + + self.wp += length; + + // Fast path using memcpy + if (length <= distance and // no overlapping buffers + (from >> 16 == from_end >> 16) and // start and and at the same circle + (to >> 16 == to_end >> 16)) + { + @memcpy(self.buffer[to & mask .. to_end & mask], self.buffer[from & mask .. from_end & mask]); + return; + } + + // Slow byte by byte + while (to < to_end) { + self.buffer[to & mask] = self.buffer[from & mask]; + to += 1; + from += 1; + } +} + +/// Returns writable part of the internal buffer of size `n` at most. Advances +/// write pointer, assumes that returned buffer will be filled with data. +pub fn getWritable(self: *Self, n: usize) []u8 { + const wp = self.wp & mask; + const len = @min(n, buffer_len - wp); + self.wp += len; + return self.buffer[wp .. wp + len]; +} + +/// Read available data. Can return part of the available data if it is +/// spread across two circles. So read until this returns zero length. +pub fn read(self: *Self) []const u8 { + return self.readAtMost(buffer_len); +} + +/// Read part of available data. Can return less than max even if there are +/// more than max decoded data. +pub fn readAtMost(self: *Self, limit: usize) []const u8 { + const rb = self.readBlock(if (limit == 0) buffer_len else limit); + defer self.rp += rb.len; + return self.buffer[rb.head..rb.tail]; +} + +const ReadBlock = struct { + head: usize, + tail: usize, + len: usize, +}; + +/// Returns position of continous read block data. +fn readBlock(self: *Self, max: usize) ReadBlock { + const r = self.rp & mask; + const w = self.wp & mask; + const n = @min( + max, + if (w >= r) w - r else buffer_len - r, + ); + return .{ + .head = r, + .tail = r + n, + .len = n, + }; +} + +/// Number of free bytes for write. +pub fn free(self: *Self) usize { + return buffer_len - (self.wp - self.rp); +} + +/// Full if largest match can't fit. 258 is largest match length. That much +/// bytes can be produced in single decode step. +pub fn full(self: *Self) bool { + return self.free() < 258 + 1; +} + +// example from: https://youtu.be/SJPvNi4HrWQ?t=3558 +test "flate.CircularBuffer writeMatch" { + var cb: Self = .{}; + + cb.writeAll("a salad; "); + try cb.writeMatch(5, 9); + try cb.writeMatch(3, 3); + + try testing.expectEqualStrings("a salad; a salsal", cb.read()); +} + +test "flate.CircularBuffer writeMatch overlap" { + var cb: Self = .{}; + + cb.writeAll("a b c "); + try cb.writeMatch(8, 4); + cb.write('d'); + + try testing.expectEqualStrings("a b c b c b c d", cb.read()); +} + +test "flate.CircularBuffer readAtMost" { + var cb: Self = .{}; + + cb.writeAll("0123456789"); + try cb.writeMatch(50, 10); + + try testing.expectEqualStrings("0123456789" ** 6, cb.buffer[cb.rp..cb.wp]); + for (0..6) |i| { + try testing.expectEqual(i * 10, cb.rp); + try testing.expectEqualStrings("0123456789", cb.readAtMost(10)); + } + try testing.expectEqualStrings("", cb.readAtMost(10)); + try testing.expectEqualStrings("", cb.read()); +} + +test "flate.CircularBuffer" { + var cb: Self = .{}; + + const data = "0123456789abcdef" ** (1024 / 16); + cb.writeAll(data); + try testing.expectEqual(@as(usize, 0), cb.rp); + try testing.expectEqual(@as(usize, 1024), cb.wp); + try testing.expectEqual(@as(usize, 1024 * 63), cb.free()); + + for (0..62 * 4) |_| + try cb.writeMatch(256, 1024); // write 62K + + try testing.expectEqual(@as(usize, 0), cb.rp); + try testing.expectEqual(@as(usize, 63 * 1024), cb.wp); + try testing.expectEqual(@as(usize, 1024), cb.free()); + + cb.writeAll(data[0..200]); + _ = cb.readAtMost(1024); // make some space + cb.writeAll(data); // overflows write position + try testing.expectEqual(@as(usize, 200 + 65536), cb.wp); + try testing.expectEqual(@as(usize, 1024), cb.rp); + try testing.expectEqual(@as(usize, 1024 - 200), cb.free()); + + const rb = cb.readBlock(Self.buffer_len); + try testing.expectEqual(@as(usize, 65536 - 1024), rb.len); + try testing.expectEqual(@as(usize, 1024), rb.head); + try testing.expectEqual(@as(usize, 65536), rb.tail); + + try testing.expectEqual(@as(usize, 65536 - 1024), cb.read().len); // read to the end of the buffer + try testing.expectEqual(@as(usize, 200 + 65536), cb.wp); + try testing.expectEqual(@as(usize, 65536), cb.rp); + try testing.expectEqual(@as(usize, 65536 - 200), cb.free()); + + try testing.expectEqual(@as(usize, 200), cb.read().len); // read the rest +} + +test "flate.CircularBuffer write overlap" { + var cb: Self = .{}; + cb.wp = cb.buffer.len - 15; + cb.rp = cb.wp; + + cb.writeAll("0123456789"); + cb.writeAll("abcdefghij"); + + try testing.expectEqual(cb.buffer.len + 5, cb.wp); + try testing.expectEqual(cb.buffer.len - 15, cb.rp); + + try testing.expectEqualStrings("0123456789abcde", cb.read()); + try testing.expectEqualStrings("fghij", cb.read()); + + try testing.expect(cb.wp == cb.rp); +} + +test "flate.CircularBuffer writeMatch/read overlap" { + var cb: Self = .{}; + cb.wp = cb.buffer.len - 15; + cb.rp = cb.wp; + + cb.writeAll("0123456789"); + try cb.writeMatch(15, 5); + + try testing.expectEqualStrings("012345678956789", cb.read()); + try testing.expectEqualStrings("5678956789", cb.read()); + + try cb.writeMatch(20, 25); + try testing.expectEqualStrings("01234567895678956789", cb.read()); +} diff --git a/lib/std/compress/flate/Lookup.zig b/lib/std/compress/flate/Lookup.zig new file mode 100644 index 000000000000..b5d1fd3c97ac --- /dev/null +++ b/lib/std/compress/flate/Lookup.zig @@ -0,0 +1,125 @@ +/// Lookup of the previous locations for the same 4 byte data. Works on hash of +/// 4 bytes data. Head contains position of the first match for each hash. Chain +/// points to the previous position of the same hash given the current location. +/// +const std = @import("std"); +const testing = std.testing; +const expect = testing.expect; +const consts = @import("consts.zig"); + +const Self = @This(); + +const prime4 = 0x9E3779B1; // 4 bytes prime number 2654435761 +const chain_len = 2 * consts.history.len; + +// Maps hash => first position +head: [consts.lookup.len]u16 = [_]u16{0} ** consts.lookup.len, +// Maps position => previous positions for the same hash value +chain: [chain_len]u16 = [_]u16{0} ** (chain_len), + +// Calculates hash of the 4 bytes from data. +// Inserts `pos` position of that hash in the lookup tables. +// Returns previous location with the same hash value. +pub fn add(self: *Self, data: []const u8, pos: u16) u16 { + if (data.len < 4) return 0; + const h = hash(data[0..4]); + return self.set(h, pos); +} + +// Retruns previous location with the same hash value given the current +// position. +pub fn prev(self: *Self, pos: u16) u16 { + return self.chain[pos]; +} + +fn set(self: *Self, h: u32, pos: u16) u16 { + const p = self.head[h]; + self.head[h] = pos; + self.chain[pos] = p; + return p; +} + +// Slide all positions in head and chain for `n` +pub fn slide(self: *Self, n: u16) void { + for (&self.head) |*v| { + v.* -|= n; + } + var i: usize = 0; + while (i < n) : (i += 1) { + self.chain[i] = self.chain[i + n] -| n; + } +} + +// Add `len` 4 bytes hashes from `data` into lookup. +// Position of the first byte is `pos`. +pub fn bulkAdd(self: *Self, data: []const u8, len: u16, pos: u16) void { + if (len == 0 or data.len < consts.match.min_length) { + return; + } + var hb = + @as(u32, data[3]) | + @as(u32, data[2]) << 8 | + @as(u32, data[1]) << 16 | + @as(u32, data[0]) << 24; + _ = self.set(hashu(hb), pos); + + var i = pos; + for (4..@min(len + 3, data.len)) |j| { + hb = (hb << 8) | @as(u32, data[j]); + i += 1; + _ = self.set(hashu(hb), i); + } +} + +// Calculates hash of the first 4 bytes of `b`. +fn hash(b: *const [4]u8) u32 { + return hashu(@as(u32, b[3]) | + @as(u32, b[2]) << 8 | + @as(u32, b[1]) << 16 | + @as(u32, b[0]) << 24); +} + +fn hashu(v: u32) u32 { + return @intCast((v *% prime4) >> consts.lookup.shift); +} + +test "flate.Lookup add/prev" { + const data = [_]u8{ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x01, 0x02, 0x03, + }; + + var h: Self = .{}; + for (data, 0..) |_, i| { + const p = h.add(data[i..], @intCast(i)); + if (i >= 8 and i < 24) { + try expect(p == i - 8); + } else { + try expect(p == 0); + } + } + + const v = Self.hash(data[2 .. 2 + 4]); + try expect(h.head[v] == 2 + 16); + try expect(h.chain[2 + 16] == 2 + 8); + try expect(h.chain[2 + 8] == 2); +} + +test "flate.Lookup bulkAdd" { + const data = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; + + // one by one + var h: Self = .{}; + for (data, 0..) |_, i| { + _ = h.add(data[i..], @intCast(i)); + } + + // in bulk + var bh: Self = .{}; + bh.bulkAdd(data, data.len, 0); + + try testing.expectEqualSlices(u16, &h.head, &bh.head); + try testing.expectEqualSlices(u16, &h.chain, &bh.chain); +} diff --git a/lib/std/compress/flate/SlidingWindow.zig b/lib/std/compress/flate/SlidingWindow.zig new file mode 100644 index 000000000000..2e935fa52bde --- /dev/null +++ b/lib/std/compress/flate/SlidingWindow.zig @@ -0,0 +1,160 @@ +//! Used in deflate (compression), holds uncompressed data form which Tokens are +//! produces. In combination with Lookup it is used to find matches in history data. +//! +const std = @import("std"); +const consts = @import("consts.zig"); + +const expect = testing.expect; +const assert = std.debug.assert; +const testing = std.testing; + +const hist_len = consts.history.len; +const buffer_len = 2 * hist_len; +const min_lookahead = consts.match.min_length + consts.match.max_length; +const max_rp = buffer_len - min_lookahead; + +const Self = @This(); + +buffer: [buffer_len]u8 = undefined, +wp: usize = 0, // write position +rp: usize = 0, // read position +fp: isize = 0, // last flush position, tokens are build from fp..rp + +/// Returns number of bytes written, or 0 if buffer is full and need to slide. +pub fn write(self: *Self, buf: []const u8) usize { + if (self.rp >= max_rp) return 0; // need to slide + + const n = @min(buf.len, buffer_len - self.wp); + @memcpy(self.buffer[self.wp .. self.wp + n], buf[0..n]); + self.wp += n; + return n; +} + +/// Slide buffer for hist_len. +/// Drops old history, preserves between hist_len and hist_len - min_lookahead. +/// Returns number of bytes removed. +pub fn slide(self: *Self) u16 { + assert(self.rp >= max_rp and self.wp >= self.rp); + const n = self.wp - hist_len; + @memcpy(self.buffer[0..n], self.buffer[hist_len..self.wp]); + self.rp -= hist_len; + self.wp -= hist_len; + self.fp -= hist_len; + return @intCast(n); +} + +/// Data from the current position (read position). Those part of the buffer is +/// not converted to tokens yet. +fn lookahead(self: *Self) []const u8 { + assert(self.wp >= self.rp); + return self.buffer[self.rp..self.wp]; +} + +/// Returns part of the lookahead buffer. If should_flush is set no lookahead is +/// preserved otherwise preserves enough data for the longest match. Returns +/// null if there is not enough data. +pub fn activeLookahead(self: *Self, should_flush: bool) ?[]const u8 { + const min: usize = if (should_flush) 0 else min_lookahead; + const lh = self.lookahead(); + return if (lh.len > min) lh else null; +} + +/// Advances read position, shrinks lookahead. +pub fn advance(self: *Self, n: u16) void { + assert(self.wp >= self.rp + n); + self.rp += n; +} + +/// Returns writable part of the buffer, where new uncompressed data can be +/// written. +pub fn writable(self: *Self) []u8 { + return self.buffer[self.wp..]; +} + +/// Notification of what part of writable buffer is filled with data. +pub fn written(self: *Self, n: usize) void { + self.wp += n; +} + +/// Finds match length between previous and current position. +/// Used in hot path! +pub fn match(self: *Self, prev_pos: u16, curr_pos: u16, min_len: u16) u16 { + const max_len: usize = @min(self.wp - curr_pos, consts.match.max_length); + // lookahead buffers from previous and current positions + const prev_lh = self.buffer[prev_pos..][0..max_len]; + const curr_lh = self.buffer[curr_pos..][0..max_len]; + + // If we alread have match (min_len > 0), + // test the first byte above previous len a[min_len] != b[min_len] + // and then all the bytes from that position to zero. + // That is likely positions to find difference than looping from first bytes. + var i: usize = min_len; + if (i > 0) { + if (max_len <= i) return 0; + while (true) { + if (prev_lh[i] != curr_lh[i]) return 0; + if (i == 0) break; + i -= 1; + } + i = min_len; + } + while (i < max_len) : (i += 1) + if (prev_lh[i] != curr_lh[i]) break; + return if (i >= consts.match.min_length) @intCast(i) else 0; +} + +/// Current position of non-compressed data. Data before rp are already converted +/// to tokens. +pub fn pos(self: *Self) u16 { + return @intCast(self.rp); +} + +/// Notification that token list is cleared. +pub fn flush(self: *Self) void { + self.fp = @intCast(self.rp); +} + +/// Part of the buffer since last flush or null if there was slide in between (so +/// fp becomes negative). +pub fn tokensBuffer(self: *Self) ?[]const u8 { + assert(self.fp <= self.rp); + if (self.fp < 0) return null; + return self.buffer[@intCast(self.fp)..self.rp]; +} + +test "flate.SlidingWindow match" { + const data = "Blah blah blah blah blah!"; + var win: Self = .{}; + try expect(win.write(data) == data.len); + try expect(win.wp == data.len); + try expect(win.rp == 0); + + // length between l symbols + try expect(win.match(1, 6, 0) == 18); + try expect(win.match(1, 11, 0) == 13); + try expect(win.match(1, 16, 0) == 8); + try expect(win.match(1, 21, 0) == 0); + + // position 15 = "blah blah!" + // position 20 = "blah!" + try expect(win.match(15, 20, 0) == 4); + try expect(win.match(15, 20, 3) == 4); + try expect(win.match(15, 20, 4) == 0); +} + +test "flate.SlidingWindow slide" { + var win: Self = .{}; + win.wp = Self.buffer_len - 11; + win.rp = Self.buffer_len - 111; + win.buffer[win.rp] = 0xab; + try expect(win.lookahead().len == 100); + try expect(win.tokensBuffer().?.len == win.rp); + + const n = win.slide(); + try expect(n == 32757); + try expect(win.buffer[win.rp] == 0xab); + try expect(win.rp == Self.hist_len - 111); + try expect(win.wp == Self.hist_len - 11); + try expect(win.lookahead().len == 100); + try expect(win.tokensBuffer() == null); +} diff --git a/lib/std/compress/flate/Token.zig b/lib/std/compress/flate/Token.zig new file mode 100644 index 000000000000..7e067daf8316 --- /dev/null +++ b/lib/std/compress/flate/Token.zig @@ -0,0 +1,327 @@ +//! Token cat be literal: single byte of data or match; reference to the slice of +//! data in the same stream represented with . Where length +//! can be 3 - 258 bytes, and distance 1 - 32768 bytes. +//! +const std = @import("std"); +const assert = std.debug.assert; +const print = std.debug.print; +const expect = std.testing.expect; +const consts = @import("consts.zig").match; + +const Token = @This(); + +pub const Kind = enum(u1) { + literal, + match, +}; + +// Distance range 1 - 32768, stored in dist as 0 - 32767 (fits u15) +dist: u15 = 0, +// Length range 3 - 258, stored in len_lit as 0 - 255 (fits u8) +len_lit: u8 = 0, +kind: Kind = .literal, + +pub fn literal(t: Token) u8 { + return t.len_lit; +} + +pub fn distance(t: Token) u16 { + return @as(u16, t.dist) + consts.min_distance; +} + +pub fn length(t: Token) u16 { + return @as(u16, t.len_lit) + consts.base_length; +} + +pub fn initLiteral(lit: u8) Token { + return .{ .kind = .literal, .len_lit = lit }; +} + +// distance range 1 - 32768, stored in dist as 0 - 32767 (u15) +// length range 3 - 258, stored in len_lit as 0 - 255 (u8) +pub fn initMatch(dist: u16, len: u16) Token { + assert(len >= consts.min_length and len <= consts.max_length); + assert(dist >= consts.min_distance and dist <= consts.max_distance); + return .{ + .kind = .match, + .dist = @intCast(dist - consts.min_distance), + .len_lit = @intCast(len - consts.base_length), + }; +} + +pub fn eql(t: Token, o: Token) bool { + return t.kind == o.kind and + t.dist == o.dist and + t.len_lit == o.len_lit; +} + +pub fn lengthCode(t: Token) u16 { + return match_lengths[match_lengths_index[t.len_lit]].code; +} + +pub fn lengthEncoding(t: Token) MatchLength { + var c = match_lengths[match_lengths_index[t.len_lit]]; + c.extra_length = t.len_lit - c.base_scaled; + return c; +} + +// Returns the distance code corresponding to a specific distance. +// Distance code is in range: 0 - 29. +pub fn distanceCode(t: Token) u8 { + var dist: u16 = t.dist; + if (dist < match_distances_index.len) { + return match_distances_index[dist]; + } + dist >>= 7; + if (dist < match_distances_index.len) { + return match_distances_index[dist] + 14; + } + dist >>= 7; + return match_distances_index[dist] + 28; +} + +pub fn distanceEncoding(t: Token) MatchDistance { + var c = match_distances[t.distanceCode()]; + c.extra_distance = t.dist - c.base_scaled; + return c; +} + +pub fn lengthExtraBits(code: u32) u8 { + return match_lengths[code - length_codes_start].extra_bits; +} + +pub fn matchLength(code: u8) MatchLength { + return match_lengths[code]; +} + +pub fn matchDistance(code: u8) MatchDistance { + return match_distances[code]; +} + +pub fn distanceExtraBits(code: u32) u8 { + return match_distances[code].extra_bits; +} + +pub fn show(t: Token) void { + if (t.kind == .literal) { + print("L('{c}'), ", .{t.literal()}); + } else { + print("M({d}, {d}), ", .{ t.distance(), t.length() }); + } +} + +// Retruns index in match_lengths table for each length in range 0-255. +const match_lengths_index = [_]u8{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, + 9, 9, 10, 10, 11, 11, 12, 12, 12, 12, + 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, + 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, + 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, + 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, + 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 27, 27, 28, +}; + +const MatchLength = struct { + code: u16, + base_scaled: u8, // base - 3, scaled to fit into u8 (0-255), same as lit_len field in Token. + base: u16, // 3-258 + extra_length: u8 = 0, + extra_bits: u4, +}; + +// match_lengths represents table from rfc (https://datatracker.ietf.org/doc/html/rfc1951#page-12) +// +// Extra Extra Extra +// Code Bits Length(s) Code Bits Lengths Code Bits Length(s) +// ---- ---- ------ ---- ---- ------- ---- ---- ------- +// 257 0 3 267 1 15,16 277 4 67-82 +// 258 0 4 268 1 17,18 278 4 83-98 +// 259 0 5 269 2 19-22 279 4 99-114 +// 260 0 6 270 2 23-26 280 4 115-130 +// 261 0 7 271 2 27-30 281 5 131-162 +// 262 0 8 272 2 31-34 282 5 163-194 +// 263 0 9 273 3 35-42 283 5 195-226 +// 264 0 10 274 3 43-50 284 5 227-257 +// 265 1 11,12 275 3 51-58 285 0 258 +// 266 1 13,14 276 3 59-66 +// +pub const length_codes_start = 257; + +const match_lengths = [_]MatchLength{ + .{ .extra_bits = 0, .base_scaled = 0, .base = 3, .code = 257 }, + .{ .extra_bits = 0, .base_scaled = 1, .base = 4, .code = 258 }, + .{ .extra_bits = 0, .base_scaled = 2, .base = 5, .code = 259 }, + .{ .extra_bits = 0, .base_scaled = 3, .base = 6, .code = 260 }, + .{ .extra_bits = 0, .base_scaled = 4, .base = 7, .code = 261 }, + .{ .extra_bits = 0, .base_scaled = 5, .base = 8, .code = 262 }, + .{ .extra_bits = 0, .base_scaled = 6, .base = 9, .code = 263 }, + .{ .extra_bits = 0, .base_scaled = 7, .base = 10, .code = 264 }, + .{ .extra_bits = 1, .base_scaled = 8, .base = 11, .code = 265 }, + .{ .extra_bits = 1, .base_scaled = 10, .base = 13, .code = 266 }, + .{ .extra_bits = 1, .base_scaled = 12, .base = 15, .code = 267 }, + .{ .extra_bits = 1, .base_scaled = 14, .base = 17, .code = 268 }, + .{ .extra_bits = 2, .base_scaled = 16, .base = 19, .code = 269 }, + .{ .extra_bits = 2, .base_scaled = 20, .base = 23, .code = 270 }, + .{ .extra_bits = 2, .base_scaled = 24, .base = 27, .code = 271 }, + .{ .extra_bits = 2, .base_scaled = 28, .base = 31, .code = 272 }, + .{ .extra_bits = 3, .base_scaled = 32, .base = 35, .code = 273 }, + .{ .extra_bits = 3, .base_scaled = 40, .base = 43, .code = 274 }, + .{ .extra_bits = 3, .base_scaled = 48, .base = 51, .code = 275 }, + .{ .extra_bits = 3, .base_scaled = 56, .base = 59, .code = 276 }, + .{ .extra_bits = 4, .base_scaled = 64, .base = 67, .code = 277 }, + .{ .extra_bits = 4, .base_scaled = 80, .base = 83, .code = 278 }, + .{ .extra_bits = 4, .base_scaled = 96, .base = 99, .code = 279 }, + .{ .extra_bits = 4, .base_scaled = 112, .base = 115, .code = 280 }, + .{ .extra_bits = 5, .base_scaled = 128, .base = 131, .code = 281 }, + .{ .extra_bits = 5, .base_scaled = 160, .base = 163, .code = 282 }, + .{ .extra_bits = 5, .base_scaled = 192, .base = 195, .code = 283 }, + .{ .extra_bits = 5, .base_scaled = 224, .base = 227, .code = 284 }, + .{ .extra_bits = 0, .base_scaled = 255, .base = 258, .code = 285 }, +}; + +// Used in distanceCode fn to get index in match_distance table for each distance in range 0-32767. +const match_distances_index = [_]u8{ + 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, + 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, +}; + +const MatchDistance = struct { + base_scaled: u16, // base - 1, same as Token dist field + base: u16, + extra_distance: u16 = 0, + code: u8, + extra_bits: u4, +}; + +// match_distances represents table from rfc (https://datatracker.ietf.org/doc/html/rfc1951#page-12) +// +// Extra Extra Extra +// Code Bits Dist Code Bits Dist Code Bits Distance +// ---- ---- ---- ---- ---- ------ ---- ---- -------- +// 0 0 1 10 4 33-48 20 9 1025-1536 +// 1 0 2 11 4 49-64 21 9 1537-2048 +// 2 0 3 12 5 65-96 22 10 2049-3072 +// 3 0 4 13 5 97-128 23 10 3073-4096 +// 4 1 5,6 14 6 129-192 24 11 4097-6144 +// 5 1 7,8 15 6 193-256 25 11 6145-8192 +// 6 2 9-12 16 7 257-384 26 12 8193-12288 +// 7 2 13-16 17 7 385-512 27 12 12289-16384 +// 8 3 17-24 18 8 513-768 28 13 16385-24576 +// 9 3 25-32 19 8 769-1024 29 13 24577-32768 +// +const match_distances = [_]MatchDistance{ + .{ .extra_bits = 0, .base_scaled = 0x0000, .code = 0, .base = 1 }, + .{ .extra_bits = 0, .base_scaled = 0x0001, .code = 1, .base = 2 }, + .{ .extra_bits = 0, .base_scaled = 0x0002, .code = 2, .base = 3 }, + .{ .extra_bits = 0, .base_scaled = 0x0003, .code = 3, .base = 4 }, + .{ .extra_bits = 1, .base_scaled = 0x0004, .code = 4, .base = 5 }, + .{ .extra_bits = 1, .base_scaled = 0x0006, .code = 5, .base = 7 }, + .{ .extra_bits = 2, .base_scaled = 0x0008, .code = 6, .base = 9 }, + .{ .extra_bits = 2, .base_scaled = 0x000c, .code = 7, .base = 13 }, + .{ .extra_bits = 3, .base_scaled = 0x0010, .code = 8, .base = 17 }, + .{ .extra_bits = 3, .base_scaled = 0x0018, .code = 9, .base = 25 }, + .{ .extra_bits = 4, .base_scaled = 0x0020, .code = 10, .base = 33 }, + .{ .extra_bits = 4, .base_scaled = 0x0030, .code = 11, .base = 49 }, + .{ .extra_bits = 5, .base_scaled = 0x0040, .code = 12, .base = 65 }, + .{ .extra_bits = 5, .base_scaled = 0x0060, .code = 13, .base = 97 }, + .{ .extra_bits = 6, .base_scaled = 0x0080, .code = 14, .base = 129 }, + .{ .extra_bits = 6, .base_scaled = 0x00c0, .code = 15, .base = 193 }, + .{ .extra_bits = 7, .base_scaled = 0x0100, .code = 16, .base = 257 }, + .{ .extra_bits = 7, .base_scaled = 0x0180, .code = 17, .base = 385 }, + .{ .extra_bits = 8, .base_scaled = 0x0200, .code = 18, .base = 513 }, + .{ .extra_bits = 8, .base_scaled = 0x0300, .code = 19, .base = 769 }, + .{ .extra_bits = 9, .base_scaled = 0x0400, .code = 20, .base = 1025 }, + .{ .extra_bits = 9, .base_scaled = 0x0600, .code = 21, .base = 1537 }, + .{ .extra_bits = 10, .base_scaled = 0x0800, .code = 22, .base = 2049 }, + .{ .extra_bits = 10, .base_scaled = 0x0c00, .code = 23, .base = 3073 }, + .{ .extra_bits = 11, .base_scaled = 0x1000, .code = 24, .base = 4097 }, + .{ .extra_bits = 11, .base_scaled = 0x1800, .code = 25, .base = 6145 }, + .{ .extra_bits = 12, .base_scaled = 0x2000, .code = 26, .base = 8193 }, + .{ .extra_bits = 12, .base_scaled = 0x3000, .code = 27, .base = 12289 }, + .{ .extra_bits = 13, .base_scaled = 0x4000, .code = 28, .base = 16385 }, + .{ .extra_bits = 13, .base_scaled = 0x6000, .code = 29, .base = 24577 }, +}; + +test "flate.Token size" { + try expect(@sizeOf(Token) == 4); +} + +// testing table https://datatracker.ietf.org/doc/html/rfc1951#page-12 +test "flate.Token MatchLength" { + var c = Token.initMatch(1, 4).lengthEncoding(); + try expect(c.code == 258); + try expect(c.extra_bits == 0); + try expect(c.extra_length == 0); + + c = Token.initMatch(1, 11).lengthEncoding(); + try expect(c.code == 265); + try expect(c.extra_bits == 1); + try expect(c.extra_length == 0); + + c = Token.initMatch(1, 12).lengthEncoding(); + try expect(c.code == 265); + try expect(c.extra_bits == 1); + try expect(c.extra_length == 1); + + c = Token.initMatch(1, 130).lengthEncoding(); + try expect(c.code == 280); + try expect(c.extra_bits == 4); + try expect(c.extra_length == 130 - 115); +} + +test "flate.Token MatchDistance" { + var c = Token.initMatch(1, 4).distanceEncoding(); + try expect(c.code == 0); + try expect(c.extra_bits == 0); + try expect(c.extra_distance == 0); + + c = Token.initMatch(192, 4).distanceEncoding(); + try expect(c.code == 14); + try expect(c.extra_bits == 6); + try expect(c.extra_distance == 192 - 129); +} + +test "flate.Token match_lengths" { + for (match_lengths, 0..) |ml, i| { + try expect(@as(u16, ml.base_scaled) + 3 == ml.base); + try expect(i + 257 == ml.code); + } + + for (match_distances, 0..) |mo, i| { + try expect(mo.base_scaled + 1 == mo.base); + try expect(i == mo.code); + } +} diff --git a/lib/std/compress/flate/bit_reader.zig b/lib/std/compress/flate/bit_reader.zig new file mode 100644 index 000000000000..40e9b76fb8e5 --- /dev/null +++ b/lib/std/compress/flate/bit_reader.zig @@ -0,0 +1,333 @@ +const std = @import("std"); +const assert = std.debug.assert; +const testing = std.testing; + +pub fn bitReader(reader: anytype) BitReader(@TypeOf(reader)) { + return BitReader(@TypeOf(reader)).init(reader); +} + +/// Bit reader used during inflate (decompression). Has internal buffer of 64 +/// bits which shifts right after bits are consumed. Uses forward_reader to fill +/// that internal buffer when needed. +/// +/// readF is the core function. Supports few different ways of getting bits +/// controlled by flags. In hot path we try to avoid checking whether we need to +/// fill buffer from forward_reader by calling fill in advance and readF with +/// buffered flag set. +/// +pub fn BitReader(comptime ReaderType: type) type { + return struct { + // Underlying reader used for filling internal bits buffer + forward_reader: ReaderType = undefined, + // Internal buffer of 64 bits + bits: u64 = 0, + // Number of bits in the buffer + nbits: u32 = 0, + + const Self = @This(); + + pub const Error = ReaderType.Error || error{EndOfStream}; + + pub fn init(rdr: ReaderType) Self { + var self = Self{ .forward_reader = rdr }; + self.fill(1) catch {}; + return self; + } + + /// Try to have `nice` bits are available in buffer. Reads from + /// forward reader if there is no `nice` bits in buffer. Returns error + /// if end of forward stream is reached and internal buffer is empty. + /// It will not error if less than `nice` bits are in buffer, only when + /// all bits are exhausted. During inflate we usually know what is the + /// maximum bits for the next step but usually that step will need less + /// bits to decode. So `nice` is not hard limit, it will just try to have + /// that number of bits available. If end of forward stream is reached + /// it may be some extra zero bits in buffer. + pub inline fn fill(self: *Self, nice: u6) !void { + if (self.nbits >= nice) { + return; // We have enought bits + } + // Read more bits from forward reader + + // Number of empty bytes in bits, round nbits to whole bytes. + const empty_bytes = + @as(u8, if (self.nbits & 0x7 == 0) 8 else 7) - // 8 for 8, 16, 24..., 7 otherwise + (self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8 + + var buf: [8]u8 = [_]u8{0} ** 8; + const bytes_read = self.forward_reader.readAll(buf[0..empty_bytes]) catch 0; + if (bytes_read > 0) { + const u: u64 = std.mem.readInt(u64, buf[0..8], .little); + self.bits |= u << @as(u6, @intCast(self.nbits)); + self.nbits += 8 * @as(u8, @intCast(bytes_read)); + return; + } + + if (self.nbits == 0) + return error.EndOfStream; + } + + /// Read exactly buf.len bytes into buf. + pub fn readAll(self: *Self, buf: []u8) !void { + assert(self.alignBits() == 0); // internal bits must be at byte boundary + + // First read from internal bits buffer. + var n: usize = 0; + while (self.nbits > 0 and n < buf.len) { + buf[n] = try self.readF(u8, flag.buffered); + n += 1; + } + // Then use forward reader for all other bytes. + try self.forward_reader.readNoEof(buf[n..]); + } + + pub const flag = struct { + pub const peek: u3 = 0b001; // dont advance internal buffer, just get bits, leave them in buffer + pub const buffered: u3 = 0b010; // assume that there is no need to fill, fill should be called before + pub const reverse: u3 = 0b100; // bit reverse readed bits + }; + + /// Alias for readF(U, 0). + pub fn read(self: *Self, comptime U: type) !U { + return self.readF(U, 0); + } + + /// Alias for readF with flag.peak set. + pub inline fn peekF(self: *Self, comptime U: type, comptime how: u3) !U { + return self.readF(U, how | flag.peek); + } + + /// Read with flags provided. + pub fn readF(self: *Self, comptime U: type, comptime how: u3) !U { + const n: u6 = @bitSizeOf(U); + switch (how) { + 0 => { // `normal` read + try self.fill(n); // ensure that there are n bits in the buffer + const u: U = @truncate(self.bits); // get n bits + try self.shift(n); // advance buffer for n + return u; + }, + (flag.peek) => { // no shift, leave bits in the buffer + try self.fill(n); + return @truncate(self.bits); + }, + flag.buffered => { // no fill, assume that buffer has enought bits + const u: U = @truncate(self.bits); + try self.shift(n); + return u; + }, + (flag.reverse) => { // same as 0 with bit reverse + try self.fill(n); + const u: U = @truncate(self.bits); + try self.shift(n); + return @bitReverse(u); + }, + (flag.peek | flag.reverse) => { + try self.fill(n); + return @bitReverse(@as(U, @truncate(self.bits))); + }, + (flag.buffered | flag.reverse) => { + const u: U = @truncate(self.bits); + try self.shift(n); + return @bitReverse(u); + }, + (flag.peek | flag.buffered) => { + return @truncate(self.bits); + }, + (flag.peek | flag.buffered | flag.reverse) => { + return @bitReverse(@as(U, @truncate(self.bits))); + }, + } + } + + /// Read n number of bits. + /// Only buffered flag can be used in how. + pub fn readN(self: *Self, n: u4, comptime how: u3) !u16 { + switch (how) { + 0 => { + try self.fill(n); + }, + flag.buffered => {}, + else => unreachable, + } + const mask: u16 = (@as(u16, 1) << n) - 1; + const u: u16 = @as(u16, @truncate(self.bits)) & mask; + try self.shift(n); + return u; + } + + /// Advance buffer for n bits. + pub fn shift(self: *Self, n: u6) !void { + if (n > self.nbits) return error.EndOfStream; + self.bits >>= n; + self.nbits -= n; + } + + /// Skip n bytes. + pub fn skipBytes(self: *Self, n: u16) !void { + for (0..n) |_| { + try self.fill(8); + try self.shift(8); + } + } + + // Number of bits to align stream to the byte boundary. + fn alignBits(self: *Self) u3 { + return @intCast(self.nbits & 0x7); + } + + /// Align stream to the byte boundary. + pub fn alignToByte(self: *Self) void { + const ab = self.alignBits(); + if (ab > 0) self.shift(ab) catch unreachable; + } + + /// Skip zero terminated string. + pub fn skipStringZ(self: *Self) !void { + while (true) { + if (try self.readF(u8, 0) == 0) break; + } + } + + /// Read deflate fixed fixed code. + /// Reads first 7 bits, and then mybe 1 or 2 more to get full 7,8 or 9 bit code. + /// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12 + /// Lit Value Bits Codes + /// --------- ---- ----- + /// 0 - 143 8 00110000 through + /// 10111111 + /// 144 - 255 9 110010000 through + /// 111111111 + /// 256 - 279 7 0000000 through + /// 0010111 + /// 280 - 287 8 11000000 through + /// 11000111 + pub fn readFixedCode(self: *Self) !u16 { + try self.fill(7 + 2); + const code7 = try self.readF(u7, flag.buffered | flag.reverse); + if (code7 <= 0b0010_111) { // 7 bits, 256-279, codes 0000_000 - 0010_111 + return @as(u16, code7) + 256; + } else if (code7 <= 0b1011_111) { // 8 bits, 0-143, codes 0011_0000 through 1011_1111 + return (@as(u16, code7) << 1) + @as(u16, try self.readF(u1, flag.buffered)) - 0b0011_0000; + } else if (code7 <= 0b1100_011) { // 8 bit, 280-287, codes 1100_0000 - 1100_0111 + return (@as(u16, code7 - 0b1100000) << 1) + try self.readF(u1, flag.buffered) + 280; + } else { // 9 bit, 144-255, codes 1_1001_0000 - 1_1111_1111 + return (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, try self.readF(u2, flag.buffered | flag.reverse)) + 144; + } + } + }; +} + +test "flate.BitReader" { + var fbs = std.io.fixedBufferStream(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 }); + var br = bitReader(fbs.reader()); + const F = BitReader(@TypeOf(fbs.reader())).flag; + + try testing.expectEqual(@as(u8, 48), br.nbits); + try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits); + + try testing.expect(try br.readF(u1, 0) == 0b0000_0001); + try testing.expect(try br.readF(u2, 0) == 0b0000_0001); + try testing.expectEqual(@as(u8, 48 - 3), br.nbits); + try testing.expectEqual(@as(u3, 5), br.alignBits()); + + try testing.expect(try br.readF(u8, F.peek) == 0b0001_1110); + try testing.expect(try br.readF(u9, F.peek) == 0b1_0001_1110); + try br.shift(9); + try testing.expectEqual(@as(u8, 36), br.nbits); + try testing.expectEqual(@as(u3, 4), br.alignBits()); + + try testing.expect(try br.readF(u4, 0) == 0b0100); + try testing.expectEqual(@as(u8, 32), br.nbits); + try testing.expectEqual(@as(u3, 0), br.alignBits()); + + try br.shift(1); + try testing.expectEqual(@as(u3, 7), br.alignBits()); + try br.shift(1); + try testing.expectEqual(@as(u3, 6), br.alignBits()); + br.alignToByte(); + try testing.expectEqual(@as(u3, 0), br.alignBits()); + + try testing.expectEqual(@as(u64, 0xc9), br.bits); + try testing.expectEqual(@as(u16, 0x9), try br.readN(4, 0)); + try testing.expectEqual(@as(u16, 0xc), try br.readN(4, 0)); +} + +test "flate.BitReader read block type 1 data" { + const data = [_]u8{ + 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 + 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, + 0x0c, 0x01, 0x02, 0x03, // + 0xaa, 0xbb, 0xcc, 0xdd, + }; + var fbs = std.io.fixedBufferStream(&data); + var br = bitReader(fbs.reader()); + const F = BitReader(@TypeOf(fbs.reader())).flag; + + try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal + try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type + + for ("Hello world\n") |c| { + try testing.expectEqual(@as(u8, c), try br.readF(u8, F.reverse) - 0x30); + } + try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block + br.alignToByte(); + try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0)); + try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0)); + try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0)); +} + +test "flate.BitReader init" { + const data = [_]u8{ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + }; + var fbs = std.io.fixedBufferStream(&data); + var br = bitReader(fbs.reader()); + + try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits); + try br.shift(8); + try testing.expectEqual(@as(u64, 0x00_08_07_06_05_04_03_02), br.bits); + try br.fill(60); // fill with 1 byte + try testing.expectEqual(@as(u64, 0x01_08_07_06_05_04_03_02), br.bits); + try br.shift(8 * 4 + 4); + try testing.expectEqual(@as(u64, 0x00_00_00_00_00_10_80_70), br.bits); + + try br.fill(60); // fill with 4 bytes (shift by 4) + try testing.expectEqual(@as(u64, 0x00_50_40_30_20_10_80_70), br.bits); + try testing.expectEqual(@as(u8, 8 * 7 + 4), br.nbits); + + try br.shift(@intCast(br.nbits)); // clear buffer + try br.fill(8); // refill with the rest of the bytes + try testing.expectEqual(@as(u64, 0x00_00_00_00_00_08_07_06), br.bits); +} + +test "flate.BitReader readAll" { + const data = [_]u8{ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + }; + var fbs = std.io.fixedBufferStream(&data); + var br = bitReader(fbs.reader()); + + try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits); + + var out: [16]u8 = undefined; + try br.readAll(out[0..]); + try testing.expect(br.nbits == 0); + try testing.expect(br.bits == 0); + + try testing.expectEqualSlices(u8, data[0..16], &out); +} + +test "flate.BitReader readFixedCode" { + const fixed_codes = @import("huffman_encoder.zig").fixed_codes; + + var fbs = std.io.fixedBufferStream(&fixed_codes); + var rdr = bitReader(fbs.reader()); + + for (0..286) |c| { + try testing.expectEqual(c, try rdr.readFixedCode()); + } + try testing.expect(rdr.nbits == 0); +} diff --git a/lib/std/compress/flate/bit_writer.zig b/lib/std/compress/flate/bit_writer.zig new file mode 100644 index 000000000000..b5d84c7e2af7 --- /dev/null +++ b/lib/std/compress/flate/bit_writer.zig @@ -0,0 +1,99 @@ +const std = @import("std"); +const assert = std.debug.assert; + +/// Bit writer for use in deflate (compression). +/// +/// Has internal bits buffer of 64 bits and internal bytes buffer of 248 bytes. +/// When we accumulate 48 bits 6 bytes are moved to the bytes buffer. When we +/// accumulate 240 bytes they are flushed to the underlying inner_writer. +/// +pub fn BitWriter(comptime WriterType: type) type { + // buffer_flush_size indicates the buffer size + // after which bytes are flushed to the writer. + // Should preferably be a multiple of 6, since + // we accumulate 6 bytes between writes to the buffer. + const buffer_flush_size = 240; + + // buffer_size is the actual output byte buffer size. + // It must have additional headroom for a flush + // which can contain up to 8 bytes. + const buffer_size = buffer_flush_size + 8; + + return struct { + inner_writer: WriterType, + + // Data waiting to be written is bytes[0 .. nbytes] + // and then the low nbits of bits. Data is always written + // sequentially into the bytes array. + bits: u64 = 0, + nbits: u32 = 0, // number of bits + bytes: [buffer_size]u8 = undefined, + nbytes: u32 = 0, // number of bytes + + const Self = @This(); + + pub const Error = WriterType.Error || error{UnfinishedBits}; + + pub fn init(writer: WriterType) Self { + return .{ .inner_writer = writer }; + } + + pub fn setWriter(self: *Self, new_writer: WriterType) void { + //assert(self.bits == 0 and self.nbits == 0 and self.nbytes == 0); + self.inner_writer = new_writer; + } + + pub fn flush(self: *Self) Error!void { + var n = self.nbytes; + while (self.nbits != 0) { + self.bytes[n] = @as(u8, @truncate(self.bits)); + self.bits >>= 8; + if (self.nbits > 8) { // Avoid underflow + self.nbits -= 8; + } else { + self.nbits = 0; + } + n += 1; + } + self.bits = 0; + _ = try self.inner_writer.write(self.bytes[0..n]); + self.nbytes = 0; + } + + pub fn writeBits(self: *Self, b: u32, nb: u32) Error!void { + self.bits |= @as(u64, @intCast(b)) << @as(u6, @intCast(self.nbits)); + self.nbits += nb; + if (self.nbits < 48) + return; + + var n = self.nbytes; + std.mem.writeInt(u64, self.bytes[n..][0..8], self.bits, .little); + n += 6; + if (n >= buffer_flush_size) { + _ = try self.inner_writer.write(self.bytes[0..n]); + n = 0; + } + self.nbytes = n; + self.bits >>= 48; + self.nbits -= 48; + } + + pub fn writeBytes(self: *Self, bytes: []const u8) Error!void { + var n = self.nbytes; + if (self.nbits & 7 != 0) { + return error.UnfinishedBits; + } + while (self.nbits != 0) { + self.bytes[n] = @as(u8, @truncate(self.bits)); + self.bits >>= 8; + self.nbits -= 8; + n += 1; + } + if (n != 0) { + _ = try self.inner_writer.write(self.bytes[0..n]); + } + self.nbytes = 0; + _ = try self.inner_writer.write(bytes); + } + }; +} diff --git a/lib/std/compress/flate/block_writer.zig b/lib/std/compress/flate/block_writer.zig new file mode 100644 index 000000000000..394f1cbec5fe --- /dev/null +++ b/lib/std/compress/flate/block_writer.zig @@ -0,0 +1,706 @@ +const std = @import("std"); +const io = std.io; +const assert = std.debug.assert; + +const hc = @import("huffman_encoder.zig"); +const consts = @import("consts.zig").huffman; +const Token = @import("Token.zig"); +const BitWriter = @import("bit_writer.zig").BitWriter; + +pub fn blockWriter(writer: anytype) BlockWriter(@TypeOf(writer)) { + return BlockWriter(@TypeOf(writer)).init(writer); +} + +/// Accepts list of tokens, decides what is best block type to write. What block +/// type will provide best compression. Writes header and body of the block. +/// +pub fn BlockWriter(comptime WriterType: type) type { + const BitWriterType = BitWriter(WriterType); + return struct { + const codegen_order = consts.codegen_order; + const end_code_mark = 255; + const Self = @This(); + + pub const Error = BitWriterType.Error; + bit_writer: BitWriterType, + + codegen_freq: [consts.codegen_code_count]u16 = undefined, + literal_freq: [consts.max_num_lit]u16 = undefined, + distance_freq: [consts.distance_code_count]u16 = undefined, + codegen: [consts.max_num_lit + consts.distance_code_count + 1]u8 = undefined, + literal_encoding: hc.LiteralEncoder = .{}, + distance_encoding: hc.DistanceEncoder = .{}, + codegen_encoding: hc.CodegenEncoder = .{}, + fixed_literal_encoding: hc.LiteralEncoder, + fixed_distance_encoding: hc.DistanceEncoder, + huff_distance: hc.DistanceEncoder, + + pub fn init(writer: WriterType) Self { + return .{ + .bit_writer = BitWriterType.init(writer), + .fixed_literal_encoding = hc.fixedLiteralEncoder(), + .fixed_distance_encoding = hc.fixedDistanceEncoder(), + .huff_distance = hc.huffmanDistanceEncoder(), + }; + } + + /// Flush intrenal bit buffer to the writer. + /// Should be called only when bit stream is at byte boundary. + /// + /// That is after final block; when last byte could be incomplete or + /// after stored block; which is aligned to the byte bounday (it has x + /// padding bits after first 3 bits). + pub fn flush(self: *Self) Error!void { + try self.bit_writer.flush(); + } + + pub fn setWriter(self: *Self, new_writer: WriterType) void { + self.bit_writer.setWriter(new_writer); + } + + fn writeCode(self: *Self, c: hc.HuffCode) Error!void { + try self.bit_writer.writeBits(c.code, c.len); + } + + // RFC 1951 3.2.7 specifies a special run-length encoding for specifying + // the literal and distance lengths arrays (which are concatenated into a single + // array). This method generates that run-length encoding. + // + // The result is written into the codegen array, and the frequencies + // of each code is written into the codegen_freq array. + // Codes 0-15 are single byte codes. Codes 16-18 are followed by additional + // information. Code bad_code is an end marker + // + // num_literals: The number of literals in literal_encoding + // num_distances: The number of distances in distance_encoding + // lit_enc: The literal encoder to use + // dist_enc: The distance encoder to use + fn generateCodegen( + self: *Self, + num_literals: u32, + num_distances: u32, + lit_enc: *hc.LiteralEncoder, + dist_enc: *hc.DistanceEncoder, + ) void { + for (self.codegen_freq, 0..) |_, i| { + self.codegen_freq[i] = 0; + } + + // Note that we are using codegen both as a temporary variable for holding + // a copy of the frequencies, and as the place where we put the result. + // This is fine because the output is always shorter than the input used + // so far. + var codegen = &self.codegen; // cache + // Copy the concatenated code sizes to codegen. Put a marker at the end. + var cgnl = codegen[0..num_literals]; + for (cgnl, 0..) |_, i| { + cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len)); + } + + cgnl = codegen[num_literals .. num_literals + num_distances]; + for (cgnl, 0..) |_, i| { + cgnl[i] = @as(u8, @intCast(dist_enc.codes[i].len)); + } + codegen[num_literals + num_distances] = end_code_mark; + + var size = codegen[0]; + var count: i32 = 1; + var out_index: u32 = 0; + var in_index: u32 = 1; + while (size != end_code_mark) : (in_index += 1) { + // INVARIANT: We have seen "count" copies of size that have not yet + // had output generated for them. + const next_size = codegen[in_index]; + if (next_size == size) { + count += 1; + continue; + } + // We need to generate codegen indicating "count" of size. + if (size != 0) { + codegen[out_index] = size; + out_index += 1; + self.codegen_freq[size] += 1; + count -= 1; + while (count >= 3) { + var n: i32 = 6; + if (n > count) { + n = count; + } + codegen[out_index] = 16; + out_index += 1; + codegen[out_index] = @as(u8, @intCast(n - 3)); + out_index += 1; + self.codegen_freq[16] += 1; + count -= n; + } + } else { + while (count >= 11) { + var n: i32 = 138; + if (n > count) { + n = count; + } + codegen[out_index] = 18; + out_index += 1; + codegen[out_index] = @as(u8, @intCast(n - 11)); + out_index += 1; + self.codegen_freq[18] += 1; + count -= n; + } + if (count >= 3) { + // 3 <= count <= 10 + codegen[out_index] = 17; + out_index += 1; + codegen[out_index] = @as(u8, @intCast(count - 3)); + out_index += 1; + self.codegen_freq[17] += 1; + count = 0; + } + } + count -= 1; + while (count >= 0) : (count -= 1) { + codegen[out_index] = size; + out_index += 1; + self.codegen_freq[size] += 1; + } + // Set up invariant for next time through the loop. + size = next_size; + count = 1; + } + // Marker indicating the end of the codegen. + codegen[out_index] = end_code_mark; + } + + const DynamicSize = struct { + size: u32, + num_codegens: u32, + }; + + // dynamicSize returns the size of dynamically encoded data in bits. + fn dynamicSize( + self: *Self, + lit_enc: *hc.LiteralEncoder, // literal encoder + dist_enc: *hc.DistanceEncoder, // distance encoder + extra_bits: u32, + ) DynamicSize { + var num_codegens = self.codegen_freq.len; + while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) { + num_codegens -= 1; + } + const header = 3 + 5 + 5 + 4 + (3 * num_codegens) + + self.codegen_encoding.bitLength(self.codegen_freq[0..]) + + self.codegen_freq[16] * 2 + + self.codegen_freq[17] * 3 + + self.codegen_freq[18] * 7; + const size = header + + lit_enc.bitLength(&self.literal_freq) + + dist_enc.bitLength(&self.distance_freq) + + extra_bits; + + return DynamicSize{ + .size = @as(u32, @intCast(size)), + .num_codegens = @as(u32, @intCast(num_codegens)), + }; + } + + // fixedSize returns the size of dynamically encoded data in bits. + fn fixedSize(self: *Self, extra_bits: u32) u32 { + return 3 + + self.fixed_literal_encoding.bitLength(&self.literal_freq) + + self.fixed_distance_encoding.bitLength(&self.distance_freq) + + extra_bits; + } + + const StoredSize = struct { + size: u32, + storable: bool, + }; + + // storedSizeFits calculates the stored size, including header. + // The function returns the size in bits and whether the block + // fits inside a single block. + fn storedSizeFits(in: ?[]const u8) StoredSize { + if (in == null) { + return .{ .size = 0, .storable = false }; + } + if (in.?.len <= consts.max_store_block_size) { + return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true }; + } + return .{ .size = 0, .storable = false }; + } + + // Write the header of a dynamic Huffman block to the output stream. + // + // num_literals: The number of literals specified in codegen + // num_distances: The number of distances specified in codegen + // num_codegens: The number of codegens used in codegen + // eof: Is it the end-of-file? (end of stream) + fn dynamicHeader( + self: *Self, + num_literals: u32, + num_distances: u32, + num_codegens: u32, + eof: bool, + ) Error!void { + const first_bits: u32 = if (eof) 5 else 4; + try self.bit_writer.writeBits(first_bits, 3); + try self.bit_writer.writeBits(num_literals - 257, 5); + try self.bit_writer.writeBits(num_distances - 1, 5); + try self.bit_writer.writeBits(num_codegens - 4, 4); + + var i: u32 = 0; + while (i < num_codegens) : (i += 1) { + const value = self.codegen_encoding.codes[codegen_order[i]].len; + try self.bit_writer.writeBits(value, 3); + } + + i = 0; + while (true) { + const code_word: u32 = @as(u32, @intCast(self.codegen[i])); + i += 1; + if (code_word == end_code_mark) { + break; + } + try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]); + + switch (code_word) { + 16 => { + try self.bit_writer.writeBits(self.codegen[i], 2); + i += 1; + }, + 17 => { + try self.bit_writer.writeBits(self.codegen[i], 3); + i += 1; + }, + 18 => { + try self.bit_writer.writeBits(self.codegen[i], 7); + i += 1; + }, + else => {}, + } + } + } + + fn storedHeader(self: *Self, length: usize, eof: bool) Error!void { + assert(length <= 65535); + const flag: u32 = if (eof) 1 else 0; + try self.bit_writer.writeBits(flag, 3); + try self.flush(); + const l: u16 = @intCast(length); + try self.bit_writer.writeBits(l, 16); + try self.bit_writer.writeBits(~l, 16); + } + + fn fixedHeader(self: *Self, eof: bool) Error!void { + // Indicate that we are a fixed Huffman block + var value: u32 = 2; + if (eof) { + value = 3; + } + try self.bit_writer.writeBits(value, 3); + } + + // Write a block of tokens with the smallest encoding. Will choose block type. + // The original input can be supplied, and if the huffman encoded data + // is larger than the original bytes, the data will be written as a + // stored block. + // If the input is null, the tokens will always be Huffman encoded. + pub fn write(self: *Self, tokens: []const Token, eof: bool, input: ?[]const u8) Error!void { + const lit_and_dist = self.indexTokens(tokens); + const num_literals = lit_and_dist.num_literals; + const num_distances = lit_and_dist.num_distances; + + var extra_bits: u32 = 0; + const ret = storedSizeFits(input); + const stored_size = ret.size; + const storable = ret.storable; + + if (storable) { + // We only bother calculating the costs of the extra bits required by + // the length of distance fields (which will be the same for both fixed + // and dynamic encoding), if we need to compare those two encodings + // against stored encoding. + var length_code: u16 = Token.length_codes_start + 8; + while (length_code < num_literals) : (length_code += 1) { + // First eight length codes have extra size = 0. + extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) * + @as(u32, @intCast(Token.lengthExtraBits(length_code))); + } + var distance_code: u16 = 4; + while (distance_code < num_distances) : (distance_code += 1) { + // First four distance codes have extra size = 0. + extra_bits += @as(u32, @intCast(self.distance_freq[distance_code])) * + @as(u32, @intCast(Token.distanceExtraBits(distance_code))); + } + } + + // Figure out smallest code. + // Fixed Huffman baseline. + var literal_encoding = &self.fixed_literal_encoding; + var distance_encoding = &self.fixed_distance_encoding; + var size = self.fixedSize(extra_bits); + + // Dynamic Huffman? + var num_codegens: u32 = 0; + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literal_encoding and the distance_encoding. + self.generateCodegen( + num_literals, + num_distances, + &self.literal_encoding, + &self.distance_encoding, + ); + self.codegen_encoding.generate(self.codegen_freq[0..], 7); + const dynamic_size = self.dynamicSize( + &self.literal_encoding, + &self.distance_encoding, + extra_bits, + ); + const dyn_size = dynamic_size.size; + num_codegens = dynamic_size.num_codegens; + + if (dyn_size < size) { + size = dyn_size; + literal_encoding = &self.literal_encoding; + distance_encoding = &self.distance_encoding; + } + + // Stored bytes? + if (storable and stored_size < size) { + try self.storedBlock(input.?, eof); + return; + } + + // Huffman. + if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) { + try self.fixedHeader(eof); + } else { + try self.dynamicHeader(num_literals, num_distances, num_codegens, eof); + } + + // Write the tokens. + try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes); + } + + pub fn storedBlock(self: *Self, input: []const u8, eof: bool) Error!void { + try self.storedHeader(input.len, eof); + try self.bit_writer.writeBytes(input); + } + + // writeBlockDynamic encodes a block using a dynamic Huffman table. + // This should be used if the symbols used have a disproportionate + // histogram distribution. + // If input is supplied and the compression savings are below 1/16th of the + // input size the block is stored. + fn dynamicBlock( + self: *Self, + tokens: []const Token, + eof: bool, + input: ?[]const u8, + ) Error!void { + const total_tokens = self.indexTokens(tokens); + const num_literals = total_tokens.num_literals; + const num_distances = total_tokens.num_distances; + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literal_encoding and the distance_encoding. + self.generateCodegen( + num_literals, + num_distances, + &self.literal_encoding, + &self.distance_encoding, + ); + self.codegen_encoding.generate(self.codegen_freq[0..], 7); + const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.distance_encoding, 0); + const size = dynamic_size.size; + const num_codegens = dynamic_size.num_codegens; + + // Store bytes, if we don't get a reasonable improvement. + + const stored_size = storedSizeFits(input); + const ssize = stored_size.size; + const storable = stored_size.storable; + if (storable and ssize < (size + (size >> 4))) { + try self.storedBlock(input.?, eof); + return; + } + + // Write Huffman table. + try self.dynamicHeader(num_literals, num_distances, num_codegens, eof); + + // Write the tokens. + try self.writeTokens(tokens, &self.literal_encoding.codes, &self.distance_encoding.codes); + } + + const TotalIndexedTokens = struct { + num_literals: u32, + num_distances: u32, + }; + + // Indexes a slice of tokens followed by an end_block_marker, and updates + // literal_freq and distance_freq, and generates literal_encoding + // and distance_encoding. + // The number of literal and distance tokens is returned. + fn indexTokens(self: *Self, tokens: []const Token) TotalIndexedTokens { + var num_literals: u32 = 0; + var num_distances: u32 = 0; + + for (self.literal_freq, 0..) |_, i| { + self.literal_freq[i] = 0; + } + for (self.distance_freq, 0..) |_, i| { + self.distance_freq[i] = 0; + } + + for (tokens) |t| { + if (t.kind == Token.Kind.literal) { + self.literal_freq[t.literal()] += 1; + continue; + } + self.literal_freq[t.lengthCode()] += 1; + self.distance_freq[t.distanceCode()] += 1; + } + // add end_block_marker token at the end + self.literal_freq[consts.end_block_marker] += 1; + + // get the number of literals + num_literals = @as(u32, @intCast(self.literal_freq.len)); + while (self.literal_freq[num_literals - 1] == 0) { + num_literals -= 1; + } + // get the number of distances + num_distances = @as(u32, @intCast(self.distance_freq.len)); + while (num_distances > 0 and self.distance_freq[num_distances - 1] == 0) { + num_distances -= 1; + } + if (num_distances == 0) { + // We haven't found a single match. If we want to go with the dynamic encoding, + // we should count at least one distance to be sure that the distance huffman tree could be encoded. + self.distance_freq[0] = 1; + num_distances = 1; + } + self.literal_encoding.generate(&self.literal_freq, 15); + self.distance_encoding.generate(&self.distance_freq, 15); + return TotalIndexedTokens{ + .num_literals = num_literals, + .num_distances = num_distances, + }; + } + + // Writes a slice of tokens to the output followed by and end_block_marker. + // codes for literal and distance encoding must be supplied. + fn writeTokens( + self: *Self, + tokens: []const Token, + le_codes: []hc.HuffCode, + oe_codes: []hc.HuffCode, + ) Error!void { + for (tokens) |t| { + if (t.kind == Token.Kind.literal) { + try self.writeCode(le_codes[t.literal()]); + continue; + } + + // Write the length + const le = t.lengthEncoding(); + try self.writeCode(le_codes[le.code]); + if (le.extra_bits > 0) { + try self.bit_writer.writeBits(le.extra_length, le.extra_bits); + } + + // Write the distance + const oe = t.distanceEncoding(); + try self.writeCode(oe_codes[oe.code]); + if (oe.extra_bits > 0) { + try self.bit_writer.writeBits(oe.extra_distance, oe.extra_bits); + } + } + // add end_block_marker at the end + try self.writeCode(le_codes[consts.end_block_marker]); + } + + // Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes + // if the results only gains very little from compression. + pub fn huffmanBlock(self: *Self, input: []const u8, eof: bool) Error!void { + // Add everything as literals + histogram(input, &self.literal_freq); + + self.literal_freq[consts.end_block_marker] = 1; + + const num_literals = consts.end_block_marker + 1; + self.distance_freq[0] = 1; + const num_distances = 1; + + self.literal_encoding.generate(&self.literal_freq, 15); + + // Figure out smallest code. + // Always use dynamic Huffman or Store + var num_codegens: u32 = 0; + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literal_encoding and the distance_encoding. + self.generateCodegen( + num_literals, + num_distances, + &self.literal_encoding, + &self.huff_distance, + ); + self.codegen_encoding.generate(self.codegen_freq[0..], 7); + const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_distance, 0); + const size = dynamic_size.size; + num_codegens = dynamic_size.num_codegens; + + // Store bytes, if we don't get a reasonable improvement. + const stored_size_ret = storedSizeFits(input); + const ssize = stored_size_ret.size; + const storable = stored_size_ret.storable; + + if (storable and ssize < (size + (size >> 4))) { + try self.storedBlock(input, eof); + return; + } + + // Huffman. + try self.dynamicHeader(num_literals, num_distances, num_codegens, eof); + const encoding = self.literal_encoding.codes[0..257]; + + for (input) |t| { + const c = encoding[t]; + try self.bit_writer.writeBits(c.code, c.len); + } + try self.writeCode(encoding[consts.end_block_marker]); + } + + // histogram accumulates a histogram of b in h. + fn histogram(b: []const u8, h: *[286]u16) void { + // Clear histogram + for (h, 0..) |_, i| { + h[i] = 0; + } + + var lh = h.*[0..256]; + for (b) |t| { + lh[t] += 1; + } + } + }; +} + +// tests +const expect = std.testing.expect; +const fmt = std.fmt; +const testing = std.testing; +const ArrayList = std.ArrayList; + +const TestCase = @import("testdata/block_writer.zig").TestCase; +const testCases = @import("testdata/block_writer.zig").testCases; + +// tests if the writeBlock encoding has changed. +test "flate.BlockWriter write" { + inline for (0..testCases.len) |i| { + try testBlock(testCases[i], .write_block); + } +} + +// tests if the writeBlockDynamic encoding has changed. +test "flate.BlockWriter dynamicBlock" { + inline for (0..testCases.len) |i| { + try testBlock(testCases[i], .write_dyn_block); + } +} + +test "flate.BlockWriter huffmanBlock" { + inline for (0..testCases.len) |i| { + try testBlock(testCases[i], .write_huffman_block); + } + try testBlock(.{ + .tokens = &[_]Token{}, + .input = "huffman-rand-max.input", + .want = "huffman-rand-max.{s}.expect", + }, .write_huffman_block); +} + +const TestFn = enum { + write_block, + write_dyn_block, // write dynamic block + write_huffman_block, + + fn to_s(self: TestFn) []const u8 { + return switch (self) { + .write_block => "wb", + .write_dyn_block => "dyn", + .write_huffman_block => "huff", + }; + } + + fn write( + comptime self: TestFn, + bw: anytype, + tok: []const Token, + input: ?[]const u8, + final: bool, + ) !void { + switch (self) { + .write_block => try bw.write(tok, final, input), + .write_dyn_block => try bw.dynamicBlock(tok, final, input), + .write_huffman_block => try bw.huffmanBlock(input.?, final), + } + try bw.flush(); + } +}; + +// testBlock tests a block against its references +// +// size +// 64K [file-name].input - input non compressed file +// 8.1K [file-name].golden - +// 78 [file-name].dyn.expect - output with writeBlockDynamic +// 78 [file-name].wb.expect - output with writeBlock +// 8.1K [file-name].huff.expect - output with writeBlockHuff +// 78 [file-name].dyn.expect-noinput - output with writeBlockDynamic when input is null +// 78 [file-name].wb.expect-noinput - output with writeBlock when input is null +// +// wb - writeBlock +// dyn - writeBlockDynamic +// huff - writeBlockHuff +// +fn testBlock(comptime tc: TestCase, comptime tfn: TestFn) !void { + if (tc.input.len != 0 and tc.want.len != 0) { + const want_name = comptime fmt.comptimePrint(tc.want, .{tfn.to_s()}); + const input = @embedFile("testdata/block_writer/" ++ tc.input); + const want = @embedFile("testdata/block_writer/" ++ want_name); + try testWriteBlock(tfn, input, want, tc.tokens); + } + + if (tfn == .write_huffman_block) { + return; + } + + const want_name_no_input = comptime fmt.comptimePrint(tc.want_no_input, .{tfn.to_s()}); + const want = @embedFile("testdata/block_writer/" ++ want_name_no_input); + try testWriteBlock(tfn, null, want, tc.tokens); +} + +// Uses writer function `tfn` to write `tokens`, tests that we got `want` as output. +fn testWriteBlock(comptime tfn: TestFn, input: ?[]const u8, want: []const u8, tokens: []const Token) !void { + var buf = ArrayList(u8).init(testing.allocator); + var bw = blockWriter(buf.writer()); + try tfn.write(&bw, tokens, input, false); + var got = buf.items; + try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result + try expect(got[0] & 0b0000_0001 == 0); // bfinal is not set + // + // Test if the writer produces the same output after reset. + buf.deinit(); + buf = ArrayList(u8).init(testing.allocator); + defer buf.deinit(); + bw.setWriter(buf.writer()); + + try tfn.write(&bw, tokens, input, true); + try bw.flush(); + got = buf.items; + + try expect(got[0] & 1 == 1); // bfinal is set + buf.items[0] &= 0b1111_1110; // remove bfinal bit, so we can run test slices + try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result +} diff --git a/lib/std/compress/flate/consts.zig b/lib/std/compress/flate/consts.zig new file mode 100644 index 000000000000..c28b40f68ef3 --- /dev/null +++ b/lib/std/compress/flate/consts.zig @@ -0,0 +1,49 @@ +pub const deflate = struct { + // Number of tokens to accumlate in deflate before starting block encoding. + // + // In zlib this depends on memlevel: 6 + memlevel, where default memlevel is + // 8 and max 9 that gives 14 or 15 bits. + pub const tokens = 1 << 15; +}; + +pub const match = struct { + pub const base_length = 3; // smallest match length per the RFC section 3.2.5 + pub const min_length = 4; // min length used in this algorithm + pub const max_length = 258; + + pub const min_distance = 1; + pub const max_distance = 32768; +}; + +pub const history = struct { + pub const len = match.max_distance; +}; + +pub const lookup = struct { + pub const bits = 15; + pub const len = 1 << bits; + pub const shift = 32 - bits; +}; + +pub const huffman = struct { + // The odd order in which the codegen code sizes are written. + pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; + // The number of codegen codes. + pub const codegen_code_count = 19; + + // The largest distance code. + pub const distance_code_count = 30; + + // Maximum number of literals. + pub const max_num_lit = 286; + + // Max number of frequencies used for a Huffman Code + // Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286). + // The largest of these is max_num_lit. + pub const max_num_frequencies = max_num_lit; + + // Biggest block size for uncompressed block. + pub const max_store_block_size = 65535; + // The special code used to mark the end of a block. + pub const end_block_marker = 256; +}; diff --git a/lib/std/compress/flate/container.zig b/lib/std/compress/flate/container.zig new file mode 100644 index 000000000000..23eec920def6 --- /dev/null +++ b/lib/std/compress/flate/container.zig @@ -0,0 +1,207 @@ +//! Container of the deflate bit stream body. Container adds header before +//! deflate bit stream and footer after. It can bi gzip, zlib or raw (no header, +//! no footer, raw bit stream). +//! +//! Zlib format is defined in rfc 1950. Header has 2 bytes and footer 4 bytes +//! addler 32 checksum. +//! +//! Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes +//! crc32 checksum and 4 bytes of uncompressed data length. +//! +//! +//! rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4 +//! rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5 +//! + +const std = @import("std"); + +pub const Container = enum { + raw, // no header or footer + gzip, // gzip header and footer + zlib, // zlib header and footer + + pub fn size(w: Container) usize { + return headerSize(w) + footerSize(w); + } + + pub fn headerSize(w: Container) usize { + return switch (w) { + .gzip => 10, + .zlib => 2, + .raw => 0, + }; + } + + pub fn footerSize(w: Container) usize { + return switch (w) { + .gzip => 8, + .zlib => 4, + .raw => 0, + }; + } + + pub const list = [_]Container{ .raw, .gzip, .zlib }; + + pub const Error = error{ + BadGzipHeader, + BadZlibHeader, + WrongGzipChecksum, + WrongGzipSize, + WrongZlibChecksum, + }; + + pub fn writeHeader(comptime wrap: Container, writer: anytype) !void { + switch (wrap) { + .gzip => { + // GZIP 10 byte header (https://datatracker.ietf.org/doc/html/rfc1952#page-5): + // - ID1 (IDentification 1), always 0x1f + // - ID2 (IDentification 2), always 0x8b + // - CM (Compression Method), always 8 = deflate + // - FLG (Flags), all set to 0 + // - 4 bytes, MTIME (Modification time), not used, all set to zero + // - XFL (eXtra FLags), all set to zero + // - OS (Operating System), 03 = Unix + const gzipHeader = [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 }; + try writer.writeAll(&gzipHeader); + }, + .zlib => { + // ZLIB has a two-byte header (https://datatracker.ietf.org/doc/html/rfc1950#page-4): + // 1st byte: + // - First four bits is the CINFO (compression info), which is 7 for the default deflate window size. + // - The next four bits is the CM (compression method), which is 8 for deflate. + // 2nd byte: + // - Two bits is the FLEVEL (compression level). Values are: 0=fastest, 1=fast, 2=default, 3=best. + // - The next bit, FDICT, is set if a dictionary is given. + // - The final five FCHECK bits form a mod-31 checksum. + // + // CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100 + const zlibHeader = [_]u8{ 0x78, 0b10_0_11100 }; + try writer.writeAll(&zlibHeader); + }, + .raw => {}, + } + } + + pub fn writeFooter(comptime wrap: Container, hasher: *Hasher(wrap), writer: anytype) !void { + var bits: [4]u8 = undefined; + switch (wrap) { + .gzip => { + // GZIP 8 bytes footer + // - 4 bytes, CRC32 (CRC-32) + // - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32 + std.mem.writeInt(u32, &bits, hasher.chksum(), .little); + try writer.writeAll(&bits); + + std.mem.writeInt(u32, &bits, hasher.bytesRead(), .little); + try writer.writeAll(&bits); + }, + .zlib => { + // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). + // 4 bytes of ADLER32 (Adler-32 checksum) + // Checksum value of the uncompressed data (excluding any + // dictionary data) computed according to Adler-32 + // algorithm. + std.mem.writeInt(u32, &bits, hasher.chksum(), .big); + try writer.writeAll(&bits); + }, + .raw => {}, + } + } + + pub fn parseHeader(comptime wrap: Container, reader: anytype) !void { + switch (wrap) { + .gzip => try parseGzipHeader(reader), + .zlib => try parseZlibHeader(reader), + .raw => {}, + } + } + + fn parseGzipHeader(reader: anytype) !void { + const magic1 = try reader.read(u8); + const magic2 = try reader.read(u8); + const method = try reader.read(u8); + const flags = try reader.read(u8); + try reader.skipBytes(6); // mtime(4), xflags, os + if (magic1 != 0x1f or magic2 != 0x8b or method != 0x08) + return error.BadGzipHeader; + // Flags description: https://www.rfc-editor.org/rfc/rfc1952.html#page-5 + if (flags != 0) { + if (flags & 0b0000_0100 != 0) { // FEXTRA + const extra_len = try reader.read(u16); + try reader.skipBytes(extra_len); + } + if (flags & 0b0000_1000 != 0) { // FNAME + try reader.skipStringZ(); + } + if (flags & 0b0001_0000 != 0) { // FCOMMENT + try reader.skipStringZ(); + } + if (flags & 0b0000_0010 != 0) { // FHCRC + try reader.skipBytes(2); + } + } + } + + fn parseZlibHeader(reader: anytype) !void { + const cm = try reader.read(u4); + const cinfo = try reader.read(u4); + _ = try reader.read(u8); + if (cm != 8 or cinfo > 7) { + return error.BadZlibHeader; + } + } + + pub fn parseFooter(comptime wrap: Container, hasher: *Hasher(wrap), reader: anytype) !void { + switch (wrap) { + .gzip => { + if (try reader.read(u32) != hasher.chksum()) return error.WrongGzipChecksum; + if (try reader.read(u32) != hasher.bytesRead()) return error.WrongGzipSize; + }, + .zlib => { + const chksum: u32 = @byteSwap(hasher.chksum()); + if (try reader.read(u32) != chksum) return error.WrongZlibChecksum; + }, + .raw => {}, + } + } + + pub fn Hasher(comptime wrap: Container) type { + const HasherType = switch (wrap) { + .gzip => std.hash.Crc32, + .zlib => std.hash.Adler32, + .raw => struct { + pub fn init() @This() { + return .{}; + } + }, + }; + + return struct { + hasher: HasherType = HasherType.init(), + bytes: usize = 0, + + const Self = @This(); + + pub fn update(self: *Self, buf: []const u8) void { + switch (wrap) { + .raw => {}, + else => { + self.hasher.update(buf); + self.bytes += buf.len; + }, + } + } + + pub fn chksum(self: *Self) u32 { + switch (wrap) { + .raw => return 0, + else => return self.hasher.final(), + } + } + + pub fn bytesRead(self: *Self) u32 { + return @truncate(self.bytes); + } + }; + } +}; diff --git a/lib/std/compress/flate/deflate.zig b/lib/std/compress/flate/deflate.zig new file mode 100644 index 000000000000..0ca998d8eb55 --- /dev/null +++ b/lib/std/compress/flate/deflate.zig @@ -0,0 +1,748 @@ +const std = @import("std"); +const io = std.io; +const assert = std.debug.assert; +const testing = std.testing; +const expect = testing.expect; +const print = std.debug.print; + +const Token = @import("Token.zig"); +const consts = @import("consts.zig"); +const BlockWriter = @import("block_writer.zig").BlockWriter; +const Container = @import("container.zig").Container; +const SlidingWindow = @import("SlidingWindow.zig"); +const Lookup = @import("Lookup.zig"); + +pub const Options = struct { + level: Level = .default, +}; + +/// Trades between speed and compression size. +/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43) +/// levels 1-3 are using different algorithm to perform faster but with less +/// compression. That is not implemented here. +pub const Level = enum(u4) { + // zig fmt: off + fast = 0xb, level_4 = 4, + level_5 = 5, + default = 0xc, level_6 = 6, + level_7 = 7, + level_8 = 8, + best = 0xd, level_9 = 9, + // zig fmt: on +}; + +/// Algorithm knobs for each level. +const LevelArgs = struct { + good: u16, // Do less lookups if we already have match of this length. + nice: u16, // Stop looking for better match if we found match with at least this length. + lazy: u16, // Don't do lazy match find if got match with at least this length. + chain: u16, // How many lookups for previous match to perform. + + pub fn get(level: Level) LevelArgs { + // zig fmt: off + return switch (level) { + .fast, .level_4 => .{ .good = 4, .lazy = 4, .nice = 16, .chain = 16 }, + .level_5 => .{ .good = 8, .lazy = 16, .nice = 32, .chain = 32 }, + .default, .level_6 => .{ .good = 8, .lazy = 16, .nice = 128, .chain = 128 }, + .level_7 => .{ .good = 8, .lazy = 32, .nice = 128, .chain = 256 }, + .level_8 => .{ .good = 32, .lazy = 128, .nice = 258, .chain = 1024 }, + .best, .level_9 => .{ .good = 32, .lazy = 258, .nice = 258, .chain = 4096 }, + }; + // zig fmt: on + } +}; + +/// Compress plain data from reader into compressed stream written to writer. +pub fn compress(comptime container: Container, reader: anytype, writer: anytype, options: Options) !void { + var c = try compressor(container, writer, options); + try c.compress(reader); + try c.finish(); +} + +/// Create compressor for writer type. +pub fn compressor(comptime container: Container, writer: anytype, options: Options) !Compressor( + container, + @TypeOf(writer), +) { + return try Compressor(container, @TypeOf(writer)).init(writer, options); +} + +/// Compressor type. +pub fn Compressor(comptime container: Container, comptime WriterType: type) type { + const TokenWriterType = BlockWriter(WriterType); + return Deflate(container, WriterType, TokenWriterType); +} + +/// Default compression algorithm. Has two steps: tokenization and token +/// encoding. +/// +/// Tokenization takes uncompressed input stream and produces list of tokens. +/// Each token can be literal (byte of data) or match (backrefernce to previous +/// data with length and distance). Tokenization accumulators 32K tokens, when +/// full or `flush` is called tokens are passed to the `block_writer`. Level +/// defines how hard (how slow) it tries to find match. +/// +/// Block writer will decide which type of deflate block to write (stored, fixed, +/// dynamic) and encode tokens to the output byte stream. Client has to call +/// `finish` to write block with the final bit set. +/// +/// Container defines type of header and footer which can be gzip, zlib or raw. +/// They all share same deflate body. Raw has no header or footer just deflate +/// body. +/// +/// Compression algorithm explained in rfc-1951 (slightly edited for this case): +/// +/// The compressor uses a chained hash table `lookup` to find duplicated +/// strings, using a hash function that operates on 4-byte sequences. At any +/// given point during compression, let XYZW be the next 4 input bytes +/// (lookahead) to be examined (not necessarily all different, of course). +/// First, the compressor examines the hash chain for XYZW. If the chain is +/// empty, the compressor simply writes out X as a literal byte and advances +/// one byte in the input. If the hash chain is not empty, indicating that the +/// sequence XYZW (or, if we are unlucky, some other 4 bytes with the same +/// hash function value) has occurred recently, the compressor compares all +/// strings on the XYZW hash chain with the actual input data sequence +/// starting at the current point, and selects the longest match. +/// +/// To improve overall compression, the compressor defers the selection of +/// matches ("lazy matching"): after a match of length N has been found, the +/// compressor searches for a longer match starting at the next input byte. If +/// it finds a longer match, it truncates the previous match to a length of +/// one (thus producing a single literal byte) and then emits the longer +/// match. Otherwise, it emits the original match, and, as described above, +/// advances N bytes before continuing. +/// +/// +/// Allocates statically ~400K (192K lookup, 128K tokens, 64K window). +/// +/// Deflate function accepts BlockWriterType so we can change that in test to test +/// just tokenization part. +/// +fn Deflate(comptime container: Container, comptime WriterType: type, comptime BlockWriterType: type) type { + return struct { + lookup: Lookup = .{}, + win: SlidingWindow = .{}, + tokens: Tokens = .{}, + wrt: WriterType, + block_writer: BlockWriterType, + level: LevelArgs, + hasher: container.Hasher() = .{}, + + // Match and literal at the previous position. + // Used for lazy match finding in processWindow. + prev_match: ?Token = null, + prev_literal: ?u8 = null, + + const Self = @This(); + + pub fn init(wrt: WriterType, options: Options) !Self { + const self = Self{ + .wrt = wrt, + .block_writer = BlockWriterType.init(wrt), + .level = LevelArgs.get(options.level), + }; + try container.writeHeader(self.wrt); + return self; + } + + const FlushOption = enum { none, flush, final }; + + // Process data in window and create tokens. If token buffer is full + // flush tokens to the token writer. In the case of `flush` or `final` + // option it will process all data from the window. In the `none` case + // it will preserve some data for the next match. + fn tokenize(self: *Self, flush_opt: FlushOption) !void { + // flush - process all data from window + const should_flush = (flush_opt != .none); + + // While there is data in active lookahead buffer. + while (self.win.activeLookahead(should_flush)) |lh| { + var step: u16 = 1; // 1 in the case of literal, match length otherwise + const pos: u16 = self.win.pos(); + const literal = lh[0]; // literal at current position + const min_len: u16 = if (self.prev_match) |m| m.length() else 0; + + // Try to find match at least min_len long. + if (self.findMatch(pos, lh, min_len)) |match| { + // Found better match than previous. + try self.addPrevLiteral(); + + // Is found match length good enough? + if (match.length() >= self.level.lazy) { + // Don't try to lazy find better match, use this. + step = try self.addMatch(match); + } else { + // Store this match. + self.prev_literal = literal; + self.prev_match = match; + } + } else { + // There is no better match at current pos then it was previous. + // Write previous match or literal. + if (self.prev_match) |m| { + // Write match from previous position. + step = try self.addMatch(m) - 1; // we already advanced 1 from previous position + } else { + // No match at previous postition. + // Write previous literal if any, and remember this literal. + try self.addPrevLiteral(); + self.prev_literal = literal; + } + } + // Advance window and add hashes. + self.windowAdvance(step, lh, pos); + } + + if (should_flush) { + // In the case of flushing, last few lookahead buffers were smaller then min match len. + // So only last literal can be unwritten. + assert(self.prev_match == null); + try self.addPrevLiteral(); + self.prev_literal = null; + + try self.flushTokens(flush_opt); + } + } + + fn windowAdvance(self: *Self, step: u16, lh: []const u8, pos: u16) void { + // current position is already added in findMatch + self.lookup.bulkAdd(lh[1..], step - 1, pos + 1); + self.win.advance(step); + } + + // Add previous literal (if any) to the tokens list. + fn addPrevLiteral(self: *Self) !void { + if (self.prev_literal) |l| try self.addToken(Token.initLiteral(l)); + } + + // Add match to the tokens list, reset prev pointers. + // Returns length of the added match. + fn addMatch(self: *Self, m: Token) !u16 { + try self.addToken(m); + self.prev_literal = null; + self.prev_match = null; + return m.length(); + } + + fn addToken(self: *Self, token: Token) !void { + self.tokens.add(token); + if (self.tokens.full()) try self.flushTokens(.none); + } + + // Finds largest match in the history window with the data at current pos. + fn findMatch(self: *Self, pos: u16, lh: []const u8, min_len: u16) ?Token { + var len: u16 = min_len; + // Previous location with the same hash (same 4 bytes). + var prev_pos = self.lookup.add(lh, pos); + // Last found match. + var match: ?Token = null; + + // How much back-references to try, performance knob. + var chain: usize = self.level.chain; + if (len >= self.level.good) { + // If we've got a match that's good enough, only look in 1/4 the chain. + chain >>= 2; + } + + // Hot path loop! + while (prev_pos > 0 and chain > 0) : (chain -= 1) { + const distance = pos - prev_pos; + if (distance > consts.match.max_distance) + break; + + const new_len = self.win.match(prev_pos, pos, len); + if (new_len > len) { + match = Token.initMatch(@intCast(distance), new_len); + if (new_len >= self.level.nice) { + // The match is good enough that we don't try to find a better one. + return match; + } + len = new_len; + } + prev_pos = self.lookup.prev(prev_pos); + } + + return match; + } + + fn flushTokens(self: *Self, flush_opt: FlushOption) !void { + // Pass tokens to the token writer + try self.block_writer.write(self.tokens.tokens(), flush_opt == .final, self.win.tokensBuffer()); + // Stored block ensures byte aligment. + // It has 3 bits (final, block_type) and then padding until byte boundary. + // After that everyting is aligned to the boundary in the stored block. + // Empty stored block is Ob000 + (0-7) bits of padding + 0x00 0x00 0xFF 0xFF. + // Last 4 bytes are byte aligned. + if (flush_opt == .flush) { + try self.block_writer.storedBlock("", false); + } + if (flush_opt != .none) { + // Safe to call only when byte aligned or it is OK to add + // padding bits (on last byte of the final block). + try self.block_writer.flush(); + } + // Reset internal tokens store. + self.tokens.reset(); + // Notify win that tokens are flushed. + self.win.flush(); + } + + // Slide win and if needed lookup tables. + fn slide(self: *Self) void { + const n = self.win.slide(); + self.lookup.slide(n); + } + + /// Compresses as much data as possible, stops when the reader becomes + /// empty. It will introduce some output latency (reading input without + /// producing all output) because some data are still in internal + /// buffers. + /// + /// It is up to the caller to call flush (if needed) or finish (required) + /// when is need to output any pending data or complete stream. + /// + pub fn compress(self: *Self, reader: anytype) !void { + while (true) { + // Fill window from reader + const buf = self.win.writable(); + if (buf.len == 0) { + try self.tokenize(.none); + self.slide(); + continue; + } + const n = try reader.readAll(buf); + self.hasher.update(buf[0..n]); + self.win.written(n); + // Process window + try self.tokenize(.none); + // Exit when no more data in reader + if (n < buf.len) break; + } + } + + /// Flushes internal buffers to the output writer. Outputs empty stored + /// block to sync bit stream to the byte boundary, so that the + /// decompressor can get all input data available so far. + /// + /// It is useful mainly in compressed network protocols, to ensure that + /// deflate bit stream can be used as byte stream. May degrade + /// compression so it should be used only when necessary. + /// + /// Completes the current deflate block and follows it with an empty + /// stored block that is three zero bits plus filler bits to the next + /// byte, followed by four bytes (00 00 ff ff). + /// + pub fn flush(self: *Self) !void { + try self.tokenize(.flush); + } + + /// Completes deflate bit stream by writing any pending data as deflate + /// final deflate block. HAS to be called once all data are written to + /// the compressor as a signal that next block has to have final bit + /// set. + /// + pub fn finish(self: *Self) !void { + try self.tokenize(.final); + try container.writeFooter(&self.hasher, self.wrt); + } + + /// Use another writer while preserving history. Most probably flush + /// should be called on old writer before setting new. + pub fn setWriter(self: *Self, new_writer: WriterType) void { + self.block_writer.setWriter(new_writer); + self.wrt = new_writer; + } + + // Writer interface + + pub const Writer = io.Writer(*Self, Error, write); + pub const Error = BlockWriterType.Error; + + /// Write `input` of uncompressed data. + /// See compress. + pub fn write(self: *Self, input: []const u8) !usize { + var fbs = io.fixedBufferStream(input); + try self.compress(fbs.reader()); + return input.len; + } + + pub fn writer(self: *Self) Writer { + return .{ .context = self }; + } + }; +} + +// Tokens store +const Tokens = struct { + list: [consts.deflate.tokens]Token = undefined, + pos: usize = 0, + + fn add(self: *Tokens, t: Token) void { + self.list[self.pos] = t; + self.pos += 1; + } + + fn full(self: *Tokens) bool { + return self.pos == self.list.len; + } + + fn reset(self: *Tokens) void { + self.pos = 0; + } + + fn tokens(self: *Tokens) []const Token { + return self.list[0..self.pos]; + } +}; + +/// Creates huffman only deflate blocks. Disables Lempel-Ziv match searching and +/// only performs Huffman entropy encoding. Results in faster compression, much +/// less memory requirements during compression but bigger compressed sizes. +pub const huffman = struct { + pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void { + var c = try huffman.compressor(container, writer); + try c.compress(reader); + try c.finish(); + } + + pub fn Compressor(comptime container: Container, comptime WriterType: type) type { + return SimpleCompressor(.huffman, container, WriterType); + } + + pub fn compressor(comptime container: Container, writer: anytype) !huffman.Compressor(container, @TypeOf(writer)) { + return try huffman.Compressor(container, @TypeOf(writer)).init(writer); + } +}; + +/// Creates store blocks only. Data are not compressed only packed into deflate +/// store blocks. That adds 9 bytes of header for each block. Max stored block +/// size is 64K. Block is emitted when flush is called on on finish. +pub const store = struct { + pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void { + var c = try store.compressor(container, writer); + try c.compress(reader); + try c.finish(); + } + + pub fn Compressor(comptime container: Container, comptime WriterType: type) type { + return SimpleCompressor(.store, container, WriterType); + } + + pub fn compressor(comptime container: Container, writer: anytype) !store.Compressor(container, @TypeOf(writer)) { + return try store.Compressor(container, @TypeOf(writer)).init(writer); + } +}; + +const SimpleCompressorKind = enum { + huffman, + store, +}; + +fn simpleCompressor( + comptime kind: SimpleCompressorKind, + comptime container: Container, + writer: anytype, +) !SimpleCompressor(kind, container, @TypeOf(writer)) { + return try SimpleCompressor(kind, container, @TypeOf(writer)).init(writer); +} + +fn SimpleCompressor( + comptime kind: SimpleCompressorKind, + comptime container: Container, + comptime WriterType: type, +) type { + const BlockWriterType = BlockWriter(WriterType); + return struct { + buffer: [65535]u8 = undefined, // because store blocks are limited to 65535 bytes + wp: usize = 0, + + wrt: WriterType, + block_writer: BlockWriterType, + hasher: container.Hasher() = .{}, + + const Self = @This(); + + pub fn init(wrt: WriterType) !Self { + const self = Self{ + .wrt = wrt, + .block_writer = BlockWriterType.init(wrt), + }; + try container.writeHeader(self.wrt); + return self; + } + + pub fn flush(self: *Self) !void { + try self.flushBuffer(false); + try self.block_writer.storedBlock("", false); + try self.block_writer.flush(); + } + + pub fn finish(self: *Self) !void { + try self.flushBuffer(true); + try self.block_writer.flush(); + try container.writeFooter(&self.hasher, self.wrt); + } + + fn flushBuffer(self: *Self, final: bool) !void { + const buf = self.buffer[0..self.wp]; + switch (kind) { + .huffman => try self.block_writer.huffmanBlock(buf, final), + .store => try self.block_writer.storedBlock(buf, final), + } + self.wp = 0; + } + + // Writes all data from the input reader of uncompressed data. + // It is up to the caller to call flush or finish if there is need to + // output compressed blocks. + pub fn compress(self: *Self, reader: anytype) !void { + while (true) { + // read from rdr into buffer + const buf = self.buffer[self.wp..]; + if (buf.len == 0) { + try self.flushBuffer(false); + continue; + } + const n = try reader.readAll(buf); + self.hasher.update(buf[0..n]); + self.wp += n; + if (n < buf.len) break; // no more data in reader + } + } + + // Writer interface + + pub const Writer = io.Writer(*Self, Error, write); + pub const Error = BlockWriterType.Error; + + // Write `input` of uncompressed data. + pub fn write(self: *Self, input: []const u8) !usize { + var fbs = io.fixedBufferStream(input); + try self.compress(fbs.reader()); + return input.len; + } + + pub fn writer(self: *Self) Writer { + return .{ .context = self }; + } + }; +} + +const builtin = @import("builtin"); + +test "flate.Deflate tokenization" { + if (builtin.target.cpu.arch == .wasm32) return error.SkipZigTest; + + const L = Token.initLiteral; + const M = Token.initMatch; + + const cases = [_]struct { + data: []const u8, + tokens: []const Token, + }{ + .{ + .data = "Blah blah blah blah blah!", + .tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') }, + }, + .{ + .data = "ABCDEABCD ABCDEABCD", + .tokens = &[_]Token{ + L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '), + L('A'), M(10, 8), + }, + }, + }; + + for (cases) |c| { + inline for (Container.list) |container| { // for each wrapping + + var cw = io.countingWriter(io.null_writer); + const cww = cw.writer(); + var df = try Deflate(container, @TypeOf(cww), TestTokenWriter).init(cww, .{}); + + _ = try df.write(c.data); + try df.flush(); + + // df.token_writer.show(); + try expect(df.block_writer.pos == c.tokens.len); // number of tokens written + try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match + + try testing.expectEqual(container.headerSize(), cw.bytes_written); + try df.finish(); + try testing.expectEqual(container.size(), cw.bytes_written); + } + } +} + +// Tests that tokens writen are equal to expected token list. +const TestTokenWriter = struct { + const Self = @This(); + + pos: usize = 0, + actual: [128]Token = undefined, + + pub fn init(_: anytype) Self { + return .{}; + } + pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void { + for (tokens) |t| { + self.actual[self.pos] = t; + self.pos += 1; + } + } + + pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {} + + pub fn get(self: *Self) []Token { + return self.actual[0..self.pos]; + } + + pub fn show(self: *Self) void { + print("\n", .{}); + for (self.get()) |t| { + t.show(); + } + } + + pub fn flush(_: *Self) !void {} +}; + +test "flate deflate file tokenization" { + if (builtin.target.cpu.arch == .wasm32) return error.SkipZigTest; + + const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 }; + const cases = [_]struct { + data: []const u8, // uncompressed content + // expected number of tokens producet in deflate tokenization + tokens_count: [levels.len]usize = .{0} ** levels.len, + }{ + .{ + .data = @embedFile("testdata/rfc1951.txt"), + .tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 }, + }, + + .{ + .data = @embedFile("testdata/block_writer/huffman-null-max.input"), + .tokens_count = .{ 257, 257, 257, 257, 257, 257 }, + }, + .{ + .data = @embedFile("testdata/block_writer/huffman-pi.input"), + .tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 }, + }, + .{ + .data = @embedFile("testdata/block_writer/huffman-text.input"), + .tokens_count = .{ 235, 234, 234, 234, 234, 234 }, + }, + .{ + .data = @embedFile("testdata/fuzz/roundtrip1.input"), + .tokens_count = .{ 333, 331, 331, 331, 331, 331 }, + }, + .{ + .data = @embedFile("testdata/fuzz/roundtrip2.input"), + .tokens_count = .{ 334, 334, 334, 334, 334, 334 }, + }, + }; + + for (cases) |case| { // for each case + const data = case.data; + + for (levels, 0..) |level, i| { // for each compression level + var original = io.fixedBufferStream(data); + + // buffer for decompressed data + var al = std.ArrayList(u8).init(testing.allocator); + defer al.deinit(); + const writer = al.writer(); + + // create compressor + const WriterType = @TypeOf(writer); + const TokenWriter = TokenDecoder(@TypeOf(writer)); + var cmp = try Deflate(.raw, WriterType, TokenWriter).init(writer, .{ .level = level }); + + // Stream uncompressed `orignal` data to the compressor. It will + // produce tokens list and pass that list to the TokenDecoder. This + // TokenDecoder uses CircularBuffer from inflate to convert list of + // tokens back to the uncompressed stream. + try cmp.compress(original.reader()); + try cmp.flush(); + const expected_count = case.tokens_count[i]; + const actual = cmp.block_writer.tokens_count; + if (expected_count == 0) { + print("actual token count {d}\n", .{actual}); + } else { + try testing.expectEqual(expected_count, actual); + } + + try testing.expectEqual(data.len, al.items.len); + try testing.expectEqualSlices(u8, data, al.items); + } + } +} + +fn TokenDecoder(comptime WriterType: type) type { + return struct { + const CircularBuffer = @import("CircularBuffer.zig"); + hist: CircularBuffer = .{}, + wrt: WriterType, + tokens_count: usize = 0, + + const Self = @This(); + + pub fn init(wrt: WriterType) Self { + return .{ .wrt = wrt }; + } + + pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void { + self.tokens_count += tokens.len; + for (tokens) |t| { + switch (t.kind) { + .literal => self.hist.write(t.literal()), + .match => try self.hist.writeMatch(t.length(), t.distance()), + } + if (self.hist.free() < 285) try self.flushWin(); + } + try self.flushWin(); + } + + pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {} + + fn flushWin(self: *Self) !void { + while (true) { + const buf = self.hist.read(); + if (buf.len == 0) break; + try self.wrt.writeAll(buf); + } + } + + pub fn flush(_: *Self) !void {} + }; +} + +test "flate.Deflate store simple compressor" { + const data = "Hello world!"; + const expected = [_]u8{ + 0x1, // block type 0, final bit set + 0xc, 0x0, // len = 12 + 0xf3, 0xff, // ~len + 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', // + //0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21, + }; + + var fbs = std.io.fixedBufferStream(data); + var al = std.ArrayList(u8).init(testing.allocator); + defer al.deinit(); + + var cmp = try store.compressor(.raw, al.writer()); + try cmp.compress(fbs.reader()); + try cmp.finish(); + try testing.expectEqualSlices(u8, &expected, al.items); + + fbs.reset(); + try al.resize(0); + + // huffman only compresoor will also emit store block for this small sample + var hc = try huffman.compressor(.raw, al.writer()); + try hc.compress(fbs.reader()); + try hc.finish(); + try testing.expectEqualSlices(u8, &expected, al.items); +} diff --git a/lib/std/compress/flate/huffman_decoder.zig b/lib/std/compress/flate/huffman_decoder.zig new file mode 100644 index 000000000000..5bfe9242c73d --- /dev/null +++ b/lib/std/compress/flate/huffman_decoder.zig @@ -0,0 +1,308 @@ +const std = @import("std"); +const testing = std.testing; + +pub const Symbol = packed struct { + pub const Kind = enum(u2) { + literal, + end_of_block, + match, + }; + + symbol: u8 = 0, // symbol from alphabet + code_bits: u4 = 0, // number of bits in code 0-15 + kind: Kind = .literal, + + code: u16 = 0, // huffman code of the symbol + next: u16 = 0, // pointer to the next symbol in linked list + // it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup + + // Sorting less than function. + pub fn asc(_: void, a: Symbol, b: Symbol) bool { + if (a.code_bits == b.code_bits) { + if (a.kind == b.kind) { + return a.symbol < b.symbol; + } + return @intFromEnum(a.kind) < @intFromEnum(b.kind); + } + return a.code_bits < b.code_bits; + } +}; + +pub const LiteralDecoder = HuffmanDecoder(286, 15, 9); +pub const DistanceDecoder = HuffmanDecoder(30, 15, 9); +pub const CodegenDecoder = HuffmanDecoder(19, 7, 7); + +pub const Error = error{ + InvalidCode, + OversubscribedHuffmanTree, + IncompleteHuffmanTree, + MissingEndOfBlockCode, +}; + +/// Creates huffman tree codes from list of code lengths (in `build`). +/// +/// `find` then finds symbol for code bits. Code can be any length between 1 and +/// 15 bits. When calling `find` we don't know how many bits will be used to +/// find symbol. When symbol is returned it has code_bits field which defines +/// how much we should advance in bit stream. +/// +/// Lookup table is used to map 15 bit int to symbol. Same symbol is written +/// many times in this table; 32K places for 286 (at most) symbols. +/// Small lookup table is optimization for faster search. +/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92) +/// with difference that we here use statically allocated arrays. +/// +fn HuffmanDecoder( + comptime alphabet_size: u16, + comptime max_code_bits: u4, + comptime lookup_bits: u4, +) type { + const lookup_shift = max_code_bits - lookup_bits; + + return struct { + // all symbols in alaphabet, sorted by code_len, symbol + symbols: [alphabet_size]Symbol = undefined, + // lookup table code -> symbol + lookup: [1 << lookup_bits]Symbol = undefined, + + const Self = @This(); + + /// Generates symbols and lookup tables from list of code lens for each symbol. + pub fn generate(self: *Self, lens: []const u4) !void { + try checkCompletnes(lens); + + // init alphabet with code_bits + for (self.symbols, 0..) |_, i| { + const cb: u4 = if (i < lens.len) lens[i] else 0; + self.symbols[i] = if (i < 256) + .{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb } + else if (i == 256) + .{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb } + else + .{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb }; + } + std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc); + + // reset lookup table + for (0..self.lookup.len) |i| { + self.lookup[i] = .{}; + } + + // assign code to symbols + // reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639 + var code: u16 = 0; + var idx: u16 = 0; + for (&self.symbols, 0..) |*sym, pos| { + //print("sym: {}\n", .{sym}); + if (sym.code_bits == 0) continue; // skip unused + sym.code = code; + + const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits)); + const next_idx = next_code >> lookup_shift; + + if (next_idx > self.lookup.len or idx >= self.lookup.len) break; + if (sym.code_bits <= lookup_bits) { + // fill small lookup table + for (idx..next_idx) |j| + self.lookup[j] = sym.*; + } else { + // insert into linked table starting at root + const root = &self.lookup[idx]; + const root_next = root.next; + root.next = @intCast(pos); + sym.next = root_next; + } + + idx = next_idx; + code = next_code; + } + //print("decoder generate, code: {d}, idx: {d}\n", .{ code, idx }); + } + + /// Given the list of code lengths check that it represents a canonical + /// Huffman code for n symbols. + /// + /// Reference: https://github.com/madler/zlib/blob/5c42a230b7b468dff011f444161c0145b5efae59/contrib/puff/puff.c#L340 + fn checkCompletnes(lens: []const u4) !void { + if (alphabet_size == 286) + if (lens[256] == 0) return error.MissingEndOfBlockCode; + + var count = [_]u16{0} ** (@as(usize, max_code_bits) + 1); + var max: usize = 0; + for (lens) |n| { + if (n == 0) continue; + if (n > max) max = n; + count[n] += 1; + } + if (max == 0) // emtpy tree + return; + + // check for an over-subscribed or incomplete set of lengths + var left: usize = 1; // one possible code of zero length + for (1..count.len) |len| { + left <<= 1; // one more bit, double codes left + if (count[len] > left) + return error.OversubscribedHuffmanTree; + left -= count[len]; // deduct count from possible codes + } + if (left > 0) { // left > 0 means incomplete + // incomplete code ok only for single length 1 code + if (max_code_bits > 7 and max == count[0] + count[1]) return; + return error.IncompleteHuffmanTree; + } + } + + /// Finds symbol for lookup table code. + pub fn find(self: *Self, code: u16) !Symbol { + // try to find in lookup table + const idx = code >> lookup_shift; + const sym = self.lookup[idx]; + if (sym.code_bits != 0) return sym; + // if not use linked list of symbols with same prefix + return self.findLinked(code, sym.next); + } + + inline fn findLinked(self: *Self, code: u16, start: u16) !Symbol { + var pos = start; + while (pos > 0) { + const sym = self.symbols[pos]; + const shift = max_code_bits - sym.code_bits; + // compare code_bits number of upper bits + if ((code ^ sym.code) >> shift == 0) return sym; + pos = sym.next; + } + return error.InvalidCode; + } + }; +} + +test "flate.HuffmanDecoder init/find" { + // example data from: https://youtu.be/SJPvNi4HrWQ?t=8423 + const code_lens = [_]u4{ 4, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2 }; + var h: CodegenDecoder = .{}; + try h.generate(&code_lens); + + const expected = [_]struct { + sym: Symbol, + code: u16, + }{ + .{ + .code = 0b00_00000, + .sym = .{ .symbol = 3, .code_bits = 2 }, + }, + .{ + .code = 0b01_00000, + .sym = .{ .symbol = 18, .code_bits = 2 }, + }, + .{ + .code = 0b100_0000, + .sym = .{ .symbol = 1, .code_bits = 3 }, + }, + .{ + .code = 0b101_0000, + .sym = .{ .symbol = 4, .code_bits = 3 }, + }, + .{ + .code = 0b110_0000, + .sym = .{ .symbol = 17, .code_bits = 3 }, + }, + .{ + .code = 0b1110_000, + .sym = .{ .symbol = 0, .code_bits = 4 }, + }, + .{ + .code = 0b1111_000, + .sym = .{ .symbol = 16, .code_bits = 4 }, + }, + }; + + // unused symbols + for (0..12) |i| { + try testing.expectEqual(0, h.symbols[i].code_bits); + } + // used, from index 12 + for (expected, 12..) |e, i| { + try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol); + try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits); + const sym_from_code = try h.find(e.code); + try testing.expectEqual(e.sym.symbol, sym_from_code.symbol); + } + + // All possible codes for each symbol. + // Lookup table has 126 elements, to cover all possible 7 bit codes. + for (0b0000_000..0b0100_000) |c| // 0..32 (32) + try testing.expectEqual(3, (try h.find(@intCast(c))).symbol); + + for (0b0100_000..0b1000_000) |c| // 32..64 (32) + try testing.expectEqual(18, (try h.find(@intCast(c))).symbol); + + for (0b1000_000..0b1010_000) |c| // 64..80 (16) + try testing.expectEqual(1, (try h.find(@intCast(c))).symbol); + + for (0b1010_000..0b1100_000) |c| // 80..96 (16) + try testing.expectEqual(4, (try h.find(@intCast(c))).symbol); + + for (0b1100_000..0b1110_000) |c| // 96..112 (16) + try testing.expectEqual(17, (try h.find(@intCast(c))).symbol); + + for (0b1110_000..0b1111_000) |c| // 112..120 (8) + try testing.expectEqual(0, (try h.find(@intCast(c))).symbol); + + for (0b1111_000..0b1_0000_000) |c| // 120...128 (8) + try testing.expectEqual(16, (try h.find(@intCast(c))).symbol); +} + +const print = std.debug.print; +const assert = std.debug.assert; +const expect = std.testing.expect; + +test "flate.HuffmanDecoder encode/decode literals" { + const LiteralEncoder = @import("huffman_encoder.zig").LiteralEncoder; + + for (1..286) |j| { // for all different number of codes + var enc: LiteralEncoder = .{}; + // create freqencies + var freq = [_]u16{0} ** 286; + freq[256] = 1; // ensure we have end of block code + for (&freq, 1..) |*f, i| { + if (i % j == 0) + f.* = @intCast(i); + } + + // encoder from freqencies + enc.generate(&freq, 15); + + // get code_lens from encoder + var code_lens = [_]u4{0} ** 286; + for (code_lens, 0..) |_, i| { + code_lens[i] = @intCast(enc.codes[i].len); + } + // generate decoder from code lens + var dec: LiteralDecoder = .{}; + try dec.generate(&code_lens); + + // expect decoder code to match original encoder code + for (dec.symbols) |s| { + if (s.code_bits == 0) continue; + const c_code: u16 = @bitReverse(@as(u15, @intCast(s.code))); + const symbol: u16 = switch (s.kind) { + .literal => s.symbol, + .end_of_block => 256, + .match => @as(u16, s.symbol) + 257, + }; + + const c = enc.codes[symbol]; + try expect(c.code == c_code); + } + + // find each symbol by code + for (enc.codes) |c| { + if (c.len == 0) continue; + + const s_code: u15 = @bitReverse(@as(u15, @intCast(c.code))); + const s = try dec.find(s_code); + try expect(s.code == s_code); + try expect(s.code_bits == c.len); + } + } +} diff --git a/lib/std/compress/flate/huffman_encoder.zig b/lib/std/compress/flate/huffman_encoder.zig new file mode 100644 index 000000000000..a8553ebb5e16 --- /dev/null +++ b/lib/std/compress/flate/huffman_encoder.zig @@ -0,0 +1,536 @@ +const std = @import("std"); +const assert = std.debug.assert; +const math = std.math; +const mem = std.mem; +const sort = std.sort; +const testing = std.testing; + +const consts = @import("consts.zig").huffman; + +const LiteralNode = struct { + literal: u16, + freq: u16, +}; + +// Describes the state of the constructed tree for a given depth. +const LevelInfo = struct { + // Our level. for better printing + level: u32, + + // The frequency of the last node at this level + last_freq: u32, + + // The frequency of the next character to add to this level + next_char_freq: u32, + + // The frequency of the next pair (from level below) to add to this level. + // Only valid if the "needed" value of the next lower level is 0. + next_pair_freq: u32, + + // The number of chains remaining to generate for this level before moving + // up to the next level + needed: u32, +}; + +// hcode is a huffman code with a bit code and bit length. +pub const HuffCode = struct { + code: u16 = 0, + len: u16 = 0, + + // set sets the code and length of an hcode. + fn set(self: *HuffCode, code: u16, length: u16) void { + self.len = length; + self.code = code; + } +}; + +pub fn HuffmanEncoder(comptime size: usize) type { + return struct { + codes: [size]HuffCode = undefined, + // Reusable buffer with the longest possible frequency table. + freq_cache: [consts.max_num_frequencies + 1]LiteralNode = undefined, + bit_count: [17]u32 = undefined, + lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate + lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate + + const Self = @This(); + + // Update this Huffman Code object to be the minimum code for the specified frequency count. + // + // freq An array of frequencies, in which frequency[i] gives the frequency of literal i. + // max_bits The maximum number of bits to use for any literal. + pub fn generate(self: *Self, freq: []u16, max_bits: u32) void { + var list = self.freq_cache[0 .. freq.len + 1]; + // Number of non-zero literals + var count: u32 = 0; + // Set list to be the set of all non-zero literals and their frequencies + for (freq, 0..) |f, i| { + if (f != 0) { + list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f }; + count += 1; + } else { + list[count] = LiteralNode{ .literal = 0x00, .freq = 0 }; + self.codes[i].len = 0; + } + } + list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 }; + + list = list[0..count]; + if (count <= 2) { + // Handle the small cases here, because they are awkward for the general case code. With + // two or fewer literals, everything has bit length 1. + for (list, 0..) |node, i| { + // "list" is in order of increasing literal value. + self.codes[node.literal].set(@as(u16, @intCast(i)), 1); + } + return; + } + self.lfs = list; + mem.sort(LiteralNode, self.lfs, {}, byFreq); + + // Get the number of literals for each bit count + const bit_count = self.bitCounts(list, max_bits); + // And do the assignment + self.assignEncodingAndSize(bit_count, list); + } + + pub fn bitLength(self: *Self, freq: []u16) u32 { + var total: u32 = 0; + for (freq, 0..) |f, i| { + if (f != 0) { + total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len)); + } + } + return total; + } + + // Return the number of literals assigned to each bit size in the Huffman encoding + // + // This method is only called when list.len >= 3 + // The cases of 0, 1, and 2 literals are handled by special case code. + // + // list: An array of the literals with non-zero frequencies + // and their associated frequencies. The array is in order of increasing + // frequency, and has as its last element a special element with frequency + // std.math.maxInt(i32) + // + // max_bits: The maximum number of bits that should be used to encode any literal. + // Must be less than 16. + // + // Returns an integer array in which array[i] indicates the number of literals + // that should be encoded in i bits. + fn bitCounts(self: *Self, list: []LiteralNode, max_bits_to_use: usize) []u32 { + var max_bits = max_bits_to_use; + const n = list.len; + const max_bits_limit = 16; + + assert(max_bits < max_bits_limit); + + // The tree can't have greater depth than n - 1, no matter what. This + // saves a little bit of work in some small cases + max_bits = @min(max_bits, n - 1); + + // Create information about each of the levels. + // A bogus "Level 0" whose sole purpose is so that + // level1.prev.needed == 0. This makes level1.next_pair_freq + // be a legitimate value that never gets chosen. + var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo); + // leaf_counts[i] counts the number of literals at the left + // of ancestors of the rightmost node at level i. + // leaf_counts[i][j] is the number of literals at the left + // of the level j ancestor. + var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32); + + { + var level = @as(u32, 1); + while (level <= max_bits) : (level += 1) { + // For every level, the first two items are the first two characters. + // We initialize the levels as if we had already figured this out. + levels[level] = LevelInfo{ + .level = level, + .last_freq = list[1].freq, + .next_char_freq = list[2].freq, + .next_pair_freq = list[0].freq + list[1].freq, + .needed = 0, + }; + leaf_counts[level][level] = 2; + if (level == 1) { + levels[level].next_pair_freq = math.maxInt(i32); + } + } + } + + // We need a total of 2*n - 2 items at top level and have already generated 2. + levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4; + + { + var level = max_bits; + while (true) { + var l = &levels[level]; + if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) { + // We've run out of both leafs and pairs. + // End all calculations for this level. + // To make sure we never come back to this level or any lower level, + // set next_pair_freq impossibly large. + l.needed = 0; + levels[level + 1].next_pair_freq = math.maxInt(i32); + level += 1; + continue; + } + + const prev_freq = l.last_freq; + if (l.next_char_freq < l.next_pair_freq) { + // The next item on this row is a leaf node. + const next = leaf_counts[level][level] + 1; + l.last_freq = l.next_char_freq; + // Lower leaf_counts are the same of the previous node. + leaf_counts[level][level] = next; + if (next >= list.len) { + l.next_char_freq = maxNode().freq; + } else { + l.next_char_freq = list[next].freq; + } + } else { + // The next item on this row is a pair from the previous row. + // next_pair_freq isn't valid until we generate two + // more values in the level below + l.last_freq = l.next_pair_freq; + // Take leaf counts from the lower level, except counts[level] remains the same. + @memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]); + levels[l.level - 1].needed = 2; + } + + l.needed -= 1; + if (l.needed == 0) { + // We've done everything we need to do for this level. + // Continue calculating one level up. Fill in next_pair_freq + // of that level with the sum of the two nodes we've just calculated on + // this level. + if (l.level == max_bits) { + // All done! + break; + } + levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq; + level += 1; + } else { + // If we stole from below, move down temporarily to replenish it. + while (levels[level - 1].needed > 0) { + level -= 1; + if (level == 0) { + break; + } + } + } + } + } + + // Somethings is wrong if at the end, the top level is null or hasn't used + // all of the leaves. + assert(leaf_counts[max_bits][max_bits] == n); + + var bit_count = self.bit_count[0 .. max_bits + 1]; + var bits: u32 = 1; + const counts = &leaf_counts[max_bits]; + { + var level = max_bits; + while (level > 0) : (level -= 1) { + // counts[level] gives the number of literals requiring at least "bits" + // bits to encode. + bit_count[bits] = counts[level] - counts[level - 1]; + bits += 1; + if (level == 0) { + break; + } + } + } + return bit_count; + } + + // Look at the leaves and assign them a bit count and an encoding as specified + // in RFC 1951 3.2.2 + fn assignEncodingAndSize(self: *Self, bit_count: []u32, list_arg: []LiteralNode) void { + var code = @as(u16, 0); + var list = list_arg; + + for (bit_count, 0..) |bits, n| { + code <<= 1; + if (n == 0 or bits == 0) { + continue; + } + // The literals list[list.len-bits] .. list[list.len-bits] + // are encoded using "bits" bits, and get the values + // code, code + 1, .... The code values are + // assigned in literal order (not frequency order). + const chunk = list[list.len - @as(u32, @intCast(bits)) ..]; + + self.lns = chunk; + mem.sort(LiteralNode, self.lns, {}, byLiteral); + + for (chunk) |node| { + self.codes[node.literal] = HuffCode{ + .code = bitReverse(u16, code, @as(u5, @intCast(n))), + .len = @as(u16, @intCast(n)), + }; + code += 1; + } + list = list[0 .. list.len - @as(u32, @intCast(bits))]; + } + } + }; +} + +fn maxNode() LiteralNode { + return LiteralNode{ + .literal = math.maxInt(u16), + .freq = math.maxInt(u16), + }; +} + +pub fn huffmanEncoder(comptime size: u32) HuffmanEncoder(size) { + return .{}; +} + +pub const LiteralEncoder = HuffmanEncoder(consts.max_num_frequencies); +pub const DistanceEncoder = HuffmanEncoder(consts.distance_code_count); +pub const CodegenEncoder = HuffmanEncoder(19); + +// Generates a HuffmanCode corresponding to the fixed literal table +pub fn fixedLiteralEncoder() LiteralEncoder { + var h: LiteralEncoder = undefined; + var ch: u16 = 0; + + while (ch < consts.max_num_frequencies) : (ch += 1) { + var bits: u16 = undefined; + var size: u16 = undefined; + switch (ch) { + 0...143 => { + // size 8, 000110000 .. 10111111 + bits = ch + 48; + size = 8; + }, + 144...255 => { + // size 9, 110010000 .. 111111111 + bits = ch + 400 - 144; + size = 9; + }, + 256...279 => { + // size 7, 0000000 .. 0010111 + bits = ch - 256; + size = 7; + }, + else => { + // size 8, 11000000 .. 11000111 + bits = ch + 192 - 280; + size = 8; + }, + } + h.codes[ch] = HuffCode{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size }; + } + return h; +} + +pub fn fixedDistanceEncoder() DistanceEncoder { + var h: DistanceEncoder = undefined; + for (h.codes, 0..) |_, ch| { + h.codes[ch] = HuffCode{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 }; + } + return h; +} + +pub fn huffmanDistanceEncoder() DistanceEncoder { + var distance_freq = [1]u16{0} ** consts.distance_code_count; + distance_freq[0] = 1; + // huff_distance is a static distance encoder used for huffman only encoding. + // It can be reused since we will not be encoding distance values. + var h: DistanceEncoder = .{}; + h.generate(distance_freq[0..], 15); + return h; +} + +fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool { + _ = context; + return a.literal < b.literal; +} + +fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool { + _ = context; + if (a.freq == b.freq) { + return a.literal < b.literal; + } + return a.freq < b.freq; +} + +test "flate.HuffmanEncoder generate a Huffman code from an array of frequencies" { + var freqs: [19]u16 = [_]u16{ + 8, // 0 + 1, // 1 + 1, // 2 + 2, // 3 + 5, // 4 + 10, // 5 + 9, // 6 + 1, // 7 + 0, // 8 + 0, // 9 + 0, // 10 + 0, // 11 + 0, // 12 + 0, // 13 + 0, // 14 + 0, // 15 + 1, // 16 + 3, // 17 + 5, // 18 + }; + + var enc = huffmanEncoder(19); + enc.generate(freqs[0..], 7); + + try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..])); + + try testing.expectEqual(@as(usize, 3), enc.codes[0].len); + try testing.expectEqual(@as(usize, 6), enc.codes[1].len); + try testing.expectEqual(@as(usize, 6), enc.codes[2].len); + try testing.expectEqual(@as(usize, 5), enc.codes[3].len); + try testing.expectEqual(@as(usize, 3), enc.codes[4].len); + try testing.expectEqual(@as(usize, 2), enc.codes[5].len); + try testing.expectEqual(@as(usize, 2), enc.codes[6].len); + try testing.expectEqual(@as(usize, 6), enc.codes[7].len); + try testing.expectEqual(@as(usize, 0), enc.codes[8].len); + try testing.expectEqual(@as(usize, 0), enc.codes[9].len); + try testing.expectEqual(@as(usize, 0), enc.codes[10].len); + try testing.expectEqual(@as(usize, 0), enc.codes[11].len); + try testing.expectEqual(@as(usize, 0), enc.codes[12].len); + try testing.expectEqual(@as(usize, 0), enc.codes[13].len); + try testing.expectEqual(@as(usize, 0), enc.codes[14].len); + try testing.expectEqual(@as(usize, 0), enc.codes[15].len); + try testing.expectEqual(@as(usize, 6), enc.codes[16].len); + try testing.expectEqual(@as(usize, 5), enc.codes[17].len); + try testing.expectEqual(@as(usize, 3), enc.codes[18].len); + + try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code); + try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code); + try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code); + try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code); + try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code); + try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code); + try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code); + try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code); + try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code); + try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code); + try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code); +} + +test "flate.HuffmanEncoder generate a Huffman code for the fixed literal table specific to Deflate" { + const enc = fixedLiteralEncoder(); + for (enc.codes) |c| { + switch (c.len) { + 7 => { + const v = @bitReverse(@as(u7, @intCast(c.code))); + try testing.expect(v <= 0b0010111); + }, + 8 => { + const v = @bitReverse(@as(u8, @intCast(c.code))); + try testing.expect((v >= 0b000110000 and v <= 0b10111111) or + (v >= 0b11000000 and v <= 11000111)); + }, + 9 => { + const v = @bitReverse(@as(u9, @intCast(c.code))); + try testing.expect(v >= 0b110010000 and v <= 0b111111111); + }, + else => unreachable, + } + } +} + +test "flate.HuffmanEncoder generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" { + const enc = fixedDistanceEncoder(); + for (enc.codes) |c| { + const v = @bitReverse(@as(u5, @intCast(c.code))); + try testing.expect(v <= 29); + try testing.expect(c.len == 5); + } +} + +// Reverse bit-by-bit a N-bit code. +fn bitReverse(comptime T: type, value: T, n: usize) T { + const r = @bitReverse(value); + return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).Int.bits - n)); +} + +test "flate bitReverse" { + const ReverseBitsTest = struct { + in: u16, + bit_count: u5, + out: u16, + }; + + const reverse_bits_tests = [_]ReverseBitsTest{ + .{ .in = 1, .bit_count = 1, .out = 1 }, + .{ .in = 1, .bit_count = 2, .out = 2 }, + .{ .in = 1, .bit_count = 3, .out = 4 }, + .{ .in = 1, .bit_count = 4, .out = 8 }, + .{ .in = 1, .bit_count = 5, .out = 16 }, + .{ .in = 17, .bit_count = 5, .out = 17 }, + .{ .in = 257, .bit_count = 9, .out = 257 }, + .{ .in = 29, .bit_count = 5, .out = 23 }, + }; + + for (reverse_bits_tests) |h| { + const v = bitReverse(u16, h.in, h.bit_count); + try std.testing.expectEqual(h.out, v); + } +} + +test "flate.HuffmanEncoder fixedLiteralEncoder codes" { + var al = std.ArrayList(u8).init(testing.allocator); + defer al.deinit(); + var bw = std.io.bitWriter(.little, al.writer()); + + const f = fixedLiteralEncoder(); + for (f.codes) |c| { + try bw.writeBits(c.code, c.len); + } + try testing.expectEqualSlices(u8, &fixed_codes, al.items); +} + +pub const fixed_codes = [_]u8{ + 0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100, + 0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100, + 0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010, + 0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010, + 0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010, + 0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010, + 0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110, + 0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110, + 0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110, + 0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110, + 0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001, + 0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001, + 0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001, + 0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001, + 0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101, + 0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101, + 0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101, + 0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101, + 0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100, + 0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110, + 0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100, + 0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101, + 0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010, + 0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110, + 0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110, + 0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111, + 0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011, + 0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101, + 0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110, + 0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100, + 0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101, + 0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010, + 0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110, + 0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000, + 0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010, + 0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100, + 0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011, + 0b10100011, +}; diff --git a/lib/std/compress/flate/inflate.zig b/lib/std/compress/flate/inflate.zig new file mode 100644 index 000000000000..b3cca8866e24 --- /dev/null +++ b/lib/std/compress/flate/inflate.zig @@ -0,0 +1,529 @@ +const std = @import("std"); +const assert = std.debug.assert; +const testing = std.testing; + +const hfd = @import("huffman_decoder.zig"); +const BitReader = @import("bit_reader.zig").BitReader; +const CircularBuffer = @import("CircularBuffer.zig"); +const Container = @import("container.zig").Container; +const Token = @import("Token.zig"); +const codegen_order = @import("consts.zig").huffman.codegen_order; + +/// Decompresses deflate bit stream `reader` and writes uncompressed data to the +/// `writer` stream. +pub fn decompress(comptime container: Container, reader: anytype, writer: anytype) !void { + var d = decompressor(container, reader); + try d.decompress(writer); +} + +/// Inflate decompressor for the reader type. +pub fn decompressor(comptime container: Container, reader: anytype) Inflate(container, @TypeOf(reader)) { + return Inflate(container, @TypeOf(reader)).init(reader); +} + +/// Inflate decompresses deflate bit stream. Reads compressed data from reader +/// provided in init. Decompressed data are stored in internal hist buffer and +/// can be accesses iterable `next` or reader interface. +/// +/// Container defines header/footer wrapper around deflate bit stream. Can be +/// gzip or zlib. +/// +/// Deflate bit stream consists of multiple blocks. Block can be one of three types: +/// * stored, non compressed, max 64k in size +/// * fixed, huffman codes are predefined +/// * dynamic, huffman code tables are encoded at the block start +/// +/// `step` function runs decoder until internal `hist` buffer is full. Client +/// than needs to read that data in order to proceed with decoding. +/// +/// Allocates 74.5K of internal buffers, most important are: +/// * 64K for history (CircularBuffer) +/// * ~10K huffman decoders (Literal and DistanceDecoder) +/// +pub fn Inflate(comptime container: Container, comptime ReaderType: type) type { + return struct { + const BitReaderType = BitReader(ReaderType); + const F = BitReaderType.flag; + + bits: BitReaderType = .{}, + hist: CircularBuffer = .{}, + // Hashes, produces checkusm, of uncompressed data for gzip/zlib footer. + hasher: container.Hasher() = .{}, + + // dynamic block huffman code decoders + lit_dec: hfd.LiteralDecoder = .{}, // literals + dst_dec: hfd.DistanceDecoder = .{}, // distances + + // current read state + bfinal: u1 = 0, + block_type: u2 = 0b11, + state: ReadState = .protocol_header, + + const ReadState = enum { + protocol_header, + block_header, + block, + protocol_footer, + end, + }; + + const Self = @This(); + + pub const Error = BitReaderType.Error || Container.Error || hfd.Error || error{ + InvalidCode, + InvalidMatch, + InvalidBlockType, + WrongStoredBlockNlen, + InvalidDynamicBlockHeader, + }; + + pub fn init(rt: ReaderType) Self { + return .{ .bits = BitReaderType.init(rt) }; + } + + fn blockHeader(self: *Self) !void { + self.bfinal = try self.bits.read(u1); + self.block_type = try self.bits.read(u2); + } + + fn storedBlock(self: *Self) !bool { + self.bits.alignToByte(); // skip padding until byte boundary + // everyting after this is byte aligned in stored block + var len = try self.bits.read(u16); + const nlen = try self.bits.read(u16); + if (len != ~nlen) return error.WrongStoredBlockNlen; + + while (len > 0) { + const buf = self.hist.getWritable(len); + try self.bits.readAll(buf); + len -= @intCast(buf.len); + } + return true; + } + + fn fixedBlock(self: *Self) !bool { + while (!self.hist.full()) { + const code = try self.bits.readFixedCode(); + switch (code) { + 0...255 => self.hist.write(@intCast(code)), + 256 => return true, // end of block + 257...285 => try self.fixedDistanceCode(@intCast(code - 257)), + else => return error.InvalidCode, + } + } + return false; + } + + // Handles fixed block non literal (length) code. + // Length code is followed by 5 bits of distance code. + fn fixedDistanceCode(self: *Self, code: u8) !void { + try self.bits.fill(5 + 5 + 13); + const length = try self.decodeLength(code); + const distance = try self.decodeDistance(try self.bits.readF(u5, F.buffered | F.reverse)); + try self.hist.writeMatch(length, distance); + } + + inline fn decodeLength(self: *Self, code: u8) !u16 { + if (code > 28) return error.InvalidCode; + const ml = Token.matchLength(code); + return if (ml.extra_bits == 0) // 0 - 5 extra bits + ml.base + else + ml.base + try self.bits.readN(ml.extra_bits, F.buffered); + } + + fn decodeDistance(self: *Self, code: u8) !u16 { + if (code > 29) return error.InvalidCode; + const md = Token.matchDistance(code); + return if (md.extra_bits == 0) // 0 - 13 extra bits + md.base + else + md.base + try self.bits.readN(md.extra_bits, F.buffered); + } + + fn dynamicBlockHeader(self: *Self) !void { + const hlit: u16 = @as(u16, try self.bits.read(u5)) + 257; // number of ll code entries present - 257 + const hdist: u16 = @as(u16, try self.bits.read(u5)) + 1; // number of distance code entries - 1 + const hclen: u8 = @as(u8, try self.bits.read(u4)) + 4; // hclen + 4 code lenths are encoded + + if (hlit > 286 or hdist > 30) + return error.InvalidDynamicBlockHeader; + + // lengths for code lengths + var cl_lens = [_]u4{0} ** 19; + for (0..hclen) |i| { + cl_lens[codegen_order[i]] = try self.bits.read(u3); + } + var cl_dec: hfd.CodegenDecoder = .{}; + try cl_dec.generate(&cl_lens); + + // literal code lengths + var lit_lens = [_]u4{0} ** (286); + var pos: usize = 0; + while (pos < hlit) { + const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse)); + try self.bits.shift(sym.code_bits); + pos += try self.dynamicCodeLength(sym.symbol, &lit_lens, pos); + } + if (pos > hlit) + return error.InvalidDynamicBlockHeader; + + // distance code lenths + var dst_lens = [_]u4{0} ** (30); + pos = 0; + while (pos < hdist) { + const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse)); + try self.bits.shift(sym.code_bits); + pos += try self.dynamicCodeLength(sym.symbol, &dst_lens, pos); + } + if (pos > hdist) + return error.InvalidDynamicBlockHeader; + + try self.lit_dec.generate(&lit_lens); + try self.dst_dec.generate(&dst_lens); + } + + // Decode code length symbol to code length. Writes decoded length into + // lens slice starting at position pos. Returns number of positions + // advanced. + fn dynamicCodeLength(self: *Self, code: u16, lens: []u4, pos: usize) !usize { + if (pos >= lens.len) + return error.InvalidDynamicBlockHeader; + + switch (code) { + 0...15 => { + // Represent code lengths of 0 - 15 + lens[pos] = @intCast(code); + return 1; + }, + 16 => { + // Copy the previous code length 3 - 6 times. + // The next 2 bits indicate repeat length + const n: u8 = @as(u8, try self.bits.read(u2)) + 3; + if (pos == 0 or pos + n > lens.len) + return error.InvalidDynamicBlockHeader; + for (0..n) |i| { + lens[pos + i] = lens[pos + i - 1]; + } + return n; + }, + // Repeat a code length of 0 for 3 - 10 times. (3 bits of length) + 17 => return @as(u8, try self.bits.read(u3)) + 3, + // Repeat a code length of 0 for 11 - 138 times (7 bits of length) + 18 => return @as(u8, try self.bits.read(u7)) + 11, + else => return error.InvalidDynamicBlockHeader, + } + } + + // In larger archives most blocks are usually dynamic, so decompression + // performance depends on this function. + fn dynamicBlock(self: *Self) !bool { + // Hot path loop! + while (!self.hist.full()) { + try self.bits.fill(15); // optimization so other bit reads can be buffered (avoiding one `if` in hot path) + const sym = try self.decodeSymbol(&self.lit_dec); + + switch (sym.kind) { + .literal => self.hist.write(sym.symbol), + .match => { // Decode match backreference + try self.bits.fill(5 + 15 + 13); // so we can use buffered reads + const length = try self.decodeLength(sym.symbol); + const dsm = try self.decodeSymbol(&self.dst_dec); + const distance = try self.decodeDistance(dsm.symbol); + try self.hist.writeMatch(length, distance); + }, + .end_of_block => return true, + } + } + return false; + } + + // Peek 15 bits from bits reader (maximum code len is 15 bits). Use + // decoder to find symbol for that code. We then know how many bits is + // used. Shift bit reader for that much bits, those bits are used. And + // return symbol. + fn decodeSymbol(self: *Self, decoder: anytype) !hfd.Symbol { + const sym = try decoder.find(try self.bits.peekF(u15, F.buffered | F.reverse)); + try self.bits.shift(sym.code_bits); + return sym; + } + + fn step(self: *Self) !void { + switch (self.state) { + .protocol_header => { + try container.parseHeader(&self.bits); + self.state = .block_header; + }, + .block_header => { + try self.blockHeader(); + self.state = .block; + if (self.block_type == 2) try self.dynamicBlockHeader(); + }, + .block => { + const done = switch (self.block_type) { + 0 => try self.storedBlock(), + 1 => try self.fixedBlock(), + 2 => try self.dynamicBlock(), + else => return error.InvalidBlockType, + }; + if (done) { + self.state = if (self.bfinal == 1) .protocol_footer else .block_header; + } + }, + .protocol_footer => { + self.bits.alignToByte(); + try container.parseFooter(&self.hasher, &self.bits); + self.state = .end; + }, + .end => {}, + } + } + + /// Replaces the inner reader with new reader. + pub fn setReader(self: *Self, new_reader: ReaderType) void { + self.bits.forward_reader = new_reader; + if (self.state == .end or self.state == .protocol_footer) { + self.state = .protocol_header; + } + } + + // Reads all compressed data from the internal reader and outputs plain + // (uncompressed) data to the provided writer. + pub fn decompress(self: *Self, writer: anytype) !void { + while (try self.next()) |buf| { + try writer.writeAll(buf); + } + } + + // Iterator interface + + /// Can be used in iterator like loop without memcpy to another buffer: + /// while (try inflate.next()) |buf| { ... } + pub fn next(self: *Self) Error!?[]const u8 { + const out = try self.get(0); + if (out.len == 0) return null; + return out; + } + + /// Returns decompressed data from internal sliding window buffer. + /// Returned buffer can be any length between 0 and `limit` bytes. 0 + /// returned bytes means end of stream reached. With limit=0 returns as + /// much data it can. It newer will be more than 65536 bytes, which is + /// size of internal buffer. + pub fn get(self: *Self, limit: usize) Error![]const u8 { + while (true) { + const out = self.hist.readAtMost(limit); + if (out.len > 0) { + self.hasher.update(out); + return out; + } + if (self.state == .end) return out; + try self.step(); + } + } + + // Reader interface + + pub const Reader = std.io.Reader(*Self, Error, read); + + /// Returns the number of bytes read. It may be less than buffer.len. + /// If the number of bytes read is 0, it means end of stream. + /// End of stream is not an error condition. + pub fn read(self: *Self, buffer: []u8) Error!usize { + const out = try self.get(buffer.len); + @memcpy(buffer[0..out.len], out); + return out.len; + } + + pub fn reader(self: *Self) Reader { + return .{ .context = self }; + } + }; +} + +test "flate.Inflate decompress" { + const cases = [_]struct { + in: []const u8, + out: []const u8, + }{ + // non compressed block (type 0) + .{ + .in = &[_]u8{ + 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen + 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data + }, + .out = "Hello world\n", + }, + // fixed code block (type 1) + .{ + .in = &[_]u8{ + 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 + 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, + }, + .out = "Hello world\n", + }, + // dynamic block (type 2) + .{ + .in = &[_]u8{ + 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2 + 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38, + 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01, + }, + .out = "ABCDEABCD ABCDEABCD", + }, + }; + for (cases) |c| { + var fb = std.io.fixedBufferStream(c.in); + var al = std.ArrayList(u8).init(testing.allocator); + defer al.deinit(); + + try decompress(.raw, fb.reader(), al.writer()); + try testing.expectEqualStrings(c.out, al.items); + } +} + +test "flate.Inflate gzip decompress" { + const cases = [_]struct { + in: []const u8, + out: []const u8, + }{ + // non compressed block (type 0) + .{ + .in = &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes) + 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen + 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data + 0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum + 0x0c, 0x00, 0x00, 0x00, // gzip footer: size + }, + .out = "Hello world\n", + }, + // fixed code block (type 1) + .{ + .in = &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes) + 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1 + 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00, + 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len) + }, + .out = "Hello world\n", + }, + // dynamic block (type 2) + .{ + .in = &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes) + 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2 + 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38, + 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01, + 0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len) + }, + .out = "ABCDEABCD ABCDEABCD", + }, + // gzip header with name + .{ + .in = &[_]u8{ + 0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, + }, + .out = "Hello world\n", + }, + }; + for (cases) |c| { + var fb = std.io.fixedBufferStream(c.in); + var al = std.ArrayList(u8).init(testing.allocator); + defer al.deinit(); + + try decompress(.gzip, fb.reader(), al.writer()); + try testing.expectEqualStrings(c.out, al.items); + } +} + +test "flate.Inflate zlib decompress" { + const cases = [_]struct { + in: []const u8, + out: []const u8, + }{ + // non compressed block (type 0) + .{ + .in = &[_]u8{ + 0x78, 0b10_0_11100, // zlib header (2 bytes) + 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen + 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data + 0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum + }, + .out = "Hello world\n", + }, + }; + for (cases) |c| { + var fb = std.io.fixedBufferStream(c.in); + var al = std.ArrayList(u8).init(testing.allocator); + defer al.deinit(); + + try decompress(.zlib, fb.reader(), al.writer()); + try testing.expectEqualStrings(c.out, al.items); + } +} + +test "flate.Inflate fuzzing tests" { + const cases = [_]struct { + input: []const u8, + out: []const u8 = "", + err: ?anyerror = null, + }{ + .{ .input = "deflate-stream", .out = @embedFile("testdata/fuzz/deflate-stream.expect") }, // 0 + .{ .input = "empty-distance-alphabet01" }, + .{ .input = "empty-distance-alphabet02" }, + .{ .input = "end-of-stream", .err = error.EndOfStream }, + .{ .input = "invalid-distance", .err = error.InvalidMatch }, + .{ .input = "invalid-tree01", .err = error.IncompleteHuffmanTree }, // 5 + .{ .input = "invalid-tree02", .err = error.IncompleteHuffmanTree }, + .{ .input = "invalid-tree03", .err = error.IncompleteHuffmanTree }, + .{ .input = "lengths-overflow", .err = error.InvalidDynamicBlockHeader }, + .{ .input = "out-of-codes", .err = error.InvalidCode }, + .{ .input = "puff01", .err = error.WrongStoredBlockNlen }, // 10 + .{ .input = "puff02", .err = error.EndOfStream }, + .{ .input = "puff03", .out = &[_]u8{0xa} }, + .{ .input = "puff04", .err = error.InvalidCode }, + .{ .input = "puff05", .err = error.EndOfStream }, + .{ .input = "puff06", .err = error.EndOfStream }, + .{ .input = "puff08", .err = error.InvalidCode }, + .{ .input = "puff09", .out = "P" }, + .{ .input = "puff10", .err = error.InvalidCode }, + .{ .input = "puff11", .err = error.InvalidMatch }, + .{ .input = "puff12", .err = error.InvalidDynamicBlockHeader }, // 20 + .{ .input = "puff13", .err = error.IncompleteHuffmanTree }, + .{ .input = "puff14", .err = error.EndOfStream }, + .{ .input = "puff15", .err = error.IncompleteHuffmanTree }, + .{ .input = "puff16", .err = error.InvalidDynamicBlockHeader }, + .{ .input = "puff17", .err = error.InvalidDynamicBlockHeader }, // 25 + .{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader }, + .{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader }, + .{ .input = "fuzz3", .err = error.InvalidMatch }, + .{ .input = "fuzz4", .err = error.OversubscribedHuffmanTree }, + .{ .input = "puff18", .err = error.OversubscribedHuffmanTree }, // 30 + .{ .input = "puff19", .err = error.OversubscribedHuffmanTree }, + .{ .input = "puff20", .err = error.OversubscribedHuffmanTree }, + .{ .input = "puff21", .err = error.OversubscribedHuffmanTree }, + .{ .input = "puff22", .err = error.OversubscribedHuffmanTree }, + .{ .input = "puff23", .err = error.InvalidDynamicBlockHeader }, // 35 + .{ .input = "puff24", .err = error.InvalidDynamicBlockHeader }, + .{ .input = "puff25", .err = error.OversubscribedHuffmanTree }, + .{ .input = "puff26", .err = error.InvalidDynamicBlockHeader }, + .{ .input = "puff27", .err = error.InvalidDynamicBlockHeader }, + }; + + inline for (cases, 0..) |c, case_no| { + var in = std.io.fixedBufferStream(@embedFile("testdata/fuzz/" ++ c.input ++ ".input")); + var out = std.ArrayList(u8).init(testing.allocator); + defer out.deinit(); + errdefer std.debug.print("test case failed {}\n", .{case_no}); + + if (c.err) |expected_err| { + try testing.expectError(expected_err, decompress(.raw, in.reader(), out.writer())); + } else { + try decompress(.raw, in.reader(), out.writer()); + try testing.expectEqualStrings(c.out, out.items); + } + } +} diff --git a/lib/std/compress/flate/testdata/block_writer.zig b/lib/std/compress/flate/testdata/block_writer.zig new file mode 100644 index 000000000000..cb8f3028d12d --- /dev/null +++ b/lib/std/compress/flate/testdata/block_writer.zig @@ -0,0 +1,606 @@ +const Token = @import("../Token.zig"); + +pub const TestCase = struct { + tokens: []const Token, + input: []const u8 = "", // File name of input data matching the tokens. + want: []const u8 = "", // File name of data with the expected output with input available. + want_no_input: []const u8 = "", // File name of the expected output when no input is available. +}; + +pub const testCases = blk: { + @setEvalBranchQuota(4096 * 2); + + const L = Token.initLiteral; + const M = Token.initMatch; + const ml = M(1, 258); // Maximum length token. Used to reduce the size of writeBlockTests + + break :blk &[_]TestCase{ + TestCase{ + .input = "huffman-null-max.input", + .want = "huffman-null-max.{s}.expect", + .want_no_input = "huffman-null-max.{s}.expect-noinput", + .tokens = &[_]Token{ + L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, L(0x0), L(0x0), + }, + }, + TestCase{ + .input = "huffman-pi.input", + .want = "huffman-pi.{s}.expect", + .want_no_input = "huffman-pi.{s}.expect-noinput", + .tokens = &[_]Token{ + L('3'), L('.'), L('1'), L('4'), L('1'), L('5'), L('9'), L('2'), + L('6'), L('5'), L('3'), L('5'), L('8'), L('9'), L('7'), L('9'), + L('3'), L('2'), L('3'), L('8'), L('4'), L('6'), L('2'), L('6'), + L('4'), L('3'), L('3'), L('8'), L('3'), L('2'), L('7'), L('9'), + L('5'), L('0'), L('2'), L('8'), L('8'), L('4'), L('1'), L('9'), + L('7'), L('1'), L('6'), L('9'), L('3'), L('9'), L('9'), L('3'), + L('7'), L('5'), L('1'), L('0'), L('5'), L('8'), L('2'), L('0'), + L('9'), L('7'), L('4'), L('9'), L('4'), L('4'), L('5'), L('9'), + L('2'), L('3'), L('0'), L('7'), L('8'), L('1'), L('6'), L('4'), + L('0'), L('6'), L('2'), L('8'), L('6'), L('2'), L('0'), L('8'), + L('9'), L('9'), L('8'), L('6'), L('2'), L('8'), L('0'), L('3'), + L('4'), L('8'), L('2'), L('5'), L('3'), L('4'), L('2'), L('1'), + L('1'), L('7'), L('0'), L('6'), L('7'), L('9'), L('8'), L('2'), + L('1'), L('4'), L('8'), L('0'), L('8'), L('6'), L('5'), L('1'), + L('3'), L('2'), L('8'), L('2'), L('3'), L('0'), L('6'), L('6'), + L('4'), L('7'), L('0'), L('9'), L('3'), L('8'), L('4'), L('4'), + L('6'), L('0'), L('9'), L('5'), L('5'), L('0'), L('5'), L('8'), + L('2'), L('2'), L('3'), L('1'), L('7'), L('2'), L('5'), L('3'), + L('5'), L('9'), L('4'), L('0'), L('8'), L('1'), L('2'), L('8'), + L('4'), L('8'), L('1'), L('1'), L('1'), L('7'), L('4'), M(127, 4), + L('4'), L('1'), L('0'), L('2'), L('7'), L('0'), L('1'), L('9'), + L('3'), L('8'), L('5'), L('2'), L('1'), L('1'), L('0'), L('5'), + L('5'), L('5'), L('9'), L('6'), L('4'), L('4'), L('6'), L('2'), + L('2'), L('9'), L('4'), L('8'), L('9'), L('5'), L('4'), L('9'), + L('3'), L('0'), L('3'), L('8'), L('1'), M(19, 4), L('2'), L('8'), + L('8'), L('1'), L('0'), L('9'), L('7'), L('5'), L('6'), L('6'), + L('5'), L('9'), L('3'), L('3'), L('4'), L('4'), L('6'), M(72, 4), + L('7'), L('5'), L('6'), L('4'), L('8'), L('2'), L('3'), L('3'), + L('7'), L('8'), L('6'), L('7'), L('8'), L('3'), L('1'), L('6'), + L('5'), L('2'), L('7'), L('1'), L('2'), L('0'), L('1'), L('9'), + L('0'), L('9'), L('1'), L('4'), M(27, 4), L('5'), L('6'), L('6'), + L('9'), L('2'), L('3'), L('4'), L('6'), M(179, 4), L('6'), L('1'), + L('0'), L('4'), L('5'), L('4'), L('3'), L('2'), L('6'), M(51, 4), + L('1'), L('3'), L('3'), L('9'), L('3'), L('6'), L('0'), L('7'), + L('2'), L('6'), L('0'), L('2'), L('4'), L('9'), L('1'), L('4'), + L('1'), L('2'), L('7'), L('3'), L('7'), L('2'), L('4'), L('5'), + L('8'), L('7'), L('0'), L('0'), L('6'), L('6'), L('0'), L('6'), + L('3'), L('1'), L('5'), L('5'), L('8'), L('8'), L('1'), L('7'), + L('4'), L('8'), L('8'), L('1'), L('5'), L('2'), L('0'), L('9'), + L('2'), L('0'), L('9'), L('6'), L('2'), L('8'), L('2'), L('9'), + L('2'), L('5'), L('4'), L('0'), L('9'), L('1'), L('7'), L('1'), + L('5'), L('3'), L('6'), L('4'), L('3'), L('6'), L('7'), L('8'), + L('9'), L('2'), L('5'), L('9'), L('0'), L('3'), L('6'), L('0'), + L('0'), L('1'), L('1'), L('3'), L('3'), L('0'), L('5'), L('3'), + L('0'), L('5'), L('4'), L('8'), L('8'), L('2'), L('0'), L('4'), + L('6'), L('6'), L('5'), L('2'), L('1'), L('3'), L('8'), L('4'), + L('1'), L('4'), L('6'), L('9'), L('5'), L('1'), L('9'), L('4'), + L('1'), L('5'), L('1'), L('1'), L('6'), L('0'), L('9'), L('4'), + L('3'), L('3'), L('0'), L('5'), L('7'), L('2'), L('7'), L('0'), + L('3'), L('6'), L('5'), L('7'), L('5'), L('9'), L('5'), L('9'), + L('1'), L('9'), L('5'), L('3'), L('0'), L('9'), L('2'), L('1'), + L('8'), L('6'), L('1'), L('1'), L('7'), M(234, 4), L('3'), L('2'), + M(10, 4), L('9'), L('3'), L('1'), L('0'), L('5'), L('1'), L('1'), + L('8'), L('5'), L('4'), L('8'), L('0'), L('7'), M(271, 4), L('3'), + L('7'), L('9'), L('9'), L('6'), L('2'), L('7'), L('4'), L('9'), + L('5'), L('6'), L('7'), L('3'), L('5'), L('1'), L('8'), L('8'), + L('5'), L('7'), L('5'), L('2'), L('7'), L('2'), L('4'), L('8'), + L('9'), L('1'), L('2'), L('2'), L('7'), L('9'), L('3'), L('8'), + L('1'), L('8'), L('3'), L('0'), L('1'), L('1'), L('9'), L('4'), + L('9'), L('1'), L('2'), L('9'), L('8'), L('3'), L('3'), L('6'), + L('7'), L('3'), L('3'), L('6'), L('2'), L('4'), L('4'), L('0'), + L('6'), L('5'), L('6'), L('6'), L('4'), L('3'), L('0'), L('8'), + L('6'), L('0'), L('2'), L('1'), L('3'), L('9'), L('4'), L('9'), + L('4'), L('6'), L('3'), L('9'), L('5'), L('2'), L('2'), L('4'), + L('7'), L('3'), L('7'), L('1'), L('9'), L('0'), L('7'), L('0'), + L('2'), L('1'), L('7'), L('9'), L('8'), M(154, 5), L('7'), L('0'), + L('2'), L('7'), L('7'), L('0'), L('5'), L('3'), L('9'), L('2'), + L('1'), L('7'), L('1'), L('7'), L('6'), L('2'), L('9'), L('3'), + L('1'), L('7'), L('6'), L('7'), L('5'), M(563, 5), L('7'), L('4'), + L('8'), L('1'), M(7, 4), L('6'), L('6'), L('9'), L('4'), L('0'), + M(488, 4), L('0'), L('0'), L('0'), L('5'), L('6'), L('8'), L('1'), + L('2'), L('7'), L('1'), L('4'), L('5'), L('2'), L('6'), L('3'), + L('5'), L('6'), L('0'), L('8'), L('2'), L('7'), L('7'), L('8'), + L('5'), L('7'), L('7'), L('1'), L('3'), L('4'), L('2'), L('7'), + L('5'), L('7'), L('7'), L('8'), L('9'), L('6'), M(298, 4), L('3'), + L('6'), L('3'), L('7'), L('1'), L('7'), L('8'), L('7'), L('2'), + L('1'), L('4'), L('6'), L('8'), L('4'), L('4'), L('0'), L('9'), + L('0'), L('1'), L('2'), L('2'), L('4'), L('9'), L('5'), L('3'), + L('4'), L('3'), L('0'), L('1'), L('4'), L('6'), L('5'), L('4'), + L('9'), L('5'), L('8'), L('5'), L('3'), L('7'), L('1'), L('0'), + L('5'), L('0'), L('7'), L('9'), M(203, 4), L('6'), M(340, 4), L('8'), + L('9'), L('2'), L('3'), L('5'), L('4'), M(458, 4), L('9'), L('5'), + L('6'), L('1'), L('1'), L('2'), L('1'), L('2'), L('9'), L('0'), + L('2'), L('1'), L('9'), L('6'), L('0'), L('8'), L('6'), L('4'), + L('0'), L('3'), L('4'), L('4'), L('1'), L('8'), L('1'), L('5'), + L('9'), L('8'), L('1'), L('3'), L('6'), L('2'), L('9'), L('7'), + L('7'), L('4'), M(117, 4), L('0'), L('9'), L('9'), L('6'), L('0'), + L('5'), L('1'), L('8'), L('7'), L('0'), L('7'), L('2'), L('1'), + L('1'), L('3'), L('4'), L('9'), M(1, 5), L('8'), L('3'), L('7'), + L('2'), L('9'), L('7'), L('8'), L('0'), L('4'), L('9'), L('9'), + M(731, 4), L('9'), L('7'), L('3'), L('1'), L('7'), L('3'), L('2'), + L('8'), M(395, 4), L('6'), L('3'), L('1'), L('8'), L('5'), M(770, 4), + M(745, 4), L('4'), L('5'), L('5'), L('3'), L('4'), L('6'), L('9'), + L('0'), L('8'), L('3'), L('0'), L('2'), L('6'), L('4'), L('2'), + L('5'), L('2'), L('2'), L('3'), L('0'), M(740, 4), M(616, 4), L('8'), + L('5'), L('0'), L('3'), L('5'), L('2'), L('6'), L('1'), L('9'), + L('3'), L('1'), L('1'), M(531, 4), L('1'), L('0'), L('1'), L('0'), + L('0'), L('0'), L('3'), L('1'), L('3'), L('7'), L('8'), L('3'), + L('8'), L('7'), L('5'), L('2'), L('8'), L('8'), L('6'), L('5'), + L('8'), L('7'), L('5'), L('3'), L('3'), L('2'), L('0'), L('8'), + L('3'), L('8'), L('1'), L('4'), L('2'), L('0'), L('6'), M(321, 4), + M(300, 4), L('1'), L('4'), L('7'), L('3'), L('0'), L('3'), L('5'), + L('9'), M(815, 5), L('9'), L('0'), L('4'), L('2'), L('8'), L('7'), + L('5'), L('5'), L('4'), L('6'), L('8'), L('7'), L('3'), L('1'), + L('1'), L('5'), L('9'), L('5'), M(854, 4), L('3'), L('8'), L('8'), + L('2'), L('3'), L('5'), L('3'), L('7'), L('8'), L('7'), L('5'), + M(896, 5), L('9'), M(315, 4), L('1'), M(329, 4), L('8'), L('0'), L('5'), + L('3'), M(395, 4), L('2'), L('2'), L('6'), L('8'), L('0'), L('6'), + L('6'), L('1'), L('3'), L('0'), L('0'), L('1'), L('9'), L('2'), + L('7'), L('8'), L('7'), L('6'), L('6'), L('1'), L('1'), L('1'), + L('9'), L('5'), L('9'), M(568, 4), L('6'), M(293, 5), L('8'), L('9'), + L('3'), L('8'), L('0'), L('9'), L('5'), L('2'), L('5'), L('7'), + L('2'), L('0'), L('1'), L('0'), L('6'), L('5'), L('4'), L('8'), + L('5'), L('8'), L('6'), L('3'), L('2'), L('7'), M(155, 4), L('9'), + L('3'), L('6'), L('1'), L('5'), L('3'), M(545, 4), M(349, 5), L('2'), + L('3'), L('0'), L('3'), L('0'), L('1'), L('9'), L('5'), L('2'), + L('0'), L('3'), L('5'), L('3'), L('0'), L('1'), L('8'), L('5'), + L('2'), M(370, 4), M(118, 4), L('3'), L('6'), L('2'), L('2'), L('5'), + L('9'), L('9'), L('4'), L('1'), L('3'), M(597, 4), L('4'), L('9'), + L('7'), L('2'), L('1'), L('7'), M(223, 4), L('3'), L('4'), L('7'), + L('9'), L('1'), L('3'), L('1'), L('5'), L('1'), L('5'), L('5'), + L('7'), L('4'), L('8'), L('5'), L('7'), L('2'), L('4'), L('2'), + L('4'), L('5'), L('4'), L('1'), L('5'), L('0'), L('6'), L('9'), + M(320, 4), L('8'), L('2'), L('9'), L('5'), L('3'), L('3'), L('1'), + L('1'), L('6'), L('8'), L('6'), L('1'), L('7'), L('2'), L('7'), + L('8'), M(824, 4), L('9'), L('0'), L('7'), L('5'), L('0'), L('9'), + M(270, 4), L('7'), L('5'), L('4'), L('6'), L('3'), L('7'), L('4'), + L('6'), L('4'), L('9'), L('3'), L('9'), L('3'), L('1'), L('9'), + L('2'), L('5'), L('5'), L('0'), L('6'), L('0'), L('4'), L('0'), + L('0'), L('9'), M(620, 4), L('1'), L('6'), L('7'), L('1'), L('1'), + L('3'), L('9'), L('0'), L('0'), L('9'), L('8'), M(822, 4), L('4'), + L('0'), L('1'), L('2'), L('8'), L('5'), L('8'), L('3'), L('6'), + L('1'), L('6'), L('0'), L('3'), L('5'), L('6'), L('3'), L('7'), + L('0'), L('7'), L('6'), L('6'), L('0'), L('1'), L('0'), L('4'), + M(371, 4), L('8'), L('1'), L('9'), L('4'), L('2'), L('9'), M(1055, 5), + M(240, 4), M(652, 4), L('7'), L('8'), L('3'), L('7'), L('4'), M(1193, 4), + L('8'), L('2'), L('5'), L('5'), L('3'), L('7'), M(522, 5), L('2'), + L('6'), L('8'), M(47, 4), L('4'), L('0'), L('4'), L('7'), M(466, 4), + L('4'), M(1206, 4), M(910, 4), L('8'), L('4'), M(937, 4), L('6'), M(800, 6), + L('3'), L('3'), L('1'), L('3'), L('6'), L('7'), L('7'), L('0'), + L('2'), L('8'), L('9'), L('8'), L('9'), L('1'), L('5'), L('2'), + M(99, 4), L('5'), L('2'), L('1'), L('6'), L('2'), L('0'), L('5'), + L('6'), L('9'), L('6'), M(1042, 4), L('0'), L('5'), L('8'), M(1144, 4), + L('5'), M(1177, 4), L('5'), L('1'), L('1'), M(522, 4), L('8'), L('2'), + L('4'), L('3'), L('0'), L('0'), L('3'), L('5'), L('5'), L('8'), + L('7'), L('6'), L('4'), L('0'), L('2'), L('4'), L('7'), L('4'), + L('9'), L('6'), L('4'), L('7'), L('3'), L('2'), L('6'), L('3'), + M(1087, 4), L('9'), L('9'), L('2'), M(1100, 4), L('4'), L('2'), L('6'), + L('9'), M(710, 6), L('7'), M(471, 4), L('4'), M(1342, 4), M(1054, 4), L('9'), + L('3'), L('4'), L('1'), L('7'), M(430, 4), L('1'), L('2'), M(43, 4), + L('4'), M(415, 4), L('1'), L('5'), L('0'), L('3'), L('0'), L('2'), + L('8'), L('6'), L('1'), L('8'), L('2'), L('9'), L('7'), L('4'), + L('5'), L('5'), L('5'), L('7'), L('0'), L('6'), L('7'), L('4'), + M(310, 4), L('5'), L('0'), L('5'), L('4'), L('9'), L('4'), L('5'), + L('8'), M(454, 4), L('9'), M(82, 4), L('5'), L('6'), M(493, 4), L('7'), + L('2'), L('1'), L('0'), L('7'), L('9'), M(346, 4), L('3'), L('0'), + M(267, 4), L('3'), L('2'), L('1'), L('1'), L('6'), L('5'), L('3'), + L('4'), L('4'), L('9'), L('8'), L('7'), L('2'), L('0'), L('2'), + L('7'), M(284, 4), L('0'), L('2'), L('3'), L('6'), L('4'), M(559, 4), + L('5'), L('4'), L('9'), L('9'), L('1'), L('1'), L('9'), L('8'), + M(1049, 4), L('4'), M(284, 4), L('5'), L('3'), L('5'), L('6'), L('6'), + L('3'), L('6'), L('9'), M(1105, 4), L('2'), L('6'), L('5'), M(741, 4), + L('7'), L('8'), L('6'), L('2'), L('5'), L('5'), L('1'), M(987, 4), + L('1'), L('7'), L('5'), L('7'), L('4'), L('6'), L('7'), L('2'), + L('8'), L('9'), L('0'), L('9'), L('7'), L('7'), L('7'), L('7'), + M(1108, 5), L('0'), L('0'), L('0'), M(1534, 4), L('7'), L('0'), M(1248, 4), + L('6'), M(1002, 4), L('4'), L('9'), L('1'), M(1055, 4), M(664, 4), L('2'), + L('1'), L('4'), L('7'), L('7'), L('2'), L('3'), L('5'), L('0'), + L('1'), L('4'), L('1'), L('4'), M(1604, 4), L('3'), L('5'), L('6'), + M(1200, 4), L('1'), L('6'), L('1'), L('3'), L('6'), L('1'), L('1'), + L('5'), L('7'), L('3'), L('5'), L('2'), L('5'), M(1285, 4), L('3'), + L('4'), M(92, 4), L('1'), L('8'), M(1148, 4), L('8'), L('4'), M(1512, 4), + L('3'), L('3'), L('2'), L('3'), L('9'), L('0'), L('7'), L('3'), + L('9'), L('4'), L('1'), L('4'), L('3'), L('3'), L('3'), L('4'), + L('5'), L('4'), L('7'), L('7'), L('6'), L('2'), L('4'), M(579, 4), + L('2'), L('5'), L('1'), L('8'), L('9'), L('8'), L('3'), L('5'), + L('6'), L('9'), L('4'), L('8'), L('5'), L('5'), L('6'), L('2'), + L('0'), L('9'), L('9'), L('2'), L('1'), L('9'), L('2'), L('2'), + L('2'), L('1'), L('8'), L('4'), L('2'), L('7'), M(575, 4), L('2'), + M(187, 4), L('6'), L('8'), L('8'), L('7'), L('6'), L('7'), L('1'), + L('7'), L('9'), L('0'), M(86, 4), L('0'), M(263, 5), L('6'), L('6'), + M(1000, 4), L('8'), L('8'), L('6'), L('2'), L('7'), L('2'), M(1757, 4), + L('1'), L('7'), L('8'), L('6'), L('0'), L('8'), L('5'), L('7'), + M(116, 4), L('3'), M(765, 5), L('7'), L('9'), L('7'), L('6'), L('6'), + L('8'), L('1'), M(702, 4), L('0'), L('0'), L('9'), L('5'), L('3'), + L('8'), L('8'), M(1593, 4), L('3'), M(1702, 4), L('0'), L('6'), L('8'), + L('0'), L('0'), L('6'), L('4'), L('2'), L('2'), L('5'), L('1'), + L('2'), L('5'), L('2'), M(1404, 4), L('7'), L('3'), L('9'), L('2'), + M(664, 4), M(1141, 4), L('4'), M(1716, 5), L('8'), L('6'), L('2'), L('6'), + L('9'), L('4'), L('5'), M(486, 4), L('4'), L('1'), L('9'), L('6'), + L('5'), L('2'), L('8'), L('5'), L('0'), M(154, 4), M(925, 4), L('1'), + L('8'), L('6'), L('3'), M(447, 4), L('4'), M(341, 5), L('2'), L('0'), + L('3'), L('9'), M(1420, 4), L('4'), L('5'), M(701, 4), L('2'), L('3'), + L('7'), M(1069, 4), L('6'), M(1297, 4), L('5'), L('6'), M(1593, 4), L('7'), + L('1'), L('9'), L('1'), L('7'), L('2'), L('8'), M(370, 4), L('7'), + L('6'), L('4'), L('6'), L('5'), L('7'), L('5'), L('7'), L('3'), + L('9'), M(258, 4), L('3'), L('8'), L('9'), M(1865, 4), L('8'), L('3'), + L('2'), L('6'), L('4'), L('5'), L('9'), L('9'), L('5'), L('8'), + M(1704, 4), L('0'), L('4'), L('7'), L('8'), M(479, 4), M(809, 4), L('9'), + M(46, 4), L('6'), L('4'), L('0'), L('7'), L('8'), L('9'), L('5'), + L('1'), M(143, 4), L('6'), L('8'), L('3'), M(304, 4), L('2'), L('5'), + L('9'), L('5'), L('7'), L('0'), M(1129, 4), L('8'), L('2'), L('2'), + M(713, 4), L('2'), M(1564, 4), L('4'), L('0'), L('7'), L('7'), L('2'), + L('6'), L('7'), L('1'), L('9'), L('4'), L('7'), L('8'), M(794, 4), + L('8'), L('2'), L('6'), L('0'), L('1'), L('4'), L('7'), L('6'), + L('9'), L('9'), L('0'), L('9'), M(1257, 4), L('0'), L('1'), L('3'), + L('6'), L('3'), L('9'), L('4'), L('4'), L('3'), M(640, 4), L('3'), + L('0'), M(262, 4), L('2'), L('0'), L('3'), L('4'), L('9'), L('6'), + L('2'), L('5'), L('2'), L('4'), L('5'), L('1'), L('7'), M(950, 4), + L('9'), L('6'), L('5'), L('1'), L('4'), L('3'), L('1'), L('4'), + L('2'), L('9'), L('8'), L('0'), L('9'), L('1'), L('9'), L('0'), + L('6'), L('5'), L('9'), L('2'), M(643, 4), L('7'), L('2'), L('2'), + L('1'), L('6'), L('9'), L('6'), L('4'), L('6'), M(1050, 4), M(123, 4), + L('5'), M(1295, 4), L('4'), M(1382, 5), L('8'), M(1370, 4), L('9'), L('7'), + M(1404, 4), L('5'), L('4'), M(1182, 4), M(575, 4), L('7'), M(1627, 4), L('8'), + L('4'), L('6'), L('8'), L('1'), L('3'), M(141, 4), L('6'), L('8'), + L('3'), L('8'), L('6'), L('8'), L('9'), L('4'), L('2'), L('7'), + L('7'), L('4'), L('1'), L('5'), L('5'), L('9'), L('9'), L('1'), + L('8'), L('5'), M(91, 4), L('2'), L('4'), L('5'), L('9'), L('5'), + L('3'), L('9'), L('5'), L('9'), L('4'), L('3'), L('1'), M(1464, 4), + L('7'), M(19, 4), L('6'), L('8'), L('0'), L('8'), L('4'), L('5'), + M(744, 4), L('7'), L('3'), M(2079, 4), L('9'), L('5'), L('8'), L('4'), + L('8'), L('6'), L('5'), L('3'), L('8'), M(1769, 4), L('6'), L('2'), + M(243, 4), L('6'), L('0'), L('9'), M(1207, 4), L('6'), L('0'), L('8'), + L('0'), L('5'), L('1'), L('2'), L('4'), L('3'), L('8'), L('8'), + L('4'), M(315, 4), M(12, 4), L('4'), L('1'), L('3'), M(784, 4), L('7'), + L('6'), L('2'), L('7'), L('8'), M(834, 4), L('7'), L('1'), L('5'), + M(1436, 4), L('3'), L('5'), L('9'), L('9'), L('7'), L('7'), L('0'), + L('0'), L('1'), L('2'), L('9'), M(1139, 4), L('8'), L('9'), L('4'), + L('4'), L('1'), M(632, 4), L('6'), L('8'), L('5'), L('5'), M(96, 4), + L('4'), L('0'), L('6'), L('3'), M(2279, 4), L('2'), L('0'), L('7'), + L('2'), L('2'), M(345, 4), M(516, 5), L('4'), L('8'), L('1'), L('5'), + L('8'), M(518, 4), M(511, 4), M(635, 4), M(665, 4), L('3'), L('9'), L('4'), + L('5'), L('2'), L('2'), L('6'), L('7'), M(1175, 6), L('8'), M(1419, 4), + L('2'), L('1'), M(747, 4), L('2'), M(904, 4), L('5'), L('4'), L('6'), + L('6'), L('6'), M(1308, 4), L('2'), L('3'), L('9'), L('8'), L('6'), + L('4'), L('5'), L('6'), M(1221, 4), L('1'), L('6'), L('3'), L('5'), + M(596, 5), M(2066, 4), L('7'), M(2222, 4), L('9'), L('8'), M(1119, 4), L('9'), + L('3'), L('6'), L('3'), L('4'), M(1884, 4), L('7'), L('4'), L('3'), + L('2'), L('4'), M(1148, 4), L('1'), L('5'), L('0'), L('7'), L('6'), + M(1212, 4), L('7'), L('9'), L('4'), L('5'), L('1'), L('0'), L('9'), + M(63, 4), L('0'), L('9'), L('4'), L('0'), M(1703, 4), L('8'), L('8'), + L('7'), L('9'), L('7'), L('1'), L('0'), L('8'), L('9'), L('3'), + M(2289, 4), L('6'), L('9'), L('1'), L('3'), L('6'), L('8'), L('6'), + L('7'), L('2'), M(604, 4), M(511, 4), L('5'), M(1344, 4), M(1129, 4), M(2050, 4), + L('1'), L('7'), L('9'), L('2'), L('8'), L('6'), L('8'), M(2253, 4), + L('8'), L('7'), L('4'), L('7'), M(1951, 5), L('8'), L('2'), L('4'), + M(2427, 4), L('8'), M(604, 4), L('7'), L('1'), L('4'), L('9'), L('0'), + L('9'), L('6'), L('7'), L('5'), L('9'), L('8'), M(1776, 4), L('3'), + L('6'), L('5'), M(309, 4), L('8'), L('1'), M(93, 4), M(1862, 4), M(2359, 4), + L('6'), L('8'), L('2'), L('9'), M(1407, 4), L('8'), L('7'), L('2'), + L('2'), L('6'), L('5'), L('8'), L('8'), L('0'), M(1554, 4), L('5'), + M(586, 4), L('4'), L('2'), L('7'), L('0'), L('4'), L('7'), L('7'), + L('5'), L('5'), M(2079, 4), L('3'), L('7'), L('9'), L('6'), L('4'), + L('1'), L('4'), L('5'), L('1'), L('5'), L('2'), M(1534, 4), L('2'), + L('3'), L('4'), L('3'), L('6'), L('4'), L('5'), L('4'), M(1503, 4), + L('4'), L('4'), L('4'), L('7'), L('9'), L('5'), M(61, 4), M(1316, 4), + M(2279, 5), L('4'), L('1'), M(1323, 4), L('3'), M(773, 4), L('5'), L('2'), + L('3'), L('1'), M(2114, 5), L('1'), L('6'), L('6'), L('1'), M(2227, 4), + L('5'), L('9'), L('6'), L('9'), L('5'), L('3'), L('6'), L('2'), + L('3'), L('1'), L('4'), M(1536, 4), L('2'), L('4'), L('8'), L('4'), + L('9'), L('3'), L('7'), L('1'), L('8'), L('7'), L('1'), L('1'), + L('0'), L('1'), L('4'), L('5'), L('7'), L('6'), L('5'), L('4'), + M(1890, 4), L('0'), L('2'), L('7'), L('9'), L('9'), L('3'), L('4'), + L('4'), L('0'), L('3'), L('7'), L('4'), L('2'), L('0'), L('0'), + L('7'), M(2368, 4), L('7'), L('8'), L('5'), L('3'), L('9'), L('0'), + L('6'), L('2'), L('1'), L('9'), M(666, 5), M(838, 4), L('8'), L('4'), + L('7'), M(979, 5), L('8'), L('3'), L('3'), L('2'), L('1'), L('4'), + L('4'), L('5'), L('7'), L('1'), M(645, 4), M(1911, 4), L('4'), L('3'), + L('5'), L('0'), M(2345, 4), M(1129, 4), L('5'), L('3'), L('1'), L('9'), + L('1'), L('0'), L('4'), L('8'), L('4'), L('8'), L('1'), L('0'), + L('0'), L('5'), L('3'), L('7'), L('0'), L('6'), M(2237, 4), M(1438, 5), + M(1922, 5), L('1'), M(1370, 4), L('7'), M(796, 4), L('5'), M(2029, 4), M(1037, 4), + L('6'), L('3'), M(2013, 5), L('4'), M(2418, 4), M(847, 5), M(1014, 5), L('8'), + M(1326, 5), M(2184, 5), L('9'), M(392, 4), L('9'), L('1'), M(2255, 4), L('8'), + L('1'), L('4'), L('6'), L('7'), L('5'), L('1'), M(1580, 4), L('1'), + L('2'), L('3'), L('9'), M(426, 6), L('9'), L('0'), L('7'), L('1'), + L('8'), L('6'), L('4'), L('9'), L('4'), L('2'), L('3'), L('1'), + L('9'), L('6'), L('1'), L('5'), L('6'), M(493, 4), M(1725, 4), L('9'), + L('5'), M(2343, 4), M(1130, 4), M(284, 4), L('6'), L('0'), L('3'), L('8'), + M(2598, 4), M(368, 4), M(901, 4), L('6'), L('2'), M(1115, 4), L('5'), M(2125, 4), + L('6'), L('3'), L('8'), L('9'), L('3'), L('7'), L('7'), L('8'), + L('7'), M(2246, 4), M(249, 4), L('9'), L('7'), L('9'), L('2'), L('0'), + L('7'), L('7'), L('3'), M(1496, 4), L('2'), L('1'), L('8'), L('2'), + L('5'), L('6'), M(2016, 4), L('6'), L('6'), M(1751, 4), L('4'), L('2'), + M(1663, 5), L('6'), M(1767, 4), L('4'), L('4'), M(37, 4), L('5'), L('4'), + L('9'), L('2'), L('0'), L('2'), L('6'), L('0'), L('5'), M(2740, 4), + M(997, 5), L('2'), L('0'), L('1'), L('4'), L('9'), M(1235, 4), L('8'), + L('5'), L('0'), L('7'), L('3'), M(1434, 4), L('6'), L('6'), L('6'), + L('0'), M(405, 4), L('2'), L('4'), L('3'), L('4'), L('0'), M(136, 4), + L('0'), M(1900, 4), L('8'), L('6'), L('3'), M(2391, 4), M(2021, 4), M(1068, 4), + M(373, 4), L('5'), L('7'), L('9'), L('6'), L('2'), L('6'), L('8'), + L('5'), L('6'), M(321, 4), L('5'), L('0'), L('8'), M(1316, 4), L('5'), + L('8'), L('7'), L('9'), L('6'), L('9'), L('9'), M(1810, 4), L('5'), + L('7'), L('4'), M(2585, 4), L('8'), L('4'), L('0'), M(2228, 4), L('1'), + L('4'), L('5'), L('9'), L('1'), M(1933, 4), L('7'), L('0'), M(565, 4), + L('0'), L('1'), M(3048, 4), L('1'), L('2'), M(3189, 4), L('0'), M(964, 4), + L('3'), L('9'), M(2859, 4), M(275, 4), L('7'), L('1'), L('5'), M(945, 4), + L('4'), L('2'), L('0'), M(3059, 5), L('9'), M(3011, 4), L('0'), L('7'), + M(834, 4), M(1942, 4), M(2736, 4), M(3171, 4), L('2'), L('1'), M(2401, 4), L('2'), + L('5'), L('1'), M(1404, 4), M(2373, 4), L('9'), L('2'), M(435, 4), L('8'), + L('2'), L('6'), M(2919, 4), L('2'), M(633, 4), L('3'), L('2'), L('1'), + L('5'), L('7'), L('9'), L('1'), L('9'), L('8'), L('4'), L('1'), + L('4'), M(2172, 5), L('9'), L('1'), L('6'), L('4'), M(1769, 5), L('9'), + M(2905, 5), M(2268, 4), L('7'), L('2'), L('2'), M(802, 4), L('5'), M(2213, 4), + M(322, 4), L('9'), L('1'), L('0'), M(189, 4), M(3164, 4), L('5'), L('2'), + L('8'), L('0'), L('1'), L('7'), M(562, 4), L('7'), L('1'), L('2'), + M(2325, 4), L('8'), L('3'), L('2'), M(884, 4), L('1'), M(1418, 4), L('0'), + L('9'), L('3'), L('5'), L('3'), L('9'), L('6'), L('5'), L('7'), + M(1612, 4), L('1'), L('0'), L('8'), L('3'), M(106, 4), L('5'), L('1'), + M(1915, 4), M(3419, 4), L('1'), L('4'), L('4'), L('4'), L('2'), L('1'), + L('0'), L('0'), M(515, 4), L('0'), L('3'), M(413, 4), L('1'), L('1'), + L('0'), L('3'), M(3202, 4), M(10, 4), M(39, 4), M(1539, 6), L('5'), L('1'), + L('6'), M(1498, 4), M(2180, 5), M(2347, 4), L('5'), M(3139, 5), L('8'), L('5'), + L('1'), L('7'), L('1'), L('4'), L('3'), L('7'), M(1542, 4), M(110, 4), + L('1'), L('5'), L('5'), L('6'), L('5'), L('0'), L('8'), L('8'), + M(954, 4), L('9'), L('8'), L('9'), L('8'), L('5'), L('9'), L('9'), + L('8'), L('2'), L('3'), L('8'), M(464, 4), M(2491, 4), L('3'), M(365, 4), + M(1087, 4), M(2500, 4), L('8'), M(3590, 5), L('3'), L('2'), M(264, 4), L('5'), + M(774, 4), L('3'), M(459, 4), L('9'), M(1052, 4), L('9'), L('8'), M(2174, 4), + L('4'), M(3257, 4), L('7'), M(1612, 4), L('0'), L('7'), M(230, 4), L('4'), + L('8'), L('1'), L('4'), L('1'), M(1338, 4), L('8'), L('5'), L('9'), + L('4'), L('6'), L('1'), M(3018, 4), L('8'), L('0'), + }, + }, + TestCase{ + .input = "huffman-rand-1k.input", + .want = "huffman-rand-1k.{s}.expect", + .want_no_input = "huffman-rand-1k.{s}.expect-noinput", + .tokens = &[_]Token{ + L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xd), L(0x85), L(0x94), L(0x25), L(0x80), L(0xaf), L(0xc2), L(0xfe), L(0x8d), + L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde), L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b), + L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73), L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4), + L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15), L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde), + L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9), L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14), + L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xd), L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c), + L(0x99), L(0xdf), L(0xfd), L(0x70), L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5), + L(0xd4), L(0x80), L(0x59), L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20), + L(0x1b), L(0x46), L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4), + L(0x7b), L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28), + L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f), L(0xa0), + L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4), L(0x75), L(0xda), + L(0xea), L(0x9b), L(0xa), L(0x64), L(0xb), L(0xe0), L(0x23), L(0x29), L(0xbd), L(0xf7), L(0xe7), L(0x83), L(0x3c), L(0xfb), + L(0xdf), L(0xb3), L(0xae), L(0x4f), L(0xa4), L(0x47), L(0x55), L(0x99), L(0xde), L(0x2f), L(0x96), L(0x6e), L(0x1c), L(0x43), + L(0x4c), L(0x87), L(0xe2), L(0x7c), L(0xd9), L(0x5f), L(0x4c), L(0x7c), L(0xe8), L(0x90), L(0x3), L(0xdb), L(0x30), L(0x95), + L(0xd6), L(0x22), L(0xc), L(0x47), L(0xb8), L(0x4d), L(0x6b), L(0xbd), L(0x24), L(0x11), L(0xab), L(0x2c), L(0xd7), L(0xbe), + L(0x6e), L(0x7a), L(0xd6), L(0x8), L(0xa3), L(0x98), L(0xd8), L(0xdd), L(0x15), L(0x6a), L(0xfa), L(0x93), L(0x30), L(0x1), + L(0x25), L(0x1d), L(0xa2), L(0x74), L(0x86), L(0x4b), L(0x6a), L(0x95), L(0xe8), L(0xe1), L(0x4e), L(0xe), L(0x76), L(0xb9), + L(0x49), L(0xa9), L(0x5f), L(0xa0), L(0xa6), L(0x63), L(0x3c), L(0x7e), L(0x7e), L(0x20), L(0x13), L(0x4f), L(0xbb), L(0x66), + L(0x92), L(0xb8), L(0x2e), L(0xa4), L(0xfa), L(0x48), L(0xcb), L(0xae), L(0xb9), L(0x3c), L(0xaf), L(0xd3), L(0x1f), L(0xe1), + L(0xd5), L(0x8d), L(0x42), L(0x6d), L(0xf0), L(0xfc), L(0x8c), L(0xc), L(0x0), L(0xde), L(0x40), L(0xab), L(0x8b), L(0x47), + L(0x97), L(0x4e), L(0xa8), L(0xcf), L(0x8e), L(0xdb), L(0xa6), L(0x8b), L(0x20), L(0x9), L(0x84), L(0x7a), L(0x66), L(0xe5), + L(0x98), L(0x29), L(0x2), L(0x95), L(0xe6), L(0x38), L(0x32), L(0x60), L(0x3), L(0xe3), L(0x9a), L(0x1e), L(0x54), L(0xe8), + L(0x63), L(0x80), L(0x48), L(0x9c), L(0xe7), L(0x63), L(0x33), L(0x6e), L(0xa0), L(0x65), L(0x83), L(0xfa), L(0xc6), L(0xba), + L(0x7a), L(0x43), L(0x71), L(0x5), L(0xf5), L(0x68), L(0x69), L(0x85), L(0x9c), L(0xba), L(0x45), L(0xcd), L(0x6b), L(0xb), + L(0x19), L(0xd1), L(0xbb), L(0x7f), L(0x70), L(0x85), L(0x92), L(0xd1), L(0xb4), L(0x64), L(0x82), L(0xb1), L(0xe4), L(0x62), + L(0xc5), L(0x3c), L(0x46), L(0x1f), L(0x92), L(0x31), L(0x1c), L(0x4e), L(0x41), L(0x77), L(0xf7), L(0xe7), L(0x87), L(0xa2), + L(0xf), L(0x6e), L(0xe8), L(0x92), L(0x3), L(0x6b), L(0xa), L(0xe7), L(0xa9), L(0x3b), L(0x11), L(0xda), L(0x66), L(0x8a), + L(0x29), L(0xda), L(0x79), L(0xe1), L(0x64), L(0x8d), L(0xe3), L(0x54), L(0xd4), L(0xf5), L(0xef), L(0x64), L(0x87), L(0x3b), + L(0xf4), L(0xc2), L(0xf4), L(0x71), L(0x13), L(0xa9), L(0xe9), L(0xe0), L(0xa2), L(0x6), L(0x14), L(0xab), L(0x5d), L(0xa7), + L(0x96), L(0x0), L(0xd6), L(0xc3), L(0xcc), L(0x57), L(0xed), L(0x39), L(0x6a), L(0x25), L(0xcd), L(0x76), L(0xea), L(0xba), + L(0x3a), L(0xf2), L(0xa1), L(0x95), L(0x5d), L(0xe5), L(0x71), L(0xcf), L(0x9c), L(0x62), L(0x9e), L(0x6a), L(0xfa), L(0xd5), + L(0x31), L(0xd1), L(0xa8), L(0x66), L(0x30), L(0x33), L(0xaa), L(0x51), L(0x17), L(0x13), L(0x82), L(0x99), L(0xc8), L(0x14), + L(0x60), L(0x9f), L(0x4d), L(0x32), L(0x6d), L(0xda), L(0x19), L(0x26), L(0x21), L(0xdc), L(0x7e), L(0x2e), L(0x25), L(0x67), + L(0x72), L(0xca), L(0xf), L(0x92), L(0xcd), L(0xf6), L(0xd6), L(0xcb), L(0x97), L(0x8a), L(0x33), L(0x58), L(0x73), L(0x70), + L(0x91), L(0x1d), L(0xbf), L(0x28), L(0x23), L(0xa3), L(0xc), L(0xf1), L(0x83), L(0xc3), L(0xc8), L(0x56), L(0x77), L(0x68), + L(0xe3), L(0x82), L(0xba), L(0xb9), L(0x57), L(0x56), L(0x57), L(0x9c), L(0xc3), L(0xd6), L(0x14), L(0x5), L(0x3c), L(0xb1), + L(0xaf), L(0x93), L(0xc8), L(0x8a), L(0x57), L(0x7f), L(0x53), L(0xfa), L(0x2f), L(0xaa), L(0x6e), L(0x66), L(0x83), L(0xfa), + L(0x33), L(0xd1), L(0x21), L(0xab), L(0x1b), L(0x71), L(0xb4), L(0x7c), L(0xda), L(0xfd), L(0xfb), L(0x7f), L(0x20), L(0xab), + L(0x5e), L(0xd5), L(0xca), L(0xfd), L(0xdd), L(0xe0), L(0xee), L(0xda), L(0xba), L(0xa8), L(0x27), L(0x99), L(0x97), L(0x69), + L(0xc1), L(0x3c), L(0x82), L(0x8c), L(0xa), L(0x5c), L(0x2d), L(0x5b), L(0x88), L(0x3e), L(0x34), L(0x35), L(0x86), L(0x37), + L(0x46), L(0x79), L(0xe1), L(0xaa), L(0x19), L(0xfb), L(0xaa), L(0xde), L(0x15), L(0x9), L(0xd), L(0x1a), L(0x57), L(0xff), + L(0xb5), L(0xf), L(0xf3), L(0x2b), L(0x5a), L(0x6a), L(0x4d), L(0x19), L(0x77), L(0x71), L(0x45), L(0xdf), L(0x4f), L(0xb3), + L(0xec), L(0xf1), L(0xeb), L(0x18), L(0x53), L(0x3e), L(0x3b), L(0x47), L(0x8), L(0x9a), L(0x73), L(0xa0), L(0x5c), L(0x8c), + L(0x5f), L(0xeb), L(0xf), L(0x3a), L(0xc2), L(0x43), L(0x67), L(0xb4), L(0x66), L(0x67), L(0x80), L(0x58), L(0xe), L(0xc1), + L(0xec), L(0x40), L(0xd4), L(0x22), L(0x94), L(0xca), L(0xf9), L(0xe8), L(0x92), L(0xe4), L(0x69), L(0x38), L(0xbe), L(0x67), + L(0x64), L(0xca), L(0x50), L(0xc7), L(0x6), L(0x67), L(0x42), L(0x6e), L(0xa3), L(0xf0), L(0xb7), L(0x6c), L(0xf2), L(0xe8), + L(0x5f), L(0xb1), L(0xaf), L(0xe7), L(0xdb), L(0xbb), L(0x77), L(0xb5), L(0xf8), L(0xcb), L(0x8), L(0xc4), L(0x75), L(0x7e), + L(0xc0), L(0xf9), L(0x1c), L(0x7f), L(0x3c), L(0x89), L(0x2f), L(0xd2), L(0x58), L(0x3a), L(0xe2), L(0xf8), L(0x91), L(0xb6), + L(0x7b), L(0x24), L(0x27), L(0xe9), L(0xae), L(0x84), L(0x8b), L(0xde), L(0x74), L(0xac), L(0xfd), L(0xd9), L(0xb7), L(0x69), + L(0x2a), L(0xec), L(0x32), L(0x6f), L(0xf0), L(0x92), L(0x84), L(0xf1), L(0x40), L(0xc), L(0x8a), L(0xbc), L(0x39), L(0x6e), + L(0x2e), L(0x73), L(0xd4), L(0x6e), L(0x8a), L(0x74), L(0x2a), L(0xdc), L(0x60), L(0x1f), L(0xa3), L(0x7), L(0xde), L(0x75), + L(0x8b), L(0x74), L(0xc8), L(0xfe), L(0x63), L(0x75), L(0xf6), L(0x3d), L(0x63), L(0xac), L(0x33), L(0x89), L(0xc3), L(0xf0), + L(0xf8), L(0x2d), L(0x6b), L(0xb4), L(0x9e), L(0x74), L(0x8b), L(0x5c), L(0x33), L(0xb4), L(0xca), L(0xa8), L(0xe4), L(0x99), + L(0xb6), L(0x90), L(0xa1), L(0xef), L(0xf), L(0xd3), L(0x61), L(0xb2), L(0xc6), L(0x1a), L(0x94), L(0x7c), L(0x44), L(0x55), + L(0xf4), L(0x45), L(0xff), L(0x9e), L(0xa5), L(0x5a), L(0xc6), L(0xa0), L(0xe8), L(0x2a), L(0xc1), L(0x8d), L(0x6f), L(0x34), + L(0x11), L(0xb9), L(0xbe), L(0x4e), L(0xd9), L(0x87), L(0x97), L(0x73), L(0xcf), L(0x3d), L(0x23), L(0xae), L(0xd5), L(0x1a), + L(0x5e), L(0xae), L(0x5d), L(0x6a), L(0x3), L(0xf9), L(0x22), L(0xd), L(0x10), L(0xd9), L(0x47), L(0x69), L(0x15), L(0x3f), + L(0xee), L(0x52), L(0xa3), L(0x8), L(0xd2), L(0x3c), L(0x51), L(0xf4), L(0xf8), L(0x9d), L(0xe4), L(0x98), L(0x89), L(0xc8), + L(0x67), L(0x39), L(0xd5), L(0x5e), L(0x35), L(0x78), L(0x27), L(0xe8), L(0x3c), L(0x80), L(0xae), L(0x79), L(0x71), L(0xd2), + L(0x93), L(0xf4), L(0xaa), L(0x51), L(0x12), L(0x1c), L(0x4b), L(0x1b), L(0xe5), L(0x6e), L(0x15), L(0x6f), L(0xe4), L(0xbb), + L(0x51), L(0x9b), L(0x45), L(0x9f), L(0xf9), L(0xc4), L(0x8c), L(0x2a), L(0xfb), L(0x1a), L(0xdf), L(0x55), L(0xd3), L(0x48), + L(0x93), L(0x27), L(0x1), L(0x26), L(0xc2), L(0x6b), L(0x55), L(0x6d), L(0xa2), L(0xfb), L(0x84), L(0x8b), L(0xc9), L(0x9e), + L(0x28), L(0xc2), L(0xef), L(0x1a), L(0x24), L(0xec), L(0x9b), L(0xae), L(0xbd), L(0x60), L(0xe9), L(0x15), L(0x35), L(0xee), + L(0x42), L(0xa4), L(0x33), L(0x5b), L(0xfa), L(0xf), L(0xb6), L(0xf7), L(0x1), L(0xa6), L(0x2), L(0x4c), L(0xca), L(0x90), + L(0x58), L(0x3a), L(0x96), L(0x41), L(0xe7), L(0xcb), L(0x9), L(0x8c), L(0xdb), L(0x85), L(0x4d), L(0xa8), L(0x89), L(0xf3), + L(0xb5), L(0x8e), L(0xfd), L(0x75), L(0x5b), L(0x4f), L(0xed), L(0xde), L(0x3f), L(0xeb), L(0x38), L(0xa3), L(0xbe), L(0xb0), + L(0x73), L(0xfc), L(0xb8), L(0x54), L(0xf7), L(0x4c), L(0x30), L(0x67), L(0x2e), L(0x38), L(0xa2), L(0x54), L(0x18), L(0xba), + L(0x8), L(0xbf), L(0xf2), L(0x39), L(0xd5), L(0xfe), L(0xa5), L(0x41), L(0xc6), L(0x66), L(0x66), L(0xba), L(0x81), L(0xef), + L(0x67), L(0xe4), L(0xe6), L(0x3c), L(0xc), L(0xca), L(0xa4), L(0xa), L(0x79), L(0xb3), L(0x57), L(0x8b), L(0x8a), L(0x75), + L(0x98), L(0x18), L(0x42), L(0x2f), L(0x29), L(0xa3), L(0x82), L(0xef), L(0x9f), L(0x86), L(0x6), L(0x23), L(0xe1), L(0x75), + L(0xfa), L(0x8), L(0xb1), L(0xde), L(0x17), L(0x4a), + }, + }, + TestCase{ + .input = "huffman-rand-limit.input", + .want = "huffman-rand-limit.{s}.expect", + .want_no_input = "huffman-rand-limit.{s}.expect-noinput", + .tokens = &[_]Token{ + L(0x61), M(1, 74), L(0xa), L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xa), L(0x85), L(0x94), L(0x25), L(0x80), + L(0xaf), L(0xc2), L(0xfe), L(0x8d), L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde), + L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b), L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73), + L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4), L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15), + L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde), L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9), + L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14), L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xa), + L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c), L(0x99), L(0xdf), L(0xfd), L(0x70), + L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5), L(0xd4), L(0x80), L(0x59), + L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20), L(0x1b), L(0x46), + L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4), L(0x7b), + L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28), + L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f), + L(0xa0), L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4), + L(0x75), L(0xda), L(0xea), L(0x9b), L(0xa), + }, + }, + TestCase{ + .input = "huffman-shifts.input", + .want = "huffman-shifts.{s}.expect", + .want_no_input = "huffman-shifts.{s}.expect-noinput", + .tokens = &[_]Token{ + L('1'), L('0'), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), + M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), + M(2, 258), M(2, 76), L(0xd), L(0xa), L('2'), L('3'), M(2, 258), M(2, 258), + M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 256), + }, + }, + TestCase{ + .input = "huffman-text-shift.input", + .want = "huffman-text-shift.{s}.expect", + .want_no_input = "huffman-text-shift.{s}.expect-noinput", + .tokens = &[_]Token{ + L('/'), L('/'), L('C'), L('o'), L('p'), L('y'), L('r'), L('i'), + L('g'), L('h'), L('t'), L('2'), L('0'), L('0'), L('9'), L('T'), + L('h'), L('G'), L('o'), L('A'), L('u'), L('t'), L('h'), L('o'), + L('r'), L('.'), L('A'), L('l'), L('l'), M(23, 5), L('r'), L('r'), + L('v'), L('d'), L('.'), L(0xd), L(0xa), L('/'), L('/'), L('U'), + L('o'), L('f'), L('t'), L('h'), L('i'), L('o'), L('u'), L('r'), + L('c'), L('c'), L('o'), L('d'), L('i'), L('g'), L('o'), L('v'), + L('r'), L('n'), L('d'), L('b'), L('y'), L('B'), L('S'), L('D'), + L('-'), L('t'), L('y'), L('l'), M(33, 4), L('l'), L('i'), L('c'), + L('n'), L('t'), L('h'), L('t'), L('c'), L('n'), L('b'), L('f'), + L('o'), L('u'), L('n'), L('d'), L('i'), L('n'), L('t'), L('h'), + L('L'), L('I'), L('C'), L('E'), L('N'), L('S'), L('E'), L('f'), + L('i'), L('l'), L('.'), L(0xd), L(0xa), L(0xd), L(0xa), L('p'), + L('c'), L('k'), L('g'), L('m'), L('i'), L('n'), M(11, 4), L('i'), + L('m'), L('p'), L('o'), L('r'), L('t'), L('"'), L('o'), L('"'), + M(13, 4), L('f'), L('u'), L('n'), L('c'), L('m'), L('i'), L('n'), + L('('), L(')'), L('{'), L(0xd), L(0xa), L(0x9), L('v'), L('r'), + L('b'), L('='), L('m'), L('k'), L('('), L('['), L(']'), L('b'), + L('y'), L('t'), L(','), L('6'), L('5'), L('5'), L('3'), L('5'), + L(')'), L(0xd), L(0xa), L(0x9), L('f'), L(','), L('_'), L(':'), + L('='), L('o'), L('.'), L('C'), L('r'), L('t'), L('('), L('"'), + L('h'), L('u'), L('f'), L('f'), L('m'), L('n'), L('-'), L('n'), + L('u'), L('l'), L('l'), L('-'), L('m'), L('x'), L('.'), L('i'), + L('n'), L('"'), M(34, 5), L('.'), L('W'), L('r'), L('i'), L('t'), + L('('), L('b'), L(')'), L(0xd), L(0xa), L('}'), L(0xd), L(0xa), + L('A'), L('B'), L('C'), L('D'), L('E'), L('F'), L('G'), L('H'), + L('I'), L('J'), L('K'), L('L'), L('M'), L('N'), L('O'), L('P'), + L('Q'), L('R'), L('S'), L('T'), L('U'), L('V'), L('X'), L('x'), + L('y'), L('z'), L('!'), L('"'), L('#'), L(0xc2), L(0xa4), L('%'), + L('&'), L('/'), L('?'), L('"'), + }, + }, + TestCase{ + .input = "huffman-text.input", + .want = "huffman-text.{s}.expect", + .want_no_input = "huffman-text.{s}.expect-noinput", + .tokens = &[_]Token{ + L('/'), L('/'), L(' '), L('z'), L('i'), L('g'), L(' '), L('v'), + L('0'), L('.'), L('1'), L('0'), L('.'), L('0'), L(0xa), L('/'), + L('/'), L(' '), L('c'), L('r'), L('e'), L('a'), L('t'), L('e'), + L(' '), L('a'), L(' '), L('f'), L('i'), L('l'), L('e'), M(5, 4), + L('l'), L('e'), L('d'), L(' '), L('w'), L('i'), L('t'), L('h'), + L(' '), L('0'), L('x'), L('0'), L('0'), L(0xa), L('c'), L('o'), + L('n'), L('s'), L('t'), L(' '), L('s'), L('t'), L('d'), L(' '), + L('='), L(' '), L('@'), L('i'), L('m'), L('p'), L('o'), L('r'), + L('t'), L('('), L('"'), L('s'), L('t'), L('d'), L('"'), L(')'), + L(';'), L(0xa), L(0xa), L('p'), L('u'), L('b'), L(' '), L('f'), + L('n'), L(' '), L('m'), L('a'), L('i'), L('n'), L('('), L(')'), + L(' '), L('!'), L('v'), L('o'), L('i'), L('d'), L(' '), L('{'), + L(0xa), L(' '), L(' '), L(' '), L(' '), L('v'), L('a'), L('r'), + L(' '), L('b'), L(' '), L('='), L(' '), L('['), L('1'), L(']'), + L('u'), L('8'), L('{'), L('0'), L('}'), L(' '), L('*'), L('*'), + L(' '), L('6'), L('5'), L('5'), L('3'), L('5'), L(';'), M(31, 5), + M(86, 6), L('f'), L(' '), L('='), L(' '), L('t'), L('r'), L('y'), + M(94, 4), L('.'), L('f'), L('s'), L('.'), L('c'), L('w'), L('d'), + L('('), L(')'), L('.'), M(144, 6), L('F'), L('i'), L('l'), L('e'), + L('('), M(43, 5), M(1, 4), L('"'), L('h'), L('u'), L('f'), L('f'), + L('m'), L('a'), L('n'), L('-'), L('n'), L('u'), L('l'), L('l'), + L('-'), L('m'), L('a'), L('x'), L('.'), L('i'), L('n'), L('"'), + L(','), M(31, 9), L('.'), L('{'), L(' '), L('.'), L('r'), L('e'), + L('a'), L('d'), M(79, 5), L('u'), L('e'), L(' '), L('}'), M(27, 6), + L(')'), M(108, 6), L('d'), L('e'), L('f'), L('e'), L('r'), L(' '), + L('f'), L('.'), L('c'), L('l'), L('o'), L('s'), L('e'), L('('), + M(183, 4), M(22, 4), L('_'), M(124, 7), L('f'), L('.'), L('w'), L('r'), + L('i'), L('t'), L('e'), L('A'), L('l'), L('l'), L('('), L('b'), + L('['), L('0'), L('.'), L('.'), L(']'), L(')'), L(';'), L(0xa), + L('}'), L(0xa), + }, + }, + TestCase{ + .input = "huffman-zero.input", + .want = "huffman-zero.{s}.expect", + .want_no_input = "huffman-zero.{s}.expect-noinput", + .tokens = &[_]Token{ L(0x30), ml, M(1, 49) }, + }, + TestCase{ + .input = "", + .want = "", + .want_no_input = "null-long-match.{s}.expect-noinput", + .tokens = &[_]Token{ + L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, + ml, ml, ml, M(1, 8), + }, + }, + }; +}; diff --git a/lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect diff --git a/lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-null-max.golden b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-null-max.golden rename to lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect diff --git a/lib/std/compress/deflate/testdata/huffman-null-max.input b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.input similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-null-max.input rename to lib/std/compress/flate/testdata/block_writer/huffman-null-max.input diff --git a/lib/std/compress/deflate/testdata/huffman-null-max.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-null-max.wb.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect diff --git a/lib/std/compress/deflate/testdata/huffman-null-max.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-null-max.wb.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-pi.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-pi.dyn.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect diff --git a/lib/std/compress/deflate/testdata/huffman-pi.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-pi.dyn.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-pi.golden b/lib/std/compress/flate/testdata/block_writer/huffman-pi.huff.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-pi.golden rename to lib/std/compress/flate/testdata/block_writer/huffman-pi.huff.expect diff --git a/lib/std/compress/deflate/testdata/huffman-pi.input b/lib/std/compress/flate/testdata/block_writer/huffman-pi.input similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-pi.input rename to lib/std/compress/flate/testdata/block_writer/huffman-pi.input diff --git a/lib/std/compress/deflate/testdata/huffman-pi.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-pi.wb.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect diff --git a/lib/std/compress/deflate/testdata/huffman-pi.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-pi.wb.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect diff --git a/lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-rand-1k.golden b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-1k.golden rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect diff --git a/lib/std/compress/deflate/testdata/huffman-rand-1k.input b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-1k.input rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input diff --git a/lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect diff --git a/lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect diff --git a/lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-rand-limit.golden b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.huff.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-limit.golden rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.huff.expect diff --git a/lib/std/compress/deflate/testdata/huffman-rand-limit.input b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.input similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-limit.input rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.input diff --git a/lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect diff --git a/lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-rand-max.golden b/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-max.golden rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect diff --git a/lib/std/compress/deflate/testdata/huffman-rand-max.input b/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-rand-max.input rename to lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input diff --git a/lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect diff --git a/lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-shifts.golden b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-shifts.golden rename to lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect diff --git a/lib/std/compress/deflate/testdata/huffman-shifts.input b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.input similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-shifts.input rename to lib/std/compress/flate/testdata/block_writer/huffman-shifts.input diff --git a/lib/std/compress/deflate/testdata/huffman-shifts.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-shifts.wb.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect diff --git a/lib/std/compress/deflate/testdata/huffman-shifts.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-shifts.wb.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect diff --git a/lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-text-shift.golden b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.huff.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-text-shift.golden rename to lib/std/compress/flate/testdata/block_writer/huffman-text-shift.huff.expect diff --git a/lib/std/compress/deflate/testdata/huffman-text-shift.input b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.input similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-text-shift.input rename to lib/std/compress/flate/testdata/block_writer/huffman-text-shift.input diff --git a/lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect diff --git a/lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-text.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-text.dyn.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect diff --git a/lib/std/compress/deflate/testdata/huffman-text.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-text.dyn.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-text.golden b/lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-text.golden rename to lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect diff --git a/lib/std/compress/deflate/testdata/huffman-text.input b/lib/std/compress/flate/testdata/block_writer/huffman-text.input similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-text.input rename to lib/std/compress/flate/testdata/block_writer/huffman-text.input diff --git a/lib/std/compress/deflate/testdata/huffman-text.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-text.wb.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect diff --git a/lib/std/compress/deflate/testdata/huffman-text.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-text.wb.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-zero.dyn.expect b/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-zero.dyn.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect diff --git a/lib/std/compress/deflate/testdata/huffman-zero.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-zero.dyn.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput diff --git a/lib/std/compress/deflate/testdata/huffman-zero.golden b/lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-zero.golden rename to lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect diff --git a/lib/std/compress/deflate/testdata/huffman-zero.input b/lib/std/compress/flate/testdata/block_writer/huffman-zero.input similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-zero.input rename to lib/std/compress/flate/testdata/block_writer/huffman-zero.input diff --git a/lib/std/compress/deflate/testdata/huffman-zero.wb.expect b/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-zero.wb.expect rename to lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect diff --git a/lib/std/compress/deflate/testdata/huffman-zero.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/huffman-zero.wb.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput diff --git a/lib/std/compress/deflate/testdata/null-long-match.dyn.expect-noinput b/lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/null-long-match.dyn.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput diff --git a/lib/std/compress/deflate/testdata/null-long-match.wb.expect-noinput b/lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput similarity index 100% rename from lib/std/compress/deflate/testdata/null-long-match.wb.expect-noinput rename to lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput diff --git a/lib/std/compress/flate/testdata/fuzz/deflate-stream.expect b/lib/std/compress/flate/testdata/fuzz/deflate-stream.expect new file mode 100644 index 000000000000..ab73c04108ca --- /dev/null +++ b/lib/std/compress/flate/testdata/fuzz/deflate-stream.expect @@ -0,0 +1,22 @@ +[ + { id: "brieflz", + name: "BriefLZ", + libraryUrl: "https://github.com/jibsen/brieflz", + license: "MIT", + revision: "bcaa6a1ee7ccf005512b5c23aa92b40cf75f9ed1", + codecs: [ { name: "brieflz" } ], }, + { id: "brotli", + name: "Brotli", + libraryUrl: "https://github.com/google/brotli", + license: "Apache 2.0", + revision: "1dd66ef114fd244778d9dcb5da09c28b49a0df33", + codecs: [ { name: "brotli", + levels: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + streaming: true } ], }, + { id: "bsc", + name: "bsc", + libraryUrl: "http://libbsc.com/", + license: "Apache 2.0", + revision: "b2b07421381b19b2fada8b291f3cdead10578abc", + codecs: [ { name: "bsc" } ] } +] diff --git a/lib/std/compress/flate/testdata/fuzz/deflate-stream.input b/lib/std/compress/flate/testdata/fuzz/deflate-stream.input new file mode 100644 index 000000000000..a0ed06f87265 --- /dev/null +++ b/lib/std/compress/flate/testdata/fuzz/deflate-stream.input @@ -0,0 +1,3 @@ +•=oÂ0†çò+Ṉ„ÄÙÚ­R»µKƒ}>W!AI@j+þ{ +¨|ˆª–¼<çóÝóÎFÎx[ÀØ´ž\õ9f;Pë%ü0৷#®¼iuûñÚV¡¸èûUWDQéûÅÚL°YFïÞtTGç¿U©î†Ÿ_Ž´¥ï|SãQë©D¢ã. - -Abstract - - This specification defines a lossless compressed data format that - compresses data using a combination of the LZ77 algorithm and Huffman - coding, with efficiency comparable to the best currently available - general-purpose compression methods. The data can be produced or - consumed, even for an arbitrarily long sequentially presented input - data stream, using only an a priori bounded amount of intermediate - storage. The format can be implemented readily in a manner not - covered by patents. - - - - - - - - -Deutsch Informational [Page 1] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - -Table of Contents - - 1. Introduction ................................................... 2 - 1.1. Purpose ................................................... 2 - 1.2. Intended audience ......................................... 3 - 1.3. Scope ..................................................... 3 - 1.4. Compliance ................................................ 3 - 1.5. Definitions of terms and conventions used ................ 3 - 1.6. Changes from previous versions ............................ 4 - 2. Compressed representation overview ............................. 4 - 3. Detailed specification ......................................... 5 - 3.1. Overall conventions ....................................... 5 - 3.1.1. Packing into bytes .................................. 5 - 3.2. Compressed block format ................................... 6 - 3.2.1. Synopsis of prefix and Huffman coding ............... 6 - 3.2.2. Use of Huffman coding in the "deflate" format ....... 7 - 3.2.3. Details of block format ............................. 9 - 3.2.4. Non-compressed blocks (BTYPE=00) ................... 11 - 3.2.5. Compressed blocks (length and distance codes) ...... 11 - 3.2.6. Compression with fixed Huffman codes (BTYPE=01) .... 12 - 3.2.7. Compression with dynamic Huffman codes (BTYPE=10) .. 13 - 3.3. Compliance ............................................... 14 - 4. Compression algorithm details ................................. 14 - 5. References .................................................... 16 - 6. Security Considerations ....................................... 16 - 7. Source code ................................................... 16 - 8. Acknowledgements .............................................. 16 - 9. Author's Address .............................................. 17 - -1. Introduction - - 1.1. Purpose - - The purpose of this specification is to define a lossless - compressed data format that: - * Is independent of CPU type, operating system, file system, - and character set, and hence can be used for interchange; - * Can be produced or consumed, even for an arbitrarily long - sequentially presented input data stream, using only an a - priori bounded amount of intermediate storage, and hence - can be used in data communications or similar structures - such as Unix filters; - * Compresses data with efficiency comparable to the best - currently available general-purpose compression methods, - and in particular considerably better than the "compress" - program; - * Can be implemented readily in a manner not covered by - patents, and hence can be practiced freely; - - - -Deutsch Informational [Page 2] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - - * Is compatible with the file format produced by the current - widely used gzip utility, in that conforming decompressors - will be able to read data produced by the existing gzip - compressor. - - The data format defined by this specification does not attempt to: - - * Allow random access to compressed data; - * Compress specialized data (e.g., raster graphics) as well - as the best currently available specialized algorithms. - - A simple counting argument shows that no lossless compression - algorithm can compress every possible input data set. For the - format defined here, the worst case expansion is 5 bytes per 32K- - byte block, i.e., a size increase of 0.015% for large data sets. - English text usually compresses by a factor of 2.5 to 3; - executable files usually compress somewhat less; graphical data - such as raster images may compress much more. - - 1.2. Intended audience - - This specification is intended for use by implementors of software - to compress data into "deflate" format and/or decompress data from - "deflate" format. - - The text of the specification assumes a basic background in - programming at the level of bits and other primitive data - representations. Familiarity with the technique of Huffman coding - is helpful but not required. - - 1.3. Scope - - The specification specifies a method for representing a sequence - of bytes as a (usually shorter) sequence of bits, and a method for - packing the latter bit sequence into bytes. - - 1.4. Compliance - - Unless otherwise indicated below, a compliant decompressor must be - able to accept and decompress any data set that conforms to all - the specifications presented here; a compliant compressor must - produce data sets that conform to all the specifications presented - here. - - 1.5. Definitions of terms and conventions used - - Byte: 8 bits stored or transmitted as a unit (same as an octet). - For this specification, a byte is exactly 8 bits, even on machines - - - -Deutsch Informational [Page 3] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - - which store a character on a number of bits different from eight. - See below, for the numbering of bits within a byte. - - String: a sequence of arbitrary bytes. - - 1.6. Changes from previous versions - - There have been no technical changes to the deflate format since - version 1.1 of this specification. In version 1.2, some - terminology was changed. Version 1.3 is a conversion of the - specification to RFC style. - -2. Compressed representation overview - - A compressed data set consists of a series of blocks, corresponding - to successive blocks of input data. The block sizes are arbitrary, - except that non-compressible blocks are limited to 65,535 bytes. - - Each block is compressed using a combination of the LZ77 algorithm - and Huffman coding. The Huffman trees for each block are independent - of those for previous or subsequent blocks; the LZ77 algorithm may - use a reference to a duplicated string occurring in a previous block, - up to 32K input bytes before. - - Each block consists of two parts: a pair of Huffman code trees that - describe the representation of the compressed data part, and a - compressed data part. (The Huffman trees themselves are compressed - using Huffman encoding.) The compressed data consists of a series of - elements of two types: literal bytes (of strings that have not been - detected as duplicated within the previous 32K input bytes), and - pointers to duplicated strings, where a pointer is represented as a - pair . The representation used in the - "deflate" format limits distances to 32K bytes and lengths to 258 - bytes, but does not limit the size of a block, except for - uncompressible blocks, which are limited as noted above. - - Each type of value (literals, distances, and lengths) in the - compressed data is represented using a Huffman code, using one code - tree for literals and lengths and a separate code tree for distances. - The code trees for each block appear in a compact form just before - the compressed data for that block. - - - - - - - - - - -Deutsch Informational [Page 4] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - -3. Detailed specification - - 3.1. Overall conventions In the diagrams below, a box like this: - - +---+ - | | <-- the vertical bars might be missing - +---+ - - represents one byte; a box like this: - - +==============+ - | | - +==============+ - - represents a variable number of bytes. - - Bytes stored within a computer do not have a "bit order", since - they are always treated as a unit. However, a byte considered as - an integer between 0 and 255 does have a most- and least- - significant bit, and since we write numbers with the most- - significant digit on the left, we also write bytes with the most- - significant bit on the left. In the diagrams below, we number the - bits of a byte so that bit 0 is the least-significant bit, i.e., - the bits are numbered: - - +--------+ - |76543210| - +--------+ - - Within a computer, a number may occupy multiple bytes. All - multi-byte numbers in the format described here are stored with - the least-significant byte first (at the lower memory address). - For example, the decimal number 520 is stored as: - - 0 1 - +--------+--------+ - |00001000|00000010| - +--------+--------+ - ^ ^ - | | - | + more significant byte = 2 x 256 - + less significant byte = 8 - - 3.1.1. Packing into bytes - - This document does not address the issue of the order in which - bits of a byte are transmitted on a bit-sequential medium, - since the final data format described here is byte- rather than - - - -Deutsch Informational [Page 5] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - - bit-oriented. However, we describe the compressed block format - in below, as a sequence of data elements of various bit - lengths, not a sequence of bytes. We must therefore specify - how to pack these data elements into bytes to form the final - compressed byte sequence: - - * Data elements are packed into bytes in order of - increasing bit number within the byte, i.e., starting - with the least-significant bit of the byte. - * Data elements other than Huffman codes are packed - starting with the least-significant bit of the data - element. - * Huffman codes are packed starting with the most- - significant bit of the code. - - In other words, if one were to print out the compressed data as - a sequence of bytes, starting with the first byte at the - *right* margin and proceeding to the *left*, with the most- - significant bit of each byte on the left as usual, one would be - able to parse the result from right to left, with fixed-width - elements in the correct MSB-to-LSB order and Huffman codes in - bit-reversed order (i.e., with the first bit of the code in the - relative LSB position). - - 3.2. Compressed block format - - 3.2.1. Synopsis of prefix and Huffman coding - - Prefix coding represents symbols from an a priori known - alphabet by bit sequences (codes), one code for each symbol, in - a manner such that different symbols may be represented by bit - sequences of different lengths, but a parser can always parse - an encoded string unambiguously symbol-by-symbol. - - We define a prefix code in terms of a binary tree in which the - two edges descending from each non-leaf node are labeled 0 and - 1 and in which the leaf nodes correspond one-for-one with (are - labeled with) the symbols of the alphabet; then the code for a - symbol is the sequence of 0's and 1's on the edges leading from - the root to the leaf labeled with that symbol. For example: - - - - - - - - - - - -Deutsch Informational [Page 6] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - - /\ Symbol Code - 0 1 ------ ---- - / \ A 00 - /\ B B 1 - 0 1 C 011 - / \ D 010 - A /\ - 0 1 - / \ - D C - - A parser can decode the next symbol from an encoded input - stream by walking down the tree from the root, at each step - choosing the edge corresponding to the next input bit. - - Given an alphabet with known symbol frequencies, the Huffman - algorithm allows the construction of an optimal prefix code - (one which represents strings with those symbol frequencies - using the fewest bits of any possible prefix codes for that - alphabet). Such a code is called a Huffman code. (See - reference [1] in Chapter 5, references for additional - information on Huffman codes.) - - Note that in the "deflate" format, the Huffman codes for the - various alphabets must not exceed certain maximum code lengths. - This constraint complicates the algorithm for computing code - lengths from symbol frequencies. Again, see Chapter 5, - references for details. - - 3.2.2. Use of Huffman coding in the "deflate" format - - The Huffman codes used for each alphabet in the "deflate" - format have two additional rules: - - * All codes of a given bit length have lexicographically - consecutive values, in the same order as the symbols - they represent; - - * Shorter codes lexicographically precede longer codes. - - - - - - - - - - - - -Deutsch Informational [Page 7] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - - We could recode the example above to follow this rule as - follows, assuming that the order of the alphabet is ABCD: - - Symbol Code - ------ ---- - A 10 - B 0 - C 110 - D 111 - - I.e., 0 precedes 10 which precedes 11x, and 110 and 111 are - lexicographically consecutive. - - Given this rule, we can define the Huffman code for an alphabet - just by giving the bit lengths of the codes for each symbol of - the alphabet in order; this is sufficient to determine the - actual codes. In our example, the code is completely defined - by the sequence of bit lengths (2, 1, 3, 3). The following - algorithm generates the codes as integers, intended to be read - from most- to least-significant bit. The code lengths are - initially in tree[I].Len; the codes are produced in - tree[I].Code. - - 1) Count the number of codes for each code length. Let - bl_count[N] be the number of codes of length N, N >= 1. - - 2) Find the numerical value of the smallest code for each - code length: - - code = 0; - bl_count[0] = 0; - for (bits = 1; bits <= MAX_BITS; bits++) { - code = (code + bl_count[bits-1]) << 1; - next_code[bits] = code; - } - - 3) Assign numerical values to all codes, using consecutive - values for all codes of the same length with the base - values determined at step 2. Codes that are never used - (which have a bit length of zero) must not be assigned a - value. - - for (n = 0; n <= max_code; n++) { - len = tree[n].Len; - if (len != 0) { - tree[n].Code = next_code[len]; - next_code[len]++; - } - - - -Deutsch Informational [Page 8] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - - } - - Example: - - Consider the alphabet ABCDEFGH, with bit lengths (3, 3, 3, 3, - 3, 2, 4, 4). After step 1, we have: - - N bl_count[N] - - ----------- - 2 1 - 3 5 - 4 2 - - Step 2 computes the following next_code values: - - N next_code[N] - - ------------ - 1 0 - 2 0 - 3 2 - 4 14 - - Step 3 produces the following code values: - - Symbol Length Code - ------ ------ ---- - A 3 010 - B 3 011 - C 3 100 - D 3 101 - E 3 110 - F 2 00 - G 4 1110 - H 4 1111 - - 3.2.3. Details of block format - - Each block of compressed data begins with 3 header bits - containing the following data: - - first bit BFINAL - next 2 bits BTYPE - - Note that the header bits do not necessarily begin on a byte - boundary, since a block does not necessarily occupy an integral - number of bytes. - - - - - -Deutsch Informational [Page 9] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - - BFINAL is set if and only if this is the last block of the data - set. - - BTYPE specifies how the data are compressed, as follows: - - 00 - no compression - 01 - compressed with fixed Huffman codes - 10 - compressed with dynamic Huffman codes - 11 - reserved (error) - - The only difference between the two compressed cases is how the - Huffman codes for the literal/length and distance alphabets are - defined. - - In all cases, the decoding algorithm for the actual data is as - follows: - - do - read block header from input stream. - if stored with no compression - skip any remaining bits in current partially - processed byte - read LEN and NLEN (see next section) - copy LEN bytes of data to output - otherwise - if compressed with dynamic Huffman codes - read representation of code trees (see - subsection below) - loop (until end of block code recognized) - decode literal/length value from input stream - if value < 256 - copy value (literal byte) to output stream - otherwise - if value = end of block (256) - break from loop - otherwise (value = 257..285) - decode distance from input stream - - move backwards distance bytes in the output - stream, and copy length bytes from this - position to the output stream. - end loop - while not last block - - Note that a duplicated string reference may refer to a string - in a previous block; i.e., the backward distance may cross one - or more block boundaries. However a distance cannot refer past - the beginning of the output stream. (An application using a - - - -Deutsch Informational [Page 10] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - - preset dictionary might discard part of the output stream; a - distance can refer to that part of the output stream anyway) - Note also that the referenced string may overlap the current - position; for example, if the last 2 bytes decoded have values - X and Y, a string reference with - adds X,Y,X,Y,X to the output stream. - - We now specify each compression method in turn. - - 3.2.4. Non-compressed blocks (BTYPE=00) - - Any bits of input up to the next byte boundary are ignored. - The rest of the block consists of the following information: - - 0 1 2 3 4... - +---+---+---+---+================================+ - | LEN | NLEN |... LEN bytes of literal data...| - +---+---+---+---+================================+ - - LEN is the number of data bytes in the block. NLEN is the - one's complement of LEN. - - 3.2.5. Compressed blocks (length and distance codes) - - As noted above, encoded data blocks in the "deflate" format - consist of sequences of symbols drawn from three conceptually - distinct alphabets: either literal bytes, from the alphabet of - byte values (0..255), or pairs, - where the length is drawn from (3..258) and the distance is - drawn from (1..32,768). In fact, the literal and length - alphabets are merged into a single alphabet (0..285), where - values 0..255 represent literal bytes, the value 256 indicates - end-of-block, and values 257..285 represent length codes - (possibly in conjunction with extra bits following the symbol - code) as follows: - - - - - - - - - - - - - - - - -Deutsch Informational [Page 11] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - - Extra Extra Extra - Code Bits Length(s) Code Bits Lengths Code Bits Length(s) - ---- ---- ------ ---- ---- ------- ---- ---- ------- - 257 0 3 267 1 15,16 277 4 67-82 - 258 0 4 268 1 17,18 278 4 83-98 - 259 0 5 269 2 19-22 279 4 99-114 - 260 0 6 270 2 23-26 280 4 115-130 - 261 0 7 271 2 27-30 281 5 131-162 - 262 0 8 272 2 31-34 282 5 163-194 - 263 0 9 273 3 35-42 283 5 195-226 - 264 0 10 274 3 43-50 284 5 227-257 - 265 1 11,12 275 3 51-58 285 0 258 - 266 1 13,14 276 3 59-66 - - The extra bits should be interpreted as a machine integer - stored with the most-significant bit first, e.g., bits 1110 - represent the value 14. - - Extra Extra Extra - Code Bits Dist Code Bits Dist Code Bits Distance - ---- ---- ---- ---- ---- ------ ---- ---- -------- - 0 0 1 10 4 33-48 20 9 1025-1536 - 1 0 2 11 4 49-64 21 9 1537-2048 - 2 0 3 12 5 65-96 22 10 2049-3072 - 3 0 4 13 5 97-128 23 10 3073-4096 - 4 1 5,6 14 6 129-192 24 11 4097-6144 - 5 1 7,8 15 6 193-256 25 11 6145-8192 - 6 2 9-12 16 7 257-384 26 12 8193-12288 - 7 2 13-16 17 7 385-512 27 12 12289-16384 - 8 3 17-24 18 8 513-768 28 13 16385-24576 - 9 3 25-32 19 8 769-1024 29 13 24577-32768 - - 3.2.6. Compression with fixed Huffman codes (BTYPE=01) - - The Huffman codes for the two alphabets are fixed, and are not - represented explicitly in the data. The Huffman code lengths - for the literal/length alphabet are: - - Lit Value Bits Codes - --------- ---- ----- - 0 - 143 8 00110000 through - 10111111 - 144 - 255 9 110010000 through - 111111111 - 256 - 279 7 0000000 through - 0010111 - 280 - 287 8 11000000 through - 11000111 - - - -Deutsch Informational [Page 12] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - - The code lengths are sufficient to generate the actual codes, - as described above; we show the codes in the table for added - clarity. Literal/length values 286-287 will never actually - occur in the compressed data, but participate in the code - construction. - - Distance codes 0-31 are represented by (fixed-length) 5-bit - codes, with possible additional bits as shown in the table - shown in Paragraph 3.2.5, above. Note that distance codes 30- - 31 will never actually occur in the compressed data. - - 3.2.7. Compression with dynamic Huffman codes (BTYPE=10) - - The Huffman codes for the two alphabets appear in the block - immediately after the header bits and before the actual - compressed data, first the literal/length code and then the - distance code. Each code is defined by a sequence of code - lengths, as discussed in Paragraph 3.2.2, above. For even - greater compactness, the code length sequences themselves are - compressed using a Huffman code. The alphabet for code lengths - is as follows: - - 0 - 15: Represent code lengths of 0 - 15 - 16: Copy the previous code length 3 - 6 times. - The next 2 bits indicate repeat length - (0 = 3, ... , 3 = 6) - Example: Codes 8, 16 (+2 bits 11), - 16 (+2 bits 10) will expand to - 12 code lengths of 8 (1 + 6 + 5) - 17: Repeat a code length of 0 for 3 - 10 times. - (3 bits of length) - 18: Repeat a code length of 0 for 11 - 138 times - (7 bits of length) - - A code length of 0 indicates that the corresponding symbol in - the literal/length or distance alphabet will not occur in the - block, and should not participate in the Huffman code - construction algorithm given earlier. If only one distance - code is used, it is encoded using one bit, not zero bits; in - this case there is a single code length of one, with one unused - code. One distance code of zero bits means that there are no - distance codes used at all (the data is all literals). - - We can now define the format of the block: - - 5 Bits: HLIT, # of Literal/Length codes - 257 (257 - 286) - 5 Bits: HDIST, # of Distance codes - 1 (1 - 32) - 4 Bits: HCLEN, # of Code Length codes - 4 (4 - 19) - - - -Deutsch Informational [Page 13] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - - (HCLEN + 4) x 3 bits: code lengths for the code length - alphabet given just above, in the order: 16, 17, 18, - 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 - - These code lengths are interpreted as 3-bit integers - (0-7); as above, a code length of 0 means the - corresponding symbol (literal/length or distance code - length) is not used. - - HLIT + 257 code lengths for the literal/length alphabet, - encoded using the code length Huffman code - - HDIST + 1 code lengths for the distance alphabet, - encoded using the code length Huffman code - - The actual compressed data of the block, - encoded using the literal/length and distance Huffman - codes - - The literal/length symbol 256 (end of data), - encoded using the literal/length Huffman code - - The code length repeat codes can cross from HLIT + 257 to the - HDIST + 1 code lengths. In other words, all code lengths form - a single sequence of HLIT + HDIST + 258 values. - - 3.3. Compliance - - A compressor may limit further the ranges of values specified in - the previous section and still be compliant; for example, it may - limit the range of backward pointers to some value smaller than - 32K. Similarly, a compressor may limit the size of blocks so that - a compressible block fits in memory. - - A compliant decompressor must accept the full range of possible - values defined in the previous section, and must accept blocks of - arbitrary size. - -4. Compression algorithm details - - While it is the intent of this document to define the "deflate" - compressed data format without reference to any particular - compression algorithm, the format is related to the compressed - formats produced by LZ77 (Lempel-Ziv 1977, see reference [2] below); - since many variations of LZ77 are patented, it is strongly - recommended that the implementor of a compressor follow the general - algorithm presented here, which is known not to be patented per se. - The material in this section is not part of the definition of the - - - -Deutsch Informational [Page 14] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - - specification per se, and a compressor need not follow it in order to - be compliant. - - The compressor terminates a block when it determines that starting a - new block with fresh trees would be useful, or when the block size - fills up the compressor's block buffer. - - The compressor uses a chained hash table to find duplicated strings, - using a hash function that operates on 3-byte sequences. At any - given point during compression, let XYZ be the next 3 input bytes to - be examined (not necessarily all different, of course). First, the - compressor examines the hash chain for XYZ. If the chain is empty, - the compressor simply writes out X as a literal byte and advances one - byte in the input. If the hash chain is not empty, indicating that - the sequence XYZ (or, if we are unlucky, some other 3 bytes with the - same hash function value) has occurred recently, the compressor - compares all strings on the XYZ hash chain with the actual input data - sequence starting at the current point, and selects the longest - match. - - The compressor searches the hash chains starting with the most recent - strings, to favor small distances and thus take advantage of the - Huffman encoding. The hash chains are singly linked. There are no - deletions from the hash chains; the algorithm simply discards matches - that are too old. To avoid a worst-case situation, very long hash - chains are arbitrarily truncated at a certain length, determined by a - run-time parameter. - - To improve overall compression, the compressor optionally defers the - selection of matches ("lazy matching"): after a match of length N has - been found, the compressor searches for a longer match starting at - the next input byte. If it finds a longer match, it truncates the - previous match to a length of one (thus producing a single literal - byte) and then emits the longer match. Otherwise, it emits the - original match, and, as described above, advances N bytes before - continuing. - - Run-time parameters also control this "lazy match" procedure. If - compression ratio is most important, the compressor attempts a - complete second search regardless of the length of the first match. - In the normal case, if the current match is "long enough", the - compressor reduces the search for a longer match, thus speeding up - the process. If speed is most important, the compressor inserts new - strings in the hash table only when no match was found, or when the - match is not "too long". This degrades the compression ratio but - saves time since there are both fewer insertions and fewer searches. - - - - - -Deutsch Informational [Page 15] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - -5. References - - [1] Huffman, D. A., "A Method for the Construction of Minimum - Redundancy Codes", Proceedings of the Institute of Radio - Engineers, September 1952, Volume 40, Number 9, pp. 1098-1101. - - [2] Ziv J., Lempel A., "A Universal Algorithm for Sequential Data - Compression", IEEE Transactions on Information Theory, Vol. 23, - No. 3, pp. 337-343. - - [3] Gailly, J.-L., and Adler, M., ZLIB documentation and sources, - available in ftp://ftp.uu.net/pub/archiving/zip/doc/ - - [4] Gailly, J.-L., and Adler, M., GZIP documentation and sources, - available as gzip-*.tar in ftp://prep.ai.mit.edu/pub/gnu/ - - [5] Schwartz, E. S., and Kallick, B. "Generating a canonical prefix - encoding." Comm. ACM, 7,3 (Mar. 1964), pp. 166-169. - - [6] Hirschberg and Lelewer, "Efficient decoding of prefix codes," - Comm. ACM, 33,4, April 1990, pp. 449-459. - -6. Security Considerations - - Any data compression method involves the reduction of redundancy in - the data. Consequently, any corruption of the data is likely to have - severe effects and be difficult to correct. Uncompressed text, on - the other hand, will probably still be readable despite the presence - of some corrupted bytes. - - It is recommended that systems using this data format provide some - means of validating the integrity of the compressed data. See - reference [3], for example. - -7. Source code - - Source code for a C language implementation of a "deflate" compliant - compressor and decompressor is available within the zlib package at - ftp://ftp.uu.net/pub/archiving/zip/zlib/. - -8. Acknowledgements - - Trademarks cited in this document are the property of their - respective owners. - - Phil Katz designed the deflate format. Jean-Loup Gailly and Mark - Adler wrote the related software described in this specification. - Glenn Randers-Pehrson converted this document to RFC and HTML format. - - - -Deutsch Informational [Page 16] - -RFC 1951 DEFLATE Compressed Data Format Specification May 1996 - - -9. Author's Address - - L. Peter Deutsch - Aladdin Enterprises - 203 Santa Margarita Ave. - Menlo Park, CA 94025 - - Phone: (415) 322-0103 (AM only) - FAX: (415) 322-1734 - EMail: - - Questions about the technical content of this specification can be - sent by email to: - - Jean-Loup Gailly and - Mark Adler - - Editorial comments on this specification can be sent by email to: - - L. Peter Deutsch and - Glenn Randers-Pehrson - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Deutsch Informational [Page 17] - diff --git a/lib/std/compress/testdata/rfc1951.txt.fixed.z.9 b/lib/std/compress/testdata/rfc1951.txt.fixed.z.9 deleted file mode 100644 index 8ea590477005..000000000000 Binary files a/lib/std/compress/testdata/rfc1951.txt.fixed.z.9 and /dev/null differ diff --git a/lib/std/compress/testdata/rfc1951.txt.z.0 b/lib/std/compress/testdata/rfc1951.txt.z.0 deleted file mode 100644 index 3f50fb68f8da..000000000000 Binary files a/lib/std/compress/testdata/rfc1951.txt.z.0 and /dev/null differ diff --git a/lib/std/compress/testdata/rfc1951.txt.z.9 b/lib/std/compress/testdata/rfc1951.txt.z.9 deleted file mode 100644 index 84e7cbe5b715..000000000000 Binary files a/lib/std/compress/testdata/rfc1951.txt.z.9 and /dev/null differ diff --git a/lib/std/compress/testdata/rfc1952.txt b/lib/std/compress/testdata/rfc1952.txt deleted file mode 100644 index a8e51b4567fd..000000000000 --- a/lib/std/compress/testdata/rfc1952.txt +++ /dev/null @@ -1,675 +0,0 @@ - - - - - - -Network Working Group P. Deutsch -Request for Comments: 1952 Aladdin Enterprises -Category: Informational May 1996 - - - GZIP file format specification version 4.3 - -Status of This Memo - - This memo provides information for the Internet community. This memo - does not specify an Internet standard of any kind. Distribution of - this memo is unlimited. - -IESG Note: - - The IESG takes no position on the validity of any Intellectual - Property Rights statements contained in this document. - -Notices - - Copyright (c) 1996 L. Peter Deutsch - - Permission is granted to copy and distribute this document for any - purpose and without charge, including translations into other - languages and incorporation into compilations, provided that the - copyright notice and this notice are preserved, and that any - substantive changes or deletions from the original are clearly - marked. - - A pointer to the latest version of this and related documentation in - HTML format can be found at the URL - . - -Abstract - - This specification defines a lossless compressed data format that is - compatible with the widely used GZIP utility. The format includes a - cyclic redundancy check value for detecting data corruption. The - format presently uses the DEFLATE method of compression but can be - easily extended to use other compression methods. The format can be - implemented readily in a manner not covered by patents. - - - - - - - - - - -Deutsch Informational [Page 1] - -RFC 1952 GZIP File Format Specification May 1996 - - -Table of Contents - - 1. Introduction ................................................... 2 - 1.1. Purpose ................................................... 2 - 1.2. Intended audience ......................................... 3 - 1.3. Scope ..................................................... 3 - 1.4. Compliance ................................................ 3 - 1.5. Definitions of terms and conventions used ................. 3 - 1.6. Changes from previous versions ............................ 3 - 2. Detailed specification ......................................... 4 - 2.1. Overall conventions ....................................... 4 - 2.2. File format ............................................... 5 - 2.3. Member format ............................................. 5 - 2.3.1. Member header and trailer ........................... 6 - 2.3.1.1. Extra field ................................... 8 - 2.3.1.2. Compliance .................................... 9 - 3. References .................................................. 9 - 4. Security Considerations .................................... 10 - 5. Acknowledgements ........................................... 10 - 6. Author's Address ........................................... 10 - 7. Appendix: Jean-Loup Gailly's gzip utility .................. 11 - 8. Appendix: Sample CRC Code .................................. 11 - -1. Introduction - - 1.1. Purpose - - The purpose of this specification is to define a lossless - compressed data format that: - - * Is independent of CPU type, operating system, file system, - and character set, and hence can be used for interchange; - * Can compress or decompress a data stream (as opposed to a - randomly accessible file) to produce another data stream, - using only an a priori bounded amount of intermediate - storage, and hence can be used in data communications or - similar structures such as Unix filters; - * Compresses data with efficiency comparable to the best - currently available general-purpose compression methods, - and in particular considerably better than the "compress" - program; - * Can be implemented readily in a manner not covered by - patents, and hence can be practiced freely; - * Is compatible with the file format produced by the current - widely used gzip utility, in that conforming decompressors - will be able to read data produced by the existing gzip - compressor. - - - - -Deutsch Informational [Page 2] - -RFC 1952 GZIP File Format Specification May 1996 - - - The data format defined by this specification does not attempt to: - - * Provide random access to compressed data; - * Compress specialized data (e.g., raster graphics) as well as - the best currently available specialized algorithms. - - 1.2. Intended audience - - This specification is intended for use by implementors of software - to compress data into gzip format and/or decompress data from gzip - format. - - The text of the specification assumes a basic background in - programming at the level of bits and other primitive data - representations. - - 1.3. Scope - - The specification specifies a compression method and a file format - (the latter assuming only that a file can store a sequence of - arbitrary bytes). It does not specify any particular interface to - a file system or anything about character sets or encodings - (except for file names and comments, which are optional). - - 1.4. Compliance - - Unless otherwise indicated below, a compliant decompressor must be - able to accept and decompress any file that conforms to all the - specifications presented here; a compliant compressor must produce - files that conform to all the specifications presented here. The - material in the appendices is not part of the specification per se - and is not relevant to compliance. - - 1.5. Definitions of terms and conventions used - - byte: 8 bits stored or transmitted as a unit (same as an octet). - (For this specification, a byte is exactly 8 bits, even on - machines which store a character on a number of bits different - from 8.) See below for the numbering of bits within a byte. - - 1.6. Changes from previous versions - - There have been no technical changes to the gzip format since - version 4.1 of this specification. In version 4.2, some - terminology was changed, and the sample CRC code was rewritten for - clarity and to eliminate the requirement for the caller to do pre- - and post-conditioning. Version 4.3 is a conversion of the - specification to RFC style. - - - -Deutsch Informational [Page 3] - -RFC 1952 GZIP File Format Specification May 1996 - - -2. Detailed specification - - 2.1. Overall conventions - - In the diagrams below, a box like this: - - +---+ - | | <-- the vertical bars might be missing - +---+ - - represents one byte; a box like this: - - +==============+ - | | - +==============+ - - represents a variable number of bytes. - - Bytes stored within a computer do not have a "bit order", since - they are always treated as a unit. However, a byte considered as - an integer between 0 and 255 does have a most- and least- - significant bit, and since we write numbers with the most- - significant digit on the left, we also write bytes with the most- - significant bit on the left. In the diagrams below, we number the - bits of a byte so that bit 0 is the least-significant bit, i.e., - the bits are numbered: - - +--------+ - |76543210| - +--------+ - - This document does not address the issue of the order in which - bits of a byte are transmitted on a bit-sequential medium, since - the data format described here is byte- rather than bit-oriented. - - Within a computer, a number may occupy multiple bytes. All - multi-byte numbers in the format described here are stored with - the least-significant byte first (at the lower memory address). - For example, the decimal number 520 is stored as: - - 0 1 - +--------+--------+ - |00001000|00000010| - +--------+--------+ - ^ ^ - | | - | + more significant byte = 2 x 256 - + less significant byte = 8 - - - -Deutsch Informational [Page 4] - -RFC 1952 GZIP File Format Specification May 1996 - - - 2.2. File format - - A gzip file consists of a series of "members" (compressed data - sets). The format of each member is specified in the following - section. The members simply appear one after another in the file, - with no additional information before, between, or after them. - - 2.3. Member format - - Each member has the following structure: - - +---+---+---+---+---+---+---+---+---+---+ - |ID1|ID2|CM |FLG| MTIME |XFL|OS | (more-->) - +---+---+---+---+---+---+---+---+---+---+ - - (if FLG.FEXTRA set) - - +---+---+=================================+ - | XLEN |...XLEN bytes of "extra field"...| (more-->) - +---+---+=================================+ - - (if FLG.FNAME set) - - +=========================================+ - |...original file name, zero-terminated...| (more-->) - +=========================================+ - - (if FLG.FCOMMENT set) - - +===================================+ - |...file comment, zero-terminated...| (more-->) - +===================================+ - - (if FLG.FHCRC set) - - +---+---+ - | CRC16 | - +---+---+ - - +=======================+ - |...compressed blocks...| (more-->) - +=======================+ - - 0 1 2 3 4 5 6 7 - +---+---+---+---+---+---+---+---+ - | CRC32 | ISIZE | - +---+---+---+---+---+---+---+---+ - - - - -Deutsch Informational [Page 5] - -RFC 1952 GZIP File Format Specification May 1996 - - - 2.3.1. Member header and trailer - - ID1 (IDentification 1) - ID2 (IDentification 2) - These have the fixed values ID1 = 31 (0x1f, \037), ID2 = 139 - (0x8b, \213), to identify the file as being in gzip format. - - CM (Compression Method) - This identifies the compression method used in the file. CM - = 0-7 are reserved. CM = 8 denotes the "deflate" - compression method, which is the one customarily used by - gzip and which is documented elsewhere. - - FLG (FLaGs) - This flag byte is divided into individual bits as follows: - - bit 0 FTEXT - bit 1 FHCRC - bit 2 FEXTRA - bit 3 FNAME - bit 4 FCOMMENT - bit 5 reserved - bit 6 reserved - bit 7 reserved - - If FTEXT is set, the file is probably ASCII text. This is - an optional indication, which the compressor may set by - checking a small amount of the input data to see whether any - non-ASCII characters are present. In case of doubt, FTEXT - is cleared, indicating binary data. For systems which have - different file formats for ascii text and binary data, the - decompressor can use FTEXT to choose the appropriate format. - We deliberately do not specify the algorithm used to set - this bit, since a compressor always has the option of - leaving it cleared and a decompressor always has the option - of ignoring it and letting some other program handle issues - of data conversion. - - If FHCRC is set, a CRC16 for the gzip header is present, - immediately before the compressed data. The CRC16 consists - of the two least significant bytes of the CRC32 for all - bytes of the gzip header up to and not including the CRC16. - [The FHCRC bit was never set by versions of gzip up to - 1.2.4, even though it was documented with a different - meaning in gzip 1.2.4.] - - If FEXTRA is set, optional extra fields are present, as - described in a following section. - - - -Deutsch Informational [Page 6] - -RFC 1952 GZIP File Format Specification May 1996 - - - If FNAME is set, an original file name is present, - terminated by a zero byte. The name must consist of ISO - 8859-1 (LATIN-1) characters; on operating systems using - EBCDIC or any other character set for file names, the name - must be translated to the ISO LATIN-1 character set. This - is the original name of the file being compressed, with any - directory components removed, and, if the file being - compressed is on a file system with case insensitive names, - forced to lower case. There is no original file name if the - data was compressed from a source other than a named file; - for example, if the source was stdin on a Unix system, there - is no file name. - - If FCOMMENT is set, a zero-terminated file comment is - present. This comment is not interpreted; it is only - intended for human consumption. The comment must consist of - ISO 8859-1 (LATIN-1) characters. Line breaks should be - denoted by a single line feed character (10 decimal). - - Reserved FLG bits must be zero. - - MTIME (Modification TIME) - This gives the most recent modification time of the original - file being compressed. The time is in Unix format, i.e., - seconds since 00:00:00 GMT, Jan. 1, 1970. (Note that this - may cause problems for MS-DOS and other systems that use - local rather than Universal time.) If the compressed data - did not come from a file, MTIME is set to the time at which - compression started. MTIME = 0 means no time stamp is - available. - - XFL (eXtra FLags) - These flags are available for use by specific compression - methods. The "deflate" method (CM = 8) sets these flags as - follows: - - XFL = 2 - compressor used maximum compression, - slowest algorithm - XFL = 4 - compressor used fastest algorithm - - OS (Operating System) - This identifies the type of file system on which compression - took place. This may be useful in determining end-of-line - convention for text files. The currently defined values are - as follows: - - - - - - -Deutsch Informational [Page 7] - -RFC 1952 GZIP File Format Specification May 1996 - - - 0 - FAT filesystem (MS-DOS, OS/2, NT/Win32) - 1 - Amiga - 2 - VMS (or OpenVMS) - 3 - Unix - 4 - VM/CMS - 5 - Atari TOS - 6 - HPFS filesystem (OS/2, NT) - 7 - Macintosh - 8 - Z-System - 9 - CP/M - 10 - TOPS-20 - 11 - NTFS filesystem (NT) - 12 - QDOS - 13 - Acorn RISCOS - 255 - unknown - - XLEN (eXtra LENgth) - If FLG.FEXTRA is set, this gives the length of the optional - extra field. See below for details. - - CRC32 (CRC-32) - This contains a Cyclic Redundancy Check value of the - uncompressed data computed according to CRC-32 algorithm - used in the ISO 3309 standard and in section 8.1.1.6.2 of - ITU-T recommendation V.42. (See http://www.iso.ch for - ordering ISO documents. See gopher://info.itu.ch for an - online version of ITU-T V.42.) - - ISIZE (Input SIZE) - This contains the size of the original (uncompressed) input - data modulo 2^32. - - 2.3.1.1. Extra field - - If the FLG.FEXTRA bit is set, an "extra field" is present in - the header, with total length XLEN bytes. It consists of a - series of subfields, each of the form: - - +---+---+---+---+==================================+ - |SI1|SI2| LEN |... LEN bytes of subfield data ...| - +---+---+---+---+==================================+ - - SI1 and SI2 provide a subfield ID, typically two ASCII letters - with some mnemonic value. Jean-Loup Gailly - is maintaining a registry of subfield - IDs; please send him any subfield ID you wish to use. Subfield - IDs with SI2 = 0 are reserved for future use. The following - IDs are currently defined: - - - -Deutsch Informational [Page 8] - -RFC 1952 GZIP File Format Specification May 1996 - - - SI1 SI2 Data - ---------- ---------- ---- - 0x41 ('A') 0x70 ('P') Apollo file type information - - LEN gives the length of the subfield data, excluding the 4 - initial bytes. - - 2.3.1.2. Compliance - - A compliant compressor must produce files with correct ID1, - ID2, CM, CRC32, and ISIZE, but may set all the other fields in - the fixed-length part of the header to default values (255 for - OS, 0 for all others). The compressor must set all reserved - bits to zero. - - A compliant decompressor must check ID1, ID2, and CM, and - provide an error indication if any of these have incorrect - values. It must examine FEXTRA/XLEN, FNAME, FCOMMENT and FHCRC - at least so it can skip over the optional fields if they are - present. It need not examine any other part of the header or - trailer; in particular, a decompressor may ignore FTEXT and OS - and always produce binary output, and still be compliant. A - compliant decompressor must give an error indication if any - reserved bit is non-zero, since such a bit could indicate the - presence of a new field that would cause subsequent data to be - interpreted incorrectly. - -3. References - - [1] "Information Processing - 8-bit single-byte coded graphic - character sets - Part 1: Latin alphabet No.1" (ISO 8859-1:1987). - The ISO 8859-1 (Latin-1) character set is a superset of 7-bit - ASCII. Files defining this character set are available as - iso_8859-1.* in ftp://ftp.uu.net/graphics/png/documents/ - - [2] ISO 3309 - - [3] ITU-T recommendation V.42 - - [4] Deutsch, L.P.,"DEFLATE Compressed Data Format Specification", - available in ftp://ftp.uu.net/pub/archiving/zip/doc/ - - [5] Gailly, J.-L., GZIP documentation, available as gzip-*.tar in - ftp://prep.ai.mit.edu/pub/gnu/ - - [6] Sarwate, D.V., "Computation of Cyclic Redundancy Checks via Table - Look-Up", Communications of the ACM, 31(8), pp.1008-1013. - - - - -Deutsch Informational [Page 9] - -RFC 1952 GZIP File Format Specification May 1996 - - - [7] Schwaderer, W.D., "CRC Calculation", April 85 PC Tech Journal, - pp.118-133. - - [8] ftp://ftp.adelaide.edu.au/pub/rocksoft/papers/crc_v3.txt, - describing the CRC concept. - -4. Security Considerations - - Any data compression method involves the reduction of redundancy in - the data. Consequently, any corruption of the data is likely to have - severe effects and be difficult to correct. Uncompressed text, on - the other hand, will probably still be readable despite the presence - of some corrupted bytes. - - It is recommended that systems using this data format provide some - means of validating the integrity of the compressed data, such as by - setting and checking the CRC-32 check value. - -5. Acknowledgements - - Trademarks cited in this document are the property of their - respective owners. - - Jean-Loup Gailly designed the gzip format and wrote, with Mark Adler, - the related software described in this specification. Glenn - Randers-Pehrson converted this document to RFC and HTML format. - -6. Author's Address - - L. Peter Deutsch - Aladdin Enterprises - 203 Santa Margarita Ave. - Menlo Park, CA 94025 - - Phone: (415) 322-0103 (AM only) - FAX: (415) 322-1734 - EMail: - - Questions about the technical content of this specification can be - sent by email to: - - Jean-Loup Gailly and - Mark Adler - - Editorial comments on this specification can be sent by email to: - - L. Peter Deutsch and - Glenn Randers-Pehrson - - - -Deutsch Informational [Page 10] - -RFC 1952 GZIP File Format Specification May 1996 - - -7. Appendix: Jean-Loup Gailly's gzip utility - - The most widely used implementation of gzip compression, and the - original documentation on which this specification is based, were - created by Jean-Loup Gailly . Since this - implementation is a de facto standard, we mention some more of its - features here. Again, the material in this section is not part of - the specification per se, and implementations need not follow it to - be compliant. - - When compressing or decompressing a file, gzip preserves the - protection, ownership, and modification time attributes on the local - file system, since there is no provision for representing protection - attributes in the gzip file format itself. Since the file format - includes a modification time, the gzip decompressor provides a - command line switch that assigns the modification time from the file, - rather than the local modification time of the compressed input, to - the decompressed output. - -8. Appendix: Sample CRC Code - - The following sample code represents a practical implementation of - the CRC (Cyclic Redundancy Check). (See also ISO 3309 and ITU-T V.42 - for a formal specification.) - - The sample code is in the ANSI C programming language. Non C users - may find it easier to read with these hints: - - & Bitwise AND operator. - ^ Bitwise exclusive-OR operator. - >> Bitwise right shift operator. When applied to an - unsigned quantity, as here, right shift inserts zero - bit(s) at the left. - ! Logical NOT operator. - ++ "n++" increments the variable n. - 0xNNN 0x introduces a hexadecimal (base 16) constant. - Suffix L indicates a long value (at least 32 bits). - - /* Table of CRCs of all 8-bit messages. */ - unsigned long crc_table[256]; - - /* Flag: has the table been computed? Initially false. */ - int crc_table_computed = 0; - - /* Make the table for a fast CRC. */ - void make_crc_table(void) - { - unsigned long c; - - - -Deutsch Informational [Page 11] - -RFC 1952 GZIP File Format Specification May 1996 - - - int n, k; - for (n = 0; n < 256; n++) { - c = (unsigned long) n; - for (k = 0; k < 8; k++) { - if (c & 1) { - c = 0xedb88320L ^ (c >> 1); - } else { - c = c >> 1; - } - } - crc_table[n] = c; - } - crc_table_computed = 1; - } - - /* - Update a running crc with the bytes buf[0..len-1] and return - the updated crc. The crc should be initialized to zero. Pre- and - post-conditioning (one's complement) is performed within this - function so it shouldn't be done by the caller. Usage example: - - unsigned long crc = 0L; - - while (read_buffer(buffer, length) != EOF) { - crc = update_crc(crc, buffer, length); - } - if (crc != original_crc) error(); - */ - unsigned long update_crc(unsigned long crc, - unsigned char *buf, int len) - { - unsigned long c = crc ^ 0xffffffffL; - int n; - - if (!crc_table_computed) - make_crc_table(); - for (n = 0; n < len; n++) { - c = crc_table[(c ^ buf[n]) & 0xff] ^ (c >> 8); - } - return c ^ 0xffffffffL; - } - - /* Return the CRC of the bytes buf[0..len-1]. */ - unsigned long crc(unsigned char *buf, int len) - { - return update_crc(0L, buf, len); - } - - - - -Deutsch Informational [Page 12] - diff --git a/lib/std/compress/testdata/rfc1952.txt.gz b/lib/std/compress/testdata/rfc1952.txt.gz deleted file mode 100644 index 17958d64f333..000000000000 Binary files a/lib/std/compress/testdata/rfc1952.txt.gz and /dev/null differ diff --git a/lib/std/compress/zlib.zig b/lib/std/compress/zlib.zig index 6708875930aa..33401ce8457c 100644 --- a/lib/std/compress/zlib.zig +++ b/lib/std/compress/zlib.zig @@ -1,282 +1,66 @@ -// -// Compressor/Decompressor for ZLIB data streams (RFC1950) +const deflate = @import("flate/deflate.zig"); +const inflate = @import("flate/inflate.zig"); -const std = @import("std"); -const io = std.io; -const fs = std.fs; -const testing = std.testing; -const mem = std.mem; -const deflate = std.compress.deflate; - -// Zlib header format as specified in RFC1950 -const ZLibHeader = packed struct { - checksum: u5, - preset_dict: u1, - compression_level: u2, - compression_method: u4, - compression_info: u4, - - const DEFLATE = 8; - const WINDOW_32K = 7; -}; - -pub fn DecompressStream(comptime ReaderType: type) type { - return struct { - const Self = @This(); - - pub const Error = ReaderType.Error || - deflate.Decompressor(ReaderType).Error || - error{ WrongChecksum, Unsupported }; - pub const Reader = io.Reader(*Self, Error, read); - - allocator: mem.Allocator, - inflater: deflate.Decompressor(ReaderType), - in_reader: ReaderType, - hasher: std.hash.Adler32, - - fn init(allocator: mem.Allocator, source: ReaderType) !Self { - // Zlib header format is specified in RFC1950 - const header_u16 = try source.readInt(u16, .big); - - // verify the header checksum - if (header_u16 % 31 != 0) - return error.BadHeader; - const header = @as(ZLibHeader, @bitCast(header_u16)); - - // The CM field must be 8 to indicate the use of DEFLATE - if (header.compression_method != ZLibHeader.DEFLATE) - return error.InvalidCompression; - // CINFO is the base-2 logarithm of the LZ77 window size, minus 8. - // Values above 7 are unspecified and therefore rejected. - if (header.compression_info > ZLibHeader.WINDOW_32K) - return error.InvalidWindowSize; - - const dictionary = null; - // TODO: Support this case - if (header.preset_dict != 0) - return error.Unsupported; - - return Self{ - .allocator = allocator, - .inflater = try deflate.decompressor(allocator, source, dictionary), - .in_reader = source, - .hasher = std.hash.Adler32.init(), - }; - } - - pub fn deinit(self: *Self) void { - self.inflater.deinit(); - } - - // Implements the io.Reader interface - pub fn read(self: *Self, buffer: []u8) Error!usize { - if (buffer.len == 0) - return 0; - - // Read from the compressed stream and update the computed checksum - const r = try self.inflater.read(buffer); - if (r != 0) { - self.hasher.update(buffer[0..r]); - return r; - } - - // We've reached the end of stream, check if the checksum matches - const hash = try self.in_reader.readInt(u32, .big); - if (hash != self.hasher.final()) - return error.WrongChecksum; - - return 0; - } - - pub fn reader(self: *Self) Reader { - return .{ .context = self }; - } - }; +/// Decompress compressed data from reader and write plain data to the writer. +pub fn decompress(reader: anytype, writer: anytype) !void { + try inflate.decompress(.zlib, reader, writer); } -pub fn decompressStream(allocator: mem.Allocator, reader: anytype) !DecompressStream(@TypeOf(reader)) { - return DecompressStream(@TypeOf(reader)).init(allocator, reader); +/// Decompressor type +pub fn Decompressor(comptime ReaderType: type) type { + return inflate.Inflate(.zlib, ReaderType); } -pub const CompressionLevel = enum(u2) { - no_compression = 0, - fastest = 1, - default = 2, - maximum = 3, -}; - -pub const CompressStreamOptions = struct { - level: CompressionLevel = .default, -}; - -pub fn CompressStream(comptime WriterType: type) type { - return struct { - const Self = @This(); - - const Error = WriterType.Error || - deflate.Compressor(WriterType).Error; - pub const Writer = io.Writer(*Self, Error, write); - - allocator: mem.Allocator, - deflator: deflate.Compressor(WriterType), - in_writer: WriterType, - hasher: std.hash.Adler32, - - fn init(allocator: mem.Allocator, dest: WriterType, options: CompressStreamOptions) !Self { - var header = ZLibHeader{ - .compression_info = ZLibHeader.WINDOW_32K, - .compression_method = ZLibHeader.DEFLATE, - .compression_level = @intFromEnum(options.level), - .preset_dict = 0, - .checksum = 0, - }; - header.checksum = @as(u5, @truncate(31 - @as(u16, @bitCast(header)) % 31)); - - try dest.writeInt(u16, @as(u16, @bitCast(header)), .big); - - const compression_level: deflate.Compression = switch (options.level) { - .no_compression => .no_compression, - .fastest => .best_speed, - .default => .default_compression, - .maximum => .best_compression, - }; - - return Self{ - .allocator = allocator, - .deflator = try deflate.compressor(allocator, dest, .{ .level = compression_level }), - .in_writer = dest, - .hasher = std.hash.Adler32.init(), - }; - } - - pub fn write(self: *Self, bytes: []const u8) Error!usize { - if (bytes.len == 0) { - return 0; - } - - const w = try self.deflator.write(bytes); - - self.hasher.update(bytes[0..w]); - return w; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } - - pub fn deinit(self: *Self) void { - self.deflator.deinit(); - } - - pub fn finish(self: *Self) !void { - const hash = self.hasher.final(); - try self.deflator.close(); - try self.in_writer.writeInt(u32, hash, .big); - } - }; +/// Create Decompressor which will read compressed data from reader. +pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) { + return inflate.decompressor(.zlib, reader); } -pub fn compressStream(allocator: mem.Allocator, writer: anytype, options: CompressStreamOptions) !CompressStream(@TypeOf(writer)) { - return CompressStream(@TypeOf(writer)).init(allocator, writer, options); -} - -fn testDecompress(data: []const u8, expected: []const u8) !void { - var in_stream = io.fixedBufferStream(data); - - var zlib_stream = try decompressStream(testing.allocator, in_stream.reader()); - defer zlib_stream.deinit(); +/// Compression level, trades between speed and compression size. +pub const Options = deflate.Options; - // Read and decompress the whole file - const buf = try zlib_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize)); - defer testing.allocator.free(buf); - - // Check against the reference - try testing.expectEqualSlices(u8, expected, buf); +/// Compress plain data from reader and write compressed data to the writer. +pub fn compress(reader: anytype, writer: anytype, options: Options) !void { + try deflate.compress(.zlib, reader, writer, options); } -// All the test cases are obtained by compressing the RFC1951 text -// -// https://tools.ietf.org/rfc/rfc1951.txt length=36944 bytes -// SHA256=5ebf4b5b7fe1c3a0c0ab9aa3ac8c0f3853a7dc484905e76e03b0b0f301350009 -test "compressed data" { - const rfc1951_txt = @embedFile("testdata/rfc1951.txt"); - - // Compressed with compression level = 0 - try testDecompress( - @embedFile("testdata/rfc1951.txt.z.0"), - rfc1951_txt, - ); - // Compressed with compression level = 9 - try testDecompress( - @embedFile("testdata/rfc1951.txt.z.9"), - rfc1951_txt, - ); - // Compressed with compression level = 9 and fixed Huffman codes - try testDecompress( - @embedFile("testdata/rfc1951.txt.fixed.z.9"), - rfc1951_txt, - ); +/// Compressor type +pub fn Compressor(comptime WriterType: type) type { + return deflate.Compressor(.zlib, WriterType); } -test "don't read past deflate stream's end" { - try testDecompress(&[_]u8{ - 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff, - 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01, - 0x83, 0x95, 0x0b, 0xf5, - }, &[_]u8{ - 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, - 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, - }); +/// Create Compressor which outputs compressed data to the writer. +pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) { + return try deflate.compressor(.zlib, writer, options); } -test "sanity checks" { - // Truncated header - try testing.expectError( - error.EndOfStream, - testDecompress(&[_]u8{0x78}, ""), - ); - // Failed FCHECK check - try testing.expectError( - error.BadHeader, - testDecompress(&[_]u8{ 0x78, 0x9D }, ""), - ); - // Wrong CM - try testing.expectError( - error.InvalidCompression, - testDecompress(&[_]u8{ 0x79, 0x94 }, ""), - ); - // Wrong CINFO - try testing.expectError( - error.InvalidWindowSize, - testDecompress(&[_]u8{ 0x88, 0x98 }, ""), - ); - // Wrong checksum - try testing.expectError( - error.WrongChecksum, - testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""), - ); - // Truncated checksum - try testing.expectError( - error.EndOfStream, - testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""), - ); -} +/// Huffman only compression. Without Lempel-Ziv match searching. Faster +/// compression, less memory requirements but bigger compressed sizes. +pub const huffman = struct { + pub fn compress(reader: anytype, writer: anytype) !void { + try deflate.huffman.compress(.zlib, reader, writer); + } -test "compress data" { - const allocator = testing.allocator; - const rfc1951_txt = @embedFile("testdata/rfc1951.txt"); + pub fn Compressor(comptime WriterType: type) type { + return deflate.huffman.Compressor(.zlib, WriterType); + } - for (std.meta.tags(CompressionLevel)) |level| { - var compressed_data = std.ArrayList(u8).init(allocator); - defer compressed_data.deinit(); + pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) { + return deflate.huffman.compressor(.zlib, writer); + } +}; - var compressor = try compressStream(allocator, compressed_data.writer(), .{ .level = level }); - defer compressor.deinit(); +// No compression store only. Compressed size is slightly bigger than plain. +pub const store = struct { + pub fn compress(reader: anytype, writer: anytype) !void { + try deflate.store.compress(.zlib, reader, writer); + } - try compressor.writer().writeAll(rfc1951_txt); - try compressor.finish(); + pub fn Compressor(comptime WriterType: type) type { + return deflate.store.Compressor(.zlib, WriterType); + } - try testDecompress(compressed_data.items, rfc1951_txt); + pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) { + return deflate.store.compressor(.zlib, writer); } -} +}; diff --git a/lib/std/debug.zig b/lib/std/debug.zig index 601a949ecf60..3b9e46be9255 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -1212,8 +1212,7 @@ pub fn readElfDebugInfo( const chdr = section_reader.readStruct(elf.Chdr) catch continue; if (chdr.ch_type != .ZLIB) continue; - var zlib_stream = std.compress.zlib.decompressStream(allocator, section_stream.reader()) catch continue; - defer zlib_stream.deinit(); + var zlib_stream = std.compress.zlib.decompressor(section_stream.reader()); const decompressed_section = try allocator.alloc(u8, chdr.ch_size); errdefer allocator.free(decompressed_section); diff --git a/lib/std/http/Client.zig b/lib/std/http/Client.zig index ed6aec55aa15..a50e814fd4cf 100644 --- a/lib/std/http/Client.zig +++ b/lib/std/http/Client.zig @@ -404,8 +404,8 @@ pub const RequestTransfer = union(enum) { /// The decompressor for response messages. pub const Compression = union(enum) { - pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Request.TransferReader); - pub const GzipDecompressor = std.compress.gzip.Decompress(Request.TransferReader); + pub const DeflateDecompressor = std.compress.zlib.Decompressor(Request.TransferReader); + pub const GzipDecompressor = std.compress.gzip.Decompressor(Request.TransferReader); pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{}); deflate: DeflateDecompressor, @@ -601,8 +601,8 @@ pub const Request = struct { pub fn deinit(req: *Request) void { switch (req.response.compression) { .none => {}, - .deflate => |*deflate| deflate.deinit(), - .gzip => |*gzip| gzip.deinit(), + .deflate => {}, + .gzip => {}, .zstd => |*zstd| zstd.deinit(), } @@ -632,8 +632,8 @@ pub const Request = struct { switch (req.response.compression) { .none => {}, - .deflate => |*deflate| deflate.deinit(), - .gzip => |*gzip| gzip.deinit(), + .deflate => {}, + .gzip => {}, .zstd => |*zstd| zstd.deinit(), } @@ -941,10 +941,10 @@ pub const Request = struct { .identity => req.response.compression = .none, .compress, .@"x-compress" => return error.CompressionNotSupported, .deflate => req.response.compression = .{ - .deflate = std.compress.zlib.decompressStream(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed, + .deflate = std.compress.zlib.decompressor(req.transferReader()), }, .gzip, .@"x-gzip" => req.response.compression = .{ - .gzip = std.compress.gzip.decompress(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed, + .gzip = std.compress.gzip.decompressor(req.transferReader()), }, .zstd => req.response.compression = .{ .zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()), diff --git a/lib/std/http/Server.zig b/lib/std/http/Server.zig index 48c4e2cbfb2a..46590417792f 100644 --- a/lib/std/http/Server.zig +++ b/lib/std/http/Server.zig @@ -195,8 +195,8 @@ pub const ResponseTransfer = union(enum) { /// The decompressor for request messages. pub const Compression = union(enum) { - pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Response.TransferReader); - pub const GzipDecompressor = std.compress.gzip.Decompress(Response.TransferReader); + pub const DeflateDecompressor = std.compress.zlib.Decompressor(Response.TransferReader); + pub const GzipDecompressor = std.compress.gzip.Decompressor(Response.TransferReader); pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Response.TransferReader, .{}); deflate: DeflateDecompressor, @@ -420,8 +420,8 @@ pub const Response = struct { switch (res.request.compression) { .none => {}, - .deflate => |*deflate| deflate.deinit(), - .gzip => |*gzip| gzip.deinit(), + .deflate => {}, + .gzip => {}, .zstd => |*zstd| zstd.deinit(), } @@ -605,10 +605,10 @@ pub const Response = struct { .identity => res.request.compression = .none, .compress, .@"x-compress" => return error.CompressionNotSupported, .deflate => res.request.compression = .{ - .deflate = std.compress.zlib.decompressStream(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed, + .deflate = std.compress.zlib.decompressor(res.transferReader()), }, .gzip, .@"x-gzip" => res.request.compression = .{ - .gzip = std.compress.gzip.decompress(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed, + .gzip = std.compress.gzip.decompressor(res.transferReader()), }, .zstd => res.request.compression = .{ .zstd = std.compress.zstd.decompressStream(res.allocator, res.transferReader()), diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index fb9d7c823c0c..ed3c6b099fd8 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1099,7 +1099,12 @@ fn unpackResource( switch (file_type) { .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()), - .@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip), + .@"tar.gz" => { + const reader = resource.reader(); + var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); + var dcp = std.compress.gzip.decompressor(br.reader()); + try unpackTarball(f, tmp_directory.handle, dcp.reader()); + }, .@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz), .@"tar.zst" => try unpackTarballCompressed(f, tmp_directory.handle, resource, ZstdWrapper), .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig index df5332d41d4b..abbb03194825 100644 --- a/src/Package/Fetch/git.zig +++ b/src/Package/Fetch/git.zig @@ -1115,8 +1115,7 @@ fn indexPackFirstPass( const entry_header = try EntryHeader.read(entry_crc32_reader.reader()); switch (entry_header) { inline .commit, .tree, .blob, .tag => |object, tag| { - var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader()); - defer entry_decompress_stream.deinit(); + var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{})); const entry_writer = entry_hashed_writer.writer(); @@ -1135,8 +1134,7 @@ fn indexPackFirstPass( }); }, inline .ofs_delta, .ref_delta => |delta| { - var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader()); - defer entry_decompress_stream.deinit(); + var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); @@ -1257,8 +1255,7 @@ fn resolveDeltaChain( fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 { const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; var buffered_reader = std.io.bufferedReader(reader); - var decompress_stream = try std.compress.zlib.decompressStream(allocator, buffered_reader.reader()); - defer decompress_stream.deinit(); + var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader()); const data = try allocator.alloc(u8, alloc_size); errdefer allocator.free(data); try decompress_stream.reader().readNoEof(data); diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index e547b797ac6e..33b86231835f 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -7276,7 +7276,20 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn else => |vector_index| @intFromEnum(vector_index) * val_bit_size, }; if (ptr_bit_off % 8 == 0) { - try self.load(dst_mcv, ptr_ty, ptr_mcv.offset(@intCast(@divExact(ptr_bit_off, 8)))); + { + const mat_ptr_mcv: MCValue = switch (ptr_mcv) { + .immediate, .register, .register_offset, .lea_frame => ptr_mcv, + else => .{ .register = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }, + }; + const mat_ptr_lock = switch (mat_ptr_mcv) { + .register => |mat_ptr_reg| self.register_manager.lockReg(mat_ptr_reg), + else => null, + }; + defer if (mat_ptr_lock) |lock| self.register_manager.unlockReg(lock); + + try self.load(dst_mcv, ptr_ty, mat_ptr_mcv.offset(@intCast(@divExact(ptr_bit_off, 8)))); + } + if (val_abi_size * 8 > val_bit_size) { if (dst_mcv.isRegister()) { try self.truncateRegister(val_ty, dst_mcv.getReg().?); diff --git a/src/link/Elf/Object.zig b/src/link/Elf/Object.zig index 395e6680a121..b29de3fc5935 100644 --- a/src/link/Elf/Object.zig +++ b/src/link/Elf/Object.zig @@ -902,9 +902,7 @@ pub fn codeDecompressAlloc(self: Object, elf_file: *Elf, atom_index: Atom.Index) switch (chdr.ch_type) { .ZLIB => { var stream = std.io.fixedBufferStream(data[@sizeOf(elf.Elf64_Chdr)..]); - var zlib_stream = std.compress.zlib.decompressStream(gpa, stream.reader()) catch - return error.InputOutput; - defer zlib_stream.deinit(); + var zlib_stream = std.compress.zlib.decompressor(stream.reader()); const size = std.math.cast(usize, chdr.ch_size) orelse return error.Overflow; const decomp = try gpa.alloc(u8, size); const nread = zlib_stream.reader().readAll(decomp) catch return error.InputOutput; diff --git a/src/objcopy.zig b/src/objcopy.zig index 8ea0eaac5956..81638d4af05f 100644 --- a/src/objcopy.zig +++ b/src/objcopy.zig @@ -1298,8 +1298,7 @@ const ElfFileHelper = struct { try compressed_stream.writer().writeAll(prefix); { - var compressor = try std.compress.zlib.compressStream(allocator, compressed_stream.writer(), .{}); - defer compressor.deinit(); + var compressor = try std.compress.zlib.compressor(compressed_stream.writer(), .{}); var buf: [8000]u8 = undefined; while (true) {