Working with flate.Decompress is maybe buggy

[mrjbq7]

Zig Version

0.15.0-dev.1405+70c6a9fba

Steps to Reproduce and Observed Behavior

With this repro.zig:

const std = @import("std");
const flate = std.compress.flate;

pub fn main() anyerror!void {
    const input_file = try std.fs.cwd().openFile("repro.zig.gz", .{});
    var input_buffer: [4096]u8 = undefined;
    var input_reader = input_file.reader(&input_buffer);

    var buffer: [flate.max_window_len]u8 = undefined;
    var decompress: flate.Decompress = .init(&input_reader.interface, .gzip, &buffer);

    const length = try decompress.reader.takeInt(u16, .big);
    std.debug.print("length: {d}\n", .{length});
}

Run it on itself compressed:

$ gzip -k repro.zig

$ zig run repro.zig
error: EndOfStream
zig/std/compress/flate/Decompress.zig:473:17: 0x1022aa5e3 in streamInner (repro)
        .end => return error.EndOfStream,
                ^
zig/std/compress/flate/Decompress.zig:251:17: 0x1022a9a0f in streamFallible (repro)
                return error.EndOfStream;
                ^
zig/std/compress/flate/Decompress.zig:173:21: 0x1022aa31f in streamIndirectInner (repro)
        else => |e| return e,
                    ^
zig/std/compress/flate/Decompress.zig:244:5: 0x1022b99a7 in streamIndirect (repro)
    return streamIndirectInner(d);
    ^
zig/std/Io/Reader.zig:1070:56: 0x1022a4e83 in fillUnbuffered (repro)
            error.ReadFailed, error.EndOfStream => |e| return e,
                                                       ^
zig/std/Io/Reader.zig:1049:5: 0x1022a3e6f in fill (repro)
    return fillUnbuffered(r, n);
    ^
zig/std/Io/Reader.zig:488:5: 0x1022a2307 in peek (repro)
    try r.fill(n);
    ^
zig/std/Io/Reader.zig:536:20: 0x1022a1d63 in take (repro)
    const result = try r.peek(n);
                   ^
zig/std/Io/Reader.zig:553:13: 0x1022a1867 in takeArray__anon_19311 (repro)
    return (try r.take(n))[0..n];
            ^
repro.zig:12:20: 0x1022a14d3 in main (repro)
    const length = try decompress.reader.takeInt(u16, .big);
                   ^

Note, if you instead make an 8mb file and compress it, and run this repro it gets a different error:

$ mkfile -n 8m foo

$ gzip foo

$ mv foo.gz repro.zig.gz

$ zig run repro.zig
error: ReadFailed
zig/std/compress/flate/Decompress.zig:480:27: 0x102d49957 in writeMatch (repro)
    if (w.end < distance) return error.InvalidMatch;
                          ^
zig/std/compress/flate/Decompress.zig:444:17: 0x102d40b07 in streamInner (repro)
                try writeMatch(w, length, distance);
                ^
zig/std/compress/flate/Decompress.zig:262:13: 0x102d3d9cb in streamFallible (repro)
            return error.ReadFailed;
            ^
zig/std/compress/flate/Decompress.zig:173:21: 0x102d3e31f in streamIndirectInner (repro)
        else => |e| return e,
                    ^
zig/std/compress/flate/Decompress.zig:244:5: 0x102d4d9a7 in streamIndirect (repro)
    return streamIndirectInner(d);
    ^
zig/std/Io/Reader.zig:1070:56: 0x102d38e83 in fillUnbuffered (repro)
            error.ReadFailed, error.EndOfStream => |e| return e,
                                                       ^
zig/std/Io/Reader.zig:1049:5: 0x102d37e6f in fill (repro)
    return fillUnbuffered(r, n);
    ^
zig/std/Io/Reader.zig:488:5: 0x102d36307 in peek (repro)
    try r.fill(n);
    ^
zig/std/Io/Reader.zig:536:20: 0x102d35d63 in take (repro)
    const result = try r.peek(n);
                   ^
zig/std/Io/Reader.zig:553:13: 0x102d35867 in takeArray__anon_19311 (repro)
    return (try r.take(n))[0..n];
            ^
repro.zig:12:20: 0x102d354d3 in main (repro)
    const length = try decompress.reader.takeInt(u16, .big);
                   ^

Expected Behavior

It should work?

[mrjbq7]

This file (https://downloads.factorcode.org/misc/panic.z) causes a crazy error:

const std = @import("std");
const flate = std.compress.flate;

pub fn main() !void {
    const input_file = try std.fs.cwd().openFile("panic.z", .{});
    var input_buffer: [4096]u8 = undefined;
    var input_reader = input_file.reader(&input_buffer);

    var buffer: [flate.max_window_len]u8 = undefined;
    var decompress: flate.Decompress = .init(&input_reader.interface, .zlib, &buffer);

    // should decompress to 4160 bytes
    var out_buffer: [4160]u8 = undefined;
    var out_writer: std.Io.Writer = .fixed(&out_buffer);

    const written = try decompress.reader.streamRemaining(&out_writer);
    std.debug.print("decompressed {} bytes\n", .{written});
}

causes this

$ zig run min.zig
thread 15118237 panic: integer does not fit in destination type
zig/std/fs/File.zig:1481:13: 0x1027e9423 in readPositional (min)
            return error.EndOfStream;
            ^
zig/std/fs/File.zig:1294:27: 0x1027e6bf7 in stream (min)
                const n = try readPositional(r, dest);
                          ^
zig/std/Io/Reader.zig:1070:56: 0x10280176b in fillUnbuffered (min)
            error.ReadFailed, error.EndOfStream => |e| return e,
                                                       ^
zig/std/Io/Reader.zig:1049:5: 0x102801417 in fill (min)
    return fillUnbuffered(r, n);
    ^
zig/std/Io/Reader.zig:488:5: 0x102800aa3 in peek (min)
    try r.fill(n);
    ^
zig/std/Io/Reader.zig:536:20: 0x1027fff3b in take (min)
    const result = try r.peek(n);
                   ^
zig/std/Io/Reader.zig:553:13: 0x1027f522b in takeArray__anon_25762 (min)
    return (try r.take(n))[0..n];
            ^
zig/std/compress/flate/Decompress.zig:606:24: 0x1027f9103 in tossBitsEnding (min)
    d.remaining_bits = @intCast(@as(usize, n) * 8 -| @as(usize, needed_bits));
                       ^
zig/std/compress/flate/Decompress.zig:586:55: 0x1027f93df in tossBits (min)
            error.EndOfStream => return tossBitsEnding(d, n),
                                                      ^
zig/std/compress/flate/Decompress.zig:613:17: 0x1027fcbc7 in takeBitsRuntime (min)
    try tossBits(d, n);
                ^
zig/std/compress/flate/Decompress.zig:193:43: 0x1027fcfdf in decodeDistance (min)
        md.base + try self.takeBitsRuntime(md.extra_bits);
                                          ^
zig/std/compress/flate/Decompress.zig:443:54: 0x1027f42eb in streamInner (min)
                const distance = try d.decodeDistance(dsm.symbol);
                                                     ^
zig/std/compress/flate/Decompress.zig:248:23: 0x1027f1157 in streamFallible (min)
    return streamInner(d, w, limit) catch |err| switch (err) {
                      ^
zig/std/compress/flate/Decompress.zig:171:23: 0x1027f1afb in streamIndirectInner (min)
    _ = streamFallible(d, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) {
                      ^
zig/std/compress/flate/Decompress.zig:244:31: 0x1028019eb in streamIndirect (min)
    return streamIndirectInner(d);
                              ^
zig/std/Io/Reader.zig:179:34: 0x102762a7b in stream (min)
    const n = try r.vtable.stream(r, w, limit);
                                 ^
zig/std/Io/Reader.zig:258:27: 0x1027e14c7 in streamRemaining (min)
        offset += r.stream(w, .unlimited) catch |err| switch (err) {
                          ^
min.zig:25:58: 0x1027e10df in main (min)
    const written = try decompress.reader.streamRemaining(&out_writer);
                                                         ^
zig/std/start.zig:627:37: 0x1027e18ff in main (min)
            const result = root.main() catch |err| {
                                    ^
???:?:?: 0x19ac72b97 in ??? (???)
???:?:?: 0x0 in ??? (???)
[1]    43635 abort      zig run min.zig

[mrjbq7]

This file (https://downloads.factorcode.org/misc/fail_read.z) causes a different error

const std = @import("std");
const flate = std.compress.flate;

pub fn main() !void {
    const input_file = try std.fs.cwd().openFile("fail_read.z", .{});
    var input_buffer: [4096]u8 = undefined;
    var input_reader = input_file.reader(&input_buffer);

    var buffer: [flate.max_window_len]u8 = undefined;
    var decompress: flate.Decompress = .init(&input_reader.interface, .zlib, &buffer);

    // should decompress to 4160 bytes
    var out_buffer: [4160]u8 = undefined;
    var out_writer: std.Io.Writer = .fixed(&out_buffer);

    const written = try decompress.reader.streamRemaining(&out_writer);
    std.debug.print("decompressed {} bytes\n", .{written});
}

has this error

error: ReadFailed
zig/std/fs/File.zig:1481:13: 0x100174093 in readPositional (repro)
            return error.EndOfStream;
            ^
zig/std/fs/File.zig:1294:27: 0x100172513 in stream (repro)
                const n = try readPositional(r, dest);
                          ^
zig/std/Io/Reader.zig:1070:56: 0x100189937 in fillUnbuffered (repro)
            error.ReadFailed, error.EndOfStream => |e| return e,
                                                       ^
zig/std/Io/Reader.zig:1049:5: 0x1001895e3 in fill (repro)
    return fillUnbuffered(r, n);
    ^
zig/std/Io/Reader.zig:488:5: 0x100188c6f in peek (repro)
    try r.fill(n);
    ^
zig/std/Io/Reader.zig:536:20: 0x100188107 in take (repro)
    const result = try r.peek(n);
                   ^
zig/std/Io/Reader.zig:553:13: 0x100187de3 in takeArray__anon_22666 (repro)
    return (try r.take(n))[0..n];
            ^
zig/std/compress/flate/Decompress.zig:463:34: 0x10017c937 in streamInner (repro)
                        .adler = try in.takeInt(u32, .little),
                                 ^
zig/std/compress/flate/Decompress.zig:254:17: 0x1001794ab in streamFallible (repro)
                return error.ReadFailed;
                ^
zig/std/compress/flate/Decompress.zig:173:21: 0x100179d63 in streamIndirectInner (repro)
        else => |e| return e,
                    ^
zig/std/compress/flate/Decompress.zig:244:5: 0x100189bff in streamIndirect (repro)
    return streamIndirectInner(d);
    ^
zig/std/Io/Reader.zig:179:15: 0x1000f2b9b in stream (repro)
    const n = try r.vtable.stream(r, w, limit);
              ^
zig/std/Io/Reader.zig:260:25: 0x10017144f in streamRemaining (repro)
            else => |e| return e,
                        ^
repro.zig:16:21: 0x1001710c7 in main (repro)
    const written = try decompress.reader.streamRemaining(&out_writer);
                    ^

[mrjbq7]

both of those errors are fixed if you add two extra bytes:

const std = @import("std");
const flate = std.compress.flate;

pub fn main() !void {
    var debug_allocator: std.heap.DebugAllocator(.{}) = .init;
    defer _ = debug_allocator.deinit();
    const gpa = debug_allocator.allocator();

    const input_file = try std.fs.cwd().openFile("panic.z", .{});
    var input_buffer: [4096]u8 = undefined;
    var input_reader = input_file.reader(&input_buffer);

    const file_contents = try input_reader.interface.allocRemaining(gpa, .unlimited);
    defer gpa.free(file_contents);

    // allocate another identical buffer with 2 extra bytes, set to 0
    const file_contents_plus_two = try gpa.alloc(u8, file_contents.len + 2);
    defer gpa.free(file_contents_plus_two);
    @memcpy(file_contents_plus_two[0..file_contents.len], file_contents);
    file_contents_plus_two[file_contents.len] = 0;
    file_contents_plus_two[file_contents.len + 1] = 0;

    var plus_two_reader: std.Io.Reader = .fixed(file_contents_plus_two);

    var buffer: [flate.max_window_len]u8 = undefined;
    var decompress: flate.Decompress = .init(&plus_two_reader, .zlib, &buffer);

    // should decompress to 4160 bytes
    var out_buffer: [4160]u8 = undefined;
    var out_writer: std.Io.Writer = .fixed(&out_buffer);

    const written = try decompress.reader.streamRemaining(&out_writer);
    std.debug.print("decompressed {} bytes\n", .{written});
}

[mrjbq7]

I can confirm those errors are still present after applying #24677.

[mrjbq7]

Actually, the OP issues at the top are fixed by #24677, but the two comments below (panic.z and fail_read.z) are not fixed.