From b019c9c010e4e047ddf4c6bc771d11dd46bcaac0 Mon Sep 17 00:00:00 2001 From: Travis Staloch <1562827+travisstaloch@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:24:32 -0700 Subject: [PATCH 1/3] add Io.Writer.AllocatingAligned to support aligned writing this commit makes it easier to write to an aligned array list. previously you needed to use an adapter such as `aligned_list.writer(allocator).adaptToNewApi(&.{});`. `Io.Writer.Allocating` becomes an `AllocatingAligned(.fromByteUnits(1))` similar to how `std.ArrayList(u8)` is a `std.array_list.Aligned(u8, null)`. * I've turned off zig fmt to make this diff easier to read. Next commit will be correctly formatted. --- lib/std/Io/Writer.zig | 61 ++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index 384faff62ee5..cb303d107a0f 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -2528,6 +2528,8 @@ pub fn Hashing(comptime Hasher: type) type { }; } +pub const Allocating = AllocatingAligned(.fromByteUnits(1)); + /// Maintains `Writer` state such that it writes to the unused capacity of an /// array list, filling it up completely before making a call through the /// vtable, causing a resize. Consequently, the same, optimized, non-generic @@ -2535,11 +2537,15 @@ pub fn Hashing(comptime Hasher: type) type { /// the hot paths when using this API. /// /// When using this API, it is not necessary to call `flush`. -pub const Allocating = struct { +// zig fmt: off +pub fn AllocatingAligned(comptime alignment: std.mem.Alignment) type { +return struct { allocator: Allocator, writer: Writer, - pub fn init(allocator: Allocator) Allocating { + const Self = @This(); + + pub fn init(allocator: Allocator) Self { return .{ .allocator = allocator, .writer = .{ @@ -2549,7 +2555,7 @@ pub const Allocating = struct { }; } - pub fn initCapacity(allocator: Allocator, capacity: usize) error{OutOfMemory}!Allocating { + pub fn initCapacity(allocator: Allocator, capacity: usize) error{OutOfMemory}!Self { return .{ .allocator = allocator, .writer = .{ @@ -2559,7 +2565,7 @@ pub const Allocating = struct { }; } - pub fn initOwnedSlice(allocator: Allocator, slice: []u8) Allocating { + pub fn initOwnedSlice(allocator: Allocator, slice: []align(alignment.toByteUnits()) u8) Self { return .{ .allocator = allocator, .writer = .{ @@ -2570,7 +2576,7 @@ pub const Allocating = struct { } /// Replaces `array_list` with empty, taking ownership of the memory. - pub fn fromArrayList(allocator: Allocator, array_list: *ArrayList(u8)) Allocating { + pub fn fromArrayList(allocator: Allocator, array_list: *std.array_list.Aligned(u8, alignment)) Self { defer array_list.* = .empty; return .{ .allocator = allocator, @@ -2583,23 +2589,23 @@ pub const Allocating = struct { } const vtable: VTable = .{ - .drain = Allocating.drain, - .sendFile = Allocating.sendFile, + .drain = Self.drain, + .sendFile = Self.sendFile, .flush = noopFlush, .rebase = growingRebase, }; - pub fn deinit(a: *Allocating) void { + pub fn deinit(a: *Self) void { a.allocator.free(a.writer.buffer); a.* = undefined; } /// Returns an array list that takes ownership of the allocated memory. /// Resets the `Allocating` to an empty state. - pub fn toArrayList(a: *Allocating) ArrayList(u8) { + pub fn toArrayList(a: *Self) std.array_list.Aligned(u8, alignment) { const w = &a.writer; - const result: ArrayList(u8) = .{ - .items = w.buffer[0..w.end], + const result: std.array_list.Aligned(u8, alignment) = .{ + .items = @alignCast(w.buffer[0..w.end]), .capacity = w.buffer.len, }; w.buffer = &.{}; @@ -2607,45 +2613,45 @@ pub const Allocating = struct { return result; } - pub fn ensureUnusedCapacity(a: *Allocating, additional_count: usize) Allocator.Error!void { + pub fn ensureUnusedCapacity(a: *Self, additional_count: usize) Allocator.Error!void { var list = a.toArrayList(); defer a.setArrayList(list); return list.ensureUnusedCapacity(a.allocator, additional_count); } - pub fn ensureTotalCapacity(a: *Allocating, new_capacity: usize) Allocator.Error!void { + pub fn ensureTotalCapacity(a: *Self, new_capacity: usize) Allocator.Error!void { var list = a.toArrayList(); defer a.setArrayList(list); return list.ensureTotalCapacity(a.allocator, new_capacity); } - pub fn toOwnedSlice(a: *Allocating) error{OutOfMemory}![]u8 { + pub fn toOwnedSlice(a: *Self) error{OutOfMemory}![]align(alignment.toByteUnits()) u8 { var list = a.toArrayList(); defer a.setArrayList(list); return list.toOwnedSlice(a.allocator); } - pub fn toOwnedSliceSentinel(a: *Allocating, comptime sentinel: u8) error{OutOfMemory}![:sentinel]u8 { + pub fn toOwnedSliceSentinel(a: *Self, comptime sentinel: u8) error{OutOfMemory}![:sentinel]align(alignment.toByteUnits()) u8 { const gpa = a.allocator; var list = @This().toArrayList(a); defer a.setArrayList(list); - return list.toOwnedSliceSentinel(gpa, sentinel); + return @alignCast(try list.toOwnedSliceSentinel(gpa, sentinel)); } - pub fn written(a: *Allocating) []u8 { - return a.writer.buffered(); + pub fn written(a: *Self) []align(alignment.toByteUnits()) u8 { + return @alignCast(a.writer.buffered()); } - pub fn shrinkRetainingCapacity(a: *Allocating, new_len: usize) void { + pub fn shrinkRetainingCapacity(a: *Self, new_len: usize) void { a.writer.end = new_len; } - pub fn clearRetainingCapacity(a: *Allocating) void { + pub fn clearRetainingCapacity(a: *Self) void { a.shrinkRetainingCapacity(0); } fn drain(w: *Writer, data: []const []const u8, splat: usize) Error!usize { - const a: *Allocating = @fieldParentPtr("writer", w); + const a: *Self = @fieldParentPtr("writer", w); const gpa = a.allocator; const pattern = data[data.len - 1]; const splat_len = pattern.len * splat; @@ -2670,7 +2676,7 @@ pub const Allocating = struct { fn sendFile(w: *Writer, file_reader: *File.Reader, limit: Limit) FileError!usize { if (File.Handle == void) return error.Unimplemented; if (limit == .nothing) return 0; - const a: *Allocating = @fieldParentPtr("writer", w); + const a: *Self = @fieldParentPtr("writer", w); const gpa = a.allocator; var list = a.toArrayList(); defer setArrayList(a, list); @@ -2685,7 +2691,7 @@ pub const Allocating = struct { } fn growingRebase(w: *Writer, preserve: usize, minimum_len: usize) Error!void { - const a: *Allocating = @fieldParentPtr("writer", w); + const a: *Self = @fieldParentPtr("writer", w); const gpa = a.allocator; var list = a.toArrayList(); defer setArrayList(a, list); @@ -2694,13 +2700,13 @@ pub const Allocating = struct { list.ensureUnusedCapacity(gpa, minimum_len) catch return error.WriteFailed; } - fn setArrayList(a: *Allocating, list: ArrayList(u8)) void { + fn setArrayList(a: *Self, list: std.array_list.Aligned(u8, alignment)) void { a.writer.buffer = list.allocatedSlice(); a.writer.end = list.items.len; } - test Allocating { - var a: Allocating = .init(testing.allocator); + test Self { + var a: Self = .init(testing.allocator); defer a.deinit(); const w = &a.writer; @@ -2711,7 +2717,8 @@ pub const Allocating = struct { try testing.expectEqualSlices(u8, "x: 42\ny: 1234\n", a.written()); } }; - +} +// zig fmt: on test "discarding sendFile" { var tmp_dir = testing.tmpDir(.{}); defer tmp_dir.cleanup(); From 35518f8ba2b8ec3f5f5395b1f3c87800d93b057a Mon Sep 17 00:00:00 2001 From: Travis Staloch <1562827+travisstaloch@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:25:40 -0700 Subject: [PATCH 2/3] removed zig fmt: off/on. whitespace only changes. --- lib/std/Io/Writer.zig | 313 +++++++++++++++++++++--------------------- 1 file changed, 156 insertions(+), 157 deletions(-) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index cb303d107a0f..0953fc619926 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -2537,188 +2537,187 @@ pub const Allocating = AllocatingAligned(.fromByteUnits(1)); /// the hot paths when using this API. /// /// When using this API, it is not necessary to call `flush`. -// zig fmt: off pub fn AllocatingAligned(comptime alignment: std.mem.Alignment) type { -return struct { - allocator: Allocator, - writer: Writer, - - const Self = @This(); + return struct { + allocator: Allocator, + writer: Writer, - pub fn init(allocator: Allocator) Self { - return .{ - .allocator = allocator, - .writer = .{ - .buffer = &.{}, - .vtable = &vtable, - }, - }; - } + const Self = @This(); - pub fn initCapacity(allocator: Allocator, capacity: usize) error{OutOfMemory}!Self { - return .{ - .allocator = allocator, - .writer = .{ - .buffer = try allocator.alloc(u8, capacity), - .vtable = &vtable, - }, - }; - } - - pub fn initOwnedSlice(allocator: Allocator, slice: []align(alignment.toByteUnits()) u8) Self { - return .{ - .allocator = allocator, - .writer = .{ - .buffer = slice, - .vtable = &vtable, - }, - }; - } + pub fn init(allocator: Allocator) Self { + return .{ + .allocator = allocator, + .writer = .{ + .buffer = &.{}, + .vtable = &vtable, + }, + }; + } - /// Replaces `array_list` with empty, taking ownership of the memory. - pub fn fromArrayList(allocator: Allocator, array_list: *std.array_list.Aligned(u8, alignment)) Self { - defer array_list.* = .empty; - return .{ - .allocator = allocator, - .writer = .{ - .vtable = &vtable, - .buffer = array_list.allocatedSlice(), - .end = array_list.items.len, - }, - }; - } + pub fn initCapacity(allocator: Allocator, capacity: usize) error{OutOfMemory}!Self { + return .{ + .allocator = allocator, + .writer = .{ + .buffer = try allocator.alloc(u8, capacity), + .vtable = &vtable, + }, + }; + } - const vtable: VTable = .{ - .drain = Self.drain, - .sendFile = Self.sendFile, - .flush = noopFlush, - .rebase = growingRebase, - }; + pub fn initOwnedSlice(allocator: Allocator, slice: []align(alignment.toByteUnits()) u8) Self { + return .{ + .allocator = allocator, + .writer = .{ + .buffer = slice, + .vtable = &vtable, + }, + }; + } - pub fn deinit(a: *Self) void { - a.allocator.free(a.writer.buffer); - a.* = undefined; - } + /// Replaces `array_list` with empty, taking ownership of the memory. + pub fn fromArrayList(allocator: Allocator, array_list: *std.array_list.Aligned(u8, alignment)) Self { + defer array_list.* = .empty; + return .{ + .allocator = allocator, + .writer = .{ + .vtable = &vtable, + .buffer = array_list.allocatedSlice(), + .end = array_list.items.len, + }, + }; + } - /// Returns an array list that takes ownership of the allocated memory. - /// Resets the `Allocating` to an empty state. - pub fn toArrayList(a: *Self) std.array_list.Aligned(u8, alignment) { - const w = &a.writer; - const result: std.array_list.Aligned(u8, alignment) = .{ - .items = @alignCast(w.buffer[0..w.end]), - .capacity = w.buffer.len, + const vtable: VTable = .{ + .drain = Self.drain, + .sendFile = Self.sendFile, + .flush = noopFlush, + .rebase = growingRebase, }; - w.buffer = &.{}; - w.end = 0; - return result; - } - pub fn ensureUnusedCapacity(a: *Self, additional_count: usize) Allocator.Error!void { - var list = a.toArrayList(); - defer a.setArrayList(list); - return list.ensureUnusedCapacity(a.allocator, additional_count); - } + pub fn deinit(a: *Self) void { + a.allocator.free(a.writer.buffer); + a.* = undefined; + } - pub fn ensureTotalCapacity(a: *Self, new_capacity: usize) Allocator.Error!void { - var list = a.toArrayList(); - defer a.setArrayList(list); - return list.ensureTotalCapacity(a.allocator, new_capacity); - } + /// Returns an array list that takes ownership of the allocated memory. + /// Resets the `Allocating` to an empty state. + pub fn toArrayList(a: *Self) std.array_list.Aligned(u8, alignment) { + const w = &a.writer; + const result: std.array_list.Aligned(u8, alignment) = .{ + .items = @alignCast(w.buffer[0..w.end]), + .capacity = w.buffer.len, + }; + w.buffer = &.{}; + w.end = 0; + return result; + } - pub fn toOwnedSlice(a: *Self) error{OutOfMemory}![]align(alignment.toByteUnits()) u8 { - var list = a.toArrayList(); - defer a.setArrayList(list); - return list.toOwnedSlice(a.allocator); - } + pub fn ensureUnusedCapacity(a: *Self, additional_count: usize) Allocator.Error!void { + var list = a.toArrayList(); + defer a.setArrayList(list); + return list.ensureUnusedCapacity(a.allocator, additional_count); + } - pub fn toOwnedSliceSentinel(a: *Self, comptime sentinel: u8) error{OutOfMemory}![:sentinel]align(alignment.toByteUnits()) u8 { - const gpa = a.allocator; - var list = @This().toArrayList(a); - defer a.setArrayList(list); - return @alignCast(try list.toOwnedSliceSentinel(gpa, sentinel)); - } + pub fn ensureTotalCapacity(a: *Self, new_capacity: usize) Allocator.Error!void { + var list = a.toArrayList(); + defer a.setArrayList(list); + return list.ensureTotalCapacity(a.allocator, new_capacity); + } - pub fn written(a: *Self) []align(alignment.toByteUnits()) u8 { - return @alignCast(a.writer.buffered()); - } + pub fn toOwnedSlice(a: *Self) error{OutOfMemory}![]align(alignment.toByteUnits()) u8 { + var list = a.toArrayList(); + defer a.setArrayList(list); + return list.toOwnedSlice(a.allocator); + } - pub fn shrinkRetainingCapacity(a: *Self, new_len: usize) void { - a.writer.end = new_len; - } + pub fn toOwnedSliceSentinel(a: *Self, comptime sentinel: u8) error{OutOfMemory}![:sentinel]align(alignment.toByteUnits()) u8 { + const gpa = a.allocator; + var list = @This().toArrayList(a); + defer a.setArrayList(list); + return @alignCast(try list.toOwnedSliceSentinel(gpa, sentinel)); + } - pub fn clearRetainingCapacity(a: *Self) void { - a.shrinkRetainingCapacity(0); - } + pub fn written(a: *Self) []align(alignment.toByteUnits()) u8 { + return @alignCast(a.writer.buffered()); + } - fn drain(w: *Writer, data: []const []const u8, splat: usize) Error!usize { - const a: *Self = @fieldParentPtr("writer", w); - const gpa = a.allocator; - const pattern = data[data.len - 1]; - const splat_len = pattern.len * splat; - var list = a.toArrayList(); - defer setArrayList(a, list); - const start_len = list.items.len; - assert(data.len != 0); - for (data) |bytes| { - list.ensureUnusedCapacity(gpa, bytes.len + splat_len + 1) catch return error.WriteFailed; - list.appendSliceAssumeCapacity(bytes); + pub fn shrinkRetainingCapacity(a: *Self, new_len: usize) void { + a.writer.end = new_len; } - if (splat == 0) { - list.items.len -= pattern.len; - } else switch (pattern.len) { - 0 => {}, - 1 => list.appendNTimesAssumeCapacity(pattern[0], splat - 1), - else => for (0..splat - 1) |_| list.appendSliceAssumeCapacity(pattern), + + pub fn clearRetainingCapacity(a: *Self) void { + a.shrinkRetainingCapacity(0); } - return list.items.len - start_len; - } - fn sendFile(w: *Writer, file_reader: *File.Reader, limit: Limit) FileError!usize { - if (File.Handle == void) return error.Unimplemented; - if (limit == .nothing) return 0; - const a: *Self = @fieldParentPtr("writer", w); - const gpa = a.allocator; - var list = a.toArrayList(); - defer setArrayList(a, list); - const pos = file_reader.logicalPos(); - const additional = if (file_reader.getSize()) |size| size - pos else |_| std.atomic.cache_line; - if (additional == 0) return error.EndOfStream; - list.ensureUnusedCapacity(gpa, limit.minInt64(additional)) catch return error.WriteFailed; - const dest = limit.slice(list.unusedCapacitySlice()); - const n = try file_reader.read(dest); - list.items.len += n; - return n; - } + fn drain(w: *Writer, data: []const []const u8, splat: usize) Error!usize { + const a: *Self = @fieldParentPtr("writer", w); + const gpa = a.allocator; + const pattern = data[data.len - 1]; + const splat_len = pattern.len * splat; + var list = a.toArrayList(); + defer setArrayList(a, list); + const start_len = list.items.len; + assert(data.len != 0); + for (data) |bytes| { + list.ensureUnusedCapacity(gpa, bytes.len + splat_len + 1) catch return error.WriteFailed; + list.appendSliceAssumeCapacity(bytes); + } + if (splat == 0) { + list.items.len -= pattern.len; + } else switch (pattern.len) { + 0 => {}, + 1 => list.appendNTimesAssumeCapacity(pattern[0], splat - 1), + else => for (0..splat - 1) |_| list.appendSliceAssumeCapacity(pattern), + } + return list.items.len - start_len; + } - fn growingRebase(w: *Writer, preserve: usize, minimum_len: usize) Error!void { - const a: *Self = @fieldParentPtr("writer", w); - const gpa = a.allocator; - var list = a.toArrayList(); - defer setArrayList(a, list); - const total = std.math.add(usize, preserve, minimum_len) catch return error.WriteFailed; - list.ensureTotalCapacity(gpa, total) catch return error.WriteFailed; - list.ensureUnusedCapacity(gpa, minimum_len) catch return error.WriteFailed; - } + fn sendFile(w: *Writer, file_reader: *File.Reader, limit: Limit) FileError!usize { + if (File.Handle == void) return error.Unimplemented; + if (limit == .nothing) return 0; + const a: *Self = @fieldParentPtr("writer", w); + const gpa = a.allocator; + var list = a.toArrayList(); + defer setArrayList(a, list); + const pos = file_reader.logicalPos(); + const additional = if (file_reader.getSize()) |size| size - pos else |_| std.atomic.cache_line; + if (additional == 0) return error.EndOfStream; + list.ensureUnusedCapacity(gpa, limit.minInt64(additional)) catch return error.WriteFailed; + const dest = limit.slice(list.unusedCapacitySlice()); + const n = try file_reader.read(dest); + list.items.len += n; + return n; + } - fn setArrayList(a: *Self, list: std.array_list.Aligned(u8, alignment)) void { - a.writer.buffer = list.allocatedSlice(); - a.writer.end = list.items.len; - } + fn growingRebase(w: *Writer, preserve: usize, minimum_len: usize) Error!void { + const a: *Self = @fieldParentPtr("writer", w); + const gpa = a.allocator; + var list = a.toArrayList(); + defer setArrayList(a, list); + const total = std.math.add(usize, preserve, minimum_len) catch return error.WriteFailed; + list.ensureTotalCapacity(gpa, total) catch return error.WriteFailed; + list.ensureUnusedCapacity(gpa, minimum_len) catch return error.WriteFailed; + } - test Self { - var a: Self = .init(testing.allocator); - defer a.deinit(); - const w = &a.writer; + fn setArrayList(a: *Self, list: std.array_list.Aligned(u8, alignment)) void { + a.writer.buffer = list.allocatedSlice(); + a.writer.end = list.items.len; + } - const x: i32 = 42; - const y: i32 = 1234; - try w.print("x: {}\ny: {}\n", .{ x, y }); + test Self { + var a: Self = .init(testing.allocator); + defer a.deinit(); + const w = &a.writer; - try testing.expectEqualSlices(u8, "x: 42\ny: 1234\n", a.written()); - } -}; + const x: i32 = 42; + const y: i32 = 1234; + try w.print("x: {}\ny: {}\n", .{ x, y }); + + try testing.expectEqualSlices(u8, "x: 42\ny: 1234\n", a.written()); + } + }; } -// zig fmt: on + test "discarding sendFile" { var tmp_dir = testing.tmpDir(.{}); defer tmp_dir.cleanup(); From 9b9ac5d025b2a7e6ee612c2c642b8698d89d1d19 Mon Sep 17 00:00:00 2001 From: Travis Staloch <1562827+travisstaloch@users.noreply.github.com> Date: Fri, 29 Aug 2025 08:56:37 -0700 Subject: [PATCH 3/3] deinit: cast to aligned buffer before freeing added a test to check for compile errors and one to check a few alignments. adding the second test exposed the deinit error i.e. Allocation alignment X does not match free alignment 1. --- lib/std/Io/Writer.zig | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index 0953fc619926..6c0555d31366 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -2595,7 +2595,9 @@ pub fn AllocatingAligned(comptime alignment: std.mem.Alignment) type { }; pub fn deinit(a: *Self) void { - a.allocator.free(a.writer.buffer); + const aligned_buffer: []align(alignment.toByteUnits()) u8 = + @alignCast(a.writer.buffer); + a.allocator.free(aligned_buffer); a.* = undefined; } @@ -2705,19 +2707,41 @@ pub fn AllocatingAligned(comptime alignment: std.mem.Alignment) type { } test Self { - var a: Self = .init(testing.allocator); - defer a.deinit(); - const w = &a.writer; + const gpa = testing.allocator; + { + var a: Self = .init(gpa); + defer a.deinit(); + const w = &a.writer; + + const x: i32 = 42; + const y: i32 = 1234; + try w.print("x: {}\ny: {}\n", .{ x, y }); - const x: i32 = 42; - const y: i32 = 1234; - try w.print("x: {}\ny: {}\n", .{ x, y }); + try testing.expectEqualSlices(u8, "x: 42\ny: 1234\n", a.written()); + } - try testing.expectEqualSlices(u8, "x: 42\ny: 1234\n", a.written()); + { // check that things compile and types are correct + var a: Self = .init(gpa); + defer a.deinit(); + var l: std.array_list.Aligned(u8, alignment) = a.toArrayList(); + _ = Self.fromArrayList(gpa, &l); + a.setArrayList(l); + const s: []align(alignment.toByteUnits()) u8 = try a.toOwnedSlice(); + defer gpa.free(s); + const s2: [:0]align(alignment.toByteUnits()) u8 = try a.toOwnedSliceSentinel(0); + defer gpa.free(s2); + } } }; } +test AllocatingAligned { + _ = AllocatingAligned(.fromByteUnits(4)); + _ = AllocatingAligned(.fromByteUnits(8)); + _ = AllocatingAligned(.fromByteUnits(16)); + _ = AllocatingAligned(.fromByteUnits(32)); +} + test "discarding sendFile" { var tmp_dir = testing.tmpDir(.{}); defer tmp_dir.cleanup();