x86_64: implement error set and enum safety

lib/compiler_rt.zig

@@ -230,6 +230,7 @@ comptime {
     _ = @import("compiler_rt/trunc.zig");
 
     // BigInt. Alphabetically sorted.
+    _ = @import("compiler_rt/divmodei4.zig");
     _ = @import("compiler_rt/udivmodei4.zig");
     _ = @import("compiler_rt/udivmodti4.zig");
 

lib/compiler_rt/divmodei4.zig

@@ -0,0 +1,50 @@
+const std = @import("std");
+const builtin = @import("builtin");
+const common = @import("common.zig");
+const udivmod = @import("udivmodei4.zig").divmod;
+
+comptime {
+    @export(&__divei4, .{ .name = "__divei4", .linkage = common.linkage, .visibility = common.visibility });
+    @export(&__modei4, .{ .name = "__modei4", .linkage = common.linkage, .visibility = common.visibility });
+}
+
+const endian = builtin.cpu.arch.endian();
+
+inline fn limb(x: []u32, i: usize) *u32 {
+    return if (endian == .little) &x[i] else &x[x.len - 1 - i];
+}
+
+inline fn neg(x: []u32) void {
+    var ov: u1 = 1;
+    for (0..x.len) |limb_index| {
+        const l = limb(x, limb_index);
+        l.*, ov = @addWithOverflow(~l.*, ov);
+    }
+}
+
+/// Mutates the arguments!
+fn divmod(q: ?[]u32, r: ?[]u32, u: []u32, v: []u32) !void {
+    const u_sign: i32 = @bitCast(u[u.len - 1]);
+    const v_sign: i32 = @bitCast(v[v.len - 1]);
+    if (u_sign < 0) neg(u);
+    if (v_sign < 0) neg(v);
+    try @call(.always_inline, udivmod, .{ q, r, u, v });
+    if (q) |x| if (u_sign ^ v_sign < 0) neg(x);
+    if (r) |x| if (u_sign < 0) neg(x);
+}
+
+pub fn __divei4(r_q: [*]u32, u_p: [*]u32, v_p: [*]u32, bits: usize) callconv(.C) void {
+    @setRuntimeSafety(builtin.is_test);
+    const u = u_p[0 .. std.math.divCeil(usize, bits, 32) catch unreachable];
+    const v = v_p[0 .. std.math.divCeil(usize, bits, 32) catch unreachable];
+    const q = r_q[0 .. std.math.divCeil(usize, bits, 32) catch unreachable];
+    @call(.always_inline, divmod, .{ q, null, u, v }) catch unreachable;
+}
+
+pub fn __modei4(r_p: [*]u32, u_p: [*]u32, v_p: [*]u32, bits: usize) callconv(.C) void {
+    @setRuntimeSafety(builtin.is_test);
+    const u = u_p[0 .. std.math.divCeil(usize, bits, 32) catch unreachable];
+    const v = v_p[0 .. std.math.divCeil(usize, bits, 32) catch unreachable];
+    const r = r_p[0 .. std.math.divCeil(usize, bits, 32) catch unreachable];
+    @call(.always_inline, divmod, .{ null, r, u, v }) catch unreachable;
+}

lib/compiler_rt/udivmodei4.zig

@@ -27,8 +27,8 @@ inline fn limb_set(x: []u32, i: usize, v: u32) void {
     }
 }
 
-// Uses Knuth's Algorithm D, 4.3.1, p. 272.
-fn divmod(q: ?[]u32, r: ?[]u32, u: []const u32, v: []const u32) !void {
+/// Uses Knuth's Algorithm D, 4.3.1, p. 272.
+pub fn divmod(q: ?[]u32, r: ?[]u32, u: []const u32, v: []const u32) !void {
     if (q) |q_| @memset(q_[0..], 0);
     if (r) |r_| @memset(r_[0..], 0);
 

src/Sema.zig

@@ -8850,7 +8850,7 @@ fn zirEnumFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
             // Use `intCast`, since it'll set up the Sema-emitted safety checks for us!
             const int_val = try sema.intCast(block, src, int_tag_ty, src, operand, src, true, true);
             const result = try block.addBitCast(dest_ty, int_val);
-            if (zcu.backendSupportsFeature(.is_named_enum_value)) {
+            if (!dest_ty.isNonexhaustiveEnum(zcu) and zcu.backendSupportsFeature(.is_named_enum_value)) {
                 const ok = try block.addUnOp(.is_named_enum_value, result);
                 try sema.addSafetyCheck(block, src, ok, .invalid_enum_value);
             }

src/Type.zig

@@ -4190,11 +4190,11 @@ pub const @"c_longlong": Type = .{ .ip_index = .c_longlong_type };
 pub const @"c_ulonglong": Type = .{ .ip_index = .c_ulonglong_type };
 pub const @"c_longdouble": Type = .{ .ip_index = .c_longdouble_type };
 
-pub const slice_const_u8: Type = .{ .ip_index = .slice_const_u8_type };
 pub const manyptr_u8: Type = .{ .ip_index = .manyptr_u8_type };
-pub const single_const_pointer_to_comptime_int: Type = .{
-    .ip_index = .single_const_pointer_to_comptime_int_type,
-};
+pub const manyptr_const_u8: Type = .{ .ip_index = .manyptr_const_u8_type };
+pub const manyptr_const_u8_sentinel_0: Type = .{ .ip_index = .manyptr_const_u8_sentinel_0_type };
+pub const single_const_pointer_to_comptime_int: Type = .{ .ip_index = .single_const_pointer_to_comptime_int_type };
+pub const slice_const_u8: Type = .{ .ip_index = .slice_const_u8_type };
 pub const slice_const_u8_sentinel_0: Type = .{ .ip_index = .slice_const_u8_sentinel_0_type };
 
 pub const vector_16_i8: Type = .{ .ip_index = .vector_16_i8_type };

src/arch/x86_64/Emit.zig

@@ -88,13 +88,32 @@ pub fn emitMir(emit: *Emit) Error!void {
                 lowered_relocs[0].lowered_inst_index == lowered_index) : ({
                 lowered_relocs = lowered_relocs[1..];
             }) switch (lowered_relocs[0].target) {
-                .inst => |target| try relocs.append(emit.lower.allocator, .{
-                    .source = start_offset,
-                    .source_offset = end_offset - 4,
-                    .target = target,
-                    .target_offset = lowered_relocs[0].off,
-                    .length = @intCast(end_offset - start_offset),
-                }),
+                .inst => |target| {
+                    const inst_length: u4 = @intCast(end_offset - start_offset);
+                    const reloc_offset, const reloc_length = reloc_offset_length: {
+                        var reloc_offset = inst_length;
+                        var op_index: usize = lowered_inst.ops.len;
+                        while (true) {
+                            op_index -= 1;
+                            const op = lowered_inst.encoding.data.ops[op_index];
+                            if (op == .none) continue;
+                            const enc_length: u4 = @intCast(
+                                std.math.divCeil(u7, @intCast(op.immBitSize()), 8) catch unreachable,
+                            );
+                            reloc_offset -= enc_length;
+                            if (op_index == lowered_relocs[0].op_index)
+                                break :reloc_offset_length .{ reloc_offset, enc_length };
+                        }
+                    };
+                    try relocs.append(emit.lower.allocator, .{
+                        .inst_offset = start_offset,
+                        .inst_length = inst_length,
+                        .source_offset = reloc_offset,
+                        .source_length = reloc_length,
+                        .target = target,
+                        .target_offset = lowered_relocs[0].off,
+                    });
+                },
                 .table => try table_relocs.append(emit.lower.allocator, .{
                     .source_offset = end_offset - 4,
                     .target_offset = lowered_relocs[0].off,
@@ -409,7 +428,7 @@ pub fn emitMir(emit: *Emit) Error!void {
                                     } } };
                                 },
                                 .pseudo_dbg_local_am => loc: {
-                                    const mem = emit.lower.mem(mir_inst.data.ax.payload);
+                                    const mem = emit.lower.mem(undefined, mir_inst.data.ax.payload);
                                     break :loc .{ mir_inst.data.ax.air_inst, .{ .plus = .{
                                         base: {
                                             loc_buf[0] = switch (mem.base()) {
@@ -466,15 +485,18 @@ pub fn emitMir(emit: *Emit) Error!void {
             }
         }
     }
-    {
-        // TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
-        // This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
-        // possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
-        // until the entire decl is correctly emitted with all JMP/CALL instructions within range.
-        for (relocs.items) |reloc| {
-            const target = code_offset_mapping[reloc.target];
-            const disp = @as(i64, @intCast(target)) - @as(i64, @intCast(reloc.source + reloc.length)) + reloc.target_offset;
-            std.mem.writeInt(i32, emit.code.items[reloc.source_offset..][0..4], @intCast(disp), .little);
+    for (relocs.items) |reloc| {
+        const target = code_offset_mapping[reloc.target];
+        const disp = @as(i64, @intCast(target)) - @as(i64, @intCast(reloc.inst_offset + reloc.inst_length)) + reloc.target_offset;
+        const inst_bytes = emit.code.items[reloc.inst_offset..][0..reloc.inst_length];
+        switch (reloc.source_length) {
+            else => unreachable,
+            inline 1, 4 => |source_length| std.mem.writeInt(
+                @Type(.{ .int = .{ .signedness = .signed, .bits = @as(u16, 8) * source_length } }),
+                inst_bytes[reloc.source_offset..][0..source_length],
+                @intCast(disp),
+                .little,
+            ),
         }
     }
     if (emit.lower.mir.table.len > 0) {
@@ -511,15 +533,17 @@ fn fail(emit: *Emit, comptime format: []const u8, args: anytype) Error {
 
 const Reloc = struct {
     /// Offset of the instruction.
-    source: u32,
+    inst_offset: u32,
+    /// Length of the instruction.
+    inst_length: u4,
     /// Offset of the relocation within the instruction.
-    source_offset: u32,
+    source_offset: u4,
+    /// Length of the relocation.
+    source_length: u4,
     /// Target of the relocation.
     target: Mir.Inst.Index,
-    /// Offset from the target instruction.
+    /// Offset from the target.
     target_offset: i32,
-    /// Length of the instruction.
-    length: u5,
 };
 
 const TableReloc = struct {

src/arch/x86_64/Encoding.zig

@@ -304,20 +304,20 @@ pub const Mnemonic = enum {
     jnc, jne, jng, jnge, jnl, jnle, jno, jnp, jns, jnz, jo, jp, jpe, jpo, jrcxz, js, jz,
     lahf, lar, lea, leave, lfence, lgdt, lidt, lldt, lmsw, loop, loope, loopne,
     lods, lodsb, lodsd, lodsq, lodsw,
-    lsl, ltr, lzcnt,
+    lsl, ltr,
     mfence, mov, movbe,
     movs, movsb, movsd, movsq, movsw,
     movsx, movsxd, movzx, mul,
     neg, nop, not,
     @"or", out, outs, outsb, outsd, outsw,
-    pause, pop, popcnt, popf, popfd, popfq, push, pushfq,
+    pause, pop, popf, popfd, popfq, push, pushfq,
     rcl, rcr,
     rdfsbase, rdgsbase, rdmsr, rdpid, rdpkru, rdpmc, rdrand, rdseed, rdssd, rdssq, rdtsc, rdtscp,
-    ret, rol, ror, rorx, rsm,
-    sahf, sal, sar, sarx, sbb,
+    ret, rol, ror, rsm,
+    sahf, sal, sar, sbb,
     scas, scasb, scasd, scasq, scasw,
     senduipi, serialize,
-    shl, shld, shlx, shr, shrd, shrx,
+    shl, shld, shr, shrd,
     stac, stc, std, sti, str, stui,
     sub, swapgs, syscall, sysenter, sysexit, sysret,
     seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,
@@ -433,14 +433,15 @@ pub const Mnemonic = enum {
     roundpd, roundps, roundsd, roundss,
     // SSE4.2
     crc32, pcmpgtq,
+    // ABM
+    lzcnt, popcnt,
     // PCLMUL
     pclmulqdq,
     // AES
     aesdec, aesdeclast, aesenc, aesenclast, aesimc, aeskeygenassist,
     // SHA
     sha1rnds4, sha1nexte, sha1msg1, sha1msg2, sha256msg1, sha256msg2, sha256rnds2,
     // AVX
-    andn, bextr, blsi, blsmsk, blsr, bzhi, tzcnt,
     vaddpd, vaddps, vaddsd, vaddss, vaddsubpd, vaddsubps,
     vaesdec, vaesdeclast, vaesenc, vaesenclast, vaesimc, vaeskeygenassist,
     vandnpd, vandnps, vandpd, vandps,
@@ -506,6 +507,10 @@ pub const Mnemonic = enum {
     vtestpd, vtestps,
     vucomisd, vucomiss, vunpckhpd, vunpckhps, vunpcklpd, vunpcklps,
     vxorpd, vxorps,
+    // BMI
+    andn, bextr, blsi, blsmsk, blsr, tzcnt,
+    // BMI2
+    bzhi, mulx, pdep, pext, rorx, sarx, shlx, shrx,
     // F16C
     vcvtph2ps, vcvtps2ph,
     // FMA