Merge pull request #21115 from Snektron/build-system-asm

compilation and build system fixes

Author: andrewrk

lib/compiler/objcopy.zig

@@ -201,9 +201,10 @@ fn cmdObjCopy(
                     if (seen_update) fatal("zig objcopy only supports 1 update for now", .{});
                     seen_update = true;
 
-                    try server.serveEmitBinPath(output, .{
-                        .flags = .{ .cache_hit = false },
-                    });
+                    // The build system already knows what the output is at this point, we
+                    // only need to communicate that the process has finished.
+                    // Use the empty error bundle to indicate that the update is done.
+                    try server.serveErrorBundle(std.zig.ErrorBundle.empty);
                 },
                 else => fatal("unsupported message: {s}", .{@tagName(hdr.tag)}),
             }

lib/std/Build.zig

@@ -2373,7 +2373,7 @@ pub const LazyPath = union(enum) {
                 // basis for not traversing up too many directories.
 
                 var file_path: Cache.Path = .{
-                    .root_dir = gen.file.step.owner.build_root,
+                    .root_dir = Cache.Directory.cwd(),
                     .sub_path = gen.file.path orelse {
                         std.debug.lockStdErr();
                         const stderr = std.io.getStdErr();

lib/std/Build/Cache.zig

@@ -896,8 +896,8 @@ pub const Manifest = struct {
         }
     }
 
-    /// Returns a hex encoded hash of the inputs.
-    pub fn final(self: *Manifest) HexDigest {
+    /// Returns a binary hash of the inputs.
+    pub fn finalBin(self: *Manifest) BinDigest {
         assert(self.manifest_file != null);
 
         // We don't close the manifest file yet, because we want to
@@ -908,7 +908,12 @@ pub const Manifest = struct {
 
         var bin_digest: BinDigest = undefined;
         self.hash.hasher.final(&bin_digest);
+        return bin_digest;
+    }
 
+    /// Returns a hex encoded hash of the inputs.
+    pub fn final(self: *Manifest) HexDigest {
+        const bin_digest = self.finalBin();
         return binToHex(bin_digest);
     }
 

lib/std/Build/Fuzz.zig

@@ -100,6 +100,15 @@ pub fn start(
 }
 
 fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {
+    rebuildTestsWorkerRunFallible(run, ttyconf, parent_prog_node) catch |err| {
+        const compile = run.producer.?;
+        log.err("step '{s}': failed to rebuild in fuzz mode: {s}", .{
+            compile.step.name, @errorName(err),
+        });
+    };
+}
+
+fn rebuildTestsWorkerRunFallible(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) !void {
     const gpa = run.step.owner.allocator;
     const stderr = std.io.getStdErr();
 
@@ -121,14 +130,9 @@ fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog
 
     const rebuilt_bin_path = result catch |err| switch (err) {
         error.MakeFailed => return,
-        else => {
-            log.err("step '{s}': failed to rebuild in fuzz mode: {s}", .{
-                compile.step.name, @errorName(err),
-            });
-            return;
-        },
+        else => |other| return other,
     };
-    run.rebuilt_executable = rebuilt_bin_path;
+    run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename);
 }
 
 fn fuzzWorkerRun(

lib/std/Build/Fuzz/WebServer.zig

@@ -8,6 +8,8 @@ const Coverage = std.debug.Coverage;
 const abi = std.Build.Fuzz.abi;
 const log = std.log;
 const assert = std.debug.assert;
+const Cache = std.Build.Cache;
+const Path = Cache.Path;
 
 const WebServer = @This();
 
@@ -31,6 +33,10 @@ coverage_mutex: std.Thread.Mutex,
 /// Signaled when `coverage_files` changes.
 coverage_condition: std.Thread.Condition,
 
+const fuzzer_bin_name = "fuzzer";
+const fuzzer_arch_os_abi = "wasm32-freestanding";
+const fuzzer_cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext";
+
 const CoverageMap = struct {
     mapped_memory: []align(std.mem.page_size) const u8,
     coverage: Coverage,
@@ -181,9 +187,18 @@ fn serveWasm(
 
     // Do the compilation every request, so that the user can edit the files
     // and see the changes without restarting the server.
-    const wasm_binary_path = try buildWasmBinary(ws, arena, optimize_mode);
+    const wasm_base_path = try buildWasmBinary(ws, arena, optimize_mode);
+    const bin_name = try std.zig.binNameAlloc(arena, .{
+        .root_name = fuzzer_bin_name,
+        .target = std.zig.system.resolveTargetQuery(std.Build.parseTargetQuery(.{
+            .arch_os_abi = fuzzer_arch_os_abi,
+            .cpu_features = fuzzer_cpu_features,
+        }) catch unreachable) catch unreachable,
+        .output_mode = .Exe,
+    });
     // std.http.Server does not have a sendfile API yet.
-    const file_contents = try std.fs.cwd().readFileAlloc(gpa, wasm_binary_path, 10 * 1024 * 1024);
+    const bin_path = try wasm_base_path.join(arena, bin_name);
+    const file_contents = try bin_path.root_dir.handle.readFileAlloc(gpa, bin_path.sub_path, 10 * 1024 * 1024);
     defer gpa.free(file_contents);
     try request.respond(file_contents, .{
         .extra_headers = &.{
@@ -197,7 +212,7 @@ fn buildWasmBinary(
     ws: *WebServer,
     arena: Allocator,
     optimize_mode: std.builtin.OptimizeMode,
-) ![]const u8 {
+) !Path {
     const gpa = ws.gpa;
 
     const main_src_path: Build.Cache.Path = .{
@@ -219,11 +234,11 @@ fn buildWasmBinary(
         ws.zig_exe_path, "build-exe", //
         "-fno-entry", //
         "-O", @tagName(optimize_mode), //
-        "-target", "wasm32-freestanding", //
-        "-mcpu", "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext", //
+        "-target", fuzzer_arch_os_abi, //
+        "-mcpu", fuzzer_cpu_features, //
         "--cache-dir", ws.global_cache_directory.path orelse ".", //
         "--global-cache-dir", ws.global_cache_directory.path orelse ".", //
-        "--name", "fuzzer", //
+        "--name", fuzzer_bin_name, //
         "-rdynamic", //
         "-fsingle-threaded", //
         "--dep", "Walk", //
@@ -251,7 +266,7 @@ fn buildWasmBinary(
     try sendMessage(child.stdin.?, .exit);
 
     const Header = std.zig.Server.Message.Header;
-    var result: ?[]const u8 = null;
+    var result: ?Path = null;
     var result_error_bundle = std.zig.ErrorBundle.empty;
 
     const stdout = poller.fifo(.stdout);
@@ -288,13 +303,17 @@ fn buildWasmBinary(
                     .extra = extra_array,
                 };
             },
-            .emit_bin_path => {
-                const EbpHdr = std.zig.Server.Message.EmitBinPath;
+            .emit_digest => {
+                const EbpHdr = std.zig.Server.Message.EmitDigest;
                 const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
                 if (!ebp_hdr.flags.cache_hit) {
                     log.info("source changes detected; rebuilt wasm component", .{});
                 }
-                result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
+                const digest = body[@sizeOf(EbpHdr)..][0..Cache.bin_digest_len];
+                result = Path{
+                    .root_dir = ws.global_cache_directory,
+                    .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)),
+                };
             },
             else => {}, // ignore other messages
         }
@@ -568,10 +587,7 @@ fn prepareTables(
     };
     errdefer gop.value_ptr.coverage.deinit(gpa);
 
-    const rebuilt_exe_path: Build.Cache.Path = .{
-        .root_dir = Build.Cache.Directory.cwd(),
-        .sub_path = run_step.rebuilt_executable.?,
-    };
+    const rebuilt_exe_path = run_step.rebuilt_executable.?;
     var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| {
         log.err("step '{s}': failed to load debug information for '{}': {s}", .{
             run_step.step.name, rebuilt_exe_path, @errorName(err),

lib/std/Build/Step.zig

@@ -317,6 +317,8 @@ const Build = std.Build;
 const Allocator = std.mem.Allocator;
 const assert = std.debug.assert;
 const builtin = @import("builtin");
+const Cache = Build.Cache;
+const Path = Cache.Path;
 
 pub fn evalChildProcess(s: *Step, argv: []const []const u8) ![]u8 {
     const run_result = try captureChildProcess(s, std.Progress.Node.none, argv);
@@ -373,7 +375,7 @@ pub fn evalZigProcess(
     argv: []const []const u8,
     prog_node: std.Progress.Node,
     watch: bool,
-) !?[]const u8 {
+) !?Path {
     if (s.getZigProcess()) |zp| update: {
         assert(watch);
         if (std.Progress.have_ipc) if (zp.progress_ipc_fd) |fd| prog_node.setIpcFd(fd);
@@ -477,7 +479,7 @@ pub fn evalZigProcess(
     return result;
 }
 
-fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {
+fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?Path {
     const b = s.owner;
     const arena = b.allocator;
 
@@ -487,7 +489,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {
     if (!watch) try sendMessage(zp.child.stdin.?, .exit);
 
     const Header = std.zig.Server.Message.Header;
-    var result: ?[]const u8 = null;
+    var result: ?Path = null;
 
     const stdout = zp.poller.fifo(.stdout);
 
@@ -531,16 +533,15 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {
                     break;
                 }
             },
-            .emit_bin_path => {
-                const EbpHdr = std.zig.Server.Message.EmitBinPath;
+            .emit_digest => {
+                const EbpHdr = std.zig.Server.Message.EmitDigest;
                 const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
                 s.result_cached = ebp_hdr.flags.cache_hit;
-                result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
-                if (watch) {
-                    // This message indicates the end of the update.
-                    stdout.discard(body.len);
-                    break;
-                }
+                const digest = body[@sizeOf(EbpHdr)..][0..Cache.bin_digest_len];
+                result = Path{
+                    .root_dir = b.cache_root,
+                    .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)),
+                };
             },
             .file_system_inputs => {
                 s.clearWatchInputs();

lib/std/Build/Step/Compile.zig

@@ -17,6 +17,7 @@ const Module = std.Build.Module;
 const InstallDir = std.Build.InstallDir;
 const GeneratedFile = std.Build.GeneratedFile;
 const Compile = @This();
+const Path = std.Build.Cache.Path;
 
 pub const base_id: Step.Id = .compile;
 
@@ -1765,7 +1766,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
 
     const zig_args = try getZigArgs(compile, false);
 
-    const maybe_output_bin_path = step.evalZigProcess(
+    const maybe_output_dir = step.evalZigProcess(
         zig_args,
         options.progress_node,
         (b.graph.incremental == true) and options.watch,
@@ -1779,53 +1780,51 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
     };
 
     // Update generated files
-    if (maybe_output_bin_path) |output_bin_path| {
-        const output_dir = fs.path.dirname(output_bin_path).?;
-
+    if (maybe_output_dir) |output_dir| {
         if (compile.emit_directory) |lp| {
-            lp.path = output_dir;
+            lp.path = b.fmt("{}", .{output_dir});
         }
 
         // -femit-bin[=path]         (default) Output machine code
         if (compile.generated_bin) |bin| {
-            bin.path = b.pathJoin(&.{ output_dir, compile.out_filename });
+            bin.path = output_dir.joinString(b.allocator, compile.out_filename) catch @panic("OOM");
         }
 
-        const sep = std.fs.path.sep;
+        const sep = std.fs.path.sep_str;
 
         // output PDB if someone requested it
         if (compile.generated_pdb) |pdb| {
-            pdb.path = b.fmt("{s}{c}{s}.pdb", .{ output_dir, sep, compile.name });
+            pdb.path = b.fmt("{}" ++ sep ++ "{s}.pdb", .{ output_dir, compile.name });
         }
 
         // -femit-implib[=path]      (default) Produce an import .lib when building a Windows DLL
         if (compile.generated_implib) |implib| {
-            implib.path = b.fmt("{s}{c}{s}.lib", .{ output_dir, sep, compile.name });
+            implib.path = b.fmt("{}" ++ sep ++ "{s}.lib", .{ output_dir, compile.name });
         }
 
         // -femit-h[=path]           Generate a C header file (.h)
         if (compile.generated_h) |lp| {
-            lp.path = b.fmt("{s}{c}{s}.h", .{ output_dir, sep, compile.name });
+            lp.path = b.fmt("{}" ++ sep ++ "{s}.h", .{ output_dir, compile.name });
         }
 
         // -femit-docs[=path]        Create a docs/ dir with html documentation
         if (compile.generated_docs) |generated_docs| {
-            generated_docs.path = b.pathJoin(&.{ output_dir, "docs" });
+            generated_docs.path = output_dir.joinString(b.allocator, "docs") catch @panic("OOM");
         }
 
         // -femit-asm[=path]         Output .s (assembly code)
         if (compile.generated_asm) |lp| {
-            lp.path = b.fmt("{s}{c}{s}.s", .{ output_dir, sep, compile.name });
+            lp.path = b.fmt("{}" ++ sep ++ "{s}.s", .{ output_dir, compile.name });
         }
 
         // -femit-llvm-ir[=path]     Produce a .ll file with optimized LLVM IR (requires LLVM extensions)
         if (compile.generated_llvm_ir) |lp| {
-            lp.path = b.fmt("{s}{c}{s}.ll", .{ output_dir, sep, compile.name });
+            lp.path = b.fmt("{}" ++ sep ++ "{s}.ll", .{ output_dir, compile.name });
         }
 
         // -femit-llvm-bc[=path]     Produce an optimized LLVM module as a .bc file (requires LLVM extensions)
         if (compile.generated_llvm_bc) |lp| {
-            lp.path = b.fmt("{s}{c}{s}.bc", .{ output_dir, sep, compile.name });
+            lp.path = b.fmt("{}" ++ sep ++ "{s}.bc", .{ output_dir, compile.name });
         }
     }
 
@@ -1841,7 +1840,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
     }
 }
 
-pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) ![]const u8 {
+pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) !Path {
     const gpa = c.step.owner.allocator;
 
     c.step.result_error_msgs.clearRetainingCapacity();