From a08cc7d2ae3bd6e90f8d26ac13f2e0652687dc30 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 15 Jun 2023 17:10:42 -0700 Subject: [PATCH 01/29] compiler: resolve library paths in the frontend search_strategy is no longer passed to Compilation at all; instead it is used in the CLI code only. When using Zig CLI mode, `-l` no longer has the ability to link statically; use positional arguments for this. The CLI has a small abstraction around library resolution handling which is used to remove some code duplication regarding static libraries, as well as handle the difference between zig cc CLI mode and zig CLI mode. Thanks to this, system libraries are now included in the cache hash, and thus changes to them will correctly cause cache misses. In the future, lib_dirs should no longer be passed to Compilation at all, because it is a frontend-only concept. Previously, -search_paths_first and -search_dylibs_first were Darwin-only arguments; they now work the same for all targets. Same thing with --sysroot. Improved the error reporting for failure to find a system library. An example error now looks like this: ``` $ zig build-exe test.zig -lfoo -L. -L/a -target x86_64-macos --sysroot /home/andy/local error: unable to find Dynamic system library 'foo' using strategy 'no_fallback'. search paths: ./libfoo.tbd ./libfoo.dylib ./libfoo.so /home/andy/local/a/libfoo.tbd /home/andy/local/a/libfoo.dylib /home/andy/local/a/libfoo.so /a/libfoo.tbd /a/libfoo.dylib /a/libfoo.so ``` closes #14963 --- src/Compilation.zig | 20 +- src/link.zig | 33 ++- src/link/Coff/lld.zig | 2 +- src/link/Elf.zig | 10 +- src/link/MachO.zig | 29 ++- src/link/MachO/zld.zig | 91 +------- src/main.zig | 515 ++++++++++++++++++++++++++++++----------- 7 files changed, 454 insertions(+), 246 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 2e269a7e80df..f966f1e17f89 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -448,6 +448,7 @@ pub const ClangPreprocessorMode = enum { stdout, }; +pub const Framework = link.Framework; pub const SystemLib = link.SystemLib; pub const CacheMode = link.CacheMode; @@ -505,7 +506,7 @@ pub const InitOptions = struct { c_source_files: []const CSourceFile = &[0]CSourceFile{}, link_objects: []LinkObject = &[0]LinkObject{}, framework_dirs: []const []const u8 = &[0][]const u8{}, - frameworks: std.StringArrayHashMapUnmanaged(SystemLib) = .{}, + frameworks: std.StringArrayHashMapUnmanaged(Framework) = .{}, system_lib_names: []const []const u8 = &.{}, system_lib_infos: []const SystemLib = &.{}, /// These correspond to the WASI libc emulated subcomponents including: @@ -644,8 +645,6 @@ pub const InitOptions = struct { entitlements: ?[]const u8 = null, /// (Darwin) size of the __PAGEZERO segment pagezero_size: ?u64 = null, - /// (Darwin) search strategy for system libraries - search_strategy: ?link.File.MachO.SearchStrategy = null, /// (Darwin) set minimum space for future expansion of the load commands headerpad_size: ?u32 = null, /// (Darwin) set enough space as if all paths were MATPATHLEN @@ -1567,7 +1566,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .install_name = options.install_name, .entitlements = options.entitlements, .pagezero_size = options.pagezero_size, - .search_strategy = options.search_strategy, .headerpad_size = options.headerpad_size, .headerpad_max_install_names = options.headerpad_max_install_names, .dead_strip_dylibs = options.dead_strip_dylibs, @@ -1727,15 +1725,18 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { // When linking mingw-w64 there are some import libs we always need. for (mingw.always_link_libs) |name| { - try comp.bin_file.options.system_libs.put(comp.gpa, name, .{}); + try comp.bin_file.options.system_libs.put(comp.gpa, name, .{ + .needed = false, + .weak = false, + .path = name, + }); } } // Generate Windows import libs. if (target.os.tag == .windows) { const count = comp.bin_file.options.system_libs.count(); try comp.work_queue.ensureUnusedCapacity(count); - var i: usize = 0; - while (i < count) : (i += 1) { + for (0..count) |i| { comp.work_queue.writeItemAssumeCapacity(.{ .windows_import_lib = i }); } } @@ -2377,7 +2378,7 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes } man.hash.addOptionalBytes(comp.bin_file.options.soname); man.hash.addOptional(comp.bin_file.options.version); - link.hashAddSystemLibs(&man.hash, comp.bin_file.options.system_libs); + try link.hashAddSystemLibs(man, comp.bin_file.options.system_libs); man.hash.addListOfBytes(comp.bin_file.options.force_undefined_symbols.keys()); man.hash.addOptional(comp.bin_file.options.allow_shlib_undefined); man.hash.add(comp.bin_file.options.bind_global_refs_locally); @@ -2395,10 +2396,9 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes // Mach-O specific stuff man.hash.addListOfBytes(comp.bin_file.options.framework_dirs); - link.hashAddSystemLibs(&man.hash, comp.bin_file.options.frameworks); + link.hashAddFrameworks(&man.hash, comp.bin_file.options.frameworks); try man.addOptionalFile(comp.bin_file.options.entitlements); man.hash.addOptional(comp.bin_file.options.pagezero_size); - man.hash.addOptional(comp.bin_file.options.search_strategy); man.hash.addOptional(comp.bin_file.options.headerpad_size); man.hash.add(comp.bin_file.options.headerpad_max_install_names); man.hash.add(comp.bin_file.options.dead_strip_dylibs); diff --git a/src/link.zig b/src/link.zig index 42322cbda882..9edba5012359 100644 --- a/src/link.zig +++ b/src/link.zig @@ -21,7 +21,16 @@ const Type = @import("type.zig").Type; const TypedValue = @import("TypedValue.zig"); /// When adding a new field, remember to update `hashAddSystemLibs`. +/// These are *always* dynamically linked. Static libraries will be +/// provided as positional arguments. pub const SystemLib = struct { + needed: bool, + weak: bool, + path: []const u8, +}; + +/// When adding a new field, remember to update `hashAddFrameworks`. +pub const Framework = struct { needed: bool = false, weak: bool = false, }; @@ -31,11 +40,23 @@ pub const SortSection = enum { name, alignment }; pub const CacheMode = enum { incremental, whole }; pub fn hashAddSystemLibs( - hh: *Cache.HashHelper, + man: *Cache.Manifest, hm: std.StringArrayHashMapUnmanaged(SystemLib), +) !void { + const keys = hm.keys(); + man.hash.addListOfBytes(keys); + for (hm.values()) |value| { + man.hash.add(value.needed); + man.hash.add(value.weak); + _ = try man.addFile(value.path, null); + } +} + +pub fn hashAddFrameworks( + hh: *Cache.HashHelper, + hm: std.StringArrayHashMapUnmanaged(Framework), ) void { const keys = hm.keys(); - hh.add(keys.len); hh.addListOfBytes(keys); for (hm.values()) |value| { hh.add(value.needed); @@ -183,9 +204,12 @@ pub const Options = struct { objects: []Compilation.LinkObject, framework_dirs: []const []const u8, - frameworks: std.StringArrayHashMapUnmanaged(SystemLib), + frameworks: std.StringArrayHashMapUnmanaged(Framework), + /// These are *always* dynamically linked. Static libraries will be + /// provided as positional arguments. system_libs: std.StringArrayHashMapUnmanaged(SystemLib), wasi_emulated_libs: []const wasi_libc.CRTFile, + // TODO: remove this. libraries are resolved by the frontend. lib_dirs: []const []const u8, rpath_list: []const []const u8, @@ -225,9 +249,6 @@ pub const Options = struct { /// (Darwin) size of the __PAGEZERO segment pagezero_size: ?u64 = null, - /// (Darwin) search strategy for system libraries - search_strategy: ?File.MachO.SearchStrategy = null, - /// (Darwin) set minimum space for future expansion of the load commands headerpad_size: ?u32 = null, diff --git a/src/link/Coff/lld.zig b/src/link/Coff/lld.zig index 800b12808133..fe2055027f8b 100644 --- a/src/link/Coff/lld.zig +++ b/src/link/Coff/lld.zig @@ -88,7 +88,7 @@ pub fn linkWithLLD(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod } } } - link.hashAddSystemLibs(&man.hash, self.base.options.system_libs); + try link.hashAddSystemLibs(&man, self.base.options.system_libs); man.hash.addListOfBytes(self.base.options.force_undefined_symbols.keys()); man.hash.addOptional(self.base.options.subsystem); man.hash.add(self.base.options.is_test); diff --git a/src/link/Elf.zig b/src/link/Elf.zig index c4371fbcbff6..3292168bce84 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1428,7 +1428,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v } man.hash.addOptionalBytes(self.base.options.soname); man.hash.addOptional(self.base.options.version); - link.hashAddSystemLibs(&man.hash, self.base.options.system_libs); + try link.hashAddSystemLibs(&man, self.base.options.system_libs); man.hash.addListOfBytes(self.base.options.force_undefined_symbols.keys()); man.hash.add(allow_shlib_undefined); man.hash.add(self.base.options.bind_global_refs_locally); @@ -1824,8 +1824,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v argv.appendAssumeCapacity("--as-needed"); var as_needed = true; - for (system_libs, 0..) |link_lib, i| { - const lib_as_needed = !system_libs_values[i].needed; + for (system_libs_values) |lib_info| { + const lib_as_needed = !lib_info.needed; switch ((@as(u2, @intFromBool(lib_as_needed)) << 1) | @intFromBool(as_needed)) { 0b00, 0b11 => {}, 0b01 => { @@ -1842,9 +1842,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v // libraries and not static libraries (the check for that needs to be earlier), // but they could be full paths to .so files, in which case we // want to avoid prepending "-l". - const ext = Compilation.classifyFileExt(link_lib); - const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{s}", .{link_lib}); - argv.appendAssumeCapacity(arg); + argv.appendAssumeCapacity(lib_info.path); } if (!as_needed) { diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 44e6acc3976d..389d275fd57e 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -58,11 +58,6 @@ const Rebase = @import("MachO/dyld_info/Rebase.zig"); pub const base_tag: File.Tag = File.Tag.macho; -pub const SearchStrategy = enum { - paths_first, - dylibs_first, -}; - /// Mode of operation of the linker. pub const Mode = enum { /// Incremental mode will preallocate segments/sections and is compatible with @@ -840,7 +835,11 @@ pub fn resolveLibSystem( // re-exports every single symbol definition. for (search_dirs) |dir| { if (try resolveLib(arena, dir, "System", ".tbd")) |full_path| { - try out_libs.put(full_path, .{ .needed = true }); + try out_libs.put(full_path, .{ + .needed = true, + .weak = false, + .path = full_path, + }); libsystem_available = true; break :blk; } @@ -850,8 +849,16 @@ pub fn resolveLibSystem( for (search_dirs) |dir| { if (try resolveLib(arena, dir, "System", ".dylib")) |libsystem_path| { if (try resolveLib(arena, dir, "c", ".dylib")) |libc_path| { - try out_libs.put(libsystem_path, .{ .needed = true }); - try out_libs.put(libc_path, .{ .needed = true }); + try out_libs.put(libsystem_path, .{ + .needed = true, + .weak = false, + .path = libsystem_path, + }); + try out_libs.put(libc_path, .{ + .needed = true, + .weak = false, + .path = libc_path, + }); libsystem_available = true; break :blk; } @@ -865,7 +872,11 @@ pub fn resolveLibSystem( const full_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "darwin", libsystem_name, }); - try out_libs.put(full_path, .{ .needed = true }); + try out_libs.put(full_path, .{ + .needed = true, + .weak = false, + .path = full_path, + }); } } diff --git a/src/link/MachO/zld.zig b/src/link/MachO/zld.zig index 39b64fe1787d..c2a814b319db 100644 --- a/src/link/MachO/zld.zig +++ b/src/link/MachO/zld.zig @@ -3410,7 +3410,6 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr // installation sources because they are always a product of the compiler version + target information. man.hash.add(stack_size); man.hash.addOptional(options.pagezero_size); - man.hash.addOptional(options.search_strategy); man.hash.addOptional(options.headerpad_size); man.hash.add(options.headerpad_max_install_names); man.hash.add(gc_sections); @@ -3418,13 +3417,13 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr man.hash.add(options.strip); man.hash.addListOfBytes(options.lib_dirs); man.hash.addListOfBytes(options.framework_dirs); - link.hashAddSystemLibs(&man.hash, options.frameworks); + link.hashAddFrameworks(&man.hash, options.frameworks); man.hash.addListOfBytes(options.rpath_list); if (is_dyn_lib) { man.hash.addOptionalBytes(options.install_name); man.hash.addOptional(options.version); } - link.hashAddSystemLibs(&man.hash, options.system_libs); + try link.hashAddSystemLibs(&man, options.system_libs); man.hash.addOptionalBytes(options.sysroot); man.hash.addListOfBytes(options.force_undefined_symbols.keys()); try man.addOptionalFile(options.entitlements); @@ -3550,84 +3549,20 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr try positionals.append(comp.libcxx_static_lib.?.full_object_path); } - // Shared and static libraries passed via `-l` flag. - var candidate_libs = std.StringArrayHashMap(link.SystemLib).init(arena); - - const system_lib_names = options.system_libs.keys(); - for (system_lib_names) |system_lib_name| { - // By this time, we depend on these libs being dynamically linked libraries and not static libraries - // (the check for that needs to be earlier), but they could be full paths to .dylib files, in which - // case we want to avoid prepending "-l". - if (Compilation.classifyFileExt(system_lib_name) == .shared_library) { - try positionals.append(system_lib_name); - continue; - } - - const system_lib_info = options.system_libs.get(system_lib_name).?; - try candidate_libs.put(system_lib_name, .{ - .needed = system_lib_info.needed, - .weak = system_lib_info.weak, - }); - } - - var lib_dirs = std.ArrayList([]const u8).init(arena); - for (options.lib_dirs) |dir| { - if (try MachO.resolveSearchDir(arena, dir, options.sysroot)) |search_dir| { - try lib_dirs.append(search_dir); - } else { - log.warn("directory not found for '-L{s}'", .{dir}); - } + { + // Add all system library paths to positionals. + const vals = options.system_libs.values(); + try positionals.ensureUnusedCapacity(vals.len); + for (vals) |info| positionals.appendAssumeCapacity(info.path); } var libs = std.StringArrayHashMap(link.SystemLib).init(arena); - // Assume ld64 default -search_paths_first if no strategy specified. - const search_strategy = options.search_strategy orelse .paths_first; - outer: for (candidate_libs.keys()) |lib_name| { - switch (search_strategy) { - .paths_first => { - // Look in each directory for a dylib (stub first), and then for archive - for (lib_dirs.items) |dir| { - for (&[_][]const u8{ ".tbd", ".dylib", ".a" }) |ext| { - if (try MachO.resolveLib(arena, dir, lib_name, ext)) |full_path| { - try libs.put(full_path, candidate_libs.get(lib_name).?); - continue :outer; - } - } - } else { - log.warn("library not found for '-l{s}'", .{lib_name}); - lib_not_found = true; - } - }, - .dylibs_first => { - // First, look for a dylib in each search dir - for (lib_dirs.items) |dir| { - for (&[_][]const u8{ ".tbd", ".dylib" }) |ext| { - if (try MachO.resolveLib(arena, dir, lib_name, ext)) |full_path| { - try libs.put(full_path, candidate_libs.get(lib_name).?); - continue :outer; - } - } - } else for (lib_dirs.items) |dir| { - if (try MachO.resolveLib(arena, dir, lib_name, ".a")) |full_path| { - try libs.put(full_path, candidate_libs.get(lib_name).?); - } else { - log.warn("library not found for '-l{s}'", .{lib_name}); - lib_not_found = true; - } - } - }, - } - } - - if (lib_not_found) { - log.warn("Library search paths:", .{}); - for (lib_dirs.items) |dir| { - log.warn(" {s}", .{dir}); - } + for (options.system_libs.values()) |v| { + try libs.put(v.path, v); } - try MachO.resolveLibSystem(arena, comp, options.sysroot, target, lib_dirs.items, &libs); + try MachO.resolveLibSystem(arena, comp, options.sysroot, target, options.lib_dirs, &libs); // frameworks var framework_dirs = std.ArrayList([]const u8).init(arena); @@ -3647,6 +3582,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr try libs.put(full_path, .{ .needed = info.needed, .weak = info.weak, + .path = full_path, }); continue :outer; } @@ -3698,11 +3634,6 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr try argv.append(try std.fmt.allocPrint(arena, "0x{x}", .{pagezero_size})); } - if (options.search_strategy) |strat| switch (strat) { - .paths_first => try argv.append("-search_paths_first"), - .dylibs_first => try argv.append("-search_dylibs_first"), - }; - if (options.headerpad_size) |headerpad_size| { try argv.append("-headerpad_size"); try argv.append(try std.fmt.allocPrint(arena, "0x{x}", .{headerpad_size})); diff --git a/src/main.zig b/src/main.zig index c29c35ede49d..83e1e2bc3878 100644 --- a/src/main.zig +++ b/src/main.zig @@ -527,11 +527,11 @@ const usage_build_generic = \\ --subsystem [subsystem] (Windows) /SUBSYSTEM: to the linker \\ --stack [size] Override default stack size \\ --image-base [addr] Set base address for executable image - \\ -weak-l[lib] (Darwin) link against system library and mark it and all referenced symbols as weak + \\ -weak-l[lib] link against system library and mark it and all referenced symbols as weak \\ -weak_library [lib] \\ -framework [name] (Darwin) link against framework \\ -needed_framework [name] (Darwin) link against framework (even if unused) - \\ -needed_library [lib] (Darwin) link against system library (even if unused) + \\ -needed_library [lib] link against system library (even if unused) \\ -weak_framework [name] (Darwin) link against framework and mark it and all referenced symbols as weak \\ -F[dir] (Darwin) add search path for frameworks \\ -install_name=[value] (Darwin) add dylib's install name @@ -716,6 +716,27 @@ const ArgsIterator = struct { } }; +/// In contrast to `link.SystemLib`, this stores arguments that may need to be +/// resolved into static libraries so that we can pass only dynamic libraries +/// as system libs to `Compilation`. +const SystemLib = struct { + needed: bool, + weak: bool, + + preferred_mode: std.builtin.LinkMode, + search_strategy: SearchStrategy, + + const SearchStrategy = enum { paths_first, mode_first, no_fallback }; + + fn fallbackMode(this: SystemLib) std.builtin.LinkMode { + assert(this.search_strategy != .no_fallback); + return switch (this.preferred_mode) { + .Dynamic => .Static, + .Static => .Dynamic, + }; + } +}; + fn buildOutputType( gpa: Allocator, arena: Allocator, @@ -854,7 +875,8 @@ fn buildOutputType( var hash_style: link.HashStyle = .both; var entitlements: ?[]const u8 = null; var pagezero_size: ?u64 = null; - var search_strategy: ?link.File.MachO.SearchStrategy = null; + var lib_search_strategy: ?SystemLib.SearchStrategy = null; + var lib_preferred_mode: ?std.builtin.LinkMode = null; var headerpad_size: ?u32 = null; var headerpad_max_install_names: bool = false; var dead_strip_dylibs: bool = false; @@ -869,11 +891,7 @@ fn buildOutputType( var llvm_m_args = std.ArrayList([]const u8).init(gpa); defer llvm_m_args.deinit(); - var system_libs = std.StringArrayHashMap(Compilation.SystemLib).init(gpa); - defer system_libs.deinit(); - - var static_libs = std.ArrayList([]const u8).init(gpa); - defer static_libs.deinit(); + var system_libs = std.StringArrayHashMap(SystemLib).init(arena); var wasi_emulated_libs = std.ArrayList(wasi_libc.CRTFile).init(gpa); defer wasi_emulated_libs.deinit(); @@ -884,8 +902,8 @@ fn buildOutputType( var extra_cflags = std.ArrayList([]const u8).init(gpa); defer extra_cflags.deinit(); - var lib_dirs = std.ArrayList([]const u8).init(gpa); - defer lib_dirs.deinit(); + // These are before resolving sysroot. + var lib_dir_args = std.ArrayList([]const u8).init(arena); var rpath_list = std.ArrayList([]const u8).init(gpa); defer rpath_list.deinit(); @@ -901,7 +919,7 @@ fn buildOutputType( var framework_dirs = std.ArrayList([]const u8).init(gpa); defer framework_dirs.deinit(); - var frameworks: std.StringArrayHashMapUnmanaged(Compilation.SystemLib) = .{}; + var frameworks: std.StringArrayHashMapUnmanaged(Compilation.Framework) = .{}; // null means replace with the test executable binary var test_exec_args = std.ArrayList(?[]const u8).init(gpa); @@ -1061,7 +1079,7 @@ fn buildOutputType( } else if (mem.eql(u8, arg, "-rpath")) { try rpath_list.append(args_iter.nextOrFatal()); } else if (mem.eql(u8, arg, "--library-directory") or mem.eql(u8, arg, "-L")) { - try lib_dirs.append(args_iter.nextOrFatal()); + try lib_dir_args.append(args_iter.nextOrFatal()); } else if (mem.eql(u8, arg, "-F")) { try framework_dirs.append(args_iter.nextOrFatal()); } else if (mem.eql(u8, arg, "-framework")) { @@ -1085,9 +1103,11 @@ fn buildOutputType( fatal("unable to parse pagezero size'{s}': {s}", .{ next_arg, @errorName(err) }); }; } else if (mem.eql(u8, arg, "-search_paths_first")) { - search_strategy = .paths_first; + lib_search_strategy = .paths_first; + lib_preferred_mode = .Dynamic; } else if (mem.eql(u8, arg, "-search_dylibs_first")) { - search_strategy = .dylibs_first; + lib_search_strategy = .mode_first; + lib_preferred_mode = .Dynamic; } else if (mem.eql(u8, arg, "-headerpad")) { const next_arg = args_iter.nextOrFatal(); headerpad_size = std.fmt.parseUnsigned(u32, eatIntPrefix(next_arg, 16), 16) catch |err| { @@ -1104,17 +1124,36 @@ fn buildOutputType( } else if (mem.eql(u8, arg, "-version-script") or mem.eql(u8, arg, "--version-script")) { version_script = args_iter.nextOrFatal(); } else if (mem.eql(u8, arg, "--library") or mem.eql(u8, arg, "-l")) { - // We don't know whether this library is part of libc or libc++ until - // we resolve the target, so we simply append to the list for now. - try system_libs.put(args_iter.nextOrFatal(), .{}); + // We don't know whether this library is part of libc + // or libc++ until we resolve the target, so we append + // to the list for now. + try system_libs.put(args_iter.nextOrFatal(), .{ + .needed = false, + .weak = false, + // -l always dynamic links. For static libraries, + // users are expected to use positional arguments + // which are always unambiguous. + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .no_fallback, + }); } else if (mem.eql(u8, arg, "--needed-library") or mem.eql(u8, arg, "-needed-l") or mem.eql(u8, arg, "-needed_library")) { const next_arg = args_iter.nextOrFatal(); - try system_libs.put(next_arg, .{ .needed = true }); + try system_libs.put(next_arg, .{ + .needed = true, + .weak = false, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .no_fallback, + }); } else if (mem.eql(u8, arg, "-weak_library") or mem.eql(u8, arg, "-weak-l")) { - try system_libs.put(args_iter.nextOrFatal(), .{ .weak = true }); + try system_libs.put(args_iter.nextOrFatal(), .{ + .needed = false, + .weak = true, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .no_fallback, + }); } else if (mem.eql(u8, arg, "-D")) { try clang_argv.append(arg); try clang_argv.append(args_iter.nextOrFatal()); @@ -1486,17 +1525,36 @@ fn buildOutputType( } else if (mem.startsWith(u8, arg, "-T")) { linker_script = arg[2..]; } else if (mem.startsWith(u8, arg, "-L")) { - try lib_dirs.append(arg[2..]); + try lib_dir_args.append(arg[2..]); } else if (mem.startsWith(u8, arg, "-F")) { try framework_dirs.append(arg[2..]); } else if (mem.startsWith(u8, arg, "-l")) { - // We don't know whether this library is part of libc or libc++ until - // we resolve the target, so we simply append to the list for now. - try system_libs.put(arg["-l".len..], .{}); + // We don't know whether this library is part of libc + // or libc++ until we resolve the target, so we append + // to the list for now. + try system_libs.put(arg["-l".len..], .{ + .needed = false, + .weak = false, + // -l always dynamic links. For static libraries, + // users are expected to use positional arguments + // which are always unambiguous. + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .no_fallback, + }); } else if (mem.startsWith(u8, arg, "-needed-l")) { - try system_libs.put(arg["-needed-l".len..], .{ .needed = true }); + try system_libs.put(arg["-needed-l".len..], .{ + .needed = true, + .weak = false, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .no_fallback, + }); } else if (mem.startsWith(u8, arg, "-weak-l")) { - try system_libs.put(arg["-weak-l".len..], .{ .weak = true }); + try system_libs.put(arg["-weak-l".len..], .{ + .needed = false, + .weak = true, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .no_fallback, + }); } else if (mem.startsWith(u8, arg, "-D")) { try clang_argv.append(arg); } else if (mem.startsWith(u8, arg, "-I")) { @@ -1642,9 +1700,22 @@ fn buildOutputType( .loption = true, }); } else if (force_static_libs) { - try static_libs.append(it.only_arg); + try system_libs.put(it.only_arg, .{ + .needed = false, + .weak = false, + .preferred_mode = .Static, + .search_strategy = .no_fallback, + }); } else { - try system_libs.put(it.only_arg, .{ .needed = needed }); + // C compilers are traditionally expected to look + // first for dynamic libraries and then fall back + // to static libraries. + try system_libs.put(it.only_arg, .{ + .needed = needed, + .weak = false, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .paths_first, + }); } }, .ignore => {}, @@ -1748,9 +1819,11 @@ fn buildOutputType( { force_static_libs = true; } else if (mem.eql(u8, linker_arg, "-search_paths_first")) { - search_strategy = .paths_first; + lib_search_strategy = .paths_first; + lib_preferred_mode = .Dynamic; } else if (mem.eql(u8, linker_arg, "-search_dylibs_first")) { - search_strategy = .dylibs_first; + lib_search_strategy = .mode_first; + lib_preferred_mode = .Dynamic; } else { try linker_args.append(linker_arg); } @@ -1828,7 +1901,7 @@ fn buildOutputType( try linker_args.append("-z"); try linker_args.append(it.only_arg); }, - .lib_dir => try lib_dirs.append(it.only_arg), + .lib_dir => try lib_dir_args.append(it.only_arg), .mcpu => target_mcpu = it.only_arg, .m => try llvm_m_args.append(it.only_arg), .dep_file => { @@ -1860,7 +1933,12 @@ fn buildOutputType( .force_undefined_symbol => { try force_undefined_symbols.put(gpa, it.only_arg, {}); }, - .weak_library => try system_libs.put(it.only_arg, .{ .weak = true }), + .weak_library => try system_libs.put(it.only_arg, .{ + .needed = false, + .weak = true, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .paths_first, + }), .weak_framework => try frameworks.put(gpa, it.only_arg, .{ .weak = true }), .headerpad_max_install_names => headerpad_max_install_names = true, .compress_debug_sections => { @@ -2156,11 +2234,26 @@ fn buildOutputType( } else if (mem.eql(u8, arg, "-needed_framework")) { try frameworks.put(gpa, linker_args_it.nextOrFatal(), .{ .needed = true }); } else if (mem.eql(u8, arg, "-needed_library")) { - try system_libs.put(linker_args_it.nextOrFatal(), .{ .needed = true }); + try system_libs.put(linker_args_it.nextOrFatal(), .{ + .weak = false, + .needed = true, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .paths_first, + }); } else if (mem.startsWith(u8, arg, "-weak-l")) { - try system_libs.put(arg["-weak-l".len..], .{ .weak = true }); + try system_libs.put(arg["-weak-l".len..], .{ + .weak = true, + .needed = false, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .paths_first, + }); } else if (mem.eql(u8, arg, "-weak_library")) { - try system_libs.put(linker_args_it.nextOrFatal(), .{ .weak = true }); + try system_libs.put(linker_args_it.nextOrFatal(), .{ + .weak = true, + .needed = false, + .preferred_mode = lib_preferred_mode orelse .Dynamic, + .search_strategy = lib_search_strategy orelse .paths_first, + }); } else if (mem.eql(u8, arg, "-compatibility_version")) { const compat_version = linker_args_it.nextOrFatal(); compatibility_version = std.SemanticVersion.parse(compat_version) catch |err| { @@ -2458,40 +2551,63 @@ fn buildOutputType( } } + // Resolve the library path arguments with respect to sysroot. + var lib_dirs = std.ArrayList([]const u8).init(arena); + if (sysroot) |root| { + for (lib_dir_args.items) |dir| { + if (fs.path.isAbsolute(dir)) { + const stripped_dir = dir[fs.path.diskDesignator(dir).len..]; + const full_path = try fs.path.join(arena, &[_][]const u8{ root, stripped_dir }); + try lib_dirs.append(full_path); + } + try lib_dirs.append(dir); + } + } else { + lib_dirs = lib_dir_args; + } + lib_dir_args = undefined; // From here we use lib_dirs instead. + // Now that we have target info, we can find out if any of the system libraries // are part of libc or libc++. We remove them from the list and communicate their // existence via flags instead. + // Similarly, if any libs in this list are statically provided, we omit + // them from the resolved list and populate the link_objects array instead. + var resolved_system_libs: std.MultiArrayList(struct { + name: []const u8, + lib: Compilation.SystemLib, + }) = .{}; + { - // Similarly, if any libs in this list are statically provided, we remove - // them from this list and populate the link_objects array instead. - const sep = fs.path.sep_str; var test_path = std.ArrayList(u8).init(gpa); defer test_path.deinit(); - var i: usize = 0; - syslib: while (i < system_libs.count()) { - const lib_name = system_libs.keys()[i]; + var checked_paths = std.ArrayList(u8).init(gpa); + defer checked_paths.deinit(); + var failed_libs = std.ArrayList(struct { + name: []const u8, + strategy: SystemLib.SearchStrategy, + checked_paths: []const u8, + preferred_mode: std.builtin.LinkMode, + }).init(arena); + + syslib: for (system_libs.keys(), system_libs.values()) |lib_name, info| { if (target_util.is_libc_lib_name(target_info.target, lib_name)) { link_libc = true; - system_libs.orderedRemoveAt(i); continue; } if (target_util.is_libcpp_lib_name(target_info.target, lib_name)) { link_libcpp = true; - system_libs.orderedRemoveAt(i); continue; } switch (target_util.classifyCompilerRtLibName(target_info.target, lib_name)) { .none => {}, .only_libunwind, .both => { link_libunwind = true; - system_libs.orderedRemoveAt(i); continue; }, .only_compiler_rt => { std.log.warn("ignoring superfluous library '{s}': this dependency is fulfilled instead by compiler-rt which zig unconditionally provides", .{lib_name}); - system_libs.orderedRemoveAt(i); continue; }, } @@ -2503,53 +2619,150 @@ fn buildOutputType( if (target_info.target.os.tag == .wasi) { if (wasi_libc.getEmulatedLibCRTFile(lib_name)) |crt_file| { try wasi_emulated_libs.append(crt_file); - system_libs.orderedRemoveAt(i); continue; } } - for (lib_dirs.items) |lib_dir_path| { - if (cross_target.isDarwin()) break; // Targeting Darwin we let the linker resolve the libraries in the correct order - test_path.clearRetainingCapacity(); - try test_path.writer().print("{s}" ++ sep ++ "{s}{s}{s}", .{ - lib_dir_path, - target_info.target.libPrefix(), - lib_name, - target_info.target.staticLibSuffix(), - }); - fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { - error.FileNotFound => continue, - else => |e| fatal("unable to search for static library '{s}': {s}", .{ - test_path.items, @errorName(e), - }), - }; - try link_objects.append(.{ .path = try arena.dupe(u8, test_path.items) }); - system_libs.orderedRemoveAt(i); - continue :syslib; - } + checked_paths.clearRetainingCapacity(); + + switch (info.search_strategy) { + .mode_first, .no_fallback => { + // check for preferred mode + for (lib_dirs.items) |lib_dir_path| { + if (try accessLibPath( + &test_path, + &checked_paths, + lib_dir_path, + lib_name, + target_info.target, + info.preferred_mode, + )) { + const path = try arena.dupe(u8, test_path.items); + switch (info.preferred_mode) { + .Static => try link_objects.append(.{ .path = path }), + .Dynamic => try resolved_system_libs.append(arena, .{ + .name = lib_name, + .lib = .{ + .needed = info.needed, + .weak = info.weak, + .path = path, + }, + }), + } + continue :syslib; + } + } + // check for fallback mode + if (info.search_strategy == .no_fallback) { + try failed_libs.append(.{ + .name = lib_name, + .strategy = info.search_strategy, + .checked_paths = try arena.dupe(u8, checked_paths.items), + .preferred_mode = info.preferred_mode, + }); + continue :syslib; + } + for (lib_dirs.items) |lib_dir_path| { + if (try accessLibPath( + &test_path, + &checked_paths, + lib_dir_path, + lib_name, + target_info.target, + info.fallbackMode(), + )) { + const path = try arena.dupe(u8, test_path.items); + switch (info.preferred_mode) { + .Static => try link_objects.append(.{ .path = path }), + .Dynamic => try resolved_system_libs.append(arena, .{ + .name = lib_name, + .lib = .{ + .needed = info.needed, + .weak = info.weak, + .path = path, + }, + }), + } + continue :syslib; + } + } + try failed_libs.append(.{ + .name = lib_name, + .strategy = info.search_strategy, + .checked_paths = try arena.dupe(u8, checked_paths.items), + .preferred_mode = info.preferred_mode, + }); + continue :syslib; + }, + .paths_first => { + for (lib_dirs.items) |lib_dir_path| { + // check for preferred mode + if (try accessLibPath( + &test_path, + &checked_paths, + lib_dir_path, + lib_name, + target_info.target, + info.preferred_mode, + )) { + const path = try arena.dupe(u8, test_path.items); + switch (info.preferred_mode) { + .Static => try link_objects.append(.{ .path = path }), + .Dynamic => try resolved_system_libs.append(arena, .{ + .name = lib_name, + .lib = .{ + .needed = info.needed, + .weak = info.weak, + .path = path, + }, + }), + } + continue :syslib; + } - // Unfortunately, in the case of MinGW we also need to look for `libfoo.a`. - if (target_info.target.isMinGW()) { - for (lib_dirs.items) |lib_dir_path| { - test_path.clearRetainingCapacity(); - try test_path.writer().print("{s}" ++ sep ++ "lib{s}.a", .{ - lib_dir_path, lib_name, + // check for fallback mode + if (try accessLibPath( + &test_path, + &checked_paths, + lib_dir_path, + lib_name, + target_info.target, + info.fallbackMode(), + )) { + const path = try arena.dupe(u8, test_path.items); + switch (info.preferred_mode) { + .Static => try link_objects.append(.{ .path = path }), + .Dynamic => try resolved_system_libs.append(arena, .{ + .name = lib_name, + .lib = .{ + .needed = info.needed, + .weak = info.weak, + .path = path, + }, + }), + } + continue :syslib; + } + } + try failed_libs.append(.{ + .name = lib_name, + .strategy = info.search_strategy, + .checked_paths = try arena.dupe(u8, checked_paths.items), + .preferred_mode = info.preferred_mode, }); - fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { - error.FileNotFound => continue, - else => |e| fatal("unable to search for static library '{s}': {s}", .{ - test_path.items, @errorName(e), - }), - }; - try link_objects.append(.{ .path = try arena.dupe(u8, test_path.items) }); - system_libs.orderedRemoveAt(i); continue :syslib; - } + }, } + @compileError("unreachable"); + } - std.log.scoped(.cli).debug("depending on system for -l{s}", .{lib_name}); - - i += 1; + if (failed_libs.items.len > 0) { + for (failed_libs.items) |f| { + std.log.err("unable to find {s} system library '{s}' using strategy '{s}'. searched paths:{s}", .{ + @tagName(f.preferred_mode), f.name, @tagName(f.strategy), f.checked_paths, + }); + } + process.exit(1); } } // libc++ depends on libc @@ -2576,7 +2789,7 @@ fn buildOutputType( } if (sysroot == null and cross_target.isNativeOs() and - (system_libs.count() != 0 or want_native_include_dirs)) + (resolved_system_libs.len != 0 or want_native_include_dirs)) { const paths = std.zig.system.NativePaths.detect(arena, target_info) catch |err| { fatal("unable to detect native system paths: {s}", .{@errorName(err)}); @@ -2613,54 +2826,8 @@ fn buildOutputType( framework_dirs.appendAssumeCapacity(framework_dir); } - for (paths.lib_dirs.items) |lib_dir| { - try lib_dirs.append(lib_dir); - } - for (paths.rpaths.items) |rpath| { - try rpath_list.append(rpath); - } - } - - { - // Resolve static libraries into full paths. - const sep = fs.path.sep_str; - - var test_path = std.ArrayList(u8).init(gpa); - defer test_path.deinit(); - - for (static_libs.items) |static_lib| { - for (lib_dirs.items) |lib_dir_path| { - test_path.clearRetainingCapacity(); - try test_path.writer().print("{s}" ++ sep ++ "{s}{s}{s}", .{ - lib_dir_path, - target_info.target.libPrefix(), - static_lib, - target_info.target.staticLibSuffix(), - }); - fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { - error.FileNotFound => continue, - else => |e| fatal("unable to search for static library '{s}': {s}", .{ - test_path.items, @errorName(e), - }), - }; - try link_objects.append(.{ .path = try arena.dupe(u8, test_path.items) }); - break; - } else { - var search_paths = std.ArrayList(u8).init(arena); - for (lib_dirs.items) |lib_dir_path| { - try search_paths.writer().print("\n {s}" ++ sep ++ "{s}{s}{s}", .{ - lib_dir_path, - target_info.target.libPrefix(), - static_lib, - target_info.target.staticLibSuffix(), - }); - } - try search_paths.appendSlice("\n suggestion: use full paths to static libraries on the command line rather than using -l and -L arguments"); - fatal("static library '{s}' not found. search paths: {s}", .{ - static_lib, search_paths.items, - }); - } - } + try lib_dirs.appendSlice(paths.lib_dirs.items); + try rpath_list.appendSlice(paths.rpaths.items); } const object_format = target_info.target.ofmt; @@ -3086,8 +3253,8 @@ fn buildOutputType( .link_objects = link_objects.items, .framework_dirs = framework_dirs.items, .frameworks = frameworks, - .system_lib_names = system_libs.keys(), - .system_lib_infos = system_libs.values(), + .system_lib_names = resolved_system_libs.items(.name), + .system_lib_infos = resolved_system_libs.items(.lib), .wasi_emulated_libs = wasi_emulated_libs.items, .link_libc = link_libc, .link_libcpp = link_libcpp, @@ -3196,7 +3363,6 @@ fn buildOutputType( .install_name = install_name, .entitlements = entitlements, .pagezero_size = pagezero_size, - .search_strategy = search_strategy, .headerpad_size = headerpad_size, .headerpad_max_install_names = headerpad_max_install_names, .dead_strip_dylibs = dead_strip_dylibs, @@ -6068,3 +6234,84 @@ fn renderOptions(color: Color) std.zig.ErrorBundle.RenderOptions { .include_reference_trace = ttyconf != .no_color, }; } + +fn accessLibPath( + test_path: *std.ArrayList(u8), + checked_paths: *std.ArrayList(u8), + lib_dir_path: []const u8, + lib_name: []const u8, + target: std.Target, + link_mode: std.builtin.LinkMode, +) !bool { + const sep = fs.path.sep_str; + + if (target.isDarwin() and link_mode == .Dynamic) tbd: { + // Prefer .tbd over .dylib. + test_path.clearRetainingCapacity(); + try test_path.writer().print("{s}" ++ sep ++ "lib{s}.tbd", .{ lib_dir_path, lib_name }); + try checked_paths.writer().print("\n {s}", .{test_path.items}); + fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { + error.FileNotFound => break :tbd, + else => |e| fatal("unable to search for tbd library '{s}': {s}", .{ + test_path.items, @errorName(e), + }), + }; + return true; + } + + main_check: { + test_path.clearRetainingCapacity(); + try test_path.writer().print("{s}" ++ sep ++ "{s}{s}{s}", .{ + lib_dir_path, + target.libPrefix(), + lib_name, + switch (link_mode) { + .Static => target.staticLibSuffix(), + .Dynamic => target.dynamicLibSuffix(), + }, + }); + try checked_paths.writer().print("\n {s}", .{test_path.items}); + fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { + error.FileNotFound => break :main_check, + else => |e| fatal("unable to search for {s} library '{s}': {s}", .{ + @tagName(link_mode), test_path.items, @errorName(e), + }), + }; + return true; + } + + // In the case of Darwin, the main check will be .dylib, so here we + // additionally check for .so files. + if (target.isDarwin() and link_mode == .Dynamic) so: { + // Prefer .tbd over .dylib. + test_path.clearRetainingCapacity(); + try test_path.writer().print("{s}" ++ sep ++ "lib{s}.so", .{ lib_dir_path, lib_name }); + try checked_paths.writer().print("\n {s}", .{test_path.items}); + fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { + error.FileNotFound => break :so, + else => |e| fatal("unable to search for tbd library '{s}': {s}", .{ + test_path.items, @errorName(e), + }), + }; + return true; + } + + // In the case of MinGW, the main check will be .lib but we also need to + // look for `libfoo.a`. + if (target.isMinGW() and link_mode == .Static) mingw: { + test_path.clearRetainingCapacity(); + try test_path.writer().print("{s}" ++ sep ++ "lib{s}.a", .{ + lib_dir_path, lib_name, + }); + try checked_paths.writer().print("\n {s}", .{test_path.items}); + fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { + error.FileNotFound => break :mingw, + else => |e| fatal("unable to search for static library '{s}': {s}", .{ + test_path.items, @errorName(e), + }), + }; + return true; + } + + return false; +} From a1e21ceec8ecb475f96c5e83ee9f8d59ceecc82d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 1 Aug 2023 17:43:09 -0700 Subject: [PATCH 02/29] frontend: fix linking to Windows DLLs as system libs --- src/Compilation.zig | 5 +++ src/main.zig | 87 ++++++++++++++++++++++++++++----------------- src/mingw.zig | 32 +++++++++++++---- 3 files changed, 85 insertions(+), 39 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index f966f1e17f89..0c8b3c524866 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -5618,6 +5618,11 @@ pub fn addLinkLib(comp: *Compilation, lib_name: []const u8) !void { // to queue up a work item to produce the DLL import library for this. const gop = try comp.bin_file.options.system_libs.getOrPut(comp.gpa, lib_name); if (!gop.found_existing and comp.getTarget().os.tag == .windows) { + gop.value_ptr.* = .{ + .needed = true, + .weak = false, + .path = undefined, + }; try comp.work_queue.writeItem(.{ .windows_import_lib = comp.bin_file.options.system_libs.count() - 1, }); diff --git a/src/main.zig b/src/main.zig index 83e1e2bc3878..797a33a20b28 100644 --- a/src/main.zig +++ b/src/main.zig @@ -28,6 +28,7 @@ const target_util = @import("target.zig"); const crash_report = @import("crash_report.zig"); const Module = @import("Module.zig"); const AstGen = @import("AstGen.zig"); +const mingw = @import("mingw.zig"); const Server = std.zig.Server; pub const std_options = struct { @@ -477,6 +478,8 @@ const usage_build_generic = \\ -needed-l[lib], Link against system library (even if unused) \\ --needed-library [lib] \\ -L[d], --library-directory [d] Add a directory to the library search path + \\ -search_paths_first Search each library search path for dynamic libs then static libs + \\ -search_dylibs_first Search for dynamic libs in each library search path, then static libs. \\ -T[script], --script [script] Use a custom linker script \\ --version-script [path] Provide a version .map file \\ --dynamic-linker [path] Set the dynamic interpreter path (usually ld.so) @@ -537,8 +540,6 @@ const usage_build_generic = \\ -install_name=[value] (Darwin) add dylib's install name \\ --entitlements [path] (Darwin) add path to entitlements file for embedding in code signature \\ -pagezero_size [value] (Darwin) size of the __PAGEZERO segment in hexadecimal notation - \\ -search_paths_first (Darwin) search each dir in library search paths for `libx.dylib` then `libx.a` - \\ -search_dylibs_first (Darwin) search `libx.dylib` in each dir in library search paths, then `libx.a` \\ -headerpad [value] (Darwin) set minimum space for future expansion of the load commands in hexadecimal notation \\ -headerpad_max_install_names (Darwin) set enough space as if all paths were MAXPATHLEN \\ -dead_strip (Darwin) remove functions and data that are unreachable by the entry point or exported symbols @@ -2567,6 +2568,34 @@ fn buildOutputType( } lib_dir_args = undefined; // From here we use lib_dirs instead. + const self_exe_path: ?[]const u8 = if (!process.can_spawn) + null + else + introspect.findZigExePath(arena) catch |err| { + fatal("unable to find zig self exe path: {s}", .{@errorName(err)}); + }; + + var zig_lib_directory: Compilation.Directory = d: { + if (override_lib_dir) |unresolved_lib_dir| { + const lib_dir = try introspect.resolvePath(arena, unresolved_lib_dir); + break :d .{ + .path = lib_dir, + .handle = fs.cwd().openDir(lib_dir, .{}) catch |err| { + fatal("unable to open zig lib directory '{s}': {s}", .{ lib_dir, @errorName(err) }); + }, + }; + } else if (builtin.os.tag == .wasi) { + break :d getWasiPreopen("/lib"); + } else if (self_exe_path) |p| { + break :d introspect.findZigLibDirFromSelfExe(arena, p) catch |err| { + fatal("unable to find zig installation directory: {s}", .{@errorName(err)}); + }; + } else { + unreachable; + } + }; + defer zig_lib_directory.handle.close(); + // Now that we have target info, we can find out if any of the system libraries // are part of libc or libc++. We remove them from the list and communicate their // existence via flags instead. @@ -2612,6 +2641,25 @@ fn buildOutputType( }, } + if (target_info.target.os.tag == .windows) { + const exists = mingw.libExists(arena, target_info.target, zig_lib_directory, lib_name) catch |err| { + fatal("failed to check zig installation for DLL import libs: {s}", .{ + @errorName(err), + }); + }; + if (exists) { + try resolved_system_libs.append(arena, .{ + .name = lib_name, + .lib = .{ + .needed = true, + .weak = false, + .path = undefined, + }, + }); + continue; + } + } + if (fs.path.isAbsolute(lib_name)) { fatal("cannot use absolute path as a system library: {s}", .{lib_name}); } @@ -2758,9 +2806,13 @@ fn buildOutputType( if (failed_libs.items.len > 0) { for (failed_libs.items) |f| { + const searched_paths = if (f.checked_paths.len == 0) " none" else f.checked_paths; std.log.err("unable to find {s} system library '{s}' using strategy '{s}'. searched paths:{s}", .{ - @tagName(f.preferred_mode), f.name, @tagName(f.strategy), f.checked_paths, + @tagName(f.preferred_mode), f.name, @tagName(f.strategy), searched_paths, }); + if (f.preferred_mode == .Dynamic and f.strategy == .no_fallback) { + std.log.info("to link statically, pass the library as a positional argument", .{}); + } } process.exit(1); } @@ -3079,35 +3131,6 @@ fn buildOutputType( } } - const self_exe_path: ?[]const u8 = if (!process.can_spawn) - null - else - introspect.findZigExePath(arena) catch |err| { - fatal("unable to find zig self exe path: {s}", .{@errorName(err)}); - }; - - var zig_lib_directory: Compilation.Directory = d: { - if (override_lib_dir) |unresolved_lib_dir| { - const lib_dir = try introspect.resolvePath(arena, unresolved_lib_dir); - break :d .{ - .path = lib_dir, - .handle = fs.cwd().openDir(lib_dir, .{}) catch |err| { - fatal("unable to open zig lib directory '{s}': {s}", .{ lib_dir, @errorName(err) }); - }, - }; - } else if (builtin.os.tag == .wasi) { - break :d getWasiPreopen("/lib"); - } else if (self_exe_path) |p| { - break :d introspect.findZigLibDirFromSelfExe(arena, p) catch |err| { - fatal("unable to find zig installation directory: {s}", .{@errorName(err)}); - }; - } else { - unreachable; - } - }; - - defer zig_lib_directory.handle.close(); - var thread_pool: ThreadPool = undefined; try thread_pool.init(.{ .allocator = gpa }); defer thread_pool.deinit(); diff --git a/src/mingw.zig b/src/mingw.zig index 39059f804a32..f4d5ae23e236 100644 --- a/src/mingw.zig +++ b/src/mingw.zig @@ -283,7 +283,7 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8) !void { defer arena_allocator.deinit(); const arena = arena_allocator.allocator(); - const def_file_path = findDef(comp, arena, lib_name) catch |err| switch (err) { + const def_file_path = findDef(arena, comp.getTarget(), comp.zig_lib_directory, lib_name) catch |err| switch (err) { error.FileNotFound => { log.debug("no {s}.def file available to make a DLL import {s}.lib", .{ lib_name, lib_name }); // In this case we will end up putting foo.lib onto the linker line and letting the linker @@ -431,10 +431,28 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8) !void { }); } -/// This function body is verbose but all it does is test 3 different paths and see if a .def file exists. -fn findDef(comp: *Compilation, allocator: Allocator, lib_name: []const u8) ![]u8 { - const target = comp.getTarget(); +pub fn libExists( + allocator: Allocator, + target: std.Target, + zig_lib_directory: Cache.Directory, + lib_name: []const u8, +) !bool { + const s = findDef(allocator, target, zig_lib_directory, lib_name) catch |err| switch (err) { + error.FileNotFound => return false, + else => |e| return e, + }; + defer allocator.free(s); + return true; +} +/// This function body is verbose but all it does is test 3 different paths and +/// see if a .def file exists. +fn findDef( + allocator: Allocator, + target: std.Target, + zig_lib_directory: Cache.Directory, + lib_name: []const u8, +) ![]u8 { const lib_path = switch (target.cpu.arch) { .x86 => "lib32", .x86_64 => "lib64", @@ -451,7 +469,7 @@ fn findDef(comp: *Compilation, allocator: Allocator, lib_name: []const u8) ![]u8 { // Try the archtecture-specific path first. const fmt_path = "libc" ++ s ++ "mingw" ++ s ++ "{s}" ++ s ++ "{s}.def"; - if (comp.zig_lib_directory.path) |p| { + if (zig_lib_directory.path) |p| { try override_path.writer().print("{s}" ++ s ++ fmt_path, .{ p, lib_path, lib_name }); } else { try override_path.writer().print(fmt_path, .{ lib_path, lib_name }); @@ -468,7 +486,7 @@ fn findDef(comp: *Compilation, allocator: Allocator, lib_name: []const u8) ![]u8 // Try the generic version. override_path.shrinkRetainingCapacity(0); const fmt_path = "libc" ++ s ++ "mingw" ++ s ++ "lib-common" ++ s ++ "{s}.def"; - if (comp.zig_lib_directory.path) |p| { + if (zig_lib_directory.path) |p| { try override_path.writer().print("{s}" ++ s ++ fmt_path, .{ p, lib_name }); } else { try override_path.writer().print(fmt_path, .{lib_name}); @@ -485,7 +503,7 @@ fn findDef(comp: *Compilation, allocator: Allocator, lib_name: []const u8) ![]u8 // Try the generic version and preprocess it. override_path.shrinkRetainingCapacity(0); const fmt_path = "libc" ++ s ++ "mingw" ++ s ++ "lib-common" ++ s ++ "{s}.def.in"; - if (comp.zig_lib_directory.path) |p| { + if (zig_lib_directory.path) |p| { try override_path.writer().print("{s}" ++ s ++ fmt_path, .{ p, lib_name }); } else { try override_path.writer().print(fmt_path, .{lib_name}); From 89c605d140bf295e052cdded4e2c453f7a143d6f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 1 Aug 2023 18:42:37 -0700 Subject: [PATCH 03/29] CLI: add --ambiguous-static-library I hate this, gonna revert it in the next commit --- src/main.zig | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/main.zig b/src/main.zig index 797a33a20b28..907e6ab6e907 100644 --- a/src/main.zig +++ b/src/main.zig @@ -477,9 +477,15 @@ const usage_build_generic = \\ -l[lib], --library [lib] Link against system library (only if actually used) \\ -needed-l[lib], Link against system library (even if unused) \\ --needed-library [lib] + \\ --ambiguous-static-library [l] Link against system library, checking for a + \\ static lib in each search path, then checking + \\ for a dynamic lib in each search path. Use of + \\ this CLI option is a code smell. \\ -L[d], --library-directory [d] Add a directory to the library search path - \\ -search_paths_first Search each library search path for dynamic libs then static libs - \\ -search_dylibs_first Search for dynamic libs in each library search path, then static libs. + \\ -search_paths_first For each library search path, check for dynamic + \\ lib then static lib before proceeding to next path. + \\ -search_dylibs_first Search for dynamic libs in all library search + \\ paths, then static libs. \\ -T[script], --script [script] Use a custom linker script \\ --version-script [path] Provide a version .map file \\ --dynamic-linker [path] Set the dynamic interpreter path (usually ld.so) @@ -1155,6 +1161,13 @@ fn buildOutputType( .preferred_mode = lib_preferred_mode orelse .Dynamic, .search_strategy = lib_search_strategy orelse .no_fallback, }); + } else if (mem.eql(u8, arg, "--ambiguous-static-library")) { + try system_libs.put(args_iter.nextOrFatal(), .{ + .needed = false, + .weak = false, + .preferred_mode = lib_preferred_mode orelse .Static, + .search_strategy = lib_search_strategy orelse .mode_first, + }); } else if (mem.eql(u8, arg, "-D")) { try clang_argv.append(arg); try clang_argv.append(args_iter.nextOrFatal()); From 8e687cb0d994568879a98a94585562294e3dea36 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 1 Aug 2023 18:43:11 -0700 Subject: [PATCH 04/29] CLI: remove --ambiguous-static-library Reverts 6232e63ca4460a953ca8453a6f7583eb910e87c8, but keeps the usage text cleanup. Let's resist adding this as long as possible. --- src/main.zig | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/main.zig b/src/main.zig index 907e6ab6e907..11b951423743 100644 --- a/src/main.zig +++ b/src/main.zig @@ -477,10 +477,6 @@ const usage_build_generic = \\ -l[lib], --library [lib] Link against system library (only if actually used) \\ -needed-l[lib], Link against system library (even if unused) \\ --needed-library [lib] - \\ --ambiguous-static-library [l] Link against system library, checking for a - \\ static lib in each search path, then checking - \\ for a dynamic lib in each search path. Use of - \\ this CLI option is a code smell. \\ -L[d], --library-directory [d] Add a directory to the library search path \\ -search_paths_first For each library search path, check for dynamic \\ lib then static lib before proceeding to next path. @@ -1161,13 +1157,6 @@ fn buildOutputType( .preferred_mode = lib_preferred_mode orelse .Dynamic, .search_strategy = lib_search_strategy orelse .no_fallback, }); - } else if (mem.eql(u8, arg, "--ambiguous-static-library")) { - try system_libs.put(args_iter.nextOrFatal(), .{ - .needed = false, - .weak = false, - .preferred_mode = lib_preferred_mode orelse .Static, - .search_strategy = lib_search_strategy orelse .mode_first, - }); } else if (mem.eql(u8, arg, "-D")) { try clang_argv.append(arg); try clang_argv.append(args_iter.nextOrFatal()); From 5c441f7f8abe31f21e53169c05c8816d8b4c89e3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 1 Aug 2023 18:45:19 -0700 Subject: [PATCH 05/29] cmake: find LLVM libraries unambiguously Before, we would end up with the LLVM_LIBRARIES variable containing, for example, "-lLLVMAnalysis;-lz" and the Zig compiler CLI would be stuck with ambiguous library names. In this branch, that ambiguity is not allowed; it will look for a dynamically linked LLVMAnalysis and fail to find it in the case that the build of LLVM is static. With this change, LLVM_LIBRARIES will contain, for example, "/full/path/to/libLLVMAnalysis.a;-lz" and it is unambiguous that the first library should be linked statically and the second dynamically. In the case of a dynamically linked system LLVM, it will look like "/full/path/to/libLLVM.so;-lz" which again is fully unambiguous. This fixes building Zig from source. --- cmake/Findllvm.cmake | 78 ++++++++++++-------------------------------- 1 file changed, 21 insertions(+), 57 deletions(-) diff --git a/cmake/Findllvm.cmake b/cmake/Findllvm.cmake index 28e2382e93de..bbf01f9563e7 100644 --- a/cmake/Findllvm.cmake +++ b/cmake/Findllvm.cmake @@ -52,6 +52,8 @@ if(ZIG_USE_LLVM_CONFIG) set(STATIC_OR_SHARED_LINK "--link-shared") elseif (ZIG_STATIC_LLVM) set(STATIC_OR_SHARED_LINK "--link-static") + else() + set(STATIC_OR_SHARED_LINK "") endif() execute_process( @@ -103,69 +105,31 @@ if(ZIG_USE_LLVM_CONFIG) break() endwhile() - if(ZIG_SHARED_LLVM OR ZIG_STATIC_LLVM) - execute_process( - COMMAND ${LLVM_CONFIG_EXE} --libfiles ${STATIC_OR_SHARED_LINK} - OUTPUT_VARIABLE LLVM_LIBRARIES_SPACES - OUTPUT_STRIP_TRAILING_WHITESPACE) - string(REPLACE " " ";" LLVM_LIBRARIES "${LLVM_LIBRARIES_SPACES}") - execute_process( - COMMAND ${LLVM_CONFIG_EXE} --libdir ${STATIC_OR_SHARED_LINK} - OUTPUT_VARIABLE LLVM_LIBDIRS_SPACES - OUTPUT_STRIP_TRAILING_WHITESPACE) - string(REPLACE " " ";" LLVM_LIBDIRS "${LLVM_LIBDIRS_SPACES}") - - execute_process( - COMMAND ${LLVM_CONFIG_EXE} --system-libs ${STATIC_OR_SHARED_LINK} - OUTPUT_VARIABLE LLVM_SYSTEM_LIBS_SPACES - OUTPUT_STRIP_TRAILING_WHITESPACE) - string(REPLACE " " ";" LLVM_SYSTEM_LIBS "${LLVM_SYSTEM_LIBS_SPACES}") + execute_process( + COMMAND ${LLVM_CONFIG_EXE} --shared-mode ${STATIC_OR_SHARED_LINK} + OUTPUT_VARIABLE LLVM_LINK_MODE + OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process( - COMMAND ${LLVM_CONFIG_EXE} --shared-mode ${STATIC_OR_SHARED_LINK} - OUTPUT_VARIABLE LLVM_LINK_MODE + execute_process( + COMMAND ${LLVM_CONFIG_EXE} --libfiles ${STATIC_OR_SHARED_LINK} + OUTPUT_VARIABLE LLVM_LIBRARIES_SPACES OUTPUT_STRIP_TRAILING_WHITESPACE) - else() - execute_process( - COMMAND ${LLVM_CONFIG_EXE} --libs - OUTPUT_VARIABLE LLVM_LIBRARIES_SPACES - OUTPUT_STRIP_TRAILING_WHITESPACE) - string(REPLACE " " ";" LLVM_LIBRARIES "${LLVM_LIBRARIES_SPACES}") + string(REPLACE " " ";" LLVM_LIBRARIES "${LLVM_LIBRARIES_SPACES}") - execute_process( - COMMAND ${LLVM_CONFIG_EXE} --libdir - OUTPUT_VARIABLE LLVM_LIBDIRS_SPACES - OUTPUT_STRIP_TRAILING_WHITESPACE) - string(REPLACE " " ";" LLVM_LIBDIRS "${LLVM_LIBDIRS_SPACES}") - - execute_process( - COMMAND ${LLVM_CONFIG_EXE} --system-libs - OUTPUT_VARIABLE LLVM_SYSTEM_LIBS_SPACES - OUTPUT_STRIP_TRAILING_WHITESPACE) - string(REPLACE " " ";" LLVM_SYSTEM_LIBS "${LLVM_SYSTEM_LIBS_SPACES}") - - execute_process( - COMMAND ${LLVM_CONFIG_EXE} --shared-mode - OUTPUT_VARIABLE LLVM_LINK_MODE + execute_process( + COMMAND ${LLVM_CONFIG_EXE} --libdir ${STATIC_OR_SHARED_LINK} + OUTPUT_VARIABLE LLVM_LIBDIRS_SPACES OUTPUT_STRIP_TRAILING_WHITESPACE) - endif() + string(REPLACE " " ";" LLVM_LIBDIRS "${LLVM_LIBDIRS_SPACES}") - if (${LLVM_LINK_MODE} STREQUAL "shared") - # We always ask for the system libs corresponding to static linking, - # since on some distros LLD is only available as a static library - # and we need these libraries to link it successfully - execute_process( - COMMAND ${LLVM_CONFIG_EXE} --system-libs --link-static - OUTPUT_VARIABLE LLVM_STATIC_SYSTEM_LIBS_SPACES - ERROR_QUIET # Some installations have no static libs, we just ignore the failure - OUTPUT_STRIP_TRAILING_WHITESPACE) - string(REPLACE " " ";" LLVM_STATIC_SYSTEM_LIBS "${LLVM_STATIC_SYSTEM_LIBS_SPACES}") + execute_process( + COMMAND ${LLVM_CONFIG_EXE} --system-libs ${STATIC_OR_SHARED_LINK} + OUTPUT_VARIABLE LLVM_SYSTEM_LIBS_SPACES + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REPLACE " " ";" LLVM_SYSTEM_LIBS "${LLVM_SYSTEM_LIBS_SPACES}") - set(LLVM_LIBRARIES ${LLVM_LIBRARIES} ${LLVM_SYSTEM_LIBS} ${LLVM_STATIC_SYSTEM_LIBS}) - else() - set(LLVM_LIBRARIES ${LLVM_LIBRARIES} ${LLVM_SYSTEM_LIBS}) - endif() + set(LLVM_LIBRARIES ${LLVM_LIBRARIES} ${LLVM_SYSTEM_LIBS}) execute_process( COMMAND ${LLVM_CONFIG_EXE} --includedir @@ -373,4 +337,4 @@ endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args(llvm DEFAULT_MSG LLVM_LIBRARIES LLVM_INCLUDE_DIRS) -mark_as_advanced(LLVM_INCLUDE_DIRS LLVM_LIBRARIES LLVM_LIBDIRS) +mark_as_advanced(LLVM_INCLUDE_DIRS LLVM_LIBRARIES LLVM_LIBDIRS LLVM_LINK_MODE) From 256074428fc987c16fe5e1b862c6e5fc8c2123a6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 1 Aug 2023 19:09:32 -0700 Subject: [PATCH 06/29] frontend: update is_libc_lib_name for mingw-w64 crt files --- src/target.zig | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/target.zig b/src/target.zig index 3076a0b0dad4..886df5dcbeb7 100644 --- a/src/target.zig +++ b/src/target.zig @@ -366,6 +366,15 @@ pub fn is_libc_lib_name(target: std.Target, name: []const u8) bool { if (eqlIgnoreCase(ignore_case, name, "m")) return true; + if (eqlIgnoreCase(ignore_case, name, "uuid")) + return true; + if (eqlIgnoreCase(ignore_case, name, "mingw32")) + return true; + if (eqlIgnoreCase(ignore_case, name, "msvcrt-os")) + return true; + if (eqlIgnoreCase(ignore_case, name, "mingwex")) + return true; + return false; } From c65a06188173a4f1c911bfd44084a9dea57fe330 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 1 Aug 2023 19:37:50 -0700 Subject: [PATCH 07/29] CLI: adjust order of operations of system libraries First, system_libs are collected into a list. This is the same as before. Next, system_libs are filtered into external_system_libs, which is the same list but without any libc, compiler_rt, etc. At this point, if there are any external system libs, native library directory paths are detected and added to lib_dirs. Finally, extern_system_libs are filtered into resolved_system_libs, which has full paths to all of the libraries. This is the list passed into Compilation. This makes the required changes noted by @ifreund in the code review for this branch. --- src/main.zig | 211 ++++++++++++++++++++++++++++----------------------- 1 file changed, 116 insertions(+), 95 deletions(-) diff --git a/src/main.zig b/src/main.zig index 11b951423743..f05c141af666 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2554,6 +2554,24 @@ fn buildOutputType( } } + if (use_lld) |opt| { + if (opt and cross_target.isDarwin()) { + fatal("LLD requested with Mach-O object format. Only the self-hosted linker is supported for this target.", .{}); + } + } + + if (want_lto) |opt| { + if (opt and cross_target.isDarwin()) { + fatal("LTO is not yet supported with the Mach-O object format. More details: https://github.com/ziglang/zig/issues/8680", .{}); + } + } + + if (comptime builtin.target.isDarwin()) { + // If we want to link against frameworks, we need system headers. + if (framework_dirs.items.len > 0 or frameworks.count() > 0) + want_native_include_dirs = true; + } + // Resolve the library path arguments with respect to sysroot. var lib_dirs = std.ArrayList([]const u8).init(arena); if (sysroot) |root| { @@ -2598,6 +2616,100 @@ fn buildOutputType( }; defer zig_lib_directory.handle.close(); + // First, remove libc, libc++, and compiler_rt libraries from the system libraries list. + // We need to know whether the set of system libraries contains anything besides these + // to decide whether to trigger native path detection logic. + var external_system_libs: std.MultiArrayList(struct { + name: []const u8, + info: SystemLib, + }) = .{}; + for (system_libs.keys(), system_libs.values()) |lib_name, info| { + if (target_util.is_libc_lib_name(target_info.target, lib_name)) { + link_libc = true; + continue; + } + if (target_util.is_libcpp_lib_name(target_info.target, lib_name)) { + link_libcpp = true; + continue; + } + switch (target_util.classifyCompilerRtLibName(target_info.target, lib_name)) { + .none => {}, + .only_libunwind, .both => { + link_libunwind = true; + continue; + }, + .only_compiler_rt => { + std.log.warn("ignoring superfluous library '{s}': this dependency is fulfilled instead by compiler-rt which zig unconditionally provides", .{lib_name}); + continue; + }, + } + + if (fs.path.isAbsolute(lib_name)) { + fatal("cannot use absolute path as a system library: {s}", .{lib_name}); + } + + if (target_info.target.os.tag == .wasi) { + if (wasi_libc.getEmulatedLibCRTFile(lib_name)) |crt_file| { + try wasi_emulated_libs.append(crt_file); + continue; + } + } + + try external_system_libs.append(arena, .{ + .name = lib_name, + .info = info, + }); + } + // After this point, external_system_libs is used instead of system_libs. + + // libc++ depends on libc + if (link_libcpp) { + link_libc = true; + } + + // Trigger native system library path detection if necessary. + if (sysroot == null and cross_target.isNativeOs() and + (external_system_libs.len != 0 or want_native_include_dirs)) + { + const paths = std.zig.system.NativePaths.detect(arena, target_info) catch |err| { + fatal("unable to detect native system paths: {s}", .{@errorName(err)}); + }; + for (paths.warnings.items) |warning| { + warn("{s}", .{warning}); + } + + const has_sysroot = if (comptime builtin.target.isDarwin()) outer: { + if (std.zig.system.darwin.isDarwinSDKInstalled(arena)) { + const sdk = std.zig.system.darwin.getDarwinSDK(arena, target_info.target) orelse + break :outer false; + native_darwin_sdk = sdk; + try clang_argv.ensureUnusedCapacity(2); + clang_argv.appendAssumeCapacity("-isysroot"); + clang_argv.appendAssumeCapacity(sdk.path); + break :outer true; + } else break :outer false; + } else false; + + try clang_argv.ensureUnusedCapacity(paths.include_dirs.items.len * 2); + const isystem_flag = if (has_sysroot) "-iwithsysroot" else "-isystem"; + for (paths.include_dirs.items) |include_dir| { + clang_argv.appendAssumeCapacity(isystem_flag); + clang_argv.appendAssumeCapacity(include_dir); + } + + try clang_argv.ensureUnusedCapacity(paths.framework_dirs.items.len * 2); + try framework_dirs.ensureUnusedCapacity(paths.framework_dirs.items.len); + const iframework_flag = if (has_sysroot) "-iframeworkwithsysroot" else "-iframework"; + for (paths.framework_dirs.items) |framework_dir| { + clang_argv.appendAssumeCapacity(iframework_flag); + clang_argv.appendAssumeCapacity(framework_dir); + framework_dirs.appendAssumeCapacity(framework_dir); + } + + try lib_dirs.appendSlice(paths.lib_dirs.items); + try rpath_list.appendSlice(paths.rpaths.items); + } + // Now that we have target info, we can find out if any of the system libraries // are part of libc or libc++. We remove them from the list and communicate their // existence via flags instead. @@ -2622,27 +2734,7 @@ fn buildOutputType( preferred_mode: std.builtin.LinkMode, }).init(arena); - syslib: for (system_libs.keys(), system_libs.values()) |lib_name, info| { - if (target_util.is_libc_lib_name(target_info.target, lib_name)) { - link_libc = true; - continue; - } - if (target_util.is_libcpp_lib_name(target_info.target, lib_name)) { - link_libcpp = true; - continue; - } - switch (target_util.classifyCompilerRtLibName(target_info.target, lib_name)) { - .none => {}, - .only_libunwind, .both => { - link_libunwind = true; - continue; - }, - .only_compiler_rt => { - std.log.warn("ignoring superfluous library '{s}': this dependency is fulfilled instead by compiler-rt which zig unconditionally provides", .{lib_name}); - continue; - }, - } - + syslib: for (external_system_libs.items(.name), external_system_libs.items(.info)) |lib_name, info| { if (target_info.target.os.tag == .windows) { const exists = mingw.libExists(arena, target_info.target, zig_lib_directory, lib_name) catch |err| { fatal("failed to check zig installation for DLL import libs: {s}", .{ @@ -2662,16 +2754,8 @@ fn buildOutputType( } } - if (fs.path.isAbsolute(lib_name)) { - fatal("cannot use absolute path as a system library: {s}", .{lib_name}); - } - - if (target_info.target.os.tag == .wasi) { - if (wasi_libc.getEmulatedLibCRTFile(lib_name)) |crt_file| { - try wasi_emulated_libs.append(crt_file); - continue; - } - } + // Checked in the first pass above while looking for libc libraries. + assert(!fs.path.isAbsolute(lib_name)); checked_paths.clearRetainingCapacity(); @@ -2819,70 +2903,7 @@ fn buildOutputType( process.exit(1); } } - // libc++ depends on libc - if (link_libcpp) { - link_libc = true; - } - - if (use_lld) |opt| { - if (opt and cross_target.isDarwin()) { - fatal("LLD requested with Mach-O object format. Only the self-hosted linker is supported for this target.", .{}); - } - } - - if (want_lto) |opt| { - if (opt and cross_target.isDarwin()) { - fatal("LTO is not yet supported with the Mach-O object format. More details: https://github.com/ziglang/zig/issues/8680", .{}); - } - } - - if (comptime builtin.target.isDarwin()) { - // If we want to link against frameworks, we need system headers. - if (framework_dirs.items.len > 0 or frameworks.count() > 0) - want_native_include_dirs = true; - } - - if (sysroot == null and cross_target.isNativeOs() and - (resolved_system_libs.len != 0 or want_native_include_dirs)) - { - const paths = std.zig.system.NativePaths.detect(arena, target_info) catch |err| { - fatal("unable to detect native system paths: {s}", .{@errorName(err)}); - }; - for (paths.warnings.items) |warning| { - warn("{s}", .{warning}); - } - - const has_sysroot = if (comptime builtin.target.isDarwin()) outer: { - if (std.zig.system.darwin.isDarwinSDKInstalled(arena)) { - const sdk = std.zig.system.darwin.getDarwinSDK(arena, target_info.target) orelse - break :outer false; - native_darwin_sdk = sdk; - try clang_argv.ensureUnusedCapacity(2); - clang_argv.appendAssumeCapacity("-isysroot"); - clang_argv.appendAssumeCapacity(sdk.path); - break :outer true; - } else break :outer false; - } else false; - - try clang_argv.ensureUnusedCapacity(paths.include_dirs.items.len * 2); - const isystem_flag = if (has_sysroot) "-iwithsysroot" else "-isystem"; - for (paths.include_dirs.items) |include_dir| { - clang_argv.appendAssumeCapacity(isystem_flag); - clang_argv.appendAssumeCapacity(include_dir); - } - - try clang_argv.ensureUnusedCapacity(paths.framework_dirs.items.len * 2); - try framework_dirs.ensureUnusedCapacity(paths.framework_dirs.items.len); - const iframework_flag = if (has_sysroot) "-iframeworkwithsysroot" else "-iframework"; - for (paths.framework_dirs.items) |framework_dir| { - clang_argv.appendAssumeCapacity(iframework_flag); - clang_argv.appendAssumeCapacity(framework_dir); - framework_dirs.appendAssumeCapacity(framework_dir); - } - - try lib_dirs.appendSlice(paths.lib_dirs.items); - try rpath_list.appendSlice(paths.rpaths.items); - } + // After this point, resolved_system_libs is used instead of external_system_libs. const object_format = target_info.target.ofmt; From e477352af3fdd34e1ff1fb0d8167ba3f7333b3b9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 1 Aug 2023 20:30:34 -0700 Subject: [PATCH 08/29] CLI: Windows: system DLLs no longer trigger native paths detection --- src/main.zig | 49 +++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/main.zig b/src/main.zig index f05c141af666..bd306fa49ec8 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2623,6 +2623,12 @@ fn buildOutputType( name: []const u8, info: SystemLib, }) = .{}; + + var resolved_system_libs: std.MultiArrayList(struct { + name: []const u8, + lib: Compilation.SystemLib, + }) = .{}; + for (system_libs.keys(), system_libs.values()) |lib_name, info| { if (target_util.is_libc_lib_name(target_info.target, lib_name)) { link_libc = true; @@ -2644,6 +2650,25 @@ fn buildOutputType( }, } + if (target_info.target.os.tag == .windows) { + const exists = mingw.libExists(arena, target_info.target, zig_lib_directory, lib_name) catch |err| { + fatal("failed to check zig installation for DLL import libs: {s}", .{ + @errorName(err), + }); + }; + if (exists) { + try resolved_system_libs.append(arena, .{ + .name = lib_name, + .lib = .{ + .needed = true, + .weak = false, + .path = undefined, + }, + }); + continue; + } + } + if (fs.path.isAbsolute(lib_name)) { fatal("cannot use absolute path as a system library: {s}", .{lib_name}); } @@ -2715,11 +2740,6 @@ fn buildOutputType( // existence via flags instead. // Similarly, if any libs in this list are statically provided, we omit // them from the resolved list and populate the link_objects array instead. - var resolved_system_libs: std.MultiArrayList(struct { - name: []const u8, - lib: Compilation.SystemLib, - }) = .{}; - { var test_path = std.ArrayList(u8).init(gpa); defer test_path.deinit(); @@ -2735,25 +2755,6 @@ fn buildOutputType( }).init(arena); syslib: for (external_system_libs.items(.name), external_system_libs.items(.info)) |lib_name, info| { - if (target_info.target.os.tag == .windows) { - const exists = mingw.libExists(arena, target_info.target, zig_lib_directory, lib_name) catch |err| { - fatal("failed to check zig installation for DLL import libs: {s}", .{ - @errorName(err), - }); - }; - if (exists) { - try resolved_system_libs.append(arena, .{ - .name = lib_name, - .lib = .{ - .needed = true, - .weak = false, - .path = undefined, - }, - }); - continue; - } - } - // Checked in the first pass above while looking for libc libraries. assert(!fs.path.isAbsolute(lib_name)); From 2725c20a723f7c2eca455a081a4d499723a21ac4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 1 Aug 2023 20:34:35 -0700 Subject: [PATCH 09/29] coff: always link uuid.lib as part of mingw libc --- src/link/Coff/lld.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/src/link/Coff/lld.zig b/src/link/Coff/lld.zig index fe2055027f8b..bd82254e6aed 100644 --- a/src/link/Coff/lld.zig +++ b/src/link/Coff/lld.zig @@ -405,6 +405,7 @@ pub fn linkWithLLD(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod try argv.append(try comp.get_libc_crt_file(arena, "mingw32.lib")); try argv.append(try comp.get_libc_crt_file(arena, "mingwex.lib")); try argv.append(try comp.get_libc_crt_file(arena, "msvcrt-os.lib")); + try argv.append(try comp.get_libc_crt_file(arena, "uuid.lib")); for (mingw.always_link_libs) |name| { if (!self.base.options.system_libs.contains(name)) { From 9e50f960875601dbaaf7245cedb7d9e429678aeb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 1 Aug 2023 21:09:12 -0700 Subject: [PATCH 10/29] MachO: put system_libs into libs not positionals Putting them in both causes collisions because the same lib ends up being linked in twice. Putting them in positionals instead of libs makes their properties about needed and weak being ignored. --- src/link/MachO/zld.zig | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/link/MachO/zld.zig b/src/link/MachO/zld.zig index c2a814b319db..40cf9b217a91 100644 --- a/src/link/MachO/zld.zig +++ b/src/link/MachO/zld.zig @@ -3549,17 +3549,12 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr try positionals.append(comp.libcxx_static_lib.?.full_object_path); } - { - // Add all system library paths to positionals. - const vals = options.system_libs.values(); - try positionals.ensureUnusedCapacity(vals.len); - for (vals) |info| positionals.appendAssumeCapacity(info.path); - } - var libs = std.StringArrayHashMap(link.SystemLib).init(arena); - for (options.system_libs.values()) |v| { - try libs.put(v.path, v); + { + const vals = options.system_libs.values(); + try libs.ensureUnusedCapacity(vals.len); + for (vals) |v| libs.putAssumeCapacity(v.path, v); } try MachO.resolveLibSystem(arena, comp, options.sysroot, target, options.lib_dirs, &libs); From e565ff305ae208b15058a462d348fde515b3e950 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 1 Aug 2023 23:14:50 -0700 Subject: [PATCH 11/29] CLI: revert -l behavior chicken out and make -l match the status quo behavior, where it looks for dynamic libraries and then falls back to static libraries. library resolution is still done in the CLI now though, and these options are added: -search_static_first Search for static libs in all library search paths, then dynamic libs. -search_dylibs_only Only search for dynamic libs. -search_static_only Only search for static libs. this matches the already existing options below: -search_paths_first For each library search path, check for dynamic lib then static lib before proceeding to next path. -search_dylibs_first Search for dynamic libs in all library search So, it is still possible to get the strict behavior by passing `-search_dylibs_only` or `-search_static_only`. This commit also makes -dynamic and -static affect the preferred link mode and search strategy. --- cmake/Findllvm.cmake | 78 +++++++++++++++++++++++++++++------------ src/main.zig | 82 ++++++++++++++++++++++++-------------------- 2 files changed, 102 insertions(+), 58 deletions(-) diff --git a/cmake/Findllvm.cmake b/cmake/Findllvm.cmake index bbf01f9563e7..28e2382e93de 100644 --- a/cmake/Findllvm.cmake +++ b/cmake/Findllvm.cmake @@ -52,8 +52,6 @@ if(ZIG_USE_LLVM_CONFIG) set(STATIC_OR_SHARED_LINK "--link-shared") elseif (ZIG_STATIC_LLVM) set(STATIC_OR_SHARED_LINK "--link-static") - else() - set(STATIC_OR_SHARED_LINK "") endif() execute_process( @@ -105,31 +103,69 @@ if(ZIG_USE_LLVM_CONFIG) break() endwhile() + if(ZIG_SHARED_LLVM OR ZIG_STATIC_LLVM) + execute_process( + COMMAND ${LLVM_CONFIG_EXE} --libfiles ${STATIC_OR_SHARED_LINK} + OUTPUT_VARIABLE LLVM_LIBRARIES_SPACES + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REPLACE " " ";" LLVM_LIBRARIES "${LLVM_LIBRARIES_SPACES}") - execute_process( - COMMAND ${LLVM_CONFIG_EXE} --shared-mode ${STATIC_OR_SHARED_LINK} - OUTPUT_VARIABLE LLVM_LINK_MODE - OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process( + COMMAND ${LLVM_CONFIG_EXE} --libdir ${STATIC_OR_SHARED_LINK} + OUTPUT_VARIABLE LLVM_LIBDIRS_SPACES + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REPLACE " " ";" LLVM_LIBDIRS "${LLVM_LIBDIRS_SPACES}") - execute_process( - COMMAND ${LLVM_CONFIG_EXE} --libfiles ${STATIC_OR_SHARED_LINK} - OUTPUT_VARIABLE LLVM_LIBRARIES_SPACES - OUTPUT_STRIP_TRAILING_WHITESPACE) - string(REPLACE " " ";" LLVM_LIBRARIES "${LLVM_LIBRARIES_SPACES}") + execute_process( + COMMAND ${LLVM_CONFIG_EXE} --system-libs ${STATIC_OR_SHARED_LINK} + OUTPUT_VARIABLE LLVM_SYSTEM_LIBS_SPACES + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REPLACE " " ";" LLVM_SYSTEM_LIBS "${LLVM_SYSTEM_LIBS_SPACES}") - execute_process( - COMMAND ${LLVM_CONFIG_EXE} --libdir ${STATIC_OR_SHARED_LINK} - OUTPUT_VARIABLE LLVM_LIBDIRS_SPACES + execute_process( + COMMAND ${LLVM_CONFIG_EXE} --shared-mode ${STATIC_OR_SHARED_LINK} + OUTPUT_VARIABLE LLVM_LINK_MODE OUTPUT_STRIP_TRAILING_WHITESPACE) - string(REPLACE " " ";" LLVM_LIBDIRS "${LLVM_LIBDIRS_SPACES}") + else() + execute_process( + COMMAND ${LLVM_CONFIG_EXE} --libs + OUTPUT_VARIABLE LLVM_LIBRARIES_SPACES + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REPLACE " " ";" LLVM_LIBRARIES "${LLVM_LIBRARIES_SPACES}") - execute_process( - COMMAND ${LLVM_CONFIG_EXE} --system-libs ${STATIC_OR_SHARED_LINK} - OUTPUT_VARIABLE LLVM_SYSTEM_LIBS_SPACES + execute_process( + COMMAND ${LLVM_CONFIG_EXE} --libdir + OUTPUT_VARIABLE LLVM_LIBDIRS_SPACES + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REPLACE " " ";" LLVM_LIBDIRS "${LLVM_LIBDIRS_SPACES}") + + execute_process( + COMMAND ${LLVM_CONFIG_EXE} --system-libs + OUTPUT_VARIABLE LLVM_SYSTEM_LIBS_SPACES + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REPLACE " " ";" LLVM_SYSTEM_LIBS "${LLVM_SYSTEM_LIBS_SPACES}") + + execute_process( + COMMAND ${LLVM_CONFIG_EXE} --shared-mode + OUTPUT_VARIABLE LLVM_LINK_MODE OUTPUT_STRIP_TRAILING_WHITESPACE) - string(REPLACE " " ";" LLVM_SYSTEM_LIBS "${LLVM_SYSTEM_LIBS_SPACES}") + endif() + + if (${LLVM_LINK_MODE} STREQUAL "shared") + # We always ask for the system libs corresponding to static linking, + # since on some distros LLD is only available as a static library + # and we need these libraries to link it successfully + execute_process( + COMMAND ${LLVM_CONFIG_EXE} --system-libs --link-static + OUTPUT_VARIABLE LLVM_STATIC_SYSTEM_LIBS_SPACES + ERROR_QUIET # Some installations have no static libs, we just ignore the failure + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REPLACE " " ";" LLVM_STATIC_SYSTEM_LIBS "${LLVM_STATIC_SYSTEM_LIBS_SPACES}") - set(LLVM_LIBRARIES ${LLVM_LIBRARIES} ${LLVM_SYSTEM_LIBS}) + set(LLVM_LIBRARIES ${LLVM_LIBRARIES} ${LLVM_SYSTEM_LIBS} ${LLVM_STATIC_SYSTEM_LIBS}) + else() + set(LLVM_LIBRARIES ${LLVM_LIBRARIES} ${LLVM_SYSTEM_LIBS}) + endif() execute_process( COMMAND ${LLVM_CONFIG_EXE} --includedir @@ -337,4 +373,4 @@ endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args(llvm DEFAULT_MSG LLVM_LIBRARIES LLVM_INCLUDE_DIRS) -mark_as_advanced(LLVM_INCLUDE_DIRS LLVM_LIBRARIES LLVM_LIBDIRS LLVM_LINK_MODE) +mark_as_advanced(LLVM_INCLUDE_DIRS LLVM_LIBRARIES LLVM_LIBDIRS) diff --git a/src/main.zig b/src/main.zig index bd306fa49ec8..8327659a6f9a 100644 --- a/src/main.zig +++ b/src/main.zig @@ -482,6 +482,10 @@ const usage_build_generic = \\ lib then static lib before proceeding to next path. \\ -search_dylibs_first Search for dynamic libs in all library search \\ paths, then static libs. + \\ -search_static_first Search for static libs in all library search + \\ paths, then dynamic libs. + \\ -search_dylibs_only Only search for dynamic libs. + \\ -search_static_only Only search for static libs. \\ -T[script], --script [script] Use a custom linker script \\ --version-script [path] Provide a version .map file \\ --dynamic-linker [path] Set the dynamic interpreter path (usually ld.so) @@ -878,8 +882,8 @@ fn buildOutputType( var hash_style: link.HashStyle = .both; var entitlements: ?[]const u8 = null; var pagezero_size: ?u64 = null; - var lib_search_strategy: ?SystemLib.SearchStrategy = null; - var lib_preferred_mode: ?std.builtin.LinkMode = null; + var lib_search_strategy: SystemLib.SearchStrategy = .paths_first; + var lib_preferred_mode: std.builtin.LinkMode = .Dynamic; var headerpad_size: ?u32 = null; var headerpad_max_install_names: bool = false; var dead_strip_dylibs: bool = false; @@ -1111,6 +1115,15 @@ fn buildOutputType( } else if (mem.eql(u8, arg, "-search_dylibs_first")) { lib_search_strategy = .mode_first; lib_preferred_mode = .Dynamic; + } else if (mem.eql(u8, arg, "-search_static_first")) { + lib_search_strategy = .mode_first; + lib_preferred_mode = .Static; + } else if (mem.eql(u8, arg, "-search_dylibs_only")) { + lib_search_strategy = .no_fallback; + lib_preferred_mode = .Dynamic; + } else if (mem.eql(u8, arg, "-search_static_only")) { + lib_search_strategy = .no_fallback; + lib_preferred_mode = .Static; } else if (mem.eql(u8, arg, "-headerpad")) { const next_arg = args_iter.nextOrFatal(); headerpad_size = std.fmt.parseUnsigned(u32, eatIntPrefix(next_arg, 16), 16) catch |err| { @@ -1136,8 +1149,8 @@ fn buildOutputType( // -l always dynamic links. For static libraries, // users are expected to use positional arguments // which are always unambiguous. - .preferred_mode = lib_preferred_mode orelse .Dynamic, - .search_strategy = lib_search_strategy orelse .no_fallback, + .preferred_mode = lib_preferred_mode, + .search_strategy = lib_search_strategy, }); } else if (mem.eql(u8, arg, "--needed-library") or mem.eql(u8, arg, "-needed-l") or @@ -1147,15 +1160,15 @@ fn buildOutputType( try system_libs.put(next_arg, .{ .needed = true, .weak = false, - .preferred_mode = lib_preferred_mode orelse .Dynamic, - .search_strategy = lib_search_strategy orelse .no_fallback, + .preferred_mode = lib_preferred_mode, + .search_strategy = lib_search_strategy, }); } else if (mem.eql(u8, arg, "-weak_library") or mem.eql(u8, arg, "-weak-l")) { try system_libs.put(args_iter.nextOrFatal(), .{ .needed = false, .weak = true, - .preferred_mode = lib_preferred_mode orelse .Dynamic, - .search_strategy = lib_search_strategy orelse .no_fallback, + .preferred_mode = lib_preferred_mode, + .search_strategy = lib_search_strategy, }); } else if (mem.eql(u8, arg, "-D")) { try clang_argv.append(arg); @@ -1388,8 +1401,12 @@ fn buildOutputType( emit_implib_arg_provided = true; } else if (mem.eql(u8, arg, "-dynamic")) { link_mode = .Dynamic; + lib_preferred_mode = .Dynamic; + lib_search_strategy = .mode_first; } else if (mem.eql(u8, arg, "-static")) { link_mode = .Static; + lib_preferred_mode = .Static; + lib_search_strategy = .no_fallback; } else if (mem.eql(u8, arg, "-fdll-export-fns")) { dll_export_fns = true; } else if (mem.eql(u8, arg, "-fno-dll-export-fns")) { @@ -1541,22 +1558,22 @@ fn buildOutputType( // -l always dynamic links. For static libraries, // users are expected to use positional arguments // which are always unambiguous. - .preferred_mode = lib_preferred_mode orelse .Dynamic, - .search_strategy = lib_search_strategy orelse .no_fallback, + .preferred_mode = lib_preferred_mode, + .search_strategy = lib_search_strategy, }); } else if (mem.startsWith(u8, arg, "-needed-l")) { try system_libs.put(arg["-needed-l".len..], .{ .needed = true, .weak = false, - .preferred_mode = lib_preferred_mode orelse .Dynamic, - .search_strategy = lib_search_strategy orelse .no_fallback, + .preferred_mode = lib_preferred_mode, + .search_strategy = lib_search_strategy, }); } else if (mem.startsWith(u8, arg, "-weak-l")) { try system_libs.put(arg["-weak-l".len..], .{ .needed = false, .weak = true, - .preferred_mode = lib_preferred_mode orelse .Dynamic, - .search_strategy = lib_search_strategy orelse .no_fallback, + .preferred_mode = lib_preferred_mode, + .search_strategy = lib_search_strategy, }); } else if (mem.startsWith(u8, arg, "-D")) { try clang_argv.append(arg); @@ -1632,7 +1649,6 @@ fn buildOutputType( var emit_llvm = false; var needed = false; var must_link = false; - var force_static_libs = false; var file_ext: ?Compilation.FileExt = null; while (it.has_next) { it.next() catch |err| { @@ -1702,22 +1718,12 @@ fn buildOutputType( .must_link = must_link, .loption = true, }); - } else if (force_static_libs) { - try system_libs.put(it.only_arg, .{ - .needed = false, - .weak = false, - .preferred_mode = .Static, - .search_strategy = .no_fallback, - }); } else { - // C compilers are traditionally expected to look - // first for dynamic libraries and then fall back - // to static libraries. try system_libs.put(it.only_arg, .{ .needed = needed, .weak = false, - .preferred_mode = lib_preferred_mode orelse .Dynamic, - .search_strategy = lib_search_strategy orelse .paths_first, + .preferred_mode = lib_preferred_mode, + .search_strategy = lib_search_strategy, }); } }, @@ -1814,13 +1820,15 @@ fn buildOutputType( mem.eql(u8, linker_arg, "-dy") or mem.eql(u8, linker_arg, "-call_shared")) { - force_static_libs = false; + lib_search_strategy = .no_fallback; + lib_preferred_mode = .Dynamic; } else if (mem.eql(u8, linker_arg, "-Bstatic") or mem.eql(u8, linker_arg, "-dn") or mem.eql(u8, linker_arg, "-non_shared") or mem.eql(u8, linker_arg, "-static")) { - force_static_libs = true; + lib_search_strategy = .no_fallback; + lib_preferred_mode = .Static; } else if (mem.eql(u8, linker_arg, "-search_paths_first")) { lib_search_strategy = .paths_first; lib_preferred_mode = .Dynamic; @@ -1939,8 +1947,8 @@ fn buildOutputType( .weak_library => try system_libs.put(it.only_arg, .{ .needed = false, .weak = true, - .preferred_mode = lib_preferred_mode orelse .Dynamic, - .search_strategy = lib_search_strategy orelse .paths_first, + .preferred_mode = lib_preferred_mode, + .search_strategy = lib_search_strategy, }), .weak_framework => try frameworks.put(gpa, it.only_arg, .{ .weak = true }), .headerpad_max_install_names => headerpad_max_install_names = true, @@ -2240,22 +2248,22 @@ fn buildOutputType( try system_libs.put(linker_args_it.nextOrFatal(), .{ .weak = false, .needed = true, - .preferred_mode = lib_preferred_mode orelse .Dynamic, - .search_strategy = lib_search_strategy orelse .paths_first, + .preferred_mode = lib_preferred_mode, + .search_strategy = lib_search_strategy, }); } else if (mem.startsWith(u8, arg, "-weak-l")) { try system_libs.put(arg["-weak-l".len..], .{ .weak = true, .needed = false, - .preferred_mode = lib_preferred_mode orelse .Dynamic, - .search_strategy = lib_search_strategy orelse .paths_first, + .preferred_mode = lib_preferred_mode, + .search_strategy = lib_search_strategy, }); } else if (mem.eql(u8, arg, "-weak_library")) { try system_libs.put(linker_args_it.nextOrFatal(), .{ .weak = true, .needed = false, - .preferred_mode = lib_preferred_mode orelse .Dynamic, - .search_strategy = lib_search_strategy orelse .paths_first, + .preferred_mode = lib_preferred_mode, + .search_strategy = lib_search_strategy, }); } else if (mem.eql(u8, arg, "-compatibility_version")) { const compat_version = linker_args_it.nextOrFatal(); From 986a3d23abcbdb61fd733d2340d4872b6ee55dca Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 1 Aug 2023 23:48:14 -0700 Subject: [PATCH 12/29] frontend: make SystemLib.path optional This can be null in two cases right now: 1. Windows DLLs that zig ships such as advapi32. 2. extern "foo" fn declarations where we find out about libraries too late TODO: make this non-optional and resolve those two cases somehow. --- src/Compilation.zig | 4 ++-- src/link.zig | 8 ++++++-- src/link/Elf.zig | 2 +- src/link/MachO/zld.zig | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 0c8b3c524866..91b31c60041f 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1728,7 +1728,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { try comp.bin_file.options.system_libs.put(comp.gpa, name, .{ .needed = false, .weak = false, - .path = name, + .path = null, }); } } @@ -5621,7 +5621,7 @@ pub fn addLinkLib(comp: *Compilation, lib_name: []const u8) !void { gop.value_ptr.* = .{ .needed = true, .weak = false, - .path = undefined, + .path = null, }; try comp.work_queue.writeItem(.{ .windows_import_lib = comp.bin_file.options.system_libs.count() - 1, diff --git a/src/link.zig b/src/link.zig index 9edba5012359..6e5921d815ba 100644 --- a/src/link.zig +++ b/src/link.zig @@ -26,7 +26,11 @@ const TypedValue = @import("TypedValue.zig"); pub const SystemLib = struct { needed: bool, weak: bool, - path: []const u8, + /// This can be null in two cases right now: + /// 1. Windows DLLs that zig ships such as advapi32. + /// 2. extern "foo" fn declarations where we find out about libraries too late + /// TODO: make this non-optional and resolve those two cases somehow. + path: ?[]const u8, }; /// When adding a new field, remember to update `hashAddFrameworks`. @@ -48,7 +52,7 @@ pub fn hashAddSystemLibs( for (hm.values()) |value| { man.hash.add(value.needed); man.hash.add(value.weak); - _ = try man.addFile(value.path, null); + if (value.path) |p| _ = try man.addFile(p, null); } } diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 3292168bce84..dd88d47fab23 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1842,7 +1842,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v // libraries and not static libraries (the check for that needs to be earlier), // but they could be full paths to .so files, in which case we // want to avoid prepending "-l". - argv.appendAssumeCapacity(lib_info.path); + argv.appendAssumeCapacity(lib_info.path.?); } if (!as_needed) { diff --git a/src/link/MachO/zld.zig b/src/link/MachO/zld.zig index 40cf9b217a91..56eee2b5463e 100644 --- a/src/link/MachO/zld.zig +++ b/src/link/MachO/zld.zig @@ -3554,7 +3554,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr { const vals = options.system_libs.values(); try libs.ensureUnusedCapacity(vals.len); - for (vals) |v| libs.putAssumeCapacity(v.path, v); + for (vals) |v| libs.putAssumeCapacity(v.path.?, v); } try MachO.resolveLibSystem(arena, comp, options.sysroot, target, options.lib_dirs, &libs); From a0e94ec576428e3b9f003c95e6538cb57f029263 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 11:04:59 -0700 Subject: [PATCH 13/29] CLI: add -search_paths_first_static to complete the API There was no previous way to set preferred mode static, search strategy paths_first. --- src/main.zig | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main.zig b/src/main.zig index 8327659a6f9a..1de6ac7449e2 100644 --- a/src/main.zig +++ b/src/main.zig @@ -477,9 +477,13 @@ const usage_build_generic = \\ -l[lib], --library [lib] Link against system library (only if actually used) \\ -needed-l[lib], Link against system library (even if unused) \\ --needed-library [lib] + \\ -weak-l[lib] link against system library marking it and all + \\ -weak_library [lib] referenced symbols as weak \\ -L[d], --library-directory [d] Add a directory to the library search path \\ -search_paths_first For each library search path, check for dynamic \\ lib then static lib before proceeding to next path. + \\ -search_paths_first_static For each library search path, check for static + \\ lib then dynamic lib before proceeding to next path. \\ -search_dylibs_first Search for dynamic libs in all library search \\ paths, then static libs. \\ -search_static_first Search for static libs in all library search @@ -536,8 +540,6 @@ const usage_build_generic = \\ --subsystem [subsystem] (Windows) /SUBSYSTEM: to the linker \\ --stack [size] Override default stack size \\ --image-base [addr] Set base address for executable image - \\ -weak-l[lib] link against system library and mark it and all referenced symbols as weak - \\ -weak_library [lib] \\ -framework [name] (Darwin) link against framework \\ -needed_framework [name] (Darwin) link against framework (even if unused) \\ -needed_library [lib] link against system library (even if unused) @@ -1112,6 +1114,9 @@ fn buildOutputType( } else if (mem.eql(u8, arg, "-search_paths_first")) { lib_search_strategy = .paths_first; lib_preferred_mode = .Dynamic; + } else if (mem.eql(u8, arg, "-search_paths_first_static")) { + lib_search_strategy = .paths_first; + lib_preferred_mode = .Static; } else if (mem.eql(u8, arg, "-search_dylibs_first")) { lib_search_strategy = .mode_first; lib_preferred_mode = .Dynamic; From aef8bcf7763cd098201d6b4b4c3e9e447188ae43 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 11:05:35 -0700 Subject: [PATCH 14/29] std.Build.Step.Compile: fine-grained system lib search control For each library you can specify the preferred mode and search strategy. The old way of setting global state is eliminated. --- lib/std/Build/Step/Compile.zig | 139 ++++++++++------------ test/link/macho/search_strategy/build.zig | 12 +- 2 files changed, 70 insertions(+), 81 deletions(-) diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 89cc2ecfd9c1..335a7e2df774 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -149,13 +149,6 @@ entitlements: ?[]const u8 = null, /// (Darwin) Size of the pagezero segment. pagezero_size: ?u64 = null, -/// (Darwin) Search strategy for searching system libraries. Either `paths_first` or `dylibs_first`. -/// The former lowers to `-search_paths_first` linker option, while the latter to `-search_dylibs_first` -/// option. -/// By default, if no option is specified, the linker assumes `paths_first` as the default -/// search strategy. -search_strategy: ?enum { paths_first, dylibs_first } = null, - /// (Darwin) Set size of the padding between the end of load commands /// and start of `__TEXT,__text` section. headerpad_size: ?u32 = null, @@ -242,7 +235,11 @@ pub const SystemLib = struct { name: []const u8, needed: bool, weak: bool, - use_pkg_config: enum { + use_pkg_config: UsePkgConfig, + preferred_link_mode: std.builtin.LinkMode, + search_strategy: SystemLib.SearchStrategy, + + pub const UsePkgConfig = enum { /// Don't use pkg-config, just pass -lfoo where foo is name. no, /// Try to get information on how to link the library from pkg-config. @@ -251,7 +248,9 @@ pub const SystemLib = struct { /// Try to get information on how to link the library from pkg-config. /// If that fails, error out. force, - }, + }; + + pub const SearchStrategy = enum { paths_first, mode_first, no_fallback }; }; const FrameworkLinkInfo = struct { @@ -718,74 +717,29 @@ pub fn defineCMacroRaw(self: *Compile, name_and_value: []const u8) void { self.c_macros.append(b.dupe(name_and_value)) catch @panic("OOM"); } -/// This one has no integration with anything, it just puts -lname on the command line. -/// Prefer to use `linkSystemLibrary` instead. +/// deprecated: use linkSystemLibrary2 pub fn linkSystemLibraryName(self: *Compile, name: []const u8) void { - const b = self.step.owner; - self.link_objects.append(.{ - .system_lib = .{ - .name = b.dupe(name), - .needed = false, - .weak = false, - .use_pkg_config = .no, - }, - }) catch @panic("OOM"); + return linkSystemLibrary2(self, name, .{ .use_pkg_config = .no }); } -/// This one has no integration with anything, it just puts -needed-lname on the command line. -/// Prefer to use `linkSystemLibraryNeeded` instead. +/// deprecated: use linkSystemLibrary2 pub fn linkSystemLibraryNeededName(self: *Compile, name: []const u8) void { - const b = self.step.owner; - self.link_objects.append(.{ - .system_lib = .{ - .name = b.dupe(name), - .needed = true, - .weak = false, - .use_pkg_config = .no, - }, - }) catch @panic("OOM"); + return linkSystemLibrary2(self, name, .{ .needed = true, .use_pkg_config = .no }); } -/// Darwin-only. This one has no integration with anything, it just puts -weak-lname on the -/// command line. Prefer to use `linkSystemLibraryWeak` instead. +/// deprecated: use linkSystemLibrary2 pub fn linkSystemLibraryWeakName(self: *Compile, name: []const u8) void { - const b = self.step.owner; - self.link_objects.append(.{ - .system_lib = .{ - .name = b.dupe(name), - .needed = false, - .weak = true, - .use_pkg_config = .no, - }, - }) catch @panic("OOM"); + return linkSystemLibrary2(self, name, .{ .weak = true, .use_pkg_config = .no }); } -/// This links against a system library, exclusively using pkg-config to find the library. -/// Prefer to use `linkSystemLibrary` instead. +/// deprecated: use linkSystemLibrary2 pub fn linkSystemLibraryPkgConfigOnly(self: *Compile, lib_name: []const u8) void { - const b = self.step.owner; - self.link_objects.append(.{ - .system_lib = .{ - .name = b.dupe(lib_name), - .needed = false, - .weak = false, - .use_pkg_config = .force, - }, - }) catch @panic("OOM"); + return linkSystemLibrary2(self, lib_name, .{ .use_pkg_config = .force }); } -/// This links against a system library, exclusively using pkg-config to find the library. -/// Prefer to use `linkSystemLibraryNeeded` instead. +/// deprecated: use linkSystemLibrary2 pub fn linkSystemLibraryNeededPkgConfigOnly(self: *Compile, lib_name: []const u8) void { - const b = self.step.owner; - self.link_objects.append(.{ - .system_lib = .{ - .name = b.dupe(lib_name), - .needed = true, - .weak = false, - .use_pkg_config = .force, - }, - }) catch @panic("OOM"); + return linkSystemLibrary2(self, lib_name, .{ .needed = true, .use_pkg_config = .force }); } /// Run pkg-config for the given library name and parse the output, returning the arguments @@ -885,21 +839,32 @@ fn runPkgConfig(self: *Compile, lib_name: []const u8) ![]const []const u8 { } pub fn linkSystemLibrary(self: *Compile, name: []const u8) void { - self.linkSystemLibraryInner(name, .{}); + self.linkSystemLibrary2(name, .{}); } +/// deprecated: use linkSystemLibrary2 pub fn linkSystemLibraryNeeded(self: *Compile, name: []const u8) void { - self.linkSystemLibraryInner(name, .{ .needed = true }); + return linkSystemLibrary2(self, name, .{ .needed = true }); } +/// deprecated: use linkSystemLibrary2 pub fn linkSystemLibraryWeak(self: *Compile, name: []const u8) void { - self.linkSystemLibraryInner(name, .{ .weak = true }); + return linkSystemLibrary2(self, name, .{ .weak = true }); } -fn linkSystemLibraryInner(self: *Compile, name: []const u8, opts: struct { +pub const LinkSystemLibraryOptions = struct { needed: bool = false, weak: bool = false, -}) void { + use_pkg_config: SystemLib.UsePkgConfig = .yes, + preferred_link_mode: std.builtin.LinkMode = .Dynamic, + search_strategy: SystemLib.SearchStrategy = .paths_first, +}; + +pub fn linkSystemLibrary2( + self: *Compile, + name: []const u8, + options: LinkSystemLibraryOptions, +) void { const b = self.step.owner; if (isLibCLibrary(name)) { self.linkLibC(); @@ -913,9 +878,11 @@ fn linkSystemLibraryInner(self: *Compile, name: []const u8, opts: struct { self.link_objects.append(.{ .system_lib = .{ .name = b.dupe(name), - .needed = opts.needed, - .weak = opts.weak, - .use_pkg_config = .yes, + .needed = options.needed, + .weak = options.weak, + .use_pkg_config = options.use_pkg_config, + .preferred_link_mode = options.preferred_link_mode, + .search_strategy = options.search_strategy, }, }) catch @panic("OOM"); } @@ -1385,6 +1352,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try transitive_deps.add(self.link_objects.items); var prev_has_cflags = false; + var prev_search_strategy: SystemLib.SearchStrategy = .paths_first; + var prev_preferred_link_mode: std.builtin.LinkMode = .Dynamic; for (transitive_deps.link_objects.items) |link_object| { switch (link_object) { @@ -1420,6 +1389,28 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { }, .system_lib => |system_lib| { + if ((system_lib.search_strategy != prev_search_strategy or + system_lib.preferred_link_mode != prev_preferred_link_mode) and + self.linkage != .static) + { + switch (system_lib.search_strategy) { + .no_fallback => switch (system_lib.preferred_link_mode) { + .Dynamic => try zig_args.append("-search_dylibs_only"), + .Static => try zig_args.append("-search_static_only"), + }, + .paths_first => switch (system_lib.preferred_link_mode) { + .Dynamic => try zig_args.append("-search_paths_first"), + .Static => try zig_args.append("-search_paths_first_static"), + }, + .mode_first => switch (system_lib.preferred_link_mode) { + .Dynamic => try zig_args.append("-search_dylibs_first"), + .Static => try zig_args.append("-search_static_first"), + }, + } + prev_search_strategy = system_lib.search_strategy; + prev_preferred_link_mode = system_lib.preferred_link_mode; + } + const prefix: []const u8 = prefix: { if (system_lib.needed) break :prefix "-needed-l"; if (system_lib.weak) break :prefix "-weak-l"; @@ -1662,10 +1653,6 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { const size = try std.fmt.allocPrint(b.allocator, "{x}", .{pagezero_size}); try zig_args.appendSlice(&[_][]const u8{ "-pagezero_size", size }); } - if (self.search_strategy) |strat| switch (strat) { - .paths_first => try zig_args.append("-search_paths_first"), - .dylibs_first => try zig_args.append("-search_dylibs_first"), - }; if (self.headerpad_size) |headerpad_size| { const size = try std.fmt.allocPrint(b.allocator, "{x}", .{headerpad_size}); try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size }); diff --git a/test/link/macho/search_strategy/build.zig b/test/link/macho/search_strategy/build.zig index 574782617b18..b128fe71cf9c 100644 --- a/test/link/macho/search_strategy/build.zig +++ b/test/link/macho/search_strategy/build.zig @@ -17,8 +17,7 @@ fn add(b: *std.Build, test_step: *std.Build.Step, optimize: std.builtin.Optimize { // -search_dylibs_first - const exe = createScenario(b, optimize, target, "search_dylibs_first"); - exe.search_strategy = .dylibs_first; + const exe = createScenario(b, optimize, target, "search_dylibs_first", .mode_first); const check = exe.checkObject(); check.checkStart(); @@ -34,8 +33,7 @@ fn add(b: *std.Build, test_step: *std.Build.Step, optimize: std.builtin.Optimize { // -search_paths_first - const exe = createScenario(b, optimize, target, "search_paths_first"); - exe.search_strategy = .paths_first; + const exe = createScenario(b, optimize, target, "search_paths_first", .paths_first); const run = b.addRunArtifact(exe); run.skip_foreign_checks = true; @@ -49,6 +47,7 @@ fn createScenario( optimize: std.builtin.OptimizeMode, target: std.zig.CrossTarget, name: []const u8, + search_strategy: std.Build.Step.Compile.SystemLib.SearchStrategy, ) *std.Build.Step.Compile { const static = b.addStaticLibrary(.{ .name = name, @@ -73,7 +72,10 @@ fn createScenario( .target = target, }); exe.addCSourceFile(.{ .file = .{ .path = "main.c" }, .flags = &.{} }); - exe.linkSystemLibraryName(name); + exe.linkSystemLibrary2(name, .{ + .use_pkg_config = .no, + .search_strategy = search_strategy, + }); exe.linkLibC(); exe.addLibraryPath(static.getEmittedBinDirectory()); exe.addLibraryPath(dylib.getEmittedBinDirectory()); From 6ce37845af63245269efa9a959920802ce715e09 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 15:19:26 -0700 Subject: [PATCH 15/29] build.zig: use InstallArtifact Options instead of mutation I expect this to have no functional change. --- build.zig | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/build.zig b/build.zig index ab3dcfc1b45d..cb62d64da2f7 100644 --- a/build.zig +++ b/build.zig @@ -204,10 +204,9 @@ pub fn build(b: *std.Build) !void { ); if (!no_bin) { - const install_exe = b.addInstallArtifact(exe, .{}); - if (flat) { - install_exe.dest_dir = .prefix; - } + const install_exe = b.addInstallArtifact(exe, .{ + .dest_dir = if (flat) .{ .override = .prefix } else .default, + }); b.getInstallStep().dependOn(&install_exe.step); } From d7e74f7cbfe3adfe70ae1d7b92509f81fce28a5a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 15:19:56 -0700 Subject: [PATCH 16/29] CI: fix path to output langref Apparently tidy doesn't exit with failure status if the input file is not found? wtf tidy --- ci/aarch64-linux-debug.sh | 2 +- ci/aarch64-linux-release.sh | 2 +- ci/x86_64-linux-debug.sh | 2 +- ci/x86_64-linux-release.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/aarch64-linux-debug.sh b/ci/aarch64-linux-debug.sh index c5dbb6f29a06..6369777e884e 100644 --- a/ci/aarch64-linux-debug.sh +++ b/ci/aarch64-linux-debug.sh @@ -72,7 +72,7 @@ stage3-debug/bin/zig build test docs \ # Look for HTML errors. # TODO: move this to a build.zig flag (-Denable-tidy) -tidy --drop-empty-elements no -qe "zig-out/doc/langref.html" +tidy --drop-empty-elements no -qe "../zig-out/doc/langref.html" # Ensure that updating the wasm binary from this commit will result in a viable build. stage3-debug/bin/zig build update-zig1 diff --git a/ci/aarch64-linux-release.sh b/ci/aarch64-linux-release.sh index cdfc5eda452a..369c14dc5f8d 100644 --- a/ci/aarch64-linux-release.sh +++ b/ci/aarch64-linux-release.sh @@ -72,7 +72,7 @@ stage3-release/bin/zig build test docs \ # Look for HTML errors. # TODO: move this to a build.zig flag (-Denable-tidy) -tidy --drop-empty-elements no -qe "zig-out/doc/langref.html" +tidy --drop-empty-elements no -qe "../zig-out/doc/langref.html" # Ensure that updating the wasm binary from this commit will result in a viable build. stage3-release/bin/zig build update-zig1 diff --git a/ci/x86_64-linux-debug.sh b/ci/x86_64-linux-debug.sh index 422cb675d188..2efca8f7746c 100755 --- a/ci/x86_64-linux-debug.sh +++ b/ci/x86_64-linux-debug.sh @@ -72,7 +72,7 @@ stage3-debug/bin/zig build test docs \ # Look for HTML errors. # TODO: move this to a build.zig flag (-Denable-tidy) -tidy --drop-empty-elements no -qe "zig-out/doc/langref.html" +tidy --drop-empty-elements no -qe "../zig-out/doc/langref.html" # Ensure that updating the wasm binary from this commit will result in a viable build. stage3-debug/bin/zig build update-zig1 diff --git a/ci/x86_64-linux-release.sh b/ci/x86_64-linux-release.sh index da2d795c7659..4dc5c379270d 100755 --- a/ci/x86_64-linux-release.sh +++ b/ci/x86_64-linux-release.sh @@ -73,7 +73,7 @@ stage3-release/bin/zig build test docs \ # Look for HTML errors. # TODO: move this to a build.zig flag (-Denable-tidy) -tidy --drop-empty-elements no -qe "zig-out/doc/langref.html" +tidy --drop-empty-elements no -qe "../zig-out/doc/langref.html" # Ensure that stage3 and stage4 are byte-for-byte identical. stage3-release/bin/zig build \ From 469af6780408cf3f2d7ce0c16e2d3b797dd525f1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 15:20:28 -0700 Subject: [PATCH 17/29] CLI: delete redundant logic and use null instead of undef The `null` value here was missed in 0a4d4eb252a73555a568a532902951a13284d2ef. I hope it is the cause of the CI failures on Windows. The fact that libc++ depends on libc is not important for the CLI and Compilation.create already handles that logic. --- src/main.zig | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/main.zig b/src/main.zig index 1de6ac7449e2..ebbe67e353ae 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1151,9 +1151,6 @@ fn buildOutputType( try system_libs.put(args_iter.nextOrFatal(), .{ .needed = false, .weak = false, - // -l always dynamic links. For static libraries, - // users are expected to use positional arguments - // which are always unambiguous. .preferred_mode = lib_preferred_mode, .search_strategy = lib_search_strategy, }); @@ -1560,9 +1557,6 @@ fn buildOutputType( try system_libs.put(arg["-l".len..], .{ .needed = false, .weak = false, - // -l always dynamic links. For static libraries, - // users are expected to use positional arguments - // which are always unambiguous. .preferred_mode = lib_preferred_mode, .search_strategy = lib_search_strategy, }); @@ -2675,7 +2669,7 @@ fn buildOutputType( .lib = .{ .needed = true, .weak = false, - .path = undefined, + .path = null, }, }); continue; @@ -2700,11 +2694,6 @@ fn buildOutputType( } // After this point, external_system_libs is used instead of system_libs. - // libc++ depends on libc - if (link_libcpp) { - link_libc = true; - } - // Trigger native system library path detection if necessary. if (sysroot == null and cross_target.isNativeOs() and (external_system_libs.len != 0 or want_native_include_dirs)) @@ -6343,13 +6332,12 @@ fn accessLibPath( // In the case of Darwin, the main check will be .dylib, so here we // additionally check for .so files. if (target.isDarwin() and link_mode == .Dynamic) so: { - // Prefer .tbd over .dylib. test_path.clearRetainingCapacity(); try test_path.writer().print("{s}" ++ sep ++ "lib{s}.so", .{ lib_dir_path, lib_name }); try checked_paths.writer().print("\n {s}", .{test_path.items}); fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { error.FileNotFound => break :so, - else => |e| fatal("unable to search for tbd library '{s}': {s}", .{ + else => |e| fatal("unable to search for so library '{s}': {s}", .{ test_path.items, @errorName(e), }), }; From ea0e6e737bb658bb6353ad6d3ab3c8cff61e051a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 15:33:30 -0700 Subject: [PATCH 18/29] CLI: reduce code size bloat Make a bunch of ArrayList objects use arena instead of gpa, eliminating the `defer` expressions, which reduces code size of zig1.wasm by 1% --- src/main.zig | 68 +++++++++++++++++++--------------------------------- 1 file changed, 24 insertions(+), 44 deletions(-) diff --git a/src/main.zig b/src/main.zig index ebbe67e353ae..87db5c80ea65 100644 --- a/src/main.zig +++ b/src/main.zig @@ -746,6 +746,18 @@ const SystemLib = struct { } }; +const CliModule = struct { + mod: *Package, + /// still in CLI arg format + deps_str: []const u8, +}; + +fn cleanupModules(modules: *std.StringArrayHashMap(CliModule)) void { + var it = modules.iterator(); + while (it.next()) |kv| kv.value_ptr.mod.destroy(modules.allocator); + modules.deinit(); +} + fn buildOutputType( gpa: Allocator, arena: Allocator, @@ -893,62 +905,30 @@ fn buildOutputType( var error_tracing: ?bool = null; var pdb_out_path: ?[]const u8 = null; var dwarf_format: ?std.dwarf.Format = null; - // e.g. -m3dnow or -mno-outline-atomics. They correspond to std.Target llvm cpu feature names. // This array is populated by zig cc frontend and then has to be converted to zig-style // CPU features. - var llvm_m_args = std.ArrayList([]const u8).init(gpa); - defer llvm_m_args.deinit(); - + var llvm_m_args = std.ArrayList([]const u8).init(arena); var system_libs = std.StringArrayHashMap(SystemLib).init(arena); - - var wasi_emulated_libs = std.ArrayList(wasi_libc.CRTFile).init(gpa); - defer wasi_emulated_libs.deinit(); - - var clang_argv = std.ArrayList([]const u8).init(gpa); - defer clang_argv.deinit(); - - var extra_cflags = std.ArrayList([]const u8).init(gpa); - defer extra_cflags.deinit(); - + var wasi_emulated_libs = std.ArrayList(wasi_libc.CRTFile).init(arena); + var clang_argv = std.ArrayList([]const u8).init(arena); + var extra_cflags = std.ArrayList([]const u8).init(arena); // These are before resolving sysroot. var lib_dir_args = std.ArrayList([]const u8).init(arena); - - var rpath_list = std.ArrayList([]const u8).init(gpa); - defer rpath_list.deinit(); - + var rpath_list = std.ArrayList([]const u8).init(arena); var symbol_wrap_set: std.StringArrayHashMapUnmanaged(void) = .{}; - - var c_source_files = std.ArrayList(Compilation.CSourceFile).init(gpa); - defer c_source_files.deinit(); - - var link_objects = std.ArrayList(Compilation.LinkObject).init(gpa); - defer link_objects.deinit(); - - var framework_dirs = std.ArrayList([]const u8).init(gpa); - defer framework_dirs.deinit(); - + var c_source_files = std.ArrayList(Compilation.CSourceFile).init(arena); + var link_objects = std.ArrayList(Compilation.LinkObject).init(arena); + var framework_dirs = std.ArrayList([]const u8).init(arena); var frameworks: std.StringArrayHashMapUnmanaged(Compilation.Framework) = .{}; - // null means replace with the test executable binary - var test_exec_args = std.ArrayList(?[]const u8).init(gpa); - defer test_exec_args.deinit(); - - var linker_export_symbol_names = std.ArrayList([]const u8).init(gpa); - defer linker_export_symbol_names.deinit(); - + var test_exec_args = std.ArrayList(?[]const u8).init(arena); + var linker_export_symbol_names = std.ArrayList([]const u8).init(arena); // Contains every module specified via --mod. The dependencies are added // after argument parsing is completed. We use a StringArrayHashMap to make // error output consistent. - var modules = std.StringArrayHashMap(struct { - mod: *Package, - deps_str: []const u8, // still in CLI arg format - }).init(gpa); - defer { - var it = modules.iterator(); - while (it.next()) |kv| kv.value_ptr.mod.destroy(gpa); - modules.deinit(); - } + var modules = std.StringArrayHashMap(CliModule).init(gpa); + defer cleanupModules(&modules); // The dependency string for the root package var root_deps_str: ?[]const u8 = null; From c94bbebb9150f68ce179caa4f6beeab0622696a6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 18:35:18 -0700 Subject: [PATCH 19/29] std.zig.system.NativePaths: simplify and integrate with Darwin SDK --- lib/std/zig/system/NativePaths.zig | 108 +++++++++-------------------- src/Compilation.zig | 6 +- src/main.zig | 25 +------ 3 files changed, 40 insertions(+), 99 deletions(-) diff --git a/lib/std/zig/system/NativePaths.zig b/lib/std/zig/system/NativePaths.zig index f9798695ad5f..61f71144ca40 100644 --- a/lib/std/zig/system/NativePaths.zig +++ b/lib/std/zig/system/NativePaths.zig @@ -1,6 +1,5 @@ const std = @import("../../std.zig"); const builtin = @import("builtin"); -const ArrayList = std.ArrayList; const Allocator = std.mem.Allocator; const process = std.process; const mem = std.mem; @@ -8,28 +7,18 @@ const mem = std.mem; const NativePaths = @This(); const NativeTargetInfo = std.zig.system.NativeTargetInfo; -include_dirs: ArrayList([:0]u8), -lib_dirs: ArrayList([:0]u8), -framework_dirs: ArrayList([:0]u8), -rpaths: ArrayList([:0]u8), -warnings: ArrayList([:0]u8), +arena: Allocator, +include_dirs: std.ArrayListUnmanaged([]const u8) = .{}, +lib_dirs: std.ArrayListUnmanaged([]const u8) = .{}, +framework_dirs: std.ArrayListUnmanaged([]const u8) = .{}, +rpaths: std.ArrayListUnmanaged([]const u8) = .{}, +warnings: std.ArrayListUnmanaged([]const u8) = .{}, -pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths { +pub fn detect(arena: Allocator, native_info: NativeTargetInfo) !NativePaths { const native_target = native_info.target; - - var self: NativePaths = .{ - .include_dirs = ArrayList([:0]u8).init(allocator), - .lib_dirs = ArrayList([:0]u8).init(allocator), - .framework_dirs = ArrayList([:0]u8).init(allocator), - .rpaths = ArrayList([:0]u8).init(allocator), - .warnings = ArrayList([:0]u8).init(allocator), - }; - errdefer self.deinit(); - + var self: NativePaths = .{ .arena = arena }; var is_nix = false; - if (process.getEnvVarOwned(allocator, "NIX_CFLAGS_COMPILE")) |nix_cflags_compile| { - defer allocator.free(nix_cflags_compile); - + if (process.getEnvVarOwned(arena, "NIX_CFLAGS_COMPILE")) |nix_cflags_compile| { is_nix = true; var it = mem.tokenizeScalar(u8, nix_cflags_compile, ' '); while (true) { @@ -58,9 +47,7 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths error.EnvironmentVariableNotFound => {}, error.OutOfMemory => |e| return e, } - if (process.getEnvVarOwned(allocator, "NIX_LDFLAGS")) |nix_ldflags| { - defer allocator.free(nix_ldflags); - + if (process.getEnvVarOwned(arena, "NIX_LDFLAGS")) |nix_ldflags| { is_nix = true; var it = mem.tokenizeScalar(u8, nix_ldflags, ' '); while (true) { @@ -89,17 +76,18 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths return self; } + // TODO: consider also adding homebrew paths + // TODO: consider also adding macports paths if (comptime builtin.target.isDarwin()) { - try self.addIncludeDir("/usr/include"); - try self.addLibDir("/usr/lib"); - try self.addFrameworkDir("/System/Library/Frameworks"); - - if (builtin.target.os.version_range.semver.min.major < 11) { - try self.addIncludeDir("/usr/local/include"); - try self.addLibDir("/usr/local/lib"); - try self.addFrameworkDir("/Library/Frameworks"); + if (std.zig.system.darwin.isDarwinSDKInstalled(arena)) sdk: { + const sdk = std.zig.system.darwin.getDarwinSDK(arena, native_target) orelse break :sdk; + try self.addLibDir(try std.fs.path.join(arena, &.{ sdk.path, "usr/lib" })); + try self.addFrameworkDir(try std.fs.path.join(arena, &.{ sdk.path, "System/Library/Frameworks" })); + try self.addIncludeDir(try std.fs.path.join(arena, &.{ sdk.path, "usr/include" })); + return self; } - + // These do not include headers, so the ones that come with the SDK are preferred. + try self.addFrameworkDir("/System/Library/Frameworks"); return self; } @@ -115,8 +103,7 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths } if (builtin.os.tag != .windows) { - const triple = try native_target.linuxTriple(allocator); - defer allocator.free(triple); + const triple = try native_target.linuxTriple(arena); const qual = native_target.ptrBitWidth(); @@ -172,69 +159,42 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths return self; } -pub fn deinit(self: *NativePaths) void { - deinitArray(&self.include_dirs); - deinitArray(&self.lib_dirs); - deinitArray(&self.framework_dirs); - deinitArray(&self.rpaths); - deinitArray(&self.warnings); - self.* = undefined; -} - -fn deinitArray(array: *ArrayList([:0]u8)) void { - for (array.items) |item| { - array.allocator.free(item); - } - array.deinit(); -} - pub fn addIncludeDir(self: *NativePaths, s: []const u8) !void { - return self.appendArray(&self.include_dirs, s); + return self.include_dirs.append(self.arena, s); } pub fn addIncludeDirFmt(self: *NativePaths, comptime fmt: []const u8, args: anytype) !void { - const item = try std.fmt.allocPrintZ(self.include_dirs.allocator, fmt, args); - errdefer self.include_dirs.allocator.free(item); - try self.include_dirs.append(item); + const item = try std.fmt.allocPrint(self.arena, fmt, args); + try self.include_dirs.append(self.arena, item); } pub fn addLibDir(self: *NativePaths, s: []const u8) !void { - return self.appendArray(&self.lib_dirs, s); + try self.lib_dirs.append(self.arena, s); } pub fn addLibDirFmt(self: *NativePaths, comptime fmt: []const u8, args: anytype) !void { - const item = try std.fmt.allocPrintZ(self.lib_dirs.allocator, fmt, args); - errdefer self.lib_dirs.allocator.free(item); - try self.lib_dirs.append(item); + const item = try std.fmt.allocPrint(self.arena, fmt, args); + try self.lib_dirs.append(self.arena, item); } pub fn addWarning(self: *NativePaths, s: []const u8) !void { - return self.appendArray(&self.warnings, s); + return self.warnings.append(self.arena, s); } pub fn addFrameworkDir(self: *NativePaths, s: []const u8) !void { - return self.appendArray(&self.framework_dirs, s); + return self.framework_dirs.append(self.arena, s); } pub fn addFrameworkDirFmt(self: *NativePaths, comptime fmt: []const u8, args: anytype) !void { - const item = try std.fmt.allocPrintZ(self.framework_dirs.allocator, fmt, args); - errdefer self.framework_dirs.allocator.free(item); - try self.framework_dirs.append(item); + const item = try std.fmt.allocPrint(self.arena, fmt, args); + try self.framework_dirs.append(self.arena, item); } pub fn addWarningFmt(self: *NativePaths, comptime fmt: []const u8, args: anytype) !void { - const item = try std.fmt.allocPrintZ(self.warnings.allocator, fmt, args); - errdefer self.warnings.allocator.free(item); - try self.warnings.append(item); + const item = try std.fmt.allocPrint(self.arena, fmt, args); + try self.warnings.append(self.arena, item); } pub fn addRPath(self: *NativePaths, s: []const u8) !void { - return self.appendArray(&self.rpaths, s); -} - -fn appendArray(self: *NativePaths, array: *ArrayList([:0]u8), s: []const u8) !void { - _ = self; - const item = try array.allocator.dupeZ(u8, s); - errdefer array.allocator.free(item); - try array.append(item); + try self.rpaths.append(self.arena, s); } diff --git a/src/Compilation.zig b/src/Compilation.zig index 91b31c60041f..094275f749c8 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -857,8 +857,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const sysroot = blk: { if (options.sysroot) |sysroot| { break :blk sysroot; - } else if (options.native_darwin_sdk) |sdk| { - break :blk sdk.path; } else { break :blk null; } @@ -4341,6 +4339,10 @@ pub fn addCCArgs( try argv.append("-ObjC++"); } + for (comp.bin_file.options.framework_dirs) |framework_dir| { + try argv.appendSlice(&.{ "-isystem", framework_dir }); + } + // According to Rich Felker libc headers are supposed to go before C language headers. // However as noted by @dimenus, appending libc headers before c_headers breaks intrinsics // and other compiler specific items. diff --git a/src/main.zig b/src/main.zig index 87db5c80ea65..3bdcf0d2ce0c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2685,34 +2685,13 @@ fn buildOutputType( warn("{s}", .{warning}); } - const has_sysroot = if (comptime builtin.target.isDarwin()) outer: { - if (std.zig.system.darwin.isDarwinSDKInstalled(arena)) { - const sdk = std.zig.system.darwin.getDarwinSDK(arena, target_info.target) orelse - break :outer false; - native_darwin_sdk = sdk; - try clang_argv.ensureUnusedCapacity(2); - clang_argv.appendAssumeCapacity("-isysroot"); - clang_argv.appendAssumeCapacity(sdk.path); - break :outer true; - } else break :outer false; - } else false; - try clang_argv.ensureUnusedCapacity(paths.include_dirs.items.len * 2); - const isystem_flag = if (has_sysroot) "-iwithsysroot" else "-isystem"; for (paths.include_dirs.items) |include_dir| { - clang_argv.appendAssumeCapacity(isystem_flag); + clang_argv.appendAssumeCapacity("-isystem"); clang_argv.appendAssumeCapacity(include_dir); } - try clang_argv.ensureUnusedCapacity(paths.framework_dirs.items.len * 2); - try framework_dirs.ensureUnusedCapacity(paths.framework_dirs.items.len); - const iframework_flag = if (has_sysroot) "-iframeworkwithsysroot" else "-iframework"; - for (paths.framework_dirs.items) |framework_dir| { - clang_argv.appendAssumeCapacity(iframework_flag); - clang_argv.appendAssumeCapacity(framework_dir); - framework_dirs.appendAssumeCapacity(framework_dir); - } - + try framework_dirs.appendSlice(paths.framework_dirs.items); try lib_dirs.appendSlice(paths.lib_dirs.items); try rpath_list.appendSlice(paths.rpaths.items); } From da91ef5c28bd11823bd84b6f54df9ca601f03e21 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 19:17:07 -0700 Subject: [PATCH 20/29] zig libc: restore functionality on macOS Regressed in 2006add8496c47804ee3b6c562f420871cb4ea0a. References to native_darwin_sdk are no longer kept in the frontend. Instead the darwin SDK is detected as part of NativePaths and as part of LibCInstallation. --- src/Compilation.zig | 134 +++++++++++++------------------ src/libc_installation.zig | 16 +++- src/link.zig | 4 +- src/link/MachO/load_commands.zig | 9 +-- src/main.zig | 4 +- 5 files changed, 77 insertions(+), 90 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 094275f749c8..8b071cff0e12 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -637,8 +637,6 @@ pub const InitOptions = struct { wasi_exec_model: ?std.builtin.WasiExecModel = null, /// (Zig compiler development) Enable dumping linker's state as JSON. enable_link_snapshots: bool = false, - /// (Darwin) Path and version of the native SDK if detected. - native_darwin_sdk: ?std.zig.system.darwin.DarwinSDK = null, /// (Darwin) Install name of the dylib install_name: ?[]const u8 = null, /// (Darwin) Path to entitlements file @@ -945,7 +943,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { options.is_native_abi, link_libc, options.libc_installation, - options.native_darwin_sdk != null, ); const must_pie = target_util.requiresPIE(options.target); @@ -1560,7 +1557,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .wasi_exec_model = wasi_exec_model, .hash_style = options.hash_style, .enable_link_snapshots = options.enable_link_snapshots, - .native_darwin_sdk = options.native_darwin_sdk, .install_name = options.install_name, .entitlements = options.entitlements, .pagezero_size = options.pagezero_size, @@ -4865,7 +4861,6 @@ fn detectLibCIncludeDirs( is_native_abi: bool, link_libc: bool, libc_installation: ?*const LibCInstallation, - has_macos_sdk: bool, ) !LibCDirs { if (!link_libc) { return LibCDirs{ @@ -4881,28 +4876,19 @@ fn detectLibCIncludeDirs( // If linking system libraries and targeting the native abi, default to // using the system libc installation. if (is_native_abi and !target.isMinGW()) { - if (target.isDarwin()) { - return if (has_macos_sdk) - // For Darwin/macOS, we are all set with getDarwinSDK found earlier. - LibCDirs{ - .libc_include_dir_list = &[0][]u8{}, - .libc_installation = null, - } - else - getZigShippedLibCIncludeDirsDarwin(arena, zig_lib_dir, target); - } const libc = try arena.create(LibCInstallation); - libc.* = LibCInstallation.findNative(.{ .allocator = arena }) catch |err| switch (err) { + libc.* = LibCInstallation.findNative(.{ .allocator = arena, .target = target }) catch |err| switch (err) { error.CCompilerExitCode, error.CCompilerCrashed, error.CCompilerCannotFindHeaders, error.UnableToSpawnCCompiler, + error.DarwinSdkNotFound, => |e| { // We tried to integrate with the native system C compiler, // however, it is not installed. So we must rely on our bundled // libc files. if (target_util.canBuildLibC(target)) { - return detectLibCFromBuilding(arena, zig_lib_dir, target, has_macos_sdk); + return detectLibCFromBuilding(arena, zig_lib_dir, target); } return e; }, @@ -4914,7 +4900,7 @@ fn detectLibCIncludeDirs( // If not linking system libraries, build and provide our own libc by // default if possible. if (target_util.canBuildLibC(target)) { - return detectLibCFromBuilding(arena, zig_lib_dir, target, has_macos_sdk); + return detectLibCFromBuilding(arena, zig_lib_dir, target); } // If zig can't build the libc for the target and we are targeting the @@ -4928,7 +4914,7 @@ fn detectLibCIncludeDirs( if (use_system_abi) { const libc = try arena.create(LibCInstallation); - libc.* = try LibCInstallation.findNative(.{ .allocator = arena, .verbose = true }); + libc.* = try LibCInstallation.findNative(.{ .allocator = arena, .verbose = true, .target = target }); return detectLibCFromLibCInstallation(arena, target, libc); } @@ -4977,69 +4963,59 @@ fn detectLibCFromBuilding( arena: Allocator, zig_lib_dir: []const u8, target: std.Target, - has_macos_sdk: bool, ) !LibCDirs { - switch (target.os.tag) { - .macos => return if (has_macos_sdk) - // For Darwin/macOS, we are all set with getDarwinSDK found earlier. - LibCDirs{ - .libc_include_dir_list = &[0][]u8{}, - .libc_installation = null, - } - else - getZigShippedLibCIncludeDirsDarwin(arena, zig_lib_dir, target), - else => { - const generic_name = target_util.libCGenericName(target); - // Some architectures are handled by the same set of headers. - const arch_name = if (target.abi.isMusl()) - musl.archNameHeaders(target.cpu.arch) - else if (target.cpu.arch.isThumb()) - // ARM headers are valid for Thumb too. - switch (target.cpu.arch) { - .thumb => "arm", - .thumbeb => "armeb", - else => unreachable, - } - else - @tagName(target.cpu.arch); - const os_name = @tagName(target.os.tag); - // Musl's headers are ABI-agnostic and so they all have the "musl" ABI name. - const abi_name = if (target.abi.isMusl()) "musl" else @tagName(target.abi); - const s = std.fs.path.sep_str; - const arch_include_dir = try std.fmt.allocPrint( - arena, - "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-{s}", - .{ zig_lib_dir, arch_name, os_name, abi_name }, - ); - const generic_include_dir = try std.fmt.allocPrint( - arena, - "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "generic-{s}", - .{ zig_lib_dir, generic_name }, - ); - const generic_arch_name = target_util.osArchName(target); - const arch_os_include_dir = try std.fmt.allocPrint( - arena, - "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-any", - .{ zig_lib_dir, generic_arch_name, os_name }, - ); - const generic_os_include_dir = try std.fmt.allocPrint( - arena, - "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-{s}-any", - .{ zig_lib_dir, os_name }, - ); + if (target.isDarwin()) + return getZigShippedLibCIncludeDirsDarwin(arena, zig_lib_dir, target); + + const generic_name = target_util.libCGenericName(target); + // Some architectures are handled by the same set of headers. + const arch_name = if (target.abi.isMusl()) + musl.archNameHeaders(target.cpu.arch) + else if (target.cpu.arch.isThumb()) + // ARM headers are valid for Thumb too. + switch (target.cpu.arch) { + .thumb => "arm", + .thumbeb => "armeb", + else => unreachable, + } + else + @tagName(target.cpu.arch); + const os_name = @tagName(target.os.tag); + // Musl's headers are ABI-agnostic and so they all have the "musl" ABI name. + const abi_name = if (target.abi.isMusl()) "musl" else @tagName(target.abi); + const s = std.fs.path.sep_str; + const arch_include_dir = try std.fmt.allocPrint( + arena, + "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-{s}", + .{ zig_lib_dir, arch_name, os_name, abi_name }, + ); + const generic_include_dir = try std.fmt.allocPrint( + arena, + "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "generic-{s}", + .{ zig_lib_dir, generic_name }, + ); + const generic_arch_name = target_util.osArchName(target); + const arch_os_include_dir = try std.fmt.allocPrint( + arena, + "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-any", + .{ zig_lib_dir, generic_arch_name, os_name }, + ); + const generic_os_include_dir = try std.fmt.allocPrint( + arena, + "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-{s}-any", + .{ zig_lib_dir, os_name }, + ); - const list = try arena.alloc([]const u8, 4); - list[0] = arch_include_dir; - list[1] = generic_include_dir; - list[2] = arch_os_include_dir; - list[3] = generic_os_include_dir; + const list = try arena.alloc([]const u8, 4); + list[0] = arch_include_dir; + list[1] = generic_include_dir; + list[2] = arch_os_include_dir; + list[3] = generic_os_include_dir; - return LibCDirs{ - .libc_include_dir_list = list, - .libc_installation = null, - }; - }, - } + return LibCDirs{ + .libc_include_dir_list = list, + .libc_installation = null, + }; } pub fn get_libc_crt_file(comp: *Compilation, arena: Allocator, basename: []const u8) ![]const u8 { diff --git a/src/libc_installation.zig b/src/libc_installation.zig index bdfe83b71ab7..4dbe9d9c57f7 100644 --- a/src/libc_installation.zig +++ b/src/libc_installation.zig @@ -33,6 +33,7 @@ pub const LibCInstallation = struct { LibCKernel32LibNotFound, UnsupportedArchitecture, WindowsSdkNotFound, + DarwinSdkNotFound, ZigIsTheCCompiler, }; @@ -171,6 +172,7 @@ pub const LibCInstallation = struct { pub const FindNativeOptions = struct { allocator: Allocator, + target: std.Target, /// If enabled, will print human-friendly errors to stderr. verbose: bool = false, @@ -181,7 +183,19 @@ pub const LibCInstallation = struct { var self: LibCInstallation = .{}; if (is_darwin) { - @panic("Darwin is handled separately via std.zig.system.darwin module"); + if (!std.zig.system.darwin.isDarwinSDKInstalled(args.allocator)) + return error.DarwinSdkNotFound; + const sdk = std.zig.system.darwin.getDarwinSDK(args.allocator, args.target) orelse + return error.DarwinSdkNotFound; + defer args.allocator.free(sdk.path); + + self.include_dir = try fs.path.join(args.allocator, &.{ + sdk.path, "usr/include", + }); + self.sys_include_dir = try fs.path.join(args.allocator, &.{ + sdk.path, "usr/include", + }); + return self; } else if (is_windows) { var sdk: ZigWindowsSDK = ZigWindowsSDK.find(args.allocator) catch |err| switch (err) { error.NotFound => return error.WindowsSdkNotFound, diff --git a/src/link.zig b/src/link.zig index 6e5921d815ba..2fea01ff89ab 100644 --- a/src/link.zig +++ b/src/link.zig @@ -231,6 +231,7 @@ pub const Options = struct { version: ?std.SemanticVersion, compatibility_version: ?std.SemanticVersion, + darwin_sdk_version: ?std.SemanticVersion = null, libc_installation: ?*const LibCInstallation, dwarf_format: ?std.dwarf.Format, @@ -241,9 +242,6 @@ pub const Options = struct { /// (Zig compiler development) Enable dumping of linker's state as JSON. enable_link_snapshots: bool = false, - /// (Darwin) Path and version of the native SDK if detected. - native_darwin_sdk: ?std.zig.system.darwin.DarwinSDK = null, - /// (Darwin) Install name for the dylib install_name: ?[]const u8 = null, diff --git a/src/link/MachO/load_commands.zig b/src/link/MachO/load_commands.zig index 10f446f19159..8f803d98cb2d 100644 --- a/src/link/MachO/load_commands.zig +++ b/src/link/MachO/load_commands.zig @@ -278,11 +278,10 @@ pub fn writeBuildVersionLC(options: *const link.Options, lc_writer: anytype) !vo const platform_version = @as(u32, @intCast(ver.major << 16 | ver.minor << 8)); break :blk platform_version; }; - const sdk_version = if (options.native_darwin_sdk) |sdk| blk: { - const ver = sdk.version; - const sdk_version = @as(u32, @intCast(ver.major << 16 | ver.minor << 8)); - break :blk sdk_version; - } else platform_version; + const sdk_version: u32 = if (options.darwin_sdk_version) |ver| + @intCast(ver.major << 16 | ver.minor << 8) + else + platform_version; const is_simulator_abi = options.target.abi == .simulator; try lc_writer.writeStruct(macho.build_version_command{ .cmdsize = cmdsize, diff --git a/src/main.zig b/src/main.zig index 3bdcf0d2ce0c..3f06469a758b 100644 --- a/src/main.zig +++ b/src/main.zig @@ -891,7 +891,6 @@ fn buildOutputType( var minor_subsystem_version: ?u32 = null; var wasi_exec_model: ?std.builtin.WasiExecModel = null; var enable_link_snapshots: bool = false; - var native_darwin_sdk: ?std.zig.system.darwin.DarwinSDK = null; var install_name: ?[]const u8 = null; var hash_style: link.HashStyle = .both; var entitlements: ?[]const u8 = null; @@ -3367,7 +3366,6 @@ fn buildOutputType( .wasi_exec_model = wasi_exec_model, .debug_compile_errors = debug_compile_errors, .enable_link_snapshots = enable_link_snapshots, - .native_darwin_sdk = native_darwin_sdk, .install_name = install_name, .entitlements = entitlements, .pagezero_size = pagezero_size, @@ -4243,10 +4241,12 @@ pub fn cmdLibC(gpa: Allocator, args: []const []const u8) !void { if (!cross_target.isNative()) { fatal("unable to detect libc for non-native target", .{}); } + const target_info = try detectNativeTargetInfo(cross_target); var libc = LibCInstallation.findNative(.{ .allocator = gpa, .verbose = true, + .target = target_info.target, }) catch |err| { fatal("unable to detect native libc: {s}", .{@errorName(err)}); }; From e582a3642bb001a7a19ecb72b128f45ab7fc08aa Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 16:25:45 -0700 Subject: [PATCH 21/29] std.zig.system.darwin: fix redundant names --- lib/std/zig/system/NativePaths.zig | 4 ++-- lib/std/zig/system/darwin.zig | 10 +++++----- src/libc_installation.zig | 4 ++-- src/main.zig | 3 --- test/link/macho/bugs/13056/build.zig | 2 +- 5 files changed, 10 insertions(+), 13 deletions(-) diff --git a/lib/std/zig/system/NativePaths.zig b/lib/std/zig/system/NativePaths.zig index 61f71144ca40..1fd92292f9bf 100644 --- a/lib/std/zig/system/NativePaths.zig +++ b/lib/std/zig/system/NativePaths.zig @@ -79,8 +79,8 @@ pub fn detect(arena: Allocator, native_info: NativeTargetInfo) !NativePaths { // TODO: consider also adding homebrew paths // TODO: consider also adding macports paths if (comptime builtin.target.isDarwin()) { - if (std.zig.system.darwin.isDarwinSDKInstalled(arena)) sdk: { - const sdk = std.zig.system.darwin.getDarwinSDK(arena, native_target) orelse break :sdk; + if (std.zig.system.darwin.isSdkInstalled(arena)) sdk: { + const sdk = std.zig.system.darwin.getSdk(arena, native_target) orelse break :sdk; try self.addLibDir(try std.fs.path.join(arena, &.{ sdk.path, "usr/lib" })); try self.addFrameworkDir(try std.fs.path.join(arena, &.{ sdk.path, "System/Library/Frameworks" })); try self.addIncludeDir(try std.fs.path.join(arena, &.{ sdk.path, "usr/include" })); diff --git a/lib/std/zig/system/darwin.zig b/lib/std/zig/system/darwin.zig index e29dc68d64dd..3857d4f46c5a 100644 --- a/lib/std/zig/system/darwin.zig +++ b/lib/std/zig/system/darwin.zig @@ -11,7 +11,7 @@ pub const macos = @import("darwin/macos.zig"); /// Therefore, we resort to the same tool used by Homebrew, namely, invoking `xcode-select --print-path` /// and checking if the status is nonzero or the returned string in nonempty. /// https://github.com/Homebrew/brew/blob/e119bdc571dcb000305411bc1e26678b132afb98/Library/Homebrew/brew.sh#L630 -pub fn isDarwinSDKInstalled(allocator: Allocator) bool { +pub fn isSdkInstalled(allocator: Allocator) bool { const argv = &[_][]const u8{ "/usr/bin/xcode-select", "--print-path" }; const result = std.ChildProcess.exec(.{ .allocator = allocator, .argv = argv }) catch return false; defer { @@ -29,7 +29,7 @@ pub fn isDarwinSDKInstalled(allocator: Allocator) bool { /// Calls `xcrun --sdk --show-sdk-path` which fetches the path to the SDK sysroot (if any). /// Subsequently calls `xcrun --sdk --show-sdk-version` which fetches version of the SDK. /// The caller needs to deinit the resulting struct. -pub fn getDarwinSDK(allocator: Allocator, target: Target) ?DarwinSDK { +pub fn getSdk(allocator: Allocator, target: Target) ?Sdk { const is_simulator_abi = target.abi == .simulator; const sdk = switch (target.os.tag) { .macos => "macosx", @@ -73,7 +73,7 @@ pub fn getDarwinSDK(allocator: Allocator, target: Target) ?DarwinSDK { }; break :version version; }; - return DarwinSDK{ + return Sdk{ .path = path, .version = version, }; @@ -96,11 +96,11 @@ fn parseSdkVersion(raw: []const u8) ?Version { return Version.parse(buffer[0..len]) catch null; } -pub const DarwinSDK = struct { +pub const Sdk = struct { path: []const u8, version: Version, - pub fn deinit(self: DarwinSDK, allocator: Allocator) void { + pub fn deinit(self: Sdk, allocator: Allocator) void { allocator.free(self.path); } }; diff --git a/src/libc_installation.zig b/src/libc_installation.zig index 4dbe9d9c57f7..2d42a03a32d7 100644 --- a/src/libc_installation.zig +++ b/src/libc_installation.zig @@ -183,9 +183,9 @@ pub const LibCInstallation = struct { var self: LibCInstallation = .{}; if (is_darwin) { - if (!std.zig.system.darwin.isDarwinSDKInstalled(args.allocator)) + if (!std.zig.system.darwin.isSdkInstalled(args.allocator)) return error.DarwinSdkNotFound; - const sdk = std.zig.system.darwin.getDarwinSDK(args.allocator, args.target) orelse + const sdk = std.zig.system.darwin.getSdk(args.allocator, args.target) orelse return error.DarwinSdkNotFound; defer args.allocator.free(sdk.path); diff --git a/src/main.zig b/src/main.zig index 3f06469a758b..3c7d2c85bedc 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2857,9 +2857,6 @@ fn buildOutputType( std.log.err("unable to find {s} system library '{s}' using strategy '{s}'. searched paths:{s}", .{ @tagName(f.preferred_mode), f.name, @tagName(f.strategy), searched_paths, }); - if (f.preferred_mode == .Dynamic and f.strategy == .no_fallback) { - std.log.info("to link statically, pass the library as a positional argument", .{}); - } } process.exit(1); } diff --git a/test/link/macho/bugs/13056/build.zig b/test/link/macho/bugs/13056/build.zig index eade752bc46a..50eb99058326 100644 --- a/test/link/macho/bugs/13056/build.zig +++ b/test/link/macho/bugs/13056/build.zig @@ -16,7 +16,7 @@ pub fn build(b: *std.Build) void { fn add(b: *std.Build, test_step: *std.Build.Step, optimize: std.builtin.OptimizeMode) void { const target: std.zig.CrossTarget = .{ .os_tag = .macos }; const target_info = std.zig.system.NativeTargetInfo.detect(target) catch unreachable; - const sdk = std.zig.system.darwin.getDarwinSDK(b.allocator, target_info.target) orelse + const sdk = std.zig.system.darwin.getSdk(b.allocator, target_info.target) orelse @panic("macOS SDK is required to run the test"); const exe = b.addExecutable(.{ From c012f5d55dc59f80a3cd4fce22443a39fcbb67e9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 16:36:58 -0700 Subject: [PATCH 22/29] std.zig.system.darwin.isSdkInstalled: fix implementation * don't assert that the child process doesn't crash * don't give a false negative on warnings printed to stderr Also fix getSdk from the same file in the same way --- lib/std/zig/system/darwin.zig | 44 +++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/lib/std/zig/system/darwin.zig b/lib/std/zig/system/darwin.zig index 3857d4f46c5a..992c0d814d79 100644 --- a/lib/std/zig/system/darwin.zig +++ b/lib/std/zig/system/darwin.zig @@ -8,27 +8,33 @@ pub const macos = @import("darwin/macos.zig"); /// Check if SDK is installed on Darwin without triggering CLT installation popup window. /// Note: simply invoking `xcrun` will inevitably trigger the CLT installation popup. -/// Therefore, we resort to the same tool used by Homebrew, namely, invoking `xcode-select --print-path` -/// and checking if the status is nonzero or the returned string in nonempty. -/// https://github.com/Homebrew/brew/blob/e119bdc571dcb000305411bc1e26678b132afb98/Library/Homebrew/brew.sh#L630 +/// Therefore, we resort to invoking `xcode-select --print-path` and checking +/// if the status is nonzero. +/// stderr from xcode-select is ignored. +/// If error.OutOfMemory occurs in Allocator, this function returns null. pub fn isSdkInstalled(allocator: Allocator) bool { - const argv = &[_][]const u8{ "/usr/bin/xcode-select", "--print-path" }; - const result = std.ChildProcess.exec(.{ .allocator = allocator, .argv = argv }) catch return false; + const result = std.process.Child.exec(.{ + .allocator = allocator, + .argv = &.{ "/usr/bin/xcode-select", "--print-path" }, + }) catch return false; + defer { allocator.free(result.stderr); allocator.free(result.stdout); } - if (result.stderr.len != 0 or result.term.Exited != 0) { - // We don't actually care if there were errors as this is best-effort check anyhow. - return false; - } - return result.stdout.len > 0; + + return switch (result.term) { + .Exited => |code| if (code == 0) result.stdout.len > 0 else false, + else => false, + }; } /// Detect SDK on Darwin. /// Calls `xcrun --sdk --show-sdk-path` which fetches the path to the SDK sysroot (if any). /// Subsequently calls `xcrun --sdk --show-sdk-version` which fetches version of the SDK. /// The caller needs to deinit the resulting struct. +/// stderr from xcrun is ignored. +/// If error.OutOfMemory occurs in Allocator, this function returns null. pub fn getSdk(allocator: Allocator, target: Target) ?Sdk { const is_simulator_abi = target.abi == .simulator; const sdk = switch (target.os.tag) { @@ -40,30 +46,28 @@ pub fn getSdk(allocator: Allocator, target: Target) ?Sdk { }; const path = path: { const argv = &[_][]const u8{ "/usr/bin/xcrun", "--sdk", sdk, "--show-sdk-path" }; - const result = std.ChildProcess.exec(.{ .allocator = allocator, .argv = argv }) catch return null; + const result = std.process.Child.exec(.{ .allocator = allocator, .argv = argv }) catch return null; defer { allocator.free(result.stderr); allocator.free(result.stdout); } - if (result.stderr.len != 0 or result.term.Exited != 0) { - // We don't actually care if there were errors as this is best-effort check anyhow - // and in the worst case the user can specify the sysroot manually. - return null; + switch (result.term) { + .Exited => |code| if (code != 0) return null, + else => return null, } const path = allocator.dupe(u8, mem.trimRight(u8, result.stdout, "\r\n")) catch return null; break :path path; }; const version = version: { const argv = &[_][]const u8{ "/usr/bin/xcrun", "--sdk", sdk, "--show-sdk-version" }; - const result = std.ChildProcess.exec(.{ .allocator = allocator, .argv = argv }) catch return null; + const result = std.process.Child.exec(.{ .allocator = allocator, .argv = argv }) catch return null; defer { allocator.free(result.stderr); allocator.free(result.stdout); } - if (result.stderr.len != 0 or result.term.Exited != 0) { - // We don't actually care if there were errors as this is best-effort check anyhow - // and in the worst case the user can specify the sysroot manually. - return null; + switch (result.term) { + .Exited => |code| if (code != 0) return null, + else => return null, } const raw_version = mem.trimRight(u8, result.stdout, "\r\n"); const version = parseSdkVersion(raw_version) orelse Version{ From 7360927afe2157df7e697d2e4c56968dd605ea08 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 20:11:59 -0700 Subject: [PATCH 23/29] CLI: add native paths only if ABI is also native --- src/main.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.zig b/src/main.zig index 3c7d2c85bedc..e3720cf903cb 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2674,7 +2674,7 @@ fn buildOutputType( // After this point, external_system_libs is used instead of system_libs. // Trigger native system library path detection if necessary. - if (sysroot == null and cross_target.isNativeOs() and + if (sysroot == null and cross_target.isNativeOs() and cross_target.isNativeAbi() and (external_system_libs.len != 0 or want_native_include_dirs)) { const paths = std.zig.system.NativePaths.detect(arena, target_info) catch |err| { From 9dd9aa49a59cf8534aaf9b44ee20d526ba8da1c6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 20:40:56 -0700 Subject: [PATCH 24/29] Compilation: fix incorrect non-optional assumption --- src/Compilation.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 8b071cff0e12..ece6fa77cbba 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2362,10 +2362,10 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes if (comp.bin_file.options.link_libc) { man.hash.add(comp.bin_file.options.libc_installation != null); if (comp.bin_file.options.libc_installation) |libc_installation| { - man.hash.addBytes(libc_installation.crt_dir.?); + man.hash.addOptionalBytes(libc_installation.crt_dir); if (target.abi == .msvc) { - man.hash.addBytes(libc_installation.msvc_lib_dir.?); - man.hash.addBytes(libc_installation.kernel32_lib_dir.?); + man.hash.addOptionalBytes(libc_installation.msvc_lib_dir); + man.hash.addOptionalBytes(libc_installation.kernel32_lib_dir); } } man.hash.addOptionalBytes(comp.bin_file.options.dynamic_linker); From 4089f96defa7dae169e2b6cdba5e545293e57cb8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 21:49:26 -0700 Subject: [PATCH 25/29] darwin: pass -iframework to clang for system frameworks --- src/Compilation.zig | 21 ++++++++++++++++++++- src/main.zig | 7 ++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index ece6fa77cbba..79110d3bc490 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -124,6 +124,7 @@ zig_lib_directory: Directory, local_cache_directory: Directory, global_cache_directory: Directory, libc_include_dir_list: []const []const u8, +libc_framework_dir_list: []const []const u8, thread_pool: *ThreadPool, /// Populated when we build the libc++ static library. A Job to build this is placed in the queue @@ -1593,6 +1594,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .cache_parent = cache, .self_exe_path = options.self_exe_path, .libc_include_dir_list = libc_dirs.libc_include_dir_list, + .libc_framework_dir_list = libc_dirs.libc_framework_dir_list, .sanitize_c = sanitize_c, .thread_pool = options.thread_pool, .clang_passthrough_mode = options.clang_passthrough_mode, @@ -4335,8 +4337,12 @@ pub fn addCCArgs( try argv.append("-ObjC++"); } + for (comp.libc_framework_dir_list) |framework_dir| { + try argv.appendSlice(&.{ "-iframework", framework_dir }); + } + for (comp.bin_file.options.framework_dirs) |framework_dir| { - try argv.appendSlice(&.{ "-isystem", framework_dir }); + try argv.appendSlice(&.{ "-F", framework_dir }); } // According to Rich Felker libc headers are supposed to go before C language headers. @@ -4821,6 +4827,7 @@ test "classifyFileExt" { const LibCDirs = struct { libc_include_dir_list: []const []const u8, libc_installation: ?*const LibCInstallation, + libc_framework_dir_list: []const []const u8, }; fn getZigShippedLibCIncludeDirsDarwin(arena: Allocator, zig_lib_dir: []const u8, target: Target) !LibCDirs { @@ -4851,6 +4858,7 @@ fn getZigShippedLibCIncludeDirsDarwin(arena: Allocator, zig_lib_dir: []const u8, return LibCDirs{ .libc_include_dir_list = list, .libc_installation = null, + .libc_framework_dir_list = &.{}, }; } @@ -4866,6 +4874,7 @@ fn detectLibCIncludeDirs( return LibCDirs{ .libc_include_dir_list = &[0][]u8{}, .libc_installation = null, + .libc_framework_dir_list = &.{}, }; } @@ -4921,11 +4930,13 @@ fn detectLibCIncludeDirs( return LibCDirs{ .libc_include_dir_list = &[0][]u8{}, .libc_installation = null, + .libc_framework_dir_list = &.{}, }; } fn detectLibCFromLibCInstallation(arena: Allocator, target: Target, lci: *const LibCInstallation) !LibCDirs { var list = try std.ArrayList([]const u8).initCapacity(arena, 5); + var framework_list = std.ArrayList([]const u8).init(arena); list.appendAssumeCapacity(lci.include_dir.?); @@ -4953,9 +4964,16 @@ fn detectLibCFromLibCInstallation(arena: Allocator, target: Target, lci: *const list.appendAssumeCapacity(config_dir); } + if (target.isDarwin()) d: { + const down1 = std.fs.path.dirname(lci.sys_include_dir.?) orelse break :d; + const down2 = std.fs.path.dirname(down1) orelse break :d; + try framework_list.append(try std.fs.path.join(arena, &.{ down2, "System", "Library", "Frameworks" })); + } + return LibCDirs{ .libc_include_dir_list = list.items, .libc_installation = lci, + .libc_framework_dir_list = framework_list.items, }; } @@ -5015,6 +5033,7 @@ fn detectLibCFromBuilding( return LibCDirs{ .libc_include_dir_list = list, .libc_installation = null, + .libc_framework_dir_list = &.{}, }; } diff --git a/src/main.zig b/src/main.zig index e3720cf903cb..a1b0216a705a 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2695,11 +2695,8 @@ fn buildOutputType( try rpath_list.appendSlice(paths.rpaths.items); } - // Now that we have target info, we can find out if any of the system libraries - // are part of libc or libc++. We remove them from the list and communicate their - // existence via flags instead. - // Similarly, if any libs in this list are statically provided, we omit - // them from the resolved list and populate the link_objects array instead. + // If any libs in this list are statically provided, we omit them from the + // resolved list and populate the link_objects array instead. { var test_path = std.ArrayList(u8).init(gpa); defer test_path.deinit(); From 4923e64199d8a16f7e0f376a1839daa70a9dfdf2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 22:08:56 -0700 Subject: [PATCH 26/29] Compilation: detect sysroot from libc installation --- src/Compilation.zig | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 79110d3bc490..a08c3e09f46c 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -853,14 +853,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { break :blk false; }; - const sysroot = blk: { - if (options.sysroot) |sysroot| { - break :blk sysroot; - } else { - break :blk null; - } - }; - const lto = blk: { if (options.want_lto) |explicit| { if (!use_lld and !options.target.isDarwin()) @@ -946,6 +938,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { options.libc_installation, ); + const sysroot = options.sysroot orelse libc_dirs.sysroot; + const must_pie = target_util.requiresPIE(options.target); const pie: bool = if (options.want_pie) |explicit| pie: { if (!explicit and must_pie) { @@ -4828,6 +4822,7 @@ const LibCDirs = struct { libc_include_dir_list: []const []const u8, libc_installation: ?*const LibCInstallation, libc_framework_dir_list: []const []const u8, + sysroot: ?[]const u8, }; fn getZigShippedLibCIncludeDirsDarwin(arena: Allocator, zig_lib_dir: []const u8, target: Target) !LibCDirs { @@ -4859,6 +4854,7 @@ fn getZigShippedLibCIncludeDirsDarwin(arena: Allocator, zig_lib_dir: []const u8, .libc_include_dir_list = list, .libc_installation = null, .libc_framework_dir_list = &.{}, + .sysroot = null, }; } @@ -4875,6 +4871,7 @@ fn detectLibCIncludeDirs( .libc_include_dir_list = &[0][]u8{}, .libc_installation = null, .libc_framework_dir_list = &.{}, + .sysroot = null, }; } @@ -4931,6 +4928,7 @@ fn detectLibCIncludeDirs( .libc_include_dir_list = &[0][]u8{}, .libc_installation = null, .libc_framework_dir_list = &.{}, + .sysroot = null, }; } @@ -4964,16 +4962,20 @@ fn detectLibCFromLibCInstallation(arena: Allocator, target: Target, lci: *const list.appendAssumeCapacity(config_dir); } + var sysroot: ?[]const u8 = null; + if (target.isDarwin()) d: { const down1 = std.fs.path.dirname(lci.sys_include_dir.?) orelse break :d; const down2 = std.fs.path.dirname(down1) orelse break :d; try framework_list.append(try std.fs.path.join(arena, &.{ down2, "System", "Library", "Frameworks" })); + sysroot = down2; } return LibCDirs{ .libc_include_dir_list = list.items, .libc_installation = lci, .libc_framework_dir_list = framework_list.items, + .sysroot = sysroot, }; } @@ -5034,6 +5036,7 @@ fn detectLibCFromBuilding( .libc_include_dir_list = list, .libc_installation = null, .libc_framework_dir_list = &.{}, + .sysroot = null, }; } From 72fb58f1078fa2409d4b9275467a99c6533eeb04 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 2 Aug 2023 22:22:41 -0700 Subject: [PATCH 27/29] CLI: fix a typo that made static/dynamic do the opposite --- src/main.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.zig b/src/main.zig index a1b0216a705a..456886c915ce 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2764,7 +2764,7 @@ fn buildOutputType( info.fallbackMode(), )) { const path = try arena.dupe(u8, test_path.items); - switch (info.preferred_mode) { + switch (info.fallbackMode()) { .Static => try link_objects.append(.{ .path = path }), .Dynamic => try resolved_system_libs.append(arena, .{ .name = lib_name, @@ -2822,7 +2822,7 @@ fn buildOutputType( info.fallbackMode(), )) { const path = try arena.dupe(u8, test_path.items); - switch (info.preferred_mode) { + switch (info.fallbackMode()) { .Static => try link_objects.append(.{ .path = path }), .Dynamic => try resolved_system_libs.append(arena, .{ .name = lib_name, From f96d773d989bdca0a7eb08965a84ae2f63ea7f4c Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 3 Aug 2023 10:02:41 +0200 Subject: [PATCH 28/29] macho: append syslibroot to search dirs when resolving libSystem --- src/link/MachO.zig | 85 ++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 389d275fd57e..1f2dd4f8b007 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -829,55 +829,50 @@ pub fn resolveLibSystem( out_libs: anytype, ) !void { // If we were given the sysroot, try to look there first for libSystem.B.{dylib, tbd}. - var libsystem_available = false; - if (syslibroot != null) blk: { - // Try stub file first. If we hit it, then we're done as the stub file - // re-exports every single symbol definition. - for (search_dirs) |dir| { - if (try resolveLib(arena, dir, "System", ".tbd")) |full_path| { - try out_libs.put(full_path, .{ - .needed = true, - .weak = false, - .path = full_path, - }); - libsystem_available = true; - break :blk; - } + if (syslibroot) |root| { + const full_dir_path = try std.fs.path.join(arena, &.{ root, "usr", "lib" }); + if (try resolveLibSystemInDirs(arena, &.{full_dir_path}, out_libs)) return; + } + + // Next, try input search dirs if we are linking on a custom host such as Nix. + if (try resolveLibSystemInDirs(arena, search_dirs, out_libs)) return; + + // As a fallback, try linking against Zig shipped stub. + const libsystem_name = try std.fmt.allocPrint(arena, "libSystem.{d}.tbd", .{ + target.os.version_range.semver.min.major, + }); + const full_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ + "libc", "darwin", libsystem_name, + }); + try out_libs.put(full_path, .{ + .needed = true, + .weak = false, + .path = full_path, + }); +} + +fn resolveLibSystemInDirs(arena: Allocator, dirs: []const []const u8, out_libs: anytype) !bool { + // Try stub file first. If we hit it, then we're done as the stub file + // re-exports every single symbol definition. + for (dirs) |dir| { + if (try resolveLib(arena, dir, "System", ".tbd")) |full_path| { + try out_libs.put(full_path, .{ .needed = true, .weak = false, .path = full_path }); + return true; } - // If we didn't hit the stub file, try .dylib next. However, libSystem.dylib - // doesn't export libc.dylib which we'll need to resolve subsequently also. - for (search_dirs) |dir| { - if (try resolveLib(arena, dir, "System", ".dylib")) |libsystem_path| { - if (try resolveLib(arena, dir, "c", ".dylib")) |libc_path| { - try out_libs.put(libsystem_path, .{ - .needed = true, - .weak = false, - .path = libsystem_path, - }); - try out_libs.put(libc_path, .{ - .needed = true, - .weak = false, - .path = libc_path, - }); - libsystem_available = true; - break :blk; - } + } + // If we didn't hit the stub file, try .dylib next. However, libSystem.dylib + // doesn't export libc.dylib which we'll need to resolve subsequently also. + for (dirs) |dir| { + if (try resolveLib(arena, dir, "System", ".dylib")) |libsystem_path| { + if (try resolveLib(arena, dir, "c", ".dylib")) |libc_path| { + try out_libs.put(libsystem_path, .{ .needed = true, .weak = false, .path = libsystem_path }); + try out_libs.put(libc_path, .{ .needed = true, .weak = false, .path = libc_path }); + return true; } } } - if (!libsystem_available) { - const libsystem_name = try std.fmt.allocPrint(arena, "libSystem.{d}.tbd", .{ - target.os.version_range.semver.min.major, - }); - const full_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ - "libc", "darwin", libsystem_name, - }); - try out_libs.put(full_path, .{ - .needed = true, - .weak = false, - .path = full_path, - }); - } + + return false; } pub fn resolveSearchDir( From d0fd67cffe664ff70d9a70dd4d2d28aba5a378e8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 3 Aug 2023 09:51:21 -0700 Subject: [PATCH 29/29] std.zig.system.NativePaths: remove bad framework dir This path actually has nothing useful in it. --- lib/std/zig/system/NativePaths.zig | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/std/zig/system/NativePaths.zig b/lib/std/zig/system/NativePaths.zig index 1fd92292f9bf..4c8f1286b872 100644 --- a/lib/std/zig/system/NativePaths.zig +++ b/lib/std/zig/system/NativePaths.zig @@ -86,8 +86,6 @@ pub fn detect(arena: Allocator, native_info: NativeTargetInfo) !NativePaths { try self.addIncludeDir(try std.fs.path.join(arena, &.{ sdk.path, "usr/include" })); return self; } - // These do not include headers, so the ones that come with the SDK are preferred. - try self.addFrameworkDir("/System/Library/Frameworks"); return self; }