From a466bdc41b4eadb1619f3bdf654978cb23e1140e Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 20 Jun 2023 11:27:51 +0200 Subject: [PATCH 01/15] macos: try auto-detecting the SDK when building on Apple for Apple Unless we detect Nix, in which case, we use Nix provided paths via env vars. --- lib/std/zig/system/NativePaths.zig | 30 ++--- src/Compilation.zig | 199 ++++++++++++++++++----------- src/libc_installation.zig | 33 ++++- src/link.zig | 4 +- src/link/MachO/load_commands.zig | 3 +- src/main.zig | 66 ++-------- 6 files changed, 180 insertions(+), 155 deletions(-) diff --git a/lib/std/zig/system/NativePaths.zig b/lib/std/zig/system/NativePaths.zig index f9798695ad5f..1e8230a055b4 100644 --- a/lib/std/zig/system/NativePaths.zig +++ b/lib/std/zig/system/NativePaths.zig @@ -6,7 +6,6 @@ const process = std.process; const mem = std.mem; const NativePaths = @This(); -const NativeTargetInfo = std.zig.system.NativeTargetInfo; include_dirs: ArrayList([:0]u8), lib_dirs: ArrayList([:0]u8), @@ -14,9 +13,11 @@ framework_dirs: ArrayList([:0]u8), rpaths: ArrayList([:0]u8), warnings: ArrayList([:0]u8), -pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths { - const native_target = native_info.target; +pub fn isNix() bool { + return process.hasEnvVarConstant("NIX_CFLAGS_COMPILE") and process.hasEnvVarConstant("NIX_LDFLAGS"); +} +pub fn detect(allocator: Allocator, native_target: std.Target) !NativePaths { var self: NativePaths = .{ .include_dirs = ArrayList([:0]u8).init(allocator), .lib_dirs = ArrayList([:0]u8).init(allocator), @@ -26,11 +27,10 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths }; errdefer self.deinit(); - var is_nix = false; - if (process.getEnvVarOwned(allocator, "NIX_CFLAGS_COMPILE")) |nix_cflags_compile| { + if (isNix()) { + const nix_cflags_compile = try process.getEnvVarOwned(allocator, "NIX_CFLAGS_COMPILE"); defer allocator.free(nix_cflags_compile); - is_nix = true; var it = mem.tokenizeScalar(u8, nix_cflags_compile, ' '); while (true) { const word = it.next() orelse break; @@ -53,16 +53,11 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths try self.addWarningFmt("Unrecognized C flag from NIX_CFLAGS_COMPILE: {s}", .{word}); } } - } else |err| switch (err) { - error.InvalidUtf8 => {}, - error.EnvironmentVariableNotFound => {}, - error.OutOfMemory => |e| return e, - } - if (process.getEnvVarOwned(allocator, "NIX_LDFLAGS")) |nix_ldflags| { + + const nix_ldflags = try process.getEnvVarOwned(allocator, "NIX_LDFLAGS"); defer allocator.free(nix_ldflags); - is_nix = true; - var it = mem.tokenizeScalar(u8, nix_ldflags, ' '); + it = mem.tokenizeScalar(u8, nix_ldflags, ' '); while (true) { const word = it.next() orelse break; if (mem.eql(u8, word, "-rpath")) { @@ -80,12 +75,7 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths break; } } - } else |err| switch (err) { - error.InvalidUtf8 => {}, - error.EnvironmentVariableNotFound => {}, - error.OutOfMemory => |e| return e, - } - if (is_nix) { + return self; } diff --git a/src/Compilation.zig b/src/Compilation.zig index 4019e43c8d13..9b3c8308b904 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -124,6 +124,7 @@ zig_lib_directory: Directory, local_cache_directory: Directory, global_cache_directory: Directory, libc_include_dir_list: []const []const u8, +libc_framework_dir_list: []const []const u8, thread_pool: *ThreadPool, /// Populated when we build the libc++ static library. A Job to build this is placed in the queue @@ -517,6 +518,7 @@ pub const InitOptions = struct { link_libc: bool = false, link_libcpp: bool = false, link_libunwind: bool = false, + want_native_include_dirs: ?bool = null, want_pic: ?bool = null, /// This means that if the output mode is an executable it will be a /// Position Independent Executable. If the output mode is not an @@ -636,8 +638,6 @@ pub const InitOptions = struct { wasi_exec_model: ?std.builtin.WasiExecModel = null, /// (Zig compiler development) Enable dumping linker's state as JSON. enable_link_snapshots: bool = false, - /// (Darwin) Path and version of the native SDK if detected. - native_darwin_sdk: ?std.zig.system.darwin.DarwinSDK = null, /// (Darwin) Install name of the dylib install_name: ?[]const u8 = null, /// (Darwin) Path to entitlements file @@ -855,14 +855,60 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { break :blk false; }; - const sysroot = blk: { - if (options.sysroot) |sysroot| { - break :blk sysroot; - } else if (options.native_darwin_sdk) |sdk| { - break :blk sdk.path; - } else { - break :blk null; + var clang_argv = std.ArrayList([]const u8).init(arena); + var lib_dirs = std.ArrayList([]const u8).init(arena); + var framework_dirs = std.ArrayList([]const u8).init(arena); + var rpath_list = std.ArrayList([]const u8).init(arena); + + try clang_argv.appendSlice(options.clang_argv); + try lib_dirs.appendSlice(options.lib_dirs); + try framework_dirs.appendSlice(options.framework_dirs); + try rpath_list.appendSlice(options.rpath_list); + + const want_native_include_dirs = options.sysroot == null and options.is_native_os and + (options.system_lib_names.len > 0 or options.want_native_include_dirs orelse false); + const darwin_native = (comptime builtin.target.isDarwin()) and options.target.isDarwin() and options.sysroot == null; + var darwin_sdk: ?std.zig.system.darwin.DarwinSDK = null; + + if (want_native_include_dirs or darwin_native) { + const paths = std.zig.system.NativePaths.detect(arena, options.target) catch |err| { + fatal("unable to detect native system paths: {s}", .{@errorName(err)}); + }; + + for (paths.warnings.items) |warning| { + std.log.warn("{s}", .{warning}); + } + + try clang_argv.ensureUnusedCapacity(paths.include_dirs.items.len * 2); + for (paths.include_dirs.items) |include_dir| { + clang_argv.appendAssumeCapacity("-isystem"); + clang_argv.appendAssumeCapacity(include_dir); + } + + try clang_argv.ensureUnusedCapacity(paths.framework_dirs.items.len * 2); + try framework_dirs.ensureUnusedCapacity(paths.framework_dirs.items.len); + for (paths.framework_dirs.items) |framework_dir| { + clang_argv.appendAssumeCapacity("-iframework"); + clang_argv.appendAssumeCapacity(framework_dir); + framework_dirs.appendAssumeCapacity(framework_dir); + } + + for (paths.lib_dirs.items) |lib_dir| { + try lib_dirs.append(lib_dir); + } + for (paths.rpaths.items) |rpath| { + try rpath_list.append(rpath); + } + + if (!std.zig.system.NativePaths.isNix() and std.zig.system.darwin.isDarwinSDKInstalled(arena)) { + darwin_sdk = std.zig.system.darwin.getDarwinSDK(arena, options.target); } + } + + const sysroot = blk: { + if (options.sysroot) |sysroot| break :blk sysroot; + if (darwin_sdk) |sdk| break :blk sdk.path; + break :blk null; }; const lto = blk: { @@ -948,7 +994,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { options.is_native_abi, link_libc, options.libc_installation, - options.native_darwin_sdk != null, + darwin_sdk, ); const must_pie = target_util.requiresPIE(options.target); @@ -1477,11 +1523,11 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .link_libunwind = link_libunwind, .objects = options.link_objects, .frameworks = options.frameworks, - .framework_dirs = options.framework_dirs, + .framework_dirs = framework_dirs.items, .system_libs = system_libs, .wasi_emulated_libs = options.wasi_emulated_libs, - .lib_dirs = options.lib_dirs, - .rpath_list = options.rpath_list, + .lib_dirs = lib_dirs.items, + .rpath_list = rpath_list.items, .symbol_wrap_set = options.symbol_wrap_set, .strip = strip, .is_native_os = options.is_native_os, @@ -1535,6 +1581,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .soname = options.soname, .version = options.version, .compatibility_version = options.compatibility_version, + .darwin_sdk_version = if (darwin_sdk) |sdk| sdk.version else null, .libc_installation = libc_dirs.libc_installation, .pic = pic, .pie = pie, @@ -1563,7 +1610,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .wasi_exec_model = wasi_exec_model, .hash_style = options.hash_style, .enable_link_snapshots = options.enable_link_snapshots, - .native_darwin_sdk = options.native_darwin_sdk, .install_name = options.install_name, .entitlements = options.entitlements, .pagezero_size = options.pagezero_size, @@ -1596,11 +1642,12 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .embed_file_work_queue = std.fifo.LinearFifo(*Module.EmbedFile, .Dynamic).init(gpa), .keep_source_files_loaded = options.keep_source_files_loaded, .use_clang = use_clang, - .clang_argv = options.clang_argv, + .clang_argv = clang_argv.items, .c_source_files = options.c_source_files, .cache_parent = cache, .self_exe_path = options.self_exe_path, .libc_include_dir_list = libc_dirs.libc_include_dir_list, + .libc_framework_dir_list = libc_dirs.libc_framework_dir_list, .sanitize_c = sanitize_c, .thread_pool = options.thread_pool, .clang_passthrough_mode = options.clang_passthrough_mode, @@ -2367,7 +2414,8 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes if (comp.bin_file.options.link_libc) { man.hash.add(comp.bin_file.options.libc_installation != null); if (comp.bin_file.options.libc_installation) |libc_installation| { - man.hash.addBytes(libc_installation.crt_dir.?); + man.hash.addOptionalBytes(libc_installation.crt_dir); + man.hash.addOptionalBytes(libc_installation.framework_dir); if (target.abi == .msvc) { man.hash.addBytes(libc_installation.msvc_lib_dir.?); man.hash.addBytes(libc_installation.kernel32_lib_dir.?); @@ -3635,6 +3683,7 @@ pub fn obtainCObjectCacheManifest(comp: *const Compilation) Cache.Manifest { // decision should not be overridden here. if (comp.bin_file.options.libc_installation != null) { man.hash.addListOfBytes(comp.libc_include_dir_list); + man.hash.addListOfBytes(comp.libc_framework_dir_list); } return man; @@ -4353,6 +4402,11 @@ pub fn addCCArgs( try argv.append(include_dir); } + for (comp.libc_framework_dir_list) |framework_dir| { + try argv.append("-iframework"); + try argv.append(framework_dir); + } + if (target.cpu.model.llvm_name) |llvm_name| { try argv.appendSlice(&[_][]const u8{ "-Xclang", "-target-cpu", "-Xclang", llvm_name, @@ -4822,40 +4876,10 @@ test "classifyFileExt" { const LibCDirs = struct { libc_include_dir_list: []const []const u8, + libc_framework_dir_list: []const []const u8, libc_installation: ?*const LibCInstallation, }; -fn getZigShippedLibCIncludeDirsDarwin(arena: Allocator, zig_lib_dir: []const u8, target: Target) !LibCDirs { - const arch_name = @tagName(target.cpu.arch); - const os_name = try std.fmt.allocPrint(arena, "{s}.{d}", .{ - @tagName(target.os.tag), - target.os.version_range.semver.min.major, - }); - const s = std.fs.path.sep_str; - const list = try arena.alloc([]const u8, 3); - - list[0] = try std.fmt.allocPrint( - arena, - "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-none", - .{ zig_lib_dir, arch_name, os_name }, - ); - list[1] = try std.fmt.allocPrint( - arena, - "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-{s}-any", - .{ zig_lib_dir, os_name }, - ); - list[2] = try std.fmt.allocPrint( - arena, - "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-macos-any", - .{zig_lib_dir}, - ); - - return LibCDirs{ - .libc_include_dir_list = list, - .libc_installation = null, - }; -} - fn detectLibCIncludeDirs( arena: Allocator, zig_lib_dir: []const u8, @@ -4863,11 +4887,12 @@ fn detectLibCIncludeDirs( is_native_abi: bool, link_libc: bool, libc_installation: ?*const LibCInstallation, - has_macos_sdk: bool, + darwin_sdk: ?std.zig.system.darwin.DarwinSDK, ) !LibCDirs { if (!link_libc) { return LibCDirs{ .libc_include_dir_list = &[0][]u8{}, + .libc_framework_dir_list = &[0][]u8{}, .libc_installation = null, }; } @@ -4879,28 +4904,22 @@ fn detectLibCIncludeDirs( // If linking system libraries and targeting the native abi, default to // using the system libc installation. if (is_native_abi and !target.isMinGW()) { - if (target.isDarwin()) { - return if (has_macos_sdk) - // For Darwin/macOS, we are all set with getDarwinSDK found earlier. - LibCDirs{ - .libc_include_dir_list = &[0][]u8{}, - .libc_installation = null, - } - else - getZigShippedLibCIncludeDirsDarwin(arena, zig_lib_dir, target); - } const libc = try arena.create(LibCInstallation); - libc.* = LibCInstallation.findNative(.{ .allocator = arena }) catch |err| switch (err) { + libc.* = LibCInstallation.findNative(.{ + .allocator = arena, + .darwin_sdk = darwin_sdk, + }) catch |err| switch (err) { error.CCompilerExitCode, error.CCompilerCrashed, error.CCompilerCannotFindHeaders, error.UnableToSpawnCCompiler, + error.DarwinSdkNotFound, => |e| { // We tried to integrate with the native system C compiler, // however, it is not installed. So we must rely on our bundled // libc files. if (target_util.canBuildLibC(target)) { - return detectLibCFromBuilding(arena, zig_lib_dir, target, has_macos_sdk); + return detectLibCFromBuilding(arena, zig_lib_dir, target); } return e; }, @@ -4912,7 +4931,7 @@ fn detectLibCIncludeDirs( // If not linking system libraries, build and provide our own libc by // default if possible. if (target_util.canBuildLibC(target)) { - return detectLibCFromBuilding(arena, zig_lib_dir, target, has_macos_sdk); + return detectLibCFromBuilding(arena, zig_lib_dir, target); } // If zig can't build the libc for the target and we are targeting the @@ -4926,24 +4945,34 @@ fn detectLibCIncludeDirs( if (use_system_abi) { const libc = try arena.create(LibCInstallation); - libc.* = try LibCInstallation.findNative(.{ .allocator = arena, .verbose = true }); + libc.* = try LibCInstallation.findNative(.{ + .allocator = arena, + .darwin_sdk = darwin_sdk, + .verbose = true, + }); return detectLibCFromLibCInstallation(arena, target, libc); } return LibCDirs{ .libc_include_dir_list = &[0][]u8{}, + .libc_framework_dir_list = &[0][]u8{}, .libc_installation = null, }; } fn detectLibCFromLibCInstallation(arena: Allocator, target: Target, lci: *const LibCInstallation) !LibCDirs { var list = try std.ArrayList([]const u8).initCapacity(arena, 5); + var framework_list = std.ArrayList([]const u8).init(arena); list.appendAssumeCapacity(lci.include_dir.?); const is_redundant = mem.eql(u8, lci.sys_include_dir.?, lci.include_dir.?); if (!is_redundant) list.appendAssumeCapacity(lci.sys_include_dir.?); + if (target.isDarwin()) { + try framework_list.append(lci.framework_dir.?); + } + if (target.os.tag == .windows) { if (std.fs.path.dirname(lci.include_dir.?)) |include_dir_parent| { const um_dir = try std.fs.path.join(arena, &[_][]const u8{ include_dir_parent, "um" }); @@ -4967,25 +4996,44 @@ fn detectLibCFromLibCInstallation(arena: Allocator, target: Target, lci: *const return LibCDirs{ .libc_include_dir_list = list.items, + .libc_framework_dir_list = framework_list.items, .libc_installation = lci, }; } -fn detectLibCFromBuilding( - arena: Allocator, - zig_lib_dir: []const u8, - target: std.Target, - has_macos_sdk: bool, -) !LibCDirs { +fn detectLibCFromBuilding(arena: Allocator, zig_lib_dir: []const u8, target: std.Target) !LibCDirs { switch (target.os.tag) { - .macos => return if (has_macos_sdk) - // For Darwin/macOS, we are all set with getDarwinSDK found earlier. - LibCDirs{ - .libc_include_dir_list = &[0][]u8{}, + .macos => { + const arch_name = @tagName(target.cpu.arch); + const os_name = try std.fmt.allocPrint(arena, "{s}.{d}", .{ + @tagName(target.os.tag), + target.os.version_range.semver.min.major, + }); + const s = std.fs.path.sep_str; + const list = try arena.alloc([]const u8, 3); + + list[0] = try std.fmt.allocPrint( + arena, + "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-none", + .{ zig_lib_dir, arch_name, os_name }, + ); + list[1] = try std.fmt.allocPrint( + arena, + "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-{s}-any", + .{ zig_lib_dir, os_name }, + ); + list[2] = try std.fmt.allocPrint( + arena, + "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-macos-any", + .{zig_lib_dir}, + ); + + return LibCDirs{ + .libc_include_dir_list = list, + .libc_framework_dir_list = &[0][]u8{}, .libc_installation = null, - } - else - getZigShippedLibCIncludeDirsDarwin(arena, zig_lib_dir, target), + }; + }, else => { const generic_name = target_util.libCGenericName(target); // Some architectures are handled by the same set of headers. @@ -5034,6 +5082,7 @@ fn detectLibCFromBuilding( return LibCDirs{ .libc_include_dir_list = list, + .libc_framework_dir_list = &[0][]u8{}, .libc_installation = null, }; }, diff --git a/src/libc_installation.zig b/src/libc_installation.zig index 8466087e15c0..67d0bfb3a4fa 100644 --- a/src/libc_installation.zig +++ b/src/libc_installation.zig @@ -11,6 +11,7 @@ const is_haiku = builtin.target.os.tag == .haiku; const log = std.log.scoped(.libc_installation); +const DarwinSDK = std.zig.system.darwin.DarwinSDK; const ZigWindowsSDK = @import("windows_sdk.zig").ZigWindowsSDK; /// See the render function implementation for documentation of the fields. @@ -21,6 +22,7 @@ pub const LibCInstallation = struct { msvc_lib_dir: ?[]const u8 = null, kernel32_lib_dir: ?[]const u8 = null, gcc_dir: ?[]const u8 = null, + framework_dir: ?[]const u8 = null, pub const FindError = error{ OutOfMemory, @@ -35,6 +37,7 @@ pub const LibCInstallation = struct { UnsupportedArchitecture, WindowsSdkNotFound, ZigIsTheCCompiler, + DarwinSdkNotFound, }; pub fn parse( @@ -121,6 +124,11 @@ pub const LibCInstallation = struct { return error.ParseError; } + if (self.framework_dir == null and target.isDarwin()) { + log.err("framework_dir may not be empty for {s}\n", .{@tagName(os_tag)}); + return error.ParseError; + } + return self; } @@ -132,6 +140,7 @@ pub const LibCInstallation = struct { const msvc_lib_dir = self.msvc_lib_dir orelse ""; const kernel32_lib_dir = self.kernel32_lib_dir orelse ""; const gcc_dir = self.gcc_dir orelse ""; + const framework_dir = self.framework_dir orelse ""; try out.print( \\# The directory that contains `stdlib.h`. @@ -160,6 +169,8 @@ pub const LibCInstallation = struct { \\# Only needed when targeting Haiku. \\gcc_dir={s} \\ + \\# The directory that contains system frameworks (Darwin only). + \\framework_dir={s} , .{ include_dir, sys_include_dir, @@ -167,6 +178,7 @@ pub const LibCInstallation = struct { msvc_lib_dir, kernel32_lib_dir, gcc_dir, + framework_dir, }); } @@ -175,6 +187,9 @@ pub const LibCInstallation = struct { /// If enabled, will print human-friendly errors to stderr. verbose: bool = false, + + /// Darwin SDK structure. + darwin_sdk: ?DarwinSDK = null, }; /// Finds the default, native libc. @@ -182,7 +197,23 @@ pub const LibCInstallation = struct { var self: LibCInstallation = .{}; if (is_darwin) { - @panic("Darwin is handled separately via std.zig.system.darwin module"); + if (args.darwin_sdk) |sdk| { + const is_pre_11 = builtin.target.os.version_range.semver.min.major < 11; + const include_path = if (is_pre_11) "/usr/local/include" else "/usr/include"; + const framework_path = if (is_pre_11) "/Library/Frameworks" else "/System/Library/Frameworks"; + self.include_dir = try std.fmt.allocPrintZ(args.allocator, "{s}{s}", .{ + sdk.path, + include_path, + }); + self.sys_include_dir = try args.allocator.dupeZ(u8, self.include_dir.?); + self.framework_dir = try std.fmt.allocPrintZ(args.allocator, "{s}{s}", .{ + sdk.path, + framework_path, + }); + } else if (std.process.can_spawn) { + try self.findNativeIncludeDirPosix(args); + // TODO find framework dir using clang + } } else if (is_windows) { if (!build_options.have_llvm) return error.WindowsSdkNotFound; diff --git a/src/link.zig b/src/link.zig index 703dfb87394b..c95fc121d9a4 100644 --- a/src/link.zig +++ b/src/link.zig @@ -213,8 +213,8 @@ pub const Options = struct { /// (Zig compiler development) Enable dumping of linker's state as JSON. enable_link_snapshots: bool = false, - /// (Darwin) Path and version of the native SDK if detected. - native_darwin_sdk: ?std.zig.system.darwin.DarwinSDK = null, + /// (Darwin) Version of the native SDK if detected. + darwin_sdk_version: ?std.SemanticVersion = null, /// (Darwin) Install name for the dylib install_name: ?[]const u8 = null, diff --git a/src/link/MachO/load_commands.zig b/src/link/MachO/load_commands.zig index 10f446f19159..c110e46cf079 100644 --- a/src/link/MachO/load_commands.zig +++ b/src/link/MachO/load_commands.zig @@ -278,8 +278,7 @@ pub fn writeBuildVersionLC(options: *const link.Options, lc_writer: anytype) !vo const platform_version = @as(u32, @intCast(ver.major << 16 | ver.minor << 8)); break :blk platform_version; }; - const sdk_version = if (options.native_darwin_sdk) |sdk| blk: { - const ver = sdk.version; + const sdk_version = if (options.darwin_sdk_version) |ver| blk: { const sdk_version = @as(u32, @intCast(ver.major << 16 | ver.minor << 8)); break :blk sdk_version; } else platform_version; diff --git a/src/main.zig b/src/main.zig index 39a7adc42446..7e3a2eaf0ff1 100644 --- a/src/main.zig +++ b/src/main.zig @@ -769,7 +769,7 @@ fn buildOutputType( var link_libc = false; var link_libcpp = false; var link_libunwind = false; - var want_native_include_dirs = false; + var want_native_include_dirs: ?bool = null; var want_pic: ?bool = null; var want_pie: ?bool = null; var want_lto: ?bool = null; @@ -849,7 +849,6 @@ fn buildOutputType( var minor_subsystem_version: ?u32 = null; var wasi_exec_model: ?std.builtin.WasiExecModel = null; var enable_link_snapshots: bool = false; - var native_darwin_sdk: ?std.zig.system.darwin.DarwinSDK = null; var install_name: ?[]const u8 = null; var hash_style: link.HashStyle = .both; var entitlements: ?[]const u8 = null; @@ -2569,58 +2568,6 @@ fn buildOutputType( } } - if (comptime builtin.target.isDarwin()) { - // If we want to link against frameworks, we need system headers. - if (framework_dirs.items.len > 0 or frameworks.count() > 0) - want_native_include_dirs = true; - } - - if (sysroot == null and cross_target.isNativeOs() and - (system_libs.count() != 0 or want_native_include_dirs)) - { - const paths = std.zig.system.NativePaths.detect(arena, target_info) catch |err| { - fatal("unable to detect native system paths: {s}", .{@errorName(err)}); - }; - for (paths.warnings.items) |warning| { - warn("{s}", .{warning}); - } - - const has_sysroot = if (comptime builtin.target.isDarwin()) outer: { - if (std.zig.system.darwin.isDarwinSDKInstalled(arena)) { - const sdk = std.zig.system.darwin.getDarwinSDK(arena, target_info.target) orelse - break :outer false; - native_darwin_sdk = sdk; - try clang_argv.ensureUnusedCapacity(2); - clang_argv.appendAssumeCapacity("-isysroot"); - clang_argv.appendAssumeCapacity(sdk.path); - break :outer true; - } else break :outer false; - } else false; - - try clang_argv.ensureUnusedCapacity(paths.include_dirs.items.len * 2); - const isystem_flag = if (has_sysroot) "-iwithsysroot" else "-isystem"; - for (paths.include_dirs.items) |include_dir| { - clang_argv.appendAssumeCapacity(isystem_flag); - clang_argv.appendAssumeCapacity(include_dir); - } - - try clang_argv.ensureUnusedCapacity(paths.framework_dirs.items.len * 2); - try framework_dirs.ensureUnusedCapacity(paths.framework_dirs.items.len); - const iframework_flag = if (has_sysroot) "-iframeworkwithsysroot" else "-iframework"; - for (paths.framework_dirs.items) |framework_dir| { - clang_argv.appendAssumeCapacity(iframework_flag); - clang_argv.appendAssumeCapacity(framework_dir); - framework_dirs.appendAssumeCapacity(framework_dir); - } - - for (paths.lib_dirs.items) |lib_dir| { - try lib_dirs.append(lib_dir); - } - for (paths.rpaths.items) |rpath| { - try rpath_list.append(rpath); - } - } - { // Resolve static libraries into full paths. const sep = fs.path.sep_str; @@ -3092,6 +3039,7 @@ fn buildOutputType( .link_libc = link_libc, .link_libcpp = link_libcpp, .link_libunwind = link_libunwind, + .want_native_include_dirs = want_native_include_dirs, .want_pic = want_pic, .want_pie = want_pie, .want_lto = want_lto, @@ -3192,7 +3140,6 @@ fn buildOutputType( .wasi_exec_model = wasi_exec_model, .debug_compile_errors = debug_compile_errors, .enable_link_snapshots = enable_link_snapshots, - .native_darwin_sdk = native_darwin_sdk, .install_name = install_name, .entitlements = entitlements, .pagezero_size = pagezero_size, @@ -4070,8 +4017,17 @@ pub fn cmdLibC(gpa: Allocator, args: []const []const u8) !void { fatal("unable to detect libc for non-native target", .{}); } + var darwin_sdk = if ((comptime builtin.target.isDarwin()) and + !std.zig.system.NativePaths.isNix() and + std.zig.system.darwin.isDarwinSDKInstalled(gpa)) + std.zig.system.darwin.getDarwinSDK(gpa, builtin.target) + else + null; + defer if (darwin_sdk) |*sdk| sdk.deinit(gpa); + var libc = LibCInstallation.findNative(.{ .allocator = gpa, + .darwin_sdk = darwin_sdk, .verbose = true, }) catch |err| { fatal("unable to detect native libc: {s}", .{@errorName(err)}); From 91c3373e9fa8493af76c82c855921a135e4e9928 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Sun, 25 Jun 2023 21:32:38 +0200 Subject: [PATCH 02/15] macos: find include dir to frameworks using clang as fallback --- src/libc_installation.zig | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/libc_installation.zig b/src/libc_installation.zig index 67d0bfb3a4fa..1045cd214254 100644 --- a/src/libc_installation.zig +++ b/src/libc_installation.zig @@ -212,7 +212,6 @@ pub const LibCInstallation = struct { }); } else if (std.process.can_spawn) { try self.findNativeIncludeDirPosix(args); - // TODO find framework dir using clang } } else if (is_windows) { if (!build_options.have_llvm) @@ -347,6 +346,19 @@ pub const LibCInstallation = struct { // search in reverse order const search_path_untrimmed = search_paths.items[search_paths.items.len - path_i - 1]; const search_path = std.mem.trimLeft(u8, search_path_untrimmed, " "); + + if (is_darwin and self.framework_dir == null) { + // Path to frameworks is reported as + // "/path/to/sdk/System/Library/Frameworks (framework directory)" + // and so we parse that before trying to open it as a directory due to the nonsensical + // suffix. + if (std.mem.indexOf(u8, search_path, "(framework directory)")) |pos| { + const framework_dir = std.mem.trimRight(u8, search_path[0..pos], " "); + self.framework_dir = try allocator.dupeZ(u8, framework_dir); + continue; + } + } + var search_dir = fs.cwd().openDir(search_path, .{}) catch |err| switch (err) { error.FileNotFound, error.NotDir, @@ -375,7 +387,11 @@ pub const LibCInstallation = struct { } } - if (self.include_dir != null and self.sys_include_dir != null) { + var success = self.include_dir != null and self.sys_include_dir != null; + if (is_darwin) { + success = success and self.framework_dir != null; + } + if (success) { // Success. return; } From aece56ea35f6b6be535112f7d4d839b680dcf6c4 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Sun, 25 Jun 2023 21:35:07 +0200 Subject: [PATCH 03/15] libc: remove unused error in libc_installation.zig --- src/Compilation.zig | 1 - src/libc_installation.zig | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 9b3c8308b904..92b6cedf11a4 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -4913,7 +4913,6 @@ fn detectLibCIncludeDirs( error.CCompilerCrashed, error.CCompilerCannotFindHeaders, error.UnableToSpawnCCompiler, - error.DarwinSdkNotFound, => |e| { // We tried to integrate with the native system C compiler, // however, it is not installed. So we must rely on our bundled diff --git a/src/libc_installation.zig b/src/libc_installation.zig index 1045cd214254..7661e3fa6c0e 100644 --- a/src/libc_installation.zig +++ b/src/libc_installation.zig @@ -37,7 +37,6 @@ pub const LibCInstallation = struct { UnsupportedArchitecture, WindowsSdkNotFound, ZigIsTheCCompiler, - DarwinSdkNotFound, }; pub fn parse( From b48c0873475b42c86d63505f49be433a4f7bf926 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 26 Jun 2023 08:57:54 +0200 Subject: [PATCH 04/15] main: fix missing type annotation --- src/main.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.zig b/src/main.zig index 7e3a2eaf0ff1..fbbe6313d2fe 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4017,7 +4017,7 @@ pub fn cmdLibC(gpa: Allocator, args: []const []const u8) !void { fatal("unable to detect libc for non-native target", .{}); } - var darwin_sdk = if ((comptime builtin.target.isDarwin()) and + var darwin_sdk: ?std.zig.system.darwin.DarwinSDK = if ((comptime builtin.target.isDarwin()) and !std.zig.system.NativePaths.isNix() and std.zig.system.darwin.isDarwinSDKInstalled(gpa)) std.zig.system.darwin.getDarwinSDK(gpa, builtin.target) From 03b7096738a4fcf56eeaf8b954d9de24f8e9d69a Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 26 Jun 2023 11:37:22 +0200 Subject: [PATCH 05/15] std: make NativePaths.isNix() error out on WASI with no libc --- lib/std/zig/system/NativePaths.zig | 26 ++++++++++++++++++++------ src/Compilation.zig | 2 +- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/lib/std/zig/system/NativePaths.zig b/lib/std/zig/system/NativePaths.zig index 1e8230a055b4..febcae30f78a 100644 --- a/lib/std/zig/system/NativePaths.zig +++ b/lib/std/zig/system/NativePaths.zig @@ -14,6 +14,9 @@ rpaths: ArrayList([:0]u8), warnings: ArrayList([:0]u8), pub fn isNix() bool { + if (builtin.os.tag == .wasi and !builtin.link_libc) { + @compileError("isNix check is not supported for WASI without libc"); + } return process.hasEnvVarConstant("NIX_CFLAGS_COMPILE") and process.hasEnvVarConstant("NIX_LDFLAGS"); } @@ -27,10 +30,11 @@ pub fn detect(allocator: Allocator, native_target: std.Target) !NativePaths { }; errdefer self.deinit(); - if (isNix()) { - const nix_cflags_compile = try process.getEnvVarOwned(allocator, "NIX_CFLAGS_COMPILE"); + var is_nix = false; + if (process.getEnvVarOwned(allocator, "NIX_CFLAGS_COMPILE")) |nix_cflags_compile| { defer allocator.free(nix_cflags_compile); + is_nix = true; var it = mem.tokenizeScalar(u8, nix_cflags_compile, ' '); while (true) { const word = it.next() orelse break; @@ -53,11 +57,16 @@ pub fn detect(allocator: Allocator, native_target: std.Target) !NativePaths { try self.addWarningFmt("Unrecognized C flag from NIX_CFLAGS_COMPILE: {s}", .{word}); } } - - const nix_ldflags = try process.getEnvVarOwned(allocator, "NIX_LDFLAGS"); + } else |err| switch (err) { + error.InvalidUtf8 => {}, + error.EnvironmentVariableNotFound => {}, + error.OutOfMemory => |e| return e, + } + if (process.getEnvVarOwned(allocator, "NIX_LDFLAGS")) |nix_ldflags| { defer allocator.free(nix_ldflags); - it = mem.tokenizeScalar(u8, nix_ldflags, ' '); + is_nix = true; + var it = mem.tokenizeScalar(u8, nix_ldflags, ' '); while (true) { const word = it.next() orelse break; if (mem.eql(u8, word, "-rpath")) { @@ -75,7 +84,12 @@ pub fn detect(allocator: Allocator, native_target: std.Target) !NativePaths { break; } } - + } else |err| switch (err) { + error.InvalidUtf8 => {}, + error.EnvironmentVariableNotFound => {}, + error.OutOfMemory => |e| return e, + } + if (is_nix) { return self; } diff --git a/src/Compilation.zig b/src/Compilation.zig index 92b6cedf11a4..3b80e3f21919 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -900,7 +900,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { try rpath_list.append(rpath); } - if (!std.zig.system.NativePaths.isNix() and std.zig.system.darwin.isDarwinSDKInstalled(arena)) { + if (darwin_native and !std.zig.system.NativePaths.isNix() and std.zig.system.darwin.isDarwinSDKInstalled(arena)) { darwin_sdk = std.zig.system.darwin.getDarwinSDK(arena, options.target); } } From 8dbddb9dc93eb909c11414ee747d39a7a5cbca59 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 27 Jun 2023 10:47:46 +0200 Subject: [PATCH 06/15] comp: move native include dirs detection to addCCArgs --- src/Compilation.zig | 68 ++++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 3b80e3f21919..ba6c14af1985 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -106,6 +106,7 @@ job_queued_compiler_rt_obj: bool = false, alloc_failure_occurred: bool = false, formatted_panics: bool = false, last_update_was_cache_hit: bool = false, +want_native_paths: bool = false, c_source_files: []const CSourceFile, clang_argv: []const []const u8, @@ -855,22 +856,28 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { break :blk false; }; - var clang_argv = std.ArrayList([]const u8).init(arena); var lib_dirs = std.ArrayList([]const u8).init(arena); var framework_dirs = std.ArrayList([]const u8).init(arena); var rpath_list = std.ArrayList([]const u8).init(arena); - try clang_argv.appendSlice(options.clang_argv); try lib_dirs.appendSlice(options.lib_dirs); try framework_dirs.appendSlice(options.framework_dirs); try rpath_list.appendSlice(options.rpath_list); - const want_native_include_dirs = options.sysroot == null and options.is_native_os and - (options.system_lib_names.len > 0 or options.want_native_include_dirs orelse false); - const darwin_native = (comptime builtin.target.isDarwin()) and options.target.isDarwin() and options.sysroot == null; - var darwin_sdk: ?std.zig.system.darwin.DarwinSDK = null; + const darwin_native = (comptime builtin.target.isDarwin()) and options.target.isDarwin() and + options.sysroot == null; + comp.want_native_paths = (options.sysroot == null and options.is_native_os and + (options.system_lib_names.len > 0 or options.want_native_include_dirs orelse false)) or + darwin_native; - if (want_native_include_dirs or darwin_native) { + const darwin_sdk: ?std.zig.system.darwin.DarwinSDK = if (darwin_native and + !std.zig.system.NativePaths.isNix() and + std.zig.system.darwin.isDarwinSDKInstalled(arena)) + std.zig.system.darwin.getDarwinSDK(arena, options.target) + else + null; + + if (comp.want_native_paths) { const paths = std.zig.system.NativePaths.detect(arena, options.target) catch |err| { fatal("unable to detect native system paths: {s}", .{@errorName(err)}); }; @@ -879,17 +886,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { std.log.warn("{s}", .{warning}); } - try clang_argv.ensureUnusedCapacity(paths.include_dirs.items.len * 2); - for (paths.include_dirs.items) |include_dir| { - clang_argv.appendAssumeCapacity("-isystem"); - clang_argv.appendAssumeCapacity(include_dir); - } - - try clang_argv.ensureUnusedCapacity(paths.framework_dirs.items.len * 2); try framework_dirs.ensureUnusedCapacity(paths.framework_dirs.items.len); for (paths.framework_dirs.items) |framework_dir| { - clang_argv.appendAssumeCapacity("-iframework"); - clang_argv.appendAssumeCapacity(framework_dir); framework_dirs.appendAssumeCapacity(framework_dir); } @@ -899,10 +897,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { for (paths.rpaths.items) |rpath| { try rpath_list.append(rpath); } - - if (darwin_native and !std.zig.system.NativePaths.isNix() and std.zig.system.darwin.isDarwinSDKInstalled(arena)) { - darwin_sdk = std.zig.system.darwin.getDarwinSDK(arena, options.target); - } } const sysroot = blk: { @@ -1642,7 +1636,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .embed_file_work_queue = std.fifo.LinearFifo(*Module.EmbedFile, .Dynamic).init(gpa), .keep_source_files_loaded = options.keep_source_files_loaded, .use_clang = use_clang, - .clang_argv = clang_argv.items, + .clang_argv = options.clang_argv, .c_source_files = options.c_source_files, .cache_parent = cache, .self_exe_path = options.self_exe_path, @@ -4397,14 +4391,38 @@ pub fn addCCArgs( try argv.append("-isystem"); try argv.append(c_headers_dir); + if (comp.want_native_paths) { + const paths = std.zig.system.NativePaths.detect(arena, target) catch |err| { + fatal("unable to detect native system paths: {s}", .{@errorName(err)}); + }; + + for (paths.warnings.items) |warning| { + std.log.warn("{s}", .{warning}); + } + + try argv.ensureUnusedCapacity(paths.include_dirs.items.len * 2); + for (paths.include_dirs.items) |include_dir| { + argv.appendAssumeCapacity("-isystem"); + argv.appendAssumeCapacity(include_dir); + } + + try argv.ensureUnusedCapacity(paths.framework_dirs.items.len * 2); + for (paths.framework_dirs.items) |framework_dir| { + argv.appendAssumeCapacity("-iframework"); + argv.appendAssumeCapacity(framework_dir); + } + } + + try argv.ensureUnusedCapacity(comp.libc_include_dir_list.len * 2); for (comp.libc_include_dir_list) |include_dir| { - try argv.append("-isystem"); - try argv.append(include_dir); + argv.appendAssumeCapacity("-isystem"); + argv.appendAssumeCapacity(include_dir); } + try argv.ensureUnusedCapacity(comp.libc_framework_dir_list.len * 2); for (comp.libc_framework_dir_list) |framework_dir| { - try argv.append("-iframework"); - try argv.append(framework_dir); + argv.appendAssumeCapacity("-iframework"); + argv.appendAssumeCapacity(framework_dir); } if (target.cpu.model.llvm_name) |llvm_name| { From 7774af94d43943d0c259640f5a27ec919ee2cb1b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 27 Jun 2023 11:07:31 +0200 Subject: [PATCH 07/15] comp: move native libc dirs detection to detectLibCDirs --- src/Compilation.zig | 96 ++++++++++++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 36 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index ba6c14af1985..652ed70bceb9 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -856,14 +856,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { break :blk false; }; - var lib_dirs = std.ArrayList([]const u8).init(arena); - var framework_dirs = std.ArrayList([]const u8).init(arena); - var rpath_list = std.ArrayList([]const u8).init(arena); - - try lib_dirs.appendSlice(options.lib_dirs); - try framework_dirs.appendSlice(options.framework_dirs); - try rpath_list.appendSlice(options.rpath_list); - const darwin_native = (comptime builtin.target.isDarwin()) and options.target.isDarwin() and options.sysroot == null; comp.want_native_paths = (options.sysroot == null and options.is_native_os and @@ -877,28 +869,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { else null; - if (comp.want_native_paths) { - const paths = std.zig.system.NativePaths.detect(arena, options.target) catch |err| { - fatal("unable to detect native system paths: {s}", .{@errorName(err)}); - }; - - for (paths.warnings.items) |warning| { - std.log.warn("{s}", .{warning}); - } - - try framework_dirs.ensureUnusedCapacity(paths.framework_dirs.items.len); - for (paths.framework_dirs.items) |framework_dir| { - framework_dirs.appendAssumeCapacity(framework_dir); - } - - for (paths.lib_dirs.items) |lib_dir| { - try lib_dirs.append(lib_dir); - } - for (paths.rpaths.items) |rpath| { - try rpath_list.append(rpath); - } - } - const sysroot = blk: { if (options.sysroot) |sysroot| break :blk sysroot; if (darwin_sdk) |sdk| break :blk sdk.path; @@ -981,7 +951,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const dll_export_fns = options.dll_export_fns orelse (is_dyn_lib or options.rdynamic); - const libc_dirs = try detectLibCIncludeDirs( + const libc_dirs = try comp.detectLibCDirs( arena, options.zig_lib_directory.path.?, options.target, @@ -1497,6 +1467,18 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { system_libs.putAssumeCapacity(lib_name, options.system_lib_infos[i]); } + var lib_dirs = std.ArrayList([]const u8).init(arena); + try lib_dirs.appendSlice(options.lib_dirs); + try lib_dirs.appendSlice(libc_dirs.libc_lib_dir_list); + + var framework_dirs = std.ArrayList([]const u8).init(arena); + try framework_dirs.appendSlice(options.framework_dirs); + try framework_dirs.appendSlice(libc_dirs.libc_framework_dir_list); + + var rpath_list = std.ArrayList([]const u8).init(arena); + try rpath_list.appendSlice(options.rpath_list); + try rpath_list.appendSlice(libc_dirs.libc_rpath_list); + const bin_file = try link.File.openPath(gpa, .{ .emit = bin_file_emit, .implib_emit = implib_emit, @@ -4894,11 +4876,14 @@ test "classifyFileExt" { const LibCDirs = struct { libc_include_dir_list: []const []const u8, + libc_lib_dir_list: []const []const u8, libc_framework_dir_list: []const []const u8, + libc_rpath_list: []const []const u8, libc_installation: ?*const LibCInstallation, }; -fn detectLibCIncludeDirs( +fn detectLibCDirs( + comp: *Compilation, arena: Allocator, zig_lib_dir: []const u8, target: Target, @@ -4910,13 +4895,15 @@ fn detectLibCIncludeDirs( if (!link_libc) { return LibCDirs{ .libc_include_dir_list = &[0][]u8{}, + .libc_lib_dir_list = &[0][]u8{}, .libc_framework_dir_list = &[0][]u8{}, + .libc_rpath_list = &[0][]u8{}, .libc_installation = null, }; } if (libc_installation) |lci| { - return detectLibCFromLibCInstallation(arena, target, lci); + return comp.detectLibCFromLibCInstallation(arena, target, lci); } // If linking system libraries and targeting the native abi, default to @@ -4942,7 +4929,7 @@ fn detectLibCIncludeDirs( }, else => |e| return e, }; - return detectLibCFromLibCInstallation(arena, target, libc); + return comp.detectLibCFromLibCInstallation(arena, target, libc); } // If not linking system libraries, build and provide our own libc by @@ -4967,19 +4954,28 @@ fn detectLibCIncludeDirs( .darwin_sdk = darwin_sdk, .verbose = true, }); - return detectLibCFromLibCInstallation(arena, target, libc); + return comp.detectLibCFromLibCInstallation(arena, target, libc); } return LibCDirs{ .libc_include_dir_list = &[0][]u8{}, + .libc_lib_dir_list = &[0][]u8{}, .libc_framework_dir_list = &[0][]u8{}, + .libc_rpath_list = &[0][]u8{}, .libc_installation = null, }; } -fn detectLibCFromLibCInstallation(arena: Allocator, target: Target, lci: *const LibCInstallation) !LibCDirs { +fn detectLibCFromLibCInstallation( + comp: *Compilation, + arena: Allocator, + target: Target, + lci: *const LibCInstallation, +) !LibCDirs { var list = try std.ArrayList([]const u8).initCapacity(arena, 5); + var lib_dir_list = std.ArrayList([]const u8).init(arena); var framework_list = std.ArrayList([]const u8).init(arena); + var rpath_list = std.ArrayList([]const u8).init(arena); list.appendAssumeCapacity(lci.include_dir.?); @@ -5011,9 +5007,33 @@ fn detectLibCFromLibCInstallation(arena: Allocator, target: Target, lci: *const list.appendAssumeCapacity(config_dir); } + if (comp.want_native_paths) { + const paths = std.zig.system.NativePaths.detect(arena, target) catch |err| { + fatal("unable to detect native system paths: {s}", .{@errorName(err)}); + }; + + for (paths.warnings.items) |warning| { + std.log.warn("{s}", .{warning}); + } + + try framework_list.ensureUnusedCapacity(paths.framework_dirs.items.len); + for (paths.framework_dirs.items) |framework_dir| { + framework_list.appendAssumeCapacity(framework_dir); + } + + for (paths.lib_dirs.items) |lib_dir| { + try lib_dir_list.append(lib_dir); + } + for (paths.rpaths.items) |rpath| { + try rpath_list.append(rpath); + } + } + return LibCDirs{ .libc_include_dir_list = list.items, + .libc_lib_dir_list = lib_dir_list.items, .libc_framework_dir_list = framework_list.items, + .libc_rpath_list = rpath_list.items, .libc_installation = lci, }; } @@ -5047,7 +5067,9 @@ fn detectLibCFromBuilding(arena: Allocator, zig_lib_dir: []const u8, target: std return LibCDirs{ .libc_include_dir_list = list, + .libc_lib_dir_list = &[0][]u8{}, .libc_framework_dir_list = &[0][]u8{}, + .libc_rpath_list = &[0][]u8{}, .libc_installation = null, }; }, @@ -5099,7 +5121,9 @@ fn detectLibCFromBuilding(arena: Allocator, zig_lib_dir: []const u8, target: std return LibCDirs{ .libc_include_dir_list = list, + .libc_lib_dir_list = &[0][]u8{}, .libc_framework_dir_list = &[0][]u8{}, + .libc_rpath_list = &[0][]u8{}, .libc_installation = null, }; }, From 6d0f8319b3f37c7c23e425c89e34cec3143f3e12 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 28 Jun 2023 09:56:43 +0200 Subject: [PATCH 08/15] comp: move all of the logic into detectLibCFromLibCInstallation --- src/Compilation.zig | 57 +++++++++++++++++---------------------------- 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 652ed70bceb9..329d01324a80 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -106,7 +106,6 @@ job_queued_compiler_rt_obj: bool = false, alloc_failure_occurred: bool = false, formatted_panics: bool = false, last_update_was_cache_hit: bool = false, -want_native_paths: bool = false, c_source_files: []const CSourceFile, clang_argv: []const []const u8, @@ -858,10 +857,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const darwin_native = (comptime builtin.target.isDarwin()) and options.target.isDarwin() and options.sysroot == null; - comp.want_native_paths = (options.sysroot == null and options.is_native_os and - (options.system_lib_names.len > 0 or options.want_native_include_dirs orelse false)) or - darwin_native; - const darwin_sdk: ?std.zig.system.darwin.DarwinSDK = if (darwin_native and !std.zig.system.NativePaths.isNix() and std.zig.system.darwin.isDarwinSDKInstalled(arena)) @@ -951,13 +946,17 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const dll_export_fns = options.dll_export_fns orelse (is_dyn_lib or options.rdynamic); - const libc_dirs = try comp.detectLibCDirs( + const want_native_paths = (options.sysroot == null and options.is_native_os and + (options.system_lib_names.len > 0 or options.want_native_include_dirs orelse false)) or + darwin_native; + const libc_dirs = try detectLibCDirs( arena, options.zig_lib_directory.path.?, options.target, options.is_native_abi, link_libc, options.libc_installation, + want_native_paths, darwin_sdk, ); @@ -4373,28 +4372,6 @@ pub fn addCCArgs( try argv.append("-isystem"); try argv.append(c_headers_dir); - if (comp.want_native_paths) { - const paths = std.zig.system.NativePaths.detect(arena, target) catch |err| { - fatal("unable to detect native system paths: {s}", .{@errorName(err)}); - }; - - for (paths.warnings.items) |warning| { - std.log.warn("{s}", .{warning}); - } - - try argv.ensureUnusedCapacity(paths.include_dirs.items.len * 2); - for (paths.include_dirs.items) |include_dir| { - argv.appendAssumeCapacity("-isystem"); - argv.appendAssumeCapacity(include_dir); - } - - try argv.ensureUnusedCapacity(paths.framework_dirs.items.len * 2); - for (paths.framework_dirs.items) |framework_dir| { - argv.appendAssumeCapacity("-iframework"); - argv.appendAssumeCapacity(framework_dir); - } - } - try argv.ensureUnusedCapacity(comp.libc_include_dir_list.len * 2); for (comp.libc_include_dir_list) |include_dir| { argv.appendAssumeCapacity("-isystem"); @@ -4883,13 +4860,13 @@ const LibCDirs = struct { }; fn detectLibCDirs( - comp: *Compilation, arena: Allocator, zig_lib_dir: []const u8, target: Target, is_native_abi: bool, link_libc: bool, libc_installation: ?*const LibCInstallation, + want_native_paths: bool, darwin_sdk: ?std.zig.system.darwin.DarwinSDK, ) !LibCDirs { if (!link_libc) { @@ -4903,7 +4880,7 @@ fn detectLibCDirs( } if (libc_installation) |lci| { - return comp.detectLibCFromLibCInstallation(arena, target, lci); + return detectLibCFromLibCInstallation(arena, target, lci, want_native_paths); } // If linking system libraries and targeting the native abi, default to @@ -4929,7 +4906,7 @@ fn detectLibCDirs( }, else => |e| return e, }; - return comp.detectLibCFromLibCInstallation(arena, target, libc); + return detectLibCFromLibCInstallation(arena, target, libc, want_native_paths); } // If not linking system libraries, build and provide our own libc by @@ -4954,7 +4931,7 @@ fn detectLibCDirs( .darwin_sdk = darwin_sdk, .verbose = true, }); - return comp.detectLibCFromLibCInstallation(arena, target, libc); + return detectLibCFromLibCInstallation(arena, target, libc, want_native_paths); } return LibCDirs{ @@ -4967,10 +4944,10 @@ fn detectLibCDirs( } fn detectLibCFromLibCInstallation( - comp: *Compilation, arena: Allocator, target: Target, lci: *const LibCInstallation, + want_native_paths: bool, ) !LibCDirs { var list = try std.ArrayList([]const u8).initCapacity(arena, 5); var lib_dir_list = std.ArrayList([]const u8).init(arena); @@ -5007,7 +4984,7 @@ fn detectLibCFromLibCInstallation( list.appendAssumeCapacity(config_dir); } - if (comp.want_native_paths) { + if (want_native_paths) { const paths = std.zig.system.NativePaths.detect(arena, target) catch |err| { fatal("unable to detect native system paths: {s}", .{@errorName(err)}); }; @@ -5016,16 +4993,24 @@ fn detectLibCFromLibCInstallation( std.log.warn("{s}", .{warning}); } + try list.ensureUnusedCapacity(paths.include_dirs.items.len); + for (paths.include_dirs.items) |include_dir| { + list.appendAssumeCapacity(include_dir); + } + try framework_list.ensureUnusedCapacity(paths.framework_dirs.items.len); for (paths.framework_dirs.items) |framework_dir| { framework_list.appendAssumeCapacity(framework_dir); } + try lib_dir_list.ensureUnusedCapacity(paths.lib_dirs.items.len); for (paths.lib_dirs.items) |lib_dir| { - try lib_dir_list.append(lib_dir); + lib_dir_list.appendAssumeCapacity(lib_dir); } + + try rpath_list.ensureUnusedCapacity(paths.rpaths.items.len); for (paths.rpaths.items) |rpath| { - try rpath_list.append(rpath); + rpath_list.appendAssumeCapacity(rpath); } } From adc61913555cebe82bbf19b4302f8ae2e142076c Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 27 Jul 2023 21:53:42 +0200 Subject: [PATCH 09/15] wip: move fetching native paths to spots which actually need them --- src/Compilation.zig | 102 +++++++++---------------------- src/link.zig | 2 + src/link/MachO.zig | 10 ++- src/link/MachO/load_commands.zig | 4 +- src/link/MachO/zld.zig | 34 ++++++++--- 5 files changed, 67 insertions(+), 85 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 329d01324a80..dea92dc59e4a 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -863,6 +863,9 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { std.zig.system.darwin.getDarwinSDK(arena, options.target) else null; + const want_native_paths = (options.sysroot == null and options.is_native_os and + (options.system_lib_names.len > 0 or options.want_native_include_dirs orelse false)) or + darwin_native; const sysroot = blk: { if (options.sysroot) |sysroot| break :blk sysroot; @@ -946,9 +949,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const dll_export_fns = options.dll_export_fns orelse (is_dyn_lib or options.rdynamic); - const want_native_paths = (options.sysroot == null and options.is_native_os and - (options.system_lib_names.len > 0 or options.want_native_include_dirs orelse false)) or - darwin_native; const libc_dirs = try detectLibCDirs( arena, options.zig_lib_directory.path.?, @@ -956,7 +956,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { options.is_native_abi, link_libc, options.libc_installation, - want_native_paths, darwin_sdk, ); @@ -1466,18 +1465,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { system_libs.putAssumeCapacity(lib_name, options.system_lib_infos[i]); } - var lib_dirs = std.ArrayList([]const u8).init(arena); - try lib_dirs.appendSlice(options.lib_dirs); - try lib_dirs.appendSlice(libc_dirs.libc_lib_dir_list); - - var framework_dirs = std.ArrayList([]const u8).init(arena); - try framework_dirs.appendSlice(options.framework_dirs); - try framework_dirs.appendSlice(libc_dirs.libc_framework_dir_list); - - var rpath_list = std.ArrayList([]const u8).init(arena); - try rpath_list.appendSlice(options.rpath_list); - try rpath_list.appendSlice(libc_dirs.libc_rpath_list); - const bin_file = try link.File.openPath(gpa, .{ .emit = bin_file_emit, .implib_emit = implib_emit, @@ -1498,11 +1485,11 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .link_libunwind = link_libunwind, .objects = options.link_objects, .frameworks = options.frameworks, - .framework_dirs = framework_dirs.items, + .framework_dirs = options.framework_dirs, .system_libs = system_libs, .wasi_emulated_libs = options.wasi_emulated_libs, - .lib_dirs = lib_dirs.items, - .rpath_list = rpath_list.items, + .lib_dirs = options.lib_dirs, + .rpath_list = options.rpath_list, .symbol_wrap_set = options.symbol_wrap_set, .strip = strip, .is_native_os = options.is_native_os, @@ -1595,6 +1582,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .force_undefined_symbols = options.force_undefined_symbols, .pdb_source_path = options.pdb_source_path, .pdb_out_path = options.pdb_out_path, + .want_native_paths = want_native_paths, }); errdefer bin_file.destroy(); comp.* = .{ @@ -4384,6 +4372,24 @@ pub fn addCCArgs( argv.appendAssumeCapacity(framework_dir); } + if (comp.bin_file.options.want_native_paths) { + const paths = std.zig.system.NativePaths.detect(arena, target) catch |err| { + fatal("unable to detect native system paths: {s}", .{@errorName(err)}); + }; + + try argv.ensureUnusedCapacity(paths.include_dirs.items.len * 2); + for (paths.include_dirs.items) |include_dir| { + argv.appendAssumeCapacity("-isystem"); + argv.appendAssumeCapacity(include_dir); + } + + try argv.ensureUnusedCapacity(paths.framework_dirs.items.len * 2); + for (paths.framework_dirs.items) |framework_dir| { + argv.appendAssumeCapacity("-iframework"); + argv.appendAssumeCapacity(framework_dir); + } + } + if (target.cpu.model.llvm_name) |llvm_name| { try argv.appendSlice(&[_][]const u8{ "-Xclang", "-target-cpu", "-Xclang", llvm_name, @@ -4853,9 +4859,7 @@ test "classifyFileExt" { const LibCDirs = struct { libc_include_dir_list: []const []const u8, - libc_lib_dir_list: []const []const u8, libc_framework_dir_list: []const []const u8, - libc_rpath_list: []const []const u8, libc_installation: ?*const LibCInstallation, }; @@ -4866,21 +4870,18 @@ fn detectLibCDirs( is_native_abi: bool, link_libc: bool, libc_installation: ?*const LibCInstallation, - want_native_paths: bool, darwin_sdk: ?std.zig.system.darwin.DarwinSDK, ) !LibCDirs { if (!link_libc) { return LibCDirs{ .libc_include_dir_list = &[0][]u8{}, - .libc_lib_dir_list = &[0][]u8{}, .libc_framework_dir_list = &[0][]u8{}, - .libc_rpath_list = &[0][]u8{}, .libc_installation = null, }; } if (libc_installation) |lci| { - return detectLibCFromLibCInstallation(arena, target, lci, want_native_paths); + return detectLibCFromLibCInstallation(arena, target, lci); } // If linking system libraries and targeting the native abi, default to @@ -4906,7 +4907,7 @@ fn detectLibCDirs( }, else => |e| return e, }; - return detectLibCFromLibCInstallation(arena, target, libc, want_native_paths); + return detectLibCFromLibCInstallation(arena, target, libc); } // If not linking system libraries, build and provide our own libc by @@ -4931,28 +4932,19 @@ fn detectLibCDirs( .darwin_sdk = darwin_sdk, .verbose = true, }); - return detectLibCFromLibCInstallation(arena, target, libc, want_native_paths); + return detectLibCFromLibCInstallation(arena, target, libc); } return LibCDirs{ .libc_include_dir_list = &[0][]u8{}, - .libc_lib_dir_list = &[0][]u8{}, .libc_framework_dir_list = &[0][]u8{}, - .libc_rpath_list = &[0][]u8{}, .libc_installation = null, }; } -fn detectLibCFromLibCInstallation( - arena: Allocator, - target: Target, - lci: *const LibCInstallation, - want_native_paths: bool, -) !LibCDirs { +fn detectLibCFromLibCInstallation(arena: Allocator, target: Target, lci: *const LibCInstallation) !LibCDirs { var list = try std.ArrayList([]const u8).initCapacity(arena, 5); - var lib_dir_list = std.ArrayList([]const u8).init(arena); var framework_list = std.ArrayList([]const u8).init(arena); - var rpath_list = std.ArrayList([]const u8).init(arena); list.appendAssumeCapacity(lci.include_dir.?); @@ -4984,41 +4976,9 @@ fn detectLibCFromLibCInstallation( list.appendAssumeCapacity(config_dir); } - if (want_native_paths) { - const paths = std.zig.system.NativePaths.detect(arena, target) catch |err| { - fatal("unable to detect native system paths: {s}", .{@errorName(err)}); - }; - - for (paths.warnings.items) |warning| { - std.log.warn("{s}", .{warning}); - } - - try list.ensureUnusedCapacity(paths.include_dirs.items.len); - for (paths.include_dirs.items) |include_dir| { - list.appendAssumeCapacity(include_dir); - } - - try framework_list.ensureUnusedCapacity(paths.framework_dirs.items.len); - for (paths.framework_dirs.items) |framework_dir| { - framework_list.appendAssumeCapacity(framework_dir); - } - - try lib_dir_list.ensureUnusedCapacity(paths.lib_dirs.items.len); - for (paths.lib_dirs.items) |lib_dir| { - lib_dir_list.appendAssumeCapacity(lib_dir); - } - - try rpath_list.ensureUnusedCapacity(paths.rpaths.items.len); - for (paths.rpaths.items) |rpath| { - rpath_list.appendAssumeCapacity(rpath); - } - } - return LibCDirs{ .libc_include_dir_list = list.items, - .libc_lib_dir_list = lib_dir_list.items, .libc_framework_dir_list = framework_list.items, - .libc_rpath_list = rpath_list.items, .libc_installation = lci, }; } @@ -5052,9 +5012,7 @@ fn detectLibCFromBuilding(arena: Allocator, zig_lib_dir: []const u8, target: std return LibCDirs{ .libc_include_dir_list = list, - .libc_lib_dir_list = &[0][]u8{}, .libc_framework_dir_list = &[0][]u8{}, - .libc_rpath_list = &[0][]u8{}, .libc_installation = null, }; }, @@ -5106,9 +5064,7 @@ fn detectLibCFromBuilding(arena: Allocator, zig_lib_dir: []const u8, target: std return LibCDirs{ .libc_include_dir_list = list, - .libc_lib_dir_list = &[0][]u8{}, .libc_framework_dir_list = &[0][]u8{}, - .libc_rpath_list = &[0][]u8{}, .libc_installation = null, }; }, diff --git a/src/link.zig b/src/link.zig index c95fc121d9a4..afd9a556dd18 100644 --- a/src/link.zig +++ b/src/link.zig @@ -247,6 +247,8 @@ pub const Options = struct { /// (Windows) .def file to specify when linking module_definition_file: ?[]const u8 = null, + want_native_paths: bool = false, + pub fn effectiveOutputMode(options: Options) std.builtin.OutputMode { return if (options.use_lld) .Obj else options.output_mode; } diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 6953cda92977..274b70d4f6e4 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -498,8 +498,16 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No sub_prog_node.activate(); defer sub_prog_node.end(); + const options = &self.base.options; const module = self.base.options.module orelse return error.LinkingWithoutZigSourceUnimplemented; + var input_rpath_list = std.ArrayList([]const u8).init(arena); + + if (options.want_native_paths) { + const paths = try std.zig.system.NativePaths.detect(arena, options.target); + try input_rpath_list.appendSlice(paths.rpaths.items); + } + if (self.lazy_syms.getPtr(.none)) |metadata| { // Most lazy symbols can be updated on first use, but // anyerror needs to wait for everything to be flushed. @@ -751,7 +759,7 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No else => {}, } - try load_commands.writeRpathLCs(self.base.allocator, &self.base.options, lc_writer); + try load_commands.writeRpathLCs(self.base.allocator, input_rpath_list.items, lc_writer); try lc_writer.writeStruct(macho.source_version_command{ .version = 0, }); diff --git a/src/link/MachO/load_commands.zig b/src/link/MachO/load_commands.zig index c110e46cf079..6a91b9ff0d80 100644 --- a/src/link/MachO/load_commands.zig +++ b/src/link/MachO/load_commands.zig @@ -247,8 +247,8 @@ const RpathIterator = struct { } }; -pub fn writeRpathLCs(gpa: Allocator, options: *const link.Options, lc_writer: anytype) !void { - var it = RpathIterator.init(gpa, options.rpath_list); +pub fn writeRpathLCs(gpa: Allocator, rpaths: []const []const u8, lc_writer: anytype) !void { + var it = RpathIterator.init(gpa, rpaths); defer it.deinit(); while (try it.next()) |rpath| { diff --git a/src/link/MachO/zld.zig b/src/link/MachO/zld.zig index 3e828984a96d..fad192320eea 100644 --- a/src/link/MachO/zld.zig +++ b/src/link/MachO/zld.zig @@ -1,5 +1,6 @@ const std = @import("std"); const build_options = @import("build_options"); +const builtin = @import("builtin"); const assert = std.debug.assert; const dwarf = std.dwarf; const fs = std.fs; @@ -3348,6 +3349,21 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr const id_symlink_basename = "zld.id"; + var input_lib_dirs = std.ArrayList([]const u8).init(arena); + var input_framework_dirs = std.ArrayList([]const u8).init(arena); + var input_rpath_list = std.ArrayList([]const u8).init(arena); + + try input_lib_dirs.appendSlice(options.lib_dirs); + try input_framework_dirs.appendSlice(options.framework_dirs); + try input_rpath_list.appendSlice(options.rpath_list); + + if (options.want_native_paths) { + const paths = try std.zig.system.NativePaths.detect(arena, target); + try input_lib_dirs.appendSlice(paths.lib_dirs.items); + try input_framework_dirs.appendSlice(paths.framework_dirs.items); + try input_rpath_list.appendSlice(paths.rpaths.items); + } + var man: Cache.Manifest = undefined; defer if (!options.disable_lld_caching) man.deinit(); @@ -3379,10 +3395,10 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr man.hash.add(gc_sections); man.hash.add(options.dead_strip_dylibs); man.hash.add(options.strip); - man.hash.addListOfBytes(options.lib_dirs); - man.hash.addListOfBytes(options.framework_dirs); + man.hash.addListOfBytes(input_lib_dirs.items); + man.hash.addListOfBytes(input_framework_dirs.items); link.hashAddSystemLibs(&man.hash, options.frameworks); - man.hash.addListOfBytes(options.rpath_list); + man.hash.addListOfBytes(input_rpath_list.items); if (is_dyn_lib) { man.hash.addOptionalBytes(options.install_name); man.hash.addOptional(options.version); @@ -3534,7 +3550,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr } var lib_dirs = std.ArrayList([]const u8).init(arena); - for (options.lib_dirs) |dir| { + for (input_lib_dirs.items) |dir| { if (try MachO.resolveSearchDir(arena, dir, options.sysroot)) |search_dir| { try lib_dirs.append(search_dir); } else { @@ -3594,7 +3610,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr // frameworks var framework_dirs = std.ArrayList([]const u8).init(arena); - for (options.framework_dirs) |dir| { + for (input_framework_dirs.items) |dir| { if (try MachO.resolveSearchDir(arena, dir, options.sysroot)) |search_dir| { try framework_dirs.append(search_dir); } else { @@ -3651,7 +3667,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr try argv.append(syslibroot); } - for (options.rpath_list) |rpath| { + for (input_rpath_list.items) |rpath| { try argv.append("-rpath"); try argv.append(rpath); } @@ -3726,7 +3742,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr try argv.append(arg); } - for (options.lib_dirs) |lib_dir| { + for (input_lib_dirs.items) |lib_dir| { try argv.append(try std.fmt.allocPrint(arena, "-L{s}", .{lib_dir})); } @@ -3741,7 +3757,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr try argv.append(arg); } - for (options.framework_dirs) |framework_dir| { + for (input_framework_dirs.items) |framework_dir| { try argv.append(try std.fmt.allocPrint(arena, "-F{s}", .{framework_dir})); } @@ -3937,7 +3953,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr try load_commands.writeDylibIdLC(zld.gpa, zld.options, lc_writer); } - try load_commands.writeRpathLCs(zld.gpa, zld.options, lc_writer); + try load_commands.writeRpathLCs(zld.gpa, input_rpath_list.items, lc_writer); try lc_writer.writeStruct(macho.source_version_command{ .version = 0, }); From 65d1f4a735775466322bd99fc7b603d397e924b7 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 28 Jul 2023 10:48:25 +0200 Subject: [PATCH 10/15] wip: store fetched NativePaths as part of link.File.Options --- lib/std/zig/system/NativePaths.zig | 16 +++++++++++++++ src/Compilation.zig | 31 ++++++++++++++++++++---------- src/link.zig | 5 +++-- src/link/Coff/lld.zig | 14 ++++++++++---- src/link/Elf.zig | 21 +++++++++++++++----- src/link/MachO.zig | 5 ++--- src/link/MachO/zld.zig | 9 ++++----- 7 files changed, 72 insertions(+), 29 deletions(-) diff --git a/lib/std/zig/system/NativePaths.zig b/lib/std/zig/system/NativePaths.zig index febcae30f78a..ca89c0a6c36d 100644 --- a/lib/std/zig/system/NativePaths.zig +++ b/lib/std/zig/system/NativePaths.zig @@ -192,6 +192,22 @@ fn deinitArray(array: *ArrayList([:0]u8)) void { array.deinit(); } +pub inline fn getIncludeDirs(self: *const NativePaths) []const [:0]const u8 { + return self.include_dirs.items; +} + +pub inline fn getLibDirs(self: *const NativePaths) []const [:0]const u8 { + return self.lib_dirs.items; +} + +pub inline fn getFrameworkDirs(self: *const NativePaths) []const [:0]const u8 { + return self.framework_dirs.items; +} + +pub inline fn getRpaths(self: *const NativePaths) []const [:0]const u8 { + return self.rpaths.items; +} + pub fn addIncludeDir(self: *NativePaths, s: []const u8) !void { return self.appendArray(&self.include_dirs, s); } diff --git a/src/Compilation.zig b/src/Compilation.zig index dea92dc59e4a..1eebf91ac7f9 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -866,6 +866,15 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const want_native_paths = (options.sysroot == null and options.is_native_os and (options.system_lib_names.len > 0 or options.want_native_include_dirs orelse false)) or darwin_native; + const native_paths = if (want_native_paths) blk: { + const paths = std.zig.system.NativePaths.detect(arena, options.target) catch |err| { + fatal("unable to detect native system paths: {s}", .{@errorName(err)}); + }; + for (paths.warnings.items) |warning| { + log.warn("{s}", .{warning}); + } + break :blk paths; + } else null; const sysroot = blk: { if (options.sysroot) |sysroot| break :blk sysroot; @@ -1582,7 +1591,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .force_undefined_symbols = options.force_undefined_symbols, .pdb_source_path = options.pdb_source_path, .pdb_out_path = options.pdb_out_path, - .want_native_paths = want_native_paths, + .native_paths = native_paths, }); errdefer bin_file.destroy(); comp.* = .{ @@ -2357,6 +2366,10 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes man.hash.add(comp.bin_file.options.rdynamic); man.hash.addListOfBytes(comp.bin_file.options.lib_dirs); man.hash.addListOfBytes(comp.bin_file.options.rpath_list); + if (comp.bin_file.options.native_paths) |paths| { + man.hash.addListOfBytes(paths.getLibDirs()); + man.hash.addListOfBytes(paths.getRpaths()); + } man.hash.addListOfBytes(comp.bin_file.options.symbol_wrap_set.keys()); man.hash.add(comp.bin_file.options.each_lib_rpath); man.hash.add(comp.bin_file.options.build_id); @@ -2406,6 +2419,8 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes // Mach-O specific stuff man.hash.addListOfBytes(comp.bin_file.options.framework_dirs); + if (comp.bin_file.options.native_paths) |paths| + man.hash.addListOfBytes(paths.getFrameworkDirs()); link.hashAddSystemLibs(&man.hash, comp.bin_file.options.frameworks); try man.addOptionalFile(comp.bin_file.options.entitlements); man.hash.addOptional(comp.bin_file.options.pagezero_size); @@ -4372,19 +4387,15 @@ pub fn addCCArgs( argv.appendAssumeCapacity(framework_dir); } - if (comp.bin_file.options.want_native_paths) { - const paths = std.zig.system.NativePaths.detect(arena, target) catch |err| { - fatal("unable to detect native system paths: {s}", .{@errorName(err)}); - }; - - try argv.ensureUnusedCapacity(paths.include_dirs.items.len * 2); - for (paths.include_dirs.items) |include_dir| { + if (comp.bin_file.options.native_paths) |paths| { + try argv.ensureUnusedCapacity(paths.getIncludeDirs().len * 2); + for (paths.getIncludeDirs()) |include_dir| { argv.appendAssumeCapacity("-isystem"); argv.appendAssumeCapacity(include_dir); } - try argv.ensureUnusedCapacity(paths.framework_dirs.items.len * 2); - for (paths.framework_dirs.items) |framework_dir| { + try argv.ensureUnusedCapacity(paths.getFrameworkDirs().len * 2); + for (paths.getFrameworkDirs()) |framework_dir| { argv.appendAssumeCapacity("-iframework"); argv.appendAssumeCapacity(framework_dir); } diff --git a/src/link.zig b/src/link.zig index afd9a556dd18..5462bb7a2ab0 100644 --- a/src/link.zig +++ b/src/link.zig @@ -16,6 +16,7 @@ const Compilation = @import("Compilation.zig"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const Liveness = @import("Liveness.zig"); const Module = @import("Module.zig"); +const NativePaths = std.zig.system.NativePaths; const InternPool = @import("InternPool.zig"); const Type = @import("type.zig").Type; const TypedValue = @import("TypedValue.zig"); @@ -204,6 +205,8 @@ pub const Options = struct { version: ?std.SemanticVersion, compatibility_version: ?std.SemanticVersion, libc_installation: ?*const LibCInstallation, + // TODO figure out if we can make it part of LibCInstallation + native_paths: ?NativePaths = null, dwarf_format: ?std.dwarf.Format, @@ -247,8 +250,6 @@ pub const Options = struct { /// (Windows) .def file to specify when linking module_definition_file: ?[]const u8 = null, - want_native_paths: bool = false, - pub fn effectiveOutputMode(options: Options) std.builtin.OutputMode { return if (options.use_lld) .Obj else options.output_mode; } diff --git a/src/link/Coff/lld.zig b/src/link/Coff/lld.zig index 800b12808133..82a129a5f49c 100644 --- a/src/link/Coff/lld.zig +++ b/src/link/Coff/lld.zig @@ -54,6 +54,12 @@ pub fn linkWithLLD(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod // See link/Elf.zig for comments on how this mechanism works. const id_symlink_basename = "lld.id"; + var input_lib_dirs = std.ArrayList([]const u8).init(arena); + try input_lib_dirs.appendSlice(self.base.options.lib_dirs); + if (self.base.options.native_paths) |paths| { + try input_lib_dirs.appendSlice(paths.getLibDirs()); + } + var man: Cache.Manifest = undefined; defer if (!self.base.options.disable_lld_caching) man.deinit(); @@ -76,7 +82,7 @@ pub fn linkWithLLD(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod man.hash.addOptionalBytes(self.base.options.entry); man.hash.addOptional(self.base.options.stack_size_override); man.hash.addOptional(self.base.options.image_base_override); - man.hash.addListOfBytes(self.base.options.lib_dirs); + man.hash.addListOfBytes(input_lib_dirs.items); man.hash.add(self.base.options.skip_linker_dependencies); if (self.base.options.link_libc) { man.hash.add(self.base.options.libc_installation != null); @@ -251,7 +257,7 @@ pub fn linkWithLLD(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod } } - for (self.base.options.lib_dirs) |lib_dir| { + for (input_lib_dirs.items) |lib_dir| { try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{lib_dir})); } @@ -491,13 +497,13 @@ pub fn linkWithLLD(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod argv.appendAssumeCapacity(crt_file.full_object_path); continue; } - if (try findLib(arena, lib_basename, self.base.options.lib_dirs)) |full_path| { + if (try findLib(arena, lib_basename, input_lib_dirs.items)) |full_path| { argv.appendAssumeCapacity(full_path); continue; } if (target.abi.isGnu()) { const fallback_name = try allocPrint(arena, "lib{s}.dll.a", .{key}); - if (try findLib(arena, fallback_name, self.base.options.lib_dirs)) |full_path| { + if (try findLib(arena, fallback_name, input_lib_dirs.items)) |full_path| { argv.appendAssumeCapacity(full_path); continue; } diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 4bb049e07403..2366793c9c82 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1362,6 +1362,17 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v // our digest. If so, we can skip linking. Otherwise, we proceed with invoking LLD. const id_symlink_basename = "lld.id"; + var input_lib_dirs = std.ArrayList([]const u8).init(arena); + var input_rpath_list = std.ArrayList([]const u8).init(arena); + + try input_lib_dirs.appendSlice(self.base.options.lib_dirs); + try input_rpath_list.appendSlice(self.base.options.rpath_list); + + if (self.base.options.native_paths) |paths| { + try input_lib_dirs.appendSlice(paths.getLibDirs()); + try input_rpath_list.appendSlice(paths.getRpaths()); + } + var man: Cache.Manifest = undefined; defer if (!self.base.options.disable_lld_caching) man.deinit(); @@ -1397,8 +1408,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v man.hash.add(self.base.options.eh_frame_hdr); man.hash.add(self.base.options.emit_relocs); man.hash.add(self.base.options.rdynamic); - man.hash.addListOfBytes(self.base.options.lib_dirs); - man.hash.addListOfBytes(self.base.options.rpath_list); + man.hash.addListOfBytes(input_lib_dirs.items); + man.hash.addListOfBytes(input_rpath_list.items); man.hash.add(self.base.options.each_lib_rpath); if (self.base.options.output_mode == .Exe) { man.hash.add(stack_size); @@ -1688,7 +1699,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v // rpaths var rpath_table = std.StringHashMap(void).init(self.base.allocator); defer rpath_table.deinit(); - for (self.base.options.rpath_list) |rpath| { + for (input_rpath_list.items) |rpath| { if ((try rpath_table.fetchPut(rpath, {})) == null) { try argv.append("-rpath"); try argv.append(rpath); @@ -1702,7 +1713,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v if (self.base.options.each_lib_rpath) { var test_path = std.ArrayList(u8).init(self.base.allocator); defer test_path.deinit(); - for (self.base.options.lib_dirs) |lib_dir_path| { + for (input_lib_dirs.items) |lib_dir_path| { for (self.base.options.system_libs.keys()) |link_lib| { test_path.clearRetainingCapacity(); const sep = fs.path.sep_str; @@ -1732,7 +1743,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v } } - for (self.base.options.lib_dirs) |lib_dir| { + for (input_lib_dirs.items) |lib_dir| { try argv.append("-L"); try argv.append(lib_dir); } diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 274b70d4f6e4..f555ab69db27 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -503,9 +503,8 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No var input_rpath_list = std.ArrayList([]const u8).init(arena); - if (options.want_native_paths) { - const paths = try std.zig.system.NativePaths.detect(arena, options.target); - try input_rpath_list.appendSlice(paths.rpaths.items); + if (options.native_paths) |paths| { + try input_rpath_list.appendSlice(paths.getRpaths()); } if (self.lazy_syms.getPtr(.none)) |metadata| { diff --git a/src/link/MachO/zld.zig b/src/link/MachO/zld.zig index fad192320eea..01769d778601 100644 --- a/src/link/MachO/zld.zig +++ b/src/link/MachO/zld.zig @@ -3357,11 +3357,10 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr try input_framework_dirs.appendSlice(options.framework_dirs); try input_rpath_list.appendSlice(options.rpath_list); - if (options.want_native_paths) { - const paths = try std.zig.system.NativePaths.detect(arena, target); - try input_lib_dirs.appendSlice(paths.lib_dirs.items); - try input_framework_dirs.appendSlice(paths.framework_dirs.items); - try input_rpath_list.appendSlice(paths.rpaths.items); + if (options.native_paths) |paths| { + try input_lib_dirs.appendSlice(paths.getLibDirs()); + try input_framework_dirs.appendSlice(paths.getFrameworkDirs()); + try input_rpath_list.appendSlice(paths.getRpaths()); } var man: Cache.Manifest = undefined; From 00854bdd78a9190f829f45ac64e536e671f2cbf8 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 28 Jul 2023 18:24:17 +0200 Subject: [PATCH 11/15] wip: replace comptime builtin.target.isDarwin() with is_native_os --- src/Compilation.zig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 1eebf91ac7f9..6edc19315f32 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -855,8 +855,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { break :blk false; }; - const darwin_native = (comptime builtin.target.isDarwin()) and options.target.isDarwin() and - options.sysroot == null; + const darwin_native = options.is_native_os and options.target.isDarwin() and options.sysroot == null; const darwin_sdk: ?std.zig.system.darwin.DarwinSDK = if (darwin_native and !std.zig.system.NativePaths.isNix() and std.zig.system.darwin.isDarwinSDKInstalled(arena)) From 9d2090cb848b2ab8db4d7aeac7cc3e6977fdc2a7 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 28 Jul 2023 20:18:57 +0200 Subject: [PATCH 12/15] wip: fix isNix on wasm32-wasi target (with no libc) --- lib/std/zig/system/NativePaths.zig | 9 ++++----- src/Compilation.zig | 2 +- src/main.zig | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/std/zig/system/NativePaths.zig b/lib/std/zig/system/NativePaths.zig index ca89c0a6c36d..516ee5c0c755 100644 --- a/lib/std/zig/system/NativePaths.zig +++ b/lib/std/zig/system/NativePaths.zig @@ -13,11 +13,10 @@ framework_dirs: ArrayList([:0]u8), rpaths: ArrayList([:0]u8), warnings: ArrayList([:0]u8), -pub fn isNix() bool { - if (builtin.os.tag == .wasi and !builtin.link_libc) { - @compileError("isNix check is not supported for WASI without libc"); - } - return process.hasEnvVarConstant("NIX_CFLAGS_COMPILE") and process.hasEnvVarConstant("NIX_LDFLAGS"); +pub fn isNix(allocator: Allocator) bool { + const cflags = process.hasEnvVar(allocator, "NIX_CFLAGS_COMPILE") catch return false; + const ldflags = process.hasEnvVar(allocator, "NIX_LDFLAGS") catch return false; + return cflags and ldflags; } pub fn detect(allocator: Allocator, native_target: std.Target) !NativePaths { diff --git a/src/Compilation.zig b/src/Compilation.zig index 6edc19315f32..84410691f87c 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -857,7 +857,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const darwin_native = options.is_native_os and options.target.isDarwin() and options.sysroot == null; const darwin_sdk: ?std.zig.system.darwin.DarwinSDK = if (darwin_native and - !std.zig.system.NativePaths.isNix() and + !std.zig.system.NativePaths.isNix(arena) and std.zig.system.darwin.isDarwinSDKInstalled(arena)) std.zig.system.darwin.getDarwinSDK(arena, options.target) else diff --git a/src/main.zig b/src/main.zig index fbbe6313d2fe..cfb45be4fd9c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4018,7 +4018,7 @@ pub fn cmdLibC(gpa: Allocator, args: []const []const u8) !void { } var darwin_sdk: ?std.zig.system.darwin.DarwinSDK = if ((comptime builtin.target.isDarwin()) and - !std.zig.system.NativePaths.isNix() and + !std.zig.system.NativePaths.isNix(gpa) and std.zig.system.darwin.isDarwinSDKInstalled(gpa)) std.zig.system.darwin.getDarwinSDK(gpa, builtin.target) else From 3fd338fbd360e2a934fb45ed1fe2567de5a8ef56 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 28 Jul 2023 20:50:33 +0200 Subject: [PATCH 13/15] std: fix error set of std.process.hasEnvVar --- lib/std/process.zig | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/std/process.zig b/lib/std/process.zig index 28d4bfcb25c0..0712aaf34153 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -351,13 +351,7 @@ test "getEnvMap" { defer env.deinit(); } -pub const GetEnvVarOwnedError = error{ - OutOfMemory, - EnvironmentVariableNotFound, - - /// See https://github.com/ziglang/zig/issues/1774 - InvalidUtf8, -}; +pub const GetEnvVarOwnedError = error{EnvironmentVariableNotFound} || HasEnvVarError; /// Caller must free returned memory. pub fn getEnvVarOwned(allocator: Allocator, key: []const u8) GetEnvVarOwnedError![]u8 { @@ -396,7 +390,13 @@ pub fn hasEnvVarConstant(comptime key: []const u8) bool { } } -pub fn hasEnvVar(allocator: Allocator, key: []const u8) error{OutOfMemory}!bool { +pub const HasEnvVarError = error{ + OutOfMemory, + /// See https://github.com/ziglang/zig/issues/1774 + InvalidUtf8, +}; + +pub fn hasEnvVar(allocator: Allocator, key: []const u8) HasEnvVarError!bool { if (builtin.os.tag == .windows) { var stack_alloc = std.heap.stackFallback(256 * @sizeOf(u16), allocator); const key_w = try std.unicode.utf8ToUtf16LeWithNull(stack_alloc.get(), key); From ac7ad6c23b45434a91451b5411ed2d061bd0be63 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 28 Jul 2023 21:05:57 +0200 Subject: [PATCH 14/15] std: fix StackFallbackAllocator usage in std.process.hasEnvVar --- lib/std/process.zig | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/std/process.zig b/lib/std/process.zig index 0712aaf34153..6bde56de4afd 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -398,9 +398,10 @@ pub const HasEnvVarError = error{ pub fn hasEnvVar(allocator: Allocator, key: []const u8) HasEnvVarError!bool { if (builtin.os.tag == .windows) { - var stack_alloc = std.heap.stackFallback(256 * @sizeOf(u16), allocator); - const key_w = try std.unicode.utf8ToUtf16LeWithNull(stack_alloc.get(), key); - defer stack_alloc.allocator.free(key_w); + var stack_allocator = std.heap.stackFallback(256 * @sizeOf(u16), allocator); + const stack_alloc = stack_allocator.get(); + const key_w = try std.unicode.utf8ToUtf16LeWithNull(stack_alloc, key); + defer stack_alloc.free(key_w); return std.os.getenvW(key_w) != null; } else if (builtin.os.tag == .wasi and !builtin.link_libc) { var envmap = getEnvMap(allocator) catch return error.OutOfMemory; From 7e1b4391a231b079518d263caedbb67077d98a3a Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Sat, 29 Jul 2023 07:43:29 +0200 Subject: [PATCH 15/15] Revert "wip: replace comptime builtin.target.isDarwin() with is_native_os" This reverts commit 00854bdd78a9190f829f45ac64e536e671f2cbf8. --- src/Compilation.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 84410691f87c..1989341cd048 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -855,7 +855,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { break :blk false; }; - const darwin_native = options.is_native_os and options.target.isDarwin() and options.sysroot == null; + const darwin_native = (comptime builtin.target.isDarwin()) and options.target.isDarwin() and + options.sysroot == null; const darwin_sdk: ?std.zig.system.darwin.DarwinSDK = if (darwin_native and !std.zig.system.NativePaths.isNix(arena) and std.zig.system.darwin.isDarwinSDKInstalled(arena))