From 51e15a9650d4634d7d228617fc2aea3ecc2ed073 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Oct 2023 23:25:04 -0700 Subject: [PATCH 01/39] std.tar: add option for omitting empty directories --- lib/std/tar.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/std/tar.zig b/lib/std/tar.zig index b71c118d52b0..9ed69730aaaf 100644 --- a/lib/std/tar.zig +++ b/lib/std/tar.zig @@ -3,6 +3,8 @@ pub const Options = struct { strip_components: u32 = 0, /// How to handle the "mode" property of files from within the tar file. mode_mode: ModeMode = .executable_bit_only, + /// Prevents creation of empty directories. + exclude_empty_directories: bool = false, /// Provide this to receive detailed error messages. /// When this is provided, some errors which would otherwise be returned immediately /// will instead be added to this structure. The API user must check the errors @@ -201,7 +203,7 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi switch (header.fileType()) { .directory => { const file_name = try stripComponents(unstripped_file_name, options.strip_components); - if (file_name.len != 0) { + if (file_name.len != 0 and !options.exclude_empty_directories) { try dir.makePath(file_name); } }, From 24c8adc6ac40e5b73ae0caee0c559c008467d63a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Oct 2023 23:25:29 -0700 Subject: [PATCH 02/39] std.zig.ErrorBundle: add some explicit error sets --- lib/std/zig/ErrorBundle.zig | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/std/zig/ErrorBundle.zig b/lib/std/zig/ErrorBundle.zig index cbde70bd7076..2b1980c58397 100644 --- a/lib/std/zig/ErrorBundle.zig +++ b/lib/std/zig/ErrorBundle.zig @@ -383,7 +383,7 @@ pub const Wip = struct { }; } - pub fn addString(wip: *Wip, s: []const u8) !u32 { + pub fn addString(wip: *Wip, s: []const u8) Allocator.Error!u32 { const gpa = wip.gpa; const index: u32 = @intCast(wip.string_bytes.items.len); try wip.string_bytes.ensureUnusedCapacity(gpa, s.len + 1); @@ -392,7 +392,7 @@ pub const Wip = struct { return index; } - pub fn printString(wip: *Wip, comptime fmt: []const u8, args: anytype) !u32 { + pub fn printString(wip: *Wip, comptime fmt: []const u8, args: anytype) Allocator.Error!u32 { const gpa = wip.gpa; const index: u32 = @intCast(wip.string_bytes.items.len); try wip.string_bytes.writer(gpa).print(fmt, args); @@ -400,12 +400,12 @@ pub const Wip = struct { return index; } - pub fn addRootErrorMessage(wip: *Wip, em: ErrorMessage) !void { + pub fn addRootErrorMessage(wip: *Wip, em: ErrorMessage) Allocator.Error!void { try wip.root_list.ensureUnusedCapacity(wip.gpa, 1); wip.root_list.appendAssumeCapacity(try addErrorMessage(wip, em)); } - pub fn addErrorMessage(wip: *Wip, em: ErrorMessage) !MessageIndex { + pub fn addErrorMessage(wip: *Wip, em: ErrorMessage) Allocator.Error!MessageIndex { return @enumFromInt(try addExtra(wip, em)); } @@ -413,15 +413,15 @@ pub const Wip = struct { return @enumFromInt(addExtraAssumeCapacity(wip, em)); } - pub fn addSourceLocation(wip: *Wip, sl: SourceLocation) !SourceLocationIndex { + pub fn addSourceLocation(wip: *Wip, sl: SourceLocation) Allocator.Error!SourceLocationIndex { return @enumFromInt(try addExtra(wip, sl)); } - pub fn addReferenceTrace(wip: *Wip, rt: ReferenceTrace) !void { + pub fn addReferenceTrace(wip: *Wip, rt: ReferenceTrace) Allocator.Error!void { _ = try addExtra(wip, rt); } - pub fn addBundleAsNotes(wip: *Wip, other: ErrorBundle) !void { + pub fn addBundleAsNotes(wip: *Wip, other: ErrorBundle) Allocator.Error!void { const gpa = wip.gpa; try wip.string_bytes.ensureUnusedCapacity(gpa, other.string_bytes.len); From ef8772938811402262060436eb481586c8a81b3e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Oct 2023 23:26:00 -0700 Subject: [PATCH 03/39] Manifest: add top-level `paths` field for inclusion rules See #14311 --- src/Manifest.zig | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/Manifest.zig b/src/Manifest.zig index 2ff54e613266..ac384b8fcb64 100644 --- a/src/Manifest.zig +++ b/src/Manifest.zig @@ -1,3 +1,4 @@ +pub const max_bytes = 10 * 1024 * 1024; pub const basename = "build.zig.zon"; pub const Hash = std.crypto.hash.sha2.Sha256; @@ -50,6 +51,7 @@ pub const multihash_len = 1 + 1 + Hash.digest_length; name: []const u8, version: std.SemanticVersion, dependencies: std.StringArrayHashMapUnmanaged(Dependency), +paths: std.StringArrayHashMapUnmanaged(void), errors: []ErrorMessage, arena_state: std.heap.ArenaAllocator.State, @@ -74,11 +76,13 @@ pub fn parse(gpa: Allocator, ast: std.zig.Ast) Error!Manifest { .name = undefined, .version = undefined, .dependencies = .{}, + .paths = .{}, .buf = .{}, }; defer p.buf.deinit(gpa); defer p.errors.deinit(gpa); defer p.dependencies.deinit(gpa); + defer p.paths.deinit(gpa); p.parseRoot(main_node_index) catch |err| switch (err) { error.ParseFailure => assert(p.errors.items.len > 0), @@ -89,6 +93,7 @@ pub fn parse(gpa: Allocator, ast: std.zig.Ast) Error!Manifest { .name = p.name, .version = p.version, .dependencies = try p.dependencies.clone(p.arena), + .paths = try p.paths.clone(p.arena), .errors = try p.arena.dupe(ErrorMessage, p.errors.items), .arena_state = arena_instance.state, }; @@ -143,6 +148,7 @@ const Parse = struct { name: []const u8, version: std.SemanticVersion, dependencies: std.StringArrayHashMapUnmanaged(Dependency), + paths: std.StringArrayHashMapUnmanaged(void), const InnerError = error{ ParseFailure, OutOfMemory }; @@ -158,6 +164,7 @@ const Parse = struct { var have_name = false; var have_version = false; + var have_included_paths = false; for (struct_init.ast.fields) |field_init| { const name_token = ast.firstToken(field_init) - 2; @@ -167,6 +174,8 @@ const Parse = struct { // that is desirable on a per-field basis. if (mem.eql(u8, field_name, "dependencies")) { try parseDependencies(p, field_init); + } else if (mem.eql(u8, field_name, "paths")) { + try parseIncludedPaths(p, field_init); } else if (mem.eql(u8, field_name, "name")) { p.name = try parseString(p, field_init); have_name = true; @@ -190,6 +199,10 @@ const Parse = struct { if (!have_version) { try appendError(p, main_token, "missing top-level 'version' field", .{}); } + + if (!have_included_paths) { + try appendError(p, main_token, "missing top-level 'paths' field", .{}); + } } fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { @@ -277,6 +290,23 @@ const Parse = struct { return dep; } + fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + + var buf: [2]Ast.Node.Index = undefined; + const array_init = ast.fullArrayInit(&buf, node) orelse { + const tok = main_tokens[node]; + return fail(p, tok, "expected paths expression to be a struct", .{}); + }; + + for (array_init.ast.elements) |elem_node| { + const path_string = try parseString(p, elem_node); + const normalized = try std.fs.path.resolve(p.arena, &.{path_string}); + try p.paths.put(p.gpa, normalized, {}); + } + } + fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { const ast = p.ast; const node_tags = ast.nodes.items(.tag); From d06da958846330ce7c40cf9c9fde1f9799cd30dd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Oct 2023 23:26:43 -0700 Subject: [PATCH 04/39] add basic build.zig.zon documentation --- doc/build.zig.zon.md | 65 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 doc/build.zig.zon.md diff --git a/doc/build.zig.zon.md b/doc/build.zig.zon.md new file mode 100644 index 000000000000..7d1af4f678ac --- /dev/null +++ b/doc/build.zig.zon.md @@ -0,0 +1,65 @@ +# build.zig.zon Documentation + +This is the manifest file for build.zig scripts. It is named build.zig.zon in +order to make it clear that it is metadata specifically pertaining to +build.zig. + +- **build root** - the directory that contains `build.zig` + +## Top-Level Fields + +### `name` + +String. Required. + +### `version` + +String. Required. + +[semver](https://semver.org/) + +### `dependencies` + +Struct. + +Each dependency must either provide a `url` and `hash`, or a `path`. + +#### `url` + +String. + +When updating this field to a new URL, be sure to delete the corresponding +`hash`, otherwise you are communicating that you expect to find the old hash at +the new URL. + +#### `hash` + +String. + +[multihash](https://multiformats.io/multihash/) + +This is computed from the file contents of the directory of files that is +obtained after fetching `url` and applying the inclusion rules given by +`paths`. + +This field is the source of truth; packages do not come from an `url`; they +come from a `hash`. `url` is just one of many possible mirrors for how to +obtain a package matching this `hash`. + +#### `path` + +String. + +When this is provided, the package is found in a directory relative to the +build root. In this case the package's hash is irrelevant and therefore not +computed. + +### `paths` + +List. Required. + +Specifies the set of files and directories that are included in this package. +Paths are relative to the build root. Use the empty string (`""`) to refer to +the build root itself. + +Only files included in the package are used to compute a package's `hash`. From 88bbec8f9b2f8f023a0177c204f51b8ac0aee83a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Oct 2023 23:27:17 -0700 Subject: [PATCH 05/39] rework package manager Organize everything around a Fetch task which does a bunch of stuff in a worker thread without touching any shared state, and then queues up Fetch tasks for its dependencies. This isn't the theoretical optimal package fetching performance because CPU cores don't necessarily map 1:1 with I/O tasks, and each fetch task contains a mixture of computations and I/O. However, it is expected for this to significantly outperform master branch, which fetches everything recursively with only one thread. The logic is now a lot more linear and easy to follow. Everything that is embarassingly parallel is done on the thread pool, and then after everything is fetched, the worker threads are joined and the main thread does the finishing touches of stitching together the dependencies.zig import files. There is only one tiny little critical section and it does not even have any error handling in it. This also lays the groundwork for #14281 because in system mode, all this fetching logic will be skipped, but the "finishing touches" mentioned above still need to be done. With this branch, that logic is separated out and no longer recursively tangled with fetching stuff. Additionally, this branch: * Implements inclusion directives in `build.zig.zon` for deciding which files belong the package (#14311). * Adds basic documentation for `build.zig.zon` files. * Adds support for fetching dependencies with the `file://` protocol scheme (#17364). * Adds a workaround for a Linux/btrfs file system bug (#17282). This commit is a work-in-progress. Still todo: 1. Hook up the CLI to the new system. 2. Restore the module table creation logic after all the fetching is done. 3. Fix compilation errors, get the tests passing, and regression test against real world projects. --- CMakeLists.txt | 2 +- src/Package.zig | 1104 +---------------------------------------- src/Package/Fetch.zig | 1012 +++++++++++++++++++++++++++++++++++++ src/Package/hash.zig | 153 ------ src/main.zig | 2 +- 5 files changed, 1017 insertions(+), 1256 deletions(-) create mode 100644 src/Package/Fetch.zig delete mode 100644 src/Package/hash.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f3337589ce7..b08dca6f04c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -528,7 +528,7 @@ set(ZIG_STAGE2_SOURCES "${CMAKE_SOURCE_DIR}/src/Liveness.zig" "${CMAKE_SOURCE_DIR}/src/Module.zig" "${CMAKE_SOURCE_DIR}/src/Package.zig" - "${CMAKE_SOURCE_DIR}/src/Package/hash.zig" + "${CMAKE_SOURCE_DIR}/src/Package/Fetch.zig" "${CMAKE_SOURCE_DIR}/src/RangeSet.zig" "${CMAKE_SOURCE_DIR}/src/Sema.zig" "${CMAKE_SOURCE_DIR}/src/TypedValue.zig" diff --git a/src/Package.zig b/src/Package.zig index 14052e3de4be..e5fa24e18d7d 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -15,9 +15,9 @@ const Compilation = @import("Compilation.zig"); const Module = @import("Module.zig"); const Cache = std.Build.Cache; const build_options = @import("build_options"); -const git = @import("git.zig"); -const computePackageHash = @import("Package/hash.zig").compute; +const Fetch = @import("Package/Fetch.zig"); +pub const build_zig_basename = "build.zig"; pub const Manifest = @import("Manifest.zig"); pub const Table = std.StringHashMapUnmanaged(*Package); @@ -213,223 +213,6 @@ pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8 return buf.toOwnedSlice(); } -pub const build_zig_basename = "build.zig"; - -/// Fetches a package and all of its dependencies recursively. Writes the -/// corresponding datastructures for the build runner into `dependencies_source`. -pub fn fetchAndAddDependencies( - pkg: *Package, - deps_pkg: *Package, - arena: Allocator, - thread_pool: *ThreadPool, - http_client: *std.http.Client, - directory: Compilation.Directory, - global_cache_directory: Compilation.Directory, - local_cache_directory: Compilation.Directory, - dependencies_source: *std.ArrayList(u8), - error_bundle: *std.zig.ErrorBundle.Wip, - all_modules: *AllModules, - root_prog_node: *std.Progress.Node, - /// null for the root package - this_hash: ?[]const u8, -) !void { - const max_bytes = 10 * 1024 * 1024; - const gpa = thread_pool.allocator; - const build_zig_zon_bytes = directory.handle.readFileAllocOptions( - arena, - Manifest.basename, - max_bytes, - null, - 1, - 0, - ) catch |err| switch (err) { - error.FileNotFound => { - // Handle the same as no dependencies. - if (this_hash) |hash| { - try dependencies_source.writer().print( - \\ pub const {} = struct {{ - \\ pub const build_root = "{}"; - \\ pub const build_zig = @import("{}"); - \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{}}; - \\ }}; - \\ - , .{ - std.zig.fmtId(hash), - std.zig.fmtEscapes(pkg.root_src_directory.path.?), - std.zig.fmtEscapes(hash), - }); - } else { - try dependencies_source.writer().writeAll( - \\pub const packages = struct {}; - \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; - \\ - ); - } - return; - }, - else => |e| return e, - }; - - var ast = try std.zig.Ast.parse(gpa, build_zig_zon_bytes, .zon); - defer ast.deinit(gpa); - - if (ast.errors.len > 0) { - const file_path = try directory.join(arena, &.{Manifest.basename}); - try main.putAstErrorsIntoBundle(gpa, ast, file_path, error_bundle); - return error.PackageFetchFailed; - } - - var manifest = try Manifest.parse(gpa, ast); - defer manifest.deinit(gpa); - - if (manifest.errors.len > 0) { - const file_path = try directory.join(arena, &.{Manifest.basename}); - for (manifest.errors) |msg| { - const str = try error_bundle.addString(msg.msg); - try Report.addErrorMessage(&ast, file_path, error_bundle, 0, str, msg.tok, msg.off); - } - return error.PackageFetchFailed; - } - - const report: Report = .{ - .ast = &ast, - .directory = directory, - .error_bundle = error_bundle, - }; - - for (manifest.dependencies.values()) |dep| { - // If the hash is invalid, let errors happen later - // We only want to add these for progress reporting - const hash = dep.hash orelse continue; - if (hash.len != hex_multihash_len) continue; - const gop = try all_modules.getOrPut(gpa, hash[0..hex_multihash_len].*); - if (!gop.found_existing) gop.value_ptr.* = null; - } - - root_prog_node.setEstimatedTotalItems(all_modules.count()); - - if (this_hash == null) { - try dependencies_source.writer().writeAll("pub const packages = struct {\n"); - } - - for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, *dep| { - var fetch_location = try FetchLocation.init(gpa, dep.*, directory, report); - defer fetch_location.deinit(gpa); - - // Directories do not provide a hash in build.zig.zon. - // Hash the path to the module rather than its contents. - const sub_mod, const found_existing = if (fetch_location == .directory) - try getDirectoryModule(gpa, fetch_location, directory, all_modules, dep, report) - else - try getCachedPackage( - gpa, - global_cache_directory, - dep.*, - all_modules, - root_prog_node, - ) orelse .{ - try fetchAndUnpack( - fetch_location, - thread_pool, - http_client, - directory, - global_cache_directory, - dep.*, - report, - all_modules, - root_prog_node, - name, - ), - false, - }; - - assert(dep.hash != null); - - switch (sub_mod) { - .zig_pkg => |sub_pkg| { - if (!found_existing) { - try sub_pkg.fetchAndAddDependencies( - deps_pkg, - arena, - thread_pool, - http_client, - sub_pkg.root_src_directory, - global_cache_directory, - local_cache_directory, - dependencies_source, - error_bundle, - all_modules, - root_prog_node, - dep.hash.?, - ); - } - - try pkg.add(gpa, name, sub_pkg); - if (deps_pkg.table.get(dep.hash.?)) |other_sub| { - // This should be the same package (and hence module) since it's the same hash - // TODO: dedup multiple versions of the same package - assert(other_sub == sub_pkg); - } else { - try deps_pkg.add(gpa, dep.hash.?, sub_pkg); - } - }, - .non_zig_pkg => |sub_pkg| { - if (!found_existing) { - try dependencies_source.writer().print( - \\ pub const {} = struct {{ - \\ pub const build_root = "{}"; - \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{}}; - \\ }}; - \\ - , .{ - std.zig.fmtId(dep.hash.?), - std.zig.fmtEscapes(sub_pkg.root_src_directory.path.?), - }); - } - }, - } - } - - if (this_hash) |hash| { - try dependencies_source.writer().print( - \\ pub const {} = struct {{ - \\ pub const build_root = "{}"; - \\ pub const build_zig = @import("{}"); - \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{ - \\ - , .{ - std.zig.fmtId(hash), - std.zig.fmtEscapes(pkg.root_src_directory.path.?), - std.zig.fmtEscapes(hash), - }); - for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { - try dependencies_source.writer().print( - " .{{ \"{}\", \"{}\" }},\n", - .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) }, - ); - } - try dependencies_source.writer().writeAll( - \\ }; - \\ }; - \\ - ); - } else { - try dependencies_source.writer().writeAll( - \\}; - \\ - \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ - \\ - ); - for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { - try dependencies_source.writer().print( - " .{{ \"{}\", \"{}\" }},\n", - .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) }, - ); - } - try dependencies_source.writer().writeAll("};\n"); - } -} - pub fn createFilePkg( gpa: Allocator, cache_directory: Compilation.Directory, @@ -450,484 +233,11 @@ pub fn createFilePkg( const hex_digest = hh.final(); const o_dir_sub_path = "o" ++ fs.path.sep_str ++ hex_digest; - try renameTmpIntoCache(cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path); + try Fetch.renameTmpIntoCache(cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path); return createWithDir(gpa, cache_directory, o_dir_sub_path, basename); } -pub const Report = struct { - ast: ?*const std.zig.Ast, - directory: Compilation.Directory, - error_bundle: *std.zig.ErrorBundle.Wip, - - fn fail( - report: Report, - tok: std.zig.Ast.TokenIndex, - comptime fmt_string: []const u8, - fmt_args: anytype, - ) error{ PackageFetchFailed, OutOfMemory } { - const msg = try report.error_bundle.printString(fmt_string, fmt_args); - return failMsg(report, tok, msg); - } - - fn failMsg( - report: Report, - tok: std.zig.Ast.TokenIndex, - msg: u32, - ) error{ PackageFetchFailed, OutOfMemory } { - const gpa = report.error_bundle.gpa; - - const file_path = try report.directory.join(gpa, &.{Manifest.basename}); - defer gpa.free(file_path); - - const eb = report.error_bundle; - - if (report.ast) |ast| { - try addErrorMessage(ast, file_path, eb, 0, msg, tok, 0); - } else { - try eb.addRootErrorMessage(.{ - .msg = msg, - .src_loc = .none, - .notes_len = 0, - }); - } - - return error.PackageFetchFailed; - } - - fn addErrorWithNotes( - report: Report, - notes_len: u32, - msg: Manifest.ErrorMessage, - ) error{OutOfMemory}!void { - const eb = report.error_bundle; - const msg_str = try eb.addString(msg.msg); - if (report.ast) |ast| { - const gpa = eb.gpa; - const file_path = try report.directory.join(gpa, &.{Manifest.basename}); - defer gpa.free(file_path); - return addErrorMessage(ast, file_path, eb, notes_len, msg_str, msg.tok, msg.off); - } else { - return eb.addRootErrorMessage(.{ - .msg = msg_str, - .src_loc = .none, - .notes_len = notes_len, - }); - } - } - - fn addErrorMessage( - ast: *const std.zig.Ast, - file_path: []const u8, - eb: *std.zig.ErrorBundle.Wip, - notes_len: u32, - msg_str: u32, - msg_tok: std.zig.Ast.TokenIndex, - msg_off: u32, - ) error{OutOfMemory}!void { - const token_starts = ast.tokens.items(.start); - const start_loc = ast.tokenLocation(0, msg_tok); - - try eb.addRootErrorMessage(.{ - .msg = msg_str, - .src_loc = try eb.addSourceLocation(.{ - .src_path = try eb.addString(file_path), - .span_start = token_starts[msg_tok], - .span_end = @as(u32, @intCast(token_starts[msg_tok] + ast.tokenSlice(msg_tok).len)), - .span_main = token_starts[msg_tok] + msg_off, - .line = @intCast(start_loc.line), - .column = @as(u32, @intCast(start_loc.column)), - .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), - }), - .notes_len = notes_len, - }); - } -}; - -pub const FetchLocation = union(enum) { - /// The relative path to a file or directory. - /// This may be a file that requires unpacking (such as a .tar.gz), - /// or the path to the root directory of a package. - file: []const u8, - directory: []const u8, - http_request: std.Uri, - git_request: std.Uri, - - pub fn init( - gpa: Allocator, - dep: Manifest.Dependency, - root_dir: Compilation.Directory, - report: Report, - ) !FetchLocation { - switch (dep.location) { - .url => |url| { - const uri = std.Uri.parse(url) catch |err| switch (err) { - error.UnexpectedCharacter => return report.fail(dep.location_tok, "failed to parse dependency location as URI", .{}), - else => return err, - }; - return initUri(uri, dep.location_tok, report); - }, - .path => |path| { - if (fs.path.isAbsolute(path)) { - return report.fail(dep.location_tok, "absolute paths are not allowed. Use a relative path instead", .{}); - } - - const is_dir = isDirectory(root_dir, path) catch |err| switch (err) { - error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{path}), - else => return err, - }; - - return if (is_dir) - .{ .directory = try gpa.dupe(u8, path) } - else - .{ .file = try gpa.dupe(u8, path) }; - }, - } - } - - pub fn initUri(uri: std.Uri, location_tok: std.zig.Ast.TokenIndex, report: Report) !FetchLocation { - if (ascii.eqlIgnoreCase(uri.scheme, "file")) { - return report.fail(location_tok, "'file' scheme is not allowed for URLs. Use '.path' instead", .{}); - } else if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) { - return .{ .http_request = uri }; - } else if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or ascii.eqlIgnoreCase(uri.scheme, "git+https")) { - return .{ .git_request = uri }; - } else { - return report.fail(location_tok, "unsupported URL scheme: {s}", .{uri.scheme}); - } - } - - pub fn deinit(f: *FetchLocation, gpa: Allocator) void { - switch (f.*) { - .file, .directory => |path| gpa.free(path), - .http_request, .git_request => {}, - } - f.* = undefined; - } - - pub fn fetch( - f: FetchLocation, - gpa: Allocator, - root_dir: Compilation.Directory, - http_client: *std.http.Client, - dep_location_tok: std.zig.Ast.TokenIndex, - report: Report, - ) !ReadableResource { - switch (f) { - .file => |file| { - const owned_path = try gpa.dupe(u8, file); - errdefer gpa.free(owned_path); - return .{ - .path = owned_path, - .resource = .{ .file = try root_dir.handle.openFile(file, .{}) }, - }; - }, - .http_request => |uri| { - var h = std.http.Headers{ .allocator = gpa }; - defer h.deinit(); - - var req = try http_client.request(.GET, uri, h, .{}); - errdefer req.deinit(); - - try req.start(.{}); - try req.wait(); - - if (req.response.status != .ok) { - return report.fail(dep_location_tok, "expected response status '200 OK' got '{} {s}'", .{ - @intFromEnum(req.response.status), - req.response.status.phrase() orelse "", - }); - } - - return .{ - .path = try gpa.dupe(u8, uri.path), - .resource = .{ .http_request = req }, - }; - }, - .git_request => |uri| { - var transport_uri = uri; - transport_uri.scheme = uri.scheme["git+".len..]; - var redirect_uri: []u8 = undefined; - var session: git.Session = .{ .transport = http_client, .uri = transport_uri }; - session.discoverCapabilities(gpa, &redirect_uri) catch |e| switch (e) { - error.Redirected => { - defer gpa.free(redirect_uri); - return report.fail(dep_location_tok, "repository moved to {s}", .{redirect_uri}); - }, - else => |other| return other, - }; - - const want_oid = want_oid: { - const want_ref = uri.fragment orelse "HEAD"; - if (git.parseOid(want_ref)) |oid| break :want_oid oid else |_| {} - - const want_ref_head = try std.fmt.allocPrint(gpa, "refs/heads/{s}", .{want_ref}); - defer gpa.free(want_ref_head); - const want_ref_tag = try std.fmt.allocPrint(gpa, "refs/tags/{s}", .{want_ref}); - defer gpa.free(want_ref_tag); - - var ref_iterator = try session.listRefs(gpa, .{ - .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, - .include_peeled = true, - }); - defer ref_iterator.deinit(); - while (try ref_iterator.next()) |ref| { - if (mem.eql(u8, ref.name, want_ref) or - mem.eql(u8, ref.name, want_ref_head) or - mem.eql(u8, ref.name, want_ref_tag)) - { - break :want_oid ref.peeled orelse ref.oid; - } - } - return report.fail(dep_location_tok, "ref not found: {s}", .{want_ref}); - }; - if (uri.fragment == null) { - const notes_len = 1; - try report.addErrorWithNotes(notes_len, .{ - .tok = dep_location_tok, - .off = 0, - .msg = "url field is missing an explicit ref", - }); - const eb = report.error_bundle; - const notes_start = try eb.reserveNotes(notes_len); - eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid) }), - })); - return error.PackageFetchFailed; - } - - var want_oid_buf: [git.fmt_oid_length]u8 = undefined; - _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{std.fmt.fmtSliceHexLower(&want_oid)}) catch unreachable; - var fetch_stream = try session.fetch(gpa, &.{&want_oid_buf}); - errdefer fetch_stream.deinit(); - - return .{ - .path = try gpa.dupe(u8, &want_oid_buf), - .resource = .{ .git_fetch_stream = fetch_stream }, - }; - }, - .directory => unreachable, // Directories do not require fetching - } - } -}; - -pub const ReadableResource = struct { - path: []const u8, - resource: union(enum) { - file: fs.File, - http_request: std.http.Client.Request, - git_fetch_stream: git.Session.FetchStream, - dir: fs.IterableDir, - }, - - /// Unpack the package into the global cache directory. - /// If `ps` does not require unpacking (for example, if it is a directory), then no caching is performed. - /// In either case, the hash is computed and returned along with the path to the package. - pub fn unpack( - rr: *ReadableResource, - allocator: Allocator, - thread_pool: *ThreadPool, - global_cache_directory: Compilation.Directory, - dep_location_tok: std.zig.Ast.TokenIndex, - report: Report, - pkg_prog_node: *std.Progress.Node, - ) !PackageLocation { - switch (rr.resource) { - inline .file, .http_request, .git_fetch_stream, .dir => |*r, tag| { - const s = fs.path.sep_str; - const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); - - const actual_hash = h: { - var tmp_directory: Compilation.Directory = d: { - const path = try global_cache_directory.join(allocator, &.{tmp_dir_sub_path}); - errdefer allocator.free(path); - - const iterable_dir = try global_cache_directory.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}); - errdefer iterable_dir.close(); - - break :d .{ - .path = path, - .handle = iterable_dir.dir, - }; - }; - defer tmp_directory.closeAndFree(allocator); - - if (tag != .dir) { - const opt_content_length = try rr.getSize(); - - var prog_reader: ProgressReader(@TypeOf(r.reader())) = .{ - .child_reader = r.reader(), - .prog_node = pkg_prog_node, - .unit = if (opt_content_length) |content_length| unit: { - const kib = content_length / 1024; - const mib = kib / 1024; - if (mib > 0) { - pkg_prog_node.setEstimatedTotalItems(@intCast(mib)); - pkg_prog_node.setUnit("MiB"); - break :unit .mib; - } else { - pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib))); - pkg_prog_node.setUnit("KiB"); - break :unit .kib; - } - } else .any, - }; - - switch (try rr.getFileType(dep_location_tok, report)) { - .tar => try unpackTarball(allocator, prog_reader.reader(), tmp_directory.handle, dep_location_tok, report), - .@"tar.gz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, dep_location_tok, report, std.compress.gzip), - .@"tar.xz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, dep_location_tok, report, std.compress.xz), - .git_pack => try unpackGitPack(allocator, &prog_reader, git.parseOid(rr.path) catch unreachable, tmp_directory.handle, dep_location_tok, report), - } - } else { - // Recursive directory copy. - var it = try r.walk(allocator); - defer it.deinit(); - while (try it.next()) |entry| { - switch (entry.kind) { - .directory => try tmp_directory.handle.makePath(entry.path), - .file => try r.dir.copyFile( - entry.path, - tmp_directory.handle, - entry.path, - .{}, - ), - .sym_link => { - var buf: [fs.MAX_PATH_BYTES]u8 = undefined; - const link_name = try r.dir.readLink(entry.path, &buf); - // TODO: if this would create a symlink to outside - // the destination directory, fail with an error instead. - try tmp_directory.handle.symLink(link_name, entry.path, .{}); - }, - else => return error.IllegalFileTypeInPackage, - } - } - } - - break :h try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle }); - }; - - const pkg_dir_sub_path = "p" ++ s ++ Manifest.hexDigest(actual_hash); - const unpacked_path = try global_cache_directory.join(allocator, &.{pkg_dir_sub_path}); - defer allocator.free(unpacked_path); - - const relative_unpacked_path = try fs.path.relative(allocator, global_cache_directory.path.?, unpacked_path); - errdefer allocator.free(relative_unpacked_path); - try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, relative_unpacked_path); - - return .{ - .hash = actual_hash, - .relative_unpacked_path = relative_unpacked_path, - }; - }, - } - } - - const FileType = enum { - tar, - @"tar.gz", - @"tar.xz", - git_pack, - }; - - pub fn getSize(rr: ReadableResource) !?u64 { - switch (rr.resource) { - .file => |f| return (try f.metadata()).size(), - // TODO: Handle case of chunked content-length - .http_request => |req| return req.response.content_length, - .git_fetch_stream => |stream| return stream.request.response.content_length, - .dir => unreachable, - } - } - - pub fn getFileType( - rr: ReadableResource, - dep_location_tok: std.zig.Ast.TokenIndex, - report: Report, - ) !FileType { - switch (rr.resource) { - .file => { - return fileTypeFromPath(rr.path) orelse - return report.fail(dep_location_tok, "unknown file type", .{}); - }, - .http_request => |req| { - const content_type = req.response.headers.getFirstValue("Content-Type") orelse - return report.fail(dep_location_tok, "missing 'Content-Type' header", .{}); - - // If the response has a different content type than the URI indicates, override - // the previously assumed file type. - if (ascii.eqlIgnoreCase(content_type, "application/x-tar")) return .tar; - - return if (ascii.eqlIgnoreCase(content_type, "application/gzip") or - ascii.eqlIgnoreCase(content_type, "application/x-gzip") or - ascii.eqlIgnoreCase(content_type, "application/tar+gzip")) - .@"tar.gz" - else if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) - .@"tar.xz" - else if (ascii.eqlIgnoreCase(content_type, "application/octet-stream")) ty: { - // support gitlab tarball urls such as https://gitlab.com///-/archive//-.tar.gz - // whose content-disposition header is: 'attachment; filename="-.tar.gz"' - const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse - return report.fail(dep_location_tok, "missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{}); - break :ty getAttachmentType(content_disposition) orelse - return report.fail(dep_location_tok, "unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition}); - } else return report.fail(dep_location_tok, "unrecognized value for 'Content-Type' header: {s}", .{content_type}); - }, - .git_fetch_stream => return .git_pack, - .dir => unreachable, - } - } - - fn fileTypeFromPath(file_path: []const u8) ?FileType { - if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; - if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; - if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; - return null; - } - - fn getAttachmentType(content_disposition: []const u8) ?FileType { - const disposition_type_end = ascii.indexOfIgnoreCase(content_disposition, "attachment;") orelse return null; - - var value_start = ascii.indexOfIgnoreCasePos(content_disposition, disposition_type_end + 1, "filename") orelse return null; - value_start += "filename".len; - if (content_disposition[value_start] == '*') { - value_start += 1; - } - if (content_disposition[value_start] != '=') return null; - value_start += 1; - - var value_end = mem.indexOfPos(u8, content_disposition, value_start, ";") orelse content_disposition.len; - if (content_disposition[value_end - 1] == '\"') { - value_end -= 1; - } - return fileTypeFromPath(content_disposition[value_start..value_end]); - } - - pub fn deinit(rr: *ReadableResource, gpa: Allocator) void { - gpa.free(rr.path); - switch (rr.resource) { - .file => |file| file.close(), - .http_request => |*req| req.deinit(), - .git_fetch_stream => |*stream| stream.deinit(), - .dir => |*dir| dir.close(), - } - rr.* = undefined; - } -}; - -pub const PackageLocation = struct { - /// For packages that require unpacking, this is the hash of the package contents. - /// For directories, this is the hash of the absolute file path. - hash: [Manifest.Hash.digest_length]u8, - relative_unpacked_path: []const u8, - - pub fn deinit(pl: *PackageLocation, allocator: Allocator) void { - allocator.free(pl.relative_unpacked_path); - pl.* = undefined; - } -}; - const hex_multihash_len = 2 * Manifest.multihash_len; const MultiHashHexDigest = [hex_multihash_len]u8; @@ -939,411 +249,3 @@ const DependencyModule = union(enum) { /// If the value is `null`, the package is a known dependency, but has not yet /// been fetched. pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?DependencyModule); - -fn ProgressReader(comptime ReaderType: type) type { - return struct { - child_reader: ReaderType, - bytes_read: u64 = 0, - prog_node: *std.Progress.Node, - unit: enum { - kib, - mib, - any, - }, - - pub const Error = ReaderType.Error; - pub const Reader = std.io.Reader(*@This(), Error, read); - - pub fn read(self: *@This(), buf: []u8) Error!usize { - const amt = try self.child_reader.read(buf); - self.bytes_read += amt; - const kib = self.bytes_read / 1024; - const mib = kib / 1024; - switch (self.unit) { - .kib => self.prog_node.setCompletedItems(@intCast(kib)), - .mib => self.prog_node.setCompletedItems(@intCast(mib)), - .any => { - if (mib > 0) { - self.prog_node.setUnit("MiB"); - self.prog_node.setCompletedItems(@intCast(mib)); - } else { - self.prog_node.setUnit("KiB"); - self.prog_node.setCompletedItems(@intCast(kib)); - } - }, - } - self.prog_node.activate(); - return amt; - } - - pub fn reader(self: *@This()) Reader { - return .{ .context = self }; - } - }; -} - -/// Get a cached package if it exists. -/// Returns `null` if the package has not been cached -/// If the package exists in the cache, returns a pointer to the package and a -/// boolean indicating whether this package has already been seen in the build -/// (i.e. whether or not its transitive dependencies have been fetched). -fn getCachedPackage( - gpa: Allocator, - global_cache_directory: Compilation.Directory, - dep: Manifest.Dependency, - all_modules: *AllModules, - root_prog_node: *std.Progress.Node, -) !?struct { DependencyModule, bool } { - const s = fs.path.sep_str; - // Check if the expected_hash is already present in the global package - // cache, and thereby avoid both fetching and unpacking. - if (dep.hash) |h| { - const hex_digest = h[0..hex_multihash_len]; - const pkg_dir_sub_path = "p" ++ s ++ hex_digest; - - var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) { - error.FileNotFound => return null, - else => |e| return e, - }; - errdefer pkg_dir.close(); - - // The compiler has a rule that a file must not be included in multiple modules, - // so we must detect if a module has been created for this package and reuse it. - const gop = try all_modules.getOrPut(gpa, hex_digest.*); - if (gop.found_existing) { - if (gop.value_ptr.*) |mod| { - return .{ mod, true }; - } - } - - root_prog_node.completeOne(); - - const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) |_| true else |_| false; - const basename = if (is_zig_mod) build_zig_basename else ""; - const pkg = try createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, basename); - - const module: DependencyModule = if (is_zig_mod) - .{ .zig_pkg = pkg } - else - .{ .non_zig_pkg = pkg }; - - try all_modules.put(gpa, hex_digest.*, module); - return .{ module, false }; - } - - return null; -} - -fn getDirectoryModule( - gpa: Allocator, - fetch_location: FetchLocation, - directory: Compilation.Directory, - all_modules: *AllModules, - dep: *Manifest.Dependency, - report: Report, -) !struct { DependencyModule, bool } { - assert(fetch_location == .directory); - - if (dep.hash != null) { - return report.fail(dep.hash_tok, "hash not allowed for directory package", .{}); - } - - const hash = try computePathHash(gpa, directory, fetch_location.directory); - const hex_digest = Manifest.hexDigest(hash); - dep.hash = try gpa.dupe(u8, &hex_digest); - - // There is no fixed location to check for directory modules. - // Instead, check whether it is already listed in all_modules. - if (all_modules.get(hex_digest)) |mod| return .{ mod.?, true }; - - var pkg_dir = directory.handle.openDir(fetch_location.directory, .{}) catch |err| switch (err) { - error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{fetch_location.directory}), - else => |e| return e, - }; - defer pkg_dir.close(); - - const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) |_| true else |_| false; - const basename = if (is_zig_mod) build_zig_basename else ""; - - const pkg = try createWithDir(gpa, directory, fetch_location.directory, basename); - const module: DependencyModule = if (is_zig_mod) - .{ .zig_pkg = pkg } - else - .{ .non_zig_pkg = pkg }; - - try all_modules.put(gpa, hex_digest, module); - return .{ module, false }; -} - -fn fetchAndUnpack( - fetch_location: FetchLocation, - thread_pool: *ThreadPool, - http_client: *std.http.Client, - directory: Compilation.Directory, - global_cache_directory: Compilation.Directory, - dep: Manifest.Dependency, - report: Report, - all_modules: *AllModules, - root_prog_node: *std.Progress.Node, - /// This does not have to be any form of canonical or fully-qualified name: it - /// is only intended to be human-readable for progress reporting. - name_for_prog: []const u8, -) !DependencyModule { - assert(fetch_location != .directory); - - const gpa = http_client.allocator; - - var pkg_prog_node = root_prog_node.start(name_for_prog, 0); - defer pkg_prog_node.end(); - pkg_prog_node.activate(); - - var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep.location_tok, report); - defer readable_resource.deinit(gpa); - - var package_location = try readable_resource.unpack( - gpa, - thread_pool, - global_cache_directory, - dep.location_tok, - report, - &pkg_prog_node, - ); - defer package_location.deinit(gpa); - - const actual_hex = Manifest.hexDigest(package_location.hash); - if (dep.hash) |h| { - if (!mem.eql(u8, h, &actual_hex)) { - return report.fail(dep.hash_tok, "hash mismatch: expected: {s}, found: {s}", .{ - h, actual_hex, - }); - } - } else { - const notes_len = 1; - try report.addErrorWithNotes(notes_len, .{ - .tok = dep.location_tok, - .off = 0, - .msg = "dependency is missing hash field", - }); - const eb = report.error_bundle; - const notes_start = try eb.reserveNotes(notes_len); - eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), - })); - return error.PackageFetchFailed; - } - - const build_zig_path = try fs.path.join(gpa, &.{ package_location.relative_unpacked_path, build_zig_basename }); - defer gpa.free(build_zig_path); - - const is_zig_mod = if (global_cache_directory.handle.access(build_zig_path, .{})) |_| true else |_| false; - const basename = if (is_zig_mod) build_zig_basename else ""; - const pkg = try createWithDir(gpa, global_cache_directory, package_location.relative_unpacked_path, basename); - const module: DependencyModule = if (is_zig_mod) - .{ .zig_pkg = pkg } - else - .{ .non_zig_pkg = pkg }; - - try all_modules.put(gpa, actual_hex, module); - return module; -} - -fn unpackTarballCompressed( - gpa: Allocator, - reader: anytype, - out_dir: fs.Dir, - dep_location_tok: std.zig.Ast.TokenIndex, - report: Report, - comptime Compression: type, -) !void { - var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); - - var decompress = try Compression.decompress(gpa, br.reader()); - defer decompress.deinit(); - - return unpackTarball(gpa, decompress.reader(), out_dir, dep_location_tok, report); -} - -fn unpackTarball( - gpa: Allocator, - reader: anytype, - out_dir: fs.Dir, - dep_location_tok: std.zig.Ast.TokenIndex, - report: Report, -) !void { - var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = gpa }; - defer diagnostics.deinit(); - - try std.tar.pipeToFileSystem(out_dir, reader, .{ - .diagnostics = &diagnostics, - .strip_components = 1, - // TODO: we would like to set this to executable_bit_only, but two - // things need to happen before that: - // 1. the tar implementation needs to support it - // 2. the hashing algorithm here needs to support detecting the is_executable - // bit on Windows from the ACLs (see the isExecutable function). - .mode_mode = .ignore, - }); - - if (diagnostics.errors.items.len > 0) { - const notes_len: u32 = @intCast(diagnostics.errors.items.len); - try report.addErrorWithNotes(notes_len, .{ - .tok = dep_location_tok, - .off = 0, - .msg = "unable to unpack tarball", - }); - const eb = report.error_bundle; - const notes_start = try eb.reserveNotes(notes_len); - for (diagnostics.errors.items, notes_start..) |item, note_i| { - switch (item) { - .unable_to_create_sym_link => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ - info.file_name, info.link_name, @errorName(info.code), - }), - })); - }, - .unsupported_file_type => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ - info.file_name, @intFromEnum(info.file_type), - }), - })); - }, - } - } - return error.InvalidTarball; - } -} - -fn unpackGitPack( - gpa: Allocator, - reader: anytype, - want_oid: git.Oid, - out_dir: fs.Dir, - dep_location_tok: std.zig.Ast.TokenIndex, - report: Report, -) !void { - // The .git directory is used to store the packfile and associated index, but - // we do not attempt to replicate the exact structure of a real .git - // directory, since that isn't relevant for fetching a package. - { - var pack_dir = try out_dir.makeOpenPath(".git", .{}); - defer pack_dir.close(); - var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true }); - defer pack_file.close(); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(reader.reader(), pack_file.writer()); - try pack_file.sync(); - - var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); - defer index_file.close(); - { - var index_prog_node = reader.prog_node.start("Index pack", 0); - defer index_prog_node.end(); - index_prog_node.activate(); - var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); - try git.indexPack(gpa, pack_file, index_buffered_writer.writer()); - try index_buffered_writer.flush(); - try index_file.sync(); - } - - { - var checkout_prog_node = reader.prog_node.start("Checkout", 0); - defer checkout_prog_node.end(); - checkout_prog_node.activate(); - var repository = try git.Repository.init(gpa, pack_file, index_file); - defer repository.deinit(); - var diagnostics: git.Diagnostics = .{ .allocator = gpa }; - defer diagnostics.deinit(); - try repository.checkout(out_dir, want_oid, &diagnostics); - - if (diagnostics.errors.items.len > 0) { - const notes_len: u32 = @intCast(diagnostics.errors.items.len); - try report.addErrorWithNotes(notes_len, .{ - .tok = dep_location_tok, - .off = 0, - .msg = "unable to unpack packfile", - }); - const eb = report.error_bundle; - const notes_start = try eb.reserveNotes(notes_len); - for (diagnostics.errors.items, notes_start..) |item, note_i| { - switch (item) { - .unable_to_create_sym_link => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ - info.file_name, info.link_name, @errorName(info.code), - }), - })); - }, - } - } - return error.InvalidGitPack; - } - } - } - - try out_dir.deleteTree(".git"); -} - -/// Compute the hash of a file path. -fn computePathHash(gpa: Allocator, dir: Compilation.Directory, path: []const u8) ![Manifest.Hash.digest_length]u8 { - const resolved_path = try std.fs.path.resolve(gpa, &.{ dir.path.?, path }); - defer gpa.free(resolved_path); - var hasher = Manifest.Hash.init(.{}); - hasher.update(resolved_path); - return hasher.finalResult(); -} - -fn isDirectory(root_dir: Compilation.Directory, path: []const u8) !bool { - var dir = root_dir.handle.openDir(path, .{}) catch |err| switch (err) { - error.NotDir => return false, - else => return err, - }; - defer dir.close(); - return true; -} - -fn renameTmpIntoCache( - cache_dir: fs.Dir, - tmp_dir_sub_path: []const u8, - dest_dir_sub_path: []const u8, -) !void { - assert(dest_dir_sub_path[1] == fs.path.sep); - var handled_missing_dir = false; - while (true) { - cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch |err| switch (err) { - error.FileNotFound => { - if (handled_missing_dir) return err; - cache_dir.makeDir(dest_dir_sub_path[0..1]) catch |mkd_err| switch (mkd_err) { - error.PathAlreadyExists => handled_missing_dir = true, - else => |e| return e, - }; - continue; - }, - error.PathAlreadyExists, error.AccessDenied => { - // Package has been already downloaded and may already be in use on the system. - cache_dir.deleteTree(tmp_dir_sub_path) catch |del_err| { - std.log.warn("unable to delete temp directory: {s}", .{@errorName(del_err)}); - }; - }, - else => |e| return e, - }; - break; - } -} - -test "getAttachmentType" { - try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); - try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attachment; filename*=\"stuff.tar.gz\"")); - try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.xz"), ReadableResource.getAttachmentType("ATTACHMENT; filename=\"stuff.tar.xz\"")); - try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.xz"), ReadableResource.getAttachmentType("attachment; FileName=\"stuff.tar.xz\"")); - try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); - - try std.testing.expect(ReadableResource.getAttachmentType("attachment FileName=\"stuff.tar.gz\"") == null); - try std.testing.expect(ReadableResource.getAttachmentType("attachment; FileName=\"stuff.tar\"") == null); - try std.testing.expect(ReadableResource.getAttachmentType("attachment; FileName\"stuff.gz\"") == null); - try std.testing.expect(ReadableResource.getAttachmentType("attachment; size=42") == null); - try std.testing.expect(ReadableResource.getAttachmentType("inline; size=42") == null); - try std.testing.expect(ReadableResource.getAttachmentType("FileName=\"stuff.tar.gz\"; attachment;") == null); - try std.testing.expect(ReadableResource.getAttachmentType("FileName=\"stuff.tar.gz\";") == null); -} diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig new file mode 100644 index 000000000000..b3b4667e4093 --- /dev/null +++ b/src/Package/Fetch.zig @@ -0,0 +1,1012 @@ +//! Represents one independent job whose responsibility is to: +//! +//! 1. Check the global zig package cache to see if the hash already exists. +//! If so, load, parse, and validate the build.zig.zon file therein, and +//! goto step 8. Likewise if the location is a relative path, treat this +//! the same as a cache hit. Otherwise, proceed. +//! 2. Fetch and unpack a URL into a temporary directory. +//! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed +//! for the file to be missing, in which case this fetched package is considered +//! to be a "naked" package. +//! 4. Apply inclusion rules of the build.zig.zon to the temporary directory by +//! deleting excluded files. If any files had errors for files that were +//! ultimately excluded, those errors should be ignored, such as failure to +//! create symlinks that weren't supposed to be included anyway. +//! 5. Compute the package hash based on the remaining files in the temporary +//! directory. +//! 6. Rename the temporary directory into the global zig package cache +//! directory. If the hash already exists, delete the temporary directory and +//! leave the zig package cache directory untouched as it may be in use by the +//! system. This is done even if the hash is invalid, in case the package with +//! the different hash is used in the future. +//! 7. Validate the computed hash against the expected hash. If invalid, +//! this job is done. +//! 8. Spawn a new fetch job for each dependency in the manifest file. Use +//! a mutex and a hash map so that redundant jobs do not get queued up. +//! +//! All of this must be done with only referring to the state inside this struct +//! because this work will be done in a dedicated thread. + +/// Try to avoid this as much as possible since arena will have less contention. +gpa: Allocator, +arena: std.heap.ArenaAllocator, +location: Location, +location_tok: std.zig.Ast.TokenIndex, +hash_tok: std.zig.Ast.TokenIndex, +global_cache: Cache.Directory, +parent_package_root: Path, +parent_manifest_ast: ?*const std.zig.Ast, +prog_node: *std.Progress.Node, +http_client: *std.http.Client, +thread_pool: *ThreadPool, +job_queue: *JobQueue, +wait_group: *WaitGroup, + +// Above this are fields provided as inputs to `run`. +// Below this are fields populated by `run`. + +/// This will either be relative to `global_cache`, or to the build root of +/// the root package. +package_root: Path, +error_bundle: std.zig.ErrorBundle.Wip, +manifest: ?Manifest, +manifest_ast: ?*std.zig.Ast, +actual_hash: Digest, +/// Fetch logic notices whether a package has a build.zig file and sets this flag. +has_build_zig: bool, +/// Indicates whether the task aborted due to an out-of-memory condition. +oom_flag: bool, + +pub const JobQueue = struct { + mutex: std.Thread.Mutex = .{}, +}; + +pub const Digest = [Manifest.Hash.digest_length]u8; +pub const MultiHashHexDigest = [hex_multihash_len]u8; + +pub const Path = struct { + root_dir: Cache.Directory, + /// The path, relative to the root dir, that this `Path` represents. + /// Empty string means the root_dir is the path. + sub_path: []const u8 = "", +}; + +pub const Location = union(enum) { + remote: Remote, + relative_path: []const u8, + + pub const Remote = struct { + url: []const u8, + /// If this is null it means the user omitted the hash field from a dependency. + /// It will be an error but the logic should still fetch and print the discovered hash. + hash: ?[hex_multihash_len]u8, + }; +}; + +pub const RunError = error{ + OutOfMemory, + /// This error code is intended to be handled by inspecting the + /// `error_bundle` field. + FetchFailed, +}; + +pub fn run(f: *Fetch) RunError!void { + const eb = &f.error_bundle; + const arena = f.arena_allocator.allocator(); + + // Check the global zig package cache to see if the hash already exists. If + // so, load, parse, and validate the build.zig.zon file therein, and skip + // ahead to queuing up jobs for dependencies. Likewise if the location is a + // relative path, treat this the same as a cache hit. Otherwise, proceed. + + const remote = switch (f.location) { + .relative_path => |sub_path| { + if (fs.path.isAbsolute(sub_path)) return f.fail( + f.location_tok, + try eb.addString("expected path relative to build root; found absolute path"), + ); + if (f.hash_tok != 0) return f.fail( + f.hash_tok, + try eb.addString("path-based dependencies are not hashed"), + ); + f.package_root = try f.parent_package_root.join(arena, sub_path); + try loadManifest(f, f.package_root); + // Package hashes are used as unique identifiers for packages, so + // we still need one for relative paths. + const hash = h: { + var hasher = Manifest.Hash.init(.{}); + // This hash is a tuple of: + // * whether it relative to the global cache directory or to the root package + // * the relative file path from there to the build root of the package + hasher.update(if (f.package_root.root_dir.handle == f.global_cache.handle) + &package_hash_prefix_cached + else + &package_hash_prefix_project); + hasher.update(f.package_root.sub_path); + break :h hasher.finalResult(); + }; + return queueJobsForDeps(f, hash); + }, + .remote => |remote| remote, + }; + const s = fs.path.sep_str; + if (remote.hash) |expected_hash| { + const pkg_sub_path = "p" ++ s ++ expected_hash; + if (f.global_cache.handle.access(pkg_sub_path, .{})) |_| { + f.package_root = .{ + .root_dir = f.global_cache, + .sub_path = pkg_sub_path, + }; + try loadManifest(f, f.package_root); + return queueJobsForDeps(f, expected_hash); + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open global package cache directory '{s}': {s}", .{ + try f.global_cache.join(arena, .{pkg_sub_path}), @errorName(e), + }), + .src_loc = .none, + .notes_len = 0, + }); + return error.FetchFailed; + }, + } + } + + // Fetch and unpack the remote into a temporary directory. + + const uri = std.Uri.parse(remote.url) catch |err| return f.fail( + f.location_tok, + "invalid URI: {s}", + .{@errorName(err)}, + ); + const rand_int = std.crypto.random.int(u64); + const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); + + var tmp_directory: Cache.Directory = .{ + .path = try f.global_cache.join(arena, &.{tmp_dir_sub_path}), + .handle = (try f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{})).dir, + }; + defer tmp_directory.handle.close(); + + var resource = try f.initResource(uri); + defer resource.deinit(); // releases more than memory + + try f.unpackResource(&resource, uri.path, tmp_directory); + + // Load, parse, and validate the unpacked build.zig.zon file. It is allowed + // for the file to be missing, in which case this fetched package is + // considered to be a "naked" package. + try loadManifest(f, .{ .root_dir = tmp_directory }); + + // Apply the manifest's inclusion rules to the temporary directory by + // deleting excluded files. If any error occurred for files that were + // ultimately excluded, those errors should be ignored, such as failure to + // create symlinks that weren't supposed to be included anyway. + + // Empty directories have already been omitted by `unpackResource`. + + const filter: Filter = .{ + .include_paths = if (f.manifest) |m| m.paths else .{}, + }; + + // Compute the package hash based on the remaining files in the temporary + // directory. + + if (builtin.os.tag == .linux and f.work_around_btrfs_bug) { + // https://github.com/ziglang/zig/issues/17095 + tmp_directory.handle.close(); + const iterable_dir = f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch + @panic("btrfs workaround failed"); + tmp_directory.handle = iterable_dir.dir; + } + + f.actual_hash = try computeHash(f, .{ .dir = tmp_directory.handle }, filter); + + // Rename the temporary directory into the global zig package cache + // directory. If the hash already exists, delete the temporary directory + // and leave the zig package cache directory untouched as it may be in use + // by the system. This is done even if the hash is invalid, in case the + // package with the different hash is used in the future. + + const dest_pkg_sub_path = "p" ++ s ++ Manifest.hexDigest(f.actual_hash); + try renameTmpIntoCache(f.global_cache.handle, tmp_dir_sub_path, dest_pkg_sub_path); + + // Validate the computed hash against the expected hash. If invalid, this + // job is done. + + const actual_hex = Manifest.hexDigest(f.actual_hash); + if (remote.hash) |declared_hash| { + if (!std.mem.eql(u8, declared_hash, &actual_hex)) { + return f.fail(f.hash_tok, "hash mismatch: manifest declares {s} but the fetched package has {s}", .{ + declared_hash, actual_hex, + }); + } + } else { + const notes_len = 1; + try f.addErrorWithNotes(notes_len, f.location_tok, "dependency is missing hash field"); + const notes_start = try eb.reserveNotes(notes_len); + eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), + })); + return error.PackageFetchFailed; + } + + // Spawn a new fetch job for each dependency in the manifest file. Use + // a mutex and a hash map so that redundant jobs do not get queued up. + return queueJobsForDeps(f, .{ .hash = f.actual_hash }); +} + +/// This function populates `f.manifest` or leaves it `null`. +fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void { + const eb = &f.error_bundle; + const arena = f.arena_allocator.allocator(); + const manifest_bytes = pkg_root.readFileAllocOptions( + arena, + Manifest.basename, + Manifest.max_bytes, + null, + 1, + 0, + ) catch |err| switch (err) { + error.FileNotFound => return, + else => |e| { + const file_path = try pkg_root.join(arena, .{Manifest.basename}); + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to load package manifest '{s}': {s}", .{ + file_path, @errorName(e), + }), + .src_loc = .none, + .notes_len = 0, + }); + }, + }; + + var ast = try std.zig.Ast.parse(arena, manifest_bytes, .zon); + f.manifest_ast = ast; + + if (ast.errors.len > 0) { + const file_path = try pkg_root.join(arena, .{Manifest.basename}); + try main.putAstErrorsIntoBundle(arena, ast, file_path, eb); + return error.PackageFetchFailed; + } + + f.manifest = try Manifest.parse(arena, ast); + + if (f.manifest.errors.len > 0) { + const file_path = try pkg_root.join(arena, .{Manifest.basename}); + const token_starts = ast.tokens.items(.start); + + for (f.manifest.errors) |msg| { + const start_loc = ast.tokenLocation(0, msg.tok); + + try eb.addRootErrorMessage(.{ + .msg = try eb.addString(msg.msg), + .src_loc = try eb.addSourceLocation(.{ + .src_path = try eb.addString(file_path), + .span_start = token_starts[msg.tok], + .span_end = @intCast(token_starts[msg.tok] + ast.tokenSlice(msg.tok).len), + .span_main = token_starts[msg.tok] + msg.off, + .line = @intCast(start_loc.line), + .column = @intCast(start_loc.column), + .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), + }), + .notes_len = 0, + }); + } + return error.PackageFetchFailed; + } +} + +fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void { + // If the package does not have a build.zig.zon file then there are no dependencies. + const manifest = f.manifest orelse return; + + const new_fetches = nf: { + // Grab the new tasks into a temporary buffer so we can unlock that mutex + // as fast as possible. + // This overallocates any fetches that get skipped by the `continue` in the + // loop below. + const new_fetches = try f.arena.alloc(Fetch, manifest.dependencies.count()); + var new_fetch_index: usize = 0; + + f.job_queue.lock(); + defer f.job_queue.unlock(); + + // It is impossible for there to be a collision here. Consider all three cases: + // * Correct hash is provided by manifest. + // - Redundant jobs are skipped in the loop below. + // * Incorrect has is provided by manifest. + // - Hash mismatch error emitted; `queueJobsForDeps` is not called. + // * Hash is not provided by manifest. + // - Hash missing error emitted; `queueJobsForDeps` is not called. + try f.job_queue.finish(hash, f, new_fetches.len); + + for (manifest.dependencies.values()) |dep| { + const location: Location = switch (dep.location) { + .url => |url| .{ .remote = .{ + .url = url, + .hash = if (dep.hash) |h| h[0..hex_multihash_len].* else null, + } }, + .path => |path| .{ .relative_path = path }, + }; + const new_fetch = &new_fetches[new_fetch_index]; + const already_done = f.job_queue.add(location, new_fetch); + if (already_done) continue; + new_fetch_index += 1; + + new_fetch.* = .{ + .gpa = f.gpa, + .arena = std.heap.ArenaAllocator.init(f.gpa), + .location = location, + .location_tok = dep.location_tok, + .hash_tok = dep.hash_tok, + .global_cache = f.global_cache, + .parent_package_root = f.package_root, + .parent_manifest_ast = f.manifest_ast.?, + .prog_node = f.prog_node, + .http_client = f.http_client, + .thread_pool = f.thread_pool, + .job_queue = f.job_queue, + .wait_group = f.wait_group, + + .package_root = undefined, + .error_bundle = .{}, + .manifest = null, + .manifest_ast = null, + .actual_hash = undefined, + .has_build_zig = false, + }; + } + + break :nf new_fetches[0..new_fetch_index]; + }; + + // Now it's time to give tasks to the thread pool. + for (new_fetches) |new_fetch| { + f.wait_group.start(); + f.thread_pool.spawn(workerRun, .{f}) catch |err| switch (err) { + error.OutOfMemory => { + new_fetch.oom_flag = true; + f.wait_group.finish(); + continue; + }, + }; + } +} + +fn workerRun(f: *Fetch) void { + defer f.wait_group.finish(); + run(f) catch |err| switch (err) { + error.OutOfMemory => f.oom_flag = true, + error.FetchFailed => {}, // See `error_bundle`. + }; +} + +fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError!void { + const ast = f.parent_manifest_ast; + const token_starts = ast.tokens.items(.start); + const start_loc = ast.tokenLocation(0, msg_tok); + const eb = &f.error_bundle; + const file_path = try f.parent_package_root.join(f.arena, Manifest.basename); + const msg_off = 0; + + try eb.addRootErrorMessage(.{ + .msg = msg_str, + .src_loc = try eb.addSourceLocation(.{ + .src_path = try eb.addString(file_path), + .span_start = token_starts[msg_tok], + .span_end = @intCast(token_starts[msg_tok] + ast.tokenSlice(msg_tok).len), + .span_main = token_starts[msg_tok] + msg_off, + .line = @intCast(start_loc.line), + .column = @intCast(start_loc.column), + .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), + }), + .notes_len = 0, + }); + + return error.FetchFailed; +} + +const Resource = union(enum) { + file: fs.File, + http_request: std.http.Client.Request, + git_fetch_stream: git.Session.FetchStream, + dir: fs.IterableDir, +}; + +const FileType = enum { + tar, + @"tar.gz", + @"tar.xz", + git_pack, + + fn fromPath(file_path: []const u8) ?FileType { + if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; + if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; + if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; + return null; + } + + /// Parameter is a content-disposition header value. + fn fromContentDisposition(cd_header: []const u8) ?FileType { + const attach_end = ascii.indexOfIgnoreCase(cd_header, "attachment;") orelse + return null; + + var value_start = ascii.indexOfIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse + return null; + value_start += "filename".len; + if (cd_header[value_start] == '*') { + value_start += 1; + } + if (cd_header[value_start] != '=') return null; + value_start += 1; + + var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len; + if (cd_header[value_end - 1] == '\"') { + value_end -= 1; + } + return fromPath(cd_header[value_start..value_end]); + } + + test fromContentDisposition { + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); + + try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null); + try std.testing.expect(fromContentDisposition("attachment; FileName=\"stuff.tar\"") == null); + try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null); + try std.testing.expect(fromContentDisposition("attachment; size=42") == null); + try std.testing.expect(fromContentDisposition("inline; size=42") == null); + try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null); + try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null); + } +}; + +fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource { + const gpa = f.gpa; + const arena = f.arena_allocator.allocator(); + const eb = &f.error_bundle; + + if (ascii.eqlIgnoreCase(uri.scheme, "file")) return .{ + .file = try f.parent_package_root.openFile(uri.path, .{}), + }; + + if (ascii.eqlIgnoreCase(uri.scheme, "http") or + ascii.eqlIgnoreCase(uri.scheme, "https")) + { + var h = std.http.Headers{ .allocator = gpa }; + defer h.deinit(); + + var req = try f.http_client.request(.GET, uri, h, .{}); + errdefer req.deinit(); // releases more than memory + + try req.start(.{}); + try req.wait(); + + if (req.response.status != .ok) { + return f.fail(f.location_tok, "expected response status '200 OK' got '{s} {s}'", .{ + @intFromEnum(req.response.status), req.response.status.phrase() orelse "", + }); + } + + return .{ .http_request = req }; + } + + if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or + ascii.eqlIgnoreCase(uri.scheme, "git+https")) + { + var transport_uri = uri; + transport_uri.scheme = uri.scheme["git+".len..]; + var redirect_uri: []u8 = undefined; + var session: git.Session = .{ .transport = f.http_client, .uri = transport_uri }; + session.discoverCapabilities(gpa, &redirect_uri) catch |e| switch (e) { + error.Redirected => { + defer gpa.free(redirect_uri); + return f.fail(f.location_tok, "repository moved to {s}", .{redirect_uri}); + }, + else => |other| return other, + }; + + const want_oid = want_oid: { + const want_ref = uri.fragment orelse "HEAD"; + if (git.parseOid(want_ref)) |oid| break :want_oid oid else |_| {} + + const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref}); + const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref}); + + var ref_iterator = try session.listRefs(gpa, .{ + .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, + .include_peeled = true, + }); + defer ref_iterator.deinit(); + while (try ref_iterator.next()) |ref| { + if (std.mem.eql(u8, ref.name, want_ref) or + std.mem.eql(u8, ref.name, want_ref_head) or + std.mem.eql(u8, ref.name, want_ref_tag)) + { + break :want_oid ref.peeled orelse ref.oid; + } + } + return f.fail(f.location_tok, "ref not found: {s}", .{want_ref}); + }; + if (uri.fragment == null) { + const notes_len = 1; + try f.addErrorWithNotes(notes_len, f.location_tok, "url field is missing an explicit ref"); + const notes_start = try eb.reserveNotes(notes_len); + eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("try .url = \"{+/}#{}\",", .{ + uri, std.fmt.fmtSliceHexLower(&want_oid), + }), + })); + return error.PackageFetchFailed; + } + + var want_oid_buf: [git.fmt_oid_length]u8 = undefined; + _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{ + std.fmt.fmtSliceHexLower(&want_oid), + }) catch unreachable; + var fetch_stream = try session.fetch(gpa, &.{&want_oid_buf}); + errdefer fetch_stream.deinit(); + + return .{ .git_fetch_stream = fetch_stream }; + } + + return f.fail(f.location_tok, "unsupported URL scheme: {s}", .{uri.scheme}); +} + +fn unpackResource( + f: *Fetch, + resource: *Resource, + uri_path: []const u8, + tmp_directory: Cache.Directory, +) RunError!void { + const file_type = switch (resource.*) { + .file => FileType.fromPath(uri_path) orelse + return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path}), + + .http_request => |req| ft: { + // Content-Type takes first precedence. + const content_type = req.response.headers.getFirstValue("Content-Type") orelse + return f.fail(f.location_tok, "missing 'Content-Type' header", .{}); + + if (ascii.eqlIgnoreCase(content_type, "application/x-tar")) + return .tar; + + if (ascii.eqlIgnoreCase(content_type, "application/gzip") or + ascii.eqlIgnoreCase(content_type, "application/x-gzip") or + ascii.eqlIgnoreCase(content_type, "application/tar+gzip")) + { + return .@"tar.gz"; + } + + if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) + return .@"tar.xz"; + + if (!ascii.eqlIgnoreCase(content_type, "application/octet-stream")) { + return f.fail(f.location_tok, "unrecognized 'Content-Type' header: '{s}'", .{ + content_type, + }); + } + + // Next, the filename from 'content-disposition: attachment' takes precedence. + if (req.response.headers.getFirstValue("Content-Disposition")) |cd_header| { + break :ft FileType.fromContentDisposition(cd_header) orelse + return f.fail( + f.location_tok, + "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", + .{cd_header}, + ); + } + + // Finally, the path from the URI is used. + break :ft FileType.fromPath(uri_path) orelse + return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path}); + }, + .git_fetch_stream => return .git_pack, + .dir => |dir| { + try f.recursiveDirectoryCopy(dir, tmp_directory.handle); + return; + }, + }; + + switch (file_type) { + .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()), + .@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip), + .@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz), + .git_pack => try unpackGitPack(f, tmp_directory.handle, resource), + } +} + +fn unpackTarballCompressed( + f: *Fetch, + out_dir: fs.Dir, + resource: *Resource, + comptime Compression: type, +) RunError!void { + const gpa = f.gpa; + const reader = resource.reader(); + var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); + + var decompress = try Compression.decompress(gpa, br.reader()); + defer decompress.deinit(); + + return unpackTarball(f, out_dir, decompress.reader()); +} + +fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { + const eb = &f.error_bundle; + + var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = f.gpa }; + defer diagnostics.deinit(); + + try std.tar.pipeToFileSystem(out_dir, reader, .{ + .diagnostics = &diagnostics, + .strip_components = 1, + // TODO: we would like to set this to executable_bit_only, but two + // things need to happen before that: + // 1. the tar implementation needs to support it + // 2. the hashing algorithm here needs to support detecting the is_executable + // bit on Windows from the ACLs (see the isExecutable function). + .mode_mode = .ignore, + .filter = .{ .exclude_empty_directories = true }, + }); + + if (diagnostics.errors.items.len > 0) { + const notes_len: u32 = @intCast(diagnostics.errors.items.len); + try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack tarball"); + const notes_start = try eb.reserveNotes(notes_len); + for (diagnostics.errors.items, notes_start..) |item, note_i| { + switch (item) { + .unable_to_create_sym_link => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ + info.file_name, info.link_name, @errorName(info.code), + }), + })); + }, + .unsupported_file_type => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ + info.file_name, @intFromEnum(info.file_type), + }), + })); + }, + } + } + return error.InvalidTarball; + } +} + +fn unpackGitPack( + f: *Fetch, + out_dir: fs.Dir, + resource: *Resource, + want_oid: git.Oid, +) !void { + const eb = &f.error_bundle; + const gpa = f.gpa; + const reader = resource.reader(); + // The .git directory is used to store the packfile and associated index, but + // we do not attempt to replicate the exact structure of a real .git + // directory, since that isn't relevant for fetching a package. + { + var pack_dir = try out_dir.makeOpenPath(".git", .{}); + defer pack_dir.close(); + var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true }); + defer pack_file.close(); + var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); + try fifo.pump(reader.reader(), pack_file.writer()); + try pack_file.sync(); + + var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); + defer index_file.close(); + { + var index_prog_node = reader.prog_node.start("Index pack", 0); + defer index_prog_node.end(); + index_prog_node.activate(); + var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); + try git.indexPack(gpa, pack_file, index_buffered_writer.writer()); + try index_buffered_writer.flush(); + try index_file.sync(); + } + + { + var checkout_prog_node = reader.prog_node.start("Checkout", 0); + defer checkout_prog_node.end(); + checkout_prog_node.activate(); + var repository = try git.Repository.init(gpa, pack_file, index_file); + defer repository.deinit(); + var diagnostics: git.Diagnostics = .{ .allocator = gpa }; + defer diagnostics.deinit(); + try repository.checkout(out_dir, want_oid, &diagnostics); + + if (diagnostics.errors.items.len > 0) { + const notes_len: u32 = @intCast(diagnostics.errors.items.len); + try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack packfile"); + const notes_start = try eb.reserveNotes(notes_len); + for (diagnostics.errors.items, notes_start..) |item, note_i| { + switch (item) { + .unable_to_create_sym_link => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ + info.file_name, info.link_name, @errorName(info.code), + }), + })); + }, + } + } + return error.InvalidGitPack; + } + } + } + + try out_dir.deleteTree(".git"); +} + +fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) RunError!void { + // Recursive directory copy. + var it = try dir.walk(f.gpa); + defer it.deinit(); + while (try it.next()) |entry| { + switch (entry.kind) { + .directory => {}, // omit empty directories + .file => { + dir.dir.copyFile( + entry.path, + tmp_dir, + entry.path, + .{}, + ) catch |err| switch (err) { + error.FileNotFound => { + if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.makePath(dirname); + try dir.dir.copyFile(entry.path, tmp_dir, entry.path, .{}); + }, + else => |e| return e, + }; + }, + .sym_link => { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const link_name = try dir.dir.readLink(entry.path, &buf); + // TODO: if this would create a symlink to outside + // the destination directory, fail with an error instead. + try tmp_dir.symLink(link_name, entry.path, .{}); + }, + else => return error.IllegalFileTypeInPackage, + } + } +} + +pub fn renameTmpIntoCache( + cache_dir: fs.Dir, + tmp_dir_sub_path: []const u8, + dest_dir_sub_path: []const u8, +) !void { + assert(dest_dir_sub_path[1] == fs.path.sep); + var handled_missing_dir = false; + while (true) { + cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch |err| switch (err) { + error.FileNotFound => { + if (handled_missing_dir) return err; + cache_dir.makeDir(dest_dir_sub_path[0..1]) catch |mkd_err| switch (mkd_err) { + error.PathAlreadyExists => handled_missing_dir = true, + else => |e| return e, + }; + continue; + }, + error.PathAlreadyExists, error.AccessDenied => { + // Package has been already downloaded and may already be in use on the system. + cache_dir.deleteTree(tmp_dir_sub_path) catch { + // Garbage files leftover in zig-cache/tmp/ is, as they say + // on Star Trek, "operating within normal parameters". + }; + }, + else => |e| return e, + }; + break; + } +} + +/// Assumes that files not included in the package have already been filtered +/// prior to calling this function. This ensures that files not protected by +/// the hash are not present on the file system. Empty directories are *not +/// hashed* and must not be present on the file system when calling this +/// function. +fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Digest { + // All the path name strings need to be in memory for sorting. + const arena = f.arena_allocator.allocator(); + const gpa = f.gpa; + + // Collect all files, recursively, then sort. + var all_files = std.ArrayList(*HashedFile).init(gpa); + defer all_files.deinit(); + + var walker = try pkg_dir.walk(gpa); + defer walker.deinit(); + + { + // The final hash will be a hash of each file hashed independently. This + // allows hashing in parallel. + var wait_group: WaitGroup = .{}; + // `computeHash` is called from a worker thread so there must not be + // any waiting without working or a deadlock could occur. + defer wait_group.waitAndWork(); + + while (try walker.next()) |entry| { + _ = filter; // TODO: apply filter rules here + + const kind: HashedFile.Kind = switch (entry.kind) { + .directory => continue, + .file => .file, + .sym_link => .sym_link, + else => return error.IllegalFileTypeInPackage, + }; + + if (std.mem.eql(u8, entry.path, build_zig_basename)) + f.has_build_zig = true; + + const hashed_file = try arena.create(HashedFile); + const fs_path = try arena.dupe(u8, entry.path); + hashed_file.* = .{ + .fs_path = fs_path, + .normalized_path = try normalizePath(arena, fs_path), + .kind = kind, + .hash = undefined, // to be populated by the worker + .failure = undefined, // to be populated by the worker + }; + wait_group.start(); + try f.thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); + + try all_files.append(hashed_file); + } + } + + std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); + + var hasher = Manifest.Hash.init(.{}); + var any_failures = false; + const eb = &f.error_bundle; + for (all_files.items) |hashed_file| { + hashed_file.failure catch |err| { + any_failures = true; + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to hash: {s}", .{@errorName(err)}), + .src_loc = try eb.addSourceLocation(.{ + .src_path = try eb.addString(hashed_file.fs_path), + .span_start = 0, + .span_end = 0, + .span_main = 0, + }), + .notes_len = 0, + }); + }; + hasher.update(&hashed_file.hash); + } + if (any_failures) return error.FetchFailed; + return hasher.finalResult(); +} + +fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void { + defer wg.finish(); + hashed_file.failure = hashFileFallible(dir, hashed_file); +} + +fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { + var buf: [8000]u8 = undefined; + var hasher = Manifest.Hash.init(.{}); + hasher.update(hashed_file.normalized_path); + switch (hashed_file.kind) { + .file => { + var file = try dir.openFile(hashed_file.fs_path, .{}); + defer file.close(); + hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) }); + while (true) { + const bytes_read = try file.read(&buf); + if (bytes_read == 0) break; + hasher.update(buf[0..bytes_read]); + } + }, + .sym_link => { + const link_name = try dir.readLink(hashed_file.fs_path, &buf); + hasher.update(link_name); + }, + } + hasher.final(&hashed_file.hash); +} + +fn isExecutable(file: fs.File) !bool { + if (builtin.os.tag == .windows) { + // TODO check the ACL on Windows. + // Until this is implemented, this could be a false negative on + // Windows, which is why we do not yet set executable_bit_only above + // when unpacking the tarball. + return false; + } else { + const stat = try file.stat(); + return (stat.mode & std.os.S.IXUSR) != 0; + } +} + +const HashedFile = struct { + fs_path: []const u8, + normalized_path: []const u8, + hash: Digest, + failure: Error!void, + kind: Kind, + + const Error = + fs.File.OpenError || + fs.File.ReadError || + fs.File.StatError || + fs.Dir.ReadLinkError; + + const Kind = enum { file, sym_link }; + + fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { + _ = context; + return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); + } +}; + +/// Make a file system path identical independently of operating system path inconsistencies. +/// This converts backslashes into forward slashes. +fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 { + const canonical_sep = '/'; + + if (fs.path.sep == canonical_sep) + return fs_path; + + const normalized = try arena.dupe(u8, fs_path); + for (normalized) |*byte| { + switch (byte.*) { + fs.path.sep => byte.* = canonical_sep, + else => continue, + } + } + return normalized; +} + +pub const Filter = struct { + include_paths: std.StringArrayHashMapUnmanaged(void) = .{}, + + /// sub_path is relative to the tarball root. + pub fn includePath(self: Filter, sub_path: []const u8) bool { + if (self.include_paths.count() == 0) return true; + if (self.include_paths.contains("")) return true; + if (self.include_paths.contains(sub_path)) return true; + + // Check if any included paths are parent directories of sub_path. + var dirname = sub_path; + while (std.fs.path.dirname(sub_path)) |next_dirname| { + if (self.include_paths.contains(sub_path)) return true; + dirname = next_dirname; + } + + return false; + } +}; + +const build_zig_basename = @import("../Package.zig").build_zig_basename; +const hex_multihash_len = 2 * Manifest.multihash_len; + +// These are random bytes. +const package_hash_prefix_cached: [8]u8 = &.{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 }; +const package_hash_prefix_project: [8]u8 = &.{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc }; + +const builtin = @import("builtin"); +const std = @import("std"); +const fs = std.fs; +const assert = std.debug.assert; +const ascii = std.ascii; +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const ThreadPool = std.Thread.Pool; +const WaitGroup = std.Thread.WaitGroup; +const Manifest = @import("../Manifest.zig"); +const Fetch = @This(); +const main = @import("../main.zig"); +const git = @import("../git.zig"); diff --git a/src/Package/hash.zig b/src/Package/hash.zig deleted file mode 100644 index b14ec70244a3..000000000000 --- a/src/Package/hash.zig +++ /dev/null @@ -1,153 +0,0 @@ -const builtin = @import("builtin"); -const std = @import("std"); -const fs = std.fs; -const ThreadPool = std.Thread.Pool; -const WaitGroup = std.Thread.WaitGroup; -const Allocator = std.mem.Allocator; - -const Hash = @import("../Manifest.zig").Hash; - -pub fn compute(thread_pool: *ThreadPool, pkg_dir: fs.IterableDir) ![Hash.digest_length]u8 { - const gpa = thread_pool.allocator; - - // We'll use an arena allocator for the path name strings since they all - // need to be in memory for sorting. - var arena_instance = std.heap.ArenaAllocator.init(gpa); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - - // TODO: delete files not included in the package prior to computing the package hash. - // for example, if the ini file has directives to include/not include certain files, - // apply those rules directly to the filesystem right here. This ensures that files - // not protected by the hash are not present on the file system. - - // Collect all files, recursively, then sort. - var all_files = std.ArrayList(*HashedFile).init(gpa); - defer all_files.deinit(); - - var walker = try pkg_dir.walk(gpa); - defer walker.deinit(); - - { - // The final hash will be a hash of each file hashed independently. This - // allows hashing in parallel. - var wait_group: WaitGroup = .{}; - defer wait_group.wait(); - - while (try walker.next()) |entry| { - const kind: HashedFile.Kind = switch (entry.kind) { - .directory => continue, - .file => .file, - .sym_link => .sym_link, - else => return error.IllegalFileTypeInPackage, - }; - const hashed_file = try arena.create(HashedFile); - const fs_path = try arena.dupe(u8, entry.path); - hashed_file.* = .{ - .fs_path = fs_path, - .normalized_path = try normalizePath(arena, fs_path), - .kind = kind, - .hash = undefined, // to be populated by the worker - .failure = undefined, // to be populated by the worker - }; - wait_group.start(); - try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); - - try all_files.append(hashed_file); - } - } - - std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); - - var hasher = Hash.init(.{}); - var any_failures = false; - for (all_files.items) |hashed_file| { - hashed_file.failure catch |err| { - any_failures = true; - std.log.err("unable to hash '{s}': {s}", .{ hashed_file.fs_path, @errorName(err) }); - }; - hasher.update(&hashed_file.hash); - } - if (any_failures) return error.PackageHashUnavailable; - return hasher.finalResult(); -} - -const HashedFile = struct { - fs_path: []const u8, - normalized_path: []const u8, - hash: [Hash.digest_length]u8, - failure: Error!void, - kind: Kind, - - const Error = - fs.File.OpenError || - fs.File.ReadError || - fs.File.StatError || - fs.Dir.ReadLinkError; - - const Kind = enum { file, sym_link }; - - fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { - _ = context; - return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); - } -}; - -/// Make a file system path identical independently of operating system path inconsistencies. -/// This converts backslashes into forward slashes. -fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 { - const canonical_sep = '/'; - - if (fs.path.sep == canonical_sep) - return fs_path; - - const normalized = try arena.dupe(u8, fs_path); - for (normalized) |*byte| { - switch (byte.*) { - fs.path.sep => byte.* = canonical_sep, - else => continue, - } - } - return normalized; -} - -fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void { - defer wg.finish(); - hashed_file.failure = hashFileFallible(dir, hashed_file); -} - -fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { - var buf: [8000]u8 = undefined; - var hasher = Hash.init(.{}); - hasher.update(hashed_file.normalized_path); - switch (hashed_file.kind) { - .file => { - var file = try dir.openFile(hashed_file.fs_path, .{}); - defer file.close(); - hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) }); - while (true) { - const bytes_read = try file.read(&buf); - if (bytes_read == 0) break; - hasher.update(buf[0..bytes_read]); - } - }, - .sym_link => { - const link_name = try dir.readLink(hashed_file.fs_path, &buf); - hasher.update(link_name); - }, - } - hasher.final(&hashed_file.hash); -} - -fn isExecutable(file: fs.File) !bool { - if (builtin.os.tag == .windows) { - // TODO check the ACL on Windows. - // Until this is implemented, this could be a false negative on - // Windows, which is why we do not yet set executable_bit_only above - // when unpacking the tarball. - return false; - } else { - const stat = try file.stat(); - return (stat.mode & std.os.S.IXUSR) != 0; - } -} diff --git a/src/main.zig b/src/main.zig index 247669440d2b..34ab5ea191f3 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4714,7 +4714,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi defer if (cleanup_build_dir) |*dir| dir.close(); const cwd_path = try process.getCwdAlloc(arena); - const build_zig_basename = if (build_file) |bf| fs.path.basename(bf) else "build.zig"; + const build_zig_basename = if (build_file) |bf| fs.path.basename(bf) else Package.build_zig_basename; const build_directory: Compilation.Directory = blk: { if (build_file) |bf| { if (fs.path.dirname(bf)) |dirname| { From d0bcc390e8f61ada470b524e3fd203c1af521a99 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Oct 2023 17:05:58 -0700 Subject: [PATCH 06/39] get `zig fetch` working with the new system * start renaming "package" to "module" (see #14307) - build system gains `main_mod_path` and `main_pkg_path` is still there but it is deprecated. * eliminate the object-oriented memory management style of what was previously `*Package`. Now it is `*Package.Module` and all pointers point to externally managed memory. * fixes to get the new Fetch.zig code working. The previous commit was work-in-progress. There are still two commented out code paths, the one that leads to `Compilation.create` and the one for `zig build` that fetches the entire dependency tree and creates the required modules for the build runner. --- build.zig | 2 +- lib/std/Build.zig | 25 +- lib/std/Build/Cache.zig | 11 + lib/std/Build/Step/Compile.zig | 15 +- src/Compilation.zig | 170 +++++---- src/Manifest.zig | 14 +- src/Module.zig | 81 ++-- src/Package.zig | 302 ++++----------- src/Package/Fetch.zig | 651 ++++++++++++++++++++++----------- src/Package/Module.zig | 32 ++ src/crash_report.zig | 22 +- src/main.zig | 308 ++++++++-------- 12 files changed, 883 insertions(+), 750 deletions(-) create mode 100644 src/Package/Module.zig diff --git a/build.zig b/build.zig index 423f80b12a1b..6f9358587236 100644 --- a/build.zig +++ b/build.zig @@ -88,7 +88,7 @@ pub fn build(b: *std.Build) !void { .name = "check-case", .root_source_file = .{ .path = "test/src/Cases.zig" }, .optimize = optimize, - .main_pkg_path = .{ .path = "." }, + .main_mod_path = .{ .path = "." }, }); check_case_exe.stack_size = stack_size; check_case_exe.single_threaded = single_threaded; diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 49afd7377825..5c80f4972c25 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -634,6 +634,9 @@ pub const ExecutableOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -652,7 +655,7 @@ pub fn addExecutable(b: *Build, options: ExecutableOptions) *Step.Compile { .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } @@ -667,6 +670,9 @@ pub const ObjectOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -683,7 +689,7 @@ pub fn addObject(b: *Build, options: ObjectOptions) *Step.Compile { .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } @@ -699,6 +705,9 @@ pub const SharedLibraryOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -717,7 +726,7 @@ pub fn addSharedLibrary(b: *Build, options: SharedLibraryOptions) *Step.Compile .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } @@ -733,6 +742,9 @@ pub const StaticLibraryOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -751,7 +763,7 @@ pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *Step.Compile .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } @@ -769,6 +781,9 @@ pub const TestOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -787,7 +802,7 @@ pub fn addTest(b: *Build, options: TestOptions) *Step.Compile { .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index f76985a01aef..bb36bb978fc9 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -9,6 +9,13 @@ pub const Directory = struct { path: ?[]const u8, handle: fs.Dir, + pub fn cwd() Directory { + return .{ + .path = null, + .handle = fs.cwd(), + }; + } + pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 { if (self.path) |p| { // TODO clean way to do this with only 1 allocation @@ -53,6 +60,10 @@ pub const Directory = struct { try writer.writeAll(fs.path.sep_str); } } + + pub fn eql(self: Directory, other: Directory) bool { + return self.handle.fd == other.handle.fd; + } }; gpa: Allocator, diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 07eb3ded0183..9e711b6f52ad 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -68,7 +68,7 @@ c_std: std.Build.CStd, /// Set via options; intended to be read-only after that. zig_lib_dir: ?LazyPath, /// Set via options; intended to be read-only after that. -main_pkg_path: ?LazyPath, +main_mod_path: ?LazyPath, exec_cmd_args: ?[]const ?[]const u8, filter: ?[]const u8, test_evented_io: bool = false, @@ -316,6 +316,9 @@ pub const Options = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -480,7 +483,7 @@ pub fn create(owner: *std.Build, options: Options) *Compile { .installed_headers = ArrayList(*Step).init(owner.allocator), .c_std = std.Build.CStd.C99, .zig_lib_dir = null, - .main_pkg_path = null, + .main_mod_path = null, .exec_cmd_args = null, .filter = options.filter, .test_runner = options.test_runner, @@ -515,8 +518,8 @@ pub fn create(owner: *std.Build, options: Options) *Compile { lp.addStepDependencies(&self.step); } - if (options.main_pkg_path) |lp| { - self.main_pkg_path = lp.dupe(self.step.owner); + if (options.main_mod_path orelse options.main_pkg_path) |lp| { + self.main_mod_path = lp.dupe(self.step.owner); lp.addStepDependencies(&self.step); } @@ -1998,8 +2001,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try zig_args.append(dir.getPath(b)); } - if (self.main_pkg_path) |dir| { - try zig_args.append("--main-pkg-path"); + if (self.main_mod_path) |dir| { + try zig_args.append("--main-mod-path"); try zig_args.append(dir.getPath(b)); } diff --git a/src/Compilation.zig b/src/Compilation.zig index 28b67ff7346c..3827789fed0a 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -273,8 +273,8 @@ const Job = union(enum) { /// The source file containing the Decl has been updated, and so the /// Decl may need its line number information updated in the debug info. update_line_number: Module.Decl.Index, - /// The main source file for the package needs to be analyzed. - analyze_pkg: *Package, + /// The main source file for the module needs to be analyzed. + analyze_mod: *Package.Module, /// one of the glibc static objects glibc_crt_file: glibc.CRTFile, @@ -414,7 +414,7 @@ pub const MiscTask = enum { compiler_rt, libssp, zig_libc, - analyze_pkg, + analyze_mod, @"musl crti.o", @"musl crtn.o", @@ -544,7 +544,7 @@ pub const InitOptions = struct { global_cache_directory: Directory, target: Target, root_name: []const u8, - main_pkg: ?*Package, + main_mod: ?*Package.Module, output_mode: std.builtin.OutputMode, thread_pool: *ThreadPool, dynamic_linker: ?[]const u8 = null, @@ -736,53 +736,53 @@ pub const InitOptions = struct { pdb_out_path: ?[]const u8 = null, }; -fn addPackageTableToCacheHash( +fn addModuleTableToCacheHash( hash: *Cache.HashHelper, arena: *std.heap.ArenaAllocator, - pkg_table: Package.Table, - seen_table: *std.AutoHashMap(*Package, void), + mod_table: Package.Module.Deps, + seen_table: *std.AutoHashMap(*Package.Module, void), hash_type: union(enum) { path_bytes, files: *Cache.Manifest }, ) (error{OutOfMemory} || std.os.GetCwdError)!void { const allocator = arena.allocator(); - const packages = try allocator.alloc(Package.Table.KV, pkg_table.count()); + const modules = try allocator.alloc(Package.Module.Deps.KV, mod_table.count()); { // Copy over the hashmap entries to our slice - var table_it = pkg_table.iterator(); + var table_it = mod_table.iterator(); var idx: usize = 0; while (table_it.next()) |entry| : (idx += 1) { - packages[idx] = .{ + modules[idx] = .{ .key = entry.key_ptr.*, .value = entry.value_ptr.*, }; } } // Sort the slice by package name - mem.sort(Package.Table.KV, packages, {}, struct { - fn lessThan(_: void, lhs: Package.Table.KV, rhs: Package.Table.KV) bool { + mem.sortUnstable(Package.Module.Deps.KV, modules, {}, struct { + fn lessThan(_: void, lhs: Package.Module.Deps.KV, rhs: Package.Module.Deps.KV) bool { return std.mem.lessThan(u8, lhs.key, rhs.key); } }.lessThan); - for (packages) |pkg| { - if ((try seen_table.getOrPut(pkg.value)).found_existing) continue; + for (modules) |mod| { + if ((try seen_table.getOrPut(mod.value)).found_existing) continue; // Finally insert the package name and path to the cache hash. - hash.addBytes(pkg.key); + hash.addBytes(mod.key); switch (hash_type) { .path_bytes => { - hash.addBytes(pkg.value.root_src_path); - hash.addOptionalBytes(pkg.value.root_src_directory.path); + hash.addBytes(mod.value.root_src_path); + hash.addOptionalBytes(mod.value.root_src_directory.path); }, .files => |man| { - const pkg_zig_file = try pkg.value.root_src_directory.join(allocator, &[_][]const u8{ - pkg.value.root_src_path, + const pkg_zig_file = try mod.value.root_src_directory.join(allocator, &[_][]const u8{ + mod.value.root_src_path, }); _ = try man.addFile(pkg_zig_file, null); }, } - // Recurse to handle the package's dependencies - try addPackageTableToCacheHash(hash, arena, pkg.value.table, seen_table, hash_type); + // Recurse to handle the module's dependencies + try addModuleTableToCacheHash(hash, arena, mod.value.deps, seen_table, hash_type); } } @@ -839,7 +839,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { break :blk true; // If we have no zig code to compile, no need for LLVM. - if (options.main_pkg == null) + if (options.main_mod == null) break :blk false; // If LLVM does not support the target, then we can't use it. @@ -869,7 +869,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { // compiler state, the second clause here can be removed so that incremental // cache mode is used for LLVM backend too. We need some fuzz testing before // that can be enabled. - const cache_mode = if ((use_llvm or options.main_pkg == null) and !options.disable_lld_caching) + const cache_mode = if ((use_llvm or options.main_mod == null) and !options.disable_lld_caching) CacheMode.whole else options.cache_mode; @@ -925,7 +925,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { if (use_llvm) { // If stage1 generates an object file, self-hosted linker is not // yet sophisticated enough to handle that. - break :blk options.main_pkg != null; + break :blk options.main_mod != null; } break :blk false; @@ -1210,7 +1210,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model); // TODO audit this and make sure everything is in it - const module: ?*Module = if (options.main_pkg) |main_pkg| blk: { + const module: ?*Module = if (options.main_mod) |main_mod| blk: { // Options that are specific to zig source files, that cannot be // modified between incremental updates. var hash = cache.hash; @@ -1223,11 +1223,12 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { // do want to namespace different source file names because they are // likely different compilations and therefore this would be likely to // cause cache hits. - hash.addBytes(main_pkg.root_src_path); - hash.addOptionalBytes(main_pkg.root_src_directory.path); + hash.addBytes(main_mod.root_src_path); + hash.addOptionalBytes(main_mod.root.root_dir.path); + hash.addBytes(main_mod.root.sub_path); { - var seen_table = std.AutoHashMap(*Package, void).init(arena); - try addPackageTableToCacheHash(&hash, &arena_allocator, main_pkg.table, &seen_table, .path_bytes); + var seen_table = std.AutoHashMap(*Package.Module, void).init(arena); + try addModuleTableToCacheHash(&hash, &arena_allocator, main_mod.deps, &seen_table, .path_bytes); } }, .whole => { @@ -1283,34 +1284,31 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .path = try options.local_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}), }; - const builtin_pkg = try Package.createWithDir( - gpa, - zig_cache_artifact_directory, - null, - "builtin.zig", - ); - errdefer builtin_pkg.destroy(gpa); + const builtin_mod = try Package.Module.create(arena, .{ + .root = .{ .root_dir = zig_cache_artifact_directory }, + .root_src_path = "builtin.zig", + }); - // When you're testing std, the main module is std. In that case, we'll just set the std - // module to the main one, since avoiding the errors caused by duplicating it is more - // effort than it's worth. - const main_pkg_is_std = m: { + // When you're testing std, the main module is std. In that case, + // we'll just set the std module to the main one, since avoiding + // the errors caused by duplicating it is more effort than it's + // worth. + const main_mod_is_std = m: { const std_path = try std.fs.path.resolve(arena, &[_][]const u8{ options.zig_lib_directory.path orelse ".", "std", "std.zig", }); - defer arena.free(std_path); const main_path = try std.fs.path.resolve(arena, &[_][]const u8{ - main_pkg.root_src_directory.path orelse ".", - main_pkg.root_src_path, + main_mod.root.root_dir.path orelse ".", + main_mod.root.sub_path, + main_mod.root_src_path, }); - defer arena.free(main_path); break :m mem.eql(u8, main_path, std_path); }; - const std_pkg = if (main_pkg_is_std) - main_pkg + const std_mod = if (main_mod_is_std) + main_mod else try Package.createWithDir( gpa, @@ -1319,16 +1317,16 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { "std.zig", ); - errdefer if (!main_pkg_is_std) std_pkg.destroy(gpa); + errdefer if (!main_mod_is_std) std_mod.destroy(gpa); - const root_pkg = if (options.is_test) root_pkg: { + const root_mod = if (options.is_test) root_mod: { const test_pkg = if (options.test_runner_path) |test_runner| test_pkg: { const test_dir = std.fs.path.dirname(test_runner); const basename = std.fs.path.basename(test_runner); const pkg = try Package.create(gpa, test_dir, basename); - // copy package table from main_pkg to root_pkg - pkg.table = try main_pkg.table.clone(gpa); + // copy module table from main_mod to root_mod + pkg.deps = try main_mod.deps.clone(gpa); break :test_pkg pkg; } else try Package.createWithDir( gpa, @@ -1338,26 +1336,26 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { ); errdefer test_pkg.destroy(gpa); - break :root_pkg test_pkg; - } else main_pkg; - errdefer if (options.is_test) root_pkg.destroy(gpa); + break :root_mod test_pkg; + } else main_mod; + errdefer if (options.is_test) root_mod.destroy(gpa); - const compiler_rt_pkg = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_pkg: { - break :compiler_rt_pkg try Package.createWithDir( + const compiler_rt_mod = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_mod: { + break :compiler_rt_mod try Package.createWithDir( gpa, options.zig_lib_directory, null, "compiler_rt.zig", ); } else null; - errdefer if (compiler_rt_pkg) |p| p.destroy(gpa); + errdefer if (compiler_rt_mod) |p| p.destroy(gpa); - try main_pkg.add(gpa, "builtin", builtin_pkg); - try main_pkg.add(gpa, "root", root_pkg); - try main_pkg.add(gpa, "std", std_pkg); + try main_mod.add(gpa, "builtin", builtin_mod); + try main_mod.add(gpa, "root", root_mod); + try main_mod.add(gpa, "std", std_mod); - if (compiler_rt_pkg) |p| { - try main_pkg.add(gpa, "compiler_rt", p); + if (compiler_rt_mod) |p| { + try main_mod.add(gpa, "compiler_rt", p); } // Pre-open the directory handles for cached ZIR code so that it does not need @@ -1395,8 +1393,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { module.* = .{ .gpa = gpa, .comp = comp, - .main_pkg = main_pkg, - .root_pkg = root_pkg, + .main_mod = main_mod, + .root_mod = root_mod, .zig_cache_artifact_directory = zig_cache_artifact_directory, .global_zir_cache = global_zir_cache, .local_zir_cache = local_zir_cache, @@ -2005,8 +2003,8 @@ fn restorePrevZigCacheArtifactDirectory(comp: *Compilation, directory: *Director // This is only for cleanup purposes; Module.deinit calls close // on the handle of zig_cache_artifact_directory. if (comp.bin_file.options.module) |module| { - const builtin_pkg = module.main_pkg.table.get("builtin").?; - module.zig_cache_artifact_directory = builtin_pkg.root_src_directory; + const builtin_mod = module.main_mod.deps.get("builtin").?; + module.zig_cache_artifact_directory = builtin_mod.root_src_directory; } } @@ -2148,8 +2146,8 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void // Make sure std.zig is inside the import_table. We unconditionally need // it for start.zig. - const std_pkg = module.main_pkg.table.get("std").?; - _ = try module.importPkg(std_pkg); + const std_mod = module.main_mod.deps.get("std").?; + _ = try module.importPkg(std_mod); // Normally we rely on importing std to in turn import the root source file // in the start code, but when using the stage1 backend that won't happen, @@ -2158,11 +2156,11 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void // Likewise, in the case of `zig test`, the test runner is the root source file, // and so there is nothing to import the main file. if (comp.bin_file.options.is_test) { - _ = try module.importPkg(module.main_pkg); + _ = try module.importPkg(module.main_mod); } - if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| { - _ = try module.importPkg(compiler_rt_pkg); + if (module.main_mod.deps.get("compiler_rt")) |compiler_rt_mod| { + _ = try module.importPkg(compiler_rt_mod); } // Put a work item in for every known source file to detect if @@ -2185,13 +2183,13 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void } } - try comp.work_queue.writeItem(.{ .analyze_pkg = std_pkg }); + try comp.work_queue.writeItem(.{ .analyze_mod = std_mod }); if (comp.bin_file.options.is_test) { - try comp.work_queue.writeItem(.{ .analyze_pkg = module.main_pkg }); + try comp.work_queue.writeItem(.{ .analyze_mod = module.main_mod }); } - if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| { - try comp.work_queue.writeItem(.{ .analyze_pkg = compiler_rt_pkg }); + if (module.main_mod.deps.get("compiler_rt")) |compiler_rt_mod| { + try comp.work_queue.writeItem(.{ .analyze_mod = compiler_rt_mod }); } } @@ -2420,19 +2418,19 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes comptime assert(link_hash_implementation_version == 10); if (comp.bin_file.options.module) |mod| { - const main_zig_file = try mod.main_pkg.root_src_directory.join(arena, &[_][]const u8{ - mod.main_pkg.root_src_path, + const main_zig_file = try mod.main_mod.root_src_directory.join(arena, &[_][]const u8{ + mod.main_mod.root_src_path, }); _ = try man.addFile(main_zig_file, null); { - var seen_table = std.AutoHashMap(*Package, void).init(arena); + var seen_table = std.AutoHashMap(*Package.Module, void).init(arena); // Skip builtin.zig; it is useless as an input, and we don't want to have to // write it before checking for a cache hit. - const builtin_pkg = mod.main_pkg.table.get("builtin").?; - try seen_table.put(builtin_pkg, {}); + const builtin_mod = mod.main_mod.deps.get("builtin").?; + try seen_table.put(builtin_mod, {}); - try addPackageTableToCacheHash(&man.hash, &arena_allocator, mod.main_pkg.table, &seen_table, .{ .files = man }); + try addModuleTableToCacheHash(&man.hash, &arena_allocator, mod.main_mod.deps, &seen_table, .{ .files = man }); } // Synchronize with other matching comments: ZigOnlyHashStuff @@ -3564,8 +3562,8 @@ fn processOneJob(comp: *Compilation, job: Job, prog_node: *std.Progress.Node) !v decl.analysis = .codegen_failure_retryable; }; }, - .analyze_pkg => |pkg| { - const named_frame = tracy.namedFrame("analyze_pkg"); + .analyze_mod => |pkg| { + const named_frame = tracy.namedFrame("analyze_mod"); defer named_frame.end(); const module = comp.bin_file.options.module.?; @@ -6379,11 +6377,11 @@ fn buildOutputFromZig( std.debug.assert(output_mode != .Exe); - var main_pkg: Package = .{ + var main_mod: Package = .{ .root_src_directory = comp.zig_lib_directory, .root_src_path = src_basename, }; - defer main_pkg.deinitTable(comp.gpa); + defer main_mod.deinitTable(comp.gpa); const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len]; const target = comp.getTarget(); const bin_basename = try std.zig.binNameAlloc(comp.gpa, .{ @@ -6404,7 +6402,7 @@ fn buildOutputFromZig( .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = &main_pkg, + .main_mod = &main_mod, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, @@ -6481,7 +6479,7 @@ pub fn build_crt_file( .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = null, + .main_mod = null, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, diff --git a/src/Manifest.zig b/src/Manifest.zig index ac384b8fcb64..4032244d6eed 100644 --- a/src/Manifest.zig +++ b/src/Manifest.zig @@ -1,6 +1,10 @@ pub const max_bytes = 10 * 1024 * 1024; pub const basename = "build.zig.zon"; pub const Hash = std.crypto.hash.sha2.Sha256; +pub const Digest = [Hash.digest_length]u8; +pub const multihash_len = 1 + 1 + Hash.digest_length; +pub const multihash_hex_digest_len = 2 * multihash_len; +pub const MultiHashHexDigest = [multihash_hex_digest_len]u8; pub const Dependency = struct { location: union(enum) { @@ -46,7 +50,6 @@ comptime { assert(@intFromEnum(multihash_function) < 127); assert(Hash.digest_length < 127); } -pub const multihash_len = 1 + 1 + Hash.digest_length; name: []const u8, version: std.SemanticVersion, @@ -122,8 +125,8 @@ test hex64 { try std.testing.expectEqualStrings("[00efcdab78563412]", s); } -pub fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 { - var result: [multihash_len * 2]u8 = undefined; +pub fn hexDigest(digest: Digest) MultiHashHexDigest { + var result: MultiHashHexDigest = undefined; result[0] = hex_charset[@intFromEnum(multihash_function) >> 4]; result[1] = hex_charset[@intFromEnum(multihash_function) & 15]; @@ -339,10 +342,9 @@ const Parse = struct { } } - const hex_multihash_len = 2 * Manifest.multihash_len; - if (h.len != hex_multihash_len) { + if (h.len != multihash_hex_digest_len) { return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{ - hex_multihash_len, h.len, + multihash_hex_digest_len, h.len, }); } diff --git a/src/Module.zig b/src/Module.zig index 8c4035cc9af7..141ee3c755ef 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -55,10 +55,10 @@ comp: *Compilation, /// Where build artifacts and incremental compilation metadata serialization go. zig_cache_artifact_directory: Compilation.Directory, /// Pointer to externally managed resource. -root_pkg: *Package, -/// Normally, `main_pkg` and `root_pkg` are the same. The exception is `zig test`, in which -/// `root_pkg` is the test runner, and `main_pkg` is the user's source file which has the tests. -main_pkg: *Package, +root_mod: *Package.Module, +/// Normally, `main_mod` and `root_mod` are the same. The exception is `zig test`, in which +/// `root_mod` is the test runner, and `main_mod` is the user's source file which has the tests. +main_mod: *Package.Module, sema_prog_node: std.Progress.Node = undefined, /// Used by AstGen worker to load and store ZIR cache. @@ -973,8 +973,8 @@ pub const File = struct { tree: Ast, /// Whether this is populated or not depends on `zir_loaded`. zir: Zir, - /// Package that this file is a part of, managed externally. - pkg: *Package, + /// Module that this file is a part of, managed externally. + mod: *Package.Module, /// Whether this file is a part of multiple packages. This is an error condition which will be reported after AstGen. multi_pkg: bool = false, /// List of references to this file, used for multi-package errors. @@ -1058,14 +1058,9 @@ pub const File = struct { .stat = file.stat, }; - const root_dir_path = file.pkg.root_src_directory.path orelse "."; - log.debug("File.getSource, not cached. pkgdir={s} sub_file_path={s}", .{ - root_dir_path, file.sub_file_path, - }); - // Keep track of inode, file size, mtime, hash so we can detect which files // have been modified when an incremental update is requested. - var f = try file.pkg.root_src_directory.handle.openFile(file.sub_file_path, .{}); + var f = try file.mod.root.openFile(file.sub_file_path, .{}); defer f.close(); const stat = try f.stat(); @@ -1134,14 +1129,12 @@ pub const File = struct { return ip.getOrPutTrailingString(mod.gpa, ip.string_bytes.items.len - start); } - /// Returns the full path to this file relative to its package. pub fn fullPath(file: File, ally: Allocator) ![]u8 { - return file.pkg.root_src_directory.join(ally, &[_][]const u8{file.sub_file_path}); + return file.mod.root.joinString(ally, file.sub_file_path); } - /// Returns the full path to this file relative to its package. pub fn fullPathZ(file: File, ally: Allocator) ![:0]u8 { - return file.pkg.root_src_directory.joinZ(ally, &[_][]const u8{file.sub_file_path}); + return file.mod.root.joinStringZ(ally, file.sub_file_path); } pub fn dumpSrc(file: *File, src: LazySrcLoc) void { @@ -2543,25 +2536,25 @@ pub fn deinit(mod: *Module) void { mod.deletion_set.deinit(gpa); - // The callsite of `Compilation.create` owns the `main_pkg`, however + // The callsite of `Compilation.create` owns the `main_mod`, however // Module owns the builtin and std packages that it adds. - if (mod.main_pkg.table.fetchRemove("builtin")) |kv| { + if (mod.main_mod.table.fetchRemove("builtin")) |kv| { gpa.free(kv.key); kv.value.destroy(gpa); } - if (mod.main_pkg.table.fetchRemove("std")) |kv| { + if (mod.main_mod.table.fetchRemove("std")) |kv| { gpa.free(kv.key); - // It's possible for main_pkg to be std when running 'zig test'! In this case, we must not + // It's possible for main_mod to be std when running 'zig test'! In this case, we must not // destroy it, since it would lead to a double-free. - if (kv.value != mod.main_pkg) { + if (kv.value != mod.main_mod) { kv.value.destroy(gpa); } } - if (mod.main_pkg.table.fetchRemove("root")) |kv| { + if (mod.main_mod.table.fetchRemove("root")) |kv| { gpa.free(kv.key); } - if (mod.root_pkg != mod.main_pkg) { - mod.root_pkg.destroy(gpa); + if (mod.root_mod != mod.main_mod) { + mod.root_mod.destroy(gpa); } mod.compile_log_text.deinit(gpa); @@ -2715,7 +2708,7 @@ pub fn astGenFile(mod: *Module, file: *File) !void { const stat = try source_file.stat(); - const want_local_cache = file.pkg == mod.main_pkg; + const want_local_cache = file.pkg == mod.main_mod; const digest = hash: { var path_hash: Cache.HashHelper = .{}; path_hash.addBytes(build_options.version); @@ -3158,23 +3151,23 @@ pub fn populateBuiltinFile(mod: *Module) !void { comp.mutex.lock(); defer comp.mutex.unlock(); - const builtin_pkg = mod.main_pkg.table.get("builtin").?; - const result = try mod.importPkg(builtin_pkg); + const builtin_mod = mod.main_mod.table.get("builtin").?; + const result = try mod.importPkg(builtin_mod); break :blk .{ .file = result.file, - .pkg = builtin_pkg, + .pkg = builtin_mod, }; }; const file = pkg_and_file.file; - const builtin_pkg = pkg_and_file.pkg; + const builtin_mod = pkg_and_file.pkg; const gpa = mod.gpa; file.source = try comp.generateBuiltinZigSource(gpa); file.source_loaded = true; - if (builtin_pkg.root_src_directory.handle.statFile(builtin_pkg.root_src_path)) |stat| { + if (builtin_mod.root_src_directory.handle.statFile(builtin_mod.root_src_path)) |stat| { if (stat.size != file.source.len) { - const full_path = try builtin_pkg.root_src_directory.join(gpa, &.{ - builtin_pkg.root_src_path, + const full_path = try builtin_mod.root_src_directory.join(gpa, &.{ + builtin_mod.root_src_path, }); defer gpa.free(full_path); @@ -3184,7 +3177,7 @@ pub fn populateBuiltinFile(mod: *Module) !void { .{ full_path, file.source.len, stat.size }, ); - try writeBuiltinFile(file, builtin_pkg); + try writeBuiltinFile(file, builtin_mod); } else { file.stat = .{ .size = stat.size, @@ -3198,7 +3191,7 @@ pub fn populateBuiltinFile(mod: *Module) !void { error.PipeBusy => unreachable, // it's not a pipe error.WouldBlock => unreachable, // not asking for non-blocking I/O - error.FileNotFound => try writeBuiltinFile(file, builtin_pkg), + error.FileNotFound => try writeBuiltinFile(file, builtin_mod), else => |e| return e, } @@ -3212,8 +3205,8 @@ pub fn populateBuiltinFile(mod: *Module) !void { file.status = .success_zir; } -fn writeBuiltinFile(file: *File, builtin_pkg: *Package) !void { - var af = try builtin_pkg.root_src_directory.handle.atomicFile(builtin_pkg.root_src_path, .{}); +fn writeBuiltinFile(file: *File, builtin_mod: *Package.Module) !void { + var af = try builtin_mod.root_src_directory.handle.atomicFile(builtin_mod.root_src_path, .{}); defer af.deinit(); try af.file.writeAll(file.source); try af.finish(); @@ -3748,7 +3741,7 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !bool { // TODO: figure out how this works under incremental changes to builtin.zig! const builtin_type_target_index: InternPool.Index = blk: { - const std_mod = mod.main_pkg.table.get("std").?; + const std_mod = mod.main_mod.table.get("std").?; if (decl.getFileScope(mod).pkg != std_mod) break :blk .none; // We're in the std module. const std_file = (try mod.importPkg(std_mod)).file; @@ -4100,13 +4093,13 @@ pub fn importFile( import_string: []const u8, ) !ImportFileResult { if (std.mem.eql(u8, import_string, "std")) { - return mod.importPkg(mod.main_pkg.table.get("std").?); + return mod.importPkg(mod.main_mod.table.get("std").?); } if (std.mem.eql(u8, import_string, "builtin")) { - return mod.importPkg(mod.main_pkg.table.get("builtin").?); + return mod.importPkg(mod.main_mod.table.get("builtin").?); } if (std.mem.eql(u8, import_string, "root")) { - return mod.importPkg(mod.root_pkg); + return mod.importPkg(mod.root_mod); } if (cur_file.pkg.table.get(import_string)) |pkg| { return mod.importPkg(pkg); @@ -4462,14 +4455,14 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) Allocator.Err // test decl with no name. Skip the part where we check against // the test name filter. if (!comp.bin_file.options.is_test) break :blk false; - if (decl_pkg != mod.main_pkg) break :blk false; + if (decl_pkg != mod.main_mod) break :blk false; try mod.test_functions.put(gpa, new_decl_index, {}); break :blk true; }, else => blk: { if (!is_named_test) break :blk false; if (!comp.bin_file.options.is_test) break :blk false; - if (decl_pkg != mod.main_pkg) break :blk false; + if (decl_pkg != mod.main_mod) break :blk false; if (comp.test_filter) |test_filter| { if (mem.indexOf(u8, ip.stringToSlice(decl_name), test_filter) == null) { break :blk false; @@ -5596,8 +5589,8 @@ pub fn populateTestFunctions( ) !void { const gpa = mod.gpa; const ip = &mod.intern_pool; - const builtin_pkg = mod.main_pkg.table.get("builtin").?; - const builtin_file = (mod.importPkg(builtin_pkg) catch unreachable).file; + const builtin_mod = mod.main_mod.table.get("builtin").?; + const builtin_file = (mod.importPkg(builtin_mod) catch unreachable).file; const root_decl = mod.declPtr(builtin_file.root_decl.unwrap().?); const builtin_namespace = mod.namespacePtr(root_decl.src_namespace); const test_functions_str = try ip.getOrPutString(gpa, "test_functions"); diff --git a/src/Package.zig b/src/Package.zig index e5fa24e18d7d..0249f8dae17c 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -1,251 +1,87 @@ -const Package = @This(); - -const builtin = @import("builtin"); -const std = @import("std"); -const fs = std.fs; -const mem = std.mem; -const Allocator = mem.Allocator; -const ascii = std.ascii; -const assert = std.debug.assert; -const log = std.log.scoped(.package); -const main = @import("main.zig"); -const ThreadPool = std.Thread.Pool; - -const Compilation = @import("Compilation.zig"); -const Module = @import("Module.zig"); -const Cache = std.Build.Cache; -const build_options = @import("build_options"); -const Fetch = @import("Package/Fetch.zig"); - +pub const Module = @import("Package/Module.zig"); +pub const Fetch = @import("Package/Fetch.zig"); pub const build_zig_basename = "build.zig"; pub const Manifest = @import("Manifest.zig"); -pub const Table = std.StringHashMapUnmanaged(*Package); - -root_src_directory: Compilation.Directory, -/// Relative to `root_src_directory`. May contain path separators. -root_src_path: []const u8, -/// The dependency table of this module. Shared dependencies such as 'std', 'builtin', and 'root' -/// are not specified in every dependency table, but instead only in the table of `main_pkg`. -/// `Module.importFile` is responsible for detecting these names and using the correct package. -table: Table = .{}, -/// Whether to free `root_src_directory` on `destroy`. -root_src_directory_owned: bool = false, - -/// Allocate a Package. No references to the slices passed are kept. -pub fn create( - gpa: Allocator, - /// Null indicates the current working directory - root_src_dir_path: ?[]const u8, - /// Relative to root_src_dir_path - root_src_path: []const u8, -) !*Package { - const ptr = try gpa.create(Package); - errdefer gpa.destroy(ptr); - - const owned_dir_path = if (root_src_dir_path) |p| try gpa.dupe(u8, p) else null; - errdefer if (owned_dir_path) |p| gpa.free(p); - - const owned_src_path = try gpa.dupe(u8, root_src_path); - errdefer gpa.free(owned_src_path); - ptr.* = .{ - .root_src_directory = .{ - .path = owned_dir_path, - .handle = if (owned_dir_path) |p| try fs.cwd().openDir(p, .{}) else fs.cwd(), - }, - .root_src_path = owned_src_path, - .root_src_directory_owned = true, - }; +pub const Path = struct { + root_dir: Cache.Directory, + /// The path, relative to the root dir, that this `Path` represents. + /// Empty string means the root_dir is the path. + sub_path: []const u8 = "", - return ptr; -} - -pub fn createWithDir( - gpa: Allocator, - directory: Compilation.Directory, - /// Relative to `directory`. If null, means `directory` is the root src dir - /// and is owned externally. - root_src_dir_path: ?[]const u8, - /// Relative to root_src_dir_path - root_src_path: []const u8, -) !*Package { - const ptr = try gpa.create(Package); - errdefer gpa.destroy(ptr); - - const owned_src_path = try gpa.dupe(u8, root_src_path); - errdefer gpa.free(owned_src_path); - - if (root_src_dir_path) |p| { - const owned_dir_path = try directory.join(gpa, &[1][]const u8{p}); - errdefer gpa.free(owned_dir_path); - - ptr.* = .{ - .root_src_directory = .{ - .path = owned_dir_path, - .handle = try directory.handle.openDir(p, .{}), - }, - .root_src_directory_owned = true, - .root_src_path = owned_src_path, - }; - } else { - ptr.* = .{ - .root_src_directory = directory, - .root_src_directory_owned = false, - .root_src_path = owned_src_path, - }; + pub fn cwd() Path { + return .{ .root_dir = Cache.Directory.cwd() }; } - return ptr; -} -/// Free all memory associated with this package. It does not destroy any packages -/// inside its table; the caller is responsible for calling destroy() on them. -pub fn destroy(pkg: *Package, gpa: Allocator) void { - gpa.free(pkg.root_src_path); - - if (pkg.root_src_directory_owned) { - // If root_src_directory.path is null then the handle is the cwd() - // which shouldn't be closed. - if (pkg.root_src_directory.path) |p| { - gpa.free(p); - pkg.root_src_directory.handle.close(); - } + pub fn join(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error!Path { + const parts: []const []const u8 = + if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path }; + return .{ + .root_dir = p.root_dir, + .sub_path = try fs.path.join(allocator, parts), + }; } - pkg.deinitTable(gpa); - gpa.destroy(pkg); -} - -/// Only frees memory associated with the table. -pub fn deinitTable(pkg: *Package, gpa: Allocator) void { - pkg.table.deinit(gpa); -} - -pub fn add(pkg: *Package, gpa: Allocator, name: []const u8, package: *Package) !void { - try pkg.table.ensureUnusedCapacity(gpa, 1); - const name_dupe = try gpa.dupe(u8, name); - pkg.table.putAssumeCapacityNoClobber(name_dupe, package); -} - -/// Compute a readable name for the package. The returned name should be freed from gpa. This -/// function is very slow, as it traverses the whole package hierarchy to find a path to this -/// package. It should only be used for error output. -pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8 { - // we'll do a breadth-first search from the root module to try and find a short name for this - // module, using a DoublyLinkedList of module/parent pairs. note that the "parent" there is - // just the first-found shortest path - a module may be children of arbitrarily many other - // modules. This path may vary between executions due to hashmap iteration order, but that - // doesn't matter too much. - var node_arena = std.heap.ArenaAllocator.init(gpa); - defer node_arena.deinit(); - const Parented = struct { - parent: ?*const @This(), - mod: *const Package, - }; - const Queue = std.DoublyLinkedList(Parented); - var to_check: Queue = .{}; - - { - const new = try node_arena.allocator().create(Queue.Node); - new.* = .{ .data = .{ .parent = null, .mod = mod.root_pkg } }; - to_check.prepend(new); + pub fn joinString(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 { + const parts: []const []const u8 = + if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path }; + return p.root_dir.join(allocator, parts); } - if (mod.main_pkg != mod.root_pkg) { - const new = try node_arena.allocator().create(Queue.Node); - // TODO: once #12201 is resolved, we may want a way of indicating a different name for this - new.* = .{ .data = .{ .parent = null, .mod = mod.main_pkg } }; - to_check.prepend(new); + pub fn joinStringZ(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 { + const parts: []const []const u8 = + if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path }; + return p.root_dir.joinZ(allocator, parts); } - // set of modules we've already checked to prevent loops - var checked = std.AutoHashMap(*const Package, void).init(gpa); - defer checked.deinit(); - - const linked = while (to_check.pop()) |node| { - const check = &node.data; - - if (checked.contains(check.mod)) continue; - try checked.put(check.mod, {}); - - if (check.mod == target) break check; - - var it = check.mod.table.iterator(); - while (it.next()) |kv| { - var new = try node_arena.allocator().create(Queue.Node); - new.* = .{ .data = .{ - .parent = check, - .mod = kv.value_ptr.*, - } }; - to_check.prepend(new); - } - } else { - // this can happen for e.g. @cImport packages - return gpa.dupe(u8, ""); - }; - - // we found a path to the module! unfortunately, we can only traverse *up* it, so we have to put - // all the names into a buffer so we can then print them in order. - var names = std.ArrayList([]const u8).init(gpa); - defer names.deinit(); - - var cur: *const Parented = linked; - while (cur.parent) |parent| : (cur = parent) { - // find cur's name in parent - var it = parent.mod.table.iterator(); - const name = while (it.next()) |kv| { - if (kv.value_ptr.* == cur.mod) { - break kv.key_ptr.*; - } - } else unreachable; - try names.append(name); + pub fn openFile( + p: Path, + sub_path: []const u8, + flags: fs.File.OpenFlags, + ) fs.File.OpenError!fs.File { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const joined_path = if (p.sub_path.len == 0) sub_path else p: { + break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ + p.sub_path, sub_path, + }) catch return error.NameTooLong; + }; + return p.root_dir.handle.openFile(joined_path, flags); } - // finally, print the names into a buffer! - var buf = std.ArrayList(u8).init(gpa); - defer buf.deinit(); - try buf.writer().writeAll("root"); - var i: usize = names.items.len; - while (i > 0) { - i -= 1; - try buf.writer().print(".{s}", .{names.items[i]}); + pub fn makeOpenPath(p: Path, sub_path: []const u8, opts: fs.OpenDirOptions) !fs.Dir { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const joined_path = if (p.sub_path.len == 0) sub_path else p: { + break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ + p.sub_path, sub_path, + }) catch return error.NameTooLong; + }; + return p.root_dir.handle.makeOpenPath(joined_path, opts); } - return buf.toOwnedSlice(); -} - -pub fn createFilePkg( - gpa: Allocator, - cache_directory: Compilation.Directory, - basename: []const u8, - contents: []const u8, -) !*Package { - const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ Manifest.hex64(rand_int); - { - var tmp_dir = try cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); - defer tmp_dir.close(); - try tmp_dir.writeFile(basename, contents); + pub fn format( + self: Path, + comptime fmt_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = options; + if (fmt_string.len > 0) + std.fmt.invalidFmtError(fmt_string, self); + if (self.root_dir.path) |p| { + try writer.writeAll(p); + try writer.writeAll(fs.path.sep_str); + } + if (self.sub_path.len > 0) { + try writer.writeAll(self.sub_path); + try writer.writeAll(fs.path.sep_str); + } } - - var hh: Cache.HashHelper = .{}; - hh.addBytes(build_options.version); - hh.addBytes(contents); - const hex_digest = hh.final(); - - const o_dir_sub_path = "o" ++ fs.path.sep_str ++ hex_digest; - try Fetch.renameTmpIntoCache(cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path); - - return createWithDir(gpa, cache_directory, o_dir_sub_path, basename); -} - -const hex_multihash_len = 2 * Manifest.multihash_len; -const MultiHashHexDigest = [hex_multihash_len]u8; - -const DependencyModule = union(enum) { - zig_pkg: *Package, - non_zig_pkg: *Package, }; -/// This is to avoid creating multiple modules for the same build.zig file. -/// If the value is `null`, the package is a known dependency, but has not yet -/// been fetched. -pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?DependencyModule); + +const Package = @This(); +const builtin = @import("builtin"); +const std = @import("std"); +const fs = std.fs; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const Cache = std.Build.Cache; diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index b3b4667e4093..55de79f039fb 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -27,59 +27,84 @@ //! All of this must be done with only referring to the state inside this struct //! because this work will be done in a dedicated thread. -/// Try to avoid this as much as possible since arena will have less contention. -gpa: Allocator, arena: std.heap.ArenaAllocator, location: Location, location_tok: std.zig.Ast.TokenIndex, hash_tok: std.zig.Ast.TokenIndex, -global_cache: Cache.Directory, -parent_package_root: Path, +parent_package_root: Package.Path, parent_manifest_ast: ?*const std.zig.Ast, prog_node: *std.Progress.Node, -http_client: *std.http.Client, -thread_pool: *ThreadPool, job_queue: *JobQueue, -wait_group: *WaitGroup, +/// If true, don't add an error for a missing hash. This flag is not passed +/// down to recursive dependencies. It's intended to be used only be the CLI. +omit_missing_hash_error: bool, // Above this are fields provided as inputs to `run`. // Below this are fields populated by `run`. /// This will either be relative to `global_cache`, or to the build root of /// the root package. -package_root: Path, +package_root: Package.Path, error_bundle: std.zig.ErrorBundle.Wip, manifest: ?Manifest, -manifest_ast: ?*std.zig.Ast, -actual_hash: Digest, +manifest_ast: std.zig.Ast, +actual_hash: Manifest.Digest, /// Fetch logic notices whether a package has a build.zig file and sets this flag. has_build_zig: bool, /// Indicates whether the task aborted due to an out-of-memory condition. oom_flag: bool, +/// Contains shared state among all `Fetch` tasks. pub const JobQueue = struct { mutex: std.Thread.Mutex = .{}, -}; - -pub const Digest = [Manifest.Hash.digest_length]u8; -pub const MultiHashHexDigest = [hex_multihash_len]u8; - -pub const Path = struct { - root_dir: Cache.Directory, - /// The path, relative to the root dir, that this `Path` represents. - /// Empty string means the root_dir is the path. - sub_path: []const u8 = "", + /// Protected by `mutex`. + table: Table = .{}, + /// `table` may be missing some tasks such as ones that failed, so this + /// field contains references to all of them. + /// Protected by `mutex`. + all_fetches: std.ArrayListUnmanaged(*Fetch) = .{}, + + http_client: *std.http.Client, + thread_pool: *ThreadPool, + wait_group: WaitGroup = .{}, + global_cache: Cache.Directory, + recursive: bool, + work_around_btrfs_bug: bool, + + pub const Table = std.AutoHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch); + + pub fn deinit(jq: *JobQueue) void { + if (jq.all_fetches.items.len == 0) return; + const gpa = jq.all_fetches.items[0].arena.child_allocator; + jq.table.deinit(gpa); + // These must be deinitialized in reverse order because subsequent + // `Fetch` instances are allocated in prior ones' arenas. + // Sorry, I know it's a bit weird, but it slightly simplifies the + // critical section. + while (jq.all_fetches.popOrNull()) |f| f.deinit(); + jq.all_fetches.deinit(gpa); + jq.* = undefined; + } }; pub const Location = union(enum) { remote: Remote, + /// A directory found inside the parent package. relative_path: []const u8, + /// Recursive Fetch tasks will never use this Location, but it may be + /// passed in by the CLI. Indicates the file contents here should be copied + /// into the global package cache. It may be a file relative to the cwd or + /// absolute, in which case it should be treated exactly like a `file://` + /// URL, or a directory, in which case it should be treated as an + /// already-unpacked directory (but still needs to be copied into the + /// global package cache and have inclusion rules applied). + path_or_url: []const u8, pub const Remote = struct { url: []const u8, /// If this is null it means the user omitted the hash field from a dependency. /// It will be an error but the logic should still fetch and print the discovered hash. - hash: ?[hex_multihash_len]u8, + hash: ?Manifest.MultiHashHexDigest, }; }; @@ -92,7 +117,11 @@ pub const RunError = error{ pub fn run(f: *Fetch) RunError!void { const eb = &f.error_bundle; - const arena = f.arena_allocator.allocator(); + const arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const cache_root = f.job_queue.global_cache; + + try eb.init(gpa); // Check the global zig package cache to see if the hash already exists. If // so, load, parse, and validate the build.zig.zon file therein, and skip @@ -111,43 +140,66 @@ pub fn run(f: *Fetch) RunError!void { ); f.package_root = try f.parent_package_root.join(arena, sub_path); try loadManifest(f, f.package_root); + if (!f.job_queue.recursive) return; // Package hashes are used as unique identifiers for packages, so // we still need one for relative paths. - const hash = h: { + const digest = h: { var hasher = Manifest.Hash.init(.{}); // This hash is a tuple of: // * whether it relative to the global cache directory or to the root package // * the relative file path from there to the build root of the package - hasher.update(if (f.package_root.root_dir.handle == f.global_cache.handle) + hasher.update(if (f.package_root.root_dir.eql(cache_root)) &package_hash_prefix_cached else &package_hash_prefix_project); hasher.update(f.package_root.sub_path); break :h hasher.finalResult(); }; - return queueJobsForDeps(f, hash); + return queueJobsForDeps(f, Manifest.hexDigest(digest)); }, .remote => |remote| remote, + .path_or_url => |path_or_url| { + if (fs.cwd().openIterableDir(path_or_url, .{})) |dir| { + var resource: Resource = .{ .dir = dir }; + return runResource(f, path_or_url, &resource, null); + } else |dir_err| { + const file_err = if (dir_err == error.NotDir) e: { + if (fs.cwd().openFile(path_or_url, .{})) |file| { + var resource: Resource = .{ .file = file }; + return runResource(f, path_or_url, &resource, null); + } else |err| break :e err; + } else dir_err; + + const uri = std.Uri.parse(path_or_url) catch |uri_err| { + return f.fail(0, try eb.printString( + "'{s}' could not be recognized as a file path ({s}) or an URL ({s})", + .{ path_or_url, @errorName(file_err), @errorName(uri_err) }, + )); + }; + var resource = try f.initResource(uri); + return runResource(f, uri.path, &resource, null); + } + }, }; + const s = fs.path.sep_str; if (remote.hash) |expected_hash| { const pkg_sub_path = "p" ++ s ++ expected_hash; - if (f.global_cache.handle.access(pkg_sub_path, .{})) |_| { + if (cache_root.handle.access(pkg_sub_path, .{})) |_| { f.package_root = .{ - .root_dir = f.global_cache, + .root_dir = cache_root, .sub_path = pkg_sub_path, }; try loadManifest(f, f.package_root); + if (!f.job_queue.recursive) return; return queueJobsForDeps(f, expected_hash); } else |err| switch (err) { error.FileNotFound => {}, else => |e| { try eb.addRootErrorMessage(.{ .msg = try eb.printString("unable to open global package cache directory '{s}': {s}", .{ - try f.global_cache.join(arena, .{pkg_sub_path}), @errorName(e), + try cache_root.join(arena, &.{pkg_sub_path}), @errorName(e), }), - .src_loc = .none, - .notes_len = 0, }); return error.FetchFailed; }, @@ -158,22 +210,50 @@ pub fn run(f: *Fetch) RunError!void { const uri = std.Uri.parse(remote.url) catch |err| return f.fail( f.location_tok, - "invalid URI: {s}", - .{@errorName(err)}, + try eb.printString("invalid URI: {s}", .{@errorName(err)}), ); + var resource = try f.initResource(uri); + return runResource(f, uri.path, &resource, remote.hash); +} + +pub fn deinit(f: *Fetch) void { + f.error_bundle.deinit(); + f.arena.deinit(); +} + +/// Consumes `resource`, even if an error is returned. +fn runResource( + f: *Fetch, + uri_path: []const u8, + resource: *Resource, + remote_hash: ?Manifest.MultiHashHexDigest, +) RunError!void { + defer resource.deinit(); + const arena = f.arena.allocator(); + const eb = &f.error_bundle; + const s = fs.path.sep_str; + const cache_root = f.job_queue.global_cache; const rand_int = std.crypto.random.int(u64); const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); + const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path}); var tmp_directory: Cache.Directory = .{ - .path = try f.global_cache.join(arena, &.{tmp_dir_sub_path}), - .handle = (try f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{})).dir, + .path = tmp_directory_path, + .handle = handle: { + const dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch |err| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to create temporary directory '{s}': {s}", .{ + tmp_directory_path, @errorName(err), + }), + }); + return error.FetchFailed; + }; + break :handle dir.dir; + }, }; defer tmp_directory.handle.close(); - var resource = try f.initResource(uri); - defer resource.deinit(); // releases more than memory - - try f.unpackResource(&resource, uri.path, tmp_directory); + try unpackResource(f, resource, uri_path, tmp_directory); // Load, parse, and validate the unpacked build.zig.zon file. It is allowed // for the file to be missing, in which case this fetched package is @@ -194,15 +274,15 @@ pub fn run(f: *Fetch) RunError!void { // Compute the package hash based on the remaining files in the temporary // directory. - if (builtin.os.tag == .linux and f.work_around_btrfs_bug) { + if (builtin.os.tag == .linux and f.job_queue.work_around_btrfs_bug) { // https://github.com/ziglang/zig/issues/17095 tmp_directory.handle.close(); - const iterable_dir = f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch + const iterable_dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch @panic("btrfs workaround failed"); tmp_directory.handle = iterable_dir.dir; } - f.actual_hash = try computeHash(f, .{ .dir = tmp_directory.handle }, filter); + f.actual_hash = try computeHash(f, tmp_directory, filter); // Rename the temporary directory into the global zig package cache // directory. If the hash already exists, delete the temporary directory @@ -211,40 +291,54 @@ pub fn run(f: *Fetch) RunError!void { // package with the different hash is used in the future. const dest_pkg_sub_path = "p" ++ s ++ Manifest.hexDigest(f.actual_hash); - try renameTmpIntoCache(f.global_cache.handle, tmp_dir_sub_path, dest_pkg_sub_path); + renameTmpIntoCache(cache_root.handle, tmp_dir_sub_path, dest_pkg_sub_path) catch |err| { + const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); + const dest = try cache_root.join(arena, &.{dest_pkg_sub_path}); + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to rename temporary directory '{s}' into package cache directory '{s}': {s}", + .{ src, dest, @errorName(err) }, + ) }); + return error.FetchFailed; + }; // Validate the computed hash against the expected hash. If invalid, this // job is done. const actual_hex = Manifest.hexDigest(f.actual_hash); - if (remote.hash) |declared_hash| { - if (!std.mem.eql(u8, declared_hash, &actual_hex)) { - return f.fail(f.hash_tok, "hash mismatch: manifest declares {s} but the fetched package has {s}", .{ - declared_hash, actual_hex, - }); + if (remote_hash) |declared_hash| { + if (!std.mem.eql(u8, &declared_hash, &actual_hex)) { + return f.fail(f.hash_tok, try eb.printString( + "hash mismatch: manifest declares {s} but the fetched package has {s}", + .{ declared_hash, actual_hex }, + )); } - } else { + } else if (!f.omit_missing_hash_error) { const notes_len = 1; - try f.addErrorWithNotes(notes_len, f.location_tok, "dependency is missing hash field"); + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("dependency is missing hash field"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); const notes_start = try eb.reserveNotes(notes_len); eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), })); - return error.PackageFetchFailed; + return error.FetchFailed; } // Spawn a new fetch job for each dependency in the manifest file. Use // a mutex and a hash map so that redundant jobs do not get queued up. - return queueJobsForDeps(f, .{ .hash = f.actual_hash }); + if (!f.job_queue.recursive) return; + return queueJobsForDeps(f, actual_hex); } /// This function populates `f.manifest` or leaves it `null`. -fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void { +fn loadManifest(f: *Fetch, pkg_root: Package.Path) RunError!void { const eb = &f.error_bundle; - const arena = f.arena_allocator.allocator(); - const manifest_bytes = pkg_root.readFileAllocOptions( + const arena = f.arena.allocator(); + const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions( arena, - Manifest.basename, + try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }), Manifest.max_bytes, null, 1, @@ -252,39 +346,39 @@ fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void { ) catch |err| switch (err) { error.FileNotFound => return, else => |e| { - const file_path = try pkg_root.join(arena, .{Manifest.basename}); + const file_path = try pkg_root.join(arena, Manifest.basename); try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to load package manifest '{s}': {s}", .{ + .msg = try eb.printString("unable to load package manifest '{}': {s}", .{ file_path, @errorName(e), }), - .src_loc = .none, - .notes_len = 0, }); + return error.FetchFailed; }, }; - var ast = try std.zig.Ast.parse(arena, manifest_bytes, .zon); - f.manifest_ast = ast; + const ast = &f.manifest_ast; + ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon); if (ast.errors.len > 0) { - const file_path = try pkg_root.join(arena, .{Manifest.basename}); - try main.putAstErrorsIntoBundle(arena, ast, file_path, eb); - return error.PackageFetchFailed; + const file_path = try std.fmt.allocPrint(arena, "{}" ++ Manifest.basename, .{pkg_root}); + try main.putAstErrorsIntoBundle(arena, ast.*, file_path, eb); + return error.FetchFailed; } - f.manifest = try Manifest.parse(arena, ast); + f.manifest = try Manifest.parse(arena, ast.*); + const manifest = &f.manifest.?; - if (f.manifest.errors.len > 0) { - const file_path = try pkg_root.join(arena, .{Manifest.basename}); + if (manifest.errors.len > 0) { + const src_path = try eb.printString("{}{s}", .{ pkg_root, Manifest.basename }); const token_starts = ast.tokens.items(.start); - for (f.manifest.errors) |msg| { + for (manifest.errors) |msg| { const start_loc = ast.tokenLocation(0, msg.tok); try eb.addRootErrorMessage(.{ .msg = try eb.addString(msg.msg), .src_loc = try eb.addSourceLocation(.{ - .src_path = try eb.addString(file_path), + .src_path = src_path, .span_start = token_starts[msg.tok], .span_end = @intCast(token_starts[msg.tok] + ast.tokenSlice(msg.tok).len), .span_main = token_starts[msg.tok] + msg.off, @@ -292,71 +386,80 @@ fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void { .column = @intCast(start_loc.column), .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), }), - .notes_len = 0, }); } - return error.PackageFetchFailed; + return error.FetchFailed; } } -fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void { +fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void { + assert(f.job_queue.recursive); + // If the package does not have a build.zig.zon file then there are no dependencies. const manifest = f.manifest orelse return; const new_fetches = nf: { + const deps = manifest.dependencies.values(); + const gpa = f.arena.child_allocator; // Grab the new tasks into a temporary buffer so we can unlock that mutex // as fast as possible. // This overallocates any fetches that get skipped by the `continue` in the // loop below. - const new_fetches = try f.arena.alloc(Fetch, manifest.dependencies.count()); + const new_fetches = try f.arena.allocator().alloc(Fetch, deps.len); var new_fetch_index: usize = 0; - f.job_queue.lock(); - defer f.job_queue.unlock(); + f.job_queue.mutex.lock(); + defer f.job_queue.mutex.unlock(); + + try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); + try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len + 1)); // It is impossible for there to be a collision here. Consider all three cases: // * Correct hash is provided by manifest. // - Redundant jobs are skipped in the loop below. - // * Incorrect has is provided by manifest. + // * Incorrect hash is provided by manifest. // - Hash mismatch error emitted; `queueJobsForDeps` is not called. // * Hash is not provided by manifest. // - Hash missing error emitted; `queueJobsForDeps` is not called. - try f.job_queue.finish(hash, f, new_fetches.len); + f.job_queue.table.putAssumeCapacityNoClobber(hash, f); - for (manifest.dependencies.values()) |dep| { + for (deps) |dep| { + const new_fetch = &new_fetches[new_fetch_index]; const location: Location = switch (dep.location) { .url => |url| .{ .remote = .{ .url = url, - .hash = if (dep.hash) |h| h[0..hex_multihash_len].* else null, + .hash = h: { + const h = dep.hash orelse break :h null; + const digest_len = @typeInfo(Manifest.MultiHashHexDigest).Array.len; + const multihash_digest = h[0..digest_len].*; + const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest); + if (gop.found_existing) continue; + gop.value_ptr.* = new_fetch; + break :h multihash_digest; + }, } }, .path => |path| .{ .relative_path = path }, }; - const new_fetch = &new_fetches[new_fetch_index]; - const already_done = f.job_queue.add(location, new_fetch); - if (already_done) continue; new_fetch_index += 1; - + f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); new_fetch.* = .{ - .gpa = f.gpa, - .arena = std.heap.ArenaAllocator.init(f.gpa), + .arena = std.heap.ArenaAllocator.init(gpa), .location = location, .location_tok = dep.location_tok, .hash_tok = dep.hash_tok, - .global_cache = f.global_cache, .parent_package_root = f.package_root, - .parent_manifest_ast = f.manifest_ast.?, + .parent_manifest_ast = &f.manifest_ast, .prog_node = f.prog_node, - .http_client = f.http_client, - .thread_pool = f.thread_pool, .job_queue = f.job_queue, - .wait_group = f.wait_group, + .omit_missing_hash_error = false, .package_root = undefined, - .error_bundle = .{}, + .error_bundle = undefined, .manifest = null, - .manifest_ast = null, + .manifest_ast = undefined, .actual_hash = undefined, .has_build_zig = false, + .oom_flag = false, }; } @@ -364,12 +467,14 @@ fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void { }; // Now it's time to give tasks to the thread pool. - for (new_fetches) |new_fetch| { - f.wait_group.start(); - f.thread_pool.spawn(workerRun, .{f}) catch |err| switch (err) { + const thread_pool = f.job_queue.thread_pool; + + for (new_fetches) |*new_fetch| { + f.job_queue.wait_group.start(); + thread_pool.spawn(workerRun, .{new_fetch}) catch |err| switch (err) { error.OutOfMemory => { new_fetch.oom_flag = true; - f.wait_group.finish(); + f.job_queue.wait_group.finish(); continue; }, }; @@ -377,43 +482,83 @@ fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void { } fn workerRun(f: *Fetch) void { - defer f.wait_group.finish(); + defer f.job_queue.wait_group.finish(); run(f) catch |err| switch (err) { error.OutOfMemory => f.oom_flag = true, - error.FetchFailed => {}, // See `error_bundle`. + error.FetchFailed => { + // Nothing to do because the errors are already reported in `error_bundle`, + // and a reference is kept to the `Fetch` task inside `all_fetches`. + }, }; } -fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError!void { - const ast = f.parent_manifest_ast; - const token_starts = ast.tokens.items(.start); - const start_loc = ast.tokenLocation(0, msg_tok); +fn srcLoc( + f: *Fetch, + tok: std.zig.Ast.TokenIndex, +) Allocator.Error!std.zig.ErrorBundle.SourceLocationIndex { + const ast = f.parent_manifest_ast orelse return .none; const eb = &f.error_bundle; - const file_path = try f.parent_package_root.join(f.arena, Manifest.basename); + const token_starts = ast.tokens.items(.start); + const start_loc = ast.tokenLocation(0, tok); + const src_path = try eb.printString("{}" ++ Manifest.basename, .{f.parent_package_root}); const msg_off = 0; + return eb.addSourceLocation(.{ + .src_path = src_path, + .span_start = token_starts[tok], + .span_end = @intCast(token_starts[tok] + ast.tokenSlice(tok).len), + .span_main = token_starts[tok] + msg_off, + .line = @intCast(start_loc.line), + .column = @intCast(start_loc.column), + .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), + }); +} +fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError { + const eb = &f.error_bundle; try eb.addRootErrorMessage(.{ .msg = msg_str, - .src_loc = try eb.addSourceLocation(.{ - .src_path = try eb.addString(file_path), - .span_start = token_starts[msg_tok], - .span_end = @intCast(token_starts[msg_tok] + ast.tokenSlice(msg_tok).len), - .span_main = token_starts[msg_tok] + msg_off, - .line = @intCast(start_loc.line), - .column = @intCast(start_loc.column), - .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), - }), - .notes_len = 0, + .src_loc = try f.srcLoc(msg_tok), }); - return error.FetchFailed; } const Resource = union(enum) { file: fs.File, http_request: std.http.Client.Request, - git_fetch_stream: git.Session.FetchStream, + git: Git, dir: fs.IterableDir, + + const Git = struct { + fetch_stream: git.Session.FetchStream, + want_oid: [git.oid_length]u8, + }; + + fn deinit(resource: *Resource) void { + switch (resource.*) { + .file => |*file| file.close(), + .http_request => |*req| req.deinit(), + .git => |*git_resource| git_resource.fetch_stream.deinit(), + .dir => |*dir| dir.close(), + } + resource.* = undefined; + } + + fn reader(resource: *Resource) std.io.AnyReader { + return .{ + .context = resource, + .readFn = read, + }; + } + + fn read(context: *const anyopaque, buffer: []u8) anyerror!usize { + const resource: *Resource = @constCast(@ptrCast(@alignCast(context))); + switch (resource.*) { + .file => |*f| return f.read(buffer), + .http_request => |*r| return r.read(buffer), + .git => |*g| return g.fetch_stream.read(buffer), + .dir => unreachable, + } + } }; const FileType = enum { @@ -468,30 +613,52 @@ const FileType = enum { }; fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource { - const gpa = f.gpa; - const arena = f.arena_allocator.allocator(); + const gpa = f.arena.child_allocator; + const arena = f.arena.allocator(); const eb = &f.error_bundle; if (ascii.eqlIgnoreCase(uri.scheme, "file")) return .{ - .file = try f.parent_package_root.openFile(uri.path, .{}), + .file = f.parent_package_root.openFile(uri.path, .{}) catch |err| { + return f.fail(f.location_tok, try eb.printString("unable to open '{}{s}': {s}", .{ + f.parent_package_root, uri.path, @errorName(err), + })); + }, }; + const http_client = f.job_queue.http_client; + if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) { var h = std.http.Headers{ .allocator = gpa }; defer h.deinit(); - var req = try f.http_client.request(.GET, uri, h, .{}); + var req = http_client.request(.GET, uri, h, .{}) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to connect to server: {s}", + .{@errorName(err)}, + )); + }; errdefer req.deinit(); // releases more than memory - try req.start(.{}); - try req.wait(); + req.start(.{}) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "HTTP request failed: {s}", + .{@errorName(err)}, + )); + }; + req.wait() catch |err| { + return f.fail(f.location_tok, try eb.printString( + "invalid HTTP response: {s}", + .{@errorName(err)}, + )); + }; if (req.response.status != .ok) { - return f.fail(f.location_tok, "expected response status '200 OK' got '{s} {s}'", .{ - @intFromEnum(req.response.status), req.response.status.phrase() orelse "", - }); + return f.fail(f.location_tok, try eb.printString( + "bad HTTP response code: '{d} {s}'", + .{ @intFromEnum(req.response.status), req.response.status.phrase() orelse "" }, + )); } return .{ .http_request = req }; @@ -503,13 +670,21 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource { var transport_uri = uri; transport_uri.scheme = uri.scheme["git+".len..]; var redirect_uri: []u8 = undefined; - var session: git.Session = .{ .transport = f.http_client, .uri = transport_uri }; - session.discoverCapabilities(gpa, &redirect_uri) catch |e| switch (e) { + var session: git.Session = .{ .transport = http_client, .uri = transport_uri }; + session.discoverCapabilities(gpa, &redirect_uri) catch |err| switch (err) { error.Redirected => { defer gpa.free(redirect_uri); - return f.fail(f.location_tok, "repository moved to {s}", .{redirect_uri}); + return f.fail(f.location_tok, try eb.printString( + "repository moved to {s}", + .{redirect_uri}, + )); + }, + else => |e| { + return f.fail(f.location_tok, try eb.printString( + "unable to discover remote git server capabilities: {s}", + .{@errorName(e)}, + )); }, - else => |other| return other, }; const want_oid = want_oid: { @@ -519,12 +694,22 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource { const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref}); const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref}); - var ref_iterator = try session.listRefs(gpa, .{ + var ref_iterator = session.listRefs(gpa, .{ .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, .include_peeled = true, - }); + }) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to list refs: {s}", + .{@errorName(err)}, + )); + }; defer ref_iterator.deinit(); - while (try ref_iterator.next()) |ref| { + while (ref_iterator.next() catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to iterate refs: {s}", + .{@errorName(err)}, + )); + }) |ref| { if (std.mem.eql(u8, ref.name, want_ref) or std.mem.eql(u8, ref.name, want_ref_head) or std.mem.eql(u8, ref.name, want_ref_tag)) @@ -532,31 +717,46 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource { break :want_oid ref.peeled orelse ref.oid; } } - return f.fail(f.location_tok, "ref not found: {s}", .{want_ref}); + return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref})); }; if (uri.fragment == null) { const notes_len = 1; - try f.addErrorWithNotes(notes_len, f.location_tok, "url field is missing an explicit ref"); + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("url field is missing an explicit ref"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); const notes_start = try eb.reserveNotes(notes_len); eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ .msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid), }), })); - return error.PackageFetchFailed; + return error.FetchFailed; } var want_oid_buf: [git.fmt_oid_length]u8 = undefined; _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{ std.fmt.fmtSliceHexLower(&want_oid), }) catch unreachable; - var fetch_stream = try session.fetch(gpa, &.{&want_oid_buf}); + var fetch_stream = session.fetch(gpa, &.{&want_oid_buf}) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to create fetch stream: {s}", + .{@errorName(err)}, + )); + }; errdefer fetch_stream.deinit(); - return .{ .git_fetch_stream = fetch_stream }; + return .{ .git = .{ + .fetch_stream = fetch_stream, + .want_oid = want_oid, + } }; } - return f.fail(f.location_tok, "unsupported URL scheme: {s}", .{uri.scheme}); + return f.fail(f.location_tok, try eb.printString( + "unsupported URL scheme: {s}", + .{uri.scheme}, + )); } fn unpackResource( @@ -565,52 +765,62 @@ fn unpackResource( uri_path: []const u8, tmp_directory: Cache.Directory, ) RunError!void { + const eb = &f.error_bundle; const file_type = switch (resource.*) { .file => FileType.fromPath(uri_path) orelse - return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path}), + return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})), .http_request => |req| ft: { // Content-Type takes first precedence. const content_type = req.response.headers.getFirstValue("Content-Type") orelse - return f.fail(f.location_tok, "missing 'Content-Type' header", .{}); + return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header")); if (ascii.eqlIgnoreCase(content_type, "application/x-tar")) - return .tar; + break :ft .tar; if (ascii.eqlIgnoreCase(content_type, "application/gzip") or ascii.eqlIgnoreCase(content_type, "application/x-gzip") or ascii.eqlIgnoreCase(content_type, "application/tar+gzip")) { - return .@"tar.gz"; + break :ft .@"tar.gz"; } if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) - return .@"tar.xz"; + break :ft .@"tar.xz"; if (!ascii.eqlIgnoreCase(content_type, "application/octet-stream")) { - return f.fail(f.location_tok, "unrecognized 'Content-Type' header: '{s}'", .{ - content_type, - }); + return f.fail(f.location_tok, try eb.printString( + "unrecognized 'Content-Type' header: '{s}'", + .{content_type}, + )); } // Next, the filename from 'content-disposition: attachment' takes precedence. if (req.response.headers.getFirstValue("Content-Disposition")) |cd_header| { - break :ft FileType.fromContentDisposition(cd_header) orelse - return f.fail( - f.location_tok, - "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", - .{cd_header}, - ); + break :ft FileType.fromContentDisposition(cd_header) orelse { + return f.fail(f.location_tok, try eb.printString( + "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", + .{cd_header}, + )); + }; } // Finally, the path from the URI is used. - break :ft FileType.fromPath(uri_path) orelse - return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path}); + break :ft FileType.fromPath(uri_path) orelse { + return f.fail(f.location_tok, try eb.printString( + "unknown file type: '{s}'", + .{uri_path}, + )); + }; }, - .git_fetch_stream => return .git_pack, - .dir => |dir| { - try f.recursiveDirectoryCopy(dir, tmp_directory.handle); - return; + + .git => .git_pack, + + .dir => |dir| return f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to copy directory '{s}': {s}", + .{ uri_path, @errorName(err) }, + )); }, }; @@ -618,7 +828,14 @@ fn unpackResource( .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()), .@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip), .@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz), - .git_pack => try unpackGitPack(f, tmp_directory.handle, resource), + .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { + error.FetchFailed => return error.FetchFailed, + error.OutOfMemory => return error.OutOfMemory, + else => |e| return f.fail(f.location_tok, try eb.printString( + "unable to unpack git files: {s}", + .{@errorName(e)}, + )), + }, } } @@ -628,11 +845,17 @@ fn unpackTarballCompressed( resource: *Resource, comptime Compression: type, ) RunError!void { - const gpa = f.gpa; + const gpa = f.arena.child_allocator; + const eb = &f.error_bundle; const reader = resource.reader(); var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); - var decompress = try Compression.decompress(gpa, br.reader()); + var decompress = Compression.decompress(gpa, br.reader()) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to decompress tarball: {s}", + .{@errorName(err)}, + )); + }; defer decompress.deinit(); return unpackTarball(f, out_dir, decompress.reader()); @@ -640,11 +863,12 @@ fn unpackTarballCompressed( fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { const eb = &f.error_bundle; + const gpa = f.arena.child_allocator; - var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = f.gpa }; + var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = gpa }; defer diagnostics.deinit(); - try std.tar.pipeToFileSystem(out_dir, reader, .{ + std.tar.pipeToFileSystem(out_dir, reader, .{ .diagnostics = &diagnostics, .strip_components = 1, // TODO: we would like to set this to executable_bit_only, but two @@ -653,12 +877,19 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { // 2. the hashing algorithm here needs to support detecting the is_executable // bit on Windows from the ACLs (see the isExecutable function). .mode_mode = .ignore, - .filter = .{ .exclude_empty_directories = true }, - }); + .exclude_empty_directories = true, + }) catch |err| return f.fail(f.location_tok, try eb.printString( + "unable to unpack tarball to temporary directory: {s}", + .{@errorName(err)}, + )); if (diagnostics.errors.items.len > 0) { const notes_len: u32 = @intCast(diagnostics.errors.items.len); - try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack tarball"); + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("unable to unpack tarball"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); const notes_start = try eb.reserveNotes(notes_len); for (diagnostics.errors.items, notes_start..) |item, note_i| { switch (item) { @@ -678,19 +909,15 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { }, } } - return error.InvalidTarball; + return error.FetchFailed; } } -fn unpackGitPack( - f: *Fetch, - out_dir: fs.Dir, - resource: *Resource, - want_oid: git.Oid, -) !void { +fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void { const eb = &f.error_bundle; - const gpa = f.gpa; - const reader = resource.reader(); + const gpa = f.arena.child_allocator; + const want_oid = resource.git.want_oid; + const reader = resource.git.fetch_stream.reader(); // The .git directory is used to store the packfile and associated index, but // we do not attempt to replicate the exact structure of a real .git // directory, since that isn't relevant for fetching a package. @@ -700,13 +927,13 @@ fn unpackGitPack( var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true }); defer pack_file.close(); var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(reader.reader(), pack_file.writer()); + try fifo.pump(reader, pack_file.writer()); try pack_file.sync(); var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); defer index_file.close(); { - var index_prog_node = reader.prog_node.start("Index pack", 0); + var index_prog_node = f.prog_node.start("Index pack", 0); defer index_prog_node.end(); index_prog_node.activate(); var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); @@ -716,7 +943,7 @@ fn unpackGitPack( } { - var checkout_prog_node = reader.prog_node.start("Checkout", 0); + var checkout_prog_node = f.prog_node.start("Checkout", 0); defer checkout_prog_node.end(); checkout_prog_node.activate(); var repository = try git.Repository.init(gpa, pack_file, index_file); @@ -727,7 +954,11 @@ fn unpackGitPack( if (diagnostics.errors.items.len > 0) { const notes_len: u32 = @intCast(diagnostics.errors.items.len); - try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack packfile"); + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("unable to unpack packfile"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); const notes_start = try eb.reserveNotes(notes_len); for (diagnostics.errors.items, notes_start..) |item, note_i| { switch (item) { @@ -748,9 +979,10 @@ fn unpackGitPack( try out_dir.deleteTree(".git"); } -fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) RunError!void { +fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) anyerror!void { + const gpa = f.arena.child_allocator; // Recursive directory copy. - var it = try dir.walk(f.gpa); + var it = try dir.walk(gpa); defer it.deinit(); while (try it.next()) |entry| { switch (entry.kind) { @@ -816,16 +1048,22 @@ pub fn renameTmpIntoCache( /// the hash are not present on the file system. Empty directories are *not /// hashed* and must not be present on the file system when calling this /// function. -fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Digest { +fn computeHash( + f: *Fetch, + tmp_directory: Cache.Directory, + filter: Filter, +) RunError!Manifest.Digest { // All the path name strings need to be in memory for sorting. - const arena = f.arena_allocator.allocator(); - const gpa = f.gpa; + const arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const eb = &f.error_bundle; + const thread_pool = f.job_queue.thread_pool; // Collect all files, recursively, then sort. var all_files = std.ArrayList(*HashedFile).init(gpa); defer all_files.deinit(); - var walker = try pkg_dir.walk(gpa); + var walker = try @as(fs.IterableDir, .{ .dir = tmp_directory.handle }).walk(gpa); defer walker.deinit(); { @@ -834,19 +1072,28 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige var wait_group: WaitGroup = .{}; // `computeHash` is called from a worker thread so there must not be // any waiting without working or a deadlock could occur. - defer wait_group.waitAndWork(); - - while (try walker.next()) |entry| { + defer thread_pool.waitAndWork(&wait_group); + + while (walker.next() catch |err| { + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to walk temporary directory '{}': {s}", + .{ tmp_directory, @errorName(err) }, + ) }); + return error.FetchFailed; + }) |entry| { _ = filter; // TODO: apply filter rules here const kind: HashedFile.Kind = switch (entry.kind) { .directory => continue, .file => .file, .sym_link => .sym_link, - else => return error.IllegalFileTypeInPackage, + else => return f.fail(f.location_tok, try eb.printString( + "package contains '{s}' which has illegal file type '{s}'", + .{ entry.path, @tagName(entry.kind) }, + )), }; - if (std.mem.eql(u8, entry.path, build_zig_basename)) + if (std.mem.eql(u8, entry.path, Package.build_zig_basename)) f.has_build_zig = true; const hashed_file = try arena.create(HashedFile); @@ -859,7 +1106,9 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige .failure = undefined, // to be populated by the worker }; wait_group.start(); - try f.thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); + try thread_pool.spawn(workerHashFile, .{ + tmp_directory.handle, hashed_file, &wait_group, + }); try all_files.append(hashed_file); } @@ -869,19 +1118,13 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige var hasher = Manifest.Hash.init(.{}); var any_failures = false; - const eb = &f.error_bundle; for (all_files.items) |hashed_file| { hashed_file.failure catch |err| { any_failures = true; try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to hash: {s}", .{@errorName(err)}), - .src_loc = try eb.addSourceLocation(.{ - .src_path = try eb.addString(hashed_file.fs_path), - .span_start = 0, - .span_end = 0, - .span_main = 0, + .msg = try eb.printString("unable to hash '{s}': {s}", .{ + hashed_file.fs_path, @errorName(err), }), - .notes_len = 0, }); }; hasher.update(&hashed_file.hash); @@ -934,7 +1177,7 @@ fn isExecutable(file: fs.File) !bool { const HashedFile = struct { fs_path: []const u8, normalized_path: []const u8, - hash: Digest, + hash: Manifest.Digest, failure: Error!void, kind: Kind, @@ -970,7 +1213,7 @@ fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 { return normalized; } -pub const Filter = struct { +const Filter = struct { include_paths: std.StringArrayHashMapUnmanaged(void) = .{}, /// sub_path is relative to the tarball root. @@ -990,12 +1233,9 @@ pub const Filter = struct { } }; -const build_zig_basename = @import("../Package.zig").build_zig_basename; -const hex_multihash_len = 2 * Manifest.multihash_len; - // These are random bytes. -const package_hash_prefix_cached: [8]u8 = &.{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 }; -const package_hash_prefix_project: [8]u8 = &.{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc }; +const package_hash_prefix_cached = [8]u8{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 }; +const package_hash_prefix_project = [8]u8{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc }; const builtin = @import("builtin"); const std = @import("std"); @@ -1010,3 +1250,4 @@ const Manifest = @import("../Manifest.zig"); const Fetch = @This(); const main = @import("../main.zig"); const git = @import("../git.zig"); +const Package = @import("../Package.zig"); diff --git a/src/Package/Module.zig b/src/Package/Module.zig new file mode 100644 index 000000000000..f2accc104d49 --- /dev/null +++ b/src/Package/Module.zig @@ -0,0 +1,32 @@ +//! Corresponds to something that Zig source code can `@import`. +//! Not to be confused with src/Module.zig which should be renamed +//! to something else. https://github.com/ziglang/zig/issues/14307 + +/// Only files inside this directory can be imported. +root: Package.Path, +/// Relative to `root`. May contain path separators. +root_src_path: []const u8, +/// The dependency table of this module. Shared dependencies such as 'std', +/// 'builtin', and 'root' are not specified in every dependency table, but +/// instead only in the table of `main_pkg`. `Module.importFile` is +/// responsible for detecting these names and using the correct package. +deps: Deps = .{}, + +pub const Deps = std.StringHashMapUnmanaged(*Module); + +pub const Tree = struct { + /// Each `Package` exposes a `Module` with build.zig as its root source file. + build_module_table: std.AutoArrayHashMapUnmanaged(MultiHashHexDigest, *Module), +}; + +pub fn create(allocator: Allocator, m: Module) Allocator.Error!*Module { + const new = try allocator.create(Module); + new.* = m; + return new; +} + +const Module = @This(); +const Package = @import("../Package.zig"); +const std = @import("std"); +const Allocator = std.mem.Allocator; +const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest; diff --git a/src/crash_report.zig b/src/crash_report.zig index d4e4b46a5350..2b33bd7fa51c 100644 --- a/src/crash_report.zig +++ b/src/crash_report.zig @@ -139,18 +139,22 @@ fn dumpStatusReport() !void { var crash_heap: [16 * 4096]u8 = undefined; -fn writeFilePath(file: *Module.File, stream: anytype) !void { - if (file.pkg.root_src_directory.path) |path| { - try stream.writeAll(path); - try stream.writeAll(std.fs.path.sep_str); +fn writeFilePath(file: *Module.File, writer: anytype) !void { + if (file.mod.root.root_dir.path) |path| { + try writer.writeAll(path); + try writer.writeAll(std.fs.path.sep_str); } - try stream.writeAll(file.sub_file_path); + if (file.mod.root.sub_path.len > 0) { + try writer.writeAll(file.mod.root.sub_path); + try writer.writeAll(std.fs.path.sep_str); + } + try writer.writeAll(file.sub_file_path); } -fn writeFullyQualifiedDeclWithFile(mod: *Module, decl: *Decl, stream: anytype) !void { - try writeFilePath(decl.getFileScope(mod), stream); - try stream.writeAll(": "); - try decl.renderFullyQualifiedDebugName(mod, stream); +fn writeFullyQualifiedDeclWithFile(mod: *Module, decl: *Decl, writer: anytype) !void { + try writeFilePath(decl.getFileScope(mod), writer); + try writer.writeAll(": "); + try decl.renderFullyQualifiedDebugName(mod, writer); } pub fn compilerPanic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, maybe_ret_addr: ?usize) noreturn { diff --git a/src/main.zig b/src/main.zig index 34ab5ea191f3..f52a4bf24af0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -416,7 +416,7 @@ const usage_build_generic = \\ dep: [[import=]name] \\ --deps [dep],[dep],... Set dependency names for the root package \\ dep: [[import=]name] - \\ --main-pkg-path Set the directory of the root package + \\ --main-mod-path Set the directory of the root module \\ -fPIC Force-enable Position Independent Code \\ -fno-PIC Force-disable Position Independent Code \\ -fPIE Force-enable Position Independent Executable @@ -765,17 +765,11 @@ const Framework = struct { }; const CliModule = struct { - mod: *Package, + mod: *Package.Module, /// still in CLI arg format deps_str: []const u8, }; -fn cleanupModules(modules: *std.StringArrayHashMap(CliModule)) void { - var it = modules.iterator(); - while (it.next()) |kv| kv.value_ptr.mod.destroy(modules.allocator); - modules.deinit(); -} - fn buildOutputType( gpa: Allocator, arena: Allocator, @@ -950,8 +944,7 @@ fn buildOutputType( // Contains every module specified via --mod. The dependencies are added // after argument parsing is completed. We use a StringArrayHashMap to make // error output consistent. - var modules = std.StringArrayHashMap(CliModule).init(gpa); - defer cleanupModules(&modules); + var modules = std.StringArrayHashMap(CliModule).init(arena); // The dependency string for the root package var root_deps_str: ?[]const u8 = null; @@ -1023,32 +1016,37 @@ fn buildOutputType( for ([_][]const u8{ "std", "root", "builtin" }) |name| { if (mem.eql(u8, mod_name, name)) { - fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{ mod_name, root_src }); + fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{ + mod_name, root_src, + }); } } var mod_it = modules.iterator(); while (mod_it.next()) |kv| { if (std.mem.eql(u8, mod_name, kv.key_ptr.*)) { - fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{ mod_name, root_src, kv.value_ptr.mod.root_src_path }); + fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{ + mod_name, root_src, kv.value_ptr.mod.root_src_path, + }); } } - try modules.ensureUnusedCapacity(1); - modules.put(mod_name, .{ - .mod = try Package.create( - gpa, - fs.path.dirname(root_src), - fs.path.basename(root_src), - ), + try modules.put(mod_name, .{ + .mod = try Package.Module.create(arena, .{ + .root = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = fs.path.dirname(root_src) orelse "", + }, + .root_src_path = fs.path.basename(root_src), + }), .deps_str = deps_str, - }) catch unreachable; + }); } else if (mem.eql(u8, arg, "--deps")) { if (root_deps_str != null) { fatal("only one --deps argument is allowed", .{}); } root_deps_str = args_iter.nextOrFatal(); - } else if (mem.eql(u8, arg, "--main-pkg-path")) { + } else if (mem.eql(u8, arg, "--main-mod-path")) { main_pkg_path = args_iter.nextOrFatal(); } else if (mem.eql(u8, arg, "-cflags")) { extra_cflags.shrinkRetainingCapacity(0); @@ -2461,19 +2459,26 @@ fn buildOutputType( var deps_it = ModuleDepIterator.init(deps_str); while (deps_it.next()) |dep| { if (dep.expose.len == 0) { - fatal("module '{s}' depends on '{s}' with a blank name", .{ kv.key_ptr.*, dep.name }); + fatal("module '{s}' depends on '{s}' with a blank name", .{ + kv.key_ptr.*, dep.name, + }); } for ([_][]const u8{ "std", "root", "builtin" }) |name| { if (mem.eql(u8, dep.expose, name)) { - fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{ dep.name, dep.expose }); + fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{ + dep.name, dep.expose, + }); } } - const dep_mod = modules.get(dep.name) orelse - fatal("module '{s}' depends on module '{s}' which does not exist", .{ kv.key_ptr.*, dep.name }); + const dep_mod = modules.get(dep.name) orelse { + fatal("module '{s}' depends on module '{s}' which does not exist", .{ + kv.key_ptr.*, dep.name, + }); + }; - try kv.value_ptr.mod.add(gpa, dep.expose, dep_mod.mod); + try kv.value_ptr.mod.deps.put(arena, dep.expose, dep_mod.mod); } } } @@ -3229,31 +3234,33 @@ fn buildOutputType( }; defer emit_implib_resolved.deinit(); - const main_pkg: ?*Package = if (root_src_file) |unresolved_src_path| blk: { + const main_mod: ?*Package.Module = if (root_src_file) |unresolved_src_path| blk: { const src_path = try introspect.resolvePath(arena, unresolved_src_path); if (main_pkg_path) |unresolved_main_pkg_path| { const p = try introspect.resolvePath(arena, unresolved_main_pkg_path); - if (p.len == 0) { - break :blk try Package.create(gpa, null, src_path); - } else { - const rel_src_path = try fs.path.relative(arena, p, src_path); - break :blk try Package.create(gpa, p, rel_src_path); - } + break :blk try Package.Module.create(arena, .{ + .root = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = p, + }, + .root_src_path = if (p.len == 0) + src_path + else + try fs.path.relative(arena, p, src_path), + }); } else { - const root_src_dir_path = fs.path.dirname(src_path); - break :blk Package.create(gpa, root_src_dir_path, fs.path.basename(src_path)) catch |err| { - if (root_src_dir_path) |p| { - fatal("unable to open '{s}': {s}", .{ p, @errorName(err) }); - } else { - return err; - } - }; + break :blk try Package.Module.create(arena, .{ + .root = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = fs.path.dirname(src_path) orelse "", + }, + .root_src_path = fs.path.basename(src_path), + }); } } else null; - defer if (main_pkg) |p| p.destroy(gpa); // Transfer packages added with --deps to the root package - if (main_pkg) |mod| { + if (main_mod) |mod| { var it = ModuleDepIterator.init(root_deps_str orelse ""); while (it.next()) |dep| { if (dep.expose.len == 0) { @@ -3269,7 +3276,7 @@ fn buildOutputType( const dep_mod = modules.get(dep.name) orelse fatal("root module depends on module '{s}' which does not exist", .{dep.name}); - try mod.add(gpa, dep.expose, dep_mod.mod); + try mod.deps.put(arena, dep.expose, dep_mod.mod); } } @@ -3310,17 +3317,18 @@ fn buildOutputType( if (arg_mode == .run) { break :l global_cache_directory; } - if (main_pkg) |pkg| { + if (main_mod != null) { // search upwards from cwd until we find directory with build.zig const cwd_path = try process.getCwdAlloc(arena); - const build_zig = "build.zig"; const zig_cache = "zig-cache"; var dirname: []const u8 = cwd_path; while (true) { - const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig }); + const joined_path = try fs.path.join(arena, &.{ + dirname, Package.build_zig_basename, + }); if (fs.cwd().access(joined_path, .{})) |_| { - const cache_dir_path = try fs.path.join(arena, &[_][]const u8{ dirname, zig_cache }); - const dir = try pkg.root_src_directory.handle.makeOpenPath(cache_dir_path, .{}); + const cache_dir_path = try fs.path.join(arena, &.{ dirname, zig_cache }); + const dir = try fs.cwd().makeOpenPath(cache_dir_path, .{}); cleanup_local_cache_dir = dir; break :l .{ .handle = dir, .path = cache_dir_path }; } else |err| switch (err) { @@ -3378,6 +3386,8 @@ fn buildOutputType( gimmeMoreOfThoseSweetSweetFileDescriptors(); + if (true) @panic("TODO restore Compilation logic"); + const comp = Compilation.create(gpa, .{ .zig_lib_directory = zig_lib_directory, .local_cache_directory = local_cache_directory, @@ -3389,7 +3399,7 @@ fn buildOutputType( .dynamic_linker = target_info.dynamic_linker.get(), .sysroot = sysroot, .output_mode = output_mode, - .main_pkg = main_pkg, + .main_mod = main_mod, .emit_bin = emit_bin_loc, .emit_h = emit_h_resolved.data, .emit_asm = emit_asm_resolved.data, @@ -4799,32 +4809,22 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi try thread_pool.init(.{ .allocator = gpa }); defer thread_pool.deinit(); - var cleanup_build_runner_dir: ?fs.Dir = null; - defer if (cleanup_build_runner_dir) |*dir| dir.close(); - - var main_pkg: Package = if (override_build_runner) |build_runner_path| + var main_mod: Package.Module = if (override_build_runner) |build_runner_path| .{ - .root_src_directory = blk: { - if (std.fs.path.dirname(build_runner_path)) |dirname| { - const dir = fs.cwd().openDir(dirname, .{}) catch |err| { - fatal("unable to open directory to build runner from argument 'build-runner', '{s}': {s}", .{ dirname, @errorName(err) }); - }; - cleanup_build_runner_dir = dir; - break :blk .{ .path = dirname, .handle = dir }; - } - - break :blk .{ .path = null, .handle = fs.cwd() }; + .root = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = fs.path.dirname(build_runner_path) orelse "", }, - .root_src_path = std.fs.path.basename(build_runner_path), + .root_src_path = fs.path.basename(build_runner_path), } else .{ - .root_src_directory = zig_lib_directory, + .root = .{ .root_dir = zig_lib_directory }, .root_src_path = "build_runner.zig", }; - var build_pkg: Package = .{ - .root_src_directory = build_directory, + var build_mod: Package.Module = .{ + .root = .{ .root_dir = build_directory }, .root_src_path = build_zig_basename, }; if (build_options.only_core_functionality) { @@ -4833,11 +4833,13 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; \\ ); - try main_pkg.add(gpa, "@dependencies", deps_pkg); + try main_mod.deps.put(arena, "@dependencies", deps_pkg); } else { var http_client: std.http.Client = .{ .allocator = gpa }; defer http_client.deinit(); + if (true) @panic("TODO restore package fetching logic"); + // Here we provide an import to the build runner that allows using reflection to find // all of the dependencies. Without this, there would be no way to use `@import` to // access dependencies by name, since `@import` requires string literals. @@ -4857,8 +4859,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi // Here we borrow main package's table and will replace it with a fresh // one after this process completes. - const fetch_result = build_pkg.fetchAndAddDependencies( - &main_pkg, + const fetch_result = build_mod.fetchAndAddDependencies( + &main_mod, arena, &thread_pool, &http_client, @@ -4886,10 +4888,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi dependencies_source.items, ); - mem.swap(Package.Table, &main_pkg.table, &deps_pkg.table); - try main_pkg.add(gpa, "@dependencies", deps_pkg); + mem.swap(Package.Table, &main_mod.table, &deps_pkg.table); + try main_mod.add(gpa, "@dependencies", deps_pkg); } - try main_pkg.add(gpa, "@build", &build_pkg); + try main_mod.add(gpa, "@build", &build_mod); const comp = Compilation.create(gpa, .{ .zig_lib_directory = zig_lib_directory, @@ -4901,7 +4903,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .is_native_abi = cross_target.isNativeAbi(), .dynamic_linker = target_info.dynamic_linker.get(), .output_mode = .Exe, - .main_pkg = &main_pkg, + .main_mod = &main_mod, .emit_bin = emit_bin, .emit_h = null, .optimize_mode = .Debug, @@ -5115,12 +5117,14 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void .tree = tree, .tree_loaded = true, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; - file.pkg = try Package.create(gpa, null, file.sub_file_path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(arena, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + }); file.zir = try AstGen.generate(gpa, file.tree); file.zir_loaded = true; @@ -5321,12 +5325,14 @@ fn fmtPathFile( .tree = tree, .tree_loaded = true, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; - file.pkg = try Package.create(gpa, null, file.sub_file_path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(fmt.arena, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + }); if (stat.size > max_src_size) return error.FileTooBig; @@ -5387,7 +5393,7 @@ pub fn putAstErrorsIntoBundle( tree: Ast, path: []const u8, wip_errors: *std.zig.ErrorBundle.Wip, -) !void { +) Allocator.Error!void { var file: Module.File = .{ .status = .never_loaded, .source_loaded = true, @@ -5402,12 +5408,15 @@ pub fn putAstErrorsIntoBundle( .tree = tree, .tree_loaded = true, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; - file.pkg = try Package.create(gpa, null, path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(gpa, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + }); + defer gpa.destroy(file.mod); file.zir = try AstGen.generate(gpa, file.tree); file.zir_loaded = true; @@ -5933,7 +5942,7 @@ pub fn cmdAstCheck( .stat = undefined, .tree = undefined, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; if (zig_source_file) |file_name| { @@ -5971,8 +5980,10 @@ pub fn cmdAstCheck( file.stat.size = source.len; } - file.pkg = try Package.create(gpa, null, file.sub_file_path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(arena, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + }); file.tree = try Ast.parse(gpa, file.source, .zig); file.tree_loaded = true; @@ -6067,7 +6078,7 @@ pub fn cmdDumpZir( .stat = undefined, .tree = undefined, .zir = try Module.loadZirCache(gpa, f), - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; @@ -6136,12 +6147,14 @@ pub fn cmdChangelist( }, .tree = undefined, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; - file.pkg = try Package.create(gpa, null, file.sub_file_path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(arena, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + }); const source = try arena.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0); const amt = try f.readAll(source); @@ -6623,8 +6636,11 @@ fn cmdFetch( args: []const []const u8, ) !void { const color: Color = .auto; - var opt_url: ?[]const u8 = null; + const work_around_btrfs_bug = builtin.os.tag == .linux and + std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND"); + var opt_path_or_url: ?[]const u8 = null; var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR"); + var recursive = false; { var i: usize = 0; @@ -6640,18 +6656,21 @@ fn cmdFetch( i += 1; override_global_cache_dir = args[i]; continue; + } else if (mem.eql(u8, arg, "--recursive")) { + recursive = true; + continue; } else { fatal("unrecognized parameter: '{s}'", .{arg}); } - } else if (opt_url != null) { + } else if (opt_path_or_url != null) { fatal("unexpected extra parameter: '{s}'", .{arg}); } else { - opt_url = arg; + opt_path_or_url = arg; } } } - const url = opt_url orelse fatal("missing url or path parameter", .{}); + const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{}); var thread_pool: ThreadPool = undefined; try thread_pool.init(.{ .allocator = gpa }); @@ -6664,19 +6683,6 @@ fn cmdFetch( const root_prog_node = progress.start("Fetch", 0); defer root_prog_node.end(); - var wip_errors: std.zig.ErrorBundle.Wip = undefined; - try wip_errors.init(gpa); - defer wip_errors.deinit(); - - var report: Package.Report = .{ - .ast = null, - .directory = .{ - .handle = fs.cwd(), - .path = null, - }, - .error_bundle = &wip_errors, - }; - var global_cache_directory: Compilation.Directory = l: { const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); break :l .{ @@ -6686,56 +6692,48 @@ fn cmdFetch( }; defer global_cache_directory.handle.close(); - var readable_resource: Package.ReadableResource = rr: { - if (fs.cwd().openIterableDir(url, .{})) |dir| { - break :rr .{ - .path = try gpa.dupe(u8, url), - .resource = .{ .dir = dir }, - }; - } else |dir_err| { - const file_err = if (dir_err == error.NotDir) e: { - if (fs.cwd().openFile(url, .{})) |f| { - break :rr .{ - .path = try gpa.dupe(u8, url), - .resource = .{ .file = f }, - }; - } else |err| break :e err; - } else dir_err; - - const uri = std.Uri.parse(url) catch |uri_err| { - fatal("'{s}' could not be recognized as a file path ({s}) or an URL ({s})", .{ - url, @errorName(file_err), @errorName(uri_err), - }); - }; - const fetch_location = try Package.FetchLocation.initUri(uri, 0, report); - const cwd: Cache.Directory = .{ - .handle = fs.cwd(), - .path = null, - }; - break :rr try fetch_location.fetch(gpa, cwd, &http_client, 0, report); - } + var job_queue: Package.Fetch.JobQueue = .{ + .http_client = &http_client, + .thread_pool = &thread_pool, + .global_cache = global_cache_directory, + .recursive = recursive, + .work_around_btrfs_bug = work_around_btrfs_bug, + }; + defer job_queue.deinit(); + + var fetch: Package.Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .path_or_url = path_or_url }, + .location_tok = 0, + .hash_tok = 0, + .parent_package_root = undefined, + .parent_manifest_ast = null, + .prog_node = root_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = null, + .manifest_ast = undefined, + .actual_hash = undefined, + .has_build_zig = false, + .oom_flag = false, }; - defer readable_resource.deinit(gpa); + defer fetch.deinit(); - var package_location = readable_resource.unpack( - gpa, - &thread_pool, - global_cache_directory, - 0, - report, - root_prog_node, - ) catch |err| { - if (wip_errors.root_list.items.len > 0) { - var errors = try wip_errors.toOwnedBundle(""); - defer errors.deinit(gpa); - errors.renderToStdErr(renderOptions(color)); - process.exit(1); - } - fatal("unable to unpack '{s}': {s}", .{ url, @errorName(err) }); + fetch.run() catch |err| switch (err) { + error.OutOfMemory => fatal("out of memory", .{}), + error.FetchFailed => {}, // error bundle checked below }; - defer package_location.deinit(gpa); - const hex_digest = Package.Manifest.hexDigest(package_location.hash); + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); + errors.renderToStdErr(renderOptions(color)); + process.exit(1); + } + + const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash); progress.done = true; progress.refresh(); From 1c0d6f9c0020188326560e058baf36b8034d76e5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Oct 2023 17:41:29 -0700 Subject: [PATCH 07/39] require inclusion directives in root manifest but not deps see #14311 --- src/Manifest.zig | 15 +++++++++++++-- src/Package/Fetch.zig | 11 +++++++++-- src/main.zig | 7 ++----- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/Manifest.zig b/src/Manifest.zig index 4032244d6eed..17d5d93fb59b 100644 --- a/src/Manifest.zig +++ b/src/Manifest.zig @@ -59,9 +59,13 @@ paths: std.StringArrayHashMapUnmanaged(void), errors: []ErrorMessage, arena_state: std.heap.ArenaAllocator.State, +pub const ParseOptions = struct { + allow_missing_paths_field: bool = false, +}; + pub const Error = Allocator.Error; -pub fn parse(gpa: Allocator, ast: std.zig.Ast) Error!Manifest { +pub fn parse(gpa: Allocator, ast: std.zig.Ast, options: ParseOptions) Error!Manifest { const node_tags = ast.nodes.items(.tag); const node_datas = ast.nodes.items(.data); assert(node_tags[0] == .root); @@ -80,6 +84,7 @@ pub fn parse(gpa: Allocator, ast: std.zig.Ast) Error!Manifest { .version = undefined, .dependencies = .{}, .paths = .{}, + .allow_missing_paths_field = options.allow_missing_paths_field, .buf = .{}, }; defer p.buf.deinit(gpa); @@ -152,6 +157,7 @@ const Parse = struct { version: std.SemanticVersion, dependencies: std.StringArrayHashMapUnmanaged(Dependency), paths: std.StringArrayHashMapUnmanaged(void), + allow_missing_paths_field: bool, const InnerError = error{ ParseFailure, OutOfMemory }; @@ -178,6 +184,7 @@ const Parse = struct { if (mem.eql(u8, field_name, "dependencies")) { try parseDependencies(p, field_init); } else if (mem.eql(u8, field_name, "paths")) { + have_included_paths = true; try parseIncludedPaths(p, field_init); } else if (mem.eql(u8, field_name, "name")) { p.name = try parseString(p, field_init); @@ -204,7 +211,11 @@ const Parse = struct { } if (!have_included_paths) { - try appendError(p, main_token, "missing top-level 'paths' field", .{}); + if (p.allow_missing_paths_field) { + try p.paths.put(p.gpa, "", {}); + } else { + try appendError(p, main_token, "missing top-level 'paths' field", .{}); + } } } diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 55de79f039fb..d6d1a5007fd3 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -38,6 +38,10 @@ job_queue: *JobQueue, /// If true, don't add an error for a missing hash. This flag is not passed /// down to recursive dependencies. It's intended to be used only be the CLI. omit_missing_hash_error: bool, +/// If true, don't fail when a manifest file is missing the `paths` field, +/// which specifies inclusion rules. This is intended to be true for the first +/// fetch task and false for the recursive dependencies. +allow_missing_paths_field: bool, // Above this are fields provided as inputs to `run`. // Below this are fields populated by `run`. @@ -365,7 +369,9 @@ fn loadManifest(f: *Fetch, pkg_root: Package.Path) RunError!void { return error.FetchFailed; } - f.manifest = try Manifest.parse(arena, ast.*); + f.manifest = try Manifest.parse(arena, ast.*, .{ + .allow_missing_paths_field = f.allow_missing_paths_field, + }); const manifest = &f.manifest.?; if (manifest.errors.len > 0) { @@ -452,6 +458,7 @@ fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void .prog_node = f.prog_node, .job_queue = f.job_queue, .omit_missing_hash_error = false, + .allow_missing_paths_field = false, .package_root = undefined, .error_bundle = undefined, @@ -481,7 +488,7 @@ fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void } } -fn workerRun(f: *Fetch) void { +pub fn workerRun(f: *Fetch) void { defer f.job_queue.wait_group.finish(); run(f) catch |err| switch (err) { error.OutOfMemory => f.oom_flag = true, diff --git a/src/main.zig b/src/main.zig index f52a4bf24af0..20a493ca675a 100644 --- a/src/main.zig +++ b/src/main.zig @@ -6640,7 +6640,6 @@ fn cmdFetch( std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND"); var opt_path_or_url: ?[]const u8 = null; var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR"); - var recursive = false; { var i: usize = 0; @@ -6656,9 +6655,6 @@ fn cmdFetch( i += 1; override_global_cache_dir = args[i]; continue; - } else if (mem.eql(u8, arg, "--recursive")) { - recursive = true; - continue; } else { fatal("unrecognized parameter: '{s}'", .{arg}); } @@ -6696,7 +6692,7 @@ fn cmdFetch( .http_client = &http_client, .thread_pool = &thread_pool, .global_cache = global_cache_directory, - .recursive = recursive, + .recursive = false, .work_around_btrfs_bug = work_around_btrfs_bug, }; defer job_queue.deinit(); @@ -6711,6 +6707,7 @@ fn cmdFetch( .prog_node = root_prog_node, .job_queue = &job_queue, .omit_missing_hash_error = true, + .allow_missing_paths_field = true, .package_root = undefined, .error_bundle = undefined, From f708c5fafc5087a2518e0dc7acf986b59673dddd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Oct 2023 21:29:08 -0700 Subject: [PATCH 08/39] CLI: finish updating module API usage Finish the work started in 4c4fb839972f66f55aa44fc0aca5f80b0608c731. Now the compiler compiles again. Wire up dependency tree fetching code in the CLI for `zig build`. Everything is hooked up except for `createDependenciesModule` is not yet implemented. --- lib/std/process.zig | 2 +- src/Autodoc.zig | 35 +++++---- src/Compilation.zig | 141 +++++++++++++++-------------------- src/Module.zig | 164 +++++++++++++++++++---------------------- src/Package.zig | 33 ++++++++- src/Package/Fetch.zig | 30 +++++++- src/Package/Module.zig | 2 +- src/Sema.zig | 12 +-- src/codegen/llvm.zig | 29 ++++---- src/glibc.zig | 2 +- src/libcxx.zig | 4 +- src/libtsan.zig | 2 +- src/libunwind.zig | 2 +- src/link/Dwarf.zig | 18 +++-- src/link/Elf.zig | 6 +- src/link/Plan9.zig | 9 ++- src/main.zig | 113 ++++++++++++++-------------- src/musl.zig | 2 +- 18 files changed, 321 insertions(+), 285 deletions(-) diff --git a/lib/std/process.zig b/lib/std/process.zig index a7f9d8d9d8a8..518def421594 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -46,7 +46,7 @@ pub fn getCwdAlloc(allocator: Allocator) ![]u8 { } } -test "getCwdAlloc" { +test getCwdAlloc { if (builtin.os.tag == .wasi) return error.SkipZigTest; const cwd = try getCwdAlloc(testing.allocator); diff --git a/src/Autodoc.zig b/src/Autodoc.zig index a70040558c16..5a241e51abcf 100644 --- a/src/Autodoc.zig +++ b/src/Autodoc.zig @@ -6,7 +6,7 @@ const Autodoc = @This(); const Compilation = @import("Compilation.zig"); const CompilationModule = @import("Module.zig"); const File = CompilationModule.File; -const Module = @import("Package.zig"); +const Module = @import("Package.zig").Module; const Tokenizer = std.zig.Tokenizer; const InternPool = @import("InternPool.zig"); const Zir = @import("Zir.zig"); @@ -98,9 +98,8 @@ pub fn generate(cm: *CompilationModule, output_dir: std.fs.Dir) !void { } fn generateZirData(self: *Autodoc, output_dir: std.fs.Dir) !void { - const root_src_dir = self.comp_module.main_pkg.root_src_directory; - const root_src_path = self.comp_module.main_pkg.root_src_path; - const joined_src_path = try root_src_dir.join(self.arena, &.{root_src_path}); + const root_src_path = self.comp_module.main_mod.root_src_path; + const joined_src_path = try self.comp_module.main_mod.root.joinString(self.arena, root_src_path); defer self.arena.free(joined_src_path); const abs_root_src_path = try std.fs.path.resolve(self.arena, &.{ ".", joined_src_path }); @@ -295,20 +294,20 @@ fn generateZirData(self: *Autodoc, output_dir: std.fs.Dir) !void { } const rootName = blk: { - const rootName = std.fs.path.basename(self.comp_module.main_pkg.root_src_path); + const rootName = std.fs.path.basename(self.comp_module.main_mod.root_src_path); break :blk rootName[0 .. rootName.len - 4]; }; const main_type_index = self.types.items.len; { - try self.modules.put(self.arena, self.comp_module.main_pkg, .{ + try self.modules.put(self.arena, self.comp_module.main_mod, .{ .name = rootName, .main = main_type_index, .table = .{}, }); try self.modules.entries.items(.value)[0].table.put( self.arena, - self.comp_module.main_pkg, + self.comp_module.main_mod, .{ .name = rootName, .value = 0, @@ -412,7 +411,7 @@ fn generateZirData(self: *Autodoc, output_dir: std.fs.Dir) !void { while (files_iterator.next()) |entry| { const sub_file_path = entry.key_ptr.*.sub_file_path; - const file_module = entry.key_ptr.*.pkg; + const file_module = entry.key_ptr.*.mod; const module_name = (self.modules.get(file_module) orelse continue).name; const file_path = std.fs.path.dirname(sub_file_path) orelse ""; @@ -986,12 +985,12 @@ fn walkInstruction( // importFile cannot error out since all files // are already loaded at this point - if (file.pkg.table.get(path)) |other_module| { + if (file.mod.deps.get(path)) |other_module| { const result = try self.modules.getOrPut(self.arena, other_module); // Immediately add this module to the import table of our // current module, regardless of wether it's new or not. - if (self.modules.getPtr(file.pkg)) |current_module| { + if (self.modules.getPtr(file.mod)) |current_module| { // TODO: apparently, in the stdlib a file gets analyzed before // its module gets added. I guess we're importing a file // that belongs to another module through its file path? @@ -1025,12 +1024,12 @@ fn walkInstruction( // TODO: Add this module as a dependency to the current module // TODO: this seems something that could be done in bulk // at the beginning or the end, or something. - const root_src_dir = other_module.root_src_directory; - const root_src_path = other_module.root_src_path; - const joined_src_path = try root_src_dir.join(self.arena, &.{root_src_path}); - defer self.arena.free(joined_src_path); - - const abs_root_src_path = try std.fs.path.resolve(self.arena, &.{ ".", joined_src_path }); + const abs_root_src_path = try std.fs.path.resolve(self.arena, &.{ + ".", + other_module.root.root_dir.path orelse ".", + other_module.root.sub_path, + other_module.root_src_path, + }); defer self.arena.free(abs_root_src_path); const new_file = self.comp_module.import_table.get(abs_root_src_path).?; @@ -5683,7 +5682,7 @@ fn writeFileTableToJson( while (it.next()) |entry| { try jsw.beginArray(); try jsw.write(entry.key_ptr.*.sub_file_path); - try jsw.write(mods.getIndex(entry.key_ptr.*.pkg) orelse 0); + try jsw.write(mods.getIndex(entry.key_ptr.*.mod) orelse 0); try jsw.endArray(); } try jsw.endArray(); @@ -5840,7 +5839,7 @@ fn addGuide(self: *Autodoc, file: *File, guide_path: []const u8, section: *Secti file.sub_file_path, "..", guide_path, }); - var guide_file = try file.pkg.root_src_directory.handle.openFile(resolved_path, .{}); + var guide_file = try file.mod.root.openFile(resolved_path, .{}); defer guide_file.close(); const guide = guide_file.reader().readAllAlloc(self.arena, 1 * 1024 * 1024) catch |err| switch (err) { diff --git a/src/Compilation.zig b/src/Compilation.zig index 3827789fed0a..29689ce6b989 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -772,12 +772,14 @@ fn addModuleTableToCacheHash( switch (hash_type) { .path_bytes => { hash.addBytes(mod.value.root_src_path); - hash.addOptionalBytes(mod.value.root_src_directory.path); + hash.addOptionalBytes(mod.value.root.root_dir.path); + hash.addBytes(mod.value.root.sub_path); }, .files => |man| { - const pkg_zig_file = try mod.value.root_src_directory.join(allocator, &[_][]const u8{ + const pkg_zig_file = try mod.value.root.joinString( + allocator, mod.value.root_src_path, - }); + ); _ = try man.addFile(pkg_zig_file, null); }, } @@ -1310,53 +1312,50 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const std_mod = if (main_mod_is_std) main_mod else - try Package.createWithDir( - gpa, - options.zig_lib_directory, - "std", - "std.zig", - ); - - errdefer if (!main_mod_is_std) std_mod.destroy(gpa); + try Package.Module.create(arena, .{ + .root = .{ + .root_dir = options.zig_lib_directory, + .sub_path = "std", + }, + .root_src_path = "std.zig", + }); const root_mod = if (options.is_test) root_mod: { - const test_pkg = if (options.test_runner_path) |test_runner| test_pkg: { - const test_dir = std.fs.path.dirname(test_runner); - const basename = std.fs.path.basename(test_runner); - const pkg = try Package.create(gpa, test_dir, basename); - - // copy module table from main_mod to root_mod - pkg.deps = try main_mod.deps.clone(gpa); - break :test_pkg pkg; - } else try Package.createWithDir( - gpa, - options.zig_lib_directory, - null, - "test_runner.zig", - ); - errdefer test_pkg.destroy(gpa); + const test_mod = if (options.test_runner_path) |test_runner| test_mod: { + const pkg = try Package.Module.create(arena, .{ + .root = .{ + .root_dir = Directory.cwd(), + .sub_path = std.fs.path.dirname(test_runner) orelse "", + }, + .root_src_path = std.fs.path.basename(test_runner), + }); + + pkg.deps = try main_mod.deps.clone(arena); + break :test_mod pkg; + } else try Package.Module.create(arena, .{ + .root = .{ + .root_dir = options.zig_lib_directory, + }, + .root_src_path = "test_runner.zig", + }); - break :root_mod test_pkg; + break :root_mod test_mod; } else main_mod; - errdefer if (options.is_test) root_mod.destroy(gpa); const compiler_rt_mod = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_mod: { - break :compiler_rt_mod try Package.createWithDir( - gpa, - options.zig_lib_directory, - null, - "compiler_rt.zig", - ); + break :compiler_rt_mod try Package.Module.create(arena, .{ + .root = .{ + .root_dir = options.zig_lib_directory, + }, + .root_src_path = "compiler_rt.zig", + }); } else null; - errdefer if (compiler_rt_mod) |p| p.destroy(gpa); - - try main_mod.add(gpa, "builtin", builtin_mod); - try main_mod.add(gpa, "root", root_mod); - try main_mod.add(gpa, "std", std_mod); - if (compiler_rt_mod) |p| { - try main_mod.add(gpa, "compiler_rt", p); - } + try main_mod.deps.put(gpa, "builtin", builtin_mod); + try main_mod.deps.put(gpa, "root", root_mod); + try main_mod.deps.put(gpa, "std", std_mod); + if (compiler_rt_mod) |m| + try main_mod.deps.put(gpa, "compiler_rt", m); // Pre-open the directory handles for cached ZIR code so that it does not need // to redundantly happen for each AstGen operation. @@ -2004,7 +2003,7 @@ fn restorePrevZigCacheArtifactDirectory(comp: *Compilation, directory: *Director // on the handle of zig_cache_artifact_directory. if (comp.bin_file.options.module) |module| { const builtin_mod = module.main_mod.deps.get("builtin").?; - module.zig_cache_artifact_directory = builtin_mod.root_src_directory; + module.zig_cache_artifact_directory = builtin_mod.root.root_dir; } } @@ -2418,9 +2417,7 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes comptime assert(link_hash_implementation_version == 10); if (comp.bin_file.options.module) |mod| { - const main_zig_file = try mod.main_mod.root_src_directory.join(arena, &[_][]const u8{ - mod.main_mod.root_src_path, - }); + const main_zig_file = try mod.main_mod.root.joinString(arena, mod.main_mod.root_src_path); _ = try man.addFile(main_zig_file, null); { var seen_table = std.AutoHashMap(*Package.Module, void).init(arena); @@ -2614,23 +2611,23 @@ fn reportMultiModuleErrors(mod: *Module) !void { errdefer for (notes[0..i]) |*n| n.deinit(mod.gpa); note.* = switch (ref) { .import => |loc| blk: { - const name = try loc.file_scope.pkg.getName(mod.gpa, mod.*); - defer mod.gpa.free(name); + //const name = try loc.file_scope.mod.getName(mod.gpa, mod.*); + //defer mod.gpa.free(name); break :blk try Module.ErrorMsg.init( mod.gpa, loc, - "imported from module {s}", - .{name}, + "imported from module {}", + .{loc.file_scope.mod.root}, ); }, .root => |pkg| blk: { - const name = try pkg.getName(mod.gpa, mod.*); - defer mod.gpa.free(name); + //const name = try pkg.getName(mod.gpa, mod.*); + //defer mod.gpa.free(name); break :blk try Module.ErrorMsg.init( mod.gpa, .{ .file_scope = file, .parent_decl_node = 0, .lazy = .entire_file }, - "root of module {s}", - .{name}, + "root of module {}", + .{pkg.root}, ); }, }; @@ -4212,17 +4209,9 @@ fn reportRetryableAstGenError( }, }; - const err_msg = if (file.pkg.root_src_directory.path) |dir_path| - try Module.ErrorMsg.create( - gpa, - src_loc, - "unable to load '{s}" ++ std.fs.path.sep_str ++ "{s}': {s}", - .{ dir_path, file.sub_file_path, @errorName(err) }, - ) - else - try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{s}': {s}", .{ - file.sub_file_path, @errorName(err), - }); + const err_msg = try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{ + file.mod.root, file.sub_file_path, @errorName(err), + }); errdefer err_msg.destroy(gpa); { @@ -4242,17 +4231,10 @@ fn reportRetryableEmbedFileError( const src_loc: Module.SrcLoc = mod.declPtr(embed_file.owner_decl).srcLoc(mod); - const err_msg = if (embed_file.pkg.root_src_directory.path) |dir_path| - try Module.ErrorMsg.create( - gpa, - src_loc, - "unable to load '{s}" ++ std.fs.path.sep_str ++ "{s}': {s}", - .{ dir_path, embed_file.sub_file_path, @errorName(err) }, - ) - else - try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{s}': {s}", .{ - embed_file.sub_file_path, @errorName(err), - }); + const err_msg = try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{ + embed_file.mod.root, embed_file.sub_file_path, @errorName(err), + }); + errdefer err_msg.destroy(gpa); { @@ -6375,13 +6357,12 @@ fn buildOutputFromZig( const tracy_trace = trace(@src()); defer tracy_trace.end(); - std.debug.assert(output_mode != .Exe); + assert(output_mode != .Exe); - var main_mod: Package = .{ - .root_src_directory = comp.zig_lib_directory, + var main_mod: Package.Module = .{ + .root = .{ .root_dir = comp.zig_lib_directory }, .root_src_path = src_basename, }; - defer main_mod.deinitTable(comp.gpa); const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len]; const target = comp.getTarget(); const bin_basename = try std.zig.binNameAlloc(comp.gpa, .{ diff --git a/src/Module.zig b/src/Module.zig index 141ee3c755ef..153edeafefa1 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -998,8 +998,8 @@ pub const File = struct { pub const Reference = union(enum) { /// The file is imported directly (i.e. not as a package) with @import. import: SrcLoc, - /// The file is the root of a package. - root: *Package, + /// The file is the root of a module. + root: *Package.Module, }; pub fn unload(file: *File, gpa: Allocator) void { @@ -1174,10 +1174,10 @@ pub const File = struct { } const pkg = switch (ref) { - .import => |loc| loc.file_scope.pkg, + .import => |loc| loc.file_scope.mod, .root => |pkg| pkg, }; - if (pkg != file.pkg) file.multi_pkg = true; + if (pkg != file.mod) file.multi_pkg = true; } /// Mark this file and every file referenced by it as multi_pkg and report an @@ -1219,7 +1219,7 @@ pub const EmbedFile = struct { bytes: [:0]const u8, stat: Cache.File.Stat, /// Package that this file is a part of, managed externally. - pkg: *Package, + mod: *Package.Module, /// The Decl that was created from the `@embedFile` to own this resource. /// This is how zig knows what other Decl objects to invalidate if the file /// changes on disk. @@ -2535,28 +2535,6 @@ pub fn deinit(mod: *Module) void { } mod.deletion_set.deinit(gpa); - - // The callsite of `Compilation.create` owns the `main_mod`, however - // Module owns the builtin and std packages that it adds. - if (mod.main_mod.table.fetchRemove("builtin")) |kv| { - gpa.free(kv.key); - kv.value.destroy(gpa); - } - if (mod.main_mod.table.fetchRemove("std")) |kv| { - gpa.free(kv.key); - // It's possible for main_mod to be std when running 'zig test'! In this case, we must not - // destroy it, since it would lead to a double-free. - if (kv.value != mod.main_mod) { - kv.value.destroy(gpa); - } - } - if (mod.main_mod.table.fetchRemove("root")) |kv| { - gpa.free(kv.key); - } - if (mod.root_mod != mod.main_mod) { - mod.root_mod.destroy(gpa); - } - mod.compile_log_text.deinit(gpa); mod.zig_cache_artifact_directory.handle.close(); @@ -2703,18 +2681,19 @@ pub fn astGenFile(mod: *Module, file: *File) !void { const gpa = mod.gpa; // In any case we need to examine the stat of the file to determine the course of action. - var source_file = try file.pkg.root_src_directory.handle.openFile(file.sub_file_path, .{}); + var source_file = try file.mod.root.openFile(file.sub_file_path, .{}); defer source_file.close(); const stat = try source_file.stat(); - const want_local_cache = file.pkg == mod.main_mod; + const want_local_cache = file.mod == mod.main_mod; const digest = hash: { var path_hash: Cache.HashHelper = .{}; path_hash.addBytes(build_options.version); path_hash.add(builtin.zig_backend); if (!want_local_cache) { - path_hash.addOptionalBytes(file.pkg.root_src_directory.path); + path_hash.addOptionalBytes(file.mod.root.root_dir.path); + path_hash.addBytes(file.mod.root.sub_path); } path_hash.addBytes(file.sub_file_path); break :hash path_hash.final(); @@ -2939,10 +2918,8 @@ pub fn astGenFile(mod: *Module, file: *File) !void { }, }; cache_file.writevAll(&iovecs) catch |err| { - const pkg_path = file.pkg.root_src_directory.path orelse "."; - const cache_path = cache_directory.path orelse "."; - log.warn("unable to write cached ZIR code for {s}/{s} to {s}/{s}: {s}", .{ - pkg_path, file.sub_file_path, cache_path, &digest, @errorName(err), + log.warn("unable to write cached ZIR code for {}{s} to {}{s}: {s}", .{ + file.mod.root, file.sub_file_path, cache_directory, &digest, @errorName(err), }); }; @@ -3147,34 +3124,24 @@ pub fn populateBuiltinFile(mod: *Module) !void { defer tracy.end(); const comp = mod.comp; - const pkg_and_file = blk: { + const builtin_mod, const file = blk: { comp.mutex.lock(); defer comp.mutex.unlock(); - const builtin_mod = mod.main_mod.table.get("builtin").?; + const builtin_mod = mod.main_mod.deps.get("builtin").?; const result = try mod.importPkg(builtin_mod); - break :blk .{ - .file = result.file, - .pkg = builtin_mod, - }; + break :blk .{ builtin_mod, result.file }; }; - const file = pkg_and_file.file; - const builtin_mod = pkg_and_file.pkg; const gpa = mod.gpa; file.source = try comp.generateBuiltinZigSource(gpa); file.source_loaded = true; - if (builtin_mod.root_src_directory.handle.statFile(builtin_mod.root_src_path)) |stat| { + if (builtin_mod.root.statFile(builtin_mod.root_src_path)) |stat| { if (stat.size != file.source.len) { - const full_path = try builtin_mod.root_src_directory.join(gpa, &.{ - builtin_mod.root_src_path, - }); - defer gpa.free(full_path); - log.warn( - "the cached file '{s}' had the wrong size. Expected {d}, found {d}. " ++ + "the cached file '{}{s}' had the wrong size. Expected {d}, found {d}. " ++ "Overwriting with correct file contents now", - .{ full_path, file.source.len, stat.size }, + .{ builtin_mod.root, builtin_mod.root_src_path, file.source.len, stat.size }, ); try writeBuiltinFile(file, builtin_mod); @@ -3206,7 +3173,7 @@ pub fn populateBuiltinFile(mod: *Module) !void { } fn writeBuiltinFile(file: *File, builtin_mod: *Package.Module) !void { - var af = try builtin_mod.root_src_directory.handle.atomicFile(builtin_mod.root_src_path, .{}); + var af = try builtin_mod.root.atomicFile(builtin_mod.root_src_path, .{}); defer af.deinit(); try af.file.writeAll(file.source); try af.finish(); @@ -3602,7 +3569,8 @@ pub fn updateEmbedFile(mod: *Module, embed_file: *EmbedFile) SemaError!void { } } -pub fn semaPkg(mod: *Module, pkg: *Package) !void { +/// https://github.com/ziglang/zig/issues/14307 +pub fn semaPkg(mod: *Module, pkg: *Package.Module) !void { const file = (try mod.importPkg(pkg)).file; return mod.semaFile(file); } @@ -3704,13 +3672,11 @@ pub fn semaFile(mod: *Module, file: *File) SemaError!void { return error.AnalysisFail; }; - const resolved_path = std.fs.path.resolve( - gpa, - if (file.pkg.root_src_directory.path) |pkg_path| - &[_][]const u8{ pkg_path, file.sub_file_path } - else - &[_][]const u8{file.sub_file_path}, - ) catch |err| { + const resolved_path = std.fs.path.resolve(gpa, &.{ + file.mod.root.root_dir.path orelse ".", + file.mod.root.sub_path, + file.sub_file_path, + }) catch |err| { try reportRetryableFileError(mod, file, "unable to resolve path: {s}", .{@errorName(err)}); return error.AnalysisFail; }; @@ -3741,8 +3707,8 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !bool { // TODO: figure out how this works under incremental changes to builtin.zig! const builtin_type_target_index: InternPool.Index = blk: { - const std_mod = mod.main_mod.table.get("std").?; - if (decl.getFileScope(mod).pkg != std_mod) break :blk .none; + const std_mod = mod.main_mod.deps.get("std").?; + if (decl.getFileScope(mod).mod != std_mod) break :blk .none; // We're in the std module. const std_file = (try mod.importPkg(std_mod)).file; const std_decl = mod.declPtr(std_file.root_decl.unwrap().?); @@ -4035,14 +4001,17 @@ pub const ImportFileResult = struct { is_pkg: bool, }; -pub fn importPkg(mod: *Module, pkg: *Package) !ImportFileResult { +/// https://github.com/ziglang/zig/issues/14307 +pub fn importPkg(mod: *Module, pkg: *Package.Module) !ImportFileResult { const gpa = mod.gpa; // The resolved path is used as the key in the import table, to detect if // an import refers to the same as another, despite different relative paths // or differently mapped package names. - const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{ - pkg.root_src_directory.path orelse ".", pkg.root_src_path, + const resolved_path = try std.fs.path.resolve(gpa, &.{ + pkg.root.root_dir.path orelse ".", + pkg.root.sub_path, + pkg.root_src_path, }); var keep_resolved_path = false; defer if (!keep_resolved_path) gpa.free(resolved_path); @@ -4076,7 +4045,7 @@ pub fn importPkg(mod: *Module, pkg: *Package) !ImportFileResult { .tree = undefined, .zir = undefined, .status = .never_loaded, - .pkg = pkg, + .mod = pkg, .root_decl = .none, }; try new_file.addReference(mod.*, .{ .root = pkg }); @@ -4093,15 +4062,15 @@ pub fn importFile( import_string: []const u8, ) !ImportFileResult { if (std.mem.eql(u8, import_string, "std")) { - return mod.importPkg(mod.main_mod.table.get("std").?); + return mod.importPkg(mod.main_mod.deps.get("std").?); } if (std.mem.eql(u8, import_string, "builtin")) { - return mod.importPkg(mod.main_mod.table.get("builtin").?); + return mod.importPkg(mod.main_mod.deps.get("builtin").?); } if (std.mem.eql(u8, import_string, "root")) { return mod.importPkg(mod.root_mod); } - if (cur_file.pkg.table.get(import_string)) |pkg| { + if (cur_file.mod.deps.get(import_string)) |pkg| { return mod.importPkg(pkg); } if (!mem.endsWith(u8, import_string, ".zig")) { @@ -4112,10 +4081,14 @@ pub fn importFile( // The resolved path is used as the key in the import table, to detect if // an import refers to the same as another, despite different relative paths // or differently mapped package names. - const cur_pkg_dir_path = cur_file.pkg.root_src_directory.path orelse "."; - const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{ - cur_pkg_dir_path, cur_file.sub_file_path, "..", import_string, + const resolved_path = try std.fs.path.resolve(gpa, &.{ + cur_file.mod.root.root_dir.path orelse ".", + cur_file.mod.root.sub_path, + cur_file.sub_file_path, + "..", + import_string, }); + var keep_resolved_path = false; defer if (!keep_resolved_path) gpa.free(resolved_path); @@ -4130,7 +4103,10 @@ pub fn importFile( const new_file = try gpa.create(File); errdefer gpa.destroy(new_file); - const resolved_root_path = try std.fs.path.resolve(gpa, &[_][]const u8{cur_pkg_dir_path}); + const resolved_root_path = try std.fs.path.resolve(gpa, &.{ + cur_file.mod.root.root_dir.path orelse ".", + cur_file.mod.root.sub_path, + }); defer gpa.free(resolved_root_path); const sub_file_path = p: { @@ -4164,7 +4140,7 @@ pub fn importFile( .tree = undefined, .zir = undefined, .status = .never_loaded, - .pkg = cur_file.pkg, + .mod = cur_file.mod, .root_decl = .none, }; return ImportFileResult{ @@ -4177,9 +4153,11 @@ pub fn importFile( pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*EmbedFile { const gpa = mod.gpa; - if (cur_file.pkg.table.get(import_string)) |pkg| { - const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{ - pkg.root_src_directory.path orelse ".", pkg.root_src_path, + if (cur_file.mod.deps.get(import_string)) |pkg| { + const resolved_path = try std.fs.path.resolve(gpa, &.{ + pkg.root.root_dir.path orelse ".", + pkg.root.sub_path, + pkg.root_src_path, }); var keep_resolved_path = false; defer if (!keep_resolved_path) gpa.free(resolved_path); @@ -4196,10 +4174,14 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb // The resolved path is used as the key in the table, to detect if a file // refers to the same as another, despite different relative paths. - const cur_pkg_dir_path = cur_file.pkg.root_src_directory.path orelse "."; - const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{ - cur_pkg_dir_path, cur_file.sub_file_path, "..", import_string, + const resolved_path = try std.fs.path.resolve(gpa, &.{ + cur_file.mod.root.root_dir.path orelse ".", + cur_file.mod.root.sub_path, + cur_file.sub_file_path, + "..", + import_string, }); + var keep_resolved_path = false; defer if (!keep_resolved_path) gpa.free(resolved_path); @@ -4207,7 +4189,10 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb errdefer assert(mod.embed_table.remove(resolved_path)); if (gop.found_existing) return gop.value_ptr.*; - const resolved_root_path = try std.fs.path.resolve(gpa, &[_][]const u8{cur_pkg_dir_path}); + const resolved_root_path = try std.fs.path.resolve(gpa, &.{ + cur_file.mod.root.root_dir.path orelse ".", + cur_file.mod.root.sub_path, + }); defer gpa.free(resolved_root_path); const sub_file_path = p: { @@ -4225,12 +4210,13 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb }; errdefer gpa.free(sub_file_path); - return newEmbedFile(mod, cur_file.pkg, sub_file_path, resolved_path, &keep_resolved_path, gop); + return newEmbedFile(mod, cur_file.mod, sub_file_path, resolved_path, &keep_resolved_path, gop); } +/// https://github.com/ziglang/zig/issues/14307 fn newEmbedFile( mod: *Module, - pkg: *Package, + pkg: *Package.Module, sub_file_path: []const u8, resolved_path: []const u8, keep_resolved_path: *bool, @@ -4241,7 +4227,7 @@ fn newEmbedFile( const new_file = try gpa.create(EmbedFile); errdefer gpa.destroy(new_file); - var file = try pkg.root_src_directory.handle.openFile(sub_file_path, .{}); + var file = try pkg.root.openFile(sub_file_path, .{}); defer file.close(); const actual_stat = try file.stat(); @@ -4268,14 +4254,14 @@ fn newEmbedFile( .sub_file_path = sub_file_path, .bytes = bytes, .stat = stat, - .pkg = pkg, + .mod = pkg, .owner_decl = undefined, // Set by Sema immediately after this function returns. }; return new_file; } pub fn detectEmbedFileUpdate(mod: *Module, embed_file: *EmbedFile) !void { - var file = try embed_file.pkg.root_src_directory.handle.openFile(embed_file.sub_file_path, .{}); + var file = try embed_file.mod.root.openFile(embed_file.sub_file_path, .{}); defer file.close(); const stat = try file.stat(); @@ -4448,21 +4434,21 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) Allocator.Err gop.key_ptr.* = new_decl_index; // Exported decls, comptime decls, usingnamespace decls, and // test decls if in test mode, get analyzed. - const decl_pkg = namespace.file_scope.pkg; + const decl_mod = namespace.file_scope.mod; const want_analysis = is_exported or switch (decl_name_index) { 0 => true, // comptime or usingnamespace decl 1 => blk: { // test decl with no name. Skip the part where we check against // the test name filter. if (!comp.bin_file.options.is_test) break :blk false; - if (decl_pkg != mod.main_mod) break :blk false; + if (decl_mod != mod.main_mod) break :blk false; try mod.test_functions.put(gpa, new_decl_index, {}); break :blk true; }, else => blk: { if (!is_named_test) break :blk false; if (!comp.bin_file.options.is_test) break :blk false; - if (decl_pkg != mod.main_mod) break :blk false; + if (decl_mod != mod.main_mod) break :blk false; if (comp.test_filter) |test_filter| { if (mem.indexOf(u8, ip.stringToSlice(decl_name), test_filter) == null) { break :blk false; @@ -5589,7 +5575,7 @@ pub fn populateTestFunctions( ) !void { const gpa = mod.gpa; const ip = &mod.intern_pool; - const builtin_mod = mod.main_mod.table.get("builtin").?; + const builtin_mod = mod.main_mod.deps.get("builtin").?; const builtin_file = (mod.importPkg(builtin_mod) catch unreachable).file; const root_decl = mod.declPtr(builtin_file.root_decl.unwrap().?); const builtin_namespace = mod.namespacePtr(root_decl.src_namespace); diff --git a/src/Package.zig b/src/Package.zig index 0249f8dae17c..47cee579790b 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -13,12 +13,13 @@ pub const Path = struct { return .{ .root_dir = Cache.Directory.cwd() }; } - pub fn join(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error!Path { + pub fn join(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path { + if (sub_path.len == 0) return p; const parts: []const []const u8 = if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path }; return .{ .root_dir = p.root_dir, - .sub_path = try fs.path.join(allocator, parts), + .sub_path = try fs.path.join(arena, parts), }; } @@ -28,7 +29,7 @@ pub const Path = struct { return p.root_dir.join(allocator, parts); } - pub fn joinStringZ(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 { + pub fn joinStringZ(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![:0]u8 { const parts: []const []const u8 = if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path }; return p.root_dir.joinZ(allocator, parts); @@ -38,7 +39,7 @@ pub const Path = struct { p: Path, sub_path: []const u8, flags: fs.File.OpenFlags, - ) fs.File.OpenError!fs.File { + ) !fs.File { var buf: [fs.MAX_PATH_BYTES]u8 = undefined; const joined_path = if (p.sub_path.len == 0) sub_path else p: { break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ @@ -58,6 +59,30 @@ pub const Path = struct { return p.root_dir.handle.makeOpenPath(joined_path, opts); } + pub fn statFile(p: Path, sub_path: []const u8) !fs.Dir.Stat { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const joined_path = if (p.sub_path.len == 0) sub_path else p: { + break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ + p.sub_path, sub_path, + }) catch return error.NameTooLong; + }; + return p.root_dir.handle.statFile(joined_path); + } + + pub fn atomicFile( + p: Path, + sub_path: []const u8, + options: fs.Dir.AtomicFileOptions, + ) !fs.AtomicFile { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const joined_path = if (p.sub_path.len == 0) sub_path else p: { + break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ + p.sub_path, sub_path, + }) catch return error.NameTooLong; + }; + return p.root_dir.handle.atomicFile(joined_path, options); + } + pub fn format( self: Path, comptime fmt_string: []const u8, diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index d6d1a5007fd3..93ac2f4f015e 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -49,7 +49,7 @@ allow_missing_paths_field: bool, /// This will either be relative to `global_cache`, or to the build root of /// the root package. package_root: Package.Path, -error_bundle: std.zig.ErrorBundle.Wip, +error_bundle: ErrorBundle.Wip, manifest: ?Manifest, manifest_ast: std.zig.Ast, actual_hash: Manifest.Digest, @@ -89,6 +89,31 @@ pub const JobQueue = struct { jq.all_fetches.deinit(gpa); jq.* = undefined; } + + /// Dumps all subsequent error bundles into the first one. + pub fn consolidateErrors(jq: *JobQueue) !void { + const root = &jq.all_fetches.items[0].error_bundle; + for (jq.all_fetches.items[1..]) |fetch| { + if (fetch.error_bundle.root_list.items.len > 0) { + try root.addBundleAsRoots(fetch.error_bundle.tmpBundle()); + } + } + } + + /// Creates the dependencies.zig file and corresponding `Module` for the + /// build runner to obtain via `@import("@dependencies")`. + pub fn createDependenciesModule( + jq: *JobQueue, + arena: Allocator, + local_cache_directory: Cache.Directory, + basename: []const u8, + ) !*Package.Module { + _ = jq; + _ = arena; + _ = local_cache_directory; + _ = basename; + @panic("TODO: createDependenciesModule"); + } }; pub const Location = union(enum) { @@ -502,7 +527,7 @@ pub fn workerRun(f: *Fetch) void { fn srcLoc( f: *Fetch, tok: std.zig.Ast.TokenIndex, -) Allocator.Error!std.zig.ErrorBundle.SourceLocationIndex { +) Allocator.Error!ErrorBundle.SourceLocationIndex { const ast = f.parent_manifest_ast orelse return .none; const eb = &f.error_bundle; const token_starts = ast.tokens.items(.start); @@ -1258,3 +1283,4 @@ const Fetch = @This(); const main = @import("../main.zig"); const git = @import("../git.zig"); const Package = @import("../Package.zig"); +const ErrorBundle = std.zig.ErrorBundle; diff --git a/src/Package/Module.zig b/src/Package/Module.zig index f2accc104d49..3502def385f8 100644 --- a/src/Package/Module.zig +++ b/src/Package/Module.zig @@ -8,7 +8,7 @@ root: Package.Path, root_src_path: []const u8, /// The dependency table of this module. Shared dependencies such as 'std', /// 'builtin', and 'root' are not specified in every dependency table, but -/// instead only in the table of `main_pkg`. `Module.importFile` is +/// instead only in the table of `main_mod`. `Module.importFile` is /// responsible for detecting these names and using the correct package. deps: Deps = .{}, diff --git a/src/Sema.zig b/src/Sema.zig index c1187c0591bd..4d49546cf7a4 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -13075,9 +13075,11 @@ fn zirImport(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air. return sema.fail(block, operand_src, "import of file outside package path: '{s}'", .{operand}); }, error.PackageNotFound => { - const name = try block.getFileScope(mod).pkg.getName(sema.gpa, mod.*); - defer sema.gpa.free(name); - return sema.fail(block, operand_src, "no package named '{s}' available within package '{s}'", .{ operand, name }); + //const name = try block.getFileScope(mod).mod.getName(sema.gpa, mod.*); + //defer sema.gpa.free(name); + return sema.fail(block, operand_src, "no package named '{s}' available within package '{}'", .{ + operand, block.getFileScope(mod).mod.root, + }); }, else => { // TODO: these errors are file system errors; make sure an update() will @@ -36415,8 +36417,8 @@ fn getBuiltinDecl(sema: *Sema, block: *Block, name: []const u8) CompileError!Mod const mod = sema.mod; const ip = &mod.intern_pool; - const std_pkg = mod.main_pkg.table.get("std").?; - const std_file = (mod.importPkg(std_pkg) catch unreachable).file; + const std_mod = mod.main_mod.deps.get("std").?; + const std_file = (mod.importPkg(std_mod) catch unreachable).file; const opt_builtin_inst = (try sema.namespaceLookupRef( block, src, diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 4355ac119143..3d15b5310be4 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -892,21 +892,24 @@ pub const Object = struct { build_options.semver.patch, }); - // We fully resolve all paths at this point to avoid lack of source line info in stack - // traces or lack of debugging information which, if relative paths were used, would - // be very location dependent. + // We fully resolve all paths at this point to avoid lack of + // source line info in stack traces or lack of debugging + // information which, if relative paths were used, would be + // very location dependent. // TODO: the only concern I have with this is WASI as either host or target, should // we leave the paths as relative then? var buf: [std.fs.MAX_PATH_BYTES]u8 = undefined; - const compile_unit_dir = blk: { - const path = d: { - const mod = options.module orelse break :d "."; - break :d mod.root_pkg.root_src_directory.path orelse "."; - }; - if (std.fs.path.isAbsolute(path)) break :blk path; - break :blk std.os.realpath(path, &buf) catch path; // If realpath fails, fallback to whatever path was + const compile_unit_dir_z = blk: { + if (options.module) |mod| { + const d = try mod.root_mod.root.joinStringZ(builder.gpa, ""); + if (std.fs.path.isAbsolute(d)) break :blk d; + const abs = std.fs.realpath(d, &buf) catch break :blk d; + builder.gpa.free(d); + break :blk try builder.gpa.dupeZ(u8, abs); + } + const cwd = try std.process.getCwd(&buf); + break :blk try builder.gpa.dupeZ(u8, cwd); }; - const compile_unit_dir_z = try builder.gpa.dupeZ(u8, compile_unit_dir); defer builder.gpa.free(compile_unit_dir_z); builder.llvm.di_compile_unit = builder.llvm.di_builder.?.createCompileUnit( @@ -2828,8 +2831,8 @@ pub const Object = struct { fn getStackTraceType(o: *Object) Allocator.Error!Type { const mod = o.module; - const std_pkg = mod.main_pkg.table.get("std").?; - const std_file = (mod.importPkg(std_pkg) catch unreachable).file; + const std_mod = mod.main_mod.deps.get("std").?; + const std_file = (mod.importPkg(std_mod) catch unreachable).file; const builtin_str = try mod.intern_pool.getOrPutString(mod.gpa, "builtin"); const std_namespace = mod.namespacePtr(mod.declPtr(std_file.root_decl.unwrap().?).src_namespace); diff --git a/src/glibc.zig b/src/glibc.zig index cf12e8ea46bf..2321063166b4 100644 --- a/src/glibc.zig +++ b/src/glibc.zig @@ -1074,7 +1074,7 @@ fn buildSharedLib( .cache_mode = .whole, .target = comp.getTarget(), .root_name = lib.name, - .main_pkg = null, + .main_mod = null, .output_mode = .Lib, .link_mode = .Dynamic, .thread_pool = comp.thread_pool, diff --git a/src/libcxx.zig b/src/libcxx.zig index 0c39469b69de..2185d7f89088 100644 --- a/src/libcxx.zig +++ b/src/libcxx.zig @@ -233,7 +233,7 @@ pub fn buildLibCXX(comp: *Compilation, prog_node: *std.Progress.Node) !void { .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = null, + .main_mod = null, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, @@ -396,7 +396,7 @@ pub fn buildLibCXXABI(comp: *Compilation, prog_node: *std.Progress.Node) !void { .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = null, + .main_mod = null, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, diff --git a/src/libtsan.zig b/src/libtsan.zig index 90d2537876c1..201f4de78599 100644 --- a/src/libtsan.zig +++ b/src/libtsan.zig @@ -202,7 +202,7 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void { .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = null, + .main_mod = null, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, diff --git a/src/libunwind.zig b/src/libunwind.zig index da248c021f4f..589634763d78 100644 --- a/src/libunwind.zig +++ b/src/libunwind.zig @@ -89,7 +89,7 @@ pub fn buildStaticLib(comp: *Compilation, prog_node: *std.Progress.Node) !void { .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = null, + .main_mod = null, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, diff --git a/src/link/Dwarf.zig b/src/link/Dwarf.zig index cbf5f350e857..931dbacdb914 100644 --- a/src/link/Dwarf.zig +++ b/src/link/Dwarf.zig @@ -1880,7 +1880,7 @@ pub fn writeDbgInfoHeader(self: *Dwarf, module: *Module, low_pc: u64, high_pc: u }, } // Write the form for the compile unit, which must match the abbrev table above. - const name_strp = try self.strtab.insert(self.allocator, module.root_pkg.root_src_path); + const name_strp = try self.strtab.insert(self.allocator, module.root_mod.root_src_path); var compile_unit_dir_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined; const compile_unit_dir = resolveCompilationDir(module, &compile_unit_dir_buffer); const comp_dir_strp = try self.strtab.insert(self.allocator, compile_unit_dir); @@ -1940,9 +1940,17 @@ fn resolveCompilationDir(module: *Module, buffer: *[std.fs.MAX_PATH_BYTES]u8) [] // be very location dependent. // TODO: the only concern I have with this is WASI as either host or target, should // we leave the paths as relative then? - const comp_dir_path = module.root_pkg.root_src_directory.path orelse "."; - if (std.fs.path.isAbsolute(comp_dir_path)) return comp_dir_path; - return std.os.realpath(comp_dir_path, buffer) catch comp_dir_path; // If realpath fails, fallback to whatever comp_dir_path was + const root_dir_path = module.root_mod.root.root_dir.path orelse "."; + const sub_path = module.root_mod.root.sub_path; + const realpath = if (std.fs.path.isAbsolute(root_dir_path)) r: { + @memcpy(buffer[0..root_dir_path.len], root_dir_path); + break :r root_dir_path; + } else std.fs.realpath(root_dir_path, buffer) catch return root_dir_path; + const len = realpath.len + 1 + sub_path.len; + if (buffer.len < len) return root_dir_path; + buffer[realpath.len] = '/'; + @memcpy(buffer[realpath.len + 1 ..][0..sub_path.len], sub_path); + return buffer[0..len]; } fn writeAddrAssumeCapacity(self: *Dwarf, buf: *std.ArrayList(u8), addr: u64) void { @@ -2664,7 +2672,7 @@ fn genIncludeDirsAndFileNames(self: *Dwarf, arena: Allocator) !struct { for (self.di_files.keys()) |dif| { const dir_path = d: { var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined; - const dir_path = dif.pkg.root_src_directory.path orelse "."; + const dir_path = try dif.mod.root.joinString(arena, dif.mod.root.sub_path); const abs_dir_path = if (std.fs.path.isAbsolute(dir_path)) dir_path else diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 24da99aab208..f75047424a6f 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -929,15 +929,15 @@ pub fn populateMissingMetadata(self: *Elf) !void { if (self.base.options.module) |module| { if (self.zig_module_index == null and !self.base.options.use_llvm) { - const index = @as(File.Index, @intCast(try self.files.addOne(gpa))); + const index: File.Index = @intCast(try self.files.addOne(gpa)); self.files.set(index, .{ .zig_module = .{ .index = index, - .path = module.main_pkg.root_src_path, + .path = module.main_mod.root_src_path, } }); self.zig_module_index = index; const zig_module = self.file(index).?.zig_module; - const name_off = try self.strtab.insert(gpa, std.fs.path.stem(module.main_pkg.root_src_path)); + const name_off = try self.strtab.insert(gpa, std.fs.path.stem(module.main_mod.root_src_path)); const symbol_index = try self.addSymbol(); try zig_module.local_symbols.append(gpa, symbol_index); const symbol_ptr = self.symbol(symbol_index); diff --git a/src/link/Plan9.zig b/src/link/Plan9.zig index b519fdda00eb..3dcef859ae6d 100644 --- a/src/link/Plan9.zig +++ b/src/link/Plan9.zig @@ -352,9 +352,12 @@ fn putFn(self: *Plan9, decl_index: Module.Decl.Index, out: FnDeclOutput) !void { // getting the full file path var buf: [std.fs.MAX_PATH_BYTES]u8 = undefined; - const dir = file.pkg.root_src_directory.path orelse try std.os.getcwd(&buf); - const sub_path = try std.fs.path.join(arena, &.{ dir, file.sub_file_path }); - try self.addPathComponents(sub_path, &a); + const full_path = try std.fs.path.join(arena, &.{ + file.mod.root.root_dir.path orelse try std.os.getcwd(&buf), + file.mod.root.sub_path, + file.sub_file_path, + }); + try self.addPathComponents(full_path, &a); // null terminate try a.append(0); diff --git a/src/main.zig b/src/main.zig index 20a493ca675a..433ae0e169e0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -897,7 +897,7 @@ fn buildOutputType( var override_local_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_LOCAL_CACHE_DIR"); var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR"); var override_lib_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_LIB_DIR"); - var main_pkg_path: ?[]const u8 = null; + var main_mod_path: ?[]const u8 = null; var clang_preprocessor_mode: Compilation.ClangPreprocessorMode = .no; var subsystem: ?std.Target.SubSystem = null; var major_subsystem_version: ?u32 = null; @@ -1047,7 +1047,7 @@ fn buildOutputType( } root_deps_str = args_iter.nextOrFatal(); } else if (mem.eql(u8, arg, "--main-mod-path")) { - main_pkg_path = args_iter.nextOrFatal(); + main_mod_path = args_iter.nextOrFatal(); } else if (mem.eql(u8, arg, "-cflags")) { extra_cflags.shrinkRetainingCapacity(0); while (true) { @@ -3236,8 +3236,8 @@ fn buildOutputType( const main_mod: ?*Package.Module = if (root_src_file) |unresolved_src_path| blk: { const src_path = try introspect.resolvePath(arena, unresolved_src_path); - if (main_pkg_path) |unresolved_main_pkg_path| { - const p = try introspect.resolvePath(arena, unresolved_main_pkg_path); + if (main_mod_path) |unresolved_main_mod_path| { + const p = try introspect.resolvePath(arena, unresolved_main_mod_path); break :blk try Package.Module.create(arena, .{ .root = .{ .root_dir = Cache.Directory.cwd(), @@ -3386,8 +3386,6 @@ fn buildOutputType( gimmeMoreOfThoseSweetSweetFileDescriptors(); - if (true) @panic("TODO restore Compilation logic"); - const comp = Compilation.create(gpa, .{ .zig_lib_directory = zig_lib_directory, .local_cache_directory = local_cache_directory, @@ -4628,6 +4626,8 @@ pub const usage_build = ; pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { + const work_around_btrfs_bug = builtin.os.tag == .linux and + std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND"); var color: Color = .auto; // We want to release all the locks before executing the child process, so we make a nice @@ -4725,7 +4725,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi const cwd_path = try process.getCwdAlloc(arena); const build_zig_basename = if (build_file) |bf| fs.path.basename(bf) else Package.build_zig_basename; - const build_directory: Compilation.Directory = blk: { + const build_root: Compilation.Directory = blk: { if (build_file) |bf| { if (fs.path.dirname(bf)) |dirname| { const dir = fs.cwd().openDir(dirname, .{}) catch |err| { @@ -4761,7 +4761,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi } } }; - child_argv.items[argv_index_build_file] = build_directory.path orelse cwd_path; + child_argv.items[argv_index_build_file] = build_root.path orelse cwd_path; var global_cache_directory: Compilation.Directory = l: { const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); @@ -4781,9 +4781,9 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .path = local_cache_dir_path, }; } - const cache_dir_path = try build_directory.join(arena, &[_][]const u8{"zig-cache"}); + const cache_dir_path = try build_root.join(arena, &[_][]const u8{"zig-cache"}); break :l .{ - .handle = try build_directory.handle.makeOpenPath("zig-cache", .{}), + .handle = try build_root.handle.makeOpenPath("zig-cache", .{}), .path = cache_dir_path, }; }; @@ -4824,74 +4824,77 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi }; var build_mod: Package.Module = .{ - .root = .{ .root_dir = build_directory }, + .root = .{ .root_dir = build_root }, .root_src_path = build_zig_basename, }; if (build_options.only_core_functionality) { - const deps_pkg = try Package.createFilePkg(gpa, local_cache_directory, "dependencies.zig", + const deps_mod = try Package.createFilePkg(gpa, local_cache_directory, "dependencies.zig", \\pub const packages = struct {}; \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; \\ ); - try main_mod.deps.put(arena, "@dependencies", deps_pkg); + try main_mod.deps.put(arena, "@dependencies", deps_mod); } else { var http_client: std.http.Client = .{ .allocator = gpa }; defer http_client.deinit(); - if (true) @panic("TODO restore package fetching logic"); - - // Here we provide an import to the build runner that allows using reflection to find - // all of the dependencies. Without this, there would be no way to use `@import` to - // access dependencies by name, since `@import` requires string literals. - var dependencies_source = std.ArrayList(u8).init(gpa); - defer dependencies_source.deinit(); - - var all_modules: Package.AllModules = .{}; - defer all_modules.deinit(gpa); - - var wip_errors: std.zig.ErrorBundle.Wip = undefined; - try wip_errors.init(gpa); - defer wip_errors.deinit(); - var progress: std.Progress = .{ .dont_print_on_dumb = true }; const root_prog_node = progress.start("Fetch Packages", 0); defer root_prog_node.end(); - // Here we borrow main package's table and will replace it with a fresh - // one after this process completes. - const fetch_result = build_mod.fetchAndAddDependencies( - &main_mod, - arena, - &thread_pool, - &http_client, - build_directory, - global_cache_directory, - local_cache_directory, - &dependencies_source, - &wip_errors, - &all_modules, - root_prog_node, - null, - ); - if (wip_errors.root_list.items.len > 0) { - var errors = try wip_errors.toOwnedBundle(""); - defer errors.deinit(gpa); + var job_queue: Package.Fetch.JobQueue = .{ + .http_client = &http_client, + .thread_pool = &thread_pool, + .global_cache = global_cache_directory, + .recursive = true, + .work_around_btrfs_bug = work_around_btrfs_bug, + }; + defer job_queue.deinit(); + + try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); + + var fetch: Package.Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .relative_path = "" }, + .location_tok = 0, + .hash_tok = 0, + .parent_package_root = build_mod.root, + .parent_manifest_ast = null, + .prog_node = root_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + .allow_missing_paths_field = true, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = null, + .manifest_ast = undefined, + .actual_hash = undefined, + .has_build_zig = false, + .oom_flag = false, + }; + job_queue.all_fetches.appendAssumeCapacity(&fetch); + + job_queue.wait_group.start(); + try job_queue.thread_pool.spawn(Package.Fetch.workerRun, .{&fetch}); + job_queue.wait_group.wait(); + + try job_queue.consolidateErrors(); + + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); errors.renderToStdErr(renderOptions(color)); process.exit(1); } - try fetch_result; - const deps_pkg = try Package.createFilePkg( - gpa, + const deps_mod = try job_queue.createDependenciesModule( + arena, local_cache_directory, "dependencies.zig", - dependencies_source.items, ); - - mem.swap(Package.Table, &main_mod.table, &deps_pkg.table); - try main_mod.add(gpa, "@dependencies", deps_pkg); + try main_mod.deps.put(arena, "@dependencies", deps_mod); } - try main_mod.add(gpa, "@build", &build_mod); + try main_mod.deps.put(arena, "@build", &build_mod); const comp = Compilation.create(gpa, .{ .zig_lib_directory = zig_lib_directory, diff --git a/src/musl.zig b/src/musl.zig index c66f464f9d68..bbb3c145bb1b 100644 --- a/src/musl.zig +++ b/src/musl.zig @@ -206,7 +206,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile, prog_node: *std.Progr .zig_lib_directory = comp.zig_lib_directory, .target = target, .root_name = "c", - .main_pkg = null, + .main_mod = null, .output_mode = .Lib, .link_mode = .Dynamic, .thread_pool = comp.thread_pool, From aed6adb6e9f5b313c7cf6158bcd8fa5adaad7ef1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Oct 2023 21:58:15 -0700 Subject: [PATCH 09/39] fix Fetch.JobQueue.consolidateErrors --- src/Package/Fetch.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 93ac2f4f015e..2746ba9c0391 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -93,9 +93,12 @@ pub const JobQueue = struct { /// Dumps all subsequent error bundles into the first one. pub fn consolidateErrors(jq: *JobQueue) !void { const root = &jq.all_fetches.items[0].error_bundle; + const gpa = root.gpa; for (jq.all_fetches.items[1..]) |fetch| { if (fetch.error_bundle.root_list.items.len > 0) { - try root.addBundleAsRoots(fetch.error_bundle.tmpBundle()); + var bundle = try fetch.error_bundle.toOwnedBundle(""); + defer bundle.deinit(gpa); + try root.addBundleAsRoots(bundle); } } } From ddb7c40037e2834c372a980a413c89b9b250988f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Oct 2023 22:02:33 -0700 Subject: [PATCH 10/39] fix inverted logic for allowing/disallowing paths field --- src/Package/Fetch.zig | 2 +- src/main.zig | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 2746ba9c0391..ba558d866c3f 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -486,7 +486,7 @@ fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void .prog_node = f.prog_node, .job_queue = f.job_queue, .omit_missing_hash_error = false, - .allow_missing_paths_field = false, + .allow_missing_paths_field = true, .package_root = undefined, .error_bundle = undefined, diff --git a/src/main.zig b/src/main.zig index 433ae0e169e0..cd7cd80da435 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4863,7 +4863,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .prog_node = root_prog_node, .job_queue = &job_queue, .omit_missing_hash_error = true, - .allow_missing_paths_field = true, + .allow_missing_paths_field = false, .package_root = undefined, .error_bundle = undefined, From 1ca442832465d9735313a326326fbe96a7ec55ed Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Oct 2023 22:16:10 -0700 Subject: [PATCH 11/39] fix recursive package fetching logic For path-relative dependencies, they always need to be added to the hash table at the end. For remote dependencies, they never need to be added. --- src/Package/Fetch.zig | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index ba558d866c3f..1c4c6ad09751 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -448,14 +448,20 @@ fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len + 1)); - // It is impossible for there to be a collision here. Consider all three cases: + // There are four cases here: // * Correct hash is provided by manifest. - // - Redundant jobs are skipped in the loop below. + // - Hash map already has the entry, no need to add it again. // * Incorrect hash is provided by manifest. // - Hash mismatch error emitted; `queueJobsForDeps` is not called. // * Hash is not provided by manifest. // - Hash missing error emitted; `queueJobsForDeps` is not called. - f.job_queue.table.putAssumeCapacityNoClobber(hash, f); + // * path-based location is used without a hash. + // - We need to add `hash` to the table now. + switch (f.location) { + .remote => assert(f.job_queue.table.get(hash) == f), + .relative_path => f.job_queue.table.putAssumeCapacityNoClobber(hash, f), + .path_or_url => unreachable, + } for (deps) |dep| { const new_fetch = &new_fetches[new_fetch_index]; From cbb9b5d9f04509260b8cea935a0441f63f33bc32 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Oct 2023 23:58:53 -0700 Subject: [PATCH 12/39] std: add unstable sorting to array hash maps closes #17426 --- lib/std/array_hash_map.zig | 33 ++++++++++++++++++++++++++++++--- lib/std/multi_array_list.zig | 2 +- lib/std/sort.zig | 2 ++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/lib/std/array_hash_map.zig b/lib/std/array_hash_map.zig index 1bb85a255d7b..75a86f63f6c3 100644 --- a/lib/std/array_hash_map.zig +++ b/lib/std/array_hash_map.zig @@ -1229,14 +1229,41 @@ pub fn ArrayHashMapUnmanaged( /// Sorts the entries and then rebuilds the index. /// `sort_ctx` must have this method: /// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool` + /// Uses a stable sorting algorithm. pub inline fn sort(self: *Self, sort_ctx: anytype) void { if (@sizeOf(ByIndexContext) != 0) @compileError("Cannot infer context " ++ @typeName(Context) ++ ", call sortContext instead."); - return self.sortContext(sort_ctx, undefined); + return sortContextInternal(self, .stable, sort_ctx, undefined); } - pub fn sortContext(self: *Self, sort_ctx: anytype, ctx: Context) void { - self.entries.sort(sort_ctx); + /// Sorts the entries and then rebuilds the index. + /// `sort_ctx` must have this method: + /// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool` + /// Uses an unstable sorting algorithm. + pub inline fn sortUnstable(self: *Self, sort_ctx: anytype) void { + if (@sizeOf(ByIndexContext) != 0) + @compileError("Cannot infer context " ++ @typeName(Context) ++ ", call sortUnstableContext instead."); + return self.sortContextInternal(.unstable, sort_ctx, undefined); + } + + pub inline fn sortContext(self: *Self, sort_ctx: anytype, ctx: Context) void { + return sortContextInternal(self, .stable, sort_ctx, ctx); + } + + pub inline fn sortUnstableContext(self: *Self, sort_ctx: anytype, ctx: Context) void { + return sortContextInternal(self, .unstable, sort_ctx, ctx); + } + + fn sortContextInternal( + self: *Self, + comptime mode: std.sort.Mode, + sort_ctx: anytype, + ctx: Context, + ) void { + switch (mode) { + .stable => self.entries.sort(sort_ctx), + .unstable => self.entries.sortUnstable(sort_ctx), + } const header = self.index_header orelse return; header.reset(); self.insertAllEntriesIntoNewHeader(if (store_hash) {} else ctx, header); diff --git a/lib/std/multi_array_list.zig b/lib/std/multi_array_list.zig index c18d2c2a5aae..11dec7803620 100644 --- a/lib/std/multi_array_list.zig +++ b/lib/std/multi_array_list.zig @@ -467,7 +467,7 @@ pub fn MultiArrayList(comptime T: type) type { /// `ctx` has the following method: /// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool` - fn sortInternal(self: Self, a: usize, b: usize, ctx: anytype, comptime mode: enum { stable, unstable }) void { + fn sortInternal(self: Self, a: usize, b: usize, ctx: anytype, comptime mode: std.sort.Mode) void { const sort_context: struct { sub_ctx: @TypeOf(ctx), slice: Slice, diff --git a/lib/std/sort.zig b/lib/std/sort.zig index abeae292475d..e110a8beb8a0 100644 --- a/lib/std/sort.zig +++ b/lib/std/sort.zig @@ -4,6 +4,8 @@ const testing = std.testing; const mem = std.mem; const math = std.math; +pub const Mode = enum { stable, unstable }; + pub const block = @import("sort/block.zig").block; pub const pdq = @import("sort/pdq.zig").pdq; pub const pdqContext = @import("sort/pdq.zig").pdqContext; From 9eb21541ec9e8b64ae0d64e2851ed8d105308a35 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Oct 2023 23:59:26 -0700 Subject: [PATCH 13/39] make Package.Path support string escape formatting --- lib/std/zig.zig | 2 +- lib/std/zig/fmt.zig | 6 +++--- src/Package.zig | 18 +++++++++++++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 63b620f67493..85d1ae4868d5 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -1,6 +1,6 @@ const std = @import("std.zig"); const tokenizer = @import("zig/tokenizer.zig"); -const fmt = @import("zig/fmt.zig"); +pub const fmt = @import("zig/fmt.zig"); const assert = std.debug.assert; pub const ErrorBundle = @import("zig/ErrorBundle.zig"); diff --git a/lib/std/zig/fmt.zig b/lib/std/zig/fmt.zig index 4afcc7ac2e17..5375b9302525 100644 --- a/lib/std/zig/fmt.zig +++ b/lib/std/zig/fmt.zig @@ -13,7 +13,7 @@ fn formatId( return writer.writeAll(bytes); } try writer.writeAll("@\""); - try formatEscapes(bytes, "", options, writer); + try stringEscape(bytes, "", options, writer); try writer.writeByte('"'); } @@ -47,7 +47,7 @@ test "isValidId" { /// Print the string as escaped contents of a double quoted or single-quoted string. /// Format `{}` treats contents as a double-quoted string. /// Format `{'}` treats contents as a single-quoted string. -fn formatEscapes( +pub fn stringEscape( bytes: []const u8, comptime fmt: []const u8, options: std.fmt.FormatOptions, @@ -90,7 +90,7 @@ fn formatEscapes( /// The format specifier must be one of: /// * `{}` treats contents as a double-quoted string. /// * `{'}` treats contents as a single-quoted string. -pub fn fmtEscapes(bytes: []const u8) std.fmt.Formatter(formatEscapes) { +pub fn fmtEscapes(bytes: []const u8) std.fmt.Formatter(stringEscape) { return .{ .data = bytes }; } diff --git a/src/Package.zig b/src/Package.zig index 47cee579790b..cb1c64ae26e7 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -89,7 +89,23 @@ pub const Path = struct { options: std.fmt.FormatOptions, writer: anytype, ) !void { - _ = options; + if (fmt_string.len == 1) { + // Quote-escape the string. + const stringEscape = std.zig.fmt.stringEscape; + const f = switch (fmt_string[0]) { + 'q' => "", + '\'' => '\'', + else => @compileError("unsupported format string: " ++ fmt_string), + }; + if (self.root_dir.path) |p| { + try stringEscape(p, f, options, writer); + if (self.sub_path.len > 0) try writer.writeAll(fs.path.sep_str); + } + if (self.sub_path.len > 0) { + try stringEscape(self.sub_path, f, options, writer); + } + return; + } if (fmt_string.len > 0) std.fmt.invalidFmtError(fmt_string, self); if (self.root_dir.path) |p| { From 4eb7b61daae4fe43b218eaaac0614f6bee1915d8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Oct 2023 23:59:47 -0700 Subject: [PATCH 14/39] package fetching: generate dependencies.zig file Only problem is that it looks like `has_build_zig` is being false when it should be true. After that is fixed then main.zig needs to create the `@dependencies` module from the generated source code. --- src/Package/Fetch.zig | 83 ++++++++++++++++++++++++++++++++++++------- src/main.zig | 19 ++++++---- 2 files changed, 84 insertions(+), 18 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 1c4c6ad09751..b688b24fe60a 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -61,6 +61,8 @@ oom_flag: bool, /// Contains shared state among all `Fetch` tasks. pub const JobQueue = struct { mutex: std.Thread.Mutex = .{}, + /// It's an array hash map so that it can be sorted before rendering the + /// dependencies.zig source file. /// Protected by `mutex`. table: Table = .{}, /// `table` may be missing some tasks such as ones that failed, so this @@ -75,7 +77,7 @@ pub const JobQueue = struct { recursive: bool, work_around_btrfs_bug: bool, - pub const Table = std.AutoHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch); + pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch); pub fn deinit(jq: *JobQueue) void { if (jq.all_fetches.items.len == 0) return; @@ -105,17 +107,74 @@ pub const JobQueue = struct { /// Creates the dependencies.zig file and corresponding `Module` for the /// build runner to obtain via `@import("@dependencies")`. - pub fn createDependenciesModule( - jq: *JobQueue, - arena: Allocator, - local_cache_directory: Cache.Directory, - basename: []const u8, - ) !*Package.Module { - _ = jq; - _ = arena; - _ = local_cache_directory; - _ = basename; - @panic("TODO: createDependenciesModule"); + pub fn createDependenciesModule(jq: *JobQueue, buf: *std.ArrayList(u8)) Allocator.Error!void { + try buf.appendSlice("pub const packages = struct {\n"); + + // Ensure the generated .zig file is deterministic. + jq.table.sortUnstable(@as(struct { + keys: []const Manifest.MultiHashHexDigest, + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return std.mem.lessThan(u8, &ctx.keys[a_index], &ctx.keys[b_index]); + } + }, .{ .keys = jq.table.keys() })); + + for (jq.table.keys()[1..], jq.table.values()[1..]) |hash, fetch| { + try buf.writer().print( + \\ pub const {} = struct {{ + \\ pub const build_root = "{q}"; + \\ + , .{ std.zig.fmtId(&hash), fetch.package_root }); + + if (fetch.has_build_zig) { + try buf.writer().print( + \\ pub const build_zig = @import("{}"); + \\ + , .{std.zig.fmtEscapes(&hash)}); + } + + if (fetch.manifest) |*manifest| { + try buf.appendSlice( + \\ pub const deps: []const struct { []const u8, []const u8 } = &.{ + \\ + ); + for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { + try buf.writer().print( + " .{{ \"{}\", \"{}\" }},\n", + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) }, + ); + } + + try buf.appendSlice( + \\ }; + \\ }; + \\ + ); + } else { + try buf.appendSlice( + \\ pub const deps: []const struct { []const u8, []const u8 } = &.{}; + \\ }; + \\ + ); + } + } + + try buf.appendSlice( + \\}; + \\ + \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ + \\ + ); + + const root_fetch = jq.all_fetches.items[0]; + const root_manifest = &root_fetch.manifest.?; + + for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| { + try buf.writer().print( + " .{{ \"{}\", \"{}\" }},\n", + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) }, + ); + } + try buf.appendSlice("};\n"); } }; diff --git a/src/main.zig b/src/main.zig index cd7cd80da435..1cdbd6124226 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4887,12 +4887,19 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi process.exit(1); } - const deps_mod = try job_queue.createDependenciesModule( - arena, - local_cache_directory, - "dependencies.zig", - ); - try main_mod.deps.put(arena, "@dependencies", deps_mod); + var buf = std.ArrayList(u8).init(gpa); + defer buf.deinit(); + try job_queue.createDependenciesModule(&buf); + if (true) { + std.debug.print("dependencies source:\n\n{s}\n", .{buf.items}); + @panic("TODO"); + } + //const deps_mod = try job_queue.createDependenciesModule( + // arena, + // local_cache_directory, + // "dependencies.zig", + //); + //try main_mod.deps.put(arena, "@dependencies", deps_mod); } try main_mod.deps.put(arena, "@build", &build_mod); From e86b7fcca3e522bede8bab06d669b75d0dca2038 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 7 Oct 2023 11:35:00 -0700 Subject: [PATCH 15/39] fix detection of build.zig file inside packages --- src/Package.zig | 10 ++++++++++ src/Package/Fetch.zig | 36 ++++++++++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/Package.zig b/src/Package.zig index cb1c64ae26e7..5de914e1146a 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -83,6 +83,16 @@ pub const Path = struct { return p.root_dir.handle.atomicFile(joined_path, options); } + pub fn access(p: Path, sub_path: []const u8, flags: fs.File.OpenFlags) !void { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const joined_path = if (p.sub_path.len == 0) sub_path else p: { + break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ + p.sub_path, sub_path, + }) catch return error.NameTooLong; + }; + return p.root_dir.handle.access(joined_path, flags); + } + pub fn format( self: Path, comptime fmt_string: []const u8, diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index b688b24fe60a..13ee477d4093 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -231,6 +231,7 @@ pub fn run(f: *Fetch) RunError!void { ); f.package_root = try f.parent_package_root.join(arena, sub_path); try loadManifest(f, f.package_root); + try checkBuildFileExistence(f); if (!f.job_queue.recursive) return; // Package hashes are used as unique identifiers for packages, so // we still need one for relative paths. @@ -279,17 +280,18 @@ pub fn run(f: *Fetch) RunError!void { if (cache_root.handle.access(pkg_sub_path, .{})) |_| { f.package_root = .{ .root_dir = cache_root, - .sub_path = pkg_sub_path, + .sub_path = try arena.dupe(u8, pkg_sub_path), }; try loadManifest(f, f.package_root); + try checkBuildFileExistence(f); if (!f.job_queue.recursive) return; return queueJobsForDeps(f, expected_hash); } else |err| switch (err) { error.FileNotFound => {}, else => |e| { try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to open global package cache directory '{s}': {s}", .{ - try cache_root.join(arena, &.{pkg_sub_path}), @errorName(e), + .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{ + cache_root, pkg_sub_path, @errorName(e), }), }); return error.FetchFailed; @@ -381,10 +383,13 @@ fn runResource( // by the system. This is done even if the hash is invalid, in case the // package with the different hash is used in the future. - const dest_pkg_sub_path = "p" ++ s ++ Manifest.hexDigest(f.actual_hash); - renameTmpIntoCache(cache_root.handle, tmp_dir_sub_path, dest_pkg_sub_path) catch |err| { + f.package_root = .{ + .root_dir = cache_root, + .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)), + }; + renameTmpIntoCache(cache_root.handle, tmp_dir_sub_path, f.package_root.sub_path) catch |err| { const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); - const dest = try cache_root.join(arena, &.{dest_pkg_sub_path}); + const dest = try cache_root.join(arena, &.{f.package_root.sub_path}); try eb.addRootErrorMessage(.{ .msg = try eb.printString( "unable to rename temporary directory '{s}' into package cache directory '{s}': {s}", .{ src, dest, @errorName(err) }, @@ -423,6 +428,25 @@ fn runResource( return queueJobsForDeps(f, actual_hex); } +/// `computeHash` gets a free check for the existence of `build.zig`, but when +/// not computing a hash, we need to do a syscall to check for it. +fn checkBuildFileExistence(f: *Fetch) RunError!void { + const eb = &f.error_bundle; + if (f.package_root.access(Package.build_zig_basename, .{})) |_| { + f.has_build_zig = true; + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to access '{}{s}': {s}", .{ + f.package_root, Package.build_zig_basename, @errorName(e), + }), + }); + return error.FetchFailed; + }, + } +} + /// This function populates `f.manifest` or leaves it `null`. fn loadManifest(f: *Fetch, pkg_root: Package.Path) RunError!void { const eb = &f.error_bundle; From a232f7e8ecc9f8afb210ffd483ca59556175a33c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 7 Oct 2023 14:37:25 -0700 Subject: [PATCH 16/39] finish implementing the auto-generated dependencies.zig module --- src/Package/Fetch.zig | 18 +++++++++++++-- src/main.zig | 53 ++++++++++++++++++++++++++++--------------- 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 13ee477d4093..7ea1fee733bc 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -108,6 +108,12 @@ pub const JobQueue = struct { /// Creates the dependencies.zig file and corresponding `Module` for the /// build runner to obtain via `@import("@dependencies")`. pub fn createDependenciesModule(jq: *JobQueue, buf: *std.ArrayList(u8)) Allocator.Error!void { + const keys = jq.table.keys(); + const values = jq.table.values(); + + if (keys.len == 0) + return createEmptyDependenciesModule(buf); + try buf.appendSlice("pub const packages = struct {\n"); // Ensure the generated .zig file is deterministic. @@ -116,9 +122,9 @@ pub const JobQueue = struct { pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { return std.mem.lessThan(u8, &ctx.keys[a_index], &ctx.keys[b_index]); } - }, .{ .keys = jq.table.keys() })); + }, .{ .keys = keys })); - for (jq.table.keys()[1..], jq.table.values()[1..]) |hash, fetch| { + for (keys[1..], values[1..]) |hash, fetch| { try buf.writer().print( \\ pub const {} = struct {{ \\ pub const build_root = "{q}"; @@ -176,6 +182,14 @@ pub const JobQueue = struct { } try buf.appendSlice("};\n"); } + + pub fn createEmptyDependenciesModule(buf: *std.ArrayList(u8)) Allocator.Error!void { + try buf.appendSlice( + \\pub const packages = struct {}; + \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; + \\ + ); + } }; pub const Location = union(enum) { diff --git a/src/main.zig b/src/main.zig index 1cdbd6124226..bf4c37781f50 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4827,13 +4827,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .root = .{ .root_dir = build_root }, .root_src_path = build_zig_basename, }; + var dependencies_zig_src = std.ArrayList(u8).init(gpa); + defer dependencies_zig_src.deinit(); if (build_options.only_core_functionality) { - const deps_mod = try Package.createFilePkg(gpa, local_cache_directory, "dependencies.zig", - \\pub const packages = struct {}; - \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; - \\ - ); - try main_mod.deps.put(arena, "@dependencies", deps_mod); + try Package.Fetch.createEmptyDependenciesModule(&dependencies_zig_src); } else { var http_client: std.http.Client = .{ .allocator = gpa }; defer http_client.deinit(); @@ -4887,19 +4884,39 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi process.exit(1); } - var buf = std.ArrayList(u8).init(gpa); - defer buf.deinit(); - try job_queue.createDependenciesModule(&buf); - if (true) { - std.debug.print("dependencies source:\n\n{s}\n", .{buf.items}); - @panic("TODO"); + try job_queue.createDependenciesModule(&dependencies_zig_src); + } + { + // Atomically create the file in a directory named after the hash of its contents. + const basename = "dependencies.zig"; + const rand_int = std.crypto.random.int(u64); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ Package.Manifest.hex64(rand_int); + { + var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); + defer tmp_dir.close(); + try tmp_dir.writeFile(basename, dependencies_zig_src.items); } - //const deps_mod = try job_queue.createDependenciesModule( - // arena, - // local_cache_directory, - // "dependencies.zig", - //); - //try main_mod.deps.put(arena, "@dependencies", deps_mod); + + var hh: Cache.HashHelper = .{}; + hh.addBytes(build_options.version); + hh.addBytes(dependencies_zig_src.items); + const hex_digest = hh.final(); + + const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest); + try Package.Fetch.renameTmpIntoCache( + local_cache_directory.handle, + tmp_dir_sub_path, + o_dir_sub_path, + ); + + const deps_mod = try Package.Module.create(arena, .{ + .root = .{ + .root_dir = local_cache_directory, + .sub_path = o_dir_sub_path, + }, + .root_src_path = basename, + }); + try main_mod.deps.put(arena, "@dependencies", deps_mod); } try main_mod.deps.put(arena, "@build", &build_mod); From 35d81c99c0f737f071e1ad0083342328dbb32acb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 7 Oct 2023 14:38:03 -0700 Subject: [PATCH 17/39] more fixes related to previous commits Package/Module API --- src/Sema.zig | 34 +++++++++++++++++++--------------- src/codegen/llvm.zig | 13 +++++-------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/Sema.zig b/src/Sema.zig index 4d49546cf7a4..5e428c9f7755 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -5732,6 +5732,8 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr const tracy = trace(@src()); defer tracy.end(); + const mod = sema.mod; + const gpa = sema.gpa; const pl_node = sema.code.instructions.items(.data)[inst].pl_node; const src = pl_node.src(); const extra = sema.code.extraData(Zir.Inst.Block, pl_node.payload_index); @@ -5741,7 +5743,7 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr if (!@import("build_options").have_llvm) return sema.fail(parent_block, src, "C import unavailable; Zig compiler built without LLVM extensions", .{}); - var c_import_buf = std.ArrayList(u8).init(sema.gpa); + var c_import_buf = std.ArrayList(u8).init(gpa); defer c_import_buf.deinit(); var comptime_reason: Block.ComptimeReason = .{ .c_import = .{ @@ -5763,25 +5765,24 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr .runtime_loop = parent_block.runtime_loop, .runtime_index = parent_block.runtime_index, }; - defer child_block.instructions.deinit(sema.gpa); + defer child_block.instructions.deinit(gpa); // Ignore the result, all the relevant operations have written to c_import_buf already. _ = try sema.analyzeBodyBreak(&child_block, body); - const mod = sema.mod; var c_import_res = mod.comp.cImport(c_import_buf.items) catch |err| return sema.fail(&child_block, src, "C import failed: {s}", .{@errorName(err)}); - defer c_import_res.deinit(mod.comp.gpa); + defer c_import_res.deinit(gpa); if (c_import_res.errors.errorMessageCount() != 0) { const msg = msg: { const msg = try sema.errMsg(&child_block, src, "C import failed", .{}); - errdefer msg.destroy(sema.gpa); + errdefer msg.destroy(gpa); if (!mod.comp.bin_file.options.link_libc) try sema.errNote(&child_block, src, msg, "libc headers not available; compilation does not link against libc", .{}); - const gop = try mod.cimport_errors.getOrPut(sema.gpa, sema.owner_decl_index); + const gop = try mod.cimport_errors.getOrPut(gpa, sema.owner_decl_index); if (!gop.found_existing) { gop.value_ptr.* = c_import_res.errors; c_import_res.errors = std.zig.ErrorBundle.empty; @@ -5790,16 +5791,19 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr }; return sema.failWithOwnedErrorMsg(&child_block, msg); } - const c_import_pkg = Package.create( - sema.gpa, - null, - c_import_res.out_zig_path, - ) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - else => unreachable, // we pass null for root_src_dir_path - }; + // All modules are intended to go into an arena with a lifetime >= the ZigUnit. + // After the other uses of `tmp_hack_arena` are eliminated, it should be + // renamed to something more appropriate such as simply `arena`. + const zu_arena = mod.tmp_hack_arena.allocator(); + const c_import_mod = try Package.Module.create(zu_arena, .{ + .root = .{ + .root_dir = Compilation.Directory.cwd(), + .sub_path = std.fs.path.dirname(c_import_res.out_zig_path) orelse "", + }, + .root_src_path = std.fs.path.basename(c_import_res.out_zig_path), + }); - const result = mod.importPkg(c_import_pkg) catch |err| + const result = mod.importPkg(c_import_mod) catch |err| return sema.fail(&child_block, src, "C import failed: {s}", .{@errorName(err)}); mod.astGenFile(result.file) catch |err| diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 3d15b5310be4..66b5bfbe0428 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -1836,14 +1836,11 @@ pub const Object = struct { } const dir_path_z = d: { var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined; - const dir_path = file.pkg.root_src_directory.path orelse "."; - const resolved_dir_path = if (std.fs.path.isAbsolute(dir_path)) - dir_path - else - std.os.realpath(dir_path, &buffer) catch dir_path; // If realpath fails, fallback to whatever dir_path was - break :d try std.fs.path.joinZ(gpa, &.{ - resolved_dir_path, std.fs.path.dirname(file.sub_file_path) orelse "", - }); + const sub_path = std.fs.path.dirname(file.sub_file_path) orelse ""; + const dir_path = try file.mod.root.joinStringZ(gpa, sub_path); + if (std.fs.path.isAbsolute(dir_path)) break :d dir_path; + const abs = std.fs.realpath(dir_path, &buffer) catch break :d dir_path; + break :d try std.fs.path.joinZ(gpa, &.{ abs, sub_path }); }; defer gpa.free(dir_path_z); const sub_file_path_z = try gpa.dupeZ(u8, std.fs.path.basename(file.sub_file_path)); From e5c2a7dbca103b0c61bebaff95c5d5a5b777bd59 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 7 Oct 2023 22:34:45 -0700 Subject: [PATCH 18/39] finish hooking up new dependency tree logic * add Module instances for each package's build.zig and attach it to the dependencies.zig module with the hash digest hex string as the name. * fix incorrectly skipping the wrong packages for creating dependencies.zig * a couple more renaming of "package" to "module" --- lib/std/Build/Cache.zig | 7 ++++ src/Module.zig | 6 ++-- src/Package.zig | 7 ++++ src/Package/Fetch.zig | 7 ++-- src/Sema.zig | 10 +++--- src/main.zig | 79 ++++++++++++++++++++++++++--------------- 6 files changed, 77 insertions(+), 39 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index bb36bb978fc9..1ad111f2e391 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -9,6 +9,13 @@ pub const Directory = struct { path: ?[]const u8, handle: fs.Dir, + pub fn clone(d: Directory, arena: Allocator) Allocator.Error!Directory { + return .{ + .path = if (d.path) |p| try arena.dupe(u8, p) else null, + .handle = d.handle, + }; + } + pub fn cwd() Directory { return .{ .path = null, diff --git a/src/Module.zig b/src/Module.zig index 153edeafefa1..41f4ec2b4101 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -4074,7 +4074,7 @@ pub fn importFile( return mod.importPkg(pkg); } if (!mem.endsWith(u8, import_string, ".zig")) { - return error.PackageNotFound; + return error.ModuleNotFound; } const gpa = mod.gpa; @@ -4120,7 +4120,7 @@ pub fn importFile( { break :p try gpa.dupe(u8, resolved_path); } - return error.ImportOutsidePkgPath; + return error.ImportOutsideModulePath; }; errdefer gpa.free(sub_file_path); @@ -4206,7 +4206,7 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb { break :p try gpa.dupe(u8, resolved_path); } - return error.ImportOutsidePkgPath; + return error.ImportOutsideModulePath; }; errdefer gpa.free(sub_file_path); diff --git a/src/Package.zig b/src/Package.zig index 5de914e1146a..84ba10bfa5f7 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -9,6 +9,13 @@ pub const Path = struct { /// Empty string means the root_dir is the path. sub_path: []const u8 = "", + pub fn clone(p: Path, arena: Allocator) Allocator.Error!Path { + return .{ + .root_dir = try p.root_dir.clone(arena), + .sub_path = try arena.dupe(u8, p.sub_path), + }; + } + pub fn cwd() Path { return .{ .root_dir = Cache.Directory.cwd() }; } diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 7ea1fee733bc..ace1871d42e2 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -109,7 +109,6 @@ pub const JobQueue = struct { /// build runner to obtain via `@import("@dependencies")`. pub fn createDependenciesModule(jq: *JobQueue, buf: *std.ArrayList(u8)) Allocator.Error!void { const keys = jq.table.keys(); - const values = jq.table.values(); if (keys.len == 0) return createEmptyDependenciesModule(buf); @@ -124,7 +123,11 @@ pub const JobQueue = struct { } }, .{ .keys = keys })); - for (keys[1..], values[1..]) |hash, fetch| { + for (keys, jq.table.values()) |hash, fetch| { + if (fetch == jq.all_fetches.items[0]) { + // The first one is a dummy package for the current project. + continue; + } try buf.writer().print( \\ pub const {} = struct {{ \\ pub const build_root = "{q}"; diff --git a/src/Sema.zig b/src/Sema.zig index 5e428c9f7755..2df8b8570e2a 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -13075,13 +13075,13 @@ fn zirImport(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air. const operand = inst_data.get(sema.code); const result = mod.importFile(block.getFileScope(mod), operand) catch |err| switch (err) { - error.ImportOutsidePkgPath => { - return sema.fail(block, operand_src, "import of file outside package path: '{s}'", .{operand}); + error.ImportOutsideModulePath => { + return sema.fail(block, operand_src, "import of file outside module path: '{s}'", .{operand}); }, - error.PackageNotFound => { + error.ModuleNotFound => { //const name = try block.getFileScope(mod).mod.getName(sema.gpa, mod.*); //defer sema.gpa.free(name); - return sema.fail(block, operand_src, "no package named '{s}' available within package '{}'", .{ + return sema.fail(block, operand_src, "no module named '{s}' available within module '{}'", .{ operand, block.getFileScope(mod).mod.root, }); }, @@ -13112,7 +13112,7 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A } const embed_file = mod.embedFile(block.getFileScope(mod), name) catch |err| switch (err) { - error.ImportOutsidePkgPath => { + error.ImportOutsideModulePath => { return sema.fail(block, operand_src, "embed of file outside package path: '{s}'", .{name}); }, else => { diff --git a/src/main.zig b/src/main.zig index bf4c37781f50..df327838079b 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4885,39 +4885,60 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi } try job_queue.createDependenciesModule(&dependencies_zig_src); - } - { - // Atomically create the file in a directory named after the hash of its contents. - const basename = "dependencies.zig"; - const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ Package.Manifest.hex64(rand_int); - { - var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); - defer tmp_dir.close(); - try tmp_dir.writeFile(basename, dependencies_zig_src.items); - } - - var hh: Cache.HashHelper = .{}; - hh.addBytes(build_options.version); - hh.addBytes(dependencies_zig_src.items); - const hex_digest = hh.final(); - const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest); - try Package.Fetch.renameTmpIntoCache( - local_cache_directory.handle, - tmp_dir_sub_path, - o_dir_sub_path, - ); + const deps_mod = m: { + // Atomically create the file in a directory named after the hash of its contents. + const basename = "dependencies.zig"; + const rand_int = std.crypto.random.int(u64); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ + Package.Manifest.hex64(rand_int); + { + var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); + defer tmp_dir.close(); + try tmp_dir.writeFile(basename, dependencies_zig_src.items); + } - const deps_mod = try Package.Module.create(arena, .{ - .root = .{ - .root_dir = local_cache_directory, - .sub_path = o_dir_sub_path, - }, - .root_src_path = basename, - }); + var hh: Cache.HashHelper = .{}; + hh.addBytes(build_options.version); + hh.addBytes(dependencies_zig_src.items); + const hex_digest = hh.final(); + + const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest); + try Package.Fetch.renameTmpIntoCache( + local_cache_directory.handle, + tmp_dir_sub_path, + o_dir_sub_path, + ); + + break :m try Package.Module.create(arena, .{ + .root = .{ + .root_dir = local_cache_directory, + .sub_path = o_dir_sub_path, + }, + .root_src_path = basename, + }); + }; + { + // We need a Module for each package's build.zig. + const hashes = job_queue.table.keys(); + const fetches = job_queue.table.values(); + try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); + for (hashes, fetches) |hash, f| { + if (f == &fetch) { + // The first one is a dummy package for the current project. + continue; + } + const m = try Package.Module.create(arena, .{ + .root = try f.package_root.clone(arena), + .root_src_path = if (f.has_build_zig) Package.build_zig_basename else "", + }); + const hash_cloned = try arena.dupe(u8, &hash); + deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); + } + } try main_mod.deps.put(arena, "@dependencies", deps_mod); } + try main_mod.deps.put(arena, "@build", &build_mod); const comp = Compilation.create(gpa, .{ From 7b25d050e64fd91ec7df17f29f3a1b84e1431241 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 7 Oct 2023 23:09:39 -0700 Subject: [PATCH 19/39] std.tar: fix creation of symlinks with omit_empty_directories --- lib/std/tar.zig | 52 ++++++++++++++++++++++++++++++++++++------- src/Package/Fetch.zig | 7 ++++++ 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/lib/std/tar.zig b/lib/std/tar.zig index 9ed69730aaaf..bdbec87e39be 100644 --- a/lib/std/tar.zig +++ b/lib/std/tar.zig @@ -31,6 +31,10 @@ pub const Options = struct { file_name: []const u8, link_name: []const u8, }, + unable_to_create_file: struct { + code: anyerror, + file_name: []const u8, + }, unsupported_file_type: struct { file_name: []const u8, file_type: Header.FileType, @@ -44,6 +48,9 @@ pub const Options = struct { d.allocator.free(info.file_name); d.allocator.free(info.link_name); }, + .unable_to_create_file => |info| { + d.allocator.free(info.file_name); + }, .unsupported_file_type => |info| { d.allocator.free(info.file_name); }, @@ -211,18 +218,34 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi if (file_size == 0 and unstripped_file_name.len == 0) return; const file_name = try stripComponents(unstripped_file_name, options.strip_components); - if (std.fs.path.dirname(file_name)) |dir_name| { - try dir.makePath(dir_name); - } - var file = try dir.createFile(file_name, .{}); - defer file.close(); + var file = dir.createFile(file_name, .{}) catch |err| switch (err) { + error.FileNotFound => again: { + const code = code: { + if (std.fs.path.dirname(file_name)) |dir_name| { + dir.makePath(dir_name) catch |code| break :code code; + break :again dir.createFile(file_name, .{}) catch |code| { + break :code code; + }; + } + break :code err; + }; + const d = options.diagnostics orelse return error.UnableToCreateFile; + try d.errors.append(d.allocator, .{ .unable_to_create_file = .{ + .code = code, + .file_name = try d.allocator.dupe(u8, file_name), + } }); + break :again null; + }, + else => |e| return e, + }; + defer if (file) |f| f.close(); var file_off: usize = 0; while (true) { const temp = try buffer.readChunk(reader, @intCast(rounded_file_size + 512 - file_off)); if (temp.len == 0) return error.UnexpectedEndOfStream; const slice = temp[0..@intCast(@min(file_size - file_off, temp.len))]; - try file.writeAll(slice); + if (file) |f| try f.writeAll(slice); file_off += slice.len; buffer.advance(slice.len); @@ -275,13 +298,26 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi }, .hard_link => return error.TarUnsupportedFileType, .symbolic_link => { + // The file system path of the symbolic link. const file_name = try stripComponents(unstripped_file_name, options.strip_components); + // The data inside the symbolic link. const link_name = header.linkName(); - dir.symLink(link_name, file_name, .{}) catch |err| { + dir.symLink(link_name, file_name, .{}) catch |err| again: { + const code = code: { + if (err == error.FileNotFound) { + if (std.fs.path.dirname(file_name)) |dir_name| { + dir.makePath(dir_name) catch |code| break :code code; + break :again dir.symLink(link_name, file_name, .{}) catch |code| { + break :code code; + }; + } + } + break :code err; + }; const d = options.diagnostics orelse return error.UnableToCreateSymLink; try d.errors.append(d.allocator, .{ .unable_to_create_sym_link = .{ - .code = err, + .code = code, .file_name = try d.allocator.dupe(u8, file_name), .link_name = try d.allocator.dupe(u8, link_name), } }); diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index ace1871d42e2..0cbfb973db73 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1041,6 +1041,13 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { }), })); }, + .unable_to_create_file => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("unable to create file '{s}': {s}", .{ + info.file_name, @errorName(info.code), + }), + })); + }, .unsupported_file_type => |info| { eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ From b7fc53c92a6edd10aa936a6fbe2d10d22c559c43 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 7 Oct 2023 23:29:35 -0700 Subject: [PATCH 20/39] dependencies.zig: omit modules without build.zig as deps --- src/main.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main.zig b/src/main.zig index df327838079b..fe6f08a54dc0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4928,9 +4928,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi // The first one is a dummy package for the current project. continue; } + if (!f.has_build_zig) + continue; const m = try Package.Module.create(arena, .{ .root = try f.package_root.clone(arena), - .root_src_path = if (f.has_build_zig) Package.build_zig_basename else "", + .root_src_path = Package.build_zig_basename, }); const hash_cloned = try arena.dupe(u8, &hash); deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); From b2fefc847341beb690e4cf49e2857fbacc16d329 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 7 Oct 2023 23:29:53 -0700 Subject: [PATCH 21/39] dependencies.zig: omit deps without hashes --- src/Package/Fetch.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 0cbfb973db73..1a0e6cb3dce1 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -147,9 +147,10 @@ pub const JobQueue = struct { \\ ); for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { + const h = dep.hash orelse continue; try buf.writer().print( " .{{ \"{}\", \"{}\" }},\n", - .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) }, + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h) }, ); } @@ -178,9 +179,10 @@ pub const JobQueue = struct { const root_manifest = &root_fetch.manifest.?; for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| { + const h = dep.hash orelse continue; try buf.writer().print( " .{{ \"{}\", \"{}\" }},\n", - .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) }, + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h) }, ); } try buf.appendSlice("};\n"); From a605bd9887a8a4220e177e42259809e0680d4221 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 7 Oct 2023 23:36:32 -0700 Subject: [PATCH 22/39] zig build: add `--fetch` argument closes #14280 --- lib/build_runner.zig | 1 + src/main.zig | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/lib/build_runner.zig b/lib/build_runner.zig index 45c1f41fd3ad..48c70646b166 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -997,6 +997,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi \\ -j Limit concurrent jobs (default is to use all CPU cores) \\ --maxrss Limit memory usage (default is to use available memory) \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss + \\ --fetch Exit after fetching dependency tree \\ \\Project-Specific Options: \\ diff --git a/src/main.zig b/src/main.zig index fe6f08a54dc0..8c13dacfa571 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4621,6 +4621,7 @@ pub const usage_build = \\ --global-cache-dir [path] Override path to global Zig cache directory \\ --zig-lib-dir [arg] Override path to Zig lib directory \\ --build-runner [file] Override path to build runner + \\ --fetch Exit after fetching dependency tree \\ -h, --help Print this help and exit \\ ; @@ -4643,6 +4644,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi var child_argv = std.ArrayList([]const u8).init(arena); var reference_trace: ?u32 = null; var debug_compile_errors = false; + var fetch_only = false; const argv_index_exe = child_argv.items.len; _ = try child_argv.addOne(); @@ -4692,6 +4694,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi } else if (mem.eql(u8, arg, "-freference-trace")) { try child_argv.append(arg); reference_trace = 256; + } else if (mem.eql(u8, arg, "--fetch")) { + fetch_only = true; } else if (mem.startsWith(u8, arg, "-freference-trace=")) { try child_argv.append(arg); const num = arg["-freference-trace=".len..]; @@ -4884,6 +4888,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi process.exit(1); } + if (fetch_only) return cleanExit(); + try job_queue.createDependenciesModule(&dependencies_zig_src); const deps_mod = m: { From 29156de12022732cb90b5bc5f5a0e33bf25d894b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 11:23:20 -0700 Subject: [PATCH 23/39] CLI: fix only_core_functionality logic --- src/Package/Fetch.zig | 10 ++--- src/main.zig | 99 +++++++++++++++++++++++++++---------------- 2 files changed, 67 insertions(+), 42 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 1a0e6cb3dce1..aa4601cb2525 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -105,13 +105,13 @@ pub const JobQueue = struct { } } - /// Creates the dependencies.zig file and corresponding `Module` for the - /// build runner to obtain via `@import("@dependencies")`. - pub fn createDependenciesModule(jq: *JobQueue, buf: *std.ArrayList(u8)) Allocator.Error!void { + /// Creates the dependencies.zig source code for the build runner to obtain + /// via `@import("@dependencies")`. + pub fn createDependenciesSource(jq: *JobQueue, buf: *std.ArrayList(u8)) Allocator.Error!void { const keys = jq.table.keys(); if (keys.len == 0) - return createEmptyDependenciesModule(buf); + return createEmptyDependenciesSource(buf); try buf.appendSlice("pub const packages = struct {\n"); @@ -188,7 +188,7 @@ pub const JobQueue = struct { try buf.appendSlice("};\n"); } - pub fn createEmptyDependenciesModule(buf: *std.ArrayList(u8)) Allocator.Error!void { + pub fn createEmptyDependenciesSource(buf: *std.ArrayList(u8)) Allocator.Error!void { try buf.appendSlice( \\pub const packages = struct {}; \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; diff --git a/src/main.zig b/src/main.zig index 8c13dacfa571..6c7469cc9393 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4831,10 +4831,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .root = .{ .root_dir = build_root }, .root_src_path = build_zig_basename, }; - var dependencies_zig_src = std.ArrayList(u8).init(gpa); - defer dependencies_zig_src.deinit(); if (build_options.only_core_functionality) { - try Package.Fetch.createEmptyDependenciesModule(&dependencies_zig_src); + try createEmptyDependenciesModule(arena, &main_mod, local_cache_directory); } else { var http_client: std.http.Client = .{ .allocator = gpa }; defer http_client.deinit(); @@ -4890,40 +4888,16 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi if (fetch_only) return cleanExit(); - try job_queue.createDependenciesModule(&dependencies_zig_src); - - const deps_mod = m: { - // Atomically create the file in a directory named after the hash of its contents. - const basename = "dependencies.zig"; - const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ - Package.Manifest.hex64(rand_int); - { - var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); - defer tmp_dir.close(); - try tmp_dir.writeFile(basename, dependencies_zig_src.items); - } + var source_buf = std.ArrayList(u8).init(gpa); + defer source_buf.deinit(); + try job_queue.createDependenciesSource(&source_buf); + const deps_mod = try createDependenciesModule( + arena, + source_buf.items, + &main_mod, + local_cache_directory, + ); - var hh: Cache.HashHelper = .{}; - hh.addBytes(build_options.version); - hh.addBytes(dependencies_zig_src.items); - const hex_digest = hh.final(); - - const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest); - try Package.Fetch.renameTmpIntoCache( - local_cache_directory.handle, - tmp_dir_sub_path, - o_dir_sub_path, - ); - - break :m try Package.Module.create(arena, .{ - .root = .{ - .root_dir = local_cache_directory, - .sub_path = o_dir_sub_path, - }, - .root_src_path = basename, - }); - }; { // We need a Module for each package's build.zig. const hashes = job_queue.table.keys(); @@ -4944,7 +4918,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); } } - try main_mod.deps.put(arena, "@dependencies", deps_mod); } try main_mod.deps.put(arena, "@build", &build_mod); @@ -6795,3 +6768,55 @@ fn cmdFetch( return cleanExit(); } + +fn createEmptyDependenciesModule( + arena: Allocator, + main_mod: *Package.Module, + local_cache_directory: Cache.Directory, +) !void { + var source = std.ArrayList(u8).init(arena); + try Package.Fetch.JobQueue.createEmptyDependenciesSource(&source); + _ = try createDependenciesModule(arena, source.items, main_mod, local_cache_directory); +} + +/// Creates the dependencies.zig file and corresponding `Package.Module` for the +/// build runner to obtain via `@import("@dependencies")`. +fn createDependenciesModule( + arena: Allocator, + source: []const u8, + main_mod: *Package.Module, + local_cache_directory: Cache.Directory, +) !*Package.Module { + // Atomically create the file in a directory named after the hash of its contents. + const basename = "dependencies.zig"; + const rand_int = std.crypto.random.int(u64); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ + Package.Manifest.hex64(rand_int); + { + var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); + defer tmp_dir.close(); + try tmp_dir.writeFile(basename, source); + } + + var hh: Cache.HashHelper = .{}; + hh.addBytes(build_options.version); + hh.addBytes(source); + const hex_digest = hh.final(); + + const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest); + try Package.Fetch.renameTmpIntoCache( + local_cache_directory.handle, + tmp_dir_sub_path, + o_dir_sub_path, + ); + + const deps_mod = try Package.Module.create(arena, .{ + .root = .{ + .root_dir = local_cache_directory, + .sub_path = o_dir_sub_path, + }, + .root_src_path = basename, + }); + try main_mod.deps.put(arena, "@dependencies", deps_mod); + return deps_mod; +} From 1fd95fc00519ee44de94fa5b73ca7a3285b92149 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 11:55:34 -0700 Subject: [PATCH 24/39] Package.Fetch: resolve instead of join relative paths This prevents bogus "error: file exists in multiple modules" errors due to file paths looking like: ``` note: root of module foo/freetype/ note: root of module foo/fontconfig/../freetype/ ``` It also enables checking for dependency paths outside the root package. --- src/Package.zig | 8 ++++++++ src/Package/Fetch.zig | 8 +++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Package.zig b/src/Package.zig index 84ba10bfa5f7..32533b2d04e8 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -30,6 +30,14 @@ pub const Path = struct { }; } + pub fn resolvePosix(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path { + if (sub_path.len == 0) return p; + return .{ + .root_dir = p.root_dir, + .sub_path = try fs.path.resolvePosix(arena, &.{ p.sub_path, sub_path }), + }; + } + pub fn joinString(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 { const parts: []const []const u8 = if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path }; diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index aa4601cb2525..3d4b58324c4e 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -248,7 +248,13 @@ pub fn run(f: *Fetch) RunError!void { f.hash_tok, try eb.addString("path-based dependencies are not hashed"), ); - f.package_root = try f.parent_package_root.join(arena, sub_path); + f.package_root = try f.parent_package_root.resolvePosix(arena, sub_path); + if (std.mem.startsWith(u8, f.package_root.sub_path, "../")) { + return f.fail( + f.location_tok, + try eb.addString("dependency path outside package"), + ); + } try loadManifest(f, f.package_root); try checkBuildFileExistence(f); if (!f.job_queue.recursive) return; From 47a413361dd6702dd0412ed58003e9ca9d8ba928 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 14:33:39 -0700 Subject: [PATCH 25/39] Package.Fetch: fix handling of relative paths --- lib/std/Build/Cache.zig | 2 +- src/Package/Fetch.zig | 81 +++++++++++++++++++++++------------------ src/main.zig | 8 +++- 3 files changed, 54 insertions(+), 37 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 1ad111f2e391..b1ceaa1b09be 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -61,7 +61,7 @@ pub const Directory = struct { writer: anytype, ) !void { _ = options; - if (fmt_string.len != 0) fmt.invalidFmtError(fmt, self); + if (fmt_string.len != 0) fmt.invalidFmtError(fmt_string, self); if (self.path) |p| { try writer.writeAll(p); try writer.writeAll(fs.path.sep_str); diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 3d4b58324c4e..9b4fc13f6f77 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -200,7 +200,7 @@ pub const JobQueue = struct { pub const Location = union(enum) { remote: Remote, /// A directory found inside the parent package. - relative_path: []const u8, + relative_path: Package.Path, /// Recursive Fetch tasks will never use this Location, but it may be /// passed in by the CLI. Indicates the file contents here should be copied /// into the global package cache. It may be a file relative to the cwd or @@ -239,8 +239,8 @@ pub fn run(f: *Fetch) RunError!void { // relative path, treat this the same as a cache hit. Otherwise, proceed. const remote = switch (f.location) { - .relative_path => |sub_path| { - if (fs.path.isAbsolute(sub_path)) return f.fail( + .relative_path => |pkg_root| { + if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail( f.location_tok, try eb.addString("expected path relative to build root; found absolute path"), ); @@ -248,31 +248,19 @@ pub fn run(f: *Fetch) RunError!void { f.hash_tok, try eb.addString("path-based dependencies are not hashed"), ); - f.package_root = try f.parent_package_root.resolvePosix(arena, sub_path); - if (std.mem.startsWith(u8, f.package_root.sub_path, "../")) { + if (std.mem.startsWith(u8, pkg_root.sub_path, "../")) { return f.fail( f.location_tok, - try eb.addString("dependency path outside package"), + try eb.printString("dependency path outside project: '{}{s}'", .{ + pkg_root.root_dir, pkg_root.sub_path, + }), ); } - try loadManifest(f, f.package_root); + f.package_root = pkg_root; + try loadManifest(f, pkg_root); try checkBuildFileExistence(f); if (!f.job_queue.recursive) return; - // Package hashes are used as unique identifiers for packages, so - // we still need one for relative paths. - const digest = h: { - var hasher = Manifest.Hash.init(.{}); - // This hash is a tuple of: - // * whether it relative to the global cache directory or to the root package - // * the relative file path from there to the build root of the package - hasher.update(if (f.package_root.root_dir.eql(cache_root)) - &package_hash_prefix_cached - else - &package_hash_prefix_project); - hasher.update(f.package_root.sub_path); - break :h hasher.finalResult(); - }; - return queueJobsForDeps(f, Manifest.hexDigest(digest)); + return queueJobsForDeps(f); }, .remote => |remote| remote, .path_or_url => |path_or_url| { @@ -310,7 +298,7 @@ pub fn run(f: *Fetch) RunError!void { try loadManifest(f, f.package_root); try checkBuildFileExistence(f); if (!f.job_queue.recursive) return; - return queueJobsForDeps(f, expected_hash); + return queueJobsForDeps(f); } else |err| switch (err) { error.FileNotFound => {}, else => |e| { @@ -450,7 +438,7 @@ fn runResource( // Spawn a new fetch job for each dependency in the manifest file. Use // a mutex and a hash map so that redundant jobs do not get queued up. if (!f.job_queue.recursive) return; - return queueJobsForDeps(f, actual_hex); + return queueJobsForDeps(f); } /// `computeHash` gets a free check for the existence of `build.zig`, but when @@ -534,27 +522,29 @@ fn loadManifest(f: *Fetch, pkg_root: Package.Path) RunError!void { } } -fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void { +fn queueJobsForDeps(f: *Fetch) RunError!void { assert(f.job_queue.recursive); // If the package does not have a build.zig.zon file then there are no dependencies. const manifest = f.manifest orelse return; const new_fetches = nf: { - const deps = manifest.dependencies.values(); + const parent_arena = f.arena.allocator(); const gpa = f.arena.child_allocator; + const cache_root = f.job_queue.global_cache; + const deps = manifest.dependencies.values(); // Grab the new tasks into a temporary buffer so we can unlock that mutex // as fast as possible. // This overallocates any fetches that get skipped by the `continue` in the // loop below. - const new_fetches = try f.arena.allocator().alloc(Fetch, deps.len); + const new_fetches = try parent_arena.alloc(Fetch, deps.len); var new_fetch_index: usize = 0; f.job_queue.mutex.lock(); defer f.job_queue.mutex.unlock(); try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); - try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len + 1)); + try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len)); // There are four cases here: // * Correct hash is provided by manifest. @@ -564,12 +554,8 @@ fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void // * Hash is not provided by manifest. // - Hash missing error emitted; `queueJobsForDeps` is not called. // * path-based location is used without a hash. - // - We need to add `hash` to the table now. - switch (f.location) { - .remote => assert(f.job_queue.table.get(hash) == f), - .relative_path => f.job_queue.table.putAssumeCapacityNoClobber(hash, f), - .path_or_url => unreachable, - } + // - Hash is added to the table based on the path alone before + // calling run(); no need to add it again. for (deps) |dep| { const new_fetch = &new_fetches[new_fetch_index]; @@ -586,7 +572,16 @@ fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void break :h multihash_digest; }, } }, - .path => |path| .{ .relative_path = path }, + .path => |rel_path| l: { + // This might produce an invalid path, which is checked for + // at the beginning of run(). + const new_root = try f.package_root.resolvePosix(parent_arena, rel_path); + const multihash_digest = relativePathDigest(new_root, cache_root); + const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest); + if (gop.found_existing) continue; + gop.value_ptr.* = new_fetch; + break :l .{ .relative_path = new_root }; + }, }; new_fetch_index += 1; f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); @@ -630,6 +625,22 @@ fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void } } +pub fn relativePathDigest( + pkg_root: Package.Path, + cache_root: Cache.Directory, +) Manifest.MultiHashHexDigest { + var hasher = Manifest.Hash.init(.{}); + // This hash is a tuple of: + // * whether it relative to the global cache directory or to the root package + // * the relative file path from there to the build root of the package + hasher.update(if (pkg_root.root_dir.eql(cache_root)) + &package_hash_prefix_cached + else + &package_hash_prefix_project); + hasher.update(pkg_root.sub_path); + return Manifest.hexDigest(hasher.finalResult()); +} + pub fn workerRun(f: *Fetch) void { defer f.job_queue.wait_group.finish(); run(f) catch |err| switch (err) { diff --git a/src/main.zig b/src/main.zig index 6c7469cc9393..5dc4b0c9c9ec 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4851,10 +4851,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi defer job_queue.deinit(); try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); + try job_queue.table.ensureUnusedCapacity(gpa, 1); var fetch: Package.Fetch = .{ .arena = std.heap.ArenaAllocator.init(gpa), - .location = .{ .relative_path = "" }, + .location = .{ .relative_path = build_mod.root }, .location_tok = 0, .hash_tok = 0, .parent_package_root = build_mod.root, @@ -4874,6 +4875,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi }; job_queue.all_fetches.appendAssumeCapacity(&fetch); + job_queue.table.putAssumeCapacityNoClobber( + Package.Fetch.relativePathDigest(build_mod.root, global_cache_directory), + &fetch, + ); + job_queue.wait_group.start(); try job_queue.thread_pool.spawn(Package.Fetch.workerRun, .{&fetch}); job_queue.wait_group.wait(); From ce052d89a85be02494b25ba734ff4483e846176c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 15:27:00 -0700 Subject: [PATCH 26/39] Manifest: fix not initializing token locations --- src/Manifest.zig | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Manifest.zig b/src/Manifest.zig index 17d5d93fb59b..7fce7c8bcb5a 100644 --- a/src/Manifest.zig +++ b/src/Manifest.zig @@ -7,13 +7,15 @@ pub const multihash_hex_digest_len = 2 * multihash_len; pub const MultiHashHexDigest = [multihash_hex_digest_len]u8; pub const Dependency = struct { - location: union(enum) { - url: []const u8, - path: []const u8, - }, + location: Location, location_tok: Ast.TokenIndex, hash: ?[]const u8, hash_tok: Ast.TokenIndex, + + pub const Location = union(enum) { + url: []const u8, + path: []const u8, + }; }; pub const ErrorMessage = struct { @@ -249,9 +251,9 @@ const Parse = struct { var dep: Dependency = .{ .location = undefined, - .location_tok = undefined, + .location_tok = 0, .hash = null, - .hash_tok = undefined, + .hash_tok = 0, }; var has_location = false; From 1ad33f53fea4109160ccfceec94efa2013f3b3b3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 16:08:08 -0700 Subject: [PATCH 27/39] give build.zig modules access to their dependencies' build.zig modules --- src/Package/Fetch.zig | 37 ++++++++++++++++++++++++++++++++----- src/main.zig | 26 +++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 9b4fc13f6f77..d241474cb9dc 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -58,6 +58,12 @@ has_build_zig: bool, /// Indicates whether the task aborted due to an out-of-memory condition. oom_flag: bool, +// This field is used by the CLI only, untouched by this file. + +/// The module for this `Fetch` tasks's package, which exposes `build.zig` as +/// the root source file. +module: ?*Package.Module, + /// Contains shared state among all `Fetch` tasks. pub const JobQueue = struct { mutex: std.Thread.Mutex = .{}, @@ -147,10 +153,10 @@ pub const JobQueue = struct { \\ ); for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { - const h = dep.hash orelse continue; + const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue; try buf.writer().print( " .{{ \"{}\", \"{}\" }},\n", - .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h) }, + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) }, ); } @@ -179,10 +185,10 @@ pub const JobQueue = struct { const root_manifest = &root_fetch.manifest.?; for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| { - const h = dep.hash orelse continue; + const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue; try buf.writer().print( " .{{ \"{}\", \"{}\" }},\n", - .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h) }, + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) }, ); } try buf.appendSlice("};\n"); @@ -258,7 +264,7 @@ pub fn run(f: *Fetch) RunError!void { } f.package_root = pkg_root; try loadManifest(f, pkg_root); - try checkBuildFileExistence(f); + if (!f.has_build_zig) try checkBuildFileExistence(f); if (!f.job_queue.recursive) return; return queueJobsForDeps(f); }, @@ -604,6 +610,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { .actual_hash = undefined, .has_build_zig = false, .oom_flag = false, + + .module = null, }; } @@ -1400,6 +1408,25 @@ const Filter = struct { } }; +pub fn depDigest( + pkg_root: Package.Path, + cache_root: Cache.Directory, + dep: Manifest.Dependency, +) ?Manifest.MultiHashHexDigest { + if (dep.hash) |h| return h[0..Manifest.multihash_hex_digest_len].*; + + switch (dep.location) { + .url => return null, + .path => |rel_path| { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&buf); + const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch + return null; + return relativePathDigest(new_root, cache_root); + }, + } +} + // These are random bytes. const package_hash_prefix_cached = [8]u8{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 }; const package_hash_prefix_project = [8]u8{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc }; diff --git a/src/main.zig b/src/main.zig index 5dc4b0c9c9ec..5a57b3c32d47 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4870,8 +4870,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .manifest = null, .manifest_ast = undefined, .actual_hash = undefined, - .has_build_zig = false, + .has_build_zig = true, .oom_flag = false, + + .module = &build_mod, }; job_queue.all_fetches.appendAssumeCapacity(&fetch); @@ -4922,6 +4924,26 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi }); const hash_cloned = try arena.dupe(u8, &hash); deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); + f.module = m; + } + + // Each build.zig module needs access to each of its + // dependencies' build.zig modules by name. + for (fetches) |f| { + const mod = f.module orelse continue; + const man = f.manifest orelse continue; + const dep_names = man.dependencies.keys(); + try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); + for (dep_names, man.dependencies.values()) |name, dep| { + const dep_digest = Package.Fetch.depDigest( + f.package_root, + global_cache_directory, + dep, + ) orelse continue; + const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue; + const name_cloned = try arena.dupe(u8, name); + mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); + } } } } @@ -6751,6 +6773,8 @@ fn cmdFetch( .actual_hash = undefined, .has_build_zig = false, .oom_flag = false, + + .module = null, }; defer fetch.deinit(); From f48ec4b8ee30f8bd42326e5b8df631a8d9234518 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 16:53:41 -0700 Subject: [PATCH 28/39] use long-lived arena for `@cImport`-generated Module --- src/Compilation.zig | 19 ++++++++----------- src/Sema.zig | 11 ++++------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 29689ce6b989..c2ce5f16f52b 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -41,8 +41,9 @@ const resinator = @import("resinator.zig"); /// General-purpose allocator. Used for both temporary and long-term storage. gpa: Allocator, -/// Arena-allocated memory used during initialization. Should be untouched until deinit. -arena_state: std.heap.ArenaAllocator.State, +/// Arena-allocated memory, mostly used during initialization. However, it can be used +/// for other things requiring the same lifetime as the `Compilation`. +arena: std.heap.ArenaAllocator, bin_file: *link.File, c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{}, win32_resource_table: if (build_options.only_core_functionality) void else std.AutoArrayHashMapUnmanaged(*Win32Resource, void) = @@ -124,7 +125,7 @@ cache_parent: *Cache, /// Path to own executable for invoking `zig clang`. self_exe_path: ?[]const u8, /// null means -fno-emit-bin. -/// This is mutable memory allocated into the Compilation-lifetime arena (`arena_state`) +/// This is mutable memory allocated into the Compilation-lifetime arena (`arena`) /// of exactly the correct size for "o/[digest]/[basename]". /// The basename is of the outputted binary file in case we don't know the directory yet. whole_bin_sub_path: ?[]u8, @@ -1661,7 +1662,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { errdefer bin_file.destroy(); comp.* = .{ .gpa = gpa, - .arena_state = arena_allocator.state, + .arena = arena_allocator, .zig_lib_directory = options.zig_lib_directory, .local_cache_directory = options.local_cache_directory, .global_cache_directory = options.global_cache_directory, @@ -1979,7 +1980,8 @@ pub fn destroy(self: *Compilation) void { if (self.owned_link_dir) |*dir| dir.close(); // This destroys `self`. - self.arena_state.promote(gpa).deinit(); + var arena_instance = self.arena; + arena_instance.deinit(); } pub fn clearMiscFailures(comp: *Compilation) void { @@ -3899,17 +3901,12 @@ pub fn obtainWin32ResourceCacheManifest(comp: *const Compilation) Cache.Manifest return man; } -test "cImport" { - _ = cImport; -} - pub const CImportResult = struct { out_zig_path: []u8, cache_hit: bool, errors: std.zig.ErrorBundle, pub fn deinit(result: *CImportResult, gpa: std.mem.Allocator) void { - gpa.free(result.out_zig_path); result.errors.deinit(gpa); } }; @@ -4054,7 +4051,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult { }; } - const out_zig_path = try comp.local_cache_directory.join(comp.gpa, &[_][]const u8{ + const out_zig_path = try comp.local_cache_directory.join(comp.arena.allocator(), &.{ "o", &digest, cimport_zig_basename, }); if (comp.verbose_cimport) { diff --git a/src/Sema.zig b/src/Sema.zig index 2df8b8570e2a..3cd9baed7844 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -5733,6 +5733,7 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr defer tracy.end(); const mod = sema.mod; + const comp = mod.comp; const gpa = sema.gpa; const pl_node = sema.code.instructions.items(.data)[inst].pl_node; const src = pl_node.src(); @@ -5770,7 +5771,7 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr // Ignore the result, all the relevant operations have written to c_import_buf already. _ = try sema.analyzeBodyBreak(&child_block, body); - var c_import_res = mod.comp.cImport(c_import_buf.items) catch |err| + var c_import_res = comp.cImport(c_import_buf.items) catch |err| return sema.fail(&child_block, src, "C import failed: {s}", .{@errorName(err)}); defer c_import_res.deinit(gpa); @@ -5779,7 +5780,7 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr const msg = try sema.errMsg(&child_block, src, "C import failed", .{}); errdefer msg.destroy(gpa); - if (!mod.comp.bin_file.options.link_libc) + if (!comp.bin_file.options.link_libc) try sema.errNote(&child_block, src, msg, "libc headers not available; compilation does not link against libc", .{}); const gop = try mod.cimport_errors.getOrPut(gpa, sema.owner_decl_index); @@ -5791,11 +5792,7 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr }; return sema.failWithOwnedErrorMsg(&child_block, msg); } - // All modules are intended to go into an arena with a lifetime >= the ZigUnit. - // After the other uses of `tmp_hack_arena` are eliminated, it should be - // renamed to something more appropriate such as simply `arena`. - const zu_arena = mod.tmp_hack_arena.allocator(); - const c_import_mod = try Package.Module.create(zu_arena, .{ + const c_import_mod = try Package.Module.create(comp.arena.allocator(), .{ .root = .{ .root_dir = Compilation.Directory.cwd(), .sub_path = std.fs.path.dirname(c_import_res.out_zig_path) orelse "", From f0cc6f17a00ac119568c6ff6a7ae50a6ced22240 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 17:21:08 -0700 Subject: [PATCH 29/39] Package.Fetch: fix not making directory paths for symlinks --- src/Package/Fetch.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index d241474cb9dc..ed82c0006a6f 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1181,7 +1181,13 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) anyer const link_name = try dir.dir.readLink(entry.path, &buf); // TODO: if this would create a symlink to outside // the destination directory, fail with an error instead. - try tmp_dir.symLink(link_name, entry.path, .{}); + tmp_dir.symLink(link_name, entry.path, .{}) catch |err| switch (err) { + error.FileNotFound => { + if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.makePath(dirname); + try tmp_dir.symLink(link_name, entry.path, .{}); + }, + else => |e| return e, + }; }, else => return error.IllegalFileTypeInPackage, } From 8f20ce74f4063351de790efc86a988ffcaf92990 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 17:21:18 -0700 Subject: [PATCH 30/39] zig fetch: require 'paths' field in the manifest --- src/main.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.zig b/src/main.zig index 5a57b3c32d47..2ec2f692ba40 100644 --- a/src/main.zig +++ b/src/main.zig @@ -6764,7 +6764,7 @@ fn cmdFetch( .prog_node = root_prog_node, .job_queue = &job_queue, .omit_missing_hash_error = true, - .allow_missing_paths_field = true, + .allow_missing_paths_field = false, .package_root = undefined, .error_bundle = undefined, From 7bae6d90648e6ef0782c7f5e8a72066742feacaf Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 17:26:02 -0700 Subject: [PATCH 31/39] fix dependencies.zig generation with no dependencies --- src/Package/Fetch.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index ed82c0006a6f..b57117ce9738 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -116,8 +116,11 @@ pub const JobQueue = struct { pub fn createDependenciesSource(jq: *JobQueue, buf: *std.ArrayList(u8)) Allocator.Error!void { const keys = jq.table.keys(); - if (keys.len == 0) + assert(keys.len != 0); // caller should have added the first one + if (keys.len == 1) { + // This is the first one. It must have no dependencies. return createEmptyDependenciesSource(buf); + } try buf.appendSlice("pub const packages = struct {\n"); From 6d84caf72771cf05997518ae2fa40a94de709de4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 17:29:55 -0700 Subject: [PATCH 32/39] move some package management related source files around --- src/Package.zig | 2 +- src/Package/Fetch.zig | 4 ++-- src/{ => Package/Fetch}/git.zig | 2 -- src/{ => Package/Fetch}/git/testdata/testrepo.idx | Bin src/{ => Package/Fetch}/git/testdata/testrepo.pack | Bin src/{ => Package}/Manifest.zig | 0 6 files changed, 3 insertions(+), 5 deletions(-) rename src/{ => Package/Fetch}/git.zig (99%) rename src/{ => Package/Fetch}/git/testdata/testrepo.idx (100%) rename src/{ => Package/Fetch}/git/testdata/testrepo.pack (100%) rename src/{ => Package}/Manifest.zig (100%) diff --git a/src/Package.zig b/src/Package.zig index 32533b2d04e8..c36830d5996e 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -1,7 +1,7 @@ pub const Module = @import("Package/Module.zig"); pub const Fetch = @import("Package/Fetch.zig"); pub const build_zig_basename = "build.zig"; -pub const Manifest = @import("Manifest.zig"); +pub const Manifest = @import("Package/Manifest.zig"); pub const Path = struct { root_dir: Cache.Directory, diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index b57117ce9738..373ab5be7844 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1449,9 +1449,9 @@ const Allocator = std.mem.Allocator; const Cache = std.Build.Cache; const ThreadPool = std.Thread.Pool; const WaitGroup = std.Thread.WaitGroup; -const Manifest = @import("../Manifest.zig"); const Fetch = @This(); const main = @import("../main.zig"); -const git = @import("../git.zig"); +const git = @import("Fetch/git.zig"); const Package = @import("../Package.zig"); +const Manifest = Package.Manifest; const ErrorBundle = std.zig.ErrorBundle; diff --git a/src/git.zig b/src/Package/Fetch/git.zig similarity index 99% rename from src/git.zig rename to src/Package/Fetch/git.zig index fd35c5c33e8a..af4317702d38 100644 --- a/src/git.zig +++ b/src/Package/Fetch/git.zig @@ -11,8 +11,6 @@ const Allocator = mem.Allocator; const Sha1 = std.crypto.hash.Sha1; const assert = std.debug.assert; -const ProgressReader = @import("Package.zig").ProgressReader; - pub const oid_length = Sha1.digest_length; pub const fmt_oid_length = 2 * oid_length; /// The ID of a Git object (an SHA-1 hash). diff --git a/src/git/testdata/testrepo.idx b/src/Package/Fetch/git/testdata/testrepo.idx similarity index 100% rename from src/git/testdata/testrepo.idx rename to src/Package/Fetch/git/testdata/testrepo.idx diff --git a/src/git/testdata/testrepo.pack b/src/Package/Fetch/git/testdata/testrepo.pack similarity index 100% rename from src/git/testdata/testrepo.pack rename to src/Package/Fetch/git/testdata/testrepo.pack diff --git a/src/Manifest.zig b/src/Package/Manifest.zig similarity index 100% rename from src/Manifest.zig rename to src/Package/Manifest.zig From 5e7c44a3211dc8b1bb0b6d3d4764f8fd0b8c665b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 17:42:49 -0700 Subject: [PATCH 33/39] Package.Fetch: tighten up check for path outside root --- src/Package/Fetch.zig | 4 +++- src/Package/Manifest.zig | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 373ab5be7844..8fe1627dbcff 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -257,7 +257,9 @@ pub fn run(f: *Fetch) RunError!void { f.hash_tok, try eb.addString("path-based dependencies are not hashed"), ); - if (std.mem.startsWith(u8, pkg_root.sub_path, "../")) { + if (std.mem.startsWith(u8, pkg_root.sub_path, "../") or + std.mem.eql(u8, pkg_root.sub_path, "..")) + { return f.fail( f.location_tok, try eb.printString("dependency path outside project: '{}{s}'", .{ diff --git a/src/Package/Manifest.zig b/src/Package/Manifest.zig index 7fce7c8bcb5a..de1870ea75a1 100644 --- a/src/Package/Manifest.zig +++ b/src/Package/Manifest.zig @@ -318,8 +318,7 @@ const Parse = struct { for (array_init.ast.elements) |elem_node| { const path_string = try parseString(p, elem_node); - const normalized = try std.fs.path.resolve(p.arena, &.{path_string}); - try p.paths.put(p.gpa, normalized, {}); + try p.paths.put(p.gpa, path_string, {}); } } From 1b372f187223c39eb32a54c96d50aff6ab08c9d6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 18:23:26 -0700 Subject: [PATCH 34/39] fix using the wrong allocator for modules --- src/Compilation.zig | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index c2ce5f16f52b..9f2b07c501a7 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1352,11 +1352,14 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { }); } else null; - try main_mod.deps.put(gpa, "builtin", builtin_mod); - try main_mod.deps.put(gpa, "root", root_mod); - try main_mod.deps.put(gpa, "std", std_mod); - if (compiler_rt_mod) |m| - try main_mod.deps.put(gpa, "compiler_rt", m); + { + try main_mod.deps.ensureUnusedCapacity(arena, 4); + main_mod.deps.putAssumeCapacity("builtin", builtin_mod); + main_mod.deps.putAssumeCapacity("root", root_mod); + main_mod.deps.putAssumeCapacity("std", std_mod); + if (compiler_rt_mod) |m| + main_mod.deps.putAssumeCapacity("compiler_rt", m); + } // Pre-open the directory handles for cached ZIR code so that it does not need // to redundantly happen for each AstGen operation. From efbfa8ef3ebdb172bf3774a54279951ebb29c4c9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 19:18:21 -0700 Subject: [PATCH 35/39] std.fs.path.resolve: add test cases for empty string Since I'm about to rely on this behavior. --- lib/std/fs/path.zig | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/std/fs/path.zig b/lib/std/fs/path.zig index 1f9632b9c1db..8457991f5994 100644 --- a/lib/std/fs/path.zig +++ b/lib/std/fs/path.zig @@ -728,15 +728,17 @@ pub fn resolvePosix(allocator: Allocator, paths: []const []const u8) Allocator.E } } -test "resolve" { +test resolve { try testResolveWindows(&[_][]const u8{ "a\\b\\c\\", "..\\..\\.." }, "."); try testResolveWindows(&[_][]const u8{"."}, "."); + try testResolveWindows(&[_][]const u8{""}, "."); try testResolvePosix(&[_][]const u8{ "a/b/c/", "../../.." }, "."); try testResolvePosix(&[_][]const u8{"."}, "."); + try testResolvePosix(&[_][]const u8{""}, "."); } -test "resolveWindows" { +test resolveWindows { try testResolveWindows( &[_][]const u8{ "Z:\\", "/usr/local", "lib\\zig\\std\\array_list.zig" }, "Z:\\usr\\local\\lib\\zig\\std\\array_list.zig", @@ -764,7 +766,7 @@ test "resolveWindows" { try testResolveWindows(&[_][]const u8{"a/b"}, "a\\b"); } -test "resolvePosix" { +test resolvePosix { try testResolvePosix(&.{ "/a/b", "c" }, "/a/b/c"); try testResolvePosix(&.{ "/a/b", "c", "//d", "e///" }, "/d/e"); try testResolvePosix(&.{ "/a/b/c", "..", "../" }, "/a"); From 4a2cf388440f6efadf9c7664b63c3949b25d6824 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 19:29:17 -0700 Subject: [PATCH 36/39] Package.Fetch: apply inclusion rules from build.zig.zon --- src/Package/Fetch.zig | 66 +++++++++++++++++++++++++++++++++++++--- src/Package/Manifest.zig | 5 ++- 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 8fe1627dbcff..e0cfc9b0a425 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1249,6 +1249,9 @@ fn computeHash( var all_files = std.ArrayList(*HashedFile).init(gpa); defer all_files.deinit(); + var all_deletions = std.ArrayList(*DeletedFile).init(gpa); + defer all_deletions.deinit(); + var walker = try @as(fs.IterableDir, .{ .dir = tmp_directory.handle }).walk(gpa); defer walker.deinit(); @@ -1267,10 +1270,25 @@ fn computeHash( ) }); return error.FetchFailed; }) |entry| { - _ = filter; // TODO: apply filter rules here + if (entry.kind == .directory) continue; + + if (!filter.includePath(entry.path)) { + // Delete instead of including in hash calculation. + const deleted_file = try arena.create(DeletedFile); + deleted_file.* = .{ + .fs_path = try arena.dupe(u8, entry.path), + .failure = undefined, // to be populated by the worker + }; + wait_group.start(); + try thread_pool.spawn(workerDeleteFile, .{ + tmp_directory.handle, deleted_file, &wait_group, + }); + try all_deletions.append(deleted_file); + continue; + } const kind: HashedFile.Kind = switch (entry.kind) { - .directory => continue, + .directory => unreachable, .file => .file, .sym_link => .sym_link, else => return f.fail(f.location_tok, try eb.printString( @@ -1295,7 +1313,6 @@ fn computeHash( try thread_pool.spawn(workerHashFile, .{ tmp_directory.handle, hashed_file, &wait_group, }); - try all_files.append(hashed_file); } } @@ -1315,6 +1332,17 @@ fn computeHash( }; hasher.update(&hashed_file.hash); } + for (all_deletions.items) |deleted_file| { + deleted_file.failure catch |err| { + any_failures = true; + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{ + deleted_file.fs_path, @errorName(err), + }), + }); + }; + } + if (any_failures) return error.FetchFailed; return hasher.finalResult(); } @@ -1324,6 +1352,11 @@ fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void { hashed_file.failure = hashFileFallible(dir, hashed_file); } +fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile, wg: *WaitGroup) void { + defer wg.finish(); + deleted_file.failure = deleteFileFallible(dir, deleted_file); +} + fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { var buf: [8000]u8 = undefined; var hasher = Manifest.Hash.init(.{}); @@ -1347,6 +1380,20 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void hasher.final(&hashed_file.hash); } +fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void { + try dir.deleteFile(deleted_file.fs_path); + // In case the file was the last remaining file in the parent directory, attempt to + // remove the parent directory. + var opt_parent = fs.path.dirname(deleted_file.fs_path); + while (opt_parent) |parent| : (opt_parent = fs.path.dirname(parent)) { + dir.deleteDir(parent) catch |err| switch (err) { + error.DirNotEmpty => return, + error.FileNotFound => return, + else => |e| return e, + }; + } +} + fn isExecutable(file: fs.File) !bool { if (builtin.os.tag == .windows) { // TODO check the ACL on Windows. @@ -1360,6 +1407,15 @@ fn isExecutable(file: fs.File) !bool { } } +const DeletedFile = struct { + fs_path: []const u8, + failure: Error!void, + + const Error = + fs.Dir.DeleteFileError || + fs.Dir.DeleteDirError; +}; + const HashedFile = struct { fs_path: []const u8, normalized_path: []const u8, @@ -1402,7 +1458,7 @@ fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 { const Filter = struct { include_paths: std.StringArrayHashMapUnmanaged(void) = .{}, - /// sub_path is relative to the tarball root. + /// sub_path is relative to the package root. pub fn includePath(self: Filter, sub_path: []const u8) bool { if (self.include_paths.count() == 0) return true; if (self.include_paths.contains("")) return true; @@ -1410,7 +1466,7 @@ const Filter = struct { // Check if any included paths are parent directories of sub_path. var dirname = sub_path; - while (std.fs.path.dirname(sub_path)) |next_dirname| { + while (std.fs.path.dirname(dirname)) |next_dirname| { if (self.include_paths.contains(sub_path)) return true; dirname = next_dirname; } diff --git a/src/Package/Manifest.zig b/src/Package/Manifest.zig index de1870ea75a1..c1b1cdfb4f3c 100644 --- a/src/Package/Manifest.zig +++ b/src/Package/Manifest.zig @@ -318,7 +318,10 @@ const Parse = struct { for (array_init.ast.elements) |elem_node| { const path_string = try parseString(p, elem_node); - try p.paths.put(p.gpa, path_string, {}); + // This is normalized so that it can be used in string comparisons + // against file system paths. + const normalized = try std.fs.path.resolve(p.arena, &.{path_string}); + try p.paths.put(p.gpa, normalized, {}); } } From cd4397783f7ae86de867bed3346ea10547a94ee1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 20:03:40 -0700 Subject: [PATCH 37/39] Package.Fetch: improved deletion algorithm Instead of every file deletion being followed by a recursive attempt to remove the parent directory, while walking the directory, every directory that will have nonzero files deleted from it is tracked in an array hash map. After all the threads have finished deleting and hashing, the parent thread sorts the "suspicious" directories by length, descending, ensuring that children appear before parents, and then iterates over the array hash map, attempting a rmdir operation on each. Any rmdir that succeeds appends the parent directory to the map so that it will be removed if empty. --- src/Package/Fetch.zig | 63 ++++++++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index e0cfc9b0a425..16ff2876f825 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1249,8 +1249,13 @@ fn computeHash( var all_files = std.ArrayList(*HashedFile).init(gpa); defer all_files.deinit(); - var all_deletions = std.ArrayList(*DeletedFile).init(gpa); - defer all_deletions.deinit(); + var deleted_files = std.ArrayList(*DeletedFile).init(gpa); + defer deleted_files.deinit(); + + // Track directories which had any files deleted from them so that empty directories + // can be deleted. + var sus_dirs: std.StringArrayHashMapUnmanaged(void) = .{}; + defer sus_dirs.deinit(gpa); var walker = try @as(fs.IterableDir, .{ .dir = tmp_directory.handle }).walk(gpa); defer walker.deinit(); @@ -1274,16 +1279,22 @@ fn computeHash( if (!filter.includePath(entry.path)) { // Delete instead of including in hash calculation. + const fs_path = try arena.dupe(u8, entry.path); + + // Also track the parent directory in case it becomes empty. + if (fs.path.dirname(fs_path)) |parent| + try sus_dirs.put(gpa, parent, {}); + const deleted_file = try arena.create(DeletedFile); deleted_file.* = .{ - .fs_path = try arena.dupe(u8, entry.path), + .fs_path = fs_path, .failure = undefined, // to be populated by the worker }; wait_group.start(); try thread_pool.spawn(workerDeleteFile, .{ tmp_directory.handle, deleted_file, &wait_group, }); - try all_deletions.append(deleted_file); + try deleted_files.append(deleted_file); continue; } @@ -1300,8 +1311,8 @@ fn computeHash( if (std.mem.eql(u8, entry.path, Package.build_zig_basename)) f.has_build_zig = true; - const hashed_file = try arena.create(HashedFile); const fs_path = try arena.dupe(u8, entry.path); + const hashed_file = try arena.create(HashedFile); hashed_file.* = .{ .fs_path = fs_path, .normalized_path = try normalizePath(arena, fs_path), @@ -1317,6 +1328,36 @@ fn computeHash( } } + { + // Sort by length, descending, so that child directories get removed first. + sus_dirs.sortUnstable(@as(struct { + keys: []const []const u8, + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return ctx.keys[b_index].len < ctx.keys[a_index].len; + } + }, .{ .keys = sus_dirs.keys() })); + + // During this loop, more entries will be added, so we must loop by index. + var i: usize = 0; + while (i < sus_dirs.count()) : (i += 1) { + const sus_dir = sus_dirs.keys()[i]; + tmp_directory.handle.deleteDir(sus_dir) catch |err| switch (err) { + error.DirNotEmpty => continue, + error.FileNotFound => continue, + else => |e| { + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to delete empty directory '{s}': {s}", + .{ sus_dir, @errorName(e) }, + ) }); + return error.FetchFailed; + }, + }; + if (fs.path.dirname(sus_dir)) |parent| { + try sus_dirs.put(gpa, parent, {}); + } + } + } + std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); var hasher = Manifest.Hash.init(.{}); @@ -1332,7 +1373,7 @@ fn computeHash( }; hasher.update(&hashed_file.hash); } - for (all_deletions.items) |deleted_file| { + for (deleted_files.items) |deleted_file| { deleted_file.failure catch |err| { any_failures = true; try eb.addRootErrorMessage(.{ @@ -1382,16 +1423,6 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void { try dir.deleteFile(deleted_file.fs_path); - // In case the file was the last remaining file in the parent directory, attempt to - // remove the parent directory. - var opt_parent = fs.path.dirname(deleted_file.fs_path); - while (opt_parent) |parent| : (opt_parent = fs.path.dirname(parent)) { - dir.deleteDir(parent) catch |err| switch (err) { - error.DirNotEmpty => return, - error.FileNotFound => return, - else => |e| return e, - }; - } } fn isExecutable(file: fs.File) !bool { From 5eb5d523b50e8e8912cf84bf021dc4c95e521a7a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 20:58:04 -0700 Subject: [PATCH 38/39] give modules friendly names for error reporting --- src/Compilation.zig | 18 ++++++------ src/Package/Module.zig | 2 ++ src/Sema.zig | 7 ++--- src/main.zig | 28 ++++++++++++++----- .../import_of_missing_package.zig | 2 +- .../compile_errors/import_outside_package.zig | 2 +- .../import_outside_package_path.zig | 2 +- test/compile_errors.zig | 2 +- 8 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 9f2b07c501a7..cd4c6ea11b68 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1290,6 +1290,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const builtin_mod = try Package.Module.create(arena, .{ .root = .{ .root_dir = zig_cache_artifact_directory }, .root_src_path = "builtin.zig", + .fully_qualified_name = "builtin", }); // When you're testing std, the main module is std. In that case, @@ -1319,6 +1320,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .sub_path = "std", }, .root_src_path = "std.zig", + .fully_qualified_name = "std", }); const root_mod = if (options.is_test) root_mod: { @@ -1329,6 +1331,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .sub_path = std.fs.path.dirname(test_runner) orelse "", }, .root_src_path = std.fs.path.basename(test_runner), + .fully_qualified_name = "root", }); pkg.deps = try main_mod.deps.clone(arena); @@ -1338,6 +1341,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .root_dir = options.zig_lib_directory, }, .root_src_path = "test_runner.zig", + .fully_qualified_name = "root", }); break :root_mod test_mod; @@ -1349,6 +1353,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .root_dir = options.zig_lib_directory, }, .root_src_path = "compiler_rt.zig", + .fully_qualified_name = "compiler_rt", }); } else null; @@ -2616,23 +2621,19 @@ fn reportMultiModuleErrors(mod: *Module) !void { errdefer for (notes[0..i]) |*n| n.deinit(mod.gpa); note.* = switch (ref) { .import => |loc| blk: { - //const name = try loc.file_scope.mod.getName(mod.gpa, mod.*); - //defer mod.gpa.free(name); break :blk try Module.ErrorMsg.init( mod.gpa, loc, - "imported from module {}", - .{loc.file_scope.mod.root}, + "imported from module {s}", + .{loc.file_scope.mod.fully_qualified_name}, ); }, .root => |pkg| blk: { - //const name = try pkg.getName(mod.gpa, mod.*); - //defer mod.gpa.free(name); break :blk try Module.ErrorMsg.init( mod.gpa, .{ .file_scope = file, .parent_decl_node = 0, .lazy = .entire_file }, - "root of module {}", - .{pkg.root}, + "root of module {s}", + .{pkg.fully_qualified_name}, ); }, }; @@ -6362,6 +6363,7 @@ fn buildOutputFromZig( var main_mod: Package.Module = .{ .root = .{ .root_dir = comp.zig_lib_directory }, .root_src_path = src_basename, + .fully_qualified_name = "root", }; const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len]; const target = comp.getTarget(); diff --git a/src/Package/Module.zig b/src/Package/Module.zig index 3502def385f8..7e6b518892b4 100644 --- a/src/Package/Module.zig +++ b/src/Package/Module.zig @@ -6,6 +6,8 @@ root: Package.Path, /// Relative to `root`. May contain path separators. root_src_path: []const u8, +/// Name used in compile errors. Looks like "root.foo.bar". +fully_qualified_name: []const u8, /// The dependency table of this module. Shared dependencies such as 'std', /// 'builtin', and 'root' are not specified in every dependency table, but /// instead only in the table of `main_mod`. `Module.importFile` is diff --git a/src/Sema.zig b/src/Sema.zig index 3cd9baed7844..a7073a5f36c3 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -5798,6 +5798,7 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr .sub_path = std.fs.path.dirname(c_import_res.out_zig_path) orelse "", }, .root_src_path = std.fs.path.basename(c_import_res.out_zig_path), + .fully_qualified_name = c_import_res.out_zig_path, }); const result = mod.importPkg(c_import_mod) catch |err| @@ -13076,10 +13077,8 @@ fn zirImport(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air. return sema.fail(block, operand_src, "import of file outside module path: '{s}'", .{operand}); }, error.ModuleNotFound => { - //const name = try block.getFileScope(mod).mod.getName(sema.gpa, mod.*); - //defer sema.gpa.free(name); - return sema.fail(block, operand_src, "no module named '{s}' available within module '{}'", .{ - operand, block.getFileScope(mod).mod.root, + return sema.fail(block, operand_src, "no module named '{s}' available within module {s}", .{ + operand, block.getFileScope(mod).mod.fully_qualified_name, }); }, else => { diff --git a/src/main.zig b/src/main.zig index 2ec2f692ba40..f0c03a69cee3 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1022,13 +1022,10 @@ fn buildOutputType( } } - var mod_it = modules.iterator(); - while (mod_it.next()) |kv| { - if (std.mem.eql(u8, mod_name, kv.key_ptr.*)) { - fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{ - mod_name, root_src, kv.value_ptr.mod.root_src_path, - }); - } + if (modules.get(mod_name)) |value| { + fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{ + mod_name, root_src, value.mod.root_src_path, + }); } try modules.put(mod_name, .{ @@ -1038,6 +1035,7 @@ fn buildOutputType( .sub_path = fs.path.dirname(root_src) orelse "", }, .root_src_path = fs.path.basename(root_src), + .fully_qualified_name = mod_name, }), .deps_str = deps_str, }); @@ -3247,6 +3245,7 @@ fn buildOutputType( src_path else try fs.path.relative(arena, p, src_path), + .fully_qualified_name = "root", }); } else { break :blk try Package.Module.create(arena, .{ @@ -3255,6 +3254,7 @@ fn buildOutputType( .sub_path = fs.path.dirname(src_path) orelse "", }, .root_src_path = fs.path.basename(src_path), + .fully_qualified_name = "root", }); } } else null; @@ -4820,16 +4820,19 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .sub_path = fs.path.dirname(build_runner_path) orelse "", }, .root_src_path = fs.path.basename(build_runner_path), + .fully_qualified_name = "root", } else .{ .root = .{ .root_dir = zig_lib_directory }, .root_src_path = "build_runner.zig", + .fully_qualified_name = "root", }; var build_mod: Package.Module = .{ .root = .{ .root_dir = build_root }, .root_src_path = build_zig_basename, + .fully_qualified_name = "root.@build", }; if (build_options.only_core_functionality) { try createEmptyDependenciesModule(arena, &main_mod, local_cache_directory); @@ -4921,6 +4924,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi const m = try Package.Module.create(arena, .{ .root = try f.package_root.clone(arena), .root_src_path = Package.build_zig_basename, + .fully_qualified_name = try std.fmt.allocPrint( + arena, + "root.@dependencies.{s}", + .{&hash}, + ), }); const hash_cloned = try arena.dupe(u8, &hash); deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); @@ -5181,6 +5189,7 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void file.mod = try Package.Module.create(arena, .{ .root = Package.Path.cwd(), .root_src_path = file.sub_file_path, + .fully_qualified_name = "root", }); file.zir = try AstGen.generate(gpa, file.tree); @@ -5389,6 +5398,7 @@ fn fmtPathFile( file.mod = try Package.Module.create(fmt.arena, .{ .root = Package.Path.cwd(), .root_src_path = file.sub_file_path, + .fully_qualified_name = "root", }); if (stat.size > max_src_size) @@ -5472,6 +5482,7 @@ pub fn putAstErrorsIntoBundle( file.mod = try Package.Module.create(gpa, .{ .root = Package.Path.cwd(), .root_src_path = file.sub_file_path, + .fully_qualified_name = "root", }); defer gpa.destroy(file.mod); @@ -6040,6 +6051,7 @@ pub fn cmdAstCheck( file.mod = try Package.Module.create(arena, .{ .root = Package.Path.cwd(), .root_src_path = file.sub_file_path, + .fully_qualified_name = "root", }); file.tree = try Ast.parse(gpa, file.source, .zig); @@ -6211,6 +6223,7 @@ pub fn cmdChangelist( file.mod = try Package.Module.create(arena, .{ .root = Package.Path.cwd(), .root_src_path = file.sub_file_path, + .fully_qualified_name = "root", }); const source = try arena.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0); @@ -6846,6 +6859,7 @@ fn createDependenciesModule( .sub_path = o_dir_sub_path, }, .root_src_path = basename, + .fully_qualified_name = "root.@dependencies", }); try main_mod.deps.put(arena, "@dependencies", deps_mod); return deps_mod; diff --git a/test/cases/compile_errors/import_of_missing_package.zig b/test/cases/compile_errors/import_of_missing_package.zig index d2a64638e871..f2cde5773418 100644 --- a/test/cases/compile_errors/import_of_missing_package.zig +++ b/test/cases/compile_errors/import_of_missing_package.zig @@ -7,4 +7,4 @@ comptime { // backend=stage2 // target=native // -// :1:21: error: no package named 'foo' available within package 'root' +// :1:21: error: no module named 'foo' available within module root diff --git a/test/cases/compile_errors/import_outside_package.zig b/test/cases/compile_errors/import_outside_package.zig index 59a75e2745a2..2a6ead5b3c48 100644 --- a/test/cases/compile_errors/import_outside_package.zig +++ b/test/cases/compile_errors/import_outside_package.zig @@ -5,4 +5,4 @@ export fn a() usize { // error // target=native // -// :2:20: error: import of file outside package path: '../../above.zig' +// :2:20: error: import of file outside module path: '../../above.zig' diff --git a/test/cases/compile_errors/import_outside_package_path.zig b/test/cases/compile_errors/import_outside_package_path.zig index 34044e3b0fba..e3d2d228a4f0 100644 --- a/test/cases/compile_errors/import_outside_package_path.zig +++ b/test/cases/compile_errors/import_outside_package_path.zig @@ -6,4 +6,4 @@ comptime { // backend=stage2 // target=native // -// :2:17: error: import of file outside package path: '../a.zig' +// :2:17: error: import of file outside module path: '../a.zig' diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 5f787683c4ba..a3bdf5b8f141 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -129,7 +129,7 @@ pub fn addCases(ctx: *Cases) !void { \\} , &[_][]const u8{ ":1:1: error: file exists in multiple modules", - ":1:1: note: root of module root.foo", + ":1:1: note: root of module foo", ":3:17: note: imported from module root", }); case.addSourceFile("foo.zig", From 95907cb79578779108f3772cb93648d38354b9ec Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 21:14:30 -0700 Subject: [PATCH 39/39] restore progress reporting for package fetching --- src/Package/Fetch.zig | 23 +++++++++++++++++------ src/main.zig | 2 +- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 16ff2876f825..3aae5a533fc3 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -539,16 +539,18 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { // If the package does not have a build.zig.zon file then there are no dependencies. const manifest = f.manifest orelse return; - const new_fetches = nf: { + const new_fetches, const prog_names = nf: { const parent_arena = f.arena.allocator(); const gpa = f.arena.child_allocator; const cache_root = f.job_queue.global_cache; + const dep_names = manifest.dependencies.keys(); const deps = manifest.dependencies.values(); // Grab the new tasks into a temporary buffer so we can unlock that mutex // as fast as possible. // This overallocates any fetches that get skipped by the `continue` in the // loop below. const new_fetches = try parent_arena.alloc(Fetch, deps.len); + const prog_names = try parent_arena.alloc([]const u8, deps.len); var new_fetch_index: usize = 0; f.job_queue.mutex.lock(); @@ -568,7 +570,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { // - Hash is added to the table based on the path alone before // calling run(); no need to add it again. - for (deps) |dep| { + for (dep_names, deps) |dep_name, dep| { const new_fetch = &new_fetches[new_fetch_index]; const location: Location = switch (dep.location) { .url => |url| .{ .remote = .{ @@ -594,6 +596,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { break :l .{ .relative_path = new_root }; }, }; + prog_names[new_fetch_index] = dep_name; new_fetch_index += 1; f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); new_fetch.* = .{ @@ -620,15 +623,18 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { }; } - break :nf new_fetches[0..new_fetch_index]; + // job_queue mutex is locked so this is OK. + f.prog_node.unprotected_estimated_total_items += new_fetch_index; + + break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] }; }; // Now it's time to give tasks to the thread pool. const thread_pool = f.job_queue.thread_pool; - for (new_fetches) |*new_fetch| { + for (new_fetches, prog_names) |*new_fetch, prog_name| { f.job_queue.wait_group.start(); - thread_pool.spawn(workerRun, .{new_fetch}) catch |err| switch (err) { + thread_pool.spawn(workerRun, .{ new_fetch, prog_name }) catch |err| switch (err) { error.OutOfMemory => { new_fetch.oom_flag = true; f.job_queue.wait_group.finish(); @@ -654,8 +660,13 @@ pub fn relativePathDigest( return Manifest.hexDigest(hasher.finalResult()); } -pub fn workerRun(f: *Fetch) void { +pub fn workerRun(f: *Fetch, prog_name: []const u8) void { defer f.job_queue.wait_group.finish(); + + var prog_node = f.prog_node.start(prog_name, 0); + defer prog_node.end(); + prog_node.activate(); + run(f) catch |err| switch (err) { error.OutOfMemory => f.oom_flag = true, error.FetchFailed => { diff --git a/src/main.zig b/src/main.zig index f0c03a69cee3..9199fe205b42 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4886,7 +4886,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi ); job_queue.wait_group.start(); - try job_queue.thread_pool.spawn(Package.Fetch.workerRun, .{&fetch}); + try job_queue.thread_pool.spawn(Package.Fetch.workerRun, .{ &fetch, "root" }); job_queue.wait_group.wait(); try job_queue.consolidateErrors();