Skip to content

package manager: write deps in a flat format, eliminating the FQN concept #17164

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 15, 2023
Merged

package manager: write deps in a flat format, eliminating the FQN concept #17164

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions lib/build_runner.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ pub fn main() !void {
global_cache_directory,
host,
&cache,
dependencies.root_deps,
);
defer builder.destroy();

Expand Down
41 changes: 26 additions & 15 deletions lib/std/Build.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,10 @@ modules: std.StringArrayHashMap(*Module),
/// A map from build root dirs to the corresponding `*Dependency`. This is shared with all child
/// `Build`s.
initialized_deps: *InitializedDepMap,
/// A mapping from dependency names to package hashes.
available_deps: AvailableDeps,

const AvailableDeps = []const struct { []const u8, []const u8 };

const InitializedDepMap = std.HashMap(InitializedDepKey, *Dependency, InitializedDepContext, std.hash_map.default_max_load_percentage);
const InitializedDepKey = struct {
Expand Down Expand Up @@ -248,6 +252,7 @@ pub fn create(
global_cache_root: Cache.Directory,
host: NativeTargetInfo,
cache: *Cache,
available_deps: AvailableDeps,
) !*Build {
const env_map = try allocator.create(EnvMap);
env_map.* = try process.getEnvMap(allocator);
Expand Down Expand Up @@ -308,6 +313,7 @@ pub fn create(
.host = host,
.modules = std.StringArrayHashMap(*Module).init(allocator),
.initialized_deps = initialized_deps,
.available_deps = available_deps,
};
try self.top_level_steps.put(allocator, self.install_tls.step.name, &self.install_tls);
try self.top_level_steps.put(allocator, self.uninstall_tls.step.name, &self.uninstall_tls);
Expand All @@ -319,14 +325,15 @@ fn createChild(
parent: *Build,
dep_name: []const u8,
build_root: Cache.Directory,
pkg_deps: AvailableDeps,
user_input_options: UserInputOptionsMap,
) !*Build {
const child = try createChildOnly(parent, dep_name, build_root, user_input_options);
const child = try createChildOnly(parent, dep_name, build_root, pkg_deps, user_input_options);
try determineAndApplyInstallPrefix(child);
return child;
}

fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory, user_input_options: UserInputOptionsMap) !*Build {
fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory, pkg_deps: AvailableDeps, user_input_options: UserInputOptionsMap) !*Build {
const allocator = parent.allocator;
const child = try allocator.create(Build);
child.* = .{
Expand Down Expand Up @@ -393,6 +400,7 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Direc
.dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }),
.modules = std.StringArrayHashMap(*Module).init(allocator),
.initialized_deps = parent.initialized_deps,
.available_deps = pkg_deps,
};
try child.top_level_steps.put(allocator, child.install_tls.step.name, &child.install_tls);
try child.top_level_steps.put(allocator, child.uninstall_tls.step.name, &child.uninstall_tls);
Expand Down Expand Up @@ -1705,20 +1713,22 @@ pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
const build_runner = @import("root");
const deps = build_runner.dependencies;

inline for (@typeInfo(deps.imports).Struct.decls) |decl| {
if (mem.startsWith(u8, decl.name, b.dep_prefix) and
mem.endsWith(u8, decl.name, name) and
decl.name.len == b.dep_prefix.len + name.len)
{
const build_zig = @field(deps.imports, decl.name);
const build_root = @field(deps.build_root, decl.name);
return dependencyInner(b, name, build_root, build_zig, args);
const pkg_hash = for (b.available_deps) |dep| {
if (mem.eql(u8, dep[0], name)) break dep[1];
} else {
const full_path = b.pathFromRoot("build.zig.zon");
std.debug.print("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file.\n", .{ name, full_path });
process.exit(1);
};

inline for (@typeInfo(deps.packages).Struct.decls) |decl| {
if (mem.eql(u8, decl.name, pkg_hash)) {
const pkg = @field(deps.packages, decl.name);
return dependencyInner(b, name, pkg.build_root, pkg.build_zig, pkg.deps, args);
}
}

const full_path = b.pathFromRoot("build.zig.zon");
std.debug.print("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file.\n", .{ name, full_path });
process.exit(1);
unreachable; // Bad @dependencies source
}

pub fn anonymousDependency(
Expand All @@ -1737,7 +1747,7 @@ pub fn anonymousDependency(
'/', '\\' => byte.* = '.',
else => continue,
};
return dependencyInner(b, name, build_root, build_zig, args);
return dependencyInner(b, name, build_root, build_zig, &.{}, args);
}

fn userValuesAreSame(lhs: UserValue, rhs: UserValue) bool {
Expand Down Expand Up @@ -1792,6 +1802,7 @@ pub fn dependencyInner(
name: []const u8,
build_root_string: []const u8,
comptime build_zig: type,
pkg_deps: AvailableDeps,
args: anytype,
) *Dependency {
const user_input_options = userInputOptionsFromArgs(b.allocator, args);
Expand All @@ -1810,7 +1821,7 @@ pub fn dependencyInner(
process.exit(1);
},
};
const sub_builder = b.createChild(name, build_root, user_input_options) catch @panic("unhandled error");
const sub_builder = b.createChild(name, build_root, pkg_deps, user_input_options) catch @panic("unhandled error");
sub_builder.runBuild(build_zig) catch @panic("unhandled error");

if (sub_builder.validateUserInputDidItFail()) {
Expand Down
1 change: 1 addition & 0 deletions lib/std/Build/Step/Options.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,7 @@ test Options {
.{ .path = "test", .handle = std.fs.cwd() },
host,
&cache,
&.{},
);
defer builder.destroy();

Expand Down
107 changes: 78 additions & 29 deletions src/Package.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,8 @@ pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8

pub const build_zig_basename = "build.zig";

/// Fetches a package and all of its dependencies recursively. Writes the
/// corresponding datastructures for the build runner into `dependencies_source`.
pub fn fetchAndAddDependencies(
pkg: *Package,
deps_pkg: *Package,
Expand All @@ -224,11 +226,11 @@ pub fn fetchAndAddDependencies(
global_cache_directory: Compilation.Directory,
local_cache_directory: Compilation.Directory,
dependencies_source: *std.ArrayList(u8),
build_roots_source: *std.ArrayList(u8),
name_prefix: []const u8,
error_bundle: *std.zig.ErrorBundle.Wip,
all_modules: *AllModules,
root_prog_node: *std.Progress.Node,
/// null for the root package
this_hash: ?[]const u8,
) !void {
const max_bytes = 10 * 1024 * 1024;
const gpa = thread_pool.allocator;
Expand All @@ -242,6 +244,28 @@ pub fn fetchAndAddDependencies(
) catch |err| switch (err) {
error.FileNotFound => {
// Handle the same as no dependencies.
if (this_hash) |hash| {
const pkg_dir_sub_path = "p" ++ fs.path.sep_str ++ hash[0..hex_multihash_len];
const build_root = try global_cache_directory.join(arena, &.{pkg_dir_sub_path});
try dependencies_source.writer().print(
\\ pub const {} = struct {{
\\ pub const build_root = "{}";
\\ pub const build_zig = @import("{}");
\\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{}};
\\ }};
\\
, .{
std.zig.fmtId(hash),
std.zig.fmtEscapes(build_root),
std.zig.fmtEscapes(hash),
});
} else {
try dependencies_source.writer().writeAll(
\\pub const packages = struct {};
\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
\\
);
}
return;
},
else => |e| return e,
Expand Down Expand Up @@ -284,23 +308,23 @@ pub fn fetchAndAddDependencies(

root_prog_node.setEstimatedTotalItems(all_modules.count());

if (this_hash == null) {
try dependencies_source.writer().writeAll("pub const packages = struct {\n");
}

const deps_list = manifest.dependencies.values();
for (manifest.dependencies.keys(), 0..) |name, i| {
const dep = deps_list[i];

const sub_prefix = try std.fmt.allocPrint(arena, "{s}{s}.", .{ name_prefix, name });
const fqn = sub_prefix[0 .. sub_prefix.len - 1];

const sub = try fetchAndUnpack(
thread_pool,
http_client,
global_cache_directory,
dep,
report,
build_roots_source,
fqn,
all_modules,
root_prog_node,
name,
);

if (!sub.found_existing) {
Expand All @@ -313,11 +337,10 @@ pub fn fetchAndAddDependencies(
global_cache_directory,
local_cache_directory,
dependencies_source,
build_roots_source,
sub_prefix,
error_bundle,
all_modules,
root_prog_node,
dep.hash.?,
);
}

Expand All @@ -329,10 +352,47 @@ pub fn fetchAndAddDependencies(
} else {
try deps_pkg.add(gpa, dep.hash.?, sub.mod);
}
}

try dependencies_source.writer().print(" pub const {s} = @import(\"{}\");\n", .{
std.zig.fmtId(fqn), std.zig.fmtEscapes(dep.hash.?),
if (this_hash) |hash| {
const pkg_dir_sub_path = "p" ++ fs.path.sep_str ++ hash[0..hex_multihash_len];
const build_root = try global_cache_directory.join(arena, &.{pkg_dir_sub_path});
try dependencies_source.writer().print(
\\ pub const {} = struct {{
\\ pub const build_root = "{}";
\\ pub const build_zig = @import("{}");
\\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{
\\
, .{
std.zig.fmtId(hash),
std.zig.fmtEscapes(build_root),
std.zig.fmtEscapes(hash),
});
for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| {
try dependencies_source.writer().print(
" .{{ \"{}\", \"{}\" }},\n",
.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) },
);
}
try dependencies_source.writer().writeAll(
\\ };
\\ };
\\
);
} else {
try dependencies_source.writer().writeAll(
\\};
\\
\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{
\\
);
for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| {
try dependencies_source.writer().print(
" .{{ \"{}\", \"{}\" }},\n",
.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) },
);
}
try dependencies_source.writer().writeAll("};\n");
}
}

Expand Down Expand Up @@ -470,10 +530,11 @@ fn fetchAndUnpack(
global_cache_directory: Compilation.Directory,
dep: Manifest.Dependency,
report: Report,
build_roots_source: *std.ArrayList(u8),
fqn: []const u8,
all_modules: *AllModules,
root_prog_node: *std.Progress.Node,
/// This does not have to be any form of canonical or fully-qualified name: it
/// is only intended to be human-readable for progress reporting.
name_for_prog: []const u8,
) !struct { mod: *Package, found_existing: bool } {
const gpa = http_client.allocator;
const s = fs.path.sep_str;
Expand All @@ -484,32 +545,27 @@ fn fetchAndUnpack(
const hex_digest = h[0..hex_multihash_len];
const pkg_dir_sub_path = "p" ++ s ++ hex_digest;

const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path});
errdefer gpa.free(build_root);

var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) {
error.FileNotFound => break :cached,
else => |e| return e,
};
errdefer pkg_dir.close();

try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{
std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root),
});

// The compiler has a rule that a file must not be included in multiple modules,
// so we must detect if a module has been created for this package and reuse it.
const gop = try all_modules.getOrPut(gpa, hex_digest.*);
if (gop.found_existing) {
if (gop.value_ptr.*) |mod| {
gpa.free(build_root);
return .{
.mod = mod,
.found_existing = true,
};
}
}

const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path});
errdefer gpa.free(build_root);

root_prog_node.completeOne();

const ptr = try gpa.create(Package);
Expand All @@ -534,7 +590,7 @@ fn fetchAndUnpack(
};
}

var pkg_prog_node = root_prog_node.start(fqn, 0);
var pkg_prog_node = root_prog_node.start(name_for_prog, 0);
defer pkg_prog_node.end();
pkg_prog_node.activate();
pkg_prog_node.context.refresh();
Expand Down Expand Up @@ -666,13 +722,6 @@ fn fetchAndUnpack(
return error.PackageFetchFailed;
}

const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path});
defer gpa.free(build_root);

try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{
std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root),
});

const mod = try createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, build_zig_basename);
try all_modules.put(gpa, actual_hex, mod);
return .{
Expand Down
22 changes: 9 additions & 13 deletions src/main.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4708,7 +4708,14 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
.root_src_directory = build_directory,
.root_src_path = build_zig_basename,
};
if (!build_options.only_core_functionality) {
if (build_options.only_core_functionality) {
const deps_pkg = try Package.createFilePkg(gpa, local_cache_directory, "dependencies.zig",
\\pub const packages = struct {};
\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
\\
);
try main_pkg.add(gpa, "@dependencies", deps_pkg);
} else {
var http_client: std.http.Client = .{ .allocator = gpa };
defer http_client.deinit();

Expand All @@ -4717,12 +4724,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
// access dependencies by name, since `@import` requires string literals.
var dependencies_source = std.ArrayList(u8).init(gpa);
defer dependencies_source.deinit();
try dependencies_source.appendSlice("pub const imports = struct {\n");

// This will go into the same package. It contains the file system paths
// to all the build.zig files.
var build_roots_source = std.ArrayList(u8).init(gpa);
defer build_roots_source.deinit();

var all_modules: Package.AllModules = .{};
defer all_modules.deinit(gpa);
Expand All @@ -4746,11 +4747,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
global_cache_directory,
local_cache_directory,
&dependencies_source,
&build_roots_source,
"",
&wip_errors,
&all_modules,
root_prog_node,
null,
);
if (wip_errors.root_list.items.len > 0) {
var errors = try wip_errors.toOwnedBundle("");
Expand All @@ -4760,10 +4760,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
}
try fetch_result;

try dependencies_source.appendSlice("};\npub const build_root = struct {\n");
try dependencies_source.appendSlice(build_roots_source.items);
try dependencies_source.appendSlice("};\n");

const deps_pkg = try Package.createFilePkg(
gpa,
local_cache_directory,
Expand Down