package manager: allow overriding dependencies with local cache

ci/aarch64-linux-debug.sh

@@ -62,6 +62,15 @@ stage3-debug/bin/zig build test docs \
   --zig-lib-dir "$PWD/../lib" \
   -Denable-superhtml
 
+# Ensure that dependency overrides function correctly
+wd=$PWD
+
+cd ../test/dependency_override
+./run-tests.sh ../../build-debug/stage3-debug/bin/zig
+cd $wd
+
+unset wd
+
 # Ensure that updating the wasm binary from this commit will result in a viable build.
 stage3-debug/bin/zig build update-zig1
 

ci/aarch64-linux-release.sh

@@ -78,6 +78,15 @@ echo "If the following command fails, it means nondeterminism has been"
 echo "introduced, making stage3 and stage4 no longer byte-for-byte identical."
 diff stage3-release/bin/zig stage4-release/bin/zig
 
+# Ensure that dependency overrides function correctly
+wd=$PWD
+
+cd ../test/dependency_override
+./run-tests.sh ../../build-release/stage3-release/bin/zig
+cd $wd
+
+unset wd
+
 # Ensure that updating the wasm binary from this commit will result in a viable build.
 stage3-release/bin/zig build update-zig1
 

ci/aarch64-macos-debug.sh

@@ -55,3 +55,12 @@ stage3-debug/bin/zig build test docs \
   -Dstatic-llvm \
   -Dskip-non-native \
   --search-prefix "$PREFIX"
+
+# Ensure that dependency overrides function correctly
+wd=$PWD
+
+cd ../test/dependency_override
+./run-tests.sh ../../build-debug/stage3-debug/bin/zig
+cd $wd
+
+unset wd

ci/aarch64-macos-release.sh

@@ -56,6 +56,15 @@ stage3-release/bin/zig build test docs \
   -Dskip-non-native \
   --search-prefix "$PREFIX"
 
+# Ensure that dependency overrides function correctly
+wd=$PWD
+
+cd ../test/dependency_override
+./run-tests.sh ../../build-release/stage3-release/bin/zig
+cd $wd
+
+unset wd
+
 # Ensure that stage3 and stage4 are byte-for-byte identical.
 stage3-release/bin/zig build \
   --prefix stage4-release \

ci/x86_64-linux-debug.sh

@@ -70,6 +70,15 @@ stage3-debug/bin/zig build test docs \
   --zig-lib-dir "$PWD/../lib" \
   -Denable-superhtml
 
+# Ensure that dependency overrides function correctly
+wd=$PWD
+
+cd ../test/dependency_override
+./run-tests.sh ../../build-debug/stage3-debug/bin/zig
+cd $wd
+
+unset wd
+
 # Ensure that updating the wasm binary from this commit will result in a viable build.
 stage3-debug/bin/zig build update-zig1
 

ci/x86_64-linux-release.sh

@@ -70,6 +70,15 @@ stage3-release/bin/zig build test docs \
   --zig-lib-dir "$PWD/../lib" \
   -Denable-superhtml
 
+# Ensure that dependency overrides function correctly
+wd=$PWD
+
+cd ../test/dependency_override
+./run-tests.sh ../../build-release/stage3-release/bin/zig
+cd $wd
+
+unset wd
+
 # Ensure that stage3 and stage4 are byte-for-byte identical.
 stage3-release/bin/zig build \
   --prefix stage4-release \

ci/x86_64-macos-release.sh

@@ -73,3 +73,12 @@ stage3/bin/zig build \
 echo "If the following command fails, it means nondeterminism has been"
 echo "introduced, making stage3 and stage4 no longer byte-for-byte identical."
 diff stage3/bin/zig stage4/bin/zig
+
+# Ensure that dependency overrides function correctly
+wd=$PWD
+
+cd ../test/dependency_override
+./run-tests.sh ../../build/stage3/bin/zig
+cd $wd
+
+unset wd

lib/compiler/build_runner.zig

@@ -374,6 +374,24 @@ pub fn main() !void {
     if (steps_menu)
         return steps(builder, stdout_writer);
 
+    inline for (dependencies.root_deps) |dep| {
+        if (!@hasDecl(dependencies.packages, dep[1])) continue;
+        const pkg = @field(dependencies.packages, dep[1]);
+        if (!@hasDecl(pkg, "local_override")) continue;
+        const allocator = single_threaded_arena.allocator();
+        const from = try std.process.getCwdAlloc(allocator);
+        const relative_path = try std.fs.path.relative(allocator, from, pkg.build_root);
+        if (pkg.local_override) {
+            if (builder.graph.system_package_mode) {
+                std.log.warn("ignoring local override of '{s}' in system mode, using '{s}'", .{
+                    dep[0], pkg.build_root,
+                });
+            } else {
+                std.log.warn("overriding dependency '{s}' with '{s}'", .{ dep[0], relative_path });
+            }
+        }
+    }
+
     var run: Run = .{
         .max_rss = max_rss,
         .max_rss_is_default = false,

src/Package/Fetch.zig

@@ -55,6 +55,7 @@ use_latest_commit: bool,
 /// This will either be relative to `global_cache`, or to the build root of
 /// the root package.
 package_root: Cache.Path,
+local_override: bool,
 error_bundle: ErrorBundle.Wip,
 manifest: ?Manifest,
 manifest_ast: std.zig.Ast,
@@ -99,6 +100,7 @@ pub const JobQueue = struct {
     thread_pool: *ThreadPool,
     wait_group: WaitGroup = .{},
     global_cache: Cache.Directory,
+    local_cache: ?Cache.Directory,
     /// If true then, no fetching occurs, and:
     /// * The `global_cache` directory is assumed to be the direct parent
     ///   directory of on-disk packages rather than having the "p/" directory
@@ -202,6 +204,11 @@ pub const JobQueue = struct {
                 }
             }
 
+            try buf.writer().print(
+                \\        pub const local_override = {};
+                \\
+            , .{fetch.local_override});
+
             try buf.writer().print(
                 \\        pub const build_root = "{q}";
                 \\
@@ -390,6 +397,35 @@ pub fn run(f: *Fetch) RunError!void {
         const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len];
         const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;
         const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];
+
+        if (f.job_queue.local_cache) |local_cache| local: {
+            const local_cache_path = if (f.job_queue.read_only) prefixed_pkg_sub_path else pkg_sub_path;
+            if (local_cache.handle.access(local_cache_path, .{})) |_| {
+                f.local_override = true;
+                if (f.job_queue.read_only) break :local;
+
+                assert(f.lazy_status != .unavailable);
+                f.package_root = .{
+                    .root_dir = local_cache,
+                    .sub_path = try arena.dupe(u8, local_cache_path),
+                };
+                try loadManifest(f, f.package_root);
+                try checkBuildFileExistence(f);
+                if (!f.job_queue.recursive) return;
+                return queueJobsForDeps(f);
+            } else |err| switch (err) {
+                error.FileNotFound => {},
+                else => |e| {
+                    try eb.addRootErrorMessage(.{
+                        .msg = try eb.printString("unable to open local package cache directory '{}{s}': {s}", .{
+                            local_cache, local_cache_path, @errorName(e),
+                        }),
+                    });
+                    return error.FetchFailed;
+                },
+            }
+        }
+
         if (cache_root.handle.access(pkg_sub_path, .{})) |_| {
             assert(f.lazy_status != .unavailable);
             f.package_root = .{
@@ -765,6 +801,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
                 .use_latest_commit = false,
 
                 .package_root = undefined,
+                .local_override = false,
                 .error_bundle = undefined,
                 .manifest = null,
                 .manifest_ast = undefined,
@@ -2312,6 +2349,7 @@ const TestFetchBuilder = struct {
             .http_client = &self.http_client,
             .thread_pool = &self.thread_pool,
             .global_cache = self.global_cache_directory,
+            .local_cache = null,
             .recursive = false,
             .read_only = false,
             .debug_hash = false,
@@ -2336,6 +2374,7 @@ const TestFetchBuilder = struct {
             .use_latest_commit = true,
 
             .package_root = undefined,
+            .local_override = false,
             .error_bundle = undefined,
             .manifest = null,
             .manifest_ast = undefined,

src/main.zig

@@ -5194,6 +5194,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
                     .http_client = &http_client,
                     .thread_pool = &thread_pool,
                     .global_cache = global_cache_directory,
+                    .local_cache = local_cache_directory,
                     .read_only = false,
                     .recursive = true,
                     .debug_hash = false,
@@ -5238,6 +5239,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
                     .use_latest_commit = false,
 
                     .package_root = undefined,
+                    .local_override = false,
                     .error_bundle = undefined,
                     .manifest = null,
                     .manifest_ast = undefined,
@@ -7115,6 +7117,7 @@ fn cmdFetch(
         .http_client = &http_client,
         .thread_pool = &thread_pool,
         .global_cache = global_cache_directory,
+        .local_cache = null,
         .recursive = false,
         .read_only = false,
         .debug_hash = debug_hash,
@@ -7140,6 +7143,7 @@ fn cmdFetch(
         .use_latest_commit = true,
 
         .package_root = undefined,
+        .local_override = false,
         .error_bundle = undefined,
         .manifest = null,
         .manifest_ast = undefined,

test/dependency_override/build.zig

@@ -0,0 +1,49 @@
+pub fn build(b: *std.Build) !void {
+    const optimize = b.standardOptimizeOption(.{});
+    const target = b.standardTargetOptions(.{});
+
+    const overridden_runtime_pkg = b.dependency("overridden_runtime", .{});
+    const overridden_buildtime_pkg = b.dependency("overridden_buildtime", .{});
+
+    const overridden_runtime_module = overridden_runtime_pkg.module("module");
+    const overridden_buildtime_module = overridden_buildtime_pkg.module("module");
+
+    const test_step = b.step("test", "check package override behavior");
+    b.default_step = test_step;
+
+    {
+        const exe = b.addExecutable(.{
+            .name = "dep-override-test-runtime",
+            .root_source_file = b.path("src/main.zig"),
+            .target = target,
+            .optimize = optimize,
+        });
+        exe.root_module.addImport("module", overridden_runtime_module);
+
+        const run = b.addRunArtifact(exe);
+
+        const step = b.step("runtime", "check error package is overridden at runtime");
+        step.dependOn(&run.step);
+
+        test_step.dependOn(&run.step);
+    }
+
+    {
+        const exe = b.addExecutable(.{
+            .name = "dep-override-test-buildtime",
+            .root_source_file = b.path("src/main.zig"),
+            .target = target,
+            .optimize = optimize,
+        });
+        exe.root_module.addImport("module", overridden_buildtime_module);
+
+        const run = b.addRunArtifact(exe);
+
+        const step = b.step("buildtime", "check error package is overridden at buildtime");
+        step.dependOn(&run.step);
+
+        test_step.dependOn(&run.step);
+    }
+}
+
+const std = @import("std");

test/dependency_override/build.zig.zon

@@ -0,0 +1,16 @@
+.{
+    .name = .dependency_override,
+    .fingerprint = 0x1d52eb5bc39f2664,
+    .version = "0.0.0",
+    .paths = .{""},
+    .dependencies = .{
+        .overridden_runtime = .{
+            .url = "https://example.com",
+            .hash = "overridden_runtime_pkg-0.0.0-AAAAAG8BAADm054BUibxV6D-KNadKUJKOmpavQp3xJkX",
+        },
+        .overridden_buildtime = .{
+            .url = "https://example.com",
+            .hash = "overridden_buildtime_pkg-0.0.0-AAAAALcBAABxm1t_JaDEv2JAbqTxTR7030GYiuw3tGUN",
+        },
+    },
+}

test/dependency_override/cache-with-bad-pkgs/p/overridden_buildtime_pkg-0.0.0-AAAAALcBAABxm1t_JaDEv2JAbqTxTR7030GYiuw3tGUN/build.zig

@@ -0,0 +1,8 @@
+pub fn build(b: *std.Build) !void {
+    _ = b.addModule("module", .{
+        .root_source_file = b.path("src/root.zig"),
+    });
+    @panic("overridden-buildtime package has not been overridden");
+}
+
+const std = @import("std");

test/dependency_override/cache-with-bad-pkgs/p/overridden_buildtime_pkg-0.0.0-AAAAALcBAABxm1t_JaDEv2JAbqTxTR7030GYiuw3tGUN/build.zig.zon

@@ -0,0 +1,5 @@
+.{
+    .name = .overridden_buildtime_pkg,
+    .version = "0.0.0",
+    .paths = .{""},
+}

test/dependency_override/cache-with-bad-pkgs/p/overridden_buildtime_pkg-0.0.0-AAAAALcBAABxm1t_JaDEv2JAbqTxTR7030GYiuw3tGUN/src/root.zig

@@ -0,0 +1,5 @@
+pub fn run() void {
+    @panic("the overridden-buildtime package has not been overridden");
+}
+
+const std = @import("std");

test/dependency_override/cache-with-bad-pkgs/p/overridden_runtime_pkg-0.0.0-AAAAAG8BAADm054BUibxV6D-KNadKUJKOmpavQp3xJkX/build.zig

@@ -0,0 +1,7 @@
+pub fn build(b: *std.Build) !void {
+    _ = b.addModule("module", .{
+        .root_source_file = b.path("src/root.zig"),
+    });
+}
+
+const std = @import("std");

test/dependency_override/cache-with-bad-pkgs/p/overridden_runtime_pkg-0.0.0-AAAAAG8BAADm054BUibxV6D-KNadKUJKOmpavQp3xJkX/build.zig.zon

@@ -0,0 +1,5 @@
+.{
+    .name = .overridden_runtime_pkg,
+    .version = "0.0.0",
+    .paths = .{""},
+}

test/dependency_override/cache-with-bad-pkgs/p/overridden_runtime_pkg-0.0.0-AAAAAG8BAADm054BUibxV6D-KNadKUJKOmpavQp3xJkX/src/root.zig

@@ -0,0 +1,5 @@
+pub fn run() void {
+    @panic("the overridden-runtime package has not been overridden");
+}
+
+const std = @import("std");

test/dependency_override/cache-with-good-pkgs/p/overridden_buildtime_pkg-0.0.0-AAAAALcBAABxm1t_JaDEv2JAbqTxTR7030GYiuw3tGUN/build.zig

@@ -0,0 +1,7 @@
+pub fn build(b: *std.Build) !void {
+    _ = b.addModule("module", .{
+        .root_source_file = b.path("src/root.zig"),
+    });
+}
+
+const std = @import("std");

test/dependency_override/cache-with-good-pkgs/p/overridden_buildtime_pkg-0.0.0-AAAAALcBAABxm1t_JaDEv2JAbqTxTR7030GYiuw3tGUN/build.zig.zon

@@ -0,0 +1,5 @@
+.{
+    .name = .overridden_buildtime_pkg,
+    .version = "0.0.0",
+    .paths = .{""},
+}

test/dependency_override/cache-with-good-pkgs/p/overridden_buildtime_pkg-0.0.0-AAAAALcBAABxm1t_JaDEv2JAbqTxTR7030GYiuw3tGUN/src/root.zig

@@ -0,0 +1,5 @@
+pub fn run() void {
+    std.debug.print("this is the overridden-buildtime package\n", .{});
+}
+
+const std = @import("std");

test/dependency_override/cache-with-good-pkgs/p/overridden_runtime_pkg-0.0.0-AAAAAG8BAADm054BUibxV6D-KNadKUJKOmpavQp3xJkX/build.zig

@@ -0,0 +1,7 @@
+pub fn build(b: *std.Build) !void {
+    _ = b.addModule("module", .{
+        .root_source_file = b.path("src/root.zig"),
+    });
+}
+
+const std = @import("std");

test/dependency_override/cache-with-good-pkgs/p/overridden_runtime_pkg-0.0.0-AAAAAG8BAADm054BUibxV6D-KNadKUJKOmpavQp3xJkX/build.zig.zon

@@ -0,0 +1,5 @@
+.{
+    .name = .overridden_runtime_pkg,
+    .version = "0.0.0",
+    .paths = .{""},
+}

test/dependency_override/cache-with-good-pkgs/p/overridden_runtime_pkg-0.0.0-AAAAAG8BAADm054BUibxV6D-KNadKUJKOmpavQp3xJkX/src/root.zig

@@ -0,0 +1,5 @@
+pub fn run() void {
+    std.debug.print("this is the overridden-runtime package\n", .{});
+}
+
+const std = @import("std");