From 1cf3608b0d33a38ddc0fdf909e02fc7685e7b44d Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Sat, 20 May 2023 03:13:25 +0200 Subject: [PATCH] Introduce experimental `fetchComposerDepsImpure` `fetchComposerDeps` works okay but since the fetching runs at evaluation time, it grinds Nix evaluator to a halt. As a bonus, we have a finer control over what is fetched so we can limit it to a shallow fetch. This patch adds an alternative implementation that moves the work to build time based on the experimental `impure-derivations` feature. Unfortunately, the feature needs to be enabled **in the daemon** and can only be used by other impure derivations. The repo derivation also will not be cached by Nix so everything will need to be re-fetched for every build. --- .github/workflows/main.yaml | 7 ++ README.md | 14 ++- overlay.nix | 1 + run-tests.sh | 7 ++ src/composer-create-repository.py | 144 ++++++++++++++++++++++++++++++ src/fetch-deps-impure.nix | 36 ++++++++ tests/flake.nix | 21 +++++ 7 files changed, 228 insertions(+), 2 deletions(-) create mode 100644 src/composer-create-repository.py create mode 100644 src/fetch-deps-impure.nix diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index d71d53a..778d147 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -13,6 +13,13 @@ jobs: - name: Install Nix uses: cachix/install-nix-action@v17 + with: + extra_nix_config: | + # `flakes` and `nix-command` for convenience + # `impure-derivations` needed for testing `fetchComposerDepsImpure` + # and it is not sufficient to enable with CLI flag: https://github.com/NixOS/nix/issues/6478 + # `ca-derivations` required by `impure-derivations` + experimental-features = flakes nix-command ca-derivations impure-derivations - name: Run integration tests run: ./run-tests.sh diff --git a/README.md b/README.md index 9896c0c..9462940 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,14 @@ This is a function that, for given source, returns a derivation with a Composer - Either `lockFile` containing an explicit path to `composer.lock` file, or `src`, which is the source directory/derivation containing the file. +### `c4.fetchComposerDepsImpure` + +This function has the same API as [`c4.fetchComposerDeps`]]#c4fetchcomposerdeps) but it fetches the dependencies at build time. There are, however, significant downsides: + +- It requires [enabling an experimental `impure-derivations`](https://nixos.org/manual/nix/stable/contributing/experimental-features.html#impure-derivations) feature [**in the daemon**](https://github.com/NixOS/nix/issues/6478) +- It can only be used by other impure derivations. +- The dependencies will be re-fetched with every build. + ### `c4.composerSetupHook` This is a [setup hook](https://nixos.org/manual/nixpkgs/stable/#ssec-setup-hooks). By adding it to `nativeBuildInputs` of a Nixpkgs derivation, the following hooks will be automatically enabled. @@ -102,10 +110,12 @@ It is controlled by the following environment variables (pass them to the deriva - It requires `composer.lock` to exist. - It currently only supports downloading packages from Git. - When the lockfile comes from a source derivation rather then a local repository, Nix’s [import from derivation](https://nixos.wiki/wiki/Import_From_Derivation) mechanism will be used, inheriting all problems of IFD. Notably, it cannot be used in Nixpkgs. -- We download the sources at evaluation time so it will block evaluation, this is especially painful since Nix currently does not support parallel evaluation. -- Nix’s fetchers will fetch the full Git ref, which will take a long time for heavy repos like https://github.com/phpstan/phpstan. +- We download the sources at evaluation time so it will block evaluation, this is especially painful since Nix currently does not support parallel evaluation. 👋 +- Nix’s fetchers will fetch the full Git ref, which will take a long time for heavy repos like https://github.com/phpstan/phpstan. 👋 - It might be somewhat slower than generated Nix files (e.g. [composer2nix]) since the Nix values need to be constructed from scratch every time. +👋 You can use the [`c4.fetchComposerDepsImpure`](#c4fetchcomposerdepsimpure) to move the work to build time and more efficient fetching but it has [other downsides](#c4fetchcomposerdepsimpure). + For more information look at Nicolas’s _[An overview of language support in Nix][nixcon-language-support-overview]_ presentation from NixCon 2019. ## How does it work? diff --git a/overlay.nix b/overlay.nix index 40d64b7..39daf77 100644 --- a/overlay.nix +++ b/overlay.nix @@ -14,5 +14,6 @@ prev: ./src/composer-setup-hook.sh; fetchComposerDeps = prev.callPackage ./src/fetch-deps.nix { }; + fetchComposerDepsImpure = prev.callPackage ./src/fetch-deps-impure.nix { }; }; } diff --git a/run-tests.sh b/run-tests.sh index e211545..6668625 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -1,6 +1,13 @@ #!/usr/bin/env bash set -x -o errexit +nix develop --no-write-lock-file ./tests#python -c black --check --diff src/composer-create-repository.py +nix develop --no-write-lock-file ./tests#python -c mypy --strict src/composer-create-repository.py + +nix build -L --no-write-lock-file --extra-experimental-features impure-derivations ./tests#composer-impure +nix build -L --no-write-lock-file --extra-experimental-features impure-derivations ./tests#grav-impure +nix build -L --no-write-lock-file --extra-experimental-features impure-derivations ./tests#non-head-rev-impure + nix build -L --no-write-lock-file ./tests#composer nix build -L --no-write-lock-file ./tests#grav nix build -L --no-write-lock-file ./tests#non-head-rev diff --git a/src/composer-create-repository.py b/src/composer-create-repository.py new file mode 100644 index 0000000..6725387 --- /dev/null +++ b/src/composer-create-repository.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +from pathlib import Path +from typing import cast, NotRequired, TypedDict +import argparse +import json +import shutil +import subprocess + +Source = TypedDict( + "Source", + { + "type": str, + "url": str, + "reference": str, + }, +) + + +class Package(TypedDict): + name: str + version: str + source: NotRequired[Source] + dist: Source + + +def clone_git_repo(url: str, rev: str, clone_target_path: Path) -> None: + subprocess.check_call( + ["git", "init"], + cwd=clone_target_path, + ) + subprocess.check_call( + ["git", "fetch", url, rev, "--depth", "1"], + cwd=clone_target_path, + ) + subprocess.check_call( + ["git", "reset", "--hard", "FETCH_HEAD"], + cwd=clone_target_path, + ) + + +def fetch_composer_package(package: Package, clone_target_path: Path) -> None: + assert ( + "source" in package and package["source"]["type"] == "git" + ), f"Package “{package['name']}” does not have source of type “git”." + + clone_git_repo( + url=package["source"]["url"], + rev=package["source"]["reference"], + clone_target_path=clone_target_path, + ) + + # Clean up git directory to ensure reproducible output + shutil.rmtree(clone_target_path / ".git") + + +def make_package( + package: Package, + clone_target_path: Path, +) -> tuple[str, dict[str, Package]]: + assert ( + package["source"]["reference"] == package["dist"]["reference"] + ), f"Package “{package['name']}” has a mismatch between “reference” keys of “dist” and “source” keys." + + # While Composer repositories only really require `name`, `version` and `source`/`dist` fields, + # we will use the original contents of the package’s entry from `composer.lock`, modifying just the sources. + # Package entries in Composer repositories correspond to `composer.json` files [1] + # and Composer appears to use them when regenerating the lockfile. + # If we just used the minimal info, stuff like `autoloading` or `bin` programs would be broken. + # + # We cannot use `source` since Composer does not support path sources: + # "PathDownloader" is a dist type downloader and can not be used to download source + # + # [1]: https://getcomposer.org/doc/05-repositories.md#packages> + + # Copy the Package so that we do not mutate the original. + package = cast(Package, dict(package)) + package.pop("source", None) + package["dist"] = { + "type": "path", + "url": str(clone_target_path / package["name"] / package["version"]), + "reference": package["dist"]["reference"], + } + + return ( + package["name"], + { + package["version"]: package, + }, + ) + + +def main( + lockfile_path: Path, + output_path: Path, +) -> None: + # We are generating a repository of type Composer + # https://getcomposer.org/doc/05-repositories.md#composer + with open(lockfile_path) as lockfile: + lock = json.load(lockfile) + repo_path = output_path / "repo" + + # We always need to fetch dev dependencies so that `composer update --lock` can update the config. + packages_to_install = lock["packages"] + lock["packages-dev"] + + for package in packages_to_install: + clone_target_path = repo_path / package["name"] / package["version"] + clone_target_path.mkdir(parents=True) + fetch_composer_package(package, clone_target_path) + + repo_manifest = { + "packages": { + package_name: metadata + for package_name, metadata in [ + make_package(package, repo_path) for package in packages_to_install + ] + } + } + with open(output_path / "packages.json", "w") as repo_manifest_file: + json.dump( + repo_manifest, + repo_manifest_file, + indent=4, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generate composer repository for offline fetching" + ) + parser.add_argument( + "lockfile_path", + help="Path to a composer lockfile", + ) + parser.add_argument( + "output_path", + help="Output path to store the repository in", + ) + + args = parser.parse_args() + + main( + lockfile_path=Path(args.lockfile_path), + output_path=Path(args.output_path), + ) diff --git a/src/fetch-deps-impure.nix b/src/fetch-deps-impure.nix new file mode 100644 index 0000000..ceb4a7c --- /dev/null +++ b/src/fetch-deps-impure.nix @@ -0,0 +1,36 @@ +{ + runCommand, + lib, + python311, + git, + cacert, +}: + +{ + src ? null, + lockFile ? null, +}: + +assert lib.assertMsg ((src == null) != (lockFile == null)) "Either “src” or “lockFile” attribute needs to be provided."; + +let + lockPath = + if lockFile != null then + # Interpolated to create a store object. + "${lockFile}" + else + "${src}/composer.lock"; +in +# We are generating a repository of type Composer +# https://getcomposer.org/doc/05-repositories.md#composer +runCommand "repo" { + __impure = true; + + nativeBuildInputs = [ + python311 + git + cacert + ]; +} '' + python3 "${./composer-create-repository.py}" ${lib.escapeShellArg lockPath} "$out" +'' diff --git a/tests/flake.nix b/tests/flake.nix index 9790abf..96e188b 100644 --- a/tests/flake.nix +++ b/tests/flake.nix @@ -12,10 +12,31 @@ c4.overlays.default ]; }; + + impurify = + pkg: + (pkg.override (prev: { + c4 = prev.c4 // { + fetchComposerDeps = prev.c4.fetchComposerDepsImpure; + }; + })).overrideAttrs (attrs: { + # Impure derivations can only be built by other impure derivations. + __impure = true; + }); in { packages.x86_64-linux.composer = pkgs.callPackage ./composer { }; + packages.x86_64-linux.composer-impure = impurify self.packages.x86_64-linux.composer; packages.x86_64-linux.grav = pkgs.callPackage ./grav { }; + packages.x86_64-linux.grav-impure = impurify self.packages.x86_64-linux.grav; packages.x86_64-linux.non-head-rev = pkgs.callPackage ./non-head-rev { }; + packages.x86_64-linux.non-head-rev-impure = impurify self.packages.x86_64-linux.non-head-rev; + + devShells.x86_64-linux.python = pkgs.mkShell { + nativeBuildInputs = [ + pkgs.python311.pkgs.black + pkgs.python311.pkgs.mypy + ]; + }; }; }