From 22af94b73c3d263d9199dfe841db6c5c367c043a Mon Sep 17 00:00:00 2001 From: Josh Spence Date: Thu, 3 Sep 2026 17:59:43 +1000 Subject: [PATCH] Fix the Nix < 2.20 NAR hash fallback for inputs with submodules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `GitInputScheme::getAccessorFromCommit()` decided whether a lock was produced with Nix < 2.20 or Nix >= 2.20 export semantics *before* mounting submodules. For an input with `submodules = true` the expected NAR hash covers the mounted tree, while both candidate hashes covered only the bare top-level repo, so neither could ever match. Instead of falling back to `git archive`/`git checkout` and warning, Nix hard-failed with a NAR hash mismatch: error: NAR hash mismatch in input 'git+ssh://git@example.org/repo.git?rev=...&shallow=1&submodules=1', expected 'sha256-CnkK...' but got 'sha256-XNEq...' `nix flake update ` recomputes rather than verifies the hash, so the failure only showed up on the read path — anyone evaluating a lock file written by Nix < 2.20 (or by Nix >= 2.20 with `nix-219-compat` enabled) for a repo with submodules could not evaluate it at all. Move the submodule mounting into a `getTree()` lambda so both candidate trees are fully mounted before their NAR hashes are compared. Since the Git filters that Nix < 2.20 applied also affect submodule contents, reproducing such a hash requires exporting the submodules with the same semantics; propagate this through the synthesized submodule inputs via a new internal `__legacyExport` attribute (not part of `allowedAttrs()`, never serialized into a lock file, following the `__final` convention). When it's absent we're a top-level input and `nix-219-compat` decides; when it's present we follow the top-level repo regardless of the setting. `GitAccessorOptions` gains a matching `legacy` field so that `makeFingerprint()` accounts for it. It is appended last, keeping the legacy cache keys byte-identical to the previous hand-concatenated `makeFingerprint(rev) + ";legacy"`. Assisted-by: Claude Opus 5 --- src/libfetchers/fetchers.cc | 4 +- src/libfetchers/git-utils.cc | 2 +- src/libfetchers/git.cc | 183 ++++++++++-------- .../include/nix/fetchers/git-utils.hh | 7 + tests/functional/fetchGitSubmodules.sh | 59 ++++++ 5 files changed, 176 insertions(+), 79 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index b1bc134ac34e..dde72d79e28a 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -105,7 +105,9 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs) auto allowedAttrs = inputScheme->allowedAttrs(); for (auto & [name, _] : attrs) - if (name != "type" && name != "__final" && allowedAttrs.count(name) == 0) + /* `__final` and `__legacyExport` are purely internal attributes, so they're not listed in + `allowedAttrs()`. */ + if (name != "type" && name != "__final" && name != "__legacyExport" && allowedAttrs.count(name) == 0) throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName); auto res = inputScheme->inputFromAttrs(settings, attrs); diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 52986a6b4509..7dc4d7c5e7a4 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -817,7 +817,7 @@ ref GitRepo::openRepo(const std::filesystem::path & path, GitRepo::Opti std::string GitAccessorOptions::makeFingerprint(const Hash & rev) const { - return "git:" + rev.gitRev() + (exportIgnore ? ";e" : "") + (smudgeLfs ? ";l" : ""); + return "git:" + rev.gitRev() + (exportIgnore ? ";e" : "") + (smudgeLfs ? ";l" : "") + (legacy ? ";legacy" : ""); } /** diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7f560e0d2bec..5b74b104cb85 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -618,6 +618,19 @@ struct GitInputScheme : InputScheme return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); } + /** + * Whether to export this input using Nix < 2.20 semantics. This is a purely internal + * attribute: it is set only on the submodule inputs synthesized by + * `getAccessorFromCommit()`, so that the export of a repository composes the export of the + * top-level repo with exports of its submodules using the *same* semantics. If it's absent, + * we're a top-level input and the `nix-219-compat` setting decides. It is deliberately not + * part of `allowedAttrs()`. + */ + std::optional getLegacyExportAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "__legacyExport"); + } + RepoInfo getRepoInfo(const Input & input) const { auto checkHashAlgorithm = [&](const std::optional & hash) { @@ -832,6 +845,7 @@ struct GitInputScheme : InputScheme .exportIgnore = getExportIgnoreAttr(input), .smudgeLfs = getLfsAttr(input), .submodules = getSubmodulesAttr(input), + .legacy = getLegacyExportAttr(input).value_or(false), }; } @@ -850,7 +864,9 @@ struct GitInputScheme : InputScheme if (!options.submodules) options.exportIgnore = true; - auto fingerprint = options.makeFingerprint(rev) + ";legacy"; + options.legacy = true; + + auto fingerprint = options.makeFingerprint(rev); auto cacheKey = makeSourcePathToHashCacheKey(fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root); @@ -1073,95 +1089,108 @@ struct GitInputScheme : InputScheme auto expectedNarHash = input.getNarHash(); - auto accessor = repo->getAccessor(rev, options, "«" + input.to_string(true) + "»"); + /* Return an accessor for the complete tree denoted by `rev`, that is, the top-level repo + with any submodules mounted into it. If `legacy` is set, Nix < 2.20 semantics are used + (i.e. `git archive` / `git checkout`, which apply Git filters, `export-ignore` and + `export-subst`); the submodules are then exported using those semantics as well. - if (settings.nix219Compat && !options.smudgeLfs) { - /* Use Nix 2.19 semantics to generate locks, but if a NAR hash is specified, support Nix >= 2.20 semantics - * as well. */ - warn("Using Nix 2.19 semantics to export Git repository '%s'.", input.to_string()); - auto accessorModern = accessor; - accessor = getLegacyGitAccessor(settings, store, repoInfo, repoDir, rev, options); - if (expectedNarHash) { - auto narHashLegacy = - fetchToStore2(settings, store, {accessor}, FetchMode::DryRun, input.getName()).second; - if (expectedNarHash != narHashLegacy) { - auto narHashModern = - fetchToStore2(settings, store, {accessorModern}, FetchMode::DryRun, input.getName()).second; - if (expectedNarHash == narHashModern) - accessor = accessorModern; + Note that submodules must be mounted before the NAR hash of the tree can be compared + against `expectedNarHash`, since that hash covers the submodule contents as well. */ + auto getTree = [&](bool legacy) -> nix::ref { + auto options2 = options; + options2.legacy = legacy; + + auto accessor = legacy ? getLegacyGitAccessor(settings, store, repoInfo, repoDir, rev, options2) + : repo->getAccessor(rev, options2, "«" + input.to_string(true) + "»"); + + /* If the repo has submodules, fetch them and return a mounted + input accessor consisting of the accessor for the top-level + repo and the accessors for the submodules. */ + if (options2.submodules) { + std::map> mounts; + + for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, options2.exportIgnore)) { + auto resolved = repo->resolveSubmoduleUrl(submodule.url); + debug( + "Git submodule %s: %s %s %s -> %s", + submodule.path, + submodule.url, + submodule.branch, + submoduleRev.gitRev(), + resolved); + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", resolved); + if (submodule.branch != "") { + // A special value of . is used to indicate that the name of the branch in the submodule + // should be the same name as the current branch in the current repository. + // https://git-scm.com/docs/gitmodules + if (submodule.branch == ".") { + attrs.insert_or_assign("ref", ref); + } else { + attrs.insert_or_assign("ref", submodule.branch); + } + } + attrs.insert_or_assign("rev", submoduleRev.gitRev()); + attrs.insert_or_assign("exportIgnore", Explicit{options2.exportIgnore}); + attrs.insert_or_assign("submodules", Explicit{true}); + attrs.insert_or_assign("lfs", Explicit{options2.smudgeLfs}); + attrs.insert_or_assign("allRefs", Explicit{true}); + /* Export the submodule using the same semantics as the top-level repo, + regardless of the `nix-219-compat` setting. */ + attrs.insert_or_assign("__legacyExport", Explicit{options2.legacy}); + auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs)); + auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); + submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string(true) + "»"); + mounts.insert_or_assign(submodule.path, submoduleAccessor); + } + + if (!mounts.empty()) { + auto newFingerprint = accessor->getFingerprint(CanonPath::root).second->append(";s"); + mounts.insert_or_assign(CanonPath::root, accessor); + auto mounted = makeMountedSourceAccessor(std::move(mounts)); + mounted->fingerprint = newFingerprint; + return mounted; } } - } else { - /* Backward compatibility hack for locks produced by Nix < 2.20 that depend on Nix applying Git filters, - * `export-ignore` or `export-subst`. Nix >= 2.20 doesn't do those, so we may get a NAR hash mismatch. If - * that happens, try again using `git archive`. */ - if (expectedNarHash) { - auto narHashNew = fetchToStore2(settings, store, {accessor}, FetchMode::DryRun, input.getName()).second; - if (expectedNarHash != narHashNew) { - auto accessorLegacy = getLegacyGitAccessor(settings, store, repoInfo, repoDir, rev, options); - auto narHashLegacy = - fetchToStore2(settings, store, {accessorLegacy}, FetchMode::DryRun, input.getName()).second; - if (expectedNarHash == narHashLegacy) { + + return accessor; + }; + + /* Use Nix 2.19 semantics if requested. If we're a submodule, follow whatever the top-level + repo is being exported with instead of the setting. */ + auto legacyExport = getLegacyExportAttr(input); + bool useLegacy = legacyExport.value_or(settings.nix219Compat) && !options.smudgeLfs; + + if (useLegacy && !legacyExport) + warn("Using Nix 2.19 semantics to export Git repository '%s'.", input.to_string()); + + auto accessor = getTree(useLegacy); + + /* Backward compatibility hack: a lock may have been produced by a Nix version that used the + other export semantics. In particular, locks produced by Nix < 2.20 depend on Nix applying + Git filters, `export-ignore` or `export-subst`, while Nix >= 2.20 doesn't do those. So if + we get a NAR hash mismatch, try again using the other semantics. */ + if (expectedNarHash) { + auto narHash = fetchToStore2(settings, store, {accessor}, FetchMode::DryRun, input.getName()).second; + if (*expectedNarHash != narHash) { + auto accessorOther = getTree(!useLegacy); + auto narHashOther = + fetchToStore2(settings, store, {accessorOther}, FetchMode::DryRun, input.getName()).second; + if (*expectedNarHash == narHashOther) { + if (!useLegacy) warn( "Git input '%s' specifies a NAR hash '%s' that was created by Nix < 2.20.\n" "Nix >= 2.20 does not apply Git filters, `export-ignore` and `export-subst` by default, which changes the NAR hash.\n" "Please update the NAR hash to '%s'.", input.to_string(), expectedNarHash->to_string(HashFormat::SRI, true), - narHashNew.to_string(HashFormat::SRI, true)); - accessor = accessorLegacy; - } + narHash.to_string(HashFormat::SRI, true)); + accessor = accessorOther; } } } - /* If the repo has submodules, fetch them and return a mounted - input accessor consisting of the accessor for the top-level - repo and the accessors for the submodules. */ - if (options.submodules) { - std::map> mounts; - - for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, options.exportIgnore)) { - auto resolved = repo->resolveSubmoduleUrl(submodule.url); - debug( - "Git submodule %s: %s %s %s -> %s", - submodule.path, - submodule.url, - submodule.branch, - submoduleRev.gitRev(), - resolved); - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "git"); - attrs.insert_or_assign("url", resolved); - if (submodule.branch != "") { - // A special value of . is used to indicate that the name of the branch in the submodule - // should be the same name as the current branch in the current repository. - // https://git-scm.com/docs/gitmodules - if (submodule.branch == ".") { - attrs.insert_or_assign("ref", ref); - } else { - attrs.insert_or_assign("ref", submodule.branch); - } - } - attrs.insert_or_assign("rev", submoduleRev.gitRev()); - attrs.insert_or_assign("exportIgnore", Explicit{options.exportIgnore}); - attrs.insert_or_assign("submodules", Explicit{true}); - attrs.insert_or_assign("lfs", Explicit{options.smudgeLfs}); - attrs.insert_or_assign("allRefs", Explicit{true}); - auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs)); - auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); - submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string(true) + "»"); - mounts.insert_or_assign(submodule.path, submoduleAccessor); - } - - if (!mounts.empty()) { - auto newFingerprint = accessor->getFingerprint(CanonPath::root).second->append(";s"); - mounts.insert_or_assign(CanonPath::root, accessor); - accessor = makeMountedSourceAccessor(std::move(mounts)); - accessor->fingerprint = newFingerprint; - } - } - assert(!origRev || origRev == rev); return {{accessor, std::move(input)}}; diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 6ffc7372df0f..f2926e6766ed 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -32,6 +32,13 @@ struct GitAccessorOptions bool smudgeLfs = false; bool submodules = false; // Currently implemented in GitInputScheme rather than GitAccessor + /** + * Whether to export the repository using Nix < 2.20 semantics, i.e. using `git archive` or + * `git checkout` (which apply Git filters, `export-ignore` and `export-subst`) rather than + * libgit2. Currently implemented in GitInputScheme rather than GitAccessor. + */ + bool legacy = false; + std::string makeFingerprint(const Hash & rev) const; }; diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh index d3f9e4da57bb..db34dedfc00d 100755 --- a/tests/functional/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -265,3 +265,62 @@ test_gitlink_without_gitmodules() { done } test_gitlink_without_gitmodules + +# The backward compatibility hack for Nix < 2.20 locks must also work for repos +# fetched with `submodules = true`. Since the NAR hash of such a repo covers the +# submodule contents, deciding whether a lock was produced with Nix < 2.20 +# semantics requires hashing the *mounted* tree, and the submodules have to be +# exported using those same semantics. +test_legacy_export_with_submodules() { + local root=$TEST_ROOT/legacySubmodulesRoot + local sub=$TEST_ROOT/legacySubmodulesSub + + rm -rf "$TEST_HOME"/.cache/nix + + createGitRepo "$sub" + # `text eol=crlf` is a Git filter, so it's applied by `git checkout` (which is + # what Nix < 2.20 used for repos with submodules) but not by libgit2. + printf "crlf text eol=crlf\n" > "$sub"/.gitattributes + printf "Hello\nWorld\n" > "$sub"/crlf + git -C "$sub" add .gitattributes crlf + git -C "$sub" commit -m "Add crlf" + + createGitRepo "$root" + git -C "$root" submodule add "$sub" sub + git -C "$root" commit -m "Add submodule" + + local rev + rev=$(git -C "$root" rev-parse HEAD) + + local input="{ type = \"git\"; url = \"file://$root\"; rev = \"$rev\"; submodules = true; }" + + # Determine the two NAR hashes in a throwaway store: `builtins.fetchTree` marks + # inputs with a `narHash` as final, so if a tree with the expected hash is + # already in the store, it is returned without running the fetcher at all, + # which would mask the behaviour we want to test below. + local scratch=$TEST_ROOT/legacySubmodulesStore + local legacyHash modernHash + legacyHash=$(nix eval --store "$scratch" --nix-219-compat --raw --expr "(builtins.fetchTree $input).narHash") + modernHash=$(nix eval --store "$scratch" --raw --expr "(builtins.fetchTree $input).narHash") + + # If the Git filter in the submodule didn't make a difference, this test + # wouldn't be testing anything. + [[ $legacyHash != "$modernHash" ]] + + # A NAR hash produced by Nix < 2.20 must still be accepted by default (with a + # warning), and must yield the filtered submodule contents. + expectStderr 0 nix eval --expr \ + "let tree = builtins.fetchTree ($input // { narHash = \"$legacyHash\"; }); in assert builtins.readFile \"\${tree}/sub/crlf\" == \"Hello\r\nWorld\r\n\"; true" \ + | grepQuiet "Please update the NAR hash to '$modernHash'" + + # Conversely, a NAR hash produced by Nix >= 2.20 must be accepted even when + # `nix-219-compat` is enabled. + nix eval --nix-219-compat --expr \ + "let tree = builtins.fetchTree ($input // { narHash = \"$modernHash\"; }); in assert builtins.readFile \"\${tree}/sub/crlf\" == \"Hello\nWorld\n\"; true" + + # A NAR hash that matches neither must still be an error. + expectStderr 102 nix eval --expr \ + "builtins.fetchTree ($input // { narHash = \"sha256-DLDvcwdcwCxnuPTxSQ6gLAyopB20lD0bOQoQB3i2hsA=\"; })" \ + | grepQuiet "NAR hash mismatch" +} +test_legacy_export_with_submodules