feat(bundler): support git gems

Signed-off-by: Roman Blanco <rblanco@redhat.com>

Author: romanblanco

hermeto/core/package_managers/bundler/gem_models.py

@@ -146,17 +146,13 @@ def download_to(self, deps_dir: RootedPath) -> None:
         git_repo_path.path.mkdir(parents=True)
 
         log.info("Cloning git repository %s", self.url)
-        repo = GitRepo.clone_from(
+        GitRepo.clone_from(
             url=str(self.url),
             to_path=git_repo_path.path,
+            bare=True,
             env={"GIT_TERMINAL_PROMPT": "0"},
         )
 
-        if self.branch is not None:
-            repo.git.checkout(self.branch)
-
-        repo.git.reset("--hard", self.ref)
-
 
 class PathDependency(_GemMetadata):
     """

hermeto/core/package_managers/bundler/main.py

@@ -33,9 +33,6 @@
 def fetch_bundler_source(request: Request) -> RequestOutput:
     """Resolve and process all bundler packages."""
     components: list[Component] = []
-    environment_variables: list[EnvironmentVariable] = (
-        _prepare_environment_variables_for_hermetic_build()
-    )
     project_files: list[ProjectFile] = []
     git_paths = []
 
@@ -48,9 +45,11 @@ def fetch_bundler_source(request: Request) -> RequestOutput:
         )
         components.extend(_comps)
         git_paths.extend(_git_paths)
-    project_files.append(
-        _prepare_for_hermetic_build(request.source_dir, request.output_dir, git_paths)
+
+    environment_variables: list[EnvironmentVariable] = (
+        _prepare_environment_variables_for_hermetic_build(git_paths)
     )
+    project_files.append(_prepare_for_hermetic_build(request.source_dir, request.output_dir))
 
     annotations = []
     if backend_annotation := create_backend_annotation(components, "bundler"):
@@ -63,17 +62,17 @@ def fetch_bundler_source(request: Request) -> RequestOutput:
     )
 
 
-# Aliases for git dependency name and git dependency name as
-# it is written to file system:
+# Aliases for git dependency name, file system name, and remote URL:
 DepName = str
 FSDepName = str
+DepURL = str
 
 
 def _resolve_bundler_package(
     package_dir: RootedPath,
     output_dir: RootedPath,
     binary_filters: BundlerBinaryFilters | None = None,
-) -> tuple[list[Component], list[tuple[DepName, FSDepName]]]:
+) -> tuple[list[Component], list[tuple[DepName, FSDepName, DepURL]]]:
     """Process a request for a single bundler package."""
     deps_dir = output_dir.join_within_root("deps", "bundler")
     deps_dir.path.mkdir(parents=True, exist_ok=True)
@@ -104,7 +103,7 @@ def _resolve_bundler_package(
         else:
             properties = []
         if isinstance(dep, GitDependency):
-            git_paths.append((dep.name, dep.repo_name + "-" + dep.ref[:12]))
+            git_paths.append((dep.name, dep.repo_name + "-" + dep.ref[:12], str(dep.url)))
 
         c = Component(name=dep.name, version=dep.version, purl=dep.purl, properties=properties)
         components.append(c)
@@ -183,16 +182,50 @@ def _get_repo_name_from_origin_remote(package_dir: RootedPath) -> str:
     return str(resolved_path)
 
 
-def _prepare_environment_variables_for_hermetic_build() -> list[EnvironmentVariable]:
-    return [
+def _prepare_environment_variables_for_hermetic_build(
+    git_paths: list[tuple[DepName, FSDepName, DepURL]] | None = None,
+) -> list[EnvironmentVariable]:
+    env_vars = [
         # Contains path to a directory where a new config could be found.
         EnvironmentVariable(name="BUNDLE_APP_CONFIG", value="${output_dir}/" + CONFIG_OVERRIDE),
     ]
-
-
-def _prepare_for_hermetic_build(
-    source_dir: RootedPath, output_dir: RootedPath, git_paths: list | None = None
-) -> ProjectFile:
+    if git_paths:
+        # Redirect git remote URLs to pre-fetched local clones via git's
+        # GIT_CONFIG_COUNT/KEY/VALUE mechanism. This injects url.insteadOf
+        # entries without replacing the global git config.
+        # See: https://git-scm.com/docs/git-config#ENVIRONMENT
+        _check_for_duplicate_git_urls(git_paths)
+        env_vars.append(EnvironmentVariable(name="GIT_CONFIG_COUNT", value=str(len(git_paths))))
+        for idx, (_packname, dirname, url) in enumerate(git_paths):
+            clone_file_url = "file://${output_dir}/deps/bundler/" + dirname + "/"
+            env_vars.append(
+                EnvironmentVariable(
+                    name=f"GIT_CONFIG_KEY_{idx}",
+                    value=f"url.{clone_file_url}.insteadOf",
+                )
+            )
+            env_vars.append(EnvironmentVariable(name=f"GIT_CONFIG_VALUE_{idx}", value=url))
+    return env_vars
+
+
+def _check_for_duplicate_git_urls(
+    git_paths: list[tuple[DepName, FSDepName, DepURL]],
+) -> None:
+    """Raise if multiple git deps share the same URL with different revisions."""
+    url_to_dirs: dict[str, set[str]] = {}
+    for _packname, dirname, url in git_paths:
+        url_to_dirs.setdefault(url, set()).add(dirname)
+    for url, dirs in url_to_dirs.items():
+        if len(dirs) > 1:
+            raise UnsupportedFeature(
+                f"Multiple git dependencies point to the same repository ({url}) "
+                f"but use different revisions: {', '.join(sorted(dirs))}. "
+                "This is not supported because git's url.insteadOf redirect "
+                "can only map a repository URL to a single local clone."
+            )
+
+
+def _prepare_for_hermetic_build(source_dir: RootedPath, output_dir: RootedPath) -> ProjectFile:
     """Prepare a package for hermetic build by injecting necessary config."""
     potential_bundle_config = source_dir.join_within_root(".bundle/config").path
     hermetic_config = dedent(
@@ -205,30 +238,6 @@ def _prepare_for_hermetic_build(
         BUNDLE_VERSION: "system"
     """
     )
-    # Note: if a package depends on a git revision then the following variables
-    # are necessary for a hermetic build:
-    #  BUNDLE_DISABLE_LOCAL_BRANCH_CHECK
-    #  BUNDLE_DISABLE_LOCAL_REVISION_CHECK
-    # because otherwise some (potentially all, depending on exact set of
-    # ecosystem components versions, environment variables and celestial
-    # alignment) Bundler versions will try to fetch the latest changes of the
-    # remotes which may be present even when instructed not to with --local
-    # flag.
-    # See https://bundler.io/guides/git.html#local-git-repos for details.
-    # (or https://github.com/rubygems/bundler-site/blob/
-    #             9ff3b76e9866524ecefe165633ffb547f0004a99/source/guides/git.html.md
-    # if the link above ceases to exist).
-    if git_paths is not None:
-        hermetic_config += 'BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true"\n'
-        hermetic_config += 'BUNDLE_DISABLE_LOCAL_REVISION_CHECK: "true"\n'
-        for packname, dirname in git_paths:
-            # "-" in variable names is deprecated in Bundler and now generates
-            # a warning and a suggestion to replace all dashes with triple
-            # underscores. Package names sometimes contain dashes:
-            varname = "BUNDLE_LOCAL." + packname.upper().replace("-", "___")
-            location = "${output_dir}/deps/bundler/" + dirname
-            config_entry = varname + f': "{location}"'
-            hermetic_config += f"{config_entry}\n"
     if potential_bundle_config.is_file():
         config_data = potential_bundle_config.read_text()
         config_data += hermetic_config

tests/integration/test_data/bundler_e2e/.build-config.yaml

@@ -1,6 +1,12 @@
 environment_variables:
 - name: BUNDLE_APP_CONFIG
   value: ${output_dir}/bundler/config_override
+- name: GIT_CONFIG_COUNT
+  value: "1"
+- name: GIT_CONFIG_KEY_0
+  value: url.file://${output_dir}/deps/bundler/json-schema-26487618a684/.insteadOf
+- name: GIT_CONFIG_VALUE_0
+  value: https://github.com/3scale/json-schema
 project_files:
 - abspath: ${test_case_tmp_path}/hermeto-output/bundler/config_override/config
   template: |2
@@ -11,6 +17,3 @@ project_files:
     BUNDLE_ALLOW_OFFLINE_INSTALL: "true"
     BUNDLE_DISABLE_VERSION_CHECK: "true"
     BUNDLE_VERSION: "system"
-    BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true"
-    BUNDLE_DISABLE_LOCAL_REVISION_CHECK: "true"
-    BUNDLE_LOCAL.JSON___SCHEMA: "${output_dir}/deps/bundler/json-schema-26487618a684"

tests/integration/test_data/bundler_e2e_missing_gemspec/.build-config.yaml

@@ -1,6 +1,12 @@
 environment_variables:
 - name: BUNDLE_APP_CONFIG
   value: ${output_dir}/bundler/config_override
+- name: GIT_CONFIG_COUNT
+  value: "1"
+- name: GIT_CONFIG_KEY_0
+  value: url.file://${output_dir}/deps/bundler/json-schema-26487618a684/.insteadOf
+- name: GIT_CONFIG_VALUE_0
+  value: https://github.com/3scale/json-schema
 project_files:
 - abspath: ${test_case_tmp_path}/hermeto-output/bundler/config_override/config
   template: |2
@@ -11,6 +17,3 @@ project_files:
     BUNDLE_ALLOW_OFFLINE_INSTALL: "true"
     BUNDLE_DISABLE_VERSION_CHECK: "true"
     BUNDLE_VERSION: "system"
-    BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true"
-    BUNDLE_DISABLE_LOCAL_REVISION_CHECK: "true"
-    BUNDLE_LOCAL.JSON___SCHEMA: "${output_dir}/deps/bundler/json-schema-26487618a684"

tests/unit/package_managers/bundler/test_main.py

@@ -6,10 +6,12 @@
 from git.repo import Repo
 
 from hermeto.core.constants import Mode
-from hermeto.core.errors import NotAGitRepo, PackageRejected
+from hermeto.core.errors import NotAGitRepo, PackageRejected, UnsupportedFeature
+from hermeto.core.models.output import EnvironmentVariable
 from hermeto.core.package_managers.bundler.main import (
     _get_main_package_name_and_version,
     _get_repo_name_from_origin_remote,
+    _prepare_environment_variables_for_hermetic_build,
     _prepare_for_hermetic_build,
 )
 from hermeto.core.package_managers.bundler.parser import (
@@ -155,6 +157,59 @@ def test_prepare_for_hermetic_build_injects_necessary_variable_into_existing_alt
     assert result.template == existing_preamble + expected_alternate_config_contents
 
 
+def test_prepare_environment_variables_generates_git_config_entries_for_git_deps() -> None:
+    git_paths = [
+        (
+            "yabeda-activejob",
+            "yabeda-activejob-1894d97a925d",
+            "https://github.com/org/yabeda-activejob.git",
+        ),
+        ("my-gem", "my-gem-abc123def456", "https://github.com/org/my-gem.git"),
+    ]
+
+    result = _prepare_environment_variables_for_hermetic_build(git_paths)
+
+    assert EnvironmentVariable(name="GIT_CONFIG_COUNT", value="2") in result
+    assert (
+        EnvironmentVariable(
+            name="GIT_CONFIG_KEY_0",
+            value="url.file://${output_dir}/deps/bundler/yabeda-activejob-1894d97a925d/.insteadOf",
+        )
+        in result
+    )
+    assert (
+        EnvironmentVariable(
+            name="GIT_CONFIG_VALUE_0",
+            value="https://github.com/org/yabeda-activejob.git",
+        )
+        in result
+    )
+    assert (
+        EnvironmentVariable(
+            name="GIT_CONFIG_KEY_1",
+            value="url.file://${output_dir}/deps/bundler/my-gem-abc123def456/.insteadOf",
+        )
+        in result
+    )
+    assert (
+        EnvironmentVariable(
+            name="GIT_CONFIG_VALUE_1",
+            value="https://github.com/org/my-gem.git",
+        )
+        in result
+    )
+
+
+def test_prepare_environment_variables_raises_on_duplicate_git_url() -> None:
+    git_paths = [
+        ("gem-a", "gem-a-aabbccdd1234", "https://github.com/org/monorepo.git"),
+        ("gem-b", "gem-b-eeff00112233", "https://github.com/org/monorepo.git"),
+    ]
+
+    with pytest.raises(UnsupportedFeature, match="same repository"):
+        _prepare_environment_variables_for_hermetic_build(git_paths)
+
+
 @mock.patch("hermeto.core.package_managers.bundler.main.get_repo_id")
 def test_get_repo_name_raises_without_git_repo(
     mock_handle_get_repo_id: mock.Mock,

tests/unit/package_managers/bundler/test_parser.py

@@ -262,6 +262,7 @@ def test_download_git_dependency_works(
     mock_git_clone.assert_called_once_with(
         url=str(dep.url),
         to_path=dep_path,
+        bare=True,
         env={"GIT_TERMINAL_PROMPT": "0"},
     )
     assert dep_path.exists()
@@ -288,6 +289,7 @@ def test_download_duplicate_git_dependency_is_skipped(
     mock_git_clone.assert_called_once_with(
         url=str(dep.url),
         to_path=dep_path,
+        bare=True,
         env={"GIT_TERMINAL_PROMPT": "0"},
     )
     assert dep_path.exists()