feat(bundler): support git gems

Signed-off-by: Roman Blanco <rblanco@redhat.com>

Author: romanblanco

hermeto/core/package_managers/bundler/main.py

@@ -28,14 +28,12 @@
 log = logging.getLogger(__name__)
 
 CONFIG_OVERRIDE = "bundler/config_override"
+GIT_CONFIG_OVERRIDE = "bundler/git_config"
 
 
 def fetch_bundler_source(request: Request) -> RequestOutput:
     """Resolve and process all bundler packages."""
     components: list[Component] = []
-    environment_variables: list[EnvironmentVariable] = (
-        _prepare_environment_variables_for_hermetic_build()
-    )
     project_files: list[ProjectFile] = []
     git_paths = []
 
@@ -48,7 +46,11 @@ def fetch_bundler_source(request: Request) -> RequestOutput:
         )
         components.extend(_comps)
         git_paths.extend(_git_paths)
-    project_files.append(
+
+    environment_variables: list[EnvironmentVariable] = (
+        _prepare_environment_variables_for_hermetic_build(has_git_deps=bool(git_paths))
+    )
+    project_files.extend(
         _prepare_for_hermetic_build(request.source_dir, request.output_dir, git_paths)
     )
 
@@ -63,17 +65,17 @@ def fetch_bundler_source(request: Request) -> RequestOutput:
     )
 
 
-# Aliases for git dependency name and git dependency name as
-# it is written to file system:
+# Aliases for git dependency name, file system name, and remote URL:
 DepName = str
 FSDepName = str
+DepURL = str
 
 
 def _resolve_bundler_package(
     package_dir: RootedPath,
     output_dir: RootedPath,
     binary_filters: BundlerBinaryFilters | None = None,
-) -> tuple[list[Component], list[tuple[DepName, FSDepName]]]:
+) -> tuple[list[Component], list[tuple[DepName, FSDepName, DepURL]]]:
     """Process a request for a single bundler package."""
     deps_dir = output_dir.join_within_root("deps", "bundler")
     deps_dir.path.mkdir(parents=True, exist_ok=True)
@@ -104,7 +106,7 @@ def _resolve_bundler_package(
         else:
             properties = []
         if isinstance(dep, GitDependency):
-            git_paths.append((dep.name, dep.repo_name + "-" + dep.ref[:12]))
+            git_paths.append((dep.name, dep.repo_name + "-" + dep.ref[:12], str(dep.url)))
 
         c = Component(name=dep.name, version=dep.version, purl=dep.purl, properties=properties)
         components.append(c)
@@ -183,16 +185,28 @@ def _get_repo_name_from_origin_remote(package_dir: RootedPath) -> str:
     return str(resolved_path)
 
 
-def _prepare_environment_variables_for_hermetic_build() -> list[EnvironmentVariable]:
-    return [
+def _prepare_environment_variables_for_hermetic_build(
+    has_git_deps: bool = False,
+) -> list[EnvironmentVariable]:
+    env_vars = [
         # Contains path to a directory where a new config could be found.
         EnvironmentVariable(name="BUNDLE_APP_CONFIG", value="${output_dir}/" + CONFIG_OVERRIDE),
     ]
+    if has_git_deps:
+        # Redirect git remote URLs to pre-fetched local clones so that bundler
+        # installs git gems into BUNDLE_PATH (not in-place from the clone).
+        # See: https://git-scm.com/docs/git-config#Documentation/git-config.txt-urlltbasegtinsteadOf
+        env_vars.append(
+            EnvironmentVariable(
+                name="GIT_CONFIG_GLOBAL", value="${output_dir}/" + GIT_CONFIG_OVERRIDE
+            )
+        )
+    return env_vars
 
 
 def _prepare_for_hermetic_build(
     source_dir: RootedPath, output_dir: RootedPath, git_paths: list | None = None
-) -> ProjectFile:
+) -> list[ProjectFile]:
     """Prepare a package for hermetic build by injecting necessary config."""
     potential_bundle_config = source_dir.join_within_root(".bundle/config").path
     hermetic_config = dedent(
@@ -205,30 +219,6 @@ def _prepare_for_hermetic_build(
         BUNDLE_VERSION: "system"
     """
     )
-    # Note: if a package depends on a git revision then the following variables
-    # are necessary for a hermetic build:
-    #  BUNDLE_DISABLE_LOCAL_BRANCH_CHECK
-    #  BUNDLE_DISABLE_LOCAL_REVISION_CHECK
-    # because otherwise some (potentially all, depending on exact set of
-    # ecosystem components versions, environment variables and celestial
-    # alignment) Bundler versions will try to fetch the latest changes of the
-    # remotes which may be present even when instructed not to with --local
-    # flag.
-    # See https://bundler.io/guides/git.html#local-git-repos for details.
-    # (or https://github.com/rubygems/bundler-site/blob/
-    #             9ff3b76e9866524ecefe165633ffb547f0004a99/source/guides/git.html.md
-    # if the link above ceases to exist).
-    if git_paths is not None:
-        hermetic_config += 'BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true"\n'
-        hermetic_config += 'BUNDLE_DISABLE_LOCAL_REVISION_CHECK: "true"\n'
-        for packname, dirname in git_paths:
-            # "-" in variable names is deprecated in Bundler and now generates
-            # a warning and a suggestion to replace all dashes with triple
-            # underscores. Package names sometimes contain dashes:
-            varname = "BUNDLE_LOCAL." + packname.upper().replace("-", "___")
-            location = "${output_dir}/deps/bundler/" + dirname
-            config_entry = varname + f': "{location}"'
-            hermetic_config += f"{config_entry}\n"
     if potential_bundle_config.is_file():
         config_data = potential_bundle_config.read_text()
         config_data += hermetic_config
@@ -241,4 +231,35 @@ def _prepare_for_hermetic_build(
     else:
         config_data = hermetic_config
     overriding_bundler_config_path = output_dir.join_within_root(CONFIG_OVERRIDE, "config").path
-    return ProjectFile(abspath=overriding_bundler_config_path, template=config_data)
+    project_files = [ProjectFile(abspath=overriding_bundler_config_path, template=config_data)]
+
+    if git_paths:
+        # Redirect each git remote URL to its pre-fetched local clone via a git
+        # url.insteadOf rewrite.  This causes bundler to clone from the local
+        # filesystem (no network) and then check out the gem into BUNDLE_PATH,
+        # so the gem is present in the runtime image after a multi-stage build.
+        #
+        # Using BUNDLE_LOCAL.* instead would make bundler call set_install_path!
+        # with the clone directory, causing it to use the gem in-place and never
+        # copy it into BUNDLE_PATH — leaving the gem absent from the runtime image.
+        url_to_dirs: dict[str, set[str]] = {}
+        for _packname, dirname, url in git_paths:
+            url_to_dirs.setdefault(url, set()).add(dirname)
+        for url, dirs in url_to_dirs.items():
+            if len(dirs) > 1:
+                raise UnsupportedFeature(
+                    f"Multiple git dependencies point to the same repository ({url}) "
+                    f"but use different revisions: {', '.join(sorted(dirs))}. "
+                    "This is not supported because git's url.insteadOf redirect "
+                    "can only map a repository URL to a single local clone."
+                )
+        git_config_lines = []
+        for _packname, dirname, url in git_paths:
+            clone_file_url = "file://${output_dir}/deps/bundler/" + dirname + "/"
+            git_config_lines.append(f'[url "{clone_file_url}"]\n\tinsteadOf = {url}\n')
+        git_config_path = output_dir.join_within_root(GIT_CONFIG_OVERRIDE).path
+        project_files.append(
+            ProjectFile(abspath=git_config_path, template="".join(git_config_lines))
+        )
+
+    return project_files

tests/integration/test_data/bundler_e2e/.build-config.yaml

@@ -1,6 +1,8 @@
 environment_variables:
 - name: BUNDLE_APP_CONFIG
   value: ${output_dir}/bundler/config_override
+- name: GIT_CONFIG_GLOBAL
+  value: ${output_dir}/bundler/git_config
 project_files:
 - abspath: ${test_case_tmp_path}/hermeto-output/bundler/config_override/config
   template: |2
@@ -11,6 +13,7 @@ project_files:
     BUNDLE_ALLOW_OFFLINE_INSTALL: "true"
     BUNDLE_DISABLE_VERSION_CHECK: "true"
     BUNDLE_VERSION: "system"
-    BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true"
-    BUNDLE_DISABLE_LOCAL_REVISION_CHECK: "true"
-    BUNDLE_LOCAL.JSON___SCHEMA: "${output_dir}/deps/bundler/json-schema-26487618a684"
+- abspath: ${test_case_tmp_path}/hermeto-output/bundler/git_config
+  template: |
+    [url "file://${output_dir}/deps/bundler/json-schema-26487618a684/"]
+    	insteadOf = https://github.com/3scale/json-schema

tests/integration/test_data/bundler_e2e_missing_gemspec/.build-config.yaml

@@ -1,6 +1,8 @@
 environment_variables:
 - name: BUNDLE_APP_CONFIG
   value: ${output_dir}/bundler/config_override
+- name: GIT_CONFIG_GLOBAL
+  value: ${output_dir}/bundler/git_config
 project_files:
 - abspath: ${test_case_tmp_path}/hermeto-output/bundler/config_override/config
   template: |2
@@ -11,6 +13,7 @@ project_files:
     BUNDLE_ALLOW_OFFLINE_INSTALL: "true"
     BUNDLE_DISABLE_VERSION_CHECK: "true"
     BUNDLE_VERSION: "system"
-    BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true"
-    BUNDLE_DISABLE_LOCAL_REVISION_CHECK: "true"
-    BUNDLE_LOCAL.JSON___SCHEMA: "${output_dir}/deps/bundler/json-schema-26487618a684"
+- abspath: ${test_case_tmp_path}/hermeto-output/bundler/git_config
+  template: |
+    [url "file://${output_dir}/deps/bundler/json-schema-26487618a684/"]
+    	insteadOf = https://github.com/3scale/json-schema

tests/unit/package_managers/bundler/test_main.py

@@ -6,7 +6,7 @@
 from git.repo import Repo
 
 from hermeto.core.constants import Mode
-from hermeto.core.errors import NotAGitRepo, PackageRejected
+from hermeto.core.errors import NotAGitRepo, PackageRejected, UnsupportedFeature
 from hermeto.core.package_managers.bundler.main import (
     _get_main_package_name_and_version,
     _get_repo_name_from_origin_remote,
@@ -83,7 +83,8 @@ def test_prepare_for_hermetic_build_injects_necessary_variable_into_empty_config
 
     result = _prepare_for_hermetic_build(rooted_tmp_path, rooted_tmp_path)
 
-    assert result.template == expected_config_contents
+    assert len(result) == 1
+    assert result[0].template == expected_config_contents
 
 
 def test_prepare_for_hermetic_build_injects_necessary_variable_into_existing_config(
@@ -115,7 +116,8 @@ def test_prepare_for_hermetic_build_injects_necessary_variable_into_existing_con
 
     result = _prepare_for_hermetic_build(rooted_tmp_path, rooted_tmp_path)
 
-    assert result.template == existing_preamble + expected_config_contents
+    assert len(result) == 1
+    assert result[0].template == existing_preamble + expected_config_contents
 
 
 def test_prepare_for_hermetic_build_injects_necessary_variable_into_existing_alternate_config(
@@ -152,7 +154,46 @@ def test_prepare_for_hermetic_build_injects_necessary_variable_into_existing_alt
         ge.return_value = str(expected_alternate_config_location.parent)
         result = _prepare_for_hermetic_build(rooted_tmp_path, rooted_tmp_path)
 
-    assert result.template == existing_preamble + expected_alternate_config_contents
+    assert len(result) == 1
+    assert result[0].template == existing_preamble + expected_alternate_config_contents
+
+
+def test_prepare_for_hermetic_build_generates_git_config_for_git_deps(
+    rooted_tmp_path: RootedPath,
+) -> None:
+    git_paths = [
+        (
+            "yabeda-activejob",
+            "yabeda-activejob-1894d97a925d",
+            "https://github.com/org/yabeda-activejob.git",
+        ),
+        ("my-gem", "my-gem-abc123def456", "https://github.com/org/my-gem.git"),
+    ]
+
+    result = _prepare_for_hermetic_build(rooted_tmp_path, rooted_tmp_path, git_paths=git_paths)
+
+    assert len(result) == 2
+    expected_git_config = dedent(
+        """\
+        [url "file://${output_dir}/deps/bundler/yabeda-activejob-1894d97a925d/"]
+        \tinsteadOf = https://github.com/org/yabeda-activejob.git
+        [url "file://${output_dir}/deps/bundler/my-gem-abc123def456/"]
+        \tinsteadOf = https://github.com/org/my-gem.git
+        """
+    )
+    assert result[1].template == expected_git_config
+
+
+def test_prepare_for_hermetic_build_raises_on_duplicate_git_url(
+    rooted_tmp_path: RootedPath,
+) -> None:
+    git_paths = [
+        ("gem-a", "gem-a-aabbccdd1234", "https://github.com/org/monorepo.git"),
+        ("gem-b", "gem-b-eeff00112233", "https://github.com/org/monorepo.git"),
+    ]
+
+    with pytest.raises(UnsupportedFeature, match="same repository"):
+        _prepare_for_hermetic_build(rooted_tmp_path, rooted_tmp_path, git_paths=git_paths)
 
 
 @mock.patch("hermeto.core.package_managers.bundler.main.get_repo_id")