feat(bundler): support git gems

Author: romanblanco

hermeto/core/package_managers/bundler/main.py

@@ -28,14 +28,12 @@
 log = logging.getLogger(__name__)
 
 CONFIG_OVERRIDE = "bundler/config_override"
+GIT_CONFIG_OVERRIDE = "bundler/git_config"
 
 
 def fetch_bundler_source(request: Request) -> RequestOutput:
     """Resolve and process all bundler packages."""
     components: list[Component] = []
-    environment_variables: list[EnvironmentVariable] = (
-        _prepare_environment_variables_for_hermetic_build()
-    )
     project_files: list[ProjectFile] = []
     git_paths = []
 
@@ -48,7 +46,11 @@ def fetch_bundler_source(request: Request) -> RequestOutput:
         )
         components.extend(_comps)
         git_paths.extend(_git_paths)
-    project_files.append(
+
+    environment_variables: list[EnvironmentVariable] = (
+        _prepare_environment_variables_for_hermetic_build(has_git_deps=bool(git_paths))
+    )
+    project_files.extend(
         _prepare_for_hermetic_build(request.source_dir, request.output_dir, git_paths)
     )
 
@@ -63,17 +65,17 @@ def fetch_bundler_source(request: Request) -> RequestOutput:
     )
 
 
-# Aliases for git dependency name and git dependency name as
-# it is written to file system:
+# Aliases for git dependency name, file system name, and remote URL:
 DepName = str
 FSDepName = str
+DepURL = str
 
 
 def _resolve_bundler_package(
     package_dir: RootedPath,
     output_dir: RootedPath,
     binary_filters: BundlerBinaryFilters | None = None,
-) -> tuple[list[Component], list[tuple[DepName, FSDepName]]]:
+) -> tuple[list[Component], list[tuple[DepName, FSDepName, DepURL]]]:
     """Process a request for a single bundler package."""
     deps_dir = output_dir.join_within_root("deps", "bundler")
     deps_dir.path.mkdir(parents=True, exist_ok=True)
@@ -104,7 +106,7 @@ def _resolve_bundler_package(
         else:
             properties = []
         if isinstance(dep, GitDependency):
-            git_paths.append((dep.name, dep.repo_name + "-" + dep.ref[:12]))
+            git_paths.append((dep.name, dep.repo_name + "-" + dep.ref[:12], str(dep.url)))
 
         c = Component(name=dep.name, version=dep.version, purl=dep.purl, properties=properties)
         components.append(c)
@@ -183,16 +185,28 @@ def _get_repo_name_from_origin_remote(package_dir: RootedPath) -> str:
     return str(resolved_path)
 
 
-def _prepare_environment_variables_for_hermetic_build() -> list[EnvironmentVariable]:
-    return [
+def _prepare_environment_variables_for_hermetic_build(
+    has_git_deps: bool = False,
+) -> list[EnvironmentVariable]:
+    env_vars = [
         # Contains path to a directory where a new config could be found.
         EnvironmentVariable(name="BUNDLE_APP_CONFIG", value="${output_dir}/" + CONFIG_OVERRIDE),
     ]
+    if has_git_deps:
+        # Redirect git remote URLs to pre-fetched local clones so that bundler
+        # installs git gems into BUNDLE_PATH (not in-place from the clone).
+        # See: https://git-scm.com/docs/git-config#Documentation/git-config.txt-urlltbasegtinsteadOf
+        env_vars.append(
+            EnvironmentVariable(
+                name="GIT_CONFIG_GLOBAL", value="${output_dir}/" + GIT_CONFIG_OVERRIDE
+            )
+        )
+    return env_vars
 
 
 def _prepare_for_hermetic_build(
     source_dir: RootedPath, output_dir: RootedPath, git_paths: list | None = None
-) -> ProjectFile:
+) -> list[ProjectFile]:
     """Prepare a package for hermetic build by injecting necessary config."""
     potential_bundle_config = source_dir.join_within_root(".bundle/config").path
     hermetic_config = dedent(
@@ -205,30 +219,6 @@ def _prepare_for_hermetic_build(
         BUNDLE_VERSION: "system"
     """
     )
-    # Note: if a package depends on a git revision then the following variables
-    # are necessary for a hermetic build:
-    #  BUNDLE_DISABLE_LOCAL_BRANCH_CHECK
-    #  BUNDLE_DISABLE_LOCAL_REVISION_CHECK
-    # because otherwise some (potentially all, depending on exact set of
-    # ecosystem components versions, environment variables and celestial
-    # alignment) Bundler versions will try to fetch the latest changes of the
-    # remotes which may be present even when instructed not to with --local
-    # flag.
-    # See https://bundler.io/guides/git.html#local-git-repos for details.
-    # (or https://github.com/rubygems/bundler-site/blob/
-    #             9ff3b76e9866524ecefe165633ffb547f0004a99/source/guides/git.html.md
-    # if the link above ceases to exist).
-    if git_paths is not None:
-        hermetic_config += 'BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true"\n'
-        hermetic_config += 'BUNDLE_DISABLE_LOCAL_REVISION_CHECK: "true"\n'
-        for packname, dirname in git_paths:
-            # "-" in variable names is deprecated in Bundler and now generates
-            # a warning and a suggestion to replace all dashes with triple
-            # underscores. Package names sometimes contain dashes:
-            varname = "BUNDLE_LOCAL." + packname.upper().replace("-", "___")
-            location = "${output_dir}/deps/bundler/" + dirname
-            config_entry = varname + f': "{location}"'
-            hermetic_config += f"{config_entry}\n"
     if potential_bundle_config.is_file():
         config_data = potential_bundle_config.read_text()
         config_data += hermetic_config
@@ -241,4 +231,24 @@ def _prepare_for_hermetic_build(
     else:
         config_data = hermetic_config
     overriding_bundler_config_path = output_dir.join_within_root(CONFIG_OVERRIDE, "config").path
-    return ProjectFile(abspath=overriding_bundler_config_path, template=config_data)
+    project_files = [ProjectFile(abspath=overriding_bundler_config_path, template=config_data)]
+
+    if git_paths:
+        # Redirect each git remote URL to its pre-fetched local clone via a git
+        # url.insteadOf rewrite.  This causes bundler to clone from the local
+        # filesystem (no network) and then check out the gem into BUNDLE_PATH,
+        # so the gem is present in the runtime image after a multi-stage build.
+        #
+        # Using BUNDLE_LOCAL.* instead would make bundler call set_install_path!
+        # with the clone directory, causing it to use the gem in-place and never
+        # copy it into BUNDLE_PATH — leaving the gem absent from the runtime image.
+        git_config_lines = []
+        for _packname, dirname, url in git_paths:
+            clone_file_url = "file://${output_dir}/deps/bundler/" + dirname + "/"
+            git_config_lines.append(f'[url "{clone_file_url}"]\n\tinsteadOf = {url}\n')
+        git_config_path = output_dir.join_within_root(GIT_CONFIG_OVERRIDE).path
+        project_files.append(
+            ProjectFile(abspath=git_config_path, template="".join(git_config_lines))
+        )
+
+    return project_files

tests/unit/package_managers/bundler/test_main.py

@@ -83,7 +83,8 @@ def test_prepare_for_hermetic_build_injects_necessary_variable_into_empty_config
 
     result = _prepare_for_hermetic_build(rooted_tmp_path, rooted_tmp_path)
 
-    assert result.template == expected_config_contents
+    assert len(result) == 1
+    assert result[0].template == expected_config_contents
 
 
 def test_prepare_for_hermetic_build_injects_necessary_variable_into_existing_config(
@@ -115,7 +116,8 @@ def test_prepare_for_hermetic_build_injects_necessary_variable_into_existing_con
 
     result = _prepare_for_hermetic_build(rooted_tmp_path, rooted_tmp_path)
 
-    assert result.template == existing_preamble + expected_config_contents
+    assert len(result) == 1
+    assert result[0].template == existing_preamble + expected_config_contents
 
 
 def test_prepare_for_hermetic_build_injects_necessary_variable_into_existing_alternate_config(
@@ -152,7 +154,26 @@ def test_prepare_for_hermetic_build_injects_necessary_variable_into_existing_alt
         ge.return_value = str(expected_alternate_config_location.parent)
         result = _prepare_for_hermetic_build(rooted_tmp_path, rooted_tmp_path)
 
-    assert result.template == existing_preamble + expected_alternate_config_contents
+    assert len(result) == 1
+    assert result[0].template == existing_preamble + expected_alternate_config_contents
+
+
+def test_prepare_for_hermetic_build_generates_git_config_for_git_deps(
+    rooted_tmp_path: RootedPath,
+) -> None:
+    git_paths = [
+        ("yabeda-activejob", "yabeda-activejob-1894d97a925d", "https://github.com/org/yabeda-activejob.git"),
+        ("my-gem", "my-gem-abc123def456", "https://github.com/org/my-gem.git"),
+    ]
+
+    result = _prepare_for_hermetic_build(rooted_tmp_path, rooted_tmp_path, git_paths=git_paths)
+
+    assert len(result) == 2
+    git_config = result[1].template
+    assert '[url "file://${output_dir}/deps/bundler/yabeda-activejob-1894d97a925d/"]' in git_config
+    assert "\tinsteadOf = https://github.com/org/yabeda-activejob.git" in git_config
+    assert '[url "file://${output_dir}/deps/bundler/my-gem-abc123def456/"]' in git_config
+    assert "\tinsteadOf = https://github.com/org/my-gem.git" in git_config
 
 
 @mock.patch("hermeto.core.package_managers.bundler.main.get_repo_id")