diff --git a/docs/design/bundler.md b/docs/design/bundler.md index 61761fb77..757e4d3d1 100644 --- a/docs/design/bundler.md +++ b/docs/design/bundler.md @@ -271,7 +271,7 @@ nokogiri-1.16.6.gem tzinfo-2.0.6.gem ``` -Notice that all the `.gem` dependencies are kept in their original format, and Git dependencies are just plain clones +Notice that all the `.gem` dependencies are kept in their original format, and Git dependencies are bare git clones of the repository placed in a folder. For Git dependencies, the folder name must match this specific [format](https://github.com/rubygems/rubygems/blob/3da9b1dda0824d1d770780352bb1d3f287cb2df5/bundler/lib/bundler/source/git.rb#L130): @@ -279,9 +279,9 @@ of the repository placed in a folder. For Git dependencies, the folder name must "#{base_name}-#{shortref_for_path(revision)}" ``` -The name of the directory **must come from the Git URL**, not the actual name of the gem, and the cloned folder must -contain unpacked source code. Any other format will cause bundler to try to re-download the repository, causing the -build to fail. +The name of the directory **must come from the Git URL**, not the actual name of the gem. The clones are bare +repositories used as local remotes via git's `url.insteadOf` mechanism (see +[Offline installs involving git dependencies](#offline-installs-involving-git-dependencies)). ##### Multiple Gems in a single repository @@ -389,8 +389,6 @@ BUNDLE_DEPLOYMENT=true BUNDLE_NO_PRUNE=true BUNDLE_ALLOW_OFFLINE_INSTALL=true BUNDLE_DISABLE_VERSION_CHECK=true -BUNDLE_DISABLE_LOCAL_BRANCH_CHECK=true -BUNDLE_DISABLE_LOCAL_REVISION_CHECK=true ``` - **BUNDLE_CACHE_PATH**: The directory that Bundler will place cached gems in when running bundle package, and that @@ -415,16 +413,6 @@ package cache. - **BUNDLE_DISABLE_VERSION_CHECK**: Stop Bundler from checking if a newer Bundler version is available on rubygems.org. -- **BUNDLE_DISABLE_LOCAL_REVISION_CHECK**: Allow Bundler to use a local git override without a -branch specified in the Gemfile - -- **BUNDLE_DISABLE_LOCAL_BRANCH_CHECK**: Allow Bundler to use a local git override without checking -if the revision present in the lockfile is present in the repository. - -- **BUNDLE_LOCAL__**: Instead of checking out the remote git repository for GEM_NAME, -the local git directory override will be used. See below for more information on Bundler's git -dependency handling. - For more information, see Bundler's [documentation](https://bundler.io/v2.5/man/bundle-config.1.html). #### Offline installation pitfalls @@ -465,30 +453,29 @@ flag](https://github.com/rubygems/rubygems/issues/8265) via configuration option making use of the deployment mode.** ##### Offline installs involving git dependencies -Bundler seems to follow a different approach when it comes to git dependencies since in its default -configuration it always tries to fetch the source from the remote to ensure the application is built -against the correct branch/revision. This argument is indirectly supported by the -[docs](https://bundler.io/guides/deploying.html#deploying-your-application), more specifically: - -> If you have run bundle pack, checked in the vendor/cache directory, and do not have any git gems, -Bundler will not contact the internet while installing your bundle. - -This is a problem for hermetic builds and as such setting `BUNDLE_DEPLOYMENT` alone doesn't help -and we need more settings. In order to overcome this behavioral trait, we need to follow the -recommendation in the [config](https://bundler.io/v2.5/man/bundle-config.1.html#LOCAL-GIT-REPOS) -docs and override each git dependency with the location on the disk we fetched the git dependency -to and tell bundler about it with the `BUNDLE_LOCAL__` configuration key. -However, this still isn't enough for Bundler to honour offline installs with git dependencies, -because then it's trying to enforce further checks as outlined in the -[docs](https://bundler.io/v2.5/man/bundle-config.1.html#LOCAL-GIT-REPOS): - ->Bundler does many checks to ensure a developer won't work with invalid references. Particularly, ->we force a developer to specify a branch in the Gemfile in order to use this feature. If the ->branch specified in the Gemfile and the current branch in the local git repository do not ->match, Bundler will abort. - -Therefore, we additionally need to enforce both `BUNDLE_DISABLE_LOCAL_BRANCH_CHECK` and -`BUNDLE_DISABLE_LOCAL_REVISION_CHECK`. +Bundler always tries to fetch git dependencies from the remote, so `BUNDLE_DEPLOYMENT` alone +doesn't prevent network access. + +To solve this without coupling to bundler's internal cache format +(which has [changed across versions](https://github.com/ruby/rubygems/commit/7d6b6316)), we use git's +[`url.insteadOf`](https://git-scm.com/docs/git-config#Documentation/git-config.txt-urlltbasegtinsteadOf) +via `GIT_CONFIG_COUNT`/`KEY`/`VALUE` environment variables to redirect each remote URL to the +pre-fetched local bare clone. This operates at the git transport layer — bundler runs its standard +flow but git silently fetches from disk instead of the network. + +For example, given a git dependency on `https://github.com/3scale/json-schema`, hermeto sets: + +``` +GIT_CONFIG_COUNT=2 +GIT_CONFIG_KEY_0=url.file:///output/deps/bundler/json-schema-26487618a684/.insteadOf +GIT_CONFIG_VALUE_0=https://github.com/3scale/json-schema +GIT_CONFIG_KEY_1=protocol.file.allow +GIT_CONFIG_VALUE_1=always +``` + +The `protocol.file.allow=always` entry ensures git permits the `file://` protocol after the URL +rewrite. See the [git documentation](https://git-scm.com/docs/git-config#Documentation/git-config.txt-protocolallow) +for details. ##### Offline installation using deployment mode Deployment mode is a way of vendoring one's code along with the dependencies. diff --git a/hermeto/core/package_managers/bundler/gem_models.py b/hermeto/core/package_managers/bundler/gem_models.py index 0a37be464..a909e74de 100644 --- a/hermeto/core/package_managers/bundler/gem_models.py +++ b/hermeto/core/package_managers/bundler/gem_models.py @@ -100,12 +100,10 @@ class GitDependency(_GemMetadata): Attributes: url: The URL of the git repository. - branch: The branch to checkout. ref: Commit hash. """ url: AcceptedUrl - branch: str | None = None ref: AcceptedGitRef @cached_property @@ -134,17 +132,13 @@ def download_to(self, deps_dir: RootedPath) -> None: git_repo_path.path.mkdir(parents=True) log.info("Cloning git repository %s", self.url) - repo = GitRepo.clone_from( + GitRepo.clone_from( url=str(self.url), to_path=git_repo_path.path, + bare=True, env={"GIT_TERMINAL_PROMPT": "0"}, ) - if self.branch is not None: - repo.git.checkout(self.branch) - - repo.git.reset("--hard", self.ref) - class PathDependency(_GemMetadata): """ diff --git a/hermeto/core/package_managers/bundler/main.py b/hermeto/core/package_managers/bundler/main.py index e733ac2ab..3c747de86 100644 --- a/hermeto/core/package_managers/bundler/main.py +++ b/hermeto/core/package_managers/bundler/main.py @@ -35,9 +35,6 @@ def fetch_bundler_source(request: Request) -> RequestOutput: """Resolve and process all bundler packages.""" components: list[Component] = [] - environment_variables: list[EnvironmentVariable] = ( - _prepare_environment_variables_for_hermetic_build() - ) project_files: list[ProjectFile] = [] git_paths = [] @@ -50,9 +47,11 @@ def fetch_bundler_source(request: Request) -> RequestOutput: ) components.extend(_comps) git_paths.extend(_git_paths) - project_files.append( - _prepare_for_hermetic_build(request.source_dir, request.output_dir, git_paths) + + environment_variables: list[EnvironmentVariable] = ( + _prepare_environment_variables_for_hermetic_build(git_paths) ) + project_files.append(_prepare_for_hermetic_build(request.source_dir, request.output_dir)) annotations = [] if backend_annotation := create_backend_annotation(components, "bundler"): @@ -65,17 +64,17 @@ def fetch_bundler_source(request: Request) -> RequestOutput: ) -# Aliases for git dependency name and git dependency name as -# it is written to file system: +# Aliases for git dependency name, file system name, and remote URL: DepName = str FSDepName = str +DepURL = str def _resolve_bundler_package( package_dir: RootedPath, output_dir: RootedPath, binary_filters: BundlerBinaryFilters | None = None, -) -> tuple[list[Component], list[tuple[DepName, FSDepName]]]: +) -> tuple[list[Component], list[tuple[DepName, FSDepName, DepURL]]]: """Process a request for a single bundler package.""" deps_dir = output_dir.join_within_root("deps", "bundler") deps_dir.path.mkdir(parents=True, exist_ok=True) @@ -110,7 +109,7 @@ def _resolve_bundler_package( files_to_download[dep.remote_location] = dep.download_location(deps_dir) case GitDependency(): dep.download_to(deps_dir) - git_paths.append((dep.name, dep.repo_name + "-" + dep.ref[:12])) + git_paths.append((dep.name, dep.repo_name + "-" + dep.ref[:12], str(dep.url))) c = Component(name=dep.name, version=dep.version, purl=dep.purl, properties=properties) components.append(c) @@ -196,16 +195,51 @@ def _get_repo_name_from_origin_remote(package_dir: RootedPath) -> str: return str(resolved_path) -def _prepare_environment_variables_for_hermetic_build() -> list[EnvironmentVariable]: - return [ +def _prepare_environment_variables_for_hermetic_build( + git_paths: list[tuple[DepName, FSDepName, DepURL]] | None = None, +) -> list[EnvironmentVariable]: + env_vars = [ # Contains path to a directory where a new config could be found. EnvironmentVariable(name="BUNDLE_APP_CONFIG", value="${output_dir}/" + CONFIG_OVERRIDE), ] + if git_paths: + # Redirect git remote URLs to pre-fetched local clones via git's + # GIT_CONFIG_COUNT/KEY/VALUE mechanism. This injects url.insteadOf + # entries without replacing the global git config. + # See: https://git-scm.com/docs/git-config#ENVIRONMENT + _check_for_duplicate_git_urls(git_paths) + # (key, value) pairs for GIT_CONFIG_KEY_N / GIT_CONFIG_VALUE_N + git_config: list[tuple[str, str]] = [] + for _packname, dirname, url in git_paths: + clone_file_url = "file://${output_dir}/deps/bundler/" + dirname + "/" + git_config.append((f"url.{clone_file_url}.insteadOf", url)) + git_config.append(("protocol.file.allow", "always")) + + env_vars.append(EnvironmentVariable(name="GIT_CONFIG_COUNT", value=str(len(git_config)))) + for idx, (key, value) in enumerate(git_config): + env_vars.append(EnvironmentVariable(name=f"GIT_CONFIG_KEY_{idx}", value=key)) + env_vars.append(EnvironmentVariable(name=f"GIT_CONFIG_VALUE_{idx}", value=value)) + return env_vars + + +def _check_for_duplicate_git_urls( + git_paths: list[tuple[DepName, FSDepName, DepURL]], +) -> None: + """Raise if multiple git deps share the same URL with different revisions.""" + url_to_dirs: dict[str, set[str]] = {} + for _packname, dirname, url in git_paths: + url_to_dirs.setdefault(url, set()).add(dirname) + for url, dirs in url_to_dirs.items(): + if len(dirs) > 1: + raise UnsupportedFeature( + f"Multiple git dependencies point to the same repository ({url}) " + f"but use different revisions: {', '.join(sorted(dirs))}. " + "This is not supported because git's url.insteadOf redirect " + "can only map a repository URL to a single local clone." + ) -def _prepare_for_hermetic_build( - source_dir: RootedPath, output_dir: RootedPath, git_paths: list | None = None -) -> ProjectFile: +def _prepare_for_hermetic_build(source_dir: RootedPath, output_dir: RootedPath) -> ProjectFile: """Prepare a package for hermetic build by injecting necessary config.""" potential_bundle_config = source_dir.join_within_root(".bundle/config").path hermetic_config = dedent( @@ -218,30 +252,6 @@ def _prepare_for_hermetic_build( BUNDLE_VERSION: "system" """ ) - # Note: if a package depends on a git revision then the following variables - # are necessary for a hermetic build: - # BUNDLE_DISABLE_LOCAL_BRANCH_CHECK - # BUNDLE_DISABLE_LOCAL_REVISION_CHECK - # because otherwise some (potentially all, depending on exact set of - # ecosystem components versions, environment variables and celestial - # alignment) Bundler versions will try to fetch the latest changes of the - # remotes which may be present even when instructed not to with --local - # flag. - # See https://bundler.io/guides/git.html#local-git-repos for details. - # (or https://github.com/rubygems/bundler-site/blob/ - # 9ff3b76e9866524ecefe165633ffb547f0004a99/source/guides/git.html.md - # if the link above ceases to exist). - if git_paths is not None: - hermetic_config += 'BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true"\n' - hermetic_config += 'BUNDLE_DISABLE_LOCAL_REVISION_CHECK: "true"\n' - for packname, dirname in git_paths: - # "-" in variable names is deprecated in Bundler and now generates - # a warning and a suggestion to replace all dashes with triple - # underscores. Package names sometimes contain dashes: - varname = "BUNDLE_LOCAL." + packname.upper().replace("-", "___") - location = "${output_dir}/deps/bundler/" + dirname - config_entry = varname + f': "{location}"' - hermetic_config += f"{config_entry}\n" if potential_bundle_config.is_file(): config_data = potential_bundle_config.read_text() config_data += hermetic_config diff --git a/hermeto/core/package_managers/bundler/scripts/lockfile_parser.rb b/hermeto/core/package_managers/bundler/scripts/lockfile_parser.rb index 2bc2d72bd..24741e9d3 100755 --- a/hermeto/core/package_managers/bundler/scripts/lockfile_parser.rb +++ b/hermeto/core/package_managers/bundler/scripts/lockfile_parser.rb @@ -39,7 +39,6 @@ version: spec.version, type: 'git', url: spec.source.uri, - branch: spec.source.branch, ref: spec.source.revision } parsed_specs << parsed_spec diff --git a/tests/integration/test_bundler.py b/tests/integration/test_bundler.py index 55438345b..647e6f639 100644 --- a/tests/integration/test_bundler.py +++ b/tests/integration/test_bundler.py @@ -70,7 +70,17 @@ def test_bundler_packages( ), [], # No additional commands are run to verify the build [], - id="bundler_e2e", + id="bundler_e2e_ruby33", + ), + pytest.param( + utils.TestParameters( + branch="bundler/e2e", + packages=({"path": ".", "type": "bundler", "binary": {}},), + check_output=True, + ), + [], + [], + id="bundler_e2e_ruby40", ), pytest.param( utils.TestParameters( diff --git a/tests/integration/test_data/bundler_e2e_missing_gemspec/.build-config.yaml b/tests/integration/test_data/bundler_e2e_missing_gemspec/.build-config.yaml index d4fc555cc..34ab6c931 100644 --- a/tests/integration/test_data/bundler_e2e_missing_gemspec/.build-config.yaml +++ b/tests/integration/test_data/bundler_e2e_missing_gemspec/.build-config.yaml @@ -1,6 +1,16 @@ environment_variables: - name: BUNDLE_APP_CONFIG value: ${output_dir}/bundler/config_override +- name: GIT_CONFIG_COUNT + value: '2' +- name: GIT_CONFIG_KEY_0 + value: url.file://${output_dir}/deps/bundler/json-schema-26487618a684/.insteadOf +- name: GIT_CONFIG_KEY_1 + value: protocol.file.allow +- name: GIT_CONFIG_VALUE_0 + value: https://github.com/3scale/json-schema +- name: GIT_CONFIG_VALUE_1 + value: always project_files: - abspath: ${test_case_tmp_path}/hermeto-output/bundler/config_override/config template: |2 @@ -11,6 +21,3 @@ project_files: BUNDLE_ALLOW_OFFLINE_INSTALL: "true" BUNDLE_DISABLE_VERSION_CHECK: "true" BUNDLE_VERSION: "system" - BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true" - BUNDLE_DISABLE_LOCAL_REVISION_CHECK: "true" - BUNDLE_LOCAL.JSON___SCHEMA: "${output_dir}/deps/bundler/json-schema-26487618a684" diff --git a/tests/integration/test_data/bundler_e2e/.build-config.yaml b/tests/integration/test_data/bundler_e2e_ruby33/.build-config.yaml similarity index 57% rename from tests/integration/test_data/bundler_e2e/.build-config.yaml rename to tests/integration/test_data/bundler_e2e_ruby33/.build-config.yaml index d4fc555cc..34ab6c931 100644 --- a/tests/integration/test_data/bundler_e2e/.build-config.yaml +++ b/tests/integration/test_data/bundler_e2e_ruby33/.build-config.yaml @@ -1,6 +1,16 @@ environment_variables: - name: BUNDLE_APP_CONFIG value: ${output_dir}/bundler/config_override +- name: GIT_CONFIG_COUNT + value: '2' +- name: GIT_CONFIG_KEY_0 + value: url.file://${output_dir}/deps/bundler/json-schema-26487618a684/.insteadOf +- name: GIT_CONFIG_KEY_1 + value: protocol.file.allow +- name: GIT_CONFIG_VALUE_0 + value: https://github.com/3scale/json-schema +- name: GIT_CONFIG_VALUE_1 + value: always project_files: - abspath: ${test_case_tmp_path}/hermeto-output/bundler/config_override/config template: |2 @@ -11,6 +21,3 @@ project_files: BUNDLE_ALLOW_OFFLINE_INSTALL: "true" BUNDLE_DISABLE_VERSION_CHECK: "true" BUNDLE_VERSION: "system" - BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true" - BUNDLE_DISABLE_LOCAL_REVISION_CHECK: "true" - BUNDLE_LOCAL.JSON___SCHEMA: "${output_dir}/deps/bundler/json-schema-26487618a684" diff --git a/tests/integration/test_data/bundler_e2e/bom.json b/tests/integration/test_data/bundler_e2e_ruby33/bom.json similarity index 100% rename from tests/integration/test_data/bundler_e2e/bom.json rename to tests/integration/test_data/bundler_e2e_ruby33/bom.json diff --git a/tests/integration/test_data/bundler_e2e/container/Containerfile b/tests/integration/test_data/bundler_e2e_ruby33/container/Containerfile similarity index 57% rename from tests/integration/test_data/bundler_e2e/container/Containerfile rename to tests/integration/test_data/bundler_e2e_ruby33/container/Containerfile index 35e753afd..93d9fe2cc 100644 --- a/tests/integration/test_data/bundler_e2e/container/Containerfile +++ b/tests/integration/test_data/bundler_e2e_ruby33/container/Containerfile @@ -1,8 +1,13 @@ -FROM mirror.gcr.io/ruby:3.3 +FROM mirror.gcr.io/ruby:3.3 AS builder # Test disabled network access RUN if getent hosts www.google.com; then echo "Has network access!"; exit 1; fi WORKDIR /src +ENV BUNDLE_PATH=/gems RUN . /tmp/hermeto.env && bundle install + +FROM mirror.gcr.io/ruby:3.3 +COPY --from=builder /gems /gems +RUN test -d /gems/ruby/*/bundler/gems/json-schema-* diff --git a/tests/integration/test_data/bundler_e2e_ruby40/.build-config.yaml b/tests/integration/test_data/bundler_e2e_ruby40/.build-config.yaml new file mode 100644 index 000000000..34ab6c931 --- /dev/null +++ b/tests/integration/test_data/bundler_e2e_ruby40/.build-config.yaml @@ -0,0 +1,23 @@ +environment_variables: +- name: BUNDLE_APP_CONFIG + value: ${output_dir}/bundler/config_override +- name: GIT_CONFIG_COUNT + value: '2' +- name: GIT_CONFIG_KEY_0 + value: url.file://${output_dir}/deps/bundler/json-schema-26487618a684/.insteadOf +- name: GIT_CONFIG_KEY_1 + value: protocol.file.allow +- name: GIT_CONFIG_VALUE_0 + value: https://github.com/3scale/json-schema +- name: GIT_CONFIG_VALUE_1 + value: always +project_files: +- abspath: ${test_case_tmp_path}/hermeto-output/bundler/config_override/config + template: |2 + + BUNDLE_CACHE_PATH: "${output_dir}/deps/bundler" + BUNDLE_DEPLOYMENT: "true" + BUNDLE_NO_PRUNE: "true" + BUNDLE_ALLOW_OFFLINE_INSTALL: "true" + BUNDLE_DISABLE_VERSION_CHECK: "true" + BUNDLE_VERSION: "system" diff --git a/tests/integration/test_data/bundler_e2e_ruby40/bom.json b/tests/integration/test_data/bundler_e2e_ruby40/bom.json new file mode 100644 index 000000000..88e210259 --- /dev/null +++ b/tests/integration/test_data/bundler_e2e_ruby40/bom.json @@ -0,0 +1,760 @@ +{ + "annotations": [ + { + "annotator": { + "organization": { + "name": "red hat" + } + }, + "subjects": [ + "pkg:gem/actioncable@6.1.7", + "pkg:gem/actionmailbox@6.1.7", + "pkg:gem/actionmailer@6.1.7", + "pkg:gem/actionpack@6.1.7", + "pkg:gem/actiontext@6.1.7", + "pkg:gem/actionview@6.1.7", + "pkg:gem/activejob@6.1.7", + "pkg:gem/activemodel@6.1.7", + "pkg:gem/activerecord@6.1.7", + "pkg:gem/activestorage@6.1.7", + "pkg:gem/activesupport@6.1.7", + "pkg:gem/addressable@2.8.7", + "pkg:gem/builder@3.3.0", + "pkg:gem/concurrent-ruby@1.3.4", + "pkg:gem/crass@1.0.6", + "pkg:gem/date@3.3.4", + "pkg:gem/erubi@1.13.0", + "pkg:gem/globalid@1.2.1", + "pkg:gem/i18n@1.14.6", + "pkg:gem/json-schema@3.0.0?vcs_url=git%2Bhttps://github.com/3scale/json-schema%4026487618a68443e94d623bb585cb464b07d36702", + "pkg:gem/loofah@2.22.0", + "pkg:gem/mail@2.8.1", + "pkg:gem/marcel@1.0.4", + "pkg:gem/method_source@1.1.0", + "pkg:gem/mini_mime@1.1.5", + "pkg:gem/minitest@5.25.1", + "pkg:gem/mygem@0.0.1", + "pkg:gem/net-imap@0.4.16", + "pkg:gem/net-pop@0.1.2", + "pkg:gem/net-protocol@0.2.2", + "pkg:gem/net-smtp@0.5.0", + "pkg:gem/nio4r@2.7.3", + "pkg:gem/nokogiri@1.16.7", + "pkg:gem/public_suffix@6.0.1", + "pkg:gem/quux@0.0.1?vcs_url=git%2Bhttps://github.com/hermetoproject/integration-tests.git%40f2d2d64600559d801b44170113a93d754052e9f7#quux", + "pkg:gem/racc@1.8.1", + "pkg:gem/rack-test@2.1.0", + "pkg:gem/rack@2.2.9", + "pkg:gem/rails-dom-testing@2.2.0", + "pkg:gem/rails-html-sanitizer@1.6.0", + "pkg:gem/rails@6.1.7", + "pkg:gem/railties@6.1.7", + "pkg:gem/rake@13.2.1", + "pkg:gem/sprockets-rails@3.5.2", + "pkg:gem/sprockets@4.2.1", + "pkg:gem/thor@1.3.2", + "pkg:gem/timeout@0.4.1", + "pkg:gem/tmp@0.1.2?vcs_url=git%2Bhttps://github.com/hermetoproject/integration-tests.git%40f2d2d64600559d801b44170113a93d754052e9f7", + "pkg:gem/tzinfo@2.0.6", + "pkg:gem/websocket-driver@0.7.6", + "pkg:gem/websocket-extensions@0.1.5", + "pkg:gem/zeitwerk@2.6.18" + ], + "text": "hermeto:backend:bundler", + "timestamp": "2025-01-01T00:00:00Z" + } + ], + "bomFormat": "CycloneDX", + "components": [ + { + "bom-ref": "pkg:gem/actioncable@6.1.7", + "name": "actioncable", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/actioncable@6.1.7", + "type": "library", + "version": "6.1.7" + }, + { + "bom-ref": "pkg:gem/actionmailbox@6.1.7", + "name": "actionmailbox", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/actionmailbox@6.1.7", + "type": "library", + "version": "6.1.7" + }, + { + "bom-ref": "pkg:gem/actionmailer@6.1.7", + "name": "actionmailer", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/actionmailer@6.1.7", + "type": "library", + "version": "6.1.7" + }, + { + "bom-ref": "pkg:gem/actionpack@6.1.7", + "name": "actionpack", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/actionpack@6.1.7", + "type": "library", + "version": "6.1.7" + }, + { + "bom-ref": "pkg:gem/actiontext@6.1.7", + "name": "actiontext", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/actiontext@6.1.7", + "type": "library", + "version": "6.1.7" + }, + { + "bom-ref": "pkg:gem/actionview@6.1.7", + "name": "actionview", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/actionview@6.1.7", + "type": "library", + "version": "6.1.7" + }, + { + "bom-ref": "pkg:gem/activejob@6.1.7", + "name": "activejob", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/activejob@6.1.7", + "type": "library", + "version": "6.1.7" + }, + { + "bom-ref": "pkg:gem/activemodel@6.1.7", + "name": "activemodel", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/activemodel@6.1.7", + "type": "library", + "version": "6.1.7" + }, + { + "bom-ref": "pkg:gem/activerecord@6.1.7", + "name": "activerecord", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/activerecord@6.1.7", + "type": "library", + "version": "6.1.7" + }, + { + "bom-ref": "pkg:gem/activestorage@6.1.7", + "name": "activestorage", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/activestorage@6.1.7", + "type": "library", + "version": "6.1.7" + }, + { + "bom-ref": "pkg:gem/activesupport@6.1.7", + "name": "activesupport", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/activesupport@6.1.7", + "type": "library", + "version": "6.1.7" + }, + { + "bom-ref": "pkg:gem/addressable@2.8.7", + "name": "addressable", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/addressable@2.8.7", + "type": "library", + "version": "2.8.7" + }, + { + "bom-ref": "pkg:gem/builder@3.3.0", + "name": "builder", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/builder@3.3.0", + "type": "library", + "version": "3.3.0" + }, + { + "bom-ref": "pkg:gem/concurrent-ruby@1.3.4", + "name": "concurrent-ruby", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/concurrent-ruby@1.3.4", + "type": "library", + "version": "1.3.4" + }, + { + "bom-ref": "pkg:gem/crass@1.0.6", + "name": "crass", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/crass@1.0.6", + "type": "library", + "version": "1.0.6" + }, + { + "bom-ref": "pkg:gem/date@3.3.4", + "name": "date", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/date@3.3.4", + "type": "library", + "version": "3.3.4" + }, + { + "bom-ref": "pkg:gem/erubi@1.13.0", + "name": "erubi", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/erubi@1.13.0", + "type": "library", + "version": "1.13.0" + }, + { + "bom-ref": "pkg:gem/globalid@1.2.1", + "name": "globalid", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/globalid@1.2.1", + "type": "library", + "version": "1.2.1" + }, + { + "bom-ref": "pkg:gem/i18n@1.14.6", + "name": "i18n", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/i18n@1.14.6", + "type": "library", + "version": "1.14.6" + }, + { + "bom-ref": "pkg:gem/json-schema@3.0.0?vcs_url=git%2Bhttps://github.com/3scale/json-schema%4026487618a68443e94d623bb585cb464b07d36702", + "name": "json-schema", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/json-schema@3.0.0?vcs_url=git%2Bhttps://github.com/3scale/json-schema%4026487618a68443e94d623bb585cb464b07d36702", + "type": "library", + "version": "3.0.0" + }, + { + "bom-ref": "pkg:gem/loofah@2.22.0", + "name": "loofah", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/loofah@2.22.0", + "type": "library", + "version": "2.22.0" + }, + { + "bom-ref": "pkg:gem/mail@2.8.1", + "name": "mail", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/mail@2.8.1", + "type": "library", + "version": "2.8.1" + }, + { + "bom-ref": "pkg:gem/marcel@1.0.4", + "name": "marcel", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/marcel@1.0.4", + "type": "library", + "version": "1.0.4" + }, + { + "bom-ref": "pkg:gem/method_source@1.1.0", + "name": "method_source", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/method_source@1.1.0", + "type": "library", + "version": "1.1.0" + }, + { + "bom-ref": "pkg:gem/mini_mime@1.1.5", + "name": "mini_mime", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/mini_mime@1.1.5", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:gem/minitest@5.25.1", + "name": "minitest", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/minitest@5.25.1", + "type": "library", + "version": "5.25.1" + }, + { + "bom-ref": "pkg:gem/mygem@0.0.1", + "name": "mygem", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/mygem@0.0.1", + "type": "library", + "version": "0.0.1" + }, + { + "bom-ref": "pkg:gem/net-imap@0.4.16", + "name": "net-imap", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/net-imap@0.4.16", + "type": "library", + "version": "0.4.16" + }, + { + "bom-ref": "pkg:gem/net-pop@0.1.2", + "name": "net-pop", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/net-pop@0.1.2", + "type": "library", + "version": "0.1.2" + }, + { + "bom-ref": "pkg:gem/net-protocol@0.2.2", + "name": "net-protocol", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/net-protocol@0.2.2", + "type": "library", + "version": "0.2.2" + }, + { + "bom-ref": "pkg:gem/net-smtp@0.5.0", + "name": "net-smtp", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/net-smtp@0.5.0", + "type": "library", + "version": "0.5.0" + }, + { + "bom-ref": "pkg:gem/nio4r@2.7.3", + "name": "nio4r", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/nio4r@2.7.3", + "type": "library", + "version": "2.7.3" + }, + { + "bom-ref": "pkg:gem/nokogiri@1.16.7", + "name": "nokogiri", + "properties": [ + { + "name": "hermeto:bundler:package:binary", + "value": "true" + }, + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/nokogiri@1.16.7", + "type": "library", + "version": "1.16.7" + }, + { + "bom-ref": "pkg:gem/public_suffix@6.0.1", + "name": "public_suffix", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/public_suffix@6.0.1", + "type": "library", + "version": "6.0.1" + }, + { + "bom-ref": "pkg:gem/quux@0.0.1?vcs_url=git%2Bhttps://github.com/hermetoproject/integration-tests.git%40f2d2d64600559d801b44170113a93d754052e9f7#quux", + "name": "quux", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/quux@0.0.1?vcs_url=git%2Bhttps://github.com/hermetoproject/integration-tests.git%40f2d2d64600559d801b44170113a93d754052e9f7#quux", + "type": "library", + "version": "0.0.1" + }, + { + "bom-ref": "pkg:gem/racc@1.8.1", + "name": "racc", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/racc@1.8.1", + "type": "library", + "version": "1.8.1" + }, + { + "bom-ref": "pkg:gem/rack-test@2.1.0", + "name": "rack-test", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/rack-test@2.1.0", + "type": "library", + "version": "2.1.0" + }, + { + "bom-ref": "pkg:gem/rack@2.2.9", + "name": "rack", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/rack@2.2.9", + "type": "library", + "version": "2.2.9" + }, + { + "bom-ref": "pkg:gem/rails-dom-testing@2.2.0", + "name": "rails-dom-testing", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/rails-dom-testing@2.2.0", + "type": "library", + "version": "2.2.0" + }, + { + "bom-ref": "pkg:gem/rails-html-sanitizer@1.6.0", + "name": "rails-html-sanitizer", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/rails-html-sanitizer@1.6.0", + "type": "library", + "version": "1.6.0" + }, + { + "bom-ref": "pkg:gem/rails@6.1.7", + "name": "rails", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/rails@6.1.7", + "type": "library", + "version": "6.1.7" + }, + { + "bom-ref": "pkg:gem/railties@6.1.7", + "name": "railties", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/railties@6.1.7", + "type": "library", + "version": "6.1.7" + }, + { + "bom-ref": "pkg:gem/rake@13.2.1", + "name": "rake", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/rake@13.2.1", + "type": "library", + "version": "13.2.1" + }, + { + "bom-ref": "pkg:gem/sprockets-rails@3.5.2", + "name": "sprockets-rails", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/sprockets-rails@3.5.2", + "type": "library", + "version": "3.5.2" + }, + { + "bom-ref": "pkg:gem/sprockets@4.2.1", + "name": "sprockets", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/sprockets@4.2.1", + "type": "library", + "version": "4.2.1" + }, + { + "bom-ref": "pkg:gem/thor@1.3.2", + "name": "thor", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/thor@1.3.2", + "type": "library", + "version": "1.3.2" + }, + { + "bom-ref": "pkg:gem/timeout@0.4.1", + "name": "timeout", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/timeout@0.4.1", + "type": "library", + "version": "0.4.1" + }, + { + "bom-ref": "pkg:gem/tmp@0.1.2?vcs_url=git%2Bhttps://github.com/hermetoproject/integration-tests.git%40f2d2d64600559d801b44170113a93d754052e9f7", + "name": "tmp", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/tmp@0.1.2?vcs_url=git%2Bhttps://github.com/hermetoproject/integration-tests.git%40f2d2d64600559d801b44170113a93d754052e9f7", + "type": "library", + "version": "0.1.2" + }, + { + "bom-ref": "pkg:gem/tzinfo@2.0.6", + "name": "tzinfo", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/tzinfo@2.0.6", + "type": "library", + "version": "2.0.6" + }, + { + "bom-ref": "pkg:gem/websocket-driver@0.7.6", + "name": "websocket-driver", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/websocket-driver@0.7.6", + "type": "library", + "version": "0.7.6" + }, + { + "bom-ref": "pkg:gem/websocket-extensions@0.1.5", + "name": "websocket-extensions", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/websocket-extensions@0.1.5", + "type": "library", + "version": "0.1.5" + }, + { + "bom-ref": "pkg:gem/zeitwerk@2.6.18", + "name": "zeitwerk", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:gem/zeitwerk@2.6.18", + "type": "library", + "version": "2.6.18" + } + ], + "metadata": { + "tools": [ + { + "name": "hermeto", + "vendor": "red hat" + } + ] + }, + "specVersion": "1.6", + "version": 1 +} diff --git a/tests/integration/test_data/bundler_e2e_ruby40/container/Containerfile b/tests/integration/test_data/bundler_e2e_ruby40/container/Containerfile new file mode 100644 index 000000000..7ae249c10 --- /dev/null +++ b/tests/integration/test_data/bundler_e2e_ruby40/container/Containerfile @@ -0,0 +1,13 @@ +FROM mirror.gcr.io/ruby:4.0 AS builder + +# Test disabled network access +RUN if getent hosts www.google.com; then echo "Has network access!"; exit 1; fi + +WORKDIR /src +ENV BUNDLE_PATH=/gems + +RUN . /tmp/hermeto.env && bundle install + +FROM mirror.gcr.io/ruby:4.0 +COPY --from=builder /gems /gems +RUN test -d /gems/ruby/*/bundler/gems/json-schema-* diff --git a/tests/unit/package_managers/bundler/test_main.py b/tests/unit/package_managers/bundler/test_main.py index 4d932ee22..3f8fe96e4 100644 --- a/tests/unit/package_managers/bundler/test_main.py +++ b/tests/unit/package_managers/bundler/test_main.py @@ -6,10 +6,12 @@ from git.repo import Repo from hermeto.core.constants import Mode -from hermeto.core.errors import NotAGitRepo, PackageRejected +from hermeto.core.errors import NotAGitRepo, PackageRejected, UnsupportedFeature +from hermeto.core.models.output import EnvironmentVariable from hermeto.core.package_managers.bundler.main import ( _get_main_package_name_and_version, _get_repo_name_from_origin_remote, + _prepare_environment_variables_for_hermetic_build, _prepare_for_hermetic_build, ) from hermeto.core.package_managers.bundler.parser import ( @@ -155,6 +157,45 @@ def test_prepare_for_hermetic_build_injects_necessary_variable_into_existing_alt assert result.template == existing_preamble + expected_alternate_config_contents +def test_prepare_environment_variables_generates_git_config_entries_for_git_deps() -> None: + git_paths = [ + ("gem-a", "gem-a-aabb11223344", "https://git.example/gem-a.git"), + ("gem-b", "gem-b-ccdd55667788", "https://git.example/gem-b.git"), + ] + + result = _prepare_environment_variables_for_hermetic_build(git_paths) + + expected = { + EnvironmentVariable( + name="BUNDLE_APP_CONFIG", value="${output_dir}/bundler/config_override" + ), + EnvironmentVariable(name="GIT_CONFIG_COUNT", value="3"), + EnvironmentVariable( + name="GIT_CONFIG_KEY_0", + value="url.file://${output_dir}/deps/bundler/gem-a-aabb11223344/.insteadOf", + ), + EnvironmentVariable(name="GIT_CONFIG_VALUE_0", value="https://git.example/gem-a.git"), + EnvironmentVariable( + name="GIT_CONFIG_KEY_1", + value="url.file://${output_dir}/deps/bundler/gem-b-ccdd55667788/.insteadOf", + ), + EnvironmentVariable(name="GIT_CONFIG_VALUE_1", value="https://git.example/gem-b.git"), + EnvironmentVariable(name="GIT_CONFIG_KEY_2", value="protocol.file.allow"), + EnvironmentVariable(name="GIT_CONFIG_VALUE_2", value="always"), + } + assert set(result) == expected + + +def test_prepare_environment_variables_raises_on_duplicate_git_url() -> None: + git_paths = [ + ("gem-a", "gem-a-aabbccdd1234", "https://git.example/monorepo.git"), + ("gem-b", "gem-b-eeff00112233", "https://git.example/monorepo.git"), + ] + + with pytest.raises(UnsupportedFeature, match="same repository"): + _prepare_environment_variables_for_hermetic_build(git_paths) + + @mock.patch("hermeto.core.package_managers.bundler.main.get_repo_id") def test_get_repo_name_raises_without_git_repo( mock_handle_get_repo_id: mock.Mock, diff --git a/tests/unit/package_managers/bundler/test_parser.py b/tests/unit/package_managers/bundler/test_parser.py index ec4994a2f..de3128c1c 100644 --- a/tests/unit/package_managers/bundler/test_parser.py +++ b/tests/unit/package_managers/bundler/test_parser.py @@ -139,7 +139,6 @@ def test_parse_gemlock( { "type": "git", "url": "https://github.com/3scale/json-schema.git", - "branch": "devel", "ref": GIT_REF, **base_dep, }, @@ -164,7 +163,6 @@ def test_parse_gemlock( name="example", version="0.1.0", url="https://github.com/3scale/json-schema.git", - branch="devel", ref=GIT_REF, ), PathDependency( @@ -215,6 +213,7 @@ def test_download_git_dependency_works( mock_git_clone.assert_called_once_with( url=str(dep.url), to_path=dep_path, + bare=True, env={"GIT_TERMINAL_PROMPT": "0"}, ) assert dep_path.exists() @@ -241,6 +240,7 @@ def test_download_duplicate_git_dependency_is_skipped( mock_git_clone.assert_called_once_with( url=str(dep.url), to_path=dep_path, + bare=True, env={"GIT_TERMINAL_PROMPT": "0"}, ) assert dep_path.exists()