Support content addressable gems in gem update, outdated, dependency, and fetch

Author: girachawda

lib/rubygems/commands/dependency_command.rb

@@ -66,7 +66,11 @@ def fetch_remote_specs(name, requirement, prerelease) # :nodoc:
       end
     end
 
-    ss.map {|tuple, source| source.fetch_spec(tuple) }
+    fetcher.decode_content_addressable_tuples(ss).map do |tuple, source|
+      spec = source.fetch_spec(tuple)
+      spec.content_address = tuple.content_address if tuple.content_address
+      spec
+    end
   end
 
   def fetch_specs(name_pattern, requirement, prerelease) # :nodoc:
@@ -85,7 +89,7 @@ def fetch_specs(name_pattern, requirement, prerelease) # :nodoc:
 
     ensure_specs specs
 
-    specs.uniq.sort
+    specs.uniq(&:full_name).sort
   end
 
   def display_pipe(specs) # :nodoc:
@@ -148,9 +152,17 @@ def ensure_specs(specs) # :nodoc:
     terminate_interaction 1
   end
 
+  def content_address_annotation(spec) # :nodoc:
+    return "" unless Gem::ContentAddress.content_addressed?(spec)
+
+    parts = ["Platform: #{spec.platform}"]
+    parts << "Ruby ABI: #{spec.ruby_abi}" if spec.ruby_abi
+    " (#{parts.join(" ")})"
+  end
+
   def print_dependencies(spec, level = 0) # :nodoc:
     response = String.new
-    response << "  " * level + "Gem #{spec.full_name}\n"
+    response << "  " * level + "Gem #{spec.full_name}#{content_address_annotation(spec)}\n"
     unless spec.dependencies.empty?
       spec.dependencies.sort_by(&:name).each do |dep|
         response << "  " * level + "  #{dep}\n"

lib/rubygems/commands/outdated_command.rb

@@ -88,7 +88,7 @@ def partition_by_cooldown(spec_tuples)
     embargoed = []
 
     with_times = spec_tuples.map do |tup, source|
-      [tup, source, source.created_at(tup.name, tup.version, tup.platform)]
+      [tup, source, source.created_at_for_tuple(tup)]
     end
 
     if !with_times.empty? && with_times.none? {|_, _, created_at| created_at }

lib/rubygems/commands/update_command.rb

@@ -194,7 +194,7 @@ def filter_cooldown_tuples(spec_tuples) # :nodoc:
     return spec_tuples unless @cooldown&.active?
 
     with_times = spec_tuples.map do |tup, source|
-      [tup, source, source.created_at(tup.name, tup.version, tup.platform)]
+      [tup, source, source.created_at_for_tuple(tup)]
     end
 
     if !with_times.empty? && with_times.none? {|_, _, created_at| created_at }

lib/rubygems/compact_index_client.rb

@@ -20,10 +20,13 @@ class Gem::CompactIndexClient
   # info returns an Array of INFO Arrays. Each INFO Array has the following indices:
   INFO_NAME = 0
   INFO_VERSION = 1
-  INFO_PLATFORM = 2
+  INFO_SUFFIX = 2
   INFO_DEPS = 3
   INFO_REQS = 4
 
+  INFO_PLATFORM = INFO_SUFFIX
+  deprecate_constant :INFO_PLATFORM
+
   def self.debug
     return unless ENV["DEBUG_COMPACT_INDEX"]
     DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") }

lib/rubygems/query_utils.rb

@@ -156,22 +156,13 @@ def show_remote_gems(name)
       if args.empty?
         matching_tuples
       else
-        decode_content_addressable_tuples(matching_tuples, latest: specs_type == :latest)
+        fetcher.decode_content_addressable_tuples(matching_tuples, latest: specs_type == :latest)
       end
     end
 
     output_query_results(spec_tuples)
   end
 
-  def decode_content_addressable_tuples(spec_tuples, latest: false)
-    spec_tuples.group_by {|_, source| source }.flat_map do |source, source_tuples|
-      next source_tuples unless source.respond_to?(:decode_content_addressable_tuples)
-
-      tuples = source_tuples.map(&:first)
-      source.decode_content_addressable_tuples(tuples, latest: latest).map {|tuple| [tuple, source] }
-    end
-  end
-
   def specs_type
     if options[:all] || options[:version].specific?
       if options[:prerelease]

lib/rubygems/source.rb

@@ -240,12 +240,12 @@ def decode_content_addressable_tuples(tuples, latest: false)
   end
 
   ##
-  # The publish time of gem +name+ at +version+ for +platform+, when this
+  # The publish time of gem +name+ at +version+ for +suffix+, when this
   # source provides it through the compact index created_at metadata.
   # Returns nil when the source, the gem or the version has no known
   # publish time.
 
-  def created_at(name, version, platform = Gem::Platform::RUBY)
+  def created_at(name, version, suffix = Gem::Platform::RUBY)
     return unless %w[http https].include?(uri.scheme)
 
     @created_at_info ||= {}
@@ -255,12 +255,12 @@ def created_at(name, version, platform = Gem::Platform::RUBY)
       []
     end
 
-    platform = (platform || Gem::Platform::RUBY).to_s
+    suffix = (suffix || Gem::Platform::RUBY).to_s
     version = version.to_s
 
     row = info.find do |row_info|
       row_info[Gem::CompactIndexClient::INFO_VERSION] == version &&
-        (row_info[Gem::CompactIndexClient::INFO_PLATFORM] || Gem::Platform::RUBY) == platform
+        (row_info[Gem::CompactIndexClient::INFO_SUFFIX] || Gem::Platform::RUBY) == suffix
     end
     return unless row
 
@@ -269,6 +269,14 @@ def created_at(name, version, platform = Gem::Platform::RUBY)
     Gem::Cooldown.parse_created_at(value)
   end
 
+  ##
+  # The publish time for +tuple+. Content-addressable tuples are looked up by
+  # content address; all other tuples are looked up by platform.
+
+  def created_at_for_tuple(tuple)
+    created_at(tuple.name, tuple.version, tuple.content_address || tuple.platform)
+  end
+
   ##
   # Downloads +spec+ and writes it to +dir+.  See also
   # Gem::RemoteFetcher#download.
@@ -403,7 +411,7 @@ def content_addressable_metadata(name, rows)
 
     available_rows = compact_index_info_rows(name).filter_map do |info_row|
       version = info_row[Gem::CompactIndexClient::INFO_VERSION]
-      suffix = info_row[Gem::CompactIndexClient::INFO_PLATFORM]
+      suffix = info_row[Gem::CompactIndexClient::INFO_SUFFIX]
 
       requirements = compact_index_requirements(info_row)
       platform = required_platform_from(requirements[:platform])

lib/rubygems/spec_fetcher.rb

@@ -91,7 +91,8 @@ def search_for_dependency(dependency, matching_platform = true, type: nil)
 
     rejected_specs = {}
 
-    list, errors = available_specs(type || dependency.identity)
+    specs_type = type || dependency.identity
+    list, errors = available_specs(specs_type)
 
     list.each do |source, specs|
       if dependency.name.is_a?(String) && specs.respond_to?(:bsearch)
@@ -100,17 +101,20 @@ def search_for_dependency(dependency, matching_platform = true, type: nil)
         specs = specs[start_index...end_index] if start_index && end_index
       end
 
+      specs = specs.select {|tup| dependency.match?(tup) }
+      specs = decode_source_content_addressable_tuples(source, specs, latest: specs_type == :latest)
+
       found[source] = specs.select do |tup|
-        if dependency.match?(tup)
-          if matching_platform && !Gem::Platform.match_gem?(tup.platform, tup.name)
-            pm = (
-              rejected_specs[dependency] ||= \
-                Gem::PlatformMismatch.new(tup.name, tup.version))
-            pm.add_platform tup.platform
-            false
-          else
-            true
-          end
+        if matching_platform && !Gem::Platform.match_gem?(tup.platform, tup.name)
+          pm = (
+            rejected_specs[dependency] ||= \
+              Gem::PlatformMismatch.new(tup.name, tup.version))
+          pm.add_platform tup.platform
+          false
+        elsif matching_platform && !ruby_abi_match?(tup)
+          false
+        else
+          true
         end
       end
     end
@@ -159,6 +163,7 @@ def spec_for_dependency(dependency, matching_platform = true)
     specs = []
     tuples.each do |tup, source|
       spec = source.fetch_spec(tup)
+      spec.content_address = tup.content_address if tup.content_address
     rescue Gem::RemoteFetcher::FetchError => e
       errors << Gem::SourceFetchProblem.new(source, e)
     else
@@ -168,6 +173,17 @@ def spec_for_dependency(dependency, matching_platform = true)
     [specs, errors]
   end
 
+  ##
+  # Decodes the content-addressable tuples in +spec_tuples+ ([tuple, source]
+  # pairs) to carry their real platform and Ruby ABI via each source.
+
+  def decode_content_addressable_tuples(spec_tuples, latest: false)
+    spec_tuples.group_by {|_, source| source }.flat_map do |source, source_tuples|
+      tuples = source_tuples.map(&:first)
+      decode_source_content_addressable_tuples(source, tuples, latest: latest).map {|tuple| [tuple, source] }
+    end
+  end
+
   ##
   # Suggests gems based on the supplied +gem_name+. Returns an array of
   # alternative gem names.
@@ -290,4 +306,20 @@ def tuples_for(source, type, gracefully_ignore = false) # :nodoc:
     raise unless gracefully_ignore
     []
   end
+
+  private
+
+  def decode_source_content_addressable_tuples(source, tuples, latest: false) # :nodoc:
+    return tuples unless source.respond_to?(:decode_content_addressable_tuples)
+
+    source.decode_content_addressable_tuples(tuples, latest: latest)
+  end
+
+  def ruby_abi_match?(tuple) # :nodoc:
+    !tuple.ruby_abi || tuple.ruby_abi == current_ruby_abi
+  end
+
+  def current_ruby_abi # :nodoc:
+    @current_ruby_abi ||= Gem.ruby_version.segments.first(2).join(".")
+  end
 end

test/rubygems/helper.rb

@@ -1037,6 +1037,31 @@ def util_gem(name, version, deps = nil, ruby_abi: nil, &block)
     [spec, cache_file]
   end
 
+  ##
+  # Builds a platform gem and serves it through compact index as a
+  # content-addressable gem. Returns the specification, gem path, and content
+  # address.
+
+  def util_setup_content_addressable_compact_index_gem(name, version, platform: "x86_64-linux", required_ruby_version: ">= 3.0", &block)
+    spec, gem_path = util_gem(name, version) do |s|
+      s.platform = platform
+      s.required_ruby_version = required_ruby_version
+      yield(s) if block
+    end
+
+    content_address = Digest::SHA256.file(gem_path).hexdigest[0, 8]
+    ca_gem_path = File.join(File.dirname(gem_path), "#{spec.name}-#{spec.version}-#{content_address}.gem")
+    FileUtils.cp gem_path, ca_gem_path
+    spec.content_address = content_address
+
+    util_setup_compact_index spec
+    @fetcher.data["#{@gem_repo}quick/Marshal.#{Gem.marshal_version}/#{spec.full_name}.gemspec.rz"] = util_zip(Marshal.dump(spec))
+    add_to_fetcher spec, ca_gem_path
+    Gem::SpecFetcher.fetcher = nil
+
+    [spec, ca_gem_path, content_address]
+  end
+
   ##
   # Gzips +data+.
 

test/rubygems/test_gem_commands_dependency_command.rb

@@ -188,6 +188,53 @@ def test_execute_remote
     assert_equal "", @stub_ui.error
   end
 
+  def test_execute_remote_content_addressable_compact_index_gem
+    spec_fetcher {}
+    util_set_arch "x86_64-linux"
+
+    _spec, _gem_path, content_address = util_setup_content_addressable_compact_index_gem(
+      "ca_dependency",
+      "1.0.0",
+      platform: "x86_64-linux",
+      required_ruby_version: "~> 3.3.0"
+    ) do |s|
+      s.add_runtime_dependency "dep_tophat", "~> 1.0"
+    end
+
+    @cmd.options[:args] = %w[ca_dependency]
+    @cmd.options[:domain] = :remote
+
+    use_ui @stub_ui do
+      @cmd.execute
+    end
+
+    assert_equal "Gem ca_dependency-1.0.0-#{content_address} (Platform: x86_64-linux Ruby ABI: 3.3)\n  dep_tophat (~> 1.0)\n\n", @stub_ui.output
+    assert_equal "", @stub_ui.error
+  end
+
+  def test_execute_remote_platform_compact_index_gem
+    spec_fetcher {}
+    util_set_arch "x86_64-linux"
+
+    spec = util_spec "platform_dependency", "1.0.0" do |s|
+      s.platform = "x86_64-linux"
+      s.add_runtime_dependency "dep_tophat", "~> 1.0"
+    end
+    util_setup_compact_index spec
+    write_marshalled_gemspecs spec
+    Gem::SpecFetcher.fetcher = nil
+
+    @cmd.options[:args] = %w[platform_dependency]
+    @cmd.options[:domain] = :remote
+
+    use_ui @stub_ui do
+      @cmd.execute
+    end
+
+    assert_equal "Gem platform_dependency-1.0.0-x86_64-linux\n  dep_tophat (~> 1.0)\n\n", @stub_ui.output
+    assert_equal "", @stub_ui.error
+  end
+
   def test_execute_remote_version
     @fetcher = Gem::FakeFetcher.new
     Gem::RemoteFetcher.fetcher = @fetcher

test/rubygems/test_gem_commands_fetch_command.rb

@@ -68,6 +68,44 @@ def test_execute_prerelease
                        "#{a2.full_name} not fetched")
   end
 
+  def test_execute_content_addressable_compact_index_gem
+    spec_fetcher {}
+    util_set_arch "x86_64-linux"
+
+    spec, _gem_path, content_address = util_setup_content_addressable_compact_index_gem(
+      "ca_fetch",
+      "1.0.0",
+      platform: "x86_64-linux"
+    )
+
+    @cmd.options[:args] = %w[ca_fetch]
+
+    execute_with_exit_code
+
+    assert_path_exist File.join(@tempdir, "ca_fetch-1.0.0-#{content_address}.gem")
+    assert_path_not_exist File.join(@tempdir, "ca_fetch-1.0.0-x86_64-linux.gem")
+    assert_equal "ca_fetch-1.0.0-#{content_address}", spec.full_name
+  end
+
+  def test_execute_platform_compact_index_gem
+    spec_fetcher {}
+    util_set_arch "x86_64-linux"
+
+    spec, gem_path = util_gem "platform_fetch", "1.0.0" do |s|
+      s.platform = "x86_64-linux"
+    end
+    util_setup_compact_index spec
+    write_marshalled_gemspecs spec
+    add_to_fetcher spec, gem_path
+    Gem::SpecFetcher.fetcher = nil
+
+    @cmd.options[:args] = %w[platform_fetch]
+
+    execute_with_exit_code
+
+    assert_path_exist File.join(@tempdir, "platform_fetch-1.0.0-x86_64-linux.gem")
+  end
+
   def test_execute_platform
     a2_spec, a2 = util_gem("a", "2")