Vendor compact_index under a dedicated namespace

Gem::Indexer (rubygems-generate_index) loads the V1-only CompactIndex
constants, either from the compact_index gem or from its own embedded
copy, so the artifice's copy of rubygems.org's V2-only implementation
can never share that name safely.

Check in a copy of rubygems.org's lib/compact_index, renamed to the
VendoredCompactIndex namespace, under spec/support/vendor/compact_index,
and load it from the artifice with a plain require. Checking the copy in
(rather than downloading it during the test run) keeps the suite
hermetic and offline, makes the namespace rewrite visible in review, and
lets parallel workers and CI runners that skip the test-deps setup load
it without falling back to the incompatible gem.

Refresh the copy with `rake vendor:compact_index`;
`rake vendor:compact_index_check` fails if the checked-in copy drifts
from the pinned upstream ref.

Assisted-By: devx/8309afec-0948-41c8-a29d-3b1ea27ef32a

Author: jenshenny

.rubocop.yml

@@ -14,6 +14,7 @@ AllCops:
     - tmp/**/*
     - lib/rubygems/vendor/**/*
     - lib/bundler/vendor/**/*
+    - spec/support/vendor/**/*
   CacheRootDirectory: tmp/rubocop
   MaxFilesInCache: 5000
 

Rakefile

@@ -164,6 +164,46 @@ namespace :vendor do
       error_message: "Vendored gems are out of sync. Please update the vendored lib patches."
     )
   end
+
+  # Pinned upstream revision of rubygems/rubygems.org that the vendored
+  # compact_index copy is generated from. Bump this (or pass COMPACT_INDEX_REF)
+  # and re-run the task to refresh.
+  COMPACT_INDEX_REF = "bdf05e24cd381402822387240f1697c0193ad171"
+  COMPACT_INDEX_FILES = %w[
+    lib/compact_index.rb
+    lib/compact_index/dependency.rb
+    lib/compact_index/gem.rb
+    lib/compact_index/gem_version.rb
+    lib/compact_index/versions_file.rb
+  ].freeze
+
+  desc "Vendor spec-suite compact_index from rubygems.org (COMPACT_INDEX_REF to override ref)"
+  task :compact_index do
+    require "open-uri"
+    require "fileutils"
+
+    ref = ENV["COMPACT_INDEX_REF"] || COMPACT_INDEX_REF
+    dest_root = File.expand_path("spec/support/vendor/compact_index", __dir__)
+
+    COMPACT_INDEX_FILES.each do |path|
+      url = "https://raw.githubusercontent.com/rubygems/rubygems.org/#{ref}/#{path}"
+      contents = URI.parse(url).open(&:read).gsub("CompactIndex", "VendoredCompactIndex")
+
+      target = File.join(dest_root, path)
+      FileUtils.mkdir_p(File.dirname(target))
+      File.write(target, contents)
+    end
+
+    puts "Vendored compact_index from rubygems.org@#{ref} into #{dest_root}"
+  end
+
+  desc "Check vendored compact_index is up to date"
+  task compact_index_check: :compact_index do
+    Spec::Rubygems.check_source_control_changes(
+      success_message: "Vendored compact_index is in sync",
+      error_message: "Vendored compact_index is out of sync. Run `rake vendor:compact_index`."
+    )
+  end
 end
 
 namespace :rubocop do

spec/support/artifice/compact_index_concurrent_download.rb

@@ -21,7 +21,7 @@ class CompactIndexConcurrentDownload < CompactIndexAPI
     etag_response do
       file = tmp("versions.list")
       FileUtils.rm_f(file)
-      file = CompactIndex::VersionsFile.new(file.to_s)
+      file = VendoredCompactIndex::VersionsFile.new(file.to_s)
       file.create(gems)
       file.contents
     end

spec/support/artifice/compact_index_partial_update_bad_digest.rb

@@ -21,7 +21,7 @@ def partial_update_bad_digest
     partial_update_bad_digest do
       file = tmp("versions.list")
       FileUtils.rm_f(file)
-      file = CompactIndex::VersionsFile.new(file.to_s)
+      file = VendoredCompactIndex::VersionsFile.new(file.to_s)
       file.create(gems)
       file.contents([], calculate_info_checksums: true)
     end
@@ -30,7 +30,7 @@ def partial_update_bad_digest
   get "/info/:name" do
     partial_update_bad_digest do
       gem = gems.find {|g| g.name == params[:name] }
-      CompactIndex.info(gem ? gem.versions : [])
+      VendoredCompactIndex.info(gem ? gem.versions : [])
     end
   end
 end

spec/support/artifice/compact_index_partial_update_no_digest_not_incremental.rb

@@ -16,7 +16,7 @@ def partial_update_no_digest
     partial_update_no_digest do
       file = tmp("versions.list")
       FileUtils.rm_f(file)
-      file = CompactIndex::VersionsFile.new(file.to_s)
+      file = VendoredCompactIndex::VersionsFile.new(file.to_s)
       file.create(gems)
       lines = file.contents([], calculate_info_checksums: true).split("\n")
       name, versions, checksum = lines.last.split(" ")
@@ -29,7 +29,7 @@ def partial_update_no_digest
   get "/info/:name" do
     partial_update_no_digest do
       gem = gems.find {|g| g.name == params[:name] }
-      lines = CompactIndex.info(gem ? gem.versions : []).split("\n")
+      lines = VendoredCompactIndex.info(gem ? gem.versions : []).split("\n")
 
       # shuffle versions so new versions are not appended to the end
       [lines.first, lines.last, *lines[1..-2]].join("\n")

spec/support/artifice/compact_index_precompiled_before.rb

@@ -6,7 +6,7 @@ class CompactIndexPrecompiledBefore < CompactIndexAPI
   get "/info/:name" do
     etag_response do
       gem = gems.find {|g| g.name == params[:name] }
-      move_ruby_variant_to_the_end(CompactIndex.info(gem ? gem.versions : []))
+      move_ruby_variant_to_the_end(VendoredCompactIndex.info(gem ? gem.versions : []))
     end
   end
 

spec/support/artifice/compact_index_range_ignored.rb

@@ -28,7 +28,7 @@ def not_modified?(_checksum)
     etag_response do
       file = tmp("versions.list")
       FileUtils.rm_f(file)
-      file = CompactIndex::VersionsFile.new(file.to_s)
+      file = VendoredCompactIndex::VersionsFile.new(file.to_s)
       file.create(gems)
       file.contents
     end

spec/support/artifice/compact_index_rate_limited.rb

@@ -32,7 +32,7 @@ def self.deq
       if RequestCounter.size == 1
         etag_response do
           gem = gems.find {|g| g.name == params[:name] }
-          CompactIndex.info(gem ? gem.versions : [])
+          VendoredCompactIndex.info(gem ? gem.versions : [])
         end
       else
         status 429

spec/support/artifice/compact_index_wrong_dependencies.rb

@@ -7,7 +7,7 @@ class CompactIndexWrongDependencies < CompactIndexAPI
     etag_response do
       gem = gems.find {|g| g.name == params[:name] }
       gem.versions.each {|gv| gv.dependencies.clear } if gem
-      CompactIndex.info(gem ? gem.versions : [])
+      VendoredCompactIndex.info(gem ? gem.versions : [])
     end
   end
 end

spec/support/artifice/compact_index_wrong_gem_checksum.rb

@@ -11,7 +11,7 @@ class CompactIndexWrongGemChecksum < CompactIndexAPI
       checksum = ENV.fetch("BUNDLER_SPEC_#{name.upcase}_CHECKSUM") { "IiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiI=" }
       versions = gem ? gem.versions : []
       versions.each {|v| v.checksum = checksum }
-      CompactIndex.info(versions)
+      VendoredCompactIndex.info(versions)
     end
   end
 end