Cap GitHub API pagination for releases and owner repos

fetch_releases and load_owner_repos_names previously used the global
auto_paginate client which fetches all pages into memory. For repos
with thousands of releases (each containing markdown bodies and asset
metadata) or orgs with thousands of repos, this can use significant
memory. Switch to manual pagination with a default cap of 10 pages
(1000 items).

Author: andrew

app/models/hosts/github.rb

@@ -216,8 +216,21 @@ def download_releases(repository)
       nil
     end
 
-    def fetch_releases(repository)
-      api_client.releases(repository.full_name).map do |release|
+    def fetch_releases(repository, max_pages: 10)
+      client = api_client(nil, auto_paginate: false)
+      releases = []
+      response = client.releases(repository.full_name, per_page: 100)
+      releases.concat(response)
+
+      last_resp = client.last_response
+      pages_fetched = 1
+      while pages_fetched < max_pages && last_resp.rels[:next]
+        last_resp = last_resp.rels[:next].get
+        releases.concat(last_resp.data)
+        pages_fetched += 1
+      end
+
+      releases.map do |release|
         {
           uuid: release.id,
           tag_name: release.tag_name,
@@ -237,8 +250,21 @@ def fetch_releases(repository)
       []
     end
 
-    def load_owner_repos_names(owner)
-      api_client.repos(owner.login, type: "all").map { |repo| repo[:full_name] }
+    def load_owner_repos_names(owner, max_pages: 10)
+      client = api_client(nil, auto_paginate: false)
+      repos = []
+      response = client.repos(owner.login, type: "all", per_page: 100)
+      repos.concat(response.map { |repo| repo[:full_name] })
+
+      last_resp = client.last_response
+      pages_fetched = 1
+      while pages_fetched < max_pages && last_resp.rels[:next]
+        last_resp = last_resp.rels[:next].get
+        repos.concat(last_resp.data.map { |repo| repo[:full_name] })
+        pages_fetched += 1
+      end
+
+      repos
     rescue *IGNORABLE_EXCEPTIONS, Octokit::NotFound
       []
     end

test/models/hosts/github_test.rb

@@ -0,0 +1,145 @@
+require "test_helper"
+
+class Hosts::GithubTest < ActiveSupport::TestCase
+  setup do
+    @host = create(:github_host)
+    @github = Hosts::Github.new(@host)
+    @repository = create(:repository, host: @host, full_name: 'testuser/testrepo', owner: 'testuser')
+  end
+
+  context 'fetch_releases' do
+    should 'fetch releases with manual pagination' do
+      release = OpenStruct.new(
+        id: 1,
+        tag_name: 'v1.0.0',
+        target_commitish: 'main',
+        name: 'Release 1.0.0',
+        body: 'First release',
+        draft: false,
+        prerelease: false,
+        created_at: 1.day.ago,
+        published_at: 1.day.ago,
+        author: OpenStruct.new(login: 'testuser'),
+        assets: []
+      )
+
+      last_response = mock('last_response')
+      last_response.stubs(:rels).returns({})
+
+      client = mock('client')
+      client.expects(:releases).with('testuser/testrepo', per_page: 100).returns([release])
+      client.stubs(:last_response).returns(last_response)
+
+      @github.stubs(:api_client).with(nil, auto_paginate: false).returns(client)
+
+      result = @github.fetch_releases(@repository)
+
+      assert_equal 1, result.length
+      assert_equal 1, result.first[:uuid]
+      assert_equal 'v1.0.0', result.first[:tag_name]
+    end
+
+    should 'stop after max_pages' do
+      release1 = OpenStruct.new(
+        id: 1, tag_name: 'v1.0', target_commitish: 'main', name: 'r', body: 'b',
+        draft: false, prerelease: false, created_at: Time.now, published_at: Time.now,
+        author: OpenStruct.new(login: 'u'), assets: []
+      )
+      release2 = OpenStruct.new(
+        id: 2, tag_name: 'v2.0', target_commitish: 'main', name: 'r2', body: 'b2',
+        draft: false, prerelease: false, created_at: Time.now, published_at: Time.now,
+        author: OpenStruct.new(login: 'u'), assets: []
+      )
+
+      page2_response = mock('page2_response')
+      page2_response.stubs(:data).returns([release2])
+      page2_response.stubs(:rels).returns({})
+
+      next_rel = mock('next_rel')
+      next_rel.stubs(:get).returns(page2_response)
+
+      first_last_response = mock('first_last_response')
+      first_last_response.stubs(:rels).returns({ next: next_rel })
+
+      client = mock('client')
+      client.expects(:releases).with('testuser/testrepo', per_page: 100).returns([release1])
+      client.stubs(:last_response).returns(first_last_response)
+
+      @github.stubs(:api_client).with(nil, auto_paginate: false).returns(client)
+
+      result = @github.fetch_releases(@repository, max_pages: 2)
+
+      assert_equal 2, result.length
+      assert_equal 'v1.0', result.first[:tag_name]
+      assert_equal 'v2.0', result.last[:tag_name]
+    end
+
+    should 'return empty array on error' do
+      client = mock('client')
+      client.expects(:releases).raises(Octokit::NotFound)
+      @github.stubs(:api_client).with(nil, auto_paginate: false).returns(client)
+
+      result = @github.fetch_releases(@repository)
+
+      assert_equal [], result
+    end
+  end
+
+  context 'load_owner_repos_names' do
+    setup do
+      @owner = OpenStruct.new(login: 'testuser')
+    end
+
+    should 'fetch repo names with manual pagination' do
+      repo = { full_name: 'testuser/repo1' }
+
+      last_response = mock('last_response')
+      last_response.stubs(:rels).returns({})
+
+      client = mock('client')
+      client.expects(:repos).with('testuser', type: 'all', per_page: 100).returns([repo])
+      client.stubs(:last_response).returns(last_response)
+
+      @github.stubs(:api_client).with(nil, auto_paginate: false).returns(client)
+
+      result = @github.load_owner_repos_names(@owner)
+
+      assert_equal ['testuser/repo1'], result
+    end
+
+    should 'stop after max_pages' do
+      repo1 = { full_name: 'testuser/repo1' }
+      repo2 = { full_name: 'testuser/repo2' }
+
+      page2_response = mock('page2_response')
+      page2_response.stubs(:data).returns([repo2])
+      page2_response.stubs(:rels).returns({})
+
+      next_rel = mock('next_rel')
+      next_rel.stubs(:get).returns(page2_response)
+
+      first_last_response = mock('first_last_response')
+      first_last_response.stubs(:rels).returns({ next: next_rel })
+
+      client = mock('client')
+      client.expects(:repos).with('testuser', type: 'all', per_page: 100).returns([repo1])
+      client.stubs(:last_response).returns(first_last_response)
+
+      @github.stubs(:api_client).with(nil, auto_paginate: false).returns(client)
+
+      result = @github.load_owner_repos_names(@owner, max_pages: 2)
+
+      assert_equal ['testuser/repo1', 'testuser/repo2'], result
+    end
+
+    should 'return empty array on error' do
+      client = mock('client')
+      client.expects(:repos).raises(Octokit::NotFound)
+      @github.stubs(:api_client).with(nil, auto_paginate: false).returns(client)
+
+      result = @github.load_owner_repos_names(@owner)
+
+      assert_equal [], result
+    end
+  end
+end