Skip to content

Commit

Permalink
fixup(FindSources): switch to new approach
Browse files Browse the repository at this point in the history 
* replace all calls to get_matching_tag to get_matching_source_url
  (3 occurances)
* rewrote get_github_info() and check_for_github_error() to raise
  a NotImplementedError. This will help consumers of these methods
  notify the change in capycli. In capycli, these methods are not
  used outside of FindSources.
* refactored the unittest test_find_golang_url_github for historical
  reasons: Due to a bug in the unittest it was the first test to
  break when I integrated the new approach. Through this the test
  became my workhorse for integrating the new approach.
  • Loading branch information
@16Martin
16Martin committed Nov 30, 2024
1 parent 669e0b9 commit b6649c2
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 49 deletions.
59 changes: 16 additions & 43 deletions capycli/bom/findsources.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,27 +200,14 @@ def get_repo_name(github_url: str) -> str:
@staticmethod
def get_github_info(repository_url: str, username: str = "",
token: str = "") -> get_github_info_type:
"""This method used to iterate through all resource pages of
GitHub's /tags API, collect the results, then return a huge
list with all results.
Removed because this approach does not scale well and we did
encounter projects with tens of thousands of tags.
"""
Query tag infos from GitHub.
In the good case a list of tags entries (= dictionaries) is returned.
In the bad case a JSON error message is returned.
"""
length_per_page = 100
page = 1
tags: List[Dict[str, Any]] = []
tag_url = "https://api.github.com/repos/" + repository_url + "/tags"
query = "?per_page=%s&page=%s" % (length_per_page, page)
tmp = FindSources.github_request(tag_url + query, username, token)
if not isinstance(tmp, list):
return tags
tags.extend(tmp)
while len(tmp) == length_per_page:
page += 1
query = "?per_page=%s&page=%s" % (length_per_page, page)
tmp = FindSources.github_request(tag_url + query, username, token)
tags.extend(tmp)
return tags
raise NotImplementedError(
"Removed with introduction of get_matchting_source_tag!")

def _get_github_repo(self, github_ref: str) -> Dict[str, Any]:
"""Fetch GitHub API object identified by @github_ref.
Expand Down Expand Up @@ -378,7 +365,7 @@ def find_github_url(self, component: Component, use_language: bool = True) -> st
if len(name_match):
for match in name_match:
tag_info = self.github_request(match["tags_url"], self.github_name, self.github_token)
source_url = self.get_matching_tag(tag_info, component.version or "", match["html_url"])
source_url = self.get_matching_source_url(component.version, match["tags_url"])
if len(name_match) == 1:
return source_url
elif source_url:
Expand Down Expand Up @@ -445,10 +432,7 @@ def find_golang_url(self, component: Component) -> str:

if repository_name.startswith("https://github.com/"):
repository_name = repository_name[len("https://github.com/"):]
tag_info = self.get_github_info(repository_name, self.github_name, self.github_token)
tag_info_checked = self.check_for_github_error(tag_info)
source_url = self.get_matching_tag(tag_info_checked, component_version,
repository_name, version_prefix or "")
source_url = self.get_matching_source_url(component_version, repository_name, version_prefix)

# component["RepositoryUrl"] = repository_name
return source_url
Expand All @@ -468,26 +452,15 @@ def get_github_source_url(self, github_url: str, version: str) -> str:

if self.verbose:
print_text(" repo_name:", repo_name)

tag_info = self.get_github_info(repo_name, self.github_name, self.github_token)
tag_info_checked = self.check_for_github_error(tag_info)
return self.get_matching_tag(tag_info_checked, version, github_url)
return self.get_matching_source_url(version, repo_name)

def check_for_github_error(self, tag_info: get_github_info_type) -> List[Dict[str, Any]]:
if isinstance(tag_info, list):
# assume valid answer
return tag_info

# check for 'rate limit exceeded' message
if "message" in tag_info:
if tag_info["message"].startswith("API rate limit exceeded"):
print_red("GitHub API rate limit exceeded - aborting!")
sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SERVICE)
if tag_info["message"].startswith("Bad credentials"):
print_red("Invalid GitHub credential provided - aborting!")
sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SERVICE)

return []
"""This method was introduced to check the output of
get_github_info() for errors.
Removed, becasue get_github_info was removed.
"""
raise NotImplementedError(
"Removed with introduction of get_matchting_source_tag!")

def get_matching_tag(self, tag_info: List[Dict[str, Any]], version: str, github_url: str,
version_prefix: str = "") -> str:
Expand Down
58 changes: 52 additions & 6 deletions tests/test_find_sources.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,19 +417,65 @@ def test_get_pkg_go_repo_url_error(self, mock_requests_get: Any) -> None:
repo_url = find_sources.get_pkg_go_repo_url('some/package')
self.assertEqual(repo_url, 'https://pkg.go.dev/some/package')

@patch('capycli.bom.findsources.FindSources.get_github_info')
@patch('capycli.bom.findsources.FindSources.get_matching_source_url')
@patch('capycli.bom.findsources.FindSources.get_pkg_go_repo_url')
@patch('capycli.bom.findsources.FindSources.get_matching_tag')
def test_find_golang_url_github(self, mock_get_github_info: Any, mock_get_matching_tag: Any) -> None:
@patch('capycli.bom.findsources.FindSources.get_github_info')
def test_find_golang_url_github(self,
mock_get_github_info: Any,
mock_get_matching_tag: Any,
mock_get_pkg_go_repo_url: Any,
mock_get_matching_source_url: Any,
) -> None:
# Mocking a GitHub scenario
mock_get_github_info.return_value = 'https://pkg.go.dev/github.com/opencontainers/runc'
mock_get_matching_tag.return_value = 'https://github.com/opencontainers/runc/archive/refs/tags/v1.0.1.zip'
runc = { # real data as of 2024-11-18
'html_url': 'https://github.com/opencontainers/runc',
'zipball_url': 'https://github.com/opencontainers/runc/archive/refs/tags/v1.0.1.zip',
}
mock_get_github_info.return_value = []
mock_get_matching_tag.return_value = runc['zipball_url']
mock_get_pkg_go_repo_url.return_value = runc['html_url']
mock_get_matching_source_url.return_value = runc['zipball_url']
find_sources = FindSources()
component = MagicMock()
component.name = 'github.com/opencontainers/runc'
component.version = 'v1.0.1'
source_url = find_sources.find_golang_url(component)

self.assertEqual(source_url, 'https://pkg.go.dev/github.com/opencontainers/runc')
# semantic versioning, sunshine and rainbows
source_url = find_sources.find_golang_url(component)
self.assertEqual(source_url, runc['zipball_url'])

# version with +incompatible
with patch.object(component, 'version', new='v1.0.1+incompatible'):
source_url = find_sources.find_golang_url(component)
self.assertEqual(source_url, runc['zipball_url'])

# '-'-separated version with commit id
with patch.object(component, 'version', new='foo-bar-ThisIsACommitId'):
source_url = find_sources.find_golang_url(component)
self.assertEqual(source_url,
runc['html_url'] + '/archive/ThisIsACommitId.zip')

# component name w/o github.com
# with patch.object(component, 'name', new='opencontainers/runc'):
# no point in testing because in this case get_pkg_go_repo_url
# would return an empty string and we would never reach the
# corresponding test in find_golang_url()
# There is test_find_golang_url_non_github() ...

# missing data for remaining tests:
# - opencontainers/runc/<version_prefix>
# - component.name.startswith('gopkg.in')
# - component.name.startswith('https://github.com')
# I think these would also fail at find_golang_url()

# WARNING it would seem when patch()-ing a MagicMock the change
# is not (always) reversed properly. Therefore this test
# goes last!
# not published on pkg.go.dev
with patch.object(mock_get_pkg_go_repo_url, 'return_value', new=''):
source_url = find_sources.find_golang_url(component)
self.assertEqual(source_url, '')

def test_find_golang_url_non_github(self) -> None:
# Mocking a non-GitHub scenario
Expand Down

0 comments on commit b6649c2

Please sign in to comment.