fixup(FindSources): switch to new approach

* replace all calls to get_matching_tag to get_matching_source_url
  (3 occurances)
* rewrote get_github_info() and check_for_github_error() to raise
  a NotImplementedError. This will help consumers of these methods
  notify the change in capycli. In capycli, these methods are not
  used outside of FindSources.
* refactored the unittest test_find_golang_url_github for historical
  reasons: Due to a bug in the unittest it was the first test to
  break when I integrated the new approach. Through this the test
  became my workhorse for integrating the new approach.

Author: 16Martin

capycli/bom/findsources.py

@@ -200,27 +200,14 @@ def get_repo_name(github_url: str) -> str:
     @staticmethod
     def get_github_info(repository_url: str, username: str = "",
                         token: str = "") -> get_github_info_type:
+        """This method used to iterate through all resource pages of
+           GitHub's /tags API, collect the results, then return a huge
+           list with all results.
+           Removed because this approach does not scale well and we did
+           encounter projects with tens of thousands of tags.
         """
-        Query tag infos from GitHub.
-
-        In the good case a list of tags entries (= dictionaries) is returned.
-        In the bad case a JSON error message is returned.
-        """
-        length_per_page = 100
-        page = 1
-        tags: List[Dict[str, Any]] = []
-        tag_url = "https://api.github.com/repos/" + repository_url + "/tags"
-        query = "?per_page=%s&page=%s" % (length_per_page, page)
-        tmp = FindSources.github_request(tag_url + query, username, token)
-        if not isinstance(tmp, list):
-            return tags
-        tags.extend(tmp)
-        while len(tmp) == length_per_page:
-            page += 1
-            query = "?per_page=%s&page=%s" % (length_per_page, page)
-            tmp = FindSources.github_request(tag_url + query, username, token)
-            tags.extend(tmp)
-        return tags
+        raise NotImplementedError(
+            "Removed with introduction of get_matchting_source_tag!")
 
     def _get_github_repo(self, github_ref: str) -> Dict[str, Any]:
         """Fetch GitHub API object identified by @github_ref.
@@ -378,7 +365,7 @@ def find_github_url(self, component: Component, use_language: bool = True) -> st
         if len(name_match):
             for match in name_match:
                 tag_info = self.github_request(match["tags_url"], self.github_name, self.github_token)
-                source_url = self.get_matching_tag(tag_info, component.version or "", match["html_url"])
+                source_url = self.get_matching_source_url(component.version, match["tags_url"])
                 if len(name_match) == 1:
                     return source_url
                 elif source_url:
@@ -445,10 +432,7 @@ def find_golang_url(self, component: Component) -> str:
 
                 if repository_name.startswith("https://github.com/"):
                     repository_name = repository_name[len("https://github.com/"):]
-                tag_info = self.get_github_info(repository_name, self.github_name, self.github_token)
-                tag_info_checked = self.check_for_github_error(tag_info)
-                source_url = self.get_matching_tag(tag_info_checked, component_version,
-                                                   repository_name, version_prefix or "")
+                source_url = self.get_matching_source_url(component_version, repository_name, version_prefix)
 
         # component["RepositoryUrl"] = repository_name
         return source_url
@@ -468,26 +452,15 @@ def get_github_source_url(self, github_url: str, version: str) -> str:
 
         if self.verbose:
             print_text("      repo_name:", repo_name)
-
-        tag_info = self.get_github_info(repo_name, self.github_name, self.github_token)
-        tag_info_checked = self.check_for_github_error(tag_info)
-        return self.get_matching_tag(tag_info_checked, version, github_url)
+        return self.get_matching_source_url(version, repo_name)
 
     def check_for_github_error(self, tag_info: get_github_info_type) -> List[Dict[str, Any]]:
-        if isinstance(tag_info, list):
-            # assume valid answer
-            return tag_info
-
-        # check for 'rate limit exceeded' message
-        if "message" in tag_info:
-            if tag_info["message"].startswith("API rate limit exceeded"):
-                print_red("GitHub API rate limit exceeded - aborting!")
-                sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SERVICE)
-            if tag_info["message"].startswith("Bad credentials"):
-                print_red("Invalid GitHub credential provided - aborting!")
-                sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SERVICE)
-
-        return []
+        """This method was introduced to check the output of
+           get_github_info() for errors.
+           Removed, becasue get_github_info was removed.
+        """
+        raise NotImplementedError(
+            "Removed with introduction of get_matchting_source_tag!")
 
     def get_matching_tag(self, tag_info: List[Dict[str, Any]], version: str, github_url: str,
                          version_prefix: str = "") -> str:

tests/test_find_sources.py

@@ -417,19 +417,65 @@ def test_get_pkg_go_repo_url_error(self, mock_requests_get: Any) -> None:
         repo_url = find_sources.get_pkg_go_repo_url('some/package')
         self.assertEqual(repo_url, 'https://pkg.go.dev/some/package')
 
-    @patch('capycli.bom.findsources.FindSources.get_github_info')
+    @patch('capycli.bom.findsources.FindSources.get_matching_source_url')
+    @patch('capycli.bom.findsources.FindSources.get_pkg_go_repo_url')
     @patch('capycli.bom.findsources.FindSources.get_matching_tag')
-    def test_find_golang_url_github(self, mock_get_github_info: Any, mock_get_matching_tag: Any) -> None:
+    @patch('capycli.bom.findsources.FindSources.get_github_info')
+    def test_find_golang_url_github(self,
+            mock_get_github_info: Any,
+            mock_get_matching_tag: Any,
+            mock_get_pkg_go_repo_url: Any,
+            mock_get_matching_source_url: Any,
+        ) -> None:
         # Mocking a GitHub scenario
-        mock_get_github_info.return_value = 'https://pkg.go.dev/github.com/opencontainers/runc'
-        mock_get_matching_tag.return_value = 'https://github.com/opencontainers/runc/archive/refs/tags/v1.0.1.zip'
+        runc = { # real data as of 2024-11-18
+            'html_url': 'https://github.com/opencontainers/runc',
+            'zipball_url': 'https://github.com/opencontainers/runc/archive/refs/tags/v1.0.1.zip',
+        }
+        mock_get_github_info.return_value = []
+        mock_get_matching_tag.return_value = runc['zipball_url']
+        mock_get_pkg_go_repo_url.return_value = runc['html_url']
+        mock_get_matching_source_url.return_value = runc['zipball_url']
         find_sources = FindSources()
         component = MagicMock()
         component.name = 'github.com/opencontainers/runc'
         component.version = 'v1.0.1'
-        source_url = find_sources.find_golang_url(component)
 
-        self.assertEqual(source_url, 'https://pkg.go.dev/github.com/opencontainers/runc')
+        # semantic versioning, sunshine and rainbows
+        source_url = find_sources.find_golang_url(component)
+        self.assertEqual(source_url, runc['zipball_url'])
+
+        # version with +incompatible
+        with patch.object(component, 'version', new='v1.0.1+incompatible'):
+            source_url = find_sources.find_golang_url(component)
+            self.assertEqual(source_url, runc['zipball_url'])
+
+        # '-'-separated version with commit id
+        with patch.object(component, 'version', new='foo-bar-ThisIsACommitId'):
+            source_url = find_sources.find_golang_url(component)
+            self.assertEqual(source_url,
+                runc['html_url'] + '/archive/ThisIsACommitId.zip')
+
+        # component name w/o github.com
+        # with patch.object(component, 'name', new='opencontainers/runc'):
+        #    no point in testing because in this case get_pkg_go_repo_url
+        #    would return an empty string and we would never reach the
+        #    corresponding test in find_golang_url()
+        #    There is test_find_golang_url_non_github() ...
+
+        # missing data for remaining tests:
+        # - opencontainers/runc/<version_prefix>
+        # - component.name.startswith('gopkg.in')
+        # - component.name.startswith('https://github.com')
+        # I think these would also fail at find_golang_url()
+
+        # WARNING it would seem when patch()-ing a MagicMock the change
+        #         is not (always) reversed properly. Therefore this test
+        #         goes last!
+        # not published on pkg.go.dev
+        with patch.object(mock_get_pkg_go_repo_url, 'return_value', new=''):
+            source_url = find_sources.find_golang_url(component)
+            self.assertEqual(source_url, '')
 
     def test_find_golang_url_non_github(self) -> None:
         # Mocking a non-GitHub scenario