feat: conduct source URL discovery using sw360 lookup, perform extensive GitLab deep search, and adapt search strategy based on different programming language

Author: Shaji

capycli/bom/findsources.py

@@ -34,7 +34,7 @@ def __init__(self):
         self.project = None
         self.sw360_url = os.environ.get("SW360ServerUrl", None)
 
-    def login(self, token: str = "", url: str = "", oauth2: bool = False) -> bool:
+    def login(self, token: str = "", url: str = "", oauth2: bool = False, exit_no_login: bool = True) -> bool:
         """Login to SW360"""
         self.sw360_url = os.environ.get("SW360ServerUrl", None)
         sw360_api_token = os.environ.get("SW360ProductionToken", None)
@@ -46,15 +46,21 @@ def login(self, token: str = "", url: str = "", oauth2: bool = False) -> bool:
             self.sw360_url = url
 
         if not self.sw360_url:
-            print_red("  No SW360 server URL specified!")
-            sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
+            if exit_no_login:
+                print_red("  No SW360 server URL specified!")
+                sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
+            else:
+                return False
 
         if self.sw360_url[-1] != "/":
             self.sw360_url += "/"
 
         if not sw360_api_token:
-            print_red("  No SW360 API token specified!")
-            sys.exit(ResultCode.RESULT_AUTH_ERROR)
+            if exit_no_login:
+                print_red("  No SW360 API token specified!")
+                sys.exit(ResultCode.RESULT_AUTH_ERROR)
+            else:
+                return False
 
         self.client = sw360.sw360_api.SW360(self.sw360_url, sw360_api_token, oauth2)
 

capycli/common/script_base.py

@@ -160,6 +160,13 @@ def register_options(self):
             help="use this token for access to SW360",
         )
 
+        self.parser.add_argument(
+            "-gt",
+            "--github_token",
+            dest="github_token",
+            help="use this token for access to github",
+        )
+
         self.parser.add_argument(
             "-oa",
             "--oauth2",

capycli/main/options.py

@@ -37,6 +37,7 @@ capycli = "capycli.main.cli:main"
 python = "^3.8" # drop support for 3.6 and 3.7 because of wheel and cli-support
 colorama = "^0.4.3"
 requests = "^2.31.0" # fix CVE-2023-32681 
+semver = "^2.9.1"
 packageurl-python = ">0.8, <1.0"
 pyjwt = "^1.7.1"
 openpyxl = "^3.0.3"
@@ -51,6 +52,7 @@ tomli = "^2.0.1"
 dateparser = "^1.1.8"
 urllib3 = "*"
 importlib-resources = "^5.12.0"
+beautifulsoup4 = "^4.11.1"
 
 [tool.poetry.dev-dependencies]
 flake8 = ">=3.7.8"

poetry.lock

@@ -7,13 +7,15 @@
 # -------------------------------------------------------------------------------
 
 import os
+import responses
 
 import capycli.common.json_support
 import capycli.common.script_base
 from capycli.bom.findsources import FindSources
 from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport
 from capycli.main.result_codes import ResultCode
 from tests.test_base import AppArguments, TestBase
+from unittest.mock import MagicMock, patch
 
 
 class TestFindSources(TestBase):
@@ -100,7 +102,7 @@ def test_find_sources(self) -> None:
         self.assertTrue("Using anonymous GitHub access" in out)
         self.assertTrue("8 components read from SBOM" in out)
         self.assertTrue("1 source files were already available" in out)
-        self.assertTrue("5 source file URLs were found" in out)
+        self.assertTrue("6 source file URLs were found" in out)
 
         sbom = CaPyCliBom.read_sbom(args.outputfile)
         self.assertIsNotNone(sbom)
@@ -191,3 +193,69 @@ def test_normalize_version(self):
                 actual = sut.to_semver_string(version)
                 self.assertEqual(actual, expected)
                 self.assertTrue(actual == expected, 'version %s is %s' % (actual, expected))
+
+    @responses.activate
+    def test_get_release_component_id(self):
+        # Mock the sw360 client
+        mock_client = MagicMock()
+        mock_client.get_release.return_value = {"_links": {"sw360:component": {"href": self.MYURL + 'components/123'}}}
+
+        # Call the method and assert the result
+        find_sources = FindSources()
+        find_sources.client = mock_client
+        component_id = find_sources.get_release_component_id("some_release_id")
+        self.assertEqual(component_id, "123")
+
+    @responses.activate
+    def test_find_source_url_from_component(self):
+        # Mock the client
+        mock_client = MagicMock()
+        mock_client.get_component.return_value = {"_embedded": {"sw360:releases": [{"_links": {"self": {"href": self.MYURL + 'releases/456'}}}]}}
+        mock_client.get_release.return_value = {"_links": {"sw360:component": {"href": self.MYURL + 'components/123'}}, "sourceCodeDownloadurl": "http://github.com/some/repo/0.0.0"}
+
+        # Call the method and assert the result
+        find_sources = FindSources()
+        find_sources.client = mock_client  # Inject the mocked client
+        source_url = find_sources.find_source_url_from_component(component_id="some_component_id")
+        self.assertEqual(source_url, "http://github.com/some/repo/0.0.0")
+
+    @patch('requests.get')
+    @patch('bs4.BeautifulSoup')
+    def test_get_pkg_go_repo_url_success(self, mock_beautifulsoup, mock_requests_get):
+        # Mocking successful response
+        mock_requests_get.return_value.text = '<div class="UnitMeta-repo"><a href="https://github.com/example/repo/1.0.0">Repo Link</a></div>'
+        mock_beautifulsoup.return_value.find.return_value = MagicMock(get=lambda x: 'https://github.com/example/repo/1.0.0')
+        find_sources = FindSources()
+        repo_url = find_sources.get_pkg_go_repo_url('example/package')
+        self.assertEqual(repo_url, 'https://github.com/example/repo/1.0.0')
+
+    @patch('requests.get', side_effect=Exception('Some error'))
+    def test_get_pkg_go_repo_url_error(self, mock_requests_get):
+        # Mocking an exception during the request
+        find_sources = FindSources()
+        repo_url = find_sources.get_pkg_go_repo_url('some/package')
+        self.assertEqual(repo_url, 'https://pkg.go.dev/some/package')
+
+    @patch('capycli.bom.findsources.FindSources.get_github_info')
+    @patch('capycli.bom.findsources.FindSources.get_matching_tag')
+    def test_find_golang_url_github(self, mock_get_github_info, mock_get_matching_tag):
+        # Mocking a GitHub scenario
+        mock_get_github_info.return_value = 'https://pkg.go.dev/github.com/opencontainers/runc'
+        mock_get_matching_tag.return_value = 'https://github.com/opencontainers/runc/archive/refs/tags/v1.0.1.zip'
+        find_sources = FindSources()
+        component = MagicMock()
+        component.name = 'github.com/opencontainers/runc'
+        component.version = 'v1.0.1'
+        source_url = find_sources.find_golang_url(component)
+
+        self.assertEqual(source_url, 'https://pkg.go.dev/github.com/opencontainers/runc')
+
+    def test_find_golang_url_non_github(self):
+        # Mocking a non-GitHub scenario
+        find_sources = FindSources()
+        component = MagicMock()
+        component.name = 'example/package'
+        component.version = 'v1.0.0'
+        source_url = find_sources.find_golang_url(component)
+
+        self.assertEqual(source_url, '')

pyproject.toml

@@ -405,7 +405,7 @@ def test_process_poetry_lock_v2(self):
         self.assertTrue("Checking meta-data:" in out)
         self.assertTrue("cli-support" in out)
         self.assertTrue(self.OUTPUTFILE2 in out)
-        self.assertTrue("34 components items written to file." in out)
+        self.assertTrue("37 components items written to file." in out)
 
         # ensure that dev dependencies are NOT listed
         self.assertTrue("flake8" not in out)