From e2e1d69f43ae633a0bafd5793ac30ea0a59fb2ea Mon Sep 17 00:00:00 2001 From: Gernot Hillier Date: Mon, 31 Jul 2023 07:30:27 +0200 Subject: [PATCH] feat(bom): new command `downloadattachments` --- ChangeLog.md | 1 + capycli/bom/download_attachments.py | 183 +++++++++++++++ capycli/bom/handle_bom.py | 7 + tests/fixtures/sbom_for_download.json | 28 ++- tests/test_bom_downloadattachments.py | 317 ++++++++++++++++++++++++++ tests/test_merge_bom.py | 4 +- 6 files changed, 537 insertions(+), 3 deletions(-) create mode 100644 capycli/bom/download_attachments.py create mode 100644 tests/test_bom_downloadattachments.py diff --git a/ChangeLog.md b/ChangeLog.md index 862e810..03459f1 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -14,6 +14,7 @@ * `project createbom` add SW360 attachment info as external references to SBOM (currently supported: source, binary, CLI, report). * `project createbom` adds SW360 project name, version and description to SBOM. +* new command `bom downloadattachments` to download CLI and report attachments ## 2.0.0 (2023-06-02) diff --git a/capycli/bom/download_attachments.py b/capycli/bom/download_attachments.py new file mode 100644 index 0000000..ff1fbfe --- /dev/null +++ b/capycli/bom/download_attachments.py @@ -0,0 +1,183 @@ +# ------------------------------------------------------------------------------- +# Copyright (c) 2020-2023 Siemens +# All Rights Reserved. +# Author: gernot.hillier@siemens.com, thomas.graf@siemens.com +# +# SPDX-License-Identifier: MIT +# ------------------------------------------------------------------------------- + +import logging +import os +import pathlib +import sys + +import sw360.sw360_api +from cyclonedx.model import ExternalReferenceType +from cyclonedx.model.bom import Bom +from cyclonedx.model.component import Component + +import capycli.common.json_support +import capycli.common.script_base +from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport, SbomWriter +from capycli.common.print import print_red, print_text, print_yellow +from capycli.common.script_support import ScriptSupport +from capycli.main.result_codes import ResultCode + +LOG = capycli.get_logger(__name__) + + +class BomDownloadAttachments(capycli.common.script_base.ScriptBase): + """ + Download SW360 attachments as specified in the SBOM. + """ + + def download_attachments(self, sbom: Bom, source_folder: str) -> Bom: + for component in sbom.components: + item_name = ScriptSupport.get_full_name_from_component(component) + print_text(" " + item_name) + + for ext_ref in component.external_references: + if not ext_ref.comment: + continue + if (not ext_ref.comment.startswith(CaPyCliBom.CLI_FILE_COMMENT) + and not ext_ref.comment.startswith(CaPyCliBom.CRT_FILE_COMMENT)): + continue + + attachment_id = ext_ref.comment.split(", sw360Id: ") + if len(attachment_id) != 2: + print_red(" No sw360Id for attachment!") + continue + attachment_id = attachment_id[1] + + release_id = CycloneDxSupport.get_property_value(component, CycloneDxSupport.CDX_PROP_SW360ID) + if not release_id: + print_red(" No sw360Id for release!") + continue + print(" ", ext_ref.url, release_id, attachment_id) + filename = os.path.join(source_folder, ext_ref.url) + + try: + at_info = self.client.get_attachment(attachment_id) + at_info = {k: v for k, v in at_info.items() + if k.startswith("check") + or k.startswith("created")} + print(at_info) + + self.client.download_release_attachment(filename, release_id, attachment_id) + ext_ref.url = filename + except sw360.sw360_api.SW360Error as swex: + print_red(" Error getting", swex.url, swex.response) + return sbom + + def have_relative_source_file_path(self, component: Component, bompath: str): + ext_ref = CycloneDxSupport.get_ext_ref( + component, ExternalReferenceType.DISTRIBUTION, CaPyCliBom.SOURCE_FILE_COMMENT) + if not ext_ref: + return + + bip = pathlib.PurePath(ext_ref.url) + try: + CycloneDxSupport.update_or_set_property( + component, + CycloneDxSupport.CDX_PROP_FILENAME, + bip.name) + file = bip.as_posix() + if os.path.isfile(file): + CycloneDxSupport.update_or_set_ext_ref( + component, + ExternalReferenceType.DISTRIBUTION, + CaPyCliBom.SOURCE_FILE_COMMENT, + "file://" + bip.relative_to(bompath).as_posix()) + except ValueError: + print_yellow( + " SBOM file is not relative to source file " + ext_ref.url) + # .relative_to + pass + + def update_local_path(self, sbom: Bom, bomfile: str): + bompath = pathlib.Path(bomfile).parent + for component in sbom.components: + self.have_relative_source_file_path(component, bompath) + + def run(self, args): + """Main method + + @params: + args - command line arguments + """ + if args.debug: + global LOG + LOG = capycli.get_logger(__name__) + else: + # suppress (debug) log output from requests and urllib + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("urllib3.connectionpool").setLevel(logging.WARNING) + + print_text( + "\n" + capycli.APP_NAME + ", " + capycli.get_app_version() + + " - Download SW360 attachments as specified in the SBOM\n") + + if args.help: + print("usage: capycli bom downloadattachments -i bom.json [-source ]") + print("") + print("optional arguments:") + print(" -h, --help show this help message and exit") + print(" -i INPUTFILE, input SBOM file to read from (JSON)") + print(" -source SOURCE source folder or additional source file") + print(" -o OUTPUTFILE output file to write to") + print(" -v be verbose") + return + + if not args.inputfile: + print_red("No input file specified!") + sys.exit(ResultCode.RESULT_COMMAND_ERROR) + + if not os.path.isfile(args.inputfile): + print_red("Input file not found!") + sys.exit(ResultCode.RESULT_FILE_NOT_FOUND) + + print_text("Loading SBOM file " + args.inputfile) + try: + bom = CaPyCliBom.read_sbom(args.inputfile) + except Exception as ex: + print_red("Error reading input SBOM file: " + repr(ex)) + sys.exit(ResultCode.RESULT_ERROR_READING_BOM) + + if args.verbose: + print_text(" " + str(len(bom.components)) + "components read from SBOM file") + + source_folder = "./" + if args.source: + source_folder = args.source + if (not source_folder) or (not os.path.isdir(source_folder)): + print_red("Target source code folder does not exist!") + sys.exit(ResultCode.RESULT_COMMAND_ERROR) + + if args.sw360_token and args.oauth2: + self.analyze_token(args.sw360_token) + + print_text(" Checking access to SW360...") + if not self.login(token=args.sw360_token, url=args.sw360_url, oauth2=args.oauth2): + print_red("ERROR: login failed!") + sys.exit(ResultCode.RESULT_AUTH_ERROR) + + print_text("Downloading source files to folder " + source_folder + " ...") + + self.download_attachments(bom, source_folder) + + if args.outputfile: + print_text("Updating path information") + self.update_local_path(bom, args.outputfile) + + print_text("Writing updated SBOM to " + args.outputfile) + try: + SbomWriter.write_to_json(bom, args.outputfile, True) + except Exception as ex: + print_red("Error writing updated SBOM file: " + repr(ex)) + sys.exit(ResultCode.RESULT_ERROR_WRITING_BOM) + + if args.verbose: + print_text(" " + str(len(bom.components)) + " components written to SBOM file") + + print("\n") diff --git a/capycli/bom/handle_bom.py b/capycli/bom/handle_bom.py index c7ffc2e..4bb4fac 100644 --- a/capycli/bom/handle_bom.py +++ b/capycli/bom/handle_bom.py @@ -15,6 +15,7 @@ import capycli.bom.create_components import capycli.bom.diff_bom import capycli.bom.download_sources +import capycli.bom.download_attachments import capycli.bom.filter_bom import capycli.bom.findsources import capycli.bom.map_bom @@ -100,6 +101,12 @@ def run_bom_command(args) -> None: app.run(args) return + if subcommand == "downloadattachments": + """Download attachments from SW360 as specified in the SBOM.""" + app = capycli.bom.download_attachments.BomDownloadAttachments() + app.run(args) + return + if subcommand == "granularity": """Check the granularity of the releases in the SBOM.""" app = capycli.bom.check_granularity.CheckGranularity() diff --git a/tests/fixtures/sbom_for_download.json b/tests/fixtures/sbom_for_download.json index 029b299..3124631 100644 --- a/tests/fixtures/sbom_for_download.json +++ b/tests/fixtures/sbom_for_download.json @@ -88,12 +88,38 @@ { "url": "https://github.com/certifi/python-certifi", "type": "website" + }, + { + "url": "CLIXML_certifi-2022.12.7.xml", + "comment": "component license information (local copy), sw360Id: 794446", + "type": "other", + "hashes": [ + { + "alg": "SHA-1", + "content": "542e87fa0acb8d9c4659145a3e1bfcd66c979f33" + } + ] + }, + { + "url": "certifi-2022.12.7_clearing_report.docx", + "comment": "clearing report (local copy), sw360Id: 63b368", + "type": "other", + "hashes": [ + { + "alg": "SHA-1", + "content": "3cd24769fa3da4af74d0118433619a130da091b0" + } + ] } ], "properties": [ { "name": "siemens:primaryLanguage", "value": "Python" + }, + { + "name": "siemens:sw360Id", + "value": "ae8c7ed" } ] } @@ -108,4 +134,4 @@ "dependsOn": [] } ] -} \ No newline at end of file +} diff --git a/tests/test_bom_downloadattachments.py b/tests/test_bom_downloadattachments.py new file mode 100644 index 0000000..e4add7c --- /dev/null +++ b/tests/test_bom_downloadattachments.py @@ -0,0 +1,317 @@ +# ------------------------------------------------------------------------------- +# Copyright (c) 2023 Siemens +# All Rights Reserved. +# Author: thomas.graf@siemens.com +# +# SPDX-License-Identifier: MIT +# ------------------------------------------------------------------------------- + +import os +import tempfile + +import responses + +from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport +from capycli.bom.download_attachments import BomDownloadAttachments +from capycli.main.result_codes import ResultCode +from cyclonedx.model import ExternalReferenceType, HashAlgorithm +from tests.test_base import AppArguments, TestBase + + +class TestBomDownloadAttachments(TestBase): + INPUTFILE = "sbom_for_download.json" + INPUTERROR = "plaintext.txt" + OUTPUTFILE = "output.json" + + @responses.activate + def setUp(self) -> None: + self.app = BomDownloadAttachments() + self.add_login_response() + self.app.login("sometoken", "https://my.server.com") + + # return super().setUp() + + def test_show_help(self) -> None: + sut = BomDownloadAttachments() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("downloadattachments") + args.help = True + + out = self.capture_stdout(sut.run, args) + self.assertTrue("usage: capycli bom downloadattachments" in out) + + def test_no_inputfile_specified(self) -> None: + try: + sut = BomDownloadAttachments() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("downloadattachments") + + sut.run(args) + self.assertTrue(False, "Failed to report missing argument") + except SystemExit as ex: + self.assertEqual(ResultCode.RESULT_COMMAND_ERROR, ex.code) + + def test_file_not_found(self) -> None: + try: + sut = BomDownloadAttachments() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("downloadattachments") + args.inputfile = "DOESNOTEXIST" + + sut.run(args) + self.assertTrue(False, "Failed to report missing file") + except SystemExit as ex: + self.assertEqual(ResultCode.RESULT_FILE_NOT_FOUND, ex.code) + + def test_error_loading_file(self) -> None: + try: + sut = BomDownloadAttachments() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("downloadattachments") + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTERROR) + + sut.run(args) + self.assertTrue(False, "Failed to report invalid file") + except SystemExit as ex: + self.assertEqual(ResultCode.RESULT_ERROR_READING_BOM, ex.code) + + @responses.activate + def test_source_folder_does_not_exist(self) -> None: + try: + sut = BomDownloadAttachments() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("downloadattachments") + + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE) + args.source = "XXX" + + sut.run(args) + self.assertTrue(False, "Failed to report missing folder") + except SystemExit as ex: + self.assertEqual(ResultCode.RESULT_COMMAND_ERROR, ex.code) + + @responses.activate + def test_simple_bom(self) -> None: + # create argparse command line argument object + bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE) + bom = CaPyCliBom.read_sbom(bom) + + # attachment info - CLI + responses.add( + method=responses.GET, + url=self.MYURL + "resource/api/attachments/794446", + body=""" + { + "filename": "CLIXML_certifi-2022.12.7.xml", + "sha1": "3cd24769fa3da4af74d0118433619a130da091b0", + "attachmentType": "COMPONENT_LICENSE_INFO_XML", + "createdBy": "thomas.graf@siemens.com", + "createdTeam": "AA", + "createdComment": "comment1", + "createdOn": "2020-10-08", + "checkStatus": "NOTCHECKED", + "_links": { + "self": { + "href": "https://my.server.com/resource/api/attachments/794446" + } + } + }""", + status=200, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + # get attachment - CLI + cli_file = self.get_cli_file_mit() + responses.add( + method=responses.GET, + url=self.MYURL + "resource/api/releases/ae8c7ed/attachments/794446", + body=cli_file, + status=200, + content_type="application/text", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + # attachment info - report + responses.add( + method=responses.GET, + url=self.MYURL + "resource/api/attachments/63b368", + body=""" + { + "filename": "certifi-2022.12.7_clearing_report.docx", + "sha1": "3cd24769fa3da4af74d0118433619a130da091b0", + "attachmentType": "CLEARING_REPORT", + "createdBy": "gernot.hillier@siemens.com", + "createdTeam": "BB", + "createdComment": "comment3", + "createdOn": "2020-10-08", + "checkedBy": "thomas.graf@siemens.com", + "checkedOn" : "2021-01-18", + "checkedComment": "comment4", + "checkStatus": "ACCEPTED", + "_links": { + "self": { + "href": "https://my.server.com/resource/api/attachments/63b368" + } + } + }""", + status=200, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + # get attachment - report + responses.add( + method=responses.GET, + url=self.MYURL + "resource/api/releases/ae8c7ed/attachments/63b368", + body="some_report_content", + status=200, + content_type="application/text", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + with tempfile.TemporaryDirectory() as tmpdirname: + try: + bom = self.app.download_attachments(bom, tmpdirname) + resultfile = os.path.join(tmpdirname, "CLIXML_certifi-2022.12.7.xml") + self.assertEqual(bom.components[0].external_references[5].url, resultfile) + self.assertTrue(os.path.isfile(resultfile), "CLI file missing") + + resultfile = os.path.join(tmpdirname, "certifi-2022.12.7_clearing_report.docx") + self.assertEqual(bom.components[0].external_references[6].url, resultfile) + self.assertTrue(os.path.isfile(resultfile), "report file missing") + return + except Exception as e: # noqa + # catch all exception to let Python cleanup the temp folder + print(e) + + self.assertTrue(False, "Error: we must never arrive here") + + @responses.activate + def test_simple_bom_download_errors(self) -> None: + # create argparse command line argument object + bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE) + bom = CaPyCliBom.read_sbom(bom) + + # attachment info - CLI, ok + responses.add( + method=responses.GET, + url=self.MYURL + "resource/api/attachments/794446", + body=""" + { + "filename": "CLIXML_certifi-2022.12.7.xml", + "sha1": "3cd24769fa3da4af74d0118433619a130da091b0", + "attachmentType": "COMPONENT_LICENSE_INFO_XML", + "_links": { + "self": { + "href": "https://my.server.com/resource/api/attachments/794446" + } + } + }""", + status=200, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + # get attachment - CLI, error + responses.add( + method=responses.GET, + url=self.MYURL + "resource/api/releases/ae8c7ed/attachments/794446", + body="cli_file", + status=500, + content_type="application/text", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + # attachment info - report, error + responses.add( + method=responses.GET, + url=self.MYURL + "resource/api/attachments/63b368", + body=""" + { + "filename": "certifi-2022.12.7_clearing_report.docx", + "sha1": "3cd24769fa3da4af74d0118433619a130da091b0", + "attachmentType": "CLEARING_REPORT", + "_links": { + "self": { + "href": "https://my.server.com/resource/api/attachments/63b368" + } + } + }""", + status=404, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + with tempfile.TemporaryDirectory() as tmpdirname: + try: + bom = self.app.download_attachments(bom, tmpdirname) + resultfile = os.path.join(tmpdirname, "CLIXML_certifi-2022.12.7.xml") + self.assertFalse(os.path.isfile(resultfile), "CLI created despite HTTP 500") + + resultfile = os.path.join(tmpdirname, "certifi-2022.12.7_clearing_report.docx") + self.assertFalse(os.path.isfile(resultfile), "report created despite HTTP 404") + return + except Exception as e: # noqa + # catch all exception to let Python cleanup the temp folder + print(e) + + self.assertTrue(False, "Error: we must never arrive here") + + @responses.activate + def test_simple_bom_no_release_id(self) -> None: + bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE) + bom = CaPyCliBom.read_sbom(bom) + bom.components[0].properties = [] + with tempfile.TemporaryDirectory() as tmpdirname: + try: + err = self.capture_stdout(self.app.download_attachments, bom, tmpdirname) + assert "No sw360Id for release" in err + + return + except Exception as e: # noqa + # catch all exception to let Python cleanup the temp folder + print(e) + + self.assertTrue(False, "Error: we must never arrive here") + + @responses.activate + def test_simple_bom_no_attachment_id(self) -> None: + bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE) + bom = CaPyCliBom.read_sbom(bom) + bom.components[0].external_references = [] + CycloneDxSupport.set_ext_ref(bom.components[0], ExternalReferenceType.OTHER, + CaPyCliBom.CLI_FILE_COMMENT, "CLIXML_foo.xml", + HashAlgorithm.SHA_1, "123") + + with tempfile.TemporaryDirectory() as tmpdirname: + try: + err = self.capture_stdout(self.app.download_attachments, bom, tmpdirname) + assert "No sw360Id for attachment" in err + + return + except Exception as e: # noqa + # catch all exception to let Python cleanup the temp folder + print(e) + + self.assertTrue(False, "Error: we must never arrive here") diff --git a/tests/test_merge_bom.py b/tests/test_merge_bom.py index b0fcc2b..65b7277 100644 --- a/tests/test_merge_bom.py +++ b/tests/test_merge_bom.py @@ -197,8 +197,8 @@ def test_merge_bom1(self) -> None: self.assertEqual("certifi", bom.components[0].name) self.assertEqual("2022.12.7", bom.components[0].version) self.assertEqual("pkg:pypi/certifi@2022.12.7", bom.components[0].purl) - self.assertEqual(6, len(bom.components[0].external_references)) - self.assertEqual(1, len(bom.components[0].properties)) + self.assertEqual(8, len(bom.components[0].external_references)) + self.assertEqual(2, len(bom.components[0].properties)) self.assertEqual(2, len(bom.components)) self.assertEqual("certifi", bom.components[1].name)