diff --git a/requirements-native.txt b/requirements-native.txt index f1f2b09b80..f0a7a06181 100644 --- a/requirements-native.txt +++ b/requirements-native.txt @@ -4,3 +4,4 @@ lxml==5.1.0 MarkupSafe==2.1.5 pyahocorasick==2.1.0 PyYAML==6.0.1 +regex==2022.8.17 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 43a0b90bc1..86c941e4b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ cryptography==42.0.5 debian-inspector==31.1.0 dockerfile-parse==1.2.0 dparse2==0.7.0 +deprecated == 1.2.18 extractcode==31.0.0 extractcode-7z==16.5.210531 extractcode-libarchive==3.5.1.210531 @@ -40,6 +41,7 @@ lxml==5.1.0 MarkupSafe==2.1.5 more-itertools==8.13.0 normality==2.3.3 +oelint_parser==8.1.0 packageurl-python==0.15.0 packaging==24.1 packvers==21.5 @@ -61,6 +63,7 @@ pyparsing==3.0.9 pytz==2022.1 PyYAML==6.0.1 rdflib==6.2.0 +regex == 2024.11.6 requests==2.31.0 saneyaml==0.6.0 semantic-version==2.8.5 diff --git a/setup-mini.cfg b/setup-mini.cfg index 9f89d63275..37fe854b56 100644 --- a/setup-mini.cfg +++ b/setup-mini.cfg @@ -87,6 +87,7 @@ install_requires = license_expression >= 30.4.1 lxml >= 4.9.2 MarkupSafe >= 2.1.2 + oelint_parser >= 8.0.0 packageurl_python >= 0.9.0 packvers >= 21.0.0 # use temp advanced patched release diff --git a/setup.cfg b/setup.cfg index 5395aacbee..fcf9c6c2e7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -87,6 +87,7 @@ install_requires = license_expression >= 30.4.1 lxml >= 4.9.2 MarkupSafe >= 2.1.2 + oelint_parser >= 8.0.0 packageurl_python >= 0.9.0 packvers >= 21.0.0 # use temp advanced patched release diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index 9cc46d0e09..fa1b288a62 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -10,6 +10,7 @@ from commoncode.system import on_linux from packagedcode import about from packagedcode import alpine +from packagedcode import bitbake from packagedcode import bower from packagedcode import build from packagedcode import build_gradle @@ -55,6 +56,8 @@ bower.BowerJsonHandler, + bitbake.BitbakeBbManifestHandler, + build_gradle.BuildGradleHandler, build.AutotoolsConfigureHandler, diff --git a/src/packagedcode/bitbake.py b/src/packagedcode/bitbake.py new file mode 100644 index 0000000000..cbf883b9f5 --- /dev/null +++ b/src/packagedcode/bitbake.py @@ -0,0 +1,194 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +import re + +from oelint_parser.cls_stash import Stash +from packageurl import PackageURL + +from packagedcode import models + +TRACE = True + +logger = logging.getLogger(__name__) + +if TRACE: + import sys + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + +class BitbakeBbManifestHandler(models.DatafileHandler): + datasource_id = 'bitbake_bb_recipe' + # note that there are .bbappend, .bbclass and bitbake.conf files. + path_patterns = ('*.bb',) + default_package_type = 'bitbake' + description = 'BitBake bb recipe manifest' + documentation_url = 'https://docs.yoctoproject.org/bitbake/bitbake-user-manual/bitbake-user-manual-metadata.html' + + @classmethod + def parse(cls, location): + + oestash = Stash(quiet=True) + + # add any bitbake like-file + # TODO: may be we should handle the bbclass and bbappend here? + oestash.AddFile(location) + + # Resolves proper cross file dependencies + oestash.Finalize() + + # collect all variables of interest. + # TODO: we should not get list values. Instead plain strings + data = { + k: ' '.join(v) if isinstance(v, (list, tuple)) else v + for k, v in oestash.ExpandVar(filename=location).items() + if v + } + name = data.get('PN') + version = data.get('PV') + description = data.get('DESCRIPTION') + homepage_url = data.get('HOMEPAGE') + download_url = data.get('PREMIRRORS') + extracted_license_statement = data.get('LICENSE') + + # The item.VarName for SRC_URI[*] from the parser are SRC_URI + # Therefore, I cannot differentiate md5, sha1, or src file location reference + # See: https://github.com/priv-kweihmann/oelint-parser/issues/3 + sha1 = data.get('SRC_URI[sha1sum]') + md5 = data.get('SRC_URI[md5sum]') + sha256 = data.get('SRC_URI[sha256sum]') + sha512 = data.get('SRC_URI[sha512sum]') + + dependencies = [] + + # Build deps: this is a list of plain BB recipes names + # https://docs.yoctoproject.org/bitbake/bitbake-user-manual/bitbake-user-manual-ref-variables.html#term-DEPENDS + build_deps = data.get('DEPENDS', '').split() + for build_dep in build_deps: + dep_purl = PackageURL( + type=cls.default_package_type, + name=build_dep, + ).to_string() + + dependency = models.DependentPackage( + purl=dep_purl, + extracted_requirement=build_dep, + scope='build', + is_runtime=False, + is_optional=True, + is_pinned=False, + is_direct=True, + ) + dependencies.append(dependency) + + # Runtime deps:this is a list of Package names with an optional (=> 12) version constraint + # https://docs.yoctoproject.org/bitbake/bitbake-user-manual/bitbake-user-manual-ref-variables.html#term-RDEPENDS + # FIXME: There are some fields such as "RDEPENDS_${PN}" so these may not be correct in all cases + for key, value in data.items(): + if not key.startswith('RDEPENDS'): + continue + if not value: + continue + for name, constraint in get_bitbake_deps(dependencies=value): + if TRACE: + logger.debug(f'RDEPENDS: name={name}, constraint={constraint}') + dep_purl = PackageURL( + type=cls.default_package_type, + name=name, + ).to_string() + + extracted_requirement = name + if constraint: + extracted_requirement += f' ({constraint})' + + dependency = models.DependentPackage( + purl=dep_purl, + extracted_requirement=extracted_requirement, + scope='install', + is_runtime=True, + is_optional=False, + is_pinned=False, + is_direct=True, + ) + + dependencies.append(dependency) + + yield models.PackageData( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + name=name, + version=version, + description=description, + homepage_url=homepage_url, + download_url=download_url, + sha1=sha1, + md5=md5, + sha256=sha256, + sha512=sha512, + extracted_license_statement=extracted_license_statement, + dependencies=dependencies, + ) + + @classmethod + def assign_package_to_resources(cls, package, resource, codebase, package_adder): + return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase, package_adder) + + +def get_bitbake_deps(dependencies): + """ + Return a list of tuple of (name, version constraint) given a BitBake + dependencies string. "version constraint" can be None. + + See https://docs.yoctoproject.org/ref-manual/variables.html?#term-RDEPENDS + For example: + >>> expected = [('ABC', None), ('abcd', '=>12312'), ('defg', None)] + >>> result = get_bitbake_deps(" ABC abcd (= > 12312) defg ") + >>> assert result == expected, result + >>> expected = [('grub', '==12.23'), ('parted', None), ('e2fsprogs-mke2fs', None)] + >>> result = get_bitbake_deps("grub (== 12.23) parted e2fsprogs-mke2fs") + >>> assert result == expected, result + """ + return [split_name_constraint(nc) for nc in split_deps(dependencies)] + + +def split_name_constraint(dependency): + """ + Return a tuple (name, version constraint) strings given a name (version + constraint) BitBake dependency string. + See https://docs.yoctoproject.org/ref-manual/variables.html?#term-RDEPENDS + For example: + >>> assert split_name_constraint(" abcd ( = 12312 ) ") == ("abcd", "=12312",) + >>> assert split_name_constraint("abcd ") == ("abcd", None) + """ + no_spaces = dependency.replace(' ', '') + if '(' in no_spaces: + name, _, constraint = no_spaces.partition('(') + constraint = constraint.rstrip(')') + return name, constraint + return no_spaces, None + + +def split_deps(dependencies): + """ + Return a list of name (version constraint) strings given a BitBake + dependencies string. + + See https://docs.yoctoproject.org/ref-manual/variables.html?#term-RDEPENDS + For example: + >>> expected = ['ABC', 'abcd (= > 12312)', 'defg', 'foo', 'bar'] + >>> result = split_deps(" ABC abcd (= > 12312) defg foo bar ") + >>> assert result == expected, result + """ + normalized_spaces = ' '.join(dependencies.split()) + name = r'\w[\w\d_-]+' + version_constraint = r'\([<>= ]+[^<>= ]+\s?\)' + splitter = re.compile(fr'({name}\s?(?:{version_constraint})?)').findall + return [s.strip() for s in splitter(normalized_spaces)] diff --git a/tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb b/tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb new file mode 100644 index 0000000000..0c6eb2e3b8 --- /dev/null +++ b/tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb @@ -0,0 +1,17 @@ +SUMMARY = "Live image install script with a second rootfs/kernel" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" +SRC_URI = "file://init-install-testfs.sh" +DEPENDS = "do_this and that" + +RDEPENDS_${PN} = "grub (== 12.23) parted e2fsprogs-mke2fs" + +S = "${WORKDIR}" + +do_install() { + install -m 0755 ${WORKDIR}/init-install-testfs.sh ${D}/install.sh +} + +INHIBIT_DEFAULT_DEPS = "1" +FILES_${PN} = " /install.sh " +COMPATIBLE_HOST = "(i.86|x86_64).*-linux" \ No newline at end of file diff --git a/tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb-expected b/tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb-expected new file mode 100644 index 0000000000..1cb5e73483 --- /dev/null +++ b/tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb-expected @@ -0,0 +1,136 @@ +[ + { + "type": "bitbake", + "namespace": null, + "name": "e2fsprogs-mke2fs", + "version": "1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "mit", + "declared_license_expression_spdx": "MIT", + "license_detections": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "rule_url": null, + "matched_text": "MIT" + } + ], + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "MIT", + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:bitbake/do_this", + "extracted_requirement": "do_this", + "scope": "build", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:bitbake/and", + "extracted_requirement": "and", + "scope": "build", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:bitbake/that", + "extracted_requirement": "that", + "scope": "build", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:bitbake/grub", + "extracted_requirement": "grub (==12.23)", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:bitbake/parted", + "extracted_requirement": "parted", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:bitbake/e2fsprogs-mke2fs", + "extracted_requirement": "e2fsprogs-mke2fs", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "bitbake_bb_recipe", + "purl": "pkg:bitbake/e2fsprogs-mke2fs@1.0" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/bitbake/netbase_6.1.bb b/tests/packagedcode/data/bitbake/netbase_6.1.bb new file mode 100644 index 0000000000..71a97e75fc --- /dev/null +++ b/tests/packagedcode/data/bitbake/netbase_6.1.bb @@ -0,0 +1,20 @@ +SUMMARY = "Basic TCP/IP networking support" +DESCRIPTION = "This package provides the necessary infrastructure for basic TCP/IP based networking" +HOMEPAGE = "http://packages.debian.org/netbase" +SECTION = "base" +LICENSE = "GPLv2" +LIC_FILES_CHKSUM = "file://debian/copyright;md5=3dd6192d306f582dee7687da3d8748ab" +PE = "1" + +SRC_URI = "${DEBIAN_MIRROR}/main/n/${BPN}/${BPN}_${PV}.tar.xz" + +SRC_URI[md5sum] = "e5871a3a5c8390557b8033cf19316a55" +SRC_URI[sha256sum] = "084d743bd84d4d9380bac4c71c51e57406dce44f5a69289bb823c903e9b035d8" + +UPSTREAM_CHECK_URI = "${DEBIAN_MIRROR}/main/n/netbase/" +do_install () { + install -d ${D}/${mandir}/man8 ${D}${sysconfdir} + install -m 0644 ${S}/etc/rpc ${D}${sysconfdir}/rpc + install -m 0644 ${S}/etc/protocols ${D}${sysconfdir}/protocols + install -m 0644 ${S}/etc/services ${D}${sysconfdir}/services +} \ No newline at end of file diff --git a/tests/packagedcode/data/bitbake/netbase_6.1.bb-expected b/tests/packagedcode/data/bitbake/netbase_6.1.bb-expected new file mode 100644 index 0000000000..c00ffa0925 --- /dev/null +++ b/tests/packagedcode/data/bitbake/netbase_6.1.bb-expected @@ -0,0 +1,69 @@ +[ + { + "type": "bitbake", + "namespace": null, + "name": "netbase", + "version": "6.1", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": "This package provides the necessary infrastructure for basic TCP/IP based networking", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "http://packages.debian.org/netbase", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "gpl-2.0", + "declared_license_expression_spdx": "GPL-2.0-only", + "license_detections": [ + { + "license_expression": "gpl-2.0", + "license_expression_spdx": "GPL-2.0-only", + "matches": [ + { + "license_expression": "gpl-2.0", + "license_expression_spdx": "GPL-2.0-only", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "gpl-2.0_bare_single_word.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/gpl-2.0_bare_single_word.RULE", + "matched_text": "GPLv2" + } + ], + "identifier": "gpl_2_0-3aa0fcde-6d7f-a8e2-6494-7f9352e6aafb" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "GPLv2", + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "bitbake_bb_recipe", + "purl": "pkg:bitbake/netbase@6.1" + } +] \ No newline at end of file diff --git a/tests/packagedcode/test_bitbake.py b/tests/packagedcode/test_bitbake.py new file mode 100644 index 0000000000..809af7a5fc --- /dev/null +++ b/tests/packagedcode/test_bitbake.py @@ -0,0 +1,38 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os.path + +from packagedcode import bitbake +from scancode_config import REGEN_TEST_FIXTURES +from packages_test_utils import PackageTester + + +class TestBitbake(PackageTester): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_parse_bitbake(self): + test_file = self.get_test_loc('bitbake/netbase_6.1.bb') + packages = bitbake.BitbakeBbManifestHandler.parse(location=test_file) + expected_loc = self.get_test_loc('bitbake/netbase_6.1.bb-expected') + self.check_packages_data( + packages_data=packages, + expected_loc=expected_loc, + regen=REGEN_TEST_FIXTURES, + ) + + def test_parse_bitbake_dependencies(self): + test_file = self.get_test_loc('bitbake/initramfs-live-install-testfs_1.0.bb') + packages = bitbake.BitbakeBbManifestHandler.parse(location=test_file) + expected_loc = self.get_test_loc('bitbake/initramfs-live-install-testfs_1.0.bb-expected') + self.check_packages_data( + packages_data=packages, + expected_loc=expected_loc, + regen=REGEN_TEST_FIXTURES, + )