Add support for bitbake recipes #1243

requirements-native.txt

@@ -4,3 +4,4 @@ lxml==5.1.0
 MarkupSafe==2.1.5
 pyahocorasick==2.1.0
 PyYAML==6.0.1
+regex==2022.8.17

requirements.txt

@@ -17,6 +17,7 @@ cryptography==42.0.5
 debian-inspector==31.1.0
 dockerfile-parse==1.2.0
 dparse2==0.7.0
+deprecated == 1.2.18
 extractcode==31.0.0
 extractcode-7z==16.5.210531
 extractcode-libarchive==3.5.1.210531
@@ -40,6 +41,7 @@ lxml==5.1.0
 MarkupSafe==2.1.5
 more-itertools==8.13.0
 normality==2.3.3
+oelint_parser==8.1.0
 packageurl-python==0.15.0
 packaging==24.1
 packvers==21.5
@@ -61,6 +63,7 @@ pyparsing==3.0.9
 pytz==2022.1
 PyYAML==6.0.1
 rdflib==6.2.0
+regex == 2024.11.6
 requests==2.31.0
 saneyaml==0.6.0
 semantic-version==2.8.5

setup-mini.cfg

@@ -87,6 +87,7 @@ install_requires =
     license_expression >= 30.4.1
     lxml >= 4.9.2
     MarkupSafe >= 2.1.2
+    oelint_parser >= 8.0.0
     packageurl_python >= 0.9.0
     packvers >= 21.0.0
     # use temp advanced patched release

setup.cfg

@@ -87,6 +87,7 @@ install_requires =
     license_expression >= 30.4.1
     lxml >= 4.9.2
     MarkupSafe >= 2.1.2
+    oelint_parser >= 8.0.0
     packageurl_python >= 0.9.0
     packvers >= 21.0.0
     # use temp advanced patched release

src/packagedcode/__init__.py

@@ -10,6 +10,7 @@
 from commoncode.system import on_linux
 from packagedcode import about
 from packagedcode import alpine
+from packagedcode import bitbake
 from packagedcode import bower
 from packagedcode import build
 from packagedcode import build_gradle
@@ -55,6 +56,8 @@
 
     bower.BowerJsonHandler,
 
+    bitbake.BitbakeBbManifestHandler,
+
     build_gradle.BuildGradleHandler,
 
     build.AutotoolsConfigureHandler,

src/packagedcode/bitbake.py

@@ -0,0 +1,194 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+import re
+
+from oelint_parser.cls_stash import Stash
+from packageurl import PackageURL
+
+from packagedcode import models
+
+TRACE = True
+
+logger = logging.getLogger(__name__)
+
+if TRACE:
+    import sys
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+
+class BitbakeBbManifestHandler(models.DatafileHandler):
+    datasource_id = 'bitbake_bb_recipe'
+    # note that there are .bbappend, .bbclass and bitbake.conf files.
+    path_patterns = ('*.bb',)
+    default_package_type = 'bitbake'
+    description = 'BitBake bb recipe manifest'
+    documentation_url = 'https://docs.yoctoproject.org/bitbake/bitbake-user-manual/bitbake-user-manual-metadata.html'
+
+    @classmethod
+    def parse(cls, location):
+
+        oestash = Stash(quiet=True)
+
+        # add any bitbake like-file
+        # TODO: may be we should handle the bbclass and bbappend here?
+        oestash.AddFile(location)
+
+        # Resolves proper cross file dependencies
+        oestash.Finalize()
+
+        # collect all variables of interest.
+        # TODO: we should not get list values. Instead plain strings
+        data = {
+            k: ' '.join(v) if isinstance(v, (list, tuple)) else v
+            for k, v in oestash.ExpandVar(filename=location).items()
+            if v
+        }
+        name = data.get('PN')
+        version = data.get('PV')
+        description = data.get('DESCRIPTION')
+        homepage_url = data.get('HOMEPAGE')
+        download_url = data.get('PREMIRRORS')
+        extracted_license_statement = data.get('LICENSE')
+
+        # The item.VarName for SRC_URI[*] from the parser are SRC_URI
+        # Therefore, I cannot differentiate md5, sha1, or src file location reference
+        # See: https://github.com/priv-kweihmann/oelint-parser/issues/3
+        sha1 = data.get('SRC_URI[sha1sum]')
+        md5 = data.get('SRC_URI[md5sum]')
+        sha256 = data.get('SRC_URI[sha256sum]')
+        sha512 = data.get('SRC_URI[sha512sum]')
+
+        dependencies = []
+
+        # Build deps: this is a list of plain BB recipes names
+        # https://docs.yoctoproject.org/bitbake/bitbake-user-manual/bitbake-user-manual-ref-variables.html#term-DEPENDS
+        build_deps = data.get('DEPENDS', '').split()
+        for build_dep in build_deps:
+            dep_purl = PackageURL(
+                type=cls.default_package_type,
+                name=build_dep,
+            ).to_string()
+
+            dependency = models.DependentPackage(
+                purl=dep_purl,
+                extracted_requirement=build_dep,
+                scope='build',
+                is_runtime=False,
+                is_optional=True,
+                is_pinned=False,
+                is_direct=True,
+            )
+            dependencies.append(dependency)
+
+        # Runtime deps:this is a list of Package names with an optional (=> 12) version constraint
+        # https://docs.yoctoproject.org/bitbake/bitbake-user-manual/bitbake-user-manual-ref-variables.html#term-RDEPENDS
+        # FIXME: There are some fields such as "RDEPENDS_${PN}" so these may not be correct in all cases
+        for key, value in data.items():
+            if not key.startswith('RDEPENDS'):
+                continue
+            if not value:
+                continue
+            for name, constraint in get_bitbake_deps(dependencies=value):
+                if TRACE:
+                    logger.debug(f'RDEPENDS: name={name}, constraint={constraint}')
+                dep_purl = PackageURL(
+                    type=cls.default_package_type,
+                    name=name,
+                ).to_string()
+
+                extracted_requirement = name
+                if constraint:
+                    extracted_requirement += f' ({constraint})'
+
+                dependency = models.DependentPackage(
+                    purl=dep_purl,
+                    extracted_requirement=extracted_requirement,
+                    scope='install',
+                    is_runtime=True,
+                    is_optional=False,
+                    is_pinned=False,
+                    is_direct=True,
+                )
+
+                dependencies.append(dependency)
+
+        yield models.PackageData(
+            datasource_id=cls.datasource_id,
+            type=cls.default_package_type,
+            name=name,
+            version=version,
+            description=description,
+            homepage_url=homepage_url,
+            download_url=download_url,
+            sha1=sha1,
+            md5=md5,
+            sha256=sha256,
+            sha512=sha512,
+            extracted_license_statement=extracted_license_statement,
+            dependencies=dependencies,
+        )
+
+    @classmethod
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
+        return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase, package_adder)
+
+
+def get_bitbake_deps(dependencies):
+    """
+    Return a list of tuple of (name, version constraint) given a BitBake
+    dependencies string. "version constraint" can be None.
+
+    See https://docs.yoctoproject.org/ref-manual/variables.html?#term-RDEPENDS
+    For example:
+    >>> expected = [('ABC', None), ('abcd', '=>12312'), ('defg', None)]
+    >>> result = get_bitbake_deps(" ABC abcd (= >  12312) defg ")
+    >>> assert result == expected, result
+    >>> expected = [('grub', '==12.23'), ('parted', None), ('e2fsprogs-mke2fs', None)]
+    >>> result = get_bitbake_deps("grub (== 12.23) parted e2fsprogs-mke2fs")
+    >>> assert result == expected, result
+    """
+    return [split_name_constraint(nc) for nc in split_deps(dependencies)]
+
+
+def split_name_constraint(dependency):
+    """
+    Return a tuple (name, version constraint) strings given a name (version
+    constraint) BitBake dependency string.
+    See https://docs.yoctoproject.org/ref-manual/variables.html?#term-RDEPENDS
+    For example:
+    >>> assert split_name_constraint(" abcd  ( = 12312 ) ") == ("abcd", "=12312",)
+    >>> assert split_name_constraint("abcd ") == ("abcd", None)
+    """
+    no_spaces = dependency.replace(' ', '')
+    if '(' in no_spaces:
+        name, _, constraint = no_spaces.partition('(')
+        constraint = constraint.rstrip(')')
+        return name, constraint
+    return no_spaces, None
+
+
+def split_deps(dependencies):
+    """
+    Return a list of name (version constraint) strings given a BitBake
+    dependencies string.
+
+    See https://docs.yoctoproject.org/ref-manual/variables.html?#term-RDEPENDS
+    For example:
+    >>> expected = ['ABC', 'abcd (= > 12312)', 'defg', 'foo', 'bar']
+    >>> result = split_deps(" ABC abcd (= >  12312) defg   foo bar ")
+    >>> assert result == expected, result
+    """
+    normalized_spaces = ' '.join(dependencies.split())
+    name = r'\w[\w\d_-]+'
+    version_constraint = r'\([<>= ]+[^<>= ]+\s?\)'
+    splitter = re.compile(fr'({name}\s?(?:{version_constraint})?)').findall
+    return [s.strip() for s in splitter(normalized_spaces)]

tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb

@@ -0,0 +1,17 @@
+SUMMARY = "Live image install script with a second rootfs/kernel"
+LICENSE = "MIT"
+LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420"
+SRC_URI = "file://init-install-testfs.sh"
+DEPENDS = "do_this and that"
+
+RDEPENDS_${PN} = "grub (== 12.23) parted e2fsprogs-mke2fs"
+
+S = "${WORKDIR}"
+
+do_install() {
+        install -m 0755 ${WORKDIR}/init-install-testfs.sh ${D}/install.sh
+}
+
+INHIBIT_DEFAULT_DEPS = "1"
+FILES_${PN} = " /install.sh "
+COMPATIBLE_HOST = "(i.86|x86_64).*-linux"