feat!(ISV-6786): consume new data structure for generating image SBOMs

Author: bclindner

docs/sboms/oci_image.md

@@ -9,14 +9,9 @@ generated by Hermeto (previously known as Cachi2), with the requirement that
 at least one SBOM is provided in total. It combines these SBOMs
 and takes them as a context of the built image.
 
-The script also parses a JSON-ified Dockerfile of the image, parses its
-content and determines which base images were used to build the image.
-It identifies builder images as well as a _parent image_ which is the
-latest image in the Dockerfile (or the base image for the stage identified
-by the build target).
-
-Additionally, you can also supply additional builder images on top of those
-already parsed from the Dockerfile.
+The script uses buildprobe (see [capo](https://github.com/konflux-ci/capo) for
+details) to determine container content and which base images were used to
+build the image.
 
 All provided SBOMs must be in the same specification! This script does not
 support combining SPDX and CycloneDX SBOMs.
@@ -28,25 +23,15 @@ mobster --verbose  generate oci-image \
 --from-syft tests/sbom/test_merge_data/cyclonedx/syft-sboms/pip-e2e-test.bom.json \
 --from-syft tests/sbom/test_merge_data/cyclonedx/syft-sboms/ubi-micro.bom.json \
 --from-hermeto tests/sbom/test_merge_data/cyclonedx/cachi2.bom.json \
---image-pullspec quay.io/foobar/examplecontainer:v10 \
---image-digest sha256:1 \
---parsed-dockerfile-path tests/data/dockerfiles/somewhat_believable_sample/parsed.json \
---dockerfile-target build \
---additional-base-image quay.io/ubi9:latest@sha256:123456789012345678901234567789012
+--metadata-path tests/data/dockerfiles/somewhat_believable_sample/metadata.yaml
 ```
 
 ## List of arguments
 - `--from-syft` -- points to an SBOM file (in a JSON format) created by Syft, can be used multiple times
 - `--from-hermeto` -- points to an SBOM file (in a JSON format) created by Hermeto
 - `--image-pullspec` -- the pullspec of the image processed in the format `<registry>/<repository>:<tag>`
 - `--image-digest` -- the digest of the image processed in the format `sha256:<digest value>`
-- `--parsed-dockerfile-path` -- points to a dockerfile processed by `dockerfile-json`
-- `--base-image-digest-file` -- points to a file with digests for images used in Dockerfile.
-  if omitted, the references will be fetched via `oras`. The expected format of the file is
-  `<registry>/<repository>:<tag> <registry>/<repository>:<tag>@sha256:<digest>`
-- `--dockerfile-target` -- if a build target was used for multi-stage build, use this argument to specify the build target
-- `--additional-base-images` -- optionally add references to other build images outside the parsed Dockerfile.
-  expects the format `<registry>/<repository>:<tag>@sha256:<digest value>`
+- `--metadata-path` -- points to Dockerfile/Containerfile metadata processed by `buildprobe`
 - `--contextualize` -- Allows SBOM contextualization (see [Contextual SBOM](#contextual-sbom))
 - `--output` -- where to save the SBOM. prints it to STDOUT if this is not specified
 - `--skip-validation` -- skips validation of the SBOM
@@ -57,7 +42,7 @@ To build an SBOM with only the OCI image, you will need to run several tools to
 get prerequisite files to use in the `mobster generate` command. These two tools are:
 
 * Syft (https://github.com/anchore/syft): for initial scanning and SBOM generation
-* dockerfile-json (https://github.com/keilerkonzept/dockerfile-json): for
+* buildprobe (provided by capo at https://github.com/konflux-ci/capo): for
   generating a human-readable Dockerfile/Containerfile manifest
 
 Download and install them before this process.
@@ -71,11 +56,11 @@ Assuming you have the Containerfile, and the OCI image stored in a repository
 syft scan quay.io/konflux-ci/mobster:latest --output spdx-json > syft.json
 ```
 
-2. Use dockerfile-json to generate a machine-readable Dockerfile/Containerfile
+2. Use buildprobe to generate a machine-readable Dockerfile/Containerfile
    description for the tool to use:
 
 ```sh
-dockerfile-json ./Containerfile > containerfile.json
+buildprobe buildah --tag="quay.io/your-image:latest" --target="" --containerfile="Containerfile" > metadata.yaml
 ```
 
 3. Run `mobster generate` with the prerequisite files and the same OCI image
@@ -87,8 +72,7 @@ mobster generate \
 	--output full-sbom.json \
 	oci-image \
 	--from-syft syft.json \
-	--parsed-dockerfile-path containerfile.json \
-	--image-pullspec quay.io/konflux-ci/mobster:latest
+	--metadata-path metadata.yaml
 ```
 
 Once the command is complete, you should see the full Mobster SBOM in

poetry.lock

@@ -36,6 +36,7 @@ dependencies = [
     "aiofiles (>=24.1.0,<25.0.0)",
     "httpx (>=0.28.1,<0.29.0)",
     "aioboto3 (>=15.2.0,<15.3.0)",
+    "pyyaml (>=6.0.3,<7.0.0)",
 ]
 
 [project.urls]

pyproject.toml

@@ -17,7 +17,7 @@
     product,
 )
 from mobster.cmd.upload import upload
-from mobster.image import ARTIFACT_PATTERN, PULLSPEC_PATTERN
+from mobster.image import PULLSPEC_PATTERN
 from mobster.release import ReleaseId
 
 
@@ -97,13 +97,6 @@ def validated_pullspec(value: str) -> str:
         )
         return value
 
-    def validated_additional_reference(value: str) -> str:
-        assert re.match(ARTIFACT_PATTERN, value), (
-            "Additional references must be in the format "
-            "<registry>/<repository>:<tag>@sha256:<digest>"
-        )
-        return value
-
     oci_image_parser = subparsers.add_parser(
         "oci-image", help="Generate an SBOM document for OCI image"
     )
@@ -130,31 +123,9 @@ def validated_additional_reference(value: str) -> str:
         help="Image digest for the OCI image in the format sha256:<digest>",
     )
     oci_image_parser.add_argument(
-        "--parsed-dockerfile-path",
+        "--metadata-path",
         type=Path,
-        help="Path to the parsed Dockerfile file",
-    )
-    oci_image_parser.add_argument(
-        "--base-image-digest-file",
-        type=Path,
-        help="Path to the file containing references "
-        "to images in the Dockerfile and their digests. "
-        "Expected format: "
-        "`<registry>/<repository>:<tag> <registry>/<repository>:<tag>@sha256:<digest>`",
-    )
-    oci_image_parser.add_argument(
-        "--dockerfile-target",
-        type=str,
-        help="The name of the build target from the Dockerfile",
-        default=None,
-    )
-    oci_image_parser.add_argument(
-        "--additional-base-image",
-        type=validated_additional_reference,
-        action="append",
-        default=[],
-        help="Base (builder) image to add, can be specified multiple times. "
-        "Expects the format <registry>/<repository>:<tag>@sha256:<digest value>",
+        help="Path to a metadata file generated by buildprobe.",
     )
     oci_image_parser.add_argument(
         "--arch",

src/mobster/cli.py

@@ -20,6 +20,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
 
         self._content: Any = None
+        self._metadata: Any = None
 
     @property
     def content(self) -> Any:

src/mobster/cmd/generate/base.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 from typing import Any
 
+import yaml
 from cyclonedx.exception import CycloneDxException
 from spdx_tools.spdx.jsonschema.document_converter import DocumentConverter
 from spdx_tools.spdx.model.document import Document
@@ -20,11 +21,10 @@
 from mobster.cmd.generate.base import GenerateCommandWithOutputTypeSelector
 from mobster.cmd.generate.oci_image.add_image import extend_sbom_with_image_reference
 from mobster.cmd.generate.oci_image.base_images_dockerfile import (
-    extend_sbom_with_base_images_from_dockerfile,
-    get_base_images_refs_from_dockerfile,
+    extend_sbom_with_base_images,
     get_digest_for_image_ref,
-    get_image_objects_from_file,
 )
+from mobster.cmd.generate.oci_image.buildprobe import SBOMMetadata
 from mobster.cmd.generate.oci_image.contextual_sbom.builder import (
     BuilderContextualizationError,
     BuilderPkgMetadata,
@@ -48,7 +48,7 @@
 from mobster.image import Image
 from mobster.log import log_elapsed
 from mobster.sbom.merge import merge_sboms
-from mobster.utils import identify_arch, load_sbom_from_json
+from mobster.utils import load_sbom_from_json
 
 logging.captureWarnings(True)  # CDX validation uses `warn()`
 LOGGER = logging.getLogger(__name__)
@@ -99,6 +99,15 @@ async def _load_and_filter_hermeto_sbom(self) -> dict[str, Any]:
         arch = self.cli_args.arch or mobster.utils.identify_arch()
         return filter_hermeto_sbom_by_arch(hermeto_sbom, arch)
 
+    def _load_metadata(self) -> None:
+        """
+        Load a metadata file from the --metadata-path argument into
+        self._metadata.
+        """
+        with open(self.cli_args.metadata_path, encoding="utf-8") as metadata_file:
+            raw_metadata = yaml.safe_load(metadata_file)
+            self._metadata = SBOMMetadata.from_dict(raw_metadata)
+
     async def _handle_bom_inputs(
         self,
     ) -> dict[str, Any]:
@@ -113,13 +122,19 @@ async def _handle_bom_inputs(
             self.cli_args.from_hermeto is None
             and self.cli_args.from_syft is None
             and self.cli_args.image_pullspec is None
+            and self.cli_args.metadata_path is None
         ):
             raise ArgumentError(
                 None,
-                "At least one of --from-syft, --from-hermeto or --image-pullspec"
-                " must be provided",
+                "At least one of --from-syft, --from-hermeto, --image-pullspec, "
+                "or --metadata-path must be provided",
             )
 
+        if self.cli_args.metadata_path is not None:
+            self._load_metadata()
+            # if we don't have an sbom provided to us, use syft to generate it
+            if self.cli_args.from_syft is None and self.cli_args.from_hermeto is None:
+                return await syft.scan_image(self._metadata.image.pullspec)
         if self.cli_args.from_syft is not None:
             # Merging Syft & Hermeto SBOMs
             if len(self.cli_args.from_syft) > 1 or self.cli_args.from_hermeto:
@@ -228,7 +243,7 @@ async def _assess_and_dispatch_contextual_workflow(
         (non-modified) SBOM is furtherly processed by mobster.
         Args:
             component_sbom_doc: The component SBOM created for this image.
-            base_images_refs: List of references from the parsed Dockerfile.
+            base_images_refs: List of references from the build.
             image_arch: CPU architecture of this image.
 
         Returns:
@@ -264,11 +279,12 @@ async def execute(self) -> Any:
         """
         LOGGER.debug("Generating SBOM document for OCI image")
 
+        # Get/merge the raw SBOM
         merged_sbom_dict = await self._handle_bom_inputs()
         sbom: Document | CycloneDX1BomWrapper
-        image_arch = identify_arch()
+        image_arch = self.cli_args.arch or mobster.utils.identify_arch()
 
-        # Parsing into objects
+        # Parse into objects
         if merged_sbom_dict.get("bomFormat") == "CycloneDX":
             if self.cli_args.contextualize:
                 raise ArgumentError(
@@ -280,9 +296,32 @@ async def execute(self) -> Any:
         else:
             raise ValueError("Unknown SBOM Format!")
 
-        # Extending with image reference
-        if self.cli_args.image_pullspec:
-            image_arch = self.cli_args.arch or mobster.utils.identify_arch()
+        base_images_refs = []
+        base_images_map: dict[str, Image] = {}
+
+        # Extend with image reference
+        if self.cli_args.metadata_path:
+            image = Image.from_image_index_url_and_digest(
+                self._metadata.image.pullspec,
+                self._metadata.image.digest,
+                arch=image_arch,
+            )
+            await extend_sbom_with_image_reference(sbom, image, False)
+            for base_image_data in self._metadata.base_images:
+                base_image = Image.from_image_index_url_and_digest(
+                    base_image_data.pullspec,
+                    base_image_data.digest,
+                )
+                base_images_refs.append(base_image_data.pullspec)
+                base_images_map[base_image_data.pullspec] = base_image
+            await extend_sbom_with_base_images(sbom, base_images_refs, base_images_map)
+            for extra_image_data in self._metadata.extra_images:
+                extra_image = Image.from_image_index_url_and_digest(
+                    extra_image_data.pullspec,
+                    extra_image_data.digest,
+                )
+                await extend_sbom_with_image_reference(sbom, extra_image, True)
+        elif self.cli_args.image_pullspec:
             if not self.cli_args.image_digest:
                 LOGGER.info(
                     "Provided pullspec but not digest."
@@ -308,37 +347,6 @@ async def execute(self) -> Any:
                 "Provided image digest but no pullspec. The digest value is ignored."
             )
 
-        base_images_refs = []
-        base_images_map: dict[str, Image] = {}
-
-        # Extending with base images references from a dockerfile
-        if self.cli_args.parsed_dockerfile_path:
-            with open(
-                self.cli_args.parsed_dockerfile_path, encoding="utf-8"
-            ) as parsed_dockerfile_io:
-                parsed_dockerfile = json.load(parsed_dockerfile_io)
-
-            base_images_refs = await get_base_images_refs_from_dockerfile(
-                parsed_dockerfile, self.cli_args.dockerfile_target
-            )
-
-            if self.cli_args.base_image_digest_file:
-                LOGGER.debug(
-                    "Supplied pre-parsed image digest file, will operate offline."
-                )
-                base_images_map = await get_image_objects_from_file(
-                    self.cli_args.base_image_digest_file
-                )
-            await extend_sbom_with_base_images_from_dockerfile(
-                sbom, base_images_refs, base_images_map
-            )
-
-        # Extending with additional base images
-        for image_ref in self.cli_args.additional_base_image:
-            image_object = Image.from_oci_artifact_reference(image_ref)
-            await extend_sbom_with_image_reference(
-                sbom, image_object, is_builder_image=True
-            )
         with log_elapsed("Contextual workflow", logging.INFO):
             contextual_sbom = await self._assess_and_dispatch_contextual_workflow(
                 sbom, base_images_refs, base_images_map, image_arch