Added ApplyPartitioning tests, Aurora packaging tests success, fixed various other tests, removed unnecessary test file

Author: bwintermann

src/finn/builder/build_dataflow_config.py

@@ -232,6 +232,7 @@ class PartitioningConfiguration:
 
     # The number of ports per device - this might change in meaning,
     # depending on the communication kernel used
+    # TODO: Should be moved into platforms.py
     ports_per_device: int = 2
 
     # What strategy to use to partition the dataflow graph

src/finn/interface/manage_tests.py

@@ -61,7 +61,10 @@ def run_test(variant: str, num_workers: str, args: str = "", no_cache_clear: boo
                     "given (please additionally pass --args "
                     "<test-name> in pytest syntax)"
                 )
-            subprocess.run(shlex.split(f"{sys.executable} -m pytest {args}", posix=IS_POSIX))
+            workers_arg = f"-n {num_workers}" if num_workers not in ["1", ""] else ""
+            subprocess.run(
+                shlex.split(f"{sys.executable} -m  pytest {workers_arg} {args}", posix=IS_POSIX)
+            )
         case "doctest":
             if args == "":
                 status(

src/finn/transformation/fpgadataflow/multifpga/aurora/prepare_aurora.py

@@ -36,6 +36,7 @@ def __init__(
         self.make_args = " ".join(
             f"{k}={v}" for k, v in partitioning_configuration.communication_kernel_arguments.items()
         )
+        self.ports = partitioning_configuration.ports_per_device
         self.aurora_storage = Path(make_build_dir("aurora_storage_")).absolute()
         self.aurora_path = get_settings().finn_deps / "AuroraFlow"
         if not self.aurora_path.exists():
@@ -65,6 +66,12 @@ def package_single(self, args: str, device: int, index: int) -> Path:
         >>> xo.parent.parent.name
         'auroraflow_build_dev0_ind0'
         """
+        if index >= self.ports:
+            raise FINNMultiFPGAConfigError(
+                f"Cannot create aurora kernel number {index+1} "
+                f"(index {index}), since the configuration/device "
+                f"has only {self.ports} networking ports."
+            )
         if self.verbosity == MFVerbosity.HIGH:
             log.info(f"Packaging AuroraFlow core (Device: {device}, Index: {index})")
 

src/finn/util/resources.py

@@ -1,15 +1,17 @@
+from numbers import Real
 from qonnx.core.modelwrapper import ModelWrapper
+from typing import cast
 
 from finn.analysis.fpgadataflow.hls_synth_res_estimation import hls_synth_res_estimation
 from finn.analysis.fpgadataflow.res_estimation import res_estimation
 from finn.util.exception import FINNUserError
 from finn.util.logging import log
 from finn.util.platforms import Platform
 
-ResourceEstimates = dict[str, dict[str, int | float]]
+ResourceEstimates = dict[str, dict[str, Real]]
 """Short alias for a resource dict."""
 
-ResourceEstimatesByIndex = dict[int, dict[str, int | float]]
+ResourceEstimatesByIndex = dict[int, dict[str, Real]]
 """Short alias for a resource dict."""
 
 
@@ -56,25 +58,44 @@ def _merge_resource_estimations(
     return result
 
 
-def get_estimated_model_resources(
-    model: ModelWrapper, fpga_part: str, considered_resources: list[str]
+def get_estimated_model_resources(  # noqa
+    model: ModelWrapper,
+    fpga_part: str,
+    considered_resources: list[str],
+    add_missing_resources: bool,
 ) -> ResourceEstimatesByIndex:
     """Gather resources of all layers based on estimates both from FINNs HWCustomOp implementation,
     as well as the HLS reports. These are then merged to produce
     the worst case resource estimates and returned.
 
-    If there are layers that don't have an estimate
-    for all considered resources, an error is raised.
+    Arguments:
+    ---------
+        `model`: The model to estimate.
+        `fpga_part`: FPGA Part identifier.
+        `considered_resources`: A list of resource types to consider.
+        `add_missing_resources`: Determines behaviour in case a resource type
+            from `considered_resources` is not found: If `True`, then missing
+            resource types are set to 0. If, for example, a layer has no `FF` estimate
+            in either FINN or HLS estimation, FF: 0 is entered for these layers. If set
+            to `False`, an error is raised instead.
     """
     estimates: ResourceEstimates = res_estimation(model, fpga_part)
     hls_estimates: ResourceEstimates = hls_synth_res_estimation(model)
     result: ResourceEstimates = _merge_resource_estimations(estimates, hls_estimates)
     resource_missing = False
+
+    # Check if resource types are missing (and add them)
     for layer in result:
         for restype in considered_resources:
             if restype not in result[layer]:
-                resource_missing = True
-                log.error(f"Node {layer} has no resource estimation for resource {restype}!")
+                if add_missing_resources:
+                    result[layer][restype] = cast("Real", 0)
+                    log.info(f"Added missing resource estimation on layer {layer} ({restype}: 0)")
+                else:
+                    resource_missing = True
+                    log.error(f"Node {layer} has no resource estimation for resource {restype}!")
+
+    # Check that estimates for all layers are available
     layer_missing = False
     est_by_index = {}
     for i, node in enumerate(model.graph.node):
@@ -84,7 +105,9 @@ def get_estimated_model_resources(
             log.error(f"No resource estimations were found for node {node.name}!")
         est_by_index[i] = result[node.name]
     if layer_missing or resource_missing:
-        raise FINNUserError("At least one node is missing one or more resource estimation numbers.")
+        raise FINNUserError(
+            "At least one node is missing one or more resource estimation numbers.\n" + str(result)
+        )
     return est_by_index
 
 

tests/fpgadataflow/multifpga/test_multifpga_aurora.py

@@ -11,44 +11,38 @@
 from fpgadataflow.multifpga.utils import get_model
 from pathlib import Path
 from test_multifpga_sdp_creation import create_sdp_ready_model_no_branches
-from typing import TYPE_CHECKING, Any, Literal
 
 from finn.builder.build_dataflow_config import (
     DataflowBuildConfig,
     MFCommunicationKernel,
     MFTopology,
     MFVerbosity,
-    MIPSolver,
     PartitioningConfiguration,
     PartitioningStrategy,
     ShellFlowType,
 )
-from finn.transformation.fpgadataflow.multifpga.assign_metadata import AssignNetworkMetadata
-from finn.transformation.fpgadataflow.multifpga.aurora_metadata import AuroraNetworkMetadata
+from finn.transformation.fpgadataflow.multifpga.aurora.metadata import AuroraNetworkMetadata
+from finn.transformation.fpgadataflow.multifpga.aurora.partitioner import AuroraPartitioner
 from finn.transformation.fpgadataflow.multifpga.communication_kernels import PrepareAuroraFlow
 from finn.transformation.fpgadataflow.multifpga.create_multi_sdp import (
     CreateMultiFPGAStreamingDataflowPartition,
 )
-from finn.transformation.fpgadataflow.multifpga.partitioner import (
-    AuroraPartitioner,
-    PartitionForMultiFPGA,
-)
-from finn.transformation.fpgadataflow.multifpga.utils import available_resources, get_device_id
+from finn.transformation.fpgadataflow.multifpga.create_network_metadata import CreateNetworkMetadata
+from finn.transformation.fpgadataflow.multifpga.partition_model import PartitionForMultiFPGA
 from finn.util.basic import make_build_dir
 from finn.util.exception import (
     FINNError,
     FINNMultiFPGAConfigError,
     FINNMultiFPGANoPartitionerSolutionError,
 )
+from finn.util.fpgadataflow import get_device_id
 from finn.util.platforms import platforms
-
-if TYPE_CHECKING:
-    from qonnx.core.modelwrapper import ModelWrapper
+from finn.util.resources import available_resources
 
 
 @pytest.mark.auroraflow
 @pytest.mark.multifpga
-@pytest.mark.parametrize("board", ["U280", "U55C", "Pynq-Z1"])
+@pytest.mark.parametrize("board", ["U280", "U55C"])
 @pytest.mark.parametrize("topology", [MFTopology.CHAIN])
 @pytest.mark.parametrize(
     "communication_kernel_args",
@@ -78,13 +72,8 @@ def test_aurora_packaging_integrated(
         """
         devices, nodes = device_node_combinations
 
-        # Check which creator we use
-        assignment_order = {MFTopology.CHAIN: "linear"}[topology]
-
         # Create an SDP ready branchless model
-        model = create_sdp_ready_model_no_branches(
-            nodes, devices, assignment_type, assignment_order, shuffle_devices
-        )
+        model = create_sdp_ready_model_no_branches(nodes, devices, assignment_type, shuffle_devices)
 
         # Create a config based on the test parameters
         cfg = DataflowBuildConfig(
@@ -108,12 +97,19 @@ def test_aurora_packaging_integrated(
             )
         )
         model = model.transform(
-            AssignNetworkMetadata(
-                communication_kernel=cfg.partitioning_configuration.communication_kernel,
-                topology=topology,
-                verbosity=MFVerbosity.NONE,
+            CreateNetworkMetadata(
+                cfg.partitioning_configuration.communication_kernel, MFVerbosity.NONE
             )
         )
+
+        # No kernels packaged yet
+        meta = AuroraNetworkMetadata.from_model(model)
+        unprepared_aurora_kernels = len(meta.get_unprepared_aurora_kernels())
+        if devices > 1:
+            assert unprepared_aurora_kernels == (devices - 1) * 2
+        else:
+            assert unprepared_aurora_kernels == 0
+
         model = model.transform(
             PrepareAuroraFlow(
                 cfg._resolve_vitis_platform(),  # noqa
@@ -124,6 +120,8 @@ def test_aurora_packaging_integrated(
 
         # Try and load the previously generated metadata from the models metadata prop
         meta = AuroraNetworkMetadata.from_model(model)
+        unprepared_aurora_kernels = len(meta.get_unprepared_aurora_kernels())
+        assert unprepared_aurora_kernels == 0
 
         # Check that the AuroraFlow storage directory got saved in the model metadata
         aurora_storage = model.get_metadata_prop("aurora_storage")
@@ -134,6 +132,10 @@ def test_aurora_packaging_integrated(
         assert aurora_storage.exists()
 
         # Check if each device had its respective kernels packaged
+        if devices == 1:
+            assert len(meta.data.keys()) == 0
+        else:
+            assert len(meta.data.keys()) == devices
         for kerneldata in meta.data.values():
             for aurora in kerneldata:
                 assert aurora.aurora_xo is not None
@@ -142,7 +144,7 @@ def test_aurora_packaging_integrated(
     @pytest.mark.multifpga
     @pytest.mark.slow
     def test_aurora_package_single(
-        self, communication_kernel_args: dict[str, str], board: str
+        self, communication_kernel_args: dict[str, str], board: str, topology: MFTopology
     ) -> None:
         """Test Aurora packaging. In detail:
         - Check that the names of the XO files produced by AuroraFlow didn't change.
@@ -154,6 +156,7 @@ def test_aurora_package_single(
             board=board,
             partitioning_configuration=PartitioningConfiguration(
                 num_fpgas=2,
+                topology=topology,
                 communication_kernel=MFCommunicationKernel.AURORA,
                 communication_kernel_arguments=communication_kernel_args,
             ),
@@ -169,10 +172,14 @@ def test_aurora_package_single(
         build_dir = prep.aurora_storage / "auroraflow_build_dev0_ind0"
         assert build_dir.exists()
         assert res.exists()
-        res = prep.package_single("", 1, 2)
-        build_dir = prep.aurora_storage / "auroraflow_build_dev1_ind2"
-        assert build_dir.exists()
-        assert res.exists()
+
+        # On devices like the U280, there are only 2 QSFPs, so an index 2 (third kernel) cannot
+        # be produced, hence the kernel should not be there.
+        with pytest.raises(FINNError):
+            res = prep.package_single("", 1, 2)
+            build_dir = prep.aurora_storage / "auroraflow_build_dev1_ind2"
+            assert build_dir.exists()
+            assert res.exists()
 
 
 @pytest.mark.auroraflow
@@ -441,33 +448,15 @@ def test_objective_regression(
         TODO: Save recent CI run data in proper infrastructure as soon as we have set it
         up, instead of hardcoding the values.
         """
-        flow_type = self.get_shell_flow_type(board)
-        model, _ = self.make_model(
-            model_type, f"test_regression_model_{'_'.join(map(str, (model_type)))}"
-        )
-        _, _, part = self.prepare_and_partition_model(
-            model,
-            board,
-            flow_type,
-            2,
-            topology,
-            strategy,
-            True,
-            model_type[0],
-            max_util=0.85,
-            ideal_util=0.75,
-            ports_per_device=2,
-            separate_iodmas=True,
-            target_fps=10,
-            mvau_wwidth_max=1024,
-            synth_clk_ns=5.0,
-            solver=None,
-        )
-        assert part.partitioner is not None
-        assert part.partitioner.model.objective.x is not None
-        assert part.partitioner.model.objective.x < 0
+        # flow_type = self.get_shell_flow_type(board)
+
+        # TODO
         raise NotImplementedError()
 
+        # assert part.partitioner is not None
+        # assert part.partitioner.model.objective.x is not None
+        # assert part.partitioner.model.objective.x < 0
+
     @pytest.mark.parametrize(
         "distribution",
         ["equal-LUT", "equal-FF"],

tests/fpgadataflow/multifpga/test_multifpga_end2end.py

@@ -1,57 +0,0 @@
-import pytest
-
-from test_multifpga_sdp_creation import create_sdp_ready_model_no_branches
-
-from finn.builder.build_dataflow_config import (
-    DataflowBuildConfig,
-    MFCommunicationKernel,
-    MFTopology,
-    PartitioningConfiguration,
-)
-from finn.util.basic import make_build_dir
-
-
-@pytest.mark.multifpga
-def test_multifpga_end2end_mobilenet() -> None:
-    raise NotImplementedError()
-
-
-@pytest.mark.multifpga
-@pytest.mark.parametrize(
-    "device_node_combinations", [(2, 2), (5, 10), (40, 100), (50, 50), (1, 10)]
-)
-@pytest.mark.parametrize("assignment_type", ["random", "equal"])
-@pytest.mark.parametrize("topology", [MFTopology.CHAIN])
-@pytest.mark.parametrize("communication_kernel", [MFCommunicationKernel.AURORA])
-@pytest.mark.parametrize("shuffle_devices", [True, False])
-def test_multifpga_end2end_artifical_network(
-    device_node_combinations: tuple[int, int],
-    assignment_type: str,
-    topology: MFTopology,
-    communication_kernel: MFCommunicationKernel,
-    shuffle_devices: bool,
-) -> None:
-    devices, nodes = device_node_combinations
-    assignment_order = None
-    if topology == MFTopology.CHAIN:
-        assignment_order = "linear"
-    else:
-        raise NotImplementedError()
-
-    # TODO: Make this model entirely from scratch
-    # TODO: Collapse all params and only pass a partitioning config
-    temps = make_build_dir("test_end2end_artificial_outputs")
-    _ = DataflowBuildConfig(
-        output_dir=temps,
-        synth_clk_period_ns=5.0,
-        partitioning_configuration=PartitioningConfiguration(
-            num_fpgas=devices, topology=topology, communication_kernel=communication_kernel
-        ),
-    )
-    model = create_sdp_ready_model_no_branches(
-        nodes, devices, assignment_type, assignment_order, shuffle_devices
-    )
-    model.set_metadata_prop("is_multifpga", "True")
-    # model = step_create_multifpga_sdp(model, cfg)
-    # model = step_prepare_network_infrastructure(model, cfg)
-    raise NotImplementedError("Assertions.")