fix: Update support for 1000 actions per pipeline and 100 actions per wave

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py

@@ -107,7 +107,7 @@ def report_final_pipeline_targets(pipeline_object):
     if number_of_targets == 0:
         LOGGER.info("Attempting to create an empty pipeline as there were no targets found")
 
-
+# pylint: disable=R0914
 def generate_pipeline_inputs(
     pipeline,
     deployment_map_source,
@@ -129,8 +129,8 @@ def generate_pipeline_inputs(
     """
     data = {}
     pipeline_object = Pipeline(pipeline)
-    regions = []
-
+    total_pipeline_actions = 2
+    # Assumption is that a Source and Build Stage is = 2 Actions
     for target in pipeline.get("targets", []):
         target_structure = TargetStructure(target)
         for raw_step in target_structure.target:
@@ -158,17 +158,22 @@ def generate_pipeline_inputs(
         # For the sake of consistency we should probably think of a target
         # consisting of multiple "waves". So if you see any reference to
         # a wave going forward it will be the individual batch of account ids.
-        pipeline_object.template_dictionary["targets"].append(
-            list(
-                target_structure.generate_waves(
-                    target=pipeline_target
-                )
-            ),
+
+        waves, wave_action_count =  target_structure.generate_waves(
+            target=pipeline_target
         )
+        pipeline_object.template_dictionary["targets"].append(list(waves))
+        # Add the actions from the waves to the total_pipeline_actions count
+        total_pipeline_actions += wave_action_count
+
+    target_structure.validate_actions_limit(
+        pipeline.get("name"),
+        total_pipeline_actions
+    )
 
     report_final_pipeline_targets(pipeline_object)
 
-    if DEPLOYMENT_ACCOUNT_REGION not in regions:
+    if DEPLOYMENT_ACCOUNT_REGION not in pipeline_object.stage_regions:
         pipeline_object.stage_regions.append(DEPLOYMENT_ACCOUNT_REGION)
 
     data["pipeline_input"] = pipeline_object.generate_input()
@@ -180,14 +185,12 @@ def generate_pipeline_inputs(
         data["pipeline_input"]["default_providers"]["source"]["properties"][
             "codeconnections_arn"
         ] = data["ssm_params"]["codeconnections_arn"]
-    data["pipeline_input"]["default_scm_branch"] = (
-        data["ssm_params"]
-        .get("default_scm_branch")
-    )
-    data["pipeline_input"]["default_scm_codecommit_account_id"] = (
-        data["ssm_params"]
-        .get("default_scm_codecommit_account_id")
-    )
+    data["pipeline_input"].update({
+        "default_scm_branch": data["ssm_params"].get("default_scm_branch"),
+        "default_scm_codecommit_account_id": data["ssm_params"].get(
+            "default_scm_codecommit_account_id"
+        )
+    })
     store_regional_parameter_config(
         pipeline_object,
         parameter_store,

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/generate_params.py

@@ -82,7 +82,7 @@ class InputPipelineWaveTarget(TypedDict):
 # When defining the pipeline, the accounts that it deploys to are mapped
 # in waves. Within each wave, it will contain a list of wave targets to
 # make sure that referencing 100 accounts for example will be broken down
-# into two waves of 50 accounts each as max supported by CodePipeline.
+# into two waves of 100 accounts each as max supported by CodePipeline.
 TargetWavesWithNestedWaveTargets = List[  # Waves
     List[  # Wave Targets
         InputPipelineWaveTarget

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/errors.py

@@ -71,3 +71,9 @@ class InsufficientWaveSizeError(Exception):
     """
     Raised when the defined wave size is less than the calculated minimum actions
     """
+
+class TooManyActionsError(Exception):
+    """
+    Raised when the Targets Deployment map configuration exceeds the theoretical
+    maximum allowed actions in a pipeline.
+    """

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/target.py

@@ -9,15 +9,17 @@
 
 import re
 import os
-
+from typing import Tuple
 import boto3
 
 # ADF imports
 from errors import (
     InvalidDeploymentMapError,
     NoAccountsFoundError,
     InsufficientWaveSizeError,
+    TooManyActionsError
 )
+
 from botocore.exceptions import ClientError
 from logger import configure_logger
 from parameter_store import ParameterStore
@@ -30,7 +32,7 @@
 AWS_ACCOUNT_ID_REGEX = re.compile(AWS_ACCOUNT_ID_REGEX_STR)
 CLOUDFORMATION_PROVIDER_NAME = "cloudformation"
 RECURSIVE_SUFFIX = "/**/*"
-
+PIPELINE_MAXIMUM_ACTIONS = 1000
 
 class TargetStructure:
     def __init__(self, target):
@@ -88,13 +90,24 @@ def _get_actions_per_target_account(
             actions_per_region += (1 + int(change_set_approval))
         return actions_per_region * regions_defined
 
-    def generate_waves(self, target):
+    def validate_actions_limit(self, pipeline_name, num_actions) -> None:
+        """Raise an exception if total amount of actions generated for the
+        pipeline would exceed the allowed threshold."""
+        if num_actions > PIPELINE_MAXIMUM_ACTIONS:
+            raise TooManyActionsError(
+                f"Pipeline {pipeline_name} has too many actions: "
+                f"{num_actions} maximum supported for any given "
+                f"Pipeline is {PIPELINE_MAXIMUM_ACTIONS} consider splitting the "
+                "Deployment Map into multiple Pipelines."
+            )
+
+    def generate_waves(self, target) -> Tuple[list, int]:
         """ Given the maximum actions allowed in a wave via wave.size property,
         reduce the accounts allocated in each wave by a factor
-        matching the number of actions necessary per account, which inturn
+        matching the number of actions necessary per account, which inturn is
         derived from the number of target regions and the specific action_type
         defined for that target. """
-        wave_size = self.wave.get('size', 50)
+        wave_size = self.wave.get('size', 100)
         actions_per_target_account = self._get_actions_per_target_account(
             regions=target.regions,
             provider=target.provider,
@@ -115,7 +128,7 @@ def generate_waves(self, target):
         wave_size = wave_size // actions_per_target_account
         waves = []
         length = len(self.account_list)
-
+        total_actions = length * actions_per_target_account
         for start_index in range(0, length, wave_size):
             end_index = min(
                 start_index + wave_size,
@@ -124,7 +137,7 @@ def generate_waves(self, target):
             waves.append(
                 self.account_list[start_index:end_index],
             )
-        return waves
+        return waves, total_actions
 
 class Target:
     """

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_target.py

@@ -10,6 +10,7 @@
 from mock import Mock, patch,call
 from .stubs import stub_target
 from ..target import Target, TargetStructure
+from ..target import TooManyActionsError
 from parameter_store import ParameterStore
 
 
@@ -217,7 +218,7 @@ def test_target_structure_respects_wave():
                 }
             )
             target.fetch_accounts_for_target()
-            waves = list(target.target_structure.generate_waves(target=target))
+            waves, total_actions = target.target_structure.generate_waves(target=target)
             assert len(waves) == 3
 
             assert len(waves[0]) == 2
@@ -277,6 +278,47 @@ def test_target_structure_respects_wave():
                 },
             ]
 
+def test_target_structure_too_many_actions():
+    """Validates that when too many targets are defined then total_actions would
+    an exception. """
+    parameter_store = Mock()
+    parameter_store.client.put_parameter.return_value = True
+    with patch.object(ParameterStore, 'fetch_parameter') as mock:
+        expected_calls = [
+            call(
+                'deployment_maps/allow_empty_target',
+                'disabled',
+            ),
+        ]
+        test_target_config = {"path": "/some/random/ou",}
+        target_structure = TargetStructure(
+            target=test_target_config,
+        )
+        for step in target_structure.target:
+            target = Target(
+                path=test_target_config.get("path")[0],
+                target_structure=target_structure,
+
+                organizations=MockOrgClient([
+                    {"Name": f"test-account-{x}", "Id": x, "Status": "ACTIVE"}
+                    for x in range(200)
+                ]),
+                step={
+                    **step,
+                    "provider": "cloudformation",
+                    "regions": ["region1", "region2", "region3", "region4"]
+                }
+            )
+            target.fetch_accounts_for_target()
+            waves, total_actions = target.target_structure.generate_waves(
+                target=target
+            )
+            with raises(TooManyActionsError):
+                target.target_structure.validate_actions_limit(
+                    "dummy_name",
+                    total_actions
+                )
+
 
 def test_target_structure_respects_multi_region():
     """ Validate behavior with multiple accounts (x5) using cloudformation
@@ -316,10 +358,10 @@ def test_target_structure_respects_multi_region():
             )
             target.fetch_accounts_for_target()
 
-            waves = list(target.target_structure.generate_waves(target=target))
+            waves, total_actions = target.target_structure.generate_waves(target=target)
 
             assert len(waves) == 3
-
+            assert total_actions == 40
             assert len(waves[0]) == 2 # x2 accounts x4 region x2 action = 16
             assert len(waves[1]) == 2 # x2 accounts x4 region x2 action = 16
             assert len(waves[2]) == 1 # x1 accounts x4 region x2 action = 8
@@ -338,7 +380,7 @@ def test_target_structure_respects_multi_action_single_region():
                 'disabled',
             ),
         ]
-        test_target_config = {"path": "/some/random/ou"}
+        test_target_config = {"path": "/some/random/ou", "wave": {"size": 20}}
         target_structure = TargetStructure(
             target=test_target_config,
         )
@@ -358,21 +400,21 @@ def test_target_structure_respects_multi_action_single_region():
                 }
             )
             target.fetch_accounts_for_target()
-            waves = list(
-                target.target_structure.generate_waves(
-                    target=target,
-                ),
+            waves, total_actions = target.target_structure.generate_waves(
+                target=target
             )
-            assert len(waves) == 2
 
-            assert len(waves[0]) == 25 # assert accts(25) region(1) action(2) = 50
-            assert len(waves[1]) == 5 # assert accnts(5) region(1) action(2) = 10
+            assert len(waves) == 3
+            assert total_actions == 60
+            assert len(waves[0]) == 10 # assert accts(20) region(1) action(2) = 20
+            assert len(waves[1]) == 10 # assert accnts(10) region(1) action(2) = 20
+            assert len(waves[1]) == 10 # assert accnts(10) region(1) action(2) = 20
 
 
 def test_target_structure_respects_multi_action_multi_region():
     """ Validate behavior with multiple accounts (x34) using cloudformation
     default actions (x2 actions) across two region (x2)
-    Limited to default 50 actions per region should split by 3 waves"""
+    Limited to default 100 actions per region should split by 2 waves"""
     parameter_store = Mock()
     parameter_store.client.put_parameter.return_value = True
     with patch.object(ParameterStore, 'fetch_parameter') as mock:
@@ -405,18 +447,18 @@ def test_target_structure_respects_multi_action_multi_region():
             )
             target.fetch_accounts_for_target()
 
-            waves = list(target.target_structure.generate_waves(target=target))
-            assert len(waves) == 3
+            waves, total_actions = target.target_structure.generate_waves(target=target)
+            assert len(waves) == 2
+            assert total_actions == 136 # x34 Accounts with x2 actions x2 regions
+            assert len(waves[0]) == 25 # assert accts(25) regions(2) actions(2) = 100
+            assert len(waves[1]) == 9 # assert accts(10) regions(2) actions(2) = 36
 
-            assert len(waves[0]) == 12 # assert accts(12) regions(2) actions(2) = 48
-            assert len(waves[1]) == 12 # assert accts(12) regions(2) actions(2) = 48
-            assert len(waves[2]) == 10 # assert accts(10) regions(2) actions(2) = 40
 
 
 def test_target_structure_respects_change_set_approval_single_region():
     """ Validate behavior with multiple accounts (x60) using cloudformation
     change_set_approval (x3 actions) across single region (x1)
-    Limited to default 50 actions per region"""
+    Limited to default 100 actions per region"""
     parameter_store = Mock()
     parameter_store.client.put_parameter.return_value = True
     with patch.object(ParameterStore, 'fetch_parameter') as mock:
@@ -452,13 +494,11 @@ def test_target_structure_respects_change_set_approval_single_region():
             )
             target.fetch_accounts_for_target()
 
-            waves = list(target.target_structure.generate_waves(target=target))
-            assert len(waves) == 4
-
-            assert len(waves[0]) == 16 # assert accts(16) regions(1) actions(3) = 48
-            assert len(waves[1]) == 16 # assert accts(16) regions(1) actions(3) = 48
-            assert len(waves[2]) == 16 # assert accts(16) regions(1) actions(3) = 48
-            assert len(waves[3]) == 12 # remaining 60 - (3 * 16) = 12
+            waves, total_actions = target.target_structure.generate_waves(target=target)
+            assert len(waves) == 2
+            assert total_actions == 180 # x60 Accounts with x3 actions
+            assert len(waves[0]) == 33 # assert accts(33) regions(1) actions(3) = 99
+            assert len(waves[1]) == 27 # assert accts(27) regions(1) actions(3) = 81
 
 
 def test_target_wave_structure_respects_exclude_config():
@@ -502,7 +542,7 @@ def test_target_wave_structure_respects_exclude_config():
                 }
             )
             target.fetch_accounts_for_target()
-            waves = list(target.target_structure.generate_waves(target=target))
+            waves, _ = target.target_structure.generate_waves(target=target)
             assert len(waves) == 3
 
             assert len(waves[0]) == 2