Retry logic for DescribeRegions while creating new account, handling non-empty default vpc v2 (#348)

* retry logic for DescribeRegions while creating new account, handling non-empty default vpc

* fixed copy/paste typo

* Fixes according to code review comments on PR #238

Co-authored-by: COLANGELO Monica <[email protected]>

Author: sbkok

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/provisioner/main.py

@@ -9,6 +9,7 @@
 import os
 from concurrent.futures import ThreadPoolExecutor
 import boto3
+import tenacity
 from organizations import Organizations
 from logger import configure_logger
 from parameter_store import ParameterStore
@@ -17,25 +18,32 @@
 
 
 LOGGER = configure_logger(__name__)
-ACCOUNTS_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'adf-accounts'))
+ACCOUNTS_FOLDER = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '..', '..', 'adf-accounts'))
+
 
 def main():
     accounts = read_config_files(ACCOUNTS_FOLDER)
     if not bool(accounts):
-        LOGGER.info(f"Found {len(accounts)} account(s) in configuration file(s). Account provisioning will not continue.")
+        LOGGER.info(
+            f"Found {len(accounts)} account(s) in configuration file(s). Account provisioning will not continue.")
         return
     LOGGER.info(f"Found {len(accounts)} account(s) in configuration file(s).")
     organizations = Organizations(boto3)
     support = Support(boto3)
     all_accounts = organizations.get_accounts()
-    parameter_store = ParameterStore(os.environ.get('AWS_REGION', 'us-east-1'), boto3)
-    adf_role_name = parameter_store.fetch_parameter('cross_account_access_role')
+    parameter_store = ParameterStore(
+        os.environ.get('AWS_REGION', 'us-east-1'), boto3)
+    adf_role_name = parameter_store.fetch_parameter(
+        'cross_account_access_role')
     for account in accounts:
         try:
-            account_id = next(acc["Id"] for acc in all_accounts if acc["Name"] == account.full_name)
-        except StopIteration: # If the account does not exist yet..
+            account_id = next(
+                acc["Id"] for acc in all_accounts if acc["Name"] == account.full_name)
+        except StopIteration:  # If the account does not exist yet..
             account_id = None
-        create_or_update_account(organizations, support, account, adf_role_name, account_id)
+        create_or_update_account(
+            organizations, support, account, adf_role_name, account_id)
 
 
 def create_or_update_account(org_session, support_session, account, adf_role_name, account_id=None):
@@ -57,24 +65,12 @@ def create_or_update_account(org_session, support_session, account, adf_role_nam
         ), 'adf_account_provisioning'
     )
 
-    LOGGER.info(f'Ensuring account {account_id} (alias {account.alias}) is in OU {account.ou_path}')
+    LOGGER.info(
+        f'Ensuring account {account_id} (alias {account.alias}) is in OU {account.ou_path}')
     org_session.move_account(account_id, account.ou_path)
     if account.delete_default_vpc:
         ec2_client = role.client('ec2')
-        all_regions = [
-            region['RegionName']
-            for region in ec2_client.describe_regions(
-                AllRegions=False,
-                Filters=[
-                    {
-                        'Name': 'opt-in-status',
-                        'Values': [
-                            'opt-in-not-required',
-                        ]
-                    }
-                ]
-            )['Regions']
-        ]
+        all_regions = get_all_regions(ec2_client)
         args = (
             (account_id, region, role)
             for region in all_regions
@@ -88,10 +84,38 @@ def create_or_update_account(org_session, support_session, account, adf_role_nam
         org_session.create_account_alias(account.alias, role)
 
     if account.tags:
-        LOGGER.info(f'Ensuring tags exist for account {account_id}: {account.tags}')
+        LOGGER.info(
+            f'Ensuring tags exist for account {account_id}: {account.tags}')
         org_session.create_account_tags(account_id, account.tags)
 
 
+@tenacity.retry(
+    stop=tenacity.stop_after_attempt(9),
+    wait=tenacity.wait_random_exponential(),
+)
+def get_all_regions(ec2_client):
+    try:
+        all_regions = [
+            region['RegionName']
+            for region in ec2_client.describe_regions(
+                AllRegions=False,
+                Filters=[
+                    {
+                        'Name': 'opt-in-status',
+                        'Values': [
+                            'opt-in-not-required',
+                        ]
+                    }
+                ]
+            )['Regions']
+        ]
+        LOGGER.info(f'Regions are: {all_regions}')
+        return all_regions
+    except Exception as ce:
+        LOGGER.info('Failed to describe regions: %s, retrying...', ce)
+        raise
+
+
 def schedule_delete_default_vpc(account_id, region, role):
     """Schedule a delete_default_vpc on a thread
     :param account_id: The account ID to remove the VPC from
@@ -101,5 +125,6 @@ def schedule_delete_default_vpc(account_id, region, role):
     ec2_client = role.client('ec2', region_name=region)
     delete_default_vpc(ec2_client, account_id, region, role)
 
+
 if __name__ == '__main__':
     main()

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/provisioner/src/vpc.py

@@ -10,33 +10,40 @@
 LOGGER = configure_logger(__name__)
 
 
-def vpc_cleanup(vpcid, role, region):
+def vpc_cleanup(account_id, vpcid, role, region):
     if not vpcid:
         return
-    ec2 = role.resource('ec2', region_name=region)
-    ec2client = ec2.meta.client
-    vpc = ec2.Vpc(vpcid)
-    # detach and delete all gateways associated with the vpc
-    for gw in vpc.internet_gateways.all():
-        vpc.detach_internet_gateway(InternetGatewayId=gw.id)
-        gw.delete()
-    # Route table associations
-    for rt in vpc.route_tables.all():
-        for rta in rt.associations:
-            if not rta.main:
-                rta.delete()
-    # Security Group
-    for sg in vpc.security_groups.all():
-        if sg.group_name != 'default':
-            sg.delete()
-    # Network interfaces
-    for subnet in vpc.subnets.all():
-        for interface in subnet.network_interfaces.all():
-            interface.delete()
-        subnet.delete()
-    # Delete vpc
-    ec2client.delete_vpc(VpcId=vpcid)
-    LOGGER.info(f"VPC {vpcid} and associated resources has been deleted.")
+    try:
+        ec2 = role.resource('ec2', region_name=region)
+        ec2client = ec2.meta.client
+        vpc = ec2.Vpc(vpcid)
+        # detach and delete all gateways associated with the vpc
+        for gw in vpc.internet_gateways.all():
+            vpc.detach_internet_gateway(InternetGatewayId=gw.id)
+            gw.delete()
+        # Route table associations
+        for rt in vpc.route_tables.all():
+            for rta in rt.associations:
+                if not rta.main:
+                    rta.delete()
+        # Security Group
+        for sg in vpc.security_groups.all():
+            if sg.group_name != 'default':
+                sg.delete()
+        # Network interfaces
+        for subnet in vpc.subnets.all():
+            for interface in subnet.network_interfaces.all():
+                interface.delete()
+            subnet.delete()
+        # Delete vpc
+        ec2client.delete_vpc(VpcId=vpcid)
+        LOGGER.info(f"VPC {vpcid} and associated resources has been deleted.")
+    except exceptions.ClientError:
+        LOGGER.warning(
+            f"WARNING: cannot delete VPC {vpcid} in account {account_id}",
+            exc_info=True,
+        )
+        raise
 
 
 def delete_default_vpc(client, account_id, region, role):
@@ -73,4 +80,4 @@ def delete_default_vpc(client, account_id, region, role):
 
     LOGGER.info(
         f"Found default VPC Id {default_vpc_id} in the {region} region")
-    vpc_cleanup(default_vpc_id, role, region)
+    vpc_cleanup(account_id, default_vpc_id, role, region)

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt

@@ -1,14 +1,15 @@
 # Install libs here that you might want in AWS CodeBuild (On Master Account)
 astroid~=2.4.2
-awscli==1.18.140
 aws-sam-cli==1.15.0
-botocore==1.17.63
+awscli==1.18.140
 boto3==1.14.63
+botocore==1.17.63
 jsii<1.20.0,>=1.16.0
 mock~=4.0.3
 pip~=20.2.3
 pylint~=2.6.0
 pytest~=6.2.1
 pyyaml>=5.3
 six~=1.15.0
+tenacity==6.3.0
 urllib3~=1.25.11