Added support for service level x-aws keys from ecs-plugin (#273)

* Added support for x-aws-role and x-aws-policies

* Added support for x-aws-autoscaling

* added info in docs

Author: JohnPreston

docs/extras.rst

@@ -7,6 +7,20 @@ Extras
 ECS ComposeX aims to make life easy to take your application to AWS ECS, with using AWS Fargate as the primary
 focus (still, allows to run on EC2 should you need to).
 
+
+Docker ECS-Plugin x-aws-keys support
+=====================================
+
+In order to keep make the integration and inter-operability of tools used by developers, we are going to add support
+for, mostly, services level x-aws keys such as **-xaws-iam-role** or **x-aws-autoscaling**.
+
+This will allow developers who might have started a journey to ECS using the docker ecs plugin to continue that journey
+with ECS ComposeX without making too many changes.
+
+In case for a similar setting, such as *x-aws-iam-policies* which in ECS Composex is under *x-iam/Policies*, these
+non conflicting settings will add up together. However, in case of conflicting information, the ECS ComposeX definition
+will prevail over the x-aws-keys.
+
 AWS AppMesh integration
 =======================
 
@@ -52,9 +66,8 @@ numbers.
         image: my-nginx
         deploy:
           replicas: 2 # by default I want 2 containers
-        x-configs:
-          scaling:
-            range: "1-10" # 1 to 10 containers to deploy for the service
+        x-scaling:
+          range: "1-10" # 1 to 10 containers to deploy for the service
             target_scaling:
               cpu_target: 80 # Means 80% average for all containers in the service.
       backend:

ecs_composex/common/compose_services.py

@@ -285,17 +285,26 @@ class ComposeService(object):
         ("x-network", dict),
     ]
 
+    ecs_plugin_aws_keys = [
+        ("x-aws-role", dict),
+        ("x-aws-policies", list),
+        ("x-aws-autoscaling", dict),
+        ("x-aws-pull_credentials", str),
+    ]
+
     def __init__(self, name, definition, volumes=None, secrets=None):
         if not isinstance(definition, dict):
             raise TypeError(
                 "The definition of a service must be", dict, "got", type(definition)
             )
         if not all(
-            key in [title[0] for title in self.keys] for key in list(definition.keys())
+            key in [title[0] for title in self.ecs_plugin_aws_keys + self.keys]
+            for key in list(definition.keys())
         ):
             raise KeyError(
                 "Only valid keys for a service definition are",
                 sorted([key[0] for key in self.keys]),
+                sorted([key[0] for key in self.ecs_plugin_aws_keys]),
                 "Got",
                 sorted(list(definition.keys())),
             )
@@ -322,6 +331,8 @@ def __init__(self, name, definition, volumes=None, secrets=None):
         self.x_iam = set_else_none("x-iam", self.definition)
         self.x_logging = {"RetentionInDays": 14, "CreateLogGroup": True}
 
+        self.import_x_aws_settings()
+
         self.replicas = 1
         self.container = None
         self.volumes = []
@@ -399,6 +410,57 @@ def set_container_definition(self):
         )
         self.container_parameters.update({self.image_param.title: self.image})
 
+    def merge_x_aws_role(self, key):
+        """
+        Method to update the service definition with the x-aws-role information if NOT defined in the composex
+        definition.
+
+        :param str key:
+        """
+        policy_def = {
+            "PolicyName": "ImportedFromXAWSRole",
+            "PolicyDocument": self.definition[key],
+        }
+        if not self.x_iam:
+            self.x_iam = {"Policies": [policy_def]}
+            LOG.info(f"Added {key} definition")
+        elif self.x_iam and keyisset("Policies", self.x_iam):
+            self.x_iam["Policies"].append(policy_def)
+            LOG.info(f"Merged {key} to existing definition")
+
+    def merge_x_policies(self, key):
+        """
+        Method to merge policies
+
+        :param str key:
+        """
+        if not self.x_iam:
+            self.x_iam = {"ManagedPolicyArns": self.definition[key]}
+            LOG.info(f"Added {key} definition")
+        elif self.x_iam and keyisset("ManagedPolicyArns", self.x_iam):
+            self.x_iam["ManagedPolicyArns"] += self.definition[key]
+            LOG.info(f"Merged {key} definition")
+
+    def import_x_aws_settings(self):
+        aws_keys = [
+            ("x-aws-role", dict, self.merge_x_aws_role),
+            ("x-aws-policies", list, self.merge_x_policies),
+            ("x-aws-autoscaling", dict, None),
+            ("x-aws-pull_credentials", str, None),
+        ]
+        for setting in aws_keys:
+            if keyisset(setting[0], self.definition) and not isinstance(
+                self.definition[setting[0]], setting[1]
+            ):
+                raise TypeError(
+                    f"{setting[0]} is of type",
+                    type(self.definition[setting[0]]),
+                    "Expected",
+                    setting[1],
+                )
+            elif keyisset(setting[0], self.definition) and setting[2]:
+                setting[2](setting[0])
+
     def define_logging(self):
         """
         Method to define logging properties
@@ -686,18 +748,22 @@ def add_policies(config, key, new_policies):
             if f"PolicyGenerated{count}" not in existing_policy_names
             else f"PolicyGenerated{count+len(existing_policy_names)}"
         )
-        name = generated_name if not keyisset("name", policy) else policy["name"]
+        name = (
+            generated_name
+            if not keyisset("PolicyName", policy)
+            else policy["PolicyName"]
+        )
         if name in existing_policy_names:
             return
-        if not keyisset("document", policy):
+        if not keyisset("PolicyDocument", policy):
             raise KeyError("You must set the policy document for the policy")
         if (
-            keyisset("Version", policy["document"])
-            and not isinstance(policy["document"]["Version"], str)
-            or not keyisset("Version", policy["document"])
+            keyisset("Version", policy["PolicyDocument"])
+            and not isinstance(policy["PolicyDocument"]["Version"], str)
+            or not keyisset("Version", policy["PolicyDocument"])
         ):
-            policy["document"]["Version"] = "2012-10-17"
-        policy_object = Policy(PolicyName=name, PolicyDocument=policy["document"])
+            policy["PolicyDocument"]["Version"] = "2012-10-17"
+        policy_object = Policy(PolicyName=name, PolicyDocument=policy["PolicyDocument"])
         existing_policies.append(policy_object)
 
 
@@ -723,7 +789,11 @@ def __init__(self, services, family_name):
         self.ignored_services = []
         self.name = family_name
         self.logical_name = re.sub(r"[^a-zA-Z0-9]+", "", family_name)
-        self.iam = {"boundary": None, "managed_policies": [], "policies": []}
+        self.iam = {
+            "PermissionsBoundary": None,
+            "ManagedPolicyArns": [],
+            "Policies": [],
+        }
         self.template = None
         self.use_xray = None
         self.stack = None
@@ -766,7 +836,7 @@ def set_xray(self):
                         "resources": {"limits": {"cpus": 0.03125, "memory": "256M"}},
                     },
                     "x-iam": {
-                        "managed_policies": [
+                        "ManagedPolicyArns": [
                             "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess"
                         ]
                     },
@@ -874,9 +944,9 @@ def sort_iam_settings(self, key, setting):
 
     def handle_iam(self):
         valid_keys = [
-            ("managed_policies", list, None),
-            ("policies", list, add_policies),
-            ("boundary", (str, Sub), handle_iam_boundary),
+            ("ManagedPolicyArns", list, None),
+            ("Policies", list, add_policies),
+            ("PermissionsBoundary", (str, Sub), handle_iam_boundary),
         ]
         iam_settings = [service.x_iam for service in self.services if service.x_iam]
         for setting in iam_settings:
@@ -885,7 +955,7 @@ def handle_iam(self):
         self.set_secrets_access()
 
     def handle_permission_boundary(self, prop_key):
-        if keyisset("boundary", self.iam) and self.template:
+        if keyisset("PermissionsBoundary", self.iam) and self.template:
             if EXEC_ROLE_T in self.template.resources:
                 add_role_boundaries(
                     self.template.resources[EXEC_ROLE_T], self.iam[prop_key]
@@ -943,17 +1013,17 @@ def assign_policies(self, role_name=None):
         role = self.template.resources[role_name]
         props = [
             (
-                "managed_policies",
+                "ManagedPolicyArns",
                 "ManagedPolicyArns",
                 list,
                 self.assign_iam_managed_policies,
             ),
-            ("policies", "Policies", list, self.assign_iam_policies),
-            ("boundary", "PermissionsBoundary", (str, Sub), None),
+            ("Policies", "Policies", list, self.assign_iam_policies),
+            ("PermissionsBoundary", "PermissionsBoundary", (str, Sub), None),
         ]
         for prop in props:
             if keyisset(prop[0], self.iam) and isinstance(self.iam[prop[0]], prop[2]):
-                if prop[0] == "boundary":
+                if prop[0] == "PermissionsBoundary":
                     self.handle_permission_boundary(prop[0])
                 elif prop[3]:
                     prop[3](role, prop)

ecs_composex/ecs/ecs_scaling.py

@@ -241,6 +241,36 @@ def handle_target_scaling(config, key, new_config):
         define_new_config(config, key, new_config)
 
 
+def handle_defined_x_aws_autoscaling(configs, service):
+    """
+    Function to sort out existing or not x-aws-autoscaling in the deploy section
+
+    :param list configs:
+    :param ecs_composex.common.compose_services.ComposeService service:
+    :return:
+    """
+    if keyisset("deploy", service.definition) and keyisset(
+        "x-aws-autoscaling", service.definition["deploy"]
+    ):
+        config = service.definition["deploy"]["x-aws-autoscaling"]
+        min_count = 1 if not keypresent("min", config) else int(config["min"])
+        max_count = 1 if not keypresent("max", config) else int(config["max"])
+        if not service.x_scaling:
+            service.x_scaling = {"range": f"{min_count}-{max_count}"}
+            if keyisset("cpu", config):
+                service.x_scaling.update(
+                    {"target_scaling": {"cpu_target": int(config["cpu"])}}
+                )
+        elif service.x_scaling:
+            LOG.warning(
+                f"Detected both x-aws-autoscaling and x-scaling for {service.name}. Priority goes to x-scaling"
+            )
+        configs.append(service.x_scaling)
+    elif not keyisset("deploy", service.definition) and service.x_scaling:
+        LOG.debug("No x-aws-autoscaling detected, proceeding as usual")
+        configs.append(service.x_scaling)
+
+
 def merge_family_services_scaling(services):
     x_scaling = {
         "range": None,
@@ -252,8 +282,10 @@ def merge_family_services_scaling(services):
     }
     x_scaling_configs = []
     for service in services:
-        if service.x_scaling:
-            x_scaling_configs.append(service.x_scaling)
+        handle_defined_x_aws_autoscaling(x_scaling_configs, service)
+
+    print(x_scaling_configs)
+
     valid_keys = [
         ("range", str, handle_range),
         ("target_scaling", dict, handle_target_scaling),
@@ -266,7 +298,6 @@ def merge_family_services_scaling(services):
                 and key[2]
             ):
                 key[2](x_scaling, key[0], config[key[0]])
-
     return x_scaling
 
 

ecs_composex/ecs/ecs_service_config.py

@@ -46,7 +46,7 @@ def __init__(self, family, settings):
         :param ecs_composex.common.settings.ComposeXSettings settings:
         """
         self.network = ServiceNetworking(family)
-        self.scaling = ServiceScaling(family.services)
+        self.scaling = ServiceScaling(family.ordered_services)
         self.use_appmesh = (
             False if not keyisset("x-appmesh", settings.compose_content) else True
         )

features/features/common.feature

@@ -42,7 +42,7 @@ Feature: common
     Then I render the docker-compose to composex to validate
     And I render all files to verify execution
     Examples:
-      | file_path                   | override_file            |
+      | file_path          | override_file            |
       | use-cases/blog.yml | use-cases/all-in-one.yml |
 
   @cluster
@@ -62,3 +62,11 @@ Feature: common
     Examples:
       | file_path                   | override_file                    |
       | use-cases/blog.features.yml | use-cases/logging/variations.yml |
+
+  @ecs-plugin-suport
+  Scenario Outline: ECS Plugin support
+    Given I use <file_path> as my docker-compose file and <override_file> as override file
+    Then I render the docker-compose to composex to validate
+    Examples:
+      | file_path                   | override_file                                    |
+      | use-cases/blog.features.yml | use-cases/ecs_plugin_support/blog.features.x.yml |

use-cases/acm/working_acm.yml

@@ -50,7 +50,7 @@ services:
     x-configs:
       use_xray: True
       iam:
-        boundary: arn:aws:iam::aws:policy/PowerUser
+        PermissionsBoundary: arn:aws:iam::aws:policy/PowerUser
 
   backend:
     image: nginx

use-cases/all-in-one.yml

@@ -32,18 +32,18 @@ services:
         target: 5000
     secrets:
       - zyx
+    x-xray: false
     x-iam:
-      policies:
-        - document:
+      Policies:
+        - PolicyName: toto
+          PolicyDocument:
             Statement:
               - Action:
                   - s3:DeleteBucket
                 Effect: Deny
                 Resource:
                   - '*'
                 Sid: SomethingStupid
-          name: toto
-    x-xray: false
   app02:
     deploy:
       labels:
@@ -72,8 +72,8 @@ services:
       - zyx
     x-configs:
     x-iam:
-      boundary: ccoe/js-developer
-      managed_policies:
+      PermissionsBoundary: ccoe/js-developer
+      ManagedPolicyArns:
         - arn:aws:iam:aws::policy/AdministratorAccess
     x-scaling:
       range: 1-5
@@ -130,7 +130,7 @@ services:
         published: 80
         target: 80
     x-iam:
-      managed_policies:
+      ManagedPolicyArns:
         - arn:aws:iam:aws::policy/ReadOnlyAccess
     x-scaling:
       range: 0-2

use-cases/appmesh/existing_mesh.yml

@@ -129,4 +129,4 @@ x-vpc:
 x-configs:
   composex:
     iam:
-      boundary: arn:aws:iam::aws:policy/PowerUser
+      PermissionsBoundary: arn:aws:iam::aws:policy/PowerUser