Merge pull request #112 from fabfuel/feature/diffing

New command: diff

Author: fabfuel

ecs_deploy/cli.py

@@ -4,11 +4,12 @@
 from time import sleep
 
 import click
+import json
 import getpass
 from datetime import datetime, timedelta
 
 from ecs_deploy import VERSION
-from ecs_deploy.ecs import DeployAction, ScaleAction, RunAction, EcsClient, \
+from ecs_deploy.ecs import DeployAction, ScaleAction, RunAction, EcsClient, DiffAction, \
     TaskPlacementError, EcsError, UpdateAction, LAUNCH_TYPE_EC2, LAUNCH_TYPE_FARGATE
 from ecs_deploy.newrelic import Deployment, NewRelicException
 from ecs_deploy.slack import SlackNotification
@@ -348,6 +349,51 @@ def run(cluster, task, count, command, env, secret, launchtype, subnet, security
         exit(1)
 
 
+@click.command()
+@click.argument('task')
+@click.argument('revision_a')
+@click.argument('revision_b')
+@click.option('--region', help='AWS region (e.g. eu-central-1)')
+@click.option('--access-key-id', help='AWS access key id')
+@click.option('--secret-access-key', help='AWS secret access key')
+@click.option('--profile', help='AWS configuration profile name')
+def diff(task, revision_a, revision_b, region, access_key_id, secret_access_key, profile):
+    """
+    Compare two task definition revisions.
+
+    \b
+    TASK is the name of your task definition (e.g. 'my-task') within ECS.
+    COUNT is the number of tasks your service should run.
+    """
+
+    try:
+        client = get_client(access_key_id, secret_access_key, region, profile)
+        action = DiffAction(client)
+
+        td_a = action.get_task_definition('%s:%s' % (task, revision_a))
+        td_b = action.get_task_definition('%s:%s' % (task, revision_b))
+
+        result = td_a.diff_raw(td_b)
+        for difference in result:
+            click.secho('%s: %s' % (difference[0], difference[1]))
+
+            if difference[0] == 'add':
+                for added in difference[2]:
+                    click.secho('    + %s: %s' % (added[0], json.dumps(added[1])))
+
+            if difference[0] == 'change':
+                click.secho('    - %s' % json.dumps(difference[2][0]))
+                click.secho('    + %s' % json.dumps(difference[2][1]))
+
+            if difference[0] == 'remove':
+                for removed in difference[2]:
+                    click.secho('    - %s: %s' % removed)
+
+    except EcsError as e:
+        click.secho('%s\n' % str(e), fg='red', err=True)
+        exit(1)
+
+
 def wait_for_finish(action, timeout, title, success_message, failure_message,
                     ignore_warnings, sleep_time=1):
     click.secho(title, nl=False)
@@ -544,6 +590,7 @@ def inspect_errors(service, failure_message, ignore_warnings, since, timeout):
 ecs.add_command(run)
 ecs.add_command(cron)
 ecs.add_command(update)
+ecs.add_command(diff)
 
 if __name__ == '__main__':  # pragma: no cover
     ecs()

ecs_deploy/ecs.py

@@ -5,6 +5,7 @@
 from boto3.session import Session
 from botocore.exceptions import ClientError, NoCredentialsError
 from dateutil.tz.tz import tzlocal
+from dictdiffer import diff
 
 JSON_LIST_REGEX = re.compile(r'^\[.*\]$')
 
@@ -237,6 +238,47 @@ def family_revision(self):
     def diff(self):
         return self._diff
 
+    def diff_raw(self, task_b):
+        containers_a = {c['name']: c for c in self.containers}
+        containers_b = {c['name']: c for c in task_b.containers}
+
+        requirements_a = sorted([r['name'] for r in self.requires_attributes])
+        requirements_b = sorted([r['name'] for r in task_b.requires_attributes])
+
+        for container in containers_a:
+            containers_a[container]['environment'] = {e['name']: e['value'] for e in containers_a[container].get('environment', {})}
+
+        for container in containers_b:
+            containers_b[container]['environment'] = {e['name']: e['value'] for e in containers_b[container].get('environment', {})}
+
+        for container in containers_a:
+            containers_a[container]['secrets'] = {e['name']: e['valueFrom'] for e in containers_a[container].get('secrets', {})}
+
+        for container in containers_b:
+            containers_b[container]['secrets'] = {e['name']: e['valueFrom'] for e in containers_b[container].get('secrets', {})}
+
+        composite_a = {
+            'containers': containers_a,
+            'volumes': self.volumes,
+            'requires_attributes': requirements_a,
+            'role_arn': self.role_arn,
+            'execution_role_arn': self.execution_role_arn,
+            'compatibilities': self.compatibilities,
+            'additional_properties': self.additional_properties,
+        }
+
+        composite_b = {
+            'containers': containers_b,
+            'volumes': task_b.volumes,
+            'requires_attributes': requirements_b,
+            'role_arn': task_b.role_arn,
+            'execution_role_arn': task_b.execution_role_arn,
+            'compatibilities': task_b.compatibilities,
+            'additional_properties': task_b.additional_properties,
+        }
+
+        return list(diff(composite_a, composite_b))
+
     def get_overrides(self):
         override = dict()
         overrides = []
@@ -670,6 +712,11 @@ def __init__(self, client):
         super(UpdateAction, self).__init__(client, None, None)
 
 
+class DiffAction(EcsAction):
+    def __init__(self, client):
+        super(DiffAction, self).__init__(client, None, None)
+
+
 class EcsError(Exception):
     pass
 

setup.py

@@ -11,7 +11,7 @@ def readme():
         return f.read()
 
 
-dependencies = ['click<7.0.0', 'botocore>=1.12.0', 'boto3>=1.4.7', 'future', 'requests']
+dependencies = ['click<7.0.0', 'botocore>=1.12.0', 'boto3>=1.4.7', 'future', 'requests', 'dictdiffer==0.8.0']
 
 setup(
     name='ecs-deploy',

tests/test_cli.py

@@ -9,7 +9,9 @@
 from ecs_deploy.ecs import EcsClient
 from ecs_deploy.newrelic import Deployment, NewRelicDeploymentException
 from tests.test_ecs import EcsTestClient, CLUSTER_NAME, SERVICE_NAME, \
-    TASK_DEFINITION_ARN_1, TASK_DEFINITION_ARN_2, TASK_DEFINITION_FAMILY_1
+    TASK_DEFINITION_ARN_1, TASK_DEFINITION_ARN_2, TASK_DEFINITION_FAMILY_1, \
+    TASK_DEFINITION_REVISION_2, TASK_DEFINITION_REVISION_1, \
+    TASK_DEFINITION_REVISION_3
 
 
 @pytest.fixture
@@ -1017,4 +1019,45 @@ def test_cron(get_client, runner):
     assert u'Successfully deregistered revision: 2' in result.output
 
 
-    print(result.output)
+@patch('ecs_deploy.cli.get_client')
+def test_diff(get_client, runner):
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key')
+    result = runner.invoke(cli.diff, (TASK_DEFINITION_FAMILY_1, str(TASK_DEFINITION_REVISION_1), str(TASK_DEFINITION_REVISION_3)))
+
+    assert not result.exception
+    assert result.exit_code == 0
+
+    assert 'change: containers.webserver.image' in result.output
+    assert '- "webserver:123"' in result.output
+    assert '+ "webserver:456"' in result.output
+    assert 'change: containers.webserver.command' in result.output
+    assert '- "run"' in result.output
+    assert '+ "execute"' in result.output
+    assert 'change: containers.webserver.environment.foo' in result.output
+    assert '- "bar"' in result.output
+    assert '+ "foobar"' in result.output
+    assert 'remove: containers.webserver.environment' in result.output
+    assert '- empty: ' in result.output
+    assert 'change: containers.webserver.secrets.baz' in result.output
+    assert '- "qux"' in result.output
+    assert '+ "foobaz"' in result.output
+    assert 'change: containers.webserver.secrets.dolor' in result.output
+    assert '- "sit"' in result.output
+    assert '+ "loremdolor"' in result.output
+    assert 'change: role_arn' in result.output
+    assert '- "arn:test:role:1"' in result.output
+    assert '+ "arn:test:another-role:1"' in result.output
+    assert 'change: execution_role_arn' in result.output
+    assert '- "arn:test:role:1"' in result.output
+    assert '+ "arn:test:another-role:1"' in result.output
+    assert 'add: containers.webserver.environment' in result.output
+    assert '+ newvar: "new value"' in result.output
+
+
+@patch('ecs_deploy.cli.get_client')
+def test_diff_without_credentials(get_client, runner):
+    get_client.return_value = EcsTestClient()
+    result = runner.invoke(cli.diff, (TASK_DEFINITION_FAMILY_1, str(TASK_DEFINITION_REVISION_1), str(TASK_DEFINITION_REVISION_3)))
+
+    assert result.exit_code == 1
+    assert u'Unable to locate credentials. Configure credentials by running "aws configure".\n\n' in result.output

tests/test_ecs.py

@@ -30,6 +30,7 @@
      u'secrets': ({"name": "baz", "valueFrom": "qux"}, {"name": "dolor", "valueFrom": "sit"})},
     {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': ()}
 ]
+
 TASK_DEFINITION_FAMILY_2 = u'test-task'
 TASK_DEFINITION_REVISION_2 = 2
 TASK_DEFINITION_ARN_2 = u'arn:aws:ecs:eu-central-1:123456789012:task-definition/%s:%s' % (TASK_DEFINITION_FAMILY_2,
@@ -42,6 +43,18 @@
     {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': ()}
 ]
 
+TASK_DEFINITION_REVISION_3 = 3
+TASK_DEFINITION_ARN_3 = u'arn:aws:ecs:eu-central-1:123456789012:task-definition/%s:%s' % (TASK_DEFINITION_FAMILY_1,
+                                                                                          TASK_DEFINITION_REVISION_3)
+TASK_DEFINITION_VOLUMES_3 = []
+TASK_DEFINITION_CONTAINERS_3 = [
+    {u'name': u'webserver', u'image': u'webserver:456', u'command': u'execute',
+     u'environment': ({"name": "foo", "value": "foobar"}, {"name": "newvar", "value": "new value"}),
+     u'secrets': ({"name": "baz", "valueFrom": "foobaz"}, {"name": "dolor", "valueFrom": "loremdolor"})},
+    {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': ()}
+]
+TASK_DEFINITION_ROLE_ARN_3 = u'arn:test:another-role:1'
+
 PAYLOAD_TASK_DEFINITION_1 = {
     u'taskDefinitionArn': TASK_DEFINITION_ARN_1,
     u'family': TASK_DEFINITION_FAMILY_1,
@@ -69,6 +82,22 @@
     u'compatibilities': [u'EC2'],
 }
 
+PAYLOAD_TASK_DEFINITION_3 = {
+    u'taskDefinitionArn': TASK_DEFINITION_ARN_3,
+    u'family': TASK_DEFINITION_FAMILY_1,
+    u'revision': TASK_DEFINITION_REVISION_3,
+    u'taskRoleArn': TASK_DEFINITION_ROLE_ARN_3,
+    u'executionRoleArn': TASK_DEFINITION_ROLE_ARN_3,
+    u'volumes': deepcopy(TASK_DEFINITION_VOLUMES_3),
+    u'containerDefinitions': deepcopy(TASK_DEFINITION_CONTAINERS_3),
+    u'status': u'active',
+    u'requiresAttributes': {},
+    u'networkMode': u'host',
+    u'placementConstraints': {},
+    u'unknownProperty': u'lorem-ipsum',
+    u'compatibilities': [u'EC2'],
+}
+
 TASK_ARN_1 = u'arn:aws:ecs:eu-central-1:123456789012:task/12345678-1234-1234-1234-123456789011'
 TASK_ARN_2 = u'arn:aws:ecs:eu-central-1:123456789012:task/12345678-1234-1234-1234-123456789012'
 
@@ -166,11 +195,17 @@
     u"taskDefinition": PAYLOAD_TASK_DEFINITION_2
 }
 
+RESPONSE_TASK_DEFINITION_3 = {
+    u"taskDefinition": PAYLOAD_TASK_DEFINITION_3
+}
+
 RESPONSE_TASK_DEFINITIONS = {
     TASK_DEFINITION_ARN_1: RESPONSE_TASK_DEFINITION,
     TASK_DEFINITION_ARN_2: RESPONSE_TASK_DEFINITION_2,
+    TASK_DEFINITION_ARN_3: RESPONSE_TASK_DEFINITION_3,
     u'test-task:1': RESPONSE_TASK_DEFINITION,
     u'test-task:2': RESPONSE_TASK_DEFINITION_2,
+    u'test-task:3': RESPONSE_TASK_DEFINITION_3,
     u'test-task': RESPONSE_TASK_DEFINITION_2,
 }
 
@@ -892,6 +927,8 @@ def describe_services(self, cluster_name, service_name):
         }
 
     def describe_task_definition(self, task_definition_arn):
+        if not self.access_key_id or not self.secret_access_key:
+            raise EcsConnectionError(u'Unable to locate credentials. Configure credentials by running "aws configure".')
         if task_definition_arn in RESPONSE_TASK_DEFINITIONS:
             return deepcopy(RESPONSE_TASK_DEFINITIONS[task_definition_arn])
         raise UnknownTaskDefinitionError('Unknown task definition arn: %s' % task_definition_arn)