Support interpolation for S3 value retrieval (#6)

Author: costimuraru

.gitignore

@@ -1,3 +1,5 @@
+*.DS_Store
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -225,6 +225,12 @@ See an example [here](https://github.com/adobe/himl#deep-merge-example).
 passphrase: "{{ssm.path(/key/coming/from/aws/secrets/store/manager).aws_profile(myprofile)}}"
 ```
 
+#### [AWS S3](https://aws.amazon.com/s3/)
+
+```yaml
+my_value: "{{s3.bucket(my-bucket).path(path/to/file.txt).base64encode(true).aws_profile(myprofile)}}"
+```
+
 #### [Vault](https://www.vaultproject.io/)
 
 Not yet implemented.

examples/complex/default.yaml

@@ -11,3 +11,6 @@ cluster:
 
 # Fetching the secret value at runtime, from a secrets store (in this case AWS SSM).
 # passphrase: "{{ssm.path(/key/coming/from/aws/secrets/store/manager).aws_profile(myprofile)}}"
+
+# Fetching the value at runtime from S3
+# my_secret: "{{s3.bucket(my-bucket).path(path/to/file.txt).base64encode(true).aws_profile(myprofile)}}"

examples/complex/env=dev/env.yaml

@@ -1 +1 @@
-env: dev
+env: dev

himl/inject_secrets.py

@@ -8,6 +8,7 @@
 # OF ANY KIND, either express or implied. See the License for the specific language
 # governing permissions and limitations under the License.
 
+import re
 from .secret_resolvers import AggregatedSecretResolver
 
 try:
@@ -43,7 +44,7 @@ def inject_secret(self, line):
         updated_line = line[2:-2]
 
         # parse each key/value (eg. path=my_pwd)
-        parts = updated_line.split('.')
+        parts = self.split_dot_not_within_parentheses(updated_line)
         if len(parts) <= 1:
             return line
 
@@ -62,3 +63,12 @@ def inject_secret(self, line):
             return self.resolver.resolve(secret_type, secret_params)
         else:
             return line
+
+    def split_dot_not_within_parentheses(self, line):
+        """
+        s3.bucket(my-bucket).path(path/to/file.txt).aws_profile(myprofile)
+        will result in:
+        ['s3', 'bucket(my-bucket)', 'path(path/to/file.txt)']
+        """
+        pattern = r'\.\s*(?![^()]*\))'
+        return re.split(pattern, line)

himl/secret_resolvers.py

@@ -8,7 +8,9 @@
 # OF ANY KIND, either express or implied. See the License for the specific language
 # governing permissions and limitations under the License.
 
+import logging
 from .simplessm import SimpleSSM
+from .simples3 import SimpleS3
 
 
 class SecretResolver:
@@ -18,6 +20,11 @@ def supports(self, secret_type):
     def resolve(self, secret_type, secret_params):
         return None
 
+    def get_param_or_exception(self, key, params):
+        if key not in params:
+            raise Exception("Could not find required key '{}' in the secret params: {}".format(key, params))
+        return params[key]
+
 
 class SSMSecretResolver(SecretResolver):
     def __init__(self, default_aws_profile=None):
@@ -29,17 +36,33 @@ def supports(self, secret_type):
     def resolve(self, secret_type, secret_params):
         aws_profile = secret_params.get("aws_profile", self.default_aws_profile)
         if not aws_profile:
-            raise Exception("Could not find the aws_profile in the secret params: {}".format(secret_params))
+            raise Exception("Could not find the aws_profile in the secret params for SSM secret: {}".format(secret_params))
 
         path = self.get_param_or_exception("path", secret_params)
         region_name = secret_params.get("region_name", "us-east-1")
         ssm = SimpleSSM(aws_profile, region_name)
         return ssm.get(path)
 
-    def get_param_or_exception(self, key, params):
-        if key not in params:
-            raise Exception("Could not find required key '{}' in the secret params: {}".format(key, params))
-        return params[key]
+
+class S3SecretResolver(SecretResolver):
+    def __init__(self, default_aws_profile=None):
+        self.default_aws_profile = default_aws_profile
+
+    def supports(self, secret_type):
+        return secret_type == "s3"
+
+    def resolve(self, secret_type, secret_params):
+        aws_profile = secret_params.get("aws_profile", self.default_aws_profile)
+        if not aws_profile:
+            raise Exception("Could not find the aws_profile in the secret params for S3 secret: {}".format(secret_params))
+
+        bucket = self.get_param_or_exception("bucket", secret_params)
+        path = self.get_param_or_exception("path", secret_params)
+        region_name = secret_params.get("region_name", "us-east-1")
+        base64Encode = secret_params.get("base64encode", False)
+        base64Encode = base64Encode == 'true'
+        s3 = SimpleS3(aws_profile, region_name)
+        return s3.get(bucket, path, base64Encode)
 
 
 # TODO - vault resolver
@@ -54,7 +77,7 @@ def resolve(self, secret_type, secret_params):
 class AggregatedSecretResolver(SecretResolver):
 
     def __init__(self, default_aws_profile=None):
-        self.secret_resolvers = (SSMSecretResolver(default_aws_profile), VaultSecretResolver())
+        self.secret_resolvers = (SSMSecretResolver(default_aws_profile), S3SecretResolver(default_aws_profile), VaultSecretResolver())
 
     def supports(self, secret_type):
         return any([resolver.supports(secret_type) for resolver in self.secret_resolvers])

himl/simples3.py

@@ -0,0 +1,45 @@
+# Copyright 2019 Adobe. All rights reserved.
+# This file is licensed to you under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+# OF ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+import boto3
+import logging
+import os
+from botocore.exceptions import ClientError
+
+
+logger = logging.getLogger(__name__)
+
+class SimpleS3(object):
+    def __init__(self, aws_profile, region_name):
+        self.aws_profile = aws_profile
+        self.region_name = region_name
+
+    def get(self, bucket_name, bucket_key, base64Encode=False):
+        try:
+            logger.info("Resolving S3 object for bucket %s, key '%s' on profile %s in region %s", 
+                bucket_name, bucket_key, self.aws_profile, self.region_name)
+            client = self.get_s3_client()
+            bucket_object = client.get_object(Bucket=bucket_name, Key=bucket_key)["Body"].read()
+            return self.parse_data(bucket_object, base64Encode)
+        except ClientError as e:
+            raise Exception(
+                'Error while trying to read S3 value for bucket_name %s, bucket_key: %s - %s' 
+                % (bucket_name, bucket_key, e.response['Error']['Code']))
+
+    def parse_data(self, bucket_object, base64Encode):
+        if base64Encode:
+            import base64
+            encodedBytes = base64.b64encode(bucket_object)
+            return str(encodedBytes, "utf-8")
+        return bucket_object
+
+    def get_s3_client(self):
+        session = boto3.session.Session(profile_name=self.aws_profile)
+        return session.client('s3')

himl/simplessm.py

@@ -8,12 +8,14 @@
 # OF ANY KIND, either express or implied. See the License for the specific language
 # governing permissions and limitations under the License.
 
-import os
-
 import boto3
+import logging
+import os
 from botocore.exceptions import ClientError
 
 
+logger = logging.getLogger(__name__)
+
 class SimpleSSM(object):
     def __init__(self, aws_profile, region_name):
         self.initial_aws_profile = os.getenv('AWS_PROFILE', None)
@@ -23,6 +25,7 @@ def __init__(self, aws_profile, region_name):
     def get(self, key):
         client = self.get_ssm_client()
         try:
+            logger.info("Resolving SSM secret for key '%s' on profile %s in region %s", key, self.aws_profile, self.region_name)
             return client.get_parameter(Name=key, WithDecryption=True).get("Parameter").get("Value")
         except ClientError as e:
             raise Exception(