add sops as a secret reslover (#155)

* add sops as a secret reslover to himl. Sops code from: https://github.com/ansible-collections/community.sops/tree/main

* add caching for sops resolver. This prevents having to touch the yubikey on every secret resolve

---------

Co-authored-by: Adrian Muraru <[email protected]>

Author: rjhaverkamp

examples/secrets/default.yaml

@@ -1,3 +1,4 @@
 ---
 secret_path_v2: "{{vault.path(/kv2_secret)}}"
 secret_key_v2: "{{vault.key(/kv2_secret/key)}}"
+sops_secret: "{{ sops.secret_file(/home/user/secrets/secret_file.yaml).secret_key(['s3']['access_key']) }}"

himl/secret_resolvers.py

@@ -11,7 +11,6 @@
 import logging
 import os
 
-
 class SecretResolver:
     def supports(self, secret_type):
         return False
@@ -67,6 +66,15 @@ def resolve(self, secret_type, secret_params):
         s3 = SimpleS3(aws_profile, region_name)
         return s3.get(bucket, path, base64Encode)
 
+class SopsSecretResolver(SecretResolver):
+    def supports(self, secret_type):
+        return secret_type == "sops"
+    
+    def resolve(self, secret_type, secret_params):
+        from .simplesops import SimpleSops
+        file = self.get_param_or_exception("secret_file", secret_params)
+        sops = SimpleSops()
+        return sops.get(secret_file=file, secret_key=secret_params.get("secret_key"))
 
 class VaultSecretResolver(SecretResolver):
     def supports(self, secret_type):
@@ -96,7 +104,7 @@ def resolve(self, secret_type, secret_params):
 class AggregatedSecretResolver(SecretResolver):
     def __init__(self, default_aws_profile=None):
         self.secret_resolvers = (SSMSecretResolver(default_aws_profile), S3SecretResolver(default_aws_profile),
-                                 VaultSecretResolver())
+                                 VaultSecretResolver(), SopsSecretResolver())
 
     def supports(self, secret_type):
         return any([resolver.supports(secret_type) for resolver in self.secret_resolvers])

himl/simplesops.py

@@ -0,0 +1,130 @@
+# Copyright (c), Edoardo Tenani <[email protected]>, 2018-2020
+# Simplified BSD License (see LICENSES/BSD-2-Clause.txt or https://opensource.org/licenses/BSD-2-Clause)
+# SPDX-License-Identifier: BSD-2-Clause
+
+from __future__ import absolute_import, division, print_function
+from functools import lru_cache
+
+import os, logging, yaml
+
+from subprocess import Popen, PIPE
+
+logger = logging.getLogger(__name__)
+
+# From https://github.com/getsops/sops/blob/master/cmd/sops/codes/codes.go
+# Should be manually updated
+SOPS_ERROR_CODES = {
+    1: "ErrorGeneric",
+    2: "CouldNotReadInputFile",
+    3: "CouldNotWriteOutputFile",
+    4: "ErrorDumpingTree",
+    5: "ErrorReadingConfig",
+    6: "ErrorInvalidKMSEncryptionContextFormat",
+    7: "ErrorInvalidSetFormat",
+    8: "ErrorConflictingParameters",
+    21: "ErrorEncryptingMac",
+    23: "ErrorEncryptingTree",
+    24: "ErrorDecryptingMac",
+    25: "ErrorDecryptingTree",
+    49: "CannotChangeKeysFromNonExistentFile",
+    51: "MacMismatch",
+    52: "MacNotFound",
+    61: "ConfigFileNotFound",
+    85: "KeyboardInterrupt",
+    91: "InvalidTreePathFormat",
+    100: "NoFileSpecified",
+    128: "CouldNotRetrieveKey",
+    111: "NoEncryptionKeyFound",
+    200: "FileHasNotBeenModified",
+    201: "NoEditorFound",
+    202: "FailedToCompareVersions",
+    203: "FileAlreadyEncrypted",
+}
+
+
+class SopsError(Exception):
+    """Extend Exception class with sops specific information"""
+
+    def __init__(self, filename, exit_code, message, decryption=True):
+        if exit_code in SOPS_ERROR_CODES:
+            exception_name = SOPS_ERROR_CODES[exit_code]
+            message = "error with file %s: %s exited with code %d: %s" % (
+                filename,
+                exception_name,
+                exit_code,
+                message,
+            )
+        else:
+            message = (
+                "could not %s file %s; Unknown sops error code: %s; message: %s"
+                % (
+                    "decrypt" if decryption else "encrypt",
+                    filename,
+                    exit_code,
+                    message,
+                )
+            )
+        super(SopsError, self).__init__(message)
+
+
+class Sops:
+    """Utility class to perform sops CLI actions"""
+
+    @lru_cache(maxsize=2048)
+    def decrypt(
+        encrypted_file,
+        decode_output=True,
+        rstrip=True,
+    ):
+        command = ["sops"]
+        env = os.environ.copy()
+
+        command.extend(["--decrypt", encrypted_file])
+        process = Popen(
+            command,
+            stdin=None,
+            stdout=PIPE,
+            stderr=PIPE,
+            env=env,
+        )
+        (output, err) = process.communicate()
+        exit_code = process.returncode
+
+        if decode_output:
+            # output is binary, we want UTF-8 string
+            output = output.decode("utf-8", errors="surrogate_or_strict")
+            # the process output is the decrypted secret; be cautious
+        if exit_code != 0:
+            raise SopsError(encrypted_file, exit_code, err, decryption=True)
+
+        if rstrip:
+            output = output.rstrip()
+        return yaml.full_load(output)
+    
+    def get_keys(self, secret_file, secret_key):
+        result = Sops.decrypt(secret_file)
+        secret_key_path = secret_key.strip("[]")
+        keys = [key.strip("'") for key in secret_key_path.split("']['")]
+        try:
+            for key in keys:
+                result = result[key]
+        except KeyError as e:
+            raise SopsError(secret_file, 128, "Encountered KeyError parsing yaml for key: %s" % secret_key, decryption=True)
+        return result
+
+
+class SimpleSops:
+    def __init__(self):
+        pass
+
+    def get(self, secret_file, secret_key):
+        try:
+            logger.info(
+                "Resolving sops secret %s from file %s", secret_key, secret_file
+            )
+            return Sops().get_keys(secret_file=secret_file, secret_key=secret_key)
+        except SopsError as e:
+            raise Exception(
+                "Error while trying to read sops value for file %s, key: %s - %s"
+                % (secret_file, secret_key, e)
+            )

setup.py

@@ -29,7 +29,7 @@
 
 setup(
     name='himl',
-    version="0.15.1",
+    version="0.16.0",
     description='A hierarchical config using yaml',
     long_description=_readme + '\n\n',
     long_description_content_type='text/markdown',