feat: add secret management with template placeholders (#93)

Author: rafafrdz

cli/src/main/scala/com/eff3ct/teckel/io/Parser.scala

@@ -27,7 +27,7 @@ package com.eff3ct.teckel.io
 import cats.effect.Async
 import com.eff3ct.teckel.api.core.Run
 import com.eff3ct.teckel.semantic.core.EvalContext
-import com.eff3ct.teckel.serializer.{ConfigMerger, VariableResolver}
+import com.eff3ct.teckel.serializer.{ConfigMerger, SecretResolver, VariableResolver}
 import fs2.io.file.{Files, Path}
 import fs2.io.stdinUtf8
 
@@ -54,13 +54,14 @@ object Parser {
             case Right(content) => content
             case Left(err)      => throw new RuntimeException(s"Failed to merge configs: ${err.message}")
           }
-          resolved = VariableResolver.resolve(merged, variables)
+          resolved = SecretResolver.resolve(VariableResolver.resolve(merged, variables))
           result <- fs2.Stream.eval(Run[F].run[O](resolved))
         } yield result
       case None =>
         Files[F]
           .readUtf8(Path(file))
           .map(content => VariableResolver.resolve(content, variables))
+          .map(content => SecretResolver.resolve(content))
           .evalMap(Run[F].run[O])
     }
 
@@ -69,6 +70,7 @@ object Parser {
   ): fs2.Stream[F, O] =
     stdinUtf8(1024)
       .map(content => VariableResolver.resolve(content, variables))
+      .map(content => SecretResolver.resolve(content))
       .evalMap(Run[F].run[O])
 
 }

docs/etl/secrets.yaml

@@ -0,0 +1,19 @@
+secrets:
+  keys:
+    db_password:
+      scope: my-keyvault
+      key: database-password
+
+input:
+  - name: dbSource
+    format: jdbc
+    options:
+      url: 'jdbc:postgresql://host:5432/db'
+      user: admin
+      password: '{{secrets.db_password}}'
+
+output:
+  - name: dbSource
+    format: parquet
+    mode: overwrite
+    path: 'data/parquet/db_export'

serializer/src/main/scala/com/eff3ct/teckel/serializer/SecretResolver.scala

@@ -0,0 +1,70 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2024 Rafael Fernandez
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package com.eff3ct.teckel.serializer
+
+import io.circe.Json
+import io.circe.yaml.parser
+
+object SecretResolver {
+
+  private val secretPattern = """\{\{secrets\.([^}]+)\}\}""".r
+
+  case class SecretKey(alias: String, scope: String, key: String)
+
+  def resolve(content: String, provider: SecretsProvider = SecretsProvider.default): String = {
+    // First, try to parse the YAML to extract the secrets.keys section
+    val secretKeys: Map[String, SecretKey] = parser
+      .parse(content)
+      .toOption
+      .flatMap(_.hcursor.downField("secrets").downField("keys").as[Map[String, Json]].toOption)
+      .map(_.flatMap { case (alias, json) =>
+        for {
+          scope <- json.hcursor.downField("scope").as[String].toOption
+          key   <- json.hcursor.downField("key").as[String].toOption
+        } yield alias -> SecretKey(alias, scope, key)
+      })
+      .getOrElse(Map.empty)
+
+    // Replace all {{secrets.alias}} placeholders
+    secretPattern.replaceAllIn(
+      content,
+      { m =>
+        val alias = m.group(1)
+        val value = secretKeys.get(alias) match {
+          case Some(sk) => provider.get(sk.scope, sk.key)
+          case None     => provider.get("", alias)
+        }
+        value match {
+          case Some(v) => java.util.regex.Matcher.quoteReplacement(v)
+          case None =>
+            throw new IllegalArgumentException(
+              s"Secret '{{secrets.$alias}}' could not be resolved. Set environment variable SECRETS__${alias.toUpperCase
+                  .replace("-", "_")} or configure a secrets provider."
+            )
+        }
+      }
+    )
+  }
+}

serializer/src/main/scala/com/eff3ct/teckel/serializer/SecretsProvider.scala

@@ -0,0 +1,40 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2024 Rafael Fernandez
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package com.eff3ct.teckel.serializer
+
+trait SecretsProvider {
+  def get(scope: String, key: String): Option[String]
+}
+
+object SecretsProvider {
+
+  object Environment extends SecretsProvider {
+    override def get(scope: String, key: String): Option[String] =
+      Option(System.getenv(s"SECRETS__${key.toUpperCase.replace("-", "_").replace(".", "_")}"))
+        .orElse(Option(System.getenv(key.toUpperCase.replace("-", "_").replace(".", "_"))))
+  }
+
+  val default: SecretsProvider = Environment
+}