concord-cli: support for remote secrets, configurable secrets providers

cli/src/main/java/com/walmartlabs/concord/cli/CliConfig.java

@@ -0,0 +1,160 @@
+package com.walmartlabs.concord.cli;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2025 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.Optional;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+public record CliConfig(Map<String, CliConfigContext> contexts) {
+
+    public static CliConfig load(Path path) throws IOException {
+        var mapper = new YAMLMapper();
+
+        JsonNode defaults = mapper.readTree(readDefaultConfig());
+
+        JsonNode cfg;
+        try (var reader = Files.newBufferedReader(path)) {
+            cfg = mapper.readTree(reader);
+        }
+
+        // merge the loaded config file with the default built-in config
+        var cfgWithDefaults = mapper.updateValue(defaults, cfg);
+
+        // merge each non-default context with the default context
+        var contexts = assertContexts(cfgWithDefaults);
+
+        var defaultCtx = contexts.get("default");
+        if (defaultCtx == null) {
+            throw new IllegalArgumentException("Missing 'default' context.");
+        }
+
+        contexts.fieldNames().forEachRemaining(ctxName -> {
+            if ("default".equals(ctxName)) {
+                return;
+            }
+
+            var ctx = contexts.get(ctxName);
+            try {
+                var mergedCtx = mapper.updateValue(defaultCtx, ctx);
+                contexts.set(ctxName, mergedCtx);
+            } catch (JsonMappingException e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+        return mapper.convertValue(cfgWithDefaults, CliConfig.class);
+    }
+
+    private static ObjectNode assertContexts(JsonNode cfg) {
+        var maybeContexts = cfg.get("contexts");
+        if (maybeContexts == null) {
+            throw new IllegalArgumentException("Missing 'contexts' object.");
+        }
+        if (!maybeContexts.isObject()) {
+            throw new IllegalArgumentException("The 'contexts' field must be an object.");
+        }
+        return (ObjectNode) maybeContexts;
+    }
+
+    public static CliConfig create() {
+        var mapper = new YAMLMapper();
+        try {
+            return mapper.readValue(readDefaultConfig(), CliConfig.class);
+        } catch (IOException e) {
+            throw new IllegalStateException("Can't parse the default CLI config file. " + e.getMessage());
+        }
+    }
+
+    public record Overrides(@Nullable Path secretStoreDir, @Nullable Path vaultDir, @Nullable String vaultId) {
+    }
+
+    public record CliConfigContext(SecretsConfiguration secrets) {
+
+        public CliConfigContext withOverrides(Overrides overrides) {
+            var secrets = this.secrets().withOverrides(overrides);
+            return new CliConfigContext(secrets);
+        }
+    }
+
+    public record SecretsConfiguration(VaultConfiguration vault,
+                                       FileSecretsProviderConfiguration local,
+                                       RemoteSecretsProviderConfiguration remote) {
+
+        public SecretsConfiguration withOverrides(Overrides overrides) {
+            var vault = this.vault().withOverrides(overrides);
+            var localFiles = this.local().withOverrides(overrides);
+            return new SecretsConfiguration(vault, localFiles, this.remote);
+        }
+
+        public record VaultConfiguration(Path dir, String id) {
+
+            public VaultConfiguration withOverrides(Overrides overrides) {
+                return new VaultConfiguration(
+                        Optional.ofNullable(overrides.vaultDir()).orElse(this.dir()),
+                        Optional.ofNullable(overrides.vaultId()).orElse(this.id()));
+            }
+        }
+
+        public record FileSecretsProviderConfiguration(boolean enabled, boolean writable, Path dir) {
+
+            public FileSecretsProviderConfiguration withOverrides(Overrides overrides) {
+                return new FileSecretsProviderConfiguration(
+                        this.enabled,
+                        this.writable,
+                        Optional.ofNullable(overrides.secretStoreDir()).orElse(this.dir()));
+            }
+        }
+
+        public record RemoteSecretsProviderConfiguration(boolean enabled,
+                                                         boolean writable,
+                                                         @Nullable String baseUrl,
+                                                         @Nullable String apiKey,
+                                                         boolean confirmAccess) {
+        }
+    }
+
+    private static String readDefaultConfig() {
+        try (var in = CliConfig.class.getResourceAsStream("defaultCliConfig.yaml")) {
+            if (in == null) {
+                throw new IllegalStateException("defaultCliConfig.yaml resource not found");
+            }
+            var ab = in.readAllBytes();
+            var s = new String(ab, UTF_8);
+
+            var dotConcordPath = Paths.get(System.getProperty("user.home")).resolve(".concord");
+            return s.replace("${configDir}", dotConcordPath.normalize().toAbsolutePath().toString());
+        } catch (IOException e) {
+            throw new IllegalStateException("Can't load the default CLI config file. " + e.getMessage());
+        }
+    }
+}

cli/src/main/java/com/walmartlabs/concord/cli/Run.java

@@ -21,9 +21,11 @@
  */
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
 import com.google.inject.Injector;
+import com.walmartlabs.concord.cli.CliConfig.CliConfigContext;
 import com.walmartlabs.concord.cli.runner.*;
 import com.walmartlabs.concord.common.ConfigurationUtils;
 import com.walmartlabs.concord.common.FileVisitor;
@@ -38,16 +40,20 @@
 import com.walmartlabs.concord.runtime.v2.NoopImportsNormalizer;
 import com.walmartlabs.concord.runtime.v2.ProjectLoaderV2;
 import com.walmartlabs.concord.runtime.v2.ProjectSerializerV2;
-import com.walmartlabs.concord.runtime.v2.model.*;
+import com.walmartlabs.concord.runtime.v2.model.Flow;
+import com.walmartlabs.concord.runtime.v2.model.ProcessDefinition;
+import com.walmartlabs.concord.runtime.v2.model.ProcessDefinitionConfiguration;
+import com.walmartlabs.concord.runtime.v2.model.Profile;
 import com.walmartlabs.concord.runtime.v2.runner.InjectorFactory;
 import com.walmartlabs.concord.runtime.v2.runner.Runner;
 import com.walmartlabs.concord.runtime.v2.runner.guice.ProcessDependenciesModule;
 import com.walmartlabs.concord.runtime.v2.runner.tasks.TaskProviders;
+import com.walmartlabs.concord.runtime.v2.runner.vm.ParallelExecutionException;
 import com.walmartlabs.concord.runtime.v2.sdk.*;
 import com.walmartlabs.concord.sdk.Constants;
 import com.walmartlabs.concord.sdk.MapUtils;
-import com.walmartlabs.concord.runtime.v2.runner.vm.ParallelExecutionException;
-import org.fusesource.jansi.Ansi;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Model.CommandSpec;
 import picocli.CommandLine.Option;
@@ -64,19 +70,21 @@
 import java.util.concurrent.Callable;
 import java.util.stream.Stream;
 
-import static org.fusesource.jansi.Ansi.Color.GREEN;
-import static org.fusesource.jansi.Ansi.Color.YELLOW;
 import static org.fusesource.jansi.Ansi.ansi;
 
 @Command(name = "run", description = "Run the current directory as a Concord process")
 public class Run implements Callable<Integer> {
 
+    private static final Logger log = LoggerFactory.getLogger(Run.class);
     @Spec
     private CommandSpec spec;
 
     @Option(names = {"-h", "--help"}, usageHelp = true, description = "display the command's help message")
     boolean helpRequested = false;
 
+    @Option(names = {"--context"}, description = "Configuration context to use")
+    String context = "default";
+
     @Option(names = {"-e", "--extra-vars"}, description = "additional process variables")
     Map<String, Object> extraVars = new LinkedHashMap<>();
 
@@ -93,13 +101,13 @@ public class Run implements Callable<Integer> {
     Path repoCacheDir = Paths.get(System.getProperty("user.home")).resolve(".concord").resolve("repoCache");
 
     @Option(names = {"--secret-dir"}, description = "secret store dir")
-    Path secretStoreDir = Paths.get(System.getProperty("user.home")).resolve(".concord").resolve("secrets");
+    Path secretStoreDir;
 
     @Option(names = {"--vault-dir"}, description = "vault dir")
-    Path vaultDir = Paths.get(System.getProperty("user.home")).resolve(".concord").resolve("vaults");
+    Path vaultDir;
 
     @Option(names = {"--vault-id"}, description = "vault id")
-    String vaultId = "default";
+    String vaultId;
 
     @Option(names = {"--imports-source"}, description = "default imports source")
     String importsSource = "https://github.com";
@@ -139,6 +147,8 @@ public class Run implements Callable<Integer> {
     public Integer call() throws Exception {
         Verbosity verbosity = new Verbosity(this.verbosity);
 
+        CliConfigContext cliConfigContext = loadCliConfig(verbosity, context);
+
         sourceDir = sourceDir.normalize().toAbsolutePath();
         Path targetDir;
 
@@ -274,7 +284,7 @@ public Integer call() throws Exception {
                 runnerCfg,
                 () -> cfg,
                 new ProcessDependenciesModule(targetDir, runnerCfg.dependencies(), cfg.debug()),
-                new CliServicesModule(secretStoreDir, targetDir, defaultTaskVars, new VaultProvider(vaultDir, vaultId), dependencyManager, verbosity))
+                new CliServicesModule(cliConfigContext, targetDir, defaultTaskVars, dependencyManager, verbosity))
                 .create();
 
         // Just to notify listeners
@@ -410,6 +420,64 @@ private DependencyManagerConfiguration getDependencyManagerConfiguration() {
         return DependencyManagerConfiguration.of(depsCacheDir);
     }
 
+    private CliConfigContext loadCliConfig(Verbosity verbosity, String context) {
+        CliConfig.Overrides overrides = new CliConfig.Overrides(secretStoreDir, vaultDir, vaultId);
+
+        Path baseDir = Paths.get(System.getProperty("user.home"), ".concord");
+        Path cfgFile = baseDir.resolve("cli.yaml");
+        if (!Files.exists(cfgFile)) {
+            cfgFile = baseDir.resolve("cli.yml");
+        }
+        if (!Files.exists(cfgFile)) {
+            CliConfig cfg = CliConfig.create();
+            return assertCliConfigContext(cfg, context).withOverrides(overrides);
+        }
+
+        if (verbosity.verbose()) {
+            log.info("Using CLI configuration file: {} (\"{}\" context)", cfgFile, context);
+        }
+
+        try {
+            CliConfig cfg = CliConfig.load(cfgFile);
+            return assertCliConfigContext(cfg, context).withOverrides(overrides);
+        } catch (Exception e) {
+            handleCliConfigErrorAndBail(cfgFile.toAbsolutePath().toString(), e);
+            return null;
+        }
+    }
+
+    private static void handleCliConfigErrorAndBail(String cfgPath, Throwable e) {
+        // unwrap runtime exceptions
+        if (e instanceof RuntimeException ex) {
+            if (ex.getCause() instanceof IllegalArgumentException) {
+                e = ex.getCause();
+            }
+        }
+
+        // handle YAML errors
+        if (e instanceof IllegalArgumentException) {
+            if (e.getCause() instanceof UnrecognizedPropertyException ex) {
+                System.out.println(ansi().fgRed().a("Invalid format of the CLI configuration file ").a(cfgPath).a(". ").a(ex.getMessage()));
+                System.exit(1);
+            }
+            System.out.println(ansi().fgRed().a("Invalid format of the CLI configuration file ").a(cfgPath).a(". ").a(e.getMessage()));
+            System.exit(1);
+        }
+
+        // all other errors
+        System.out.println(ansi().fgRed().a("Failed to read the CLI configuration file ").a(cfgPath).a(". ").a(e.getMessage()));
+        System.exit(1);
+    }
+
+    private static CliConfigContext assertCliConfigContext(CliConfig config, String context) {
+        CliConfigContext result = config.contexts().get(context);
+        if (result == null) {
+            System.out.println(ansi().fgRed().a("Configuration context not found: ").a(context).a(". Check the CLI configuration file."));
+            System.exit(1);
+        }
+        return result;
+    }
+
     private static void dumpArguments(Map<String, Object> args) {
         ObjectMapper om = new ObjectMapper(new YAMLFactory().disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER));
         try {
@@ -446,7 +514,7 @@ public void visit(Path sourceFile, Path dstFile) {
             }
 
             if (currentCount == notifyOnCount) {
-                System.out.println(ansi().fgBrightBlack().a("Copying files into the target directory..."));
+                System.out.println(ansi().fgBrightBlack().a("Copying files into ./target/ directory..."));
                 currentCount = -1;
                 return;
             }