Skip to content

Commit ec10218

Browse files
ibodrovibodrov
authored
concord-cli: support for remote secrets, configurable secrets providers (#1143)
1 parent 0a4d5d5 commit ec10218

File tree

15 files changed

+1057
-202
lines changed

15 files changed

+1057
-202
lines changed

‎cli/src/main/java/com/walmartlabs/concord/cli/CliConfig.java‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
package com.walmartlabs.concord.cli;
2+
3+
/*-
4+
* *****
5+
* Concord
6+
* -----
7+
* Copyright (C) 2017 - 2025 Walmart Inc.
8+
* -----
9+
* Licensed under the Apache License, Version 2.0 (the "License");
10+
* you may not use this file except in compliance with the License.
11+
* You may obtain a copy of the License at
12+
*
13+
* http://www.apache.org/licenses/LICENSE-2.0
14+
*
15+
* Unless required by applicable law or agreed to in writing, software
16+
* distributed under the License is distributed on an "AS IS" BASIS,
17+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18+
* See the License for the specific language governing permissions and
19+
* limitations under the License.
20+
* =====
21+
*/
22+
23+
import com.fasterxml.jackson.databind.JsonMappingException;
24+
import com.fasterxml.jackson.databind.JsonNode;
25+
import com.fasterxml.jackson.databind.node.ObjectNode;
26+
import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
27+
28+
import javax.annotation.Nullable;
29+
import java.io.IOException;
30+
import java.nio.file.Files;
31+
import java.nio.file.Path;
32+
import java.nio.file.Paths;
33+
import java.util.Map;
34+
import java.util.Optional;
35+
36+
import static java.nio.charset.StandardCharsets.UTF_8;
37+
38+
public record CliConfig(Map<String, CliConfigContext> contexts) {
39+
40+
public static CliConfig load(Path path) throws IOException {
41+
var mapper = new YAMLMapper();
42+
43+
JsonNode defaults = mapper.readTree(readDefaultConfig());
44+
45+
JsonNode cfg;
46+
try (var reader = Files.newBufferedReader(path)) {
47+
cfg = mapper.readTree(reader);
48+
}
49+
50+
// merge the loaded config file with the default built-in config
51+
var cfgWithDefaults = mapper.updateValue(defaults, cfg);
52+
53+
// merge each non-default context with the default context
54+
var contexts = assertContexts(cfgWithDefaults);
55+
56+
var defaultCtx = contexts.get("default");
57+
if (defaultCtx == null) {
58+
throw new IllegalArgumentException("Missing 'default' context.");
59+
}
60+
61+
contexts.fieldNames().forEachRemaining(ctxName -> {
62+
if ("default".equals(ctxName)) {
63+
return;
64+
}
65+
66+
var ctx = contexts.get(ctxName);
67+
try {
68+
var mergedCtx = mapper.updateValue(defaultCtx, ctx);
69+
contexts.set(ctxName, mergedCtx);
70+
} catch (JsonMappingException e) {
71+
throw new RuntimeException(e);
72+
}
73+
});
74+
75+
return mapper.convertValue(cfgWithDefaults, CliConfig.class);
76+
}
77+
78+
private static ObjectNode assertContexts(JsonNode cfg) {
79+
var maybeContexts = cfg.get("contexts");
80+
if (maybeContexts == null) {
81+
throw new IllegalArgumentException("Missing 'contexts' object.");
82+
}
83+
if (!maybeContexts.isObject()) {
84+
throw new IllegalArgumentException("The 'contexts' field must be an object.");
85+
}
86+
return (ObjectNode) maybeContexts;
87+
}
88+
89+
public static CliConfig create() {
90+
var mapper = new YAMLMapper();
91+
try {
92+
return mapper.readValue(readDefaultConfig(), CliConfig.class);
93+
} catch (IOException e) {
94+
throw new IllegalStateException("Can't parse the default CLI config file. " + e.getMessage());
95+
}
96+
}
97+
98+
public record Overrides(@Nullable Path secretStoreDir, @Nullable Path vaultDir, @Nullable String vaultId) {
99+
}
100+
101+
public record CliConfigContext(SecretsConfiguration secrets) {
102+
103+
public CliConfigContext withOverrides(Overrides overrides) {
104+
var secrets = this.secrets().withOverrides(overrides);
105+
return new CliConfigContext(secrets);
106+
}
107+
}
108+
109+
public record SecretsConfiguration(VaultConfiguration vault,
110+
FileSecretsProviderConfiguration local,
111+
RemoteSecretsProviderConfiguration remote) {
112+
113+
public SecretsConfiguration withOverrides(Overrides overrides) {
114+
var vault = this.vault().withOverrides(overrides);
115+
var localFiles = this.local().withOverrides(overrides);
116+
return new SecretsConfiguration(vault, localFiles, this.remote);
117+
}
118+
119+
public record VaultConfiguration(Path dir, String id) {
120+
121+
public VaultConfiguration withOverrides(Overrides overrides) {
122+
return new VaultConfiguration(
123+
Optional.ofNullable(overrides.vaultDir()).orElse(this.dir()),
124+
Optional.ofNullable(overrides.vaultId()).orElse(this.id()));
125+
}
126+
}
127+
128+
public record FileSecretsProviderConfiguration(boolean enabled, boolean writable, Path dir) {
129+
130+
public FileSecretsProviderConfiguration withOverrides(Overrides overrides) {
131+
return new FileSecretsProviderConfiguration(
132+
this.enabled,
133+
this.writable,
134+
Optional.ofNullable(overrides.secretStoreDir()).orElse(this.dir()));
135+
}
136+
}
137+
138+
public record RemoteSecretsProviderConfiguration(boolean enabled,
139+
boolean writable,
140+
@Nullable String baseUrl,
141+
@Nullable String apiKey,
142+
boolean confirmAccess) {
143+
}
144+
}
145+
146+
private static String readDefaultConfig() {
147+
try (var in = CliConfig.class.getResourceAsStream("defaultCliConfig.yaml")) {
148+
if (in == null) {
149+
throw new IllegalStateException("defaultCliConfig.yaml resource not found");
150+
}
151+
var ab = in.readAllBytes();
152+
var s = new String(ab, UTF_8);
153+
154+
var dotConcordPath = Paths.get(System.getProperty("user.home")).resolve(".concord");
155+
return s.replace("${configDir}", dotConcordPath.normalize().toAbsolutePath().toString());
156+
} catch (IOException e) {
157+
throw new IllegalStateException("Can't load the default CLI config file. " + e.getMessage());
158+
}
159+
}
160+
}

‎cli/src/main/java/com/walmartlabs/concord/cli/Run.java‎

Lines changed: 73 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,11 @@
2121
*/
2222

2323
import com.fasterxml.jackson.databind.ObjectMapper;
24+
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
2425
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
2526
import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
2627
import com.google.inject.Injector;
28+
import com.walmartlabs.concord.cli.CliConfig.CliConfigContext;
2729
import com.walmartlabs.concord.cli.runner.*;
2830
import com.walmartlabs.concord.common.ConfigurationUtils;
2931
import com.walmartlabs.concord.common.FileVisitor;
@@ -50,6 +52,8 @@
5052
import com.walmartlabs.concord.runtime.v2.sdk.*;
5153
import com.walmartlabs.concord.sdk.Constants;
5254
import com.walmartlabs.concord.sdk.MapUtils;
55+
import org.slf4j.Logger;
56+
import org.slf4j.LoggerFactory;
5357
import picocli.CommandLine.Command;
5458
import picocli.CommandLine.Model.CommandSpec;
5559
import picocli.CommandLine.Option;
@@ -71,12 +75,16 @@
7175
@Command(name = "run", description = "Run the current directory as a Concord process")
7276
public class Run implements Callable<Integer> {
7377

78+
private static final Logger log = LoggerFactory.getLogger(Run.class);
7479
@Spec
7580
private CommandSpec spec;
7681

7782
@Option(names = {"-h", "--help"}, usageHelp = true, description = "display the command's help message")
7883
boolean helpRequested = false;
7984

85+
@Option(names = {"--context"}, description = "Configuration context to use")
86+
String context = "default";
87+
8088
@Option(names = {"-e", "--extra-vars"}, description = "additional process variables")
8189
Map<String, Object> extraVars = new LinkedHashMap<>();
8290

@@ -93,13 +101,13 @@ public class Run implements Callable<Integer> {
93101
Path repoCacheDir = Paths.get(System.getProperty("user.home")).resolve(".concord").resolve("repoCache");
94102

95103
@Option(names = {"--secret-dir"}, description = "secret store dir")
96-
Path secretStoreDir = Paths.get(System.getProperty("user.home")).resolve(".concord").resolve("secrets");
104+
Path secretStoreDir;
97105

98106
@Option(names = {"--vault-dir"}, description = "vault dir")
99-
Path vaultDir = Paths.get(System.getProperty("user.home")).resolve(".concord").resolve("vaults");
107+
Path vaultDir;
100108

101109
@Option(names = {"--vault-id"}, description = "vault id")
102-
String vaultId = "default";
110+
String vaultId;
103111

104112
@Option(names = {"--imports-source"}, description = "default imports source")
105113
String importsSource = "https://github.com";
@@ -139,6 +147,8 @@ public class Run implements Callable<Integer> {
139147
public Integer call() throws Exception {
140148
Verbosity verbosity = new Verbosity(this.verbosity);
141149

150+
CliConfigContext cliConfigContext = loadCliConfig(verbosity, context);
151+
142152
sourceDir = sourceDir.normalize().toAbsolutePath();
143153
Path targetDir;
144154

@@ -274,7 +284,7 @@ public Integer call() throws Exception {
274284
runnerCfg,
275285
() -> cfg,
276286
new ProcessDependenciesModule(targetDir, runnerCfg.dependencies(), cfg.debug()),
277-
new CliServicesModule(secretStoreDir, targetDir, defaultTaskVars, new VaultProvider(vaultDir, vaultId), dependencyManager, verbosity))
287+
new CliServicesModule(cliConfigContext, targetDir, defaultTaskVars, dependencyManager, verbosity))
278288
.create();
279289

280290
// Just to notify listeners
@@ -410,6 +420,64 @@ private DependencyManagerConfiguration getDependencyManagerConfiguration() {
410420
return DependencyManagerConfiguration.of(depsCacheDir);
411421
}
412422

423+
private CliConfigContext loadCliConfig(Verbosity verbosity, String context) {
424+
CliConfig.Overrides overrides = new CliConfig.Overrides(secretStoreDir, vaultDir, vaultId);
425+
426+
Path baseDir = Paths.get(System.getProperty("user.home"), ".concord");
427+
Path cfgFile = baseDir.resolve("cli.yaml");
428+
if (!Files.exists(cfgFile)) {
429+
cfgFile = baseDir.resolve("cli.yml");
430+
}
431+
if (!Files.exists(cfgFile)) {
432+
CliConfig cfg = CliConfig.create();
433+
return assertCliConfigContext(cfg, context).withOverrides(overrides);
434+
}
435+
436+
if (verbosity.verbose()) {
437+
log.info("Using CLI configuration file: {} (\"{}\" context)", cfgFile, context);
438+
}
439+
440+
try {
441+
CliConfig cfg = CliConfig.load(cfgFile);
442+
return assertCliConfigContext(cfg, context).withOverrides(overrides);
443+
} catch (Exception e) {
444+
handleCliConfigErrorAndBail(cfgFile.toAbsolutePath().toString(), e);
445+
return null;
446+
}
447+
}
448+
449+
private static void handleCliConfigErrorAndBail(String cfgPath, Throwable e) {
450+
// unwrap runtime exceptions
451+
if (e instanceof RuntimeException ex) {
452+
if (ex.getCause() instanceof IllegalArgumentException) {
453+
e = ex.getCause();
454+
}
455+
}
456+
457+
// handle YAML errors
458+
if (e instanceof IllegalArgumentException) {
459+
if (e.getCause() instanceof UnrecognizedPropertyException ex) {
460+
System.out.println(ansi().fgRed().a("Invalid format of the CLI configuration file ").a(cfgPath).a(". ").a(ex.getMessage()));
461+
System.exit(1);
462+
}
463+
System.out.println(ansi().fgRed().a("Invalid format of the CLI configuration file ").a(cfgPath).a(". ").a(e.getMessage()));
464+
System.exit(1);
465+
}
466+
467+
// all other errors
468+
System.out.println(ansi().fgRed().a("Failed to read the CLI configuration file ").a(cfgPath).a(". ").a(e.getMessage()));
469+
System.exit(1);
470+
}
471+
472+
private static CliConfigContext assertCliConfigContext(CliConfig config, String context) {
473+
CliConfigContext result = config.contexts().get(context);
474+
if (result == null) {
475+
System.out.println(ansi().fgRed().a("Configuration context not found: ").a(context).a(". Check the CLI configuration file."));
476+
System.exit(1);
477+
}
478+
return result;
479+
}
480+
413481
private static void dumpArguments(Map<String, Object> args) {
414482
ObjectMapper om = new ObjectMapper(new YAMLFactory().disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER));
415483
try {
@@ -446,7 +514,7 @@ public void visit(Path sourceFile, Path dstFile) {
446514
}
447515

448516
if (currentCount == notifyOnCount) {
449-
System.out.println(ansi().fgBrightBlack().a("Copying files into the target directory..."));
517+
System.out.println(ansi().fgBrightBlack().a("Copying files into ./target/ directory..."));
450518
currentCount = -1;
451519
return;
452520
}

0 commit comments

Comments
 (0)