runtime-v2, concord-cli: add support for resources.concordCli

cli/src/main/java/com/walmartlabs/concord/cli/Run.java

@@ -28,6 +28,7 @@
 import com.walmartlabs.concord.common.ConfigurationUtils;
 import com.walmartlabs.concord.common.FileVisitor;
 import com.walmartlabs.concord.common.IOUtils;
+import com.walmartlabs.concord.common.ResourceUtils;
 import com.walmartlabs.concord.dependencymanager.DependencyManager;
 import com.walmartlabs.concord.dependencymanager.DependencyManagerConfiguration;
 import com.walmartlabs.concord.dependencymanager.DependencyManagerRepositories;
@@ -39,6 +40,7 @@
 import com.walmartlabs.concord.runtime.v2.ProjectLoaderV2;
 import com.walmartlabs.concord.runtime.v2.ProjectSerializerV2;
 import com.walmartlabs.concord.runtime.v2.model.*;
+import com.walmartlabs.concord.runtime.v2.model.Resources.ConcordCliResources;
 import com.walmartlabs.concord.runtime.v2.runner.InjectorFactory;
 import com.walmartlabs.concord.runtime.v2.runner.Runner;
 import com.walmartlabs.concord.runtime.v2.runner.guice.ProcessDependenciesModule;
@@ -63,6 +65,8 @@
 import java.util.concurrent.Callable;
 import java.util.stream.Stream;
 
+import static com.walmartlabs.concord.runtime.v2.model.Resources.DEFAULT_CONCORD_CLI_INCLUDES;
+
 @Command(name = "run", description = "Run the current directory as a Concord process")
 public class Run implements Callable<Integer> {
 
@@ -145,6 +149,11 @@ public Integer call() throws Exception {
 
             Files.copy(src, targetDir.resolve("concord.yml"), StandardCopyOption.REPLACE_EXISTING);
         } else if (Files.isDirectory(sourceDir)) {
+            ConcordCliResources resources = getConcordCliResources(sourceDir);
+            if (verbosity.verbose()) {
+                System.out.println("Using CLI resources:\n\t" + resources);
+            }
+
             targetDir = sourceDir.resolve("target");
             if (cleanup && Files.exists(targetDir)) {
                 if (verbosity.verbose()) {
@@ -154,7 +163,8 @@ public Integer call() throws Exception {
             }
 
             // copy everything into target except target
-            IOUtils.copy(sourceDir, targetDir, "^target$", new CopyNotifier(verbosity.verbose() ? 0 : 100), StandardCopyOption.REPLACE_EXISTING);
+            CopyNotifier notifier = new CopyNotifier(verbosity.verbose() ? 0 : 100);
+            ResourceUtils.copyResources(sourceDir, resources.includes(), resources.excludes(), targetDir, notifier, StandardCopyOption.REPLACE_EXISTING);
         } else {
             throw new IllegalArgumentException("Not a directory or single Concord YAML file: " + sourceDir);
         }
@@ -296,6 +306,26 @@ public Integer call() throws Exception {
         return 0;
     }
 
+    private static ConcordCliResources getConcordCliResources(Path sourceDir) {
+        // try to load the root concord.y?ml and get the concordCli resources config from there
+
+        Path rootConcordFile = sourceDir.resolve("concord.yml");
+        if (!Files.exists(rootConcordFile)) {
+            rootConcordFile = sourceDir.resolve("concord.yaml");
+        }
+        if (!Files.exists(rootConcordFile)) {
+            return ImmutableConcordCliResources.builder().build();
+        }
+
+        try {
+            ProjectLoaderV2.Result result = new ProjectLoaderV2(new NoopImportManager())
+                    .loadFromFile(rootConcordFile);
+            return result.getProjectDefinition().resources().concordCli();
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to load root Concord file from " + rootConcordFile, e);
+        }
+    }
+
     @SuppressWarnings("unchecked")
     private static ProcessInfo processInfo(Map<String, Object> args, List<String> profiles) {
         Object processInfoObject = args.get("processInfo");

common/src/main/java/com/walmartlabs/concord/common/ResourceUtils.java

@@ -0,0 +1,154 @@
+package com.walmartlabs.concord.common;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2025 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import com.walmartlabs.concord.sdk.Constants;
+
+import java.io.IOException;
+import java.nio.file.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public final class ResourceUtils {
+
+    public static void copyResources(Path baseDir, List<String> includePatterns, Path destDir, CopyOption... options) throws IOException {
+        copyResources(baseDir, includePatterns, List.of(), destDir, null, options);
+    }
+
+    /**
+     * Copies Concord resources from baseDir to destDir.
+     * @param baseDir source
+     * @param includePatterns list of paths (glob, regexes, file paths) to include
+     * @param excludePatterns list of paths (glob, regexes, file paths) to exclude
+     * @param destDir destination
+     * @param visitor a FileVisitor to apply after each copied file (can be null)
+     * @param options array of CopyOptions
+     * @throws IOException
+     */
+    public static void copyResources(Path baseDir, List<String> includePatterns, List<String> excludePatterns, Path destDir, FileVisitor visitor, CopyOption... options) throws IOException {
+        var paths = findResources(baseDir, includePatterns, excludePatterns);
+        for (var fileName : Constants.Files.PROJECT_ROOT_FILE_NAMES) {
+            var p = baseDir.resolve(fileName);
+            paths.add(p);
+        }
+        copy(paths, baseDir, destDir, visitor, options);
+    }
+
+    public static List<Path> findResources(Path baseDir, List<String> includePatterns) throws IOException {
+        return findResources(baseDir, includePatterns, List.of());
+    }
+
+    /**
+     * Finds paths in baseDir that match includePatterns and do not match excludePatterns.
+     * @param baseDir source
+     * @param includePatterns list of paths (glob, regexes, file paths) to include
+     * @param excludePatterns list of paths (glob, regexes, file paths) to exclude
+     * @return list of absolute paths
+     * @throws IOException
+     */
+    public static List<Path> findResources(Path baseDir, List<String> includePatterns, List<String> excludePatterns) throws IOException {
+        var result = new ArrayList<Path>();
+
+        var includeMatchers = includePatterns.stream().map(p -> parsePattern(baseDir, p)).toList();
+        var excludeMatchers = excludePatterns.stream().map(p -> parsePattern(baseDir, p)).toList();
+
+        try (var walker = Files.walk(baseDir)) {
+            walker.filter(candidate -> !Files.isDirectory(candidate))
+                    .filter(candidate ->
+                            includeMatchers.stream().anyMatch(m -> m.matches(candidate)) &&
+                            excludeMatchers.stream().noneMatch(pattern -> pattern.matches(candidate)))
+                    .forEach(result::add);
+        }
+
+        return result;
+    }
+
+    private static void copy(Collection<Path> files, Path baseDir, Path dest, FileVisitor visitor, CopyOption... options) throws IOException {
+        for (var f : files) {
+            if (Files.notExists(f)) {
+                continue;
+            }
+
+            var src = baseDir.relativize(f);
+            var dst = dest.resolve(src);
+
+            var dstParent = dst.getParent();
+            if (dstParent != null && Files.notExists(dstParent)) {
+                Files.createDirectories(dstParent);
+            }
+
+            if (Files.isSymbolicLink(f)) {
+                Path link = Files.readSymbolicLink(f);
+                Path target = f.getParent().resolve(link).normalize();
+
+                if (!target.startsWith(baseDir)) {
+                    throw new IOException("Symlinks outside the base directory are not supported: " + baseDir + " -> " + target);
+                }
+
+                if (Files.notExists(target)) {
+                    continue;
+                }
+
+                Files.createSymbolicLink(dst, link);
+            } else {
+                Files.copy(f, dst, options);
+            }
+
+            if (visitor != null) {
+                visitor.visit(src, dst);
+            }
+        }
+    }
+
+    static PathMatcher parsePattern(Path baseDir, String pattern) {
+        pattern = pattern.trim();
+
+        String normalizedPattern;
+        if (pattern.startsWith("glob:")) {
+            normalizedPattern = "glob:" + concat(baseDir, pattern.substring("glob:".length()));
+        } else if (pattern.startsWith("regex:")) {
+            normalizedPattern = "regex:" + concat(baseDir, pattern.substring("regex:".length()));
+        } else {
+            normalizedPattern = "glob:" + concat(baseDir, pattern
+                    .replace("*", "\\*")
+                    .replace("{", "\\{")
+                    .replace("}", "\\}")
+                    .replace("[", "\\[")
+                    .replace("]", "\\]")
+                    .replace("!", "\\!")
+                    .replace("?", "\\?"));
+        }
+
+        return FileSystems.getDefault().getPathMatcher(normalizedPattern);
+    }
+
+    private static String concat(Path path, String str) {
+        var separator = "/";
+        if (str.startsWith("/")) {
+            separator = "";
+        }
+        return path.toAbsolutePath() + separator + str;
+    }
+
+    private ResourceUtils() {
+    }
+}

common/src/test/java/com/walmartlabs/concord/common/ResourceUtilTest.java

@@ -0,0 +1,54 @@
+package com.walmartlabs.concord.common;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2025 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.Paths;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class ResourceUtilTest {
+
+    @Test
+    public void testParsePattern() {
+        var baseDir = Paths.get("/tmp/foo");
+
+        var pattern = ResourceUtils.parsePattern(baseDir, "glob:*.txt");
+        assertTrue(pattern.matches(Paths.get("/tmp/foo/bar.txt")));
+        assertTrue(pattern.matches(Paths.get("/tmp/foo/bar!.txt")));
+        assertFalse(pattern.matches(Paths.get("/tmp/foo/baz.tmp")));
+
+        pattern = ResourceUtils.parsePattern(baseDir, "glob:concord/{**/,}{*.,}concord.{yml,yaml}");
+        assertTrue(pattern.matches(Paths.get("/tmp/foo/concord/foo.concord.yml")));
+        assertTrue(pattern.matches(Paths.get("/tmp/foo/concord/bar.concord.yaml")));
+        assertFalse(pattern.matches(Paths.get("/tmp/foo/qux.concord.yaml")));
+
+        pattern = ResourceUtils.parsePattern(baseDir, "regex:.*\\.qux");
+        assertTrue(pattern.matches(Paths.get("/tmp/foo/bar.qux")));
+        assertFalse(pattern.matches(Paths.get("/tmp/foo/qux")));
+
+        pattern = ResourceUtils.parsePattern(baseDir, "baz.txt");
+        assertTrue(pattern.matches(Paths.get("/tmp/foo/baz.txt")));
+        assertFalse(pattern.matches(Paths.get("/tmp/foo/qux.txt")));
+    }
+}