From 605e4e3dad411c9a84edf7e49e4604db596b0e7e Mon Sep 17 00:00:00 2001 From: amithkb Date: Mon, 21 Apr 2025 08:20:48 +0530 Subject: [PATCH 1/4] filter to state downloads --- .../server/process/StateManagerUtils.java | 77 ++++++++++++++++ .../process/state/ProcessStateManager.java | 6 +- .../server/process/StateManagerUtilsTest.java | 90 +++++++++++++++++++ 3 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 server/impl/src/main/java/com/walmartlabs/concord/server/process/StateManagerUtils.java create mode 100644 server/impl/src/test/java/com/walmartlabs/concord/server/process/StateManagerUtilsTest.java diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/process/StateManagerUtils.java b/server/impl/src/main/java/com/walmartlabs/concord/server/process/StateManagerUtils.java new file mode 100644 index 0000000000..3f781058b3 --- /dev/null +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/process/StateManagerUtils.java @@ -0,0 +1,77 @@ +package com.walmartlabs.concord.server.process; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.walmartlabs.concord.server.security.Roles; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.walmartlabs.concord.sdk.Constants.Files.CONFIGURATION_FILE_NAME; + +public final class StateManagerUtils { + + private static final Map> STATE_FILTER = Map.of(CONFIGURATION_FILE_NAME, List.of("arguments")); + private static final List ALLOWED_EXTENSIONS = List.of("json", "yaml", "yml"); + + public static InputStream stateFilter(String file, InputStream in) { + // do not apply filter for admins + if (Roles.isAdmin()) { + return in; + } + + return filter(file, in); + } + + public static InputStream filter(String file, InputStream in) { + try { + String extension = getFileExtension(file); + if (!ALLOWED_EXTENSIONS.contains(extension) || STATE_FILTER.get(file) == null) { + // only filter for allowed extension files + return in; + } + + byte[] inputBytes = in.readAllBytes(); + if (inputBytes.length == 0) { + return new ByteArrayInputStream(inputBytes); + } + + Map map = switch (extension) { + case "json" -> new ObjectMapper().readValue(inputBytes, Map.class); + case "yaml", "yml" -> new ObjectMapper(new YAMLFactory()).readValue(inputBytes, Map.class); + default -> null; + }; + + if (map == null) { + return new ByteArrayInputStream(inputBytes); + } + + Map filteredMap = STATE_FILTER.get(file).stream() + .filter(map::containsKey) + .collect(HashMap::new, (m, key) -> m.put(key, map.get(key)), HashMap::putAll); + + byte[] data = switch (extension) { + case "json" -> new ObjectMapper().writeValueAsBytes(filteredMap); + case "yaml", "yml" -> new ObjectMapper(new YAMLFactory()).writeValueAsBytes(filteredMap); + default -> throw new IllegalArgumentException("Unsupported file extension: " + extension); + }; + + return new ByteArrayInputStream(data); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static String getFileExtension(String fileName) { + if (fileName.lastIndexOf(".") != -1 && fileName.lastIndexOf(".") != 0) + return fileName.substring(fileName.lastIndexOf(".") + 1); + else return ""; + } + + private StateManagerUtils() { + } +} diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/process/state/ProcessStateManager.java b/server/impl/src/main/java/com/walmartlabs/concord/server/process/state/ProcessStateManager.java index 61892e3013..83050dcaca 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/process/state/ProcessStateManager.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/process/state/ProcessStateManager.java @@ -34,6 +34,7 @@ import com.walmartlabs.concord.server.cfg.SecretStoreConfiguration; import com.walmartlabs.concord.server.policy.PolicyException; import com.walmartlabs.concord.server.policy.PolicyManager; +import com.walmartlabs.concord.server.process.StateManagerUtils; import com.walmartlabs.concord.server.process.logs.ProcessLogManager; import com.walmartlabs.concord.server.sdk.PartialProcessKey; import com.walmartlabs.concord.server.sdk.ProcessKey; @@ -423,8 +424,9 @@ public boolean export(ProcessKey processKey, ItemConsumer consumer) { int unixMode = rs.getInt(2); boolean encrypted = rs.getBoolean(3); try (InputStream in = rs.getBinaryStream(4); - InputStream processed = encrypted ? decrypt(in) : in) { - consumer.accept(n, unixMode, processed); + InputStream processed = encrypted ? decrypt(in) : in; + InputStream filtered = StateManagerUtils.stateFilter(n, processed)) { + consumer.accept(n, unixMode, filtered); } } } diff --git a/server/impl/src/test/java/com/walmartlabs/concord/server/process/StateManagerUtilsTest.java b/server/impl/src/test/java/com/walmartlabs/concord/server/process/StateManagerUtilsTest.java new file mode 100644 index 0000000000..91a330baab --- /dev/null +++ b/server/impl/src/test/java/com/walmartlabs/concord/server/process/StateManagerUtilsTest.java @@ -0,0 +1,90 @@ +package com.walmartlabs.concord.server.process; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +public class StateManagerUtilsTest { + + @Test + void testFilterWithJsonFile() throws Exception { + String jsonInput = "{\"arguments\": \"value\", \"otherKey\": \"otherValue\"}"; + InputStream inputStream = new ByteArrayInputStream(jsonInput.getBytes(StandardCharsets.UTF_8)); + + InputStream result = StateManagerUtils.filter("_main.json", inputStream); + String text = new String(result.readAllBytes(), StandardCharsets.UTF_8); + + ObjectMapper mapper = new ObjectMapper(); + Map resultMap = mapper.readValue(text, Map.class); + + assertEquals(1, resultMap.size()); + assertEquals("value", resultMap.get("arguments")); + } + + @Test + void testFilterWithYamlFile() throws Exception { + String yamlInput = "arguments: value\notherKey: otherValue"; + InputStream inputStream = new ByteArrayInputStream(yamlInput.getBytes(StandardCharsets.UTF_8)); + + InputStream result = StateManagerUtils.filter("test.yaml", inputStream); + + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + Map resultMap = mapper.readValue(result, Map.class); + + // return the same if to filter items not present + assertEquals(2, resultMap.size()); + assertEquals("otherValue", resultMap.get("otherKey")); + } + + @Test + void testFilterWithUnsupportedExtension() throws Exception { + String input = "key: value"; + InputStream inputStream = new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8)); + + InputStream result = StateManagerUtils.filter("file.txt", inputStream); + String text = new String(result.readAllBytes(), StandardCharsets.UTF_8); + + assertEquals(input, text); + } + + // Test case for empty input + @Test + void testFilterWithEmptyInput() throws Exception { + String input = ""; + InputStream inputStream = new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8)); + + InputStream result = StateManagerUtils.filter("_main.json", inputStream); + String text = new String(result.readAllBytes(), StandardCharsets.UTF_8); + + assertEquals(input, text); + } + + // Test case for null filtering rules + @Test + void testFilterWithNullRules() throws Exception { + String jsonInput = "{\"key\": \"value\"}"; + InputStream inputStream = new ByteArrayInputStream(jsonInput.getBytes(StandardCharsets.UTF_8)); + + InputStream result = StateManagerUtils.filter("unknown.json", inputStream); + String text = new String(result.readAllBytes(), StandardCharsets.UTF_8); + + assertEquals(jsonInput, text); + } + + // Test case for null input + @Test + void testFilterWithNullInput() throws Exception { + String jsonInput = "{\"key\": \"value\"}"; + InputStream inputStream = null; + InputStream result = StateManagerUtils.filter("unknown.json", inputStream); + + assertNull(result); + } +} From 1e908c52ca336fc728999e723b389d7a98926ad2 Mon Sep 17 00:00:00 2001 From: amithkb Date: Mon, 21 Apr 2025 09:20:57 +0530 Subject: [PATCH 2/4] change to filter only before zip --- .../server/process/state/ProcessStateManager.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/process/state/ProcessStateManager.java b/server/impl/src/main/java/com/walmartlabs/concord/server/process/state/ProcessStateManager.java index 83050dcaca..7bb31502c1 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/process/state/ProcessStateManager.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/process/state/ProcessStateManager.java @@ -424,9 +424,8 @@ public boolean export(ProcessKey processKey, ItemConsumer consumer) { int unixMode = rs.getInt(2); boolean encrypted = rs.getBoolean(3); try (InputStream in = rs.getBinaryStream(4); - InputStream processed = encrypted ? decrypt(in) : in; - InputStream filtered = StateManagerUtils.stateFilter(n, processed)) { - consumer.accept(n, unixMode, filtered); + InputStream processed = encrypted ? decrypt(in) : in) { + consumer.accept(n, unixMode, processed); } } } @@ -803,10 +802,11 @@ private ZipConsumer(ZipArchiveOutputStream dst) { public void accept(String name, int unixMode, InputStream src) { ZipArchiveEntry entry = new ZipArchiveEntry(name); entry.setUnixMode(unixMode); - + // filter before zip to download + InputStream filtered = StateManagerUtils.stateFilter(name, src); try { dst.putArchiveEntry(entry); - IOUtils.copy(src, dst); + IOUtils.copy(filtered, dst); dst.closeArchiveEntry(); } catch (IOException e) { throw new RuntimeException(e); From 91f1ce890b2b202898e88abf9a598dcc47ae0dd5 Mon Sep 17 00:00:00 2001 From: amithkb Date: Mon, 21 Apr 2025 15:58:17 +0530 Subject: [PATCH 3/4] up --- .../server/process/ProcessResource.java | 21 +++++++++++++++++-- .../server/process/StateManagerUtils.java | 10 --------- .../process/state/ProcessStateManager.java | 12 ++++++----- .../server/process/StateManagerUtilsTest.java | 14 ++++++------- 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/process/ProcessResource.java b/server/impl/src/main/java/com/walmartlabs/concord/server/process/ProcessResource.java index 98725d417f..2e1a7a5031 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/process/ProcessResource.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/process/ProcessResource.java @@ -747,15 +747,17 @@ public void appendLog(@PathParam("id") UUID instanceId, InputStream data) { content = @Content(mediaType = "application/zip", schema = @Schema(type = "string", format = "binary")) ) - public Response downloadState(@PathParam("id") UUID instanceId) { + public Response downloadState(@PathParam("id") UUID instanceId, @Context HttpServletRequest request) { ProcessEntry entry = assertProcess(PartialProcessKey.from(instanceId)); ProcessKey processKey = new ProcessKey(entry.instanceId(), entry.createdAt()); assertProcessAccess(entry, "state"); + boolean applyFilter = assertApplyFilter(request); + StreamingOutput out = output -> { try (ZipArchiveOutputStream dst = new ZipArchiveOutputStream(output)) { - stateManager.export(processKey, new ProcessStateManager.FilteringConsumer(zipTo(dst), s -> { + stateManager.export(processKey, new ProcessStateManager.FilteringConsumer(zipTo(dst, applyFilter), s -> { if (!isSessionResource(s)) { return true; } @@ -769,6 +771,21 @@ public Response downloadState(@PathParam("id") UUID instanceId) { .build(); } + private boolean assertApplyFilter(HttpServletRequest request) { + // do not filter request from admins + if (Roles.isAdmin()) { + return false; + } + + // do not filter request from agent + String userAgent = request.getHeader("User-Agent"); + if (userAgent.startsWith("Concord-Agent")) { + return false; + } + + return true; + } + /** * Downloads a single file from the current state snapshot of a process. */ diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/process/StateManagerUtils.java b/server/impl/src/main/java/com/walmartlabs/concord/server/process/StateManagerUtils.java index 3f781058b3..5fce5484cb 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/process/StateManagerUtils.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/process/StateManagerUtils.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.walmartlabs.concord.server.security.Roles; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -19,15 +18,6 @@ public final class StateManagerUtils { private static final List ALLOWED_EXTENSIONS = List.of("json", "yaml", "yml"); public static InputStream stateFilter(String file, InputStream in) { - // do not apply filter for admins - if (Roles.isAdmin()) { - return in; - } - - return filter(file, in); - } - - public static InputStream filter(String file, InputStream in) { try { String extension = getFileExtension(file); if (!ALLOWED_EXTENSIONS.contains(extension) || STATE_FILTER.get(file) == null) { diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/process/state/ProcessStateManager.java b/server/impl/src/main/java/com/walmartlabs/concord/server/process/state/ProcessStateManager.java index 7bb31502c1..051d9be3c6 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/process/state/ProcessStateManager.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/process/state/ProcessStateManager.java @@ -511,8 +511,8 @@ public static ItemConsumer copyTo(Path dst, String[] ignored, OpenOption... opti * * @param dst archive stream. */ - public static ItemConsumer zipTo(ZipArchiveOutputStream dst) { - return new ZipConsumer(dst); + public static ItemConsumer zipTo(ZipArchiveOutputStream dst, boolean filterContents) { + return new ZipConsumer(dst, filterContents); } public static ItemConsumer exclude(ItemConsumer delegate, String... patterns) { @@ -793,9 +793,11 @@ public void accept(String name, int unixMode, InputStream src) { public static final class ZipConsumer implements ItemConsumer { private final ZipArchiveOutputStream dst; + private final boolean filterContents; - private ZipConsumer(ZipArchiveOutputStream dst) { + private ZipConsumer(ZipArchiveOutputStream dst, boolean filterContents) { this.dst = dst; + this.filterContents = filterContents; } @Override @@ -803,10 +805,10 @@ public void accept(String name, int unixMode, InputStream src) { ZipArchiveEntry entry = new ZipArchiveEntry(name); entry.setUnixMode(unixMode); // filter before zip to download - InputStream filtered = StateManagerUtils.stateFilter(name, src); + InputStream processed = filterContents ? StateManagerUtils.stateFilter(name, src) : src; try { dst.putArchiveEntry(entry); - IOUtils.copy(filtered, dst); + IOUtils.copy(processed, dst); dst.closeArchiveEntry(); } catch (IOException e) { throw new RuntimeException(e); diff --git a/server/impl/src/test/java/com/walmartlabs/concord/server/process/StateManagerUtilsTest.java b/server/impl/src/test/java/com/walmartlabs/concord/server/process/StateManagerUtilsTest.java index 91a330baab..ad1ec3a14d 100644 --- a/server/impl/src/test/java/com/walmartlabs/concord/server/process/StateManagerUtilsTest.java +++ b/server/impl/src/test/java/com/walmartlabs/concord/server/process/StateManagerUtilsTest.java @@ -18,7 +18,7 @@ void testFilterWithJsonFile() throws Exception { String jsonInput = "{\"arguments\": \"value\", \"otherKey\": \"otherValue\"}"; InputStream inputStream = new ByteArrayInputStream(jsonInput.getBytes(StandardCharsets.UTF_8)); - InputStream result = StateManagerUtils.filter("_main.json", inputStream); + InputStream result = StateManagerUtils.stateFilter("_main.json", inputStream); String text = new String(result.readAllBytes(), StandardCharsets.UTF_8); ObjectMapper mapper = new ObjectMapper(); @@ -33,12 +33,12 @@ void testFilterWithYamlFile() throws Exception { String yamlInput = "arguments: value\notherKey: otherValue"; InputStream inputStream = new ByteArrayInputStream(yamlInput.getBytes(StandardCharsets.UTF_8)); - InputStream result = StateManagerUtils.filter("test.yaml", inputStream); + InputStream result = StateManagerUtils.stateFilter("test.yaml", inputStream); ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); Map resultMap = mapper.readValue(result, Map.class); - // return the same if to filter items not present + // return the same if to stateFilter items not present assertEquals(2, resultMap.size()); assertEquals("otherValue", resultMap.get("otherKey")); } @@ -48,7 +48,7 @@ void testFilterWithUnsupportedExtension() throws Exception { String input = "key: value"; InputStream inputStream = new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8)); - InputStream result = StateManagerUtils.filter("file.txt", inputStream); + InputStream result = StateManagerUtils.stateFilter("file.txt", inputStream); String text = new String(result.readAllBytes(), StandardCharsets.UTF_8); assertEquals(input, text); @@ -60,7 +60,7 @@ void testFilterWithEmptyInput() throws Exception { String input = ""; InputStream inputStream = new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8)); - InputStream result = StateManagerUtils.filter("_main.json", inputStream); + InputStream result = StateManagerUtils.stateFilter("_main.json", inputStream); String text = new String(result.readAllBytes(), StandardCharsets.UTF_8); assertEquals(input, text); @@ -72,7 +72,7 @@ void testFilterWithNullRules() throws Exception { String jsonInput = "{\"key\": \"value\"}"; InputStream inputStream = new ByteArrayInputStream(jsonInput.getBytes(StandardCharsets.UTF_8)); - InputStream result = StateManagerUtils.filter("unknown.json", inputStream); + InputStream result = StateManagerUtils.stateFilter("unknown.json", inputStream); String text = new String(result.readAllBytes(), StandardCharsets.UTF_8); assertEquals(jsonInput, text); @@ -83,7 +83,7 @@ void testFilterWithNullRules() throws Exception { void testFilterWithNullInput() throws Exception { String jsonInput = "{\"key\": \"value\"}"; InputStream inputStream = null; - InputStream result = StateManagerUtils.filter("unknown.json", inputStream); + InputStream result = StateManagerUtils.stateFilter("unknown.json", inputStream); assertNull(result); } From e27ddf02fd2f5ea4300f45f68d2042e95dfe31dd Mon Sep 17 00:00:00 2001 From: amithkb Date: Mon, 21 Apr 2025 18:30:41 +0530 Subject: [PATCH 4/4] fix bouncy versions --- agent-operator/pom.xml | 7 +++++-- targetplatform/pom.xml | 9 ++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/agent-operator/pom.xml b/agent-operator/pom.xml index 3f45888696..4c085e0d4f 100644 --- a/agent-operator/pom.xml +++ b/agent-operator/pom.xml @@ -39,13 +39,16 @@ org.bouncycastle - bcprov-ext-jdk18on + bcprov-jdk18on org.bouncycastle bcpkix-jdk18on - + + org.bouncycastle + bcutil-jdk18on + com.fasterxml.jackson.core jackson-core diff --git a/targetplatform/pom.xml b/targetplatform/pom.xml index ca9d447522..bacee5c0f9 100644 --- a/targetplatform/pom.xml +++ b/targetplatform/pom.xml @@ -39,7 +39,6 @@ 1.0 1.11.475 1.80 - 1.78.1 1.0.3 1.14.9 1.9.4 @@ -1103,17 +1102,17 @@ org.bouncycastle - bcprov-ext-jdk18on - ${bouncycastle.ext.version} + bcprov-jdk18on + ${bouncycastle.version} org.bouncycastle - bcprov-jdk18on + bcpkix-jdk18on ${bouncycastle.version} org.bouncycastle - bcpkix-jdk18on + bcutil-jdk18on ${bouncycastle.version}