Skip to content

Commit 2034ae3

Browse files
ibodrovibodrov
authored
it: do not archive deps.dir into payloads (#1214)
Explicitly specify plugin versions. Avoids classpath conflicts when multiple versions of the same task are required.
1 parent 9b928c2 commit 2034ae3

File tree

78 files changed

+355
-177
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+355
-177
lines changed

‎it/common/pom.xml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,4 +71,16 @@
7171
<artifactId>annotations</artifactId>
7272
</dependency>
7373
</dependencies>
74+
75+
<build>
76+
<resources>
77+
<resource>
78+
<filtering>true</filtering>
79+
<directory>${project.basedir}/src/main/filtered-resources</directory>
80+
<includes>
81+
<include>**/*</include>
82+
</includes>
83+
</resource>
84+
</resources>
85+
</build>
7486
</project>

‎it/runtime-v2/src/test/filtered-resources/version.properties‎ renamed to ‎it/common/src/main/filtered-resources/com/walmartlabs/concord/it/common/version.properties‎

File renamed without changes.

‎it/common/src/main/java/com/walmartlabs/concord/it/common/ITUtils.java‎

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@
2121
*/
2222

2323
import com.walmartlabs.concord.common.PathUtils;
24+
import com.walmartlabs.concord.common.TemporaryPath;
2425
import com.walmartlabs.concord.common.ZipUtils;
25-
import com.walmartlabs.concord.sdk.Constants;
2626
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
2727
import org.eclipse.jgit.api.Git;
2828
import org.eclipse.jgit.api.errors.GitAPIException;
@@ -36,27 +36,24 @@
3636
import java.nio.file.Files;
3737
import java.nio.file.Path;
3838
import java.nio.file.Paths;
39+
import java.nio.file.StandardOpenOption;
3940
import java.nio.file.attribute.PosixFilePermissions;
4041
import java.util.Random;
4142
import java.util.concurrent.ThreadLocalRandom;
43+
import java.util.stream.Stream;
4244

4345
public final class ITUtils {
4446

4547
private static final char[] RANDOM_CHARS = "abcdef0123456789".toCharArray();
4648

4749
public static byte[] archive(URI uri) throws IOException {
48-
return archive(uri, null);
49-
}
50-
51-
public static byte[] archive(URI uri, String depsDir) throws IOException {
52-
ByteArrayOutputStream out = new ByteArrayOutputStream();
53-
try (ZipArchiveOutputStream zip = new ZipArchiveOutputStream(out)) {
54-
ZipUtils.zip(zip, Paths.get(uri));
55-
if (depsDir != null) {
56-
ZipUtils.zip(zip, Constants.Files.LIBRARIES_DIR_NAME + "/", Paths.get(depsDir));
50+
try (TemporaryPath tmpDir = preprocessDir(uri)) {
51+
ByteArrayOutputStream out = new ByteArrayOutputStream();
52+
try (ZipArchiveOutputStream zip = new ZipArchiveOutputStream(out)) {
53+
ZipUtils.zip(zip, tmpDir.path());
5754
}
55+
return out.toByteArray();
5856
}
59-
return out.toByteArray();
6057
}
6158

6259
public static String resourceToString(Class<?> klass, String resource) throws Exception {
@@ -109,6 +106,31 @@ public static String randomPwd() {
109106
return "pwd_" + randomString() + "A!";
110107
}
111108

109+
private static TemporaryPath preprocessDir(URI uri) throws IOException {
110+
Path src = Paths.get(uri);
111+
112+
// copy files from the specified URI to a temporary directory
113+
TemporaryPath tmpDir = PathUtils.tempDir("test");
114+
PathUtils.copy(src, tmpDir.path());
115+
116+
// find and replace all PROJECT_VERSION strings with the current ${project.version}
117+
try (Stream<Path> yamlFiles = Files.walk(tmpDir.path())) {
118+
for (Path yamlFile : yamlFiles.filter(f -> {
119+
String fileName = f.getFileName().toString().toLowerCase();
120+
return fileName.endsWith(".yaml") || fileName.endsWith(".yml") || fileName.endsWith(".json");
121+
}).toList()) {
122+
String content = Files.readString(yamlFile);
123+
if (!content.contains("PROJECT_VERSION")) {
124+
continue;
125+
}
126+
content = content.replaceAll("PROJECT_VERSION", Version.PROJECT_VERSION);
127+
Files.writeString(yamlFile, content, StandardOpenOption.TRUNCATE_EXISTING);
128+
}
129+
}
130+
131+
return tmpDir;
132+
}
133+
112134
private ITUtils() {
113135
}
114136
}

‎it/common/src/main/java/com/walmartlabs/concord/it/common/Version.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package com.walmartlabs.concord.it.common;
2+
3+
/*-
4+
* *****
5+
* Concord
6+
* -----
7+
* Copyright (C) 2017 - 2018 Walmart Inc.
8+
* -----
9+
* Licensed under the Apache License, Version 2.0 (the "License");
10+
* you may not use this file except in compliance with the License.
11+
* You may obtain a copy of the License at
12+
*
13+
* http://www.apache.org/licenses/LICENSE-2.0
14+
*
15+
* Unless required by applicable law or agreed to in writing, software
16+
* distributed under the License is distributed on an "AS IS" BASIS,
17+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18+
* See the License for the specific language governing permissions and
19+
* limitations under the License.
20+
* =====
21+
*/
22+
23+
import java.io.IOException;
24+
import java.io.InputStream;
25+
import java.util.Properties;
26+
27+
public final class Version {
28+
29+
public static final String PROJECT_VERSION;
30+
31+
static {
32+
try (InputStream in = Version.class.getResourceAsStream("version.properties")) {
33+
Properties props = new Properties();
34+
props.load(in);
35+
PROJECT_VERSION = props.getProperty("project.version");
36+
} catch (IOException e) {
37+
throw new RuntimeException(e);
38+
}
39+
}
40+
41+
private Version() {
42+
}
43+
}

‎it/runtime-v2/src/test/java/com/walmartlabs/concord/it/runtime/v2/ITConstants.java‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,29 +20,10 @@
2020
* =====
2121
*/
2222

23-
import java.io.IOException;
24-
import java.io.InputStream;
25-
import java.util.Properties;
26-
2723
public final class ITConstants {
2824

29-
public static final String PROJECT_VERSION;
3025
public static final long DEFAULT_TEST_TIMEOUT = 120000;
3126

32-
static {
33-
PROJECT_VERSION = getProperties("version.properties").getProperty("project.version");
34-
}
35-
36-
private static Properties getProperties(String path) {
37-
try (InputStream in = ClassLoader.getSystemResourceAsStream(path)) {
38-
Properties props = new Properties();
39-
props.load(in);
40-
return props;
41-
} catch (IOException e) {
42-
throw new RuntimeException(e);
43-
}
44-
}
45-
4627
private ITConstants() {
4728
}
4829
}

‎it/runtime-v2/src/test/java/com/walmartlabs/concord/it/runtime/v2/NodeRosterIT.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import com.walmartlabs.concord.client2.HostEntry;
2727
import com.walmartlabs.concord.client2.NodeRosterHostsApi;
2828
import com.walmartlabs.concord.client2.ProcessEntry;
29+
import com.walmartlabs.concord.it.common.Version;
2930
import org.junit.jupiter.api.Test;
3031
import org.junit.jupiter.api.extension.RegisterExtension;
3132

@@ -46,7 +47,7 @@ public void test() throws Exception {
4647
// run the Ansible flow first to get some data
4748

4849
String concordYml = resourceToString(NodeRosterIT.class.getResource("noderoster/ansible.yml"))
49-
.replaceAll("PROJECT_VERSION", ITConstants.PROJECT_VERSION);
50+
.replaceAll("PROJECT_VERSION", Version.PROJECT_VERSION);
5051

5152
ConcordProcess proc = concord.processes().start(new Payload()
5253
.concordYml(concordYml)
@@ -67,7 +68,7 @@ public void test() throws Exception {
6768
// run the Node Roster flow next to test the plugin
6869

6970
concordYml = resourceToString(ProcessIT.class.getResource("noderoster/noderoster.yml"))
70-
.replaceAll("PROJECT_VERSION", ITConstants.PROJECT_VERSION);
71+
.replaceAll("PROJECT_VERSION", Version.PROJECT_VERSION);
7172

7273
proc = concord.processes().start(new Payload()
7374
.concordYml(concordYml));

‎it/runtime-v2/src/test/java/com/walmartlabs/concord/it/runtime/v2/ProcessIT.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import ca.ibodrov.concord.testcontainers.junit5.ConcordRule;
2626
import com.walmartlabs.concord.client2.*;
2727
import com.walmartlabs.concord.common.ConfigurationUtils;
28+
import com.walmartlabs.concord.it.common.Version;
2829
import com.walmartlabs.concord.sdk.Constants;
2930
import com.walmartlabs.concord.sdk.MapUtils;
3031
import org.junit.jupiter.api.Test;
@@ -347,7 +348,7 @@ public void testCheckpointsParallel() throws Exception {
347348
@Test
348349
public void testNoStateAfterCheckpoint() throws Exception {
349350
String concordYml = resourceToString(ProcessIT.class.getResource("checkpointState/concord.yml"))
350-
.replaceAll("PROJECT_VERSION", ITConstants.PROJECT_VERSION);
351+
.replaceAll("PROJECT_VERSION", Version.PROJECT_VERSION);
351352

352353
Payload payload = new Payload().concordYml(concordYml);
353354

@@ -418,7 +419,7 @@ public void testForkCheckpoints() throws Exception {
418419
@Test
419420
public void testCheckpointsWith3rdPartyClasses() throws Exception {
420421
String concordYml = resourceToString(NodeRosterIT.class.getResource("checkpointClasses/concord.yml"))
421-
.replaceAll("PROJECT_VERSION", ITConstants.PROJECT_VERSION);
422+
.replaceAll("PROJECT_VERSION", Version.PROJECT_VERSION);
422423

423424
ConcordProcess proc = concord.processes().start(new Payload()
424425
.concordYml(concordYml));

‎it/runtime-v2/src/test/java/com/walmartlabs/concord/it/runtime/v2/SmtpIT.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import com.icegreen.greenmail.junit5.GreenMailExtension;
2929
import com.icegreen.greenmail.util.ServerSetup;
3030
import com.walmartlabs.concord.client2.ProcessEntry;
31+
import com.walmartlabs.concord.it.common.Version;
3132
import org.junit.jupiter.api.Test;
3233
import org.junit.jupiter.api.extension.RegisterExtension;
3334
import org.testcontainers.Testcontainers;
@@ -61,7 +62,7 @@ public void test() throws Exception {
6162

6263
// SMTP host and port must be accessible by the process
6364
// i.e. when running in a container the host must point to the docker host's address
64-
concordYml = concordYml.replaceAll("PROJECT_VERSION", ITConstants.PROJECT_VERSION)
65+
concordYml = concordYml.replaceAll("PROJECT_VERSION", Version.PROJECT_VERSION)
6566
.replaceAll("SMTP_HOST", concord.hostAddressAccessibleByContainers())
6667
.replaceAll("SMTP_PORT", String.valueOf(mailServer.getSmtp().getPort()));
6768

‎it/server/pom.xml‎

Lines changed: 1 addition & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -268,67 +268,20 @@
268268
<configuration>
269269
<overWriteIfNewer>true</overWriteIfNewer>
270270
<artifactItems>
271-
<artifactItem>
272-
<groupId>com.walmartlabs.concord.plugins.basic</groupId>
273-
<artifactId>ansible-tasks</artifactId>
274-
<version>${project.version}</version>
275-
</artifactItem>
276271
<artifactItem>
277272
<groupId>com.walmartlabs.concord.plugins.basic</groupId>
278273
<artifactId>ansible-template</artifactId>
279274
<version>${project.version}</version>
280275
<destFileName>ansible-template.jar</destFileName>
281276
</artifactItem>
282-
<artifactItem>
283-
<groupId>com.walmartlabs.concord.plugins.basic</groupId>
284-
<artifactId>concord-tasks</artifactId>
285-
<version>${project.version}</version>
286-
</artifactItem>
287-
<artifactItem>
288-
<groupId>com.walmartlabs.concord.plugins.basic</groupId>
289-
<artifactId>dynamic-tasks</artifactId>
290-
<version>${project.version}</version>
291-
</artifactItem>
292277

293-
<!-- TODO this should be automated -->
294-
<artifactItem>
295-
<groupId>com.walmartlabs.concord.plugins.basic</groupId>
296-
<artifactId>noderoster-tasks</artifactId>
297-
<version>${project.version}</version>
298-
</artifactItem>
299-
<artifactItem>
300-
<groupId>com.walmartlabs.concord.server.plugins.noderoster</groupId>
301-
<artifactId>concord-noderoster-plugin-client2</artifactId>
302-
<version>${project.version}</version>
303-
</artifactItem>
304-
305-
<!-- TODO this should be automated -->
306-
<artifactItem>
307-
<groupId>com.walmartlabs.concord.plugins.basic</groupId>
308-
<artifactId>smtp-tasks</artifactId>
309-
<version>${project.version}</version>
310-
</artifactItem>
311-
<artifactItem>
312-
<groupId>org.apache.commons</groupId>
313-
<artifactId>commons-email</artifactId>
314-
</artifactItem>
315-
<artifactItem>
316-
<groupId>com.sun.mail</groupId>
317-
<artifactId>javax.mail</artifactId>
318-
</artifactItem>
319-
<dependency>
320-
<groupId>com.sun.activation</groupId>
321-
<artifactId>javax.activation</artifactId>
322-
</dependency>
323-
324-
<!-- used by DependenciesIT as an example dependency -->
278+
<!-- used by DependenciesIT as example dependencies -->
325279
<artifactItem>
326280
<groupId>com.walmartlabs.concord.plugins.basic</groupId>
327281
<artifactId>example-tasks</artifactId>
328282
<version>${project.version}</version>
329283
<destFileName>example.jar</destFileName>
330284
</artifactItem>
331-
332285
<dependency>
333286
<groupId>com.walmartlabs.concord.plugins.basic</groupId>
334287
<artifactId>ansible-tasks</artifactId>

‎it/server/src/test/java/com/walmartlabs/concord/it/server/AnsibleEventIT.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public class AnsibleEventIT extends AbstractServerIT {
3838
@SuppressWarnings("unchecked")
3939
public void testEvent() throws Exception {
4040
URI uri = AnsibleEventIT.class.getResource("ansibleEvent").toURI();
41-
byte[] payload = archive(uri, ITConstants.DEPENDENCIES_DIR);
41+
byte[] payload = archive(uri);
4242

4343
// ---
4444

@@ -81,7 +81,7 @@ public void testEvent() throws Exception {
8181
@Test
8282
public void testIgnoredFailures() throws Exception {
8383
URI uri = AnsibleEventIT.class.getResource("ansibleIgnoredFailures").toURI();
84-
byte[] payload = archive(uri, ITConstants.DEPENDENCIES_DIR);
84+
byte[] payload = archive(uri);
8585

8686
// ---
8787

@@ -123,7 +123,7 @@ public void testIgnoredFailures() throws Exception {
123123
@Test
124124
public void testFailedHosts() throws Exception {
125125
URI uri = AnsibleEventIT.class.getResource("ansibleFailedHosts").toURI();
126-
byte[] payload = archive(uri, ITConstants.DEPENDENCIES_DIR);
126+
byte[] payload = archive(uri);
127127

128128
// ---
129129

0 commit comments

Comments
 (0)