Skip to content

Commit d4ee23d

Browse files
novarxnovarx
authored
Merge branch 'citrusframework:main' into main
2 parents 018c3fc + 38476ea commit d4ee23d

File tree

21 files changed

+918
-81
lines changed

21 files changed

+918
-81
lines changed

‎core/citrus-base/src/main/java/org/citrusframework/sharding/Shard.java

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/*
2+
* Copyright 2024 the original author or authors.
3+
*
4+
* Licensed to the Apache Software Foundation (ASF) under one or more
5+
* contributor license agreements. See the NOTICE file distributed with
6+
* this work for additional information regarding copyright ownership.
7+
* The ASF licenses this file to You under the Apache License, Version 2.0
8+
* (the "License"); you may not use this file except in compliance with
9+
* the License. You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
*/
19+
20+
package org.citrusframework.sharding;
21+
22+
import java.util.ArrayList;
23+
import java.util.List;
24+
import java.util.Random;
25+
import java.util.stream.Stream;
26+
27+
import static java.lang.Math.ceil;
28+
import static java.lang.Math.min;
29+
import static java.util.Collections.shuffle;
30+
import static java.util.stream.Collectors.toCollection;
31+
32+
/**
33+
* A utility class for implementing sharded test loading and execution, additionally to traditional test loading. It
34+
* enhances the citrus framework with the ability to divide test cases into shards. Their loading and execution is still
35+
* managed by the traditional test loaders and executors.
36+
* <p>
37+
* This class is part of the Citrus framework, designed to streamline the process of sharding test cases for efficient
38+
* and scalable testing.
39+
* <p>
40+
* Configuration happens via environment variables. See {@link ShardingConfiguration} for more information.
41+
*
42+
* @see ShardingConfiguration
43+
*/
44+
public final class Shard {
45+
46+
private Shard() {
47+
throw new IllegalArgumentException("Utility class shall not be instantiated!");
48+
}
49+
50+
/**
51+
* Creates a sharded stream from the input stream using the default sharding configuration. Note that the initial
52+
* stream will be terminated!
53+
*
54+
* @param <T> The type of elements in the stream.
55+
* @param input The input stream to be sharded.
56+
* @return A sharded stream based on the default sharding configuration.
57+
*/
58+
public static <T> Stream<T> createShard(Stream<T> input) {
59+
return createShard(input, new ShardingConfiguration());
60+
}
61+
62+
/**
63+
* Creates a sharded stream from the input stream using the provided sharding configuration. Note that the initial
64+
* stream will be terminated!
65+
*
66+
* @param <T> The type of elements in the stream.
67+
* @param input The input stream to be sharded.
68+
* @param shardingConfiguration The configuration for sharding.
69+
* @return A sharded stream based on the provided sharding configuration.
70+
*/
71+
public static <T> Stream<T> createShard(Stream<T> input, ShardingConfiguration shardingConfiguration) {
72+
return createShard(input, shardingConfiguration, false);
73+
}
74+
75+
76+
/**
77+
* Creates a sharded stream from the input stream using the provided sharding configuration and a flag to determine
78+
* whether the stream should be parallel. Note that the initial stream will be terminated!
79+
*
80+
* @param <T> The type of elements in the stream.
81+
* @param input The input stream to be sharded.
82+
* @param shardingConfiguration The configuration for sharding.
83+
* @param parallel A flag indicating whether the resulting stream should be parallel.
84+
* @return A sharded stream based on the provided sharding configuration.
85+
*/
86+
public static <T> Stream<T> createShard(Stream<T> input, ShardingConfiguration shardingConfiguration, boolean parallel) {
87+
List<T> itemList = input.collect(toCollection(ArrayList::new));
88+
89+
var random = new Random(shardingConfiguration.getSeed());
90+
shuffle(itemList, random);
91+
92+
int shardSize = (int) ceil(itemList.size() / (double) shardingConfiguration.getTotalNumberOfShards());
93+
int startIndex = shardingConfiguration.getShardNumber() * shardSize;
94+
int endIndex = min(itemList.size(), startIndex + shardSize);
95+
96+
var shardedItems = itemList.subList(startIndex, endIndex);
97+
98+
if (parallel) {
99+
return shardedItems.parallelStream();
100+
} else {
101+
return shardedItems.stream();
102+
}
103+
}
104+
}

‎core/citrus-base/src/main/java/org/citrusframework/sharding/ShardingConfiguration.java

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
/*
2+
* Copyright 2024 the original author or authors.
3+
*
4+
* Licensed to the Apache Software Foundation (ASF) under one or more
5+
* contributor license agreements. See the NOTICE file distributed with
6+
* this work for additional information regarding copyright ownership.
7+
* The ASF licenses this file to You under the Apache License, Version 2.0
8+
* (the "License"); you may not use this file except in compliance with
9+
* the License. You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
*/
19+
20+
package org.citrusframework.sharding;
21+
22+
import org.citrusframework.exceptions.CitrusRuntimeException;
23+
import org.citrusframework.util.SystemProvider;
24+
25+
import java.util.Optional;
26+
27+
import static java.lang.Integer.parseInt;
28+
import static java.lang.String.valueOf;
29+
30+
/**
31+
* A configuration class for sharded test loading and execution withing the citrus framework. It uses environment
32+
* variables and system properties to configure the sharding behavior.
33+
* <p>
34+
* This class is part of the Citrus framework, designed to streamline the process of sharding test cases for efficient
35+
* and scalable testing.
36+
* <p>
37+
* <h3>Configuration Example:</h3>
38+
* <p>To configure the sharding behavior, set the following environment variables or system properties:</p>
39+
* <ul>
40+
* <li><b>Total number of shards:</b>
41+
* <ul>
42+
* <li>Environment Variable: <code>CITRUS_SHARDING_TOTAL</code></li>
43+
* <li>System Property: <code>citrus.sharding.total</code></li>
44+
* <li>Description: Specifies the total number of shards into which the test cases will be divided.</li>
45+
* </ul>
46+
* </li>
47+
* <li><b>Shard number:</b>
48+
* <ul>
49+
* <li>Environment Variable: <code>CITRUS_SHARDING_NUMBER</code></li>
50+
* <li>System Property: <code>citrus.sharding.number</code></li>
51+
* <li>
52+
* Description: Indicates the specific shard number of the current test loader. This should be a value between
53+
* 0 and the total number of shards minus one.
54+
* </li>
55+
* </ul>
56+
* </li>
57+
* <li><b>Shard seed:</b>
58+
* <ul>
59+
* <li>Environment Variable: <code>CITRUS_SHARDING_SEED</code></li>
60+
* <li>System Property: <code>citrus.sharding.seed</code></li>
61+
* <li>
62+
* Description: Specifies a seed value used for shuffling test cases within a shard. Providing a consistent
63+
* seed value ensures the same shuffling order across different executions.
64+
* </li>
65+
* </ul>
66+
* </li>
67+
* </ul>
68+
*
69+
* <h3>Example Usage:</h3>
70+
* <p>To configure a system with 4 total shards and assign this instance to shard number 1 (second shard, since
71+
* numbering starts at 0), set the environment variables or system properties as follows:</p>
72+
* <ul>
73+
* <li>
74+
* Set <code>CITRUS_SHARDING_TOTAL</code> or <code>citrus.sharding.total</code> to 4.
75+
* </li>
76+
* <li>
77+
* Set <code>CITRUS_SHARDING_NUMBER</code> or <code>citrus.sharding.number</code> to 1.
78+
* </li>
79+
* <li>
80+
* Optionally, set a seed for shuffling test cases using <code>CITRUS_SHARDING_SEED</code>
81+
* or <code>citrus.sharding.seed</code>. The total number of shards will be used as see by default.
82+
* </li>
83+
* </ul>
84+
*
85+
* @see Shard
86+
*/
87+
public final class ShardingConfiguration {
88+
89+
public static final String TOTAL_SHARD_NUMBER_PROPERTY_NAME = "citrus.sharding.total";
90+
public static final String TOTAL_SHARD_NUMBER_ENV_VAR_NAME = TOTAL_SHARD_NUMBER_PROPERTY_NAME.replace(".", "_").toUpperCase();
91+
92+
public static final String SHARD_NUMBER_PROPERTY_NAME = "citrus.sharding.number";
93+
public static final String SHARD_NUMBER_ENV_VAR_NAME = SHARD_NUMBER_PROPERTY_NAME.replace(".", "_").toUpperCase();
94+
95+
public static final String SHARD_SEED_PROPERTY_NAME = "citrus.sharding.seed";
96+
public static final String SHARD_SEED_ENV_VAR_NAME = SHARD_SEED_PROPERTY_NAME.replace(".", "_").toUpperCase();
97+
98+
private final int totalNumberOfShards;
99+
private final int shardNumber;
100+
private final String seed;
101+
102+
/**
103+
* Default sharding configuration which initializes the sharding with system properties and environment variables.
104+
*/
105+
public ShardingConfiguration() {
106+
this(new SystemProvider());
107+
}
108+
109+
/**
110+
* Constructor that allows for injecting a custom {@link SystemProvider}. This is primarily intended for testing
111+
* purposes, enabling the mocking and overriding of system environment and properties.
112+
*
113+
* @param systemProvider a provider for system environment variables and properties.
114+
*/
115+
protected ShardingConfiguration(SystemProvider systemProvider) {
116+
this(getTotalNumberOfShards(systemProvider), getShardNumber(systemProvider), systemProvider);
117+
}
118+
119+
/**
120+
* Create a new sharding configuration with explicit total number of shards and shard number.
121+
*
122+
* @param totalNumberOfShards the total number of shards to be used.
123+
* @param shardNumber the specific shard number for this loader, zero-based.
124+
*/
125+
public ShardingConfiguration(int totalNumberOfShards, int shardNumber) {
126+
this(totalNumberOfShards, shardNumber, new SystemProvider());
127+
}
128+
129+
/**
130+
* Constructor that sets the total number of shards, shard number, and allows for injecting a
131+
* custom {@link SystemProvider}. Primarily used for testing purposes.
132+
*
133+
* @param totalNumberOfShards the total number of shards.
134+
* @param shardNumber the shard number for this loader, zero-based.
135+
* @param systemProvider a provider for system environment variables and properties.
136+
*/
137+
protected ShardingConfiguration(int totalNumberOfShards, int shardNumber, SystemProvider systemProvider) {
138+
this.totalNumberOfShards = totalNumberOfShards;
139+
this.shardNumber = shardNumber;
140+
141+
seed = getSeedOrDefaultValue(systemProvider, totalNumberOfShards);
142+
143+
sanitizeConfiguration();
144+
}
145+
146+
private static int getTotalNumberOfShards(SystemProvider systemProvider) {
147+
return extractEnvOrProperty(systemProvider, TOTAL_SHARD_NUMBER_ENV_VAR_NAME, TOTAL_SHARD_NUMBER_PROPERTY_NAME, 1, "Failed to calculate number of total shards, received string instead of number!");
148+
}
149+
150+
private static int getShardNumber(SystemProvider systemProvider) {
151+
return extractEnvOrProperty(systemProvider, SHARD_NUMBER_ENV_VAR_NAME, SHARD_NUMBER_PROPERTY_NAME, 0, "Failed to calculate shard number, received string instead of number!");
152+
}
153+
154+
private static String getSeedOrDefaultValue(SystemProvider systemProvider, int totalNumberOfShards) {
155+
return extractEnvOrProperty(systemProvider, SHARD_SEED_ENV_VAR_NAME, SHARD_SEED_PROPERTY_NAME)
156+
.orElseGet(() -> valueOf(totalNumberOfShards));
157+
}
158+
159+
private static int extractEnvOrProperty(SystemProvider systemProvider, String envVarName, String fallbackPropertyName, int defaultValue, String numberParseErrorMessage) {
160+
try {
161+
return parseInt(extractEnvOrProperty(systemProvider, envVarName, fallbackPropertyName)
162+
.orElseGet(() -> valueOf(defaultValue)));
163+
} catch (NumberFormatException e) {
164+
throw new CitrusRuntimeException(numberParseErrorMessage, e);
165+
}
166+
}
167+
168+
private static Optional<String> extractEnvOrProperty(SystemProvider systemProvider, String envVarName, String fallbackPropertyName) {
169+
return systemProvider.getEnv(envVarName)
170+
.or(() -> systemProvider.getProperty(fallbackPropertyName));
171+
}
172+
173+
private void sanitizeConfiguration() {
174+
if (totalNumberOfShards <= 0) {
175+
throw new CitrusRuntimeException("Number of total shards must be configured!");
176+
} else if (shardNumber < 0) {
177+
throw new CitrusRuntimeException("Shard number cannot be negative!");
178+
} else if (shardNumber >= totalNumberOfShards) {
179+
throw new CitrusRuntimeException("Shard number must be less than the total number of shards!");
180+
}
181+
}
182+
183+
public int getTotalNumberOfShards() {
184+
return totalNumberOfShards;
185+
}
186+
187+
public int getShardNumber() {
188+
return shardNumber;
189+
}
190+
191+
public int getSeed() {
192+
return seed.hashCode();
193+
}
194+
}

‎core/citrus-base/src/main/java/org/citrusframework/util/SystemProvider.java

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright 2024 the original author or authors.
3+
*
4+
* Licensed to the Apache Software Foundation (ASF) under one or more
5+
* contributor license agreements. See the NOTICE file distributed with
6+
* this work for additional information regarding copyright ownership.
7+
* The ASF licenses this file to You under the Apache License, Version 2.0
8+
* (the "License"); you may not use this file except in compliance with
9+
* the License. You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
*/
19+
20+
package org.citrusframework.util;
21+
22+
import java.util.Optional;
23+
24+
import static java.util.Optional.ofNullable;
25+
26+
public final class SystemProvider {
27+
28+
public Optional<String> getEnv(String envVarName) {
29+
return ofNullable(System.getenv(envVarName));
30+
}
31+
32+
public Optional<String> getProperty(String propertyName) {
33+
return ofNullable(System.getProperty(propertyName));
34+
}
35+
}

0 commit comments

Comments
 (0)