Merge branch 'citrusframework:main' into main

core/citrus-base/src/main/java/org/citrusframework/sharding/Shard.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2024 the original author or authors.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.citrusframework.sharding;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.stream.Stream;
+
+import static java.lang.Math.ceil;
+import static java.lang.Math.min;
+import static java.util.Collections.shuffle;
+import static java.util.stream.Collectors.toCollection;
+
+/**
+ * A utility class for implementing sharded test loading and execution, additionally to traditional test loading. It
+ * enhances the citrus framework with the ability to divide test cases into shards. Their loading and execution is still
+ * managed by the traditional test loaders and executors.
+ * <p>
+ * This class is part of the Citrus framework, designed to streamline the process of sharding test cases for efficient
+ * and scalable testing.
+ * <p>
+ * Configuration happens via environment variables. See {@link ShardingConfiguration} for more information.
+ *
+ * @see ShardingConfiguration
+ */
+public final class Shard {
+
+    private Shard() {
+        throw new IllegalArgumentException("Utility class shall not be instantiated!");
+    }
+
+    /**
+     * Creates a sharded stream from the input stream using the default sharding configuration. Note that the initial
+     * stream will be terminated!
+     *
+     * @param <T>   The type of elements in the stream.
+     * @param input The input stream to be sharded.
+     * @return A sharded stream based on the default sharding configuration.
+     */
+    public static <T> Stream<T> createShard(Stream<T> input) {
+        return createShard(input, new ShardingConfiguration());
+    }
+
+    /**
+     * Creates a sharded stream from the input stream using the provided sharding configuration. Note that the initial
+     * stream will be terminated!
+     *
+     * @param <T>                   The type of elements in the stream.
+     * @param input                 The input stream to be sharded.
+     * @param shardingConfiguration The configuration for sharding.
+     * @return A sharded stream based on the provided sharding configuration.
+     */
+    public static <T> Stream<T> createShard(Stream<T> input, ShardingConfiguration shardingConfiguration) {
+        return createShard(input, shardingConfiguration, false);
+    }
+
+
+    /**
+     * Creates a sharded stream from the input stream using the provided sharding configuration and a flag to determine
+     * whether the stream should be parallel. Note that the initial stream will be terminated!
+     *
+     * @param <T>                   The type of elements in the stream.
+     * @param input                 The input stream to be sharded.
+     * @param shardingConfiguration The configuration for sharding.
+     * @param parallel              A flag indicating whether the resulting stream should be parallel.
+     * @return A sharded stream based on the provided sharding configuration.
+     */
+    public static <T> Stream<T> createShard(Stream<T> input, ShardingConfiguration shardingConfiguration, boolean parallel) {
+        List<T> itemList = input.collect(toCollection(ArrayList::new));
+
+        var random = new Random(shardingConfiguration.getSeed());
+        shuffle(itemList, random);
+
+        int shardSize = (int) ceil(itemList.size() / (double) shardingConfiguration.getTotalNumberOfShards());
+        int startIndex = shardingConfiguration.getShardNumber() * shardSize;
+        int endIndex = min(itemList.size(), startIndex + shardSize);
+
+        var shardedItems = itemList.subList(startIndex, endIndex);
+
+        if (parallel) {
+            return shardedItems.parallelStream();
+        } else {
+            return shardedItems.stream();
+        }
+    }
+}

core/citrus-base/src/main/java/org/citrusframework/sharding/ShardingConfiguration.java

@@ -0,0 +1,194 @@
+/*
+ * Copyright 2024 the original author or authors.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.citrusframework.sharding;
+
+import org.citrusframework.exceptions.CitrusRuntimeException;
+import org.citrusframework.util.SystemProvider;
+
+import java.util.Optional;
+
+import static java.lang.Integer.parseInt;
+import static java.lang.String.valueOf;
+
+/**
+ * A configuration class for sharded test loading and execution withing the citrus framework. It uses environment
+ * variables and system properties to configure the sharding behavior.
+ * <p>
+ * This class is part of the Citrus framework, designed to streamline the process of sharding test cases for efficient
+ * and scalable testing.
+ * <p>
+ * <h3>Configuration Example:</h3>
+ * <p>To configure the sharding behavior, set the following environment variables or system properties:</p>
+ * <ul>
+ *   <li><b>Total number of shards:</b>
+ *     <ul>
+ *       <li>Environment Variable: <code>CITRUS_SHARDING_TOTAL</code></li>
+ *       <li>System Property: <code>citrus.sharding.total</code></li>
+ *       <li>Description: Specifies the total number of shards into which the test cases will be divided.</li>
+ *     </ul>
+ *   </li>
+ *   <li><b>Shard number:</b>
+ *     <ul>
+ *       <li>Environment Variable: <code>CITRUS_SHARDING_NUMBER</code></li>
+ *       <li>System Property: <code>citrus.sharding.number</code></li>
+ *       <li>
+ *           Description: Indicates the specific shard number of the current test loader. This should be a value between
+ *           0 and the total number of shards minus one.
+ *       </li>
+ *     </ul>
+ *   </li>
+ *   <li><b>Shard seed:</b>
+ *     <ul>
+ *       <li>Environment Variable: <code>CITRUS_SHARDING_SEED</code></li>
+ *       <li>System Property: <code>citrus.sharding.seed</code></li>
+ *       <li>
+ *           Description: Specifies a seed value used for shuffling test cases within a shard. Providing a consistent
+ *           seed value ensures the same shuffling order across different executions.
+ *       </li>
+ *     </ul>
+ *   </li>
+ * </ul>
+ *
+ * <h3>Example Usage:</h3>
+ * <p>To configure a system with 4 total shards and assign this instance to shard number 1 (second shard, since
+ * numbering starts at 0), set the environment variables or system properties as follows:</p>
+ * <ul>
+ *   <li>
+ *       Set <code>CITRUS_SHARDING_TOTAL</code> or <code>citrus.sharding.total</code> to 4.
+ *   </li>
+ *   <li>
+ *       Set <code>CITRUS_SHARDING_NUMBER</code> or <code>citrus.sharding.number</code> to 1.
+ *   </li>
+ *   <li>
+ *       Optionally, set a seed for shuffling test cases using <code>CITRUS_SHARDING_SEED</code>
+ *       or <code>citrus.sharding.seed</code>. The total number of shards will be used as see by default.
+ *   </li>
+ * </ul>
+ *
+ * @see Shard
+ */
+public final class ShardingConfiguration {
+
+    public static final String TOTAL_SHARD_NUMBER_PROPERTY_NAME = "citrus.sharding.total";
+    public static final String TOTAL_SHARD_NUMBER_ENV_VAR_NAME = TOTAL_SHARD_NUMBER_PROPERTY_NAME.replace(".", "_").toUpperCase();
+
+    public static final String SHARD_NUMBER_PROPERTY_NAME = "citrus.sharding.number";
+    public static final String SHARD_NUMBER_ENV_VAR_NAME = SHARD_NUMBER_PROPERTY_NAME.replace(".", "_").toUpperCase();
+
+    public static final String SHARD_SEED_PROPERTY_NAME = "citrus.sharding.seed";
+    public static final String SHARD_SEED_ENV_VAR_NAME = SHARD_SEED_PROPERTY_NAME.replace(".", "_").toUpperCase();
+
+    private final int totalNumberOfShards;
+    private final int shardNumber;
+    private final String seed;
+
+    /**
+     * Default sharding configuration which initializes the sharding with system properties and environment variables.
+     */
+    public ShardingConfiguration() {
+        this(new SystemProvider());
+    }
+
+    /**
+     * Constructor that allows for injecting a custom {@link SystemProvider}. This is primarily intended for testing
+     * purposes, enabling the mocking and overriding of system environment and properties.
+     *
+     * @param systemProvider a provider for system environment variables and properties.
+     */
+    protected ShardingConfiguration(SystemProvider systemProvider) {
+        this(getTotalNumberOfShards(systemProvider), getShardNumber(systemProvider), systemProvider);
+    }
+
+    /**
+     * Create a new sharding configuration with explicit total number of shards and shard number.
+     *
+     * @param totalNumberOfShards the total number of shards to be used.
+     * @param shardNumber         the specific shard number for this loader, zero-based.
+     */
+    public ShardingConfiguration(int totalNumberOfShards, int shardNumber) {
+        this(totalNumberOfShards, shardNumber, new SystemProvider());
+    }
+
+    /**
+     * Constructor that sets the total number of shards, shard number, and allows for injecting a
+     * custom {@link SystemProvider}. Primarily used for testing purposes.
+     *
+     * @param totalNumberOfShards the total number of shards.
+     * @param shardNumber         the shard number for this loader, zero-based.
+     * @param systemProvider      a provider for system environment variables and properties.
+     */
+    protected ShardingConfiguration(int totalNumberOfShards, int shardNumber, SystemProvider systemProvider) {
+        this.totalNumberOfShards = totalNumberOfShards;
+        this.shardNumber = shardNumber;
+
+        seed = getSeedOrDefaultValue(systemProvider, totalNumberOfShards);
+
+        sanitizeConfiguration();
+    }
+
+    private static int getTotalNumberOfShards(SystemProvider systemProvider) {
+        return extractEnvOrProperty(systemProvider, TOTAL_SHARD_NUMBER_ENV_VAR_NAME, TOTAL_SHARD_NUMBER_PROPERTY_NAME, 1, "Failed to calculate number of total shards, received string instead of number!");
+    }
+
+    private static int getShardNumber(SystemProvider systemProvider) {
+        return extractEnvOrProperty(systemProvider, SHARD_NUMBER_ENV_VAR_NAME, SHARD_NUMBER_PROPERTY_NAME, 0, "Failed to calculate shard number, received string instead of number!");
+    }
+
+    private static String getSeedOrDefaultValue(SystemProvider systemProvider, int totalNumberOfShards) {
+        return extractEnvOrProperty(systemProvider, SHARD_SEED_ENV_VAR_NAME, SHARD_SEED_PROPERTY_NAME)
+                .orElseGet(() -> valueOf(totalNumberOfShards));
+    }
+
+    private static int extractEnvOrProperty(SystemProvider systemProvider, String envVarName, String fallbackPropertyName, int defaultValue, String numberParseErrorMessage) {
+        try {
+            return parseInt(extractEnvOrProperty(systemProvider, envVarName, fallbackPropertyName)
+                    .orElseGet(() -> valueOf(defaultValue)));
+        } catch (NumberFormatException e) {
+            throw new CitrusRuntimeException(numberParseErrorMessage, e);
+        }
+    }
+
+    private static Optional<String> extractEnvOrProperty(SystemProvider systemProvider, String envVarName, String fallbackPropertyName) {
+        return systemProvider.getEnv(envVarName)
+                .or(() -> systemProvider.getProperty(fallbackPropertyName));
+    }
+
+    private void sanitizeConfiguration() {
+        if (totalNumberOfShards <= 0) {
+            throw new CitrusRuntimeException("Number of total shards must be configured!");
+        } else if (shardNumber < 0) {
+            throw new CitrusRuntimeException("Shard number cannot be negative!");
+        } else if (shardNumber >= totalNumberOfShards) {
+            throw new CitrusRuntimeException("Shard number must be less than the total number of shards!");
+        }
+    }
+
+    public int getTotalNumberOfShards() {
+        return totalNumberOfShards;
+    }
+
+    public int getShardNumber() {
+        return shardNumber;
+    }
+
+    public int getSeed() {
+        return seed.hashCode();
+    }
+}

core/citrus-base/src/main/java/org/citrusframework/util/SystemProvider.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2024 the original author or authors.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.citrusframework.util;
+
+import java.util.Optional;
+
+import static java.util.Optional.ofNullable;
+
+public final class SystemProvider {
+
+    public Optional<String> getEnv(String envVarName) {
+        return ofNullable(System.getenv(envVarName));
+    }
+
+    public Optional<String> getProperty(String propertyName) {
+        return ofNullable(System.getProperty(propertyName));
+    }
+}