Merge branch 'master' into ib/misc-security

Author: ibodrov

CHANGELOG.md

@@ -1,5 +1,62 @@
 # Change Log
 
+## [2.25.1] - 2025-04-26
+
+### Changed
+
+- runtime-v2: set lowercase loop mode value in schema
+([#1122](https://github.com/walmartlabs/concord/pull/1122));
+- http-tasks: refactor tests, support for parallel tests
+([#1123](https://github.com/walmartlabs/concord/pull/1123));
+- project: update bouncycastle dependency versions
+([#1127](https://github.com/walmartlabs/concord/pull/1127));
+- concord-server: handle invalid regex in dispatcher
+requirements
+([#1128](https://github.com/walmartlabs/concord/pull/1128));
+- project: downgrade nimbus-jose-jwt
+([#1129](https://github.com/walmartlabs/concord/pull/1129)).
+
+
+
+## [2.25.0] - 2025-03-23
+
+### Changed
+
+- concord-agent: replace simple immutables with records
+([#1091](https://github.com/walmartlabs/concord/pull/1091));
+- concord-server, concord-server-db: generate UUIDs on
+the server, use UUID v7
+([#1106](https://github.com/walmartlabs/concord/pull/1106));
+- concord-server: upgrade Shiro to 2.x
+([#1107](https://github.com/walmartlabs/concord/pull/1107));
+- concord-server, concord-client2: uncomment valid code
+([#1108](https://github.com/walmartlabs/concord/pull/1108));
+- concord-server: remove more @Named usage
+([#1110](https://github.com/walmartlabs/concord/pull/1110));
+- concord-server: remove deprecated SecretEntry
+([#1111](https://github.com/walmartlabs/concord/pull/1111));
+- concord-server: remove deprecated /logs/* endpoint
+([#1113](https://github.com/walmartlabs/concord/pull/1113));
+- project: update dependencies
+([#1114](https://github.com/walmartlabs/concord/pull/1114));
+- project: update README
+([#1115](https://github.com/walmartlabs/concord/pull/1115));
+- docker-images: update JDK versions
+([#1116](https://github.com/walmartlabs/concord/pull/1116));
+- project: update dependencies
+([#1118](https://github.com/walmartlabs/concord/pull/1118));
+- project: remove unused dependencies
+([#1119](https://github.com/walmartlabs/concord/pull/1119)).
+
+### Breaking
+
+- project: remove client1
+([#1013](https://github.com/walmartlabs/concord/pull/1013));
+- concord-server: remove deprecated features
+([#1112](https://github.com/walmartlabs/concord/pull/1112)).
+
+
+
 ## [2.24.0] - 2025-03-15
 
 ### Added
@@ -63,6 +120,7 @@ array
 ([#1090](https://github.com/walmartlabs/concord/pull/1090)).
 
 
+
 ## [2.22.0] - 2025-02-14
 
 ### Added

NOTES.md

@@ -12,13 +12,17 @@ Some classes require explicit binding using `Multibinder.newSetBinder`:
 - BackgroundTask
 - Component
 - ContextHandlerConfigurator
+- CustomBeanMethodResolver
 - CustomEnqueueProcessor
+- CustomTaskMethodResolver
 - ExceptionMapper
+- ExecutionListener
 - Filter
 - FilterChainConfigurator
 - FilterHolder
 - GaugeProvider
 - HttpServlet
+- LogAppender
 - ModeProcessor
 - PolicyApplier
 - ProcessEventListener
@@ -32,4 +36,6 @@ Some classes require explicit binding using `Multibinder.newSetBinder`:
 - SecretStore
 - ServletContextListener
 - ServletHolder
-- UserInfoProvider
+- TaskCallListener
+- TaskProvider
+- UserInfoProvider

README.md

@@ -67,7 +67,7 @@ npm run start
 
 Prerequisites:
 
-- Git 2.3+
+- Git 2.18+
 - Docker, listening on `tcp://127.0.0.1:2375`;
 - Ansible 2.6.0+ must be installed and available in `$PATH`.
   See [the official documentation](http://docs.ansible.com/ansible/intro_installation.html);
@@ -107,17 +107,31 @@ See the [examples](examples) directory.
   ```
   $ ./mvnw release:prepare release:perform
   ```
-- push the new tag:
+- update and commit the CHANGELOG.md file
+  ```
+  $ git add CHANGELOG.md
+  $ git commit -m 'update changelog'
+  ```
+- push the new tag and the master branch:
   ```
   $ git push origin RELEASE_TAG
+  $ git push origin master
   ```
-- sync to [Central](https://central.sonatype.com/);
 - build and push the Docker images:
   ```
   $ git checkout RELEASE_TAG
   $ ./mvnw -f docker-images clean package -Pdocker
   $ ./docker-images/push.sh RELEASE_TAG
   ```
+- sync to [Sonatype](https://oss.sonatype.org/);
+- check the Central repository if the sync is complete:
+  ```
+  https://repo.maven.apache.org/maven2/com/walmartlabs/concord/parent/RELEASE_TAG
+  ```
+- once the sync is complete, push the `latest` Docker images:
+  ```
+  $ ./docker-images/push.sh latest
+  ```
 
 ## Development Notes
 

agent-operator/README.md

@@ -49,7 +49,7 @@ present in your minikube instance):
 5. Create the custom resource:
   ```
   $ minikube kubectl -- create -f deploy/crds/crd.yml -n default
-  $ minikube kubectl -- create -f deploy/crds/cr.yml -n default
+  $ minikube kubectl -- create -f deploy/crds/example.agentpool.yml -n default
   ```
 6. Start the operator:
   ```
@@ -103,7 +103,7 @@ minikube kubectl -- logs -f <pod_name> -c <container_name> -n default
   $ minikube kubectl -- create -f deploy/operator.yml
   ```
 4. Check the operator's pod logs;
-5. Deploy one or more CRs using `deploy/crds/cr.yml` as a template.
+5. Deploy one or more CRs using `deploy/crds/example.agentpool.yml` as a template.
 
 ## How To Release New Versions
 

agent-operator/deploy/crds/cr.yml …erator/deploy/crds/example.agentpool.ymlagent-operator/deploy/crds/cr.yml renamed to agent-operator/deploy/crds/example.agentpool.yml agent-operator/deploy/crds/cr.yml renamed to agent-operator/deploy/crds/example.agentpool.yml

@@ -12,7 +12,7 @@
 apiVersion: concord.walmartlabs.com/v1alpha1
 kind: AgentPool
 metadata:
-  name: example-agentpool
+  name: example
 spec:
   queueSelector:
     agent:
@@ -116,10 +116,10 @@ spec:
               name: process-tmp
           resources:
             requests:
-              cpu: "1"
+              cpu: 1
               memory: "1G"
             limits:
-              cpu: "2"
+              cpu: 2
               memory: "2G"
           env:
             - name: CONCORD_TMP_DIR
@@ -150,7 +150,7 @@ spec:
             preStop:
               exec:
                 command:
-                  - "sh"
+                  - "/bin/bash"
                   - "/opt/concord/hooks/preStopHook.sh"
       volumes:
         - name: cfg

agent-operator/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>com.walmartlabs.concord</groupId>
         <artifactId>parent</artifactId>
-        <version>2.24.1-SNAPSHOT</version>
+        <version>2.25.2-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -21,14 +21,6 @@
         </dependency>
 
         <!-- k8s stuff -->
-        <dependency>
-            <groupId>io.fabric8</groupId>
-            <artifactId>crd-generator-api</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.fabric8</groupId>
-            <artifactId>crd-generator-apt</artifactId>
-        </dependency>
         <dependency>
             <groupId>io.fabric8</groupId>
             <artifactId>kubernetes-client</artifactId>
@@ -47,13 +39,16 @@
         </dependency>
         <dependency>
             <groupId>org.bouncycastle</groupId>
-            <artifactId>bcprov-ext-jdk15on</artifactId>
+            <artifactId>bcprov-jdk18on</artifactId>
         </dependency>
         <dependency>
             <groupId>org.bouncycastle</groupId>
-            <artifactId>bcpkix-jdk15on</artifactId>
+            <artifactId>bcpkix-jdk18on</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcutil-jdk18on</artifactId>
         </dependency>
-
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-core</artifactId>
@@ -89,12 +84,6 @@
             <artifactId>jaxb-impl</artifactId>
         </dependency>
 
-        <!-- required at runtime by the fabric8 client, but not included automatically for some reason -->
-        <dependency>
-            <groupId>commons-collections</groupId>
-            <artifactId>commons-collections</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>org.immutables</groupId>
             <artifactId>value</artifactId>
@@ -107,11 +96,6 @@
             <artifactId>junit-jupiter-api</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter-engine</artifactId>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 
     <build>

agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/Operator.java

@@ -20,8 +20,6 @@
  * =====
  */
 
-
-import com.walmartlabs.concord.agentoperator.agent.AgentClientFactory;
 import com.walmartlabs.concord.agentoperator.crd.AgentPool;
 import com.walmartlabs.concord.agentoperator.crd.AgentPoolList;
 import com.walmartlabs.concord.agentoperator.scheduler.AutoScalerFactory;

agent-operator/src/main/java/com/walmartlabs/concord/agentoperator/scheduler/DefaultAutoScaler.java

@@ -30,7 +30,6 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.function.Function;
 
 public class DefaultAutoScaler implements AutoScaler {
@@ -93,7 +92,8 @@ public AgentPoolInstance apply(AgentPoolInstance i) throws IOException {
 
         AgentPoolConfiguration cfg = i.getResource().getSpec();
 
-        if (!canBeScaledUp.apply(i) && !canBeScaledDown.apply(i)) {
+        boolean canBeScaled = canBeScaledUp.apply(i) || canBeScaledDown.apply(i);
+        if (!canBeScaled) {
             // was updated recently, skipping
             return i;
         }
@@ -222,7 +222,6 @@ private int tryScaleDown(AgentPoolConfiguration cfg, AgentPoolInstance i, int po
     private static int getProcessCount(AgentPoolConfiguration cfg, List<ProcessQueueEntry> processQueueEntries) {
         return (int) processQueueEntries.stream()
                 .map(ProcessQueueEntry::getRequirements)
-                .filter(Objects::nonNull)
                 .filter(a -> isEmpty(cfg.getQueueSelector()) || Matcher.matches(a, cfg.getQueueSelector()))
                 .count();
     }

agent-operator/src/main/resources/prestop-hook.sh

@@ -1,30 +1,40 @@
-workers="1"
-num_retries=0
+#!/usr/bin/env bash
+
+echo "PreStop hook started at $(date)"
+
 MAX_RETRIES=5
+RETRY_DELAY=1
+
+current_workers="1"
+num_retries=0
 
-while [ "$workers" != "0" ] && [ "$num_retries" -lt "$MAX_RETRIES" ]
+while [ "$current_workers" != "0" ] && [ "$num_retries" -lt "$MAX_RETRIES" ]
 do
-  echo "[$HOSTNAME]: Agent is still executing a process.. enabling maintenance mode and checking the number of workers"
-  mmode_response=`wget -qO - --post-data="" http://127.0.0.1:8010/maintenance-mode`
+  echo "[$HOSTNAME]: Agent is still executing a process.. enabling maintenance mode and checking the number of current_workers"
 
-  mmode_enabled=`echo $mmode_response | sed -n 's/^.*\"maintenanceMode\":\([a-z]*\).*$/\1/p'`
-  if [ "$mmode_enabled" == "true" ]; then
-    echo "[$HOSTNAME]: Maintenance mode enabled: $mmode_enabled"
+  response=$(exec 3<>/dev/tcp/127.0.0.1/8010;
+             echo -e "POST /maintenance-mode HTTP/1.1\r\nHost: 127.0.0.1:8010\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" >&3;
+             cat <&3;
+             exec 3>&-)
 
-    workers=`echo $mmode_response | sed -n 's/^.*\"workersAlive\":\([0-9]*\).*$/\1/p'`
-    echo "[$HOSTNAME]: Number of workers: $workers"
+  mmode_response=$(echo "$response" | sed -n '/^\r*$/,$p' | tail -n +2)
+  mmode_enabled=$(echo "$mmode_response" | sed -n 's/^.*\"maintenanceMode\":\([a-z]*\).*$/\1/p')
 
+  if [ "$mmode_enabled" == "true" ]; then
+    echo "[$HOSTNAME]: Maintenance mode enabled: $mmode_enabled"
+    current_workers=$(echo "$mmode_response" | sed -n 's/^.*\"workersAlive\":\([0-9]*\).*$/\1/p')
+    echo "[$HOSTNAME]: Number of current_workers: $current_workers"
   else
     echo "[$HOSTNAME]: trouble enabling maintenance mode"
-    num_retries=`expr $num_retries + 1`
+    num_retries=$(("$num_retries" + 1))
   fi
 
-sleep 5
+  sleep ${RETRY_DELAY}
 done
 
 if [ "$num_retries" -ge "$MAX_RETRIES" ]; then
   echo "[$HOSTNAME]: Number of retries to enable exceeded $MAX_RETRIES times. Exiting ..."
   exit 1
 fi
 
-echo "[$HOSTNAME]: There are no processes running on this agent. Terminating..."
+echo "[$HOSTNAME]: There are no processes running on this agent. Terminating..."

agent/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>com.walmartlabs.concord</groupId>
         <artifactId>parent</artifactId>
-        <version>2.24.1-SNAPSHOT</version>
+        <version>2.25.2-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -85,10 +85,6 @@
             <groupId>com.fasterxml.jackson.datatype</groupId>
             <artifactId>jackson-datatype-jdk8</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.datatype</groupId>
-            <artifactId>jackson-datatype-jsr310</artifactId>
-        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>
@@ -105,10 +101,6 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-compress</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.jboss.spec.javax.ws.rs</groupId>
-            <artifactId>jboss-jaxrs-api_2.1_spec</artifactId>
-        </dependency>
         <dependency>
             <groupId>com.typesafe</groupId>
             <artifactId>config</artifactId>
@@ -144,11 +136,6 @@
             <artifactId>junit-jupiter-api</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter-engine</artifactId>
-            <scope>test</scope>
-        </dependency>
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>