Merge remote-tracking branch 'upstream/trunk' into kip848_await_async_commit_legacy

Author: lucasbru

build.gradle

@@ -114,6 +114,13 @@ ext {
   repo = file("$rootDir/.git").isDirectory() ? Grgit.open(currentDir: project.getRootDir()) : null
 
   commitId = determineCommitId()
+
+  addParametersForTests = { name, options ->
+    // -parameters generates arguments with parameter names in TestInfo#getDisplayName.
+    // ref: https://github.com/junit-team/junit5/blob/4c0dddad1b96d4a20e92a2cd583954643ac56ac0/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTest.java#L161-L164
+    if (name == "compileTestJava" || name == "compileTestScala")
+      options.compilerArgs << "-parameters"
+  }
 }
 
 allprojects {
@@ -278,11 +285,9 @@ subprojects {
     // --source/--target 8 is deprecated in Java 20, suppress warning until Java 8 support is dropped in Kafka 4.0
     if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_20))
       options.compilerArgs << "-Xlint:-options"
-  }
 
-  // -parameters generates arguments with parameter names in TestInfo#getDisplayName.
-  // ref: https://github.com/junit-team/junit5/blob/4c0dddad1b96d4a20e92a2cd583954643ac56ac0/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTest.java#L161-L164
-  compileTestJava.options.compilerArgs.add "-parameters"
+    addParametersForTests(name, options)
+  }
 
   // We should only set this if Java version is < 9 (--release is recommended for >= 9), but the Scala plugin for IntelliJ sets
   // `-target` incorrectly if this is unset
@@ -703,6 +708,8 @@ subprojects {
     if (versions.baseScala == "2.13" || JavaVersion.current().isJava9Compatible())
       scalaCompileOptions.additionalParameters += ["-release", String.valueOf(minJavaVersion)]
 
+    addParametersForTests(name, options)
+
     configure(scalaCompileOptions.forkOptions) {
       memoryMaximumSize = defaultMaxHeapSize
       jvmArgs = defaultJvmArgs
@@ -1837,6 +1844,7 @@ project(':storage') {
     testImplementation project(':clients').sourceSets.test.output
     testImplementation project(':core')
     testImplementation project(':core').sourceSets.test.output
+    testImplementation project(':server')
     testImplementation project(':server-common')
     testImplementation project(':server-common').sourceSets.test.output
     testImplementation libs.hamcrest
@@ -2025,6 +2033,7 @@ project(':tools') {
     testImplementation project(':connect:runtime')
     testImplementation project(':connect:runtime').sourceSets.test.output
     testImplementation project(':storage:storage-api').sourceSets.main.output
+    testImplementation project(':group-coordinator')
     testImplementation libs.junitJupiter
     testImplementation libs.mockitoCore
     testImplementation libs.mockitoJunitJupiter // supports MockitoExtension
@@ -2177,10 +2186,13 @@ project(':streams') {
     testCompileOnly project(':streams:test-utils')
 
     testImplementation project(':clients').sourceSets.test.output
+    testImplementation project(':server')
     testImplementation project(':core')
     testImplementation project(':tools')
     testImplementation project(':core').sourceSets.test.output
     testImplementation project(':storage')
+    testImplementation project(':group-coordinator')
+    testImplementation project(':transaction-coordinator')
     testImplementation project(':server-common')
     testImplementation project(':server-common').sourceSets.test.output
     testImplementation project(':server')
@@ -2192,6 +2204,7 @@ project(':streams') {
     testImplementation libs.hamcrest
     testImplementation libs.mockitoCore
     testImplementation libs.mockitoJunitJupiter // supports MockitoExtension
+    testImplementation project(':group-coordinator')
 
     testRuntimeOnly project(':streams:test-utils')
     testRuntimeOnly libs.slf4jlog4j
@@ -2328,6 +2341,7 @@ project(':streams:streams-scala') {
       // So we make sure to not include it in the dependencies.
       api libs.scalaCollectionCompat
     }
+    testImplementation project(':group-coordinator')
     testImplementation project(':core')
     testImplementation project(':core').sourceSets.test.output
     testImplementation project(':server-common').sourceSets.test.output
@@ -2991,12 +3005,15 @@ project(':connect:runtime') {
 
     testImplementation project(':clients').sourceSets.test.output
     testImplementation project(':core')
+    testImplementation project(':server')
     testImplementation project(':metadata')
     testImplementation project(':core').sourceSets.test.output
     testImplementation project(':server-common')
     testImplementation project(':server')
+    testImplementation project(':group-coordinator')
     testImplementation project(':storage')
     testImplementation project(':connect:test-plugins')
+    testImplementation project(':group-coordinator')
 
     testImplementation libs.easymock
     testImplementation libs.junitJupiterApi
@@ -3207,6 +3224,7 @@ project(':connect:mirror') {
     testImplementation project(':connect:runtime').sourceSets.test.output
     testImplementation project(':core')
     testImplementation project(':core').sourceSets.test.output
+    testImplementation project(':server')
 
     testRuntimeOnly project(':connect:runtime')
     testRuntimeOnly libs.slf4jlog4j

checkstyle/import-control-core.xml

@@ -37,7 +37,7 @@
   <allow pkg="kafka.serializer" />
   <allow pkg="org.apache.kafka.common" />
   <allow pkg="org.mockito" class="AssignmentsManagerTest"/>
-
+  <allow pkg="org.apache.kafka.server"/>
   <!-- see KIP-544 for why KafkaYammerMetrics should be used instead of the global default yammer metrics registry
        https://cwiki.apache.org/confluence/display/KAFKA/KIP-544%3A+Make+metrics+exposed+via+JMX+configurable -->
   <disallow class="com.yammer.metrics.Metrics" />

checkstyle/import-control-server-common.xml

@@ -111,9 +111,13 @@
                 <allow class="org.apache.kafka.server.util.ShutdownableThread" />
             </subpackage>
         </subpackage>
+        <subpackage name="config">
+            <allow pkg="org.apache.kafka.server"/>
+        </subpackage>
     </subpackage>
 
     <subpackage name="admin">
         <allow pkg="org.apache.kafka.server.common" />
     </subpackage>
+
 </import-control>

checkstyle/import-control-server.xml

@@ -82,6 +82,9 @@
       <allow class="org.apache.kafka.server.authorizer.AuthorizableRequestContext" />
       <allow pkg="org.apache.kafka.server.telemetry" />
     </subpackage>
+    <subpackage name="config">
+      <allow pkg="org.apache.kafka.server" />
+    </subpackage>
   </subpackage>
 
   <subpackage name="security">

checkstyle/import-control-storage.xml

@@ -111,6 +111,7 @@
         <allow pkg="org.apache.kafka.server.log" />
         <allow pkg="org.apache.kafka.server.log.remote" />
         <allow pkg="org.apache.kafka.server.log.remote.storage" />
+        <allow pkg="org.apache.kafka.server.config" />
 
         <allow pkg="org.apache.kafka.test" />
         <subpackage name="actions">

checkstyle/import-control.xml

@@ -303,6 +303,7 @@
     <allow pkg="kafka.admin" />
     <allow pkg="kafka.server" />
     <allow pkg="org.apache.kafka.storage.internals" />
+    <allow pkg="org.apache.kafka.server.config" />
     <allow pkg="org.apache.kafka.server.common" />
     <allow pkg="org.apache.kafka.clients" />
     <allow pkg="org.apache.kafka.clients.admin" />
@@ -323,8 +324,8 @@
 
     <subpackage name="consumer">
       <allow pkg="org.apache.kafka.tools"/>
-
       <subpackage name="group">
+        <allow pkg="org.apache.kafka.coordinator.group"/>
         <allow pkg="kafka.api"/>
         <allow pkg="kafka.security"/>
         <allow pkg="kafka.zk" />
@@ -413,6 +414,8 @@
       <allow pkg="org.apache.kafka.tools" />
       <allow pkg="org.apache.kafka.server.config" />
       <allow class="org.apache.kafka.storage.internals.log.CleanerConfig" />
+      <allow class="org.apache.kafka.coordinator.transaction.TransactionLogConfigs" />
+      <allow pkg="org.apache.kafka.coordinator.group" />
     </subpackage>
 
     <subpackage name="test">
@@ -483,6 +486,8 @@
     <allow pkg="org.apache.kafka.connect.components"/>
     <allow pkg="org.apache.kafka.clients" />
     <allow pkg="org.apache.kafka.test"/>
+    <!-- for testing -->
+    <allow pkg="org.apache.kafka.coordinator.group" />
 
     <subpackage name="source">
       <allow pkg="org.apache.kafka.connect.connector" />
@@ -532,6 +537,7 @@
       <!-- for tests -->
       <allow pkg="org.apache.kafka.connect.integration" />
       <allow pkg="org.apache.kafka.connect.mirror" />
+      <allow pkg="org.apache.kafka.server.config" />
       <allow pkg="kafka.server" />
       <subpackage name="rest">
         <allow pkg="javax.ws.rs" />
@@ -595,6 +601,7 @@
       <allow pkg="com.fasterxml.jackson.annotation" />
       <allow pkg="com.fasterxml.jackson.databind" />
       <subpackage name="clusters">
+        <allow pkg="org.apache.kafka.server.config" />
         <allow pkg="kafka.cluster" />
         <allow pkg="kafka.server" />
         <allow pkg="kafka.zk" />

checkstyle/suppressions.xml

@@ -99,7 +99,7 @@
               files="(AbstractRequest|AbstractResponse|KerberosLogin|WorkerSinkTaskTest|TransactionManagerTest|SenderTest|KafkaAdminClient|ConsumerCoordinatorTest|KafkaAdminClientTest|KafkaRaftClientTest).java"/>
 
     <suppress checks="NPathComplexity"
-              files="(ConsumerCoordinator|BufferPool|MetricName|Node|ConfigDef|RecordBatch|SslFactory|SslTransportLayer|MetadataResponse|KerberosLogin|Selector|Sender|Serdes|TokenInformation|Agent|Values|PluginUtils|MiniTrogdorCluster|TasksRequest|KafkaProducer|AbstractStickyAssignor|KafkaRaftClient|Authorizer|FetchSessionHandler|RecordAccumulator).java"/>
+              files="(ConsumerCoordinator|BufferPool|MetricName|Node|ConfigDef|RecordBatch|SslFactory|SslTransportLayer|MetadataResponse|KerberosLogin|Selector|Sender|Serdes|TokenInformation|Agent|Values|PluginUtils|MiniTrogdorCluster|TasksRequest|KafkaProducer|AbstractStickyAssignor|KafkaRaftClient|Authorizer|FetchSessionHandler|RecordAccumulator|Shell).java"/>
 
     <suppress checks="(JavaNCSS|CyclomaticComplexity|MethodLength)"
               files="CoordinatorClient.java"/>

clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java

@@ -44,11 +44,7 @@ public InFlightRequests(int maxInFlightRequestsPerConnection) {
      */
     public void add(NetworkClient.InFlightRequest request) {
         String destination = request.destination;
-        Deque<NetworkClient.InFlightRequest> reqs = this.requests.get(destination);
-        if (reqs == null) {
-            reqs = new ArrayDeque<>();
-            this.requests.put(destination, reqs);
-        }
+        Deque<NetworkClient.InFlightRequest> reqs = this.requests.computeIfAbsent(destination, k -> new ArrayDeque<>());
         reqs.addFirst(request);
         inFlightRequestCount.incrementAndGet();
     }

clients/src/main/java/org/apache/kafka/clients/KafkaClient.java

@@ -70,7 +70,7 @@ public interface KafkaClient extends Closeable {
 
     /**
      * Check if the connection of the node has failed, based on the connection state. Such connection failure are
-     * usually transient and can be resumed in the next {@link #ready(org.apache.kafka.common.Node, long)} }
+     * usually transient and can be resumed in the next {@link #ready(org.apache.kafka.common.Node, long)}
      * call, but there are cases where transient failures needs to be caught and re-acted upon.
      *
      * @param node the node to check

clients/src/main/java/org/apache/kafka/clients/admin/AdminClient.java

@@ -25,7 +25,7 @@
  *
  * Client code should use the newer {@link Admin} interface in preference to this class.
  *
- * This class may be removed in a later release, but has not be marked as deprecated to avoid unnecessary noise.
+ * This class may be removed in a later release, but has not been marked as deprecated to avoid unnecessary noise.
  */
 public abstract class AdminClient implements Admin {