Merge branch 'master' into HDDS-11463

Author: slfan1989

.github/workflows/ci.yml

@@ -111,7 +111,7 @@ jobs:
   build:
     needs:
       - build-info
-    if: needs.build-info.outputs.needs-build == 'true'
+    if: needs.build-info.outputs.needs-build == 'true' || needs.build-info.outputs.needs-integration-tests == 'true'
     uses: ./.github/workflows/check.yml
     with:
       java-version: ${{ needs.build-info.outputs.java-version }}
@@ -265,7 +265,10 @@ jobs:
   integration:
     needs:
       - build-info
+      - build
       - basic
+      - dependency
+      - license
     if: needs.build-info.outputs.needs-integration-tests == 'true'
     uses: ./.github/workflows/check.yml
     secrets: inherit

.github/workflows/intermittent-test-check.yml

@@ -30,7 +30,7 @@ on:
         required: false
       submodule:
         description: Submodule
-        default: 'AUTO'
+        default: 'ozone-integration-test'
         required: true
       iterations:
         description: Number of Iterations per split
@@ -94,54 +94,11 @@ jobs:
     with:
       repo: ${{ github.event.inputs.ratis-repo || format('{0}/ratis', github.repository_owner) }}
       ref: ${{ github.event.inputs.ratis-ref }}
-  find-tests:
-    if: ${{ always() && github.event.inputs.submodule == 'AUTO' }}
-    needs:
-      - prepare-job
-    runs-on: ubuntu-24.04
-    outputs:
-      modules: ${{ steps.modules.outputs.modules }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.inputs.ref }}
-      - name: Cache for maven dependencies
-        uses: actions/cache/restore@v4
-        with:
-          path: |
-            ~/.m2/repository/*/*/*
-            !~/.m2/repository/org/apache/ozone
-          key: maven-repo-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            maven-repo-
-      - name: Setup java
-        uses: actions/setup-java@v4
-        with:
-          distribution: 'temurin'
-          java-version: ${{ github.event.inputs.java-version }}
-      - name: Find tests
-        run: |
-          # find tests to be run in splits by running them with very short timeout
-          hadoop-ozone/dev-support/checks/junit.sh -DexcludedGroups="native|slow|unhealthy" -DskipShade \
-              -Dtest="$TEST_CLASS,Abstract*Test*\$*" \
-              -Dsurefire.fork.timeout=1 -Dmaven-surefire-plugin.version=${{ env.SUREFIRE_VERSION }} \
-              || true # ignore errors
-        env:
-          ITERATIONS: 1
-      - name: Find modules
-        id: modules
-        run: |
-          grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' target/unit/output.log | grep -B1 'Running org.apache'
-          modules=$(grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' target/unit/output.log | grep -B1 'Running org.apache' \
-              | grep surefire | cut -f2 -d'@' | awk '{ print $1 }' | sed 's/^/:/' | xargs | sed -e 's/ /,/g')
-          echo "modules=$modules" >> $GITHUB_OUTPUT
-        if: ${{ !cancelled() }}
   build:
     if: ${{ always() }}
     needs:
       - prepare-job
       - ratis
-      - find-tests
     runs-on: ubuntu-24.04
     timeout-minutes: 60
     steps:
@@ -181,11 +138,7 @@ jobs:
             args="$args -Dgrpc.protobuf-compile.version=${{ needs.ratis.outputs.protobuf-version }}"
           fi
 
-          if [[ "${{ github.event.inputs.submodule }}" != "AUTO" ]]; then
-            args="$args -am -pl :${{ github.event.inputs.submodule }}"
-          elif [[ -n "${{ needs.find-tests.outputs.modules }}" ]]; then
-            args="$args -am -pl ${{ needs.find-tests.outputs.modules }}"
-          fi
+          args="$args -am -pl :${{ github.event.inputs.submodule }}"
 
           hadoop-ozone/dev-support/checks/build.sh $args
       - name: Store Maven repo for tests
@@ -201,7 +154,6 @@ jobs:
       - prepare-job
       - ratis
       - build
-      - find-tests
     name: Run-Split
     runs-on: ubuntu-24.04
     strategy:
@@ -256,11 +208,7 @@ jobs:
             args="$args -Dgrpc.protobuf-compile.version=${{ needs.ratis.outputs.protobuf-version }}"
           fi
 
-          if [[ "${{ github.event.inputs.submodule }}" != "AUTO" ]]; then
-            args="$args -pl :${{ github.event.inputs.submodule }}"
-          elif [[ -n "${{ needs.find-tests.outputs.modules }}" ]]; then
-            args="$args -pl ${{ needs.find-tests.outputs.modules }}"
-          fi
+          args="$args -pl :${{ github.event.inputs.submodule }}"
 
           if [ "$TEST_METHOD" = "ALL" ]; then
             echo "Running all tests from $TEST_CLASS"
@@ -292,12 +240,12 @@ jobs:
         uses: actions/download-artifact@v4
       - name: Count failures
         run: |
-          failures=$(find . -name 'summary.txt' | grep -v 'iteration' | xargs grep -v 'exit code: 0' | wc -l)
+          failures=$(find . -name 'summary.txt' | grep --text -v 'iteration' | xargs grep --text -v 'exit code: 0' | wc -l)
           echo "Total failures: $failures"
           if [[ $failures -gt 0 ]]; then
             echo ""
             echo "Failed runs:"
-            grep 'exit code: 1' */summary.txt | grep -o 'split.*teration [0-9]*' | sed -e 's/.summary.txt:/ /' -e 's/-/ /' | sort -g -k2 -k4
+            grep --text 'exit code: 1' */summary.txt | grep --text -o 'split.*teration [0-9]*' | sed -e 's/.summary.txt:/ /' -e 's/-/ /' | sort -g -k2 -k4
             echo ""
             exit 1
           fi

.mvn/extensions.xml

@@ -29,6 +29,6 @@
   <extension>
     <groupId>com.gradle</groupId>
     <artifactId>common-custom-user-data-maven-extension</artifactId>
-    <version>2.0.2</version>
+    <version>2.0.3</version>
   </extension>
 </extensions>

dev-support/byteman/skip-notify-group-remove.btm

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+RULE skip notifyGroupRemove
+CLASS org.apache.hadoop.ozone.container.common.transport.server.ratis.ContainerStateMachine
+METHOD notifyGroupRemove
+AT ENTRY
+IF TRUE
+DO
+    System.out.println("[" + java.time.LocalDateTime.now() + "] BYTEMAN: " +
+    "Skip notifyGroupRemove in ContainerStateMachine");
+    return;
+ENDRULE

dev-support/byteman/skip-put-block.btm

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+RULE Block putBlock
+CLASS org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl
+METHOD putBlock
+AT ENTRY
+IF TRUE
+DO
+    System.out.println("[" + java.time.LocalDateTime.now() + "] BYTEMAN: " +
+    "Blocking putBlock in BlockManagerImpl");
+    return 0;
+ENDRULE

hadoop-hdds/client/pom.xml

@@ -110,16 +110,6 @@
 
   <build>
     <plugins>
-      <plugin>
-        <groupId>com.coderplus.maven.plugins</groupId>
-        <artifactId>copy-rename-maven-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>rename-generated-config</id>
-            <phase>process-classes</phase>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <groupId>com.github.spotbugs</groupId>
         <artifactId>spotbugs-maven-plugin</artifactId>

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java

@@ -17,8 +17,6 @@
 
 package org.apache.hadoop.hdds.scm.storage;
 
-import static org.apache.hadoop.hdds.client.ReplicationConfig.getLegacyFactor;
-
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import java.io.EOFException;
@@ -32,7 +30,6 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.BlockData;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
@@ -114,6 +111,8 @@ public class BlockInputStream extends BlockExtendedInputStream {
 
   private final Function<BlockID, BlockLocationInfo> refreshFunction;
 
+  private BlockData blockData;
+
   public BlockInputStream(
       BlockLocationInfo blockInfo,
       Pipeline pipeline,
@@ -155,7 +154,6 @@ public synchronized void initialize() throws IOException {
       return;
     }
 
-    BlockData blockData = null;
     List<ChunkInfo> chunks = null;
     IOException catchEx = null;
     do {
@@ -295,10 +293,7 @@ private void setPipeline(Pipeline pipeline) throws IOException {
     boolean okForRead =
         pipeline.getType() == HddsProtos.ReplicationType.STAND_ALONE
             || pipeline.getType() == HddsProtos.ReplicationType.EC;
-    Pipeline readPipeline = okForRead ? pipeline : Pipeline.newBuilder(pipeline)
-        .setReplicationConfig(StandaloneReplicationConfig.getInstance(
-            getLegacyFactor(pipeline.getReplicationConfig())))
-        .build();
+    Pipeline readPipeline = okForRead ? pipeline : pipeline.copyForRead();
     pipelineRef.set(readPipeline);
   }
 
@@ -552,8 +547,7 @@ public long getLength() {
     return length;
   }
 
-  @VisibleForTesting
-  synchronized int getChunkIndex() {
+  public synchronized int getChunkIndex() {
     return chunkIndex;
   }
 
@@ -616,9 +610,12 @@ private void handleReadError(IOException cause) throws IOException {
     refreshBlockInfo(cause);
   }
 
-  @VisibleForTesting
   public synchronized List<ChunkInputStream> getChunkStreams() {
     return chunkStreams;
   }
 
+  public BlockData getStreamBlockData() {
+    return blockData;
+  }
+
 }

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java

@@ -742,4 +742,8 @@ public synchronized void unbuffer() {
   public ByteBuffer[] getCachedBuffers() {
     return BufferUtils.getReadOnlyByteBuffers(buffers);
   }
+
+  public ChunkInfo getChunkInfo() {
+    return chunkInfo;
+  }
 }

hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/client/TestHddsClientUtils.java

@@ -362,6 +362,7 @@ void testVerifyKeyName() throws IllegalArgumentException {
 
     for (String name : validNames) {
       HddsClientUtils.verifyKeyName(name);
+      HddsClientUtils.verifyKeyName(name + OzoneConsts.FS_FILE_COPYING_TEMP_SUFFIX);
     }
   }
 

hadoop-hdds/common/pom.xml

@@ -47,11 +47,6 @@
       <groupId>com.github.stephenc.jcip</groupId>
       <artifactId>jcip-annotations</artifactId>
     </dependency>
-    <dependency>
-      <groupId>com.google.errorprone</groupId>
-      <artifactId>error_prone_annotations</artifactId>
-      <optional>true</optional>
-    </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
@@ -76,10 +71,6 @@
       <groupId>info.picocli</groupId>
       <artifactId>picocli</artifactId>
     </dependency>
-    <dependency>
-      <groupId>io.dropwizard.metrics</groupId>
-      <artifactId>metrics-core</artifactId>
-    </dependency>
     <dependency>
       <groupId>io.jaegertracing</groupId>
       <artifactId>jaeger-core</artifactId>
@@ -101,10 +92,6 @@
       <groupId>jakarta.xml.bind</groupId>
       <artifactId>jakarta.xml.bind-api</artifactId>
     </dependency>
-    <dependency>
-      <groupId>javax.annotation</groupId>
-      <artifactId>javax.annotation-api</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>
@@ -113,10 +100,6 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.ozone</groupId>
-      <artifactId>hdds-annotation-processing</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-config</artifactId>
@@ -165,14 +148,6 @@
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcprov-jdk18on</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.jetbrains.kotlin</groupId>
-      <artifactId>kotlin-stdlib</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.reflections</groupId>
-      <artifactId>reflections</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
@@ -262,20 +237,6 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <groupId>com.coderplus.maven.plugins</groupId>
-        <artifactId>copy-rename-maven-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>rename-generated-config</id>
-            <phase>process-classes</phase>
-          </execution>
-          <execution>
-            <id>rename-generated-test-config</id>
-            <phase>process-test-classes</phase>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <groupId>com.github.spotbugs</groupId>
         <artifactId>spotbugs-maven-plugin</artifactId>