Merge branch 'apache:master' into HDDS-11463

Author: slfan1989

.asf.yaml

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 github:
-  description: "Scalable, redundant, and distributed object store for Apache Hadoop"
+  description: "Scalable, reliable, distributed storage system optimized for data analytics and object store workloads."
   homepage: https://ozone.apache.org
   labels:
     - hadoop

.github/workflows/ci.yml

@@ -29,7 +29,8 @@ on:
 env:
   FAIL_FAST: ${{ github.event_name == 'pull_request' }}
   # Minimum required Java version for running Ozone is defined in pom.xml (javac.version).
-  TEST_JAVA_VERSION: 17 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
+  TEST_JAVA_VERSION: 21 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
+  MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml --show-version
   MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
   HADOOP_IMAGE: ghcr.io/apache/hadoop
   OZONE_IMAGE: ghcr.io/apache/ozone
@@ -257,7 +258,7 @@ jobs:
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             maven-repo-
-        if: ${{ !contains('author,bats,docs', matrix.check) }}
+        if: ${{ !contains('author,bats', matrix.check) }}
       - name: Download Ratis repo
         if: ${{ inputs.ratis_args != '' }}
         uses: actions/download-artifact@v4
@@ -343,6 +344,15 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: ${{ needs.build-info.outputs.sha }}
+      - name: Cache for maven dependencies
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
       - name: Download compiled Ozone binaries
         uses: actions/download-artifact@v4
         with:
@@ -486,6 +496,15 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: ${{ needs.build-info.outputs.sha }}
+      - name: Cache for maven dependencies
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
       - name: Download compiled Ozone binaries
         uses: actions/download-artifact@v4
         with:
@@ -530,6 +549,15 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: ${{ needs.build-info.outputs.sha }}
+      - name: Cache for maven dependencies
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
       - name: Download compiled Ozone binaries
         uses: actions/download-artifact@v4
         with:

.github/workflows/populate-cache.yml

@@ -73,7 +73,7 @@ jobs:
 
       - name: Fetch dependencies
         if: steps.restore-cache.outputs.cache-hit != 'true'
-        run: mvn --batch-mode --no-transfer-progress --show-version -Pgo-offline -Pdist clean verify
+        run: mvn --batch-mode --no-transfer-progress --show-version -Pgo-offline -Pdist -Drocks_tools_native clean verify
 
       - name: Delete Ozone jars from repo
         if: steps.restore-cache.outputs.cache-hit != 'true'

.mvn/extensions.xml

@@ -24,7 +24,7 @@
   <extension>
     <groupId>com.gradle</groupId>
     <artifactId>develocity-maven-extension</artifactId>
-    <version>1.22.1</version>
+    <version>1.23</version>
   </extension>
   <extension>
     <groupId>com.gradle</groupId>

dev-support/ci/maven-settings.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0http://maven.apache.org/xsd/settings-1.0.0.xsd">
+  <mirrors>
+    <mirror>
+      <id>block-snapshots1</id>
+      <mirrorOf>apache.snapshots</mirrorOf>
+      <name>Block access to Apache Snapshots</name>
+      <url>https://repository.apache.org/snapshots</url>
+      <blocked>true</blocked>
+    </mirror>
+    <mirror>
+      <id>block-snapshots2</id>
+      <mirrorOf>apache.snapshots.https</mirrorOf>
+      <name>Block access to Apache Snapshots</name>
+      <url>https://repository.apache.org/content/repositories/snapshots</url>
+      <blocked>true</blocked>
+    </mirror>
+  </mirrors>
+</settings>

dev-support/pom.xml

@@ -12,27 +12,25 @@
   See the License for the specific language governing permissions and
   limitations under the License. See accompanying LICENSE file.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
   <parent>
-    <artifactId>ozone-main</artifactId>
     <groupId>org.apache.ozone</groupId>
+    <artifactId>ozone-main</artifactId>
     <version>2.0.0-SNAPSHOT</version>
   </parent>
-  <modelVersion>4.0.0</modelVersion>
   <artifactId>ozone-dev-support</artifactId>
-  <description>Helper module for sharing resources among projects</description>
   <name>Apache Ozone Dev Support</name>
+  <description>Helper module for sharing resources among projects</description>
 
   <properties>
     <failIfNoTests>false</failIfNoTests>
   </properties>
   <build>
     <resources>
       <resource>
-        <directory>${project.build.directory}/extra-resources</directory>
         <targetPath>META-INF</targetPath>
+        <directory>${project.build.directory}/extra-resources</directory>
         <includes>
           <include>LICENSE.txt</include>
           <include>NOTICE.txt</include>
@@ -54,10 +52,10 @@
         <executions>
           <execution>
             <id>copy-resources</id>
-            <phase>validate</phase>
             <goals>
               <goal>copy-resources</goal>
             </goals>
+            <phase>validate</phase>
             <configuration>
               <outputDirectory>${project.build.directory}/extra-resources</outputDirectory>
               <resources>
@@ -77,21 +75,21 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-remote-resources-plugin</artifactId>
-        <executions>
-          <execution>
-            <phase>process-resources</phase>
-            <goals>
-              <goal>bundle</goal>
-            </goals>
-          </execution>
-        </executions>
         <configuration>
           <resourcesDirectory>${project.build.outputDirectory}</resourcesDirectory>
           <includes>
             <include>META-INF/LICENSE.txt</include>
             <include>META-INF/NOTICE.txt</include>
           </includes>
         </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>bundle</goal>
+            </goals>
+            <phase>process-resources</phase>
+          </execution>
+        </executions>
       </plugin>
     </plugins>
   </build>

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java

@@ -16,6 +16,7 @@
  */
 package org.apache.hadoop.hdds.cli;
 
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -25,6 +26,7 @@
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.security.UserGroupInformation;
 import picocli.CommandLine;
 import picocli.CommandLine.ExitCode;
 import picocli.CommandLine.Model.CommandSpec;
@@ -48,25 +50,20 @@ public class GenericCli implements Callable<Void>, GenericParentCommand {
   private String configurationPath;
 
   private final CommandLine cmd;
+  private OzoneConfiguration conf;
+  private UserGroupInformation user;
 
   public GenericCli() {
-    this(null);
+    this(CommandLine.defaultFactory());
   }
 
-  public GenericCli(Class<?> type) {
-    this(type, CommandLine.defaultFactory());
-  }
-
-  public GenericCli(Class<?> type, CommandLine.IFactory factory) {
+  public GenericCli(CommandLine.IFactory factory) {
     cmd = new CommandLine(this, factory);
     cmd.setExecutionExceptionHandler((ex, commandLine, parseResult) -> {
       printError(ex);
       return EXECUTION_ERROR_EXIT_CODE;
     });
 
-    if (type != null) {
-      SubcommandWithParent.addSubcommands(getCmd(), type);
-    }
     ExtensibleParentCommand.addSubcommands(cmd);
   }
 
@@ -122,6 +119,20 @@ public OzoneConfiguration createOzoneConfiguration() {
     return ozoneConf;
   }
 
+  public OzoneConfiguration getOzoneConf() {
+    if (conf == null) {
+      conf = createOzoneConfiguration();
+    }
+    return conf;
+  }
+
+  public UserGroupInformation getUser() throws IOException {
+    if (user == null) {
+      user = UserGroupInformation.getCurrentUser();
+    }
+    return user;
+  }
+
   @VisibleForTesting
   public picocli.CommandLine getCmd() {
     return cmd;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/SubcommandWithParent.java

@@ -58,9 +58,9 @@ public class CodecBuffer implements UncheckedAutoCloseable {
   private static class Factory {
     private static volatile BiFunction<ByteBuf, Object, CodecBuffer> constructor
         = CodecBuffer::new;
-    static void set(BiFunction<ByteBuf, Object, CodecBuffer> f) {
+    static void set(BiFunction<ByteBuf, Object, CodecBuffer> f, String name) {
       constructor = f;
-      LOG.info("Successfully set constructor to " + f);
+      LOG.info("Successfully set constructor to {}: {}", name, f);
     }
 
     static CodecBuffer newCodecBuffer(ByteBuf buf) {
@@ -89,7 +89,7 @@ protected void finalize() {
    * Note that there is a severe performance penalty for leak detection.
    */
   public static void enableLeakDetection() {
-    Factory.set(LeakDetector::newCodecBuffer);
+    Factory.set(LeakDetector::newCodecBuffer, "LeakDetector::newCodecBuffer");
   }
 
   /** The size of a buffer. */

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java

@@ -248,10 +248,7 @@ public List<ByteBuffer> asByteBufferList() {
 
   @Override
   public long writeTo(GatheringByteChannel channel) throws IOException {
-    long written = 0;
-    for (ByteBuffer buf : buffers) {
-      written += BufferUtils.writeFully(channel, buf);
-    }
+    final long written = BufferUtils.writeFully(channel, buffers);
     findCurrent();
     return written;
   }