Merge branch 'apache:master' into HDDS-11463

Author: slfan1989

CONTRIBUTING.md

@@ -90,6 +90,10 @@ When creating a new jira for any kind of new feature, improvement or bug, please
      * If it is follow up of another issue, please link the previous jira to it so that context is preserve.
   3. **Jira examples:** [HDDS-9272](https://issues.apache.org/jira/browse/HDDS-9272), [HDDS-9322](https://issues.apache.org/jira/browse/HDDS-9322), [HDDS-9291](https://issues.apache.org/jira/browse/HDDS-9291), [HDDS-8940](https://issues.apache.org/jira/browse/HDDS-8940), [HDDS-9282](https://issues.apache.org/jira/browse/HDDS-9282)
 
+## New feature development
+
+For large feature development changes, we use a process called "Ozone Enhancement Proposals" (OEP). This process is designed to ensure that major changes to Ozone are well-designed and have community consensus. If you are planning to propose a significant change, please read the [Ozone Enhancement Proposals](https://ozone.apache.org/docs/edge/design/ozone-enhancement-proposals.html) documentation and create a design document before you start coding. Please note that we only accept design documents in Markdown format; PDF or Google Docs are no longer accepted.
+
 ## Contribute your modifications
 
 We use GitHub pull requests for contributing changes to the repository. The main workflow is as follows:

README.md

@@ -1,3 +1,16 @@
+<div align="center">
+  <a href="https://ozone.apache.org">
+    <img src="https://www.apache.org/logos/res/ozone/default.png" alt="Apache Ozone Logo" />
+  </a>
+</div>
+
+[![License](https://img.shields.io/:license-Apache%202-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0.txt)
+[![Docker Pulls](https://img.shields.io/docker/pulls/apache/ozone.svg)](https://hub.docker.com/r/apache/ozone)
+[![Docker Stars](https://img.shields.io/docker/stars/apache/ozone.svg)](https://hub.docker.com/r/apache/ozone)
+[![Contributors](https://img.shields.io/github/contributors/apache/ozone)](https://github.com/apache/ozone/graphs/contributors)
+[![Commit Activity](https://img.shields.io/github/commit-activity/m/apache/ozone)](https://github.com/apache/ozone/commits/master)
+[![OSSRank](https://shields.io/endpoint?url=https://ossrank.com/shield/3018)](https://ossrank.com/p/3018-apache-ozone)
+
 Apache Ozone
 ===
 

dev-support/byteman/fail-create-bucket.btm

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This script instruments ozone manager to fail a CreateBucket request for a specific name
+#
+
+RULE Crash OM with CreateBucket
+CLASS org.apache.hadoop.ozone.om.request.bucket.OMBucketCreateRequest
+METHOD validateAndUpdateCache
+AT ENTRY
+IF TRUE
+DO
+  traceln("--> crashing CreateBucket request");
+  THROW new RuntimeException("Byteman crashes OM");
+ENDRULE

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java

@@ -255,7 +255,7 @@ public final class HddsConfigKeys {
 
   public static final String HDDS_SECRET_KEY_EXPIRY_DURATION =
       "hdds.secret.key.expiry.duration";
-  public static final String HDDS_SECRET_KEY_EXPIRY_DURATION_DEFAULT = "7d";
+  public static final String HDDS_SECRET_KEY_EXPIRY_DURATION_DEFAULT = "9d";
 
   public static final String HDDS_SECRET_KEY_ROTATE_DURATION =
       "hdds.secret.key.rotate.duration";

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java

@@ -104,7 +104,7 @@ public class ScmConfig extends ReconfigurableConfig {
 
   @Config(key = "hdds.scm.block.deletion.per-interval.max",
       type = ConfigType.INT,
-      defaultValue = "100000",
+      defaultValue = "500000",
       reconfigurable = true,
       tags = { ConfigTag.SCM, ConfigTag.DELETION},
       description =

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java

@@ -456,6 +456,11 @@ public final class ScmConfigKeys {
       "ozone.scm.block.deletion.max.retry";
   public static final int OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT = 4096;
 
+  public static final String OZONE_SCM_BLOCK_DELETION_PER_DN_DISTRIBUTION_FACTOR =
+      "ozone.scm.block.deletion.per.dn.distribution.factor";
+
+  public static final int OZONE_SCM_BLOCK_DELETION_PER_DN_DISTRIBUTION_FACTOR_DEFAULT = 8;
+
   public static final String OZONE_SCM_SEQUENCE_ID_BATCH_SIZE =
       "ozone.scm.sequence.id.batch.size";
   public static final int OZONE_SCM_SEQUENCE_ID_BATCH_SIZE_DEFAULT = 1000;
@@ -521,7 +526,7 @@ public final class ScmConfigKeys {
 
   public static final String OZONE_SCM_HA_RAFT_SEGMENT_SIZE =
           "ozone.scm.ha.ratis.segment.size";
-  public static final String OZONE_SCM_HA_RAFT_SEGMENT_SIZE_DEFAULT = "4MB";
+  public static final String OZONE_SCM_HA_RAFT_SEGMENT_SIZE_DEFAULT = "64MB";
 
   public static final String OZONE_SCM_HA_RAFT_SEGMENT_PRE_ALLOCATED_SIZE =
           "ozone.scm.ha.ratis.segment.preallocated.size";
@@ -617,7 +622,7 @@ public final class ScmConfigKeys {
   public static final String OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL =
       "ozone.scm.ha.dbtransactionbuffer.flush.interval";
   public static final long
-      OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL_DEFAULT = 600 * 1000L;
+      OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL_DEFAULT = 60 * 1000L;
 
   public static final String NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY =
       "net.topology.node.switch.mapping.impl";

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java

@@ -41,7 +41,8 @@ public enum HDDSLayoutFeature implements LayoutFeature {
   HADOOP_PRC_PORTS_IN_DATANODEDETAILS(7, "Adding Hadoop RPC ports " +
                                      "to DatanodeDetails."),
   HBASE_SUPPORT(8, "Datanode RocksDB Schema Version 3 has an extra table " +
-          "for the last chunk of blocks to support HBase.)");
+          "for the last chunk of blocks to support HBase.)"),
+  WITNESSED_CONTAINER_DB_PROTO_VALUE(9, "ContainerID table schema to use value type as proto");
 
   //////////////////////////////  //////////////////////////////
 

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundService.java

@@ -77,11 +77,11 @@ protected CompletableFuture<Void> getFuture() {
   }
 
   @VisibleForTesting
-  public ExecutorService getExecutorService() {
+  public synchronized ExecutorService getExecutorService() {
     return this.exec;
   }
 
-  public void setPoolSize(int size) {
+  public synchronized void setPoolSize(int size) {
     if (size <= 0) {
       throw new IllegalArgumentException("Pool size must be positive.");
     }
@@ -108,6 +108,7 @@ public void runPeriodicalTaskNow() throws Exception {
     while (!tasks.isEmpty()) {
       tasks.poll().call();
     }
+    execTaskCompletion();
   }
 
   // start service
@@ -125,15 +126,30 @@ protected synchronized void setInterval(long newInterval, TimeUnit newUnit) {
     this.unit = newUnit;
   }
 
+  protected synchronized long getIntervalMillis() {
+    return this.unit.toMillis(interval);
+  }
+
   public abstract BackgroundTaskQueue getTasks();
 
+  protected void execTaskCompletion() { }
+
   /**
    * Run one or more background tasks concurrently.
    * Wait until all tasks to return the result.
    */
   public class PeriodicalTask implements Runnable {
     @Override
-    public synchronized void run() {
+    public void run() {
+      // wait for previous set of tasks to complete
+      try {
+        future.join();
+      } catch (RuntimeException e) {
+        LOG.error("Background service execution failed.", e);
+      } finally {
+        execTaskCompletion();
+      }
+
       if (LOG.isDebugEnabled()) {
         LOG.debug("Running background service : {}", serviceName);
       }
@@ -146,35 +162,36 @@ public synchronized void run() {
       if (LOG.isDebugEnabled()) {
         LOG.debug("Number of background tasks to execute : {}", tasks.size());
       }
-
-      while (!tasks.isEmpty()) {
-        BackgroundTask task = tasks.poll();
-        future = future.thenCombine(CompletableFuture.runAsync(() -> {
-          long startTime = System.nanoTime();
-          try {
-            BackgroundTaskResult result = task.call();
-            if (LOG.isDebugEnabled()) {
-              LOG.debug("task execution result size {}", result.getSize());
-            }
-          } catch (Throwable e) {
-            LOG.error("Background task execution failed", e);
-            if (e instanceof Error) {
-              throw (Error) e;
-            }
-          } finally {
-            long endTime = System.nanoTime();
-            if (endTime - startTime > serviceTimeoutInNanos) {
-              LOG.warn("{} Background task execution took {}ns > {}ns(timeout)",
-                  serviceName, endTime - startTime, serviceTimeoutInNanos);
+      synchronized (BackgroundService.this) {
+        while (!tasks.isEmpty()) {
+          BackgroundTask task = tasks.poll();
+          future = future.thenCombine(CompletableFuture.runAsync(() -> {
+            long startTime = System.nanoTime();
+            try {
+              BackgroundTaskResult result = task.call();
+              if (LOG.isDebugEnabled()) {
+                LOG.debug("task execution result size {}", result.getSize());
+              }
+            } catch (Throwable e) {
+              LOG.error("Background task execution failed", e);
+              if (e instanceof Error) {
+                throw (Error) e;
+              }
+            } finally {
+              long endTime = System.nanoTime();
+              if (endTime - startTime > serviceTimeoutInNanos) {
+                LOG.warn("{} Background task execution took {}ns > {}ns(timeout)",
+                    serviceName, endTime - startTime, serviceTimeoutInNanos);
+              }
             }
-          }
-        }, exec), (Void1, Void) -> null);
+          }, exec).exceptionally(e -> null), (Void1, Void) -> null);
+        }
       }
     }
   }
 
   // shutdown and make sure all threads are properly released.
-  public void shutdown() {
+  public synchronized void shutdown() {
     LOG.info("Shutting down service {}", this.serviceName);
     exec.shutdown();
     try {