add bulkdelete command for bulk deletion.

* Only works with hadoop 3.4.1 or later releases with the bulkdelete
API

Review more bulkdelete settings in storediag performance

Author: steveloughran

README.md

@@ -123,6 +123,12 @@ Retrieves metadata from an S3 Bucket (v2 SDK only) by probing the store.
 
 See [bucketmetadata](src/main/site/bucketmetadata.md) for details.
 
+## Command `bulkdelete`
+
+Retrieves metadata from an S3 Bucket (v2 SDK only) by probing the store.
+
+See [bucketmetadata](src/main/site/bulkdelete.md) for details.
+
 ## Command `cloudup` -upload and download files; optimised for cloud storage 
 
 See [cloudup](src/main/site/cloudup.md)

src/main/java/help.java

@@ -75,6 +75,7 @@ public static void main(String[] args) {
     println("========================");
     println("");
     printCommand("bucketmetadata", "Print bucket metadata from store");
+    printCommand("bulkdelete", "bulk delete objects/files");
     printCommand("regions2", "emulate region lookup of AWS v2 SDK");
     println("");
     println("See https://github.com/steveloughran/cloudstore");

src/main/java/org/apache/hadoop/fs/store/CommonParameters.java

@@ -67,6 +67,9 @@ public final class CommonParameters {
 
   public static final String CSVFILE = "csv";
 
+  public static final String DEBUG = "debug";
+
+  public static final String DEBUG_USAGE = "debug with extra logging";
   public static final String FLUSH = "flush";
 
   public static final String HFLUSH = "hflush";
@@ -77,11 +80,10 @@ public final class CommonParameters {
 
   public static final String OVERWRITE = "overwrite";
 
-  public static final String UPDATE = "update";
+  public static final String PAGE = "page";
 
-  public static final String DEBUG = "debug";
+  public static final String UPDATE = "update";
 
-  public static final String DEBUG_USAGE = "debug with extra logging";
 
   private CommonParameters() {
   }

src/main/java/org/apache/hadoop/fs/store/StoreDurationInfo.java

@@ -29,7 +29,7 @@
  * This allows it to be used in a try-with-resources clause, and have the
  * duration automatically logged.
  *
- * Base on the S3A one; adds an empty constructor which doesn't do
+ * Based on the hadoop one; adds an empty constructor which doesn't do
  * any logging.
  */
 public class StoreDurationInfo

src/main/java/org/apache/hadoop/fs/store/StoreEntryPoint.java

@@ -650,7 +650,7 @@ protected Configuration createPreconfiguredConfig()
 
     maybeAddXMLFileOption(conf);
     maybePatchDefined(conf, DEFINE);
-    conf.set(IOSTATISTICS_LOGGING_LEVEL, "info");
+    // conf.set(IOSTATISTICS_LOGGING_LEVEL, "info");
 
     return conf;
   }

src/main/java/org/apache/hadoop/fs/store/StoreUtils.java

@@ -18,8 +18,13 @@
 
 package org.apache.hadoop.fs.store;
 
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
 import java.io.IOException;
 import java.lang.reflect.Array;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
@@ -198,7 +203,8 @@ public static String sanitize(final String value, boolean hide) {
     return String.format("\"%s\" [%d]", safe, len);
   }
 
-  public static class StringPair implements Map.Entry<String, String>{
+  public static final class StringPair implements Map.Entry<String, String> {
+
     private String key, value;
 
     private StringPair(final String left, final String right) {
@@ -221,4 +227,23 @@ public String setValue(final String value) {
     }
   }
 
+  /**
+   * Read lines, skipping those with # or blank.
+   * (generated by copilot.)
+   * @param file path to file
+   * @return the list of lines
+   * @throws IOException failure to read
+   */
+  public static List<String> readLines(File file) throws IOException {
+    List<String> lines = new ArrayList<>();
+    try (BufferedReader br = new BufferedReader(new FileReader(file))) {
+      String line;
+      while ((line = br.readLine()) != null) {
+        if (!line.trim().isEmpty() && !line.trim().startsWith("#")) {
+          lines.add(line);
+        }
+      }
+    }
+    return lines;
+  }
 }

src/main/java/org/apache/hadoop/fs/store/commands/DirectoryTree.java

@@ -77,8 +77,7 @@ public int run(String[] args) throws Exception {
     List<String> paths = processArgs(args, 1, 1, USAGE);
     final Configuration conf = createPreconfiguredConfig();
 
-    int threads = getOptional(THREADS).map(Integer::valueOf).orElse(
-        DEFAULT_THREADS);
+    int threads = getIntOption(THREADS, DEFAULT_THREADS);
 
     final Path source = new Path(paths.get(0));
     heading("Deleting __temporary directories under %s with thread count %d",

src/main/java/org/apache/hadoop/fs/store/commands/EnvEntry.java

@@ -20,7 +20,7 @@
 
 /**
  * Environment/property entry, with
- * methods to convert to a given format,
+ * methods to convert to a given format.
  */
 public class EnvEntry {
   final String name;

src/main/java/org/apache/hadoop/fs/store/commands/package-info.java

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.store.commands;

src/main/java/org/apache/hadoop/fs/store/diag/S3ADiagnosticsInfo.java

@@ -180,6 +180,8 @@ public class S3ADiagnosticsInfo extends StoreDiagnosticsInfo {
 
   public static final String BULK_DELETE_PAGE_SIZE = "fs.s3a.bulk.delete.page.size";
 
+  public static final int BULK_DELETE_PAGE_SIZE_DEFAULT = 250;
+
   public static final String CONNECTION_SSL_ENABLED = "fs.s3a.connection.ssl.enabled";
 
   public static final String SERVER_SIDE_ENCRYPTION_ALGORITHM =
@@ -278,6 +280,8 @@ public class S3ADiagnosticsInfo extends StoreDiagnosticsInfo {
 
   public static final String CHANNEL_MODE = "fs.s3a.ssl.channel.mode";
 
+  public static final String MULTIOBJECTDELETE_ENABLE = "fs.s3a.multiobjectdelete.enable";
+
   private static final Object[][] options = {
       /* Core auth */
       {ACCESS_KEY, true, true},
@@ -340,7 +344,7 @@ public class S3ADiagnosticsInfo extends StoreDiagnosticsInfo {
       {DISABLE_CACHE, false, false},
       {"fs.s3a.list.version", false, false},
       {"fs.s3a.max.total.tasks", false, false},
-      {"fs.s3a.multiobjectdelete.enable", false, false},
+      {MULTIOBJECTDELETE_ENABLE, false, false},
       {FS_S3A_MULTIPART_SIZE, false, false},
       {MULTIPART_UPLOADS_ENABLED, false, false},
       {MULTIPART_PURGE, false, false},
@@ -1370,7 +1374,7 @@ CONNECTION_TTL, ofMinutes(5),
         requestTimeout,
         true, "");
 
-    printout.subheading("Other options");
+    printout.subheading("Misc options");
     printout.println();
 
     int bucketProbe = conf.getInt(BUCKET_PROBE, 0);
@@ -1390,6 +1394,23 @@ CONNECTION_TTL, ofMinutes(5),
 
     reviewReadPolicy(printout, conf);
 
+    printout.subheading("Bulk Delete behavior");
+    boolean multi = conf.getBoolean(MULTIOBJECTDELETE_ENABLE, true);
+    int pageSize = conf.getInt(BULK_DELETE_PAGE_SIZE, BULK_DELETE_PAGE_SIZE_DEFAULT);
+    printout.println("%s = %s", MULTIOBJECTDELETE_ENABLE, multi);
+    printout.println("%s = %d", BULK_DELETE_PAGE_SIZE, pageSize);
+    printout.println();
+    if (multi) {
+      printout.println("Multi object delete is enabled; page size is %d", pageSize);
+      hint(printout, pageSize > 500,
+          "The page size is > 500. This is dangerous on heavily loaded stores\n"
+              + "See HADOOP-16823. Large DeleteObject requests are their own Thundering Herd");
+      hint(printout, pageSize < 100,
+          "The page size is <100. This will slow down renames, deletes and other bulk operations.");
+    } else {
+      printout.println("Multi object delete is disabled.");
+      printout.println("This should only be done when working with third party stores.");
+    }
   }
 
   /**
@@ -1400,7 +1421,7 @@ private void reviewReadPolicy(final Printout printout,
     // look at seek policy and warn of risks
     final String fadvise =
         conf.getTrimmed(INPUT_FADVISE, INPUT_FADV_NORMAL);
-    printout.heading("Seek policy: %s", fadvise);
+    printout.subheading("Seek policy: %s", fadvise);
     switch (fadvise) {
     case INPUT_FADV_NORMAL:
     case OptionSets.EnhancedOpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE: