release 2023-08-21

Author: steveloughran

BUILDING.md

@@ -18,7 +18,7 @@ With maven, with profiles for many different hadoop versions.
 
 To build a production release
 1. Use java8
-1. And compile against a shipping hadoop version, with `-Pextra` for the extra stuff
+2. And compile against a shipping hadoop version, with `-Pextra` for the extra stuff
 
 
 ```bash
@@ -30,10 +30,8 @@ mvn clean install -Phadoop-3.3.2 -Pextra
 To publish the release use the gui or the github command line through the `fish` shell.
 
 ```bash
-set -gx now (date '+%Y-%m-%d')
-echo [$now]
-git add .
-git status
+set -gx now (date '+%Y-%m-%d'); echo [$now]
+git add .; git status
 git commit -S -m "release $now"
 git push
 gh release create tag-release-$now -t release-$now -n "release of $now" -d target/cloudstore-1.0.jar

src/main/java/org/apache/hadoop/fs/store/diag/S3ADiagnosticsInfo.java

@@ -167,8 +167,17 @@ public class S3ADiagnosticsInfo extends StoreDiagnosticsInfo {
    */
   public static final String PREFETCH_BLOCK_COUNT_KEY = "fs.s3a.prefetch.block.count";
 
+  /**
+   * Signing.
+   */
   public static final String SIGNING_ALGORITHM = "fs.s3a.signing-algorithm";
 
+  /**
+   * Signing; allows use of v2 signing on sdk v1,
+   * -D fs.s3a.signing-algorithm=S3SignerType
+   */
+  public static final String SIGNING_V2_ALGORITHM = "S3SignerType";
+
   private static final Object[][] options = {
       /* Core auth */
       {ACCESS_KEY, true, true},
@@ -721,7 +730,7 @@ protected void validateConfig(final Printout printout,
     printout.heading("Endpoint validation");
     String endpoint = conf.getTrimmed(ENDPOINT, "").toLowerCase(Locale.ROOT);
     String region = conf.getTrimmed(REGION, "").toLowerCase(Locale.ROOT);
-    String signing = conf.getTrimmed(SIGNING_ALGORITHM, "").toLowerCase(Locale.ROOT);
+    String signing = conf.getTrimmed(SIGNING_ALGORITHM, "");
     String bucket = getFsURI().getHost();
     boolean privateLink = false;
     boolean isIpv4 = false;
@@ -732,10 +741,12 @@ protected void validateConfig(final Printout printout,
     printout.println("%s = \"%s\"", ENDPOINT, endpoint);
     printout.println("%s = \"%s\"", REGION, region);
     printout.println("%s = \"%s\"", PATH_STYLE_ACCESS, pathStyleAccess);
-    printout.println("%s = \"%s\"", SIGNING_ALGORITHM, pathStyleAccess);
-    printout.println("%s = \"%s\"", SECURE_CONNECTIONS, signing);
+    printout.println("%s = \"%s\"", SIGNING_ALGORITHM, signing);
+    printout.println("%s = \"%s\"", SECURE_CONNECTIONS, secureConnections);
 
     boolean isUsingAws = false;
+    boolean isUsingV2Signing = SIGNING_V2_ALGORITHM.equals(signing);
+
     if (endpoint.isEmpty()) {
       isUsingAws = true;
       printout.println("Central us-east endpoint will be used. "
@@ -759,7 +770,7 @@ protected void validateConfig(final Printout printout,
       printout.println("See https://issues.apache.org/jira/browse/HADOOP-17705 for a workaround");
     } else if (!endpoint.contains(".amazonaws.")) {
       isUsingAws = false;
-      printout.println("This does not appear to be an amazon endpoint, unless it is a VPN addresss.");
+      printout.println("This does not appear to be an amazon endpoint, unless it is a VPN address.");
       isIpv4 = isIpV4String(endpoint);
 
       if (region.isEmpty()) {
@@ -808,12 +819,24 @@ protected void validateConfig(final Printout printout,
         printout.warn("You should probably set %s to true", PATH_STYLE_ACCESS);
       }
     }
-    if (!privateLink && isUsingAws) {
+    if (isUsingAws && !privateLink) {
+      printout.println("");
+      printout.println("This client is configured to connect to to AWS S3");
+      printout.println("");
       printout.println("Important: if you are working with a third party store,");
-      printout.println(" this client is still trying to connect to to AWS S3");
       printout.println("Expect failure until %s is set to the private endpoint", ENDPOINT);
     }
 
+    if (isUsingV2Signing) {
+      if (isUsingAws) {
+        printout.warn("The signing algorithm is %s; this is not supported on newer AWS buckets or the v2 AWS SDK", SIGNING_V2_ALGORITHM);
+      } else {
+        printout.println("The signing algorithm is %s; this is required for some third-party S3 stores", SIGNING_V2_ALGORITHM);
+        printout.warn("The signing algorithm is not yet available through the v2 AWS SDK");
+      }
+    }
+
+
     printout.heading("Bucket Name validation");
 
     printout.println("bucket name = \"%s\"", bucket);