HADOOP-19654. Upgrade AWS SDK to 2.35.4 (apache#7882)

AWS SDK upgraded to 2.35.4.

This SDK has changed checksum/checksum headers handling significantly,
causing problems with third party stores, and, in some combinations
AWS S3 itself.

The S3A connector has retained old behavior; options to change
these settings are now available.

The default settings are chosen for maximum compatiblity and performance.

fs.s3a.request.md5.header:       true
fs.s3a.checksum.generation:      false
fs.s3a.create.checksum.algorithm: ""

Consult the documentation for more details.

Contributed by Steve Loughran

Author: steveloughran

LICENSE-binary

@@ -366,8 +366,9 @@ org.lz4:lz4-java:1.7.1
 org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.1.8.2
 org.yaml:snakeyaml:2.0
-org.wildfly.openssl:wildfly-openssl:1.1.3.Final
-
+org.wildfly.openssl:wildfly-openssl:2.2.5.Final
+software.amazon.awssdk:bundle:2.35.4
+software.amazon.s3.analyticsaccelerator:analyticsaccelerator-s3:1.3.0
 
 --------------------------------------------------------------------------------
 This product bundles various third-party components under other open source

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.fs.contract;
 
+import org.assertj.core.api.Assertions;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -28,6 +29,7 @@
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.readNBytes;
 
 /**
  * Contract tests for {@link org.apache.hadoop.fs.CanUnbuffer#unbuffer}.
@@ -136,10 +138,12 @@ protected void validateFileContents(FSDataInputStream stream, int length,
                                       int startIndex)
           throws IOException {
     byte[] streamData = new byte[length];
-    assertEquals("failed to read expected number of bytes from "
-            + "stream. This may be transient",
-        length, stream.read(streamData));
+    final int read = readNBytes(stream, streamData, 0, length);
+    Assertions.assertThat(read)
+        .describedAs("failed to read expected number of bytes from stream. %s", stream)
+        .isEqualTo(length);
     byte[] validateFileBytes;
+
     if (startIndex == 0 && length == fileBytes.length) {
       validateFileBytes = fileBytes;
     } else {

hadoop-project/pom.xml

@@ -190,7 +190,11 @@
     <exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
     <make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
     <surefire.fork.timeout>900</surefire.fork.timeout>
-    <aws-java-sdk.version>1.12.782</aws-java-sdk.version>
+    <aws-java-sdk.version>1.12.720</aws-java-sdk.version>
+    <aws-java-sdk-v2.version>2.35.4</aws-java-sdk-v2.version>
+    <amazon-s3-encryption-client-java.version>3.1.1</amazon-s3-encryption-client-java.version>
+    <amazon-s3-analyticsaccelerator-s3.version>1.3.0</amazon-s3-analyticsaccelerator-s3.version>
+    <aws.eventstream.version>1.0.1</aws.eventstream.version>
     <hsqldb.version>2.7.1</hsqldb.version>
     <frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
     <jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java

@@ -18,34 +18,44 @@
 
 package org.apache.hadoop.fs.s3a;
 
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.SdkBaseException;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import software.amazon.awssdk.core.exception.SdkException;
+import org.apache.hadoop.util.Preconditions;
 
 import java.io.IOException;
 
 /**
- * IOException equivalent of an {@link AmazonClientException}.
+ * IOException equivalent of an {@link SdkException}.
  */
 public class AWSClientIOException extends IOException {
 
   private final String operation;
 
   public AWSClientIOException(String operation,
-      SdkBaseException cause) {
+      SdkException cause) {
     super(cause);
     Preconditions.checkArgument(operation != null, "Null 'operation' argument");
     Preconditions.checkArgument(cause != null, "Null 'cause' argument");
     this.operation = operation;
   }
 
-  public AmazonClientException getCause() {
-    return (AmazonClientException) super.getCause();
+  public SdkException getCause() {
+    return (SdkException) super.getCause();
   }
 
   @Override
   public String getMessage() {
     return operation + ": " + getCause().getMessage();
   }
 
+  /**
+   * Query inner cause for retryability.
+   * @return what the cause says.
+   */
+  public boolean retryable() {
+    return getCause().retryable();
+  }
+
+  public String getOperation() {
+    return operation;
+  }
 }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java

@@ -24,6 +24,12 @@
  * Status code 443, no response from server. This is considered idempotent.
  */
 public class AWSNoResponseException extends AWSServiceIOException {
+
+  /**
+   * Constructor.
+   * @param operation operation in progress.
+   * @param cause inner cause
+   */
   public AWSNoResponseException(String operation,
       AmazonServiceException cause) {
     super(operation, cause);