diff --git a/codebuild/cd/fips-where-available-jar-build.yml b/codebuild/cd/fips-where-available-jar-build.yml index c85f0510f..d5c5a66a8 100644 --- a/codebuild/cd/fips-where-available-jar-build.yml +++ b/codebuild/cd/fips-where-available-jar-build.yml @@ -2,7 +2,6 @@ version: 0.2 phases: install: commands: - - sudo add-apt-repository ppa:openjdk-r/ppa - sudo apt-get update -y - sudo apt-get install openjdk-11-jdk-headless maven wget unzip -y -f build: diff --git a/codebuild/cd/release-android-maven-central.yml b/codebuild/cd/release-android-maven-central.yml index 347908dcf..5db350b5f 100644 --- a/codebuild/cd/release-android-maven-central.yml +++ b/codebuild/cd/release-android-maven-central.yml @@ -4,7 +4,6 @@ version: 0.2 phases: install: commands: - - sudo add-apt-repository ppa:openjdk-r/ppa - sudo apt-get update -y # Java 17 for AGP 8.x diff --git a/codebuild/cd/release-maven-central.yml b/codebuild/cd/release-maven-central.yml index e4d3f18c3..3e2aa5a8b 100644 --- a/codebuild/cd/release-maven-central.yml +++ b/codebuild/cd/release-maven-central.yml @@ -4,7 +4,6 @@ version: 0.2 phases: install: commands: - - sudo add-apt-repository ppa:openjdk-r/ppa - sudo apt-get update -y - sudo apt-get install openjdk-11-jdk-headless maven wget unzip -y -f pre_build: diff --git a/codebuild/linux-integration-tests.yml b/codebuild/linux-integration-tests.yml index 159727787..224ba11c8 100644 --- a/codebuild/linux-integration-tests.yml +++ b/codebuild/linux-integration-tests.yml @@ -9,8 +9,6 @@ env: phases: install: commands: - - sudo add-apt-repository ppa:openjdk-r/ppa - - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt-get update -y - sudo apt-get install cmake -y build: diff --git a/codebuild/mqtt5-java-canary-test.yml b/codebuild/mqtt5-java-canary-test.yml index 1c5fa3ca8..2f9b9a86b 100644 --- a/codebuild/mqtt5-java-canary-test.yml +++ b/codebuild/mqtt5-java-canary-test.yml @@ -15,8 +15,6 @@ env: phases: install: commands: - - add-apt-repository ppa:openjdk-r/ppa - - add-apt-repository ppa:ubuntu-toolchain-r/test - apt-get update -y - sudo apt-get install cmake -y - git submodule update --init --recursive diff --git a/codebuild/test-aws-java-sdk-v2.yml b/codebuild/test-aws-java-sdk-v2.yml index 008f502a8..5839fd2a1 100644 --- a/codebuild/test-aws-java-sdk-v2.yml +++ b/codebuild/test-aws-java-sdk-v2.yml @@ -5,8 +5,6 @@ phases: runtime-versions: java: "corretto8" commands: - - add-apt-repository ppa:openjdk-r/ppa - - add-apt-repository ppa:ubuntu-toolchain-r/test - apt-get update -y - apt-get install gcc-7 cmake ninja-build -y - git submodule update --init --recursive diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index 7397f2fc0..ab764f57e 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit 7397f2fc077caf4d404c8708201fa52853a52847 +Subproject commit ab764f57ec3cab7866c0f8b7d9e9772b7cc1b9ee diff --git a/crt/s2n b/crt/s2n index f5e5e8303..a71ea1f97 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit f5e5e83031be60691f22442373fb8371274fcd56 +Subproject commit a71ea1f9764ee9ae91b47c34faa0447c4fe11d0a diff --git a/src/main/java/software/amazon/awssdk/crt/checksums/XXHash.java b/src/main/java/software/amazon/awssdk/crt/checksums/XXHash.java index 7d9262ea0..b68af0da1 100644 --- a/src/main/java/software/amazon/awssdk/crt/checksums/XXHash.java +++ b/src/main/java/software/amazon/awssdk/crt/checksums/XXHash.java @@ -120,7 +120,7 @@ public void update(int b) { } /** - * Update xxhash state with a subrange of input + * Update xxhash state with a subrange of input. * @param input input to update with * @param offset to start update with * @param length of data diff --git a/src/main/java/software/amazon/awssdk/crt/s3/ChecksumAlgorithm.java b/src/main/java/software/amazon/awssdk/crt/s3/ChecksumAlgorithm.java index 5803a5dad..7bef01362 100644 --- a/src/main/java/software/amazon/awssdk/crt/s3/ChecksumAlgorithm.java +++ b/src/main/java/software/amazon/awssdk/crt/s3/ChecksumAlgorithm.java @@ -22,7 +22,17 @@ public enum ChecksumAlgorithm { SHA256(4), - CRC64NVME(5); + CRC64NVME(5), + + SHA512(6), + + XXHASH64(7), + + XXHASH3(8), + + XXHASH128(9), + + MD5(10); ChecksumAlgorithm(int nativeValue) { this.nativeValue = nativeValue; @@ -49,6 +59,11 @@ private static Map buildEnumMapping() { enumMapping.put(SHA1.getNativeValue(), SHA1); enumMapping.put(SHA256.getNativeValue(), SHA256); enumMapping.put(CRC64NVME.getNativeValue(), CRC64NVME); + enumMapping.put(SHA512.getNativeValue(), SHA512); + enumMapping.put(XXHASH64.getNativeValue(), XXHASH64); + enumMapping.put(XXHASH3.getNativeValue(), XXHASH3); + enumMapping.put(XXHASH128.getNativeValue(), XXHASH128); + enumMapping.put(MD5.getNativeValue(), MD5); return enumMapping; } diff --git a/src/main/java/software/amazon/awssdk/crt/s3/ChecksumConfig.java b/src/main/java/software/amazon/awssdk/crt/s3/ChecksumConfig.java index d3d9e1dc2..77721cf94 100644 --- a/src/main/java/software/amazon/awssdk/crt/s3/ChecksumConfig.java +++ b/src/main/java/software/amazon/awssdk/crt/s3/ChecksumConfig.java @@ -114,7 +114,7 @@ public boolean getValidateChecksum() { * The list of algorithms for user to pick up when validate the checksum. Client * will pick up the algorithm from the list with the priority based on * performance, and the algorithm sent by server. The priority based on - * performance is [CRC64NVME, CRC32C, CRC32, SHA1, SHA256]. + * performance is [XXHASH128, XXHASH3,CRC64NVME, CRC32C, CRC32, XXHASH64, SHA512, SHA256, SHA1]. * * If the response checksum was validated by client, the result will indicate * which algorithm was picked. diff --git a/src/main/java/software/amazon/awssdk/crt/s3/S3Client.java b/src/main/java/software/amazon/awssdk/crt/s3/S3Client.java index 4009961da..ba46922d5 100644 --- a/src/main/java/software/amazon/awssdk/crt/s3/S3Client.java +++ b/src/main/java/software/amazon/awssdk/crt/s3/S3Client.java @@ -168,6 +168,12 @@ public S3MetaRequest makeMetaRequest(S3MetaRequestOptions options) { throw new IllegalArgumentException("S3Client.makeMetaRequest has invalid options; Operation name must be set for MetaRequestType.DEFAULT."); } + if (options.getChecksumConfig() != null && options.getChecksumAlgorithm() == ChecksumAlgorithm.MD5) { + Log.log(Log.LogLevel.Error, Log.LogSubject.S3Client, + "S3Client.makeMetaRequest has invalid options; MD5 not supported as checksum algorithm."); + throw new IllegalArgumentException("S3Client.makeMetaRequest has invalid options; MD5 not supported as checksum algorithm."); + } + S3MetaRequest metaRequest = new S3MetaRequest(); S3MetaRequestResponseHandlerNativeAdapter responseHandlerNativeAdapter = new S3MetaRequestResponseHandlerNativeAdapter( options.getResponseHandler()); diff --git a/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java b/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java index c63c29985..82179d31b 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java @@ -307,6 +307,39 @@ public void testS3ClientCreateDestroyHttpProxyEnvironmentVariableSetting() { } } + @Test + public void testS3ClientCreateMD5MetaRequest() { + skipIfAndroid(); + skipIfNetworkUnavailable(); + + S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION) + .withComputeContentMd5(true) + .withMemoryLimitInBytes(5L * 1024 * 1024 * 1024); + try (S3Client client = createS3Client(clientOptions)) { + S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() { + @Override + public int onResponseBody(ByteBuffer bodyBytesIn, long objectRangeStart, long objectRangeEnd) { + return 0; + } + + @Override + public void onFinished(S3FinishedResponseContext context) { + } + }; + + HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) }; + HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null); + + S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() + .withMetaRequestType(MetaRequestType.GET_OBJECT) + .withChecksumAlgorithm(ChecksumAlgorithm.MD5) + .withHttpRequest(httpRequest) + .withResponseHandler(responseHandler); + + Assert.assertThrows(IllegalArgumentException.class, () -> { client.makeMetaRequest(metaRequestOptions); }); + } + } + @Test public void testS3Get() { skipIfAndroid(); @@ -1360,9 +1393,13 @@ public void onFinished(S3FinishedResponseContext context) { } }; ArrayList algorList = new ArrayList(); + algorList.add(ChecksumAlgorithm.XXHASH128); + algorList.add(ChecksumAlgorithm.XXHASH3); algorList.add(ChecksumAlgorithm.CRC64NVME); algorList.add(ChecksumAlgorithm.CRC32C); algorList.add(ChecksumAlgorithm.CRC32); + algorList.add(ChecksumAlgorithm.XXHASH64); + algorList.add(ChecksumAlgorithm.SHA512); algorList.add(ChecksumAlgorithm.SHA1); algorList.add(ChecksumAlgorithm.SHA256); ChecksumConfig validateChecksumConfig = new ChecksumConfig().withValidateChecksum(true) @@ -1381,13 +1418,29 @@ public void onFinished(S3FinishedResponseContext context) { } @Test - public void testS3PutTrailerChecksums() throws Exception { + public void testS3PutTrailerChecksumsCRC64() throws Exception { skipIfAndroid(); skipIfNetworkUnavailable(); Assume.assumeTrue(hasAwsCredentials()); testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC64NVME, ChecksumLocation.TRAILER, false, false); } + @Test + public void testS3PutTrailerChecksumsSHA512() throws Exception { + skipIfAndroid(); + skipIfNetworkUnavailable(); + Assume.assumeTrue(hasAwsCredentials()); + testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.SHA512, ChecksumLocation.TRAILER, false, false); + } + + @Test + public void testS3PutTrailerChecksumsXXHASH3_64() throws Exception { + skipIfAndroid(); + skipIfNetworkUnavailable(); + Assume.assumeTrue(hasAwsCredentials()); + testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.XXHASH3, ChecksumLocation.TRAILER, false, false); + } + @Test public void testS3PutHeaderChecksums() throws Exception { skipIfAndroid();