Skip to content

Commit 14001cb

Browse files
THausherrTHausherr
committed
TIKA-4524: migrate to aws v2
1 parent 73c3951 commit 14001cb

2 files changed

Lines changed: 53 additions & 25 deletions

File tree

‎tika-integration-tests/tika-pipes-s3-integration-tests/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,11 @@
7474
<artifactId>log4j-slf4j2-impl</artifactId>
7575
<scope>test</scope>
7676
</dependency>
77+
<dependency>
78+
<groupId>software.amazon.awssdk</groupId>
79+
<artifactId>s3</artifactId>
80+
<scope>test</scope>
81+
</dependency>
7782
</dependencies>
7883

7984
<build>

‎tika-integration-tests/tika-pipes-s3-integration-tests/src/test/java/org/apache/tika/pipes/s3/tests/S3PipeIntegrationTest.java‎

Lines changed: 48 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -18,19 +18,18 @@
1818

1919
import java.io.File;
2020
import java.io.InputStream;
21+
import java.net.URI;
22+
import java.net.URISyntaxException;
2123
import java.nio.charset.StandardCharsets;
24+
import java.security.MessageDigest;
25+
import java.security.NoSuchAlgorithmException;
2226
import java.time.Duration;
2327
import java.time.temporal.ChronoUnit;
28+
import java.util.Base64;
2429
import java.util.HashSet;
2530
import java.util.Set;
2631

27-
import com.amazonaws.auth.AWSStaticCredentialsProvider;
28-
import com.amazonaws.auth.BasicAWSCredentials;
29-
import com.amazonaws.client.builder.AwsClientBuilder;
30-
import com.amazonaws.regions.Regions;
31-
import com.amazonaws.services.s3.AmazonS3;
32-
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
33-
import com.amazonaws.services.s3.model.S3Object;
32+
import org.apache.commons.io.FileUtils;
3433
import org.apache.commons.io.IOUtils;
3534
import org.jetbrains.annotations.NotNull;
3635
import org.junit.jupiter.api.AfterAll;
@@ -42,9 +41,20 @@
4241
import org.slf4j.LoggerFactory;
4342
import org.testcontainers.containers.DockerComposeContainer;
4443
import org.testcontainers.junit.jupiter.Testcontainers;
45-
import org.testcontainers.shaded.org.apache.commons.io.FileUtils;
4644
import org.testcontainers.shaded.org.hamcrest.MatcherAssert;
4745
import org.testcontainers.shaded.org.hamcrest.Matchers;
46+
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
47+
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
48+
import software.amazon.awssdk.core.ResponseInputStream;
49+
import software.amazon.awssdk.core.sync.RequestBody;
50+
import software.amazon.awssdk.regions.Region;
51+
import software.amazon.awssdk.services.s3.S3Client;
52+
import software.amazon.awssdk.services.s3.S3Configuration;
53+
import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;
54+
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
55+
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
56+
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
57+
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
4858

4959
import org.apache.tika.cli.TikaCLI;
5060
import org.apache.tika.pipes.HandlerConfig;
@@ -65,28 +75,39 @@ class S3PipeIntegrationTest {
6575
private static final String FETCH_BUCKET = "fetch-bucket";
6676
private static final String EMIT_BUCKET = "emit-bucket";
6777

68-
private static final String REGION = Regions.US_EAST_1.getName();
78+
private static final Region REGION = Region.US_EAST_1;
6979

70-
private AmazonS3 s3Client;
80+
private S3Client s3Client;
7181

7282
private final File testFileFolder = new File("target", "test-files");
7383
private final Set<String> testFiles = new HashSet<>();
7484

75-
private void createTestFiles() {
85+
private void createTestFiles() throws NoSuchAlgorithmException {
7686
if (testFileFolder.mkdirs()) {
7787
LOG.info("Created test folder: {}", testFileFolder);
7888
}
7989
int numDocs = 42;
8090
for (int i = 0; i < numDocs; ++i) {
8191
String nextFileName = "test-" + i + ".html";
8292
testFiles.add(nextFileName);
83-
s3Client.putObject(FETCH_BUCKET, nextFileName,
84-
"<html><body>body-of-" + nextFileName + "</body></html>");
93+
String s = "<html><body>body-of-" + nextFileName + "</body></html>";
94+
byte[] bytes = s.getBytes(StandardCharsets.US_ASCII);
95+
// checksum must be done explicitely, or we get an exception:
96+
// "The provided 'x-amz-content-sha256' header does not match what was computed"
97+
// https://github.com/minio/minio/issues/17662
98+
// https://github.com/minio/minio/issues/20845
99+
MessageDigest md = MessageDigest.getInstance("SHA256");
100+
byte [] hash = md.digest(bytes);
101+
String enc64 = Base64.getEncoder().encodeToString(hash);
102+
PutObjectRequest request = PutObjectRequest.builder().bucket(FETCH_BUCKET).key(nextFileName).
103+
checksumAlgorithm(ChecksumAlgorithm.SHA256).checksumSHA256(enc64).build();
104+
RequestBody requestBody = RequestBody.fromBytes(bytes);
105+
s3Client.putObject(request, requestBody);
85106
}
86107
}
87108

88109
@BeforeAll
89-
void setupMinio() {
110+
void setupMinio() throws URISyntaxException {
90111
minioContainer.start();
91112
initializeS3Client();
92113
}
@@ -96,20 +117,21 @@ void closeMinio() {
96117
minioContainer.close();
97118
}
98119

99-
private void initializeS3Client() {
100-
AwsClientBuilder.EndpointConfiguration endpoint =
101-
new AwsClientBuilder.EndpointConfiguration(MINIO_ENDPOINT, REGION);
102-
s3Client = AmazonS3ClientBuilder.standard().withCredentials(
103-
new AWSStaticCredentialsProvider(new BasicAWSCredentials(ACCESS_KEY, SECRET_KEY)))
104-
.withEndpointConfiguration(endpoint).withPathStyleAccessEnabled(true).build();
120+
private void initializeS3Client() throws URISyntaxException {
121+
AwsBasicCredentials awsCreds = AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY);
122+
// https://github.com/aws/aws-sdk-java-v2/discussions/3536
123+
StaticCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(awsCreds);
124+
S3Configuration s3c = S3Configuration.builder().pathStyleAccessEnabled(true).build(); // SO11228792
125+
s3Client = S3Client.builder().serviceConfiguration(s3c).region(REGION).
126+
credentialsProvider(credentialsProvider).endpointOverride(new URI(MINIO_ENDPOINT)).build();
105127
}
106128

107129
@Test
108130
void s3PipelineIteratorS3FetcherAndS3Emitter() throws Exception {
109131

110132
// create s3 bucket for fetches and for emits
111-
s3Client.createBucket(FETCH_BUCKET);
112-
s3Client.createBucket(EMIT_BUCKET);
133+
s3Client.createBucket(CreateBucketRequest.builder().bucket(FETCH_BUCKET).build());
134+
s3Client.createBucket(CreateBucketRequest.builder().bucket(EMIT_BUCKET).build());
113135

114136
// create some test files and insert into fetch bucket
115137
createTestFiles();
@@ -139,9 +161,10 @@ void s3PipelineIteratorS3FetcherAndS3Emitter() throws Exception {
139161
}
140162

141163
for (String testFile : testFiles) {
142-
S3Object object = s3Client.getObject(EMIT_BUCKET, testFile + ".json");
164+
GetObjectRequest objectRequest = GetObjectRequest.builder().bucket(EMIT_BUCKET).key(testFile + ".json").build();
165+
ResponseInputStream<GetObjectResponse> object = s3Client.getObject(objectRequest);
143166
Assertions.assertNotNull(object);
144-
String data = IOUtils.toString(object.getObjectContent(), StandardCharsets.UTF_8);
167+
String data = IOUtils.toString(object, StandardCharsets.UTF_8);
145168
MatcherAssert.assertThat(
146169
"Should be able to read the parsed body of the HTML file as the body of the document",
147170
data, Matchers.containsString("body-of-" + testFile));
@@ -159,6 +182,6 @@ private String createTikaConfigXml(File tikaConfigFile, File log4jPropFile,
159182
.replace("{EMIT_BUCKET}", EMIT_BUCKET).replace("{FETCH_BUCKET}", FETCH_BUCKET)
160183
.replace("{ACCESS_KEY}", ACCESS_KEY).replace("{SECRET_KEY}", SECRET_KEY)
161184
.replace("{ENDPOINT_CONFIGURATION_SERVICE}", MINIO_ENDPOINT)
162-
.replace("{REGION}", REGION);
185+
.replace("{REGION}", REGION.id());
163186
}
164187
}

0 commit comments

Comments
 (0)