Skip to content

Commit e8b92a5

Browse files
afalkoafalko
authored
Merge pull request #800 from tarishij17/fix-s3
Fetch all items incase of S3 image tag store
2 parents 02fcadf + 0e0a228 commit e8b92a5

File tree

3 files changed

+73
-7
lines changed

3 files changed

+73
-7
lines changed

‎dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/CommandLine.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,6 @@ static ArgumentParser getArgumentParser() {
9393
"(default: Dockerfile,docker-compose)");
9494
parser.addArgument("-r", "--" + RATE_LIMIT_PR_CREATION)
9595
.type(String.class)
96-
.setDefault("")
9796
.required(false)
9897
.help("Use RateLimiting when sending PRs. RateLimiting is enabled only if this value is set it's disabled by default.");
9998
return parser;

‎dockerfile-image-update/src/main/java/com/salesforce/dockerfileimageupdate/storage/S3BackedImageTagStore.java‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.amazonaws.services.s3.AmazonS3;
44
import com.amazonaws.services.s3.model.ListObjectsV2Result;
5+
import com.amazonaws.services.s3.model.ListObjectsV2Request;
56
import com.amazonaws.services.s3.model.S3Object;
67
import com.amazonaws.services.s3.model.S3ObjectInputStream;
78
import com.amazonaws.services.s3.model.S3ObjectSummary;
@@ -26,6 +27,7 @@ public class S3BackedImageTagStore implements ImageTagStore {
2627
private final AmazonS3 s3;
2728
private final String store;
2829

30+
2931
public S3BackedImageTagStore(AmazonS3 s3, @NonNull String store) {
3032
this.s3 = s3;
3133
this.store = store;
@@ -58,8 +60,7 @@ public void updateStore(String img, String tag) throws IOException {
5860
public List<ImageTagStoreContent> getStoreContent(DockerfileGitHubUtil dockerfileGitHubUtil, String storeName) throws InterruptedException {
5961
List<ImageTagStoreContent> imageNamesWithTag;
6062
Map<String, Date> imageNameWithAccessTime = new HashMap<>();
61-
ListObjectsV2Result result = getS3Objects();
62-
List<S3ObjectSummary> objects = result.getObjectSummaries();
63+
List<S3ObjectSummary> objects = getS3Objects();
6364
for (S3ObjectSummary os : objects) {
6465
Date lastModified = os.getLastModified();
6566
String key = os.getKey();
@@ -108,8 +109,21 @@ private String convertS3ObjectKeyToImageString(String key) {
108109
return key.replace(S3_FILE_KEY_PATH_DELIMITER, '/');
109110
}
110111

111-
private ListObjectsV2Result getS3Objects() {
112-
return s3.listObjectsV2(store);
112+
private List<S3ObjectSummary> getS3Objects() {
113+
ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(store);
114+
ListObjectsV2Result listObjectsV2Result;
115+
List<S3ObjectSummary> objectSummaries = null;
116+
117+
do {
118+
listObjectsV2Result = s3.listObjectsV2(request);
119+
if (objectSummaries == null)
120+
objectSummaries = listObjectsV2Result.getObjectSummaries();
121+
else
122+
objectSummaries.addAll(listObjectsV2Result.getObjectSummaries());
123+
request.setContinuationToken(listObjectsV2Result.getNextContinuationToken());
124+
} while(listObjectsV2Result.isTruncated());
125+
126+
return objectSummaries;
113127
}
114128

115129
private S3Object getS3Object(String store, String key) {

‎dockerfile-image-update/src/test/java/com/salesforce/dockerfileimageupdate/storage/S3BackedImageTagStoreTest.java‎

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
package com.salesforce.dockerfileimageupdate.storage;
22

3+
34
import com.amazonaws.services.s3.AmazonS3;
45
import com.amazonaws.services.s3.model.ListObjectsV2Result;
6+
import com.amazonaws.services.s3.model.ListObjectsV2Request;
57
import com.amazonaws.services.s3.model.S3Object;
68
import com.amazonaws.services.s3.model.S3ObjectInputStream;
79
import com.amazonaws.services.s3.model.S3ObjectSummary;
810
import com.salesforce.dockerfileimageupdate.utils.DockerfileGitHubUtil;
911
import org.testng.annotations.Test;
12+
import static org.mockito.ArgumentMatchers.any;
1013

1114
import java.io.ByteArrayInputStream;
1215
import java.io.IOException;
@@ -36,12 +39,60 @@ public void testUpdateStoreThrowsExceptionWhenBucketDoesNotExist() throws IOExce
3639
verify(amazonS3, times(0)).putObject("store", "image", "tag");
3740
}
3841

42+
@Test
43+
public void testGetStoreContentReturnsStoreContentWithTruncatedResults() throws InterruptedException {
44+
AmazonS3 amazonS3 = mock(AmazonS3.class);
45+
S3BackedImageTagStore s3BackedImageTagStore = spy(new S3BackedImageTagStore(amazonS3, "store"));
46+
DockerfileGitHubUtil dockerfileGitHubUtil = mock(DockerfileGitHubUtil.class);
47+
ListObjectsV2Result listObjectsV2Result1 = mock(ListObjectsV2Result.class);
48+
ListObjectsV2Result listObjectsV2Result2 = mock(ListObjectsV2Result.class);
49+
50+
S3ObjectSummary s3ObjectSummary = mock(S3ObjectSummary.class);
51+
List<S3ObjectSummary> s3ObjectSummaryList = new ArrayList<>();
52+
s3ObjectSummaryList.add(s3ObjectSummary);
53+
54+
Date date = mock(Date.class);
55+
S3Object s3Object = mock(S3Object.class);
56+
S3Object s3Object2 = mock(S3Object.class);
57+
String tag = "tag";
58+
String tag2 = "tag2";
59+
byte tagBytes[] = tag.getBytes();
60+
byte tagBytes2[] = tag2.getBytes();
61+
S3ObjectInputStream objectContent = new S3ObjectInputStream(new ByteArrayInputStream(tagBytes), null);
62+
S3ObjectInputStream objectContent2 = new S3ObjectInputStream(new ByteArrayInputStream(tagBytes2), null);
63+
s3Object.setObjectContent(objectContent);
64+
s3Object2.setObjectContent(objectContent2);
65+
66+
when(amazonS3.listObjectsV2(any(ListObjectsV2Request.class))).thenReturn(listObjectsV2Result1, listObjectsV2Result2);
67+
when(listObjectsV2Result1.getObjectSummaries()).thenReturn(s3ObjectSummaryList);
68+
when(listObjectsV2Result1.isTruncated()).thenReturn(true);
69+
when(listObjectsV2Result2.getObjectSummaries()).thenReturn(s3ObjectSummaryList);
70+
when(listObjectsV2Result2.isTruncated()).thenReturn(false);
71+
when(s3ObjectSummary.getLastModified()).thenReturn(date , date);
72+
when(s3ObjectSummary.getKey()).thenReturn("domain!namespace!image", "domain!namespace!image2");
73+
when(amazonS3.getObject("store", "domain!namespace!image")).thenReturn(s3Object);
74+
when(amazonS3.getObject("store", "domain!namespace!image2")).thenReturn(s3Object2);
75+
when(s3Object.getObjectContent()).thenReturn(objectContent);
76+
when(s3Object2.getObjectContent()).thenReturn(objectContent2);
77+
78+
List<ImageTagStoreContent> actualResult = s3BackedImageTagStore.getStoreContent(dockerfileGitHubUtil, "store");
79+
80+
verify(amazonS3).getObject("store", "domain!namespace!image");
81+
verify(amazonS3).getObject("store", "domain!namespace!image2");
82+
assertEquals(actualResult.size(), 2);
83+
assertEquals(actualResult.get(0).getImageName(), "domain/namespace/image");
84+
assertEquals(actualResult.get(0).getTag(), "tag");
85+
assertEquals(actualResult.get(1).getImageName(), "domain/namespace/image2");
86+
assertEquals(actualResult.get(1).getTag(), "tag2");
87+
}
88+
3989
@Test
4090
public void testGetStoreContentReturnsStoreContent() throws InterruptedException {
4191
AmazonS3 amazonS3 = mock(AmazonS3.class);
4292
S3BackedImageTagStore s3BackedImageTagStore = spy(new S3BackedImageTagStore(amazonS3, "store"));
4393
DockerfileGitHubUtil dockerfileGitHubUtil = mock(DockerfileGitHubUtil.class);
4494
ListObjectsV2Result listObjectsV2Result = mock(ListObjectsV2Result.class);
95+
4596
S3ObjectSummary s3ObjectSummary = mock(S3ObjectSummary.class);
4697
List<S3ObjectSummary> s3ObjectSummaryListList = Collections.singletonList(s3ObjectSummary);
4798
Date date = mock(Date.class);
@@ -51,8 +102,9 @@ public void testGetStoreContentReturnsStoreContent() throws InterruptedException
51102
S3ObjectInputStream objectContent = new S3ObjectInputStream(new ByteArrayInputStream(tagBytes), null);
52103
s3Object.setObjectContent(objectContent);
53104

54-
when(amazonS3.listObjectsV2("store")).thenReturn(listObjectsV2Result);
105+
when(amazonS3.listObjectsV2(any(ListObjectsV2Request.class))).thenReturn(listObjectsV2Result);
55106
when(listObjectsV2Result.getObjectSummaries()).thenReturn(s3ObjectSummaryListList);
107+
when(listObjectsV2Result.isTruncated()).thenReturn(false);
56108
when(s3ObjectSummary.getLastModified()).thenReturn(date);
57109
when(s3ObjectSummary.getKey()).thenReturn("domain!namespace!image");
58110
when(amazonS3.getObject("store", "domain!namespace!image")).thenReturn(s3Object);
@@ -97,8 +149,9 @@ public void testGetStoreContentReturnsStoreContentSorted() throws InterruptedExc
97149
when(s3ObjectSummaryIterator.next()).thenReturn(s3ObjectSummary, s3ObjectSummary);
98150
when(s3ObjectSummaryIterator.hasNext()).thenReturn(true, true, false);
99151
when(s3ObjectSummaryList.iterator()).thenReturn(s3ObjectSummaryIterator);
100-
when(amazonS3.listObjectsV2("store")).thenReturn(listObjectsV2Result);
152+
when(amazonS3.listObjectsV2(any(ListObjectsV2Request.class))).thenReturn(listObjectsV2Result);
101153
when(listObjectsV2Result.getObjectSummaries()).thenReturn(s3ObjectSummaryList);
154+
when(listObjectsV2Result.isTruncated()).thenReturn(false);
102155
when(s3ObjectSummary.getLastModified()).thenReturn(date1, date2);
103156
when(s3ObjectSummary.getKey()).thenReturn(key1, key2);
104157
when(amazonS3.getObject("store", key1)).thenReturn(s3Object1);

0 commit comments

Comments
 (0)