Skip to content

Commit 35fea54

Browse files
timuralpgaul
timuralp
authored and
gaul
committed
Alias middleware to remap backend buckets
Alias middleware implements a way to remap backend buckets to a configurable front-end name. The mappings are configured as: s3proxy.alias-blobstore.<s3proxy bucket> = <backend bucket> A single bucket cannot be mapped to multiple names.
1 parent 2521db9 commit 35fea54

File tree

4 files changed

+487
-0
lines changed

4 files changed

+487
-0
lines changed

‎src/main/java/org/gaul/s3proxy/AliasBlobStore.java

+290
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,290 @@
1+
/*
2+
* Copyright 2014-2021 Andrew Gaul <[email protected]>
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.gaul.s3proxy;
18+
19+
import static java.util.Objects.requireNonNull;
20+
21+
import static com.google.common.base.Preconditions.checkArgument;
22+
23+
import java.util.HashMap;
24+
import java.util.List;
25+
import java.util.Map;
26+
import java.util.Properties;
27+
28+
import com.google.common.collect.BiMap;
29+
import com.google.common.collect.ImmutableBiMap;
30+
import com.google.common.collect.ImmutableList;
31+
32+
import org.jclouds.blobstore.BlobStore;
33+
import org.jclouds.blobstore.domain.Blob;
34+
import org.jclouds.blobstore.domain.BlobMetadata;
35+
import org.jclouds.blobstore.domain.ContainerAccess;
36+
import org.jclouds.blobstore.domain.MultipartPart;
37+
import org.jclouds.blobstore.domain.MultipartUpload;
38+
import org.jclouds.blobstore.domain.MutableStorageMetadata;
39+
import org.jclouds.blobstore.domain.PageSet;
40+
import org.jclouds.blobstore.domain.StorageMetadata;
41+
import org.jclouds.blobstore.domain.internal.MutableStorageMetadataImpl;
42+
import org.jclouds.blobstore.domain.internal.PageSetImpl;
43+
import org.jclouds.blobstore.options.CopyOptions;
44+
import org.jclouds.blobstore.options.CreateContainerOptions;
45+
import org.jclouds.blobstore.options.GetOptions;
46+
import org.jclouds.blobstore.options.ListContainerOptions;
47+
import org.jclouds.blobstore.options.PutOptions;
48+
import org.jclouds.blobstore.util.ForwardingBlobStore;
49+
import org.jclouds.domain.Location;
50+
import org.jclouds.io.Payload;
51+
52+
/**
53+
* This class implements a middleware to alias buckets to a different name.
54+
* The aliases are configured as:
55+
* s3proxy.alias-blobstore.&lt;alias name&gt; = &lt;backend bucket&gt;
56+
*
57+
* The aliases appear in bucket listings if the configured
58+
* backend buckets are present. Requests for all other buckets are unaffected.
59+
*/
60+
public final class AliasBlobStore extends ForwardingBlobStore {
61+
private final BiMap<String, String> aliases;
62+
63+
private AliasBlobStore(BlobStore delegate,
64+
BiMap<String, String> aliases) {
65+
super(delegate);
66+
this.aliases = requireNonNull(aliases);
67+
}
68+
69+
static BlobStore newAliasBlobStore(BlobStore delegate,
70+
BiMap<String, String> aliases) {
71+
return new AliasBlobStore(delegate, aliases);
72+
}
73+
74+
private MultipartUpload getDelegateMpu(MultipartUpload mpu) {
75+
return MultipartUpload.create(
76+
getContainer(mpu.containerName()),
77+
mpu.blobName(),
78+
mpu.id(),
79+
mpu.blobMetadata(),
80+
mpu.putOptions());
81+
}
82+
83+
public static ImmutableBiMap<String, String> parseAliases(
84+
Properties properties) {
85+
Map<String, String> backendBuckets = new HashMap<>();
86+
for (String key : properties.stringPropertyNames()) {
87+
if (key.startsWith(S3ProxyConstants.PROPERTY_ALIAS_BLOBSTORE)) {
88+
String virtualBucket = key.substring(
89+
S3ProxyConstants.PROPERTY_ALIAS_BLOBSTORE.length() + 1);
90+
String backendBucket = properties.getProperty(key);
91+
checkArgument(
92+
!backendBuckets.containsKey(backendBucket),
93+
"Backend bucket %s is aliased twice",
94+
backendBucket);
95+
backendBuckets.put(backendBucket, virtualBucket);
96+
}
97+
}
98+
return ImmutableBiMap.copyOf(backendBuckets).inverse();
99+
}
100+
101+
private String getContainer(String container) {
102+
return this.aliases.getOrDefault(container, container);
103+
}
104+
105+
@Override
106+
public boolean createContainerInLocation(Location location,
107+
String container) {
108+
return this.delegate().createContainerInLocation(location,
109+
getContainer(container));
110+
}
111+
112+
@Override
113+
public boolean createContainerInLocation(
114+
Location location, String container,
115+
CreateContainerOptions options) {
116+
return delegate().createContainerInLocation(
117+
location, getContainer(container), options);
118+
}
119+
120+
@Override
121+
public boolean containerExists(String container) {
122+
return delegate().containerExists(getContainer(container));
123+
}
124+
125+
@Override
126+
public ContainerAccess getContainerAccess(String container) {
127+
return delegate().getContainerAccess(getContainer(container));
128+
}
129+
130+
@Override
131+
public void setContainerAccess(String container,
132+
ContainerAccess containerAccess) {
133+
delegate().setContainerAccess(getContainer(container), containerAccess);
134+
}
135+
136+
@Override
137+
public PageSet<? extends StorageMetadata> list() {
138+
PageSet<? extends StorageMetadata> upstream = this.delegate().list();
139+
ImmutableList.Builder<StorageMetadata> results =
140+
new ImmutableList.Builder<>();
141+
for (StorageMetadata sm : upstream) {
142+
if (aliases.containsValue(sm.getName())) {
143+
MutableStorageMetadata bucketAlias =
144+
new MutableStorageMetadataImpl();
145+
bucketAlias.setName(aliases.inverse().get(sm.getName()));
146+
bucketAlias.setCreationDate(sm.getCreationDate());
147+
bucketAlias.setETag(sm.getETag());
148+
bucketAlias.setId(sm.getProviderId());
149+
bucketAlias.setLastModified(sm.getLastModified());
150+
bucketAlias.setLocation(sm.getLocation());
151+
bucketAlias.setSize(sm.getSize());
152+
bucketAlias.setTier(sm.getTier());
153+
bucketAlias.setType(sm.getType());
154+
// TODO: the URI should be rewritten to use the alias
155+
bucketAlias.setUri(sm.getUri());
156+
bucketAlias.setUserMetadata(sm.getUserMetadata());
157+
results.add(bucketAlias);
158+
} else {
159+
results.add(sm);
160+
}
161+
}
162+
return new PageSetImpl<>(results.build(), upstream.getNextMarker());
163+
}
164+
165+
@Override
166+
public PageSet<? extends StorageMetadata> list(String container) {
167+
return delegate().list(getContainer(container));
168+
}
169+
170+
@Override
171+
public PageSet<? extends StorageMetadata> list(
172+
String container, ListContainerOptions options) {
173+
return delegate().list(getContainer(container), options);
174+
}
175+
176+
@Override
177+
public void clearContainer(String container) {
178+
delegate().clearContainer(getContainer(container));
179+
}
180+
181+
@Override
182+
public void clearContainer(String container, ListContainerOptions options) {
183+
delegate().clearContainer(getContainer(container), options);
184+
}
185+
186+
@Override
187+
public void deleteContainer(String container) {
188+
delegate().deleteContainer(getContainer(container));
189+
}
190+
191+
@Override
192+
public boolean deleteContainerIfEmpty(String container) {
193+
return delegate().deleteContainerIfEmpty(getContainer(container));
194+
}
195+
196+
@Override
197+
public boolean directoryExists(String container, String directory) {
198+
return delegate().directoryExists(getContainer(container), directory);
199+
}
200+
201+
@Override
202+
public void createDirectory(String container, String directory) {
203+
delegate().createDirectory(getContainer(container), directory);
204+
}
205+
206+
@Override
207+
public void deleteDirectory(String container, String directory) {
208+
delegate().deleteDirectory(getContainer(container), directory);
209+
}
210+
211+
@Override
212+
public boolean blobExists(String container, String name) {
213+
return delegate().blobExists(getContainer(container), name);
214+
}
215+
216+
@Override
217+
public BlobMetadata blobMetadata(String container, String name) {
218+
return delegate().blobMetadata(getContainer(container), name);
219+
}
220+
221+
@Override
222+
public Blob getBlob(String containerName, String blobName) {
223+
return delegate().getBlob(getContainer(containerName), blobName);
224+
}
225+
226+
@Override
227+
public Blob getBlob(String containerName, String blobName,
228+
GetOptions getOptions) {
229+
return delegate().getBlob(getContainer(containerName), blobName,
230+
getOptions);
231+
}
232+
233+
@Override
234+
public String putBlob(String containerName, Blob blob) {
235+
return delegate().putBlob(getContainer(containerName), blob);
236+
}
237+
238+
@Override
239+
public String putBlob(final String containerName, Blob blob,
240+
final PutOptions options) {
241+
return delegate().putBlob(getContainer(containerName), blob,
242+
options);
243+
}
244+
245+
@Override
246+
public void removeBlob(final String containerName, final String blobName) {
247+
delegate().removeBlob(getContainer(containerName), blobName);
248+
}
249+
250+
@Override
251+
public void removeBlobs(final String containerName,
252+
final Iterable<String> blobNames) {
253+
delegate().removeBlobs(getContainer(containerName), blobNames);
254+
}
255+
256+
@Override
257+
public String copyBlob(final String fromContainer, final String fromName,
258+
final String toContainer, final String toName,
259+
final CopyOptions options) {
260+
return delegate().copyBlob(getContainer(fromContainer), fromName,
261+
getContainer(toContainer), toName, options);
262+
}
263+
264+
@Override
265+
public MultipartUpload initiateMultipartUpload(
266+
String container, BlobMetadata blobMetadata, PutOptions options) {
267+
MultipartUpload mpu = delegate().initiateMultipartUpload(
268+
getContainer(container), blobMetadata, options);
269+
return MultipartUpload.create(container, blobMetadata.getName(),
270+
mpu.id(), mpu.blobMetadata(), mpu.putOptions());
271+
}
272+
273+
@Override
274+
public void abortMultipartUpload(MultipartUpload mpu) {
275+
delegate().abortMultipartUpload(getDelegateMpu(mpu));
276+
}
277+
278+
@Override
279+
public String completeMultipartUpload(final MultipartUpload mpu,
280+
final List<MultipartPart> parts) {
281+
return delegate().completeMultipartUpload(getDelegateMpu(mpu), parts);
282+
}
283+
284+
@Override
285+
public MultipartPart uploadMultipartPart(MultipartUpload mpu,
286+
int partNumber, Payload payload) {
287+
return delegate().uploadMultipartPart(getDelegateMpu(mpu), partNumber,
288+
payload);
289+
}
290+
}

‎src/main/java/org/gaul/s3proxy/Main.java

+8
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import java.util.concurrent.TimeUnit;
3434

3535
import com.google.common.base.Strings;
36+
import com.google.common.collect.ImmutableBiMap;
3637
import com.google.common.collect.ImmutableList;
3738
import com.google.common.collect.ImmutableMap;
3839
import com.google.common.collect.Maps;
@@ -224,6 +225,13 @@ private static BlobStore parseMiddlewareProperties(BlobStore blobStore,
224225
blobStore = ReadOnlyBlobStore.newReadOnlyBlobStore(blobStore);
225226
}
226227

228+
ImmutableBiMap<String, String> aliases = AliasBlobStore.parseAliases(
229+
properties);
230+
if (!aliases.isEmpty()) {
231+
System.err.println("Using alias backend");
232+
blobStore = AliasBlobStore.newAliasBlobStore(blobStore, aliases);
233+
}
234+
227235
ImmutableMap<String, Integer> shards =
228236
ShardedBlobStore.parseBucketShards(properties);
229237
ImmutableMap<String, String> prefixes =

‎src/main/java/org/gaul/s3proxy/S3ProxyConstants.java

+3
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ public final class S3ProxyConstants {
7878
/** Probability of eventual consistency, between 0.0 and 1.0. */
7979
public static final String PROPERTY_EVENTUAL_CONSISTENCY_PROBABILITY =
8080
"s3proxy.eventual-consistency.probability";
81+
/** Alias a backend bucket to an alternate name. */
82+
public static final String PROPERTY_ALIAS_BLOBSTORE =
83+
"s3proxy.alias-blobstore";
8184
/** Discard object data. */
8285
public static final String PROPERTY_NULL_BLOBSTORE =
8386
"s3proxy.null-blobstore";

0 commit comments

Comments
 (0)