Skip to content

Google Cloud Storage whitelist buckets by regex #85

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
damiano1996 wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Google Cloud Storage whitelist buckets by regex #85

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
671743b
Regular expression to whitelist gcs buckets.
damiano1996 Mar 25, 2023
3c1cf18
gcs-layout.json updated
damiano1996 Mar 25, 2023
afa7a1a
Property for whitelist mode added
damiano1996 Mar 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 21 additions & 9 deletions plugins/gcs/src/main/java/com/dremio/plugins/gcs/GCSConf.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ public enum AuthMode {
@Tag(2) @DisplayMetadata(label = "Automatic/Service Account") AUTO
}

public enum AllowlistedBucketsMode {
@Tag(1) @DisplayMetadata(label = "List Buckets") LIST,
@Tag(2) @DisplayMetadata(label = "Regular Expression") REGEX,
}

@Tag(2)
public AuthMode authMode = AuthMode.SERVICE_ACCOUNT_KEYS;
Expand All @@ -76,51 +80,59 @@ public enum AuthMode {
public boolean allowCreateDrop;

@Tag(7)
public AllowlistedBucketsMode allowlistedBucketsMode = AllowlistedBucketsMode.LIST;

@Tag(8)
@NotMetadataImpacting
@DisplayMetadata(label = "Allowlisted buckets")
@DisplayMetadata(label = "Enter the name of each bucket to be included")
public List<String> bucketWhitelist;

@Tag(9)
@NotMetadataImpacting
@DisplayMetadata(label = "Enter a regular expression pattern to match bucket names. Only matching buckets will be allowed")
public String bucketWhitelistRegexFilter = ".*";

@Tag(10)
@NotMetadataImpacting
@DisplayMetadata(label = "Enable asynchronous access when possible")
public boolean asyncEnabled = true;

@Tag(10)
@Tag(11)
@NotMetadataImpacting
@DisplayMetadata(label = "Enable local caching when possible")
public boolean cachingEnable = true;

@Tag(11)
@Tag(12)
@NotMetadataImpacting
@Min(value = 1, message = "Max percent of total available cache space must be between 1 and 100")
@Max(value = 100, message = "Max percent of total available cache space must be between 1 and 100")
@DisplayMetadata(label = "Max percent of total available cache space to use when possible")
public int cachePercent = 70;

@Tag(12)
@Tag(13)
@DisplayMetadata(label = "Private Key ID")
public String privateKeyId = "";

@Tag(13)
@Tag(14)
@Secret
@DisplayMetadata(label = "Private Key")
public String privateKey = "";

@Tag(14)
@Tag(15)
@DisplayMetadata(label = "Client Email")
public String clientEmail = "";

@Tag(15)
@Tag(16)
@DisplayMetadata(label = "Client ID")
public String clientId = "";


@Tag(16)
@Tag(17)
@NotMetadataImpacting
@DisplayMetadata(label = "Default CTAS Format")
public DefaultCtasFormatSelection defaultCtasFormat = DefaultCtasFormatSelection.ICEBERG;

@Tag(17)
@Tag(18)
@NotMetadataImpacting
@DisplayMetadata(label = "Enable partition column inference")
public boolean isPartitionInferenceEnabled = false;
Expand Down
57 changes: 42 additions & 15 deletions plugins/gcs/src/main/java/com/dremio/plugins/gcs/GoogleBucketFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem;
import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemConfiguration;
import com.google.cloud.storage.BucketInfo;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.Storage.BlobListOption;
import com.google.cloud.storage.Storage.BucketListOption;
Expand All @@ -67,6 +68,7 @@ public class GoogleBucketFileSystem extends ContainerFileSystem implements MayPr
private static final Logger logger = LoggerFactory.getLogger(GoogleBucketFileSystem.class);

private static final String EMPTY_STRING = "";
private static final String ANY_STRING_REGEX = ".*";
private static final String CONF_ACCOUNT_EMAIL = "fs.gs.auth.service.account.email";
private static final String CONF_PRIVATE_KEY_ID = "fs.gs.auth.service.account.private.key.id";
private static final String CONF_PRIVATE_KEY = "fs.gs.auth.service.account.private.key";
Expand All @@ -78,7 +80,9 @@ public class GoogleBucketFileSystem extends ContainerFileSystem implements MayPr
public static final String DREMIO_CLIENT_ID = "dremio.gcs.clientId";
public static final String DREMIO_PRIVATE_KEY_ID = "dremio.gcs.privateKeyId";
public static final String DREMIO_PRIVATE_KEY = "dremio.gcs.privateKey";
public static final String DREMIO_WHITELIST_MODE = "dremio.gcs.whitelisted.mode";
public static final String DREMIO_WHITELIST_BUCKETS = "dremio.gcs.whitelisted.buckets";
public static final String DREMIO_WHITELIST_BUCKETS_REGEX = "dremio.gcs.whitelisted.regex";

private static final List<String> UNIQUE_PROPERTIES = ImmutableList.<String>of(
CONF_PROJECT_ID,
Expand Down Expand Up @@ -138,12 +142,19 @@ protected void setup(Configuration conf) throws IOException {
gcsConf.authMode = AuthMode.AUTO;
}

gcsConf.projectId = conf.get(DREMIO_PROJECT_ID, EMPTY_STRING);
gcsConf.projectId = conf.get(DREMIO_PROJECT_ID, EMPTY_STRING);
if (gcsConf.projectId.equals(EMPTY_STRING)) {
gcsConf.projectId = conf.get("fs.gs.project.id", EMPTY_STRING);
}
gcsConf.asyncEnabled = true;
gcsConf.bucketWhitelist = getWhiteListBuckets(conf);

if (conf.getBoolean(DREMIO_WHITELIST_MODE, true)) {
gcsConf.allowlistedBucketsMode = GCSConf.AllowlistedBucketsMode.LIST;
gcsConf.bucketWhitelist = getWhiteListBuckets(conf);
} else {
gcsConf.allowlistedBucketsMode = GCSConf.AllowlistedBucketsMode.REGEX;
gcsConf.bucketWhitelistRegexFilter = conf.get(DREMIO_WHITELIST_BUCKETS_REGEX, ANY_STRING_REGEX);
}

this.connectionConf = gcsConf;

Expand Down Expand Up @@ -185,7 +196,7 @@ protected void setup(Configuration conf) throws IOException {
}

private List<String> getWhiteListBuckets(Configuration conf) {
String bucketList = conf.get(DREMIO_WHITELIST_BUCKETS,"");
String bucketList = conf.get(DREMIO_WHITELIST_BUCKETS, EMPTY_STRING);
return Arrays.stream(bucketList.split(","))
.map(String::trim)
.filter(input -> !Strings.isNullOrEmpty(input))
Expand All @@ -194,20 +205,36 @@ private List<String> getWhiteListBuckets(Configuration conf) {

@Override
protected Stream<ContainerCreator> getContainerCreators() throws IOException {
final Stream<String> bucketNames;
if (connectionConf.bucketWhitelist != null && !connectionConf.bucketWhitelist.isEmpty()) {
bucketNames = connectionConf.bucketWhitelist.stream();
} else {
try {
bucketNames = StreamSupport.stream(storage.list(BucketListOption.pageSize(100)).iterateAll().spliterator(), false).map(b -> b.getName());
} catch (StorageException se) {
throw UserException.validationError(se)
.message("Failed to list buckets.")
.build(logger);
}
return getBucketNames().map(GCSContainerCreator::new);
}

private Stream<String> getBucketNames() {
switch (connectionConf.allowlistedBucketsMode) {
case LIST:
default:
if (connectionConf.bucketWhitelist != null && !connectionConf.bucketWhitelist.isEmpty()) {
return connectionConf.bucketWhitelist.stream();
}
break;
case REGEX:
if (connectionConf.bucketWhitelistRegexFilter != null && !connectionConf.bucketWhitelistRegexFilter.equals("")) {
return getStorageBucketNameStream().filter(s -> s.matches(connectionConf.bucketWhitelistRegexFilter));
}
break;
}
return getStorageBucketNameStream();
}

return bucketNames.map(b -> new GCSContainerCreator(b));
private Stream<String> getStorageBucketNameStream() {
try {
return StreamSupport
.stream(storage.list(BucketListOption.pageSize(100)).iterateAll().spliterator(), false)
.map(BucketInfo::getName);
} catch (StorageException se) {
throw UserException.validationError(se)
.message("Failed to list buckets.")
.build(logger);
}
}

private final class FileSystemSupplierImpl extends FileSystemSupplier {
Expand Down
62 changes: 37 additions & 25 deletions plugins/gcs/src/main/java/com/dremio/plugins/gcs/GoogleStoragePlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,43 +50,42 @@
*/
@Options
public class GoogleStoragePlugin extends DirectorySupportLackingFileSystemPlugin<GCSConf> {
private static final Logger logger = LoggerFactory.getLogger(GoogleStoragePlugin.class);

public static final BooleanValidator ASYNC_READS = new BooleanValidator("store.gcs.async", true);
public static final String GCS_OUTPUT_STREAM_UPLOAD_CHUNK_SIZE_DEFAULT = "8388608";
private static final Logger logger = LoggerFactory.getLogger(GoogleStoragePlugin.class);

public GoogleStoragePlugin(
GCSConf config,
SabotContext context,
String name,
Provider<StoragePluginId> idProvider) {
GCSConf config,
SabotContext context,
String name,
Provider<StoragePluginId> idProvider) {
super(config, context, name, idProvider);
}

@Override
protected List<Property> getProperties() {
List<Property> properties = new ArrayList<>();
properties.add(new Property(String.format("fs.%s.impl", DREMIO_GCS_SCHEME),GoogleBucketFileSystem.class.getName()));
properties.add(new Property(String.format("fs.%s.impl.disable.cache", DREMIO_GCS_SCHEME),"true"));
properties.add(new Property(String.format("fs.%s.impl", DREMIO_GCS_SCHEME), GoogleBucketFileSystem.class.getName()));
properties.add(new Property(String.format("fs.%s.impl.disable.cache", DREMIO_GCS_SCHEME), "true"));
properties.add(new Property(GoogleHadoopFileSystemConfiguration.GCS_OUTPUT_STREAM_UPLOAD_CHUNK_SIZE.getKey(),
GCS_OUTPUT_STREAM_UPLOAD_CHUNK_SIZE_DEFAULT));
GCS_OUTPUT_STREAM_UPLOAD_CHUNK_SIZE_DEFAULT));

GCSConf conf = getConfig();

if ("".equals(conf.projectId)) {
throw UserException.validationError()
.message("Failure creating GCS connection. You must provide Project ID")
.build(logger);
.message("Failure creating GCS connection. You must provide Project ID")
.build(logger);
}
switch (conf.authMode) {
case SERVICE_ACCOUNT_KEYS:
if ("".equals(conf.clientEmail) ||
"".equals(conf.clientId) ||
"".equals(conf.privateKey) ||
"".equals(conf.privateKeyId)) {
"".equals(conf.clientId) ||
"".equals(conf.privateKey) ||
"".equals(conf.privateKeyId)) {
throw UserException.validationError()
.message("Failure creating GCS connection. You must provide Private Key ID, Private Key, Client E-mail and Client ID.")
.build(logger);
.message("Failure creating GCS connection. You must provide Private Key ID, Private Key, Client E-mail and Client ID.")
.build(logger);
}
break;
case AUTO:
Expand All @@ -107,10 +106,23 @@ protected List<Property> getProperties() {
properties.add(new Property(GoogleBucketFileSystem.DREMIO_KEY_FILE, "false"));
break;
}

properties.add(new Property(GoogleBucketFileSystem.DREMIO_PROJECT_ID, conf.projectId));
properties.add(new Property(GoogleBucketFileSystem.DREMIO_WHITELIST_BUCKETS,
(conf.bucketWhitelist != null && !conf.bucketWhitelist.isEmpty()) ? String.join(",", conf.bucketWhitelist) : ""));
if(conf.getProperties() != null) {

switch (conf.allowlistedBucketsMode) {
case LIST:
default:
properties.add(new Property(GoogleBucketFileSystem.DREMIO_WHITELIST_MODE, "true"));
properties.add(new Property(GoogleBucketFileSystem.DREMIO_WHITELIST_BUCKETS,
(conf.bucketWhitelist != null && !conf.bucketWhitelist.isEmpty()) ? String.join(",", conf.bucketWhitelist) : ""));
break;
case REGEX:
properties.add(new Property(GoogleBucketFileSystem.DREMIO_WHITELIST_MODE, "false"));
properties.add(new Property(GoogleBucketFileSystem.DREMIO_WHITELIST_BUCKETS_REGEX, conf.bucketWhitelistRegexFilter));
break;
}

if (conf.getProperties() != null) {
properties.addAll(conf.getProperties());
}
return properties;
Expand All @@ -128,8 +140,8 @@ public CreateTableEntry createNewTable(
final String containerName = tableSchemaPath.getPathComponents().get(1);
if (tableSchemaPath.size() == 2) {
throw UserException.validationError()
.message("Creating buckets is not supported (name: %s)", containerName)
.build(logger);
.message("Creating buckets is not supported (name: %s)", containerName)
.build(logger);
}

final CreateTableEntry entry = super.createNewTable(tableSchemaPath, config, icebergProps, writerOptions, storageOptions, isResultsTable);
Expand All @@ -138,8 +150,8 @@ public CreateTableEntry createNewTable(

if (!fs.containerExists(containerName)) {
throw UserException.validationError()
.message("Cannot create the table because '%s' container does not exist.", containerName)
.build(logger);
.message("Cannot create the table because '%s' container does not exist.", containerName)
.build(logger);
}
return entry;
}
Expand All @@ -155,11 +167,11 @@ public SourceState getState() {
GoogleBucketFileSystem fs = getSystemUserFS().unwrap(GoogleBucketFileSystem.class);
fs.refreshFileSystems();
List<ContainerFailure> failures = fs.getSubFailures();
if(failures.isEmpty()) {
if (failures.isEmpty()) {
return SourceState.GOOD;
}
StringBuilder sb = new StringBuilder();
for(ContainerFailure f : failures) {
for (ContainerFailure f : failures) {
sb.append(f.getName());
sb.append(": ");
sb.append(f.getException().getMessage());
Expand Down
38 changes: 31 additions & 7 deletions plugins/gcs/src/main/resources/gcs-layout.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,15 +121,39 @@
]
},
{
"name": "Bucket Allowlist",
"elements": [
{
"propName": "config.bucketWhitelist[]",
"uiType": "value_list",
"emptyLabel": "No allowlisted buckets added",
"addLabel": "Add bucket",
"validate": {
"isRequired": false
}
"propName": "config.allowlistedBucketsMode",
"uiType": "container_selection",
"options": [
{
"value": "LIST",
"container": {
"elements": [
{
"propName": "config.bucketWhitelist[]",
"uiType": "value_list",
"emptyLabel": "No allowlisted buckets added",
"addLabel": "Add bucket",
"validate": {
"isRequired": false
}
}
]
}
},
{
"value": "REGEX",
"container": {
"elements": [
{
"propName": "config.bucketWhitelistRegexFilter"
}
]
}
}
]
}
]
},
Expand Down