Skip to content

Commit 52ea809

Browse files
vintmdvintmd
authored
pass through cos ranger policy url and auth md5 (#71)
change the version to v8.1.6 and begin to release Co-authored-by: alantong(佟明达) <[email protected]>
1 parent ba4cbb7 commit 52ea809

File tree

6 files changed

+109
-34
lines changed

6 files changed

+109
-34
lines changed

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>com.qcloud.cos</groupId>
88
<artifactId>hadoop-cos</artifactId>
9-
<version>8.1.5</version>
9+
<version>8.1.6</version>
1010
<packaging>jar</packaging>
1111

1212
<name>Apache Hadoop Tencent Cloud COS Support</name>

‎src/main/java/org/apache/hadoop/fs/CosFileSystem.java

Lines changed: 37 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -116,15 +116,19 @@ public void initialize(URI uri, Configuration conf) throws IOException {
116116
// network version start from the 2.7.
117117
// sdk version start from the 1.0.4.
118118
this.actualImplFS = getActualFileSystemByClassName(posixBucketFSImpl);
119-
if (this.actualImplFS instanceof CHDFSHadoopFileSystemAdapter) {
119+
120+
// judge normal impl first, skip the class nodef error when only use normal bucket
121+
if (this.actualImplFS instanceof CosNFileSystem) {
122+
this.nativeStore.isPosixBucket(true);
123+
((CosNFileSystem) this.actualImplFS).withStore(this.nativeStore).withBucket(bucket)
124+
.withPosixBucket(isPosixFSStore).withRangerCredentialsClient(rangerCredentialsClient);
125+
} else if (this.actualImplFS instanceof CHDFSHadoopFileSystemAdapter) {
126+
// judge whether ranger client contains policy url or other config need to pass to ofs
127+
this.passThroughRangerConfig();
120128
// before the init, must transfer the config and disable the range in ofs
121129
this.transferOfsConfig();
122130
this.nativeStore.close();
123131
this.nativeStore = null;
124-
} else if (this.actualImplFS instanceof CosNFileSystem) {
125-
this.nativeStore.isPosixBucket(true);
126-
((CosNFileSystem) this.actualImplFS).withStore(this.nativeStore).withBucket(bucket)
127-
.withPosixBucket(isPosixFSStore).withRangerCredentialsClient(rangerCredentialsClient);
128132
} else {
129133
// Another class
130134
throw new IOException(
@@ -202,7 +206,7 @@ public boolean delete(Path f, boolean recursive) throws IOException {
202206
@Override
203207
public FileStatus getFileStatus(Path f) throws IOException {
204208
LOG.debug("Get file status: {}.", f);
205-
checkPermission(f, RangerAccessType.READ);
209+
// keep same not change ranger permission here
206210
return this.actualImplFS.getFileStatus(f);
207211
}
208212

@@ -367,6 +371,25 @@ public NativeFileSystemStore getStore() {
367371
return this.nativeStore;
368372
}
369373

374+
// pass ofs ranger client config to ofs
375+
private void passThroughRangerConfig() {
376+
if (!this.rangerCredentialsClient.isEnableRangerPluginPermissionCheck()) {
377+
LOG.info("not enable ranger plugin permission check");
378+
return;
379+
}
380+
// todo: alantong, ofs java sdk decide the key
381+
if (this.rangerCredentialsClient.getRangerPolicyUrl() != null) {
382+
String policyUrlKey = Constants.COSN_CONFIG_TRANSFER_PREFIX.
383+
concat(Constants.COSN_POSIX_BUCKET_RANGER_POLICY_URL);
384+
this.getConf().set(policyUrlKey, this.rangerCredentialsClient.getRangerPolicyUrl());
385+
}
386+
if (this.rangerCredentialsClient.getAuthJarMd5() != null) {
387+
String authJarMd5Key = Constants.COSN_CONFIG_TRANSFER_PREFIX.
388+
concat(Constants.COSN_POSIX_BUCKET_RANGER_AUTH_JAR_MD5);
389+
this.getConf().set(authJarMd5Key, this.rangerCredentialsClient.getAuthJarMd5());
390+
}
391+
}
392+
370393
// exclude the ofs original config, filter the ofs config with COSN_CONFIG_TRANSFER_PREFIX
371394
private void transferOfsConfig() {
372395
// 1. list to get transfer prefix ofs config
@@ -404,11 +427,18 @@ public String getCanonicalServiceName() {
404427
return this.rangerCredentialsClient.doGetCanonicalServiceName();
405428
}
406429

407-
408430
private void checkPermission(Path f, RangerAccessType rangerAccessType) throws IOException {
409431
this.rangerCredentialsClient.doCheckPermission(f, rangerAccessType, getOwnerId(), getWorkingDirectory());
410432
}
411433

434+
/**
435+
* @param conf
436+
* @throws IOException
437+
*/
438+
private void checkCustomAuth(Configuration conf) throws IOException {
439+
this.rangerCredentialsClient.doCheckCustomAuth(conf);
440+
}
441+
412442
private String getOwnerId() {
413443
UserGroupInformation currentUgi;
414444
try {
@@ -430,14 +460,6 @@ private String getOwnerId() {
430460
return shortUserName;
431461
}
432462

433-
/**
434-
* @param conf
435-
* @throws IOException
436-
*/
437-
private void checkCustomAuth(Configuration conf) throws IOException {
438-
this.rangerCredentialsClient.doCheckCustomAuth(conf);
439-
}
440-
441463
@Override
442464
public void close() throws IOException {
443465
LOG.info("begin to close cos file system");

‎src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
@InterfaceStability.Unstable
1313
public class CosNConfigKeys extends CommonConfigurationKeys {
1414
public static final String USER_AGENT = "fs.cosn.user.agent";
15-
public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v8.1.5";
15+
public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v8.1.6";
1616

1717
public static final String TENCENT_EMR_VERSION_KEY = "fs.emr.version";
1818

@@ -97,6 +97,14 @@ public class CosNConfigKeys extends CommonConfigurationKeys {
9797
public static final String FILESTATUS_LIST_MAX_KEYS = "fs.cosn.filestatus.list_max_keys";
9898
public static final int DEFAULT_FILESTATUS_LIST_MAX_KEYS = 2;
9999

100+
// used by normal bucket to control max keys of list status
101+
public static final String LISTSTATUS_LIST_MAX_KEYS = "fs.cosn.liststatus.list_max_keys";
102+
public static final int DEFAULT_LISTSTATUS_LIST_MAX_KEYS = 999;
103+
104+
// used by posix bucket to control max keys of list status
105+
public static final String LISTSTATUS_POSIX_BUCKET__LIST_MAX_KEYS = "fs.cosn.liststatus.posix_bucket.list_max_keys";
106+
public static final int DEFAULT_LISTSTATUS_POSIX_BUCKET_LIST_MAX_KEYS = 5000;
107+
100108
// used for double check complete mpu in case of return cos client exception but status is 200 ok.
101109
public static final String COSN_COMPLETE_MPU_CHECK = "fs.cosn.complete.mpu.check";
102110
public static final boolean DEFAULT_COSN_COMPLETE_MPU_CHECK_ENABLE = true;

‎src/main/java/org/apache/hadoop/fs/CosNFileSystem.java

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,6 @@ public class CosNFileSystem extends FileSystem {
3434

3535
static final String SCHEME = "cosn";
3636
static final String PATH_DELIMITER = Path.SEPARATOR;
37-
static final int BUCKET_LIST_LIMIT = 999;
38-
static final int POSIX_BUCKET_LIST_LIMIT = 5000;
3937
static final Charset METADATA_ENCODING = StandardCharsets.UTF_8;
4038
// The length of name:value pair should be less than or equal to 1024 bytes.
4139
static final int MAX_XATTR_SIZE = 1024;
@@ -357,7 +355,11 @@ public boolean delete(Path f, boolean recursive) throws IOException {
357355
}
358356
// how to tell the result
359357
if (!isPosixBucket) {
360-
internalRecursiveDelete(key);
358+
int listMaxLength = this.getConf().getInt(
359+
CosNConfigKeys.LISTSTATUS_LIST_MAX_KEYS,
360+
CosNConfigKeys.DEFAULT_LISTSTATUS_LIST_MAX_KEYS
361+
);
362+
internalRecursiveDelete(key, listMaxLength);
361363
} else {
362364
internalAutoRecursiveDelete(key);
363365
}
@@ -372,14 +374,14 @@ public boolean delete(Path f, boolean recursive) throws IOException {
372374
return true;
373375
}
374376

375-
private void internalRecursiveDelete(String key) throws IOException {
377+
private void internalRecursiveDelete(String key, int listMaxLength) throws IOException {
376378
CosNDeleteFileContext deleteFileContext = new CosNDeleteFileContext();
377379
int deleteToFinishes = 0;
378380

379381
String priorLastKey = null;
380382
do {
381383
CosNPartialListing listing =
382-
nativeStore.list(key, this.BUCKET_LIST_LIMIT, priorLastKey, true);
384+
nativeStore.list(key, listMaxLength, priorLastKey, true);
383385
for (FileMetadata file : listing.getFiles()) {
384386
checkPermission(new Path(file.getKey()), RangerAccessType.DELETE);
385387
this.boundedCopyThreadPool.execute(new CosNDeleteFileTask(
@@ -486,12 +488,18 @@ public FileStatus getFileStatus(Path f) throws IOException {
486488
public FileStatus[] listStatus(Path f) throws IOException {
487489
Path absolutePath = makeAbsolute(f);
488490
String key = pathToKey(absolutePath);
489-
int listMaxLength = CosNFileSystem.BUCKET_LIST_LIMIT;
491+
492+
int listMaxLength = this.getConf().getInt(
493+
CosNConfigKeys.LISTSTATUS_LIST_MAX_KEYS,
494+
CosNConfigKeys.DEFAULT_LISTSTATUS_LIST_MAX_KEYS
495+
);
490496
if (isPosixBucket) {
491-
listMaxLength = CosNFileSystem.POSIX_BUCKET_LIST_LIMIT;
497+
listMaxLength = this.getConf().getInt(
498+
CosNConfigKeys.LISTSTATUS_POSIX_BUCKET__LIST_MAX_KEYS,
499+
CosNConfigKeys.DEFAULT_LISTSTATUS_POSIX_BUCKET_LIST_MAX_KEYS
500+
);
492501
}
493502

494-
495503
if (key.length() > 0) {
496504
FileMetadata meta = nativeStore.retrieveMetadata(key);
497505
if (meta != null && meta.isFile()) {
@@ -784,16 +792,21 @@ public boolean rename(Path src, Path dst) throws IOException {
784792
}
785793

786794
if (!isPosixBucket) {
787-
return internalCopyAndDelete(src, dst, srcFileStatus.isDirectory());
795+
int listMaxLength = this.getConf().getInt(
796+
CosNConfigKeys.LISTSTATUS_LIST_MAX_KEYS,
797+
CosNConfigKeys.DEFAULT_LISTSTATUS_LIST_MAX_KEYS
798+
);
799+
return internalCopyAndDelete(src, dst, srcFileStatus.isDirectory(), listMaxLength);
788800
} else {
789801
return internalRename(src, dst);
790802
}
791803
}
792804

793-
private boolean internalCopyAndDelete(Path srcPath, Path dstPath, boolean isDir) throws IOException {
805+
private boolean internalCopyAndDelete(Path srcPath, Path dstPath,
806+
boolean isDir, int listMaxLength) throws IOException {
794807
boolean result = false;
795808
if (isDir) {
796-
result = this.copyDirectory(srcPath, dstPath);
809+
result = this.copyDirectory(srcPath, dstPath, listMaxLength);
797810
} else {
798811
result = this.copyFile(srcPath, dstPath);
799812
}
@@ -822,7 +835,7 @@ private boolean copyFile(Path srcPath, Path dstPath) throws IOException {
822835
return true;
823836
}
824837

825-
private boolean copyDirectory(Path srcPath, Path dstPath) throws IOException {
838+
private boolean copyDirectory(Path srcPath, Path dstPath, int listMaxLength) throws IOException {
826839
String srcKey = pathToKey(srcPath);
827840
if (!srcKey.endsWith(PATH_DELIMITER)) {
828841
srcKey += PATH_DELIMITER;
@@ -849,7 +862,7 @@ private boolean copyDirectory(Path srcPath, Path dstPath) throws IOException {
849862
String priorLastKey = null;
850863
do {
851864
CosNPartialListing objectList = this.nativeStore.list(srcKey,
852-
this.BUCKET_LIST_LIMIT, priorLastKey, true);
865+
listMaxLength, priorLastKey, true);
853866
for (FileMetadata file : objectList.getFiles()) {
854867
checkPermission(new Path(file.getKey()), RangerAccessType.DELETE);
855868
this.boundedCopyThreadPool.execute(new CosNCopyFileTask(

‎src/main/java/org/apache/hadoop/fs/RangerCredentialsClient.java

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,7 @@
44
import org.apache.hadoop.conf.Configuration;
55
import org.apache.hadoop.fs.auth.RangerCredentialsProvider;
66
import org.apache.hadoop.fs.cosn.ranger.client.RangerQcloudObjectStorageClient;
7-
import org.apache.hadoop.fs.cosn.ranger.security.authorization.AccessType;
8-
import org.apache.hadoop.fs.cosn.ranger.security.authorization.PermissionRequest;
9-
import org.apache.hadoop.fs.cosn.ranger.security.authorization.PermissionResponse;
10-
import org.apache.hadoop.fs.cosn.ranger.security.authorization.ServiceType;
7+
import org.apache.hadoop.fs.cosn.ranger.security.authorization.*;
118
import org.apache.hadoop.fs.cosn.ranger.security.sts.GetSTSResponse;
129
import org.apache.hadoop.security.token.Token;
1310
import org.slf4j.Logger;
@@ -29,6 +26,10 @@ public class RangerCredentialsClient {
2926

3027
private boolean enableRangerPluginPermissionCheck = false;
3128

29+
private String rangerPolicyUrl;
30+
31+
private String authJarMd5;
32+
3233
public RangerCredentialsClient() {
3334
}
3435

@@ -168,6 +169,20 @@ private void initRangerClientImpl(Configuration conf) throws IOException {
168169
(RangerQcloudObjectStorageClient) rangerClientImplClass.newInstance();
169170
tmpClient.init(conf);
170171
RangerCredentialsClient.rangerQcloudObjectStorageStorageClient = tmpClient;
172+
173+
// set ranger policy url and other auth info.
174+
// when use posix mode to query bucket. server side also auth the policy url.
175+
// so need to pass these configurations to ofs java sdk which carried on when mount fs.
176+
RangerAuthPolicyResponse rangerAuthPolicyResp =
177+
rangerQcloudObjectStorageStorageClient.getRangerAuthPolicy();
178+
if (rangerAuthPolicyResp != null) {
179+
if (rangerAuthPolicyResp.getRangerPolicyUrl() != null) {
180+
this.rangerPolicyUrl = rangerAuthPolicyResp.getRangerPolicyUrl();
181+
}
182+
if (rangerAuthPolicyResp.getAuthJarMd5() != null) {
183+
this.authJarMd5 = rangerAuthPolicyResp.getAuthJarMd5();
184+
}
185+
}
171186
} catch (Exception e) {
172187
log.error(String.format("init %s failed", CosNConfigKeys.COSN_RANGER_PLUGIN_CLIENT_IMPL), e);
173188
throw new IOException(String.format("init %s failed",
@@ -176,6 +191,19 @@ private void initRangerClientImpl(Configuration conf) throws IOException {
176191
}
177192
}
178193
}
194+
} // end of init ranger impl
195+
196+
// must call after init
197+
public String getRangerPolicyUrl() {
198+
return this.rangerPolicyUrl;
199+
}
200+
201+
public String getAuthJarMd5() {
202+
return this.authJarMd5;
203+
}
179204

205+
public boolean isEnableRangerPluginPermissionCheck() {
206+
return this.enableRangerPluginPermissionCheck;
180207
}
208+
181209
}

‎src/main/java/org/apache/hadoop/fs/cosn/Constants.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,4 +37,8 @@ private Constants() {
3737
public static final String COSN_POSIX_BUCKET_FS_CHDFS_IMPL="com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter";
3838

3939
public static final String CUSTOM_AUTHENTICATION = "custom authentication";
40+
41+
// posix bucket ranger config need to pass through
42+
public static final String COSN_POSIX_BUCKET_RANGER_POLICY_URL = "fs.ofs.cosn.ranger.policy.url";
43+
public static final String COSN_POSIX_BUCKET_RANGER_AUTH_JAR_MD5 = "fs.ofs.cosn.ranger.auth.jar.md5";
4044
}

0 commit comments

Comments
 (0)