Skip to content

Commit 428de4b

Browse files
yuyang733yuyang733
authored
Merge pull request #25 from vintmd/zuoyebang400
add the key requel prefix list and head requestid
2 parents fe5a425 + 81b8c9b commit 428de4b

File tree

7 files changed

+100
-13
lines changed

7 files changed

+100
-13
lines changed

‎compile.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
base_dir=$(cd `dirname $0`;pwd)
44
cd ${base_dir}
5-
hadoop_version_array=("2.6.5" "2.7.5" "2.8.5" "3.1.0" "3.3.0")
5+
hadoop_version_array=("2.6.0" "2.6.5" "2.7.5" "2.8.5" "3.1.0" "3.3.0")
66

77
origin_version=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec)
88

‎pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>com.qcloud.cos</groupId>
88
<artifactId>hadoop-cos</artifactId>
9-
<version>5.9.0</version>
9+
<version>5.9.1</version>
1010
<packaging>jar</packaging>
1111

1212
<name>Apache Hadoop Tencent Qcloud COS Support</name>
@@ -41,7 +41,7 @@
4141
<maven.compiler.source>1.7</maven.compiler.source>
4242
<maven.compiler.target>1.7</maven.compiler.target>
4343
<hadoop.version>3.3.0</hadoop.version>
44-
<cos_api.version>5.6.32</cos_api.version>
44+
<cos_api.version>5.6.35</cos_api.version>
4545
<google.guava.version>24.1.1-jre</google.guava.version>
4646
<commons_lang3.version>3.1</commons_lang3.version>
4747
<junit.version>4.8</junit.version>

‎src/main/java/org/apache/hadoop/fs/CosFileSystem.java

+10-2
Original file line numberDiff line numberDiff line change
@@ -420,7 +420,8 @@ public FileStatus getFileStatus(Path f) throws IOException {
420420
return newDirectory(absolutePath);
421421
}
422422

423-
FileMetadata meta = store.retrieveMetadata(key);
423+
CosResultInfo headInfo = new CosResultInfo();
424+
FileMetadata meta = store.retrieveMetadata(key, headInfo);
424425
if (meta != null) {
425426
if (meta.isFile()) {
426427
LOG.debug("Retrieve the cos key [{}] to find that it is a file.", key);
@@ -435,12 +436,19 @@ public FileStatus getFileStatus(Path f) throws IOException {
435436
key += PATH_DELIMITER;
436437
}
437438
LOG.debug("List the cos key [{}] to judge whether it is a directory or not.", key);
438-
PartialListing listing = store.list(key, 1);
439+
CosResultInfo listInfo = new CosResultInfo();
440+
PartialListing listing = store.list(key, 1, listInfo);
439441
if (listing.getFiles().length > 0 || listing.getCommonPrefixes().length > 0) {
440442
LOG.debug("List the cos key [{}] to find that it is a directory.", key);
441443
return newDirectory(absolutePath);
442444
}
443445

446+
if (listInfo.isKeySameToPrefix()) {
447+
LOG.info("List the cos key [{}] same to prefix, head-id:[{}], " +
448+
"list-id:[{}], list-type:[{}], thread-id:[{}], thread-name:[{}]",
449+
key, headInfo.getRequestID(), listInfo.getRequestID(),
450+
listInfo.isKeySameToPrefix(), Thread.currentThread().getId(), Thread.currentThread().getName());
451+
}
444452
LOG.debug("Can not find the cos key [{}] on COS.", key);
445453

446454
throw new FileNotFoundException("No such file or directory '" + absolutePath + "'");

‎src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
@InterfaceStability.Unstable
1111
public class CosNConfigKeys extends CommonConfigurationKeys {
1212
public static final String USER_AGENT = "fs.cosn.user.agent";
13-
public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v5.8.7";
13+
public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v5.9.1";
1414

1515
public static final String TENCENT_EMR_VERSION_KEY = "fs.emr.version";
1616

‎src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java

+49-7
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,11 @@ public int compare(PartETag o1, PartETag o2) {
449449
}
450450

451451
private FileMetadata queryObjectMetadata(String key) throws IOException {
452+
return queryObjectMetadata(key, null);
453+
}
454+
455+
private FileMetadata queryObjectMetadata(String key,
456+
CosResultInfo info) throws IOException {
452457
LOG.debug("Query Object metadata. cos key: {}.", key);
453458
GetObjectMetadataRequest getObjectMetadataRequest =
454459
new GetObjectMetadataRequest(bucketName, key);
@@ -494,10 +499,17 @@ private FileMetadata queryObjectMetadata(String key) throws IOException {
494499
new FileMetadata(key, fileSize, mtime, !key.endsWith(PATH_DELIMITER),
495500
ETag, crc64ecm, crc32cm, versionId,
496501
objectMetadata.getStorageClass(), userMetadata);
502+
// record the last request result info
503+
if (info != null) {
504+
info.setRequestID(objectMetadata.getRequestId());
505+
}
497506
LOG.debug("Retrieve the file metadata. cos key: {}, ETag:{}, length:{}, crc64ecm: {}.", key,
498507
objectMetadata.getETag(), objectMetadata.getContentLength(), objectMetadata.getCrc64Ecma());
499508
return fileMetadata;
500509
} catch (CosServiceException e) {
510+
if (info != null) {
511+
info.setRequestID(e.getRequestId());
512+
}
501513
if (e.getStatusCode() != 404) {
502514
String errorMsg =
503515
String.format("Retrieve the file metadata file failure. " +
@@ -510,19 +522,26 @@ private FileMetadata queryObjectMetadata(String key) throws IOException {
510522

511523
@Override
512524
public FileMetadata retrieveMetadata(String key) throws IOException {
525+
return retrieveMetadata(key, null);
526+
}
527+
528+
// this method only used in getFileStatus to get the head request result info
529+
@Override
530+
public FileMetadata retrieveMetadata(String key,
531+
CosResultInfo info) throws IOException {
513532
if (key.endsWith(PATH_DELIMITER)) {
514533
key = key.substring(0, key.length() - 1);
515534
}
516535

517536
if (!key.isEmpty()) {
518-
FileMetadata fileMetadata = queryObjectMetadata(key);
537+
FileMetadata fileMetadata = queryObjectMetadata(key, info);
519538
if (fileMetadata != null) {
520539
return fileMetadata;
521540
}
522541
}
523542
// judge if the key is directory
524543
key = key + PATH_DELIMITER;
525-
return queryObjectMetadata(key);
544+
return queryObjectMetadata(key, info);
526545
}
527546

528547
@Override
@@ -782,15 +801,26 @@ public boolean retrieveBlock(String key, long byteRangeStart,
782801

783802
@Override
784803
public PartialListing list(String prefix, int maxListingLength) throws IOException {
785-
return list(prefix, maxListingLength, null, false);
804+
return list(prefix, maxListingLength, null);
805+
}
806+
807+
@Override
808+
public PartialListing list(String prefix, int maxListingLength, CosResultInfo info) throws IOException {
809+
return list(prefix, maxListingLength, null, false, info);
786810
}
787811

788812
@Override
789813
public PartialListing list(String prefix, int maxListingLength,
790814
String priorLastKey,
791815
boolean recurse) throws IOException {
816+
return list(prefix, maxListingLength, priorLastKey, recurse, null);
817+
}
792818

793-
return list(prefix, recurse ? null : PATH_DELIMITER, maxListingLength, priorLastKey);
819+
@Override
820+
public PartialListing list(String prefix, int maxListingLength,
821+
String priorLastKey,
822+
boolean recurse, CosResultInfo info) throws IOException {
823+
return list(prefix, recurse ? null : PATH_DELIMITER, maxListingLength, priorLastKey, info);
794824
}
795825

796826
/**
@@ -806,7 +836,7 @@ public PartialListing list(String prefix, int maxListingLength,
806836

807837
private PartialListing list(String prefix, String delimiter,
808838
int maxListingLength,
809-
String priorLastKey) throws IOException {
839+
String priorLastKey, CosResultInfo info) throws IOException {
810840
LOG.debug("List the cos key prefix: {}, max listing length: {}, delimiter: {}, prior last key: {}.",
811841
prefix,
812842
delimiter, maxListingLength, priorLastKey);
@@ -846,12 +876,14 @@ private PartialListing list(String prefix, String delimiter,
846876
ArrayList<FileMetadata> commonPrefixArray =
847877
new ArrayList<FileMetadata>();
848878
List<COSObjectSummary> summaries = objectListing.getObjectSummaries();
879+
boolean isKeySamePrefix = false;
849880
for (COSObjectSummary cosObjectSummary : summaries) {
850881
String filePath = cosObjectSummary.getKey();
851882
if (!filePath.startsWith(PATH_DELIMITER)) {
852883
filePath = PATH_DELIMITER + filePath;
853884
}
854885
if (filePath.equals(prefix)) {
886+
isKeySamePrefix = true;
855887
continue;
856888
}
857889
long mtime = 0;
@@ -890,10 +922,20 @@ private PartialListing list(String prefix, String delimiter,
890922

891923
// 如果truncated为false, 则表明已经遍历完
892924
if (!objectListing.isTruncated()) {
893-
return new PartialListing(null, fileMetadata, commonPrefixMetaData);
925+
PartialListing ret = new PartialListing(null, fileMetadata, commonPrefixMetaData);
926+
if (info != null) {
927+
info.setRequestID(objectListing.getRequestId());
928+
info.setKeySameToPrefix(isKeySamePrefix);
929+
}
930+
return ret;
894931
} else {
895-
return new PartialListing(objectListing.getNextMarker(),
932+
PartialListing ret = new PartialListing(objectListing.getNextMarker(),
896933
fileMetadata, commonPrefixMetaData);
934+
if (info != null) {
935+
info.setRequestID(objectListing.getRequestId());
936+
info.setKeySameToPrefix(isKeySamePrefix);
937+
}
938+
return ret;
897939
}
898940
}
899941

‎src/main/java/org/apache/hadoop/fs/CosResultInfo.java

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package org.apache.hadoop.fs;
2+
3+
/**
4+
* Used to record the cos client query result
5+
*/
6+
public class CosResultInfo {
7+
private String requestID;
8+
private boolean isKeySameToPrefix;
9+
10+
CosResultInfo() {
11+
requestID = "";
12+
isKeySameToPrefix = false;
13+
}
14+
15+
public void setRequestID(String requestID) {
16+
this.requestID = requestID;
17+
}
18+
public String getRequestID() {
19+
return this.requestID;
20+
}
21+
22+
public boolean isKeySameToPrefix() {
23+
return this.isKeySameToPrefix;
24+
}
25+
26+
public void setKeySameToPrefix(boolean isKeySameToPrefix) {
27+
this.isKeySameToPrefix = isKeySameToPrefix;
28+
}
29+
}

‎src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java

+8
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ PartETag uploadPart(InputStream inputStream, String key, String uploadId,
4545

4646
FileMetadata retrieveMetadata(String key) throws IOException;
4747

48+
FileMetadata retrieveMetadata(String key, CosResultInfo info) throws IOException;
49+
4850
byte[] retrieveAttribute(String key, String attribute) throws IOException;
4951

5052
void storeDirAttribute(String key, String attribute, byte[] value) throws IOException;
@@ -69,10 +71,16 @@ boolean retrieveBlock(String key, long byteRangeStart, long blockSize,
6971

7072
PartialListing list(String prefix, int maxListingLength) throws IOException;
7173

74+
PartialListing list(String prefix, int maxListingLength, CosResultInfo info) throws IOException;
75+
7276
PartialListing list(String prefix, int maxListingLength,
7377
String priorLastKey, boolean recursive)
7478
throws IOException;
7579

80+
PartialListing list(String prefix, int maxListingLength,
81+
String priorLastKey, boolean recursive, CosResultInfo info)
82+
throws IOException;
83+
7684
void delete(String key) throws IOException;
7785

7886
void copy(String srcKey, String dstKey) throws IOException;

0 commit comments

Comments
 (0)