new listversions command to list versions of files to a tsv file

Author: steveloughran

README.md

@@ -448,6 +448,37 @@ Parsed private key -entry length 28 lines
 factory com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.CredentialFactory@d706f19
 ```
 
+## Command `iampolicy`
+
+Generate AWS IAM policy for a given bucket
+```
+hadoop jar cloudstore-1.0.jar iampolicy s3a://example-bucket/
+
+{
+  "Version" : "2012-10-17",
+  "Statement" : [ {
+    "Sid" : "7",
+    "Effect" : "Allow",
+    "Action" : [ "s3:GetBucketLocation", "s3:ListBucket*" ],
+    "Resource" : "arn:aws:s3:::example-bucket"
+  }, {
+    "Sid" : "8",
+    "Effect" : "Allow",
+    "Action" : [ "s3:Get*", "s3:PutObject", "s3:PutObjectAcl", "s3:DeleteObject", "s3:AbortMultipartUpload" ],
+    "Resource" : "arn:aws:s3:::example-bucket/*"
+  }, {
+    "Sid" : "1",
+    "Effect" : "Allow",
+    "Action" : "kms:*",
+    "Resource" : "*"
+  } ]
+}
+
+```
+Notes:
+* KMS policy is always added in case data is encrypted/decrypted with S3-KMS; it is not need if this is not the case.
+* Read and Write access up the tree is needed. Maybe if you enable directory marker retention writing from root becomes optional.
+* "s3:GetBucketLocation" is used by the bucket existence v2 check. If the probe is at 0, it is never called.
 
 
 ## Command `list`

pom.xml

@@ -218,24 +218,30 @@
         </exclusion>
       </exclusions>
     </dependency>
-<!--
-
     <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-aws</artifactId>
-      <version>${hadoop.version}</version>
-      <type>test-jar</type>
-      <scope>test</scope>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>aws-java-sdk-bundle</artifactId>
+      <version>1.12.316</version>
+      <scope>compile</scope>
     </dependency>
+    <!--
 
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-azure</artifactId>
-      <version>${hadoop.version}</version>
-      <type>test-jar</type>
-      <scope>test</scope>
-    </dependency>
--->
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-aws</artifactId>
+          <version>${hadoop.version}</version>
+          <type>test-jar</type>
+          <scope>test</scope>
+        </dependency>
+
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-azure</artifactId>
+          <version>${hadoop.version}</version>
+          <type>test-jar</type>
+          <scope>test</scope>
+        </dependency>
+    -->
 
   </dependencies>
 
@@ -322,7 +328,7 @@
 
     <!-- latest hadoop release -->
     <profile>
-      <id>hadoop-latest</id>
+      <id>latest</id>
       <properties>
         <hadoop.version>3.3.5</hadoop.version>
       </properties>

src/main/extra/listversions.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.hadoop.fs.s3a.extra.ListObjects;
+import org.apache.hadoop.fs.s3a.extra.ListVersions;
+import org.apache.hadoop.fs.store.commands.Command;
+
+public class listversions extends Command {
+
+  public static void main(String[] args) throws Exception {
+    ListVersions.main(args);
+  }
+
+  public static void help() {
+    printCommand("listversions", "list all versions of S3 objects under a path");
+  }
+
+}