[JIRA-ODS-1624] Preliminary commits in advance of switching to a supported S3 SDK (#1120)

* Prelim commit - Removing all references to backup file system cleanup and lifecycle config

* Addressing comments

* Minor change, adding another clear test call

Author: sinchanah-dot

priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java

@@ -99,119 +99,6 @@ public void setS3Client(AmazonS3 client) {
         s3Client = client;
     }
 
-    @Override
-    public void cleanup() {
-
-        AmazonS3 s3Client = getS3Client();
-        String clusterPath = pathProvider.get().clusterPrefix("");
-        logger.debug("Bucket: {}", config.getBackupPrefix());
-        BucketLifecycleConfiguration lifeConfig =
-                s3Client.getBucketLifecycleConfiguration(config.getBackupPrefix());
-        logger.debug("Got bucket:{} lifecycle.{}", config.getBackupPrefix(), lifeConfig);
-        if (lifeConfig == null) {
-            lifeConfig = new BucketLifecycleConfiguration();
-            List<Rule> rules = Lists.newArrayList();
-            lifeConfig.setRules(rules);
-        }
-
-        List<Rule> rules = lifeConfig.getRules();
-
-        if (updateLifecycleRule(config, rules, clusterPath)) {
-            if (rules.size() > 0) {
-                lifeConfig.setRules(rules);
-                s3Client.setBucketLifecycleConfiguration(config.getBackupPrefix(), lifeConfig);
-            } else s3Client.deleteBucketLifecycleConfiguration(config.getBackupPrefix());
-        }
-    }
-
-    // Dummy class to get Prefix. - Why oh why AWS you can't give the details!!
-    private class PrefixVisitor implements LifecyclePredicateVisitor {
-        String prefix;
-
-        @Override
-        public void visit(LifecyclePrefixPredicate lifecyclePrefixPredicate) {
-            prefix = lifecyclePrefixPredicate.getPrefix();
-        }
-
-        @Override
-        public void visit(LifecycleTagPredicate lifecycleTagPredicate) {}
-
-        @Override
-        public void visit(
-                LifecycleObjectSizeGreaterThanPredicate lifecycleObjectSizeGreaterThanPredicate) {}
-
-        @Override
-        public void visit(LifecycleAndOperator lifecycleAndOperator) {}
-
-        @Override
-        public void visit(
-                LifecycleObjectSizeLessThanPredicate lifecycleObjectSizeLessThanPredicate) {}
-    }
-
-    private Optional<Rule> getBucketLifecycleRule(List<Rule> rules, String prefix) {
-        if (rules == null || rules.isEmpty()) return Optional.empty();
-
-        for (Rule rule : rules) {
-            String rulePrefix = "";
-            if (rule.getFilter() != null) {
-                PrefixVisitor prefixVisitor = new PrefixVisitor();
-                rule.getFilter().getPredicate().accept(prefixVisitor);
-                rulePrefix = prefixVisitor.prefix;
-            } else if (rule.getPrefix() != null) {
-                // Being backwards compatible, here.
-                rulePrefix = rule.getPrefix();
-            }
-            if (prefix.equalsIgnoreCase(rulePrefix)) {
-                return Optional.of(rule);
-            }
-        }
-
-        return Optional.empty();
-    }
-
-    private boolean updateLifecycleRule(IConfiguration config, List<Rule> rules, String prefix) {
-        Optional<Rule> rule = getBucketLifecycleRule(rules, prefix);
-        // No need to update the rule as it never existed and retention is not set.
-        if (!rule.isPresent() && config.getBackupRetentionDays() <= 0) return false;
-
-        // Rule not required as retention days is zero or negative.
-        if (rule.isPresent() && config.getBackupRetentionDays() <= 0) {
-            logger.warn(
-                    "Removing the rule for backup retention on prefix: {} as retention is set to [{}] days. Only positive values are supported by S3!!",
-                    prefix,
-                    config.getBackupRetentionDays());
-            rules.remove(rule.get());
-            return true;
-        }
-
-        // Rule present and is current.
-        if (rule.isPresent()
-                && rule.get().getExpirationInDays() == config.getBackupRetentionDays()
-                && rule.get().getStatus().equalsIgnoreCase(BucketLifecycleConfiguration.ENABLED)) {
-            logger.info(
-                    "Cleanup rule already set on prefix: {} with retention period: [{}] days",
-                    prefix,
-                    config.getBackupRetentionDays());
-            return false;
-        }
-
-        if (!rule.isPresent()) {
-            // Create a new rule
-            rule = Optional.of(new BucketLifecycleConfiguration.Rule());
-            rules.add(rule.get());
-        }
-
-        rule.get().setStatus(BucketLifecycleConfiguration.ENABLED);
-        rule.get().setExpirationInDays(config.getBackupRetentionDays());
-        rule.get().setFilter(new LifecycleFilter(new LifecyclePrefixPredicate(prefix)));
-        rule.get().setId(prefix);
-        logger.info(
-                "Setting cleanup rule for prefix: {} with retention period: [{}] days",
-                prefix,
-                config.getBackupRetentionDays());
-        return true;
-    }
-
     void checkSuccessfulUpload(
             CompleteMultipartUploadResult resultS3MultiPartUploadComplete, Path localPath)
             throws BackupRestoreException {

priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java

@@ -120,9 +120,6 @@ default String getShard() {
      */
     Iterator<String> listFileSystem(String prefix, String delimiter, String marker);
 
-    /** Runs cleanup or set retention */
-    void cleanup();
-
     /** Give the file system a chance to terminate any thread pools, etc. */
     void shutdown();
 

priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java

@@ -57,7 +57,7 @@ public void setupTest(List<String> files) {
         }
     }
 
-    private void clearTest() {
+    public void clearTest() {
         flist.clear();
         downloadedFiles.clear();
         uploadedFiles.clear();
@@ -111,11 +111,6 @@ public void deleteFiles(List<Path> remotePaths) throws BackupRestoreException {
                         });
     }
 
-    @Override
-    public void cleanup() {
-        clearTest();
-    }
-
     @Override
     protected void downloadFileImpl(AbstractBackupPath path, String suffix)
             throws BackupRestoreException {

priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java

@@ -58,11 +58,6 @@ public Iterator<String> listFileSystem(String prefix, String delimiter, String m
         return Collections.emptyIterator();
     }
 
-    @Override
-    public void cleanup() {
-        // TODO Auto-generated method stub
-    }
-
     @Override
     protected void downloadFileImpl(AbstractBackupPath path, String suffix)
             throws BackupRestoreException {}

priam/src/test/java/com/netflix/priam/backup/TestBackup.java

@@ -70,7 +70,7 @@ public static void cleanup() throws IOException {
 
     @Test
     public void testIncrementalBackup() throws Exception {
-        filesystem.cleanup();
+        filesystem.clearTest();
         generateIncrementalFiles();
         IncrementalBackup backup = injector.getInstance(IncrementalBackup.class);
         backup.execute();
@@ -85,7 +85,7 @@ public void testIncrementalBackup() throws Exception {
 
     @Test
     public void testIncrementalBackupOfSecondaryIndexes() throws Exception {
-        filesystem.cleanup();
+        filesystem.clearTest();
         generateIncrementalFiles();
         IncrementalBackup backup = injector.getInstance(IncrementalBackup.class);
         File secondaryIndexBackupDir =
@@ -121,7 +121,7 @@ public void testClusterSpecificColumnFamiliesSkippedFrom21() throws Exception {
 
     private void testClusterSpecificColumnFamiliesSkipped(String[] columnFamilyDirs)
             throws Exception {
-        filesystem.cleanup();
+        filesystem.clearTest();
         File tmp = new File("target/data/");
         if (tmp.exists()) cleanup(tmp);
         // Generate "data"

priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java

@@ -154,42 +154,6 @@ public void testFileUploadCompleteFailure() throws Exception {
         }
     }
 
-    @Test
-    public void testCleanupAdd() throws Exception {
-        MockAmazonS3Client.setRuleAvailable(false);
-        S3FileSystem fs = injector.getInstance(S3FileSystem.class);
-        fs.cleanup();
-        Assert.assertEquals(1, MockAmazonS3Client.bconf.getRules().size());
-        BucketLifecycleConfiguration.Rule rule = MockAmazonS3Client.bconf.getRules().get(0);
-        Assert.assertEquals("casstestbackup/" + region + "/fake-app/", rule.getId());
-        Assert.assertEquals(configuration.getBackupRetentionDays(), rule.getExpirationInDays());
-    }
-
-    @Test
-    public void testCleanupIgnore() throws Exception {
-        MockAmazonS3Client.setRuleAvailable(true);
-        S3FileSystem fs = injector.getInstance(S3FileSystem.class);
-        fs.cleanup();
-        Assert.assertEquals(1, MockAmazonS3Client.bconf.getRules().size());
-        BucketLifecycleConfiguration.Rule rule = MockAmazonS3Client.bconf.getRules().get(0);
-        Assert.assertEquals("casstestbackup/" + region + "/fake-app/", rule.getId());
-        Assert.assertEquals(configuration.getBackupRetentionDays(), rule.getExpirationInDays());
-    }
-
-    @Test
-    public void testCleanupUpdate() throws Exception {
-        MockAmazonS3Client.setRuleAvailable(true);
-        S3FileSystem fs = injector.getInstance(S3FileSystem.class);
-        String clusterPrefix = "casstestbackup/" + region + "/fake-app/";
-        MockAmazonS3Client.updateRule(
-                MockAmazonS3Client.getBucketLifecycleConfig(clusterPrefix, 2));
-        fs.cleanup();
-        Assert.assertEquals(1, MockAmazonS3Client.bconf.getRules().size());
-        BucketLifecycleConfiguration.Rule rule = MockAmazonS3Client.bconf.getRules().get(0);
-        Assert.assertEquals("casstestbackup/" + region + "/fake-app/", rule.getId());
-        Assert.assertEquals(configuration.getBackupRetentionDays(), rule.getExpirationInDays());
-    }
-
     @Test
     public void testDeleteObjects() throws Exception {
         S3FileSystem fs = injector.getInstance(S3FileSystem.class);

priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java

@@ -139,7 +139,7 @@ private String getRemoteFromLocal(String localPath) throws ParseException {
     @After
     public void cleanup() {
         BackupFileUtils.cleanupDir(Paths.get(configuration.getDataFileLocation()));
-        backupFileSystem.cleanup();
+        backupFileSystem.clearTest();
     }
 
     private List<String> getAllFiles() {