Add multi stage cleanup for orphaned packages

Author: kraih

cavil.conf

@@ -56,6 +56,12 @@
   # Number of packages per cleanup process
   cleanup_bucket_average => 400,
 
+  # Number of days to keep reviewed packages when not part of a product before cleanup
+  days_to_keep_orphaned_packages => 30,
+
+  # Number of days to keep duplicate reviewed packages when not part of a product before cleanup
+  days_to_keep_orphaned_duplicate_packages => 7,
+
   # Maximum number of files to show in reports per risk level
   min_files_short_report => 20,
 

lib/Cavil/Model/Packages.pm

@@ -397,6 +397,11 @@ sub mark_matched_for_reindex ($self, $pid, $priority = 0) {
   $self->minion->enqueue(reindex_matched_later => [$pid] => {priority => $priority});
 }
 
+sub need_cleanup ($self) {
+  return $self->pg->db->query('SELECT id FROM bot_packages WHERE obsolete IS TRUE AND cleaned IS NULL ORDER BY ID')
+    ->arrays->flatten->to_array;
+}
+
 sub name_suggestions ($self, $partial) {
   my $like = '%' . $partial . '%';
   return $self->pg->db->select(
@@ -426,6 +431,26 @@ sub obs_import ($self, $id, $data, $priority = 5) {
   );
 }
 
+sub obsolete_duplicate_new ($self) {
+  my $db = $self->pg->db;
+
+  # Mark all duplicate new packages as obsolete (same external_link and name)
+  $db->query(
+    q{
+      UPDATE bot_packages
+      SET obsolete = true, state = 'obsolete'
+      WHERE id IN (
+        SELECT a.id FROM (
+          SELECT id, ROW_NUMBER() OVER (PARTITION BY external_link, name ORDER BY id DESC) row_no
+          FROM bot_packages
+          WHERE state = 'new' AND external_link IS NOT NULL
+        ) AS a
+        WHERE row_no > 1
+      );
+    }
+  );
+}
+
 sub obsolete_if_not_in_product ($self, $id) {
   my $db = $self->pg->db;
   return undef if $db->query('select 1 from bot_package_products where package = ? limit 1', $id)->array;
@@ -436,6 +461,35 @@ sub obsolete_if_not_in_product ($self, $id) {
   return 1;
 }
 
+sub obsolete_old_packages ($self, $days_to_keep_orphaned, $days_to_keep_orphaned_duplicates) {
+  my $db = $self->pg->db;
+
+  # Mark duplicate old packages not in products as obsolete
+  $db->query(
+    "UPDATE bot_packages SET obsolete = true WHERE id IN (
+       SELECT id FROM (
+         SELECT id, ROW_NUMBER() OVER (PARTITION BY name ORDER BY id DESC) row_no
+         FROM bot_packages LEFT JOIN bot_package_products ON bot_package_products.package = bot_packages.id
+         WHERE state != 'new' AND checksum IS NOT NULL
+           AND imported < NOW() - (INTERVAL '1 days' * ?)
+           AND bot_package_products.product IS NULL
+       ) AS a
+       WHERE row_no > 1
+     )", $days_to_keep_orphaned_duplicates
+  );
+
+  # Mark all old packages not in products as obsolete
+  $db->query(
+    "UPDATE bot_packages SET obsolete = true WHERE id IN (
+       SELECT id
+       FROM bot_packages LEFT JOIN bot_package_products ON bot_package_products.package = bot_packages.id
+       WHERE state != 'new' AND obsolete != true AND checksum IS NOT NULL
+         AND imported < NOW() - (INTERVAL '1 days' * ?)
+         AND bot_package_products.product IS NULL
+     )", $days_to_keep_orphaned
+  );
+}
+
 sub reindex ($self, $id, @args) {
 
   # Protect from race conditions (even before creating a background job)

lib/Cavil/Task/Cleanup.pm

@@ -26,27 +26,11 @@ sub register ($self, $app, $config) {
 }
 
 sub _cleanup ($job) {
-  my $app = $job->app;
-  my $db  = $app->pg->db;
+  my $app  = $job->app;
+  my $pkgs = $app->packages;
 
-  # Mark all duplicate new packages as obsolete (same external_link and name)
-  $db->query(
-    q{
-      UPDATE bot_packages
-      SET obsolete = true, state = 'obsolete'
-      WHERE id IN (
-        SELECT a.id FROM (
-          SELECT id, ROW_NUMBER() OVER (PARTITION BY external_link, name ORDER BY id DESC) row_no
-          FROM bot_packages
-          WHERE state = 'new' AND external_link IS NOT NULL
-        ) AS a
-        WHERE row_no > 1
-      );
-    }
-  );
-
-  my $ids = $db->query('SELECT id FROM bot_packages WHERE obsolete IS TRUE AND cleaned IS NULL ORDER BY ID')
-    ->arrays->flatten->to_array;
+  $pkgs->obsolete_duplicate_new;
+  my $ids     = $pkgs->need_cleanup;
   my $buckets = Cavil::Util::buckets($ids, $app->config->{cleanup_bucket_average});
 
   my $minion = $app->minion;
@@ -59,22 +43,10 @@ sub _cleanup_batch ($job, @ids) {
 }
 
 sub _obsolete ($job) {
-  my $app = $job->app;
-  my $log = $app->log;
-  my $db  = $job->app->pg->db;
-
-  my $leave_untagged_imports = 7;
-
-  my $list = $db->query(
-    "update bot_packages set obsolete = true where id in
-       (select id from bot_packages
-        left join bot_package_products on bot_package_products.package=bot_packages.id
-        where state != 'new' and checksum is not null and
-        imported < now() - Interval '$leave_untagged_imports days' and
-        bot_package_products.product is null
-       )"
-  );
-
+  my $app    = $job->app;
+  my $config = $app->config;
+  $app->packages->obsolete_old_packages($config->{days_to_keep_orphaned_packages},
+    $config->{days_to_keep_orphaned_duplicate_packages});
   $app->minion->enqueue('cleanup' => [] => {parents => [$job->id]});
 }
 

staging/start.pl

@@ -42,6 +42,8 @@
   acceptable_risk      => 3,
   index_bucket_average => 100,
   cleanup_bucket_average => 50,
+  days_to_keep_orphaned_packages => 30,
+  days_to_keep_orphaned_duplicate_packages => 7,
   min_files_short_report => 20,
   max_email_url_size   => 2048,
   max_task_memory      => 5_000_000_000,

t/cleanup_duplicates.t t/cleanup_new_duplicates.tt/cleanup_duplicates.t renamed to t/cleanup_new_duplicates.t

@@ -30,7 +30,7 @@ use Mojo::URL;
 
 plan skip_all => 'set TEST_ONLINE to enable this test' unless $ENV{TEST_ONLINE};
 
-my $cavil_test = Cavil::Test->new(online => $ENV{TEST_ONLINE}, schema => 'cleanup_duplicates_test');
+my $cavil_test = Cavil::Test->new(online => $ENV{TEST_ONLINE}, schema => 'cleanup_new_duplicates_test');
 my $config     = $cavil_test->default_config;
 my $t          = Test::Mojo->new(Cavil => $config);
 $cavil_test->no_fixtures($t->app);
@@ -190,7 +190,7 @@ subtest 'Clean up duplicates' => sub {
   ok !$t->app->pg->db->select('file_snippets',   [\'count(*)'], {package => $one_id})->array->[0], 'no file snippets';
 
   # Second package (cleaned up)
-  is $t->app->packages->find($one_id)->{state}, 'obsolete', 'right state';
+  is $t->app->packages->find($two_id)->{state}, 'obsolete', 'right state';
   ok $t->app->packages->find($two_id)->{obsolete}, 'obsolete';
   ok $t->app->packages->find($two_id)->{cleaned},  'cleanup done';
   is $t->app->packages->find($two_id)->{result}, undef, 'right result';