Centralize data retention policy

Also prune page data.

Author: cpeel

SETUP/dp.cron.template

@@ -10,6 +10,7 @@
 01  0 * * * JOB=RecordProjectStateCounts; <<PHP_CLI_EXECUTABLE>> <<CODE_DIR>>/crontab/run_background_job.php $JOB
 # See SETUP/ARCHIVING.md to set up archiving before enabling the line below
 #10  0 * * * JOB=ArchiveProjects; <<PHP_CLI_EXECUTABLE>> <<CODE_DIR>>/crontab/run_background_job.php $JOB
+30  0 * * * JOB=PrunePageData; <<PHP_CLI_EXECUTABLE>> <<CODE_DIR>>/crontab/run_background_job.php $JOB
 45  0 * * * JOB=CleanDownloadTemp; <<PHP_CLI_EXECUTABLE>> <<CODE_DIR>>/crontab/run_background_job.php $JOB
 50  0 * * * JOB=CleanUploadsTrash; <<PHP_CLI_EXECUTABLE>> <<CODE_DIR>>/crontab/run_background_job.php $JOB
 55  0 * * * JOB=ExtendSiteTallyGoals; <<PHP_CLI_EXECUTABLE>> <<CODE_DIR>>/crontab/run_background_job.php $JOB

crontab/ArchiveProjects.inc

@@ -4,7 +4,6 @@ include_once($relPath.'archiving.inc');
 // Find projects that were posted to PG a while ago and archive them.
 class ArchiveProjects extends BackgroundJob
 {
-    private int $days_to_retain = 100;
     public bool $requires_web_context = true;
 
     public function work()
@@ -14,11 +13,12 @@ class ArchiveProjects extends BackgroundJob
             SELECT *
             FROM projects
             WHERE
-                modifieddate <= UNIX_TIMESTAMP() - (24 * 60 * 60) * {$this->days_to_retain}
+                modifieddate <= UNIX_TIMESTAMP() - (24 * 60 * 60) * %d
                 AND archived = '0'
                 AND state = '%s'
             ORDER BY modifieddate
             ",
+            SiteConfig::get()->days_to_retain_projects_before_archiving,
             DPDatabase::escape(PROJ_SUBMIT_PG_POSTED)
         );
         $result = DPDatabase::query($sql);

crontab/PruneJobLogs.inc

@@ -6,6 +6,6 @@ class PruneJobLogs extends BackgroundJob
 {
     public function work()
     {
-        prune_job_log_entries(365 + 30); // ~13 months
+        prune_job_log_entries(SiteConfig::get()->days_to_retain_job_logs);
     }
 }

crontab/PruneNonActivatedUsers.inc

@@ -5,6 +5,6 @@ class PruneNonActivatedUsers extends BackgroundJob
 {
     public function work()
     {
-        NonactivatedUser::prune(90); // ~3 months
+        NonactivatedUser::prune(SiteConfig::get()->days_to_retain_pending_accounts);
     }
 }

crontab/PrunePageData.inc

@@ -0,0 +1,68 @@
+<?php
+
+// Find projects that were posted to PG a while ago and prune page data.
+class PrunePageData extends BackgroundJob
+{
+    public function work()
+    {
+        $database = SiteConfig::get()->archive_db_name ? SiteConfig::get()->archive_db_name : SiteConfig::get()->db_name;
+
+        // We need to iteratively delete data in batches by projectid because:
+        // 1. we base the pruning on the project's posted time
+        // 2. there are indexes for these tables based on projectid
+        // 3. to possibly limit our runtime
+        $sql = sprintf(
+            "
+            SELECT projectid
+            FROM projects
+            WHERE
+                modifieddate <= UNIX_TIMESTAMP() - (24 * 60 * 60) * %d
+                AND state = '%s'
+            ORDER BY modifieddate
+            ",
+            SiteConfig::get()->days_to_retain_page_data_after_posting,
+            DPDatabase::escape(PROJ_SUBMIT_PG_POSTED)
+        );
+        $result = DPDatabase::query($sql);
+        $num_projects = mysqli_num_rows($result);
+
+        echo "Pruning data for $num_projects projects...\n";
+
+        $num_projects_pruned = 0;
+        while ([$project_id] = mysqli_fetch_row($result)) {
+            if ($this->watch->read() >= $this->web_context_max_runtime_s) {
+                break;
+            }
+
+            // first wordcheck_events
+            $sql = sprintf(
+                "
+                DELETE FROM $database.wordcheck_events
+                WHERE projectid = '%s'
+                ",
+                DPDatabase::escape($project_id)
+            );
+            DPDatabase::query($sql);
+
+            // then page_events
+            $sql = sprintf(
+                "
+                DELETE FROM $database.page_events
+                WHERE projectid = '%s'
+                ",
+                DPDatabase::escape($project_id)
+            );
+            DPDatabase::query($sql);
+
+            $num_projects_pruned += 1;
+        }
+
+        $leftover_projects = $num_projects - $num_projects_pruned;
+        if ($leftover_projects) {
+            echo "Reached runtime limit, skipping pruning of page data for remaining $leftover_projects projects.\n";
+            $this->stop_message = "Pruned page data for $num_projects_pruned projects, ran out of time to prune page data for remaining $leftover_projects";
+        } else {
+            $this->stop_message = "Pruned page data for $num_projects_pruned projects";
+        }
+    }
+}

pinc/SiteConfig.inc

@@ -21,6 +21,12 @@ class SiteConfig
     public ?string $archive_db_name;
     public ?string $archive_projects_dir;
 
+    // data retention configuration
+    public int $days_to_retain_projects_before_archiving = 100;
+    public int $days_to_retain_page_data_after_posting = 365;
+    public int $days_to_retain_pending_accounts = 90;
+    public int $days_to_retain_job_logs = 365 + 30; // ~13 months
+
     // directories & URLs
     public string $code_dir;
     public string $code_url;