Merge pull request #5721 from Countly/SER-1987-ss-online-users-no-ttl

[SER-1987] ttl collection clean up job

Author: ArtursKadikis

api/api.js

@@ -313,6 +313,7 @@ plugins.connectToAllDatabases().then(function() {
             jobs.job('api:clearAutoTasks').replace().schedule('every 1 day');
             jobs.job('api:task').replace().schedule('every 5 minutes');
             jobs.job('api:userMerge').replace().schedule('every 10 minutes');
+            jobs.job("api:ttlCleanup").replace().schedule("every 1 minute");
             //jobs.job('api:appExpire').replace().schedule('every 1 day');
         }, 10000);
 

api/jobs/ttlCleanup.js

@@ -0,0 +1,48 @@
+const plugins = require("../../plugins/pluginManager.js");
+const common = require('../utils/common');
+const job = require("../parts/jobs/job.js");
+const log = require("../utils/log.js")("job:ttlCleanup");
+
+/**
+ * Class for job of cleaning expired records inside ttl collections
+ */
+class TTLCleanup extends job.Job {
+    /**
+     * Run the job
+     */
+    async run() {
+        log.d("Started running TTL clean up job");
+        for (let i = 0; i < plugins.ttlCollections.length; i++) {
+            const {
+                db = "countly",
+                collection,
+                property,
+                expireAfterSeconds = 0
+            } = plugins.ttlCollections[i];
+            let dbInstance;
+            switch (db) {
+            case "countly": dbInstance = common.db; break;
+            case "countly_drill": dbInstance = common.drillDb; break;
+            case "countly_out": dbInstance = common.outDb; break;
+            }
+            if (!dbInstance) {
+                log.e("Invalid db selection:", db);
+                continue;
+            }
+
+            log.d("Started cleaning up", collection);
+            const result = await dbInstance.collection(collection).deleteMany({
+                [property]: {
+                    $lte: new Date(Date.now() - expireAfterSeconds * 1000)
+                }
+            });
+            log.d("Finished cleaning up", result.deletedCount, "records from", collection);
+
+            // Sleep 1 second to prevent sending too many deleteMany queries
+            await new Promise(res => setTimeout(res, 1000));
+        }
+        log.d("Finished running TTL clean up job");
+    }
+}
+
+module.exports = TTLCleanup;

plugins/pluginManager.js

@@ -74,7 +74,11 @@ var pluginManager = function pluginManager() {
         countly_out: "../api/configs/config.db_out.js",
         countly_fs: "../api/configs/config.db_fs.js"
     };
-
+    /**
+     * TTL collections to clean up periodically
+     * @type {{collection: string, db: mongodb.Db, property: string, expireAfterSeconds: number}[]}
+     */
+    this.ttlCollections = [];
     /**
      *  Custom configuration files for different databases for docker env
      */