Merge pull request #5708 from Countly/SER-1985-ss-logger-plugin-needs-to-be-reworked

[SER-1985] logger: replaced capped collection with periodic clean up job

Author: ArtursKadikis

plugins/logger/api/api.js

@@ -4,7 +4,8 @@ var exported = {},
     automaticStateManager = require('./helpers/automaticStateManager'),
     log = require('../../../api/utils/log.js')('logger:api'),
     { validateRead } = require('../../../api/utils/rights.js');
-
+const JOB = require('../../../api/parts/jobs');
+const MAX_NUMBER_OF_LOG_ENTRIES = 1000;
 const FEATURE_NAME = 'logger';
 
 var RequestLoggerStateEnum = {
@@ -20,6 +21,13 @@ plugins.setConfigs("logger", {
 });
 
 (function() {
+    plugins.register("/master", function() {
+        setTimeout(() => {
+            JOB.job('logger:clear', { max: MAX_NUMBER_OF_LOG_ENTRIES })
+                .replace()
+                .schedule("every 5 minutes");
+        }, 10000);
+    });
 
     plugins.register("/permissions/features", function(ob) {
         ob.features.push(FEATURE_NAME);
@@ -345,46 +353,32 @@ plugins.setConfigs("logger", {
             return true;
         }
         if (params.qstring.method === 'collection_info') {
-            validateRead(params, FEATURE_NAME, async function(parameters) {
-                try {
-                    var stats = await common.db.collection('logs' + parameters.app_id).aggregate([ { $collStats: { storageStats: { } } } ]).toArray();
-                    common.returnOutput(parameters, {capped: stats?.[0]?.storageStats?.capped, max: stats?.[0]?.storageStats?.max});
-                }
-                catch (ex) {
-                    console.log("Failed fetching logs collection info: ", ex);
-                    common.returnMessage(parameters, 400, 'Error fetching collection info');
-                }
+            validateRead(params, FEATURE_NAME, function(parameters) {
+                common.db.collection('logs' + parameters.app_id).stats(function(err, stats) {
+                    if (err) {
+                        console.log("Failed fetching logs collection info", err);
+                        return common.returnMessage(parameters, 400, 'Error fetching collection info');
+                    }
+                    common.returnOutput(parameters, stats && {size: stats.size, count: stats.count, max: MAX_NUMBER_OF_LOG_ENTRIES} || {});
+                });
             });
             return true;
         }
     });
 
-    plugins.register("/i/apps/create", function(ob) {
-        var appId = ob.appId;
-        common.db.command({"convertToCapped": 'logs' + appId, size: 10000000, max: 1000}, function(err) {
-            if (err) {
-                common.db.createCollection('logs' + appId, {capped: true, size: 10000000, max: 1000}, function() {});
-            }
-        });
-    });
-
     plugins.register("/i/apps/delete", function(ob) {
         var appId = ob.appId;
         common.db.collection('logs' + appId).drop(function() {});
     });
 
     plugins.register("/i/apps/reset", function(ob) {
         var appId = ob.appId;
-        common.db.collection('logs' + appId).drop(function() {
-            common.db.createCollection('logs' + appId, {capped: true, size: 10000000, max: 1000}, function() {});
-        });
+        common.db.collection('logs' + appId).drop(function() {});
     });
 
     plugins.register("/i/apps/clear_all", function(ob) {
         var appId = ob.appId;
-        common.db.collection('logs' + appId).drop(function() {
-            common.db.createCollection('logs' + appId, {capped: true, size: 10000000, max: 1000}, function() {});
-        });
+        common.db.collection('logs' + appId).drop(function() {});
     });
 }(exported));
 

plugins/logger/api/jobs/clear.js

@@ -0,0 +1,48 @@
+/**
+ * @typedef {import("mongodb").Db} Database
+ * @typedef {import("mongodb").ObjectId} ObjectId
+ */
+const JOB = require("../../../../api/parts/jobs/job.js");
+const log = require("../../../../api/utils/log.js")("job:logger:clear");
+
+const DEFAULT_MAX_ENTRIES = 1000;
+
+/**
+ * clears logs
+ */
+class ClearJob extends JOB.Job {
+    /**
+     * Cleans up the logs{APPID} collection
+     * @param {Database} db mongodb database instance
+     */
+    async run(db) {
+        let max = this.data.max;
+        if (typeof max !== "number") {
+            log.e(
+                "Maximum number of log entries required. Falling back to default value:",
+                DEFAULT_MAX_ENTRIES
+            );
+            max = DEFAULT_MAX_ENTRIES;
+        }
+
+        log.d("Started: cleaning logs before the last", max);
+        const appIds = await db.collection("apps").find().project({ _id: 1 }).toArray();
+        const loggerCollections = appIds.map(({_id}) => "logs" + _id.toString());
+
+        for (const colName of loggerCollections) {
+            const col = db.collection(colName);
+            // find the first entry we want to keep
+            const firstEntry = await col.find({}).project({ _id: 1 }).sort({ _id: -1 })
+                .skip(max - 1).limit(1).toArray();
+            if (!firstEntry[0]) {
+                continue;
+            }
+            // delete all previous entries
+            const result = await col.deleteMany({ _id: { $lt: firstEntry[0]._id } });
+            log.d("Cleaned up", result.deletedCount, "entries from", colName);
+        }
+        log.d("Finished cleaning logs");
+    }
+}
+
+module.exports = ClearJob;

plugins/logger/install.js

@@ -1,53 +0,0 @@
-var async = require('async'),
-    pluginManager = require('../pluginManager.js');
-
-console.log("Installing logger plugin");
-pluginManager.dbConnection().then((countlyDb) => {
-    countlyDb.collection('apps').find({}).toArray(function(err, apps) {
-
-        if (!apps || err) {
-            countlyDb.close();
-            return;
-        }
-        function upgrade(app, done) {
-            console.log("Creating logs collection for " + app.name);
-            function cb() {
-                done();
-            }
-
-            countlyDb.command({ "listCollections": 1, "filter": { "name": "logs" + app._id } }, function(err, res) {
-                if (err) {
-                    console.log(err);
-                    cb();
-                }
-                else {
-                    //check if collection capped
-                    if (res && res.cursor && res.cursor.firstBatch && res.cursor.firstBatch.length > 0) {
-                        //collection exists
-                        if (!res.cursor.firstBatch[0].options.capped) {
-                            console.log("converting to the capped");
-                            countlyDb.command({ "convertToCapped": 'logs' + app._id, size: 10000000, max: 1000 },
-                                function(err) {
-                                    if (err) {
-                                        console.log(err);
-                                    }
-                                    cb();
-                                });
-                        }
-                        else {
-                            cb();
-                        }
-                    }
-                    else { //collection does not exist
-                        console.log("collection does not exist");
-                        countlyDb.createCollection('logs' + app._id, { capped: true, size: 10000000, max: 1000 }, cb);
-                    }
-                }
-            });
-        }
-        async.forEach(apps, upgrade, function() {
-            console.log("Logger plugin installation finished");
-            countlyDb.close();
-        });
-    });
-});