Merge branch 'master' into anna/master

Author: Cookiezaurs

CHANGELOG.md

@@ -1,3 +1,41 @@
+## Version 24.05.11
+Fixes:
+- [cache] Use a cursor without timeout
+
+## Version 24.05.10
+Fixes:
+- [alerts] Alerts improvements
+- [core] Various fixes for frontend to support running countly from subdirectory
+- [logs]  Show collected problems on logger
+
+Enterprise fixes:
+- [data-manager] Fixes n UI to allow events starting with "/"
+- [flows] Fixes for flows step generation
+- [surveys] Other is not allowed as a valid answer for required questions
+
+## Version 24.05.9
+Fixes:
+- [crashes] Fix crashes template paths and add data check
+
+Enterprise fixes:
+- [license] Fix for License warning appears is Flex server
+- [drill] Fix typo in view regeneration api response
+
+## Version 24.05.8
+Fixes:
+- [scripts] script to check settings limits
+- [crashes] Update custom field cleanup script with streaming
+- [core] Skip data masking on internal appUsers data refetching query
+- [core][bugfix] Correct validation on "is any plugin masked"
+- [scripts] Delete custom events scripts
+- [dashboards] Show edit option also for invalid widgets
+- [core] batcher stats added
+- [dependencies] Bump xlsx-write-stream from 1.0.2 to 1.0.3
+
+Enterprise fixes:
+- [data-manager] changes to allow skipping query rewriting using passed property
+- [os] fix for changing default repository of CentOS 8 to vault because OS reached EOL
+  
 ## Version 24.05.7
 Fixes:
 - [countly-request]Fix countly-request get and post methods

api/config.sample.js

@@ -99,6 +99,7 @@ var countlyConfig = {
     * @property {string} algorithm - name of the algorithm to use for encryption. The algorithm is dependent on OpenSSL, examples are 'aes192', etc. On recent OpenSSL releases, openssl list-cipher-algorithms will display the available cipher algorithms. Default value is aes-256-cbc
     * @property {string} input_encoding - how encryption input is encoded. Used as output for decrypting. Default utf-8.
     * @property {string} output_encoding - how encryption output is encoded. Used as input for decrypting. Default hex.
+    * @property {string} reports_key - key used for encryption of reports links
     */
     encryption: {},
 

api/parts/data/batcher.js

@@ -4,7 +4,24 @@ const plugins = require('../../../plugins/pluginManager.js');
 const log = require('../../utils/log.js')("batcher");
 const common = require('../../utils/common.js');
 
-
+var batcherStats = {
+    key: 'BATCHER_STATS',
+    pid: process.pid,
+    insert_queued: 0,
+    insert_processing: 0,
+    insert_errored_fallback: 0,
+    insert_errored_no_fallback: 0,
+    insert_errored_no_fallback_last_error: "",
+    update_queued: 0,
+    update_processing: 0,
+    update_errored_fallback: 0,
+    update_errored_no_fallback: 0,
+    update_errored_no_fallback_last_error: "",
+};
+
+setInterval(function() {
+    log.i('%j', batcherStats);
+}, 10000);
 /**
  *  Class for batching database insert operations
  *  @example
@@ -56,13 +73,16 @@ class InsertBatcher {
         if (this.data[db][collection].length) {
             var docs = this.data[db][collection];
             this.data[db][collection] = [];
+            batcherStats.insert_queued -= docs.length;
+            batcherStats.insert_processing += docs.length;
             try {
                 await new Promise((resolve, reject) => {
                     this.dbs[db].collection(collection).insertMany(docs, {ordered: false, ignore_errors: [11000]}, function(err, res) {
                         if (err) {
                             reject(err);
                             return;
                         }
+                        batcherStats.insert_processing -= docs.length;
                         resolve(res);
                     });
                 });
@@ -75,7 +95,10 @@ class InsertBatcher {
                 //trying to rollback operations to try again on next iteration
                 if (ex.writeErrors && ex.writeErrors.length) {
                     for (let i = 0; i < ex.writeErrors.length; i++) {
+                        batcherStats.insert_processing--;
                         if (no_fallback_errors.indexOf(ex.writeErrors[i].code) !== -1) {
+                            batcherStats.insert_errored_no_fallback++;
+                            batcherStats.insert_errored_no_fallback_last_error = ex.writeErrors[i].errmsg;
                             //dispatch failure
                             if (notify_fallback_errors.indexOf(ex.writeErrors[i].code) !== -1) {
                                 var index0 = ex.writeErrors[i].index;
@@ -85,6 +108,7 @@ class InsertBatcher {
                             //we could record in diagnostic data
                             continue;
                         }
+                        batcherStats.insert_errored_fallback++;
                         let index = ex.writeErrors[i].index;
                         if (docs[index]) {
                             this.data[db][collection].push(docs[index]);
@@ -136,8 +160,10 @@ class InsertBatcher {
                 for (let i = 0; i < doc.length; i++) {
                     this.data[db][collection].push(doc[i]);
                 }
+                batcherStats.insert_queued += doc.length;
             }
             else {
+                batcherStats.insert_queued++;
                 this.data[db][collection].push(doc);
             }
             if (!this.process) {
@@ -212,13 +238,16 @@ class WriteBatcher {
                 }
             }
             this.data[db][collection] = {};
+            batcherStats.update_queued -= queries.length;
+            batcherStats.update_processing += queries.length;
             try {
                 await new Promise((resolve, reject) => {
                     this.dbs[db].collection(collection).bulkWrite(queries, {ordered: false, ignore_errors: [11000]}, function(err, res) {
                         if (err) {
                             reject(err);
                             return;
                         }
+                        batcherStats.update_processing -= queries.length;
                         resolve(res);
                     });
                 });
@@ -231,8 +260,10 @@ class WriteBatcher {
                 //trying to rollback operations to try again on next iteration
                 if (ex.writeErrors && ex.writeErrors.length) {
                     for (let i = 0; i < ex.writeErrors.length; i++) {
-
+                        batcherStats.update_processing--;
                         if (no_fallback_errors.indexOf(ex.writeErrors[i].code) !== -1) {
+                            batcherStats.update_errored_no_fallback++;
+                            batcherStats.update_errored_no_fallback_last_error = ex.writeErrors[i].errmsg;
                             //dispatch failure
                             if (notify_errors.indexOf(ex.writeErrors[i].code) !== -1) {
                                 var index0 = ex.writeErrors[i].index;
@@ -242,6 +273,7 @@ class WriteBatcher {
                             //we could record in diagnostic data
                             continue;
                         }
+                        batcherStats.update_errored_fallback++;
                         let index = ex.writeErrors[i].index;
                         if (queries[index]) {
                             //if we don't have anything for this document yet just use query
@@ -316,6 +348,7 @@ class WriteBatcher {
             }
             if (!this.data[db][collection][id]) {
                 this.data[db][collection][id] = {id: id, value: operation};
+                batcherStats.update_queued++;
             }
             else {
                 this.data[db][collection][id].value = common.mergeQuery(this.data[db][collection][id].value, operation);
@@ -653,4 +686,4 @@ function getId() {
     return crypto.randomBytes(16).toString("hex");
 }
 
-module.exports = {WriteBatcher, ReadBatcher, InsertBatcher};
+module.exports = {WriteBatcher, ReadBatcher, InsertBatcher};

api/parts/data/cache.js

@@ -971,7 +971,7 @@ class StreamedCollection {
             let [col, last] = await createCollection(this.db, this.name, 1e7);
 
             this.col = col;
-            this.stream = col.find({_id: {$gt: last}}, {tailable: true, awaitData: true, numberOfRetries: -1}).stream();
+            this.stream = col.find({_id: {$gt: last}}, {tailable: true, awaitData: true, noCursorTimeout: true, numberOfRetries: -1}).stream();
 
             this.stream.on('data', doc => {
                 if (this.inserts.indexOf(doc._id.toString()) !== -1) {

api/parts/data/fetch.js

@@ -1589,7 +1589,7 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) {
         options = {};
     }
 
-    if (typeof options === "undefined") {
+    if (!options) {
         options = {};
     }
 
@@ -1679,7 +1679,7 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) {
 
         var zeroDocs = [zeroIdToFetch];
         var monthDocs = [monthIdToFetch];
-        if (!(options && options.dontBreak)) {
+        if (!options.dontBreak) {
             for (let i = 0; i < common.base64.length; i++) {
                 zeroDocs.push(zeroIdToFetch + "_" + common.base64[i]);
                 monthDocs.push(monthIdToFetch + "_" + common.base64[i]);

bin/scripts/expire-data/countly_single_app_expireDataBatches.js

@@ -171,6 +171,77 @@ function prepareIterationList(collections, seconds) {
     return listed;
 
 }
+
+function processDrillCollection(collection, seconds, callback) {
+    var listed = [];
+
+    if (start === 0) {
+        getMinTs(function(err, minTs) {
+            if (err) {
+                console.log("ERROR: Could not fetch min ts for collection " + collection.collection);
+                callback(err);
+            }
+            else {
+                console.log("Min ts for collection " + collection.collection + ": " + minTs);
+                generateIterationList(minTs);
+                processCollection();
+            }
+        });
+    }
+    else {
+        generateIterationList(start);
+        processCollection();
+    }
+
+    function getMinTs(cb) {
+        collection.db.collection(collection.collection).findOne({}, { sort: { ts: 1 }, projection: { ts: 1 } }, function(err, doc) {
+            if (err) {
+                return callback(err);
+            }
+            if (doc && doc.ts) {
+                cb(null, doc.ts);
+            }
+            else {
+                cb(null, end);
+            }
+        });
+    }
+
+    function generateIterationList(z) {
+        z = (start === 0 && z) ? z : start;
+        if (timeSpan === 0 && z === 0) {
+            listed.push({"collection": collection.collection, "db": collection.db, "start": 0, "end": end, "query": {"ts": {"$lt": end}}});
+        }
+        else if (timeSpan === 0) {
+            listed.push({"collection": collection.collection, "db": collection.db, "start": z, "end": end, "query": {"ts": {"$gte": z, "$lt": end}}});
+        }
+        else {
+            if (seconds) {
+                z = Math.floor(z / 1000);
+                for (; z <= Math.floor(end / 1000); z += timeSpan) {
+                    listed.push({"collection": collection.collection, "db": collection.db, "start": z, "end": Math.min(z + timeSpan, end), "seconds": true});
+                }
+            }
+            else {
+                for (; z <= end; z += timeSpan * 1000) {
+                    listed.push({"collection": collection.collection, "db": collection.db, "start": z, "end": Math.min(z + timeSpan * 1000, end)});
+                }
+            }
+        }
+    }
+
+    function processCollection() {
+        async.eachLimit(listed, paralelCn, eventIterator, function(err) {
+            if (err) {
+                console.log("ERROR: Error while processing drill collection: " + collection.collection);
+                return callback(err);
+            }
+            console.log('Finished processing collection ' + collection.collection);
+            callback();
+        });
+    }
+}
+
 function processDrillCollections(db, drill_db, callback) {
     if (process && process.drill_events) {
         var collections = [];
@@ -192,10 +263,19 @@ function processDrillCollections(db, drill_db, callback) {
                     collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update(eventData.list[i] + APP_ID).digest('hex')});
                 }
             }
-            var iteratorList = prepareIterationList(collections);
-            async.eachLimit(iteratorList, paralelCn, eventIterator, function() {
-                console.log('Drill collections processed');
-                callback();
+
+            async.eachSeries(collections, function(collection, done) {
+                processDrillCollection(collection, false, function(err) {
+                    if (err) {
+                        console.log("ERROR: Error while processing drill collection: " + collection.collection);
+                    }
+                    done(err);
+                });
+            }, function(err) {
+                if (err) {
+                    console.log("ERROR: Error processing collections.");
+                }
+                callback(err);
             });
         });
     }
@@ -224,15 +304,22 @@ Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_dril
                     }
                 }
             }
-            var iteratorList = prepareIterationList(processCols, true);
-            async.eachLimit(iteratorList, paralelCn, eventIterator, function() {
-                if (errorCn > 0) {
-                    console.log("There were errors. Please recheck logs for those.");
-                }
-                console.log('finished');
+            if (processCols.length === 0) {
+                console.log("Finished");
                 db.close();
                 db_drill.close();
-            });
+            }
+            else {
+                var iteratorList = prepareIterationList(processCols, true);
+                async.eachLimit(iteratorList, paralelCn, eventIterator, function() {
+                    if (errorCn > 0) {
+                        console.log("There were errors. Please recheck logs for those.");
+                    }
+                    console.log('finished');
+                    db.close();
+                    db_drill.close();
+                });
+            }
         });
     }
 });

bin/scripts/expire-data/delete_custom_events.js

@@ -12,7 +12,7 @@ const common = require('../../../api/utils/common.js');
 const drillCommon = require('../../../plugins/drill/api/common.js');
 
 const APP_ID = "";
-const EVENTS = []; //leave empty to delete all custom events
+const EVENTS = []; //If empty, no events will be deleted
 
 Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
     console.log("Connected to databases...");
@@ -28,11 +28,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
         //GET EVENTS
         var events = EVENTS;
         if (!events.length) {
-            events = await countlyDb.collection("events").findOne({_id: app._id}, {_id: 0, list: 1});
-            events = (events && events.list) || [];
-        }
-        if (!events.length) {
-            close("No events found");
+            close("No events to delete");
         }
         else {
             //DELETE EVENTS