Merge branch 'SER-683' of https://github.com/Countly/countly-server into SER-683

Author: ayasayadi1

.eslintrc.json

@@ -237,9 +237,6 @@
                 "for-direction": [
                     "off"
                 ],
-                "no-console": [
-                    "off"
-                ],
                 "no-control-regex": [
                     "off"
                 ],
@@ -343,4 +340,4 @@
             }
         }
     ]
-}
+}

.github/dependabot.yml

@@ -202,9 +202,9 @@ updates:
     interval: daily
   open-pull-requests-limit: 10
   reviewers:
-  - iartem
+  - cihadtekin
   assignees:
-  - iartem
+  - cihadtekin
   labels:
   - dependencies
   versioning-strategy: increase-if-necessary

.github/workflows/main.yml

@@ -155,6 +155,11 @@ jobs:
         shell: bash
         run: "sed -i 's/port: 3001,/port: 3001, workers: 1,/' /opt/countly/api/config.js"
 
+      - name: NPM install
+        shell: bash
+        working-directory: /opt/countly
+        run: npm install
+
       - name: Run tests
         shell: bash
         working-directory: /opt/countly
@@ -202,6 +207,11 @@ jobs:
         shell: bash
         run: "sed -i 's/port: 3001,/port: 3001, workers: 1,/' /opt/countly/api/config.js"
 
+      - name: NPM install
+        shell: bash
+        working-directory: /opt/countly
+        run: npm install
+
       - name: Install plugins
         shell: bash
         run: |
@@ -255,6 +265,11 @@ jobs:
         shell: bash
         run: "sed -i 's/mongosh --quiet/mongosh --host mongodb --quiet/' /opt/countly/bin/backup/import_events.sh && sed -i 's/mongoimport --db/mongoimport --host mongodb --db/' /opt/countly/bin/backup/import_events.sh"
 
+      - name: NPM install
+        shell: bash
+        working-directory: /opt/countly
+        run: npm install
+
       - name: Prepare environment
         shell: bash
         working-directory: /opt/countly
@@ -332,6 +347,11 @@ jobs:
         shell: bash
         run: "sed -i 's/mongosh --quiet/mongosh --host mongodb --quiet/' /opt/countly/bin/backup/import_events.sh && sed -i 's/mongoimport --db/mongoimport --host mongodb --db/' /opt/countly/bin/backup/import_events.sh"
 
+      - name: NPM install
+        shell: bash
+        working-directory: /opt/countly
+        run: npm install
+
       - name: Prepare environment
         shell: bash
         working-directory: /opt/countly

CHANGELOG.md

@@ -1,3 +1,121 @@
+## Version 24.05.11
+Fixes:
+- [cache] Use a cursor without timeout
+
+## Version 24.05.10
+Fixes:
+- [alerts] Alerts improvements
+- [core] Various fixes for frontend to support running countly from subdirectory
+- [logs]  Show collected problems on logger
+
+Enterprise fixes:
+- [data-manager] Fixes n UI to allow events starting with "/"
+- [flows] Fixes for flows step generation
+- [surveys] Other is not allowed as a valid answer for required questions
+
+## Version 24.05.9
+Fixes:
+- [crashes] Fix crashes template paths and add data check
+
+Enterprise fixes:
+- [license] Fix for License warning appears is Flex server
+- [drill] Fix typo in view regeneration api response
+
+## Version 24.05.8
+Fixes:
+- [scripts] script to check settings limits
+- [crashes] Update custom field cleanup script with streaming
+- [core] Skip data masking on internal appUsers data refetching query
+- [core][bugfix] Correct validation on "is any plugin masked"
+- [scripts] Delete custom events scripts
+- [dashboards] Show edit option also for invalid widgets
+- [core] batcher stats added
+- [dependencies] Bump xlsx-write-stream from 1.0.2 to 1.0.3
+
+Enterprise fixes:
+- [data-manager] changes to allow skipping query rewriting using passed property
+- [os] fix for changing default repository of CentOS 8 to vault because OS reached EOL
+  
+## Version 24.05.7
+Fixes:
+- [countly-request]Fix countly-request get and post methods
+- [core] Improvements to prevent XSS
+- [push] objects are not allowed for the properties inside "data" key of the payload in new FCM API
+- [populator] Fixes for handling bulk requests
+
+Enterprise fixes:
+- [adjust] Added adjust plugin
+- [surveys] UI changes for table
+
+## Version 24.05.6
+Fixes:
+- [data-manager] Fix for event deletion timeout
+- [alerts] Fix for special character problem in value part for all data types
+- [scripts] Fix for old views script
+- [server-stats] Add monthly data points endpoint
+
+Enterprise fixes:
+- [config-transfer] Correct check when adding config transfer menu item
+- [users] Changes to show sample document download for all users
+- [funnels] Prevent autofill from Chrome on inputs
+- [drill] Fix for case when it is not possible to download table in drill
+- [surveys] Update popup height attribute in survey widgets
+- [license] Get correct license metric values
+- [okta] Add config extender code for OKTA
+
+## Version 24.05.5
+Fixes:
+- [push] p12 file parsing bug fixed
+- [views] script to delete old views
+
+Enterprise fixes:
+- [active_users] Fixes for Active users recalculation
+
+## Version 24.05.4
+Fixes:
+- [core] Make sure Unicode characters are escaped for events segment values on incoming data
+- [dashboard] Small UI fixes for dealing with unexpected appid values
+
+Features:
+- [core]  Script for getting document counts of drill collections
+
+Enterprise fixes:
+- [drill] Make sure Unicode characters are escaped for events segment values on incoming data
+
+## Version 24.05.3
+Fixes:
+- [core] Bugfix for newsletter check
+- [core] Presets UI fixes
+- [core] Security Vulnerabilities and bugs
+- [events] Fix event chart selection
+- [push] Hotfix: removed the projection from push token query
+- [star-rating] Text and link sanitization
+
+Features:
+- [core]  Run once option for crash removal script
+- [populator] Add warning in case salt is set to application
+
+Enterprise fixes:
+- [block] Add user prop support for crash blocking and isset operator overall
+- [drill] Fixes for query builder when dealing with values starting with an empty string
+
+
+## Version 24.05.2
+Fixes:
+- [dashboards] Disable intercom for custom dashboards
+- [data-manager] Fix duplicate Create Event Group button
+- [push] Push config is not being loaded properly
+- [push] Push FCM SDK is not using proxy while sending messages
+- [core] Fix height inconsistency in datepicker popup
+- [core] Fixes to prevent scheduling duplicated jobs
+- [core] Making sure "Exclude current day" works everywhere
+- [dashboards] Fix issues in note widget
+- [flows] Nullcheck in default flows creation function.
+
+Enterprise fixes:
+- [flows] Nullcheck in default flows creation function
+- [drill] Nullcheck when fetching snapshots data
+
 ## Version 24.05.1
 Fixes:
 - [populator] Bugfix for - Can't edit populator template

api/config.sample.js

@@ -106,7 +106,7 @@ var countlyConfig = {
     * Specifies where to store files. Value "fs" means file system or basically storing files on hard drive. Another currently supported option is "gridfs" storing files in MongoDB database using GridFS. By default fallback to "fs";
     * @type {string} [default=fs]
     */
-    fileStorage: "fs",
+    fileStorage: "gridfs",
     /**
     * Specifies after how long time configurations are reloded from data base. Default value is 10000 (10 seconds)
     * @type {integer} [default=10000]
@@ -120,7 +120,7 @@ var countlyConfig = {
     /** 
      * Share same database connection pool between databases
      */
-    shared_connection: false,
+    shared_connection: true,
     /**
      * Simple SMTP mail sender configuration.
      * Can only be used when you don't have custom mailer extend ({@code countly/extend/mail.js}).

api/lib/countly.common.js

@@ -384,7 +384,17 @@ function getPeriodObject(prmPeriod, bucket) {
 
     var period = prmPeriod || _period;
 
-    var excludeCurrentDay = period.excludeCurrentDay || false;
+    if (typeof period === 'string' && period.indexOf(",") !== -1) {
+        try {
+            period = JSON.parse(period);
+        }
+        catch (SyntaxError) {
+            console.log("period JSON parse failed");
+            period = "30days";
+        }
+    }
+
+    var excludeCurrentDay = period.exclude_current_day || false;
 
     if (period.period) {
         period = period.period;
@@ -396,16 +406,6 @@ function getPeriodObject(prmPeriod, bucket) {
         period = [period.since, endTimestamp.clone().valueOf()];
     }
 
-    if (period && typeof period === 'string' && period.indexOf(",") !== -1) {
-        try {
-            period = JSON.parse(period);
-        }
-        catch (SyntaxError) {
-            console.log("period JSON parse failed");
-            period = "30days";
-        }
-    }
-
     if (Array.isArray(period)) {
         if ((period[0] + "").length === 10) {
             period[0] *= 1000;

api/parts/data/batcher.js

@@ -4,7 +4,24 @@ const plugins = require('../../../plugins/pluginManager.js');
 const log = require('../../utils/log.js')("batcher");
 const common = require('../../utils/common.js');
 
-
+var batcherStats = {
+    key: 'BATCHER_STATS',
+    pid: process.pid,
+    insert_queued: 0,
+    insert_processing: 0,
+    insert_errored_fallback: 0,
+    insert_errored_no_fallback: 0,
+    insert_errored_no_fallback_last_error: "",
+    update_queued: 0,
+    update_processing: 0,
+    update_errored_fallback: 0,
+    update_errored_no_fallback: 0,
+    update_errored_no_fallback_last_error: "",
+};
+
+setInterval(function() {
+    log.i('%j', batcherStats);
+}, 10000);
 /**
  *  Class for batching database insert operations
  *  @example
@@ -56,13 +73,16 @@ class InsertBatcher {
         if (this.data[db][collection].length) {
             var docs = this.data[db][collection];
             this.data[db][collection] = [];
+            batcherStats.insert_queued -= docs.length;
+            batcherStats.insert_processing += docs.length;
             try {
                 await new Promise((resolve, reject) => {
                     this.dbs[db].collection(collection).insertMany(docs, {ordered: false, ignore_errors: [11000]}, function(err, res) {
                         if (err) {
                             reject(err);
                             return;
                         }
+                        batcherStats.insert_processing -= docs.length;
                         resolve(res);
                     });
                 });
@@ -75,7 +95,10 @@ class InsertBatcher {
                 //trying to rollback operations to try again on next iteration
                 if (ex.writeErrors && ex.writeErrors.length) {
                     for (let i = 0; i < ex.writeErrors.length; i++) {
+                        batcherStats.insert_processing--;
                         if (no_fallback_errors.indexOf(ex.writeErrors[i].code) !== -1) {
+                            batcherStats.insert_errored_no_fallback++;
+                            batcherStats.insert_errored_no_fallback_last_error = ex.writeErrors[i].errmsg;
                             //dispatch failure
                             if (notify_fallback_errors.indexOf(ex.writeErrors[i].code) !== -1) {
                                 var index0 = ex.writeErrors[i].index;
@@ -85,6 +108,7 @@ class InsertBatcher {
                             //we could record in diagnostic data
                             continue;
                         }
+                        batcherStats.insert_errored_fallback++;
                         let index = ex.writeErrors[i].index;
                         if (docs[index]) {
                             this.data[db][collection].push(docs[index]);
@@ -136,8 +160,10 @@ class InsertBatcher {
                 for (let i = 0; i < doc.length; i++) {
                     this.data[db][collection].push(doc[i]);
                 }
+                batcherStats.insert_queued += doc.length;
             }
             else {
+                batcherStats.insert_queued++;
                 this.data[db][collection].push(doc);
             }
             if (!this.process) {
@@ -212,13 +238,16 @@ class WriteBatcher {
                 }
             }
             this.data[db][collection] = {};
+            batcherStats.update_queued -= queries.length;
+            batcherStats.update_processing += queries.length;
             try {
                 await new Promise((resolve, reject) => {
                     this.dbs[db].collection(collection).bulkWrite(queries, {ordered: false, ignore_errors: [11000]}, function(err, res) {
                         if (err) {
                             reject(err);
                             return;
                         }
+                        batcherStats.update_processing -= queries.length;
                         resolve(res);
                     });
                 });
@@ -231,8 +260,10 @@ class WriteBatcher {
                 //trying to rollback operations to try again on next iteration
                 if (ex.writeErrors && ex.writeErrors.length) {
                     for (let i = 0; i < ex.writeErrors.length; i++) {
-
+                        batcherStats.update_processing--;
                         if (no_fallback_errors.indexOf(ex.writeErrors[i].code) !== -1) {
+                            batcherStats.update_errored_no_fallback++;
+                            batcherStats.update_errored_no_fallback_last_error = ex.writeErrors[i].errmsg;
                             //dispatch failure
                             if (notify_errors.indexOf(ex.writeErrors[i].code) !== -1) {
                                 var index0 = ex.writeErrors[i].index;
@@ -242,6 +273,7 @@ class WriteBatcher {
                             //we could record in diagnostic data
                             continue;
                         }
+                        batcherStats.update_errored_fallback++;
                         let index = ex.writeErrors[i].index;
                         if (queries[index]) {
                             //if we don't have anything for this document yet just use query
@@ -316,6 +348,7 @@ class WriteBatcher {
             }
             if (!this.data[db][collection][id]) {
                 this.data[db][collection][id] = {id: id, value: operation};
+                batcherStats.update_queued++;
             }
             else {
                 this.data[db][collection][id].value = common.mergeQuery(this.data[db][collection][id].value, operation);
@@ -653,4 +686,4 @@ function getId() {
     return crypto.randomBytes(16).toString("hex");
 }
 
-module.exports = {WriteBatcher, ReadBatcher, InsertBatcher};
+module.exports = {WriteBatcher, ReadBatcher, InsertBatcher};

api/parts/data/cache.js

@@ -971,7 +971,7 @@ class StreamedCollection {
             let [col, last] = await createCollection(this.db, this.name, 1e7);
 
             this.col = col;
-            this.stream = col.find({_id: {$gt: last}}, {tailable: true, awaitData: true, numberOfRetries: -1}).stream();
+            this.stream = col.find({_id: {$gt: last}}, {tailable: true, awaitData: true, noCursorTimeout: true, numberOfRetries: -1}).stream();
 
             this.stream.on('data', doc => {
                 if (this.inserts.indexOf(doc._id.toString()) !== -1) {