Merge branch 'release.24.10' into 24.05->24.10-merge

Author: Cookiezaurs

.eslintrc.json

@@ -237,9 +237,6 @@
                 "for-direction": [
                     "off"
                 ],
-                "no-console": [
-                    "off"
-                ],
                 "no-control-regex": [
                     "off"
                 ],
@@ -343,4 +340,4 @@
             }
         }
     ]
-}
+}

.github/workflows/main.yml

@@ -160,6 +160,11 @@ jobs:
         shell: bash
         run: "sed -i 's/port: 3001,/port: 3001, workers: 1,/' /opt/countly/api/config.js"
 
+      - name: NPM install
+        shell: bash
+        working-directory: /opt/countly
+        run: npm install
+
       - name: Run tests
         shell: bash
         working-directory: /opt/countly
@@ -207,6 +212,11 @@ jobs:
         shell: bash
         run: "sed -i 's/port: 3001,/port: 3001, workers: 1,/' /opt/countly/api/config.js"
 
+      - name: NPM install
+        shell: bash
+        working-directory: /opt/countly
+        run: npm install
+
       - name: Install plugins
         shell: bash
         run: |
@@ -260,6 +270,11 @@ jobs:
         shell: bash
         run: "sed -i 's/mongosh --quiet/mongosh --host mongodb --quiet/' /opt/countly/bin/backup/import_events.sh && sed -i 's/mongoimport --db/mongoimport --host mongodb --db/' /opt/countly/bin/backup/import_events.sh"
 
+      - name: NPM install
+        shell: bash
+        working-directory: /opt/countly
+        run: npm install
+
       - name: Prepare environment
         shell: bash
         working-directory: /opt/countly
@@ -338,6 +353,11 @@ jobs:
         shell: bash
         run: "sed -i 's/mongosh --quiet/mongosh --host mongodb --quiet/' /opt/countly/bin/backup/import_events.sh && sed -i 's/mongoimport --db/mongoimport --host mongodb --db/' /opt/countly/bin/backup/import_events.sh"
 
+      - name: NPM install
+        shell: bash
+        working-directory: /opt/countly
+        run: npm install
+
       - name: Prepare environment
         shell: bash
         working-directory: /opt/countly

CHANGELOG.md

@@ -1,42 +1,30 @@
-## Version 24.05.x
-- [feedback] Uniformize drawer internal name input texts
-- [star-rating] Fix rating score and responses table sorting
 
-## Version 24.05.30
+## Version 24.10.x
 Features:
 - [crashed] Fix unescaped SDK logs
 - [dashboards] Added the option to set a refresh rate for dashboards, allowing data to update more frequently for selected dashboards
 - [star-rating] Added missing columns to Rating Widgets table edit
 - [ui] Fix alignment of drawers title and close icon
+- [feedback] Uniformize drawer internal name input texts
+- [star-rating] Fix rating score and responses table sorting
 
-## Version 24.05.29
+## Version 24.10.9
 Fixes:
 - [core] Allow downloading data also from other databases in dbviewer
 - [crash_symbolication] Symbolication server api end point test fix
+- [crashes] Remove memory addresses from stack trace grouping
 - [push] Fixed push notifications title and content text and variables combination
 - [reports] Correctly match event for email report if event key contains '.'
-
-## Version 24.05.28
-Enterprise Fixes:
-- [cohorts] Fixed issue with combining multiple cohorts
-  
-## Version 24.05.27
-Fixes:
-- [crashes] Remove memory addresses from stack trace grouping
+- [script] Refined delete_custom_events.js to clean up faulty/dead events completely.
 - [user-management] Prevent global admin from self-revoke and self-delete
 
 Enterprise Fixes:
-- [cohorts] Fixed issue with combining multiple cohorts
-
-## Version 24.05.26
-Enterprise fixes:
 - [ab-testing] Fixed bug with variant user filtering
+- [cohorts] Fixed issue with combining multiple cohorts
+- [drill] Do not recheck old collections on app_user data deletion if querying from old collections is disabled
 - [license] Fixed issue with handling invalid date periods
 
-Fixes:
-- [script] Refined delete_custom_events.js to clean up faulty/dead events completely.
-
-## Version 24.05.25
+## Version 24.10.8
 Fixes:
 - [app-management] Unescaping HTML for created/updated application names
 - [countly-edge] Add import from Countly Edge Server
@@ -47,27 +35,23 @@ Features:
 
 Enterprise fixes:
 - [ab-testing] Mismatching user counts between ab-testing and user profiles
-
-## Version 24.05.24
-Enterprise fixes:
 - [ldap] Fixed issues that would lead to configuration options not being picked up
 
-## Version 24.05.23
-Features:
-- [hooks] Added remote config changes to internal actions
-- [system-utility] New endpoint: /take-heap-snapshot.
-- [system-utility] Using nodejs fs to write profiler files instead of gridfs.
-
+## Version 24.10.7
 Fixes:
+- [data-manager] Modifying existing values when segment values want to be updated in the Data Manager
 - [drill] Fix for UI error when push plugin is not enabled
 
 Enterprise fixes:
 - [drill] Fixed empty events list in drill section
- 
-## Version 24.05.22
+
 Features:
 - [core] Add self tracking capability
+- [hooks] Added remote config changes to internal actions
+- [system-utility] New endpoint: /take-heap-snapshot.
+- [system-utility] Using nodejs fs to write profiler files instead of gridfs.
 
+## Version 24.10.6
 Fixes:
 - [push] Using apns-id header as message result in debug mode
 - [server-stats] Fix data point calculation in job
@@ -76,6 +60,7 @@ Fixes:
 
 Enterprise fixes:
 - [drill] Encoding url component before changing history state
+- [drill] Fixed drill meta regeneration
 - [drill] [license] Update license loader to enable supplying db client
 - [users] Format data points displayed in user sidebar
 - [cohorts] Unescape drill texts in cohort component
@@ -88,7 +73,7 @@ Enterprise Dependencies:
 - Bump nanoid in /plugins/cognito from 2.1.11 to 3.3.8
 - Bump shortid in /plugins/cognito from 2.2.16 to 2.2.17
 
-## Version 24.05.21
+## Version 24.10.5
 Fixes:
 - [core] Fixed a bug causing events to not being loaded when there's an escaped character in the event name
 - [core] Fixed a bug that was causing drill to crash when there's a percentage symbol in the event name
@@ -97,15 +82,14 @@ Fixes:
 - [surveys] "Select one" text in the widget can be edited now
 - [system-utility] Fixed: Mongo error (code: 26) in some Countly instances when the profiler gets run for the first time
 
-
 Dependencies:
 - Bump countly-sdk-nodejs from 24.10.0 to 24.10.1
 - Bump countly-sdk-web from 24.11.2 to 24.11.4
 - Bump express-rate-limit from 7.4.1 to 7.5.0
 - Bump puppeteer from 23.10.4 to 23.11.1
 - Bump sass from 1.81.0 to 1.83.4
 
-## Version 24.05.20
+## Version 24.10.4
 Fixes:
 - [push] Fixed bug where IOS credentials get mixed up while sending messages from different apps at the same time
 - [push] Fixed bug where it crashes in connection pool growth because of a type mismatch in an if condition
@@ -125,8 +109,10 @@ Dependencies:
 - Bump nodemailer from 6.9.15 to 6.9.16
 - Bump puppeteer from 23.8.0 to 23.9.0
 - Bump tslib from 2.7.0 to 2.8.1
+- Bump express from 4.21.1 to 4.21.2
+- Bump mocha from 10.2.0 to 10.8.2
 
-## Version 24.05.19
+## Version 24.10.3
 Fixes:
 - [dashboards] Fixing issue where dashboard widgets go into single column
 
@@ -138,49 +124,78 @@ Security:
 - Bump cross-spawn from 7.0.3 to 7.0.6 in /ui-tests
 - Bump cross-spawn from 7.0.3 to 7.0.6 in /plugins/hooks
 
-## Version 24.05.18
+## Version 24.10.2
 Fixes:
+- [core] Correct aggregated collection cleanup on event omitting
 - [core] Fixed bug where changing passwords results in the loss of the "Global Admin" role
 - [core] Fixed bug where exporting incoming data logs could result in "Incorrect parameter \"data\" error
+- [core] Removed use of commands which needs admin rights from report manager.
 - [crash] Fixed bug in crash ingestion for scenarios where the "app version" is not a string.
 - [script] Fixing bug with "delete_old_members" script that led to malformed requests
 
 Enterprise fixes:
 - [nps] Fixed bug that showed the wrong nps preview title
 
-## Version 24.05.17
+## Version 24.10.1
 Fixes:
+- [core] Replaced "Users" with "Sessions" label on technology home widgets
 - [push] Improved ability to observe push related errors
+- [push] Replaced push plugin with an earlier version of the plugin
 
 Enterprise fixes:
 - [cohorts] Fixed issues with nightly cleanup
 - [data-manager] Fixed UI bug where rules were not visible when editing "Merge by regex" transformations
 - [drill] Fixed wrong pie chart label  tooltip in dashboard widget
 - [flows] Fixed bug in case of null data in schema
+- [license] Fixed bug with MAU type of licenses that would prevent the server from starting
 - [nps] Fixed bug in the editor where the "internal name" field was not mandatory
+- [nps] Fixed bug where it was possible to submit empty nps surveys
+- [ratings] Fixed bug with user consent
 - [ratings] Fixed UI bug where "Internal name" was not a mandatory field
 
 Security:
+- Bumped cookie-parser from 1.4.6 to 1.4.7
+- Bumped express-rate-limit from 7.4.0 to 7.4.1
+- Bumped moment-timezone from 0.5.45 to 0.5.46
+- Bumped sass from 1.79.3 to 1.79.4
 - Fixing minor vulnerability that would allow for unauthorized file upload
 
 Enterprise Features:
 - [block] Added a way to filter crashes by their error (stacktrace)
 
-## Version 24.05.16
+## Version 24.10
 Fixes:
-- [core] Replaced "Users" with "Sessions" label on technology home widgets
-- [push] Replaced push plugin with an earlier version of the plugin
+- [core] Interpreting carrier value of "--" as an unknown value
+- [crash] Fixed issues with visualisation of Flutter stack traces
+- [dashoards] If a user is given access to a dashboard, they will now be able to see all widgets even if they don't have access to the underlying data
+- [density] UI fixed
+- [events] Fixed issue where slower loading data would replace newest event selection
 
 Enterprise fixes:
-- [license] Fixed bug with MAU type of licenses that would prevent the server from starting
-- [nps] Fixed bug where it was possible to submit empty nps surveys
-- [ratings] Fixed bug with user consent
+- [crashes-jira] Preventing crashing when jira authentification fails
+- [formulas] Removing HTML from tooltip localization
 
-Security:
-- Bumped cookie-parser from 1.4.6 to 1.4.7
-- Bumped express-rate-limit from 7.4.0 to 7.4.1
-- Bumped moment-timezone from 0.5.45 to 0.5.46
-- Bumped sass from 1.79.3 to 1.79.4
+Features:
+- Making user guide icons stand out more before first use
+- [alerts] UI improvements
+- [alerts] small improvements in email notifications
+- [code] Unifying bottom menu for bulk operations
+- [core] Aggregated events data collections merged together
+- [nps] UI improvements
+- [surveys] UI improvements
+- upgraded mongoDB to version 7
+- upgraded nodejs to version 20
+
+Enterprise Features:
+- [crash_symbolication] Added ability to set custom domain override for the symbolication server
+- [drill] Drill "by query" vizualization improved
+- [drill] Drill data collections merged together
+- [events] Added a quick transition to drill
+- [hooks] Added audit log entries for hook "create", "edit", "delete", "update" actions
+- [users] "Last Session" column in User Profiles is now sortable
+- [users] Crashes in the crashes tab can now be sorted
+- [users] UI improvements
+- [views] Added a quick transition to drill
 
 ## Version 24.05.15
 Enterprise fixes:

api/api.js

@@ -156,9 +156,9 @@ plugins.connectToAllDatabases().then(function() {
             console.log("Successfully stored batch state");
         }
         catch (ex) {
-            console.log("Could not store batch state");
+            console.log("Could not store batch state", ex);
         }
-        process.exit(code);
+        process.exit(typeof code === "number" ? code : 1);
     }
 
     /**

api/config.sample.js

@@ -107,7 +107,7 @@ var countlyConfig = {
     * Specifies where to store files. Value "fs" means file system or basically storing files on hard drive. Another currently supported option is "gridfs" storing files in MongoDB database using GridFS. By default fallback to "fs";
     * @type {string} [default=fs]
     */
-    fileStorage: "fs",
+    fileStorage: "gridfs",
     /**
     * Specifies after how long time configurations are reloded from data base. Default value is 10000 (10 seconds)
     * @type {integer} [default=10000]
@@ -121,7 +121,7 @@ var countlyConfig = {
     /** 
      * Share same database connection pool between databases
      */
-    shared_connection: false,
+    shared_connection: true,
     /**
      * Simple SMTP mail sender configuration.
      * Can only be used when you don't have custom mailer extend ({@code countly/extend/mail.js}).

api/jobs/topEvents.js

@@ -138,14 +138,19 @@ class TopEventsJob extends job.Job {
      * The errors of all functions will be caught here.
      */
     async getAllApps() {
+        log.d("Fetching all apps");
         try {
             const getAllApps = await new Promise((res, rej) => common.db.collection("apps").find({}, { _id: 1, timezone: 1 }).toArray((err, apps) => err ? rej(err) : res(apps)));
-            await Promise.all(getAllApps.map((app) => this.getAppEvents(app)));
+            for (var z = 0; z < getAllApps.length; z++) {
+                //Calculating for each app serially.
+                await this.getAppEvents(getAllApps[z]);
+            }
         }
         catch (error) {
             log.e("TopEvents Job has a error: ", error);
             throw error;
         }
+        log.d("Finished processing");
     }
 
     /**
@@ -178,6 +183,7 @@ class TopEventsJob extends job.Job {
      * @param {Object} app - saveAppEvents object
      */
     async getAppEvents(app) {
+        log.d(app._id + ": Fetching app events");
         const getEvents = await new Promise((res, rej) => common.db.collection("events").findOne({ _id: app._id }, (errorEvents, result) => errorEvents ? rej(errorEvents) : res(result)));
         if (getEvents && 'list' in getEvents) {
             const eventMap = this.eventsFilter(getEvents.list);
@@ -199,6 +205,7 @@ class TopEventsJob extends job.Job {
                     let totalDuration = 0;
                     let prevTotalDuration = 0;
                     for (const event of eventMap) {
+                        log.d("    getting event data for event: " + event + " (" + period + ")");
                         const collectionNameEvents = this.eventsCollentions({ event, id: app._id });
                         await this.getEventsCount({ collectionNameEvents, ob, data, event });
                         totalCount += data[event].data.count.total;
@@ -208,11 +215,16 @@ class TopEventsJob extends job.Job {
                         totalDuration += data[event].data.duration.total;
                         prevTotalDuration += data[event].data.duration["prev-total"];
                     }
+                    log.d("    getting session count (" + period + ")");
                     await this.getSessionCount({ ob, sessionData, usersData, usersCollectionName });
+                    log.d("    saving data (" + period + ")");
                     await this.saveAppEvents({ app, data, sessionData, usersData, period, totalCount, prevTotalCount, totalSum, prevTotalSum, totalDuration, prevTotalDuration });
                 }
             }
         }
+        else {
+            log.d("    No events found for app");
+        }
 
     }
 

api/lib/countly.common.js

@@ -1533,9 +1533,6 @@ countlyCommon.extractTwoLevelData = function(db, rangeArray, clearFunction, data
         periodObj = countlyCommon.periodObj;
     }
 
-    if (!rangeArray) {
-        return {"chartData": tableData};
-    }
     var periodMin = 0,
         periodMax = 0,
         dataObj = {},
@@ -1544,6 +1541,10 @@ countlyCommon.extractTwoLevelData = function(db, rangeArray, clearFunction, data
         propertyFunctions = underscore.pluck(dataProperties, "func"),
         propertyValue = 0;
 
+    if (!rangeArray) {
+        return {"chartData": tableData};
+    }
+
     if (!periodObj.isSpecialPeriod) {
         periodMin = periodObj.periodMin;
         periodMax = (periodObj.periodMax + 1);