Merge pull request #5803 from Countly/next

bump next to this branch for tests

Author: kanwarujjaval

.github/workflows/main.yml

@@ -6,7 +6,7 @@ name: CI
 on:
   # Triggers the workflow on push or pull request events but only for the master branch
   pull_request:
-    branches: [ master, next ]
+    branches: [ master, next, release.24.10 ]
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
@@ -302,7 +302,7 @@ jobs:
           cd ui-tests
           npm install
           xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" \
-          npm run cy:run:dashboard --headless --no-sandbox --disable-gpu --disable-dev-shm-usage
+          npm run cy:run:dashboard
 
       - name: Upload UI tests artifacts
         if: ${{ failure() }}
@@ -381,7 +381,7 @@ jobs:
           cd ui-tests
           npm install
           xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" \
-          npm run cy:run:onboarding --headless --no-sandbox --disable-gpu --disable-dev-shm-usage
+          npm run cy:run:onboarding
 
       - name: Upload UI tests artifacts
         if: ${{ failure() }}

CHANGELOG.md

@@ -1,3 +1,42 @@
+## Version 24.10.2
+Fixes:
+- [core] Correct aggregated collection cleanup on event omitting
+- [core] Fixed bug where changing passwords results in the loss of the "Global Admin" role
+- [core] Fixed bug where exporting incoming data logs could result in "Incorrect parameter \"data\" error
+- [core] Removed use of commands which needs admin rights from report manager.
+- [crash] Fixed bug in crash ingestion for scenarios where the "app version" is not a string.
+- [script] Fixing bug with "delete_old_members" script that led to malformed requests
+
+Enterprise fixes:
+- [nps] Fixed bug that showed the wrong nps preview title
+
+## Version 24.10.1
+Fixes:
+- [core] Replaced "Users" with "Sessions" label on technology home widgets
+- [push] Improved ability to observe push related errors
+- [push] Replaced push plugin with an earlier version of the plugin
+
+Enterprise fixes:
+- [cohorts] Fixed issues with nightly cleanup
+- [data-manager] Fixed UI bug where rules were not visible when editing "Merge by regex" transformations
+- [drill] Fixed wrong pie chart label  tooltip in dashboard widget
+- [flows] Fixed bug in case of null data in schema
+- [license] Fixed bug with MAU type of licenses that would prevent the server from starting
+- [nps] Fixed bug in the editor where the "internal name" field was not mandatory
+- [nps] Fixed bug where it was possible to submit empty nps surveys
+- [ratings] Fixed bug with user consent
+- [ratings] Fixed UI bug where "Internal name" was not a mandatory field
+
+Security:
+- Bumped cookie-parser from 1.4.6 to 1.4.7
+- Bumped express-rate-limit from 7.4.0 to 7.4.1
+- Bumped moment-timezone from 0.5.45 to 0.5.46
+- Bumped sass from 1.79.3 to 1.79.4
+- Fixing minor vulnerability that would allow for unauthorized file upload
+
+Enterprise Features:
+- [block] Added a way to filter crashes by their error (stacktrace)
+
 ## Version 24.10
 Fixes:
 - [core] Interpreting carrier value of "--" as an unknown value
@@ -34,6 +73,7 @@ Enterprise Features:
 
 ## Version 24.05.15
 Enterprise fixes:
+- [ab-testing] Fixed JSON.parse issue preventing creation of AB tests
 - [nps] Fixed UI issues in the widget editor related to the "user consent" section
 - [ratings] Fixed rendering issue for escaped values
 

api/api.js

@@ -313,6 +313,7 @@ plugins.connectToAllDatabases().then(function() {
             jobs.job('api:clearAutoTasks').replace().schedule('every 1 day');
             jobs.job('api:task').replace().schedule('every 5 minutes');
             jobs.job('api:userMerge').replace().schedule('every 10 minutes');
+            jobs.job("api:ttlCleanup").replace().schedule("every 1 minute");
             //jobs.job('api:appExpire').replace().schedule('every 1 day');
         }, 10000);
 

api/jobs/task.js

@@ -56,6 +56,10 @@ class MonitorJob extends job.Job {
                 return true;
             }
 
+            if (task.dirty) {
+                return true;
+            }
+
             if ((now + duration - lastStart) / 1000 >= interval) {
                 return true;
             }
@@ -74,7 +78,8 @@ class MonitorJob extends job.Job {
                         taskmanager.rerunTask({
                             db: common.db,
                             id: task._id,
-                            autoUpdate: true
+                            autoUpdate: true,
+                            dirty: task.dirty
                         }, function(e) {
                             if (e) {
                                 log.e(e, e.stack);

api/jobs/ttlCleanup.js

@@ -0,0 +1,48 @@
+const plugins = require("../../plugins/pluginManager.js");
+const common = require('../utils/common');
+const job = require("../parts/jobs/job.js");
+const log = require("../utils/log.js")("job:ttlCleanup");
+
+/**
+ * Class for job of cleaning expired records inside ttl collections
+ */
+class TTLCleanup extends job.Job {
+    /**
+     * Run the job
+     */
+    async run() {
+        log.d("Started running TTL clean up job");
+        for (let i = 0; i < plugins.ttlCollections.length; i++) {
+            const {
+                db = "countly",
+                collection,
+                property,
+                expireAfterSeconds = 0
+            } = plugins.ttlCollections[i];
+            let dbInstance;
+            switch (db) {
+            case "countly": dbInstance = common.db; break;
+            case "countly_drill": dbInstance = common.drillDb; break;
+            case "countly_out": dbInstance = common.outDb; break;
+            }
+            if (!dbInstance) {
+                log.e("Invalid db selection:", db);
+                continue;
+            }
+
+            log.d("Started cleaning up", collection);
+            const result = await dbInstance.collection(collection).deleteMany({
+                [property]: {
+                    $lte: new Date(Date.now() - expireAfterSeconds * 1000)
+                }
+            });
+            log.d("Finished cleaning up", result.deletedCount, "records from", collection);
+
+            // Sleep 1 second to prevent sending too many deleteMany queries
+            await new Promise(res => setTimeout(res, 1000));
+        }
+        log.d("Finished running TTL clean up job");
+    }
+}
+
+module.exports = TTLCleanup;

api/parts/data/fetch.js

@@ -1323,7 +1323,7 @@ fetch.fetchEvents = function(params) {
 */
 fetch.fetchTimeObj = function(collection, params, isCustomEvent, options) {
     fetchTimeObj(collection, params, isCustomEvent, options, function(output) {
-        if (params?.qstring?.event) {
+        if (params.qstring?.event) {
             output.eventName = params.qstring.event;
         }
         common.returnOutput(params, output);