Merge branch 'release.24.10' into next

Author: ArtursKadikis

CHANGELOG.md

@@ -35,6 +35,18 @@ Enterprise Features:
 - [users] UI improvements
 - [views] Added a quick transition to drill
 
+## Version 24.05.17
+Fixes:
+- [push] Improved ability to observe push related errors
+
+Enterprise fixes:
+- [cohorts] Fixed issues with nightly cleanup
+- [data-manager] Fixed UI bug where rules were not visible when editing "Merge by regex" transformations
+- [drill] Fixed wrong pie chart label  tooltip in dashboard widget
+- [flows] Fixed bug in case of null data in schema
+- [nps] Fixed bug in the editor where the "internal name" field was not mandatory
+- [ratings] Fixed UI bug where "Internal name" was not a mandatory field
+
 ## Version 24.05.16
 Fixes:
 - [core] Replaced "Users" with "Sessions" label on technology home widgets

bin/scripts/expire-data/delete_custom_events_regex.js

@@ -6,7 +6,6 @@
  */
 
 
-const { ObjectId } = require('mongodb');
 const pluginManager = require('../../../plugins/pluginManager.js');
 const common = require('../../../api/utils/common.js');
 const drillCommon = require('../../../plugins/drill/api/common.js');
@@ -25,7 +24,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
 
     //GET APP
     try {
-        const app = await countlyDb.collection("apps").findOne({_id: ObjectId(APP_ID)}, {_id: 1, name: 1});
+        const app = await countlyDb.collection("apps").findOne({_id: countlyDb.ObjectID(APP_ID)}, {_id: 1, name: 1});
         console.log("App:", app.name);
         //GET EVENTS
         var events = [];
@@ -51,6 +50,27 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
                     }
                 ]).toArray();
                 events = events.length ? events[0].list : [];
+                const metaEvents = await drillDb.collection("drill_meta").aggregate([
+                    {
+                        $match: {
+                            'app_id': app._id + "",
+                            "type": "e",
+                            "e": { $regex: regex, $options: CASE_INSENSITIVE ? "i" : "", $nin: events }
+                        }
+                    },
+                    {
+                        $group: {
+                            _id: "$e"
+                        }
+                    },
+                    {
+                        $project: {
+                            _id: 0,
+                            e: "$_id"
+                        }
+                    }
+                ]).toArray();
+                events = events.concat(metaEvents.map(e => e.e));
             }
             catch (err) {
                 close("Invalid regex");
@@ -86,6 +106,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
         close(err);
     }
 
+
     async function deleteDrillEvents(appId, events) {
         for (let i = 0; i < events.length; i++) {
             var collectionName = drillCommon.getCollectionName(events[i], appId);

plugins/data-manager/frontend/public/javascripts/countly.views.js

@@ -1148,11 +1148,12 @@
                 }
                 if (doc.actionType === 'EVENT_MERGE' && doc.isRegexMerge === true) {
                     doc.actionType = 'merge-regex';
+                    doc.eventTransformTargetRegex = doc.transformTarget[0];
                 }
                 else {
                     doc.actionType = doc.actionType.split('_')[1].toLowerCase();
                 }
-                doc.isExistingEvent = 'true';
+                doc.isExistingEvent = doc.isExistingEvent ? 'true' : 'false';
                 // doc.tab;
                 // delete doc.transformType;
                 doc.name = countlyCommon.unescapeHtml(doc.name);

plugins/push/README.md

@@ -0,0 +1,12 @@
+# Countly Push Plugin
+
+These URLs need to be accessible by the Countly instance:
+
+- for Android:
+	- fcm.googleapis.com
+	- accounts.google.com
+- for IOS:
+	- api.development.push.apple.com
+	- api.push.apple.com
+- for Huawei
+	- push-api.cloud.huawei.com

plugins/push/api/api.js

@@ -83,6 +83,10 @@ plugins.register('/master', function() {
     common.dbUniqueMap.users.push(common.dbMap['messaging-enabled'] = DBMAP.MESSAGING_ENABLED);
     fields(platforms, true).forEach(f => common.dbUserMap[f] = f);
     PUSH.cache = common.cache.cls(PUSH_CACHE_GROUP);
+    setTimeout(() => {
+        const jobManager = require('../../../api/parts/jobs');
+        jobManager.job("push:clear-stats").replace().schedule("at 3:00 am every 7 days");
+    }, 10000);
 });
 
 plugins.register('/master/runners', runners => {
@@ -361,6 +365,7 @@ plugins.register('/i/app_users/export', ({app_id, uids, export_commands, dbargs,
  * @apiDefine PushMessageBody
  *
  * @apiBody {ObjectID} app Application ID
+ * @apiBody {Boolean} saveStats Store each individual push records into push_stats for debugging
  * @apiBody {String[]} platforms Array of platforms to send to
  * @apiBody {String="draft"} [status] Message status, only set to draft when creating or editing a draft message, don't set otherwise
  * @apiBody {Object} filter={} User profile filter to limit recipients of this message
@@ -410,6 +415,7 @@ plugins.register('/i/app_users/export', ({app_id, uids, export_commands, dbargs,
  *
  * @apiSuccess {ObjectID} _id Message ID
  * @apiSuccess {ObjectID} app Application ID
+ * @apiSuccess {Boolean} saveStats Store each individual push records into push_stats for debugging
  * @apiSuccess {String[]} platforms Array of platforms to send to
  * @apiSuccess {Number} state Message state, for internal use
  * @apiSuccess {String="created", "inactive", "draft", "scheduled", "sending", "sent", "stopped", "failed"} [status] Message status: "created" is for messages yet to be scheduled (put into queue), "inactive" - cannot be scheduled (approval required for push approver plugin), "draft", "scheduled", "sending", "sent", "stopped" - automated message has been stopped, "failed" - failed to send all notifications

plugins/push/api/jobs/clear-stats.js

@@ -0,0 +1,27 @@
+'use strict';
+/**
+ * @typedef {import("mongodb").Db} MongoDb
+ */
+
+const { Job } = require('../../../../api/parts/jobs/job.js');
+const log = require('../../../../api/utils/log.js')('job:push:clear-stats');
+
+const EXPIRY = 30 * 24 * 60 * 60 * 1000; // 30 days
+
+/**
+ * Clears push_stats collection
+ */
+class ClearStatsJob extends Job {
+    /**
+     * Clears push_stats based on EXPIRY and MAX_RECORDS
+     * @param {MongoDb} db - db connection
+     */
+    async run(db) {
+        log.d('Clearing push_stats');
+        await db.collection("push_stats").deleteMany({
+            d: { $lte: new Date(Date.now() - EXPIRY) }
+        });
+    }
+}
+
+module.exports = ClearStatsJob;

plugins/push/api/jobs/util/resultor.js

@@ -1,6 +1,23 @@
 const { FRAME, FRAME_NAME } = require('../../send/proto'),
     { DoFinish } = require('./do_finish'),
     { ERROR, TriggerKind, State, Status, PushError, Result } = require('../../send/data');
+/**
+ * @typedef {import("mongodb").ObjectId} ObjectId
+ */
+
+/**
+ * PushStat object (collection: push_stats)
+ * @typedef  {Object}    PushStat
+ * @property {ObjectId}  a - application id
+ * @property {ObjectId}  m - message id from "messages" collection
+ * @property {string}    u - uid from app_users{appId}
+ * @property {string}    t - token from "push_{appId}" collection
+ * @property {string=}   r - id returned from provider
+ * @property {Date}      d - date this message sent to this user
+ * @property {string=}   e - error message
+ * @property {string}    p - platform: "a" for android, "i" for ios and "h" for huawei
+ * @property {string}    f - token type: "p" for production
+ */
 
 /**
  * Stream responsible for handling sending results:
@@ -34,7 +51,10 @@ class Resultor extends DoFinish {
         this.fatalErrors = {}; // {mid: []}
         this.toDelete = []; // [push id, push id, ...]
         this.count = 0; // number of results cached
-        this.last = null; // time of last data from 
+        this.last = null; // time of last data from
+
+        /** @type {PushStat[]} */
+        this.pushStats = [];
 
         this.data.on('app', app => {
             this.changed[app._id] = {};
@@ -118,11 +138,17 @@ class Resultor extends DoFinish {
                         if (id < 0) {
                             return;
                         }
-                        let {p, m, pr} = this.data.pushes[id],
+                        const p = this.data.pushes[id];
+                        let {p: platform, m, pr} = p,
                             msg = this.data.message(m),
                             result,
                             rp, rl;
 
+                        // additional fields to keep this in push_stats
+                        if (msg && msg.saveStats) {
+                            this.pushStats.push({ a: p.a, m: p.m, p: p.p, f: p.f, u: p.u, t: p.t, d: new Date, r: null, e: results.toString() });
+                        }
+
                         if (msg) {
                             result = msg.result;
                             result.lastRun.processed++;
@@ -131,7 +157,7 @@ class Resultor extends DoFinish {
                         else {
                             result = this.noMessage[m] || (this.noMessage[m] = new Result());
                         }
-                        rp = result.sub(p, undefined, PLATFORM[p].parent);
+                        rp = result.sub(platform, undefined, PLATFORM[platform].parent);
                         rl = rp.sub(pr.la || 'default');
 
                         result.processed++;
@@ -141,8 +167,8 @@ class Resultor extends DoFinish {
                         rl.recordError(results.message, 1);
                         rl.processed++;
 
-                        if (PLATFORM[p].parent) {
-                            rp = result.sub(PLATFORM[p].parent),
+                        if (PLATFORM[platform].parent) {
+                            rp = result.sub(PLATFORM[platform].parent),
                             rl = rp.sub(pr.la || 'default');
                             rp.recordError(results.message, 1);
                             rp.processed++;
@@ -159,29 +185,39 @@ class Resultor extends DoFinish {
             }
             else {
                 results.forEach(res => {
-                    let id, token;
-                    if (typeof res === 'string') {
-                        this.log.d('Ok for %s', id);
-                        id = res;
-                    }
-                    else {
+                    let id, resultId, token;
+
+                    if (Array.isArray(res)) {
                         this.log.d('New token for %s', id);
                         id = res[0];
                         token = res[1];
                     }
+                    else {
+                        id = res;
+                    }
+
+                    if (typeof id !== "string") {
+                        resultId = id.r;
+                        id = id.p;
+                    }
 
                     let p = this.data.pushes[id];
                     if (!p) { // 2 or more resultors on one pool
                         return;
                     }
 
-                    this.data.decSending(p.m);
-
-                    let m = this.data.message(p.m),
+                    let msg = this.data.message(p.m),
                         result, rp, rl;
 
-                    if (m) {
-                        result = m.result;
+                    // additional fields to keep this in push_stats
+                    if (msg && msg.saveStats) {
+                        this.pushStats.push({ a: p.a, m: p.m, p: p.p, f: p.f, u: p.u, t: p.t, d: new Date, r: resultId, e: null });
+                    }
+
+                    this.data.decSending(p.m);
+
+                    if (msg) {
+                        result = msg.result;
                         result.lastRun.processed++;
                     }
                     else {
@@ -220,14 +256,6 @@ class Resultor extends DoFinish {
                 });
                 this.log.d('Added %d results', results.length);
             }
-
-            // // in case no more data is expected, we can safely close the stream
-            // if (this.check()) {
-            //     for (let _ in this.state.pushes) {
-            //         return;
-            //     }
-            //     this.do_flush(() => this.end());
-            // }
         }
         else if (frame & FRAME.ERROR) {
             let error = results.messageError(),
@@ -241,28 +269,35 @@ class Resultor extends DoFinish {
                         return;
                     }
                     this.log.d('Error %d %s for %s', results.type, results.name, id);
-                    let {m, p, pr} = this.data.pushes[id],
+                    const p = this.data.pushes[id];
+                    let {m, p: platform, pr} = p,
                         result, rp, rl;
+                    let msg = this.data.message(m);
+
+                    // additional fields to keep this in push_stats
+                    if (msg && msg.saveStats) {
+                        this.pushStats.push({ a: p.a, m: p.m, p: p.p, f: p.f, u: p.u, t: p.t, d: new Date, r: null, e: results.toString() });
+                    }
+
                     mids[m] = (mids[m] || 0) + 1;
                     delete this.data.pushes[id];
                     this.toDelete.push(id);
 
-                    let msg = this.data.message(m);
                     if (msg) {
                         result = msg.result;
                     }
                     else {
                         result = this.noMessage[m] || (this.noMessage[m] = new Result());
                     }
 
-                    rp = result.sub(p, undefined, PLATFORM[p].parent);
+                    rp = result.sub(platform, undefined, PLATFORM[platform].parent);
                     rl = rp.sub(pr.la || 'default');
 
                     rp.processed++;
                     rl.processed++;
 
-                    if (PLATFORM[p].parent) {
-                        rp = result.sub(PLATFORM[p].parent),
+                    if (PLATFORM[platform].parent) {
+                        rp = result.sub(PLATFORM[platform].parent),
                         rl = rp.sub(pr.la || 'default');
                         rp.processed++;
                         rl.processed++;
@@ -514,6 +549,11 @@ class Resultor extends DoFinish {
             }
         }
 
+        if (this.pushStats.length) {
+            promises.push(this.db.collection("push_stats").insertMany(this.pushStats));
+            this.pushStats = [];
+        }
+
         Promise.all(promises).then(() => {
             this.log.d('do_flush done');
             callback();