Merge branch 'next' into nginx-optimise-test

Author: kanwarujjaval

bin/scripts/data-reports/compare_drill_aggregated.js

@@ -7,9 +7,12 @@
     * node compare_drill_aggregated.js
 */
 var period = "7days"; //Chose any of formats: "Xdays" ("7days","100days") or  ["1-1-2024", "1-10-2024"], 
-var app_list = []; //List with apps
+var app_list = []; //List with apps ""
 //Example var eventMap = {"6075f94b7e5e0d392902520c":["Logout","Login"],"6075f94b7e5e0d392902520d":["Logout","Login","Buy"]};
 var eventMap = {}; //If left empty will run for all alls/events.
+
+var union_with_old_collection = true; //False if all sessions are stored in drill_events collection
+
 var verbose = false; //true to show more output
 
 
@@ -151,11 +154,40 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
                                                         }
                                                     }
                                                     if (haveAnything) {
-                                                        console.log("    " + JSON.stringify(report));
+                                                        let aggCount = totals.c || 0;
+                                                        let drillCount = drillData.totals.c || 0;
+                                                        let percentageDiff = 0;
+                                                        if (drillCount !== 0) {
+                                                            percentageDiff = ((drillCount - aggCount) / drillCount) * 100;
+                                                        }
+                                                        else {
+                                                            if (aggCount !== 0) {
+                                                                // If drillCount is 0, and aggCount is not 0, show a large difference
+                                                                percentageDiff = (aggCount > 0 ? 100 : -100); // 100% or -100% depending on the sign of aggCount
+                                                            }
+                                                            else {
+                                                                percentageDiff = 0; // Both counts are 0, no difference
+                                                            }
+                                                        }
+
+                                                        console.log("----------------------------------------------");
+                                                        console.log("- Application name:", app.name);
+                                                        console.log("- Event name:", event);
+                                                        console.log("- Counts in Aggregated data:", aggCount);
+                                                        console.log("- Counts in Drill data:", drillCount);
+                                                        console.log("- Percentage difference between Drill data and Aggregated data:", percentageDiff.toFixed(2) + "%");
+                                                        console.log("----------------------------------------------");
                                                         endReport[app._id]["bad"]++;
                                                         endReport[app._id]["events"] = endReport[app._id]["events"] || {};
-                                                        endReport[app._id]["events"][event] = {"e": event, report: report};
+                                                        endReport[app._id]["events"][event] = {
+                                                            "e": event,
+                                                            "aggregated_count": aggCount,
+                                                            "drill_count": drillCount,
+                                                            "percentage_difference": percentageDiff.toFixed(2),
+                                                            "report": report
+                                                        };
                                                     }
+
                                                     resolve2();
                                                 });
                                             }
@@ -164,6 +196,25 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
                                 }).then(function() {
                                     console.log("Finished processing app: ", app.name);
                                     resolve();
+
+                                    //Complete CSV after processing the apps
+                                    console.log("\nSummary Report (CSV-like):");
+                                    console.log("App,Event,Aggregated,Drill,% Difference");
+                                    // var csvRows = ["App,Event,Aggregated,Drill,% Difference"]; 
+                                    for (var appId in endReport) {
+                                        var appData = endReport[appId];
+                                        var appName = appData.name;
+                                        if (appData.events) {
+                                            for (var event in appData.events) {
+                                                var eventData = appData.events[event];
+                                                var row = `${appName},${event},${eventData.aggregated_count},${eventData.drill_count},${eventData.percentage_difference}`;
+                                                console.log(row);
+                                                //csvRows.push(row);
+                                            }
+                                        }
+                                    }
+
+
                                 }).catch(function(eee) {
                                     console.log("Error processing app: ", app.name);
                                     console.log(eee);
@@ -207,14 +258,17 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
         }
         endDate = endDate.valueOf() - endDate.utcOffset() * 60000;
 
-        let collection = "drill_events" + crypto.createHash('sha1').update(options.event + options.app_id).digest('hex');
-        var query = {"ts": {"$gte": startDate, "$lt": endDate}};
-        var pipeline = [
-            {"$match": query},
-        ];
+        var query = {"ts": {"$gte": startDate, "$lt": endDate}, "a": options.app_id, "e": options.event};
+        var pipeline = [];
+        pipeline.push({"$match": query});
+        if (union_with_old_collection) {
+            let collection = "drill_events" + crypto.createHash('sha1').update(options.event + options.app_id).digest('hex');
+            var query2 = {"ts": {"$gte": startDate, "$lt": endDate}};
+            pipeline.push({"$unionWith": { "coll": collection, "pipeline": [{"$match": query2}] }});
+        }
 
         pipeline.push({"$group": {"_id": "$d", "c": {"$sum": "$c"}, "s": {"$sum": "$s"}, "dur": {"$sum": "$dur"}}});
-        options.drillDb.collection(collection).aggregate(pipeline, {"allowDiskUse": true}).toArray(function(err, data) {
+        options.drillDb.collection("drill_events").aggregate(pipeline, {"allowDiskUse": true}).toArray(function(err, data) {
             if (err) {
                 console.log(err);
             }