From 5b5c53a306552abe77e9881292e34c95f6b9b629 Mon Sep 17 00:00:00 2001
From: Meghana M R <86944692+mrmeghana@users.noreply.github.com>
Date: Fri, 24 Oct 2025 17:21:35 +0530
Subject: [PATCH 1/2] Update delete_custom_events.js to
 delete_x_days_inactive_events

---
 .../expire-data/delete_custom_events.js       | 275 +++++++++---------
 1 file changed, 133 insertions(+), 142 deletions(-)

diff --git a/bin/scripts/expire-data/delete_custom_events.js b/bin/scripts/expire-data/delete_custom_events.js
index addb9b4d26c..97ad9414dbf 100644
--- a/bin/scripts/expire-data/delete_custom_events.js
+++ b/bin/scripts/expire-data/delete_custom_events.js
@@ -1,144 +1,135 @@
 /**
- *  Description: Deletes custom events from countly and drill databases
- *  Server: countly
- *  Path: $(countly dir)/bin/scripts/expire-data
- *  Command: node delete_custom_events.js
+ *  Description: Deletes obsolete custom events (not received for >90 days) from Countly and Drill DBs for an app.
+ *  Usage: node delete_x_days_inactive_events.js
  */
-
-const { ObjectId } = require('mongodb');
-const pluginManager = require('../../../plugins/pluginManager.js');
-const common = require('../../../api/utils/common.js');
-const drillCommon = require('../../../plugins/drill/api/common.js');
-
-const DRY_RUN = true;
-const APP_ID = "";
-const EVENTS = []; // If empty, no events will be deleted . The format has to be "event1","event2"
-
-Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
-    console.log("Connected to databases...");
-
-    //SET COMMON DB AND DRILL DB
-    common.db = countlyDb;
-    common.drillDb = drillDb;
-    //GET APP
-    try {
-        const app = await countlyDb.collection("apps").findOne({_id: new ObjectId(APP_ID)}, {_id: 1, name: 1});
-        console.log("App:", app.name);
-        //GET EVENTS
-        let events = EVENTS;
-        if (!events.length) {
-            console.log("Fetching events from drill_meta...");
-            const metaEvents = await drillDb.collection("drill_meta").aggregate([
-                {
-                    $match: {
-                        'app_id': app._id + "",
-                        "type": "e",
-                        "e": { $nin: events }
-                    }
-                },
-                {
-                    $group: {
-                        _id: "$e"
-                    }
-                },
-                {
-                    $project: {
-                        _id: 0,
-                        e: "$_id"
-                    }
-                }
-            ]).toArray();
-            events = metaEvents.map(e => e.e);
-        }
-        console.log("Events to delete:", events);
-        if (DRY_RUN) {
-            console.log("DRY_RUN enabled. No changes will be made.");
-            return close();
-        }
-        if (!events.length) {
-            return close("No events to delete");
-        }
-        try {
-            //DELETE EVENTS
-            console.log(1 + ") Deleting drill events:");
-            await deleteDrillEvents(app._id, events);
-            console.log(2 + ") Deleting countly events:");
-            await deleteCountlyEvents(app._id, events);
-            console.log(3 + ") Deleting event times:");
-            await deleteEventTimes(app._id, events);
-            console.log(4 + ") Deleting event groups:");
-            await deleteEventGroups(app._id, events);
-            console.log(5 + ") Deleting event keys:");
-            await deleteEventKeys(app._id, events);
-            close();
-        }
-        catch (err) {
-            close(err);
-        }
-    }
-    catch (err) {
-        console.log("App not found");
-        close(err);
-    }
-
-    async function deleteDrillEvents(appId, events) {
-        for (let i = 0; i < events.length; i++) {
-            var collectionName = drillCommon.getCollectionName(events[i], appId);
-            await drillDb.collection(collectionName).drop();
-            console.log("Dropped collection:", collectionName);
-        }
-        //delete from aggregated drill event collection
-        await drillDb.collection('drill_events').remove({'a': appId + "", 'e': {$in: events}});
-        console.log("Cleared from drill_events");
-        await drillDb.collection('drill_bookmarks').remove({'app_id': appId, 'event_key': {$in: events}});
-        console.log("Cleared drill_bookmarks");
-        await drillDb.collection("drill_meta").remove({'app_id': (appId + ""), "type": "e", "e": {$in: events}});
-        console.log("Cleared drill_meta");
-    }
-
-    async function deleteCountlyEvents(appId, events) {
-        for (let i = 0; i < events.length; i++) {
-            var collectionName = 'events' + drillCommon.getEventHash(events[i], appId);
-            await countlyDb.collection(collectionName).drop();
-            console.log("Dropped collection:", collectionName);
-
-            //clear from merged collection
-            await countlyDb.collection("events_data").remove({'_id': {"$regex": "^" + appId + "_" + drillCommon.getEventHash(events[i], appId) + "_.*"}});
-            console.log("Cleared from agregated collection");
-        }
-    }
-
-    async function deleteEventTimes(appId, events) {
-        await countlyDb.collection("eventTimes" + appId).update({}, {"$pull": {"e": {"e": {$in: events}}}}, {"multi": true});
-        console.log("Cleared eventTimes");
-        await countlyDb.collection("timelineStatus").remove({'app_id': (appId + ""), "event": {$in: events}});
-        console.log("Cleared timelineStatus");
-    }
-
-    async function deleteEventKeys(appId, events) {
-        const unsetQuery = {};
-        for (let i = 0; i < events.length; i++) {
-            unsetQuery[`segments.${events[i]}`] = 1;
-            unsetQuery[`map.${events[i]}`] = 1;
-            unsetQuery[`omitted_segments.${events[i]}`] = 1;
-        }
-        await countlyDb.collection("events").updateOne({_id: appId}, {$pull: {list: {$in: events}}, $unset: unsetQuery}, {$pull: {"overview": {eventKey: {$in: events}}}});
-        console.log("Cleared events:", events);
-    }
-
-    async function deleteEventGroups(appId, events) {
-        await countlyDb.collection("event_groups").update({'app_id': (appId + "")}, {"$pull": {"source_events": {$in: events}}}, {"multi": true});
-        console.log("Cleared event_groups");
-    }
-
-    function close(err) {
-        if (err) {
-            console.log("Finished with errors:", err);
-        }
-        else {
-            console.log("Finished successfully.");
-        }
-        countlyDb.close();
-        drillDb.close();
-    }
-});
+ const { ObjectId } = require('mongodb');
+ const pluginManager = require('../../../plugins/pluginManager.js');
+ const common = require('../../../api/utils/common.js');
+ const drillCommon = require('../../../plugins/drill/api/common.js');
+//double-check below details before running the script
+ const DRY_RUN = true; // Set to false to actually delete
+ const APP_ID = ""; // Replace with your app's ObjectID string
+ const DAYS = 90; // Number of days for active events
+ function daysAgo(days) {
+     return Math.floor(Date.now() / 1000) - days * 24 * 60 * 60; // eventTimes uses Unix timestamps (seconds)
+ }
+ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
+     console.log("Connected to databases...");
+     common.db = countlyDb;
+     common.drillDb = drillDb;
+     try {
+         const app = await countlyDb.collection("apps").findOne({_id: new ObjectId(APP_ID)}, {_id: 1, name: 1});
+         if (!app) {
+             console.log("App not found");
+             return close();
+         }
+         console.log("App:", app.name);
+         // Get eventTimes data
+         const eventTimesColl = "eventTimes" + APP_ID;
+         const allDocs = await countlyDb.collection(eventTimesColl).find({}).toArray();
+ 
+         let obsoleteEvents = [];
+         if (allDocs.length) {
+             for (const doc of allDocs) {
+                 if (doc.e && Array.isArray(doc.e)) {
+                     for (const eventObj of doc.e) {
+                         // eventObj.e is event key, eventObj.ts is last timestamp (Unix seconds)
+                         if (eventObj.t && Math.floor(eventObj.t / 1000) < daysAgo(DAYS)) {
+                             obsoleteEvents.push(eventObj.e);
+                         }
+                     }
+                 }
+             }
+         }
+         obsoleteEvents = [...new Set(obsoleteEvents)];
+         console.log("Obsolete events to delete:", obsoleteEvents);
+         if (DRY_RUN) {
+             console.log("DRY_RUN enabled. No changes will be made.");
+             return close();
+         }
+         if (!obsoleteEvents.length) {
+             return close("No obsolete events to delete");
+         }
+         // Delete obsolete events
+         await deleteDrillEvents(app._id, obsoleteEvents);
+         await deleteCountlyEvents(app._id, obsoleteEvents);
+         await deleteEventTimes(app._id, obsoleteEvents);
+         await deleteEventGroups(app._id, obsoleteEvents);
+         await deleteEventKeys(app._id, obsoleteEvents);
+         close();
+ 
+     } catch (err) {
+         console.log("App not found or error occurred");
+         close(err);
+     }
+     async function deleteDrillEvents(appId, events) {
+         for (let i = 0; i < events.length; i++) {
+             var collectionName = drillCommon.getCollectionName(events[i], appId);
+             try {
+                 await drillDb.collection(collectionName).drop();
+                 console.log("Dropped Drill collection:", collectionName);
+             } catch (ex) {
+                 // Collection may not exist
+                 console.log("Could not drop collection (may not exist):", collectionName);
+             }
+         }
+         //delete from aggregated drill event collection
+         await drillDb.collection('drill_events').deleteMany({'a': appId + "", 'e': {$in: events}});
+         console.log("Cleared from drill_events");
+         await drillDb.collection('drill_bookmarks').deleteMany({'app_id': appId, 'event_key': {$in: events}});
+         console.log("Cleared drill_bookmarks");
+         await drillDb.collection("drill_meta").deleteMany({'app_id': (appId + ""), "type": "e", "e": {$in: events}});
+         console.log("Cleared drill_meta");
+     }
+     async function deleteCountlyEvents(appId, events) {
+         for (let i = 0; i < events.length; i++) {
+             var collectionName = 'events' + drillCommon.getEventHash(events[i], appId);
+             try {
+                 await countlyDb.collection(collectionName).drop();
+                 console.log("Dropped Countly collection:", collectionName);
+             } catch (ex) {
+                 // Collection may not exist
+                 console.log("Could not drop collection (may not exist):", collectionName);
+             }
+             //clear from merged collection
+             await countlyDb.collection("events_data").deleteMany({'_id': {"$regex": "^" + appId + "_" + drillCommon.getEventHash(events[i], appId) + "_.*"}});
+             console.log("Cleared from aggregated events_data for:", events[i]);
+         }
+     }
+     async function deleteEventTimes(appId, events) {
+         await countlyDb.collection("eventTimes" + appId).updateMany({}, {"$pull": {"e": {"e": {$in: events}}}});
+         console.log("Cleared eventTimes");
+         await countlyDb.collection("timelineStatus").deleteMany({'app_id': (appId + ""), "event": {$in: events}});
+         console.log("Cleared timelineStatus");
+     }
+     async function deleteEventKeys(appId, events) {
+         const unsetQuery = {};
+         for (let i = 0; i < events.length; i++) {
+             unsetQuery[`segments.${events[i]}`] = "";
+             unsetQuery[`map.${events[i]}`] = "";
+             unsetQuery[`omitted_segments.${events[i]}`] = "";
+         }
+         // Remove from "list" and "overview" arrays, and unset event segments/maps
+         await countlyDb.collection("events").updateOne(
+             {_id: appId},
+             {
+                 $pull: {
+                     list: {$in: events},
+                     overview: {eventKey: {$in: events}}
+                 },
+                 $unset: unsetQuery
+             }
+         );
+         console.log("Cleared events:", events);
+     }
+     async function deleteEventGroups(appId, events) {
+         await countlyDb.collection("event_groups").updateMany({'app_id': (appId + "")}, {"$pull": {"source_events": {$in: events}}});
+         console.log("Cleared event_groups");
+     }
+     function close(err) {
+         if (err) console.log("Finished with errors:", err);
+         else console.log("Finished successfully.");
+         countlyDb.close();
+         drillDb.close();
+     }
+ });

From a9e73c9a19f978f383e2f148dbb079c798f0d790 Mon Sep 17 00:00:00 2001
From: Meghana M R <86944692+mrmeghana@users.noreply.github.com>
Date: Fri, 24 Oct 2025 17:43:03 +0530
Subject: [PATCH 2/2] Update delete_custom_events.js

---
 .../expire-data/delete_custom_events.js       | 265 +++++++++---------
 1 file changed, 134 insertions(+), 131 deletions(-)

diff --git a/bin/scripts/expire-data/delete_custom_events.js b/bin/scripts/expire-data/delete_custom_events.js
index 97ad9414dbf..4540bffe575 100644
--- a/bin/scripts/expire-data/delete_custom_events.js
+++ b/bin/scripts/expire-data/delete_custom_events.js
@@ -2,134 +2,137 @@
  *  Description: Deletes obsolete custom events (not received for >90 days) from Countly and Drill DBs for an app.
  *  Usage: node delete_x_days_inactive_events.js
  */
- const { ObjectId } = require('mongodb');
- const pluginManager = require('../../../plugins/pluginManager.js');
- const common = require('../../../api/utils/common.js');
- const drillCommon = require('../../../plugins/drill/api/common.js');
-//double-check below details before running the script
- const DRY_RUN = true; // Set to false to actually delete
- const APP_ID = ""; // Replace with your app's ObjectID string
- const DAYS = 90; // Number of days for active events
- function daysAgo(days) {
-     return Math.floor(Date.now() / 1000) - days * 24 * 60 * 60; // eventTimes uses Unix timestamps (seconds)
- }
- Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
-     console.log("Connected to databases...");
-     common.db = countlyDb;
-     common.drillDb = drillDb;
-     try {
-         const app = await countlyDb.collection("apps").findOne({_id: new ObjectId(APP_ID)}, {_id: 1, name: 1});
-         if (!app) {
-             console.log("App not found");
-             return close();
-         }
-         console.log("App:", app.name);
-         // Get eventTimes data
-         const eventTimesColl = "eventTimes" + APP_ID;
-         const allDocs = await countlyDb.collection(eventTimesColl).find({}).toArray();
- 
-         let obsoleteEvents = [];
-         if (allDocs.length) {
-             for (const doc of allDocs) {
-                 if (doc.e && Array.isArray(doc.e)) {
-                     for (const eventObj of doc.e) {
-                         // eventObj.e is event key, eventObj.ts is last timestamp (Unix seconds)
-                         if (eventObj.t && Math.floor(eventObj.t / 1000) < daysAgo(DAYS)) {
-                             obsoleteEvents.push(eventObj.e);
-                         }
-                     }
-                 }
-             }
-         }
-         obsoleteEvents = [...new Set(obsoleteEvents)];
-         console.log("Obsolete events to delete:", obsoleteEvents);
-         if (DRY_RUN) {
-             console.log("DRY_RUN enabled. No changes will be made.");
-             return close();
-         }
-         if (!obsoleteEvents.length) {
-             return close("No obsolete events to delete");
-         }
-         // Delete obsolete events
-         await deleteDrillEvents(app._id, obsoleteEvents);
-         await deleteCountlyEvents(app._id, obsoleteEvents);
-         await deleteEventTimes(app._id, obsoleteEvents);
-         await deleteEventGroups(app._id, obsoleteEvents);
-         await deleteEventKeys(app._id, obsoleteEvents);
-         close();
- 
-     } catch (err) {
-         console.log("App not found or error occurred");
-         close(err);
-     }
-     async function deleteDrillEvents(appId, events) {
-         for (let i = 0; i < events.length; i++) {
-             var collectionName = drillCommon.getCollectionName(events[i], appId);
-             try {
-                 await drillDb.collection(collectionName).drop();
-                 console.log("Dropped Drill collection:", collectionName);
-             } catch (ex) {
-                 // Collection may not exist
-                 console.log("Could not drop collection (may not exist):", collectionName);
-             }
-         }
-         //delete from aggregated drill event collection
-         await drillDb.collection('drill_events').deleteMany({'a': appId + "", 'e': {$in: events}});
-         console.log("Cleared from drill_events");
-         await drillDb.collection('drill_bookmarks').deleteMany({'app_id': appId, 'event_key': {$in: events}});
-         console.log("Cleared drill_bookmarks");
-         await drillDb.collection("drill_meta").deleteMany({'app_id': (appId + ""), "type": "e", "e": {$in: events}});
-         console.log("Cleared drill_meta");
-     }
-     async function deleteCountlyEvents(appId, events) {
-         for (let i = 0; i < events.length; i++) {
-             var collectionName = 'events' + drillCommon.getEventHash(events[i], appId);
-             try {
-                 await countlyDb.collection(collectionName).drop();
-                 console.log("Dropped Countly collection:", collectionName);
-             } catch (ex) {
-                 // Collection may not exist
-                 console.log("Could not drop collection (may not exist):", collectionName);
-             }
-             //clear from merged collection
-             await countlyDb.collection("events_data").deleteMany({'_id': {"$regex": "^" + appId + "_" + drillCommon.getEventHash(events[i], appId) + "_.*"}});
-             console.log("Cleared from aggregated events_data for:", events[i]);
-         }
-     }
-     async function deleteEventTimes(appId, events) {
-         await countlyDb.collection("eventTimes" + appId).updateMany({}, {"$pull": {"e": {"e": {$in: events}}}});
-         console.log("Cleared eventTimes");
-         await countlyDb.collection("timelineStatus").deleteMany({'app_id': (appId + ""), "event": {$in: events}});
-         console.log("Cleared timelineStatus");
-     }
-     async function deleteEventKeys(appId, events) {
-         const unsetQuery = {};
-         for (let i = 0; i < events.length; i++) {
-             unsetQuery[`segments.${events[i]}`] = "";
-             unsetQuery[`map.${events[i]}`] = "";
-             unsetQuery[`omitted_segments.${events[i]}`] = "";
-         }
-         // Remove from "list" and "overview" arrays, and unset event segments/maps
-         await countlyDb.collection("events").updateOne(
-             {_id: appId},
-             {
-                 $pull: {
-                     list: {$in: events},
-                     overview: {eventKey: {$in: events}}
-                 },
-                 $unset: unsetQuery
-             }
-         );
-         console.log("Cleared events:", events);
-     }
-     async function deleteEventGroups(appId, events) {
-         await countlyDb.collection("event_groups").updateMany({'app_id': (appId + "")}, {"$pull": {"source_events": {$in: events}}});
-         console.log("Cleared event_groups");
-     }
-     function close(err) {
-         if (err) console.log("Finished with errors:", err);
-         else console.log("Finished successfully.");
-         countlyDb.close();
-         drillDb.close();
-     }
- });
+const { ObjectId } = require('mongodb');
+const pluginManager = require('../../../plugins/pluginManager.js');
+const common = require('../../../api/utils/common.js');
+const drillCommon = require('../../../plugins/drill/api/common.js');
+// double-check below details before running the script
+const DRY_RUN = true; // Set to false to actually delete
+const APP_ID = ""; // Replace with your app's ObjectID string
+const DAYS = 90; // Number of days for active events
+function daysAgo(days) {
+    // eventTimes uses Unix timestamps (seconds)
+    return Math.floor(Date.now() / 1000) - days * 24 * 60 * 60;
+}
+Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]){
+    console.log("Connected to databases...");
+    common.db = countlyDb;
+    common.drillDb = drillDb;
+    try {
+        const app = await countlyDb.collection("apps").findOne({ _id: new ObjectId(APP_ID) }, { _id: 1, name: 1 });
+        if (!app) {
+            console.log("App not found");
+            return close();
+        }
+        console.log("App:", app.name);
+        // Get eventTimes data
+        const eventTimesColl = "eventTimes" + APP_ID;
+        const allDocs = await countlyDb.collection(eventTimesColl).find({}).toArray();
+
+        let obsoleteEvents = [];
+        if (allDocs.length) {
+            for (const doc of allDocs) {
+                if (doc.e && Array.isArray(doc.e)) {
+                    for (const eventObj of doc.e) {
+                        // eventObj.e is event key, eventObj.ts is last timestamp (Unix seconds)
+                        if (eventObj.t && Math.floor(eventObj.t / 1000) < daysAgo(DAYS)) {
+                            obsoleteEvents.push(eventObj.e);
+                        }
+                    }
+                }
+            }
+        }
+        obsoleteEvents = [...new Set(obsoleteEvents)];
+        console.log("Obsolete events to delete:", obsoleteEvents);
+        if (DRY_RUN) {
+            console.log("DRY_RUN enabled. No changes will be made.");
+            return close();
+        }
+        if (!obsoleteEvents.length) {
+            return close("No obsolete events to delete");
+        }
+        // Delete obsolete events
+        await deleteDrillEvents(app._id, obsoleteEvents);
+        await deleteCountlyEvents(app._id, obsoleteEvents);
+        await deleteEventTimes(app._id, obsoleteEvents);
+        await deleteEventGroups(app._id, obsoleteEvents);
+        await deleteEventKeys(app._id, obsoleteEvents);
+        close();
+    } catch (err) {
+        console.log("App not found or error occurred");
+        close(err);
+    }
+    async function deleteDrillEvents(appId, events) {
+        for (let i = 0; i < events.length; i++) {
+            var collectionName = drillCommon.getCollectionName(events[i], appId);
+            try {
+                await drillDb.collection(collectionName).drop();
+                console.log("Dropped Drill collection:", collectionName);
+            } catch (ex) {
+                // Collection may not exist
+                console.log("Could not drop collection (may not exist):", collectionName);
+            }
+        }
+        // delete from aggregated drill event collection
+        await drillDb.collection('drill_events').deleteMany({ 'a': appId + "", 'e': { $in: events } });
+        console.log("Cleared from drill_events");
+        await drillDb.collection('drill_bookmarks').deleteMany({ 'app_id': appId, 'event_key': { $in: events } });
+        console.log("Cleared drill_bookmarks");
+        await drillDb.collection("drill_meta").deleteMany({ 'app_id': (appId + ""), "type": "e", "e": { $in: events } });
+        console.log("Cleared drill_meta");
+    }
+    async function deleteCountlyEvents(appId, events) {
+        for (let i = 0; i < events.length; i++) {
+            var collectionName = 'events' + drillCommon.getEventHash(events[i], appId);
+            try {
+                await countlyDb.collection(collectionName).drop();
+                console.log("Dropped Countly collection:", collectionName);
+            } catch (ex) {
+                // Collection may not exist
+                console.log("Could not drop collection (may not exist):", collectionName);
+            }
+            // clear from merged collection
+            await countlyDb.collection("events_data").deleteMany({ '_id': { "$regex": "^" + appId + "_" + drillCommon.getEventHash(events[i], appId) + "_.*" } });
+            console.log("Cleared from aggregated events_data for:", events[i]);
+        }
+    }
+    async function deleteEventTimes(appId, events) {
+        await countlyDb.collection("eventTimes" + appId).updateMany({}, { "$pull": { "e": { "e": { $in: events } } } });
+        console.log("Cleared eventTimes");
+        await countlyDb.collection("timelineStatus").deleteMany({ 'app_id': (appId + ""), "event": { $in: events } });
+        console.log("Cleared timelineStatus");
+    }
+    async function deleteEventKeys(appId, events) {
+        const unsetQuery = {};
+        for (let i = 0; i < events.length; i++) {
+            unsetQuery[`segments.${events[i]}`] = "";
+            unsetQuery[`map.${events[i]}`] = "";
+            unsetQuery[`omitted_segments.${events[i]}`] = "";
+        }
+        // Remove from "list" and "overview" arrays, and unset event segments/maps
+        await countlyDb.collection("events").updateOne(
+            { _id: appId },
+            {
+                $pull: {
+                    list: { $in: events },
+                    overview: { eventKey: { $in: events } }
+                },
+                $unset: unsetQuery
+            }
+        );
+        console.log("Cleared events:", events);
+    }
+    async function deleteEventGroups(appId, events) {
+        await countlyDb.collection("event_groups").updateMany({ 'app_id': (appId + "") }, { "$pull": { "source_events": { $in: events } } });
+        console.log("Cleared event_groups");
+    }
+    function close(err) {
+        if (err) {
+            console.log("Finished with errors:", err);
+        } else {
+            console.log("Finished successfully.");
+        }
+        countlyDb.close();
+        drillDb.close();
+    }
+});