Skip to content

Update delete_custom_events.js to delete_x_days_inactive_events #6851

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
+82 −88
Closed

Update delete_custom_events.js to delete_x_days_inactive_events #6851

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5b5c53a
Update delete_custom_events.js to delete_x_days_inactive_events
mrmeghana Oct 24, 2025
a9e73c9
Update delete_custom_events.js
mrmeghana Oct 24, 2025
becac0b
Merge branch 'master' into mrmeghana-patch-8
ar2rsawseen Oct 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 82 additions & 88 deletions bin/scripts/expire-data/delete_custom_events.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,141 +1,135 @@
/**
* Description: Deletes custom events from countly and drill databases
* Server: countly
* Path: $(countly dir)/bin/scripts/expire-data
* Command: node delete_custom_events.js
* Description: Deletes obsolete custom events (not received for >90 days) from Countly and Drill DBs for an app.
* Usage: node delete_x_days_inactive_events.js
*/

const { ObjectId } = require('mongodb');
const pluginManager = require('../../../plugins/pluginManager.js');
const common = require('../../../api/utils/common.js');
const drillCommon = require('../../../plugins/drill/api/common.js');

const DRY_RUN = true;
const APP_ID = "";
const EVENTS = []; // If empty, no events will be deleted . The format has to be "event1","event2"

Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
// double-check below details before running the script
const DRY_RUN = true; // Set to false to actually delete
const APP_ID = ""; // Replace with your app's ObjectID string
const DAYS = 90; // Number of days for active events
function daysAgo(days) {
// eventTimes uses Unix timestamps (seconds)
return Math.floor(Date.now() / 1000) - days * 24 * 60 * 60;
}
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]){
console.log("Connected to databases...");

//SET COMMON DB AND DRILL DB
common.db = countlyDb;
common.drillDb = drillDb;
//GET APP
try {
const app = await countlyDb.collection("apps").findOne({_id: new ObjectId(APP_ID)}, {_id: 1, name: 1});
const app = await countlyDb.collection("apps").findOne({ _id: new ObjectId(APP_ID) }, { _id: 1, name: 1 });
if (!app) {
console.log("App not found");
return close();
}
console.log("App:", app.name);
//GET EVENTS
let events = EVENTS;
if (!events.length) {
console.log("Fetching events from drill_meta...");
const metaEvents = await drillDb.collection("drill_meta").aggregate([
{
$match: {
'app_id': app._id + "",
"type": "e",
"e": { $nin: events }
}
},
{
$group: {
_id: "$e"
}
},
{
$project: {
_id: 0,
e: "$_id"
// Get eventTimes data
const eventTimesColl = "eventTimes" + APP_ID;
const allDocs = await countlyDb.collection(eventTimesColl).find({}).toArray();

let obsoleteEvents = [];
if (allDocs.length) {
for (const doc of allDocs) {
if (doc.e && Array.isArray(doc.e)) {
for (const eventObj of doc.e) {
// eventObj.e is event key, eventObj.ts is last timestamp (Unix seconds)
if (eventObj.t && Math.floor(eventObj.t / 1000) < daysAgo(DAYS)) {
obsoleteEvents.push(eventObj.e);
}
}
}
]).toArray();
events = metaEvents.map(e => e.e);
}
}
console.log("Events to delete:", events);
obsoleteEvents = [...new Set(obsoleteEvents)];
console.log("Obsolete events to delete:", obsoleteEvents);
if (DRY_RUN) {
console.log("DRY_RUN enabled. No changes will be made.");
return close();
}
if (!events.length) {
return close("No events to delete");
}
try {
//DELETE EVENTS
console.log(1 + ") Deleting drill events:");
await deleteDrillEvents(app._id, events);
console.log(2 + ") Deleting countly events:");
await deleteCountlyEvents(app._id, events);
console.log(3 + ") Deleting event times:");
await deleteEventTimes(app._id, events);
console.log(4 + ") Deleting event groups:");
await deleteEventGroups(app._id, events);
console.log(5 + ") Deleting event keys:");
await deleteEventKeys(app._id, events);
close();
if (!obsoleteEvents.length) {
return close("No obsolete events to delete");
}
catch (err) {
close(err);
}
}
catch (err) {
console.log("App not found");
// Delete obsolete events
await deleteDrillEvents(app._id, obsoleteEvents);
await deleteCountlyEvents(app._id, obsoleteEvents);
await deleteEventTimes(app._id, obsoleteEvents);
await deleteEventGroups(app._id, obsoleteEvents);
await deleteEventKeys(app._id, obsoleteEvents);
close();
} catch (err) {
console.log("App not found or error occurred");
close(err);
}

async function deleteDrillEvents(appId, events) {
for (let i = 0; i < events.length; i++) {
var collectionName = drillCommon.getCollectionName(events[i], appId);
await drillDb.collection(collectionName).drop();
console.log("Dropped collection:", collectionName);
try {
await drillDb.collection(collectionName).drop();
console.log("Dropped Drill collection:", collectionName);
} catch (ex) {
// Collection may not exist
console.log("Could not drop collection (may not exist):", collectionName);
}
}
//delete from aggregated drill event collection
await drillDb.collection('drill_events').remove({'a': appId + "", 'e': {$in: events}});
// delete from aggregated drill event collection
await drillDb.collection('drill_events').deleteMany({ 'a': appId + "", 'e': { $in: events } });
console.log("Cleared from drill_events");
await drillDb.collection('drill_bookmarks').remove({'app_id': appId, 'event_key': {$in: events}});
await drillDb.collection('drill_bookmarks').deleteMany({ 'app_id': appId, 'event_key': { $in: events } });
console.log("Cleared drill_bookmarks");
await drillDb.collection("drill_meta").remove({'app_id': (appId + ""), "type": "e", "e": {$in: events}});
await drillDb.collection("drill_meta").deleteMany({ 'app_id': (appId + ""), "type": "e", "e": { $in: events } });
console.log("Cleared drill_meta");
}

async function deleteCountlyEvents(appId, events) {
for (let i = 0; i < events.length; i++) {
var collectionName = 'events' + drillCommon.getEventHash(events[i], appId);
await countlyDb.collection(collectionName).drop();
console.log("Dropped collection:", collectionName);

//clear from merged collection
await countlyDb.collection("events_data").remove({'_id': {"$regex": "^" + appId + "_" + drillCommon.getEventHash(events[i], appId) + "_.*"}});
console.log("Cleared from agregated collection");
try {
await countlyDb.collection(collectionName).drop();
console.log("Dropped Countly collection:", collectionName);
} catch (ex) {
// Collection may not exist
console.log("Could not drop collection (may not exist):", collectionName);
}
// clear from merged collection
await countlyDb.collection("events_data").deleteMany({ '_id': { "$regex": "^" + appId + "_" + drillCommon.getEventHash(events[i], appId) + "_.*" } });
console.log("Cleared from aggregated events_data for:", events[i]);
}
}

async function deleteEventTimes(appId, events) {
await countlyDb.collection("eventTimes" + appId).update({}, {"$pull": {"e": {"e": {$in: events}}}}, {"multi": true});
await countlyDb.collection("eventTimes" + appId).updateMany({}, { "$pull": { "e": { "e": { $in: events } } } });
console.log("Cleared eventTimes");
await countlyDb.collection("timelineStatus").remove({'app_id': (appId + ""), "event": {$in: events}});
await countlyDb.collection("timelineStatus").deleteMany({ 'app_id': (appId + ""), "event": { $in: events } });
console.log("Cleared timelineStatus");
}

async function deleteEventKeys(appId, events) {
const unsetQuery = {};
for (let i = 0; i < events.length; i++) {
unsetQuery[`segments.${events[i]}`] = 1;
unsetQuery[`map.${events[i]}`] = 1;
unsetQuery[`omitted_segments.${events[i]}`] = 1;
unsetQuery[`segments.${events[i]}`] = "";
unsetQuery[`map.${events[i]}`] = "";
unsetQuery[`omitted_segments.${events[i]}`] = "";
}
await countlyDb.collection("events").updateOne({_id: appId}, {$pull: {list: {$in: events}}, $unset: unsetQuery}, {$pull: {"overview": {eventKey: {$in: events}}}});
// Remove from "list" and "overview" arrays, and unset event segments/maps
await countlyDb.collection("events").updateOne(
{ _id: appId },
{
$pull: {
list: { $in: events },
overview: { eventKey: { $in: events } }
},
$unset: unsetQuery
}
);
console.log("Cleared events:", events);
}

async function deleteEventGroups(appId, events) {
await countlyDb.collection("event_groups").update({'app_id': (appId + "")}, {"$pull": {"source_events": {$in: events}}}, {"multi": true});
await countlyDb.collection("event_groups").updateMany({ 'app_id': (appId + "") }, { "$pull": { "source_events": { $in: events } } });
console.log("Cleared event_groups");
}

function close(err) {
if (err) {
console.log("Finished with errors:", err);
}
else {
} else {
console.log("Finished successfully.");
}
countlyDb.close();
Expand Down
Loading