Skip to content

Added changes to scripts in /expire-data/ to deal with new merged collections #5599

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Sep 25, 2024
Merged

Added changes to scripts in /expire-data/ to deal with new merged collections #5599

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
df636e0
Added changes to # scripts in /expire-data/ to deal with new merged c…
Cookiezaurs Sep 24, 2024
82758cc
changes for /expire-data folder
Cookiezaurs Sep 24, 2024
aa75079
Merge branch 'next' into anna/next
Cookiezaurs Sep 24, 2024
ad5340c
Merge branch 'next' into anna/next
Cookiezaurs Sep 25, 2024
2cf3532
eslint
Cookiezaurs Sep 25, 2024
0ef7c03
Changes for countly_single_app_expireDataBatches.js
Cookiezaurs Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 53 additions & 55 deletions bin/scripts/expire-data/countly_multi_app_expireData.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,73 +8,71 @@
var EXPIRE_AFTER = 60 * 60 * 24 * 365; //1 year in seconds
var INDEX_NAME = "cd_1";

var async = require('async'),
Promise = require("bluebird"),
var Promise = require("bluebird"),
plugins = require('../../../plugins/pluginManager.js');

//var db = plugins.dbConnection("countly");
//var db_drill = plugins.dbConnection("countly_drill");

Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_drill")]).spread(function(db, db_drill) {
db_drill.collections(function(err, collections) {
collections = collections || [];
function eventIterator(coll, done) {
var collection = coll.collectionName;
console.log("processing", collection);
db_drill.collection(collection).indexes(function(err, indexes) {
if (err) {
console.log(err);
}
if (!err && indexes) {
var hasIndex = false;
var dropIndex = false;
for (var i = 0; i < indexes.length; i++) {
if (indexes[i].name == INDEX_NAME) {
if (indexes[i].expireAfterSeconds == EXPIRE_AFTER) {
//print("skipping", c)
hasIndex = true;
}
//has index but incorrect expire time, need to be reindexed
else {
dropIndex = true;
}
break;
}
}
if (dropIndex) {
console.log("modifying index", collection);
db_drill.command({
"collMod": collection,
"index": {
"keyPattern": {"cd": 1},
expireAfterSeconds: EXPIRE_AFTER
}
}, function(err) {
if (err) {
console.log(err);
}
done();
});

var collection = "drill_events";
console.log("processing", collection);
db_drill.collection(collection).indexes(function(err, indexes) {
if (err) {
console.log(err);
}
if (!err && indexes) {
var hasIndex = false;
var dropIndex = false;
for (var i = 0; i < indexes.length; i++) {
if (indexes[i].name == INDEX_NAME) {
if (indexes[i].expireAfterSeconds == EXPIRE_AFTER) {
//print("skipping", c)
hasIndex = true;
}
else if (!hasIndex) {
console.log("creating index", collection);
db_drill.collection(collection).createIndex({"cd": 1}, {expireAfterSeconds: EXPIRE_AFTER, "background": true}, function() {
done();
});
}
//has index but incorrect expire time, need to be reindexed
else {
done();
dropIndex = true;
}
break;
}
else {
}
if (dropIndex) {
console.log("modifying index", collection);
db_drill.command({
"collMod": collection,
"index": {
"keyPattern": {"cd": 1},
expireAfterSeconds: EXPIRE_AFTER
}
}, function(err) {
if (err) {
console.log(err);
}
done();
}
});
});

}
else if (!hasIndex) {
console.log("creating index", collection);
db_drill.collection(collection).createIndex({"cd": 1}, {expireAfterSeconds: EXPIRE_AFTER, "background": true}, function() {
done();
});
}
else {
done();
}
}
else {
done();
}
async.eachSeries(collections, eventIterator, function() {
db.close();
db_drill.close();
});
});


function done() {
db.close();
db_drill.close();
}

});
44 changes: 9 additions & 35 deletions bin/scripts/expire-data/countly_single_app_expireDataBatches.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ var timeout = 500; //timeout in miliseconds between deletion. (One second ====


var async = require('async'),
crypto = require('crypto'),
Promise = require("bluebird"),
plugins = require('../../../plugins/pluginManager.js');

Expand Down Expand Up @@ -109,6 +108,9 @@ function eventIterator(fr, done) {
console.log('Processing range: ' + JSON.stringify({"ts": {"$gte": fr.start, "$lt": fr.end}}) + ' for ' + fr.collection);
var query = {};
query["ts"] = {"$gte": fr.start, "$lt": fr.end};
if (collection === 'drill_events') {
query["a"] = APP_ID;
}
if (fr.query) {
for (var key in fr.query) {
query[key] = fr.query.key;
Expand Down Expand Up @@ -242,41 +244,13 @@ function processDrillCollection(collection, seconds, callback) {
}
}

function processDrillCollections(db, drill_db, callback) {
function processDrillCollections(drill_db, callback) {
if (process && process.drill_events) {
var collections = [];
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_session" + APP_ID).digest('hex')});
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_crash" + APP_ID).digest('hex')});
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_view" + APP_ID).digest('hex')});
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_action" + APP_ID).digest('hex')});
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_apm_device" + APP_ID).digest('hex')});
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_apm_network" + APP_ID).digest('hex')});
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_nps" + APP_ID).digest('hex')});
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_survey" + APP_ID).digest('hex')});
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_push_action" + APP_ID).digest('hex')});
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_star_rating" + APP_ID).digest('hex')});
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_push_sent" + APP_ID).digest('hex')});
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_consent" + APP_ID).digest('hex')});
db.collection("events").findOne({'_id': db.ObjectID(APP_ID)}, {list: 1}, function(err, eventData) {
if (eventData && eventData.list) {
for (var i = 0; i < eventData.list.length; i++) {
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update(eventData.list[i] + APP_ID).digest('hex')});
}
processDrillCollection({"collection": "drill_events", db: drill_db}, false, function(err) {
if (err) {
console.log("ERROR: Error while processing drill collection: drill_events");
}

async.eachSeries(collections, function(collection, done) {
processDrillCollection(collection, false, function(err) {
if (err) {
console.log("ERROR: Error while processing drill collection: " + collection.collection);
}
done(err);
});
}, function(err) {
if (err) {
console.log("ERROR: Error processing collections.");
}
callback(err);
});
callback(err);
});
}
else {
Expand All @@ -292,7 +266,7 @@ Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_dril
db_drill.close();
}
else {
processDrillCollections(db, db_drill, function() {
processDrillCollections(db_drill, function() {
var processCols = [];
for (var key in process) {
if (key !== 'drill_events') {
Expand Down
28 changes: 26 additions & 2 deletions bin/scripts/expire-data/db_free_storage_calculation.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,32 @@ Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_dril
done();
});
}, function() {
console.log(result);
close();
//checking aggregated data collection:
drill.collection("drill_events").stats({scale: SCALE}, function(err, stats) {
if (stats.count > 0) {
var avg = stats.storageSize / stats.count;
result.available_for_reuse += stats.freeStorageSize || 0;
drill.collection("drill_events").find({cd: {$lt: new Date(Date.now() - (DATA_RETENTION_IN_SECONDS * 1000))}}).count(function(err, count) {
if (err) {
console.log(err);
}
if (count > 0) {
result.count += count;
result.total_will_be_free += Math.round(avg * count);
result.drill_events = {};
result.drill_events.available_for_reuse = stats.freeStorageSize || 0;
result.drill_events.total_will_be_free = Math.round(avg * count);
result.drill_events.count = count;
}
console.log("Result: ", result);
close();
});
}
else {
console.log(result);
close();
}
});
});
});
});
Expand Down
7 changes: 7 additions & 0 deletions bin/scripts/expire-data/delete_custom_events.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
await drillDb.collection(collectionName).drop();
console.log("Dropped collection:", collectionName);
}
//delete from aggregated drill event collection
await drillDb.collection('drill_events').remove({'a': appId + "", 'e': {$in: events}});
console.log("Cleared from drill_events");
await drillDb.collection('drill_bookmarks').remove({'app_id': appId, 'event_key': {$in: events}});
console.log("Cleared drill_bookmarks");
await drillDb.collection("drill_meta").remove({'app_id': (appId + ""), "type": "e", "e": {$in: events}});
Expand All @@ -72,6 +75,10 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
var collectionName = 'events' + drillCommon.getEventHash(events[i], appId);
await countlyDb.collection(collectionName).drop();
console.log("Dropped collection:", collectionName);

//clear from merged collection
await countlyDb.collection("events_data").remove({'_id': {"$regex": "^" + appId + "_" + drillCommon.getEventHash(events[i], appId) + "_.*"}});
console.log("Cleared from agregated collection");
}
}

Expand Down
5 changes: 5 additions & 0 deletions bin/scripts/expire-data/delete_custom_events_regex.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
await drillDb.collection(collectionName).drop();
console.log("Dropped collection:", collectionName);
}
await drillDb.collection('drill_events').remove({'a': appId + "", 'e': {$in: events}});
console.log("Cleared from drill_events");
await drillDb.collection('drill_bookmarks').remove({'app_id': appId, 'event_key': {$in: events}});
console.log("Cleared drill_bookmarks");
await drillDb.collection("drill_meta").remove({'app_id': (appId + ""), "type": "e", "e": {$in: events}});
Expand All @@ -103,6 +105,9 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
var collectionName = 'events' + drillCommon.getEventHash(events[i], appId);
await countlyDb.collection(collectionName).drop();
console.log("Dropped collection:", collectionName);
//clear from merged collection
await countlyDb.collection("events_data").remove({'_id': {"$regex": "^" + appId + "_" + drillCommon.getEventHash(events[i], appId) + "_.*"}});
console.log("Cleared from agregated collection");
}
}

Expand Down
51 changes: 51 additions & 0 deletions bin/scripts/expire-data/mongo_expireDataPeriod.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/**
* Deletes data in drill based on period
* Modify start and end before running script.
* Path: any
* Command: mongo < mongo_expireDataPeriod.js
*/

var COUNTLY_DRILL = 'countly_drill',
start = 1157738337000, //min timestamp
end = 1657738337000; //max timestamp

var PROCESS = [
/^drill_events\.*/
];

/* global Mongo, print*/

var conn = new Mongo();

var query = {"ts": {"$gte": start, "$lt": end}};
/**
// Enable for auth

var authDB = conn.getDB('admin');
authDB.auth('<username>', '<password>');

**/

var drill = conn.getDB(COUNTLY_DRILL);

var collections = drill.getCollectionNames();

collections.forEach(function(c) {
var process = false;
PROCESS.forEach(function(r) {
if (typeof r === 'string' && r === c) {
process = true;
}
else if (typeof r === 'object' && r.test(c)) {
process = true;
}
});
if (process) {
print('clearing data from collection:' + c);
var rr = drill.getCollection(c).remove(query);
print(JSON.stringify(rr));
}
});

print('Done with cleanup');

Loading
Loading