Skip to content

Commit ec433a1

Browse files
CookiezaursCookiezaurs
authored
Merge pull request #5599 from Countly/anna/next
Added changes to scripts in /expire-data/ to deal with new merged collections
2 parents 201c41b + 0ef7c03 commit ec433a1

10 files changed

+564
-92
lines changed

‎bin/scripts/expire-data/countly_multi_app_expireData.js‎

Lines changed: 53 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -8,73 +8,71 @@
88
var EXPIRE_AFTER = 60 * 60 * 24 * 365; //1 year in seconds
99
var INDEX_NAME = "cd_1";
1010

11-
var async = require('async'),
12-
Promise = require("bluebird"),
11+
var Promise = require("bluebird"),
1312
plugins = require('../../../plugins/pluginManager.js');
1413

1514
//var db = plugins.dbConnection("countly");
1615
//var db_drill = plugins.dbConnection("countly_drill");
1716

1817
Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_drill")]).spread(function(db, db_drill) {
19-
db_drill.collections(function(err, collections) {
20-
collections = collections || [];
21-
function eventIterator(coll, done) {
22-
var collection = coll.collectionName;
23-
console.log("processing", collection);
24-
db_drill.collection(collection).indexes(function(err, indexes) {
25-
if (err) {
26-
console.log(err);
27-
}
28-
if (!err && indexes) {
29-
var hasIndex = false;
30-
var dropIndex = false;
31-
for (var i = 0; i < indexes.length; i++) {
32-
if (indexes[i].name == INDEX_NAME) {
33-
if (indexes[i].expireAfterSeconds == EXPIRE_AFTER) {
34-
//print("skipping", c)
35-
hasIndex = true;
36-
}
37-
//has index but incorrect expire time, need to be reindexed
38-
else {
39-
dropIndex = true;
40-
}
41-
break;
42-
}
43-
}
44-
if (dropIndex) {
45-
console.log("modifying index", collection);
46-
db_drill.command({
47-
"collMod": collection,
48-
"index": {
49-
"keyPattern": {"cd": 1},
50-
expireAfterSeconds: EXPIRE_AFTER
51-
}
52-
}, function(err) {
53-
if (err) {
54-
console.log(err);
55-
}
56-
done();
57-
});
5818

19+
var collection = "drill_events";
20+
console.log("processing", collection);
21+
db_drill.collection(collection).indexes(function(err, indexes) {
22+
if (err) {
23+
console.log(err);
24+
}
25+
if (!err && indexes) {
26+
var hasIndex = false;
27+
var dropIndex = false;
28+
for (var i = 0; i < indexes.length; i++) {
29+
if (indexes[i].name == INDEX_NAME) {
30+
if (indexes[i].expireAfterSeconds == EXPIRE_AFTER) {
31+
//print("skipping", c)
32+
hasIndex = true;
5933
}
60-
else if (!hasIndex) {
61-
console.log("creating index", collection);
62-
db_drill.collection(collection).createIndex({"cd": 1}, {expireAfterSeconds: EXPIRE_AFTER, "background": true}, function() {
63-
done();
64-
});
65-
}
34+
//has index but incorrect expire time, need to be reindexed
6635
else {
67-
done();
36+
dropIndex = true;
6837
}
38+
break;
6939
}
70-
else {
40+
}
41+
if (dropIndex) {
42+
console.log("modifying index", collection);
43+
db_drill.command({
44+
"collMod": collection,
45+
"index": {
46+
"keyPattern": {"cd": 1},
47+
expireAfterSeconds: EXPIRE_AFTER
48+
}
49+
}, function(err) {
50+
if (err) {
51+
console.log(err);
52+
}
7153
done();
72-
}
73-
});
54+
});
55+
56+
}
57+
else if (!hasIndex) {
58+
console.log("creating index", collection);
59+
db_drill.collection(collection).createIndex({"cd": 1}, {expireAfterSeconds: EXPIRE_AFTER, "background": true}, function() {
60+
done();
61+
});
62+
}
63+
else {
64+
done();
65+
}
66+
}
67+
else {
68+
done();
7469
}
75-
async.eachSeries(collections, eventIterator, function() {
76-
db.close();
77-
db_drill.close();
78-
});
7970
});
71+
72+
73+
function done() {
74+
db.close();
75+
db_drill.close();
76+
}
77+
8078
});

‎bin/scripts/expire-data/countly_single_app_expireDataBatches.js‎

Lines changed: 9 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ var timeout = 500; //timeout in miliseconds between deletion. (One second ====
3939

4040

4141
var async = require('async'),
42-
crypto = require('crypto'),
4342
Promise = require("bluebird"),
4443
plugins = require('../../../plugins/pluginManager.js');
4544

@@ -109,6 +108,9 @@ function eventIterator(fr, done) {
109108
console.log('Processing range: ' + JSON.stringify({"ts": {"$gte": fr.start, "$lt": fr.end}}) + ' for ' + fr.collection);
110109
var query = {};
111110
query["ts"] = {"$gte": fr.start, "$lt": fr.end};
111+
if (collection === 'drill_events') {
112+
query["a"] = APP_ID;
113+
}
112114
if (fr.query) {
113115
for (var key in fr.query) {
114116
query[key] = fr.query.key;
@@ -242,41 +244,13 @@ function processDrillCollection(collection, seconds, callback) {
242244
}
243245
}
244246

245-
function processDrillCollections(db, drill_db, callback) {
247+
function processDrillCollections(drill_db, callback) {
246248
if (process && process.drill_events) {
247-
var collections = [];
248-
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_session" + APP_ID).digest('hex')});
249-
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_crash" + APP_ID).digest('hex')});
250-
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_view" + APP_ID).digest('hex')});
251-
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_action" + APP_ID).digest('hex')});
252-
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_apm_device" + APP_ID).digest('hex')});
253-
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_apm_network" + APP_ID).digest('hex')});
254-
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_nps" + APP_ID).digest('hex')});
255-
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_survey" + APP_ID).digest('hex')});
256-
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_push_action" + APP_ID).digest('hex')});
257-
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_star_rating" + APP_ID).digest('hex')});
258-
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_push_sent" + APP_ID).digest('hex')});
259-
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update("[CLY]_consent" + APP_ID).digest('hex')});
260-
db.collection("events").findOne({'_id': db.ObjectID(APP_ID)}, {list: 1}, function(err, eventData) {
261-
if (eventData && eventData.list) {
262-
for (var i = 0; i < eventData.list.length; i++) {
263-
collections.push({'db': drill_db, 'collection': "drill_events" + crypto.createHash('sha1').update(eventData.list[i] + APP_ID).digest('hex')});
264-
}
249+
processDrillCollection({"collection": "drill_events", db: drill_db}, false, function(err) {
250+
if (err) {
251+
console.log("ERROR: Error while processing drill collection: drill_events");
265252
}
266-
267-
async.eachSeries(collections, function(collection, done) {
268-
processDrillCollection(collection, false, function(err) {
269-
if (err) {
270-
console.log("ERROR: Error while processing drill collection: " + collection.collection);
271-
}
272-
done(err);
273-
});
274-
}, function(err) {
275-
if (err) {
276-
console.log("ERROR: Error processing collections.");
277-
}
278-
callback(err);
279-
});
253+
callback(err);
280254
});
281255
}
282256
else {
@@ -292,7 +266,7 @@ Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_dril
292266
db_drill.close();
293267
}
294268
else {
295-
processDrillCollections(db, db_drill, function() {
269+
processDrillCollections(db_drill, function() {
296270
var processCols = [];
297271
for (var key in process) {
298272
if (key !== 'drill_events') {

‎bin/scripts/expire-data/db_free_storage_calculation.js‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,32 @@ Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_dril
7575
done();
7676
});
7777
}, function() {
78-
console.log(result);
79-
close();
78+
//checking aggregated data collection:
79+
drill.collection("drill_events").stats({scale: SCALE}, function(err, stats) {
80+
if (stats.count > 0) {
81+
var avg = stats.storageSize / stats.count;
82+
result.available_for_reuse += stats.freeStorageSize || 0;
83+
drill.collection("drill_events").find({cd: {$lt: new Date(Date.now() - (DATA_RETENTION_IN_SECONDS * 1000))}}).count(function(err, count) {
84+
if (err) {
85+
console.log(err);
86+
}
87+
if (count > 0) {
88+
result.count += count;
89+
result.total_will_be_free += Math.round(avg * count);
90+
result.drill_events = {};
91+
result.drill_events.available_for_reuse = stats.freeStorageSize || 0;
92+
result.drill_events.total_will_be_free = Math.round(avg * count);
93+
result.drill_events.count = count;
94+
}
95+
console.log("Result: ", result);
96+
close();
97+
});
98+
}
99+
else {
100+
console.log(result);
101+
close();
102+
}
103+
});
80104
});
81105
});
82106
});

‎bin/scripts/expire-data/delete_custom_events.js‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
6161
await drillDb.collection(collectionName).drop();
6262
console.log("Dropped collection:", collectionName);
6363
}
64+
//delete from aggregated drill event collection
65+
await drillDb.collection('drill_events').remove({'a': appId + "", 'e': {$in: events}});
66+
console.log("Cleared from drill_events");
6467
await drillDb.collection('drill_bookmarks').remove({'app_id': appId, 'event_key': {$in: events}});
6568
console.log("Cleared drill_bookmarks");
6669
await drillDb.collection("drill_meta").remove({'app_id': (appId + ""), "type": "e", "e": {$in: events}});
@@ -72,6 +75,10 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
7275
var collectionName = 'events' + drillCommon.getEventHash(events[i], appId);
7376
await countlyDb.collection(collectionName).drop();
7477
console.log("Dropped collection:", collectionName);
78+
79+
//clear from merged collection
80+
await countlyDb.collection("events_data").remove({'_id': {"$regex": "^" + appId + "_" + drillCommon.getEventHash(events[i], appId) + "_.*"}});
81+
console.log("Cleared from agregated collection");
7582
}
7683
}
7784

‎bin/scripts/expire-data/delete_custom_events_regex.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,8 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
9292
await drillDb.collection(collectionName).drop();
9393
console.log("Dropped collection:", collectionName);
9494
}
95+
await drillDb.collection('drill_events').remove({'a': appId + "", 'e': {$in: events}});
96+
console.log("Cleared from drill_events");
9597
await drillDb.collection('drill_bookmarks').remove({'app_id': appId, 'event_key': {$in: events}});
9698
console.log("Cleared drill_bookmarks");
9799
await drillDb.collection("drill_meta").remove({'app_id': (appId + ""), "type": "e", "e": {$in: events}});
@@ -103,6 +105,9 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
103105
var collectionName = 'events' + drillCommon.getEventHash(events[i], appId);
104106
await countlyDb.collection(collectionName).drop();
105107
console.log("Dropped collection:", collectionName);
108+
//clear from merged collection
109+
await countlyDb.collection("events_data").remove({'_id': {"$regex": "^" + appId + "_" + drillCommon.getEventHash(events[i], appId) + "_.*"}});
110+
console.log("Cleared from agregated collection");
106111
}
107112
}
108113

‎bin/scripts/expire-data/mongo_expireDataPeriod.js‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/**
2+
* Deletes data in drill based on period
3+
* Modify start and end before running script.
4+
* Path: any
5+
* Command: mongo < mongo_expireDataPeriod.js
6+
*/
7+
8+
var COUNTLY_DRILL = 'countly_drill',
9+
start = 1157738337000, //min timestamp
10+
end = 1657738337000; //max timestamp
11+
12+
var PROCESS = [
13+
/^drill_events\.*/
14+
];
15+
16+
/* global Mongo, print*/
17+
18+
var conn = new Mongo();
19+
20+
var query = {"ts": {"$gte": start, "$lt": end}};
21+
/**
22+
// Enable for auth
23+
24+
var authDB = conn.getDB('admin');
25+
authDB.auth('<username>', '<password>');
26+
27+
**/
28+
29+
var drill = conn.getDB(COUNTLY_DRILL);
30+
31+
var collections = drill.getCollectionNames();
32+
33+
collections.forEach(function(c) {
34+
var process = false;
35+
PROCESS.forEach(function(r) {
36+
if (typeof r === 'string' && r === c) {
37+
process = true;
38+
}
39+
else if (typeof r === 'object' && r.test(c)) {
40+
process = true;
41+
}
42+
});
43+
if (process) {
44+
print('clearing data from collection:' + c);
45+
var rr = drill.getCollection(c).remove(query);
46+
print(JSON.stringify(rr));
47+
}
48+
});
49+
50+
print('Done with cleanup');
51+

0 commit comments

Comments
 (0)