Skip to content

Commit 895f84f

Browse files
cihadtekincihadtekin
authored
Merge branch 'next' into SER-1557
2 parents aba9395 + 736397e commit 895f84f

File tree

57 files changed

+2090
-526
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+2090
-526
lines changed

‎api/lib/countly.common.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1533,9 +1533,6 @@ countlyCommon.extractTwoLevelData = function(db, rangeArray, clearFunction, data
15331533
periodObj = countlyCommon.periodObj;
15341534
}
15351535

1536-
if (!rangeArray) {
1537-
return {"chartData": tableData};
1538-
}
15391536
var periodMin = 0,
15401537
periodMax = 0,
15411538
dataObj = {},
@@ -1544,6 +1541,10 @@ countlyCommon.extractTwoLevelData = function(db, rangeArray, clearFunction, data
15441541
propertyFunctions = underscore.pluck(dataProperties, "func"),
15451542
propertyValue = 0;
15461543

1544+
if (!rangeArray) {
1545+
return {"chartData": tableData};
1546+
}
1547+
15471548
if (!periodObj.isSpecialPeriod) {
15481549
periodMin = periodObj.periodMin;
15491550
periodMax = (periodObj.periodMax + 1);

‎api/parts/data/batcher.js‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ class WriteBatcher {
232232
updateOne: {
233233
filter: {_id: this.data[db][collection][key].id},
234234
update: this.data[db][collection][key].value,
235-
upsert: true
235+
upsert: this.data[db][collection][key].upsert
236236
}
237237
});
238238
}
@@ -340,18 +340,25 @@ class WriteBatcher {
340340
* @param {string} id - id of the document
341341
* @param {object} operation - operation
342342
* @param {string} db - name of the database for which to write data
343+
* @param {object=} options - options for operation ((upsert: false) - if you don't want to upsert document)
343344
*/
344-
add(collection, id, operation, db = "countly") {
345+
add(collection, id, operation, db = "countly", options) {
345346
if (!this.shared || cluster.isMaster) {
346347
if (!this.data[db][collection]) {
347348
this.data[db][collection] = {};
348349
}
349350
if (!this.data[db][collection][id]) {
350-
this.data[db][collection][id] = {id: id, value: operation};
351+
this.data[db][collection][id] = {id: id, value: operation, upsert: true};
352+
if (options && options.upsert === false) {
353+
this.data[db][collection][id].upsert = false;
354+
}
351355
batcherStats.update_queued++;
352356
}
353357
else {
354358
this.data[db][collection][id].value = common.mergeQuery(this.data[db][collection][id].value, operation);
359+
if (options && options.upsert === false) {
360+
this.data[db][collection][id].upsert = this.data[db][collection][id].upsert || false;
361+
}
355362
}
356363
if (!this.process) {
357364
this.flush(db, collection);

‎api/utils/taskmanager.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -607,6 +607,7 @@ taskmanager.checkResult = function(options, callback) {
607607
* @param {funciton} callback - callback for the result
608608
*/
609609
taskmanager.checkIfRunning = function(options, callback) {
610+
options = options || {};
610611
options.db = options.db || common.db;
611612
var query = {};
612613
if (options.id) {
@@ -982,6 +983,7 @@ taskmanager.rerunTask = function(options, callback) {
982983
reqData = {};
983984
}
984985
if (reqData.uri) {
986+
reqData.json = reqData.json || {};
985987
reqData.json.task_id = options.id;
986988
reqData.strictSSL = false;
987989
if (reqData.json && reqData.json.period && Array.isArray(reqData.json.period)) {

‎bin/scripts/fix-data/fix_drill_custom_props.js‎

Lines changed: 52 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -34,39 +34,65 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
3434
//PROCESS COLLECTIONS FOR EACH APP
3535
for (let i = 0; i < apps.length; i++) {
3636
console.log("Processing app: " + apps[i].name);
37-
var collectionName = drillCommon.getCollectionName("[CLY]_session", apps[i]._id + "");
38-
console.log("Processing collection: " + collectionName);
37+
var collectionName0 = drillCommon.getCollectionName("[CLY]_session", apps[i]._id + "");
3938

40-
const cursor = drillDb.collection(collectionName).find(query_drill, {"_id": 1, "custom": 1});
41-
//FOR EACH DOCUMENT
42-
var updates = [];
43-
while (await cursor.hasNext()) {
44-
var doc = await cursor.next();
45-
if (doc.custom) {
46-
var updateDoc = {};
47-
let updateMe = false;
48-
for (var key in doc.custom) {
49-
if (doc.custom[key] && typeof doc.custom[key] === "object") {
50-
var specialKeys = ["$set", "$addToSet", "$push", "$pull", "$inc", "$min", "$max", "$setOnce"];
51-
for (var z = 0; z < specialKeys.length; z++) {
52-
if (doc.custom[key][specialKeys[z]]) {
53-
updateDoc["custom." + key] = doc.custom[key][specialKeys[z]];
54-
updateMe = true;
39+
var collections = ["drill_events", collectionName0];
40+
41+
for (var z1 = 0; z1 < collections.length; z1++) {
42+
var collectionName = collections[z1];
43+
console.log("Processing collection: " + collectionName);
44+
45+
if (collectionName === "drill_events") {
46+
query_drill.a = apps[i]._id + "";
47+
query_drill.e = "[CLY]_session";
48+
}
49+
else {
50+
delete query_drill.a;
51+
delete query_drill.e;
52+
}
53+
54+
const cursor = drillDb.collection(collectionName).find(query_drill, {"_id": 1, "custom": 1});
55+
//FOR EACH DOCUMENT
56+
var updates = [];
57+
while (await cursor.hasNext()) {
58+
var doc = await cursor.next();
59+
if (doc.custom) {
60+
var updateDoc = {};
61+
let updateMe = false;
62+
for (var key in doc.custom) {
63+
if (doc.custom[key] && typeof doc.custom[key] === "object") {
64+
var specialKeys = ["$set", "$addToSet", "$push", "$pull", "$inc", "$min", "$max", "$setOnce"];
65+
for (var z = 0; z < specialKeys.length; z++) {
66+
if (doc.custom[key][specialKeys[z]]) {
67+
updateDoc["custom." + key] = doc.custom[key][specialKeys[z]];
68+
updateMe = true;
69+
}
5570
}
5671
}
5772
}
73+
if (updateMe) {
74+
updates.push({
75+
'updateOne': {
76+
'filter': { '_id': doc._id },
77+
'update': { '$set': updateDoc },
78+
'upsert': false
79+
}
80+
});
81+
}
5882
}
59-
if (updateMe) {
60-
updates.push({
61-
'updateOne': {
62-
'filter': { '_id': doc._id },
63-
'update': { '$set': updateDoc },
64-
'upsert': false
65-
}
66-
});
83+
if (updates.length === 500) {
84+
if (dry_run) {
85+
console.log("DRY RUN: Would update " + updates.length + " docs in " + collectionName);
86+
console.log(JSON.stringify(updates));
87+
}
88+
else {
89+
console.log("updating");
90+
await drillDb.collection(collectionName).bulkWrite(updates, {"ordered": false});
91+
}
92+
updates = [];
6793
}
6894
}
69-
if (updates.length === 500) {
95+
if (updates.length > 0) {
7096
if (dry_run) {
7197
console.log("DRY RUN: Would update " + updates.length + " docs in " + collectionName);
7298
console.log(JSON.stringify(updates));
@@ -78,16 +104,6 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
78104
updates = [];
79105
}
80106
}
81-
if (updates.length > 0) {
82-
if (dry_run) {
83-
console.log("DRY RUN: Would update " + updates.length + " docs in " + collectionName);
84-
console.log(JSON.stringify(updates));
85-
}
86-
else {
87-
await drillDb.collection(collectionName).bulkWrite(updates, {"ordered": false});
88-
}
89-
updates = [];
90-
}
91107
}
92108
}
93109
catch (err) {

‎bin/scripts/fix-data/fix_null_uids.js‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
4242
Promise.each(users, function(user) {
4343
return new Promise(function(resolve) {
4444
var device_id;
45-
var events = [];
45+
var events = [{"key": ""}];
4646
if (user.did) {
4747
device_id = user.did;
4848
}
@@ -91,7 +91,12 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
9191
}
9292
else {
9393
var collection = "drill_events" + crypto.createHash('sha1').update(event.key + app._id).digest('hex');
94-
drillDb.collection(collection).aggregate([{$match: {did: device_id, uid: {"$exists": true}}}, {"$sort": {"ts": -1}}, {$limit: 1}, {$project: {did: 1, uid: 1}}], {"allowDiskUse": true}, function(err, res) {
94+
var query = {did: device_id, uid: {"$exists": true}};
95+
if (event.key === "") {
96+
collection = "drill_events";
97+
query.a = app._id + "";
98+
}
99+
drillDb.collection(collection).aggregate([{$match: query}, {"$sort": {"ts": -1}}, {$limit: 1}, {$project: {did: 1, uid: 1}}], {"allowDiskUse": true}, function(err, res) {
95100
if (err) {
96101
console.log(err);
97102
resolve1();

‎bin/scripts/fix-data/recheck_merges.js‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
6969
}
7070

7171
async function getDrillCollections(app_id) {
72-
var collections = [];
72+
var collections = ["drill_events"];
7373
try {
7474
var events = await countlyDb.collection("events").findOne({_id: common.db.ObjectID(app_id)});
7575
var list = ["[CLY]_session", "[CLY]_crash", "[CLY]_view", "[CLY]_action", "[CLY]_push_action", "[CLY]_push_sent", "[CLY]_star_rating", "[CLY]_nps", "[CLY]_survey", "[CLY]_apm_network", "[CLY]_apm_device", "[CLY]_consent"];
@@ -97,15 +97,19 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
9797
for (let i = 0; i < collections.length; i++) {
9898
const collection = collections[i].collectionName;
9999
try {
100-
const events = await drillDb.collection(collection).find({uid: old_uid}, {uid: 1, _id: 0}).limit(1).toArray();
100+
var query = {uid: old_uid};
101+
if (collection === "drill_events") {
102+
query = {uid: old_uid, 'a': app._id + ""};
103+
}
104+
const events = await drillDb.collection(collection).find(query, {uid: 1, _id: 0}).limit(1).toArray();
101105
if (!events || !events.length) {
102106
continue;
103107
}
104108
if (events && events[0]) {
105109
console.log("Found at least one event with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "updating to new uid", new_uid);
106110
try {
107111
if (!DRY_RUN) {
108-
await drillDb.collection(collection).update({uid: old_uid}, {'$set': {uid: new_uid}}, {multi: true});
112+
await drillDb.collection(collection).update(query, {'$set': {uid: new_uid}}, {multi: true});
109113
}
110114
}
111115
catch (err) {

‎bin/scripts/fix-data/recheck_merges_new.js‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
5151
}
5252

5353
async function getDrillCollections(app_id) {
54-
var collections = [];
54+
var collections = ["drill_events"];
5555
try {
5656
var events = await countlyDb.collection("events").findOne({_id: common.db.ObjectID(app_id)});
5757
var list = ["[CLY]_session", "[CLY]_crash", "[CLY]_view", "[CLY]_action", "[CLY]_push_action", "[CLY]_push_sent", "[CLY]_star_rating", "[CLY]_nps", "[CLY]_survey", "[CLY]_apm_network", "[CLY]_apm_device", "[CLY]_consent"];
@@ -80,15 +80,19 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
8080
for (let i = 0; i < collections.length; i++) {
8181
const collection = collections[i].collectionName;
8282
try {
83-
const events = await drillDb.collection(collection).find({uid: old_uid}, {uid: 1, _id: 0}).limit(1).toArray();
83+
var query = {uid: old_uid};
84+
if (collection === "drill_events") {
85+
query = {uid: old_uid, 'a': app._id + ""};
86+
}
87+
const events = await drillDb.collection(collection).find(query, {uid: 1, _id: 0}).limit(1).toArray();
8488
if (!events || !events.length) {
8589
continue;
8690
}
8791
if (events && events[0]) {
8892
console.log("Found at least one event with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "updating to new uid", new_uid);
8993
try {
9094
if (!DRY_RUN) {
91-
await drillDb.collection(collection).update({uid: old_uid}, {'$set': {uid: new_uid}}, {multi: true});
95+
await drillDb.collection(collection).update(query, {'$set': {uid: new_uid}}, {multi: true});
9296
}
9397
}
9498
catch (err) {

‎bin/scripts/modify-data/delete/delete_user_properties.js‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,23 @@ Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_dril
4242
done();
4343
}
4444
}, function() {
45-
db.close();
46-
dbDrill.close();
47-
console.log("done");
45+
//delete property from merged drill events collection
46+
var unset = {};
47+
if (PROPERTY.startsWith("custom") || PROPERTY.startsWith("cmp")) {
48+
unset[PROPERTY] = "";
49+
}
50+
else {
51+
unset["up." + PROPERTY] = "";
52+
}
53+
54+
dbDrill.collection("drill_events").updateMany({"a": (APP_ID + "")}, {$unset: unset}, function(err) {
55+
if (err) {
56+
console.log("Error", err);
57+
}
58+
db.close();
59+
dbDrill.close();
60+
console.log("done");
61+
});
4862
});
4963
}
5064
});

‎bin/scripts/performance-monitoring/apm_events_delete.js‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,19 @@ if (dry_run) {
1717

1818
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
1919
try {
20-
console.log('Deleting APM events for app_id: ' + APP_ID);
21-
await Promise.all([
22-
countlyDb.collection("apm").remove({app_id: APP_ID}),
23-
drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_apm_network" + APP_ID).digest('hex')).drop(),
24-
drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_apm_device" + APP_ID).digest('hex')).drop(),
25-
]);
26-
console.log("All done");
20+
if (!APP_ID) {
21+
console.log("Please set APP_ID");
22+
}
23+
else {
24+
console.log('Deleting APM events for app_id: ' + APP_ID);
25+
await Promise.all([
26+
countlyDb.collection("apm").remove({app_id: APP_ID}),
27+
drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_apm_network" + APP_ID).digest('hex')).drop(),
28+
drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_apm_device" + APP_ID).digest('hex')).drop(),
29+
drillDb.collection("drill_events").remove({"a": APP_ID, "e": {$in: ["[CLY]_apm_device", "[CLY]_apm_network"]}})
30+
]);
31+
console.log("All done");
32+
}
2733
}
2834
catch (error) {
2935
console.log("ERROR: ");

0 commit comments

Comments
 (0)