Skip to content

Commit 79c14ad

Browse files
ayasayadi1ayasayadi1
committed
Merge branch 'next' of https://github.com/Countly/countly-server into SER-1160
2 parents ce3a480 + d8b7ea3 commit 79c14ad

File tree

23 files changed

+788
-180
lines changed

23 files changed

+788
-180
lines changed

‎bin/scripts/data-cleanup/delete_old_views.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,9 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
9090
let view = await cursor.next();
9191
//Find one drill entry for the view with timestamp greater than expiration date
9292
var drillEntry = await drillSession.client.db("countly_drill").collection(collectionName).findOne({"sg.name": view.view, "ts": { $gt: expiration_timestamp }}, {ts: 1});
93+
var drillEntry2 = await drillSession.client.db("countly_drill").collection("drill_events").findOne({"a": app._id + "", "e": event, "sg.name": view.view, "ts": { $gt: expiration_timestamp }}, {ts: 1});
9394
//If no entry found, delete the view
94-
if (!drillEntry) {
95+
if (!drillEntry && !drillEntry2) {
9596
console.log("Deleting view: ", view.view);
9697
if (!DRY_RUN) {
9798
await new Promise(function(resolve) {

‎bin/scripts/data-cleanup/remove_old_crashes_sync.js‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ Promise.all(
6767
count = await drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_crash" + app._id).digest('hex')).countDocuments({ ts: { $lt: lastUnixTimestamp * 1000 } });
6868
console.log("", count, "drill crashes to be deleted via command");
6969
console.log("", "", "", `db.drill_events${crypto.createHash('sha1').update("[CLY]_crash" + app._id).digest('hex')}.deleteMany({ ts: { $lt: ${lastUnixTimestamp * 1000} } })`);
70+
var count2 = await drillDb.collection("drill_events").countDocuments({"a": app._id + "", "e": "[CLY]_crash", ts: { $lt: lastUnixTimestamp * 1000 } });
71+
console.log("", count2, "drill crashes to be deleted via command");
72+
console.log("", "", "", `db.drill_events.deleteMany({"a": ${app._id}, "e": "[CLY]_crash", ts: { $lt: ${lastUnixTimestamp * 1000} } })`);
7073
checkThreshold(count);
7174
}
7275
else {
@@ -77,8 +80,12 @@ Promise.all(
7780
res = await db.collection(`app_crashes${app._id}`).deleteMany({ ts: { $lt: lastUnixTimestamp } });
7881
console.log("", res, "crashes deleted");
7982
await sleep(SLEEP);
83+
//deleting from old collection
8084
res = await drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_crash" + app._id).digest('hex')).deleteMany({ ts: { $lt: lastUnixTimestamp * 1000 } });
8185
console.log("", res, "drill crashes deleted");
86+
//deleting from amerged drill collection
87+
res = await drillDb.collection("drill_events").deleteMany({"a": app._id + "", "e": "[CLY]_crash", ts: { $lt: lastUnixTimestamp * 1000 } });
88+
console.log("", res, "drill crashes deleted");
8289
await sleep(SLEEP);
8390
}
8491
}

‎bin/scripts/data-reports/check_user_mismatch.js‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,17 +14,25 @@ var query = {"ts": {"$gt": 1709762400000}}; //Change this query to set date rang
1414
var eventKey = "[CLY]_session"; //Write in your event key
1515
var appID = "6075f94b7e5e0d392902520c"; //Write in YOUR app ID
1616

17+
var union_with_old_collection = false; //False if all sessions are stored in drill_events collection
18+
1719
console.log("Running for:" + appID + " " + eventKey + " " + JSON.stringify(query));
1820

1921
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
2022

2123
var pipeline = [];
22-
pipeline.push({"$match": query});
23-
pipeline.push({"$group": {"_id": null, "uid": {"$addToSet": "$uid"}}});
24+
var query2 = JSON.parse(JSON.stringify(query));
25+
query2.e = eventKey;
26+
query2.a = appID;
27+
pipeline.push({"$match": query2});
2428

25-
var collection = "drill_events" + crypto.createHash('sha1').update(eventKey + appID).digest('hex');
29+
if (union_with_old_collection) {
30+
var collection = "drill_events" + crypto.createHash('sha1').update(eventKey + appID).digest('hex');
31+
pipeline.push({"$unionWith": { "coll": collection, "pipeline": [{"$match": query}] }});
32+
}
33+
pipeline.push({"$group": {"_id": null, "uid": {"$addToSet": "$uid"}}});
2634

27-
drillDb.collection(collection).aggregate(pipeline, function(err, res) {
35+
drillDb.collection("drill_events").aggregate(pipeline, function(err, res) {
2836
if (err) {
2937
console.log(err);
3038
countlyDb.close();
@@ -67,6 +75,9 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
6775
console.log("Users missing in app_users collection: " + missingOnes.length);
6876
console.log(JSON.stringify(missingOnes));
6977
}
78+
else {
79+
console.log("All users matched");
80+
}
7081

7182
countlyDb.close();
7283
drillDb.close();

‎bin/scripts/data-reports/drill_data_cd.js‎

Lines changed: 36 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,14 @@
77
*/
88

99
//period to check
10-
var startDate = new Date("2024-04-2T00:00:00");
11-
var endDate = new Date("2024-04-3T00:00:00");
10+
var startDate = new Date("2021-04-02T00:00:00");
11+
var endDate = new Date("2025-04-03T00:00:00");
1212
var apps = []; //Put in your APP ID like ["3469834986y34968y206y2"]
1313

14-
var internal_events = [];
15-
var verbose = true;
16-
1714
var Promise = require("bluebird");
18-
var crypto = require('crypto');
1915
var pluginManager = require("../../../plugins/pluginManager");
16+
var results = {};
17+
var dates = {};
2018

2119
function output_data(dates, results) {
2220
var datesList = Object.keys(dates).sort(function(a, b) {
@@ -70,80 +68,48 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
7068
queryApps = {_id: {$in: apps}};
7169
}
7270

73-
countlyDb.collection("apps").find(queryApps).toArray(function(err, apps) {
71+
countlyDb.collection("apps").find(queryApps).toArray(function(err, apps2) {
7472
if (err) {
7573
console.log(err);
7674
countlyDb.close();
7775
drillDb.close();
7876
}
7977
else {
80-
Promise.each(apps, function(app) {
81-
console.log('------' + app.name + '------');
82-
return new Promise(function(resolve) {
83-
var dates = {};
84-
var results = {};
85-
//fetch events list
86-
countlyDb.collection("events").findOne({"_id": app._id}, {"list": 1}, function(err, events) {
87-
events = events || [];
88-
var list = events.list || [];
89-
if (verbose) {
90-
console.log(list.length + " events found");
91-
}
92-
for (var z = 0; z < internal_events.length; z++) {
93-
if (list.indexOf(internal_events[z]) === -1) {
94-
list.push(internal_events[z]);
95-
}
78+
var query = {"cd": {"$gte": startDate, "$lt": endDate}};
79+
if (apps.length > 0) {
80+
81+
var a = [];
82+
for (var i = 0; i < apps2.length; i++) {
83+
a.push(apps2[i]._id + "");
84+
}
85+
query.a = {$in: a};
86+
}
87+
var pipeline = [{"$match": query}, {"$group": {"_id": {"a": "$a", "e": "$e", "d": "$d"}, "c": {"$sum": 1}}}];
88+
drillDb.collection("drill_events").aggregate(pipeline, {"allowDiskUse": true}).toArray(function(err, data) {
89+
if (err) {
90+
console.log(err);
91+
}
92+
if (data && data.length > 0) {
93+
results["_total"] = results["_total"] || {};
94+
95+
for (var z = 0; z < data.length; z++) {
96+
var event = data[z]._id.e;
97+
if (!dates[data[z]._id.d]) {
98+
dates[data[z]._id.d] = true;
9699
}
97-
Promise.each(list, function(event) {
98-
return new Promise(function(resolve2) {
99-
//get hashed drill collection name
100-
let collection = "drill_events" + crypto.createHash('sha1').update(event + app._id).digest('hex');
101-
if (verbose) {
102-
console.log(collection);
103-
}
104-
var pipeline = [
105-
{"$match": {"cd": {"$gte": startDate, "$lt": endDate}}},
106-
{"$group": {"_id": "$d", "c": {"$sum": 1}}}
107-
];
108-
drillDb.collection(collection).aggregate(pipeline, {"allowDiskUse": true}).toArray(function(err, data) {
109-
if (err) {
110-
console.log(err);
111-
}
112-
if (data.length > 0) {
113-
results["_total"] = results["_total"] || {};
114-
results[event] = {};
115-
for (var z = 0; z < data.length; z++) {
116-
if (!dates[data[z]._id]) {
117-
dates[data[z]._id] = true;
118-
}
119-
results[event][data[z]._id] = data[z].c;
120-
results["_total"][data[z]._id] = results["_total"][data[z]._id] || 0;
121-
results["_total"][data[z]._id] += data[z].c || 0;
122-
}
123-
}
124-
resolve2();
125-
});
126-
});
127-
}).then(function() {
128-
//output
129-
//sort dates
130-
output_data(dates, results);
131-
resolve();
132-
}).catch(function(err) {
133-
output_data(dates, results);
134-
console.log(err);
135-
resolve();
136-
});
137-
});
138-
});
139-
}).then(function() {
140-
countlyDb.close();
141-
drillDb.close();
142-
}).catch(function(err) {
143-
console.log(err);
100+
results[data[z]._id.e] = {};
101+
results[event][data[z]._id.d] = data[z].c;
102+
results["_total"][data[z]._id.d] = results["_total"][data[z]._id.d] || 0;
103+
results["_total"][data[z]._id.d] += data[z].c || 0;
104+
}
105+
output_data(dates, results);
106+
}
107+
144108
countlyDb.close();
145109
drillDb.close();
146110
});
111+
112+
147113
}
148114
});
149115
});

‎bin/scripts/data-reports/old_scripts/drill_data_cd.js‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
/**
2+
* Check for which dates data was recorded
3+
* requires index on cd collections or else will be slower
4+
* Server: countly
5+
* Path: $(countly dir)/bin/scripts/data-reports
6+
* Command: node drill_data_cd.js
7+
*/
8+
9+
//period to check
10+
var startDate = new Date("2024-04-2T00:00:00");
11+
var endDate = new Date("2024-04-3T00:00:00");
12+
var apps = []; //Put in your APP ID like ["3469834986y34968y206y2"]
13+
14+
var internal_events = [];
15+
var verbose = true;
16+
17+
var Promise = require("bluebird");
18+
var crypto = require('crypto');
19+
var pluginManager = require("../../../plugins/pluginManager");
20+
21+
function output_data(dates, results) {
22+
var datesList = Object.keys(dates).sort(function(a, b) {
23+
var d1 = a.split(":");
24+
var d2 = b.split(":");
25+
for (var z = 0; z < d1.length; z++) {
26+
d1[z] = parseInt(d1[z]);
27+
d2[z] = parseInt(d2[z]);
28+
}
29+
30+
for (var k = 0; k < d1.length; k++) {
31+
if (d1[k] > d2[k]) {
32+
return 1;
33+
}
34+
else if (d1[k] < d2[k]) {
35+
return -1;
36+
}
37+
}
38+
return 1;
39+
40+
});
41+
//output header line
42+
console.log("Event," + datesList.join(","));
43+
for (var key in results) {
44+
if (key !== "_total") {
45+
var line = key;
46+
for (var z = 0; z < datesList.length; z++) {
47+
line += "," + (results[key][datesList[z]] || 0);
48+
}
49+
console.log(line);
50+
}
51+
//output total
52+
53+
54+
55+
}
56+
var line2 = "Total";
57+
for (var z1 = 0; z1 < datesList.length; z1++) {
58+
line2 += "," + (results["_total"][datesList[z1]] || 0);
59+
}
60+
console.log(line2);
61+
62+
}
63+
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
64+
//get apps
65+
var queryApps = {};
66+
if (apps && apps.length) {
67+
for (var z = 0; z < apps.length; z++) {
68+
apps[z] = countlyDb.ObjectID(apps[z]);
69+
}
70+
queryApps = {_id: {$in: apps}};
71+
}
72+
73+
countlyDb.collection("apps").find(queryApps).toArray(function(err, apps) {
74+
if (err) {
75+
console.log(err);
76+
countlyDb.close();
77+
drillDb.close();
78+
}
79+
else {
80+
Promise.each(apps, function(app) {
81+
console.log('------' + app.name + '------');
82+
return new Promise(function(resolve) {
83+
var dates = {};
84+
var results = {};
85+
//fetch events list
86+
countlyDb.collection("events").findOne({"_id": app._id}, {"list": 1}, function(err, events) {
87+
events = events || [];
88+
var list = events.list || [];
89+
if (verbose) {
90+
console.log(list.length + " events found");
91+
}
92+
for (var z = 0; z < internal_events.length; z++) {
93+
if (list.indexOf(internal_events[z]) === -1) {
94+
list.push(internal_events[z]);
95+
}
96+
}
97+
Promise.each(list, function(event) {
98+
return new Promise(function(resolve2) {
99+
//get hashed drill collection name
100+
let collection = "drill_events" + crypto.createHash('sha1').update(event + app._id).digest('hex');
101+
if (verbose) {
102+
console.log(collection);
103+
}
104+
var pipeline = [
105+
{"$match": {"cd": {"$gte": startDate, "$lt": endDate}}},
106+
{"$group": {"_id": "$d", "c": {"$sum": 1}}}
107+
];
108+
drillDb.collection(collection).aggregate(pipeline, {"allowDiskUse": true}).toArray(function(err, data) {
109+
if (err) {
110+
console.log(err);
111+
}
112+
if (data.length > 0) {
113+
results["_total"] = results["_total"] || {};
114+
results[event] = {};
115+
for (var z = 0; z < data.length; z++) {
116+
if (!dates[data[z]._id]) {
117+
dates[data[z]._id] = true;
118+
}
119+
results[event][data[z]._id] = data[z].c;
120+
results["_total"][data[z]._id] = results["_total"][data[z]._id] || 0;
121+
results["_total"][data[z]._id] += data[z].c || 0;
122+
}
123+
}
124+
resolve2();
125+
});
126+
});
127+
}).then(function() {
128+
//output
129+
//sort dates
130+
output_data(dates, results);
131+
resolve();
132+
}).catch(function(err) {
133+
output_data(dates, results);
134+
console.log(err);
135+
resolve();
136+
});
137+
});
138+
});
139+
}).then(function() {
140+
countlyDb.close();
141+
drillDb.close();
142+
}).catch(function(err) {
143+
console.log(err);
144+
countlyDb.close();
145+
drillDb.close();
146+
});
147+
}
148+
});
149+
});

‎bin/scripts/data-reports/event_list_no_data.js‎ renamed to ‎bin/scripts/data-reports/old_scripts/event_list_no_data.js‎

File renamed without changes.

0 commit comments

Comments
 (0)