Skip to content

Commit da2ff21

Browse files
ar2rsawseenar2rsawseen
authored
Merge branch 'master' into dependabot/npm_and_yarn/plugins/web/ua-parser-js-1.0.39
2 parents 0dc3709 + 220db14 commit da2ff21

File tree

22 files changed

+473
-362
lines changed

22 files changed

+473
-362
lines changed

‎CHANGELOG.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,13 @@
1+
## Version 24.05.12
2+
Fixes:
3+
- [dashboards] Fixes for dashboards grid
4+
- [dasboards] UI fix for dashboard widget action menu
5+
- [push] Refactored fcm API related code
6+
- [reports] Use config for encryption key in reports
7+
8+
Enterprise fixes:
9+
- [retention] Fixes for byval retention query calculation
10+
111
## Version 24.05.11
212
Fixes:
313
- [cache] Use a cursor without timeout

‎bin/scripts/fix-data/regenerate_aggregated_data.js‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
/*
2+
* Scrript triggers data regeneration for events and sessions.
3+
* It stores progress in county.data_regeneration_progress collection.
4+
* Collection can be dropped once regeneration is complete. Data in there ensures that if script dies unexpectedly, then upon staring again it will not regenerate collections again.
5+
* mongosh countly
6+
* db.data_regeneration_progress.drop();
7+
*
8+
* script path: {countly}/bin/scripts/fix-data/
9+
*
10+
* To run script:
11+
* node regenerate_aggregated_data.js
12+
*/
13+
//Adjust settings below before running script
14+
var regenerate_events = true; //if true will regenerate all custom events aggregated data
15+
var regenerate_sessions = false;
16+
var period = "732days"; //any valid period
17+
18+
//Each app IDis listed as string, for example var appList = ["6075f94b7e5e0d392902520c",6075f94b7e5e0d392902520d]
19+
var appList = [];//If left empty, will run for all apps.
20+
//For each app defined there only listed events will be regenerated. If left empty, all events will be regenerated.
21+
//Example var eventMap = {"6075f94b7e5e0d392902520c":["Logout","Login"],"6075f94b7e5e0d392902520d":["Logout","Login","Buy"]};
22+
var eventMap = {}; //If left empty will run for all alls/events.
23+
24+
25+
//End of adjustable settings
26+
27+
const common = require("../../../api/utils/common.js");
28+
const pluginManager = require('../../../plugins/pluginManager.js');
29+
const asyncjs = require('async');
30+
const drill = require('../../../plugins/drill/api/parts/data/drill.js');
31+
32+
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
33+
console.log("Connected to databases...");
34+
common.db = countlyDb;
35+
common.drillDb = drillDb;
36+
//get all apps
37+
try {
38+
var query = {};
39+
if (appList && appList.length) {
40+
query._id = {$in: appList.map(app_id=>common.db.ObjectID(app_id))};
41+
}
42+
const apps = await countlyDb.collection("apps").find(query, {_id: 1, name: 1, timezone: 1}).toArray();
43+
if (!apps || !apps.length) {
44+
return close();
45+
}
46+
try {
47+
//for each app serially process users
48+
asyncjs.eachSeries(apps, async function(app) {
49+
console.log("Processing app: ", app.name);
50+
//get all drill collections for this app
51+
var skipped_ec = 0;
52+
if (regenerate_events) {
53+
var events = await countlyDb.collection("events").findOne({_id: app._id}, {'list': 1});
54+
if (events && events.list && events.list.length) {
55+
events.list = events.list.filter(function(ee) {
56+
if (ee.indexOf("[CLY]_") === 0) {
57+
return false;
58+
}
59+
else if (eventMap && eventMap[app._id + ""]) {
60+
return eventMap[app._id + ""].indexOf(ee) > -1;
61+
}
62+
else {
63+
return true;
64+
}
65+
});
66+
for (var z = 0; z < events.list.length; z++) {
67+
var qq = {_id: app._id + ""};
68+
qq[events.list[z]] = {$exists: true};
69+
var doc = await countlyDb.collection("data_regeneration_progress").findOne(qq);
70+
if (!doc) {
71+
var event = events.list[z];
72+
console.log(" Processing event: ", event);
73+
var params = {
74+
appTimezone: app.timezone,
75+
time: common.initTimeObj(app.timezone),
76+
qstring: {
77+
app_id: app._id + "",
78+
event: event,
79+
period: period
80+
}
81+
};
82+
try {
83+
await new Promise((resolve)=>{
84+
drill.calculateEvents(params, function() {
85+
resolve();
86+
});
87+
});
88+
var update = {};
89+
update[event] = Date.now();
90+
await countlyDb.collection("data_regeneration_progress").updateOne({_id: app._id + ""}, {"$set": update}, {"upsert": true});
91+
92+
}
93+
catch (err) {
94+
console.log(err);
95+
}
96+
}
97+
else {
98+
skipped_ec++;
99+
}
100+
}
101+
}
102+
else {
103+
console.log(" No events found for app: ", app.name);
104+
}
105+
if (skipped_ec) {
106+
console.log(" Skipped ", skipped_ec, " events as they are marked as recalculated");
107+
}
108+
}
109+
if (regenerate_sessions) {
110+
var doc2 = await countlyDb.collection("data_regeneration_progress").findOne({_id: app._id + "", "[CLY]_session": {$exists: true}});
111+
if (!doc2) {
112+
console.log(" Processing sessions");
113+
var params2 = {
114+
appTimezone: app.timezone,
115+
time: common.initTimeObj(app.timezone),
116+
qstring: {
117+
app_id: app._id + "",
118+
period: period
119+
}
120+
};
121+
try {
122+
await new Promise((resolve)=>{
123+
drill.calculateSessions(params2, function() {
124+
resolve();
125+
});
126+
});
127+
await countlyDb.collection("data_regeneration_progress").updateOne({_id: app._id + ""}, {"$set": { "[CLY]_session": Date.now()}}, {"upsert": true});
128+
}
129+
catch (err) {
130+
console.log(err);
131+
}
132+
}
133+
else {
134+
console.log(" Sessions already processed for app: ", app.name);
135+
}
136+
}
137+
138+
}, function(err) {
139+
return close(err);
140+
});
141+
}
142+
catch (err) {
143+
return close(err);
144+
}
145+
}
146+
catch (err) {
147+
return close(err);
148+
}
149+
150+
function close(err) {
151+
if (err) {
152+
console.log(err);
153+
console.log('EXITED WITH ERROR');
154+
}
155+
console.log("Closing connections...");
156+
countlyDb.close();
157+
drillDb.close();
158+
console.log("DONE");
159+
}
160+
161+
});

‎bin/scripts/fix-data/user-merge.js‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
/**
2+
* Description: This script is used to merge users based on username.
3+
* Server: countly
4+
* Path: $(countly dir)/bin/scripts/fix-data
5+
* Command: node user-merge.js
6+
*/
7+
var pluginManager = require("../../../plugins/pluginManager.js");
8+
var appUsers = require("../../../api/parts/mgmt/app_users.js");
9+
var common = require("../../../api/utils/common.js");
10+
11+
console.log("Merging app users");
12+
13+
var APP_ID = "";
14+
var COLLECTION_NAME = "app_users" + APP_ID;
15+
16+
var RETRY_LIMIT = 3;
17+
var UPDATE_COUNTER = 0;
18+
19+
//Number of requests to be made before checking record count in app_user_merges
20+
var UPDATE_LIMIT = 100;
21+
//Number of records in app_user_merges after which script will sleep
22+
var RECORD_COUNT_LIMIT = 10;
23+
//Cooldown period if record count exceeds limit
24+
var RECORD_OVERLOAD_SLEEP = 2000;
25+
//Cooldown period between requests
26+
var COOLDOWN_PERIOD = 1000;
27+
28+
const sleep = m => new Promise((r) => {
29+
//console.log("Cooling period for " + m + " seconds!");
30+
setTimeout(r, m);
31+
});
32+
33+
pluginManager.dbConnection("countly").then(async(countlyDb) => {
34+
try {
35+
36+
common.db = countlyDb;
37+
38+
await cursor();
39+
40+
console.log("Total updates on the server - ", UPDATE_COUNTER);
41+
console.log("Script ran successfully!");
42+
common.db.close();
43+
process.exit(1);
44+
}
45+
catch (e) {
46+
console.log("Error while running script ", e);
47+
common.db.close();
48+
process.exit(1);
49+
}
50+
51+
async function cursor() {
52+
53+
const duplicates = await common.db.collection(COLLECTION_NAME).aggregate([
54+
{
55+
$group: {
56+
_id: "$username",
57+
count: { $sum: 1 }
58+
}
59+
},
60+
{
61+
$match: {
62+
count: { $gt: 1 },
63+
_id: { $ne: null }
64+
}
65+
}
66+
]).toArray();
67+
68+
console.log("Found", duplicates.length, "duplicate username groups.");
69+
70+
for (var i = 0; i < duplicates.length; i++) {
71+
72+
var mainUser = null;
73+
var mergedUsersUIDs = [];
74+
75+
var query = {
76+
username: duplicates[i]._id
77+
};
78+
79+
var projections = {};
80+
81+
var sort = { ls: -1 };
82+
83+
var cursor = common.db.collection(COLLECTION_NAME).find(query).project(projections).sort(sort);
84+
85+
while (await cursor.hasNext()) {
86+
var doc = await cursor.next();
87+
88+
if (doc.uid && doc.uid !== "") {
89+
if (!mainUser) {
90+
mainUser = doc;
91+
}
92+
else {
93+
await mergeUsers(mainUser, doc);
94+
mergedUsersUIDs.push(doc.uid);
95+
}
96+
}
97+
}
98+
99+
if (mergedUsersUIDs.length > 0) {
100+
console.log("Total", mergedUsersUIDs.length, "users merged into user", mainUser.uid, ": (", mergedUsersUIDs.join(", "), ")");
101+
}
102+
}
103+
}
104+
105+
async function mergeUsers(mainUser, user) {
106+
var retryCounter = 1;
107+
var success = false;
108+
109+
while (!success && retryCounter < RETRY_LIMIT) {
110+
await new Promise(function(resolve) {
111+
var newUser = JSON.parse(JSON.stringify(mainUser));
112+
113+
appUsers.merge(APP_ID, newUser, newUser._id, user._id, newUser.did, user.did, function(err) {
114+
if (err) {
115+
console.log("Error while merging - ", err);
116+
retryCounter += 1;
117+
}
118+
else {
119+
success = true;
120+
}
121+
122+
resolve();
123+
});
124+
});
125+
await sleep(COOLDOWN_PERIOD);
126+
}
127+
128+
if (success) {
129+
if (retryCounter > 1) {
130+
console.log("User ", user.uid, " merged successfully after ", retryCounter, " retries.");
131+
}
132+
UPDATE_COUNTER += 1;
133+
if (UPDATE_COUNTER % UPDATE_LIMIT === 0) {
134+
await checkRecordCount();
135+
}
136+
}
137+
else {
138+
console.log("Retry limit exceeded for users ", mainUser.uid, " and ", user.uid);
139+
}
140+
}
141+
142+
async function checkRecordCount() {
143+
var recordCount = await common.db.collection("app_user_merges").countDocuments();
144+
console.log("Record count in app_user_merges: ", recordCount);
145+
146+
while (recordCount > RECORD_COUNT_LIMIT) {
147+
console.log("Record count exceeds limit. Sleeping for " + RECORD_OVERLOAD_SLEEP / 1000 + "seconds.");
148+
await sleep(RECORD_OVERLOAD_SLEEP);
149+
recordCount = await common.db.collection("app_user_merges").countDocuments();
150+
}
151+
}
152+
});

‎plugins/alerts/api/alertModules/cohorts.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ const commonLib = require("../parts/common-lib.js");
99
const { ObjectId } = require('mongodb');
1010

1111
module.exports.check = async function({ alertConfigs: alert, done, scheduledTo: date }) {
12-
const app = await common.db.collection("apps").findOne({ _id: ObjectId(alert.selectedApps[0]) });
12+
const app = await common.readBatcher.getOne("apps", { _id: new ObjectId(alert.selectedApps[0]) });
1313
if (!app) {
1414
log.e(`App ${alert.selectedApps[0]} couldn't be found`);
1515
return done();
@@ -22,7 +22,7 @@ module.exports.check = async function({ alertConfigs: alert, done, scheduledTo:
2222

2323
if (compareType === commonLib.COMPARE_TYPE_ENUM.MORE_THAN) {
2424
if (metricValue > compareValue) {
25-
await commonLib.trigger({ alert, app, metricValue, date });
25+
await commonLib.trigger({ alert, app, metricValue, date }, log);
2626
}
2727
}
2828
else {
@@ -38,7 +38,7 @@ module.exports.check = async function({ alertConfigs: alert, done, scheduledTo:
3838
: change <= -compareValue;
3939

4040
if (shouldTrigger) {
41-
await commonLib.trigger({ alert, app, date, metricValue, metricValueBefore });
41+
await commonLib.trigger({ alert, app, date, metricValue, metricValueBefore }, log);
4242
}
4343
}
4444

‎plugins/alerts/api/alertModules/dataPoints.js‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ module.exports.check = async function({ alertConfigs: alert, done, scheduledTo:
1414
const selectedApp = alert.selectedApps[0];
1515
let apps;
1616
if (selectedApp === "all") {
17-
apps = await common.db.collection("apps").find().toArray();
17+
apps = await common.readBatcher.getMany("apps", {});
1818
}
1919
else {
20-
apps = [await common.db.collection("apps").findOne({ _id: ObjectId(selectedApp) })];
20+
apps = [await common.readBatcher.getOne("apps", { _id: new ObjectId(selectedApp) })];
2121
}
2222

2323
for (let app of apps) {
@@ -33,7 +33,7 @@ module.exports.check = async function({ alertConfigs: alert, done, scheduledTo:
3333

3434
if (compareType === commonLib.COMPARE_TYPE_ENUM.MORE_THAN) {
3535
if (metricValue > compareValue) {
36-
await commonLib.trigger({ alert, app, metricValue, date });
36+
await commonLib.trigger({ alert, app, metricValue, date }, log);
3737
}
3838
}
3939
else {
@@ -49,7 +49,7 @@ module.exports.check = async function({ alertConfigs: alert, done, scheduledTo:
4949
: change <= -compareValue;
5050

5151
if (shouldTrigger) {
52-
await commonLib.trigger({ alert, app, date, metricValue, metricValueBefore });
52+
await commonLib.trigger({ alert, app, date, metricValue, metricValueBefore }, log);
5353
}
5454
}
5555
}

0 commit comments

Comments
 (0)