Skip to content

Commit ad5340c

Browse files
CookiezaursCookiezaurs
authored
Merge branch 'next' into anna/next
2 parents aa75079 + 201c41b commit ad5340c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+3174
-3027
lines changed

‎CHANGELOG.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,13 @@
1+
## Version 24.05.12
2+
Fixes:
3+
- [dashboards] Fixes for dashboards grid
4+
- [dasboards] UI fix for dashboard widget action menu
5+
- [push] Refactored fcm API related code
6+
- [reports] Use config for encryption key in reports
7+
8+
Enterprise fixes:
9+
- [retention] Fixes for byval retention query calculation
10+
111
## Version 24.05.11
212
Fixes:
313
- [cache] Use a cursor without timeout

‎api/config.sample.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ var countlyConfig = {
9999
* @property {string} algorithm - name of the algorithm to use for encryption. The algorithm is dependent on OpenSSL, examples are 'aes192', etc. On recent OpenSSL releases, openssl list-cipher-algorithms will display the available cipher algorithms. Default value is aes-256-cbc
100100
* @property {string} input_encoding - how encryption input is encoded. Used as output for decrypting. Default utf-8.
101101
* @property {string} output_encoding - how encryption output is encoded. Used as input for decrypting. Default hex.
102+
* @property {string} reports_key - key used for encryption of reports links
102103
*/
103104
encryption: {},
104105

‎api/parts/data/fetch.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1614,7 +1614,7 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) {
16141614
options = {};
16151615
}
16161616

1617-
if (typeof options === "undefined") {
1617+
if (!options) {
16181618
options = {};
16191619
}
16201620

@@ -1704,7 +1704,7 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) {
17041704

17051705
var zeroDocs = [zeroIdToFetch];
17061706
var monthDocs = [monthIdToFetch];
1707-
if (!(options && options.dontBreak)) {
1707+
if (!options.dontBreak) {
17081708
for (let i = 0; i < common.base64.length; i++) {
17091709
zeroDocs.push(zeroIdToFetch + "_" + common.base64[i]);
17101710
monthDocs.push(monthIdToFetch + "_" + common.base64[i]);

‎bin/scripts/expire-data/countly_single_app_expireDataBatches.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ function processDrillCollection(collection, seconds, callback) {
209209

210210
function generateIterationList(z) {
211211
z = (start === 0 && z) ? z : start;
212-
if (timeSpan === 0 && start === 0) {
212+
if (timeSpan === 0 && z === 0) {
213213
listed.push({"collection": collection.collection, "db": collection.db, "start": 0, "end": end, "query": {"ts": {"$lt": end}}});
214214
}
215215
else if (timeSpan === 0) {

‎bin/scripts/fix-data/regenerate_aggregated_data.js‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
/*
2+
* Scrript triggers data regeneration for events and sessions.
3+
* It stores progress in county.data_regeneration_progress collection.
4+
* Collection can be dropped once regeneration is complete. Data in there ensures that if script dies unexpectedly, then upon staring again it will not regenerate collections again.
5+
* mongosh countly
6+
* db.data_regeneration_progress.drop();
7+
*
8+
* script path: {countly}/bin/scripts/fix-data/
9+
*
10+
* To run script:
11+
* node regenerate_aggregated_data.js
12+
*/
13+
//Adjust settings below before running script
14+
var regenerate_events = true; //if true will regenerate all custom events aggregated data
15+
var regenerate_sessions = false;
16+
var period = "732days"; //any valid period
17+
18+
//Each app IDis listed as string, for example var appList = ["6075f94b7e5e0d392902520c",6075f94b7e5e0d392902520d]
19+
var appList = [];//If left empty, will run for all apps.
20+
//For each app defined there only listed events will be regenerated. If left empty, all events will be regenerated.
21+
//Example var eventMap = {"6075f94b7e5e0d392902520c":["Logout","Login"],"6075f94b7e5e0d392902520d":["Logout","Login","Buy"]};
22+
var eventMap = {}; //If left empty will run for all alls/events.
23+
24+
25+
//End of adjustable settings
26+
27+
const common = require("../../../api/utils/common.js");
28+
const pluginManager = require('../../../plugins/pluginManager.js');
29+
const asyncjs = require('async');
30+
const drill = require('../../../plugins/drill/api/parts/data/drill.js');
31+
32+
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
33+
console.log("Connected to databases...");
34+
common.db = countlyDb;
35+
common.drillDb = drillDb;
36+
//get all apps
37+
try {
38+
var query = {};
39+
if (appList && appList.length) {
40+
query._id = {$in: appList.map(app_id=>common.db.ObjectID(app_id))};
41+
}
42+
const apps = await countlyDb.collection("apps").find(query, {_id: 1, name: 1, timezone: 1}).toArray();
43+
if (!apps || !apps.length) {
44+
return close();
45+
}
46+
try {
47+
//for each app serially process users
48+
asyncjs.eachSeries(apps, async function(app) {
49+
console.log("Processing app: ", app.name);
50+
//get all drill collections for this app
51+
var skipped_ec = 0;
52+
if (regenerate_events) {
53+
var events = await countlyDb.collection("events").findOne({_id: app._id}, {'list': 1});
54+
if (events && events.list && events.list.length) {
55+
events.list = events.list.filter(function(ee) {
56+
if (ee.indexOf("[CLY]_") === 0) {
57+
return false;
58+
}
59+
else if (eventMap && eventMap[app._id + ""]) {
60+
return eventMap[app._id + ""].indexOf(ee) > -1;
61+
}
62+
else {
63+
return true;
64+
}
65+
});
66+
for (var z = 0; z < events.list.length; z++) {
67+
var qq = {_id: app._id + ""};
68+
qq[events.list[z]] = {$exists: true};
69+
var doc = await countlyDb.collection("data_regeneration_progress").findOne(qq);
70+
if (!doc) {
71+
var event = events.list[z];
72+
console.log(" Processing event: ", event);
73+
var params = {
74+
appTimezone: app.timezone,
75+
time: common.initTimeObj(app.timezone),
76+
qstring: {
77+
app_id: app._id + "",
78+
event: event,
79+
period: period
80+
}
81+
};
82+
try {
83+
await new Promise((resolve)=>{
84+
drill.calculateEvents(params, function() {
85+
resolve();
86+
});
87+
});
88+
var update = {};
89+
update[event] = Date.now();
90+
await countlyDb.collection("data_regeneration_progress").updateOne({_id: app._id + ""}, {"$set": update}, {"upsert": true});
91+
92+
}
93+
catch (err) {
94+
console.log(err);
95+
}
96+
}
97+
else {
98+
skipped_ec++;
99+
}
100+
}
101+
}
102+
else {
103+
console.log(" No events found for app: ", app.name);
104+
}
105+
if (skipped_ec) {
106+
console.log(" Skipped ", skipped_ec, " events as they are marked as recalculated");
107+
}
108+
}
109+
if (regenerate_sessions) {
110+
var doc2 = await countlyDb.collection("data_regeneration_progress").findOne({_id: app._id + "", "[CLY]_session": {$exists: true}});
111+
if (!doc2) {
112+
console.log(" Processing sessions");
113+
var params2 = {
114+
appTimezone: app.timezone,
115+
time: common.initTimeObj(app.timezone),
116+
qstring: {
117+
app_id: app._id + "",
118+
period: period
119+
}
120+
};
121+
try {
122+
await new Promise((resolve)=>{
123+
drill.calculateSessions(params2, function() {
124+
resolve();
125+
});
126+
});
127+
await countlyDb.collection("data_regeneration_progress").updateOne({_id: app._id + ""}, {"$set": { "[CLY]_session": Date.now()}}, {"upsert": true});
128+
}
129+
catch (err) {
130+
console.log(err);
131+
}
132+
}
133+
else {
134+
console.log(" Sessions already processed for app: ", app.name);
135+
}
136+
}
137+
138+
}, function(err) {
139+
return close(err);
140+
});
141+
}
142+
catch (err) {
143+
return close(err);
144+
}
145+
}
146+
catch (err) {
147+
return close(err);
148+
}
149+
150+
function close(err) {
151+
if (err) {
152+
console.log(err);
153+
console.log('EXITED WITH ERROR');
154+
}
155+
console.log("Closing connections...");
156+
countlyDb.close();
157+
drillDb.close();
158+
console.log("DONE");
159+
}
160+
161+
});

‎bin/scripts/fix-data/user-merge.js‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
/**
2+
* Description: This script is used to merge users based on username.
3+
* Server: countly
4+
* Path: $(countly dir)/bin/scripts/fix-data
5+
* Command: node user-merge.js
6+
*/
7+
var pluginManager = require("../../../plugins/pluginManager.js");
8+
var appUsers = require("../../../api/parts/mgmt/app_users.js");
9+
var common = require("../../../api/utils/common.js");
10+
11+
console.log("Merging app users");
12+
13+
var APP_ID = "";
14+
var COLLECTION_NAME = "app_users" + APP_ID;
15+
16+
var RETRY_LIMIT = 3;
17+
var UPDATE_COUNTER = 0;
18+
19+
//Number of requests to be made before checking record count in app_user_merges
20+
var UPDATE_LIMIT = 100;
21+
//Number of records in app_user_merges after which script will sleep
22+
var RECORD_COUNT_LIMIT = 10;
23+
//Cooldown period if record count exceeds limit
24+
var RECORD_OVERLOAD_SLEEP = 2000;
25+
//Cooldown period between requests
26+
var COOLDOWN_PERIOD = 1000;
27+
28+
const sleep = m => new Promise((r) => {
29+
//console.log("Cooling period for " + m + " seconds!");
30+
setTimeout(r, m);
31+
});
32+
33+
pluginManager.dbConnection("countly").then(async(countlyDb) => {
34+
try {
35+
36+
common.db = countlyDb;
37+
38+
await cursor();
39+
40+
console.log("Total updates on the server - ", UPDATE_COUNTER);
41+
console.log("Script ran successfully!");
42+
common.db.close();
43+
process.exit(1);
44+
}
45+
catch (e) {
46+
console.log("Error while running script ", e);
47+
common.db.close();
48+
process.exit(1);
49+
}
50+
51+
async function cursor() {
52+
53+
const duplicates = await common.db.collection(COLLECTION_NAME).aggregate([
54+
{
55+
$group: {
56+
_id: "$username",
57+
count: { $sum: 1 }
58+
}
59+
},
60+
{
61+
$match: {
62+
count: { $gt: 1 },
63+
_id: { $ne: null }
64+
}
65+
}
66+
]).toArray();
67+
68+
console.log("Found", duplicates.length, "duplicate username groups.");
69+
70+
for (var i = 0; i < duplicates.length; i++) {
71+
72+
var mainUser = null;
73+
var mergedUsersUIDs = [];
74+
75+
var query = {
76+
username: duplicates[i]._id
77+
};
78+
79+
var projections = {};
80+
81+
var sort = { ls: -1 };
82+
83+
var cursor = common.db.collection(COLLECTION_NAME).find(query).project(projections).sort(sort);
84+
85+
while (await cursor.hasNext()) {
86+
var doc = await cursor.next();
87+
88+
if (doc.uid && doc.uid !== "") {
89+
if (!mainUser) {
90+
mainUser = doc;
91+
}
92+
else {
93+
await mergeUsers(mainUser, doc);
94+
mergedUsersUIDs.push(doc.uid);
95+
}
96+
}
97+
}
98+
99+
if (mergedUsersUIDs.length > 0) {
100+
console.log("Total", mergedUsersUIDs.length, "users merged into user", mainUser.uid, ": (", mergedUsersUIDs.join(", "), ")");
101+
}
102+
}
103+
}
104+
105+
async function mergeUsers(mainUser, user) {
106+
var retryCounter = 1;
107+
var success = false;
108+
109+
while (!success && retryCounter < RETRY_LIMIT) {
110+
await new Promise(function(resolve) {
111+
var newUser = JSON.parse(JSON.stringify(mainUser));
112+
113+
appUsers.merge(APP_ID, newUser, newUser._id, user._id, newUser.did, user.did, function(err) {
114+
if (err) {
115+
console.log("Error while merging - ", err);
116+
retryCounter += 1;
117+
}
118+
else {
119+
success = true;
120+
}
121+
122+
resolve();
123+
});
124+
});
125+
await sleep(COOLDOWN_PERIOD);
126+
}
127+
128+
if (success) {
129+
if (retryCounter > 1) {
130+
console.log("User ", user.uid, " merged successfully after ", retryCounter, " retries.");
131+
}
132+
UPDATE_COUNTER += 1;
133+
if (UPDATE_COUNTER % UPDATE_LIMIT === 0) {
134+
await checkRecordCount();
135+
}
136+
}
137+
else {
138+
console.log("Retry limit exceeded for users ", mainUser.uid, " and ", user.uid);
139+
}
140+
}
141+
142+
async function checkRecordCount() {
143+
var recordCount = await common.db.collection("app_user_merges").countDocuments();
144+
console.log("Record count in app_user_merges: ", recordCount);
145+
146+
while (recordCount > RECORD_COUNT_LIMIT) {
147+
console.log("Record count exceeds limit. Sleeping for " + RECORD_OVERLOAD_SLEEP / 1000 + "seconds.");
148+
await sleep(RECORD_OVERLOAD_SLEEP);
149+
recordCount = await common.db.collection("app_user_merges").countDocuments();
150+
}
151+
}
152+
});

‎frontend/express/public/javascripts/countly/vue/components/date.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1138,6 +1138,9 @@
11381138
},
11391139
methods: {
11401140
loadValue: function(value) {
1141+
if (!value) {
1142+
return;
1143+
}
11411144
var changes = this.valueToInputState(value),
11421145
self = this;
11431146
changes.label = getRangeLabel(changes, this.type);

‎frontend/express/public/javascripts/countly/vue/components/dropdown.js‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -547,7 +547,7 @@
547547
template: '<cly-dropdown class="cly-vue-more-options" ref="dropdown" :widthSameAsTrigger="widthSameAsTrigger" :placement="placement" :disabled="disabled" @hide="toggleArrowState" @show="toggleArrowState" v-on="$listeners">\
548548
<template v-slot:trigger>\
549549
<slot name="trigger">\
550-
<el-button :data-test-id="testId + \'-more-option-button\'" :size="size" :icon="icon" :type="type" >\
550+
<el-button :data-test-id="testId + \'-more-option-button\'" :size="size" :icon="icon" :type="type" :disabled="disabledButton">\
551551
<span :data-test-id="testId + \'-more-option-text\'" v-if="text">{{text}}</span>\
552552
<i v-if="showArrows" style="display:inline-block; margin: 0px 0px 0px 8px;" :class="iconClass"></i>\
553553
</el-button>\
@@ -579,6 +579,10 @@
579579
type: Boolean,
580580
default: false
581581
},
582+
disabledButton: {
583+
type: Boolean,
584+
default: false,
585+
},
582586
placement: {
583587
type: String,
584588
default: 'bottom-end'

0 commit comments

Comments
 (0)