Skip to content

Commit 736397e

Browse files
CookiezaursCookiezaurs
authored
Merge pull request #5595 from Countly/anna/next
Updated scripts
2 parents 327c010 + 55248f7 commit 736397e

File tree

9 files changed

+115
-54
lines changed

9 files changed

+115
-54
lines changed

‎bin/scripts/fix-data/fix_drill_custom_props.js‎

Lines changed: 52 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -34,39 +34,65 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
3434
//PROCESS COLLECTIONS FOR EACH APP
3535
for (let i = 0; i < apps.length; i++) {
3636
console.log("Processing app: " + apps[i].name);
37-
var collectionName = drillCommon.getCollectionName("[CLY]_session", apps[i]._id + "");
38-
console.log("Processing collection: " + collectionName);
37+
var collectionName0 = drillCommon.getCollectionName("[CLY]_session", apps[i]._id + "");
3938

40-
const cursor = drillDb.collection(collectionName).find(query_drill, {"_id": 1, "custom": 1});
41-
//FOR EACH DOCUMENT
42-
var updates = [];
43-
while (await cursor.hasNext()) {
44-
var doc = await cursor.next();
45-
if (doc.custom) {
46-
var updateDoc = {};
47-
let updateMe = false;
48-
for (var key in doc.custom) {
49-
if (doc.custom[key] && typeof doc.custom[key] === "object") {
50-
var specialKeys = ["$set", "$addToSet", "$push", "$pull", "$inc", "$min", "$max", "$setOnce"];
51-
for (var z = 0; z < specialKeys.length; z++) {
52-
if (doc.custom[key][specialKeys[z]]) {
53-
updateDoc["custom." + key] = doc.custom[key][specialKeys[z]];
54-
updateMe = true;
39+
var collections = ["drill_events", collectionName0];
40+
41+
for (var z1 = 0; z1 < collections.length; z1++) {
42+
var collectionName = collections[z1];
43+
console.log("Processing collection: " + collectionName);
44+
45+
if (collectionName === "drill_events") {
46+
query_drill.a = apps[i]._id + "";
47+
query_drill.e = "[CLY]_session";
48+
}
49+
else {
50+
delete query_drill.a;
51+
delete query_drill.e;
52+
}
53+
54+
const cursor = drillDb.collection(collectionName).find(query_drill, {"_id": 1, "custom": 1});
55+
//FOR EACH DOCUMENT
56+
var updates = [];
57+
while (await cursor.hasNext()) {
58+
var doc = await cursor.next();
59+
if (doc.custom) {
60+
var updateDoc = {};
61+
let updateMe = false;
62+
for (var key in doc.custom) {
63+
if (doc.custom[key] && typeof doc.custom[key] === "object") {
64+
var specialKeys = ["$set", "$addToSet", "$push", "$pull", "$inc", "$min", "$max", "$setOnce"];
65+
for (var z = 0; z < specialKeys.length; z++) {
66+
if (doc.custom[key][specialKeys[z]]) {
67+
updateDoc["custom." + key] = doc.custom[key][specialKeys[z]];
68+
updateMe = true;
69+
}
5570
}
5671
}
5772
}
73+
if (updateMe) {
74+
updates.push({
75+
'updateOne': {
76+
'filter': { '_id': doc._id },
77+
'update': { '$set': updateDoc },
78+
'upsert': false
79+
}
80+
});
81+
}
5882
}
59-
if (updateMe) {
60-
updates.push({
61-
'updateOne': {
62-
'filter': { '_id': doc._id },
63-
'update': { '$set': updateDoc },
64-
'upsert': false
65-
}
66-
});
83+
if (updates.length === 500) {
84+
if (dry_run) {
85+
console.log("DRY RUN: Would update " + updates.length + " docs in " + collectionName);
86+
console.log(JSON.stringify(updates));
87+
}
88+
else {
89+
console.log("updating");
90+
await drillDb.collection(collectionName).bulkWrite(updates, {"ordered": false});
91+
}
92+
updates = [];
6793
}
6894
}
69-
if (updates.length === 500) {
95+
if (updates.length > 0) {
7096
if (dry_run) {
7197
console.log("DRY RUN: Would update " + updates.length + " docs in " + collectionName);
7298
console.log(JSON.stringify(updates));
@@ -78,16 +104,6 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
78104
updates = [];
79105
}
80106
}
81-
if (updates.length > 0) {
82-
if (dry_run) {
83-
console.log("DRY RUN: Would update " + updates.length + " docs in " + collectionName);
84-
console.log(JSON.stringify(updates));
85-
}
86-
else {
87-
await drillDb.collection(collectionName).bulkWrite(updates, {"ordered": false});
88-
}
89-
updates = [];
90-
}
91107
}
92108
}
93109
catch (err) {

‎bin/scripts/fix-data/fix_null_uids.js‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
4242
Promise.each(users, function(user) {
4343
return new Promise(function(resolve) {
4444
var device_id;
45-
var events = [];
45+
var events = [{"key": ""}];
4646
if (user.did) {
4747
device_id = user.did;
4848
}
@@ -91,7 +91,12 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
9191
}
9292
else {
9393
var collection = "drill_events" + crypto.createHash('sha1').update(event.key + app._id).digest('hex');
94-
drillDb.collection(collection).aggregate([{$match: {did: device_id, uid: {"$exists": true}}}, {"$sort": {"ts": -1}}, {$limit: 1}, {$project: {did: 1, uid: 1}}], {"allowDiskUse": true}, function(err, res) {
94+
var query = {did: device_id, uid: {"$exists": true}};
95+
if (event.key === "") {
96+
collection = "drill_events";
97+
query.a = app._id + "";
98+
}
99+
drillDb.collection(collection).aggregate([{$match: query}, {"$sort": {"ts": -1}}, {$limit: 1}, {$project: {did: 1, uid: 1}}], {"allowDiskUse": true}, function(err, res) {
95100
if (err) {
96101
console.log(err);
97102
resolve1();

‎bin/scripts/fix-data/recheck_merges.js‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
6969
}
7070

7171
async function getDrillCollections(app_id) {
72-
var collections = [];
72+
var collections = ["drill_events"];
7373
try {
7474
var events = await countlyDb.collection("events").findOne({_id: common.db.ObjectID(app_id)});
7575
var list = ["[CLY]_session", "[CLY]_crash", "[CLY]_view", "[CLY]_action", "[CLY]_push_action", "[CLY]_push_sent", "[CLY]_star_rating", "[CLY]_nps", "[CLY]_survey", "[CLY]_apm_network", "[CLY]_apm_device", "[CLY]_consent"];
@@ -97,15 +97,19 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
9797
for (let i = 0; i < collections.length; i++) {
9898
const collection = collections[i].collectionName;
9999
try {
100-
const events = await drillDb.collection(collection).find({uid: old_uid}, {uid: 1, _id: 0}).limit(1).toArray();
100+
var query = {uid: old_uid};
101+
if (collection === "drill_events") {
102+
query = {uid: old_uid, 'a': app._id + ""};
103+
}
104+
const events = await drillDb.collection(collection).find(query, {uid: 1, _id: 0}).limit(1).toArray();
101105
if (!events || !events.length) {
102106
continue;
103107
}
104108
if (events && events[0]) {
105109
console.log("Found at least one event with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "updating to new uid", new_uid);
106110
try {
107111
if (!DRY_RUN) {
108-
await drillDb.collection(collection).update({uid: old_uid}, {'$set': {uid: new_uid}}, {multi: true});
112+
await drillDb.collection(collection).update(query, {'$set': {uid: new_uid}}, {multi: true});
109113
}
110114
}
111115
catch (err) {

‎bin/scripts/fix-data/recheck_merges_new.js‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
5151
}
5252

5353
async function getDrillCollections(app_id) {
54-
var collections = [];
54+
var collections = ["drill_events"];
5555
try {
5656
var events = await countlyDb.collection("events").findOne({_id: common.db.ObjectID(app_id)});
5757
var list = ["[CLY]_session", "[CLY]_crash", "[CLY]_view", "[CLY]_action", "[CLY]_push_action", "[CLY]_push_sent", "[CLY]_star_rating", "[CLY]_nps", "[CLY]_survey", "[CLY]_apm_network", "[CLY]_apm_device", "[CLY]_consent"];
@@ -80,15 +80,19 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
8080
for (let i = 0; i < collections.length; i++) {
8181
const collection = collections[i].collectionName;
8282
try {
83-
const events = await drillDb.collection(collection).find({uid: old_uid}, {uid: 1, _id: 0}).limit(1).toArray();
83+
var query = {uid: old_uid};
84+
if (collection === "drill_events") {
85+
query = {uid: old_uid, 'a': app._id + ""};
86+
}
87+
const events = await drillDb.collection(collection).find(query, {uid: 1, _id: 0}).limit(1).toArray();
8488
if (!events || !events.length) {
8589
continue;
8690
}
8791
if (events && events[0]) {
8892
console.log("Found at least one event with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "updating to new uid", new_uid);
8993
try {
9094
if (!DRY_RUN) {
91-
await drillDb.collection(collection).update({uid: old_uid}, {'$set': {uid: new_uid}}, {multi: true});
95+
await drillDb.collection(collection).update(query, {'$set': {uid: new_uid}}, {multi: true});
9296
}
9397
}
9498
catch (err) {

‎bin/scripts/modify-data/delete/delete_user_properties.js‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,23 @@ Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_dril
4242
done();
4343
}
4444
}, function() {
45-
db.close();
46-
dbDrill.close();
47-
console.log("done");
45+
//delete property from merged drill events collection
46+
var unset = {};
47+
if (PROPERTY.startsWith("custom") || PROPERTY.startsWith("cmp")) {
48+
unset[PROPERTY] = "";
49+
}
50+
else {
51+
unset["up." + PROPERTY] = "";
52+
}
53+
54+
dbDrill.collection("drill_events").updateMany({"a": (APP_ID + "")}, {$unset: unset}, function(err) {
55+
if (err) {
56+
console.log("Error", err);
57+
}
58+
db.close();
59+
dbDrill.close();
60+
console.log("done");
61+
});
4862
});
4963
}
5064
});

‎bin/scripts/performance-monitoring/apm_events_delete.js‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,19 @@ if (dry_run) {
1717

1818
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
1919
try {
20-
console.log('Deleting APM events for app_id: ' + APP_ID);
21-
await Promise.all([
22-
countlyDb.collection("apm").remove({app_id: APP_ID}),
23-
drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_apm_network" + APP_ID).digest('hex')).drop(),
24-
drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_apm_device" + APP_ID).digest('hex')).drop(),
25-
]);
26-
console.log("All done");
20+
if (!APP_ID) {
21+
console.log("Please set APP_ID");
22+
}
23+
else {
24+
console.log('Deleting APM events for app_id: ' + APP_ID);
25+
await Promise.all([
26+
countlyDb.collection("apm").remove({app_id: APP_ID}),
27+
drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_apm_network" + APP_ID).digest('hex')).drop(),
28+
drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_apm_device" + APP_ID).digest('hex')).drop(),
29+
drillDb.collection("drill_events").remove({"a": APP_ID, "e": {$in: ["[CLY]_apm_device", "[CLY]_apm_network"]}})
30+
]);
31+
console.log("All done");
32+
}
2733
}
2834
catch (error) {
2935
console.log("ERROR: ");

‎plugins/crashes/frontend/public/templates/overview.html‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@
4646
<cly-diff-helper v-if="(canUserDelete || canUserUpdate)" :diff="selectedCrashgroups" @discard="scope.unpatch()" :isModal=true>
4747
<template v-slot:main>
4848
<div class="bu-mr-0 bu-is-flex bu-is-justify-content-flex-end bu-is-align-items-center cly-vue-user-selected" style="height: 100%;">
49+
<span class="selected-count-blue bu-pl-1 text-medium">
50+
<span style="background-color:#0166D6; color:white; padding:3px 7px; border-radius:4px;">{{selectedCrashgroups.length}}</span><span class="bu-is-lowercase text-medium color-cool-gray-50 bu-pl-1">{{i18n('common.selected')}}</span>
51+
</span>
52+
<span class="vertical-divider bu-mr-4 bu-ml-4"></span>
4953
<cly-more-options v-if="canUserUpdate" class="bu-mr-3" size="small" text="Change Status" type="default" icon="cly-io cly-io-refresh bu-mr-2" :showArrows=true @command="setSelectedAs($event)" >
5054
<el-dropdown-item command="resolved" icon="cly-io cly-io-check-circle">{{ i18n('crashes.action-resolved') }}</el-dropdown-item>
5155
<el-dropdown-item command="resolving" icon="cly-io cly-io-clock">{{ i18n('crashes.action-resolving') }}</el-dropdown-item>

‎plugins/data-manager/frontend/public/templates/event-groups.html‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,10 @@
5151
<cly-diff-helper v-if="canUserUpdate || canUserDelete" class="action-bar" :diff="scope.diff" @discard="scope.unpatch()" :isModal=true>
5252
<template v-slot:main>
5353
<div class="bu-mr-0 bu-is-flex bu-is-justify-content-flex-end bu-is-align-items-center cly-vue-user-selected" style="height: 100%;">
54+
<span class="selected-count-blue bu-pl-1 text-medium">
55+
<span style="background-color:#0166D6; color:white; padding:3px 7px; border-radius:4px;">{{scope.diff.length}}</span><span class="bu-is-lowercase text-medium color-cool-gray-50 bu-pl-1">{{i18n('common.selected')}}</span>
56+
</span>
57+
<span class="vertical-divider bu-mr-4 bu-ml-4"></span>
5458
<cly-more-options size="small" :text=" i18n('data-manager.change-visibility') " type="default" icon="cly-io cly-io-eye bu-mr-2" :widthSameAsTrigger="true" :showArrows=true @command="handleChangeVisibility($event, scope.diff)">
5559
<el-dropdown-item command="visible"><i class="cly-io cly-io-eye"></i>{{ i18n('data-manager.visible') }}</el-dropdown-item>
5660
<el-dropdown-item command="hidden"><i class="cly-io cly-io-eye-off"></i>{{ i18n('data-manager.hidden') }}</el-dropdown-item>

‎plugins/data-manager/frontend/public/templates/events-default.html‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,10 @@
136136
<cly-diff-helper v-if="canUserUpdate || canUserDelete" :diff="scope.diff" @discard="scope.unpatch()" :isModal=true>
137137
<template v-slot:main>
138138
<div class="bu-mr-0 bu-is-flex bu-is-justify-content-flex-end bu-is-align-items-center cly-vue-user-selected" style="height: 100%;">
139+
<span class="selected-count-blue bu-pl-1 text-medium"></span>
140+
<span style="background-color:#0166D6; color:white; padding:3px 7px; border-radius:4px;">{{scope.diff.length}}</span><span class="bu-is-lowercase text-medium color-cool-gray-50 bu-pl-1">{{i18n('common.selected')}}</span>
141+
</span>
142+
<span class="vertical-divider bu-mr-4 bu-ml-4"></span>
139143
<cly-more-options class="bu-mr-3" size="small" :text="i18n('data-manager.change-category')" type="default" icon="cly-io cly-io-hashtag bu-mr-2" :widthSameAsTrigger="true" :showArrows=true @command="handleChangeCategory($event, scope.diff)">
140144
<el-dropdown-item command=null><i class="cly-io cly-io-hashtag"></i>{{ i18n('data-manager.uncategorized') }}</el-dropdown-item>
141145
<el-dropdown-item :command="cat._id" :key="idx" v-for="(cat, idx) in categories"><i class="cly-io cly-io-hashtag"></i>{{cat.name}}</el-dropdown-item>

0 commit comments

Comments
 (0)