Skip to content

Updated scripts #5595

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Sep 23, 2024
Merged

Updated scripts #5595

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
7fa9008
Changes for apm_events_delete script to delete data also from new dri…
Cookiezaurs Sep 23, 2024
1498856
Changes for delete_users_properties.js
Cookiezaurs Sep 23, 2024
14e1aef
Changes for merges rechecking scripts to support new drill_events col…
Cookiezaurs Sep 23, 2024
1df29aa
Fixes for fix_null_uids.js to support new drill_events collection
Cookiezaurs Sep 23, 2024
566421c
Changes for fix_drill_costom_props.js to support new drill_events col…
Cookiezaurs Sep 23, 2024
0faea92
Add "selected" count to crashes overview table diff helper
Cookiezaurs Sep 23, 2024
be2e41e
Added selected count in data manager
Cookiezaurs Sep 23, 2024
7f545de
Merge branch 'next' into anna/next
Cookiezaurs Sep 23, 2024
55248f7
eslint
Cookiezaurs Sep 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 52 additions & 36 deletions bin/scripts/fix-data/fix_drill_custom_props.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,39 +34,65 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
//PROCESS COLLECTIONS FOR EACH APP
for (let i = 0; i < apps.length; i++) {
console.log("Processing app: " + apps[i].name);
var collectionName = drillCommon.getCollectionName("[CLY]_session", apps[i]._id + "");
console.log("Processing collection: " + collectionName);
var collectionName0 = drillCommon.getCollectionName("[CLY]_session", apps[i]._id + "");

const cursor = drillDb.collection(collectionName).find(query_drill, {"_id": 1, "custom": 1});
//FOR EACH DOCUMENT
var updates = [];
while (await cursor.hasNext()) {
var doc = await cursor.next();
if (doc.custom) {
var updateDoc = {};
let updateMe = false;
for (var key in doc.custom) {
if (doc.custom[key] && typeof doc.custom[key] === "object") {
var specialKeys = ["$set", "$addToSet", "$push", "$pull", "$inc", "$min", "$max", "$setOnce"];
for (var z = 0; z < specialKeys.length; z++) {
if (doc.custom[key][specialKeys[z]]) {
updateDoc["custom." + key] = doc.custom[key][specialKeys[z]];
updateMe = true;
var collections = ["drill_events", collectionName0];

for (var z1 = 0; z1 < collections.length; z1++) {
var collectionName = collections[z1];
console.log("Processing collection: " + collectionName);

if (collectionName === "drill_events") {
query_drill.a = apps[i]._id + "";
query_drill.e = "[CLY]_session";
}
else {
delete query_drill.a;
delete query_drill.e;
}

const cursor = drillDb.collection(collectionName).find(query_drill, {"_id": 1, "custom": 1});
//FOR EACH DOCUMENT
var updates = [];
while (await cursor.hasNext()) {
var doc = await cursor.next();
if (doc.custom) {
var updateDoc = {};
let updateMe = false;
for (var key in doc.custom) {
if (doc.custom[key] && typeof doc.custom[key] === "object") {
var specialKeys = ["$set", "$addToSet", "$push", "$pull", "$inc", "$min", "$max", "$setOnce"];
for (var z = 0; z < specialKeys.length; z++) {
if (doc.custom[key][specialKeys[z]]) {
updateDoc["custom." + key] = doc.custom[key][specialKeys[z]];
updateMe = true;
}
}
}
}
if (updateMe) {
updates.push({
'updateOne': {
'filter': { '_id': doc._id },
'update': { '$set': updateDoc },
'upsert': false
}
});
}
}
if (updateMe) {
updates.push({
'updateOne': {
'filter': { '_id': doc._id },
'update': { '$set': updateDoc },
'upsert': false
}
});
if (updates.length === 500) {
if (dry_run) {
console.log("DRY RUN: Would update " + updates.length + " docs in " + collectionName);
console.log(JSON.stringify(updates));
}
else {
console.log("updating");
await drillDb.collection(collectionName).bulkWrite(updates, {"ordered": false});
}
updates = [];
}
}
if (updates.length === 500) {
if (updates.length > 0) {
if (dry_run) {
console.log("DRY RUN: Would update " + updates.length + " docs in " + collectionName);
console.log(JSON.stringify(updates));
Expand All @@ -78,16 +104,6 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
updates = [];
}
}
if (updates.length > 0) {
if (dry_run) {
console.log("DRY RUN: Would update " + updates.length + " docs in " + collectionName);
console.log(JSON.stringify(updates));
}
else {
await drillDb.collection(collectionName).bulkWrite(updates, {"ordered": false});
}
updates = [];
}
}
}
catch (err) {
Expand Down
9 changes: 7 additions & 2 deletions bin/scripts/fix-data/fix_null_uids.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
Promise.each(users, function(user) {
return new Promise(function(resolve) {
var device_id;
var events = [];
var events = [{"key": ""}];
if (user.did) {
device_id = user.did;
}
Expand Down Expand Up @@ -91,7 +91,12 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
}
else {
var collection = "drill_events" + crypto.createHash('sha1').update(event.key + app._id).digest('hex');
drillDb.collection(collection).aggregate([{$match: {did: device_id, uid: {"$exists": true}}}, {"$sort": {"ts": -1}}, {$limit: 1}, {$project: {did: 1, uid: 1}}], {"allowDiskUse": true}, function(err, res) {
var query = {did: device_id, uid: {"$exists": true}};
if (event.key === "") {
collection = "drill_events";
query.a = app._id + "";
}
drillDb.collection(collection).aggregate([{$match: query}, {"$sort": {"ts": -1}}, {$limit: 1}, {$project: {did: 1, uid: 1}}], {"allowDiskUse": true}, function(err, res) {
if (err) {
console.log(err);
resolve1();
Expand Down
10 changes: 7 additions & 3 deletions bin/scripts/fix-data/recheck_merges.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
}

async function getDrillCollections(app_id) {
var collections = [];
var collections = ["drill_events"];
try {
var events = await countlyDb.collection("events").findOne({_id: common.db.ObjectID(app_id)});
var list = ["[CLY]_session", "[CLY]_crash", "[CLY]_view", "[CLY]_action", "[CLY]_push_action", "[CLY]_push_sent", "[CLY]_star_rating", "[CLY]_nps", "[CLY]_survey", "[CLY]_apm_network", "[CLY]_apm_device", "[CLY]_consent"];
Expand Down Expand Up @@ -97,15 +97,19 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
for (let i = 0; i < collections.length; i++) {
const collection = collections[i].collectionName;
try {
const events = await drillDb.collection(collection).find({uid: old_uid}, {uid: 1, _id: 0}).limit(1).toArray();
var query = {uid: old_uid};
if (collection === "drill_events") {
query = {uid: old_uid, 'a': app._id + ""};
}
const events = await drillDb.collection(collection).find(query, {uid: 1, _id: 0}).limit(1).toArray();
if (!events || !events.length) {
continue;
}
if (events && events[0]) {
console.log("Found at least one event with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "updating to new uid", new_uid);
try {
if (!DRY_RUN) {
await drillDb.collection(collection).update({uid: old_uid}, {'$set': {uid: new_uid}}, {multi: true});
await drillDb.collection(collection).update(query, {'$set': {uid: new_uid}}, {multi: true});
}
}
catch (err) {
Expand Down
10 changes: 7 additions & 3 deletions bin/scripts/fix-data/recheck_merges_new.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
}

async function getDrillCollections(app_id) {
var collections = [];
var collections = ["drill_events"];
try {
var events = await countlyDb.collection("events").findOne({_id: common.db.ObjectID(app_id)});
var list = ["[CLY]_session", "[CLY]_crash", "[CLY]_view", "[CLY]_action", "[CLY]_push_action", "[CLY]_push_sent", "[CLY]_star_rating", "[CLY]_nps", "[CLY]_survey", "[CLY]_apm_network", "[CLY]_apm_device", "[CLY]_consent"];
Expand Down Expand Up @@ -80,15 +80,19 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
for (let i = 0; i < collections.length; i++) {
const collection = collections[i].collectionName;
try {
const events = await drillDb.collection(collection).find({uid: old_uid}, {uid: 1, _id: 0}).limit(1).toArray();
var query = {uid: old_uid};
if (collection === "drill_events") {
query = {uid: old_uid, 'a': app._id + ""};
}
const events = await drillDb.collection(collection).find(query, {uid: 1, _id: 0}).limit(1).toArray();
if (!events || !events.length) {
continue;
}
if (events && events[0]) {
console.log("Found at least one event with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "updating to new uid", new_uid);
try {
if (!DRY_RUN) {
await drillDb.collection(collection).update({uid: old_uid}, {'$set': {uid: new_uid}}, {multi: true});
await drillDb.collection(collection).update(query, {'$set': {uid: new_uid}}, {multi: true});
}
}
catch (err) {
Expand Down
20 changes: 17 additions & 3 deletions bin/scripts/modify-data/delete/delete_user_properties.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,23 @@ Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_dril
done();
}
}, function() {
db.close();
dbDrill.close();
console.log("done");
//delete property from merged drill events collection
var unset = {};
if (PROPERTY.startsWith("custom") || PROPERTY.startsWith("cmp")) {
unset[PROPERTY] = "";
}
else {
unset["up." + PROPERTY] = "";
}

dbDrill.collection("drill_events").updateMany({"a": (APP_ID + "")}, {$unset: unset}, function(err) {
if (err) {
console.log("Error", err);
}
db.close();
dbDrill.close();
console.log("done");
});
});
}
});
Expand Down
20 changes: 13 additions & 7 deletions bin/scripts/performance-monitoring/apm_events_delete.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,19 @@ if (dry_run) {

Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
try {
console.log('Deleting APM events for app_id: ' + APP_ID);
await Promise.all([
countlyDb.collection("apm").remove({app_id: APP_ID}),
drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_apm_network" + APP_ID).digest('hex')).drop(),
drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_apm_device" + APP_ID).digest('hex')).drop(),
]);
console.log("All done");
if (!APP_ID) {
console.log("Please set APP_ID");
}
else {
console.log('Deleting APM events for app_id: ' + APP_ID);
await Promise.all([
countlyDb.collection("apm").remove({app_id: APP_ID}),
drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_apm_network" + APP_ID).digest('hex')).drop(),
drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_apm_device" + APP_ID).digest('hex')).drop(),
drillDb.collection("drill_events").remove({"a": APP_ID, "e": {$in: ["[CLY]_apm_device", "[CLY]_apm_network"]}})
]);
console.log("All done");
}
}
catch (error) {
console.log("ERROR: ");
Expand Down
4 changes: 4 additions & 0 deletions plugins/crashes/frontend/public/templates/overview.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@
<cly-diff-helper v-if="(canUserDelete || canUserUpdate)" :diff="selectedCrashgroups" @discard="scope.unpatch()" :isModal=true>
<template v-slot:main>
<div class="bu-mr-0 bu-is-flex bu-is-justify-content-flex-end bu-is-align-items-center cly-vue-user-selected" style="height: 100%;">
<span class="selected-count-blue bu-pl-1 text-medium">
<span style="background-color:#0166D6; color:white; padding:3px 7px; border-radius:4px;">{{selectedCrashgroups.length}}</span><span class="bu-is-lowercase text-medium color-cool-gray-50 bu-pl-1">{{i18n('common.selected')}}</span>
</span>
<span class="vertical-divider bu-mr-4 bu-ml-4"></span>
<cly-more-options v-if="canUserUpdate" class="bu-mr-3" size="small" text="Change Status" type="default" icon="cly-io cly-io-refresh bu-mr-2" :showArrows=true @command="setSelectedAs($event)" >
<el-dropdown-item command="resolved" icon="cly-io cly-io-check-circle">{{ i18n('crashes.action-resolved') }}</el-dropdown-item>
<el-dropdown-item command="resolving" icon="cly-io cly-io-clock">{{ i18n('crashes.action-resolving') }}</el-dropdown-item>
Expand Down
4 changes: 4 additions & 0 deletions plugins/data-manager/frontend/public/templates/event-groups.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@
<cly-diff-helper v-if="canUserUpdate || canUserDelete" class="action-bar" :diff="scope.diff" @discard="scope.unpatch()" :isModal=true>
<template v-slot:main>
<div class="bu-mr-0 bu-is-flex bu-is-justify-content-flex-end bu-is-align-items-center cly-vue-user-selected" style="height: 100%;">
<span class="selected-count-blue bu-pl-1 text-medium">
<span style="background-color:#0166D6; color:white; padding:3px 7px; border-radius:4px;">{{scope.diff.length}}</span><span class="bu-is-lowercase text-medium color-cool-gray-50 bu-pl-1">{{i18n('common.selected')}}</span>
</span>
<span class="vertical-divider bu-mr-4 bu-ml-4"></span>
<cly-more-options size="small" :text=" i18n('data-manager.change-visibility') " type="default" icon="cly-io cly-io-eye bu-mr-2" :widthSameAsTrigger="true" :showArrows=true @command="handleChangeVisibility($event, scope.diff)">
<el-dropdown-item command="visible"><i class="cly-io cly-io-eye"></i>{{ i18n('data-manager.visible') }}</el-dropdown-item>
<el-dropdown-item command="hidden"><i class="cly-io cly-io-eye-off"></i>{{ i18n('data-manager.hidden') }}</el-dropdown-item>
Expand Down
4 changes: 4 additions & 0 deletions plugins/data-manager/frontend/public/templates/events-default.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,10 @@
<cly-diff-helper v-if="canUserUpdate || canUserDelete" :diff="scope.diff" @discard="scope.unpatch()" :isModal=true>
<template v-slot:main>
<div class="bu-mr-0 bu-is-flex bu-is-justify-content-flex-end bu-is-align-items-center cly-vue-user-selected" style="height: 100%;">
<span class="selected-count-blue bu-pl-1 text-medium"></span>
<span style="background-color:#0166D6; color:white; padding:3px 7px; border-radius:4px;">{{scope.diff.length}}</span><span class="bu-is-lowercase text-medium color-cool-gray-50 bu-pl-1">{{i18n('common.selected')}}</span>
</span>
<span class="vertical-divider bu-mr-4 bu-ml-4"></span>
<cly-more-options class="bu-mr-3" size="small" :text="i18n('data-manager.change-category')" type="default" icon="cly-io cly-io-hashtag bu-mr-2" :widthSameAsTrigger="true" :showArrows=true @command="handleChangeCategory($event, scope.diff)">
<el-dropdown-item command=null><i class="cly-io cly-io-hashtag"></i>{{ i18n('data-manager.uncategorized') }}</el-dropdown-item>
<el-dropdown-item :command="cat._id" :key="idx" v-for="(cat, idx) in categories"><i class="cly-io cly-io-hashtag"></i>{{cat.name}}</el-dropdown-item>
Expand Down
Loading