Skip to content

Changes for scirpts /data-reports to support merged drill colelctions #5600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 25, 2024
Merged

Changes for scirpts /data-reports to support merged drill colelctions #5600

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8d59e73
Changes for scipts /data-reports to support merged drill colelctions
Cookiezaurs Sep 25, 2024
4239c02
Merge branch 'next' into anna/next
Cookiezaurs Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions bin/scripts/data-reports/check_user_mismatch.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,25 @@ var query = {"ts": {"$gt": 1709762400000}}; //Change this query to set date rang
var eventKey = "[CLY]_session"; //Write in your event key
var appID = "6075f94b7e5e0d392902520c"; //Write in YOUR app ID

var union_with_old_collection = false; //False if all sessions are stored in drill_events collection

console.log("Running for:" + appID + " " + eventKey + " " + JSON.stringify(query));

Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {

var pipeline = [];
pipeline.push({"$match": query});
pipeline.push({"$group": {"_id": null, "uid": {"$addToSet": "$uid"}}});
var query2 = JSON.parse(JSON.stringify(query));
query2.e = eventKey;
query2.a = appID;
pipeline.push({"$match": query2});

var collection = "drill_events" + crypto.createHash('sha1').update(eventKey + appID).digest('hex');
if (union_with_old_collection) {
var collection = "drill_events" + crypto.createHash('sha1').update(eventKey + appID).digest('hex');
pipeline.push({"$unionWith": { "coll": collection, "pipeline": [{"$match": query}] }});
}
pipeline.push({"$group": {"_id": null, "uid": {"$addToSet": "$uid"}}});

drillDb.collection(collection).aggregate(pipeline, function(err, res) {
drillDb.collection("drill_events").aggregate(pipeline, function(err, res) {
if (err) {
console.log(err);
countlyDb.close();
Expand Down Expand Up @@ -67,6 +75,9 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
console.log("Users missing in app_users collection: " + missingOnes.length);
console.log(JSON.stringify(missingOnes));
}
else {
console.log("All users matched");
}

countlyDb.close();
drillDb.close();
Expand Down
106 changes: 36 additions & 70 deletions bin/scripts/data-reports/drill_data_cd.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,14 @@
*/

//period to check
var startDate = new Date("2024-04-2T00:00:00");
var endDate = new Date("2024-04-3T00:00:00");
var startDate = new Date("2021-04-02T00:00:00");
var endDate = new Date("2025-04-03T00:00:00");
var apps = []; //Put in your APP ID like ["3469834986y34968y206y2"]

var internal_events = [];
var verbose = true;

var Promise = require("bluebird");
var crypto = require('crypto');
var pluginManager = require("../../../plugins/pluginManager");
var results = {};
var dates = {};

function output_data(dates, results) {
var datesList = Object.keys(dates).sort(function(a, b) {
Expand Down Expand Up @@ -70,80 +68,48 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
queryApps = {_id: {$in: apps}};
}

countlyDb.collection("apps").find(queryApps).toArray(function(err, apps) {
countlyDb.collection("apps").find(queryApps).toArray(function(err, apps2) {
if (err) {
console.log(err);
countlyDb.close();
drillDb.close();
}
else {
Promise.each(apps, function(app) {
console.log('------' + app.name + '------');
return new Promise(function(resolve) {
var dates = {};
var results = {};
//fetch events list
countlyDb.collection("events").findOne({"_id": app._id}, {"list": 1}, function(err, events) {
events = events || [];
var list = events.list || [];
if (verbose) {
console.log(list.length + " events found");
}
for (var z = 0; z < internal_events.length; z++) {
if (list.indexOf(internal_events[z]) === -1) {
list.push(internal_events[z]);
}
var query = {"cd": {"$gte": startDate, "$lt": endDate}};
if (apps.length > 0) {

var a = [];
for (var i = 0; i < apps2.length; i++) {
a.push(apps2[i]._id + "");
}
query.a = {$in: a};
}
var pipeline = [{"$match": query}, {"$group": {"_id": {"a": "$a", "e": "$e", "d": "$d"}, "c": {"$sum": 1}}}];
drillDb.collection("drill_events").aggregate(pipeline, {"allowDiskUse": true}).toArray(function(err, data) {
if (err) {
console.log(err);
}
if (data && data.length > 0) {
results["_total"] = results["_total"] || {};

for (var z = 0; z < data.length; z++) {
var event = data[z]._id.e;
if (!dates[data[z]._id.d]) {
dates[data[z]._id.d] = true;
}
Promise.each(list, function(event) {
return new Promise(function(resolve2) {
//get hashed drill collection name
let collection = "drill_events" + crypto.createHash('sha1').update(event + app._id).digest('hex');
if (verbose) {
console.log(collection);
}
var pipeline = [
{"$match": {"cd": {"$gte": startDate, "$lt": endDate}}},
{"$group": {"_id": "$d", "c": {"$sum": 1}}}
];
drillDb.collection(collection).aggregate(pipeline, {"allowDiskUse": true}).toArray(function(err, data) {
if (err) {
console.log(err);
}
if (data.length > 0) {
results["_total"] = results["_total"] || {};
results[event] = {};
for (var z = 0; z < data.length; z++) {
if (!dates[data[z]._id]) {
dates[data[z]._id] = true;
}
results[event][data[z]._id] = data[z].c;
results["_total"][data[z]._id] = results["_total"][data[z]._id] || 0;
results["_total"][data[z]._id] += data[z].c || 0;
}
}
resolve2();
});
});
}).then(function() {
//output
//sort dates
output_data(dates, results);
resolve();
}).catch(function(err) {
output_data(dates, results);
console.log(err);
resolve();
});
});
});
}).then(function() {
countlyDb.close();
drillDb.close();
}).catch(function(err) {
console.log(err);
results[data[z]._id.e] = {};
results[event][data[z]._id.d] = data[z].c;
results["_total"][data[z]._id.d] = results["_total"][data[z]._id.d] || 0;
results["_total"][data[z]._id.d] += data[z].c || 0;
}
output_data(dates, results);
}

countlyDb.close();
drillDb.close();
});


}
});
});
149 changes: 149 additions & 0 deletions bin/scripts/data-reports/old_scripts/drill_data_cd.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
/**
* Check for which dates data was recorded
* requires index on cd collections or else will be slower
* Server: countly
* Path: $(countly dir)/bin/scripts/data-reports
* Command: node drill_data_cd.js
*/

//period to check
var startDate = new Date("2024-04-2T00:00:00");
var endDate = new Date("2024-04-3T00:00:00");
var apps = []; //Put in your APP ID like ["3469834986y34968y206y2"]

var internal_events = [];
var verbose = true;

var Promise = require("bluebird");
var crypto = require('crypto');
var pluginManager = require("../../../plugins/pluginManager");

function output_data(dates, results) {
var datesList = Object.keys(dates).sort(function(a, b) {
var d1 = a.split(":");
var d2 = b.split(":");
for (var z = 0; z < d1.length; z++) {
d1[z] = parseInt(d1[z]);
d2[z] = parseInt(d2[z]);
}

for (var k = 0; k < d1.length; k++) {
if (d1[k] > d2[k]) {
return 1;
}
else if (d1[k] < d2[k]) {
return -1;
}
}
return 1;

});
//output header line
console.log("Event," + datesList.join(","));
for (var key in results) {
if (key !== "_total") {
var line = key;
for (var z = 0; z < datesList.length; z++) {
line += "," + (results[key][datesList[z]] || 0);
}
console.log(line);
}
//output total



}
var line2 = "Total";
for (var z1 = 0; z1 < datesList.length; z1++) {
line2 += "," + (results["_total"][datesList[z1]] || 0);
}
console.log(line2);

}
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
//get apps
var queryApps = {};
if (apps && apps.length) {
for (var z = 0; z < apps.length; z++) {
apps[z] = countlyDb.ObjectID(apps[z]);
}
queryApps = {_id: {$in: apps}};
}

countlyDb.collection("apps").find(queryApps).toArray(function(err, apps) {
if (err) {
console.log(err);
countlyDb.close();
drillDb.close();
}
else {
Promise.each(apps, function(app) {
console.log('------' + app.name + '------');
return new Promise(function(resolve) {
var dates = {};
var results = {};
//fetch events list
countlyDb.collection("events").findOne({"_id": app._id}, {"list": 1}, function(err, events) {
events = events || [];
var list = events.list || [];
if (verbose) {
console.log(list.length + " events found");
}
for (var z = 0; z < internal_events.length; z++) {
if (list.indexOf(internal_events[z]) === -1) {
list.push(internal_events[z]);
}
}
Promise.each(list, function(event) {
return new Promise(function(resolve2) {
//get hashed drill collection name
let collection = "drill_events" + crypto.createHash('sha1').update(event + app._id).digest('hex');
if (verbose) {
console.log(collection);
}
var pipeline = [
{"$match": {"cd": {"$gte": startDate, "$lt": endDate}}},
{"$group": {"_id": "$d", "c": {"$sum": 1}}}
];
drillDb.collection(collection).aggregate(pipeline, {"allowDiskUse": true}).toArray(function(err, data) {
if (err) {
console.log(err);
}
if (data.length > 0) {
results["_total"] = results["_total"] || {};
results[event] = {};
for (var z = 0; z < data.length; z++) {
if (!dates[data[z]._id]) {
dates[data[z]._id] = true;
}
results[event][data[z]._id] = data[z].c;
results["_total"][data[z]._id] = results["_total"][data[z]._id] || 0;
results["_total"][data[z]._id] += data[z].c || 0;
}
}
resolve2();
});
});
}).then(function() {
//output
//sort dates
output_data(dates, results);
resolve();
}).catch(function(err) {
output_data(dates, results);
console.log(err);
resolve();
});
});
});
}).then(function() {
countlyDb.close();
drillDb.close();
}).catch(function(err) {
console.log(err);
countlyDb.close();
drillDb.close();
});
}
});
});
0 ...cripts/data-reports/event_list_no_data.js → ...reports/old_scripts/event_list_no_data.js
View file
Open in desktop
File renamed without changes.
Loading