Merge pull request #5715 from Countly/anna/next

[core] Ability to mark reports as 'dirty' to make sure they are regenerated in full on job. Used in drill and formula reports on cohort update.

Author: Cookiezaurs

api/jobs/task.js

@@ -56,6 +56,10 @@ class MonitorJob extends job.Job {
                 return true;
             }
 
+            if (task.dirty) {
+                return true;
+            }
+
             if ((now + duration - lastStart) / 1000 >= interval) {
                 return true;
             }
@@ -74,7 +78,8 @@ class MonitorJob extends job.Job {
                         taskmanager.rerunTask({
                             db: common.db,
                             id: task._id,
-                            autoUpdate: true
+                            autoUpdate: true,
+                            dirty: task.dirty
                         }, function(e) {
                             if (e) {
                                 log.e(e, e.stack);

api/utils/taskmanager.js

@@ -323,6 +323,89 @@ taskmanager.createTask = function(options, callback) {
     }
 };
 
+var checkIfAllRulesMatch = function(rules, data) {
+    var match = true;
+    for (var key in rules) {
+        if (data[key]) {
+            if (rules[key] === data[key]) {
+                continue;
+            }
+            else {
+                if (typeof rules[key] === "object") {
+                    if (!checkIfAllRulesMatch(rules[key], data[key])) {
+                        return false;
+                    }
+                }
+                else {
+                    if (data[key].$in) {
+                        if (data[key].$in.indexOf(rules[key]) === -1) {
+                            return false;
+                        }
+                    }
+                    else if (data[key].$nin) {
+                        if (data[key].$nin.indexOf(rules[key]) !== -1) {
+                            return false;
+                        }
+                    }
+                    else {
+                        return false;
+                    }
+                }
+            }
+        }
+        else {
+            return false;
+        }
+    }
+    return match;
+};
+
+taskmanager.markReportsDirtyBasedOnRule = function(options, callback) {
+    common.db.collection("long_tasks").find({
+        autoRefresh: true,
+    }).toArray(function(err, tasks) {
+        var ids_to_mark_dirty = [];
+        if (err) {
+            log.e("Error while fetching tasks", err);
+            if (callback && typeof callback === "function") {
+                callback();
+            }
+            return;
+
+        }
+        tasks = tasks || [];
+        for (var z = 0; z < tasks.length; z++) {
+            try {
+                var req = JSON.parse(tasks[z].request);
+                if (checkIfAllRulesMatch(options.rules, req.json.queryObject)) {
+                    ids_to_mark_dirty.push(tasks[z]._id);
+                }
+            }
+            catch (e) {
+                log.e(' got error while process task request parse', e);
+            }
+
+        }
+        if (ids_to_mark_dirty.length > 0) {
+            common.db.collection("long_tasks").updateMany({_id: {$in: ids_to_mark_dirty}}, {$set: {dirty: new Date().getTime()}}, function(err3) {
+                if (err3) {
+                    log.e("Error while updating reports", err3);
+                }
+                if (callback && typeof callback === "function") {
+                    callback();
+                }
+            });
+        }
+        else {
+            if (callback && typeof callback === "function") {
+                callback();
+            }
+        }
+
+
+    });
+
+};
 /**
 * Save result from the task
 * @param {object} options - options for the task
@@ -388,6 +471,9 @@ taskmanager.saveResult = function(options, data, callback) {
         options.db.collection("long_tasks").update({_id: options.subtask}, updateObj, {'upsert': false}, function() {});
     }
     options.db.collection("long_tasks").findOne({_id: options.id}, function(error, task) {
+        if (task && task.dirty && task.dirty < task.start) {
+            update.dirty = false;
+        }
         if (options.gridfs || (task && task.gridfs)) {
             //let's store it in gridfs
             update.data = {};
@@ -886,6 +972,8 @@ taskmanager.errorResults = function(options, callback) {
 * @param {object} options - options for the task
 * @param {object} options.db - database connection
 * @param {string} options.id - id of the task result
+* @param {boolean} options.autoUpdate - if auto update is needed or not
+* @param {boolean} options.dirty - if dirty is true then it means some part of report is wrong. It should be regenerated fully.
 * @param {funciton} callback - callback for the result
 */
 taskmanager.rerunTask = function(options, callback) {
@@ -990,7 +1078,7 @@ taskmanager.rerunTask = function(options, callback) {
                     reqData.json.period = JSON.stringify(reqData.json.period);
                 }
                 options.subtask = res.subtask;
-                reqData.json.autoUpdate = options.autoUpdate || false;
+                reqData.json.autoUpdate = ((!options.dirty) && (options.autoUpdate || false)); //If dirty  set autoUpdate to false
                 if (!reqData.json.api_key && res.creator) {
                     options.db.collection("members").findOne({_id: common.db.ObjectID(res.creator)}, function(err1, member) {
                         if (member && member.api_key) {