Skip to content

Commit 3dd72ed

Browse files
pnrgencpnrgenc
authored
Merge pull request #6681 from Countly/compliance-hub-clickhouse-support
feat: adding clickhouse support to the compliance-hub plugin
2 parents b918445 + 54e5c8e commit 3dd72ed

File tree

5 files changed

+511
-117
lines changed

5 files changed

+511
-117
lines changed

‎plugins/compliance-hub/api/api.js‎

Lines changed: 93 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ var plugin = {},
66
fetch = require('../../../api/parts/data/fetch.js'),
77
plugins = require('../../pluginManager.js'),
88
log = common.log('compliance-hub:api'),
9+
consentQueries = require('./queries/consents'),
910
{ validateRead } = require('../../../api/utils/rights.js');
1011

1112
const FEATURE_NAME = 'compliance_hub';
@@ -54,128 +55,119 @@ const FEATURE_NAME = 'compliance_hub';
5455
});
5556
break;
5657
}
57-
case 'searchDrill': {
58+
case 'search': {
5859
if (!params.qstring.app_id) {
5960
common.returnMessage(params, 400, 'Missing parameter "app_id"');
6061
return false;
6162
}
6263
validateRead(params, FEATURE_NAME, function() {
63-
var query = params.qstring.query || {};
64-
if (typeof query === "string" && query.length) {
65-
try {
66-
query = JSON.parse(query);
64+
try {
65+
var columns = ["device_id", "device_id", "uid", "type", "after", "ts"];
66+
var checkOb;
67+
if (params.qstring.iSortCol_0 && params.qstring.sSortDir_0 && columns[params.qstring.iSortCol_0]) {
68+
checkOb = {};
69+
checkOb[columns[params.qstring.iSortCol_0]] = (params.qstring.sSortDir_0 === "asc") ? 1 : -1;
6770
}
68-
catch (ex) {
69-
query = {};
71+
else if (params.qstring.sort && typeof params.qstring.sort === 'object' && Object.keys(params.qstring.sort).length) {
72+
checkOb = params.qstring.sort;
7073
}
71-
}
72-
query = query || {};
73-
query.a = params.qstring.app_id;
74-
query.e = "[CLY]_consent";
75-
76-
log.e(JSON.stringify(query));
77-
78-
common.drillDb.collection("drill_events").count(query, function(err, total) {
79-
if (err) {
80-
common.returnMessage(params, 400, err);
74+
else {
75+
checkOb = {};
8176
}
82-
else if (total > 0) {
83-
params.qstring.query = params.qstring.query || params.qstring.filter || {};
84-
params.qstring.project = params.qstring.project || params.qstring.projection || {};
85-
86-
params.qstring.query = params.qstring.query || {};
87-
params.qstring.query.a = params.qstring.app_id;
88-
params.qstring.query.e = "[CLY]_consent";
89-
90-
if (typeof params.qstring.query === "string" && params.qstring.query.length) {
91-
try {
92-
params.qstring.query = JSON.parse(params.qstring.query);
93-
}
94-
catch (ex) {
95-
params.qstring.query = {};
77+
var queryParams = {
78+
appID: params.qstring.app_id,
79+
period: params.qstring.period,
80+
qstring: params.qstring,
81+
sSearch: params.qstring.sSearch,
82+
sort: checkOb,
83+
project: params.qstring.project || params.qstring.projection || {}
84+
};
85+
86+
let rawFilter = params.qstring.filter || params.qstring.query;
87+
if (rawFilter) {
88+
try {
89+
if (typeof rawFilter === 'string' && rawFilter.length) {
90+
rawFilter = JSON.parse(rawFilter);
9691
}
92+
queryParams.query = rawFilter;
9793
}
98-
99-
if (params.qstring.sSearch && params.qstring.sSearch !== "") {
100-
try {
101-
params.qstring.query.did = {"$regex": new RegExp(".*" + params.qstring.sSearch + ".*", 'i')};
102-
}
103-
catch {
104-
console.log('Could not use as regex: ' + params.qstring.sSearch);
105-
}
106-
}
107-
108-
var columns = ["device_id", "device_id", "uid", "type", "after", "ts"];
109-
var checkOb;
110-
if (params.qstring.iSortCol_0 && params.qstring.sSortDir_0 && columns[params.qstring.iSortCol_0]) {
111-
checkOb = {};
112-
checkOb[columns[params.qstring.iSortCol_0]] = (params.qstring.sSortDir_0 === "asc") ? 1 : -1;
94+
catch (e) {
95+
log.e('Cannot parse filter/query', e);
11396
}
114-
params.qstring.sort = checkOb || params.qstring.sort || {};
97+
}
11598

116-
if (params.qstring.period) {
117-
countlyCommon.getPeriodObj(params);
118-
params.qstring.query.ts = countlyCommon.getTimestampRangeQuery(params, false);
99+
const adapter = params.qstring.db_override &&
100+
params.qstring.db_override !== 'compare' &&
101+
params.qstring.db_override !== 'config'
102+
? params.qstring.db_override
103+
: undefined;
104+
const isClickHouse = adapter === 'clickhouse';
105+
if (isClickHouse) {
106+
queryParams.limit = parseInt(params.qstring.limit) || parseInt(params.qstring.iDisplayLength) || 20;
107+
if (params.qstring.cursor) {
108+
queryParams.cursor = params.qstring.cursor;
119109
}
120-
121-
params.qstring.project = params.qstring.project || {};
122-
if (typeof params.qstring.project === "string" && params.qstring.project.length) {
123-
try {
124-
params.qstring.project = JSON.parse(params.qstring.project);
125-
}
126-
catch (ex) {
127-
params.qstring.project = {};
128-
}
110+
if (params.qstring.paginationMode) {
111+
queryParams.paginationMode = params.qstring.paginationMode;
129112
}
130-
131-
params.qstring.sort = params.qstring.sort || {};
132-
if (typeof params.qstring.sort === "string" && params.qstring.sort.length) {
133-
try {
134-
params.qstring.sort = JSON.parse(params.qstring.sort);
135-
}
136-
catch (ex) {
137-
params.qstring.sort = {};
138-
}
113+
if (!queryParams.paginationMode) {
114+
queryParams.paginationMode = 'snapshot';
139115
}
140-
141-
params.qstring.limit = parseInt(params.qstring.limit) || parseInt(params.qstring.iDisplayLength) || 0;
142-
params.qstring.skip = parseInt(params.qstring.skip) || parseInt(params.qstring.iDisplayStart) || 0;
143-
144-
var cursor = common.drillDb.collection("drill_events").find(params.qstring.query, params.qstring.project);
145-
cursor.count(function(countErr, count) {
146-
if (countErr) {
147-
log.e(countErr);
148-
}
149-
if (Object.keys(params.qstring.sort).length) {
150-
cursor.sort(params.qstring.sort);
151-
}
152-
153-
if (params.qstring.skip) {
154-
cursor.skip(params.qstring.skip);
155-
}
156-
157-
if (params.qstring.limit) {
158-
cursor.limit(params.qstring.limit);
159-
}
160-
161-
cursor.toArray(function(toArrayErr, items) {
162-
if (toArrayErr) {
163-
common.returnMessage(params, 400, toArrayErr);
164-
}
165-
else {
166-
common.returnOutput(params, {sEcho: params.qstring.sEcho, iTotalRecords: total, iTotalDisplayRecords: count, aaData: items});
167-
}
168-
});
169-
});
170116
}
171117
else {
172-
common.returnOutput(params, {sEcho: params.qstring.sEcho, iTotalRecords: total, iTotalDisplayRecords: total, aaData: []});
118+
queryParams.limit = parseInt(params.qstring.limit) || parseInt(params.qstring.iDisplayLength) || 20;
119+
queryParams.skip = parseInt(params.qstring.skip) || parseInt(params.qstring.iDisplayStart) || 0;
173120
}
174-
});
121+
consentQueries.fetchConsentsList(queryParams, { adapter: adapter, comparisonMode: params.qstring.comparison })
122+
.then(function(res) {
123+
var result;
124+
125+
if (isClickHouse) {
126+
result = {
127+
sEcho: params.qstring.sEcho,
128+
iTotalRecords: res.total || 0,
129+
iTotalDisplayRecords: res.total || 0,
130+
aaData: res.data || []
131+
};
132+
if (res.hasNextPage) {
133+
result.hasNextPage = true;
134+
result.nextCursor = res.nextCursor;
135+
}
136+
if (res.paginationMode) {
137+
result.paginationMode = res.paginationMode;
138+
}
139+
if (res.isApproximate !== undefined) {
140+
result.isApproximate = res.isApproximate;
141+
}
142+
}
143+
else {
144+
result = {
145+
sEcho: params.qstring.sEcho,
146+
iTotalRecords: res.total || 0,
147+
iTotalDisplayRecords: res.filteredTotal || res.total || 0,
148+
aaData: res.data || []
149+
};
150+
}
151+
common.returnOutput(params, result);
152+
})
153+
.catch(function(e) {
154+
log.e(e);
155+
common.returnMessage(params, 400, 'Error. Please check logs.');
156+
});
157+
}
158+
catch (e) {
159+
log.e(e);
160+
common.returnMessage(params, 400, 'Error. Please check logs.');
161+
}
175162
});
176163
break;
177164
}
178-
case 'search': {
165+
/*
166+
Internal info:
167+
searchOld endpoint uses consent_history
168+
we keep this endpoint as a backup in case we need to use old data
169+
*/
170+
case 'searchOld': {
179171
if (!params.qstring.app_id) {
180172
common.returnMessage(params, 400, 'Missing parameter "app_id"');
181173
return false;

0 commit comments

Comments
 (0)