Skip to content

Commit 9f3f490

Browse files
turtledreamsturtledreams
authored
Merge pull request #102 from Countly/MemoryOnlyStorage
Memory Only Storage Option
2 parents 97a4ccc + ae89395 commit 9f3f490

File tree

8 files changed

+386
-82
lines changed

8 files changed

+386
-82
lines changed

‎CHANGELOG.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
## xx.xx.xx
2-
* Default max segmentation value count changed from 30 to 100
2+
- Default max segmentation value count changed from 30 to 100
3+
- Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file.
4+
- Added a new init time config option (conf.storage_type) which can make user set among these storage options:
5+
- File Storage
6+
- Memory Only Storage
37

48
## 22.06.0
59
- Fixed a bug where remote config requests were rejected

‎lib/countly-bulk.js‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ var cc = require("./countly-common");
2525
var BulkUser = require("./countly-bulk-user");
2626
var CountlyStorage = require("./countly-storage");
2727

28+
const StorageTypes = cc.storageTypeEnums;
29+
2830
/**
2931
* @lends module:lib/countly-bulk
3032
* Initialize CountlyBulk server object
@@ -47,6 +49,7 @@ var CountlyStorage = require("./countly-storage");
4749
* @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted
4850
* @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread
4951
* @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length
52+
* @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied
5053
* @example
5154
* var server = new CountlyBulk({
5255
* app_key: "{YOUR-API-KEY}",
@@ -72,6 +75,7 @@ function CountlyBulk(conf) {
7275
var maxBreadcrumbCount = 100;
7376
var maxStackTraceLinesPerThread = 30;
7477
var maxStackTraceLineLength = 200;
78+
var storageType = StorageTypes.FILE;
7579

7680
cc.debugBulk = conf.debug || false;
7781
if (!conf.app_key) {
@@ -101,8 +105,9 @@ function CountlyBulk(conf) {
101105
conf.maxBreadcrumbCount = conf.max_breadcrumb_count || maxBreadcrumbCount;
102106
conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread;
103107
conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength;
108+
conf.storage_type = conf.storage_type || storageType;
104109

105-
CountlyStorage.setStoragePath(conf.storage_path, true, conf.persist_queue);
110+
CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.persist_queue);
106111

107112
this.conf = conf;
108113
/**

‎lib/countly-common.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,12 @@ var cc = {
5151
ACTION: '[CLY]_action',
5252
},
5353

54+
storageTypeEnums: {
55+
FILE: "file",
56+
MEMORY: "memory",
57+
CUSTOM: "custom",
58+
},
59+
5460
/**
5561
* Get current timestamp
5662
* @returns {number} unix timestamp in seconds

‎lib/countly-storage.js‎

Lines changed: 157 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,133 @@ var defaultPath = "../data/"; // Default path
88
var defaultBulkPath = "../bulk_data/"; // Default path
99
var asyncWriteLock = false;
1010
var asyncWriteQueue = [];
11+
let storageMethod = {};
12+
const StorageTypes = cc.storageTypeEnums;
13+
14+
// Memory-only storage methods
15+
const memoryStorage = {
16+
/**
17+
* Save value in memory
18+
* @param {String} key - key for value to store
19+
* @param {varies} value - value to store
20+
* @param {Function} callback - callback to call when done storing
21+
*/
22+
storeSet: function(key, value, callback) {
23+
if (key) {
24+
cc.log(cc.logLevelEnums.DEBUG, `storeSet, Setting key: [${key}] & value: [${value}]!`);
25+
__data[key] = value;
26+
if (typeof callback === "function") {
27+
callback(null);
28+
}
29+
}
30+
else {
31+
cc.log(cc.logLevelEnums.WARNING, `storeSet, Provioded key: [${key}] is null!`);
32+
}
33+
},
34+
/**
35+
* Get value from memory
36+
* @param {String} key - key of value to get
37+
* @param {varies} def - default value to use if not set
38+
* @returns {varies} value for the key
39+
*/
40+
storeGet: function(key, def) {
41+
cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from memory with key: [${key}].`);
42+
return typeof __data[key] !== "undefined" ? __data[key] : def;
43+
},
44+
/**
45+
* Remove value from memory
46+
* @param {String} key - key of value to remove
47+
*/
48+
storeRemove: function(key) {
49+
delete __data[key];
50+
},
51+
};
52+
53+
// File storage methods
54+
const fileStorage = {
55+
/**
56+
* Save value in storage
57+
* @param {String} key - key for value to store
58+
* @param {varies} value - value to store
59+
* @param {Function} callback - callback to call when done storing
60+
*/
61+
storeSet: function(key, value, callback) {
62+
__data[key] = value;
63+
if (!asyncWriteLock) {
64+
asyncWriteLock = true;
65+
writeFile(key, value, callback);
66+
}
67+
else {
68+
asyncWriteQueue.push([key, value, callback]);
69+
}
70+
},
71+
/**
72+
* Get value from storage
73+
* @param {String} key - key of value to get
74+
* @param {varies} def - default value to use if not set
75+
* @returns {varies} value for the key
76+
*/
77+
storeGet: function(key, def) {
78+
cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`);
79+
if (typeof __data[key] === "undefined") {
80+
var ob = readFile(key);
81+
var obLen;
82+
// check if the 'read object' is empty or not
83+
try {
84+
obLen = Object.keys(ob).length;
85+
}
86+
catch (error) {
87+
// if we can not even asses length set it to 0 so we can return the default value
88+
obLen = 0;
89+
}
90+
91+
// if empty or falsy set default value
92+
if (!ob || obLen === 0) {
93+
__data[key] = def;
94+
}
95+
// else set the value read file has
96+
else {
97+
__data[key] = ob[key];
98+
}
99+
}
100+
return __data[key];
101+
},
102+
storeRemove: function(key) {
103+
delete __data[key];
104+
var filePath = path.resolve(__dirname, `${getStoragePath()}__${key}.json`);
105+
fs.access(filePath, fs.constants.F_OK, (accessErr) => {
106+
if (accessErr) {
107+
cc.log(cc.logLevelEnums.WARNING, `storeRemove, No file found with key: [${key}]. Nothing to remove.`);
108+
return;
109+
}
110+
fs.unlink(filePath, (err) => {
111+
if (err) {
112+
cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err.message}].`);
113+
}
114+
else {
115+
cc.log(cc.logLevelEnums.INFO, `storeRemove, Successfully removed file with key: [${key}].`);
116+
}
117+
});
118+
});
119+
},
120+
};
121+
122+
/**
123+
* Sets the storage method, by default sets file storage and storage path.
124+
* @param {String} userPath - User provided storage path
125+
* @param {StorageTypes} storageType - Whether to use memory only storage or not
126+
* @param {Boolean} isBulk - Whether the storage is for bulk data
127+
* @param {Boolean} persistQueue - Whether to persist the queue until processed
128+
*/
129+
var initStorage = function(userPath, storageType, isBulk = false, persistQueue = false) {
130+
if (storageType === StorageTypes.MEMORY) {
131+
storageMethod = memoryStorage;
132+
}
133+
else {
134+
storageMethod = fileStorage;
135+
setStoragePath(userPath, isBulk, persistQueue);
136+
}
137+
};
11138

12139
/**
13140
* Sets the storage path, defaulting to a specified path if none is provided.
@@ -63,15 +190,21 @@ var resetStorage = function() {
63190
*/
64191
var readFile = function(key) {
65192
var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`);
66-
67193
// try reading data file
68194
var data;
69195
try {
70-
data = fs.readFileSync(dir);
196+
data = fs.readFileSync(dir, 'utf8'); // read file as string
71197
}
72198
catch (ex) {
73199
// there was no file, probably new init
74-
data = null;
200+
cc.log(cc.logLevelEnums.WARN, `readFile, File not found for key: [${key}]. Returning null.`);
201+
return null; // early exit if file doesn't exist
202+
}
203+
204+
// early exit if file is empty or whitespace
205+
if (!data.trim()) {
206+
cc.log(cc.logLevelEnums.WARN, `readFile, File is empty or contains only whitespace for key: [${key}]. Returning null.`);
207+
return null;
75208
}
76209

77210
try {
@@ -84,7 +217,7 @@ var readFile = function(key) {
84217
// backup corrupted file data
85218
fs.writeFile(path.resolve(__dirname, `${getStoragePath()}__${key}.${cc.getTimestamp()}${Math.random()}.json`), data, () => { });
86219
// start with new clean object
87-
data = null;
220+
return null; // return null in case of corrupted data
88221
}
89222
return data;
90223
};
@@ -139,62 +272,39 @@ var writeFile = function(key, value, callback) {
139272
});
140273
};
141274

142-
/**
143-
* Save value in storage
144-
* @param {String} key - key for value to store
145-
* @param {varies} value - value to store
146-
* @param {Function} callback - callback to call when done storing
147-
*/
148275
var storeSet = function(key, value, callback) {
149-
__data[key] = value;
150-
if (!asyncWriteLock) {
151-
asyncWriteLock = true;
152-
writeFile(key, value, callback);
153-
}
154-
else {
155-
asyncWriteQueue.push([key, value, callback]);
156-
}
276+
storageMethod.storeSet(key, value, callback);
157277
};
158278

159-
/**
160-
* Get value from storage
161-
* @param {String} key - key of value to get
162-
* @param {varies} def - default value to use if not set
163-
* @returns {varies} value for the key
164-
*/
165279
var storeGet = function(key, def) {
166-
cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`);
167-
if (typeof __data[key] === "undefined") {
168-
var ob = readFile(key);
169-
var obLen;
170-
// check if the 'read object' is empty or not
171-
try {
172-
obLen = Object.keys(ob).length;
173-
}
174-
catch (error) {
175-
// if we can not even asses length set it to 0 so we can return the default value
176-
obLen = 0;
177-
}
280+
return storageMethod.storeGet(key, def);
281+
};
178282

179-
// if empty or falsy set default value
180-
if (!ob || obLen === 0) {
181-
__data[key] = def;
182-
}
183-
// else set the value read file has
184-
else {
185-
__data[key] = ob[key];
186-
}
283+
var storeRemove = function(key) {
284+
storageMethod.storeRemove(key);
285+
};
286+
287+
/**
288+
* Disclaimer: This method is mainly for testing purposes.
289+
* @returns {StorageTypes} Returns the active storage type for the SDK
290+
*/
291+
var getStorageType = function() {
292+
if (storageMethod === memoryStorage) {
293+
return StorageTypes.MEMORY;
187294
}
188-
return __data[key];
295+
return StorageTypes.FILE;
189296
};
190297

191298
module.exports = {
192-
writeFile,
193-
storeGet,
299+
initStorage,
194300
storeSet,
301+
storeGet,
302+
storeRemove,
303+
writeFile,
195304
forceStore,
196305
getStoragePath,
197306
setStoragePath,
198307
resetStorage,
199308
readFile,
309+
getStorageType,
200310
};

‎lib/countly.js‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ var Bulk = require("./countly-bulk");
2929
var CountlyStorage = require("./countly-storage");
3030

3131
var Countly = {};
32+
const StorageTypes = cc.storageTypeEnums;
3233

3334
Countly.Bulk = Bulk;
3435
(function() {
@@ -71,7 +72,7 @@ Countly.Bulk = Bulk;
7172
var maxStackTraceLinesPerThread = 30;
7273
var maxStackTraceLineLength = 200;
7374
var deviceIdType = null;
74-
75+
var storageType = StorageTypes.FILE;
7576
/**
7677
* Array with list of available features that you can require consent for
7778
*/
@@ -122,6 +123,7 @@ Countly.Bulk = Bulk;
122123
* @param {string} conf.metrics._density - screen density of the device
123124
* @param {string} conf.metrics._locale - locale or language of the device in ISO format
124125
* @param {string} conf.metrics._store - source from where the user/device/installation came from
126+
* @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied
125127
* @example
126128
* Countly.init({
127129
* app_key: "{YOUR-APP-KEY}",
@@ -166,12 +168,11 @@ Countly.Bulk = Bulk;
166168
Countly.maxBreadcrumbCount = conf.max_breadcrumb_count || Countly.max_breadcrumb_count || conf.max_logs || Countly.max_logs || maxBreadcrumbCount;
167169
Countly.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || Countly.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread;
168170
Countly.maxStackTraceLineLength = conf.max_stack_trace_line_length || Countly.max_stack_trace_line_length || maxStackTraceLineLength;
169-
171+
conf.storage_type = conf.storage_type || storageType;
170172
// Common module debug value is set to init time debug value
171173
cc.debug = conf.debug;
172174

173-
// Set the storage path
174-
CountlyStorage.setStoragePath(conf.storage_path);
175+
CountlyStorage.initStorage(conf.storage_path, conf.storage_type);
175176

176177
// clear stored device ID if flag is set
177178
if (conf.clear_stored_device_id) {

0 commit comments

Comments
 (0)