diff --git a/.gitignore b/.gitignore index c3b9412..93f1361 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -test/*.json node_modules npm-debug.log diff --git a/.travis.yml b/.travis.yml index 8c27bcb..8fd8ffe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,15 +8,9 @@ node_js: before_install: - travis_retry npm install -g npm@2.14.5 - travis_retry npm install --loglevel=http - - > - openssl aes-256-cbc \ - -K $encrypted_4fb8e306b739_key \ - -iv $encrypted_4fb8e306b739_iv \ - -in test/config.json.enc \ - -out test/config.json -d script: - - npm test + - npm run test-as-mock matrix: allow_failures: diff --git a/README.md b/README.md index 56dca1d..32b170d 100644 --- a/README.md +++ b/README.md @@ -158,11 +158,13 @@ See the [Loggly search guide][search] for more details on how to effectively sea ``` ## Run Tests -All of the node-loggly tests are written in [vows][vows], and cover all of the use cases described above. You will need to add your Loggly username, password, subdomain, and a two test inputs to test/data/test-config.json before running tests. When configuring the test inputs on Loggly, the first test input should be named 'test' using the HTTP service. The second input should be name 'test_json' using the HTTP service with the JSON logging option enabled: + +### Run Tests by sending events to your Loggly Account +All of the node-loggly tests are written in [vows][vows], and cover all of the use cases described above. You will need to add your Loggly username, password, subdomain, and your loggly token to test/config.json before running tests. ``` js { - "token": "your-really-long-token-you-got-when-you-created-an-http-input", + "token": "your-loggly-token", "subdomain": "your-subdomain", "auth": { "username": "your-username", @@ -176,6 +178,12 @@ Once you have valid Loggly credentials you can run tests with [vows][vows]: ``` bash $ npm test ``` +### Run Tests with Mock HTTP Request +To mock the HTTP requests and run test cases in your local machine you can run the following command +```bash + $ npm run test-as-mock +``` + #### Author: [Charlie Robbins](http://www.github.com/indexzero) #### Contributors: [Marak Squires](http://github.com/marak), [hij1nx](http://github.com/hij1nx), [Kord Campbell](http://loggly.com), [Erik Hedenström](http://github.com/ehedenst), diff --git a/lib/loggly/client.js b/lib/loggly/client.js index 81b4838..9f18f30 100644 --- a/lib/loggly/client.js +++ b/lib/loggly/client.js @@ -46,6 +46,7 @@ var Loggly = exports.Loggly = function (options) { } events.EventEmitter.call(this); + this.subdomain = options.subdomain; this.token = options.token; this.host = options.host || 'logs-01.loggly.com'; @@ -54,7 +55,7 @@ var Loggly = exports.Loggly = function (options) { this.proxy = options.proxy || null; this.userAgent = 'node-loggly ' + loggly.version; this.useTagHeader = 'useTagHeader' in options ? options.useTagHeader : true; - + this.isBulk = options.isBulk || false; // // Set the tags on this instance. // @@ -101,7 +102,6 @@ Loggly.prototype.log = function (msg, tags, callback) { // Remark: Have some extra logic for detecting if we want to make a bulk // request to loggly // - var isBulk = Array.isArray(msg); function serialize(msg) { if (msg instanceof Object) { return self.json ? stringify(msg) : common.serialize(msg); @@ -111,13 +111,14 @@ Loggly.prototype.log = function (msg, tags, callback) { } } - msg = isBulk ? msg.map(serialize).join('\n') : serialize(msg); + msg = serialize(msg); logOptions = { - uri: isBulk ? this.urls.bulk : this.urls.log, + uri: this.isBulk ? this.urls.bulk : this.urls.log, method: 'POST', body: msg, proxy: this.proxy, + isBulk: this.isBulk, headers: { host: this.host, accept: '*/*', diff --git a/lib/loggly/common.js b/lib/loggly/common.js index 633f715..785563d 100644 --- a/lib/loggly/common.js +++ b/lib/loggly/common.js @@ -6,6 +6,13 @@ * */ +// +// Variables for Bulk +// +var arrSize = 100, + arrMsg = [], + timerFunction = null; + var https = require('https'), util = require('util'), request = require('request'), @@ -69,6 +76,7 @@ common.loggly = function () { method, auth, proxy, + isBulk, uri; // @@ -88,6 +96,7 @@ common.loggly = function () { uri = args[0].uri; requestBody = args[0].body; auth = args[0].auth; + isBulk = args[0].isBulk; headers = args[0].headers; proxy = args[0].proxy; } @@ -109,41 +118,81 @@ common.loggly = function () { if (callback) { callback(err) } } } - var requestOptions = { - uri: uri, + uri: isBulk ? uri + '/tag/' + headers['X-LOGGLY-TAG'] : uri, method: method, - headers: headers || {}, + headers: isBulk ? {} : headers || {}, // Set headers empty for bulk proxy: proxy }; if (auth) { requestOptions.headers.authorization = 'Basic ' + new Buffer(auth.username + ':' + auth.password).toString('base64'); } - if (requestBody) { requestOptions.body = requestBody; } - - try { - request(requestOptions, function (err, res, body) { - if (err) { - return onError(err); - } - - var statusCode = res.statusCode.toString(); - if (Object.keys(failCodes).indexOf(statusCode) !== -1) { - return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); - } - - success(res, body); - }); + function sendLogs() { + try { + request(requestOptions, function (err, res, body) { + if (err) { + return onError(err); + } + var statusCode = res.statusCode.toString(); + if (Object.keys(failCodes).indexOf(statusCode) !== -1) { + return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); + } + success(res, body); + }); + } + catch (ex) { + onError(ex); + } } - catch (ex) { - onError(ex); + function sendBulkLogs() { + if (arrMsg.length === 0) { + return; + } + // + // Join Array Message with new line ('\n') character + // + requestOptions.body = arrMsg.join('\n'); + try { + request(requestOptions, function (err, res, body) { + if (err) { + return onError(err); + } + var statusCode = res.statusCode.toString(); + if (Object.keys(failCodes).indexOf(statusCode) !== -1) { + return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); + } + success(res, body); + }); + } + catch (ex) { + onError(ex); + } + finally { + // + // Empty the array + // + arrMsg.length = 0; + } + } + if (isBulk === true) { + if (timerFunction === null) { + timerFunction = setInterval(function () { + sendBulkLogs(); + },30000); + } + arrMsg.push(requestBody); + if (arrMsg.length === arrSize) { + sendBulkLogs(); + } + } + else { + sendLogs(); } }; - // // ### function serialize (obj, key) // #### @obj {Object|literal} Object to serialize diff --git a/package.json b/package.json index 5282934..4d829fb 100644 --- a/package.json +++ b/package.json @@ -20,12 +20,15 @@ }, "devDependencies": { "common-style": "^3.1.0", - "vows": "0.8.x" + "vows": "0.8.x", + "nock": "~7.2.2" }, "main": "./lib/loggly", "scripts": { "pretest": "common lib/**/*.js lib/*.js test/helpers.js", - "test": "vows test/*-test.js --spec" + "test": "vows test/*-test.js --spec", + "pretest-as-mock": "common lib/**/*.js lib/*.js test-as-mock/helpers.js", + "test-as-mock": "vows test-as-mock/*-test.js --spec" }, "license": "MIT", "engines": { diff --git a/test-as-mock/common-test.js b/test-as-mock/common-test.js new file mode 100644 index 0000000..ac5d3f4 --- /dev/null +++ b/test-as-mock/common-test.js @@ -0,0 +1,32 @@ +/* + * common-test.js: Tests for Loggly `common` utility module + * + * (C) 2010 Charlie Robbins + * MIT LICENSE + * + */ + +var path = require('path'), + vows = require('vows'), + assert = require('assert'), + common = require('../lib/loggly/common'); + +vows.describe('node-loggly/common').addBatch({ + "When using the common module": { + "the clone() method": { + topic: function () { + this.obj = { + name: 'common', + deep: { + first: 'first', + second: 'second' + } + }; + return common.clone(this.obj); + }, + "should return a deep clone of the object": function (clone) { + assert.isFalse(this.obj.deep === clone.deep); + } + } + } +}).export(module); \ No newline at end of file diff --git a/test-as-mock/config.json b/test-as-mock/config.json new file mode 100644 index 0000000..21ec805 --- /dev/null +++ b/test-as-mock/config.json @@ -0,0 +1,8 @@ +{ + "token": "test", + "subdomain": "testSubdomain", + "auth": { + "username": "test", + "password": "test" + } + } diff --git a/test-as-mock/customer-test.js b/test-as-mock/customer-test.js new file mode 100644 index 0000000..15f1ca1 --- /dev/null +++ b/test-as-mock/customer-test.js @@ -0,0 +1,43 @@ +/* + * search-test.js: Tests for Loggly search requests + * + * (C) 2010 Charlie Robbins + * MIT LICENSE + * + */ +var path = require('path'), + vows = require('vows'), + assert = require('assert'), + nock = require('nock'), + helpers = require('./helpers'); + +var options = {}, + testContext = {}, + config = helpers.loadConfig(), + loggly = require('../lib/loggly').createClient(config); + +vows.describe('node-loggly/customer').addBatch({ + "When using the node-loggly client": { + "the customer() method": { + topic: function() { + nock("https://" + config.subdomain + ".loggly.com") + .get('/apiv2/customer') + .reply(200, { + "tokens": ["test", "test2"], + "subdomain": config.subdomain, + "subscription": { + "key1": "value1" + } + }); + loggly.customer(this.callback); + + }, + "should return a valid customer": function(err, customer) { + assert.isNull(err); + assert.isArray(customer.tokens); + assert.isString(customer.subdomain); + assert.isObject(customer.subscription); + } + } + } +}).export(module); diff --git a/test-as-mock/helpers.js b/test-as-mock/helpers.js new file mode 100644 index 0000000..7ac92e0 --- /dev/null +++ b/test-as-mock/helpers.js @@ -0,0 +1,48 @@ +/* + * helpers.js: Test helpers for node-loggly + * + * (C) 2010 Charlie Robbins + * MIT LICENSE + * + */ + +var fs = require('fs'), + util = require('util'), + path = require('path'), + vows = require('vows'), + assert = require('assert'), + loggly = require('../lib/loggly'); + +var helpers = exports; + +helpers.validConfig = function (config) { + return config + && config.subdomain !== 'test-subdomain' + && config.auth + && config.auth.username !== 'test-username' + && config.auth.password !== 'test-password' + && config.token; +}; + +helpers.loadConfig = function () { + try { + var config = require('./config'); + if (!helpers.validConfig(config)) { + throw new Error(util.format('test/config.json: invalid data %j', config)); + } + + helpers.config = config || {}; + return helpers.config; + } + catch (ex) { + console.log('Error parsing test/config.json'); + throw ex; + } +}; + +helpers.assertSearch = function (err, results) { + assert.isNull(err); + assert.isObject(results); + assert.isArray(results.events); + assert.isTrue(typeof results.total_events !== 'undefined'); +}; diff --git a/test-as-mock/log-test.js b/test-as-mock/log-test.js new file mode 100644 index 0000000..4caee30 --- /dev/null +++ b/test-as-mock/log-test.js @@ -0,0 +1,203 @@ +/* + * log-test.js: Tests for vanilla logging with no authentication. + * + * (C) 2010 Charlie Robbins + * MIT LICENSE + * + */ +var path = require('path'), + vows = require('vows'), + assert = require('assert'), + nock = require('nock'), + helpers = require('./helpers'); + +var config = helpers.loadConfig(), + loggly = require('../lib/loggly').createClient({ + subdomain: config.subdomain, + token: config.token + }), + logglyJSON = require('../lib/loggly').createClient({ + subdomain: config.subdomain, + token: config.token, + json: true + }); + +vows.describe('node-loggly/inputs (no auth)').addBatch({ + "When using the node-loggly client without authentication": { + "the log() method": { + "to a 'text' input": { + "when passed a callback": { + topic: function() { + + nock("https://logs-01.loggly.com") + .post('/inputs/' + config.token) + .reply(200, { + "response": "ok" + }); + + loggly.log( + 'this is a test logging message from /test/input-test.js', + this.callback + ); + }, + "should log messages to loggly": function(err, result) { + assert.isNull(err); + assert.isObject(result); + assert.equal(result.response, 'ok'); + } + }, + } + } + } + }) + .addBatch({ + "When using the node-loggly client without authentication": { + "the log() method": { + "to a 'text' input": { + "when not passed a callback": { + topic: function() { + nock("https://logs-01.loggly.com") + .post('/inputs/' + config.token).reply(200, { + "response": "ok" + }); + loggly.log('this is a test logging message from /test/input-test.js'); + loggly.on('log', this.callback.bind(null, null)); + }, + "should log messages to loggly": function(err, result) { + assert.isNull(err); + assert.isObject(result); + assert.equal(result.response, 'ok'); + } + } + }, + } + } + }) + .addBatch({ + "When using the node-loggly client without authentication": { + "the log() method": { + "to a 'json' input": { + "when passed a callback": { + topic: function() { + + nock("https://logs-01.loggly.com") + .post('/inputs/' + config.token).reply(200, { + "response": "ok" + }); + logglyJSON.log({ + timestamp: new Date().getTime(), + message: 'this is a test logging message from /test/input-test.js' + }, this.callback); + }, + "should log messages to loggly": function(err, result) { + assert.isNull(err); + assert.isObject(result); + assert.equal(result.response, 'ok'); + } + } + } + } + } + }) + .addBatch({ + "When using the node-loggly client without authentication": { + "the log() method": { + "to a 'json' input": { + "when not passed a callback": { + topic: function() { + nock("https://logs-01.loggly.com") + .post('/inputs/' + config.token).reply(200, { + "response": "ok" + }); + logglyJSON.log({ + timestamp: new Date().getTime(), + message: 'this is a test logging message from /test/input-test.js' + }); + logglyJSON.on('log', this.callback.bind(null, null)); + }, + "should log messages to loggly": function(err, result) { + assert.isNull(err); + assert.isObject(result); + assert.equal(result.response, 'ok'); + } + } + } + } + } + }).addBatch({ + "When using the node-loggly client without authentication": { + "the log() method": { + "to a 'json' input with a single tag": { + "when not passed a callback": { + topic: function() { + nock("https://logs-01.loggly.com") + .post('/inputs/' + config.token).reply(200, { + "response": "ok" + }); + logglyJSON.log({ + timestamp: new Date().getTime(), + message: 'this is a test logging message from /test/input-test.js' + }, "WOOOO-TAG"); + logglyJSON.on('log', this.callback.bind(null, null)); + }, + "should log messages to loggly": function(err, result) { + assert.isNull(err); + assert.isObject(result); + assert.equal(result.response, 'ok'); + } + } + } + } + } + }).addBatch({ + "When using the node-loggly client without authentication": { + "the log() method": { + "to a 'json' input with tags that exist as an array": { + "when not passed a callback": { + topic: function() { + nock("https://logs-01.loggly.com") + .post('/inputs/' + config.token).reply(200, { + "response": "ok" + }); + logglyJSON.log({ + timestamp: new Date().getTime(), + message: 'this is a test logging message from /test/input-test.js' + }, ["tag", "tag2"]); + logglyJSON.on('log', this.callback.bind(null, null)); + }, + "should log messages to loggly": function(err, result) { + assert.isNull(err); + assert.isObject(result); + assert.equal(result.response, 'ok'); + } + } + } + } + } + }).addBatch({ + "When using the node-loggly client without authentication": { + "the log() method": { + "takes an array": { + "when not passed a callback": { + topic: function() { + nock("https://logs-01.loggly.com") + .post('/inputs/' + config.token).reply(200, { + "response": "ok" + }); + logglyJSON.log([{ + work: 'it harder' + }, { + make: 'it better' + }]); + logglyJSON.on('log', this.callback.bind(null, null)); + }, + "should log messages to loggly": function(err, result) { + assert.isNull(err); + assert.isObject(result); + assert.equal(result.response, 'ok'); + } + } + } + } + } + }).export(module); diff --git a/test-as-mock/search-test.js b/test-as-mock/search-test.js new file mode 100644 index 0000000..27d3e9f --- /dev/null +++ b/test-as-mock/search-test.js @@ -0,0 +1,195 @@ +/* + * search-test.js: Tests for Loggly search requests + * + * (C) 2010 Charlie Robbins + * MIT LICENSE + * + */ +var path = require('path'), + vows = require('vows'), + nock = require('nock'), + assert = require('assert'), + helpers = require('./helpers'); + +var options = {}, + testContext = {}, + config = helpers.loadConfig(), + loggly = require('../lib/loggly').createClient(config); + +vows.describe('node-loggly/search').addBatch({ + "When using the node-loggly client": { + "the search() method": { + "when searching without chaining": { + topic: function() { + nock("https://" + config.subdomain + ".loggly.com", { + reqheaders: { + 'authorization': 'Basic ' + new Buffer(config.auth.username + ":" + config.auth.password).toString('base64') + } + }) + .get('/apiv2/search') + .query({ + q: 'logging message' + }) + .reply(200, { + total_events: 1, + page: 0, + events: [{ + raw: 'this is a test logging message from /test/input-test.js', + logtypes: [], + timestamp: 1456830373968, + unparsed: null, + logmsg: 'this is a test logging message from /test/input-test.js', + id: '9ce38479-df9d-11e5-802c-12a650209768', + tags: [], + event: {} + }, ], + callback: '', + rsid: { + status: 'SCHEDULED', + date_from: 1455971607000, + elapsed_time: 0.026120901107788086, + date_to: 1456835607000, + id: '897886661' + } + + }); + nock("https://" + config.subdomain + ".loggly.com", { + reqheaders: { + 'authorization': 'Basic ' + new Buffer(config.auth.username + ":" + config.auth.password).toString('base64') + } + }) + .get('/apiv2/events') + .query({ + rsid: '897886661' + }) + .reply(200, { + total_events: 1, + page: 0, + events: [{ + raw: 'this is a test logging message from /test/input-test.js', + logtypes: [], + timestamp: 1456830373968, + unparsed: null, + logmsg: 'this is a test logging message from /test/input-test.js', + id: '9ce38479-df9d-11e5-802c-12a650209768', + tags: [], + event: {} + }, ], + callback: '', + rsid: { + status: 'SCHEDULED', + date_from: 1455971607000, + elapsed_time: 0.026120901107788086, + date_to: 1456835607000, + id: '897886661' + } + + }); + loggly.search('logging message', this.callback) + }, + "should return a set of valid search results": function(err, results) { + helpers.assertSearch(err, results); + } + }, + "when searching with chaining": { + topic: function() { + nock("https://" + config.subdomain + ".loggly.com", { + reqheaders: { + 'authorization': 'Basic ' + new Buffer(config.auth.username + ":" + config.auth.password).toString('base64') + } + }) + .get('/apiv2/search') + .query({ + q: 'logging message', + callback: '' + }) + .reply(200, { + total_events: 1, + page: 0, + events: [{ + raw: 'this is a test logging message from /test/input-test.js', + logtypes: [], + timestamp: 1456830373968, + unparsed: null, + logmsg: 'this is a test logging message from /test/input-test.js', + id: '9ce38479-df9d-11e5-802c-12a650209768', + tags: [], + event: {} + }], + callback: '', + rsid: { + status: 'SCHEDULED', + date_from: 1455971607000, + elapsed_time: 0.026120901107788086, + date_to: 1456835607000, + id: '897886661' + } + }); + nock("https://" + config.subdomain + ".loggly.com", { + reqheaders: { + 'authorization': 'Basic ' + new Buffer(config.auth.username + ":" + config.auth.password).toString('base64') + } + }).get('/apiv2/events') + .query({ + rsid: '897886661' + }) + .reply(200, { + total_events: 1, + page: 0, + events: [{ + raw: 'this is a test logging message from /test/input-test.js', + logtypes: [], + timestamp: 1456830373968, + unparsed: null, + logmsg: 'this is a test logging message from /test/input-test.js', + id: '9ce38479-df9d-11e5-802c-12a650209768', + tags: [], + event: {} + }], + callback: '', + rsid: { + status: 'SCHEDULED', + date_from: 1455971607000, + elapsed_time: 0.026120901107788086, + date_to: 1456835607000, + id: '' + } + }); + loggly.search('logging message') + .run(this.callback); + }, + "should return a set of valid search results": function(err, results) { + helpers.assertSearch(err, results); + } + } + }, + "the _checkRange() method": { + "with invalid options set": { + "should correct them": function() { + var search = loggly.search({ + query: 'invalid logging message', + from: 'now', + until: '-1d' + }) + ._checkRange(); + + assert.equal(search.options.from, 'now'); + assert.equal(search.options.until, '-1d'); + } + }, + "with valid options set": { + "should not modify them": function() { + var search = loggly.search({ + query: 'valid logging message', + from: '-2M', + until: 'now' + }) + ._checkRange(); + + assert.equal(search.options.from, '-2M'); + assert.equal(search.options.until, 'now'); + } + } + } + } +}).export(module); diff --git a/test/config.json b/test/config.json new file mode 100644 index 0000000..21072ae --- /dev/null +++ b/test/config.json @@ -0,0 +1,8 @@ + { + "token": "your-loggly-token", + "subdomain": "your-subdomain", + "auth": { + "username": "your-username", + "password": "your-password" + } + } diff --git a/test/config.json.enc b/test/config.json.enc deleted file mode 100644 index c188c20..0000000 --- a/test/config.json.enc +++ /dev/null @@ -1 +0,0 @@ -N�4�Q��/�#m8ێ�d��CU�l�����]����JQ�Y��R�H�����P�[�h�`0�܆��S�_�SzX>0?�q<�Y�+�%?5��t]�H�g��V]5*�$��Z㈮T���׹�Ep<�pf�@��a�?�ߨ��A&.t�޷4�HN+�[�lc� \ No newline at end of file