diff --git a/changelog.md b/changelog.md index dd7c0bb..fbc2c1f 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,10 @@ # Change Log All notable changes to this project will be documented in this file. For change log formatting, see http://keepachangelog.com/ +## 4.2.0 +- Adds `safe-regex2` to check for unsafe regexes in SampleStream filter input +- Throws an error on lines bigger than 1KB to prevent potential resource exhaustion + ## 4.1.0 - Adds option to include given headers in requests @@ -12,7 +16,7 @@ All notable changes to this project will be documented in this file. For change - Use github actions for running tests ## 3.0.0 2022-08-11 -- Include request method in stream when generating a path. This will introduce a breaking change. More info in the [#55](https://github.com/mapbox/aws-log-replay/pull/55) PR description +- Include request method in stream when generating a path. This will introduce a breaking change. More info in the [#55](https://github.com/mapbox/aws-log-replay/pull/55) PR description - Request method will be used if passed to RequestStream. Only `GET` or `HEAD` requests are allowed to be replayed ## 2.6.1 2022-05-27 diff --git a/index.js b/index.js index 2a355f2..9d9ff81 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ var url = require('url'); var stream = require('stream'); var crypto = require('crypto'); var parallel = require('parallel-stream'); +const safe = require('safe-regex2'); module.exports = {}; module.exports.RequestStream = RequestStream; @@ -58,8 +59,8 @@ function GeneratePath(type, keepReferer = false) { const obj = { path, method }; if (referer) obj.referer = referer; generatePath.push(obj); - } - } + } + } } else if (type.toLowerCase() == 'lb') { if (line.indexOf('Amazon Route 53 Health Check Service') > -1) return callback(); parts = line.split(/\s+/g); @@ -97,7 +98,7 @@ function RequestStream(options) { referer = data['referer']; if (referer && typeof referer !== 'string') referer = referer.toString('utf8'); - + pathname = data['path']; if (pathname && typeof pathname !== 'string') pathname = pathname.toString('utf8'); if (!pathname || pathname.indexOf('/') !== 0) return callback(); @@ -107,7 +108,7 @@ function RequestStream(options) { var gotOptions = { method: method || 'GET', prefixUrl: options.baseurl, - https: { + https: { rejectUnauthorized: options.strictSSL === false ? false : true }, responseType: 'buffer', @@ -134,10 +135,10 @@ function RequestStream(options) { got(url, gotOptions) .then(({ statusCode, body, timings }) => { - this.push({ - url: url.toString(), - elapsedTime: timings.phases.total, - statusCode, + this.push({ + url: url.toString(), + elapsedTime: timings.phases.total, + statusCode, body }); callback(); @@ -172,17 +173,22 @@ function SampleStream(options) { sampleStream.count = 0; sampleStream.threshold = Math.round(parseFloat(options.rate) * Math.pow(2, 16)); if (options.filter) { + if (!safe(options.filter)) throw new Error('Unsafe regex provided'); sampleStream.filterFunction = new RegExp(options.filter); } sampleStream._transform = function(line, enc, callback) { if (!line) return callback(); + // Check if line is larger than 1KB + if (Buffer.byteLength(line, 'utf8') > 1024) { + return callback(new Error('Log line exceeds 1KB')); + } if (sampleStream.filterFunction && !sampleStream.filterFunction.test(line)) return callback(); - + var hash = crypto.createHash('md5').update('cloudfront-log-read-salt-' + sampleStream.count).digest().readUInt16LE(0); if (hash < sampleStream.threshold) sampleStream.push(line); sampleStream.count++; - + callback(); }; diff --git a/package-lock.json b/package-lock.json index f86bde5..3a61875 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,18 +1,19 @@ { "name": "@mapbox/aws-log-replay", - "version": "4.1.0", + "version": "4.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mapbox/aws-log-replay", - "version": "4.1.0", + "version": "4.2.0", "license": "BSD-2-Clause", "dependencies": { "agentkeepalive": "^3.5.1", "got": "^11.8.6", "minimist": "^1.2.8", "parallel-stream": "^1.1.2", + "safe-regex2": "^5.0.0", "split": "^1.0.1" }, "bin": { @@ -1428,6 +1429,15 @@ "through": "~2.3.4" } }, + "node_modules/ret": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.5.0.tgz", + "integrity": "sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/rimraf": { "version": "2.6.3", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", @@ -1502,6 +1512,25 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "dev": true }, + "node_modules/safe-regex2": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/safe-regex2/-/safe-regex2-5.0.0.tgz", + "integrity": "sha512-YwJwe5a51WlK7KbOJREPdjNrpViQBI3p4T50lfwPuDhZnE3XGVTlGvi+aolc5+RvxDD6bnUmjVsU9n1eboLUYw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "ret": "~0.5.0" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", diff --git a/package.json b/package.json index 5655173..9c2fc1a 100644 --- a/package.json +++ b/package.json @@ -10,12 +10,13 @@ "name": "@mapbox/aws-log-replay", "license": "BSD-2-Clause", "author": "Mapbox", - "version": "4.1.0", + "version": "4.2.0", "dependencies": { "agentkeepalive": "^3.5.1", "got": "^11.8.6", "minimist": "^1.2.8", "parallel-stream": "^1.1.2", + "safe-regex2": "^5.0.0", "split": "^1.0.1" }, "scripts": { diff --git a/test/SampleStream.test.js b/test/SampleStream.test.js index 0dc30cf..dca48d5 100644 --- a/test/SampleStream.test.js +++ b/test/SampleStream.test.js @@ -35,3 +35,29 @@ var expectedFiltered = [299, 607, 904, 1218, 1509, 1788, 2085, 2384, 2716]; for (rate = 1; rate < 10; rate++) { tape('filtered, sample rate ' + (rate * 0.1).toFixed(1), testFunc.bind(null, rate, 'a.json', expectedFiltered[rate - 1])); } + +tape('throws error for unsafe regex', function(t) { + try { + reader.SampleStream({ rate: 0.5, filter: /(a+)+b/ }); + t.fail('Should have thrown for unsafe regex'); + } catch (err) { + t.equal(err.message, 'Unsafe regex provided', 'throws correct error for unsafe regex'); + t.end(); + } +}); + +tape('throws error for log lines larger than 1KB', function(t) { + var sample = reader.SampleStream({ rate: 0.5, filter: /test/ }); + var splitStream = split(); + splitStream.pipe(sample); + + sample.on('error', function(err) { + t.equal(err.message, 'Log line exceeds 1KB', 'throws correct error for oversized log line'); + t.end(); + }); + + // Create a line larger than 1KB + var bigLine = 'A'.repeat(1234) + '\n'; + splitStream.write(bigLine); + splitStream.end(); +}); \ No newline at end of file