Skip to content

Commit e5b3816

Browse files
nellhnellh
authored
Merge pull request #687 from DaNish808/more-memory-#677
Prevent out-of-memory error
2 parents 39cec46 + 306db5a commit e5b3816

File tree

15 files changed

+218
-179
lines changed

15 files changed

+218
-179
lines changed

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
"jshint": "^2.9.6",
3939
"minimatch": "3.0.4",
4040
"nifti-js": "^1.0.1",
41+
"p-limit": "^2.1.0",
4142
"pako": "^1.0.6",
4243
"path": "^0.12.7",
4344
"pluralize": "^7.0.0",

‎tests/bids.spec.js‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,7 @@ describe('BIDS example datasets ', function() {
4646
'tests/data/bids-examples-' + global.test_version + '/' + path + '/',
4747
options,
4848
function(issues) {
49-
var errors = issues.errors
5049
var warnings = issues.warnings
51-
assert.deepEqual(
52-
errors.filter(issue => issue.key !== 'EMPTY_FILE'),
53-
[],
54-
)
5550
var session_flag = false
5651
for (var warning in warnings) {
5752
if (warnings[warning]['code'] === '38') {

‎tests/utils/files.spec.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ describe('validateMisc', () => {
5858
})
5959
it('returns issues for empty files (0kb)', done => {
6060
const files = groupFileTypes(filelist, {})
61+
utils.collectSummary(filelist, {})
6162

6263
validateMisc(files.misc).then(issues => {
6364
assert.ok(issues.length > 0)

‎utils/files/validateMisc.js‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,20 @@
1-
const readFile = require('./readFile')
21
const Issue = require('../issues/issue')
32

3+
function createIssueForEmpty(file) {
4+
const size = typeof window !== 'undefined' ? file.size : file.stats.size
5+
return size <= 0 && new Issue({ code: 99, file: file })
6+
}
7+
function clearNonIssues(x) {
8+
return x instanceof Issue
9+
}
10+
411
/**
512
* validateMisc
613
*
714
* takes a list of files and returns an issue for each file
815
*/
916
module.exports = function validateMisc(miscFiles) {
10-
const issuePromises = miscFiles.reduce(
11-
(issues, file) => [
12-
...issues,
13-
readFile(file, false, null).catch(err => {
14-
if (err instanceof Issue) return err
15-
throw err
16-
}),
17-
],
18-
[],
19-
)
20-
return (
21-
Promise.all(issuePromises)
22-
// remove non-issues
23-
.then(res => res.filter(o => o instanceof Issue))
17+
return Promise.resolve(
18+
miscFiles.map(createIssueForEmpty).filter(clearNonIssues),
2419
)
2520
}

‎utils/index.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ var modalities = require('./modalities')
99
var options = require('./options')
1010
var type = require('./type')
1111
const collectSummary = require('./summary/collectSummary')
12+
const limit = require('./promise_limiter')
1213

1314
module.exports = {
1415
array: array,
@@ -21,4 +22,5 @@ module.exports = {
2122
options: options,
2223
type: type,
2324
collectSummary: collectSummary,
25+
limit,
2426
}

‎utils/json.js‎

Lines changed: 46 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,61 @@
1-
var Issue = require('./issues').Issue
2-
var JSHINT = require('jshint').JSHINT
1+
const Issue = require('./issues').Issue
2+
const { JSHINT } = require('jshint')
33

4-
module.exports = {
5-
/**
6-
* Parse
7-
*
8-
* Similar to native JSON.parse but uses
9-
* a callback structure, jshint for more
10-
* thorough error reporting and error formatting
11-
* of the rest of the validator.
12-
*/
13-
parse: function(file, contents, callback) {
14-
var jsObj = null
15-
var err = null
4+
/**
5+
* Similar to native JSON.parse but returns a promise and
6+
* runs jshint for more thorough error reporting
7+
*/
8+
function parse(file, contents) {
9+
return new Promise(resolve => {
10+
let jsObj
11+
let err
1612
try {
1713
jsObj = JSON.parse(contents)
1814
} catch (exception) {
1915
err = exception
2016
} finally {
2117
if (err) {
22-
this.jshint(file, contents, function(issues) {
23-
callback(issues, null)
18+
jshint(file, contents, function(issues) {
19+
resolve({ issues, parsed: null })
2420
})
2521
} else {
26-
callback([], jsObj)
22+
resolve({ issues: [], parsed: jsObj })
2723
}
2824
}
29-
},
25+
})
26+
}
3027

31-
/**
32-
* JSHint
33-
*
34-
* Checks known invalid JSON file
35-
* content in order to produce a
36-
* verbose error message.
37-
*/
38-
jshint: function(file, contents, callback) {
39-
var issues = []
40-
if (!JSHINT(contents)) {
41-
var out = JSHINT.data()
42-
for (var i = 0; out.errors.length > i; ++i) {
43-
var error = out.errors[i]
44-
if (error) {
45-
issues.push(
46-
new Issue({
47-
code: 27,
48-
file: file,
49-
line: error.line ? error.line : null,
50-
character: error.character ? error.character : null,
51-
reason: error.reason ? error.reason : null,
52-
evidence: error.evidence ? error.evidence : null,
53-
}),
54-
)
55-
}
28+
/**
29+
* JSHint
30+
*
31+
* Checks known invalid JSON file
32+
* content in order to produce a
33+
* verbose error message.
34+
*/
35+
function jshint(file, contents, callback) {
36+
var issues = []
37+
if (!JSHINT(contents)) {
38+
var out = JSHINT.data()
39+
for (var i = 0; out.errors.length > i; ++i) {
40+
var error = out.errors[i]
41+
if (error) {
42+
issues.push(
43+
new Issue({
44+
code: 27,
45+
file: file,
46+
line: error.line ? error.line : null,
47+
character: error.character ? error.character : null,
48+
reason: error.reason ? error.reason : null,
49+
evidence: error.evidence ? error.evidence : null,
50+
}),
51+
)
5652
}
5753
}
58-
callback(issues)
59-
},
54+
}
55+
callback(issues)
56+
}
57+
58+
module.exports = {
59+
parse,
60+
jshint,
6061
}

‎utils/options.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,17 +52,17 @@ module.exports = {
5252
if (issues) {
5353
callback(issues, null)
5454
} else {
55-
json.parse(file, contents, function(issues, jsObj) {
55+
json.parse(file, contents).then(({ issues, parsed: jsObj }) => {
5656
if (issues && issues.length > 0) {
5757
callback(issues, null)
5858
} else {
59-
var parsed = {
59+
const parsedConfig = {
6060
ignore: jsObj.ignore ? jsObj.ignore : [],
6161
warn: jsObj.warn ? jsObj.warn : [],
6262
error: jsObj.error ? jsObj.error : [],
6363
ignoredFiles: jsObj.ignoredFiles ? jsObj.ignoredFiles : [],
6464
}
65-
callback(null, parsed)
65+
callback(null, parsedConfig)
6666
}
6767
})
6868
}

‎utils/promise_limiter.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
/* limits promises to LIMIT to prevent memory overuse */
2+
3+
const pLimit = require('p-limit')
4+
const LIMIT = 200
5+
6+
module.exports = pLimit(LIMIT)

‎validators/bval/validate.js‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,23 +5,26 @@ const validate = (files, bContentsDict, annexed, dir) => {
55
let issues = []
66
// validate bval
77
const bvalPromises = files.map(function(file) {
8-
return new Promise((resolve, reject) => {
9-
utils.files
10-
.readFile(file, annexed, dir)
11-
.then(contents => {
12-
bContentsDict[file.relativePath] = contents
13-
bval(file, contents, function(bvalIssues) {
14-
issues = issues.concat(bvalIssues)
15-
resolve()
16-
})
17-
})
18-
.catch(err =>
19-
utils.issues.redirect(err, reject, () => {
20-
issues.push(err)
21-
resolve()
22-
}),
23-
)
24-
})
8+
return utils.limit(
9+
() =>
10+
new Promise((resolve, reject) => {
11+
utils.files
12+
.readFile(file, annexed, dir)
13+
.then(contents => {
14+
bContentsDict[file.relativePath] = contents
15+
bval(file, contents, function(bvalIssues) {
16+
issues = issues.concat(bvalIssues)
17+
resolve()
18+
})
19+
})
20+
.catch(err =>
21+
utils.issues.redirect(err, reject, () => {
22+
issues.push(err)
23+
resolve()
24+
}),
25+
)
26+
}),
27+
)
2528
})
2629
return Promise.all(bvalPromises).then(() => issues)
2730
}

‎validators/bvec/validate.js‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,23 +5,26 @@ const validate = (files, bContentsDict, annexed, dir) => {
55
// validate bvec
66
let issues = []
77
const bvecPromises = files.map(function(file) {
8-
return new Promise((resolve, reject) => {
9-
utils.files
10-
.readFile(file, annexed, dir)
11-
.then(contents => {
12-
bContentsDict[file.relativePath] = contents
13-
bvec(file, contents, function(bvecIssues) {
14-
issues = issues.concat(bvecIssues)
15-
resolve()
16-
})
17-
})
18-
.catch(err =>
19-
utils.issues.redirect(err, reject, () => {
20-
issues.push(err)
21-
resolve()
22-
}),
23-
)
24-
})
8+
return utils.limit(
9+
() =>
10+
new Promise((resolve, reject) => {
11+
utils.files
12+
.readFile(file, annexed, dir)
13+
.then(contents => {
14+
bContentsDict[file.relativePath] = contents
15+
bvec(file, contents, function(bvecIssues) {
16+
issues = issues.concat(bvecIssues)
17+
resolve()
18+
})
19+
})
20+
.catch(err =>
21+
utils.issues.redirect(err, reject, () => {
22+
issues.push(err)
23+
resolve()
24+
}),
25+
)
26+
}),
27+
)
2528
})
2629
return Promise.all(bvecPromises).then(() => issues)
2730
}

0 commit comments

Comments
 (0)