Skip to content

Commit 90ac639

Browse files
chrisgorgochrisgorgo
authored
Merge pull request #368 from chrisfilo/enh/rel_path_exclude
Relative path and directory traversal optimization
2 parents 3c59e14 + dcee48b commit 90ac639

File tree

7 files changed

+116
-39
lines changed

7 files changed

+116
-39
lines changed

‎tests/type.spec.js‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,30 @@ describe('utils.type.isAssociatedData', function () {
242242
});
243243
});
244244

245+
describe('utils.type.isStimuliData', function () {
246+
it('should return false for unknown root directories', function () {
247+
var badFilenames = [
248+
"/images/picture.jpeg",
249+
"/temporary/test.json"
250+
];
251+
252+
badFilenames.forEach(function (path) {
253+
assert.equal(utils.type.isStimuliData(path), false);
254+
});
255+
});
256+
257+
it('should return true for stimuli data directories and any files within', function () {
258+
var goodFilenames = [
259+
"/stimuli/sub-01/mov.avi",
260+
"/stimuli/text.pdf"
261+
];
262+
263+
goodFilenames.forEach(function (path) {
264+
assert(utils.type.isStimuliData(path));
265+
});
266+
});
267+
});
268+
245269
describe('utils.type.getPathValues', function () {
246270
it('should return the correct path values from a valid file path', function () {
247271
assert.equal(utils.type.getPathValues('/sub-22/ses-1/func/sub-22_ses-1_task-rest_acq-prefrontal_physio.tsv.gz').sub, 22);

‎utils/files.js‎

Lines changed: 71 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
var nifti = require('nifti-js');
44
var Issue = require('./issues').Issue;
5+
var type = require('./type');
56

67
/**
78
* If the current environment is server side
@@ -21,8 +22,7 @@ var fileUtils = {
2122
newFile: newFile,
2223
readFile: readFile,
2324
readDir: readDir,
24-
readNiftiHeader: readNiftiHeader,
25-
relativePath: relativePath
25+
readNiftiHeader: readNiftiHeader
2626
};
2727

2828
// implementations ----------------------------------------------------------------
@@ -65,6 +65,8 @@ function readFile (file, callback) {
6565
}
6666
}
6767

68+
69+
6870
/**
6971
* Read Directory
7072
*
@@ -76,33 +78,77 @@ function readFile (file, callback) {
7678
* object to the callback.
7779
*/
7880
function readDir (dir, callback) {
81+
var filesObj = {};
82+
var filesList = [];
7983
if (fs) {
80-
var files = getFiles(dir);
81-
var filesObj = {};
82-
var str = dir.substr(dir.lastIndexOf('/') + 1) + '$';
83-
var subpath = dir.replace(new RegExp(str), '');
84-
for (var i = 0; i < files.length; i++) {
85-
filesObj[i] = {
86-
name: files[i].substr(files[i].lastIndexOf('/') + 1),
87-
path: files[i],
88-
relativePath: files[i].replace(subpath, '')
89-
};
90-
}
91-
callback(filesObj);
84+
filesList = preprocessNode(dir);
9285
} else {
93-
callback(dir);
86+
filesList = preprocessBrowser(dir);
87+
}
88+
// converting array to object
89+
for (var j = 0; j < filesList.length; j++) {
90+
filesObj[j] = filesList[j];
91+
}
92+
callback(filesObj);
93+
}
94+
95+
/**
96+
* Preprocess file objects from a browser
97+
*
98+
* 1. Filters out ignored files and folder.
99+
* 2. Adds 'relativePath' field of each file object.
100+
*/
101+
function preprocessBrowser(filesObj) {
102+
var filesList = [];
103+
for (var i = 0; i < filesObj.length; i++) {
104+
var fileObj = filesObj[i];
105+
fileObj.relativePath = harmonizeRelativePath(fileObj.webkitRelativePath);
106+
if (type.isIgnoredPath(fileObj.relativePath)) {
107+
continue;
108+
}
109+
filesList.push(fileObj);
94110
}
111+
return filesList;
95112
}
96113

97-
function getFiles (dir, files_){
114+
/**
115+
* Preprocess directory path from a Node CLI
116+
*
117+
* 1. Recursively travers the directory tree
118+
* 2. Filters out ignored files and folder.
119+
* 3. Harmonizes the 'relativePath' field
120+
*/
121+
function preprocessNode(dir) {
122+
var str = dir.substr(dir.lastIndexOf('/') + 1) + '$';
123+
var rootpath = dir.replace(new RegExp(str), '');
124+
return getFiles(dir, [], rootpath);
125+
}
126+
127+
/**
128+
* Recursive helper function for 'preprocessNode'
129+
*/
130+
function getFiles(dir, files_, rootpath){
98131
files_ = files_ || [];
99132
var files = fs.readdirSync(dir);
100133
for (var i = 0; i < files.length; i++) {
101-
var name = dir + '/' + files[i];
102-
if (fs.lstatSync(name).isDirectory()) {
103-
getFiles(name, files_);
134+
var fullPath = dir + '/' + files[i];
135+
var relativePath = fullPath.replace(rootpath, '');
136+
relativePath = harmonizeRelativePath(relativePath);
137+
if (type.isIgnoredPath(relativePath)){
138+
continue;
139+
}
140+
var fileName = files[i];
141+
142+
var fileObj = {
143+
name: fileName,
144+
path: fullPath,
145+
relativePath: relativePath
146+
};
147+
148+
if (fs.lstatSync(fullPath).isDirectory()) {
149+
getFiles(fullPath, files_, rootpath);
104150
} else {
105-
files_.push(name);
151+
files_.push(fileObj);
106152
}
107153
}
108154
return files_;
@@ -233,15 +279,14 @@ function parseNIfTIHeader (buffer, file) {
233279
* Takes a file and returns the correct relative path property
234280
* base on the environment.
235281
*/
236-
function relativePath (file) {
237-
var relPath = (typeof window != 'undefined' ? file.webkitRelativePath : file.relativePath);
282+
function harmonizeRelativePath(path) {
238283

239284
// This hack uniforms relative paths for command line calls to 'BIDS-examples/ds001/' and 'BIDS-examples/ds001'
240-
if (relPath[0] !== '/') {
241-
var pathParts = relPath.split('/');
242-
relPath = '/' + pathParts.slice(1).join('/');
285+
if (path[0] !== '/') {
286+
var pathParts = path.split('/');
287+
path = '/' + pathParts.slice(1).join('/');
243288
}
244-
return relPath;
289+
return path;
245290
}
246291

247292
/**

‎utils/type.js‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ module.exports = {
1919
isBIDS: function (path) {
2020
return (
2121
this.isTopLevel(path) ||
22-
this.isAssociatedData(path) ||
22+
this.isStimuliData(path) ||
2323
this.isSessionLevel(path) ||
2424
this.isSubjectLevel(path) ||
2525
this.isAnat(path) ||
@@ -65,6 +65,11 @@ module.exports = {
6565
return associatedData.test(path);
6666
},
6767

68+
isStimuliData: function (path) {
69+
var stimuliDataRe = new RegExp('^\\/(?:stimuli)\\/(?:.*)$');
70+
return stimuliDataRe.test(path);
71+
},
72+
6873
/**
6974
* Check if file is phenotypic data.
7075
*/
@@ -194,6 +199,12 @@ module.exports = {
194199
return !isNaN(parseFloat(n)) && isFinite(n);
195200
},
196201

202+
isIgnoredPath: function (path) {
203+
var ignoredDirsRe = new RegExp('^\\/(derivatives|sourcedata|code).*$');
204+
var ignoreHiddenRe = new RegExp('^.*\\/[\\.].+$');
205+
return conditionalMatch(ignoredDirsRe, path) || conditionalMatch(ignoreHiddenRe, path);
206+
},
207+
197208
/**
198209
* Get Path Values
199210
*

‎validators/bids.js‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ BIDS = {
106106
for (var key in fileList) {
107107
if (fileList.hasOwnProperty(key)) {
108108
var file = fileList[key];
109-
var path = utils.files.relativePath(file);
109+
var path = file.relativePath;
110110
if (path) {
111111
path = path.split('/');
112112
path = path.reverse();
@@ -181,7 +181,7 @@ BIDS = {
181181

182182

183183
async.eachOfLimit(fileList, 200, function (file) {
184-
var completename = utils.files.relativePath(file);
184+
var completename = file.relativePath;
185185
if(!(completename.startsWith('/derivatives') || completename.startsWith('/code') || completename.startsWith('/sourcedata'))) {
186186
for (var re_index = 0; re_index < illegalchar_regex_list.length; re_index++) {
187187
var err_regex = illegalchar_regex_list[re_index][0];
@@ -202,8 +202,7 @@ BIDS = {
202202

203203
// validate individual files
204204
async.eachOfLimit(fileList, 200, function (file, key, cb) {
205-
var path = utils.files.relativePath(file);
206-
file.relativePath = path;
205+
var path = file.relativePath;
207206

208207
// check for subject directory presence
209208
if (path.startsWith('/sub-')) {
@@ -216,7 +215,7 @@ BIDS = {
216215
}
217216

218217
// ignore associated data
219-
if (utils.type.isAssociatedData(file.relativePath)) {
218+
if (utils.type.isStimuliData(file.relativePath)) {
220219
process.nextTick(cb);
221220
}
222221

@@ -360,7 +359,7 @@ BIDS = {
360359
}
361360

362361
// collect sessions & subjects
363-
if (!utils.type.isAssociatedData(file.relativePath) && utils.type.isBIDS(file.relativePath)) {
362+
if (!utils.type.isStimuliData(file.relativePath) && utils.type.isBIDS(file.relativePath)) {
364363
var pathValues = utils.type.getPathValues(file.relativePath);
365364

366365
if (pathValues.sub && summary.subjects.indexOf(pathValues.sub) === -1) {
@@ -491,8 +490,6 @@ BIDS = {
491490

492491
// validates if sub/ses-id in filename matches with ses/sub directory file is saved
493492
async.eachOfLimit(fileList, 200, function (file) {
494-
var path = utils.files.relativePath(file);
495-
file.relativePath = path;
496493
var values = getPathandFileValues(file.relativePath);
497494

498495
var pathValues = values[0];

‎validators/headerFields.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ var headerField = function headerField(headers, field) {
6767
var filename;
6868
var header = headers[header_index][1];
6969
var match;
70-
var path = utils.files.relativePath(file);
70+
var path = file.relativePath;
7171
var subject;
7272

7373

‎validators/nii.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ module.exports = function NIFTI (header, file, jsonContentsDict, bContentsDict,
227227
var intendedForFile = path.split("/")[1] + "/" + intendedFor[key];
228228
var onTheList = false;
229229
async.eachOfLimit(fileList, 200, function (file) {
230-
var filePath = file.path ? file.path : file.webkitRelativePath;
230+
var filePath = file.relativePath;
231231
if (filePath.endsWith(intendedForFile)){
232232
onTheList = true;
233233
}

‎validators/session.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ var session = function missingSessionFiles(fileList) {
1919
continue;
2020
}
2121

22-
var path = utils.files.relativePath(file);
23-
if (!utils.type.isBIDS(path) || utils.type.isAssociatedData(path)) {
22+
var path = file.relativePath;
23+
if (!utils.type.isBIDS(path) || utils.type.isStimuliData(path)) {
2424
continue;
2525
}
2626
var subject;

0 commit comments

Comments
 (0)