Skip to content

Commit d9fc387

Browse files
david-nishinellh
david-nishi
authored and
nellh
committed
Add gitTreeMode (#833)
Adds an optimization reading dataset metadata from git --gitTreeMode and --gitRef <ref> allow you to run validation against a git-annex tree and check a particular commit. This mode is disabled by default and ignores any uncommitted changes when enabled.
1 parent 392af3c commit d9fc387

File tree

9 files changed

+370
-44
lines changed

9 files changed

+370
-44
lines changed

‎bids-validator/bin/bids-validator‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,22 @@ var argv = require('yargs')
2525
'ignoreSymlinks',
2626
'Skip any symlinked directories when validating a dataset',
2727
)
28+
.boolean('gitTreeMode')
29+
.describe(
30+
'gitTreeMode',
31+
'Improve performance using git metadata. Does not capture changes not known to git.',
32+
)
33+
.option('gitRef', {
34+
describe:
35+
'Targets files at a given branch, tag, or commit hash. Use with --gitTreeMode. [default: "HEAD"]',
36+
type: 'string',
37+
})
38+
.implies('gitRef', 'gitTreeMode')
2839
.option('config', {
2940
alias: 'c',
3041
describe:
3142
'Optional configuration file. See https://github.com/bids-standard/bids-validator for more info',
43+
default: {},
3244
})
3345
.epilogue(
3446
'This tool checks if a dataset in a given directory is \

‎bids-validator/tests/bids.spec.js‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,11 @@ function assertErrorCode(errors, expected_error_code) {
5151
assert(matchingErrors.length > 0)
5252
}
5353

54-
// Default validate.BIDS options
55-
const options = { ignoreNiftiHeaders: true, json: true }
56-
const enableNiftiHeaders = { json: true }
57-
5854
describe('BIDS example datasets ', function() {
55+
// Default validate.BIDS options
56+
const options = { ignoreNiftiHeaders: true, json: true }
57+
const enableNiftiHeaders = { json: true }
58+
5959
describe('basic example dataset tests', () => {
6060
getDirectories(
6161
dataDirectory + 'bids-examples-' + global.test_version + '/',

‎bids-validator/tests/cli.spec.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
const assert = require('assert')
1+
const { assert } = require('chai')
22
const { spawn } = require('child_process')
33
const dir = process.cwd()
44
const data_dir = dir + '/bids-validator/tests/data/'

‎bids-validator/utils/__tests__/gitTreeMode.spec.js‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
const { assert } = require('chai')
2+
const {
3+
readLsTreeLines,
4+
readCatFileLines,
5+
processFiles,
6+
} = require('../files/readDir')
7+
const ignore = require('ignore')
8+
9+
describe('gitTreeMode functions', () => {
10+
describe('readLsTreeLines', () => {
11+
it('will handle regular files', () => {
12+
const lsTreeLines = [
13+
'100644 blob longkeystring 1000000\tfile/path',
14+
'100644 blob anotherlongkeystring 1\tpath/to/file',
15+
]
16+
17+
const output = readLsTreeLines(lsTreeLines)
18+
assert.hasAllKeys(output, ['files', 'symlinkFilenames', 'symlinkObjects'])
19+
assert.isEmpty(output.symlinkFilenames)
20+
assert.isEmpty(output.symlinkObjects)
21+
assert.equal(output.files[0].path, 'file/path')
22+
assert.equal(output.files[0].size, 1000000)
23+
assert.equal(output.files[1].key, 'anotherlongkeystring')
24+
assert.isString(output.files[1].id)
25+
})
26+
27+
it('will handle symlinked files', () => {
28+
const lsTreeLines = [
29+
'120000 blob e886cd8566b5e97db1fc41bb9364fc22cbe81426 199\tsymlink/filepath',
30+
'120000 blob e2cd091677489a0377d9062347c32d3efebf4322 199\they/jude/dont/be/afraid',
31+
]
32+
const expected = {
33+
files: [],
34+
symlinkFilenames: ['symlink/filepath', 'hey/jude/dont/be/afraid'],
35+
symlinkObjects: [
36+
'e886cd8566b5e97db1fc41bb9364fc22cbe81426',
37+
'e2cd091677489a0377d9062347c32d3efebf4322',
38+
],
39+
}
40+
assert.deepEqual(readLsTreeLines(lsTreeLines), expected)
41+
})
42+
})
43+
44+
describe('readCatFileLines', () => {
45+
it('creates file objects from git cat-file output', () => {
46+
const catFileOutput = [
47+
'hash blob 140',
48+
'.git/annex/objects/Mv/99/SHA256E-s54--42c98d14dbe3d066d35897a61154e39ced478cd1f0ec6159ba5f2361c4919878.json/SHA256E-s54--42c98d14dbe3d066d35897a61154e39ced478cd1f0ec6159ba5f2361c4919878.json',
49+
'otherhash blob 140',
50+
'.git/annex/objects/QV/mW/SHA256E-s99--bbef536348750373727d3b5856398d7377e5d7e23875eed026b83d12cee6f885.json/SHA256E-s99--bbef536348750373727d3b5856398d7377e5d7e23875eed026b83d12cee6f885.json',
51+
]
52+
const symlinkFilenames = ['path/to/file/a', 'path/to/file/b']
53+
const output = readCatFileLines(catFileOutput, symlinkFilenames)
54+
assert.equal(output[0].path, symlinkFilenames[0])
55+
assert.equal(output[0].size, 54)
56+
assert.equal(
57+
output[1].key,
58+
'SHA256E-s99--bbef536348750373727d3b5856398d7377e5d7e23875eed026b83d12cee6f885.json',
59+
)
60+
assert.isString(output[1].id)
61+
})
62+
})
63+
64+
describe('processFiles', () => {
65+
const ig = ignore()
66+
.add('.*')
67+
.add('/derivatives')
68+
it('aggregates, filters, and augments the files given to it', () => {
69+
const filesA = [
70+
{
71+
path: '.DS_Store',
72+
size: 1000000,
73+
id: 'athousandthousand',
74+
key: 'amillion',
75+
},
76+
{
77+
path: 'path/to/a',
78+
size: 100,
79+
id: '4yhct827ty08q4uv507829',
80+
key: 'oiweurykjsvmxcnvjqweir',
81+
},
82+
{
83+
path: 'path/to/b',
84+
size: 99,
85+
id: 'q',
86+
key: '213494759827349237492759493045982734982',
87+
},
88+
]
89+
const filesB = [
90+
{
91+
path: 'path/to/c',
92+
size: 98,
93+
id: 'ididid',
94+
key: 'o',
95+
},
96+
{
97+
path: 'path/to/d',
98+
size: 1,
99+
id: 'none',
100+
key: 'hairpin',
101+
},
102+
{
103+
path: 'derivatives/to/derivative_file',
104+
size: 1,
105+
id: 'gone',
106+
key: 'with_the_wind',
107+
},
108+
]
109+
const expected = [
110+
{
111+
path: '/path/to/dataset/path/to/a',
112+
size: 100,
113+
id: '4yhct827ty08q4uv507829',
114+
key: 'oiweurykjsvmxcnvjqweir',
115+
relativePath: '/path/to/a',
116+
name: 'a',
117+
},
118+
{
119+
path: '/path/to/dataset/path/to/b',
120+
size: 99,
121+
id: 'q',
122+
key: '213494759827349237492759493045982734982',
123+
relativePath: '/path/to/b',
124+
name: 'b',
125+
},
126+
{
127+
path: '/path/to/dataset/path/to/c',
128+
size: 98,
129+
id: 'ididid',
130+
key: 'o',
131+
relativePath: '/path/to/c',
132+
name: 'c',
133+
},
134+
{
135+
path: '/path/to/dataset/path/to/d',
136+
size: 1,
137+
id: 'none',
138+
key: 'hairpin',
139+
relativePath: '/path/to/d',
140+
name: 'd',
141+
},
142+
]
143+
const output = processFiles('/path/to/dataset', ig, filesA, filesB)
144+
const fileNames = output.map(file => file.name)
145+
assert(!fileNames.includes('.DS_Store'), 'filters out ignored files')
146+
assert(!fileNames.includes('derivative_file'), 'filters out ignored directories')
147+
assert.deepEqual(fileNames, ['a', 'b', 'c', 'd'], 'aggregates files')
148+
assert.isString(output[0].relativePath, 'adds relativePath to files')
149+
assert.isString(output[1].relativePath, 'adds name to files')
150+
})
151+
})
152+
})

‎bids-validator/utils/files/__tests__/readDir-examples.spec.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ describe('readDir.js - examples integration', () => {
2020
})
2121
it('correctly follows symlinks for subjects with followSymbolicLink: true', async done => {
2222
readDir('bids-validator/tests/data/symlinked_subject', {
23-
followSymbolicDirectories: true,
23+
ignoreSymlinks: false,
2424
}).then(files => {
2525
expect(Object.keys(files)).toHaveLength(12)
2626
expect(Object.values(files).map(f => f.name)).toEqual([
@@ -42,7 +42,7 @@ describe('readDir.js - examples integration', () => {
4242
})
4343
it('correctly does not follow symlinks for subjects with followSymbolicLink: false', async done => {
4444
readDir('bids-validator/tests/data/symlinked_subject', {
45-
followSymbolicDirectories: false,
45+
ignoreSymlinks: true,
4646
}).then(files => {
4747
expect(Object.keys(files)).toHaveLength(6)
4848
expect(Object.values(files).map(f => f.name)).toEqual([
@@ -58,7 +58,7 @@ describe('readDir.js - examples integration', () => {
5858
})
5959
it('returns file objects with the expected shape', async done => {
6060
readDir('bids-validator/tests/data/symlinked_subject', {
61-
followSymbolicDirectories: false,
61+
ignoreSymlinks: true,
6262
}).then(files => {
6363
expect(Object.keys(files)).toHaveLength(6)
6464
Object.values(files).forEach(f => {

‎bids-validator/utils/files/collectDirectorySize.js‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,11 @@ const collectDirectorySize = fileList => {
66
keys.forEach(key => {
77
const file = fileList[key]
88
// collect file stats
9-
if (typeof window !== 'undefined' && file.size) {
9+
if (file.size) {
10+
// from File api in browser
1011
size += file.size
12+
// or from git-annex metadata when in gitTreeMode
13+
if (typeof window === 'undefined') file.stats = { size: file.size }
1114
} else {
1215
file.stats = getFileStats(file)
1316
size += file.stats.size

0 commit comments

Comments
 (0)