From 36c231c85611dabe021cb7e0de145a0041296978 Mon Sep 17 00:00:00 2001 From: cneuromod_bot Date: Mon, 6 Jan 2025 11:20:04 -0500 Subject: [PATCH 1/6] allow wildcards in paths for config files --- src/issues/datasetIssues.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/issues/datasetIssues.ts b/src/issues/datasetIssues.ts index 3bb52df7..3b1ef2a5 100644 --- a/src/issues/datasetIssues.ts +++ b/src/issues/datasetIssues.ts @@ -1,3 +1,5 @@ +import { default as ignore } from '@ignore' +import type { Ignore } from '@ignore' import { nonSchemaIssues } from './list.ts' import type { Issue, IssueDefinition, IssueFile, Severity } from '../types/issues.ts' export type { Issue, IssueDefinition, IssueFile, Severity } @@ -41,7 +43,13 @@ export class DatasetIssues { if (!value) { continue } - found = found.filter((x) => x[key as keyof Issue] === value) + if (key === 'location'){ + // @ts-expect-error + const key_ignore = ignore().add(value as string) + found = found.filter((x) => x[key] && key_ignore.ignores(x[key].slice(1, x[key].length))) + } else { + found = found.filter((x) => x[key as keyof Issue] === value) + } } return found } From 38c84ece89271922f303f6a8148ecc65d8197eda Mon Sep 17 00:00:00 2001 From: cneuromod_bot Date: Mon, 6 Jan 2025 15:42:21 -0500 Subject: [PATCH 2/6] config glob patterns not prefixed with slash, fix tests that mistakenly created a file not starting with slash --- src/issues/datasetIssues.ts | 2 +- src/validators/filenameIdentify.test.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/issues/datasetIssues.ts b/src/issues/datasetIssues.ts index 3b1ef2a5..d2e98624 100644 --- a/src/issues/datasetIssues.ts +++ b/src/issues/datasetIssues.ts @@ -43,7 +43,7 @@ export class DatasetIssues { if (!value) { continue } - if (key === 'location'){ + if (key === 'location' && typeof value === "string" && !value.startsWith('/')){ // @ts-expect-error const key_ignore = ignore().add(value as string) found = found.filter((x) => x[key] && key_ignore.ignores(x[key].slice(1, x[key].length))) diff --git a/src/validators/filenameIdentify.test.ts b/src/validators/filenameIdentify.test.ts index 78ca37eb..ab665f97 100644 --- a/src/validators/filenameIdentify.test.ts +++ b/src/validators/filenameIdentify.test.ts @@ -70,7 +70,7 @@ Deno.test('test hasMatch', async (t) => { const parts = tmpFile.split('/') const file = new BIDSFileDeno( parts.slice(0, parts.length - 1).join('/'), - parts[parts.length - 1], + '/' + parts[parts.length - 1], ignore, ) @@ -82,6 +82,7 @@ Deno.test('test hasMatch', async (t) => { code: 'NOT_INCLUDED', }).length, 1, + `${context.file.path} ${tmpFile}` ) Deno.removeSync(tmpFile) }) From 9e073faa3b6a6d43dac83105d84e8294545aba41 Mon Sep 17 00:00:00 2001 From: cneuromod_bot Date: Mon, 6 Jan 2025 15:45:28 -0500 Subject: [PATCH 3/6] describe feature in the docs --- docs/user_guide/command-line.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/user_guide/command-line.md b/docs/user_guide/command-line.md index 47c93437..c8db5d14 100644 --- a/docs/user_guide/command-line.md +++ b/docs/user_guide/command-line.md @@ -84,6 +84,8 @@ warnings, errors or ignored. } ``` +When a configuration specifies `location` without leading `/` it is interpreted +as a glob pattern following gitignore syntax. The issues are partial matches of the [Issues] that the validator accumulates. Pass the `--json` flag to see the issues in detail. From 79b6466c690af70f0ff9be96b051174bbcb8830d Mon Sep 17 00:00:00 2001 From: cneuromod_bot Date: Mon, 6 Jan 2025 16:33:56 -0500 Subject: [PATCH 4/6] fix tests path manip following Chris\' suggestions. --- src/validators/filenameIdentify.test.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/validators/filenameIdentify.test.ts b/src/validators/filenameIdentify.test.ts index ab665f97..387693f4 100644 --- a/src/validators/filenameIdentify.test.ts +++ b/src/validators/filenameIdentify.test.ts @@ -1,4 +1,5 @@ import { assertEquals } from '@std/assert' +import { SEPARATOR_PATTERN } from '@std/path' import { BIDSContext } from '../schema/context.ts' import { _findRuleMatches, datatypeFromDirectory, hasMatch } from './filenameIdentify.ts' import { BIDSFileDeno } from '../files/deno.ts' @@ -67,12 +68,8 @@ Deno.test('test hasMatch', async (t) => { await t.step('No match', async () => { const tmpFile = Deno.makeTempFileSync() - const parts = tmpFile.split('/') - const file = new BIDSFileDeno( - parts.slice(0, parts.length - 1).join('/'), - '/' + parts[parts.length - 1], - ignore, - ) + const [ dir, base ] = tmpFile.split(SEPARATOR_PATTERN) + const file = new BIDSFileDeno(dir, `/${base}`, ignore) const context = new BIDSContext(file) await hasMatch(schema, context) @@ -82,7 +79,6 @@ Deno.test('test hasMatch', async (t) => { code: 'NOT_INCLUDED', }).length, 1, - `${context.file.path} ${tmpFile}` ) Deno.removeSync(tmpFile) }) From 9bbac823b40ab8c261d1ce39661f0b10c5401fad Mon Sep 17 00:00:00 2001 From: cneuromod_bot Date: Mon, 6 Jan 2025 16:55:07 -0500 Subject: [PATCH 5/6] add basic test for glob patterns --- src/issues/datasetIssues.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/issues/datasetIssues.test.ts b/src/issues/datasetIssues.test.ts index 651cdbda..a58b4c0e 100644 --- a/src/issues/datasetIssues.test.ts +++ b/src/issues/datasetIssues.test.ts @@ -25,6 +25,16 @@ Deno.test('DatasetIssues management class', async (t) => { assertEquals(foundIssue[0].code, 'TEST_FILES_ERROR') }) + await t.step('get issues with glob pattern', () => { + const issues = new DatasetIssues() + issues.add({ code: 'TEST_FILES_ERROR', location: '/acq-mprage_T1w.json' }, 'Test issue') + issues.add({ code: 'TEST_FILES_ERROR', location: '/acq-memprage_T1w.json' }, 'Test issue') + issues.add({ code: 'TEST_FILES_ERROR', location: '/acq-mb1_bold.json' }, 'Test issue') + issues.add({ code: 'TEST_FILES_ERROR', location: '/acq-mb4_bold.json' }, 'Test issue') + const foundIssue = issues.get({ location: '*_bold.json' }) + assertEquals(foundIssue.length, 2) + }) + await t.step('test groupBy', () => { const issues = new DatasetIssues() issues.add({ code: 'NOT_INCLUDED', location: '/file_1' }) From 84bf2bcc7fa9e96b6938d5b4a09ac19d95421b6f Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Thu, 16 Jan 2025 15:17:31 -0600 Subject: [PATCH 6/6] bump to latest ignore version for fixed typing --- deno.json | 2 +- deno.lock | 24 ++++++++++++++++-------- src/files/ignore.ts | 1 - src/issues/datasetIssues.ts | 2 -- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/deno.json b/deno.json index bd997341..1515c8f9 100644 --- a/deno.json +++ b/deno.json @@ -32,7 +32,7 @@ "@cliffy/command": "jsr:@effigies/cliffy-command@1.0.0-dev.8", "@cliffy/table": "jsr:@effigies/cliffy-table@1.0.0-dev.5", "@hed/validator": "npm:hed-validator@3.15.5", - "@ignore": "npm:ignore@6.0.2", + "@ignore": "npm:ignore@7.0.3", "@libs/xml": "jsr:@libs/xml@6.0.1", "@mango/nifti": "npm:@bids/nifti-reader-js@0.6.9", "@std/assert": "jsr:@std/assert@1.0.7", diff --git a/deno.lock b/deno.lock index c90229f9..4f881d2f 100644 --- a/deno.lock +++ b/deno.lock @@ -1,9 +1,10 @@ { "version": "4", "specifiers": { - "jsr:@bids/schema@0.11.3+2": "0.11.3+1", + "jsr:@bids/schema@1.0.0": "1.0.0", "jsr:@cliffy/flags@1.0.0-rc.7": "1.0.0-rc.7", "jsr:@cliffy/internal@1.0.0-rc.7": "1.0.0-rc.7", + "jsr:@cliffy/table@1.0.0-rc.7": "1.0.0-rc.7", "jsr:@effigies/cliffy-command@1.0.0-dev.8": "1.0.0-dev.8", "jsr:@effigies/cliffy-table@1.0.0-dev.5": "1.0.0-dev.5", "jsr:@effigies/cliffy-table@^1.0.0-dev.5": "1.0.0-dev.5", @@ -36,11 +37,11 @@ "npm:@bids/nifti-reader-js@0.6.9": "0.6.9", "npm:ajv@8.17.1": "8.17.1", "npm:hed-validator@3.15.5": "3.15.5", - "npm:ignore@6.0.2": "6.0.2" + "npm:ignore@7.0.3": "7.0.3" }, "jsr": { - "@bids/schema@0.11.3+1": { - "integrity": "331d9975fe35175e5fc4990e97abf6459f564bc630309ae80ceee98211123cfb" + "@bids/schema@1.0.0": { + "integrity": "866fe0f636b73e08bf6ba739941821a24199cc72d0f312a81328ec7ecafaceee" }, "@cliffy/flags@1.0.0-rc.7": { "integrity": "318d9be98f6a6417b108e03dec427dea96cdd41a15beb21d2554ae6da450a781", @@ -51,11 +52,18 @@ "@cliffy/internal@1.0.0-rc.7": { "integrity": "10412636ab3e67517d448be9eaab1b70c88eba9be22617b5d146257a11cc9b17" }, + "@cliffy/table@1.0.0-rc.7": { + "integrity": "9fdd9776eda28a0b397981c400eeb1aa36da2371b43eefe12e6ff555290e3180", + "dependencies": [ + "jsr:@std/fmt@~1.0.2" + ] + }, "@effigies/cliffy-command@1.0.0-dev.8": { "integrity": "d6489c66bf7f603225a426b26b62eaee32bf6da04b69056bcb015a1f17190087", "dependencies": [ "jsr:@cliffy/flags", "jsr:@cliffy/internal", + "jsr:@cliffy/table", "jsr:@effigies/cliffy-table@^1.0.0-dev.5", "jsr:@std/fmt@~1.0.2", "jsr:@std/text" @@ -231,8 +239,8 @@ "ieee754@1.2.1": { "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" }, - "ignore@6.0.2": { - "integrity": "sha512-InwqeHHN2XpumIkMvpl/DCJVrAHgCsG5+cn1XlnLWGwtZBm8QJfSusItfrwx81CTp5agNZqpKU2J/ccC5nGT4A==" + "ignore@7.0.3": { + "integrity": "sha512-bAH5jbK/F3T3Jls4I0SO1hmPR0dKU0a7+SY6n1yzRtG54FLO8d6w/nxLFX2Nb7dBu6cCWXPaAME6cYqFUMmuCA==" }, "inherits@2.0.3": { "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" @@ -402,7 +410,7 @@ }, "workspace": { "dependencies": [ - "jsr:@bids/schema@0.11.3+2", + "jsr:@bids/schema@1.0.0", "jsr:@effigies/cliffy-command@1.0.0-dev.8", "jsr:@effigies/cliffy-table@1.0.0-dev.5", "jsr:@libs/xml@6.0.1", @@ -416,7 +424,7 @@ "npm:@bids/nifti-reader-js@0.6.9", "npm:ajv@8.17.1", "npm:hed-validator@3.15.5", - "npm:ignore@6.0.2" + "npm:ignore@7.0.3" ] } } diff --git a/src/files/ignore.ts b/src/files/ignore.ts index 3f362c08..36b3e2c0 100644 --- a/src/files/ignore.ts +++ b/src/files/ignore.ts @@ -31,7 +31,6 @@ export class FileIgnoreRules { config: string[], addDefaults: boolean = true, ) { - // @ts-expect-error this.#ignore = ignore() if (addDefaults) { this.#ignore.add(defaultIgnores) diff --git a/src/issues/datasetIssues.ts b/src/issues/datasetIssues.ts index d2e98624..1e806498 100644 --- a/src/issues/datasetIssues.ts +++ b/src/issues/datasetIssues.ts @@ -1,5 +1,4 @@ import { default as ignore } from '@ignore' -import type { Ignore } from '@ignore' import { nonSchemaIssues } from './list.ts' import type { Issue, IssueDefinition, IssueFile, Severity } from '../types/issues.ts' export type { Issue, IssueDefinition, IssueFile, Severity } @@ -44,7 +43,6 @@ export class DatasetIssues { continue } if (key === 'location' && typeof value === "string" && !value.startsWith('/')){ - // @ts-expect-error const key_ignore = ignore().add(value as string) found = found.filter((x) => x[key] && key_ignore.ignores(x[key].slice(1, x[key].length))) } else {