diff --git a/.vscode/launch.json b/.vscode/launch.json index 46b03f4..961c3b7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -18,7 +18,7 @@ "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", "request": "launch", "skipFiles": ["/**"], - "type": "pwa-node" + "type": "node" }, { "name": "Attach to Node Functions", diff --git a/README.md b/README.md index d6c1196..4f98b22 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,13 @@ -Summary -======= - -Product | Validator API Endpoints ---- | --- -Description | Node.JS app that provides API end points to validate IATI XML files, is used by https://validator.iatistandard.org/ -Website | [https://developer.iatistandard.org/](https://developer.iatistandard.org/) -Related | [IATI/validator-services](https://github.com/IATI/validator-services), [IATI/validator-web](https://github.com/IATI/validator-web) -Documentation | [https://developer.iatistandard.org/](https://developer.iatistandard.org/) -Technical Issues | https://github.com/IATI/js-validator-api/issues -Support | https://iatistandard.org/en/guidance/get-support/ +# Summary +| Product | Validator API Endpoints | +| ---------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| Description | Node.JS app that provides API end points to validate IATI XML files, is used by https://validator.iatistandard.org/ | +| Website | [https://developer.iatistandard.org/](https://developer.iatistandard.org/) | +| Related | [IATI/validator-services](https://github.com/IATI/validator-services), [IATI/validator-web](https://github.com/IATI/validator-web) | +| Documentation | [https://developer.iatistandard.org/](https://developer.iatistandard.org/) | +| Technical Issues | https://github.com/IATI/js-validator-api/issues | +| Support | https://iatistandard.org/en/guidance/get-support/ | # IATI JavaScript Validator API @@ -29,15 +27,15 @@ See OpenAPI specification `postman/schemas/index.yaml`. To view locally in Swagg - [Docker Engine](https://docs.docker.com/engine/install/#server) - [Docker Compose](https://docs.docker.com/compose/install/) - `xmllint` - - There are two ways to run this locally: directly, or using the `docker compose` setup. The docker build includes the `xmllint` tool. If you run it directly, you may need to install this tool. - - On Ubuntu it is in `libxml2-utils`, so you'll need something like `sudo apt install libxml2-utils`. + - There are two ways to run this locally: directly, or using the `docker compose` setup. The docker build includes the `xmllint` tool. If you run it directly, you may need to install this tool. + - On Ubuntu it is in `libxml2-utils`, so you'll need something like `sudo apt install libxml2-utils`. ## Getting Started 1. Clone this repository 1. Follow instructions for nvm/node prerequisties above 1. Setup the environment variables using the `.env` file (instructions below) - * If running directly (option 1), this will mean setting up a `redis` database. + - If running directly (option 1), this will mean setting up a `redis` database. ### Option 1: running directly @@ -50,7 +48,6 @@ See OpenAPI specification `postman/schemas/index.yaml`. To view locally in Swagg 4. Run `npm run docker:start` to run the Function inside a Docker container using docker-compose.yml to create a Redis instance. 1. The API end points will be available on `localhost:8080` - ### Test setup To test the basic setup, you can run the following command: @@ -83,7 +80,6 @@ You should see something like (if the file is a valid IATI file): If it has warnings or errors, you'll see them listed. - ## Environment Variables ### Set Up @@ -117,8 +113,8 @@ DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME=x-functions-key DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE= DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS=5000 -- URL and API Key for datastore services, used by the advisory system to check for the - existence of IATI Identifiers in the Datastore +- URL and API Key for datastore services, used by the advisory system to check for the + existence of IATI Identifiers in the Datastore ### App config defaults (set in `config/config.js`) @@ -181,7 +177,7 @@ Locally - Install newman globally `npm i -g newman` - Start function `npm start` -- Run Tests `npm int:test` +- Run Tests `npm run int:test` In Docker container diff --git a/integration-tests/js-validator-api-tests.postman_collection.json b/integration-tests/js-validator-api-tests.postman_collection.json index 0d44822..d6d621c 100644 --- a/integration-tests/js-validator-api-tests.postman_collection.json +++ b/integration-tests/js-validator-api-tests.postman_collection.json @@ -2915,6 +2915,83 @@ } }, "response": [] + }, + { + "name": "iati-org-schema-error-broken-root-tag", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 422\", function () { pm.response.to.have.status(422); });", + "const jsonData = pm.response.json();", + "pm.test(\"The file is invalid\", function () {", + " pm.expect(jsonData.valid).to.eql(false);", + "});", + "pm.test(\"fileType value is iati-organisations\", function () {", + " pm.expect(jsonData.fileType).to.eql('iati-organisations');", + "});", + "pm.test(\"version is 2.02\", function () {", + " pm.expect(jsonData.iatiVersion).to.eql('2.02');", + "});", + "pm.test(\"Two schema errors in first group\", function () {", + " pm.expect(jsonData.errors[0].errors[0].errors.length).to.eql(2);", + "});", + "pm.test(\"Error ID should be 0.3.1\", function () {", + " pm.expect(jsonData.errors[0].errors[0].errors[0].id).to.eql(\"0.3.1\");", + "});", + "" + ], + "type": "text/javascript", + "packages": {} + } + }, + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript", + "packages": {} + } + } + ], + "protocolProfileBehavior": { + "followRedirects": false, + "followOriginalHttpMethod": false, + "followAuthorizationHeader": false + }, + "request": { + "method": "POST", + "header": [ + { + "key": "Accept", + "value": "application/json" + }, + { + "key": "Content-Type", + "value": "application/xml" + } + ], + "body": { + "mode": "file", + "file": { + "src": "/home/s/dev/js-validator-api/integration-tests/test-files/iati-org-schema-errors-broken-root-element.xml" + } + }, + "url": { + "raw": "{{baseURL}}/pub/validate", + "host": [ + "{{baseURL}}" + ], + "path": [ + "pub", + "validate" + ] + } + }, + "response": [] } ], "event": [ diff --git a/integration-tests/test-files/iati-org-schema-errors-broken-root-element.xml b/integration-tests/test-files/iati-org-schema-errors-broken-root-element.xml new file mode 100644 index 0000000..c072ef3 --- /dev/null +++ b/integration-tests/test-files/iati-org-schema-errors-broken-root-element.xml @@ -0,0 +1,34 @@ + + + TEST-ORG-010 + + Office for Government Policy Coordination + + + Office for Government Policy Coordination + + + + + TEST-ORG-051 + + State Government International Cooperation Agency + + + Office for Government Policy Coordination + + + + + TEST-ORG-060 + + Office of Ministry of Justice + + + Office for Government Policy Coordination + + + \ No newline at end of file diff --git a/services/rulesValidator.js b/services/rulesValidator.js index 007a094..7b0826c 100644 --- a/services/rulesValidator.js +++ b/services/rulesValidator.js @@ -58,7 +58,7 @@ const getRuleMethodName = (ruleName) => { const getFullContext = (xml, xpathContext, lineCount) => { const elementNode = select(xpathContext.xpath, xml).find( - (element) => element.lineNumber + lineCount === xpathContext.lineNumber + (element) => element.lineNumber + lineCount === xpathContext.lineNumber, ); const parentElement = elementNode.parentNode; @@ -88,7 +88,7 @@ class Rules { if (path.nodeName === 'reference') { text = `For the ${parentNodeName} "${select( 'string(../title/narrative)', - path + path, )}"`; } else if (parentNodeName === 'transaction') { const transactionType = @@ -96,10 +96,10 @@ class Rules { text = `For the ${transactionType || parentNodeName} of ${select( 'string(../transaction-date/@iso-date)', - path + path, )} with value ${select('string(../value/@currency)', path)}${select( 'string(../value)', - path + path, )}`; } return { @@ -111,8 +111,8 @@ class Rules { columnNumber: path.columnNumber, text, }; - }) - ) + }), + ), ); } if ('prefix' in oneCase) { @@ -129,7 +129,7 @@ class Rules { })); } return {}; - }) + }), ); } ['less', 'more', 'start', 'date', 'end'].forEach((timeCase) => { @@ -141,15 +141,15 @@ class Rules { if ('idCondition' in oneCase) { if (oneCase.idCondition === 'NOT_EXISTING_ORG_ID') { this.idCondition = this.pathMatchesText.every( - (pathText) => !idSets['ORG-ID'].has(pathText) + (pathText) => !idSets['ORG-ID'].has(pathText), ); } if (oneCase.idCondition === 'NOT_EXISTING_ORG_ID_PREFIX') { this.idCondition = this.pathMatchesText.every( (pathText) => !Array.from(idSets['ORG-ID']).some((orgId) => - pathText.startsWith(`${orgId}-`) - ) + pathText.startsWith(`${orgId}-`), + ), ); } } @@ -258,7 +258,7 @@ class Rules { this.addFailureContext(currency); result = false; } - }) + }), ); return result; default: @@ -277,19 +277,19 @@ class Rules { strictSum(oneCase) { const computedSum = Number( - this.pathMatchesText.reduce((acc, val) => Number(acc) + Number(val), 0).toFixed(4) + this.pathMatchesText.reduce((acc, val) => Number(acc) + Number(val), 0).toFixed(4), ); const limitSum = Number(oneCase.sum); if (computedSum !== limitSum) { const elements = Array.from( - new Set(this.pathMatches.map((path) => path.ownerElement.nodeName)) + new Set(this.pathMatches.map((path) => path.ownerElement.nodeName)), ).join(', '); const vocabularies = Array.from( new Set( this.pathMatches.map((path) => - select('string(@vocabulary | ../@vocabulary)', path.ownerElement) - ) - ) + select('string(@vocabulary | ../@vocabulary)', path.ownerElement), + ), + ), ).join(', '); const text = `The sum is ${computedSum} for ${elements} in vocabulary ${vocabularies}`; this.failContext.push({ @@ -350,7 +350,7 @@ class Rules { regex(oneCase, allMatches) { const regEx = new RegExp(oneCase.regex); return this.pathMatchesText.every( - (pathMatchText) => pathMatchText === '' || regEx.test(pathMatchText) === allMatches + (pathMatchText) => pathMatchText === '' || regEx.test(pathMatchText) === allMatches, ); } @@ -377,7 +377,7 @@ class Rules { } // get text matches for start into Array const startMatchesText = _.flatten( - oneCase.prefix.map((path) => select(path, this.element)) + oneCase.prefix.map((path) => select(path, this.element)), ).map((match) => getText(match)); // every path match (e.g. iati-identifier), must start with at least one (some) start match (e.g. reporting-org/@ref) @@ -386,7 +386,7 @@ class Rules { const separator = _.has(oneCase, 'separator') ? oneCase.separator : ''; const textWithSep = startText + separator; return pathMatchText.startsWith(textWithSep); - }) + }), ); } @@ -409,7 +409,7 @@ class Rules { noSpaces() { return this.pathMatchesText.every( - (pathMatchText) => pathMatchText === pathMatchText.trim() + (pathMatchText) => pathMatchText === pathMatchText.trim(), ); } } @@ -474,7 +474,7 @@ const testRuleLoop = (contextXpath, element, oneCase, idSets, lineCount = 0) => } }); results.push( - testRule(contextXpath, element, subRule, subCaseTest, idSets, lineCount) + testRule(contextXpath, element, subRule, subCaseTest, idSets, lineCount), ); }); }); @@ -512,11 +512,11 @@ const testRuleset = (ruleset, xml, idSets, lineCount = 0) => { theCases.forEach((oneCase) => { if (rule === 'loop') { result = result.concat( - testRuleLoop(contextXpath, element, oneCase, idSets, lineCount) + testRuleLoop(contextXpath, element, oneCase, idSets, lineCount), ); } else { result.push( - testRule(contextXpath, element, rule, oneCase, idSets, lineCount) + testRule(contextXpath, element, rule, oneCase, idSets, lineCount), ); } }); @@ -547,7 +547,7 @@ const createPathsContext = (caseContext, xpathContext, concatenate) => { `${acc}${i === 0 ? 'For ' : ' and '}${xpathContext.xpath}/${path.xpath} = '${ path.value }' at line: ${path.lineNumber}, column: ${path.columnNumber}`, - '' + '', ); return [{ text }]; } @@ -581,7 +581,7 @@ const standardiseResultFormat = (result, showDetails, xml, lineCount) => { elementContext = `<${xpathContext.xpath.split('/').pop()}>`; } context.push({ - text: `For ${elementContext} at line: ${xpathContext.lineNumber}, column: ${xpathContext.columnNumber}` + text: `For ${elementContext} at line: ${xpathContext.lineNumber}, column: ${xpathContext.columnNumber}`, }); break; case 'dateNow': @@ -661,8 +661,8 @@ const standardiseResultFormat = (result, showDetails, xml, lineCount) => { caseContext.prefix.map((prefixPath) => caseContext.paths.map((casePath) => ({ text: `For prefix: ${xpathContext.xpath}/${prefixPath.xpath} = '${prefixPath.value}' at line: ${prefixPath.lineNumber}, column: ${prefixPath.columnNumber} and ${xpathContext.xpath}/${casePath.xpath} = '${casePath.value}' at line: ${casePath.lineNumber}, column: ${casePath.columnNumber}`, - })) - ) + })), + ), ); } break; @@ -699,9 +699,25 @@ const standardiseResultFormat = (result, showDetails, xml, lineCount) => { }; const validateSchema = (xmlString, schema, identifier, title, showDetails, lineOffset = 0) => { - const xmlDoc = libxml.parseXml(xmlString); + let xmlDoc = null; + + try { + xmlDoc = libxml.parseXml(xmlString); + } catch (error) { + return [ + { + id: '0.3.1', + category: 'schema', + severity: 'critical', + message: error.message, + context: [ + { text: `At line ${lineOffset + (error.line === undefined ? 0 : error.line)}` }, + ], + }, + ]; + } - if (!xmlDoc.validate(schema)) { + if (xmlDoc != null && !xmlDoc.validate(schema)) { const curSchemaErrors = xmlDoc.validationErrors.reduce((acc, error) => { let errContext; const errorDetail = error; @@ -809,7 +825,7 @@ const splitXMLTransform = (root, elementName) => { // push single subelement, wrapped in root this.push( - `${rootOpen}${spacer}${doc.slice(0, closeIndex + close.length)}${rootClose}` + `${rootOpen}${spacer}${doc.slice(0, closeIndex + close.length)}${rootClose}`, ); // remove subelement that's been pushed @@ -844,7 +860,7 @@ const validateIATI = async ( idSets, schema, showDetails = false, - showElementMeta = false + showElementMeta = false, ) => { let schemaErrors = []; const ruleErrors = {}; @@ -934,13 +950,13 @@ const validateIATI = async ( result, showDetails, singleActivityDoc, - lineCount - ) + lineCount, + ), ); } return acc; }, - [] + [], ); if (errors.length > 0) { @@ -961,7 +977,7 @@ const validateIATI = async ( await pipeline( Readable.from(xml), splitXMLTransform(fileType, fileDefinition[fileType].subRoot), - processActivity() + processActivity(), ); // if no iati child elements found, evaluate schema errors at file level