Skip to content

Apply validation functions to mapping. #192

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rossjones merged 19 commits into from
Apr 29, 2025
Merged

Apply validation functions to mapping. #192

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
3927fb3
Add validation functions to mapping input
rossjones Apr 7, 2025
579df1d
Appease the linter!
rossjones Apr 7, 2025
2ab7285
Ensure required fields are included in mapping
rossjones Apr 7, 2025
e3a61c6
Appease the linter
rossjones Apr 7, 2025
32334fc
Show warnings+errors in review page
rossjones Apr 7, 2025
6244e87
Validation failures are errors
rossjones Apr 8, 2025
76130c2
Store errors as index, field and message
rossjones Apr 8, 2025
1062b2a
Add short rows as a warning
rossjones Apr 22, 2025
edef69e
Add error row to errors when flag is set
rossjones Apr 22, 2025
009e636
Merge branch 'main' into validate-mapping
rossjones Apr 23, 2025
8ebbd5b
Remove extraneous closing tbody tags
rossjones Apr 24, 2025
666d10d
Clear error on successful submission
rossjones Apr 24, 2025
7e82f2d
When mapping include all fields for validation
rossjones Apr 24, 2025
b2877d6
Fix ordering of errors in test
rossjones Apr 24, 2025
f2cbe70
Autoselect submitted values on mapping error
rossjones Apr 24, 2025
e3ad256
Use slugs to link errors to the correct row
rossjones Apr 25, 2025
6a13173
Happy path validation test
rossjones Apr 28, 2025
9596c79
Missing values and incorrect type tests
rossjones Apr 28, 2025
f46bd9a
Test structural warnings
rossjones Apr 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 24 additions & 3 deletions lib/importer/nunjucks/importer/macros/field_mapper.njk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,35 @@
{% macro importerFieldMapper(data, caption='', columnTitle='Column', examplesTitle='Example values', fieldsTitle='Fields') %}
{% set fields = data['importer.session']['fields'] %}
{% set headings = importerGetHeaders(data) %}
{% set error = headings.error %}
{% set headingError = headings.error %}
{% set error = importerError(data) %}

{% if error %}
{% if headingError %}
<p id="mapping-error" class="govuk-error-message">
<span class="govuk-visually-hidden">Error:</span> {{ error.text }}
<span class="govuk-visually-hidden">Error:</span> {{ headingError.text }}
</p>
{% endif %}

{% if error %}
<div class="govuk-error-summary" data-module="govuk-error-summary">
<div role="alert">
<h2 class="govuk-error-summary__title">
{{ error.text }}
</h2>
<div class="govuk-error-summary__body">
<ul class="govuk-list govuk-error-summary__list">
{% for e in error.extra %}
<li>
<a href="#">{{ e }}</a>
Copy link
Copy Markdown
Member

@simonwo simonwo Apr 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why the anchor here?

Copy link
Copy Markdown
Contributor Author

@rossjones rossjones Apr 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think in the design system it's intended to jump to the anchor where the error occurs, but I think I just forgot to go back and implement that by adding ids to the fields. Will sort it..

Copy link
Copy Markdown
Contributor Author

@rossjones rossjones Apr 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in e3ad256 but not sure scrolling down the page without a visual indicator on the erroring row is that helpful, but not sure I know how to visually indicate the error in a table.

</li>
{% endfor %}
</ul>
</div>
</div>
</div>

{% endif %}


<table class="govuk-table">
{% if caption %}
Expand Down
46 changes: 27 additions & 19 deletions lib/importer/src/attribute-types.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,13 @@
// An attribute type is a function from an input value to a result.

class AttributeMappingResult {
constructor(value, warnings, errors) {
constructor(value, errors) {
this.value = value;
this.warnings = warnings || [];
this.errors = errors;
this.errors = errors || [];
}

get valid() {
return !this.errors;
return this.errors.length == 0;
}

get empty() {
Expand All @@ -22,20 +21,20 @@ class AttributeMappingResult {

// These helpers define the three kinds of results:

function emptyMapping(warnings) {
return new AttributeMappingResult(undefined, warnings);
function emptyMapping(errors) {
return new AttributeMappingResult(undefined, errors);
}

function successfulMapping(outputValue, warnings) {
return new AttributeMappingResult(outputValue, warnings);
function successfulMapping(outputValue, errors) {
return new AttributeMappingResult(outputValue, errors);
}

function failedMapping(warnings, errors) {
return new AttributeMappingResult(undefined, warnings, errors);
function failedMapping(errors) {
return new AttributeMappingResult(undefined, errors);
}

// Create an optional version of an existing type, that allows empty strings or
// undefined values and maps then to "undefined", and converts any validation errors to warnings
// undefined values and maps then to "undefined"
exports.optionalType = (baseType) => {
return (inputValue) => {
if (inputValue !== undefined && inputValue != "") {
Expand All @@ -44,7 +43,7 @@ exports.optionalType = (baseType) => {
return result;
} else {
// Convert any errors into warnings, as this is an optional field
return emptyMapping(result.warnings.concat(result.errors));
return emptyMapping(result.errors);
}
}
else {
Expand All @@ -60,7 +59,7 @@ exports.requiredType = (baseType) => {
return baseType(inputValue);
}
else {
return failedMapping([], ["A value must be provided"]);
return failedMapping(["A value must be provided"]);
}
};
}
Expand All @@ -74,19 +73,28 @@ exports.basicStringType = (inputValue) => {
exports.basicNumberType = (inputValue) => {
const result = parseFloat(inputValue);
if (isNaN(result)) {
return failedMapping([], ["This is not a valid number"]);
return failedMapping(["This is not a valid number"]);
} else {
return successfulMapping(result);
}
};

exports.typeFromName = (name) => {
if (!name) return null
exports.mapperForField = (field) => {
if (!field) return null

switch (name.toLowerCase()) {
let m = exports.basicStringType;

switch (field.type.toLowerCase()) {
case "number":
return exports.basicNumberType
m = exports.basicNumberType
break
default:
return exports.basicStringType
m = exports.basicStringType
}

if (field.required) {
return exports.requiredType(m)
}

return exports.optionalType(m)
Comment on lines -83 to +99
Copy link
Copy Markdown
Member

@simonwo simonwo Apr 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Something here doesn't seem to be working. If I have config:

{
  "serviceName": "Upload monthly pension return",
  "fields": [
    {
      "name": "Title",
      "type": "text",
      "required": false
    },
    {
      "name": "First name",
      "type": "text",
      "required": false
    },
    {
      "name": "Surname",
      "type": "text",
      "required": false
    },
    {
      "name": "Employee number",
      "type": "text",
      "required": true
    },
    {
      "name": "Employment start date",
      "type": "text",
      "required": false
    },
    {
      "name": "Salary",
      "type": "text",
      "required": true
    },
    {
      "name": "Contribution percentage",
      "type": "text",
      "required": true
    },
    {
      "name": "Payment date",
      "type": "text",
      "required": false
    }
  ],
  "uploadPath": ""
}

and I upload this CSV and map the columns in the obvious way:

Employee number,Title,First name,Surname,Employment start date,Salary,Contribution percentage,Payment date
1,Mr,Jon,Arbuckle,2025-01-01,,,2025-04-01
2,Mr,Odie,Arbuckle,2025-02-01,123,12%,2025-04-01

Then I am told that I have uploaded two rows and there are no errors. But there should be errors because one of the rows is missing values for required fields.

image

More tests needed?

Copy link
Copy Markdown
Contributor Author

@rossjones rossjones Apr 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Resolved in 7e82f2d, tests to follow.

}
23 changes: 10 additions & 13 deletions lib/importer/src/attribute-types.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,21 @@ test('required basic string', () => {

let r = t(undefined);
expect(r.valid).toBeFalsy();
expect(r.warnings).toMatchObject([]);
expect(r.errors).toMatchObject(["A value must be provided"]);

r = t("");
expect(r.valid).toBeFalsy();
expect(r.warnings).toMatchObject([]);
expect(r.errors).toMatchObject(["A value must be provided"]);

r = t("foo");
expect(r.valid).toBeTruthy();
expect(r.value).toEqual("foo");
expect(r.warnings).toMatchObject([]);
expect(r.errors).toMatchObject([]);

r = t(123);
expect(r.valid).toBeTruthy();
expect(r.value).toEqual("123");
expect(r.warnings).toMatchObject([]);
expect(r.errors).toMatchObject([]);
});

test('optional basic string', () => {
Expand All @@ -31,50 +29,49 @@ test('optional basic string', () => {

let r = t(undefined);
expect(r.valid).toBeTruthy();
expect(r.warnings).toMatchObject([]);
expect(r.errors).toMatchObject([]);
expect(r.value).toEqual(undefined);

r = t("");
expect(r.valid).toBeTruthy();
expect(r.warnings).toMatchObject([]);
expect(r.errors).toMatchObject([]);
expect(r.value).toEqual(undefined);

r = t("foo");
expect(r.valid).toBeTruthy();
expect(r.value).toEqual("foo");
expect(r.warnings).toMatchObject([]);
expect(r.errors).toMatchObject([]);

r = t(123);
expect(r.valid).toBeTruthy();
expect(r.value).toEqual("123");
expect(r.warnings).toMatchObject([]);
expect(r.errors).toMatchObject([]);
});

test('required basic number', () => {
const t = attributeTypes.requiredType(attributeTypes.basicNumberType);

let r = t("foo");
expect(r.valid).toBeFalsy();
expect(r.warnings).toMatchObject([]);
expect(r.errors).toMatchObject(["This is not a valid number"]);

r = t(123);
expect(r.valid).toBeTruthy();
expect(r.value).toEqual(123);
expect(r.warnings).toMatchObject([]);
expect(r.errors).toMatchObject([]);
});

test('optional basic number', () => {
// Also tests that optionalType converts errors into warnings
const t = attributeTypes.optionalType(attributeTypes.basicNumberType);

let r = t("foo");
expect(r.valid).toBeTruthy();
expect(r.valid).toBeFalsy();
expect(r.value).toBeUndefined();
expect(r.warnings).toMatchObject(["This is not a valid number"]);
expect(r.errors).toMatchObject(["This is not a valid number"]);

r = t(123);
expect(r.valid).toBeTruthy();
expect(r.value).toEqual(123);
expect(r.warnings).toMatchObject([]);
expect(r.errors).toMatchObject([]);
});
27 changes: 20 additions & 7 deletions lib/importer/src/backend.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@ function mapCellValue(cell) {
return cell.v;
}

exports.SessionPerformMappingJob = (sid, range, mapping) => {
exports.SessionPerformMappingJob = (sid, range, mapping, includeErrorRow = false) => {
assert(sessionStore.get(sid));
assert(sessionStore.get(sid).wb.Sheets[range.sheet]);
assert(range.end.row >= range.start.row);
Expand All @@ -499,13 +499,26 @@ exports.SessionPerformMappingJob = (sid, range, mapping) => {
const attrMap = Object.entries(mapping.attributeMappings);
const attrTypes = mapping.attributeTypes || {};


// TODO: Replace this with a map/sum once range is a real type.
let expectedColumnCount = 0;
for (let rowIdx = range.start.row; rowIdx <= range.end.row; rowIdx++) {
let row = getMergedRow(data, merges, rowIdx);
if (row.length > expectedColumnCount) expectedColumnCount = row.length;
}

for (let rowIdx = range.start.row; rowIdx <= range.end.row; rowIdx++) {
let row = getMergedRow(data, merges, rowIdx);
let record = {};
let foundSomeValues = false;
let rowWarnings = [];
let rowErrors = [];

if (row) {
if (row.length < expectedColumnCount) {
rowWarnings.push("Row has fewer columns than expected, wanted " + expectedColumnCount + " but found " + + row.length)
}

attrMap.forEach((element) => {
const [attr, m] = element;

Expand All @@ -523,6 +536,8 @@ exports.SessionPerformMappingJob = (sid, range, mapping) => {
});
}

const rowData = includeErrorRow ? row.map((x) => mapCellValue(x)) : undefined;

if (foundSomeValues) {
// Only if we found something do we validate and map the types
let mappedRecord = {};
Expand All @@ -533,10 +548,6 @@ exports.SessionPerformMappingJob = (sid, range, mapping) => {

const result = attrType(inputVal);

result.warnings.forEach((text) => {
rowWarnings.push([attr, text]);
});

if (result.valid) {
// Succeeded, but maybe an empty result
if (result.value !== undefined) {
Expand All @@ -545,19 +556,22 @@ exports.SessionPerformMappingJob = (sid, range, mapping) => {
} else {
// Failed
result.errors.forEach((text) => {
rowErrors.push([attr, text]);
rowErrors.push({ row: rowIdx, field: attr, message: text, data: rowData });
});
}
});

if (rowErrors.length == 0) {
records.push(mappedRecord);
}
} else {
rowWarnings.push("Row is empty")
}

if (rowWarnings.length > 0) {
warnings.set(rowIdx, rowWarnings);
}

if (rowErrors.length > 0) {
errors.set(rowIdx, rowErrors);
}
Expand Down Expand Up @@ -596,7 +610,6 @@ exports.JobGetWarnings = (jid) => {
exports.JobGetErrors = (jid) => {
assert(jobStore.has(jid));
let job = jobStore.get(jid);

return job.errors;
}

Expand Down
Loading
Loading