Initial implementation of format picking in the UI.

Still to do:

 - Skip the format selection step if it's not needed
 - Improve text on the format selection step
 - Use guessed types to preselect formats if we can

Author: alaric-rd

lib/importer/govuk-prototype-kit.config.json

@@ -30,6 +30,11 @@
       "path": "/templates/mapping.html",
       "type": "nunjucks"
     },
+    {
+      "name": "Choose field formats",
+      "path": "/templates/format.html",
+      "type": "nunjucks"
+    },
     {
       "name": "Review your data",
       "path": "/templates/review.html",

lib/importer/nunjucks/importer/macros/field_mapper.njk

@@ -46,6 +46,7 @@
       </div>
     {% endif %}
 
+<input type="hidden" name="mappings-present" value="yes">
 <table class="govuk-table">
   {% if params.caption %}
   <caption class="govuk-table__caption govuk-table__caption--m">{{params.caption}}</caption>
@@ -67,7 +68,7 @@
       <th scope="row" class="govuk-table__header">{{ h.name }}</th>
       <td class="govuk-table__cell">{{ h.examples }}</td>
       <td class="govuk-table__cell govuk-table__cell--numeric">
-         <select class="govuk-select" style="float: right;" name="{{h.index}}">
+         <select class="govuk-select" style="float: right;" name="field-{{h.index}}">
                 <option name=""></option>
                 {% for field in fields %}
                     <option value="{{field.name}}" {% if field.name == currentValue or (mappingsLen == 0 and field.name == h.name) %}selected{% endif %}>

lib/importer/nunjucks/importer/macros/format_picker.njk

@@ -0,0 +1,87 @@
+{#
+    dudkFormatPicker shows each column heading mapped to a field whose type has
+    format options from the current spreadsheet, alongside a drop-down containing
+    available formats.  This allows the user to choose a format for
+    each column that needs one, to select how the value mapping should be applied.
+
+    The resulting data to be submitted to the 'importerMapDataPath'
+    will be a map of column indices to format names.
+
+    It accepts a data object which is taken from the prototype kit's
+    session data which is made available on every page, and contains the
+    data submitted from forms to the backend, and also the current
+    data import session.
+#}
+
+{% macro dudkFormatPicker(params) %}
+    {% set fields = params.data['importer.session']['fields'] %}
+    {% set mapping = params.data['importer.session']['mapping'] %}
+    {% set headings = importerGetHeaders(params.data) %}
+    {% set headingError = headings.error %}
+    {% set error = importerError(params.data) %}
+
+    {% if headingError %}
+    <p id="mapping-error" class="govuk-error-message">
+      <span class="govuk-visually-hidden">Error:</span> {{ headingError.text }}
+    </p>
+    {% endif %}
+
+    {% if error %}
+      <div class="govuk-error-summary" data-module="govuk-error-summary">
+        <div role="alert">
+          <h2 class="govuk-error-summary__title">
+            {{ error.text }}
+          </h2>
+          <div class="govuk-error-summary__body">
+            <ul class="govuk-list govuk-error-summary__list">
+              {% for e in error.extra %}
+              <li>
+                <a href="#field-{{ e | slugify }}">{{ e }}</a>
+              </li>
+              {% endfor %}
+            </ul>
+          </div>
+        </div>
+      </div>
+    {% endif %}
+
+<input type="hidden" name="formats-present" value="yes">
+<table class="govuk-table">
+  {% if params.caption %}
+  <caption class="govuk-table__caption govuk-table__caption--m">{{params.caption}}</caption>
+  {% endif %}
+  <thead class="govuk-table__head">
+    <tr class="govuk-table__row">
+      <th scope="col" class="govuk-table__header">{{params.columnTitle}}</th>
+      <th scope="col" class="govuk-table__header">{{params.examplesTitle}}</th>
+      <th scope="col" class="govuk-table__header" style="padding-left: 1.5em">{{params.fieldsTitle}}</th>
+    </tr>
+  </thead>
+  <tbody class="govuk-table__body">
+    {% set mappingsLen = mapping | length %}
+    {% for h in headings.data %}
+      {% set hIndex = loop.index0 %}
+      {% set currentValue = importerErrorMappingData(error, hIndex) %}
+      {% set possibleFormats = importerPossibleColumnFormats(params.data, hIndex) %}
+
+    <tr class="govuk-table__row" id="field-{{ h.name | slugify }}">
+      <th scope="row" class="govuk-table__header">{{ h.name }}</th>
+      <td class="govuk-table__cell">{{ h.examples }}</td>
+      <td class="govuk-table__cell govuk-table__cell--numeric">
+         {% if possibleFormats %}
+         <select class="govuk-select" style="float: right;" name="format-{{h.index}}">
+                <option name=""></option>
+                {% for format in possibleFormats %}
+                    <option value="{{format.name}}" {% if format.name == currentValue %}selected{% endif %}>
+                      {{format.displayName}}
+                    </option>
+                {% endfor %}
+         </select>
+         {% endif %}
+      </td>
+    </tr>
+    {% endfor %}
+  </tbody>
+</table>
+{% endmacro %}
+

lib/importer/src/dudk/backend.js

@@ -6,7 +6,8 @@ const guesser = require('./types/guesser');
 const store = require("./session_store")
 const errorTypes = require('./util/errors');
 
-// An implementation of the interface described in https://struct.register-dynamics.co.uk/trac/wiki/DataImporter/API
+// An implementation of the interface described in
+// https://struct.register-dynamics.co.uk/trac/wiki/DataImporter/API
 
 let sessionStore = function () {
   if (process.env.NODE_ENV == "development") {
@@ -53,14 +54,20 @@ exports.SessionSetFile = (sid, filename) => {
     filename: filename,
     wb: wb,
     sheetNames: Array.from(Object.keys(wb.Sheets)),
+    jobs: new Map(),
+    // State we store for the frontend (we don't do anything with it, just store it and spit it back on demand)
     headerRanges: {}, // A map from sheetname to the header range
     footerRanges: {}, // A map from sheetname to the footer range
-    mappingRules: {},
-    jobs: new Map(),
+    mappingRules: {}, // A map from column index to attribute name
+    formatChoices: {}, // A map from column index to format
   }
   sessionStore.set(sid, session);
 };
 
+///
+/// Storage of UI state for the frontend
+///
+
 exports.SessionGetMappingRules = (sid) => {
   let session = sessionStore.get(sid);
   return session.mappingRules;
@@ -70,11 +77,6 @@ exports.SessionSetMappingRules = (sid, rules) => {
   sessionStore.apply(sid, (s) => { s.mappingRules = rules })
 };
 
-exports.SessionGetSheets = (sid) => {
-  let session = sessionStore.get(sid);
-  return session.sheetNames
-};
-
 exports.SessionSetHeaderRange = (sid, range) => {
   assert(range != null, "Null range passed to SessionSetHeaderRange");
   assert(range.sheet != null, "Range with null sheet passed to SessionSetHeaderRange");
@@ -98,6 +100,24 @@ exports.SessionGetFooterRange = (sid, sheetName) => {
   return session.footerRanges[sheetName]
 };
 
+exports.SessionGetFormatChoices = (sid) => {
+  let session = sessionStore.get(sid);
+  return session.formatChoices;
+};
+
+exports.SessionSetFormatChoices = (sid, choices) => {
+  sessionStore.apply(sid, (s) => { s.formatChoices = choices })
+};
+
+///
+/// Information about the loaded input file
+///
+
+exports.SessionGetSheets = (sid) => {
+  let session = sessionStore.get(sid);
+  return session.sheetNames
+};
+
 function getDimensions(sid) {
   let session = sessionStore.get(sid);
   let sheetDimensions = new Map();
@@ -533,7 +553,7 @@ exports.SessionGuessTypes = (sid, range) => {
 // field "formats" that maps from format names (as returned by SessionGetTypes)
 // to objects with strings field "displayName" and "description".
 exports.SessionGetSupportedTypes = (sid) => {
-  return types.supportedTypes;
+  return attributeTypes.supportedTypes;
 };
 
 // Given guesses as returned by SessionGuessTypes and a list of domain-model

lib/importer/src/dudk/sheets.js

@@ -253,10 +253,11 @@ exports.GetColumnValues = (sid, sheet, columnIndex, cellWidth = 30, count = 10)
 };
 
 
-// Convert source mapping (a map from column index -> attribute name) into a mapping for the backend
-// TODO: This should be in the front-end and we should set expectations for what the mapping should
-// look like when sending to the backend.
-const RewriteMapping = (mapping, fields) => {
+// Convert source mapping (a map from column index -> attribute name) and format
+// choices (a map from column index -> format) into a mapping for the backend
+// TODO: This should be in the front-end and we should set expectations for what
+// the mapping should look like when sending to the backend.
+const RewriteMapping = (mapping, formats, fields) => {
   let rewrittenMapping = new Map();
 
   const attrTypes = {}
@@ -268,8 +269,7 @@ const RewriteMapping = (mapping, fields) => {
 
     const f = fields.find((x) => x.name == attributeName)
     if (f) {
-      // FIXME: Actually let the user pick a format, for types that need one. Hardcoding "false" for now.
-      attrTypes[f.name] = attributeTypes.mapperForField(f, false)
+      attrTypes[f.name] = attributeTypes.mapperForField(f, formats[columnIndex])
     }
   }
 
@@ -282,15 +282,15 @@ const RewriteMapping = (mapping, fields) => {
 // Uses the session ID provided, which must contain a sheet name and a
 // mapping to perform the mapping of the data across the remaining
 // rows in the sheet to return an array of objects.
-exports.MapData = (sid, sheet, mapping, fields, previewLimit = DEFAULT_PREVIEW_LIMIT) => {
+exports.MapData = (sid, sheet, mapping, formats, fields, previewLimit = DEFAULT_PREVIEW_LIMIT) => {
   const hRange = backend.SessionGetHeaderRange(sid, sheet)
   const fRange = backend.SessionGetFooterRange(sid, sheet)
 
   // Construct the range to be mapped - everything but the first row
   const rowRange = backend.SessionSuggestDataRange(sid, hRange, fRange);
 
   // Convert source mapping (a map from column index -> attribute name) into a mapping for the backend
-  let rewrittenMapping = RewriteMapping(mapping, fields)
+  let rewrittenMapping = RewriteMapping(mapping, formats, fields)
 
   // Apply the mapping
   const backendJid = backend.SessionPerformMappingJob(sid, rowRange, rewrittenMapping);

lib/importer/src/dudk/types/attribute-types.js

@@ -74,19 +74,19 @@ exports.mapperForField = (field,format) => {
     case "native":
       // Date with no format spec relies on pre-parsed date values
       // being provided by spreadsheet formats
-      m = dates.MakeCombinedDateType([]);
+      m = dates.makeCombinedDateType([]);
       break;
     case "ymd":
-      m = dates.MakeCombinedDateType(['year','month','day']);
+      m = dates.makeCombinedDateType(['year','month','day']);
       break;
     case "ydm":
-      m = dates.MakeCombinedDateType(['year','day','month']);
+      m = dates.makeCombinedDateType(['year','day','month']);
       break;
     case "dmy":
-      m = dates.MakeCombinedDateType(['day','month','year']);
+      m = dates.makeCombinedDateType(['day','month','year']);
       break;
     case "mdy":
-      m = dates.MakeCombinedDateType(['month','day','year']);
+      m = dates.makeCombinedDateType(['month','day','year']);
       break;
     default:
       throw new Error("Unknown date format", format);

lib/importer/src/functions.js

@@ -1,6 +1,7 @@
 
 const sheets_lib = require("./dudk/sheets.js");
 const session_lib = require("./session.js");
+const backend_lib = require("./dudk/backend.js");
 
 const IMPORTER_SESSION_KEY = "importer.session";
 const IMPORTER_ERROR_KEY = "importer.error";
@@ -48,10 +49,39 @@ const importerErrorMappingData = (error, key) => {
         return ""
     }
 
+    // ABS FIXME: Do we need to remove a field- or mapping- prefix from k before parseInt?
     const m = new Map(Object.entries(error.data).map(([k, v]) => [parseInt(k), v]))
     return m.get(key) || ""
 }
 
+//--------------------------------------------------------------------
+// As long as column->field mappings have been set up, this function will return
+// a list of possible formats for a specified column (by index). The return
+// value will be false if the column is mapped to a field whose type does not
+// require formats, otherwise it will be a list of objects with 'name' (internal
+// code), 'displayName' (human-facing name) and 'description' (longer description) fields.
+// --------------------------------------------------------------------
+const importerPossibleColumnFormats = (data, index) => {
+    const session_data = data[IMPORTER_SESSION_KEY];
+    const session = new session_lib.Session(session_data);
+    const supportedTypes = backend_lib.SessionGetSupportedTypes(session.backendSid);
+    const mappings = backend_lib.SessionGetMappingRules(session.backendSid);
+    const columnField = mappings[index];
+    const columnDef = session.fields.find((field) => field.name == columnField);
+    const columnTypeName = columnDef.type;
+    const columnType = supportedTypes.get(columnTypeName);
+    if (columnType.formats) {
+        const options = Array.from(columnType.formats.entries()).map((fmtEntry) => ({
+            name: fmtEntry[0],
+            displayName: fmtEntry[1].displayName,
+            description: fmtEntry[1].description
+        }));
+        return options;
+    } else {
+        return false;
+    }
+}
+
 //--------------------------------------------------------------------
 // Allows a template to obtain `count` rows from the start of the data
 // range.
@@ -156,7 +186,7 @@ const importerMappedData = (data) => {
     const session_data = data[IMPORTER_SESSION_KEY];
     const session = new session_lib.Session(session_data)
 
-    const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.fields);
+    const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.formats, session.fields);
     const headers = session.fields;
 
     return {
@@ -186,7 +216,7 @@ const data_sum = (data, column) => {
     const session_data = data[IMPORTER_SESSION_KEY];
     const session = new session_lib.Session(session_data)
 
-    const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.fields);
+    const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.formats, session.fields);
     const headers = session.fields;
 
     const idx = headers.findIndex((x) => x.name == column)
@@ -206,7 +236,7 @@ const data_avg = (data, column) => {
     const session_data = data[IMPORTER_SESSION_KEY];
     const session = new session_lib.Session(session_data)
 
-    const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.fields);
+    const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.formats, session.fields);
     const headers = session.fields;
 
     const idx = headers.findIndex((x) => x.name == column)
@@ -247,6 +277,7 @@ const parseNumberFromString = (s) => {
 module.exports = {
     importerError,
     importerErrorMappingData,
+    importerPossibleColumnFormats,
     importSheetPreview,
     importerGetRows,
     importerGetHeaders,
@@ -257,3 +288,9 @@ module.exports = {
     data_sum,
     data_avg
 }
+
+/*
+Local Variables:
+js-indent-level: 4
+End:
+*/