Added some tweaks to review page to accomodate tables,

moved the check for footer rows in SessionPerformMappingJob from if(row) to its
parent for loop, as we were having an issue where an empty row in the table footer
(i.e. the row immediately below the table) was giving an error.

Author: OwainJones

lib/importer/src/dudk/backend.js

@@ -247,9 +247,11 @@ exports.SessionSuggestDataRange = (sid, headerRange, footerRange) => {
   const dimensions = getDimensions(sid);
 
   if (headerRange) {
+    console.debug(headerRange)
     const sheetName = headerRange.sheet;
     const sheetDimensions = dimensions.sheetDimensions.get(sheetName);
     if (footerRange) {
+      console.debug(footerRange)
       // FIXME: Ensure footer is in same sheet as header, and is below the
       // header.  Not sure how fussy we need to be about the columns in
       // the footer. Perhaps they should at least overlap the columns in
@@ -785,13 +787,13 @@ exports.SessionPerformMappingJob = (sid, range, mapping, includeErrorRow = false
     let row = getMergedRow(data, merges, rowIdx);
     let recordCells = {};
     let foundSomeValues = false;
+    // Ensure we exclude footer rows from the validation as they may not
+    // have values in them.
+    if (rowIdx == range.end.row && footerRange) {
+      continue;
+    }
 
     if (row) {
-      // Ensure we exclude footer rows from the validation as they may not
-      // have the same number of columns as the header row.
-      if (rowIdx == range.end.row && footerRange) {
-        continue;
-      }
 
       if (row.length < expectedColumnCount) {
         warnings.push({row: rowIdx, field: "", type: errorTypes.ValidationError.ShortRow({ expected: expectedColumnCount, actual: row.length})});
@@ -880,6 +882,9 @@ exports.JobGetSummary = (sid, jid) => {
   assert(session.jobs.has(jid), `No such job ${jid} when getting job summary`);
 
   let job = session.jobs.get(jid);
+  console.debug(job)
+  console.debug(job.records)
+  console.debug(job.records.length)
 
   return {
     recordCount: job.records.length,

lib/importer/src/dudk/sheets.js

@@ -257,6 +257,22 @@ exports.GetTableHeader = (sid, sheet, table) => {
 
 }
 
+exports.GetTableFooter = (sid, sheet, table) => {
+  const tableRangeContainer = backend.SessionGetInputTableRanges(sid, sheet).get(table);  
+  const sheetDimensions = backend.SessionGetInputDimensions(sid)
+    .sheetDimensions.get(sheet);
+  assert(tableRangeContainer, `Table Range ${table} not found in ${sheet}` )
+  const tableRange = tableRangeContainer.range 
+
+  const footerRange = { 
+    start: { row: tableRange.end.row + 1, column: tableRange.start.column },
+    end: {row: tableRange.end.row + 1, column: tableRange.end.column }
+  }
+  
+  return footerRange;
+
+}
+
 
 
 // Given a session id and a sheet returns up to count rows.
@@ -293,7 +309,7 @@ exports.GetRowsTable = (sid, sheet, start = 0, count = 10, table=null) => {
 };
 
 exports.GetRowRangeFromEnd = (sid, sheet, count = 10, table = null) => {
-  if (table) {
+  if (table && table != null) {
     const tableRangeContainer = backend.SessionGetInputTableRanges(sid, sheet).get(table); 
     const tableRange = tableRangeContainer.range
     console.debug('Table range:', tableRange)
@@ -412,6 +428,7 @@ exports.MapData = (sid, sheet, mapping, fields, previewLimit = DEFAULT_PREVIEW_L
 
   // Construct the range to be mapped - everything but the first row
   const rowRange = backend.SessionSuggestDataRange(sid, hRange, fRange);
+  console.debug('Final row range:', rowRange)
 
   // Convert source mapping (a map from column index -> attribute name) into a mapping for the backend
   let rewrittenMapping = RewriteMapping(mapping, fields)

lib/importer/src/functions.js

@@ -77,7 +77,8 @@ const importerErrorMappingData = (error, key) => {
 const importerGetRows = (data, start, count) => {
     const session_data = data[IMPORTER_SESSION_KEY];
     const session = new session_lib.Session(session_data)
-    if (session.table & session.table != null) {
+    if (session.table && session.table != null) {
+        console.debug('showing table rows')
         return sheets_lib.GetRowsTable(session.backendSid, session.sheet, start, count, session.table)
     }
     return sheets_lib.GetRows(session.backendSid, session.sheet, start, count);
@@ -90,7 +91,9 @@ const importerGetRows = (data, start, count) => {
 const importerGetTrailingRows = (data, count) => {
     const session_data = data[IMPORTER_SESSION_KEY];
     const session = new session_lib.Session(session_data)
-    if (session.table & session.table != null) {
+    console.debug('Table name for footer:', session.table)
+    if (session.table && session.table != null) {
+        console.debug('showing table footer')
         return sheets_lib.GetTrailingRowsTable(session.backendSid, session.sheet, session.table, count);
     }
     return sheets_lib.GetTrailingRows(session.backendSid, session.sheet, count);

lib/importer/src/index.js

@@ -241,6 +241,25 @@ exports.Initialise = (config, router, prototypeKit) => {
 
       // Ensure the session is persisted. Currently in session, eventually another way
       request.session.data[IMPORTER_SESSION_KEY] = session;
+
+      tableList = []
+
+      tableList = sheets_lib.ListTables(session.backendSid, session.sheet)
+      console.debug('Table List:', tableList)
+      console.debug(tableList.length)
+
+      if (tableList.length != 0) {
+        
+        redirectOnwards(request, response);
+        return;
+      }
+
+      
+      if ("other" in request.query) {
+        const otherPage = decodeURIComponent(request.query.other)
+        if (otherPage) { request.query.next = request.query.other }
+        console.debug(request.query.other)
+      }
       redirectOnwards(request, response);
     },
   );
@@ -273,6 +292,10 @@ exports.Initialise = (config, router, prototypeKit) => {
         session.table = null
         //redirectOnwards(request, response); //if no table is selected we don't need to persist one
         //return;
+        // Ensure the session is persisted. Currently in session, eventually another way
+        request.session.data[IMPORTER_SESSION_KEY] = session;
+        redirectOnwards(request, response); 
+        return;
       } else {
         session.table = request.body.table;
         let check = sheets_lib.GetTablePreview(session.backendSid, session.sheet, session.table);
@@ -285,28 +308,39 @@ exports.Initialise = (config, router, prototypeKit) => {
         // Get header range, for tables this will check the column names against the first row of the table.
         // If they match that first row will be designated as the headerRange, otherwise it will be null.
         headerRange = sheets_lib.GetTableHeader(session.backendSid, session.sheet, session.table)
-        console.debug(headerRange)
+        console.debug('Table header:', headerRange)
         if (headerRange != null) {
           session.headerRange = headerRange //overwrite assumed header range with table header range
         }
-      }
-  
-      // The user
+
+        //Set a footer range, this may be overridden later
+        footerRange = sheets_lib.GetTableFooter(session.backendSid, session.sheet, session.table)
+        console.debug('Assumed footer:', footerRange)
+        if (footerRange != null) {
+          session.footerRange = footerRange 
+        }
+
+        // The user
 
-      // Ensure the session is persisted. Currently in session, eventually another way
-      request.session.data[IMPORTER_SESSION_KEY] = session;
+        // Ensure the session is persisted. Currently in session, eventually another way
+        request.session.data[IMPORTER_SESSION_KEY] = session;
 
-      if (headerRange == null) {
-        redirectOnwards(request, response); 
-        return;
+        if (!headerRange | headerRange == null) {
+          redirectOnwards(request, response); 
+          return;
+        }
+
+        if ("other" in request.query) {
+          const otherPage = decodeURIComponent(request.query.other)
+          if (otherPage) { request.query.next = request.query.other }
+          console.debug(request.query.other)
+        }
+        redirectOnwards(request, response);
       }
+  
+     
+      
 
-      if ("other" in request.query) {
-        const otherPage = decodeURIComponent(request.query.other)
-        if (otherPage) { request.query.next = request.query.other }
-        console.debug(request.query.other)
-      }
-      redirectOnwards(request, response);
     },
   );
 
@@ -371,10 +405,15 @@ exports.Initialise = (config, router, prototypeKit) => {
         redirectOnwards(request, response);
         return;
       }
-
+      
+      previewRange = {}
       // We will assume we asked for the default of 10 rows, and calculate the offset
       // we need to apply to the selection range so that it is valid
-      let previewRange = sheets_lib.GetRowRangeFromEnd(session.backendSid, session.sheet, 10);
+      if (session.table && session.table != null) {
+        previewRange = sheets_lib.GetRowRangeFromEnd(session.backendSid, session.sheet, 10, session.table);
+      } else {
+        previewRange = sheets_lib.GetRowRangeFromEnd(session.backendSid, session.sheet, 10);
+      };
       selectionRange.start.row += previewRange.start.row - 1;
       selectionRange.end.row += previewRange.start.row - 1;
 
@@ -386,6 +425,7 @@ exports.Initialise = (config, router, prototypeKit) => {
       }
 
       session.footerRange = selectionRange;
+      console.debug('Footer range:', session.footerRange)
 
       // Ensure the session is persisted. Currently in session, eventually another way
       request.session.data[IMPORTER_SESSION_KEY] = session;

lib/importer/templates/select_sheet.html

@@ -10,7 +10,7 @@
 {% block content %}
 <div class="govuk-grid-row">
   <div class="govuk-grid-column-full">
-    <form action="{{ importerSelectSheetPath('/select_table') }}" method="post">
+    <form action="{{ importerSelectSheetPath('/select_table', '/select_header_row') }}" method="post">
       <div class="govuk-form-group">
         <fieldset class="govuk-fieldset">
           <legend class="govuk-fieldset__legend govuk-fieldset__legend--l">