From 2f85c49386a81070aceb3818f959b34262d6930d Mon Sep 17 00:00:00 2001
From: Alaric Snell-Pym <alaric@register-dynamics.co.uk>
Date: Tue, 23 Sep 2025 09:50:42 +0100
Subject: [PATCH 01/11] Initial implementation of format picking in the UI.

Still to do:

 - Skip the format selection step if it's not needed
 - Improve text on the format selection step
 - Use guessed types to preselect formats if we can
---
 lib/importer/govuk-prototype-kit.config.json  |  5 ++
 .../nunjucks/importer/macros/field_mapper.njk |  3 +-
 .../importer/macros/format_picker.njk         | 87 +++++++++++++++++++
 lib/importer/src/dudk/backend.js              | 38 ++++++--
 lib/importer/src/dudk/sheets.js               | 16 ++--
 .../src/dudk/types/attribute-types.js         | 10 +--
 lib/importer/src/functions.js                 | 43 ++++++++-
 lib/importer/src/index.js                     | 70 +++++++++------
 lib/importer/src/session.js                   |  8 ++
 lib/importer/templates/mapping.html           |  2 +-
 prototypes/basic/app/config.json              |  2 +-
 prototypes/basic/app/views/mapping.html       |  2 +-
 12 files changed, 232 insertions(+), 54 deletions(-)
 create mode 100644 lib/importer/nunjucks/importer/macros/format_picker.njk

diff --git a/lib/importer/govuk-prototype-kit.config.json b/lib/importer/govuk-prototype-kit.config.json
index 2e85f535..616a929d 100644
--- a/lib/importer/govuk-prototype-kit.config.json
+++ b/lib/importer/govuk-prototype-kit.config.json
@@ -30,6 +30,11 @@
       "path": "/templates/mapping.html",
       "type": "nunjucks"
     },
+    {
+      "name": "Choose field formats",
+      "path": "/templates/format.html",
+      "type": "nunjucks"
+    },
     {
       "name": "Review your data",
       "path": "/templates/review.html",
diff --git a/lib/importer/nunjucks/importer/macros/field_mapper.njk b/lib/importer/nunjucks/importer/macros/field_mapper.njk
index c6737e10..86824c19 100644
--- a/lib/importer/nunjucks/importer/macros/field_mapper.njk
+++ b/lib/importer/nunjucks/importer/macros/field_mapper.njk
@@ -46,6 +46,7 @@
       </div>
     {% endif %}
 
+<input type="hidden" name="mappings-present" value="yes">
 <table class="govuk-table">
   {% if params.caption %}
   <caption class="govuk-table__caption govuk-table__caption--m">{{params.caption}}</caption>
@@ -67,7 +68,7 @@
       <th scope="row" class="govuk-table__header">{{ h.name }}</th>
       <td class="govuk-table__cell">{{ h.examples }}</td>
       <td class="govuk-table__cell govuk-table__cell--numeric">
-         <select class="govuk-select" style="float: right;" name="{{h.index}}">
+         <select class="govuk-select" style="float: right;" name="field-{{h.index}}">
                 <option name=""></option>
                 {% for field in fields %}
                     <option value="{{field.name}}" {% if field.name == currentValue or (mappingsLen == 0 and field.name == h.name) %}selected{% endif %}>
diff --git a/lib/importer/nunjucks/importer/macros/format_picker.njk b/lib/importer/nunjucks/importer/macros/format_picker.njk
new file mode 100644
index 00000000..08db6d9f
--- /dev/null
+++ b/lib/importer/nunjucks/importer/macros/format_picker.njk
@@ -0,0 +1,87 @@
+{#
+    dudkFormatPicker shows each column heading mapped to a field whose type has
+    format options from the current spreadsheet, alongside a drop-down containing
+    available formats.  This allows the user to choose a format for
+    each column that needs one, to select how the value mapping should be applied.
+
+    The resulting data to be submitted to the 'importerMapDataPath'
+    will be a map of column indices to format names.
+
+    It accepts a data object which is taken from the prototype kit's
+    session data which is made available on every page, and contains the
+    data submitted from forms to the backend, and also the current
+    data import session.
+#}
+
+{% macro dudkFormatPicker(params) %}
+    {% set fields = params.data['importer.session']['fields'] %}
+    {% set mapping = params.data['importer.session']['mapping'] %}
+    {% set headings = importerGetHeaders(params.data) %}
+    {% set headingError = headings.error %}
+    {% set error = importerError(params.data) %}
+
+    {% if headingError %}
+    <p id="mapping-error" class="govuk-error-message">
+      <span class="govuk-visually-hidden">Error:</span> {{ headingError.text }}
+    </p>
+    {% endif %}
+
+    {% if error %}
+      <div class="govuk-error-summary" data-module="govuk-error-summary">
+        <div role="alert">
+          <h2 class="govuk-error-summary__title">
+            {{ error.text }}
+          </h2>
+          <div class="govuk-error-summary__body">
+            <ul class="govuk-list govuk-error-summary__list">
+              {% for e in error.extra %}
+              <li>
+                <a href="#field-{{ e | slugify }}">{{ e }}</a>
+              </li>
+              {% endfor %}
+            </ul>
+          </div>
+        </div>
+      </div>
+    {% endif %}
+
+<input type="hidden" name="formats-present" value="yes">
+<table class="govuk-table">
+  {% if params.caption %}
+  <caption class="govuk-table__caption govuk-table__caption--m">{{params.caption}}</caption>
+  {% endif %}
+  <thead class="govuk-table__head">
+    <tr class="govuk-table__row">
+      <th scope="col" class="govuk-table__header">{{params.columnTitle}}</th>
+      <th scope="col" class="govuk-table__header">{{params.examplesTitle}}</th>
+      <th scope="col" class="govuk-table__header" style="padding-left: 1.5em">{{params.fieldsTitle}}</th>
+    </tr>
+  </thead>
+  <tbody class="govuk-table__body">
+    {% set mappingsLen = mapping | length %}
+    {% for h in headings.data %}
+      {% set hIndex = loop.index0 %}
+      {% set currentValue = importerErrorMappingData(error, hIndex) %}
+      {% set possibleFormats = importerPossibleColumnFormats(params.data, hIndex) %}
+
+    <tr class="govuk-table__row" id="field-{{ h.name | slugify }}">
+      <th scope="row" class="govuk-table__header">{{ h.name }}</th>
+      <td class="govuk-table__cell">{{ h.examples }}</td>
+      <td class="govuk-table__cell govuk-table__cell--numeric">
+         {% if possibleFormats %}
+         <select class="govuk-select" style="float: right;" name="format-{{h.index}}">
+                <option name=""></option>
+                {% for format in possibleFormats %}
+                    <option value="{{format.name}}" {% if format.name == currentValue %}selected{% endif %}>
+                      {{format.displayName}}
+                    </option>
+                {% endfor %}
+         </select>
+         {% endif %}
+      </td>
+    </tr>
+    {% endfor %}
+  </tbody>
+</table>
+{% endmacro %}
+
diff --git a/lib/importer/src/dudk/backend.js b/lib/importer/src/dudk/backend.js
index fa6b85de..92e7ad84 100644
--- a/lib/importer/src/dudk/backend.js
+++ b/lib/importer/src/dudk/backend.js
@@ -6,7 +6,8 @@ const guesser = require('./types/guesser');
 const store = require("./session_store")
 const errorTypes = require('./util/errors');
 
-// An implementation of the interface described in https://struct.register-dynamics.co.uk/trac/wiki/DataImporter/API
+// An implementation of the interface described in
+// https://struct.register-dynamics.co.uk/trac/wiki/DataImporter/API
 
 let sessionStore = function () {
   if (process.env.NODE_ENV == "development") {
@@ -53,14 +54,20 @@ exports.SessionSetFile = (sid, filename) => {
     filename: filename,
     wb: wb,
     sheetNames: Array.from(Object.keys(wb.Sheets)),
+    jobs: new Map(),
+    // State we store for the frontend (we don't do anything with it, just store it and spit it back on demand)
     headerRanges: {}, // A map from sheetname to the header range
     footerRanges: {}, // A map from sheetname to the footer range
-    mappingRules: {},
-    jobs: new Map(),
+    mappingRules: {}, // A map from column index to attribute name
+    formatChoices: {}, // A map from column index to format
   }
   sessionStore.set(sid, session);
 };
 
+///
+/// Storage of UI state for the frontend
+///
+
 exports.SessionGetMappingRules = (sid) => {
   let session = sessionStore.get(sid);
   return session.mappingRules;
@@ -70,11 +77,6 @@ exports.SessionSetMappingRules = (sid, rules) => {
   sessionStore.apply(sid, (s) => { s.mappingRules = rules })
 };
 
-exports.SessionGetSheets = (sid) => {
-  let session = sessionStore.get(sid);
-  return session.sheetNames
-};
-
 exports.SessionSetHeaderRange = (sid, range) => {
   assert(range != null, "Null range passed to SessionSetHeaderRange");
   assert(range.sheet != null, "Range with null sheet passed to SessionSetHeaderRange");
@@ -98,6 +100,24 @@ exports.SessionGetFooterRange = (sid, sheetName) => {
   return session.footerRanges[sheetName]
 };
 
+exports.SessionGetFormatChoices = (sid) => {
+  let session = sessionStore.get(sid);
+  return session.formatChoices;
+};
+
+exports.SessionSetFormatChoices = (sid, choices) => {
+  sessionStore.apply(sid, (s) => { s.formatChoices = choices })
+};
+
+///
+/// Information about the loaded input file
+///
+
+exports.SessionGetSheets = (sid) => {
+  let session = sessionStore.get(sid);
+  return session.sheetNames
+};
+
 function getDimensions(sid) {
   let session = sessionStore.get(sid);
   let sheetDimensions = new Map();
@@ -533,7 +553,7 @@ exports.SessionGuessTypes = (sid, range) => {
 // field "formats" that maps from format names (as returned by SessionGetTypes)
 // to objects with strings field "displayName" and "description".
 exports.SessionGetSupportedTypes = (sid) => {
-  return types.supportedTypes;
+  return attributeTypes.supportedTypes;
 };
 
 // Given guesses as returned by SessionGuessTypes and a list of domain-model
diff --git a/lib/importer/src/dudk/sheets.js b/lib/importer/src/dudk/sheets.js
index f8841a31..4e4ab2e3 100644
--- a/lib/importer/src/dudk/sheets.js
+++ b/lib/importer/src/dudk/sheets.js
@@ -253,10 +253,11 @@ exports.GetColumnValues = (sid, sheet, columnIndex, cellWidth = 30, count = 10)
 };
 
 
-// Convert source mapping (a map from column index -> attribute name) into a mapping for the backend
-// TODO: This should be in the front-end and we should set expectations for what the mapping should
-// look like when sending to the backend.
-const RewriteMapping = (mapping, fields) => {
+// Convert source mapping (a map from column index -> attribute name) and format
+// choices (a map from column index -> format) into a mapping for the backend
+// TODO: This should be in the front-end and we should set expectations for what
+// the mapping should look like when sending to the backend.
+const RewriteMapping = (mapping, formats, fields) => {
   let rewrittenMapping = new Map();
 
   const attrTypes = {}
@@ -268,8 +269,7 @@ const RewriteMapping = (mapping, fields) => {
 
     const f = fields.find((x) => x.name == attributeName)
     if (f) {
-      // FIXME: Actually let the user pick a format, for types that need one. Hardcoding "false" for now.
-      attrTypes[f.name] = attributeTypes.mapperForField(f, false)
+      attrTypes[f.name] = attributeTypes.mapperForField(f, formats[columnIndex])
     }
   }
 
@@ -282,7 +282,7 @@ const RewriteMapping = (mapping, fields) => {
 // Uses the session ID provided, which must contain a sheet name and a
 // mapping to perform the mapping of the data across the remaining
 // rows in the sheet to return an array of objects.
-exports.MapData = (sid, sheet, mapping, fields, previewLimit = DEFAULT_PREVIEW_LIMIT) => {
+exports.MapData = (sid, sheet, mapping, formats, fields, previewLimit = DEFAULT_PREVIEW_LIMIT) => {
   const hRange = backend.SessionGetHeaderRange(sid, sheet)
   const fRange = backend.SessionGetFooterRange(sid, sheet)
 
@@ -290,7 +290,7 @@ exports.MapData = (sid, sheet, mapping, fields, previewLimit = DEFAULT_PREVIEW_L
   const rowRange = backend.SessionSuggestDataRange(sid, hRange, fRange);
 
   // Convert source mapping (a map from column index -> attribute name) into a mapping for the backend
-  let rewrittenMapping = RewriteMapping(mapping, fields)
+  let rewrittenMapping = RewriteMapping(mapping, formats, fields)
 
   // Apply the mapping
   const backendJid = backend.SessionPerformMappingJob(sid, rowRange, rewrittenMapping);
diff --git a/lib/importer/src/dudk/types/attribute-types.js b/lib/importer/src/dudk/types/attribute-types.js
index 08d85182..7f6b8373 100644
--- a/lib/importer/src/dudk/types/attribute-types.js
+++ b/lib/importer/src/dudk/types/attribute-types.js
@@ -74,19 +74,19 @@ exports.mapperForField = (field,format) => {
     case "native":
       // Date with no format spec relies on pre-parsed date values
       // being provided by spreadsheet formats
-      m = dates.MakeCombinedDateType([]);
+      m = dates.makeCombinedDateType([]);
       break;
     case "ymd":
-      m = dates.MakeCombinedDateType(['year','month','day']);
+      m = dates.makeCombinedDateType(['year','month','day']);
       break;
     case "ydm":
-      m = dates.MakeCombinedDateType(['year','day','month']);
+      m = dates.makeCombinedDateType(['year','day','month']);
       break;
     case "dmy":
-      m = dates.MakeCombinedDateType(['day','month','year']);
+      m = dates.makeCombinedDateType(['day','month','year']);
       break;
     case "mdy":
-      m = dates.MakeCombinedDateType(['month','day','year']);
+      m = dates.makeCombinedDateType(['month','day','year']);
       break;
     default:
       throw new Error("Unknown date format", format);
diff --git a/lib/importer/src/functions.js b/lib/importer/src/functions.js
index 827b802a..18d7a530 100644
--- a/lib/importer/src/functions.js
+++ b/lib/importer/src/functions.js
@@ -1,6 +1,7 @@
 
 const sheets_lib = require("./dudk/sheets.js");
 const session_lib = require("./session.js");
+const backend_lib = require("./dudk/backend.js");
 
 const IMPORTER_SESSION_KEY = "importer.session";
 const IMPORTER_ERROR_KEY = "importer.error";
@@ -48,10 +49,39 @@ const importerErrorMappingData = (error, key) => {
         return ""
     }
 
+    // ABS FIXME: Do we need to remove a field- or mapping- prefix from k before parseInt?
     const m = new Map(Object.entries(error.data).map(([k, v]) => [parseInt(k), v]))
     return m.get(key) || ""
 }
 
+//--------------------------------------------------------------------
+// As long as column->field mappings have been set up, this function will return
+// a list of possible formats for a specified column (by index). The return
+// value will be false if the column is mapped to a field whose type does not
+// require formats, otherwise it will be a list of objects with 'name' (internal
+// code), 'displayName' (human-facing name) and 'description' (longer description) fields.
+// --------------------------------------------------------------------
+const importerPossibleColumnFormats = (data, index) => {
+    const session_data = data[IMPORTER_SESSION_KEY];
+    const session = new session_lib.Session(session_data);
+    const supportedTypes = backend_lib.SessionGetSupportedTypes(session.backendSid);
+    const mappings = backend_lib.SessionGetMappingRules(session.backendSid);
+    const columnField = mappings[index];
+    const columnDef = session.fields.find((field) => field.name == columnField);
+    const columnTypeName = columnDef.type;
+    const columnType = supportedTypes.get(columnTypeName);
+    if (columnType.formats) {
+        const options = Array.from(columnType.formats.entries()).map((fmtEntry) => ({
+            name: fmtEntry[0],
+            displayName: fmtEntry[1].displayName,
+            description: fmtEntry[1].description
+        }));
+        return options;
+    } else {
+        return false;
+    }
+}
+
 //--------------------------------------------------------------------
 // Allows a template to obtain `count` rows from the start of the data
 // range.
@@ -156,7 +186,7 @@ const importerMappedData = (data) => {
     const session_data = data[IMPORTER_SESSION_KEY];
     const session = new session_lib.Session(session_data)
 
-    const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.fields);
+    const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.formats, session.fields);
     const headers = session.fields;
 
     return {
@@ -200,7 +230,7 @@ const data_sum = (data, column) => {
     const session_data = data[IMPORTER_SESSION_KEY];
     const session = new session_lib.Session(session_data)
 
-    const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.fields);
+    const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.formats, session.fields);
     const headers = session.fields;
 
     const idx = headers.findIndex((x) => x.name == column)
@@ -220,7 +250,7 @@ const data_avg = (data, column) => {
     const session_data = data[IMPORTER_SESSION_KEY];
     const session = new session_lib.Session(session_data)
 
-    const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.fields);
+    const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.formats, session.fields);
     const headers = session.fields;
 
     const idx = headers.findIndex((x) => x.name == column)
@@ -261,6 +291,7 @@ const parseNumberFromString = (s) => {
 module.exports = {
     importerError,
     importerErrorMappingData,
+    importerPossibleColumnFormats,
     importSheetPreview,
     importerGetRows,
     importerGetHeaders,
@@ -273,3 +304,9 @@ module.exports = {
     data_sum,
     data_avg
 }
+
+/*
+Local Variables:
+js-indent-level: 4
+End:
+*/
diff --git a/lib/importer/src/index.js b/lib/importer/src/index.js
index 74e3a1e2..2ff46c7d 100644
--- a/lib/importer/src/index.js
+++ b/lib/importer/src/index.js
@@ -385,35 +385,55 @@ exports.Initialise = (config, router, prototypeKit) => {
 
       request.session.data['reference_number'] = session.id.match(/\d+/g).join("").substring(0, 8);
 
-      session.mapping = request.body;
+      // This request might set up field mappings, field formats, or both (or
+      // even neither, but that would be weird) so we update those parts of the
+      // session relevant to what we've been sent. Hidden fields called
+      // "formats-present" or "mappings-present" are used to tell us what parts
+      // of the session we should be updating. We can't just use "absence of any
+      // fields" because there's no way to distinguish "clearing the relevant
+      // part of the session out" from "not updating that part of the session",
+      // which might cause us footguns in future.
+
+      if(request.body['formats-present'] == 'yes') {
+        const formats = Object.fromEntries(Object.entries(request.body)
+                                             .filter((f) => f[0].startsWith("format-"))
+                                           .map((f) => [f[0].substring(7),f[1]])); // 7 = length of "format-"
+        session.formats = formats;
+      }
+
+      if(request.body['mappings-present'] == 'yes') {
+        session.mapping = Object.fromEntries(Object.entries(request.body)
+                                  .filter((f) => f[0].startsWith("field-"))
+                                  .map((f) => [f[0].substring(6),f[1]])); // 6 = length of "field-"
 
-      const values = new Array();
+        const values = new Array();
 
-      for (const value of Object.values(session.mapping)) {
-        if (value != '') {
-          values.push(value)
+        for (const value of Object.values(session.mapping)) {
+          if (value != '') {
+            values.push(value)
+          }
         }
-      }
 
-      // For each key in the config that is required, we need to ensure that the field name is present
-      // in the values array. If not then we will
-      const required_fields = plugin_config.fields
-        .filter((f) => f.required)
-        .filter((f) => !values.includes(f.name))
-        .map((f) => f.name)
-
-      // If required_fields has any values left, then they are required fields not present in the
-      // mapping. In which case we should return an error for the user.
-      if (required_fields.length > 0) {
-        request.session.data[IMPORTER_ERROR_KEY] = "The following fields are required"
-        request.session.data[IMPORTER_ERROR_EXTRA_KEY] = required_fields
-        request.session.data[IMPORTER_ERROR_DATA_KEY] = session.mapping
-        response.redirect(request.get('Referrer'));
-        return;
-      } else {
-        delete request.session.data[IMPORTER_ERROR_KEY]
-        delete request.session.data[IMPORTER_ERROR_EXTRA_KEY]
-        delete request.session.data[IMPORTER_ERROR_DATA_KEY]
+        // For each key in the config that is required, we need to ensure that the field name is present
+        // in the values array. If not then we will
+        const required_fields = plugin_config.fields
+              .filter((f) => f.required)
+              .filter((f) => !values.includes(f.name))
+              .map((f) => f.name)
+
+        // If required_fields has any values left, then they are required fields not present in the
+        // mapping. In which case we should return an error for the user.
+        if (required_fields.length > 0) {
+          request.session.data[IMPORTER_ERROR_KEY] = "The following fields are required"
+          request.session.data[IMPORTER_ERROR_EXTRA_KEY] = required_fields
+          request.session.data[IMPORTER_ERROR_DATA_KEY] = session.mapping
+          response.redirect(request.get('Referrer'));
+          return;
+        } else {
+          delete request.session.data[IMPORTER_ERROR_KEY]
+          delete request.session.data[IMPORTER_ERROR_EXTRA_KEY]
+          delete request.session.data[IMPORTER_ERROR_DATA_KEY]
+        }
       }
 
       // Ensure the session is persisted. Currently in session, eventually another way
diff --git a/lib/importer/src/session.js b/lib/importer/src/session.js
index 1f2a8254..47e0f662 100644
--- a/lib/importer/src/session.js
+++ b/lib/importer/src/session.js
@@ -57,6 +57,14 @@ class Session {
     backend.SessionSetMappingRules(this.backendSid, rules)
   }
 
+  get formats() {
+    return backend.SessionGetFormatChoices(this.backendSid)
+  }
+
+  set formats(fmt) {
+    backend.SessionSetFormatChoices(this.backendSid, fmt)
+  }
+
 }
 
 class Range {
diff --git a/lib/importer/templates/mapping.html b/lib/importer/templates/mapping.html
index 68df3985..37e0a2ac 100644
--- a/lib/importer/templates/mapping.html
+++ b/lib/importer/templates/mapping.html
@@ -20,7 +20,7 @@ <h1 class="govuk-heading-l">Identify columns</h1>
 
         <h2 class="govuk-heading-m">{{sheet}}</h2>
 
-        <form action="{{ importerMapDataPath('/review') }}" method="post">
+        <form action="{{ importerMapDataPath('/format') }}" method="post">
 
             {{ dudkFieldMapper({data: data}) }}
 
diff --git a/prototypes/basic/app/config.json b/prototypes/basic/app/config.json
index 35a32895..2b107a74 100644
--- a/prototypes/basic/app/config.json
+++ b/prototypes/basic/app/config.json
@@ -24,7 +24,7 @@
         {
             "name": "Employment start date",
             "required": false,
-            "type": "text"
+            "type": "date"
         },
         {
             "name": "Salary",
diff --git a/prototypes/basic/app/views/mapping.html b/prototypes/basic/app/views/mapping.html
index 6dac044c..a6ccbd2c 100644
--- a/prototypes/basic/app/views/mapping.html
+++ b/prototypes/basic/app/views/mapping.html
@@ -49,7 +49,7 @@ <h1 class="govuk-heading-l">Identify columns</h1>
 
         <h2 class="govuk-heading-m">{{sheet}}</h2>
 
-        <form action="{{ importerMapDataPath('/review') }}" method="post">
+        <form action="{{ importerMapDataPath('/format') }}" method="post">
 
             {{ dudkFieldMapper({data: data}) }}
 

From 8ecfdaf8d294c1990f7f56901983de20a212771e Mon Sep 17 00:00:00 2001
From: Alaric Snell-Pym <alaric@register-dynamics.co.uk>
Date: Tue, 23 Sep 2025 17:20:35 +0100
Subject: [PATCH 02/11] Skip the formats page if no columns are mapped to
 attributes whose types require formats

---
 lib/importer/src/dudk/backend.js        | 50 +++++++++++++++---
 lib/importer/src/index.js               | 12 +++++
 lib/importer/src/session.js             |  1 +
 lib/importer/templates/format.html      | 34 +++++++++++++
 lib/importer/templates/mapping.html     |  2 +-
 prototypes/basic/app/views/format.html  | 67 +++++++++++++++++++++++++
 prototypes/basic/app/views/mapping.html |  2 +-
 7 files changed, 158 insertions(+), 10 deletions(-)
 create mode 100644 lib/importer/templates/format.html
 create mode 100644 prototypes/basic/app/views/format.html

diff --git a/lib/importer/src/dudk/backend.js b/lib/importer/src/dudk/backend.js
index 92e7ad84..24b7d4de 100644
--- a/lib/importer/src/dudk/backend.js
+++ b/lib/importer/src/dudk/backend.js
@@ -56,6 +56,7 @@ exports.SessionSetFile = (sid, filename) => {
     sheetNames: Array.from(Object.keys(wb.Sheets)),
     jobs: new Map(),
     // State we store for the frontend (we don't do anything with it, just store it and spit it back on demand)
+    attributeDefs: [], // A list of objects with {name:string, required:boolean, type:string} defining the attributes we output
     headerRanges: {}, // A map from sheetname to the header range
     footerRanges: {}, // A map from sheetname to the footer range
     mappingRules: {}, // A map from column index to attribute name
@@ -68,14 +69,9 @@ exports.SessionSetFile = (sid, filename) => {
 /// Storage of UI state for the frontend
 ///
 
-exports.SessionGetMappingRules = (sid) => {
-  let session = sessionStore.get(sid);
-  return session.mappingRules;
-};
-
-exports.SessionSetMappingRules = (sid, rules) => {
-  sessionStore.apply(sid, (s) => { s.mappingRules = rules })
-};
+exports.SessionSetAttributes = (sid, attrDefs) => {
+  sessionStore.apply(sid, (s) => { s.attribudeDefs = attrDefs })
+}
 
 exports.SessionSetHeaderRange = (sid, range) => {
   assert(range != null, "Null range passed to SessionSetHeaderRange");
@@ -100,6 +96,15 @@ exports.SessionGetFooterRange = (sid, sheetName) => {
   return session.footerRanges[sheetName]
 };
 
+exports.SessionGetMappingRules = (sid) => {
+  let session = sessionStore.get(sid);
+  return session.mappingRules;
+};
+
+exports.SessionSetMappingRules = (sid, rules) => {
+  sessionStore.apply(sid, (s) => { s.mappingRules = rules })
+};
+
 exports.SessionGetFormatChoices = (sid) => {
   let session = sessionStore.get(sid);
   return session.formatChoices;
@@ -109,6 +114,35 @@ exports.SessionSetFormatChoices = (sid, choices) => {
   sessionStore.apply(sid, (s) => { s.formatChoices = choices })
 };
 
+///
+/// Frontend helpers
+///
+
+
+// Returns an array of column indexes that need a format to be chosen - requires
+// that SessionSetAttributes and SessionSetMappingRules have both happened so we
+// have the required information.
+exports.SessionGetColumnsNeedingFormats = (sid) => {
+  const session = sessionStore.get(sid);
+  const attributes = session.attribudeDefs;
+  const mappings = session.mappingRules;
+  const supportedTypes = attributeTypes.supportedTypes;
+
+  let columnsNeedingFormats = new Array();
+
+  Object.entries(mappings).forEach((entry) => {
+    const [colIdx, columnAttribute] = entry;
+    const columnDef = attributes.find((attr) => attr.name == columnAttribute);
+    const columnTypeName = columnDef.type;
+    const columnType = supportedTypes.get(columnTypeName);
+    if (columnType.formats) {
+      columnsNeedingFormats.push(colIdx);
+    }
+  });
+
+  return columnsNeedingFormats;
+};
+
 ///
 /// Information about the loaded input file
 ///
diff --git a/lib/importer/src/index.js b/lib/importer/src/index.js
index 2ff46c7d..bf774921 100644
--- a/lib/importer/src/index.js
+++ b/lib/importer/src/index.js
@@ -2,6 +2,7 @@ const multer = require("multer");
 const conf = require("./config.js");
 const fs = require("node:fs");
 const path = require("node:path");
+const backend = require("./dudk/backend.js");
 const session_lib = require("./session.js");
 const sheets_lib = require("./dudk/sheets.js");
 const tpl_functions = require("./functions.js")
@@ -434,6 +435,17 @@ exports.Initialise = (config, router, prototypeKit) => {
           delete request.session.data[IMPORTER_ERROR_EXTRA_KEY]
           delete request.session.data[IMPORTER_ERROR_DATA_KEY]
         }
+
+        // If we are processing mappings, AND we are given two next-page paths
+        // to consider, then we should select the other next page if there are
+        // no format choices to be made
+        if ("other" in request.query) {
+          const columnsNeedingFormats = backend.SessionGetColumnsNeedingFormats(session.backendSid);
+          if(columnsNeedingFormats.length == 0) {
+            const otherPage = decodeURIComponent(request.query.other)
+            if (otherPage) { request.query.next = request.query.other }
+          }
+        }
       }
 
       // Ensure the session is persisted. Currently in session, eventually another way
diff --git a/lib/importer/src/session.js b/lib/importer/src/session.js
index 47e0f662..4e9482ad 100644
--- a/lib/importer/src/session.js
+++ b/lib/importer/src/session.js
@@ -14,6 +14,7 @@ class Session {
       backend.SessionSetFile(this.backendSid, p.filename);
     }
 
+    backend.SessionSetAttributes(this.backendSid, p.fields);
     this.fields = p.fields;
     this.sheet = p.sheet ?? "";
   }
diff --git a/lib/importer/templates/format.html b/lib/importer/templates/format.html
new file mode 100644
index 00000000..333253d2
--- /dev/null
+++ b/lib/importer/templates/format.html
@@ -0,0 +1,34 @@
+{% extends "layouts/main.html" %}
+
+{% block pageTitle %} {{ serviceName }} – GOV.UK Prototype Kit {% endblock %}
+
+{% block beforeContent %}
+{{ govukBackLink({ text: "Back", href: "javascript:window.history.back()" }) }}
+{% endblock %}
+
+{% from "importer/macros/format_picker.njk" import dudkFormatPicker %}
+
+{% block content %}
+<div class="govuk-grid-row">
+    <div class="govuk-grid-column-full">
+        <h1 class="govuk-heading-l">Identify columns</h1>
+        <p class="govuk-body">
+            Select a format for any fields that need it. FIXME: Write better text here...
+        </p>
+
+        <h2 class="govuk-heading-m">{{sheet}}</h2>
+
+        <form action="{{ importerMapDataPath('/review') }}" method="post">
+
+            {{ dudkFormatPicker({data: data}) }}
+
+            <div class="govuk-grid-column-two-thirds">
+                <div class="govuk-button-group">
+                    {{ govukButton({ text: "Submit" }) }}
+                </div>
+            </div>
+
+        </form>
+    </div>
+</div>
+{% endblock %}
diff --git a/lib/importer/templates/mapping.html b/lib/importer/templates/mapping.html
index 37e0a2ac..5665c496 100644
--- a/lib/importer/templates/mapping.html
+++ b/lib/importer/templates/mapping.html
@@ -20,7 +20,7 @@ <h1 class="govuk-heading-l">Identify columns</h1>
 
         <h2 class="govuk-heading-m">{{sheet}}</h2>
 
-        <form action="{{ importerMapDataPath('/format') }}" method="post">
+        <form action="{{ importerMapDataPath('/format','/review') }}" method="post">
 
             {{ dudkFieldMapper({data: data}) }}
 
diff --git a/prototypes/basic/app/views/format.html b/prototypes/basic/app/views/format.html
new file mode 100644
index 00000000..ede0d7da
--- /dev/null
+++ b/prototypes/basic/app/views/format.html
@@ -0,0 +1,67 @@
+{% extends "layouts/main.html" %}
+
+{% block pageTitle %} {{ serviceName }} – GOV.UK Prototype Kit {% endblock %}
+
+{% block beforeContent %}
+{{
+    govukBreadcrumbs({
+      items: [
+        {
+          text: "Home",
+          href: "/"
+        },
+        {
+            text: "Upload spreadsheet",
+            href: "/upload"
+        },
+        {
+          text: "Select sheet",
+          href: "/select_sheet"
+        },
+        {
+            text: "Select header row",
+            href: "/select_header_row"
+        },
+        {
+            text: "Select footer row",
+            href: "/select_footer_row"
+        },
+        {
+            text: "Identify columns",
+            href: "/mapping"
+        },
+        {
+	    text: "Choose formats",
+            href: "/format"
+        }
+      ]
+    })
+  }}
+{% endblock %}
+
+{% from "importer/macros/format_picker.njk" import dudkFormatPicker %}
+
+{% block content %}
+<div class="govuk-grid-row">
+    <div class="govuk-grid-column-full">
+        <h1 class="govuk-heading-l">Choose formats</h1>
+        <p class="govuk-body">
+            Select a format for any fields that need it. FIXME: Write better text here...
+        </p>
+
+        <h2 class="govuk-heading-m">{{sheet}}</h2>
+
+        <form action="{{ importerMapDataPath('/review') }}" method="post">
+
+            {{ dudkFormatPicker({data: data}) }}
+
+            <div class="govuk-grid-column-two-thirds">
+                <div class="govuk-button-group">
+                    {{ govukButton({ text: "Submit" }) }}
+                </div>
+            </div>
+
+        </form>
+    </div>
+</div>
+{% endblock %}
diff --git a/prototypes/basic/app/views/mapping.html b/prototypes/basic/app/views/mapping.html
index a6ccbd2c..c8cf6044 100644
--- a/prototypes/basic/app/views/mapping.html
+++ b/prototypes/basic/app/views/mapping.html
@@ -49,7 +49,7 @@ <h1 class="govuk-heading-l">Identify columns</h1>
 
         <h2 class="govuk-heading-m">{{sheet}}</h2>
 
-        <form action="{{ importerMapDataPath('/format') }}" method="post">
+        <form action="{{ importerMapDataPath('/format','/review') }}" method="post">
 
             {{ dudkFieldMapper({data: data}) }}
 

From da0d82dfb5e9b5477fef3c2e6c99e312d7726c01 Mon Sep 17 00:00:00 2001
From: Alaric Snell-Pym <alaric@register-dynamics.co.uk>
Date: Thu, 25 Sep 2025 14:08:02 +0100
Subject: [PATCH 03/11] Using guessed formats to guide the user in picking
 formats.

Includes a few bug fixes to the guesser!
---
 .../importer/macros/format_picker.njk         | 13 +++-
 .../views/error-handling/server-error.njk     |  5 --
 lib/importer/src/dudk/backend.js              | 78 ++++++++++---------
 lib/importer/src/dudk/backend.test.js         |  2 +-
 .../src/dudk/types/attribute-types.js         |  2 +-
 lib/importer/src/dudk/types/guesser.js        | 19 +++++
 lib/importer/src/functions.js                 | 62 ++++++++++-----
 lib/importer/templates/format.html            |  5 +-
 prototypes/basic/app/views/format.html        |  5 +-
 9 files changed, 122 insertions(+), 69 deletions(-)

diff --git a/lib/importer/nunjucks/importer/macros/format_picker.njk b/lib/importer/nunjucks/importer/macros/format_picker.njk
index 08db6d9f..d442993b 100644
--- a/lib/importer/nunjucks/importer/macros/format_picker.njk
+++ b/lib/importer/nunjucks/importer/macros/format_picker.njk
@@ -59,10 +59,11 @@
   </thead>
   <tbody class="govuk-table__body">
     {% set mappingsLen = mapping | length %}
+    {% set possibleFormatsByColumn = importerPossibleColumnFormats(params.data) %}
     {% for h in headings.data %}
       {% set hIndex = loop.index0 %}
       {% set currentValue = importerErrorMappingData(error, hIndex) %}
-      {% set possibleFormats = importerPossibleColumnFormats(params.data, hIndex) %}
+      {% set possibleFormats = possibleFormatsByColumn[hIndex] %}
 
     <tr class="govuk-table__row" id="field-{{ h.name | slugify }}">
       <th scope="row" class="govuk-table__header">{{ h.name }}</th>
@@ -70,10 +71,14 @@
       <td class="govuk-table__cell govuk-table__cell--numeric">
          {% if possibleFormats %}
          <select class="govuk-select" style="float: right;" name="format-{{h.index}}">
-                <option name=""></option>
                 {% for format in possibleFormats %}
-                    <option value="{{format.name}}" {% if format.name == currentValue %}selected{% endif %}>
-                      {{format.displayName}}
+                    <option value="{{format.name}}"
+                            {% if format.name == currentValue %}selected{% endif %}>
+                      {% if format.likely %}
+                         {{format.displayName}} (*)
+                      {% else %}
+                         {{format.displayName}}
+                      {% endif %}
                     </option>
                 {% endfor %}
          </select>
diff --git a/lib/importer/nunjucks/views/error-handling/server-error.njk b/lib/importer/nunjucks/views/error-handling/server-error.njk
index f3b424d4..9726d16e 100644
--- a/lib/importer/nunjucks/views/error-handling/server-error.njk
+++ b/lib/importer/nunjucks/views/error-handling/server-error.njk
@@ -45,11 +45,6 @@
 {% endblock %}
 
 
-{% block footer %}
-  {{ govukFooter({}) }}
-{% endblock %}
-
-
 {% block pageScripts %}
 <script>
   ;(() => {
diff --git a/lib/importer/src/dudk/backend.js b/lib/importer/src/dudk/backend.js
index 24b7d4de..33fbbc3c 100644
--- a/lib/importer/src/dudk/backend.js
+++ b/lib/importer/src/dudk/backend.js
@@ -70,7 +70,7 @@ exports.SessionSetFile = (sid, filename) => {
 ///
 
 exports.SessionSetAttributes = (sid, attrDefs) => {
-  sessionStore.apply(sid, (s) => { s.attribudeDefs = attrDefs })
+  sessionStore.apply(sid, (s) => { s.attributeDefs = attrDefs })
 }
 
 exports.SessionSetHeaderRange = (sid, range) => {
@@ -124,7 +124,7 @@ exports.SessionSetFormatChoices = (sid, choices) => {
 // have the required information.
 exports.SessionGetColumnsNeedingFormats = (sid) => {
   const session = sessionStore.get(sid);
-  const attributes = session.attribudeDefs;
+  const attributes = session.attributeDefs;
   const mappings = session.mappingRules;
   const supportedTypes = attributeTypes.supportedTypes;
 
@@ -132,17 +132,49 @@ exports.SessionGetColumnsNeedingFormats = (sid) => {
 
   Object.entries(mappings).forEach((entry) => {
     const [colIdx, columnAttribute] = entry;
-    const columnDef = attributes.find((attr) => attr.name == columnAttribute);
-    const columnTypeName = columnDef.type;
-    const columnType = supportedTypes.get(columnTypeName);
-    if (columnType.formats) {
-      columnsNeedingFormats.push(colIdx);
+    if(columnAttribute != "") {
+      const columnDef = attributes.find((attr) => attr.name == columnAttribute);
+      const columnTypeName = columnDef.type;
+      const columnType = supportedTypes.get(columnTypeName);
+      if (columnType.formats) {
+        columnsNeedingFormats.push(colIdx);
+      }
     }
   });
 
   return columnsNeedingFormats;
 };
 
+// Given guesses as returned by SessionGuessTypes and a list of domain-model
+// fields (with name and type fields), returns the type guesses with each column
+// augmented with a `fields` field - a Map mapping likely field names for this
+// column to an array of possible formats for that field.
+
+exports.SessionSuggestFields = (sid, typeGuesses, domainModelFields) => {
+  let result = [];
+
+  typeGuesses.forEach((colTypeGuess) => {
+    let fieldFormats = new Map(); // Map from field names to likely formats
+    // Examine every field, and check if they have a type that this column might
+    // contain
+    domainModelFields.forEach((field) => {
+      if(colTypeGuess.types.has(field.type)) {
+        // Add this field to the possibilities for the column, using the guessed
+        // formats
+        fieldFormats.set(field.name, colTypeGuess.types.get(field.type));
+      }
+    });
+
+    // Extend the column type guess with the "fields" field
+    let typeGuessesAndFields = Object.assign({}, colTypeGuess);
+    typeGuessesAndFields.fields = fieldFormats;
+    result.push(typeGuessesAndFields);
+  });
+
+  return result;
+};
+
+
 ///
 /// Information about the loaded input file
 ///
@@ -514,7 +546,7 @@ exports.SessionGuessTypes = (sid, range) => {
   }
 
   // How many columns do we have?
-  const columns = range.end.column-range.start.column;
+  const columns = (range.end.column-range.start.column)+1;
 
   let guesses = new Array(columns);
   if (range.end.row < range.start.row) {
@@ -535,7 +567,7 @@ exports.SessionGuessTypes = (sid, range) => {
   const merges = sheet["!merges"];
 
   // Examine each row in turn, looking at the columns in parallel
-  for (let row = range.start.row; row < range.end.row; row++) {
+  for (let row = range.start.row; row <= range.end.row; row++) {
     let rowData = extractColsFromRow(getMergedRow(data, merges, row),
                                      range.start.column, range.end.column + 1);
     for(let col = 0; col < columns; col++) {
@@ -590,34 +622,6 @@ exports.SessionGetSupportedTypes = (sid) => {
   return attributeTypes.supportedTypes;
 };
 
-// Given guesses as returned by SessionGuessTypes and a list of domain-model
-// fields (with name and type fields), returns the type guesses with each column
-// augmented with a `fields` field - a Map mapping likely field names for this
-// column to an array of possible formats for that field.
-
-exports.SessionSuggestFields = (sid, typeGuesses, domainModelFields) => {
-  let result = [];
-
-  typeGuesses.forEach((colTypeGuess) => {
-    let fieldFormats = new Map(); // Map from field names to likely formats
-    // Examine every field, and check if they have a type that this column might
-    // contain
-    domainModelFields.forEach((field) => {
-      if(colTypeGuess.types.has(field.type)) {
-        // Add this field to the possibilities for the column, using the guessed
-        // formats
-        fieldFormats.set(field.name, colTypeGuess.types.get(field.type));
-      }
-    });
-
-    // Extend the column type guess with the "fields" field
-    let typeGuessesAndFields = Object.assign({}, colTypeGuess);
-    typeGuessesAndFields.fields = fieldFormats;
-    result.push(typeGuessesAndFields);
-  });
-
-  return result;
-};
 
 // Return the unique values in each column in the range. Return no more than
 // maxValues values for any given column. Return format is an array, one entry
diff --git a/lib/importer/src/dudk/backend.test.js b/lib/importer/src/dudk/backend.test.js
index 6d471fc3..09aa0cb7 100644
--- a/lib/importer/src/dudk/backend.test.js
+++ b/lib/importer/src/dudk/backend.test.js
@@ -603,7 +603,7 @@ test('guessing types', () => {
   const dataRange = {
     sheet: 'Sheet1',
     start: { row: 1, column: 0 },
-    end: { row: 6, column: 9 }
+    end: { row: 6, column: 8 }
   };
 
   const guesses = backend.SessionGuessTypes(sid, dataRange);
diff --git a/lib/importer/src/dudk/types/attribute-types.js b/lib/importer/src/dudk/types/attribute-types.js
index 7f6b8373..20ec7fad 100644
--- a/lib/importer/src/dudk/types/attribute-types.js
+++ b/lib/importer/src/dudk/types/attribute-types.js
@@ -111,7 +111,7 @@ exports.supportedTypes = new Map([
   ["boolean", {displayName: "Y/N", description: "A 'Yes' or 'No'"}],
   ["date", {displayName: "Date", description: "A full date (eg Year, Month and Day number)",
             formats: new Map([
-              ["native", {displayName: "Spreadsheet", description: "A date cell in a spreadsheet"}],
+              ["native", {displayName: "Spreadsheet Date", description: "A date cell in a spreadsheet"}],
               ["ymd", {displayName: "Y/M/D", description: "A date written in Year, Month, Day order"}],
               ["ydm", {displayName: "Y/D/M", description: "A date written in Year, Day, Month order"}],
               ["dmy", {displayName: "D/M/Y", description: "A date written in Day, Month, Year order"}],
diff --git a/lib/importer/src/dudk/types/guesser.js b/lib/importer/src/dudk/types/guesser.js
index e7c251a5..aeab34e4 100644
--- a/lib/importer/src/dudk/types/guesser.js
+++ b/lib/importer/src/dudk/types/guesser.js
@@ -35,6 +35,18 @@ const TYPE_PATTERNS = [
 
 exports.TYPE_PATTERNS = TYPE_PATTERNS;
 
+// Does the format string look like a date?  See "Common Date-Time Formats" at
+// https://docs.sheetjs.com/docs/csf/features/dates for the source of these
+// patterns.
+const formatStringPartRE = new RegExp("[^a-zA-Z]+");
+// Note that m and mm can also represent minutes in time formats
+const formatStringDateParts = ["yy","yyyy","m","mm","mmm","mmmm","mmmmm","d","dd","ddd","dddd"];
+function isDateFormat(fmt) {
+  const parts = fmt.split(formatStringPartRE);
+  // Is every token in the format string valid for a date format?
+  return parts.every((part) => formatStringDateParts.includes(part));
+}
+
 // Given a sheet.js field, return two values. First is a list of possible types
 // for it, as a map from type names to either:
 
@@ -64,6 +76,13 @@ exports.ListPossibleTypes = (field) => {
   case "z": // Empty cell
     // Either way, we can't narrow the set of possible types, but it's a blank cell
     return [false, true];
+  case "d": // Date or time or both
+    if(isDateFormat(field.z)) {
+      possibleTypes.set("date", ["native"]);
+    }
+    // FIXME: Add cases for time formats, eg "h:mm", when we support them
+    return [possibleTypes, isBlank];
+    break;
   case "s":
     // String
     if(isBlank) {
diff --git a/lib/importer/src/functions.js b/lib/importer/src/functions.js
index 18d7a530..d96f3f30 100644
--- a/lib/importer/src/functions.js
+++ b/lib/importer/src/functions.js
@@ -56,30 +56,54 @@ const importerErrorMappingData = (error, key) => {
 
 //--------------------------------------------------------------------
 // As long as column->field mappings have been set up, this function will return
-// a list of possible formats for a specified column (by index). The return
-// value will be false if the column is mapped to a field whose type does not
-// require formats, otherwise it will be a list of objects with 'name' (internal
-// code), 'displayName' (human-facing name) and 'description' (longer description) fields.
+// an array indexed on column, with each element being list of possible formats
+// for that column. The array element for a column will be false if the column is mapped to a
+// field whose type does not require formats, otherwise it will be a list of
+// objects with 'name' (internal code), 'displayName' (human-facing name) and
+// 'description' (longer description) fields.
 // --------------------------------------------------------------------
-const importerPossibleColumnFormats = (data, index) => {
+const importerPossibleColumnFormats = (data) => {
+    // FIXME: This should probably move into the backend, as a SessionSuggestFormats function
     const session_data = data[IMPORTER_SESSION_KEY];
     const session = new session_lib.Session(session_data);
     const supportedTypes = backend_lib.SessionGetSupportedTypes(session.backendSid);
     const mappings = backend_lib.SessionGetMappingRules(session.backendSid);
-    const columnField = mappings[index];
-    const columnDef = session.fields.find((field) => field.name == columnField);
-    const columnTypeName = columnDef.type;
-    const columnType = supportedTypes.get(columnTypeName);
-    if (columnType.formats) {
-        const options = Array.from(columnType.formats.entries()).map((fmtEntry) => ({
-            name: fmtEntry[0],
-            displayName: fmtEntry[1].displayName,
-            description: fmtEntry[1].description
-        }));
-        return options;
-    } else {
-        return false;
-    }
+    const hRange = backend_lib.SessionGetHeaderRange(session.backendSid, session.sheet)
+    const fRange = backend_lib.SessionGetFooterRange(session.backendSid, session.sheet)
+    const dataRange = backend_lib.SessionSuggestDataRange(session.backendSid, hRange, fRange);
+    const typeGuesses = backend_lib.SessionGuessTypes(session.backendSid, dataRange);
+
+    return Object.entries(mappings).map((entry) => {
+        const [colIndex, columnField] = entry;
+        if (!columnField) return false; // Skip unmapped columns
+        const columnGuesses = typeGuesses[colIndex];
+        const columnDef = session.fields.find((field) => field.name == columnField);
+        const columnTypeName = columnDef.type;
+        const columnType = supportedTypes.get(columnTypeName);
+        let likelyFormats = new Set(); // Names of formats this column is likely to have, according to the guesser
+        if (columnGuesses.types && columnGuesses.types.has(columnTypeName)) {
+            const guessedFormats = columnGuesses.types.get(columnTypeName);
+            if(guessedFormats && columnType.formats) {
+                likelyFormats = new Set(Array.from(columnType.formats.keys()).filter((fmtName) => {
+                    return guessedFormats.includes(fmtName);
+                }));
+            }
+        }
+
+        if (columnType.formats) {
+            const options = Array.from(columnType.formats.entries()).map((fmtEntry) => {
+                return {
+                    name: fmtEntry[0],
+                    displayName: fmtEntry[1].displayName,
+                    description: fmtEntry[1].description,
+                    likely: likelyFormats.has(fmtEntry[0])
+                };
+            });
+            return options;
+        } else {
+            return false;
+        }
+    });
 }
 
 //--------------------------------------------------------------------
diff --git a/lib/importer/templates/format.html b/lib/importer/templates/format.html
index 333253d2..cc0f2bcb 100644
--- a/lib/importer/templates/format.html
+++ b/lib/importer/templates/format.html
@@ -13,7 +13,10 @@
     <div class="govuk-grid-column-full">
         <h1 class="govuk-heading-l">Identify columns</h1>
         <p class="govuk-body">
-            Select a format for any fields that need it. FIXME: Write better text here...
+            One or more columns may contain data represented in one of a number
+            of formats, such as dates. Please select the format your file uses
+            in the table below. Formats that appear to match the data found in
+            your file are marked with an asterisk (*).
         </p>
 
         <h2 class="govuk-heading-m">{{sheet}}</h2>
diff --git a/prototypes/basic/app/views/format.html b/prototypes/basic/app/views/format.html
index ede0d7da..19613df2 100644
--- a/prototypes/basic/app/views/format.html
+++ b/prototypes/basic/app/views/format.html
@@ -46,7 +46,10 @@
     <div class="govuk-grid-column-full">
         <h1 class="govuk-heading-l">Choose formats</h1>
         <p class="govuk-body">
-            Select a format for any fields that need it. FIXME: Write better text here...
+            One or more columns may contain data represented in one of a number
+            of formats, such as dates. Please select the format your file uses
+            in the table below. Formats that appear to match the data found in
+            your file are marked with an asterisk (*).
         </p>
 
         <h2 class="govuk-heading-m">{{sheet}}</h2>

From 2f2417f688099c163807b2dade4f1f1812a453c7 Mon Sep 17 00:00:00 2001
From: Alaric Snell-Pym <alaric@register-dynamics.co.uk>
Date: Thu, 25 Sep 2025 14:30:01 +0100
Subject: [PATCH 04/11] Show the mapped field and type in the format picker
 page, as per the mockup

---
 .../importer/macros/format_picker.njk         |  8 ++++++
 lib/importer/src/functions.js                 | 26 +++++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/lib/importer/nunjucks/importer/macros/format_picker.njk b/lib/importer/nunjucks/importer/macros/format_picker.njk
index d442993b..5c9e617c 100644
--- a/lib/importer/nunjucks/importer/macros/format_picker.njk
+++ b/lib/importer/nunjucks/importer/macros/format_picker.njk
@@ -54,20 +54,28 @@
     <tr class="govuk-table__row">
       <th scope="col" class="govuk-table__header">{{params.columnTitle}}</th>
       <th scope="col" class="govuk-table__header">{{params.examplesTitle}}</th>
+      <th scope="col" class="govuk-table__header">{{params.fieldAndTypeTitle}}</th>
       <th scope="col" class="govuk-table__header" style="padding-left: 1.5em">{{params.fieldsTitle}}</th>
     </tr>
   </thead>
   <tbody class="govuk-table__body">
     {% set mappingsLen = mapping | length %}
     {% set possibleFormatsByColumn = importerPossibleColumnFormats(params.data) %}
+    {% set mappedColumnDetails = importerMappedColumnDetails(params.data) %}
     {% for h in headings.data %}
       {% set hIndex = loop.index0 %}
       {% set currentValue = importerErrorMappingData(error, hIndex) %}
       {% set possibleFormats = possibleFormatsByColumn[hIndex] %}
+      {% set mapping = mappedColumnDetails[hIndex] %}
 
     <tr class="govuk-table__row" id="field-{{ h.name | slugify }}">
       <th scope="row" class="govuk-table__header">{{ h.name }}</th>
       <td class="govuk-table__cell">{{ h.examples }}</td>
+      <td class="govuk-table__cell">
+          {% if mapping %}
+              {{ mapping.fieldName }} ({{ mapping.typeName }})
+          {% endif %}
+      </td>
       <td class="govuk-table__cell govuk-table__cell--numeric">
          {% if possibleFormats %}
          <select class="govuk-select" style="float: right;" name="format-{{h.index}}">
diff --git a/lib/importer/src/functions.js b/lib/importer/src/functions.js
index d96f3f30..1b372552 100644
--- a/lib/importer/src/functions.js
+++ b/lib/importer/src/functions.js
@@ -54,6 +54,31 @@ const importerErrorMappingData = (error, key) => {
     return m.get(key) || ""
 }
 
+//--------------------------------------------------------------------
+// As long as column->field mappings have been set up, this function will return
+// an array indexed on column, with each element being of the form { fieldName:
+// <field name>, typeName: <type name> } if the column is to be mapped to a
+// field, or false if not.
+//--------------------------------------------------------------------
+const importerMappedColumnDetails = (data) => {
+    const session_data = data[IMPORTER_SESSION_KEY];
+    const session = new session_lib.Session(session_data);
+    const supportedTypes = backend_lib.SessionGetSupportedTypes(session.backendSid);
+    const mappings = backend_lib.SessionGetMappingRules(session.backendSid);
+
+    return Object.entries(mappings).map((entry) => {
+        const [colIndex, columnField] = entry;
+        if (!columnField) return false; // Skip unmapped columns
+        const columnDef = session.fields.find((field) => field.name == columnField);
+        const columnTypeName = columnDef.type;
+        const columnType = supportedTypes.get(columnTypeName);
+        return {
+            fieldName: columnField,
+            typeName: columnType.displayName
+        };
+    });
+};
+
 //--------------------------------------------------------------------
 // As long as column->field mappings have been set up, this function will return
 // an array indexed on column, with each element being list of possible formats
@@ -316,6 +341,7 @@ module.exports = {
     importerError,
     importerErrorMappingData,
     importerPossibleColumnFormats,
+    importerMappedColumnDetails,
     importSheetPreview,
     importerGetRows,
     importerGetHeaders,

From d43b0878b5738ee7d92d08a5fea3803bcdf21ea0 Mon Sep 17 00:00:00 2001
From: Alaric Snell-Pym <alaric@register-dynamics.co.uk>
Date: Thu, 25 Sep 2025 14:31:40 +0100
Subject: [PATCH 05/11] Appease linter

---
 lib/importer/src/functions.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/importer/src/functions.js b/lib/importer/src/functions.js
index 1b372552..f62d54b4 100644
--- a/lib/importer/src/functions.js
+++ b/lib/importer/src/functions.js
@@ -67,7 +67,7 @@ const importerMappedColumnDetails = (data) => {
     const mappings = backend_lib.SessionGetMappingRules(session.backendSid);
 
     return Object.entries(mappings).map((entry) => {
-        const [colIndex, columnField] = entry;
+        const columnField = entry[1];
         if (!columnField) return false; // Skip unmapped columns
         const columnDef = session.fields.find((field) => field.name == columnField);
         const columnTypeName = columnDef.type;

From e1553a634d305217219b05b04b5bb603fc189391 Mon Sep 17 00:00:00 2001
From: Alaric Snell-Pym <alaric@register-dynamics.co.uk>
Date: Thu, 25 Sep 2025 17:45:27 +0100
Subject: [PATCH 06/11] Fix playwright test

---
 prototypes/basic/tests/helpers/mapping_page.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/prototypes/basic/tests/helpers/mapping_page.js b/prototypes/basic/tests/helpers/mapping_page.js
index 0cd2ca1e..74b870f2 100644
--- a/prototypes/basic/tests/helpers/mapping_page.js
+++ b/prototypes/basic/tests/helpers/mapping_page.js
@@ -39,7 +39,7 @@ export class MappingPage {
     setMapping = async(columnName, fieldName) => {
         let cols = await this.getColumnNames()
         let idx = cols.findIndex((x)=>x == columnName);
-        await this.p.locator(`[name='${idx}']`).selectOption(fieldName)
+        await this.p.locator(`[name='field-${idx}']`).selectOption(fieldName)
     }
 
     submit = async() => {

From d387fa5336efc1d9531e89fa525edc7ee63fd509 Mon Sep 17 00:00:00 2001
From: Alaric Snell-Pym <alaric@register-dynamics.co.uk>
Date: Thu, 16 Oct 2025 11:32:50 +0100
Subject: [PATCH 07/11] Code review feedback

---
 .../nunjucks/views/error-handling/server-error.njk |  5 +++++
 lib/importer/src/dudk/backend.js                   |  3 +--
 lib/importer/src/dudk/types/guesser.js             | 14 ++++++++++----
 lib/importer/src/functions.js                      |  7 ++-----
 4 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/lib/importer/nunjucks/views/error-handling/server-error.njk b/lib/importer/nunjucks/views/error-handling/server-error.njk
index 9726d16e..f3b424d4 100644
--- a/lib/importer/nunjucks/views/error-handling/server-error.njk
+++ b/lib/importer/nunjucks/views/error-handling/server-error.njk
@@ -45,6 +45,11 @@
 {% endblock %}
 
 
+{% block footer %}
+  {{ govukFooter({}) }}
+{% endblock %}
+
+
 {% block pageScripts %}
 <script>
   ;(() => {
diff --git a/lib/importer/src/dudk/backend.js b/lib/importer/src/dudk/backend.js
index 33fbbc3c..dbb625e0 100644
--- a/lib/importer/src/dudk/backend.js
+++ b/lib/importer/src/dudk/backend.js
@@ -130,8 +130,7 @@ exports.SessionGetColumnsNeedingFormats = (sid) => {
 
   let columnsNeedingFormats = new Array();
 
-  Object.entries(mappings).forEach((entry) => {
-    const [colIdx, columnAttribute] = entry;
+  Object.entries(mappings).forEach((colIdx, columnAttribute) => {
     if(columnAttribute != "") {
       const columnDef = attributes.find((attr) => attr.name == columnAttribute);
       const columnTypeName = columnDef.type;
diff --git a/lib/importer/src/dudk/types/guesser.js b/lib/importer/src/dudk/types/guesser.js
index aeab34e4..3d63b962 100644
--- a/lib/importer/src/dudk/types/guesser.js
+++ b/lib/importer/src/dudk/types/guesser.js
@@ -77,11 +77,17 @@ exports.ListPossibleTypes = (field) => {
     // Either way, we can't narrow the set of possible types, but it's a blank cell
     return [false, true];
   case "d": // Date or time or both
-    if(isDateFormat(field.z)) {
-      possibleTypes.set("date", ["native"]);
+    if(field.z) {
+      if(isDateFormat(field.z)) {
+        possibleTypes.set("date", ["native"]);
+      }
+      // FIXME: Add cases for time formats, eg "h:mm", when we support them
+      return [possibleTypes, isBlank];
+    } else {
+      // Field format code in field.z should always be present, but let's fail
+      // quietly if not rather than ruining the whole import
+      return [possibleTypes, isBlank];
     }
-    // FIXME: Add cases for time formats, eg "h:mm", when we support them
-    return [possibleTypes, isBlank];
     break;
   case "s":
     // String
diff --git a/lib/importer/src/functions.js b/lib/importer/src/functions.js
index f62d54b4..00f804ef 100644
--- a/lib/importer/src/functions.js
+++ b/lib/importer/src/functions.js
@@ -98,8 +98,7 @@ const importerPossibleColumnFormats = (data) => {
     const dataRange = backend_lib.SessionSuggestDataRange(session.backendSid, hRange, fRange);
     const typeGuesses = backend_lib.SessionGuessTypes(session.backendSid, dataRange);
 
-    return Object.entries(mappings).map((entry) => {
-        const [colIndex, columnField] = entry;
+    return Object.entries(mappings).map((colIndex, columnField) => {
         if (!columnField) return false; // Skip unmapped columns
         const columnGuesses = typeGuesses[colIndex];
         const columnDef = session.fields.find((field) => field.name == columnField);
@@ -109,9 +108,7 @@ const importerPossibleColumnFormats = (data) => {
         if (columnGuesses.types && columnGuesses.types.has(columnTypeName)) {
             const guessedFormats = columnGuesses.types.get(columnTypeName);
             if(guessedFormats && columnType.formats) {
-                likelyFormats = new Set(Array.from(columnType.formats.keys()).filter((fmtName) => {
-                    return guessedFormats.includes(fmtName);
-                }));
+                likelyFormats = new Set(Array.from(columnType.formats.keys()).filter((fmtName) => guessedFormats.includes(fmtName)));
             }
         }
 

From 59a17999a98fa90f9555b56b08a2c7f94c1e0643 Mon Sep 17 00:00:00 2001
From: Alaric Snell-Pym <alaric@register-dynamics.co.uk>
Date: Thu, 23 Oct 2025 17:29:38 +0100
Subject: [PATCH 08/11] Fix bugs in review feedback (turns out you can't
 deconstruct things quite as hoped)

---
 lib/importer/src/dudk/backend.js | 4 ++--
 lib/importer/src/functions.js    | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/importer/src/dudk/backend.js b/lib/importer/src/dudk/backend.js
index dbb625e0..27b08527 100644
--- a/lib/importer/src/dudk/backend.js
+++ b/lib/importer/src/dudk/backend.js
@@ -130,7 +130,7 @@ exports.SessionGetColumnsNeedingFormats = (sid) => {
 
   let columnsNeedingFormats = new Array();
 
-  Object.entries(mappings).forEach((colIdx, columnAttribute) => {
+  for (const [colIdx, columnAttribute] of Object.entries(mappings)) {
     if(columnAttribute != "") {
       const columnDef = attributes.find((attr) => attr.name == columnAttribute);
       const columnTypeName = columnDef.type;
@@ -139,7 +139,7 @@ exports.SessionGetColumnsNeedingFormats = (sid) => {
         columnsNeedingFormats.push(colIdx);
       }
     }
-  });
+  }
 
   return columnsNeedingFormats;
 };
diff --git a/lib/importer/src/functions.js b/lib/importer/src/functions.js
index 00f804ef..a2d13450 100644
--- a/lib/importer/src/functions.js
+++ b/lib/importer/src/functions.js
@@ -98,7 +98,8 @@ const importerPossibleColumnFormats = (data) => {
     const dataRange = backend_lib.SessionSuggestDataRange(session.backendSid, hRange, fRange);
     const typeGuesses = backend_lib.SessionGuessTypes(session.backendSid, dataRange);
 
-    return Object.entries(mappings).map((colIndex, columnField) => {
+    return Object.entries(mappings).map((entry) => {
+        const [colIndex, columnField] = entry;
         if (!columnField) return false; // Skip unmapped columns
         const columnGuesses = typeGuesses[colIndex];
         const columnDef = session.fields.find((field) => field.name == columnField);

From bacc101f3a1cbe0e178b0726f8b2edb20807a5d9 Mon Sep 17 00:00:00 2001
From: Alaric Snell-Pym <alaric@register-dynamics.co.uk>
Date: Thu, 23 Oct 2025 17:30:17 +0100
Subject: [PATCH 09/11] Review feedback: Don't imply separators in user-visible
 date format names

---
 lib/importer/src/dudk/types/attribute-types.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/importer/src/dudk/types/attribute-types.js b/lib/importer/src/dudk/types/attribute-types.js
index 20ec7fad..c509c438 100644
--- a/lib/importer/src/dudk/types/attribute-types.js
+++ b/lib/importer/src/dudk/types/attribute-types.js
@@ -112,9 +112,9 @@ exports.supportedTypes = new Map([
   ["date", {displayName: "Date", description: "A full date (eg Year, Month and Day number)",
             formats: new Map([
               ["native", {displayName: "Spreadsheet Date", description: "A date cell in a spreadsheet"}],
-              ["ymd", {displayName: "Y/M/D", description: "A date written in Year, Month, Day order"}],
-              ["ydm", {displayName: "Y/D/M", description: "A date written in Year, Day, Month order"}],
-              ["dmy", {displayName: "D/M/Y", description: "A date written in Day, Month, Year order"}],
-              ["mdy", {displayName: "M/D/Y", description: "A date written in Month, Day, Year order"}]
+              ["ymd", {displayName: "Year, Month, Day", description: "A date written in Year, Month, Day order"}],
+              ["ydm", {displayName: "Year, Day, Month", description: "A date written in Year, Day, Month order"}],
+              ["dmy", {displayName: "Day, Month, Year", description: "A date written in Day, Month, Year order"}],
+              ["mdy", {displayName: "Month, Day, Year", description: "A date written in Month, Day, Year order"}]
             ])}]
 ]);

From 5f448a13dc099132659b0a2be823e41e7263cdef Mon Sep 17 00:00:00 2001
From: Alaric Snell-Pym <alaric@register-dynamics.co.uk>
Date: Thu, 23 Oct 2025 17:31:25 +0100
Subject: [PATCH 10/11] Review feedback (part 1): Extend type/format guesses to
 return both "safe" and "maybe" options

"Safe" options are valid for EVERY value in the column.
"Maybe" options are valid for SOME values in the column (but not every).

The frontend needs update to reflect this distinction, only showing
safe/maybe options (rather than ALL options as currently).
---
 lib/importer/src/dudk/backend.js      | 104 ++++++++++++++++----
 lib/importer/src/dudk/backend.test.js | 131 ++++++++++++++++----------
 2 files changed, 168 insertions(+), 67 deletions(-)

diff --git a/lib/importer/src/dudk/backend.js b/lib/importer/src/dudk/backend.js
index 27b08527..59ae47c6 100644
--- a/lib/importer/src/dudk/backend.js
+++ b/lib/importer/src/dudk/backend.js
@@ -147,26 +147,34 @@ exports.SessionGetColumnsNeedingFormats = (sid) => {
 // Given guesses as returned by SessionGuessTypes and a list of domain-model
 // fields (with name and type fields), returns the type guesses with each column
 // augmented with a `fields` field - a Map mapping likely field names for this
-// column to an array of possible formats for that field.
+// column to an array of safe formats for that field - and a `maybeFields` field
+// - the same but for "maybe" fields that might be a partial match for the data.
 
 exports.SessionSuggestFields = (sid, typeGuesses, domainModelFields) => {
   let result = [];
 
   typeGuesses.forEach((colTypeGuess) => {
-    let fieldFormats = new Map(); // Map from field names to likely formats
+    let safeFieldFormats = new Map(); // Map from field names to safe formats
+    let maybeFieldFormats = new Map(); // Map from field names to possible formats
     // Examine every field, and check if they have a type that this column might
     // contain
     domainModelFields.forEach((field) => {
-      if(colTypeGuess.types.has(field.type)) {
-        // Add this field to the possibilities for the column, using the guessed
+      if(colTypeGuess.safeTypes.has(field.type)) {
+        // Add this field to the safe possibilities for the column, using the guessed
         // formats
-        fieldFormats.set(field.name, colTypeGuess.types.get(field.type));
+        safeFieldFormats.set(field.name, colTypeGuess.safeTypes.get(field.type));
+      }
+      if(colTypeGuess.maybeTypes.has(field.type)) {
+        // Add this field to the maybe possibilities for the column, using the guessed
+        // formats
+        maybeFieldFormats.set(field.name, colTypeGuess.maybeTypes.get(field.type));
       }
     });
 
     // Extend the column type guess with the "fields" field
     let typeGuessesAndFields = Object.assign({}, colTypeGuess);
-    typeGuessesAndFields.fields = fieldFormats;
+    typeGuessesAndFields.safeFields = safeFieldFormats;
+    typeGuessesAndFields.maybeFields = maybeFieldFormats;
     result.push(typeGuessesAndFields);
   });
 
@@ -532,10 +540,12 @@ exports.SessionGetInputSampleRows = (sid, range, startCount, middleCount, endCou
 // interpreted as extra text values.
 
 // The return value is an array, indexed on the relative column number within
-// the range; each element is an object with elements "types" listing a range of
-// possible types in a map whose values are either 'false' for types without
-// formats, or an Array of possible formats for that type; and a field "full"
-// indicating if the column has no missing values.
+// the range; each element is an object with elements "safeTypes" listing a
+// range of types that all cells in the column match, in a map whose values are
+// either 'false' for types without formats, or an Array of possible formats for
+// that type; "maybeTypes" in the same format, but listing type/format pairs
+// which *some* but not all cells match; and a field "full" indicating if the
+// column has no missing values.
 exports.SessionGuessTypes = (sid, range) => {
   assert(sessionStore.has(sid), `No such session ${sid} when guessing types`);
   assert(sessionStore.get(sid).wb.Sheets[range.sheet], `No such sheet ${range.sheet} in session ${sid} when guessing types`);
@@ -556,7 +566,8 @@ exports.SessionGuessTypes = (sid, range) => {
 
   // Initialise state, that we will refine as we go
   for (let col=0;col<columns;col++) {
-    guesses[col] = {types: undefined,
+    guesses[col] = {safeTpes: undefined,
+                    maybeTypes: undefined,
                     full:true}; // This is unset if we find a blank/missing
                                 // value
   }
@@ -577,32 +588,87 @@ exports.SessionGuessTypes = (sid, range) => {
         // Blank cell, so mark the column as not full but don't reduce the list of guessed types
         guesses[col].full = false;
       } else if(possibleCellTypes) {
-        if(guesses[col].types) {
-          // Remove any types in the current guess that aren't possibly valid for
+        // Check if this is the first time we've seen this column. safeTypes and
+        // maybeTypes are both set at the same time, so we can use safeTypes
+        // being set as a proxy for both.
+        if(guesses[col].safeTypes) {
+          // Remove any safeTypes in the current guess that aren't possibly valid for
           // this cell
-          let newGuesses = new Map();
-          guesses[col].types.forEach((formats, typeName) => {
+          let newSafeGuesses = new Map();
+          guesses[col].safeTypes.forEach((formats, typeName) => {
             if(possibleCellTypes.has(typeName)) {
               if(formats) {
                 // Type with formats, so find the intersection of the current possible formats and this cell's possible formats
                 let possibleCellFormats = possibleCellTypes.get(typeName);
                 let newPossibleFormats = formats.filter(value => possibleCellFormats.includes(value));
-                newGuesses.set(typeName, newPossibleFormats);
+                newSafeGuesses.set(typeName, newPossibleFormats);
               } else {
                 // Type without formats
-                newGuesses.set(typeName, formats);
+                newSafeGuesses.set(typeName, formats);
               }
             }
+            // Add any types in the current guess to maybeTypes
+            possibleCellTypes.forEach((formats, typeName) => {
+              const maybe = guesses[col].maybeTypes;
+              if(formats) {
+                // Type with formats, so make sure we've added all the formats to the list
+                if(!maybe.has(typeName)) {
+                  maybe.set(typeName, []);
+                }
+                formats.forEach((format) => {
+                  if(!maybe.get(typeName).includes(format)) {
+                    maybe.get(typeName).push(format);
+                  }
+                });
+              } else {
+                // Type without formats, just set it to false
+                maybe.set(typeName, false);
+              }
+              guesses[col].maybeTypes = maybe;
+            });
           });
-          guesses[col].types = newGuesses;
+          guesses[col].safeTypes = newSafeGuesses;
         } else {
           // First cell examined, so start off with its possible types
-          guesses[col].types = possibleCellTypes;
+          guesses[col].safeTypes = possibleCellTypes;
+          guesses[col].maybeTypes = possibleCellTypes;
         }
       }
     }
   }
 
+  // Now we must go back through the guesses and remove any safe types/formats
+  // from the maybe types/formats, so "maybe" ONLY contains options that are
+  // valid for some *but not all* values; every safe type/format will be listed
+  // as a maybe type/format due to the process used above.
+
+  guesses.forEach((colGuesses) => {
+    let maybeTypes = colGuesses.maybeTypes;
+    colGuesses.safeTypes.forEach((safeFormats, typeName) => {
+      if(safeFormats) {
+        // It's a type with formats
+
+        // typeName will always be present in maybeTypes at this point
+        let formats = maybeTypes.get(typeName);
+        safeFormats.forEach((format) => {
+          const index = formats.indexOf(format);
+          // index will never be -1
+          formats.splice(index, 1);
+        });
+        // If this is the last format for this type in maybeTypes, remove it. We
+        // won't get any other safeTypes for this maybeType so there's no chance
+        // of the above code being invoked again, and failing because typeName is
+        // no longe rin maybeTypes.
+        if(formats.length == 0) {
+          maybeTypes.delete(typeName);
+        }
+      } else {
+        // It's a type without formats
+        maybeTypes.delete(typeName);
+      }
+    });
+  });
+
   return guesses;
 };
 
diff --git a/lib/importer/src/dudk/backend.test.js b/lib/importer/src/dudk/backend.test.js
index 09aa0cb7..06f40a69 100644
--- a/lib/importer/src/dudk/backend.test.js
+++ b/lib/importer/src/dudk/backend.test.js
@@ -609,15 +609,33 @@ test('guessing types', () => {
   const guesses = backend.SessionGuessTypes(sid, dataRange);
   expect(guesses).toMatchObject([
     // Order is actually irrelevant for all formats lists, so this test over-specifies a bit
-    {full:true, types:new Map([["number", false],["text", false]])},
-    {full:true, types:new Map([["date", ["ymd"]],["text", false]])},
-    {full:true, types:new Map([["date", ["ydm"]],["text", false]])},
-    {full:true, types:new Map([["date", ["dmy"]],["text", false]])},
-    {full:true, types:new Map([["date", ["mdy"]],["text", false]])},
-    {full:true, types:new Map([["date", ["ymd","ydm"]],["text", false]])},
-    {full:false, types:new Map([["number", false],["text", false]])},
-    {full:true, types:new Map([["text", false]])},
-    {full:false, types:new Map([["text", false]])}
+    {full:true,
+     safeTypes:new Map([["number", false],["text", false]]),
+     maybeTypes:new Map([])},
+    {full:true,
+     safeTypes:new Map([["date", ["ymd"]],["text", false]]),
+     maybeTypes: new Map([["date", ["ydm","dmy","mdy"]]])},
+    {full:true,
+     safeTypes:new Map([["date", ["ydm"]],["text", false]]),
+     maybeTypes: new Map([["date", ["ymd","dmy","mdy"]]])},
+    {full:true,
+     safeTypes:new Map([["date", ["dmy"]],["text", false]]),
+     maybeTypes: new Map([["date", ["mdy", "ymd"]]])},
+    {full:true,
+     safeTypes:new Map([["date", ["mdy"]],["text", false]]),
+     maybeTypes: new Map([["date", ["dmy"]]])},
+    {full:true,
+     safeTypes:new Map([["date", ["ymd","ydm"]],["text", false]]),
+     maybeTypes: new Map([])},
+    {full:false,
+     safeTypes:new Map([["number", false],["text", false]]),
+     maybeTypes: new Map([])},
+    {full:true,
+     safeTypes:new Map([["text", false]]),
+     maybeTypes: new Map([])},
+    {full:false,
+     safeTypes:new Map([["text", false]]),
+     maybeTypes: new Map([])}
   ]);
 
   const fieldSuggestions = backend.SessionSuggestFields(sid, guesses, [
@@ -645,48 +663,65 @@ test('guessing types', () => {
 
   expect(fieldSuggestions).toMatchObject([
     // Order is actually irrelevant for all formats lists, so this test over-specifies a bit
-    {full:true, types:new Map([["number", false],["text", false]]),
-     fields: new Map([
-       ["numberField",false],
-       ["textField",false]
+    // ABS FIXME add maybeFields
+    {safeFields: new Map([
+      ["numberField",false],
+      ["textField",false]
+    ]),
+     maybeFields: new Map([])},
+
+    {safeFields: new Map([
+      ["dateField",["ymd"]],
+      ["textField",false]
+    ]),
+     maybeFields: new Map([
+       ["dateField",["ydm","dmy","mdy"]]
      ])},
-    {full:true, types:new Map([["date", ["ymd"]],["text", false]]),
-     fields: new Map([
-       ["dateField",["ymd"]],
-       ["textField",false]
-     ])},
-    {full:true, types:new Map([["date", ["ydm"]],["text", false]]),
-     fields: new Map([
-       ["dateField",["ydm"]],
-       ["textField",false]
-     ])},
-    {full:true, types:new Map([["date", ["dmy"]],["text", false]]),
-     fields: new Map([
-       ["dateField",["dmy"]],
-       ["textField",false]
-     ])},
-    {full:true, types:new Map([["date", ["mdy"]],["text", false]]),
-     fields: new Map([
-       ["dateField",["mdy"]],
-       ["textField",false]
-     ])},
-    {full:true, types:new Map([["date", ["ymd","ydm"]],["text", false]]),
-     fields: new Map([
-       ["dateField",["ymd","ydm"]],
-       ["textField",false]
+
+    {safeFields: new Map([
+      ["dateField",["ydm"]],
+      ["textField",false]
+    ]),
+     maybeFields: new Map([
+       ["dateField",["ymd","dmy","mdy"]]
      ])},
-    {full:false, types:new Map([["number", false],["text", false]]),
-     fields: new Map([
-       ["numberField",false],
-       ["textField",false]
+
+    {safeFields: new Map([
+      ["dateField",["dmy"]],
+      ["textField",false]
+    ]),
+     maybeFields: new Map([
+       ["dateField",["mdy","ymd"]]
      ])},
-    {full:true, types:new Map([["text", false]]),
-     fields: new Map([
-       ["textField",false]
+
+    {safeFields: new Map([
+      ["dateField",["mdy"]],
+      ["textField",false]
+    ]),
+     maybeFields: new Map([
+       ["dateField",["dmy"]]
      ])},
-    {full:false, types:new Map([["text", false]]),
-     fields: new Map([
-       ["textField",false]
-     ])}
+
+    {safeFields: new Map([
+      ["dateField",["ymd","ydm"]],
+      ["textField",false]
+    ]),
+     maybeFields: new Map([])},
+
+    {safeFields: new Map([
+      ["numberField",false],
+      ["textField",false]
+    ]),
+     maybeFields: new Map([])},
+
+    {safeFields: new Map([
+      ["textField",false]
+    ]),
+     maybeFields: new Map([])},
+
+    {safeFields: new Map([
+      ["textField",false]
+    ]),
+     maybeFields: new Map([])}
   ]);
 });

From 43ec5592436f7a4c846edd0a27e20def9b6db934 Mon Sep 17 00:00:00 2001
From: Alaric Snell-Pym <alaric@register-dynamics.co.uk>
Date: Tue, 28 Oct 2025 16:21:51 +0000
Subject: [PATCH 11/11] Update UI to reflect new distinction between
 safe/maybe/unsafe formats and types

---
 .../importer/macros/format_picker.njk         | 20 ++++--
 lib/importer/src/functions.js                 | 69 +++++++++++--------
 lib/importer/src/index.js                     |  2 +-
 3 files changed, 56 insertions(+), 35 deletions(-)

diff --git a/lib/importer/nunjucks/importer/macros/format_picker.njk b/lib/importer/nunjucks/importer/macros/format_picker.njk
index 5c9e617c..c9170f10 100644
--- a/lib/importer/nunjucks/importer/macros/format_picker.njk
+++ b/lib/importer/nunjucks/importer/macros/format_picker.njk
@@ -79,16 +79,22 @@
       <td class="govuk-table__cell govuk-table__cell--numeric">
          {% if possibleFormats %}
          <select class="govuk-select" style="float: right;" name="format-{{h.index}}">
-                {% for format in possibleFormats %}
+            <optgroup label="Matches every row">
+                {% for format in possibleFormats[0] %}
                     <option value="{{format.name}}"
-                            {% if format.name == currentValue %}selected{% endif %}>
-                      {% if format.likely %}
-                         {{format.displayName}} (*)
-                      {% else %}
-                         {{format.displayName}}
-                      {% endif %}
+                      {% if format.name == currentValue %}selected{% endif %}>
+                      {{format.displayName}}
                     </option>
                 {% endfor %}
+            </optgroup>
+            <optgroup label="Matches some rows">
+                {% for format in possibleFormats[1] %}
+                    <option value="{{format.name}}"
+                      {% if format.name == currentValue %}selected{% endif %}>
+                      {{format.displayName}}
+                    </option>
+                {% endfor %}
+            </optgroup>
          </select>
          {% endif %}
       </td>
diff --git a/lib/importer/src/functions.js b/lib/importer/src/functions.js
index a2d13450..8cae1103 100644
--- a/lib/importer/src/functions.js
+++ b/lib/importer/src/functions.js
@@ -82,10 +82,13 @@ const importerMappedColumnDetails = (data) => {
 //--------------------------------------------------------------------
 // As long as column->field mappings have been set up, this function will return
 // an array indexed on column, with each element being list of possible formats
-// for that column. The array element for a column will be false if the column is mapped to a
-// field whose type does not require formats, otherwise it will be a list of
-// objects with 'name' (internal code), 'displayName' (human-facing name) and
-// 'description' (longer description) fields.
+// for that column. The array element for a column will be false if the column
+// is mapped to a field whose type does not require formats, otherwise it will
+// be TWO lists of objects with 'name' (internal code), 'displayName'
+// (human-facing name) and 'description' (longer description) fields. The first
+// is "safe" formats that will work for all the values in the column, the second
+// are "maybe" formats that will work for some of them. Those that work for none
+// of them are not present at all.
 // --------------------------------------------------------------------
 const importerPossibleColumnFormats = (data) => {
     // FIXME: This should probably move into the backend, as a SessionSuggestFormats function
@@ -105,11 +108,20 @@ const importerPossibleColumnFormats = (data) => {
         const columnDef = session.fields.find((field) => field.name == columnField);
         const columnTypeName = columnDef.type;
         const columnType = supportedTypes.get(columnTypeName);
-        let likelyFormats = new Set(); // Names of formats this column is likely to have, according to the guesser
-        if (columnGuesses.types && columnGuesses.types.has(columnTypeName)) {
-            const guessedFormats = columnGuesses.types.get(columnTypeName);
+
+        let safeFormats = new Set(); // Names of formats this column is likely to have, according to the guesser
+        if (columnGuesses.safeTypes && columnGuesses.safeTypes.has(columnTypeName)) {
+            const guessedFormats = columnGuesses.safeTypes.get(columnTypeName);
+            if(guessedFormats && columnType.formats) {
+                safeFormats = new Set(Array.from(columnType.formats.keys()).filter((fmtName) => guessedFormats.includes(fmtName)));
+            }
+        }
+
+        let maybeFormats = new Set(); // Names of formats this column merely might have, according to the guesser
+        if (columnGuesses.maybeTypes && columnGuesses.maybeTypes.has(columnTypeName)) {
+            const guessedFormats = columnGuesses.maybeTypes.get(columnTypeName);
             if(guessedFormats && columnType.formats) {
-                likelyFormats = new Set(Array.from(columnType.formats.keys()).filter((fmtName) => guessedFormats.includes(fmtName)));
+                maybeFormats = new Set(Array.from(columnType.formats.keys()).filter((fmtName) => guessedFormats.includes(fmtName)));
             }
         }
 
@@ -119,10 +131,13 @@ const importerPossibleColumnFormats = (data) => {
                     name: fmtEntry[0],
                     displayName: fmtEntry[1].displayName,
                     description: fmtEntry[1].description,
-                    likely: likelyFormats.has(fmtEntry[0])
+                    safe: safeFormats.has(fmtEntry[0]),
+                    maybe: maybeFormats.has(fmtEntry[0])
                 };
             });
-            return options;
+            const formats = [options.filter((fmtEntry) => fmtEntry.safe),
+                             options.filter((fmtEntry) => fmtEntry.maybe)];
+            return formats;
         } else {
             return false;
         }
@@ -158,19 +173,19 @@ const importerGetTrailingRows = (data, count) => {
 //     "All {n} rows of {sheet}"
 //--------------------------------------------------------------------
 const importerGetTableCaption =
-    (data, prefix, rowsAskedFor, sheetName = null) => {
-        const session_data = data[IMPORTER_SESSION_KEY];
-        const session = new session_lib.Session(session_data)
+      (data, prefix, rowsAskedFor, sheetName = null) => {
+          const session_data = data[IMPORTER_SESSION_KEY];
+          const session = new session_lib.Session(session_data)
 
-        const sheet = (sheetName ??= session.sheet);
-        const rowCount = sheets_lib.GetTotalRows(session.backendSid, sheet);
+          const sheet = (sheetName ??= session.sheet);
+          const rowCount = sheets_lib.GetTotalRows(session.backendSid, sheet);
 
-        if (rowCount > rowsAskedFor) {
-            return `${prefix} ${rowsAskedFor} rows of '${sheet}'`;
-        }
+          if (rowCount > rowsAskedFor) {
+              return `${prefix} ${rowsAskedFor} rows of '${sheet}'`;
+          }
 
-        return `All rows of '${sheet}'`;
-    }
+          return `All rows of '${sheet}'`;
+      }
 
 //--------------------------------------------------------------------
 // In the absence of a step to choose headers, we will instead provide
@@ -208,8 +223,8 @@ const importerGetHeaders = (data) => {
                 session.backendSid,
                 session.sheet,
                 index,
-          /* cellWidth */ 20,
-          /* count */ 5,
+                /* cellWidth */ 20,
+                /* count */ 5,
             ).inputValues;
 
             let exampleString = examples.join(", ").trim();
@@ -306,8 +321,8 @@ const data_avg = (data, column) => {
     }
 
     const numbers = mapResults.resultRecords
-        .filter((x) => x !== undefined && x[idx] !== undefined)
-        .map((x) => parseNumberFromString(x[idx]))
+          .filter((x) => x !== undefined && x[idx] !== undefined)
+          .map((x) => parseNumberFromString(x[idx]))
 
 
     const avg = numbers.reduce((acc, i) => acc + i, 0) / numbers.length
@@ -354,7 +369,7 @@ module.exports = {
 }
 
 /*
-Local Variables:
-js-indent-level: 4
-End:
+  Local Variables:
+  js-indent-level: 4
+  End:
 */
diff --git a/lib/importer/src/index.js b/lib/importer/src/index.js
index bf774921..88328bb7 100644
--- a/lib/importer/src/index.js
+++ b/lib/importer/src/index.js
@@ -442,7 +442,7 @@ exports.Initialise = (config, router, prototypeKit) => {
         if ("other" in request.query) {
           const columnsNeedingFormats = backend.SessionGetColumnsNeedingFormats(session.backendSid);
           if(columnsNeedingFormats.length == 0) {
-            const otherPage = decodeURIComponent(request.query.other)
+            const otherPage = decodeURIComponent(request.query.other);
             if (otherPage) { request.query.next = request.query.other }
           }
         }