From 35985fc55e875e150c4ea689d5ecc5c9ee7dffb0 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Wed, 25 Jun 2025 09:10:17 +0100 Subject: [PATCH 1/3] Exploring how to debug views --- lib/importer/src/debug.js | 23 +++++++++++++++++ lib/importer/src/dudk/sheets.js | 1 - lib/importer/src/functions.js | 1 - lib/importer/src/index.js | 40 +++++++++++++++++++++++++++++ lib/importer/src/session.js | 1 - lib/importer/views/debug/index.html | 17 ++++++++++++ lib/importer/views/debug/valid.html | 18 +++++++++++++ prototypes/basic/app/routes.js | 1 - 8 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 lib/importer/src/debug.js create mode 100644 lib/importer/views/debug/index.html create mode 100644 lib/importer/views/debug/valid.html diff --git a/lib/importer/src/debug.js b/lib/importer/src/debug.js new file mode 100644 index 00000000..204cc446 --- /dev/null +++ b/lib/importer/src/debug.js @@ -0,0 +1,23 @@ + +const fs = require("node:fs") +const path = require("node:path") + +const session_lib = require("./session.js"); + +exports.CreateSessionWithFileFixture = (plugin_config, request, filename) => { + const sourceFile = path.join(path.resolve(process.cwd()), "../../fixtures/", filename); + const targetFile = path.join(plugin_config.uploadPath, filename); + + fs.copyFileSync(sourceFile, targetFile); + + request.file = { + fieldname: "file", + originalName: filename, + mimetype: "text/csv", + filename: filename, + path: targetFile, + destination: plugin_config.uploadPath, + }; + + return session_lib.CreateSession(plugin_config, request); +} diff --git a/lib/importer/src/dudk/sheets.js b/lib/importer/src/dudk/sheets.js index 5514d5ba..3d5f2f10 100644 --- a/lib/importer/src/dudk/sheets.js +++ b/lib/importer/src/dudk/sheets.js @@ -269,7 +269,6 @@ exports.MapData = (sid, sheet, mapping, fields, previewLimit = DEFAULT_PREVIEW_L // Convert source mapping (a map from column index -> attribute name) into a mapping for the backend let rewrittenMapping = RewriteMapping(mapping, fields) - // Apply the mapping const backendJid = backend.SessionPerformMappingJob(sid, rowRange, rewrittenMapping); diff --git a/lib/importer/src/functions.js b/lib/importer/src/functions.js index be4ec2eb..ddf9a47a 100644 --- a/lib/importer/src/functions.js +++ b/lib/importer/src/functions.js @@ -158,7 +158,6 @@ const importerGetHeaders = (data) => { const importerMappedData = (data) => { const session_data = data[IMPORTER_SESSION_KEY]; const session = new session_lib.Session(session_data) - const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.fields); const headers = session.fields; return { diff --git a/lib/importer/src/index.js b/lib/importer/src/index.js index ce5db786..f52f0896 100644 --- a/lib/importer/src/index.js +++ b/lib/importer/src/index.js @@ -479,6 +479,46 @@ exports.Initialise = (config, router, prototypeKit) => { } } + //-------------------------------------------------------------------- + // Debug routes for testing presentation + //-------------------------------------------------------------------- + const debug_lib = require("./debug.js") + const funcs_lib = require("./functions.js") + + // debug routes ... + router.get('/importer/debug', function (request, response) { + response.render("debug/index.html", {}) + }) + + + // Valid, with headers, only warnings + router.get('/importer/debug/valid', function (request, response) { + const sess = debug_lib.CreateSessionWithFileFixture(plugin_config, request, "validation-test.csv") + sess.sheet = "Sheet1" + sess.headerRange = { + sheet: "Sheet1", + start: { row: 0, column: 0 }, end: { row: 0, column: 7 } + } + + sess.mapping = { + 0: "Employee number", + 1: "Title", + 2: "First name", + 3: "Surname", + 4: "Employment start date", + 5: "Salary", + 6: "Contribution percentage", + } + + const data = {} + data[IMPORTER_SESSION_KEY] = sess + const res = funcs_lib.importerMappedData(data) + + response.render("debug/valid.html", res) + }) + + + //-------------------------------------------------------------------- // Review the processing for the current session before continuing. diff --git a/lib/importer/src/session.js b/lib/importer/src/session.js index 426990ab..ea6ed34e 100644 --- a/lib/importer/src/session.js +++ b/lib/importer/src/session.js @@ -82,7 +82,6 @@ exports.CreateSession = (config, request) => { createResponse.error = err; return createResponse; } - createResponse.id = getFilenameHash(file.filename); createResponse.session = new Session({ id: createResponse.id, diff --git a/lib/importer/views/debug/index.html b/lib/importer/views/debug/index.html new file mode 100644 index 00000000..a9d758d5 --- /dev/null +++ b/lib/importer/views/debug/index.html @@ -0,0 +1,17 @@ +{% extends "../layouts/main.html" %} + +{% set pageName="Debug" %} + +{% block content %} + +
+
+

Debug views

+ +

+ /fixtures/validation-test.csv with headers specified and missing row warnings +

+
+
+ +{% endblock %} diff --git a/lib/importer/views/debug/valid.html b/lib/importer/views/debug/valid.html new file mode 100644 index 00000000..5e4a17dd --- /dev/null +++ b/lib/importer/views/debug/valid.html @@ -0,0 +1,18 @@ +{% extends "../layouts/main.html" %} + +{% set pageName="Debug - Valid" %} + +{% block beforeContent %} + Back +{% endblock %} + +{% block content %} + +
+
+

Valid, with headers, only warnings

+ +
+
+ +{% endblock %} diff --git a/prototypes/basic/app/routes.js b/prototypes/basic/app/routes.js index 8cf1a45b..d733561c 100644 --- a/prototypes/basic/app/routes.js +++ b/prototypes/basic/app/routes.js @@ -5,7 +5,6 @@ const govukPrototypeKit = require("govuk-prototype-kit"); const router = govukPrototypeKit.requests.setupRouter(); - // Below 3 lines added by the Data Upload Design Kit plugin. // If you uninstall the plugin, remove the 3 lines below. const importer = require("@register-dynamics/importer"); From 7816516a2b2692a5a8359bddc2f2311daa8b4bd6 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Thu, 26 Jun 2025 10:17:17 +0100 Subject: [PATCH 2/3] Fix issue with use of incorrect type when creating session --- lib/importer/src/debug.js | 19 +++++++++++++++---- lib/importer/src/functions.js | 1 + lib/importer/src/index.js | 11 +++++++---- lib/importer/views/debug/valid.html | 4 ++++ 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/lib/importer/src/debug.js b/lib/importer/src/debug.js index 204cc446..f1612e02 100644 --- a/lib/importer/src/debug.js +++ b/lib/importer/src/debug.js @@ -9,15 +9,26 @@ exports.CreateSessionWithFileFixture = (plugin_config, request, filename) => { const targetFile = path.join(plugin_config.uploadPath, filename); fs.copyFileSync(sourceFile, targetFile); + const fstats = fs.statSync(targetFile) + + let encoding = "utf8"; + let mimetype = "text/csv" + + if (!filename.toLowerCase().endsWith(".csv")) { + encoding = "7bit"; + mimetype = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + } request.file = { fieldname: "file", - originalName: filename, - mimetype: "text/csv", - filename: filename, + originalname: filename, + mimetype: mimetype, + encoding: encoding, path: targetFile, + size: fstats.size, destination: plugin_config.uploadPath, + filename: filename }; - return session_lib.CreateSession(plugin_config, request); + return session_lib.CreateSession(plugin_config, request) } diff --git a/lib/importer/src/functions.js b/lib/importer/src/functions.js index ddf9a47a..fd1f58ea 100644 --- a/lib/importer/src/functions.js +++ b/lib/importer/src/functions.js @@ -160,6 +160,7 @@ const importerMappedData = (data) => { const session = new session_lib.Session(session_data) const mapResults = sheets_lib.MapData(session.backendSid, session.sheet, session.mapping, session.fields); const headers = session.fields; + return { rows: mapResults.resultRecords, headers: headers, diff --git a/lib/importer/src/index.js b/lib/importer/src/index.js index f52f0896..e2e4f8f3 100644 --- a/lib/importer/src/index.js +++ b/lib/importer/src/index.js @@ -493,7 +493,9 @@ exports.Initialise = (config, router, prototypeKit) => { // Valid, with headers, only warnings router.get('/importer/debug/valid', function (request, response) { - const sess = debug_lib.CreateSessionWithFileFixture(plugin_config, request, "validation-test.csv") + + const sess = debug_lib.CreateSessionWithFileFixture(plugin_config, request, "validation-test.csv").session + sess.sheet = "Sheet1" sess.headerRange = { sheet: "Sheet1", @@ -510,9 +512,10 @@ exports.Initialise = (config, router, prototypeKit) => { 6: "Contribution percentage", } - const data = {} - data[IMPORTER_SESSION_KEY] = sess - const res = funcs_lib.importerMappedData(data) + const local_data = {} + local_data[IMPORTER_SESSION_KEY] = sess + + const res = funcs_lib.importerMappedData(local_data) response.render("debug/valid.html", res) }) diff --git a/lib/importer/views/debug/valid.html b/lib/importer/views/debug/valid.html index 5e4a17dd..6a8c4028 100644 --- a/lib/importer/views/debug/valid.html +++ b/lib/importer/views/debug/valid.html @@ -12,6 +12,10 @@

Valid, with headers, only warnings

+

+{{res}} +

+
From 4ea026a8bbb6b9b0a2937b695c11d3e2fed8a104 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Tue, 1 Jul 2025 08:53:35 +0100 Subject: [PATCH 3/3] stash --- lib/importer/nunjucks/importer/macros/table_view.njk | 2 +- lib/importer/src/functions.js | 1 + lib/importer/src/index.js | 7 +++++-- lib/importer/views/debug/valid.html | 4 +++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/importer/nunjucks/importer/macros/table_view.njk b/lib/importer/nunjucks/importer/macros/table_view.njk index 7c076096..34aa4df7 100644 --- a/lib/importer/nunjucks/importer/macros/table_view.njk +++ b/lib/importer/nunjucks/importer/macros/table_view.njk @@ -4,7 +4,7 @@ {% set rows = data.rows %} {% set moreRowsAvailable = data.extraRecordCount > 0 %} {% set moreRowsCount = data.extraRecordCount %} - +

.{{data}}.

{% set tableHeaders = importerHeaderRowDisplay(session, headerMode) %} diff --git a/lib/importer/src/functions.js b/lib/importer/src/functions.js index fd1f58ea..7a6ddd83 100644 --- a/lib/importer/src/functions.js +++ b/lib/importer/src/functions.js @@ -174,6 +174,7 @@ const importerMappedData = (data) => { } const importerHeaderRowDisplay = (data, mode) => { + console.log(data) const session_data = data[IMPORTER_SESSION_KEY]; const session = new session_lib.Session(session_data) diff --git a/lib/importer/src/index.js b/lib/importer/src/index.js index e2e4f8f3..b7152948 100644 --- a/lib/importer/src/index.js +++ b/lib/importer/src/index.js @@ -499,7 +499,8 @@ exports.Initialise = (config, router, prototypeKit) => { sess.sheet = "Sheet1" sess.headerRange = { sheet: "Sheet1", - start: { row: 0, column: 0 }, end: { row: 0, column: 7 } + start: { row: 0, column: 0 }, + end: { row: 0, column: 7 } } sess.mapping = { @@ -517,7 +518,9 @@ exports.Initialise = (config, router, prototypeKit) => { const res = funcs_lib.importerMappedData(local_data) - response.render("debug/valid.html", res) + // request.session.data[IMPORTER_SESSION_KEY] = sess + + response.render("debug/valid.html", { debug_data: { result: res, session: sess } }) }) diff --git a/lib/importer/views/debug/valid.html b/lib/importer/views/debug/valid.html index 6a8c4028..50a09852 100644 --- a/lib/importer/views/debug/valid.html +++ b/lib/importer/views/debug/valid.html @@ -1,5 +1,7 @@ {% extends "../layouts/main.html" %} +{% from "importer/macros/table_view.njk" import importerTableView %} + {% set pageName="Debug - Valid" %} {% block beforeContent %} @@ -13,7 +15,7 @@

Valid, with headers, only warnings

-{{res}} + {{ importerTableView(debug_data.session, debug_data.result, caption="test", headerMode="none") }}