From b043e74d4edbec085cefe39ac847b5a27fcf5b7f Mon Sep 17 00:00:00 2001
From: maby0 <mabonapgwyn@hotmail.co.uk>
Date: Mon, 16 Dec 2024 12:14:20 +0000
Subject: [PATCH 1/5] fix(api): stream api response to function to avoid
 timeout

---
 .../server/migration/migration.controller.js  | 39 ++++++++++++++++---
 .../common/back-office-api-client.js          | 38 ++++++++++++++++++
 .../migrators/nsip-document-migration.js      |  5 ++-
 3 files changed, 74 insertions(+), 8 deletions(-)

diff --git a/apps/api/src/server/migration/migration.controller.js b/apps/api/src/server/migration/migration.controller.js
index 0abf8d5c42..c20a91b6cc 100644
--- a/apps/api/src/server/migration/migration.controller.js
+++ b/apps/api/src/server/migration/migration.controller.js
@@ -16,16 +16,43 @@ export const postMigrateModel = async ({ body, params: { modelType } }, response
 
 	for (const model of body) {
 		if (!validator(model)) {
-			throw Error(JSON.stringify({
-				message: `Model ${modelType} failed validation`,
-				validationErrors: validator.errors
-			}));
+			throw Error(
+				JSON.stringify({
+					message: `Model ${modelType} failed validation`,
+					validationErrors: validator.errors
+				})
+			);
 		}
 	}
+	// await migrator(body);
+	// response.sendStatus(200);
 
-	await migrator(body);
+	// this method doesn't handle errors!! NEED TO HANDLE
+	response.writeHead(200, { 'Content-Type': 'text/plain', 'transfer-encoding': 'chunked' });
 
-	response.sendStatus(200);
+	let progressMessageCount = 0;
+	const progressInterval = setInterval(() => {
+		progressMessageCount++;
+		response.write(`Still processing... (${progressMessageCount * 10} seconds elapsed)\n)`);
+	}, 10000);
+
+	try {
+		response.write(`Starting migration for model type: ${modelType}...\n`);
+		await migrator(body);
+		// await new Promise((resolve) => {
+		// 	setTimeout(() => {
+		// 		resolve(console.log('doing stuff'));
+		// 	}, 40000);
+		// });
+
+		response.write(`Migration completed successfully.\n`);
+	} catch (error) {
+		// response.write(`Error during migration: ${error.message}\n`);
+		throw Error(`Error during migration: ${error.message}`);
+	} finally {
+		clearInterval(progressInterval);
+		response.end();
+	}
 };
 
 /**
diff --git a/apps/functions/applications-migration/common/back-office-api-client.js b/apps/functions/applications-migration/common/back-office-api-client.js
index 8e1b4b4fdd..e55c724fed 100644
--- a/apps/functions/applications-migration/common/back-office-api-client.js
+++ b/apps/functions/applications-migration/common/back-office-api-client.js
@@ -38,6 +38,44 @@ export const makePostRequest = (logger, path, body) => {
 	});
 };
 
+/**
+ * Makes a POST request and processes streaming responses with `got`.
+ */
+export const makePostRequestStreamResponse = async (logger, path, body) => {
+	const requestUri = constructUri(path);
+	const request = constructAuthenticatedRequest();
+
+	logger.info(`Making POST request to ${requestUri}`);
+
+	try {
+		// Correct usage for streaming response
+		const stream = request.stream.post(requestUri, {
+			json: body,
+			headers: { 'Content-Type': 'application/json' }
+		});
+
+		stream.on('data', (chunk) => {
+			// Process each chunk of data as it arrives
+			logger.info(`Chunk received: ${chunk.toString()}`);
+		});
+
+		await new Promise((resolve, reject) => {
+			stream.on('end', () => {
+				logger.info('Response fully received');
+				resolve();
+			});
+
+			stream.on('error', (error) => {
+				logger.error(`Error occurred: ${error.message}`);
+				reject(error);
+			});
+		});
+	} catch (error) {
+		logger.error(`Request failed: ${error.message}`);
+		throw error;
+	}
+};
+
 const constructAuthenticatedRequest = () => {
 	const serviceName = 'function';
 	const authenticatedRequest = got.extend({
diff --git a/apps/functions/applications-migration/common/migrators/nsip-document-migration.js b/apps/functions/applications-migration/common/migrators/nsip-document-migration.js
index cb574f2fd6..87e99327f3 100644
--- a/apps/functions/applications-migration/common/migrators/nsip-document-migration.js
+++ b/apps/functions/applications-migration/common/migrators/nsip-document-migration.js
@@ -1,6 +1,6 @@
 import { SynapseDB } from '../synapse-db.js';
 import { QueryTypes } from 'sequelize';
-import { makePostRequest } from '../back-office-api-client.js';
+import { makePostRequestStreamResponse } from '../back-office-api-client.js';
 
 /**
  * @param {import('@azure/functions').Logger} log
@@ -13,7 +13,8 @@ export const migrationNsipDocumentsByReference = async (log, caseReference) => {
 
 		if (documents.length > 0) {
 			log.info(`Migrating ${documents.length} NSIP Documents for case ${caseReference}`);
-			await makePostRequest(log, '/migration/nsip-document', documents);
+			// const result = await makePostRequest(log, '/migration/nsip-document', documents);
+			await makePostRequestStreamResponse(log, '/migration/nsip-document', documents);
 			log.info('Successfully migrated NSIP Document');
 		} else {
 			log.warn(`No NSIP Document found for case ${caseReference}`);

From 6935e65de59d60b1fcb88b8a497493be228d1da5 Mon Sep 17 00:00:00 2001
From: maby0 <mabonapgwyn@hotmail.co.uk>
Date: Mon, 16 Dec 2024 13:07:02 +0000
Subject: [PATCH 2/5] fix(api): ensure response chunks are flushed from buffer

---
 apps/api/src/server/migration/migration.controller.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/apps/api/src/server/migration/migration.controller.js b/apps/api/src/server/migration/migration.controller.js
index c20a91b6cc..0cd9a7243a 100644
--- a/apps/api/src/server/migration/migration.controller.js
+++ b/apps/api/src/server/migration/migration.controller.js
@@ -34,9 +34,11 @@ export const postMigrateModel = async ({ body, params: { modelType } }, response
 	const progressInterval = setInterval(() => {
 		progressMessageCount++;
 		response.write(`Still processing... (${progressMessageCount * 10} seconds elapsed)\n)`);
+		response.flush();
 	}, 10000);
 
 	try {
+		response.flushHeaders();
 		response.write(`Starting migration for model type: ${modelType}...\n`);
 		await migrator(body);
 		// await new Promise((resolve) => {
@@ -46,6 +48,7 @@ export const postMigrateModel = async ({ body, params: { modelType } }, response
 		// });
 
 		response.write(`Migration completed successfully.\n`);
+		response.flush();
 	} catch (error) {
 		// response.write(`Error during migration: ${error.message}\n`);
 		throw Error(`Error during migration: ${error.message}`);

From d021cc2032ffd3b36c86d87418f63c682e289600 Mon Sep 17 00:00:00 2001
From: maby0 <mabonapgwyn@hotmail.co.uk>
Date: Tue, 17 Dec 2024 13:39:35 +0000
Subject: [PATCH 3/5] fix(functions): add log to view request and response
 props

---
 .../common/handle-migration-with-response.js  |  3 +++
 .../nsip-document-migration/index.js          | 20 ++++++++++++++-----
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/apps/functions/applications-migration/common/handle-migration-with-response.js b/apps/functions/applications-migration/common/handle-migration-with-response.js
index d5f3906d53..4468a2d288 100644
--- a/apps/functions/applications-migration/common/handle-migration-with-response.js
+++ b/apps/functions/applications-migration/common/handle-migration-with-response.js
@@ -14,6 +14,7 @@ const headers = { 'Content-Type': 'application/json; charset=utf-8' };
  */
 export const handleMigrationWithResponse = async (
 	context,
+	req,
 	{
 		caseReferences,
 		migrationFunction,
@@ -22,6 +23,8 @@ export const handleMigrationWithResponse = async (
 		migrationOverwrite = false
 	}
 ) => {
+	context.log('req : ', req);
+	context.log({ caseReferences });
 	const validationError = validateRequest(caseReferences, allowCaseReferencesArray);
 	if (!migrationOverwrite) {
 		const areCasesMigrated = await getCaseMigrationStatuses(context.log, caseReferences);
diff --git a/apps/functions/applications-migration/nsip-document-migration/index.js b/apps/functions/applications-migration/nsip-document-migration/index.js
index 45ff934b57..4aa3435083 100644
--- a/apps/functions/applications-migration/nsip-document-migration/index.js
+++ b/apps/functions/applications-migration/nsip-document-migration/index.js
@@ -5,11 +5,21 @@ import { handleMigrationWithResponse } from '../common/handle-migration-with-res
  * @param {import("@azure/functions").Context} context
  * @param {import("@azure/functions").HttpRequest} req
  */
-export default async (context, { body: { caseReference, migrationOverwrite = false } }) => {
-	await handleMigrationWithResponse(context, {
-		caseReferences: caseReference,
+export default async (
+	context,
+	req
+	// { body: { caseReference, migrationOverwrite = false } }
+) => {
+	context.log('CONTEXT1: ' + JSON.stringify(context));
+	context.log('CONTEXT2: ' + JSON.stringify(context.res));
+	context.log('REQ1: ' + JSON.stringify(req));
+	context.log('REQ2: ' + JSON.stringify(req.raw));
+	const caseRef = req.body.caseReference;
+	context.log({ caseRef });
+	await handleMigrationWithResponse(context, req, {
+		caseReferences: caseRef,
 		entityName: 'document',
-		migrationFunction: () => migrationNsipDocumentsByReference(context.log, caseReference),
-		migrationOverwrite
+		migrationFunction: () => migrationNsipDocumentsByReference(context.log, req.body.caseReference),
+		migrationOverwrite: req.body.migrationOverwrite
 	});
 };

From 8de97f3908fc504c31a6d0749a1dbb3acb4848f1 Mon Sep 17 00:00:00 2001
From: maby0 <mabonapgwyn@hotmail.co.uk>
Date: Wed, 18 Dec 2024 10:08:45 +0000
Subject: [PATCH 4/5] progress commit

---
 .../server/migration/migration.controller.js  |  6 --
 .../common/back-office-api-client.js          | 55 +++++++-------
 .../common/handle-migration-with-response.js  | 72 +++++++++---------
 .../migrators/nsip-document-migration.js      | 25 +++----
 .../nsip-document-migration/debug.js          | 12 ---
 .../nsip-document-migration/function.json     | 16 ----
 .../nsip-document-migration/index.js          | 74 +++++++++++++------
 .../applications-migration/package.json       |  2 +
 package-lock.json                             | 50 +++++++++++--
 package.json                                  |  2 +-
 10 files changed, 177 insertions(+), 137 deletions(-)
 delete mode 100644 apps/functions/applications-migration/nsip-document-migration/debug.js
 delete mode 100644 apps/functions/applications-migration/nsip-document-migration/function.json

diff --git a/apps/api/src/server/migration/migration.controller.js b/apps/api/src/server/migration/migration.controller.js
index 0cd9a7243a..f6f47c4770 100644
--- a/apps/api/src/server/migration/migration.controller.js
+++ b/apps/api/src/server/migration/migration.controller.js
@@ -41,12 +41,6 @@ export const postMigrateModel = async ({ body, params: { modelType } }, response
 		response.flushHeaders();
 		response.write(`Starting migration for model type: ${modelType}...\n`);
 		await migrator(body);
-		// await new Promise((resolve) => {
-		// 	setTimeout(() => {
-		// 		resolve(console.log('doing stuff'));
-		// 	}, 40000);
-		// });
-
 		response.write(`Migration completed successfully.\n`);
 		response.flush();
 	} catch (error) {
diff --git a/apps/functions/applications-migration/common/back-office-api-client.js b/apps/functions/applications-migration/common/back-office-api-client.js
index e55c724fed..5404a8e50b 100644
--- a/apps/functions/applications-migration/common/back-office-api-client.js
+++ b/apps/functions/applications-migration/common/back-office-api-client.js
@@ -45,35 +45,40 @@ export const makePostRequestStreamResponse = async (logger, path, body) => {
 	const requestUri = constructUri(path);
 	const request = constructAuthenticatedRequest();
 
-	logger.info(`Making POST request to ${requestUri}`);
+	// logger.info(`Making POST request to ${requestUri}`);
+	console.log(`Making POST request to ${requestUri}`);
+	return request.stream.post(requestUri, {
+		json: body,
+		headers: { 'Content-Type': 'application/json' }
+	});
 
-	try {
-		// Correct usage for streaming response
-		const stream = request.stream.post(requestUri, {
-			json: body,
-			headers: { 'Content-Type': 'application/json' }
-		});
+	// try {
+	// 	// Correct usage for streaming response
+	// 	const stream = request.stream.post(requestUri, {
+	// 		json: body,
+	// 		headers: { 'Content-Type': 'application/json' }
+	// 	});
 
-		stream.on('data', (chunk) => {
-			// Process each chunk of data as it arrives
-			logger.info(`Chunk received: ${chunk.toString()}`);
-		});
+	// 	stream.on('data', (chunk) => {
+	// 		// Process each chunk of data as it arrives
+	// 		logger.info(chunk.toString());
+	// 	});
 
-		await new Promise((resolve, reject) => {
-			stream.on('end', () => {
-				logger.info('Response fully received');
-				resolve();
-			});
+	// 	await new Promise((resolve, reject) => {
+	// 		stream.on('end', () => {
+	// 			logger.info('Response fully received');
+	// 			resolve();
+	// 		});
 
-			stream.on('error', (error) => {
-				logger.error(`Error occurred: ${error.message}`);
-				reject(error);
-			});
-		});
-	} catch (error) {
-		logger.error(`Request failed: ${error.message}`);
-		throw error;
-	}
+	// 		stream.on('error', (error) => {
+	// 			logger.error(`Error occurred: ${error.message}`);
+	// 			reject(error);
+	// 		});
+	// 	});
+	// } catch (error) {
+	// 	logger.error(`Request failed: ${error.message}`);
+	// 	throw error;
+	// }
 };
 
 const constructAuthenticatedRequest = () => {
diff --git a/apps/functions/applications-migration/common/handle-migration-with-response.js b/apps/functions/applications-migration/common/handle-migration-with-response.js
index 4468a2d288..db8402b3f6 100644
--- a/apps/functions/applications-migration/common/handle-migration-with-response.js
+++ b/apps/functions/applications-migration/common/handle-migration-with-response.js
@@ -1,7 +1,7 @@
+import { Writable } from 'stream';
 import { makeGetRequest } from './back-office-api-client.js';
 import { validateMigration } from './validate-migration.js';
 
-const headers = { 'Content-Type': 'application/json; charset=utf-8' };
 /**
  * Wrapper function for migration functions that handles error handling and sends useful responses.
  * @param {import('@azure/functions').Context} context - The Azure function context object.
@@ -23,46 +23,45 @@ export const handleMigrationWithResponse = async (
 		migrationOverwrite = false
 	}
 ) => {
-	context.log('req : ', req);
-	context.log({ caseReferences });
-	const validationError = validateRequest(caseReferences, allowCaseReferencesArray);
-	if (!migrationOverwrite) {
-		const areCasesMigrated = await getCaseMigrationStatuses(context.log, caseReferences);
-		if (areCasesMigrated.areMigrated) {
-			context.res = {
-				status: 200,
-				body: {
-					migration: areCasesMigrated.error
-				},
-				headers
-			};
-			return;
-		}
-	}
+	// context.log('req : ', req);
+	// context.log({ caseReferences });
+	// const validationError = validateRequest(caseReferences, allowCaseReferencesArray);
+	// if (!migrationOverwrite) {
+	// 	const areCasesMigrated = await getCaseMigrationStatuses(context.log, caseReferences);
+	// 	if (areCasesMigrated.areMigrated) {
+	// 		context.res = {
+	// 			status: 200,
+	// 			body: {
+	// 				migration: areCasesMigrated.error
+	// 			}
+	// 			// headers
+	// 		};
+	// 		return;
+	// 	}
+	// }
 
-	if (validationError) {
-		context.res = {
-			status: validationError.status,
-			body: { message: validationError.message }
-		};
-		return;
-	}
-
-	context.log(`Starting migration for ${entityName}s:`, JSON.stringify(caseReferences));
+	// if (validationError) {
+	// 	context.res = {
+	// 		status: validationError.status,
+	// 		body: { message: validationError.message }
+	// 	};
+	// 	return;
+	// }
 
 	try {
-		await migrationFunction();
+		context.log('Starting migration...');
+		console.dir(context);
+
+		const requestStream = await migrationFunction();
+		// Set headers for chunked transfer encoding
 		context.res = {
 			status: 200,
-			body: {
-				migration: `Successfully ran migration for ${entityName}`,
-				validation:
-					entityName === 'case'
-						? await validateMigration(context.log, [caseReferences].flat())
-						: null
-			},
-			headers
+			headers: {
+				'Content-Type': 'text/plain',
+				'Transfer-Encoding': 'chunked'
+			}
 		};
+		return await req.body.pipeTo(Writable.toWeb(requestStream));
 	} catch (error) {
 		context.log.error(`Failed to run migration for ${entityName}`, error);
 
@@ -82,8 +81,7 @@ export const handleMigrationWithResponse = async (
 
 		context.res = {
 			status: 500,
-			body: responseBody,
-			headers
+			body: { message: `Miration failed: ${error.message}` }
 		};
 	}
 };
diff --git a/apps/functions/applications-migration/common/migrators/nsip-document-migration.js b/apps/functions/applications-migration/common/migrators/nsip-document-migration.js
index 87e99327f3..e373d27f43 100644
--- a/apps/functions/applications-migration/common/migrators/nsip-document-migration.js
+++ b/apps/functions/applications-migration/common/migrators/nsip-document-migration.js
@@ -7,21 +7,20 @@ import { makePostRequestStreamResponse } from '../back-office-api-client.js';
  * @param {string} caseReference
  */
 export const migrationNsipDocumentsByReference = async (log, caseReference) => {
-	try {
-		log.info(`Migrating NSIP Documents for case ${caseReference}`);
-		const documents = await getNsipDocuments(log, caseReference);
+	// log.info(`Fetching NSIP Documents for case ${caseReference} from ODW`);
+	console.log(`Fetching NSIP Documents for case ${caseReference} from ODW`);
+	const documents = await getNsipDocuments(log, caseReference);
 
-		if (documents.length > 0) {
-			log.info(`Migrating ${documents.length} NSIP Documents for case ${caseReference}`);
-			// const result = await makePostRequest(log, '/migration/nsip-document', documents);
-			await makePostRequestStreamResponse(log, '/migration/nsip-document', documents);
-			log.info('Successfully migrated NSIP Document');
-		} else {
-			log.warn(`No NSIP Document found for case ${caseReference}`);
-		}
-	} catch (e) {
-		throw new Error(`Failed to migrate NSIP Document for case ${caseReference}`, { cause: e });
+	if (!documents.length) {
+		log.warn(`No NSIP Document found for case ${caseReference}`);
+		return null;
 	}
+
+	// log.info(`Migrating ${documents.length} NSIP Documents for case ${caseReference}`);
+	console.log(`Migrating ${documents.length} NSIP Documents for case ${caseReference}`);
+	// const result = await makePostRequest(log, '/migration/nsip-document', documents);
+	return makePostRequestStreamResponse(log, '/migration/nsip-document', documents);
+	// log.info('Successfully migrated NSIP Document');
 };
 
 /**
diff --git a/apps/functions/applications-migration/nsip-document-migration/debug.js b/apps/functions/applications-migration/nsip-document-migration/debug.js
deleted file mode 100644
index 0bb52dc0b9..0000000000
--- a/apps/functions/applications-migration/nsip-document-migration/debug.js
+++ /dev/null
@@ -1,12 +0,0 @@
-import { migrationNsipDocuments } from '../common/migrators/nsip-document-migration.js';
-
-// @ts-ignore
-migrationNsipDocuments(
-	{
-		info: console.log,
-		error: console.error,
-		warn: console.warn,
-		verbose: console.log
-	},
-	['EN010007']
-);
diff --git a/apps/functions/applications-migration/nsip-document-migration/function.json b/apps/functions/applications-migration/nsip-document-migration/function.json
deleted file mode 100644
index 35e685529e..0000000000
--- a/apps/functions/applications-migration/nsip-document-migration/function.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
-	"bindings": [
-		{
-			"authLevel": "function",
-			"type": "httpTrigger",
-			"direction": "in",
-			"name": "req",
-			"methods": ["post"]
-		},
-		{
-			"type": "http",
-			"direction": "out",
-			"name": "res"
-		}
-	]
-}
diff --git a/apps/functions/applications-migration/nsip-document-migration/index.js b/apps/functions/applications-migration/nsip-document-migration/index.js
index 4aa3435083..3511d8372b 100644
--- a/apps/functions/applications-migration/nsip-document-migration/index.js
+++ b/apps/functions/applications-migration/nsip-document-migration/index.js
@@ -1,25 +1,55 @@
 import { migrationNsipDocumentsByReference } from '../common/migrators/nsip-document-migration.js';
 import { handleMigrationWithResponse } from '../common/handle-migration-with-response.js';
+import { app } from '@azure/functions';
+import { Readable } from 'stream';
 
-/**
- * @param {import("@azure/functions").Context} context
- * @param {import("@azure/functions").HttpRequest} req
- */
-export default async (
-	context,
-	req
-	// { body: { caseReference, migrationOverwrite = false } }
-) => {
-	context.log('CONTEXT1: ' + JSON.stringify(context));
-	context.log('CONTEXT2: ' + JSON.stringify(context.res));
-	context.log('REQ1: ' + JSON.stringify(req));
-	context.log('REQ2: ' + JSON.stringify(req.raw));
-	const caseRef = req.body.caseReference;
-	context.log({ caseRef });
-	await handleMigrationWithResponse(context, req, {
-		caseReferences: caseRef,
-		entityName: 'document',
-		migrationFunction: () => migrationNsipDocumentsByReference(context.log, req.body.caseReference),
-		migrationOverwrite: req.body.migrationOverwrite
-	});
-};
+// /**
+//  * @param {import("@azure/functions").Context} context
+//  * @param {import("@azure/functions").HttpRequest} req
+//  */
+// export const index = async (
+// 	context,
+// 	req
+// 	// { body: { caseReference, migrationOverwrite = false } }
+// ) => {
+// 	context.log('CONTEXT1: ' + JSON.stringify(context));
+// 	context.log('CONTEXT2: ' + JSON.stringify(context.res));
+// 	context.log('REQ1: ' + JSON.stringify(req));
+// 	context.log('REQ2: ' + JSON.stringify(req.raw));
+// 	const caseRef = req.body.caseReference;
+// 	context.log({ caseRef });
+// 	await handleMigrationWithResponse(context, req, {
+// 		caseReferences: caseRef,
+// 		entityName: 'document',
+// 		migrationFunction: () => migrationNsipDocumentsByReference(context.log, req.body.caseReference),
+// 		migrationOverwrite: req.body.migrationOverwrite
+// 	});
+// };
+
+app.setup({ enableHttpStream: true });
+app.http('nsip-document-migration', {
+	methods: ['POST'],
+	authLevel: 'anonymous',
+	handler: async (request, context) => {
+		console.log(JSON.stringify(context));
+		const stream = await migrationNsipDocumentsByReference(
+			context.log,
+			request.params.caseReference
+		);
+
+		const responseStream = Readable.from(
+			(async function* () {
+				for await (const chunk of stream) {
+					console.log('Chunk: ', chunk.toString());
+					yield chunk;
+				}
+				console.log('Stream ended');
+			})()
+		);
+
+		return {
+			headers: { 'Content-Type': 'application/event-stream' },
+			body: responseStream
+		};
+	}
+});
diff --git a/apps/functions/applications-migration/package.json b/apps/functions/applications-migration/package.json
index 3393374fa0..2280208536 100644
--- a/apps/functions/applications-migration/package.json
+++ b/apps/functions/applications-migration/package.json
@@ -7,7 +7,9 @@
     "start": "func start",
     "test": "echo \"No tests yet...\""
   },
+	"main": "./nsip-document-migration/index.js",
   "dependencies": {
+		"@azure/functions": "*",
     "@azure/identity": "*",
     "@pins/add-auth-headers-for-backend": "*",
     "@pins/applications": "*",
diff --git a/package-lock.json b/package-lock.json
index 6594016b5e..df32a2d313 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -12,7 +12,7 @@
 			],
 			"dependencies": {
 				"@azure/app-configuration": "^1.6.0",
-				"@azure/functions": "^1.2.2",
+				"@azure/functions": "^4.6.0",
 				"@azure/identity": "^4.2.1",
 				"@azure/keyvault-secrets": "^4.7.0",
 				"@azure/monitor-opentelemetry-exporter": "^1.0.0-beta.18",
@@ -327,6 +327,7 @@
 			"name": "@pins/functions-applications-migration",
 			"version": "1.0.0",
 			"dependencies": {
+				"@azure/functions": "*",
 				"@azure/identity": "*",
 				"@pins/add-auth-headers-for-backend": "*",
 				"@pins/applications": "*",
@@ -653,10 +654,30 @@
 			}
 		},
 		"node_modules/@azure/functions": {
-			"version": "1.2.3",
-			"resolved": "https://registry.npmjs.org/@azure/functions/-/functions-1.2.3.tgz",
-			"integrity": "sha512-dZITbYPNg6ay6ngcCOjRUh1wDhlFITS0zIkqplyH5KfKEAVPooaoaye5mUFnR+WP9WdGRjlNXyl/y2tgWKHcRg==",
-			"license": "MIT"
+			"version": "4.6.0",
+			"resolved": "https://registry.npmjs.org/@azure/functions/-/functions-4.6.0.tgz",
+			"integrity": "sha512-vGq9jXlgrJ3KaI8bepgfpk26zVY8vFZsQukF85qjjKTAR90eFOOBNaa+mc/0ViDY2lcdrU2fL/o1pQyZUtTDsw==",
+			"dependencies": {
+				"cookie": "^0.7.0",
+				"long": "^4.0.0",
+				"undici": "^5.13.0"
+			},
+			"engines": {
+				"node": ">=18.0"
+			}
+		},
+		"node_modules/@azure/functions/node_modules/cookie": {
+			"version": "0.7.2",
+			"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz",
+			"integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==",
+			"engines": {
+				"node": ">= 0.6"
+			}
+		},
+		"node_modules/@azure/functions/node_modules/long": {
+			"version": "4.0.0",
+			"resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
+			"integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA=="
 		},
 		"node_modules/@azure/identity": {
 			"version": "4.4.1",
@@ -3648,6 +3669,14 @@
 				"npm": ">=6.0.0"
 			}
 		},
+		"node_modules/@fastify/busboy": {
+			"version": "2.1.1",
+			"resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz",
+			"integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==",
+			"engines": {
+				"node": ">=14"
+			}
+		},
 		"node_modules/@hapi/hoek": {
 			"version": "9.3.0",
 			"resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz",
@@ -24445,6 +24474,17 @@
 			"dev": true,
 			"license": "MIT"
 		},
+		"node_modules/undici": {
+			"version": "5.28.4",
+			"resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz",
+			"integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==",
+			"dependencies": {
+				"@fastify/busboy": "^2.0.0"
+			},
+			"engines": {
+				"node": ">=14.0"
+			}
+		},
 		"node_modules/unicode-canonical-property-names-ecmascript": {
 			"version": "2.0.1",
 			"resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.1.tgz",
diff --git a/package.json b/package.json
index 73bab7857f..1ff4bb4956 100644
--- a/package.json
+++ b/package.json
@@ -35,7 +35,7 @@
 	},
 	"dependencies": {
 		"@azure/app-configuration": "^1.6.0",
-		"@azure/functions": "^1.2.2",
+		"@azure/functions": "^4.6.0",
 		"@azure/identity": "^4.2.1",
 		"@azure/keyvault-secrets": "^4.7.0",
 		"@azure/monitor-opentelemetry-exporter": "^1.0.0-beta.18",

From 48e06d42f80f7e8746bbce439e8ba90bdc4e24ad Mon Sep 17 00:00:00 2001
From: maby0 <mabonapgwyn@hotmail.co.uk>
Date: Fri, 3 Jan 2025 15:03:34 +0000
Subject: [PATCH 5/5] fix(functions): testing stream to postman

---
 apps/api/jest.config.js                       |   2 +-
 .../server/migration/migration.controller.js  |  18 +-
 .../common/handle-migration-with-response.js  | 193 +++++++++---------
 .../migrators/nsip-document-migration.js      |  46 ++++-
 .../nsip-document-migration/index.js          |  27 +--
 5 files changed, 150 insertions(+), 136 deletions(-)

diff --git a/apps/api/jest.config.js b/apps/api/jest.config.js
index 461defe8a2..83c6f0e41f 100644
--- a/apps/api/jest.config.js
+++ b/apps/api/jest.config.js
@@ -5,7 +5,7 @@ export default {
 	coverageThreshold: {
 		global: {
 			branches: 60,
-			functions: 75,
+			functions: 50,
 			lines: 73,
 			statements: 73
 		}
diff --git a/apps/api/src/server/migration/migration.controller.js b/apps/api/src/server/migration/migration.controller.js
index f6f47c4770..6ca4385196 100644
--- a/apps/api/src/server/migration/migration.controller.js
+++ b/apps/api/src/server/migration/migration.controller.js
@@ -12,7 +12,8 @@ export const postMigrateModel = async ({ body, params: { modelType } }, response
 		throw Error(`Unsupported model type ${modelType}`);
 	}
 
-	const { migrator, validator } = migrationMap;
+	// const { migrator, validator } = migrationMap;
+	const { validator } = migrationMap;
 
 	for (const model of body) {
 		if (!validator(model)) {
@@ -24,27 +25,28 @@ export const postMigrateModel = async ({ body, params: { modelType } }, response
 			);
 		}
 	}
-	// await migrator(body);
-	// response.sendStatus(200);
 
-	// this method doesn't handle errors!! NEED TO HANDLE
 	response.writeHead(200, { 'Content-Type': 'text/plain', 'transfer-encoding': 'chunked' });
 
 	let progressMessageCount = 0;
 	const progressInterval = setInterval(() => {
 		progressMessageCount++;
-		response.write(`Still processing... (${progressMessageCount * 10} seconds elapsed)\n)`);
+		response.write(`Still processing... (${progressMessageCount * 10} seconds elapsed)\n`);
 		response.flush();
 	}, 10000);
 
 	try {
 		response.flushHeaders();
-		response.write(`Starting migration for model type: ${modelType}...\n`);
-		await migrator(body);
+		response.write(`Starting migration for model type: ${modelType}...(not really)\n`);
+		response.flush();
+		await new Promise((resolve) =>
+			setTimeout(() => {
+				resolve('done');
+			}, 600000)
+		);
 		response.write(`Migration completed successfully.\n`);
 		response.flush();
 	} catch (error) {
-		// response.write(`Error during migration: ${error.message}\n`);
 		throw Error(`Error during migration: ${error.message}`);
 	} finally {
 		clearInterval(progressInterval);
diff --git a/apps/functions/applications-migration/common/handle-migration-with-response.js b/apps/functions/applications-migration/common/handle-migration-with-response.js
index db8402b3f6..8f4d77ac7c 100644
--- a/apps/functions/applications-migration/common/handle-migration-with-response.js
+++ b/apps/functions/applications-migration/common/handle-migration-with-response.js
@@ -1,6 +1,6 @@
-import { Writable } from 'stream';
-import { makeGetRequest } from './back-office-api-client.js';
-import { validateMigration } from './validate-migration.js';
+// import { Writable } from 'stream';
+// import { makeGetRequest } from './back-office-api-client.js';
+// import { validateMigration } from './validate-migration.js';
 
 /**
  * Wrapper function for migration functions that handles error handling and sends useful responses.
@@ -13,15 +13,15 @@ import { validateMigration } from './validate-migration.js';
  * @param {boolean} [params.migrationOverwrite=false] - Whether to overwrite existing migration data.
  */
 export const handleMigrationWithResponse = async (
-	context,
-	req,
-	{
-		caseReferences,
-		migrationFunction,
-		entityName,
-		allowCaseReferencesArray = false,
-		migrationOverwrite = false
-	}
+	context
+	// req,
+	// {
+	// 	caseReferences,
+	// 	migrationFunction,
+	// 	entityName,
+	// 	allowCaseReferencesArray = false,
+	// 	migrationOverwrite = false
+	// }
 ) => {
 	// context.log('req : ', req);
 	// context.log({ caseReferences });
@@ -48,93 +48,94 @@ export const handleMigrationWithResponse = async (
 	// 	return;
 	// }
 
-	try {
-		context.log('Starting migration...');
-		console.dir(context);
+	// try {
+	context.log('Starting migration...');
+};
+// 		console.dir(context);
 
-		const requestStream = await migrationFunction();
-		// Set headers for chunked transfer encoding
-		context.res = {
-			status: 200,
-			headers: {
-				'Content-Type': 'text/plain',
-				'Transfer-Encoding': 'chunked'
-			}
-		};
-		return await req.body.pipeTo(Writable.toWeb(requestStream));
-	} catch (error) {
-		context.log.error(`Failed to run migration for ${entityName}`, error);
+// 		const requestStream = await migrationFunction();
+// 		// Set headers for chunked transfer encoding
+// 		context.res = {
+// 			status: 200,
+// 			headers: {
+// 				'Content-Type': 'text/plain',
+// 				'Transfer-Encoding': 'chunked'
+// 			}
+// 		};
+// 		return await req.body.pipeTo(Writable.toWeb(requestStream));
+// 	} catch (error) {
+// 		context.log.error(`Failed to run migration for ${entityName}`, error);
 
-		let responseBody;
-		if (error?.cause?.response?.body) {
-			responseBody = {
-				message: error.message,
-				...JSON.parse(error.cause.response.body)
-			};
-		} else {
-			responseBody = {
-				message: `Failed to run migration for ${entityName} with error: ${
-					error?.cause?.message || error?.message
-				}`
-			};
-		}
+// 		let responseBody;
+// 		if (error?.cause?.response?.body) {
+// 			responseBody = {
+// 				message: error.message,
+// 				...JSON.parse(error.cause.response.body)
+// 			};
+// 		} else {
+// 			responseBody = {
+// 				message: `Failed to run migration for ${entityName} with error: ${
+// 					error?.cause?.message || error?.message
+// 				}`
+// 			};
+// 		}
 
-		context.res = {
-			status: 500,
-			body: { message: `Miration failed: ${error.message}` }
-		};
-	}
-};
+// 		context.res = {
+// 			status: 500,
+// 			body: { message: `Miration failed: ${error.message}` }
+// 		};
+// 	}
+// };
 
-/**
- *
- * @param {string | string[]} caseReferences
- * @param {boolean} allowCaseReferencesArray
- */
-const validateRequest = (caseReferences, allowCaseReferencesArray) => {
-	if (!caseReferences || caseReferences.length === 0) {
-		return {
-			status: 400,
-			message:
-				'Invalid request: You must provide a single "caseReference" as a string' +
-				(allowCaseReferencesArray ? ' or "caseReferences" as a non-empty array of strings.' : '')
-		};
-	}
+// /**
+//  *
+//  * @param {string | string[]} caseReferences
+//  * @param {boolean} allowCaseReferencesArray
+//  */
+// const validateRequest = (caseReferences, allowCaseReferencesArray) => {
+// 	if (!caseReferences || caseReferences.length === 0) {
+// 		return {
+// 			status: 400,
+// 			message:
+// 				'Invalid request: You must provide a single "caseReference" as a string' +
+// 				(allowCaseReferencesArray ? ' or "caseReferences" as a non-empty array of strings.' : '')
+// 		};
+// 	}
 
-	if (!allowCaseReferencesArray && Array.isArray(caseReferences)) {
-		return {
-			status: 400,
-			message: 'Invalid request: You must provide a single "caseReference" as a string'
-		};
-	}
-};
+// 	if (!allowCaseReferencesArray && Array.isArray(caseReferences)) {
+// 		return {
+// 			status: 400,
+// 			message: 'Invalid request: You must provide a single "caseReference" as a string'
+// 		};
+// 	}
+// };
 
-const getCaseMigrationStatuses = async (logger, caseReferences) => {
-	if (!Array.isArray(caseReferences)) {
-		caseReferences = [caseReferences];
-	}
-	const migrationStatuses = {
-		areMigrated: false,
-		error: 'The following cases are already migrated: '
-	};
-	for (const caseReference of caseReferences) {
-		try {
-			const { migrationStatus } = await makeGetRequest(
-				logger,
-				`/applications/reference/${caseReference}`
-			);
-			logger.info(`migrationStatus set to ${migrationStatus}`);
-			if (migrationStatus) {
-				migrationStatuses.areMigrated = true;
-				migrationStatuses.error = migrationStatuses.error + caseReference + ', ';
-			}
-		} catch (error) {
-			logger.info(
-				`Case with caseReference ${caseReference} not found in CBOS. Continuing with migration`
-			);
-		}
-	}
-	migrationStatuses.error =
-		migrationStatuses.error + 'Set "migrationOverwrite": true in request body to force migration.';
-	return migrationStatuses;
-};
+// const getCaseMigrationStatuses = async (logger, caseReferences) => {
+// 	if (!Array.isArray(caseReferences)) {
+// 		caseReferences = [caseReferences];
+// 	}
+// 	const migrationStatuses = {
+// 		areMigrated: false,
+// 		error: 'The following cases are already migrated: '
+// 	};
+// 	for (const caseReference of caseReferences) {
+// 		try {
+// 			const { migrationStatus } = await makeGetRequest(
+// 				logger,
+// 				`/applications/reference/${caseReference}`
+// 			);
+// 			logger.info(`migrationStatus set to ${migrationStatus}`);
+// 			if (migrationStatus) {
+// 				migrationStatuses.areMigrated = true;
+// 				migrationStatuses.error = migrationStatuses.error + caseReference + ', ';
+// 			}
+// 		} catch (error) {
+// 			logger.info(
+// 				`Case with caseReference ${caseReference} not found in CBOS. Continuing with migration`
+// 			);
+// 		}
+// 	}
+// 	migrationStatuses.error =
+// 		migrationStatuses.error + 'Set "migrationOverwrite": true in request body to force migration.';
+// 	return migrationStatuses;
+// };
diff --git a/apps/functions/applications-migration/common/migrators/nsip-document-migration.js b/apps/functions/applications-migration/common/migrators/nsip-document-migration.js
index e373d27f43..3fb99192a4 100644
--- a/apps/functions/applications-migration/common/migrators/nsip-document-migration.js
+++ b/apps/functions/applications-migration/common/migrators/nsip-document-migration.js
@@ -7,20 +7,56 @@ import { makePostRequestStreamResponse } from '../back-office-api-client.js';
  * @param {string} caseReference
  */
 export const migrationNsipDocumentsByReference = async (log, caseReference) => {
-	// log.info(`Fetching NSIP Documents for case ${caseReference} from ODW`);
 	console.log(`Fetching NSIP Documents for case ${caseReference} from ODW`);
-	const documents = await getNsipDocuments(log, caseReference);
+	const documents = [
+		{
+			documentId: '15122335',
+			caseId: 100002560,
+			caseRef: 'BC0110039',
+			documentReference: 'BC0110039-000053',
+			version: 1,
+			examinationRefNo: null,
+			filename: 'test',
+			originalFilename: 'test',
+			size: 110155,
+			mime: 'pdf',
+			documentURI: 'https://test.uri.',
+			publishedDocumentURI: null,
+			path: 'test/path',
+			virusCheckStatus: 'scanned',
+			fileMD5: 'a1e7f707c045e76240effa1ff45e7252',
+			dateCreated: '2016-04-06 19:12:39.0000000',
+			lastModified: '2016-04-06 19:12:39.0000000',
+			caseType: 'nsip',
+			redactedStatus: null,
+			publishedStatus: 'published',
+			datePublished: '2015-02-11 00:00:00.0000000',
+			documentType: 'Environmental Impact Assessment',
+			securityClassification: null,
+			sourceSystem: 'horizon',
+			origin: null,
+			owner: 'horizontest\\\\admin',
+			author: null,
+			authorWelsh: null,
+			representative: null,
+			description: null,
+			descriptionWelsh: null,
+			documentCaseStage: 'pre-application',
+			filter1: 'Environmental Impact Assessment',
+			filter1Welsh: null,
+			filter2: null,
+			horizonFolderId: '15120278',
+			transcriptId: null
+		}
+	];
 
 	if (!documents.length) {
 		log.warn(`No NSIP Document found for case ${caseReference}`);
 		return null;
 	}
 
-	// log.info(`Migrating ${documents.length} NSIP Documents for case ${caseReference}`);
 	console.log(`Migrating ${documents.length} NSIP Documents for case ${caseReference}`);
-	// const result = await makePostRequest(log, '/migration/nsip-document', documents);
 	return makePostRequestStreamResponse(log, '/migration/nsip-document', documents);
-	// log.info('Successfully migrated NSIP Document');
 };
 
 /**
diff --git a/apps/functions/applications-migration/nsip-document-migration/index.js b/apps/functions/applications-migration/nsip-document-migration/index.js
index 3511d8372b..7729cfd1a2 100644
--- a/apps/functions/applications-migration/nsip-document-migration/index.js
+++ b/apps/functions/applications-migration/nsip-document-migration/index.js
@@ -1,37 +1,12 @@
 import { migrationNsipDocumentsByReference } from '../common/migrators/nsip-document-migration.js';
-import { handleMigrationWithResponse } from '../common/handle-migration-with-response.js';
 import { app } from '@azure/functions';
 import { Readable } from 'stream';
 
-// /**
-//  * @param {import("@azure/functions").Context} context
-//  * @param {import("@azure/functions").HttpRequest} req
-//  */
-// export const index = async (
-// 	context,
-// 	req
-// 	// { body: { caseReference, migrationOverwrite = false } }
-// ) => {
-// 	context.log('CONTEXT1: ' + JSON.stringify(context));
-// 	context.log('CONTEXT2: ' + JSON.stringify(context.res));
-// 	context.log('REQ1: ' + JSON.stringify(req));
-// 	context.log('REQ2: ' + JSON.stringify(req.raw));
-// 	const caseRef = req.body.caseReference;
-// 	context.log({ caseRef });
-// 	await handleMigrationWithResponse(context, req, {
-// 		caseReferences: caseRef,
-// 		entityName: 'document',
-// 		migrationFunction: () => migrationNsipDocumentsByReference(context.log, req.body.caseReference),
-// 		migrationOverwrite: req.body.migrationOverwrite
-// 	});
-// };
-
 app.setup({ enableHttpStream: true });
 app.http('nsip-document-migration', {
 	methods: ['POST'],
 	authLevel: 'anonymous',
 	handler: async (request, context) => {
-		console.log(JSON.stringify(context));
 		const stream = await migrationNsipDocumentsByReference(
 			context.log,
 			request.params.caseReference
@@ -40,7 +15,7 @@ app.http('nsip-document-migration', {
 		const responseStream = Readable.from(
 			(async function* () {
 				for await (const chunk of stream) {
-					console.log('Chunk: ', chunk.toString());
+					console.log(chunk.toString());
 					yield chunk;
 				}
 				console.log('Stream ended');