From 189983c354047fa5076444dcbe4697112cf9e187 Mon Sep 17 00:00:00 2001 From: Sergio N <71926587+SergioNR@users.noreply.github.com> Date: Thu, 8 Jan 2026 12:34:01 +0100 Subject: [PATCH 01/12] Replace logInfo with console.log for logging --- server/controllers/transcriptionController.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/controllers/transcriptionController.js b/server/controllers/transcriptionController.js index 3afa5d4..669d482 100644 --- a/server/controllers/transcriptionController.js +++ b/server/controllers/transcriptionController.js @@ -11,7 +11,7 @@ export const processPendingTranscriptionJobs = async () => { const pendingTranscriptionJobs = await getPendingTranscriptionJobsFromDb(); if (pendingTranscriptionJobs.length === 0) { - logInfo('No pending transcription jobs found'); + console.log('No pending transcription jobs found'); return; } From 48a311d91af0eef5e98172d3cae652ac3dd57f43 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sun, 11 Jan 2026 14:54:44 +0100 Subject: [PATCH 02/12] UPDATE prisma schema to allow for one-to-many relationship between transcription jobs <--> analysis entry --- prisma/schema.prisma | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prisma/schema.prisma b/prisma/schema.prisma index c75a00b..50aff9a 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -125,7 +125,7 @@ model AnalysisEntry { updated_at DateTime @default(now()) @updatedAt transcription_segments Json? full_transcript String? - transcriptionJob TranscriptionJob? + transcriptionJob TranscriptionJob[] @@index([analysis_id, status]) } @@ -133,7 +133,7 @@ model AnalysisEntry { model TranscriptionJob { id Int @id @unique @default(autoincrement()) AnalysisEntry AnalysisEntry @relation(fields: [analysis_entry_id], references: [id]) - analysis_entry_id String @unique + analysis_entry_id String status TranscriptionJobStatus @default(PENDING) created_at DateTime @default(now()) updated_at DateTime @default(now()) @updatedAt From 1f0fc250fd45931c28c898a1ebf55c6db7eaf8bd Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sun, 11 Jan 2026 14:55:14 +0100 Subject: [PATCH 03/12] REFACTOR pendingTranscriptionJob to a fire and forget function All errors will be handled in the actual processing function --- server/cron/getPendingTranscriptionJobScheduler.js | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/server/cron/getPendingTranscriptionJobScheduler.js b/server/cron/getPendingTranscriptionJobScheduler.js index a35aaa0..9f43075 100644 --- a/server/cron/getPendingTranscriptionJobScheduler.js +++ b/server/cron/getPendingTranscriptionJobScheduler.js @@ -1,11 +1,6 @@ import { CronJob } from 'cron'; import { processPendingTranscriptionJobs } from '../controllers/transcriptionController.js'; -import { logError } from '../config/loggerFunctions.js'; -export const getPendingTranscriptionJobScheduler = new CronJob('*/15 * * * *', async () => { - try { - await processPendingTranscriptionJobs(); - } catch (error) { - logError('Error processing transcription request', error); - } +export const getPendingTranscriptionJobScheduler = new CronJob('* * * * *', async () => { + processPendingTranscriptionJobs(); }); From 1485ccbb020ae37d8b6c682127478181d7bc769d Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sun, 11 Jan 2026 21:44:24 +0100 Subject: [PATCH 04/12] ADD no-return-await exception to eslint config --- .eslintrc.cjs | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index b0a4d2d..9009f8a 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -47,5 +47,6 @@ module.exports = { 'no-plusplus': 'off', 'no-await-in-loop': 'off', 'no-restricted-syntax': 'off', + 'no-return-await': 'off' }, }; From 9236c7ff81ebb57f3bec4acf94d824ff6d00e969 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sun, 11 Jan 2026 21:44:42 +0100 Subject: [PATCH 05/12] FIX typo in S3 client --- server/integrations/s3-client/s3.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/integrations/s3-client/s3.js b/server/integrations/s3-client/s3.js index 4d3c89a..0af61ee 100644 --- a/server/integrations/s3-client/s3.js +++ b/server/integrations/s3-client/s3.js @@ -12,7 +12,7 @@ export const externalS3Client = new S3Client({ }); export const internalS3Client = new S3Client({ - region: process.env.S3_REGION, // irrelevant since miniIO doesnt takei into account + region: process.env.S3_REGION, // irrelevant since miniIO doesnt take into account endpoint: process.env.S3_INTERNAL_ENDPOINT, // Container network endpoint forcePathStyle: true, // Required for MinIO path-style URLs credentials: { From 483078de25f3b25a7e26d4fa0125a14b33f269ac Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sun, 11 Jan 2026 21:46:45 +0100 Subject: [PATCH 06/12] UPDATE processing transcriptions to 5 minutes --- server/cron/getPendingTranscriptionJobScheduler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/cron/getPendingTranscriptionJobScheduler.js b/server/cron/getPendingTranscriptionJobScheduler.js index 9f43075..4200c94 100644 --- a/server/cron/getPendingTranscriptionJobScheduler.js +++ b/server/cron/getPendingTranscriptionJobScheduler.js @@ -1,6 +1,6 @@ import { CronJob } from 'cron'; import { processPendingTranscriptionJobs } from '../controllers/transcriptionController.js'; -export const getPendingTranscriptionJobScheduler = new CronJob('* * * * *', async () => { +export const getPendingTranscriptionJobScheduler = new CronJob('*/5 * * * *', async () => { processPendingTranscriptionJobs(); }); From 732d7b3280239a3960d3aef04441dfa4e1d43dde Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sun, 11 Jan 2026 21:48:30 +0100 Subject: [PATCH 07/12] REFACTOR transcription job processing process Jobs will now be taken one by one sequentially to prevent hogging the system resources - taking one every 5 minutes - which should be enough for transcription to be completed --- server/controllers/transcriptionController.js | 89 ++++++++++++++----- server/models/transcriptionModel.js | 19 ++-- 2 files changed, 73 insertions(+), 35 deletions(-) diff --git a/server/controllers/transcriptionController.js b/server/controllers/transcriptionController.js index 669d482..7f5b5b6 100644 --- a/server/controllers/transcriptionController.js +++ b/server/controllers/transcriptionController.js @@ -1,45 +1,88 @@ import { logError, logInfo } from '../config/loggerFunctions.js'; import { transcribeRecording } from '../integrations/whisper-asr-webservice/transcribe.js'; import { - getPendingTranscriptionJobsFromDb, + getFirstTranscriptionJobFromDbByStatus, storeNormalizedTranscriptionInDb, updateStatusSingleTranscriptionJobInDb, } from '../models/transcriptionModel.js'; import { cleanUpTranscriptSegments } from '../utils/transcription/transcriptionNormalizer.js'; export const processPendingTranscriptionJobs = async () => { - const pendingTranscriptionJobs = await getPendingTranscriptionJobsFromDb(); + // check if there is an ongoing transcription job - if (pendingTranscriptionJobs.length === 0) { - console.log('No pending transcription jobs found'); - return; + let inProgressTranscriptionJob; + + try { + inProgressTranscriptionJob = await getFirstTranscriptionJobFromDbByStatus('IN_PROGRESS'); + } catch (error) { + return logError('error fetching first transcription job from DB', error); + } + if (inProgressTranscriptionJob) { + return console.log(`Transcription job id: ${inProgressTranscriptionJob.id} is in progress Skipping new jobs.`); // * Set as console log for registering but not cluttering logs + } + + let pendingTranscriptionJob; + + try { + pendingTranscriptionJob = await getFirstTranscriptionJobFromDbByStatus('PENDING'); + } catch (error) { + return logError('Error fetching first pending transcription job from DB', error); + } + + if (!pendingTranscriptionJob) { + return console.log('No pending transcription jobs found.'); // * Set as console log for registering but not cluttering logs } - for (const transcriptionJob of pendingTranscriptionJobs) { - logInfo(`Processing transcription job for analysis entry ID: ${transcriptionJob.analysis_entry_id}`); + const { + id: transcriptionJobId, + analysis_entry_id: analysisEntryId, + } = pendingTranscriptionJob; - const { analysis_entry_id: analysisEntryId } = transcriptionJob; + logInfo(`Processing transcription job ${transcriptionJobId} for analysis entry ID: ${analysisEntryId}`); - try { - // Mark job as IN_PROGRESS before making async call to prevent re-queuing - await updateStatusSingleTranscriptionJobInDb(analysisEntryId, 'IN_PROGRESS'); - logInfo(`Marked transcription job ${analysisEntryId} as IN_PROGRESS`); + try { // Mark job as IN_PROGRESS before making async call to prevent re-queuing + await updateStatusSingleTranscriptionJobInDb(transcriptionJobId, 'IN_PROGRESS'); + logInfo(`Marked transcription job ${transcriptionJobId} for analysis entry ID ${analysisEntryId} as IN_PROGRESS`); + } catch (error) { + return logError(`error updating status for transcription job ${transcriptionJobId} to IN_PROGRESS`, error); + } - const transcriptionJobResult = await transcribeRecording(transcriptionJob); + let transcriptionJobResult; - const { segments, text: fullText } = transcriptionJobResult; + try { + transcriptionJobResult = await transcribeRecording(pendingTranscriptionJob); + } catch (error) { + logError(`error transcribing recording for transcription job ${transcriptionJobId}`, error); + return await updateStatusSingleTranscriptionJobInDb(transcriptionJobId, 'PENDING'); + } + const { + segments, + text: fullText, + } = transcriptionJobResult; - const cleanedUpSegments = await cleanUpTranscriptSegments(segments); + let cleanedUpSegments; - await storeNormalizedTranscriptionInDb(analysisEntryId, fullText, cleanedUpSegments); + try { + cleanedUpSegments = await cleanUpTranscriptSegments(segments); + } catch (error) { + logError(`error cleaning up transcript segments for transcription job ${transcriptionJobId}`, error); + return await updateStatusSingleTranscriptionJobInDb(transcriptionJobId, 'PENDING'); + } - await updateStatusSingleTranscriptionJobInDb(analysisEntryId, 'COMPLETED'); + try { + await storeNormalizedTranscriptionInDb(analysisEntryId, transcriptionJobId, fullText, cleanedUpSegments); + } catch (error) { + logError(`error inserting normalized transcription from transcription job ${transcriptionJobId} in analysis entry ${analysisEntryId}`, error); + return await updateStatusSingleTranscriptionJobInDb(transcriptionJobId, 'PENDING'); + } - logInfo(`Transcription job ${analysisEntryId} completed successfully`); - } catch (error) { - // Mark job back as PENDING to allow retry - await updateStatusSingleTranscriptionJobInDb(analysisEntryId, 'PENDING'); - logError(`Error processing transcription job, ${analysisEntryId}`, error); - } + try { + await updateStatusSingleTranscriptionJobInDb(transcriptionJobId, 'COMPLETED'); + logInfo(`Marked transcription job ${transcriptionJobId} for analysis entry ID ${analysisEntryId} as COMPLETED`); + } catch (error) { + logError(`error updating status for transcription job ${transcriptionJobId} to COMPLETED`, error); + return await updateStatusSingleTranscriptionJobInDb(transcriptionJobId, 'PENDING'); } + + return logInfo(`Transcription job ${transcriptionJobId} for analysis entry ID ${analysisEntryId} completed successfully`); }; diff --git a/server/models/transcriptionModel.js b/server/models/transcriptionModel.js index ba52834..ec70f38 100644 --- a/server/models/transcriptionModel.js +++ b/server/models/transcriptionModel.js @@ -12,9 +12,9 @@ export const insertTranscriptionJobInDb = async (transcriptionRequest) => { }); }; -export const updateStatusSingleTranscriptionJobInDb = async (analysisEntryId, status) => { +export const updateStatusSingleTranscriptionJobInDb = async (transcriptionJobId, status) => { const whereClause = { - analysis_entry_id: analysisEntryId, + id: transcriptionJobId, }; await prisma.transcriptionJob.update({ @@ -25,7 +25,7 @@ export const updateStatusSingleTranscriptionJobInDb = async (analysisEntryId, st }); }; -export const storeNormalizedTranscriptionInDb = async (analysisEntryId, fullText, normalizedSegments) => { +export const storeNormalizedTranscriptionInDb = async (analysisEntryId, transcriptionJobId, fullText, normalizedSegments) => { const whereClause = { id: analysisEntryId, }; @@ -35,23 +35,19 @@ export const storeNormalizedTranscriptionInDb = async (analysisEntryId, fullText data: { full_transcript: fullText, transcription_segments: normalizedSegments, - transcriptionJob: { - update: { - status: 'COMPLETED', - }, - }, }, }); }; -export const getPendingTranscriptionJobsFromDb = async () => { +export const getFirstTranscriptionJobFromDbByStatus = async (status) => { const whereClause = { - status: 'PENDING', + status: status, }; - const pendingTranscriptionJobs = await prisma.transcriptionJob.findMany({ + const pendingTranscriptionJobs = await prisma.transcriptionJob.findFirst({ where: whereClause, select: { + id: true, analysis_entry_id: true, language_code: true, AnalysisEntry: { @@ -60,7 +56,6 @@ export const getPendingTranscriptionJobsFromDb = async () => { }, }, }, - take: 10, }); return pendingTranscriptionJobs; From 3a681ab071863948ccd0a0544b6f22e596d5e731 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Sun, 11 Jan 2026 22:05:58 +0100 Subject: [PATCH 08/12] REMOVE unused parameter in transcription model --- server/controllers/transcriptionController.js | 2 +- server/models/transcriptionModel.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/controllers/transcriptionController.js b/server/controllers/transcriptionController.js index 7f5b5b6..3441599 100644 --- a/server/controllers/transcriptionController.js +++ b/server/controllers/transcriptionController.js @@ -70,7 +70,7 @@ export const processPendingTranscriptionJobs = async () => { } try { - await storeNormalizedTranscriptionInDb(analysisEntryId, transcriptionJobId, fullText, cleanedUpSegments); + await storeNormalizedTranscriptionInDb(analysisEntryId, fullText, cleanedUpSegments); } catch (error) { logError(`error inserting normalized transcription from transcription job ${transcriptionJobId} in analysis entry ${analysisEntryId}`, error); return await updateStatusSingleTranscriptionJobInDb(transcriptionJobId, 'PENDING'); diff --git a/server/models/transcriptionModel.js b/server/models/transcriptionModel.js index ec70f38..0d0e433 100644 --- a/server/models/transcriptionModel.js +++ b/server/models/transcriptionModel.js @@ -25,7 +25,7 @@ export const updateStatusSingleTranscriptionJobInDb = async (transcriptionJobId, }); }; -export const storeNormalizedTranscriptionInDb = async (analysisEntryId, transcriptionJobId, fullText, normalizedSegments) => { +export const storeNormalizedTranscriptionInDb = async (analysisEntryId, fullText, normalizedSegments) => { const whereClause = { id: analysisEntryId, }; From aecf7a947edadb83761121013d36dd77f71674d3 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Mon, 12 Jan 2026 09:57:03 +0100 Subject: [PATCH 09/12] HOTFIX: update prisma.schema to accomodate for recently pushed changes --- .../migration.sql | 5 +++++ prisma/schema.prisma | 2 ++ 2 files changed, 7 insertions(+) create mode 100644 prisma/migrations/20260112085631_remove_analysis_entry_id_uniqueness_to_allow_for_transcription_reruns/migration.sql diff --git a/prisma/migrations/20260112085631_remove_analysis_entry_id_uniqueness_to_allow_for_transcription_reruns/migration.sql b/prisma/migrations/20260112085631_remove_analysis_entry_id_uniqueness_to_allow_for_transcription_reruns/migration.sql new file mode 100644 index 0000000..134e669 --- /dev/null +++ b/prisma/migrations/20260112085631_remove_analysis_entry_id_uniqueness_to_allow_for_transcription_reruns/migration.sql @@ -0,0 +1,5 @@ +-- DropIndex +DROP INDEX "TranscriptionJob_analysis_entry_id_key"; + +-- CreateIndex +CREATE INDEX "TranscriptionJob_status_idx" ON "TranscriptionJob"("status"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 50aff9a..309e711 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -138,6 +138,8 @@ model TranscriptionJob { created_at DateTime @default(now()) updated_at DateTime @default(now()) @updatedAt language_code String + + @@index([status]) } enum TranscriptionJobStatus { From 4641324036a0deae52ae36ebd0fbe77092db7648 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Mon, 12 Jan 2026 09:57:17 +0100 Subject: [PATCH 10/12] PIN prisma to v6 --- .vscode/settings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ac583ab..d35d858 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,4 @@ { - "prisma.fileWatcher": true + "prisma.fileWatcher": true, + "prisma.pinToPrisma6": true } \ No newline at end of file From 966865447d79937d5c92fd427d09d32ff4df56c7 Mon Sep 17 00:00:00 2001 From: Sergio N Date: Mon, 12 Jan 2026 10:08:11 +0100 Subject: [PATCH 11/12] HOTFIX redaction in loggers --- server/config/logger.js | 11 ++++++++--- server/config/loggerFunctions.js | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/server/config/logger.js b/server/config/logger.js index 3f94ffe..24be7d4 100644 --- a/server/config/logger.js +++ b/server/config/logger.js @@ -1,12 +1,11 @@ import pino from 'pino'; const redactOptions = { - paths: ['context.userData.email', 'context.userData.password'], + paths: ['context.email', 'context.password'], censor: '[REDACTED]', }; const transport = pino.transport({ - redact: redactOptions, targets: [ { target: 'pino-pretty', @@ -18,4 +17,10 @@ const transport = pino.transport({ ], }); -export const logger = pino(transport); +// Apply redaction at the logger level so sensitive fields are removed +export const logger = pino( + { + redact: redactOptions, + }, + transport, +); diff --git a/server/config/loggerFunctions.js b/server/config/loggerFunctions.js index c12bbe3..4d0fd9e 100644 --- a/server/config/loggerFunctions.js +++ b/server/config/loggerFunctions.js @@ -16,7 +16,7 @@ export const logError = (errorMessage, error, additionalInfo = 'N/A') => { sendErrorLogsToTelegram(errorMessage, error); } catch (err) { - return; + return; // Fail silently } }; From cabe0361a511e632f9f38a37e7d34124087e5aec Mon Sep 17 00:00:00 2001 From: Sergio N Date: Mon, 12 Jan 2026 10:15:52 +0100 Subject: [PATCH 12/12] REMOVE unneeded await in pendingTranscriptionScheduler --- server/cron/getPendingTranscriptionJobScheduler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/cron/getPendingTranscriptionJobScheduler.js b/server/cron/getPendingTranscriptionJobScheduler.js index 4200c94..19aa2ab 100644 --- a/server/cron/getPendingTranscriptionJobScheduler.js +++ b/server/cron/getPendingTranscriptionJobScheduler.js @@ -1,6 +1,6 @@ import { CronJob } from 'cron'; import { processPendingTranscriptionJobs } from '../controllers/transcriptionController.js'; -export const getPendingTranscriptionJobScheduler = new CronJob('*/5 * * * *', async () => { +export const getPendingTranscriptionJobScheduler = new CronJob('*/5 * * * *', () => { processPendingTranscriptionJobs(); });