From 761e19def38596ac84c42273449f108f0a1396e8 Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Sat, 27 Sep 2025 21:41:16 -0400 Subject: [PATCH 01/19] fix(emailController): validate required fields and process HTML attachments before sending email refactor(emailController): enhance email handling and subscription logic; improve error handling and HTML formatting feat(emailController): enhance email sending logic to support multiple recipients with BCC; improve response messages feat(routes): add email template router to API for enhanced email management feat(emailTemplateController): enhance getAllEmailTemplates with pagination, sorting, and field selection; add updated_by field to model feat(emailController, emailTemplateController): implement email batch processing for sending and managing emails feat(emailBatch): add retry functionality for failed batch items and update routes --- src/controllers/emailBatchController.js | 178 ++++++ .../emailBatchDashboardController.js | 32 ++ src/controllers/emailController.js | 288 ++++++++-- src/controllers/emailTemplateController.js | 511 ++++++++++++++++++ src/models/EmailTemplateModel.js | 73 +++ src/models/emailBatch.js | 126 +++++ src/models/emailBatchItem.js | 87 +++ src/models/emailSubcriptionList.js | 20 +- src/routes/emailBatchDashboardRoutes.js | 14 + src/routes/emailBatchRoutes.js | 20 + src/routes/emailRouter.js | 18 +- src/routes/emailTemplateRouter.js | 14 + src/routes/templateRoutes.js | 24 + src/server.js | 3 + src/services/emailBatchDashboardService.js | 391 ++++++++++++++ src/services/emailBatchProcessor.js | 197 +++++++ src/services/emailBatchService.js | 286 ++++++++++ src/startup/routes.js | 8 +- src/utilities/emailSender.js | 37 +- 19 files changed, 2258 insertions(+), 69 deletions(-) create mode 100644 src/controllers/emailBatchController.js create mode 100644 src/controllers/emailBatchDashboardController.js create mode 100644 src/controllers/emailTemplateController.js create mode 100644 src/models/EmailTemplateModel.js create mode 100644 src/models/emailBatch.js create mode 100644 src/models/emailBatchItem.js create mode 100644 src/routes/emailBatchDashboardRoutes.js create mode 100644 src/routes/emailBatchRoutes.js create mode 100644 src/routes/emailTemplateRouter.js create mode 100644 src/routes/templateRoutes.js create mode 100644 src/services/emailBatchDashboardService.js create mode 100644 src/services/emailBatchProcessor.js create mode 100644 src/services/emailBatchService.js diff --git a/src/controllers/emailBatchController.js b/src/controllers/emailBatchController.js new file mode 100644 index 000000000..064eeb9d2 --- /dev/null +++ b/src/controllers/emailBatchController.js @@ -0,0 +1,178 @@ +/** + * Simplified Email Batch Controller - Production Ready + * Focus: Essential batch management endpoints + */ + +const EmailBatchService = require('../services/emailBatchService'); +const emailBatchProcessor = require('../services/emailBatchProcessor'); +const logger = require('../startup/logger'); + +/** + * Get all batches with pagination and filtering + */ +const getBatches = async (req, res) => { + try { + const { page = 1, limit = 20, status, dateFrom, dateTo } = req.query; + + const filters = { status, dateFrom, dateTo }; + const result = await EmailBatchService.getBatches( + filters, + parseInt(page, 10), + parseInt(limit, 10), + ); + + res.status(200).json({ + success: true, + data: result, + }); + } catch (error) { + logger.logException(error, 'Error getting batches'); + res.status(500).json({ + success: false, + message: 'Error getting batches', + error: error.message, + }); + } +}; + +/** + * Get batch details with items + */ +const getBatchDetails = async (req, res) => { + try { + const { batchId } = req.params; + const result = await EmailBatchService.getBatchWithItems(batchId); + + if (!result) { + return res.status(404).json({ + success: false, + message: 'Batch not found', + }); + } + + res.status(200).json({ + success: true, + data: result, + }); + } catch (error) { + logger.logException(error, 'Error getting batch details'); + res.status(500).json({ + success: false, + message: 'Error getting batch details', + error: error.message, + }); + } +}; + +/** + * Get dashboard statistics + */ +const getDashboardStats = async (req, res) => { + try { + const stats = await EmailBatchService.getDashboardStats(); + + res.status(200).json({ + success: true, + data: stats, + }); + } catch (error) { + logger.logException(error, 'Error getting dashboard stats'); + res.status(500).json({ + success: false, + message: 'Error getting dashboard stats', + error: error.message, + }); + } +}; + +/** + * Get processor status + */ +const getProcessorStatus = async (req, res) => { + try { + const status = emailBatchProcessor.getStatus(); + + res.status(200).json({ + success: true, + data: status, + }); + } catch (error) { + logger.logException(error, 'Error getting processor status'); + res.status(500).json({ + success: false, + message: 'Error getting processor status', + error: error.message, + }); + } +}; + +// Retry a failed batch item +const retryBatchItem = async (req, res) => { + try { + const { itemId } = req.params; + + // Find the batch item + const EmailBatchItem = require('../models/emailBatchItem'); + const item = await EmailBatchItem.findById(itemId); + + if (!item) { + return res.status(404).json({ + success: false, + message: 'Batch item not found', + }); + } + + // Check if item is already being processed + if (item.status === 'SENDING') { + return res.status(400).json({ + success: false, + message: 'Batch item is currently being processed', + }); + } + + // Only allow retry for FAILED or PENDING items + if (item.status !== 'FAILED' && item.status !== 'PENDING') { + return res.status(400).json({ + success: false, + message: 'Only failed or pending items can be retried', + }); + } + + // Reset the item status to PENDING for retry + item.status = 'PENDING'; + item.attempts = 0; + item.error = null; + item.failedAt = null; + item.lastAttemptedAt = null; + await item.save(); + + // Use the processor's retry method + const emailBatchProcessorService = require('../services/emailBatchProcessor'); + await emailBatchProcessorService.retryBatchItem(itemId); + + res.json({ + success: true, + message: 'Batch item retry initiated', + data: { + itemId: item._id, + status: item.status, + attempts: item.attempts, + }, + }); + } catch (error) { + logger.logException(error, 'Error retrying batch item'); + res.status(500).json({ + success: false, + message: 'Error retrying batch item', + error: error.message, + }); + } +}; + +module.exports = { + getBatches, + getBatchDetails, + getDashboardStats, + getProcessorStatus, + retryBatchItem, +}; diff --git a/src/controllers/emailBatchDashboardController.js b/src/controllers/emailBatchDashboardController.js new file mode 100644 index 000000000..e5eee32a1 --- /dev/null +++ b/src/controllers/emailBatchDashboardController.js @@ -0,0 +1,32 @@ +/** + * Simplified Email Batch Dashboard Controller - Production Ready + * Focus: Essential dashboard endpoints only + */ + +const EmailBatchService = require('../services/emailBatchService'); +const logger = require('../startup/logger'); + +/** + * Get dashboard statistics + */ +const getDashboardStats = async (req, res) => { + try { + const stats = await EmailBatchService.getDashboardStats(); + + res.status(200).json({ + success: true, + data: stats, + }); + } catch (error) { + logger.logException(error, 'Error getting dashboard stats'); + res.status(500).json({ + success: false, + message: 'Error getting dashboard stats', + error: error.message, + }); + } +}; + +module.exports = { + getDashboardStats, +}; diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index c71abf7e2..5ca2e8c78 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -5,6 +5,8 @@ const emailSender = require('../utilities/emailSender'); const { hasPermission } = require('../utilities/permissions'); const EmailSubcriptionList = require('../models/emailSubcriptionList'); const userProfile = require('../models/userProfile'); +const EmailBatchService = require('../services/emailBatchService'); +const emailBatchProcessor = require('../services/emailBatchProcessor'); const frontEndUrl = process.env.FRONT_END_URL || 'http://localhost:3000'; const jwtSecret = process.env.JWT_SECRET || 'EmailSecret'; @@ -28,8 +30,10 @@ const handleContentToNonOC = (htmlContent, email) => ${htmlContent} -

Thank you for subscribing to our email updates!

-

If you would like to unsubscribe, please click here

+

+ If you would like to unsubscribe from these emails, please click + here +

`; @@ -63,8 +67,7 @@ const sendEmail = async (req, res) => { return; } try { - const { to, subject, html } = req.body; - // Validate required fields + const { to, subject, html, useBatch = true } = req.body; if (!subject || !html || !to) { const missingFields = []; if (!subject) missingFields.push('Subject'); @@ -75,13 +78,80 @@ const sendEmail = async (req, res) => { .send(`${missingFields.join(' and ')} ${missingFields.length > 1 ? 'are' : 'is'} required`); } - await emailSender(to, subject, html) - .then(() => { - res.status(200).send(`Email sent successfully to ${to}`); - }) - .catch(() => { - res.status(500).send('Error sending email'); - }); + const { html: processedHtml, attachments } = extractImagesAndCreateAttachments(html); + + try { + // Convert to array if it's a string + const recipientsArray = Array.isArray(to) ? to : [to]; + + if (useBatch) { + // Use new batch system for better tracking and user experience + const user = await userProfile.findById(req.body.requestor.requestorId); + if (!user) { + return res.status(400).send('User not found'); + } + + // Create batch for this email send (this already adds recipients internally) + console.log('📧 Creating batch for email send...'); + const batch = await EmailBatchService.createSingleSendBatch( + { + to: recipientsArray, + subject, + html: processedHtml, + attachments, + }, + user, + ); + + console.log('✅ Batch created with recipients:', batch.batchId); + + // Start processing the batch + console.log('🚀 Starting batch processing...'); + emailBatchProcessor.processBatch(batch.batchId).catch((error) => { + console.error('❌ Error processing batch:', error); + }); + + // Get dynamic counts for response + const counts = await batch.getEmailCounts(); + + res.status(200).json({ + success: true, + message: `Email batch created successfully for ${recipientsArray.length} recipient(s)`, + data: { + batchId: batch.batchId, + status: batch.status, + subject: batch.subject, + recipients: recipientsArray, + ...counts, + createdAt: batch.createdAt, + }, + }); + } else { + // Legacy direct sending (fallback) + if (recipientsArray.length === 1) { + // Single recipient - use TO field + await emailSender(to, subject, processedHtml, attachments); + } else { + // Multiple recipients - use BCC to hide recipient list + // Send to self (sender) as primary recipient, then BCC all actual recipients + const senderEmail = req.body.fromEmail || 'updates@onecommunityglobal.org'; + await emailSender( + senderEmail, + subject, + processedHtml, + attachments, + null, + null, + recipientsArray, + ); + } + + res.status(200).send(`Email sent successfully to ${recipientsArray.length} recipient(s)`); + } + } catch (emailError) { + console.error('Error sending email:', emailError); + res.status(500).send('Error sending email'); + } } catch (error) { return res.status(500).send('Error sending email'); } @@ -94,42 +164,146 @@ const sendEmailToAll = async (req, res) => { return; } try { - const { subject, html } = req.body; + const { subject, html, useBatch = true } = req.body; if (!subject || !html) { return res.status(400).send('Subject and HTML content are required'); } const { html: processedHtml, attachments } = extractImagesAndCreateAttachments(html); + if (useBatch) { + // Use new batch system for broadcast emails + const user = await userProfile.findById(req.body.requestor.requestorId); + if (!user) { + return res.status(400).send('User not found'); + } + + // Get all recipients + const users = await userProfile.find({ + firstName: { $ne: '' }, + email: { $ne: null }, + isActive: true, + emailSubscriptions: true, + }); + + const emailSubscribers = await EmailSubcriptionList.find({ + email: { $exists: true, $ne: '' }, + isConfirmed: true, + emailSubscriptions: true, + }); + + const totalRecipients = users.length + emailSubscribers.length; + console.log('# sendEmailToAll total recipients:', totalRecipients); + + if (totalRecipients === 0) { + return res.status(400).send('No recipients found'); + } + + // Create batch for broadcast + const batch = await EmailBatchService.createBatch({ + name: `Broadcast - ${subject}`, + description: `Broadcast email to all subscribers (${totalRecipients} recipients)`, + createdBy: user._id, + createdByName: `${user.firstName} ${user.lastName}`, + createdByEmail: user.email, + subject, + htmlContent: processedHtml, + attachments, + metadata: { + type: 'broadcast', + originalRequest: req.body, + priority: 'NORMAL', + }, + }); + + // Add HGN users + if (users.length > 0) { + const hgnRecipients = users.map((hgnUser) => ({ + email: hgnUser.email, + name: `${hgnUser.firstName} ${hgnUser.lastName}`, + personalizedContent: handleContentToOC(processedHtml), + emailType: 'TO', + tags: ['hgn_user'], + })); + await EmailBatchService.addRecipients(batch.batchId, hgnRecipients); + } + + // Add email subscribers + if (emailSubscribers.length > 0) { + const subscriberRecipients = emailSubscribers.map((subscriber) => ({ + email: subscriber.email, + personalizedContent: handleContentToNonOC(processedHtml, subscriber.email), + emailType: 'TO', + tags: ['email_subscriber'], + })); + await EmailBatchService.addRecipients(batch.batchId, subscriberRecipients); + } + + // Start processing the batch + emailBatchProcessor.processBatch(batch.batchId).catch((error) => { + console.error('Error processing broadcast batch:', error); + }); + + // Get dynamic counts for response + const counts = await batch.getEmailCounts(); + + return res.status(200).json({ + success: true, + message: `Broadcast email batch created successfully for ${totalRecipients} recipient(s)`, + data: { + batchId: batch.batchId, + status: batch.status, + subject: batch.subject, + recipients: { + hgnUsers: users.length, + emailSubscribers: emailSubscribers.length, + total: totalRecipients, + }, + ...counts, + createdBy: batch.createdBy, + createdAt: batch.createdAt, + estimatedCompletion: new Date(Date.now() + totalRecipients * 2000), // 2 seconds per email estimate + }, + }); + } + // Legacy direct sending (fallback) + // HGN Users logic const users = await userProfile.find({ - firstName: '', + firstName: { $ne: '' }, email: { $ne: null }, isActive: true, emailSubscriptions: true, }); - if (users.length === 0) { - return res.status(404).send('No users found'); - } - const recipientEmails = users.map((user) => user.email); - console.log('# sendEmailToAll to', recipientEmails.join(',')); - if (recipientEmails.length === 0) { - throw new Error('No recipients defined'); + if (users.length > 0) { + const recipientEmails = users.map((user) => user.email); + console.log('# sendEmailToAll to HGN users:', recipientEmails.length); + const emailContentToOCmembers = handleContentToOC(processedHtml); + await Promise.all( + recipientEmails.map((email) => + emailSender(email, subject, emailContentToOCmembers, attachments), + ), + ); + } else { + console.log('# sendEmailToAll: No HGN users found with email subscriptions'); } - const emailContentToOCmembers = handleContentToOC(processedHtml); - await Promise.all( - recipientEmails.map((email) => - emailSender(email, subject, emailContentToOCmembers, attachments), - ), - ); - const emailSubscribers = await EmailSubcriptionList.find({ email: { $exists: true, $ne: '' } }); + const emailSubscribers = await EmailSubcriptionList.find({ + email: { $exists: true, $ne: '' }, + isConfirmed: true, + emailSubscriptions: true, + }); console.log('# sendEmailToAll emailSubscribers', emailSubscribers.length); - await Promise.all( - emailSubscribers.map(({ email }) => { - const emailContentToNonOCmembers = handleContentToNonOC(processedHtml, email); - return emailSender(email, subject, emailContentToNonOCmembers, attachments); - }), - ); + + if (emailSubscribers.length > 0) { + await Promise.all( + emailSubscribers.map(({ email }) => { + const emailContentToNonOCmembers = handleContentToNonOC(processedHtml, email); + return emailSender(email, subject, emailContentToNonOCmembers, attachments); + }), + ); + } else { + console.log('# sendEmailToAll: No confirmed email subscribers found'); + } return res.status(200).send('Email sent successfully'); } catch (error) { console.error('Error sending email:', error); @@ -165,26 +339,38 @@ const addNonHgnEmailSubscription = async (req, res) => { return res.status(400).send('Email already exists'); } - // Save to DB immediately - const newEmailList = new EmailSubcriptionList({ email }); + // Save to DB immediately with confirmation pending + const newEmailList = new EmailSubcriptionList({ + email, + isConfirmed: false, + emailSubscriptions: true, + }); await newEmailList.save(); // Optional: Still send confirmation email const payload = { email }; - const token = jwt.sign(payload, jwtSecret, { expiresIn: 360 }); + const token = jwt.sign(payload, jwtSecret, { expiresIn: '360' }); const emailContent = `

Thank you for subscribing to our email updates!

-

Click here to confirm your email

+

Click here to confirm your email

`; - emailSender(email, 'HGN Email Subscription', emailContent); - return res.status(200).send('Email subscribed successfully'); + try { + await emailSender(email, 'HGN Email Subscription', emailContent); + return res.status(200).send('Email subscribed successfully'); + } catch (emailError) { + console.error('Error sending confirmation email:', emailError); + // Still return success since the subscription was saved to DB + return res + .status(200) + .send('Email subscribed successfully (confirmation email failed to send)'); + } } catch (error) { console.error('Error adding email subscription:', error); res.status(500).send('Error adding email subscription'); @@ -209,15 +395,29 @@ const confirmNonHgnEmailSubscription = async (req, res) => { return res.status(400).send('Invalid token'); } try { - const newEmailList = new EmailSubcriptionList({ email }); - await newEmailList.save(); + // Update existing subscription to confirmed, or create new one + const existingSubscription = await EmailSubcriptionList.findOne({ email }); + if (existingSubscription) { + existingSubscription.isConfirmed = true; + existingSubscription.confirmedAt = new Date(); + existingSubscription.emailSubscriptions = true; + await existingSubscription.save(); + } else { + const newEmailList = new EmailSubcriptionList({ + email, + isConfirmed: true, + confirmedAt: new Date(), + emailSubscriptions: true, + }); + await newEmailList.save(); + } } catch (error) { if (error.code === 11000) { return res.status(200).send('Email already exists'); } } // console.log('email', email); - return res.status(200).send('Email subsribed successfully'); + return res.status(200).send('Email subscribed successfully'); } catch (error) { console.error('Error updating email subscriptions:', error); return res.status(500).send('Error updating email subscriptions'); @@ -233,7 +433,7 @@ const removeNonHgnEmailSubscription = async (req, res) => { return res.status(400).send('Email is required'); } - // Try to delete the email + // Try to delete the email subscription completely const deletedEntry = await EmailSubcriptionList.findOneAndDelete({ email: { $eq: email }, }); @@ -243,7 +443,7 @@ const removeNonHgnEmailSubscription = async (req, res) => { return res.status(404).send('Email not found or already unsubscribed'); } - return res.status(200).send('Email unsubscribed successfully'); + return res.status(200).send('Email unsubscribed and removed from subscription list'); } catch (error) { return res.status(500).send('Server error while unsubscribing'); } diff --git a/src/controllers/emailTemplateController.js b/src/controllers/emailTemplateController.js new file mode 100644 index 000000000..c47ef2e35 --- /dev/null +++ b/src/controllers/emailTemplateController.js @@ -0,0 +1,511 @@ +const EmailTemplate = require('../models/EmailTemplateModel'); + +// Get all email templates with pagination and optimization +exports.getAllEmailTemplates = async (req, res) => { + try { + const { search, page, limit, sortBy, sortOrder, fields, includeVariables } = req.query; + + const query = {}; + const sort = {}; + + // Parse pagination parameters - let frontend decide defaults + const pageNum = page ? Math.max(1, parseInt(page, 10)) : 1; + const limitNum = limit ? parseInt(limit, 10) : null; // No restrictions, let frontend decide + const skip = limitNum && pageNum ? (pageNum - 1) * limitNum : 0; + + // Add search functionality with text index + if (search && search.trim()) { + query.$or = [ + { name: { $regex: search.trim(), $options: 'i' } }, + { subject: { $regex: search.trim(), $options: 'i' } }, + ]; + } + + // No filtering - removed variable filtering as requested + + // Build sort object - let frontend decide sort field and order + if (sortBy) { + const sortDirection = sortOrder === 'desc' ? -1 : 1; + sort[sortBy] = sortDirection; + } else { + // Default sort only if frontend doesn't specify + sort.created_at = -1; + } + + // Execute optimized query with pagination + let queryBuilder = EmailTemplate.find(query); + + // Let components decide which fields to include + if (fields) { + // Parse comma-separated fields and always include _id + const fieldList = fields.split(',').map((field) => field.trim()); + if (!fieldList.includes('_id')) { + fieldList.unshift('_id'); + } + queryBuilder = queryBuilder.select(fieldList.join(' ')); + } else if (includeVariables === 'true') { + // Include all fields including variables if requested + // Don't use select('') as it excludes all fields, use no select() to include all + } else { + // Default minimal fields for list view + queryBuilder = queryBuilder.select('_id name created_at updated_at created_by updated_by'); + } + + // Populate user fields if they're in the selection + if (includeVariables === 'true' || !fields || fields.includes('created_by')) { + queryBuilder = queryBuilder.populate('created_by', 'firstName lastName'); + } + if (includeVariables === 'true' || !fields || fields.includes('updated_by')) { + queryBuilder = queryBuilder.populate('updated_by', 'firstName lastName'); + } + + queryBuilder = queryBuilder.sort(sort).skip(skip); + + // Only apply limit if specified + if (limitNum) { + queryBuilder = queryBuilder.limit(limitNum); + } + + const [templates, totalCount] = await Promise.all([ + queryBuilder.lean(), // Use lean() for better performance + EmailTemplate.countDocuments(query), + ]); + + // Transform templates based on what components requested + let processedTemplates; + if (includeVariables === 'true') { + // Return full template data including variables + processedTemplates = templates.map((template) => ({ + _id: template._id, + name: template.name, + subject: template.subject, + content: template.content, + variables: template.variables || [], + created_by: template.created_by, + updated_by: template.updated_by, + created_at: template.created_at, + updated_at: template.updated_at, + })); + } else if (fields) { + // Return only requested fields + processedTemplates = templates.map((template) => { + const fieldList = fields.split(',').map((field) => field.trim()); + const result = { _id: template._id }; + fieldList.forEach((field) => { + if (template[field] !== undefined) { + result[field] = template[field]; + } + }); + return result; + }); + } else { + // Default minimal fields for list view + processedTemplates = templates.map((template) => ({ + _id: template._id, + name: template.name, + created_by: template.created_by, + updated_by: template.updated_by, + created_at: template.created_at, + updated_at: template.updated_at, + })); + } + + // Calculate pagination info + const totalPages = limitNum ? Math.ceil(totalCount / limitNum) : 1; + const hasNextPage = limitNum ? pageNum < totalPages : false; + const hasPrevPage = pageNum > 1; + + res.status(200).json({ + success: true, + templates: processedTemplates, + pagination: { + currentPage: pageNum, + totalPages, + totalCount, + limit: limitNum, + hasNextPage, + hasPrevPage, + }, + }); + } catch (error) { + console.error('Error fetching email templates:', error); + res.status(500).json({ + success: false, + message: 'Error fetching email templates.', + error: error.message, + }); + } +}; + +// Get a single email template by ID +exports.getEmailTemplateById = async (req, res) => { + try { + const { id } = req.params; + const template = await EmailTemplate.findById(id) + .populate('created_by', 'firstName lastName') + .populate('updated_by', 'firstName lastName'); + + if (!template) { + return res.status(404).json({ + success: false, + message: 'Email template not found.', + }); + } + + res.status(200).json({ + success: true, + template, + }); + } catch (error) { + console.error('Error fetching email template:', error); + res.status(500).json({ + success: false, + message: 'Error fetching email template.', + error: error.message, + }); + } +}; + +// Create a new email template +exports.createEmailTemplate = async (req, res) => { + try { + const { name, subject, html_content: htmlContent, variables } = req.body; + const userId = req.body.requestor?.requestorId; + + // Validate required fields + if (!name || !subject || !htmlContent) { + return res.status(400).json({ + success: false, + message: 'Name, subject, and HTML content are required.', + }); + } + + // Check if template with the same name already exists + const existingTemplate = await EmailTemplate.findOne({ name }); + if (existingTemplate) { + return res.status(400).json({ + success: false, + message: 'Email template with this name already exists.', + }); + } + + // Validate variables if provided + if (variables && variables.length > 0) { + const invalidVariable = variables.find((variable) => !variable.name || !variable.label); + if (invalidVariable) { + return res.status(400).json({ + success: false, + message: 'Variable name and label are required for all variables.', + }); + } + } + + // Create new email template + const template = new EmailTemplate({ + name, + subject, + html_content: htmlContent, + variables: variables || [], + created_by: userId, + updated_by: userId, // Set updated_by to same as created_by for new templates + }); + + await template.save(); + + // Populate created_by and updated_by fields for response + await template.populate('created_by', 'firstName lastName'); + await template.populate('updated_by', 'firstName lastName'); + + res.status(201).json({ + success: true, + message: 'Email template created successfully.', + template, + }); + } catch (error) { + console.error('Error creating email template:', error); + res.status(500).json({ + success: false, + message: 'Error creating email template.', + error: error.message, + }); + } +}; + +// Update an email template +exports.updateEmailTemplate = async (req, res) => { + try { + const { id } = req.params; + const { name, subject, html_content: htmlContent, variables } = req.body; + + // Validate required fields + if (!name || !subject || !htmlContent) { + return res.status(400).json({ + success: false, + message: 'Name, subject, and HTML content are required.', + }); + } + + // Get current template to check if name is actually changing + const currentTemplate = await EmailTemplate.findById(id); + if (!currentTemplate) { + return res.status(404).json({ + success: false, + message: 'Email template not found.', + }); + } + + // Only check for duplicate names if the name is actually changing + if (currentTemplate.name !== name) { + const existingTemplate = await EmailTemplate.findOne({ + name, + _id: { $ne: id }, + }); + if (existingTemplate) { + return res.status(400).json({ + success: false, + message: 'Another email template with this name already exists.', + }); + } + } + + // Validate variables if provided + if (variables && variables.length > 0) { + const invalidVariable = variables.find((variable) => !variable.name || !variable.label); + if (invalidVariable) { + return res.status(400).json({ + success: false, + message: 'Variable name and label are required for all variables.', + }); + } + } + + // Update template + const updateData = { + name, + subject, + html_content: htmlContent, + variables: variables || [], + updated_by: req.body.requestor?.requestorId, // Set who updated the template + }; + + const template = await EmailTemplate.findByIdAndUpdate(id, updateData, { + new: true, + runValidators: true, + }) + .populate('created_by', 'firstName lastName') + .populate('updated_by', 'firstName lastName'); + + if (!template) { + return res.status(404).json({ + success: false, + message: 'Email template not found.', + }); + } + + res.status(200).json({ + success: true, + message: 'Email template updated successfully.', + template, + }); + } catch (error) { + console.error('Error updating email template:', error); + res.status(500).json({ + success: false, + message: 'Error updating email template.', + error: error.message, + }); + } +}; + +// Delete an email template +exports.deleteEmailTemplate = async (req, res) => { + try { + const { id } = req.params; + const template = await EmailTemplate.findByIdAndDelete(id); + + if (!template) { + return res.status(404).json({ + success: false, + message: 'Email template not found.', + }); + } + + res.status(200).json({ + success: true, + message: 'Email template deleted successfully.', + }); + } catch (error) { + console.error('Error deleting email template:', error); + res.status(500).json({ + success: false, + message: 'Error deleting email template.', + error: error.message, + }); + } +}; + +// Send email using template +exports.sendEmailWithTemplate = async (req, res) => { + try { + const { id } = req.params; + const { recipients, variableValues, broadcastToAll } = req.body; + + // Validate required fields + if (!broadcastToAll && (!recipients || !Array.isArray(recipients) || recipients.length === 0)) { + return res.status(400).json({ + success: false, + message: 'Recipients array is required when not broadcasting to all.', + }); + } + + // Get template + const template = await EmailTemplate.findById(id); + if (!template) { + return res.status(404).json({ + success: false, + message: 'Email template not found.', + }); + } + + // Validate all variables (since all are required by default) + const missingVariable = template.variables.find( + (variable) => !variableValues || !variableValues[variable.name], + ); + if (missingVariable) { + return res.status(400).json({ + success: false, + message: `Variable '${missingVariable.label}' is missing.`, + }); + } + + // Replace variables in subject and content + let processedSubject = template.subject; + let processedContent = template.html_content; + + if (variableValues) { + Object.entries(variableValues).forEach(([varName, varValue]) => { + const regex = new RegExp(`{{${varName}}}`, 'g'); + processedSubject = processedSubject.replace(regex, varValue); + processedContent = processedContent.replace(regex, varValue); + }); + } + + if (broadcastToAll) { + // Use existing broadcast functionality + const { sendEmailToAll } = require('./emailController'); + + // Create a mock request object for sendEmailToAll + const mockReq = { + body: { + requestor: req.body.requestor || req.user, // Pass the user making the request + subject: processedSubject, + html: processedContent, + }, + }; + + // Create a mock response object to capture the result + let broadcastResult = null; + const mockRes = { + status: (code) => ({ + send: (message) => { + broadcastResult = { code, message }; + }, + }), + }; + + await sendEmailToAll(mockReq, mockRes); + + if (broadcastResult && broadcastResult.code === 200) { + res.status(200).json({ + success: true, + message: 'Email template broadcasted successfully to all users.', + broadcasted: true, + }); + } else { + res.status(broadcastResult?.code || 500).json({ + success: false, + message: broadcastResult?.message || 'Error broadcasting email template.', + }); + } + } else { + // Send to specific recipients using batch system + try { + const EmailBatchService = require('../services/emailBatchService'); + const emailBatchProcessor = require('../services/emailBatchProcessor'); + const userProfile = require('../models/userProfile'); + + // Get user information + const user = await userProfile.findById(req.body.requestor.requestorId); + if (!user) { + return res.status(400).json({ + success: false, + message: 'User not found', + }); + } + + // Create batch for template email (this already adds recipients internally) + console.log('📧 Creating batch for template email...'); + const batch = await EmailBatchService.createSingleSendBatch( + { + to: recipients, + subject: processedSubject, + html: processedContent, + attachments: null, + }, + user, + ); + + console.log('✅ Template batch created with recipients:', batch.batchId); + console.log('📊 Batch details:', { + id: batch._id, + batchId: batch.batchId, + status: batch.status, + createdBy: batch.createdBy, + }); + + // Start processing the batch + console.log('🚀 Starting template batch processing...'); + emailBatchProcessor.processBatch(batch.batchId).catch((error) => { + console.error('❌ Error processing template batch:', error); + }); + + // Get dynamic counts for response + const counts = await batch.getEmailCounts(); + + res.status(200).json({ + success: true, + message: `Email template batch created successfully for ${recipients.length} recipient(s).`, + data: { + batchId: batch.batchId, + status: batch.status, + subject: batch.subject, + recipients, + ...counts, + template: { + id: template._id, + name: template.name, + subject: template.subject, + }, + createdBy: batch.createdBy, + createdAt: batch.createdAt, + estimatedCompletion: new Date(Date.now() + recipients.length * 2000), // 2 seconds per email estimate + }, + }); + } catch (emailError) { + console.error('Error creating template batch:', emailError); + res.status(500).json({ + success: false, + message: 'Error creating template batch.', + error: emailError.message, + }); + } + } + } catch (error) { + console.error('Error in sendEmailWithTemplate:', error); + res.status(500).json({ + success: false, + message: 'Error processing email template.', + error: error.message, + }); + } +}; diff --git a/src/models/EmailTemplateModel.js b/src/models/EmailTemplateModel.js new file mode 100644 index 000000000..be392b8db --- /dev/null +++ b/src/models/EmailTemplateModel.js @@ -0,0 +1,73 @@ +const mongoose = require('mongoose'); + +const emailTemplateSchema = new mongoose.Schema( + { + name: { + type: String, + required: true, + unique: true, + trim: true, + }, + subject: { + type: String, + required: true, + trim: true, + }, + html_content: { + type: String, + required: true, + }, + variables: [ + { + name: { + type: String, + required: true, + trim: true, + }, + label: { + type: String, + required: true, + trim: true, + }, + type: { + type: String, + enum: ['text', 'url', 'number', 'textarea', 'image'], + default: 'text', + }, + }, + ], + created_by: { + type: mongoose.Schema.Types.ObjectId, + ref: 'userProfile', + }, + updated_by: { + type: mongoose.Schema.Types.ObjectId, + ref: 'userProfile', + }, + }, + { + timestamps: { + createdAt: 'created_at', + updatedAt: 'updated_at', + }, + }, +); + +// Indexes for better search performance +emailTemplateSchema.index({ name: 1 }); +emailTemplateSchema.index({ created_at: -1 }); +emailTemplateSchema.index({ updated_at: -1 }); +emailTemplateSchema.index({ created_by: 1 }); +emailTemplateSchema.index({ updated_by: 1 }); + +// Text index for full-text search +emailTemplateSchema.index({ + name: 'text', + subject: 'text', +}); + +// Compound indexes for common queries +emailTemplateSchema.index({ created_by: 1, created_at: -1 }); +emailTemplateSchema.index({ name: 1, created_at: -1 }); + +module.exports = mongoose.model('EmailTemplate', emailTemplateSchema); diff --git a/src/models/emailBatch.js b/src/models/emailBatch.js new file mode 100644 index 000000000..b092cf6fd --- /dev/null +++ b/src/models/emailBatch.js @@ -0,0 +1,126 @@ +/** + * Simplified Email Batch Model - Production Ready + * Focus: Store batch information with email body for efficiency + */ + +const mongoose = require('mongoose'); + +const { Schema } = mongoose; + +const EmailBatchSchema = new Schema({ + // Core identification + batchId: { type: String, required: true, unique: true, index: true }, + subject: { type: String, required: true }, + htmlContent: { type: String, required: true }, // Store email body in batch + + // Status tracking + status: { + type: String, + enum: ['PENDING', 'PROCESSING', 'COMPLETED', 'FAILED'], + default: 'PENDING', + index: true, + }, + + // Creator reference only + createdBy: { type: Schema.Types.ObjectId, ref: 'userProfile', required: true }, + + // Timing + createdAt: { type: Date, default: Date.now, index: true }, + startedAt: Date, + completedAt: Date, + updatedAt: { type: Date, default: Date.now }, +}); + +// Update timestamps +EmailBatchSchema.pre('save', function (next) { + this.updatedAt = new Date(); + next(); +}); + +// Calculate email counts dynamically from batch items with multiple recipients +EmailBatchSchema.methods.getEmailCounts = async function () { + const EmailBatchItem = require('./emailBatchItem'); + + const counts = await EmailBatchItem.aggregate([ + { $match: { batchId: this._id } }, + { + $group: { + _id: null, + total: { $sum: { $cond: [{ $isArray: '$recipients' }, { $size: '$recipients' }, 0] } }, + sent: { + $sum: { + $cond: [ + { $and: [{ $eq: ['$status', 'SENT'] }, { $isArray: '$recipients' }] }, + { $size: '$recipients' }, + 0, + ], + }, + }, + failed: { + $sum: { + $cond: [ + { $and: [{ $eq: ['$status', 'FAILED'] }, { $isArray: '$recipients' }] }, + { $size: '$recipients' }, + 0, + ], + }, + }, + pending: { + $sum: { + $cond: [ + { $and: [{ $in: ['$status', ['PENDING', 'SENDING']] }, { $isArray: '$recipients' }] }, + { $size: '$recipients' }, + 0, + ], + }, + }, + }, + }, + ]); + + if (counts.length > 0) { + const count = counts[0]; + return { + totalEmails: count.total || 0, + sentEmails: count.sent || 0, + failedEmails: count.failed || 0, + pendingEmails: count.pending || 0, + progress: count.total > 0 ? Math.round((count.sent / count.total) * 100) : 0, + }; + } + + return { + totalEmails: 0, + sentEmails: 0, + failedEmails: 0, + pendingEmails: 0, + progress: 0, + }; +}; + +// Update status based on email counts +EmailBatchSchema.methods.updateStatus = async function () { + const counts = await this.getEmailCounts(); + + if (counts.pendingEmails === 0 && counts.totalEmails > 0) { + // All emails processed + if (counts.failedEmails === 0) { + this.status = 'COMPLETED'; + } else if (counts.sentEmails === 0) { + this.status = 'FAILED'; + } else { + this.status = 'COMPLETED'; // Partial success + } + this.completedAt = new Date(); + } else if (counts.totalEmails > 0) { + this.status = 'PROCESSING'; + if (!this.startedAt) { + this.startedAt = new Date(); + } + } + + await this.save(); + return this; +}; + +module.exports = mongoose.model('EmailBatch', EmailBatchSchema, 'emailBatches'); diff --git a/src/models/emailBatchItem.js b/src/models/emailBatchItem.js new file mode 100644 index 000000000..9850922a3 --- /dev/null +++ b/src/models/emailBatchItem.js @@ -0,0 +1,87 @@ +/** + * Simplified Email Batch Item Model - Production Ready + * Focus: Store multiple recipients per batch item, no names or priority + */ + +const mongoose = require('mongoose'); + +const { Schema } = mongoose; + +const EmailBatchItemSchema = new Schema({ + // Batch reference + batchId: { type: Schema.Types.ObjectId, ref: 'EmailBatch', required: true, index: true }, + + // Multiple recipients in one batch item (emails only) + // _id: false prevents MongoDB from generating unnecessary _id fields for each recipient + recipients: [ + { + _id: false, // Prevent MongoDB from generating _id for each recipient + email: { type: String, required: true }, + }, + ], + + // Email type for the batch item + emailType: { + type: String, + enum: ['TO', 'CC', 'BCC'], + default: 'BCC', // Use BCC for multiple recipients + }, + + // Status tracking (for the entire batch item) + status: { + type: String, + enum: ['PENDING', 'SENDING', 'SENT', 'FAILED'], + default: 'PENDING', + index: true, + }, + + // Processing info + attempts: { type: Number, default: 0 }, + lastAttemptedAt: Date, + sentAt: Date, + failedAt: Date, + error: String, + + // Timestamps + createdAt: { type: Date, default: Date.now }, + updatedAt: { type: Date, default: Date.now }, +}); + +// Update timestamps +EmailBatchItemSchema.pre('save', function (next) { + this.updatedAt = new Date(); + next(); +}); + +// Update status with proper attempt tracking +EmailBatchItemSchema.methods.updateStatus = async function (newStatus, errorMessage = null) { + this.status = newStatus; + this.lastAttemptedAt = new Date(); + + // Only increment attempts for actual sending attempts (SENDING status) + if (newStatus === 'SENDING') { + this.attempts += 1; + } + + if (newStatus === 'SENT') { + this.sentAt = new Date(); + this.failedAt = null; + this.error = null; + } else if (newStatus === 'FAILED') { + this.failedAt = new Date(); + this.error = errorMessage; + } + + await this.save(); + + // Update parent batch status + const EmailBatch = require('./emailBatch'); + const batch = await EmailBatch.findById(this.batchId); + if (batch) { + await batch.updateStatus(); + } + + return this; +}; + +module.exports = mongoose.model('EmailBatchItem', EmailBatchItemSchema, 'emailBatchItems'); diff --git a/src/models/emailSubcriptionList.js b/src/models/emailSubcriptionList.js index 6fbc0804a..f28feaf13 100644 --- a/src/models/emailSubcriptionList.js +++ b/src/models/emailSubcriptionList.js @@ -1,5 +1,5 @@ /* eslint-disable quotes */ -const mongoose = require("mongoose"); +const mongoose = require('mongoose'); const { Schema } = mongoose; @@ -9,6 +9,22 @@ const emailSubscriptionSchema = new Schema({ type: Boolean, default: true, }, + isConfirmed: { + type: Boolean, + default: false, + }, + subscribedAt: { + type: Date, + default: Date.now, + }, + confirmedAt: { + type: Date, + default: null, + }, }); -module.exports = mongoose.model("emailSubscriptions", emailSubscriptionSchema, "emailSubscriptions"); +module.exports = mongoose.model( + 'emailSubscriptions', + emailSubscriptionSchema, + 'emailSubscriptions', +); diff --git a/src/routes/emailBatchDashboardRoutes.js b/src/routes/emailBatchDashboardRoutes.js new file mode 100644 index 000000000..99ad2fdb1 --- /dev/null +++ b/src/routes/emailBatchDashboardRoutes.js @@ -0,0 +1,14 @@ +/** + * Simplified Email Batch Dashboard Routes - Production Ready + * Focus: Essential dashboard endpoints only + */ + +const express = require('express'); + +const router = express.Router(); +const emailBatchDashboardController = require('../controllers/emailBatchDashboardController'); + +// Dashboard routes +router.get('/dashboard', emailBatchDashboardController.getDashboardStats); + +module.exports = router; diff --git a/src/routes/emailBatchRoutes.js b/src/routes/emailBatchRoutes.js new file mode 100644 index 000000000..dbe298d7f --- /dev/null +++ b/src/routes/emailBatchRoutes.js @@ -0,0 +1,20 @@ +/** + * Simplified Email Batch Routes - Production Ready + * Focus: Essential endpoints only + */ + +const express = require('express'); + +const router = express.Router(); +const emailBatchController = require('../controllers/emailBatchController'); + +// Batch management routes +router.get('/batches', emailBatchController.getBatches); +router.get('/batches/:batchId', emailBatchController.getBatchDetails); +router.get('/dashboard', emailBatchController.getDashboardStats); +router.get('/status', emailBatchController.getProcessorStatus); + +// Retry operations +router.post('/retry-item/:itemId', emailBatchController.retryBatchItem); + +module.exports = router; diff --git a/src/routes/emailRouter.js b/src/routes/emailRouter.js index 8c520009b..66b75d159 100644 --- a/src/routes/emailRouter.js +++ b/src/routes/emailRouter.js @@ -11,19 +11,13 @@ const { const routes = function () { const emailRouter = express.Router(); - emailRouter.route('/send-emails') - .post(sendEmail); - emailRouter.route('/broadcast-emails') - .post(sendEmailToAll); + emailRouter.route('/send-emails').post(sendEmail); + emailRouter.route('/broadcast-emails').post(sendEmailToAll); - emailRouter.route('/update-email-subscriptions') - .post(updateEmailSubscriptions); - emailRouter.route('/add-non-hgn-email-subscription') - .post(addNonHgnEmailSubscription); - emailRouter.route('/confirm-non-hgn-email-subscription') - .post(confirmNonHgnEmailSubscription); - emailRouter.route('/remove-non-hgn-email-subscription') - .post(removeNonHgnEmailSubscription); + emailRouter.route('/update-email-subscriptions').post(updateEmailSubscriptions); + emailRouter.route('/add-non-hgn-email-subscription').post(addNonHgnEmailSubscription); + emailRouter.route('/confirm-non-hgn-email-subscription').post(confirmNonHgnEmailSubscription); + emailRouter.route('/remove-non-hgn-email-subscription').post(removeNonHgnEmailSubscription); return emailRouter; }; diff --git a/src/routes/emailTemplateRouter.js b/src/routes/emailTemplateRouter.js new file mode 100644 index 000000000..758b43357 --- /dev/null +++ b/src/routes/emailTemplateRouter.js @@ -0,0 +1,14 @@ +const express = require('express'); +const emailTemplateController = require('../controllers/emailTemplateController'); + +const router = express.Router(); + +// Email template routes +router.get('/email-templates', emailTemplateController.getAllEmailTemplates); +router.get('/email-templates/:id', emailTemplateController.getEmailTemplateById); +router.post('/email-templates', emailTemplateController.createEmailTemplate); +router.put('/email-templates/:id', emailTemplateController.updateEmailTemplate); +router.delete('/email-templates/:id', emailTemplateController.deleteEmailTemplate); +router.post('/email-templates/:id/send', emailTemplateController.sendEmailWithTemplate); + +module.exports = router; diff --git a/src/routes/templateRoutes.js b/src/routes/templateRoutes.js new file mode 100644 index 000000000..0ee712653 --- /dev/null +++ b/src/routes/templateRoutes.js @@ -0,0 +1,24 @@ +/** + * Template Routes - API endpoints for template management + */ + +const express = require('express'); + +const router = express.Router(); +const templateController = require('../controllers/templateController'); + +// Template statistics (must come before /:id routes) +router.get('/stats', templateController.getTemplateStats); + +// Template CRUD operations +router.get('/', templateController.getAllTemplates); +router.get('/:id', templateController.getTemplateById); +router.post('/', templateController.createTemplate); +router.put('/:id', templateController.updateTemplate); +router.delete('/:id', templateController.deleteTemplate); + +// Template rendering and sending +router.post('/:id/render', templateController.renderTemplate); +router.post('/:id/send', templateController.sendTemplateEmail); + +module.exports = router; diff --git a/src/server.js b/src/server.js index 17a132336..389b8bc69 100644 --- a/src/server.js +++ b/src/server.js @@ -9,6 +9,9 @@ require('./startup/db')(); require('./cronjobs/userProfileJobs')(); require('./jobs/analyticsAggregation').scheduleDaily(); require('./cronjobs/bidWinnerJobs')(); + +// Email batch system is initialized automatically when needed + const websocketRouter = require('./websockets/webSocketRouter'); const port = process.env.PORT || 4500; diff --git a/src/services/emailBatchDashboardService.js b/src/services/emailBatchDashboardService.js new file mode 100644 index 000000000..7fe8cd69f --- /dev/null +++ b/src/services/emailBatchDashboardService.js @@ -0,0 +1,391 @@ +/** + * Enhanced Email Batch Dashboard Service + * + * FEATURES: + * - Caching for performance + * - Pagination for large datasets + * - Real-time updates + * - Performance analytics + * - Memory optimization + */ + +const EmailBatch = require('../models/emailBatch'); +const EmailBatchItem = require('../models/emailBatchItem'); +const logger = require('../startup/logger'); + +class EmailBatchDashboardService { + constructor() { + this.cache = new Map(); + this.cacheTimeout = 30000; // 30 seconds + this.maxCacheSize = 100; + } + + /** + * Get dashboard data with caching and performance optimization + */ + async getDashboardData(filters = {}) { + const cacheKey = this.generateCacheKey('dashboard', filters); + + // Check cache first + if (this.cache.has(cacheKey)) { + const cached = this.cache.get(cacheKey); + if (Date.now() - cached.timestamp < this.cacheTimeout) { + return cached.data; + } + } + + try { + const data = await this.fetchDashboardData(filters); + + // Cache the result + this.setCache(cacheKey, data); + + return data; + } catch (error) { + logger.logException(error, 'Error fetching dashboard data'); + throw error; + } + } + + /** + * Fetch dashboard data with optimized queries + */ + async fetchDashboardData(filters) { + const query = this.buildQuery(filters); + + // Use aggregation pipeline for better performance + const [overviewStats, emailStats, performanceStats, recentBatches] = await Promise.all([ + this.getOverviewStats(query), + this.getEmailStats(query), + this.getPerformanceStats(query), + this.getRecentBatches(query, 10), + ]); + + return { + overview: overviewStats, + emailStats, + performance: performanceStats, + recentBatches, + filters, + timestamp: new Date(), + }; + } + + /** + * Get overview statistics + */ + static async getOverviewStats(query) { + const stats = await EmailBatch.aggregate([ + { $match: query }, + { + $group: { + _id: null, + totalBatches: { $sum: 1 }, + pendingBatches: { $sum: { $cond: [{ $eq: ['$status', 'PENDING'] }, 1, 0] } }, + processingBatches: { $sum: { $cond: [{ $eq: ['$status', 'PROCESSING'] }, 1, 0] } }, + completedBatches: { $sum: { $cond: [{ $eq: ['$status', 'COMPLETED'] }, 1, 0] } }, + failedBatches: { $sum: { $cond: [{ $eq: ['$status', 'FAILED'] }, 1, 0] } }, + }, + }, + ]); + + return ( + stats[0] || { + totalBatches: 0, + pendingBatches: 0, + processingBatches: 0, + completedBatches: 0, + failedBatches: 0, + } + ); + } + + /** + * Get email statistics + */ + static async getEmailStats(query) { + const stats = await EmailBatch.aggregate([ + { $match: query }, + { + $group: { + _id: null, + totalEmails: { $sum: '$totalEmails' }, + sentEmails: { $sum: '$sentEmails' }, + failedEmails: { $sum: '$failedEmails' }, + pendingEmails: { $sum: '$pendingEmails' }, + }, + }, + ]); + + const result = stats[0] || { + totalEmails: 0, + sentEmails: 0, + failedEmails: 0, + pendingEmails: 0, + }; + + result.successRate = + result.totalEmails > 0 ? Math.round((result.sentEmails / result.totalEmails) * 100) : 0; + + return result; + } + + /** + * Get performance statistics + */ + static async getPerformanceStats(query) { + const stats = await EmailBatch.aggregate([ + { + $match: { + ...query, + status: 'COMPLETED', + startedAt: { $exists: true }, + }, + }, + { + $project: { + processingTime: { $subtract: ['$completedAt', '$startedAt'] }, + totalEmails: 1, + createdAt: 1, + }, + }, + { + $group: { + _id: null, + avgProcessingTime: { $avg: '$processingTime' }, + avgEmailsPerBatch: { $avg: '$totalEmails' }, + totalProcessingTime: { $sum: '$processingTime' }, + batchCount: { $sum: 1 }, + }, + }, + ]); + + const result = stats[0] || { + avgProcessingTime: null, + avgEmailsPerBatch: 0, + totalProcessingTime: 0, + batchCount: 0, + }; + + return { + avgProcessingTime: result.avgProcessingTime + ? Math.round(result.avgProcessingTime / 1000) + : null, + avgEmailsPerBatch: Math.round(result.avgEmailsPerBatch || 0), + totalProcessingTime: result.totalProcessingTime + ? Math.round(result.totalProcessingTime / 1000) + : 0, + batchCount: result.batchCount, + }; + } + + /** + * Get recent batches with pagination + */ + static async getRecentBatches(query, limit = 10) { + const batches = await EmailBatch.find(query) + .sort({ createdAt: -1 }) + .limit(limit) + .populate('createdBy', 'firstName lastName email') + .select( + 'batchId name status totalEmails sentEmails failedEmails progress createdAt completedAt subject createdBy', + ); + + return batches.map((batch) => ({ + batchId: batch.batchId, + name: batch.name, + status: batch.status, + totalEmails: batch.totalEmails, + sentEmails: batch.sentEmails, + failedEmails: batch.failedEmails, + progress: batch.progress, + createdAt: batch.createdAt, + completedAt: batch.completedAt, + subject: batch.subject, + createdBy: batch.createdBy, + })); + } + + /** + * Get batch details with pagination for large batches + */ + async getBatchDetails(batchId, page = 1, limit = 100) { + const cacheKey = this.generateCacheKey('batchDetails', { batchId, page, limit }); + + if (this.cache.has(cacheKey)) { + const cached = this.cache.get(cacheKey); + if (Date.now() - cached.timestamp < this.cacheTimeout) { + return cached.data; + } + } + + try { + const batch = await EmailBatch.findOne({ batchId }).populate( + 'createdBy', + 'firstName lastName email', + ); + + if (!batch) { + throw new Error('Batch not found'); + } + + // Get batch items with pagination + const skip = (page - 1) * limit; + const [batchItems, totalItems] = await Promise.all([ + EmailBatchItem.find({ batchId: batch._id }) + .select('recipients status attempts sentAt failedAt error createdAt') + .sort({ createdAt: 1 }) + .limit(limit) + .skip(skip), + EmailBatchItem.countDocuments({ batchId: batch._id }), + ]); + + const data = { + batch: { + batchId: batch.batchId, + name: batch.name, + description: batch.description, + status: batch.status, + subject: batch.subject, + htmlContent: batch.htmlContent, + attachments: batch.attachments || [], + metadata: batch.metadata, + createdBy: batch.createdBy, + createdByName: batch.createdByName, + createdByEmail: batch.createdByEmail, + createdAt: batch.createdAt, + updatedAt: batch.updatedAt, + startedAt: batch.startedAt, + completedAt: batch.completedAt, + }, + statistics: { + totalEmails: batch.totalEmails, + sentEmails: batch.sentEmails, + failedEmails: batch.failedEmails, + pendingEmails: batch.pendingEmails, + progress: batch.progress, + }, + items: batchItems, + pagination: { + page, + limit, + total: totalItems, + pages: Math.ceil(totalItems / limit), + }, + }; + + this.setCache(cacheKey, data); + return data; + } catch (error) { + logger.logException(error, 'Error fetching batch details'); + throw error; + } + } + + /** + * Get batches with advanced filtering and pagination + */ + async getBatches(filters = {}, page = 1, limit = 20) { + const query = this.buildQuery(filters); + const skip = (page - 1) * limit; + + const [batches, total] = await Promise.all([ + EmailBatch.find(query) + .sort({ createdAt: -1 }) + .limit(limit) + .skip(skip) + .populate('createdBy', 'firstName lastName email') + .select( + 'batchId name status totalEmails sentEmails failedEmails progress createdAt completedAt subject createdBy', + ), + EmailBatch.countDocuments(query), + ]); + + return { + batches: batches.map((batch) => ({ + batchId: batch.batchId, + name: batch.name, + status: batch.status, + totalEmails: batch.totalEmails, + sentEmails: batch.sentEmails, + failedEmails: batch.failedEmails, + progress: batch.progress, + createdAt: batch.createdAt, + completedAt: batch.completedAt, + subject: batch.subject, + createdBy: batch.createdBy, + })), + pagination: { + page, + limit, + total, + pages: Math.ceil(total / limit), + }, + }; + } + + /** + * Build query from filters + */ + static buildQuery(filters) { + const query = {}; + + if (filters.dateFrom) { + query.createdAt = { $gte: new Date(filters.dateFrom) }; + } + if (filters.dateTo) { + query.createdAt = { ...query.createdAt, $lte: new Date(filters.dateTo) }; + } + if (filters.status) { + query.status = filters.status; + } + if (filters.type) { + query['metadata.type'] = filters.type; + } + if (filters.createdBy) { + query.createdBy = filters.createdBy; + } + + return query; + } + + /** + * Cache management + */ + static generateCacheKey(prefix, params) { + return `${prefix}_${JSON.stringify(params)}`; + } + + setCache(key, data) { + // Implement LRU cache + if (this.cache.size >= this.maxCacheSize) { + const firstKey = this.cache.keys().next().value; + this.cache.delete(firstKey); + } + + this.cache.set(key, { + data, + timestamp: Date.now(), + }); + } + + /** + * Clear cache + */ + clearCache() { + this.cache.clear(); + } + + /** + * Get cache statistics + */ + getCacheStats() { + return { + size: this.cache.size, + maxSize: this.maxCacheSize, + timeout: this.cacheTimeout, + }; + } +} + +module.exports = new EmailBatchDashboardService(); diff --git a/src/services/emailBatchProcessor.js b/src/services/emailBatchProcessor.js new file mode 100644 index 000000000..28d9be05b --- /dev/null +++ b/src/services/emailBatchProcessor.js @@ -0,0 +1,197 @@ +/** + * Simplified Email Batch Processor - Production Ready + * Focus: Efficient processing with email body from batch record + */ + +const EmailBatch = require('../models/emailBatch'); +const EmailBatchItem = require('../models/emailBatchItem'); +const emailSender = require('../utilities/emailSender'); +const logger = require('../startup/logger'); + +class EmailBatchProcessor { + constructor() { + this.processingBatches = new Set(); + this.maxRetries = 3; + this.retryDelay = 2000; // 2 seconds + } + + /** + * Process a batch + */ + async processBatch(batchId) { + if (this.processingBatches.has(batchId)) { + return; // Already processing + } + + this.processingBatches.add(batchId); + + try { + console.log('🔍 Looking for batch with batchId:', batchId); + const batch = await EmailBatch.findOne({ batchId }); + if (!batch) { + console.error('❌ Batch not found with batchId:', batchId); + throw new Error('Batch not found'); + } + console.log('✅ Found batch:', batch.batchId, 'Status:', batch.status); + + if (batch.status === 'COMPLETED' || batch.status === 'FAILED') { + return; + } + + // Update batch status + batch.status = 'PROCESSING'; + batch.startedAt = new Date(); + await batch.save(); + + // Process batch items (each item contains multiple recipients) + await this.processBatchItems(batch); + + // Update final status + await batch.updateStatus(); + logger.logInfo(`Batch ${batchId} processed successfully`); + } catch (error) { + logger.logException(error, `Error processing batch ${batchId}`); + + // Mark batch as failed + try { + const batch = await EmailBatch.findOne({ batchId }); + if (batch) { + batch.status = 'FAILED'; + batch.completedAt = new Date(); + await batch.save(); + } + } catch (updateError) { + logger.logException(updateError, 'Error updating batch status to failed'); + } + } finally { + this.processingBatches.delete(batchId); + } + } + + /** + * Process all items in a batch + */ + async processBatchItems(batch) { + const items = await EmailBatchItem.find({ + batchId: batch._id, + status: 'PENDING', + }); + + // Process items in parallel with concurrency limit + const processPromises = items.map((item) => this.processItem(item, batch)); + await Promise.all(processPromises); + } + + /** + * Process a single batch item with multiple recipients + */ + async processItem(item, batch) { + const processWithRetry = async (attempt = 1) => { + try { + // Update to SENDING status (this increments attempts) + await item.updateStatus('SENDING'); + + // Extract recipient emails from the batch item + const recipientEmails = item.recipients.map((recipient) => recipient.email); + + // Use the existing emailSender with batched recipients and email body from batch + await emailSender( + recipientEmails, // Array of emails for batching + batch.subject, + batch.htmlContent, // Use email body from batch record + null, // attachments + null, // cc + null, // replyTo + null, // bcc + { + type: 'batch_send', + batchId: batch.batchId, + itemId: item._id, + emailType: item.emailType, + recipientCount: recipientEmails.length, + }, + ); + + // Mark as sent + await item.updateStatus('SENT'); + logger.logInfo( + `Email batch sent successfully to ${recipientEmails.length} recipients (attempt ${item.attempts})`, + ); + // Success + } catch (error) { + logger.logException( + error, + `Failed to send email batch to ${item.recipients.length} recipients (attempt ${attempt})`, + ); + + if (attempt >= this.maxRetries) { + await item.updateStatus('FAILED', error.message); + logger.logError( + `Permanently failed to send email batch to ${item.recipients.length} recipients after ${this.maxRetries} attempts`, + ); + return; + } + + // Wait before retry + await EmailBatchProcessor.sleep(this.retryDelay); + return processWithRetry(attempt + 1); + } + }; + + return processWithRetry(); + } + + /** + * Sleep utility + */ + static sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); + } + + /** + * Retry a specific batch item + */ + async retryBatchItem(itemId) { + try { + const item = await EmailBatchItem.findById(itemId); + if (!item) { + throw new Error('Batch item not found'); + } + + const batch = await EmailBatch.findById(item.batchId); + if (!batch) { + throw new Error('Parent batch not found'); + } + + // Process the specific item + await this.processItem(item, batch); + + return { + success: true, + itemId: item._id, + status: item.status, + }; + } catch (error) { + logger.logException(error, 'Error retrying batch item'); + throw error; + } + } + + /** + * Get processor status + */ + getStatus() { + return { + isRunning: true, + processingBatches: Array.from(this.processingBatches), + maxRetries: this.maxRetries, + }; + } +} + +// Create singleton instance +const emailBatchProcessor = new EmailBatchProcessor(); + +module.exports = emailBatchProcessor; diff --git a/src/services/emailBatchService.js b/src/services/emailBatchService.js new file mode 100644 index 000000000..9a8d3502d --- /dev/null +++ b/src/services/emailBatchService.js @@ -0,0 +1,286 @@ +/** + * Simplified Email Batch Service - Production Ready + * Focus: Efficient batching with email body storage + */ + +const { v4: uuidv4 } = require('uuid'); +const EmailBatch = require('../models/emailBatch'); +const EmailBatchItem = require('../models/emailBatchItem'); +const logger = require('../startup/logger'); + +class EmailBatchService { + constructor() { + this.batchSize = 50; // Match emailSender batch size for efficiency + } + + /** + * Create a new email batch with email body + */ + static async createBatch(batchData) { + try { + const batch = new EmailBatch({ + batchId: batchData.batchId || uuidv4(), + subject: batchData.subject, + htmlContent: batchData.htmlContent, // Store email body + createdBy: batchData.createdBy, + }); + + await batch.save(); + console.log('💾 Batch saved successfully:', { + id: batch._id, + batchId: batch.batchId, + status: batch.status, + }); + return batch; + } catch (error) { + logger.logException(error, 'Error creating batch'); + throw error; + } + } + + /** + * Add recipients to a batch with efficient batching + */ + static async addRecipients(batchId, recipients, batchConfig = {}) { + try { + const batch = await EmailBatch.findOne({ batchId }); + if (!batch) { + throw new Error('Batch not found'); + } + + const batchSize = batchConfig.batchSize || 50; + const emailType = batchConfig.emailType || 'BCC'; + + // Create batch items with multiple recipients per item + const batchItems = []; + + for (let i = 0; i < recipients.length; i += batchSize) { + const recipientChunk = recipients.slice(i, i + batchSize); + + const batchItem = { + batchId: batch._id, + recipients: recipientChunk.map((recipient) => ({ + email: recipient.email, // Only email, no name + })), + emailType, + status: 'PENDING', + }; + + batchItems.push(batchItem); + } + + await EmailBatchItem.insertMany(batchItems); + + return batch; + } catch (error) { + logger.logException(error, 'Error adding recipients to batch'); + throw error; + } + } + + /** + * Create a single send batch (most common use case) + */ + static async createSingleSendBatch(emailData, user) { + try { + // Handle both 'to' field and direct recipients array + let recipients; + if (emailData.to) { + recipients = Array.isArray(emailData.to) ? emailData.to : [emailData.to]; + } else if (Array.isArray(emailData.recipients)) { + recipients = emailData.recipients; + } else { + throw new Error('No recipients provided'); + } + + // Create batch with email body + const batch = await this.createBatch({ + batchId: uuidv4(), + subject: emailData.subject, + htmlContent: emailData.html, // Store email body in batch + createdBy: user._id || user.requestorId, + }); + + // Add recipients with efficient batching + const batchConfig = { + batchSize: 50, // Use standard batch size + emailType: recipients.length === 1 ? 'TO' : 'BCC', // Single recipient uses TO, multiple use BCC + }; + + // Convert recipients to proper format + const recipientObjects = recipients.map((email) => ({ email })); + console.log('📧 Adding recipients to batch:', recipientObjects.length, 'recipients'); + await this.addRecipients(batch.batchId, recipientObjects, batchConfig); + + return batch; + } catch (error) { + logger.logException(error, 'Error creating single send batch'); + throw error; + } + } + + /** + * Get batch with items and dynamic counts + */ + static async getBatchWithItems(batchId) { + try { + const batch = await EmailBatch.findOne({ batchId }).populate( + 'createdBy', + 'firstName lastName email', + ); + + if (!batch) { + return null; + } + + const items = await EmailBatchItem.find({ batchId: batch._id }).sort({ createdAt: 1 }); + + // Get dynamic counts + const counts = await batch.getEmailCounts(); + + // Return batch items as-is (each item contains multiple recipients) + const transformedItems = items.map((item) => ({ + _id: item._id, + recipients: item.recipients || [], + status: item.status, + attempts: item.attempts || 0, + lastAttemptedAt: item.lastAttemptedAt, + sentAt: item.sentAt, + failedAt: item.failedAt, + error: item.error, + errorCode: item.errorCode, + emailType: item.emailType, + createdAt: item.createdAt, + updatedAt: item.updatedAt, + })); + + return { + batch: { + ...batch.toObject(), + ...counts, + }, + items: transformedItems, + }; + } catch (error) { + logger.logException(error, 'Error getting batch with items'); + throw error; + } + } + + /** + * Get all batches with pagination and dynamic counts + */ + static async getBatches(filters = {}, page = 1, limit = 20) { + try { + const query = {}; + + if (filters.status) query.status = filters.status; + if (filters.dateFrom) query.createdAt = { $gte: new Date(filters.dateFrom) }; + if (filters.dateTo) query.createdAt = { ...query.createdAt, $lte: new Date(filters.dateTo) }; + + const skip = (page - 1) * limit; + + const [batches, total] = await Promise.all([ + EmailBatch.find(query) + .sort({ createdAt: -1 }) + .limit(limit) + .skip(skip) + .populate('createdBy', 'firstName lastName email'), + EmailBatch.countDocuments(query), + ]); + + // Add dynamic counts to each batch + const batchesWithCounts = await Promise.all( + batches.map(async (batch) => { + const counts = await batch.getEmailCounts(); + return { + ...batch.toObject(), + ...counts, + }; + }), + ); + + return { + batches: batchesWithCounts, + pagination: { + page, + limit, + total, + pages: Math.ceil(total / limit), + }, + }; + } catch (error) { + logger.logException(error, 'Error getting batches'); + throw error; + } + } + + /** + * Get dashboard statistics with dynamic calculations + */ + static async getDashboardStats() { + try { + const [totalBatches, pendingBatches, processingBatches, completedBatches, failedBatches] = + await Promise.all([ + EmailBatch.countDocuments(), + EmailBatch.countDocuments({ status: 'PENDING' }), + EmailBatch.countDocuments({ status: 'PROCESSING' }), + EmailBatch.countDocuments({ status: 'COMPLETED' }), + EmailBatch.countDocuments({ status: 'FAILED' }), + ]); + + // Calculate email stats dynamically from batch items + const emailStats = await EmailBatchItem.aggregate([ + { + $group: { + _id: null, + totalEmails: { + $sum: { $cond: [{ $isArray: '$recipients' }, { $size: '$recipients' }, 0] }, + }, + sentEmails: { + $sum: { + $cond: [ + { $and: [{ $eq: ['$status', 'SENT'] }, { $isArray: '$recipients' }] }, + { $size: '$recipients' }, + 0, + ], + }, + }, + failedEmails: { + $sum: { + $cond: [ + { $and: [{ $eq: ['$status', 'FAILED'] }, { $isArray: '$recipients' }] }, + { $size: '$recipients' }, + 0, + ], + }, + }, + }, + }, + ]); + + const stats = emailStats[0] || { totalEmails: 0, sentEmails: 0, failedEmails: 0 }; + const successRate = + stats.totalEmails > 0 ? Math.round((stats.sentEmails / stats.totalEmails) * 100) : 0; + + return { + overview: { + totalBatches, + pendingBatches, + processingBatches, + completedBatches, + failedBatches, + }, + emailStats: { + ...stats, + successRate, + }, + }; + } catch (error) { + logger.logException(error, 'Error getting dashboard stats'); + throw error; + } + } +} + +module.exports = EmailBatchService; diff --git a/src/startup/routes.js b/src/startup/routes.js index 9fb2af275..731294e38 100644 --- a/src/startup/routes.js +++ b/src/startup/routes.js @@ -91,7 +91,6 @@ const tag = require('../models/tag'); const educationTask = require('../models/educationTask'); const injujrySeverity = require('../models/bmdashboard/injujrySeverity'); - const bidoverview_Listing = require('../models/lbdashboard/bidoverview/Listing'); const bidoverview_Bid = require('../models/lbdashboard/bidoverview/Bid'); const bidoverview_User = require('../models/lbdashboard/bidoverview/User'); @@ -153,6 +152,8 @@ const rolePresetRouter = require('../routes/rolePresetRouter')(rolePreset); const ownerMessageRouter = require('../routes/ownerMessageRouter')(ownerMessage); const emailRouter = require('../routes/emailRouter')(); +const emailBatchRouter = require('../routes/emailBatchRoutes'); +const emailBatchDashboardRouter = require('../routes/emailBatchDashboardRoutes'); const reasonRouter = require('../routes/reasonRouter')(reason, userProfile); const mouseoverTextRouter = require('../routes/mouseoverTextRouter')(mouseoverText); @@ -279,6 +280,7 @@ const collaborationRouter = require('../routes/collaborationRouter'); const registrationRouter = require('../routes/registrationRouter')(registration); const templateRouter = require('../routes/templateRouter'); +const emailTemplateRouter = require('../routes/emailTemplateRouter'); const projectMaterialRouter = require('../routes/projectMaterialroutes'); @@ -333,6 +335,8 @@ module.exports = function (app) { app.use('/api', mouseoverTextRouter); app.use('/api', permissionChangeLogRouter); app.use('/api', emailRouter); + app.use('/api/email-batches', emailBatchRouter); + app.use('/api/email-batches', emailBatchDashboardRouter); app.use('/api', isEmailExistsRouter); app.use('/api', faqRouter); app.use('/api', mapLocationRouter); @@ -360,6 +364,7 @@ module.exports = function (app) { app.use('/api/costs', costsRouter); app.use('/api', hoursPledgedRoutes); app.use('/api', templateRouter); + app.use('/api', emailTemplateRouter); app.use('/api/help-categories', helpCategoryRouter); app.use('/api', tagRouter); @@ -415,7 +420,6 @@ module.exports = function (app) { app.use('/api/bm', bmIssueRouter); app.use('/api/bm', bmInjuryRouter); - app.use('/api/lb', bidPropertyRouter); app.use('/api/lb', userBidRouter); diff --git a/src/utilities/emailSender.js b/src/utilities/emailSender.js index ec571cbed..c7112a5dc 100644 --- a/src/utilities/emailSender.js +++ b/src/utilities/emailSender.js @@ -87,9 +87,11 @@ const sendWithRetry = async (batch, retries = 3, baseDelay = 1000) => { for (let attempt = 1; attempt <= retries; attempt += 1) { try { + // eslint-disable-next-line no-await-in-loop await sendEmail(batch); if (isBsAssignment) { + // eslint-disable-next-line no-await-in-loop await EmailHistory.findOneAndUpdate( { uniqueKey: key }, { @@ -113,6 +115,7 @@ const sendWithRetry = async (batch, retries = 3, baseDelay = 1000) => { logger.logException(err, `Batch to ${batch.to || '(empty)'} attempt ${attempt}`); if (attempt === retries && isBsAssignment) { + // eslint-disable-next-line no-await-in-loop await EmailHistory.findOneAndUpdate( { uniqueKey: key }, { @@ -133,34 +136,46 @@ const sendWithRetry = async (batch, retries = 3, baseDelay = 1000) => { } } - if (attempt < retries) await sleep(baseDelay * attempt); // backoff + if (attempt < retries) { + // eslint-disable-next-line no-await-in-loop + await sleep(baseDelay * attempt); // backoff + } } return false; }; const worker = async () => { + let allSuccessful = true; + // eslint-disable-next-line no-constant-condition while (true) { // atomically pull next batch const batch = queue.shift(); if (!batch) break; // queue drained for this worker - const success = await sendWithRetry(batch); - if (!success) { - throw new Error(`Failed to send email to ${batch.to} after all retry attempts`); + // eslint-disable-next-line no-await-in-loop + const result = await sendWithRetry(batch); + if (result === false) { + allSuccessful = false; + } + if (config.rateLimitDelay) { + // eslint-disable-next-line no-await-in-loop + await sleep(config.rateLimitDelay); // pacing } - if (config.rateLimitDelay) await sleep(config.rateLimitDelay); // pacing } + return allSuccessful; }; const processQueue = async () => { - if (isProcessing || queue.length === 0) return; + if (isProcessing || queue.length === 0) return true; isProcessing = true; try { const n = Math.max(1, Number(config.concurrency) || 1); const workers = Array.from({ length: n }, () => worker()); - await Promise.all(workers); // drain-until-empty with N workers + const results = await Promise.all(workers); // drain-until-empty with N workers + // Return true if all workers succeeded, false if any failed + return results.every((result) => result !== false); } finally { isProcessing = false; } @@ -236,8 +251,12 @@ const emailSender = ( setImmediate(async () => { try { - await processQueue(); - resolve('Emails processed successfully'); + const result = await processQueue(); + if (result === false) { + reject(new Error('Email sending failed after all retries')); + } else { + resolve('Emails processed successfully'); + } } catch (error) { reject(error); } From a58da848bb22a3e8709c35ea9ea8fe61d56c9bfb Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Wed, 29 Oct 2025 01:46:53 -0400 Subject: [PATCH 02/19] feat(emailBatch): integrate email announcement job processor and enhance email batch auditing features - Added email announcement job processor to server initialization. - Introduced new audit trail endpoints in emailBatchController for tracking email and batch actions. - Updated email batch item status logic to reflect QUEUED state instead of PENDING. - Removed deprecated emailBatchDashboardController and related routes. - Enhanced email batch model to support comprehensive auditing and recipient management. --- src/config/emailJobConfig.js | 58 +++ src/controllers/emailBatchController.js | 98 ++++- .../emailBatchDashboardController.js | 32 -- src/controllers/emailController.js | 86 +++- src/controllers/emailTemplateController.js | 10 +- src/jobs/emailAnnouncementJobProcessor.js | 198 +++++++++ src/models/email.js | 227 ++++++++++ src/models/emailBatch.js | 239 ++++++----- src/models/emailBatchAudit.js | 124 ++++++ src/models/emailBatchItem.js | 87 ---- src/models/emailSubcriptionList.js | 4 +- src/routes/emailBatchDashboardRoutes.js | 14 - src/routes/emailBatchRoutes.js | 5 + src/server.js | 1 + src/services/emailAnnouncementService.js | 302 ++++++++++++++ src/services/emailBatchAuditService.js | 259 ++++++++++++ src/services/emailBatchDashboardService.js | 391 ------------------ src/services/emailBatchProcessor.js | 71 +++- src/services/emailBatchService.js | 61 ++- src/startup/routes.js | 2 - 20 files changed, 1568 insertions(+), 701 deletions(-) create mode 100644 src/config/emailJobConfig.js delete mode 100644 src/controllers/emailBatchDashboardController.js create mode 100644 src/jobs/emailAnnouncementJobProcessor.js create mode 100644 src/models/email.js create mode 100644 src/models/emailBatchAudit.js delete mode 100644 src/models/emailBatchItem.js delete mode 100644 src/routes/emailBatchDashboardRoutes.js create mode 100644 src/services/emailAnnouncementService.js create mode 100644 src/services/emailBatchAuditService.js delete mode 100644 src/services/emailBatchDashboardService.js diff --git a/src/config/emailJobConfig.js b/src/config/emailJobConfig.js new file mode 100644 index 000000000..c1649ef9b --- /dev/null +++ b/src/config/emailJobConfig.js @@ -0,0 +1,58 @@ +/** + * Email Job Queue Configuration + * Centralized configuration for email announcement job queue system + */ + +const EMAIL_JOB_CONFIG = { + // Processing intervals + CRON_INTERVAL: '0 * * * * *', // Every minute at 0 seconds + MAX_CONCURRENT_BATCHES: 3, + + // Retry configuration + DEFAULT_MAX_RETRIES: 3, + RETRY_DELAYS: [60000, 300000, 900000], // 1min, 5min, 15min + + // Status enums + EMAIL_STATUSES: { + QUEUED: 'QUEUED', // Waiting in queue + SENDING: 'SENDING', // Currently sending + SENT: 'SENT', // All emails successfully sent + PROCESSED: 'PROCESSED', // Processing finished (mixed results) + FAILED: 'FAILED', // Failed to send + CANCELLED: 'CANCELLED', // Cancelled by user + }, + + EMAIL_BATCH_STATUSES: { + QUEUED: 'QUEUED', // Waiting to send + SENDING: 'SENDING', // Currently sending + SENT: 'SENT', // Successfully delivered + FAILED: 'FAILED', // Delivery failed + RESENDING: 'RESENDING', // Resending delivery + }, + + EMAIL_BATCH_AUDIT_ACTIONS: { + // Email-level actions (main batch) + EMAIL_CREATED: 'EMAIL_CREATED', + EMAIL_QUEUED: 'EMAIL_QUEUED', + EMAIL_SENDING: 'EMAIL_SENDING', + EMAIL_SENT: 'EMAIL_SENT', + EMAIL_PROCESSED: 'EMAIL_PROCESSED', + EMAIL_FAILED: 'EMAIL_FAILED', + EMAIL_CANCELLED: 'EMAIL_CANCELLED', + + // Email batch item-level actions + EMAIL_BATCH_QUEUED: 'EMAIL_BATCH_QUEUED', + EMAIL_BATCH_SENDING: 'EMAIL_BATCH_SENDING', + EMAIL_BATCH_SENT: 'EMAIL_BATCH_SENT', + EMAIL_BATCH_FAILED: 'EMAIL_BATCH_FAILED', + EMAIL_BATCH_RESENDING: 'EMAIL_BATCH_RESENDING', + }, + + EMAIL_TYPES: { + TO: 'TO', + CC: 'CC', + BCC: 'BCC', + }, +}; + +module.exports = { EMAIL_JOB_CONFIG }; diff --git a/src/controllers/emailBatchController.js b/src/controllers/emailBatchController.js index 064eeb9d2..752ccedbb 100644 --- a/src/controllers/emailBatchController.js +++ b/src/controllers/emailBatchController.js @@ -5,6 +5,7 @@ const EmailBatchService = require('../services/emailBatchService'); const emailBatchProcessor = require('../services/emailBatchProcessor'); +const EmailBatchAuditService = require('../services/emailBatchAuditService'); const logger = require('../startup/logger'); /** @@ -112,8 +113,8 @@ const retryBatchItem = async (req, res) => { const { itemId } = req.params; // Find the batch item - const EmailBatchItem = require('../models/emailBatchItem'); - const item = await EmailBatchItem.findById(itemId); + const EmailBatch = require('../models/emailBatch'); + const item = await EmailBatch.findById(itemId); if (!item) { return res.status(404).json({ @@ -130,16 +131,16 @@ const retryBatchItem = async (req, res) => { }); } - // Only allow retry for FAILED or PENDING items - if (item.status !== 'FAILED' && item.status !== 'PENDING') { + // Only allow retry for FAILED or QUEUED items + if (item.status !== 'FAILED' && item.status !== 'QUEUED') { return res.status(400).json({ success: false, message: 'Only failed or pending items can be retried', }); } - // Reset the item status to PENDING for retry - item.status = 'PENDING'; + // Reset the item status to QUEUED for retry + item.status = 'QUEUED'; item.attempts = 0; item.error = null; item.failedAt = null; @@ -169,10 +170,95 @@ const retryBatchItem = async (req, res) => { } }; +/** + * Get audit trail for a specific batch + */ +const getEmailAuditTrail = async (req, res) => { + try { + const { emailId } = req.params; + const { page = 1, limit = 50, action } = req.query; + + const auditTrail = await EmailBatchAuditService.getEmailAuditTrail( + emailId, + parseInt(page, 10), + parseInt(limit, 10), + action, + ); + + res.status(200).json({ + success: true, + data: auditTrail, + }); + } catch (error) { + logger.logException(error, 'Error getting email audit trail'); + res.status(500).json({ + success: false, + message: 'Error getting email audit trail', + error: error.message, + }); + } +}; + +/** + * Get audit trail for a specific batch item + */ +const getEmailBatchAuditTrail = async (req, res) => { + try { + const { emailBatchId } = req.params; + const { page = 1, limit = 50, action } = req.query; + + const auditTrail = await EmailBatchAuditService.getEmailBatchAuditTrail( + emailBatchId, + parseInt(page, 10), + parseInt(limit, 10), + action, + ); + + res.status(200).json({ + success: true, + data: auditTrail, + }); + } catch (error) { + logger.logException(error, 'Error getting email batch audit trail'); + res.status(500).json({ + success: false, + message: 'Error getting email batch audit trail', + error: error.message, + }); + } +}; + +/** + * Get audit statistics + */ +const getAuditStats = async (req, res) => { + try { + const { dateFrom, dateTo, action } = req.query; + + const filters = { dateFrom, dateTo, action }; + const stats = await EmailBatchAuditService.getAuditStats(filters); + + res.status(200).json({ + success: true, + data: stats, + }); + } catch (error) { + logger.logException(error, 'Error getting audit stats'); + res.status(500).json({ + success: false, + message: 'Error getting audit stats', + error: error.message, + }); + } +}; + module.exports = { getBatches, getBatchDetails, getDashboardStats, getProcessorStatus, retryBatchItem, + getEmailAuditTrail, + getEmailBatchAuditTrail, + getAuditStats, }; diff --git a/src/controllers/emailBatchDashboardController.js b/src/controllers/emailBatchDashboardController.js deleted file mode 100644 index e5eee32a1..000000000 --- a/src/controllers/emailBatchDashboardController.js +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Simplified Email Batch Dashboard Controller - Production Ready - * Focus: Essential dashboard endpoints only - */ - -const EmailBatchService = require('../services/emailBatchService'); -const logger = require('../startup/logger'); - -/** - * Get dashboard statistics - */ -const getDashboardStats = async (req, res) => { - try { - const stats = await EmailBatchService.getDashboardStats(); - - res.status(200).json({ - success: true, - data: stats, - }); - } catch (error) { - logger.logException(error, 'Error getting dashboard stats'); - res.status(500).json({ - success: false, - message: 'Error getting dashboard stats', - error: error.message, - }); - } -}; - -module.exports = { - getDashboardStats, -}; diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index 5ca2e8c78..b558eb50a 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -1,12 +1,11 @@ // emailController.js const jwt = require('jsonwebtoken'); const cheerio = require('cheerio'); -const emailSender = require('../utilities/emailSender'); +const emailAnnouncementService = require('../services/emailAnnouncementService'); const { hasPermission } = require('../utilities/permissions'); const EmailSubcriptionList = require('../models/emailSubcriptionList'); const userProfile = require('../models/userProfile'); const EmailBatchService = require('../services/emailBatchService'); -const emailBatchProcessor = require('../services/emailBatchProcessor'); const frontEndUrl = process.env.FRONT_END_URL || 'http://localhost:3000'; const jwtSecret = process.env.JWT_SECRET || 'EmailSecret'; @@ -105,11 +104,10 @@ const sendEmail = async (req, res) => { console.log('✅ Batch created with recipients:', batch.batchId); - // Start processing the batch - console.log('🚀 Starting batch processing...'); - emailBatchProcessor.processBatch(batch.batchId).catch((error) => { - console.error('❌ Error processing batch:', error); - }); + // REMOVED: Immediate processing - batch will be processed by cron job + // emailBatchProcessor.processBatch(batch.batchId).catch((error) => { + // console.error('❌ Error processing batch:', error); + // }); // Get dynamic counts for response const counts = await batch.getEmailCounts(); @@ -127,15 +125,27 @@ const sendEmail = async (req, res) => { }, }); } else { - // Legacy direct sending (fallback) + // Legacy direct sending (fallback) - using new announcement service if (recipientsArray.length === 1) { // Single recipient - use TO field - await emailSender(to, subject, processedHtml, attachments); + await emailAnnouncementService.sendAnnouncement( + to, + subject, + processedHtml, + attachments, + null, + null, + null, + { + announcementType: 'direct_send', + priority: 'NORMAL', + }, + ); } else { // Multiple recipients - use BCC to hide recipient list // Send to self (sender) as primary recipient, then BCC all actual recipients const senderEmail = req.body.fromEmail || 'updates@onecommunityglobal.org'; - await emailSender( + await emailAnnouncementService.sendAnnouncement( senderEmail, subject, processedHtml, @@ -143,6 +153,10 @@ const sendEmail = async (req, res) => { null, null, recipientsArray, + { + announcementType: 'direct_send', + priority: 'NORMAL', + }, ); } @@ -239,10 +253,10 @@ const sendEmailToAll = async (req, res) => { await EmailBatchService.addRecipients(batch.batchId, subscriberRecipients); } - // Start processing the batch - emailBatchProcessor.processBatch(batch.batchId).catch((error) => { - console.error('Error processing broadcast batch:', error); - }); + // REMOVED: Immediate processing - batch will be processed by cron job + // emailBatchProcessor.processBatch(batch.batchId).catch((error) => { + // console.error('Error processing broadcast batch:', error); + // }); // Get dynamic counts for response const counts = await batch.getEmailCounts(); @@ -266,7 +280,7 @@ const sendEmailToAll = async (req, res) => { }, }); } - // Legacy direct sending (fallback) + // Legacy direct sending (fallback) - using new announcement service // HGN Users logic const users = await userProfile.find({ firstName: { $ne: '' }, @@ -281,7 +295,19 @@ const sendEmailToAll = async (req, res) => { const emailContentToOCmembers = handleContentToOC(processedHtml); await Promise.all( recipientEmails.map((email) => - emailSender(email, subject, emailContentToOCmembers, attachments), + emailAnnouncementService.sendAnnouncement( + email, + subject, + emailContentToOCmembers, + attachments, + null, + null, + null, + { + announcementType: 'broadcast_hgn', + priority: 'NORMAL', + }, + ), ), ); } else { @@ -298,7 +324,19 @@ const sendEmailToAll = async (req, res) => { await Promise.all( emailSubscribers.map(({ email }) => { const emailContentToNonOCmembers = handleContentToNonOC(processedHtml, email); - return emailSender(email, subject, emailContentToNonOCmembers, attachments); + return emailAnnouncementService.sendAnnouncement( + email, + subject, + emailContentToNonOCmembers, + attachments, + null, + null, + null, + { + announcementType: 'broadcast_subscriber', + priority: 'NORMAL', + }, + ); }), ); } else { @@ -362,7 +400,19 @@ const addNonHgnEmailSubscription = async (req, res) => { `; try { - await emailSender(email, 'HGN Email Subscription', emailContent); + await emailAnnouncementService.sendAnnouncement( + email, + 'HGN Email Subscription', + emailContent, + null, + null, + null, + null, + { + announcementType: 'subscription_confirmation', + priority: 'NORMAL', + }, + ); return res.status(200).send('Email subscribed successfully'); } catch (emailError) { console.error('Error sending confirmation email:', emailError); diff --git a/src/controllers/emailTemplateController.js b/src/controllers/emailTemplateController.js index c47ef2e35..35390e623 100644 --- a/src/controllers/emailTemplateController.js +++ b/src/controllers/emailTemplateController.js @@ -431,7 +431,6 @@ exports.sendEmailWithTemplate = async (req, res) => { // Send to specific recipients using batch system try { const EmailBatchService = require('../services/emailBatchService'); - const emailBatchProcessor = require('../services/emailBatchProcessor'); const userProfile = require('../models/userProfile'); // Get user information @@ -463,11 +462,10 @@ exports.sendEmailWithTemplate = async (req, res) => { createdBy: batch.createdBy, }); - // Start processing the batch - console.log('🚀 Starting template batch processing...'); - emailBatchProcessor.processBatch(batch.batchId).catch((error) => { - console.error('❌ Error processing template batch:', error); - }); + // REMOVED: Immediate processing - batch will be processed by cron job + // emailBatchProcessor.processBatch(batch.batchId).catch((error) => { + // console.error('❌ Error processing template batch:', error); + // }); // Get dynamic counts for response const counts = await batch.getEmailCounts(); diff --git a/src/jobs/emailAnnouncementJobProcessor.js b/src/jobs/emailAnnouncementJobProcessor.js new file mode 100644 index 000000000..00e0cdce4 --- /dev/null +++ b/src/jobs/emailAnnouncementJobProcessor.js @@ -0,0 +1,198 @@ +/** + * Email Announcement Job Processor + * Cron-based processor for email announcement job queue + */ + +const { CronJob } = require('cron'); +const Email = require('../models/email'); +const emailBatchProcessor = require('../services/emailBatchProcessor'); +const EmailBatchAuditService = require('../services/emailBatchAuditService'); +const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +const logger = require('../startup/logger'); + +class EmailAnnouncementJobProcessor { + constructor() { + this.isProcessing = false; + this.maxConcurrentBatches = EMAIL_JOB_CONFIG.MAX_CONCURRENT_BATCHES; + this.processingInterval = EMAIL_JOB_CONFIG.CRON_INTERVAL; + this.cronJob = null; + } + + /** + * Start the job processor + */ + start() { + if (this.cronJob) { + logger.logInfo('Email announcement job processor is already running'); + return; + } + + this.cronJob = new CronJob( + EMAIL_JOB_CONFIG.CRON_INTERVAL, + async () => { + await this.processPendingBatches(); + }, + null, + false, // Don't start immediately + 'America/Los_Angeles', + ); + + this.cronJob.start(); + logger.logInfo(`Email announcement job processor started - runs every minute`); + } + + /** + * Stop the job processor + */ + stop() { + if (this.cronJob) { + this.cronJob.stop(); + this.cronJob = null; + logger.logInfo('Email announcement job processor stopped'); + } + } + + /** + * Process pending batches + */ + async processPendingBatches() { + if (this.isProcessing) { + logger.logInfo('Email job processor is already running, skipping this cycle'); + return; + } + + this.isProcessing = true; + + try { + // Get batches ready for processing + const pendingBatches = await Email.find({ + status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.QUEUED, + }) + .sort({ createdAt: 1 }) // FIFO order + .limit(this.maxConcurrentBatches); + + if (pendingBatches.length === 0) { + logger.logInfo('No pending email batches to process'); + return; + } + + logger.logInfo(`Processing ${pendingBatches.length} email batches`); + + // Process each batch + const processingPromises = pendingBatches.map((batch) => + EmailAnnouncementJobProcessor.processBatchWithAuditing(batch), + ); + + await Promise.allSettled(processingPromises); + } catch (error) { + logger.logException(error, 'Error processing announcement batches'); + } finally { + this.isProcessing = false; + } + } + + /** + * Process a single batch with comprehensive auditing + */ + static async processBatchWithAuditing(batch) { + const startTime = Date.now(); + + try { + // Log batch start + await EmailBatchAuditService.logEmailStarted(batch._id, { + batchId: batch.batchId, + subject: batch.subject, + recipientCount: await batch.getEmailCounts(), + }); + + // Update batch status + batch.status = EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING; + batch.startedAt = new Date(); + await batch.save(); + + // Process using existing emailBatchProcessor + await emailBatchProcessor.processBatch(batch.batchId); + + const processingTime = Date.now() - startTime; + + // Log completion + await EmailBatchAuditService.logEmailCompleted(batch._id, { + batchId: batch.batchId, + processingTime, + finalCounts: await batch.getEmailCounts(), + }); + + logger.logInfo(`Successfully processed batch ${batch.batchId} in ${processingTime}ms`); + } catch (error) { + const processingTime = Date.now() - startTime; + + // Log error + await EmailBatchAuditService.logEmailFailed(batch._id, error, { + batchId: batch.batchId, + processingTime, + errorMessage: error.message, + }); + + logger.logException(error, `Failed to process batch ${batch.batchId}`); + } + } + + /** + * Get processor status + */ + getStatus() { + return { + isRunning: !!this.cronJob, + isProcessing: this.isProcessing, + maxConcurrentBatches: this.maxConcurrentBatches, + cronInterval: this.processingInterval, + nextRun: this.cronJob ? this.cronJob.nextDate() : null, + }; + } + + /** + * Get pending batches count + */ + static async getPendingBatchesCount() { + return Email.countDocuments({ + status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.QUEUED, + }); + } + + /** + * Get processing statistics + */ + static async getProcessingStats() { + const stats = await Email.aggregate([ + { + $group: { + _id: '$status', + count: { $sum: 1 }, + avgProcessingTime: { + $avg: { + $cond: [ + { $ne: ['$processingStartedAt', null] }, + { $subtract: ['$completedAt', '$processingStartedAt'] }, + null, + ], + }, + }, + }, + }, + ]); + + return { + statusCounts: stats.reduce((acc, stat) => { + acc[stat._id] = stat.count; + return acc; + }, {}), + averageProcessingTime: stats[0]?.avgProcessingTime || 0, + lastUpdated: new Date(), + }; + } +} + +// Create singleton instance +const emailAnnouncementJobProcessor = new EmailAnnouncementJobProcessor(); + +module.exports = emailAnnouncementJobProcessor; diff --git a/src/models/email.js b/src/models/email.js new file mode 100644 index 000000000..aac0bdc6b --- /dev/null +++ b/src/models/email.js @@ -0,0 +1,227 @@ +const mongoose = require('mongoose'); +const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); + +const { Schema } = mongoose; + +const EmailSchema = new Schema({ + batchId: { + type: String, + required: [true, 'batchId is required'], + unique: true, + index: true, + }, + subject: { + type: String, + required: [true, 'Subject is required'], + maxlength: [200, 'Subject cannot exceed 200 characters'], + validate: { + validator(v) { + return v && v.trim().length > 0; + }, + message: 'Subject cannot be empty or whitespace only', + }, + }, + htmlContent: { + type: String, + required: [true, 'HTML content is required'], + validate: { + validator(v) { + return v && v.trim().length > 0; + }, + message: 'HTML content cannot be empty or whitespace only', + }, + }, + status: { + type: String, + enum: Object.values(EMAIL_JOB_CONFIG.EMAIL_STATUSES), + default: EMAIL_JOB_CONFIG.EMAIL_STATUSES.QUEUED, + index: true, + }, + createdBy: { + type: Schema.Types.ObjectId, + ref: 'userProfile', + required: [true, 'createdBy is required'], + validate: { + validator(v) { + return mongoose.Types.ObjectId.isValid(v); + }, + message: 'Invalid createdBy ObjectId', + }, + }, + createdAt: { type: Date, default: () => new Date(), index: true }, + startedAt: { + type: Date, + validate: { + validator(v) { + return !v || v >= this.createdAt; + }, + message: 'startedAt cannot be before createdAt', + }, + }, + completedAt: { + type: Date, + validate: { + validator(v) { + return !v || v >= (this.startedAt || this.createdAt); + }, + message: 'completedAt cannot be before startedAt or createdAt', + }, + }, + updatedAt: { type: Date, default: () => new Date() }, + lastStuckFixAttempt: { type: Date }, // For preventing infinite retries +}); + +// Update timestamps and validate basic constraints +EmailSchema.pre('save', function (next) { + this.updatedAt = new Date(); + + // Validate timestamp consistency + if (this.startedAt && this.completedAt && this.startedAt > this.completedAt) { + return next(new Error('startedAt cannot be after completedAt')); + } + + next(); +}); + +// Add indexes for better performance +EmailSchema.index({ status: 1, createdAt: 1 }); +EmailSchema.index({ createdBy: 1, createdAt: -1 }); +EmailSchema.index({ startedAt: 1 }); +EmailSchema.index({ completedAt: 1 }); + +// Calculate email counts dynamically from batch items with multiple recipients +EmailSchema.methods.getEmailCounts = async function () { + try { + const EmailBatch = require('./emailBatch'); + + // Validate this._id exists + if (!this._id) { + throw new Error('Email ID is required for counting'); + } + + const counts = await EmailBatch.aggregate([ + { $match: { batchId: this._id } }, // EmailBatch.batchId references Email._id (ObjectId) + { + $group: { + _id: null, + total: { $sum: { $cond: [{ $isArray: '$recipients' }, { $size: '$recipients' }, 0] } }, + sent: { + $sum: { + $cond: [ + { + $and: [ + { $eq: ['$status', EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENT] }, + { $isArray: '$recipients' }, + ], + }, + { $size: '$recipients' }, + 0, + ], + }, + }, + failed: { + $sum: { + $cond: [ + { + $and: [ + { $eq: ['$status', EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.FAILED] }, + { $isArray: '$recipients' }, + ], + }, + { $size: '$recipients' }, + 0, + ], + }, + }, + pending: { + $sum: { + $cond: [ + { + $and: [ + { + $in: [ + '$status', + [ + EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED, + EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENDING, + EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.RESENDING, + ], + ], + }, + { $isArray: '$recipients' }, + ], + }, + { $size: '$recipients' }, + 0, + ], + }, + }, + cancelled: { + $sum: { + $cond: [ + { + $and: [ + { $eq: ['$status', EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.CANCELLED] }, + { $isArray: '$recipients' }, + ], + }, + { $size: '$recipients' }, + 0, + ], + }, + }, + }, + }, + ]); + + if (counts.length > 0) { + const count = counts[0]; + + // Protect against negative counts (data corruption edge case) + const totalEmails = Math.max(0, count.total || 0); + const sentEmails = Math.max(0, Math.min(count.sent || 0, totalEmails)); + const failedEmails = Math.max(0, Math.min(count.failed || 0, totalEmails)); + const pendingEmails = Math.max(0, Math.min(count.pending || 0, totalEmails)); + const cancelledEmails = Math.max(0, Math.min(count.cancelled || 0, totalEmails)); + + // Validate counts don't exceed total + const calculatedTotal = sentEmails + failedEmails + pendingEmails + cancelledEmails; + if (calculatedTotal !== totalEmails) { + console.warn( + `Email count mismatch for ${this._id}: calculated=${calculatedTotal}, total=${totalEmails}`, + ); + } + + return { + totalEmails, + sentEmails, + failedEmails, + pendingEmails, + cancelledEmails, + progress: totalEmails > 0 ? Math.round((sentEmails / totalEmails) * 100) : 0, + }; + } + + return { + totalEmails: 0, + sentEmails: 0, + failedEmails: 0, + pendingEmails: 0, + cancelledEmails: 0, + progress: 0, + }; + } catch (error) { + // Log error and return safe defaults + console.error('Error calculating email counts:', error); + return { + totalEmails: 0, + sentEmails: 0, + failedEmails: 0, + pendingEmails: 0, + cancelledEmails: 0, + progress: 0, + }; + } +}; + +module.exports = mongoose.model('Email', EmailSchema, 'emails'); diff --git a/src/models/emailBatch.js b/src/models/emailBatch.js index b092cf6fd..7e33d10a7 100644 --- a/src/models/emailBatch.js +++ b/src/models/emailBatch.js @@ -1,126 +1,163 @@ -/** - * Simplified Email Batch Model - Production Ready - * Focus: Store batch information with email body for efficiency - */ - const mongoose = require('mongoose'); +const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); const { Schema } = mongoose; const EmailBatchSchema = new Schema({ - // Core identification - batchId: { type: String, required: true, unique: true, index: true }, - subject: { type: String, required: true }, - htmlContent: { type: String, required: true }, // Store email body in batch + // Batch reference + batchId: { + type: Schema.Types.ObjectId, + ref: 'Email', + required: [true, 'batchId is required'], + index: true, + validate: { + validator(v) { + return mongoose.Types.ObjectId.isValid(v); + }, + message: 'Invalid batchId ObjectId', + }, + }, - // Status tracking + // Multiple recipients in one batch item (emails only) + recipients: { + type: [ + { + _id: false, // Prevent MongoDB from generating _id for each recipient + email: { + type: String, + required: [true, 'Email is required'], + validate: { + validator(v) { + // Basic email format validation + const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + return emailRegex.test(v); + }, + message: 'Invalid email format', + }, + }, + }, + ], + required: [true, 'Recipients array is required'], + validate: { + validator(v) { + // Ensure at least one recipient and not too many + return v && v.length > 0 && v.length <= 1000; // Max 1000 recipients per batch + }, + message: 'Recipients must have 1-1000 email addresses', + }, + }, + + // Email type for the batch item (uses config enum) + emailType: { + type: String, + enum: Object.values(EMAIL_JOB_CONFIG.EMAIL_TYPES), + default: EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC, // Use BCC for multiple recipients + required: [true, 'Email type is required'], + }, + + // Status tracking (for the entire batch item) - uses config enum status: { type: String, - enum: ['PENDING', 'PROCESSING', 'COMPLETED', 'FAILED'], - default: 'PENDING', + enum: Object.values(EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES), + default: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED, index: true, + required: [true, 'Status is required'], + }, + + // Processing info + attempts: { + type: Number, + default: 0, + min: [0, 'Attempts cannot be negative'], + }, + lastAttemptedAt: { + type: Date, + validate: { + validator(v) { + return !v || v >= this.createdAt; + }, + message: 'lastAttemptedAt cannot be before createdAt', + }, + }, + sentAt: { + type: Date, + validate: { + validator(v) { + return !v || v >= this.createdAt; + }, + message: 'sentAt cannot be before createdAt', + }, + }, + failedAt: { + type: Date, + validate: { + validator(v) { + return !v || v >= this.createdAt; + }, + message: 'failedAt cannot be before createdAt', + }, }, - // Creator reference only - createdBy: { type: Schema.Types.ObjectId, ref: 'userProfile', required: true }, + // ERROR TRACKING + lastError: { + type: String, + maxlength: [500, 'Error message cannot exceed 500 characters'], + }, + lastErrorAt: { + type: Date, + validate: { + validator(v) { + return !v || v >= this.createdAt; + }, + message: 'lastErrorAt cannot be before createdAt', + }, + }, + errorCode: { + type: String, + maxlength: [1000, 'Error code cannot exceed 1000 characters'], + }, - // Timing - createdAt: { type: Date, default: Date.now, index: true }, - startedAt: Date, - completedAt: Date, - updatedAt: { type: Date, default: Date.now }, + // Timestamps + createdAt: { type: Date, default: () => new Date(), index: true }, + updatedAt: { type: Date, default: () => new Date() }, }); -// Update timestamps +// Update timestamps and validate basic constraints EmailBatchSchema.pre('save', function (next) { this.updatedAt = new Date(); + + // Validate timestamp consistency + if (this.sentAt && this.failedAt) { + return next(new Error('Cannot have both sentAt and failedAt')); + } + + // Validate status consistency with timestamps + if (this.status === EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENT && !this.sentAt) { + this.sentAt = new Date(); + } + if (this.status === EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.FAILED && !this.failedAt) { + this.failedAt = new Date(); + } + next(); }); -// Calculate email counts dynamically from batch items with multiple recipients -EmailBatchSchema.methods.getEmailCounts = async function () { - const EmailBatchItem = require('./emailBatchItem'); - - const counts = await EmailBatchItem.aggregate([ - { $match: { batchId: this._id } }, - { - $group: { - _id: null, - total: { $sum: { $cond: [{ $isArray: '$recipients' }, { $size: '$recipients' }, 0] } }, - sent: { - $sum: { - $cond: [ - { $and: [{ $eq: ['$status', 'SENT'] }, { $isArray: '$recipients' }] }, - { $size: '$recipients' }, - 0, - ], - }, - }, - failed: { - $sum: { - $cond: [ - { $and: [{ $eq: ['$status', 'FAILED'] }, { $isArray: '$recipients' }] }, - { $size: '$recipients' }, - 0, - ], - }, - }, - pending: { - $sum: { - $cond: [ - { $and: [{ $in: ['$status', ['PENDING', 'SENDING']] }, { $isArray: '$recipients' }] }, - { $size: '$recipients' }, - 0, - ], - }, - }, - }, - }, - ]); - - if (counts.length > 0) { - const count = counts[0]; - return { - totalEmails: count.total || 0, - sentEmails: count.sent || 0, - failedEmails: count.failed || 0, - pendingEmails: count.pending || 0, - progress: count.total > 0 ? Math.round((count.sent / count.total) * 100) : 0, - }; - } +// Add indexes for better performance +EmailBatchSchema.index({ batchId: 1, status: 1 }); // For batch queries by status +EmailBatchSchema.index({ status: 1, createdAt: 1 }); // For status-based queries +EmailBatchSchema.index({ batchId: 1, createdAt: -1 }); // For batch history +EmailBatchSchema.index({ lastAttemptedAt: 1 }); // For retry logic +EmailBatchSchema.index({ attempts: 1, status: 1 }); // For retry queries - return { - totalEmails: 0, - sentEmails: 0, - failedEmails: 0, - pendingEmails: 0, - progress: 0, - }; +// Get recipient count for this batch item +EmailBatchSchema.methods.getRecipientCount = function () { + return this.recipients ? this.recipients.length : 0; }; -// Update status based on email counts -EmailBatchSchema.methods.updateStatus = async function () { - const counts = await this.getEmailCounts(); - - if (counts.pendingEmails === 0 && counts.totalEmails > 0) { - // All emails processed - if (counts.failedEmails === 0) { - this.status = 'COMPLETED'; - } else if (counts.sentEmails === 0) { - this.status = 'FAILED'; - } else { - this.status = 'COMPLETED'; // Partial success - } - this.completedAt = new Date(); - } else if (counts.totalEmails > 0) { - this.status = 'PROCESSING'; - if (!this.startedAt) { - this.startedAt = new Date(); - } - } - - await this.save(); - return this; +// Check if this batch item is in a final state +EmailBatchSchema.methods.isFinalState = function () { + const { EMAIL_BATCH_STATUSES } = EMAIL_JOB_CONFIG; + return [EMAIL_BATCH_STATUSES.SENT, EMAIL_BATCH_STATUSES.FAILED].includes(this.status); }; module.exports = mongoose.model('EmailBatch', EmailBatchSchema, 'emailBatches'); diff --git a/src/models/emailBatchAudit.js b/src/models/emailBatchAudit.js new file mode 100644 index 000000000..bebb0f234 --- /dev/null +++ b/src/models/emailBatchAudit.js @@ -0,0 +1,124 @@ +const mongoose = require('mongoose'); +const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); + +const { Schema } = mongoose; + +const EmailBatchAuditSchema = new Schema({ + // Reference to the main email + emailId: { + type: Schema.Types.ObjectId, + ref: 'Email', + required: [true, 'emailId is required'], + index: true, + validate: { + validator(v) { + return mongoose.Types.ObjectId.isValid(v); + }, + message: 'Invalid emailId ObjectId', + }, + }, + + // Reference to specific email batch item + emailBatchId: { + type: Schema.Types.ObjectId, + ref: 'EmailBatch', + index: true, + validate: { + validator(v) { + return !v || mongoose.Types.ObjectId.isValid(v); + }, + message: 'Invalid emailBatchId ObjectId', + }, + }, + + // Action performed (uses config enum) + action: { + type: String, + enum: Object.values(EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS), + required: [true, 'Action is required'], + index: true, + }, + + // Action details + details: { + type: String, + required: [true, 'Details are required'], + maxlength: [1000, 'Details cannot exceed 1000 characters'], + validate: { + validator(v) { + return v && v.trim().length > 0; + }, + message: 'Details cannot be empty or whitespace only', + }, + }, + + // Error information (if applicable) + error: { + type: String, + maxlength: [1000, 'Error message cannot exceed 1000 characters'], + }, + errorCode: { + type: String, + maxlength: [50, 'Error code cannot exceed 50 characters'], + }, + + // Contextual metadata (flexible object for additional data) + metadata: { + type: Schema.Types.Mixed, + default: {}, + validate: { + validator(v) { + // Limit metadata size to prevent abuse + const sizeInBytes = Buffer.byteLength(JSON.stringify(v), 'utf8'); + return sizeInBytes <= 10000; // 10KB limit + }, + message: 'Metadata cannot exceed 10KB', + }, + }, + + // Timestamps + timestamp: { + type: Date, + default: () => new Date(), + index: true, + required: [true, 'Timestamp is required'], + }, + + // User who triggered the action (if applicable) + triggeredBy: { + type: Schema.Types.ObjectId, + ref: 'userProfile', + validate: { + validator(v) { + return !v || mongoose.Types.ObjectId.isValid(v); + }, + message: 'Invalid triggeredBy ObjectId', + }, + }, + + // Processing context + processingContext: { + attemptNumber: { + type: Number, + min: [0, 'Attempt number cannot be negative'], + }, + retryDelay: { + type: Number, + min: [0, 'Retry delay cannot be negative'], + }, + processingTime: { + type: Number, + min: [0, 'Processing time cannot be negative'], + }, + }, +}); + +// Indexes for efficient querying +EmailBatchAuditSchema.index({ emailId: 1, timestamp: 1 }); +EmailBatchAuditSchema.index({ emailBatchId: 1, timestamp: 1 }); +EmailBatchAuditSchema.index({ action: 1, timestamp: 1 }); +EmailBatchAuditSchema.index({ timestamp: -1 }); +EmailBatchAuditSchema.index({ triggeredBy: 1, timestamp: -1 }); // For user audit queries +EmailBatchAuditSchema.index({ emailId: 1, action: 1 }); // For action-specific queries + +module.exports = mongoose.model('EmailBatchAudit', EmailBatchAuditSchema, 'emailBatchAudits'); diff --git a/src/models/emailBatchItem.js b/src/models/emailBatchItem.js deleted file mode 100644 index 9850922a3..000000000 --- a/src/models/emailBatchItem.js +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Simplified Email Batch Item Model - Production Ready - * Focus: Store multiple recipients per batch item, no names or priority - */ - -const mongoose = require('mongoose'); - -const { Schema } = mongoose; - -const EmailBatchItemSchema = new Schema({ - // Batch reference - batchId: { type: Schema.Types.ObjectId, ref: 'EmailBatch', required: true, index: true }, - - // Multiple recipients in one batch item (emails only) - // _id: false prevents MongoDB from generating unnecessary _id fields for each recipient - recipients: [ - { - _id: false, // Prevent MongoDB from generating _id for each recipient - email: { type: String, required: true }, - }, - ], - - // Email type for the batch item - emailType: { - type: String, - enum: ['TO', 'CC', 'BCC'], - default: 'BCC', // Use BCC for multiple recipients - }, - - // Status tracking (for the entire batch item) - status: { - type: String, - enum: ['PENDING', 'SENDING', 'SENT', 'FAILED'], - default: 'PENDING', - index: true, - }, - - // Processing info - attempts: { type: Number, default: 0 }, - lastAttemptedAt: Date, - sentAt: Date, - failedAt: Date, - error: String, - - // Timestamps - createdAt: { type: Date, default: Date.now }, - updatedAt: { type: Date, default: Date.now }, -}); - -// Update timestamps -EmailBatchItemSchema.pre('save', function (next) { - this.updatedAt = new Date(); - next(); -}); - -// Update status with proper attempt tracking -EmailBatchItemSchema.methods.updateStatus = async function (newStatus, errorMessage = null) { - this.status = newStatus; - this.lastAttemptedAt = new Date(); - - // Only increment attempts for actual sending attempts (SENDING status) - if (newStatus === 'SENDING') { - this.attempts += 1; - } - - if (newStatus === 'SENT') { - this.sentAt = new Date(); - this.failedAt = null; - this.error = null; - } else if (newStatus === 'FAILED') { - this.failedAt = new Date(); - this.error = errorMessage; - } - - await this.save(); - - // Update parent batch status - const EmailBatch = require('./emailBatch'); - const batch = await EmailBatch.findById(this.batchId); - if (batch) { - await batch.updateStatus(); - } - - return this; -}; - -module.exports = mongoose.model('EmailBatchItem', EmailBatchItemSchema, 'emailBatchItems'); diff --git a/src/models/emailSubcriptionList.js b/src/models/emailSubcriptionList.js index f28feaf13..263234124 100644 --- a/src/models/emailSubcriptionList.js +++ b/src/models/emailSubcriptionList.js @@ -3,7 +3,7 @@ const mongoose = require('mongoose'); const { Schema } = mongoose; -const emailSubscriptionSchema = new Schema({ +const emailSubscriptionListSchema = new Schema({ email: { type: String, required: true, unique: true }, emailSubscriptions: { type: Boolean, @@ -25,6 +25,6 @@ const emailSubscriptionSchema = new Schema({ module.exports = mongoose.model( 'emailSubscriptions', - emailSubscriptionSchema, + emailSubscriptionListSchema, 'emailSubscriptions', ); diff --git a/src/routes/emailBatchDashboardRoutes.js b/src/routes/emailBatchDashboardRoutes.js deleted file mode 100644 index 99ad2fdb1..000000000 --- a/src/routes/emailBatchDashboardRoutes.js +++ /dev/null @@ -1,14 +0,0 @@ -/** - * Simplified Email Batch Dashboard Routes - Production Ready - * Focus: Essential dashboard endpoints only - */ - -const express = require('express'); - -const router = express.Router(); -const emailBatchDashboardController = require('../controllers/emailBatchDashboardController'); - -// Dashboard routes -router.get('/dashboard', emailBatchDashboardController.getDashboardStats); - -module.exports = router; diff --git a/src/routes/emailBatchRoutes.js b/src/routes/emailBatchRoutes.js index dbe298d7f..a43bd8b0e 100644 --- a/src/routes/emailBatchRoutes.js +++ b/src/routes/emailBatchRoutes.js @@ -17,4 +17,9 @@ router.get('/status', emailBatchController.getProcessorStatus); // Retry operations router.post('/retry-item/:itemId', emailBatchController.retryBatchItem); +// Audit operations +router.get('/audit/email/:emailId', emailBatchController.getEmailAuditTrail); +router.get('/audit/email-batch/:emailBatchId', emailBatchController.getEmailBatchAuditTrail); +router.get('/audit/stats', emailBatchController.getAuditStats); + module.exports = router; diff --git a/src/server.js b/src/server.js index 389b8bc69..d2f3b447e 100644 --- a/src/server.js +++ b/src/server.js @@ -2,6 +2,7 @@ require('dotenv').config(); const http = require('http'); require('./jobs/dailyMessageEmailNotification'); +require('./jobs/emailAnnouncementJobProcessor').start(); // Start email announcement job processor const { app, logger } = require('./app'); const TimerWebsockets = require('./websockets').default; const MessagingWebSocket = require('./websockets/lbMessaging/messagingSocket').default; diff --git a/src/services/emailAnnouncementService.js b/src/services/emailAnnouncementService.js new file mode 100644 index 000000000..e9d8cc91a --- /dev/null +++ b/src/services/emailAnnouncementService.js @@ -0,0 +1,302 @@ +/** + * Email Announcement Service + * Enhanced email service specifically tuned for announcement use cases + * Provides better tracking, analytics, and announcement-specific features + */ + +const nodemailer = require('nodemailer'); +const { google } = require('googleapis'); +const logger = require('../startup/logger'); + +class EmailAnnouncementService { + constructor() { + this.config = { + email: process.env.REACT_APP_EMAIL, + clientId: process.env.REACT_APP_EMAIL_CLIENT_ID, + clientSecret: process.env.REACT_APP_EMAIL_CLIENT_SECRET, + redirectUri: process.env.REACT_APP_EMAIL_CLIENT_REDIRECT_URI, + refreshToken: process.env.REACT_APP_EMAIL_REFRESH_TOKEN, + batchSize: 50, + concurrency: 3, + rateLimitDelay: 1000, + }; + + this.OAuth2Client = new google.auth.OAuth2( + this.config.clientId, + this.config.clientSecret, + this.config.redirectUri, + ); + this.OAuth2Client.setCredentials({ refresh_token: this.config.refreshToken }); + + // Create the email transporter + this.transporter = nodemailer.createTransport({ + service: 'gmail', + auth: { + type: 'OAuth2', + user: this.config.email, + clientId: this.config.clientId, + clientSecret: this.config.clientSecret, + }, + }); + + this.queue = []; + this.isProcessing = false; + } + + /** + * Normalize email field (convert to array) + */ + static normalize(field) { + if (!field) return []; + if (Array.isArray(field)) return field; + return String(field).split(','); + } + + /** + * Send email with enhanced announcement tracking + */ + async sendEmail(mailOptions) { + try { + const accessTokenResp = await this.OAuth2Client.getAccessToken(); + const token = typeof accessTokenResp === 'object' ? accessTokenResp?.token : accessTokenResp; + + if (!token) { + throw new Error('NO_OAUTH_ACCESS_TOKEN'); + } + + mailOptions.auth = { + type: 'OAuth2', + user: this.config.email, + clientId: this.config.clientId, + clientSecret: this.config.clientSecret, + refreshToken: this.config.refreshToken, + accessToken: token, + }; + + const result = await this.transporter.sendMail(mailOptions); + + // Enhanced logging for announcements + if (process.env.NODE_ENV === 'local') { + logger.logInfo(`Announcement email sent: ${JSON.stringify(result)}`); + } + + return result; + } catch (error) { + console.error('Error sending announcement email:', error); + logger.logException(error, `Error sending announcement email: ${mailOptions.to}`); + + throw error; + } + } + + /** + * Send email with retry logic and announcement-specific handling + */ + async sendWithRetry(batch, retries = 3, baseDelay = 1000) { + /* eslint-disable no-await-in-loop */ + for (let attempt = 1; attempt <= retries; attempt += 1) { + try { + const gmailResponse = await this.sendEmail(batch); + + // Store Gmail response for audit logging + batch.gmailResponse = gmailResponse; + return true; + } catch (err) { + logger.logException( + err, + `Announcement batch to ${batch.to || '(empty)'} attempt ${attempt}`, + ); + } + + if (attempt < retries) { + await EmailAnnouncementService.sleep(baseDelay * attempt); // Exponential backoff + } + } + /* eslint-enable no-await-in-loop */ + return false; + } + + /** + * Process email queue with announcement-specific optimizations + */ + async processQueue() { + if (this.isProcessing || this.queue.length === 0) { + return true; + } + + this.isProcessing = true; + + try { + const batches = this.queue.splice(0, this.config.concurrency); + const promises = batches.map((batch) => this.sendWithRetry(batch)); + + await Promise.all(promises); + + if (this.queue.length > 0) { + await EmailAnnouncementService.sleep(this.config.rateLimitDelay); + return this.processQueue(); + } + + // Return the last successful Gmail response for audit logging + const lastBatch = batches[batches.length - 1]; + return lastBatch?.gmailResponse || true; + } catch (error) { + logger.logException(error, 'Error processing announcement email queue'); + return false; + } finally { + this.isProcessing = false; + } + } + + /** + * Send announcement email with enhanced features + * @param {string|string[]} recipients - Email recipients + * @param {string} subject - Email subject + * @param {string} message - HTML message content + * @param {Object[]|null} attachments - Email attachments + * @param {string[]|null} cc - CC recipients + * @param {string|null} replyTo - Reply-to address + * @param {string[]|null} emailBccs - BCC recipients + * @param {Object} opts - Options including announcement-specific metadata + * @returns {Promise} Processing result + */ + async sendAnnouncement( + recipients, + subject, + message, + attachments = null, + cc = null, + replyTo = null, + emailBccs = null, + opts = {}, + ) { + const announcementType = opts.announcementType || 'general'; + const priority = opts.priority || 'NORMAL'; + const isUrgent = priority === 'URGENT'; + const isPasswordReset = announcementType === 'password_reset'; + + // Check if email sending is enabled + if ( + !process.env.sendEmail || + (String(process.env.sendEmail).toLowerCase() === 'false' && !isPasswordReset) + ) { + return Promise.resolve('EMAIL_SENDING_DISABLED'); + } + + return new Promise((resolve, reject) => { + const recipientsArray = Array.isArray(recipients) ? recipients : [recipients]; + + // Enhanced metadata for announcements + const enhancedMeta = { + ...opts, + announcementType, + priority, + timestamp: new Date(), + recipientCount: recipientsArray.length, + isUrgent, + }; + + // Process recipients in batches + for (let i = 0; i < recipientsArray.length; i += this.config.batchSize) { + const batchRecipients = recipientsArray.slice(i, i + this.config.batchSize); + + this.queue.push({ + from: this.config.email, + to: batchRecipients.length ? batchRecipients.join(',') : '', + bcc: emailBccs ? emailBccs.join(',') : '', + subject, + html: message, + attachments, + cc, + replyTo, + meta: enhancedMeta, + }); + } + + // Process queue immediately for urgent announcements + if (isUrgent) { + // Move urgent emails to front of queue + const urgentBatches = this.queue.filter((batch) => batch.meta.isUrgent); + const normalBatches = this.queue.filter((batch) => !batch.meta.isUrgent); + this.queue = [...urgentBatches, ...normalBatches]; + } + + setImmediate(async () => { + try { + const result = await this.processQueue(); + if (result === false) { + reject(new Error('Announcement email sending failed after all retries')); + } else { + // Return the last successful Gmail response for audit logging + const lastBatch = this.queue[this.queue.length - 1]; + if (lastBatch && lastBatch.gmailResponse) { + resolve(lastBatch.gmailResponse); + } else { + resolve(result); + } + } + } catch (error) { + reject(error); + } + }); + }); + } + + /** + * Send announcement summary notification + */ + async sendAnnouncementSummary(recipientEmail, summary) { + const summaryHtml = ` +

Announcement Summary

+

Total Recipients: ${summary.totalRecipients}

+

Successfully Sent: ${summary.sent}

+

Failed: ${summary.failed}

+

Success Rate: ${summary.successRate}%

+

Processing Time: ${summary.processingTime}

+ ${summary.errors.length > 0 ? `

Errors:

    ${summary.errors.map((e) => `
  • ${e}
    • `).join('')}
` : ''} + `; + + return this.sendAnnouncement( + recipientEmail, + 'Announcement Summary Report', + summaryHtml, + null, + null, + null, + null, + { + announcementType: 'summary', + priority: 'LOW', + }, + ); + } + + /** + * Sleep utility + */ + static sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); + } + + /** + * Get service status + */ + getStatus() { + return { + isProcessing: this.isProcessing, + queueLength: this.queue.length, + config: { + batchSize: this.config.batchSize, + concurrency: this.config.concurrency, + rateLimitDelay: this.config.rateLimitDelay, + }, + }; + } +} + +// Create singleton instance +const emailAnnouncementService = new EmailAnnouncementService(); + +module.exports = emailAnnouncementService; diff --git a/src/services/emailBatchAuditService.js b/src/services/emailBatchAuditService.js new file mode 100644 index 000000000..924b12c04 --- /dev/null +++ b/src/services/emailBatchAuditService.js @@ -0,0 +1,259 @@ +/** + * Email Batch Audit Service + * Centralized audit management for email batch operations + */ + +const EmailBatchAudit = require('../models/emailBatchAudit'); +const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +const logger = require('../startup/logger'); + +class EmailBatchAuditService { + /** + * Log an action to the audit trail + */ + static async logAction( + emailId, + action, + details, + metadata = {}, + error = null, + triggeredBy = null, + emailBatchId = null, + ) { + try { + const audit = new EmailBatchAudit({ + emailId, + emailBatchId, + action, + details, + metadata, + error: error?.message, + errorCode: error?.code, + triggeredBy, + }); + + await audit.save(); + return audit; + } catch (err) { + logger.logException(err, 'Error logging audit action'); + throw err; + } + } + + /** + * Get complete audit trail for an email (main batch) + */ + static async getEmailAuditTrail(emailId, page = 1, limit = 50, action = null) { + const query = { emailId }; + + // Add action filter if provided + if (action) { + query.action = action; + } + + const skip = (page - 1) * limit; + + const auditTrail = await EmailBatchAudit.find(query) + .sort({ timestamp: 1 }) + .populate('triggeredBy', 'firstName lastName email') + .populate('emailBatchId', 'recipients emailType') + .skip(skip) + .limit(limit); + + const totalCount = await EmailBatchAudit.countDocuments(query); + const totalPages = Math.ceil(totalCount / limit); + + return { + auditTrail, + totalCount, + page, + totalPages, + limit, + }; + } + + /** + * Get audit trail for a specific email batch item + */ + static async getEmailBatchAuditTrail(emailBatchId, page = 1, limit = 50, action = null) { + const query = { emailBatchId }; + + // Add action filter if provided + if (action) { + query.action = action; + } + + const skip = (page - 1) * limit; + + const auditTrail = await EmailBatchAudit.find(query) + .sort({ timestamp: 1 }) + .populate('triggeredBy', 'firstName lastName email') + .populate('emailId', 'subject batchId') + .skip(skip) + .limit(limit); + + const totalCount = await EmailBatchAudit.countDocuments(query); + const totalPages = Math.ceil(totalCount / limit); + + return { + auditTrail, + totalCount, + page, + totalPages, + limit, + }; + } + + /** + * Get system-wide audit statistics + */ + static async getAuditStats(dateFrom = null, dateTo = null) { + const matchStage = {}; + if (dateFrom || dateTo) { + matchStage.timestamp = {}; + if (dateFrom) matchStage.timestamp.$gte = new Date(dateFrom); + if (dateTo) matchStage.timestamp.$lte = new Date(dateTo); + } + + return EmailBatchAudit.aggregate([ + { $match: matchStage }, + { + $group: { + _id: '$action', + count: { $sum: 1 }, + avgProcessingTime: { + $avg: '$processingContext.processingTime', + }, + }, + }, + ]); + } + + /** + * Log email creation + */ + static async logEmailCreated(emailId, createdBy, metadata = {}) { + return this.logAction( + emailId, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.BATCH_CREATED, + `Email created with ID: ${emailId}`, + metadata, + null, + createdBy, + ); + } + + /** + * Log email processing start + */ + static async logEmailStarted(emailId, metadata = {}) { + return this.logAction( + emailId, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.BATCH_STARTED, + `Email processing started`, + metadata, + ); + } + + /** + * Log email processing completion + */ + static async logEmailCompleted(emailId, metadata = {}) { + return this.logAction( + emailId, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.BATCH_COMPLETED, + `Email processing completed successfully`, + metadata, + ); + } + + /** + * Log email processing failure + */ + static async logEmailFailed(emailId, error, metadata = {}) { + return this.logAction( + emailId, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.BATCH_FAILED, + `Email processing failed`, + metadata, + error, + ); + } + + /** + * Log email batch item sent with essential delivery tracking + */ + static async logEmailBatchSent(emailId, emailBatchId, metadata = {}, gmailResponse = null) { + const includeApiDetails = + process.env.NODE_ENV === 'development' || process.env.LOG_API_DETAILS === 'true'; + + const enhancedMetadata = { + ...metadata, + // Include essential delivery tracking details + ...(includeApiDetails && gmailResponse + ? { + deliveryStatus: { + messageId: gmailResponse.messageId, + accepted: gmailResponse.accepted, + rejected: gmailResponse.rejected, + }, + quotaInfo: { + quotaRemaining: gmailResponse.quotaRemaining, + quotaResetTime: gmailResponse.quotaResetTime, + }, + } + : {}), + }; + + return this.logAction( + emailId, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.ITEM_SENT, + `Email batch item sent successfully`, + enhancedMetadata, + null, + null, + emailBatchId, + ); + } + + /** + * Log email batch item failure with optional Gmail API metadata + */ + static async logEmailBatchFailed(emailId, emailBatchId, error, metadata = {}) { + const includeApiDetails = true; + + const enhancedMetadata = { + ...metadata, + // Include essential error tracking details + ...(includeApiDetails && error?.gmailResponse + ? { + deliveryStatus: { + messageId: error.gmailResponse.messageId, + accepted: error.gmailResponse.accepted, + rejected: error.gmailResponse.rejected, + }, + quotaInfo: { + quotaRemaining: error.gmailResponse.quotaRemaining, + quotaResetTime: error.gmailResponse.quotaResetTime, + }, + errorDetails: { + errorCode: error.gmailResponse.errorCode, + errorMessage: error.gmailResponse.errorMessage, + }, + } + : {}), + }; + + return this.logAction( + emailId, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.ITEM_FAILED, + `Email batch item failed to send`, + enhancedMetadata, + error, + null, + emailBatchId, + ); + } +} + +module.exports = EmailBatchAuditService; diff --git a/src/services/emailBatchDashboardService.js b/src/services/emailBatchDashboardService.js deleted file mode 100644 index 7fe8cd69f..000000000 --- a/src/services/emailBatchDashboardService.js +++ /dev/null @@ -1,391 +0,0 @@ -/** - * Enhanced Email Batch Dashboard Service - * - * FEATURES: - * - Caching for performance - * - Pagination for large datasets - * - Real-time updates - * - Performance analytics - * - Memory optimization - */ - -const EmailBatch = require('../models/emailBatch'); -const EmailBatchItem = require('../models/emailBatchItem'); -const logger = require('../startup/logger'); - -class EmailBatchDashboardService { - constructor() { - this.cache = new Map(); - this.cacheTimeout = 30000; // 30 seconds - this.maxCacheSize = 100; - } - - /** - * Get dashboard data with caching and performance optimization - */ - async getDashboardData(filters = {}) { - const cacheKey = this.generateCacheKey('dashboard', filters); - - // Check cache first - if (this.cache.has(cacheKey)) { - const cached = this.cache.get(cacheKey); - if (Date.now() - cached.timestamp < this.cacheTimeout) { - return cached.data; - } - } - - try { - const data = await this.fetchDashboardData(filters); - - // Cache the result - this.setCache(cacheKey, data); - - return data; - } catch (error) { - logger.logException(error, 'Error fetching dashboard data'); - throw error; - } - } - - /** - * Fetch dashboard data with optimized queries - */ - async fetchDashboardData(filters) { - const query = this.buildQuery(filters); - - // Use aggregation pipeline for better performance - const [overviewStats, emailStats, performanceStats, recentBatches] = await Promise.all([ - this.getOverviewStats(query), - this.getEmailStats(query), - this.getPerformanceStats(query), - this.getRecentBatches(query, 10), - ]); - - return { - overview: overviewStats, - emailStats, - performance: performanceStats, - recentBatches, - filters, - timestamp: new Date(), - }; - } - - /** - * Get overview statistics - */ - static async getOverviewStats(query) { - const stats = await EmailBatch.aggregate([ - { $match: query }, - { - $group: { - _id: null, - totalBatches: { $sum: 1 }, - pendingBatches: { $sum: { $cond: [{ $eq: ['$status', 'PENDING'] }, 1, 0] } }, - processingBatches: { $sum: { $cond: [{ $eq: ['$status', 'PROCESSING'] }, 1, 0] } }, - completedBatches: { $sum: { $cond: [{ $eq: ['$status', 'COMPLETED'] }, 1, 0] } }, - failedBatches: { $sum: { $cond: [{ $eq: ['$status', 'FAILED'] }, 1, 0] } }, - }, - }, - ]); - - return ( - stats[0] || { - totalBatches: 0, - pendingBatches: 0, - processingBatches: 0, - completedBatches: 0, - failedBatches: 0, - } - ); - } - - /** - * Get email statistics - */ - static async getEmailStats(query) { - const stats = await EmailBatch.aggregate([ - { $match: query }, - { - $group: { - _id: null, - totalEmails: { $sum: '$totalEmails' }, - sentEmails: { $sum: '$sentEmails' }, - failedEmails: { $sum: '$failedEmails' }, - pendingEmails: { $sum: '$pendingEmails' }, - }, - }, - ]); - - const result = stats[0] || { - totalEmails: 0, - sentEmails: 0, - failedEmails: 0, - pendingEmails: 0, - }; - - result.successRate = - result.totalEmails > 0 ? Math.round((result.sentEmails / result.totalEmails) * 100) : 0; - - return result; - } - - /** - * Get performance statistics - */ - static async getPerformanceStats(query) { - const stats = await EmailBatch.aggregate([ - { - $match: { - ...query, - status: 'COMPLETED', - startedAt: { $exists: true }, - }, - }, - { - $project: { - processingTime: { $subtract: ['$completedAt', '$startedAt'] }, - totalEmails: 1, - createdAt: 1, - }, - }, - { - $group: { - _id: null, - avgProcessingTime: { $avg: '$processingTime' }, - avgEmailsPerBatch: { $avg: '$totalEmails' }, - totalProcessingTime: { $sum: '$processingTime' }, - batchCount: { $sum: 1 }, - }, - }, - ]); - - const result = stats[0] || { - avgProcessingTime: null, - avgEmailsPerBatch: 0, - totalProcessingTime: 0, - batchCount: 0, - }; - - return { - avgProcessingTime: result.avgProcessingTime - ? Math.round(result.avgProcessingTime / 1000) - : null, - avgEmailsPerBatch: Math.round(result.avgEmailsPerBatch || 0), - totalProcessingTime: result.totalProcessingTime - ? Math.round(result.totalProcessingTime / 1000) - : 0, - batchCount: result.batchCount, - }; - } - - /** - * Get recent batches with pagination - */ - static async getRecentBatches(query, limit = 10) { - const batches = await EmailBatch.find(query) - .sort({ createdAt: -1 }) - .limit(limit) - .populate('createdBy', 'firstName lastName email') - .select( - 'batchId name status totalEmails sentEmails failedEmails progress createdAt completedAt subject createdBy', - ); - - return batches.map((batch) => ({ - batchId: batch.batchId, - name: batch.name, - status: batch.status, - totalEmails: batch.totalEmails, - sentEmails: batch.sentEmails, - failedEmails: batch.failedEmails, - progress: batch.progress, - createdAt: batch.createdAt, - completedAt: batch.completedAt, - subject: batch.subject, - createdBy: batch.createdBy, - })); - } - - /** - * Get batch details with pagination for large batches - */ - async getBatchDetails(batchId, page = 1, limit = 100) { - const cacheKey = this.generateCacheKey('batchDetails', { batchId, page, limit }); - - if (this.cache.has(cacheKey)) { - const cached = this.cache.get(cacheKey); - if (Date.now() - cached.timestamp < this.cacheTimeout) { - return cached.data; - } - } - - try { - const batch = await EmailBatch.findOne({ batchId }).populate( - 'createdBy', - 'firstName lastName email', - ); - - if (!batch) { - throw new Error('Batch not found'); - } - - // Get batch items with pagination - const skip = (page - 1) * limit; - const [batchItems, totalItems] = await Promise.all([ - EmailBatchItem.find({ batchId: batch._id }) - .select('recipients status attempts sentAt failedAt error createdAt') - .sort({ createdAt: 1 }) - .limit(limit) - .skip(skip), - EmailBatchItem.countDocuments({ batchId: batch._id }), - ]); - - const data = { - batch: { - batchId: batch.batchId, - name: batch.name, - description: batch.description, - status: batch.status, - subject: batch.subject, - htmlContent: batch.htmlContent, - attachments: batch.attachments || [], - metadata: batch.metadata, - createdBy: batch.createdBy, - createdByName: batch.createdByName, - createdByEmail: batch.createdByEmail, - createdAt: batch.createdAt, - updatedAt: batch.updatedAt, - startedAt: batch.startedAt, - completedAt: batch.completedAt, - }, - statistics: { - totalEmails: batch.totalEmails, - sentEmails: batch.sentEmails, - failedEmails: batch.failedEmails, - pendingEmails: batch.pendingEmails, - progress: batch.progress, - }, - items: batchItems, - pagination: { - page, - limit, - total: totalItems, - pages: Math.ceil(totalItems / limit), - }, - }; - - this.setCache(cacheKey, data); - return data; - } catch (error) { - logger.logException(error, 'Error fetching batch details'); - throw error; - } - } - - /** - * Get batches with advanced filtering and pagination - */ - async getBatches(filters = {}, page = 1, limit = 20) { - const query = this.buildQuery(filters); - const skip = (page - 1) * limit; - - const [batches, total] = await Promise.all([ - EmailBatch.find(query) - .sort({ createdAt: -1 }) - .limit(limit) - .skip(skip) - .populate('createdBy', 'firstName lastName email') - .select( - 'batchId name status totalEmails sentEmails failedEmails progress createdAt completedAt subject createdBy', - ), - EmailBatch.countDocuments(query), - ]); - - return { - batches: batches.map((batch) => ({ - batchId: batch.batchId, - name: batch.name, - status: batch.status, - totalEmails: batch.totalEmails, - sentEmails: batch.sentEmails, - failedEmails: batch.failedEmails, - progress: batch.progress, - createdAt: batch.createdAt, - completedAt: batch.completedAt, - subject: batch.subject, - createdBy: batch.createdBy, - })), - pagination: { - page, - limit, - total, - pages: Math.ceil(total / limit), - }, - }; - } - - /** - * Build query from filters - */ - static buildQuery(filters) { - const query = {}; - - if (filters.dateFrom) { - query.createdAt = { $gte: new Date(filters.dateFrom) }; - } - if (filters.dateTo) { - query.createdAt = { ...query.createdAt, $lte: new Date(filters.dateTo) }; - } - if (filters.status) { - query.status = filters.status; - } - if (filters.type) { - query['metadata.type'] = filters.type; - } - if (filters.createdBy) { - query.createdBy = filters.createdBy; - } - - return query; - } - - /** - * Cache management - */ - static generateCacheKey(prefix, params) { - return `${prefix}_${JSON.stringify(params)}`; - } - - setCache(key, data) { - // Implement LRU cache - if (this.cache.size >= this.maxCacheSize) { - const firstKey = this.cache.keys().next().value; - this.cache.delete(firstKey); - } - - this.cache.set(key, { - data, - timestamp: Date.now(), - }); - } - - /** - * Clear cache - */ - clearCache() { - this.cache.clear(); - } - - /** - * Get cache statistics - */ - getCacheStats() { - return { - size: this.cache.size, - maxSize: this.maxCacheSize, - timeout: this.cacheTimeout, - }; - } -} - -module.exports = new EmailBatchDashboardService(); diff --git a/src/services/emailBatchProcessor.js b/src/services/emailBatchProcessor.js index 28d9be05b..8d04c2783 100644 --- a/src/services/emailBatchProcessor.js +++ b/src/services/emailBatchProcessor.js @@ -1,17 +1,19 @@ /** - * Simplified Email Batch Processor - Production Ready - * Focus: Efficient processing with email body from batch record + * Enhanced Email Batch Processor - Production Ready with Audit Integration + * Focus: Efficient processing with email body from batch record and comprehensive auditing */ +const Email = require('../models/email'); const EmailBatch = require('../models/emailBatch'); -const EmailBatchItem = require('../models/emailBatchItem'); -const emailSender = require('../utilities/emailSender'); +const emailAnnouncementService = require('./emailAnnouncementService'); +const EmailBatchAuditService = require('./emailBatchAuditService'); +const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); const logger = require('../startup/logger'); class EmailBatchProcessor { constructor() { this.processingBatches = new Set(); - this.maxRetries = 3; + this.maxRetries = EMAIL_JOB_CONFIG.DEFAULT_MAX_RETRIES; this.retryDelay = 2000; // 2 seconds } @@ -27,19 +29,22 @@ class EmailBatchProcessor { try { console.log('🔍 Looking for batch with batchId:', batchId); - const batch = await EmailBatch.findOne({ batchId }); + const batch = await Email.findOne({ batchId }); if (!batch) { console.error('❌ Batch not found with batchId:', batchId); throw new Error('Batch not found'); } console.log('✅ Found batch:', batch.batchId, 'Status:', batch.status); - if (batch.status === 'COMPLETED' || batch.status === 'FAILED') { + if ( + batch.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT || + batch.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED + ) { return; } // Update batch status - batch.status = 'PROCESSING'; + batch.status = EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING; batch.startedAt = new Date(); await batch.save(); @@ -54,9 +59,9 @@ class EmailBatchProcessor { // Mark batch as failed try { - const batch = await EmailBatch.findOne({ batchId }); + const batch = await Email.findOne({ batchId }); if (batch) { - batch.status = 'FAILED'; + batch.status = EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED; batch.completedAt = new Date(); await batch.save(); } @@ -72,9 +77,9 @@ class EmailBatchProcessor { * Process all items in a batch */ async processBatchItems(batch) { - const items = await EmailBatchItem.find({ + const items = await EmailBatch.find({ batchId: batch._id, - status: 'PENDING', + status: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED, }); // Process items in parallel with concurrency limit @@ -89,13 +94,13 @@ class EmailBatchProcessor { const processWithRetry = async (attempt = 1) => { try { // Update to SENDING status (this increments attempts) - await item.updateStatus('SENDING'); + await item.updateStatus(EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENDING); // Extract recipient emails from the batch item const recipientEmails = item.recipients.map((recipient) => recipient.email); - // Use the existing emailSender with batched recipients and email body from batch - await emailSender( + // Use the new emailAnnouncementService with enhanced announcement features + const gmailResponse = await emailAnnouncementService.sendAnnouncement( recipientEmails, // Array of emails for batching batch.subject, batch.htmlContent, // Use email body from batch record @@ -104,16 +109,30 @@ class EmailBatchProcessor { null, // replyTo null, // bcc { - type: 'batch_send', + announcementType: 'batch_send', batchId: batch.batchId, itemId: item._id, emailType: item.emailType, recipientCount: recipientEmails.length, + priority: 'NORMAL', }, ); // Mark as sent - await item.updateStatus('SENT'); + await item.updateStatus(EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENT); + + // Log successful send to audit + await EmailBatchAuditService.logEmailBatchSent( + batch._id, + item._id, + { + recipientCount: recipientEmails.length, + emailType: item.emailType, + attempt: item.attempts, + }, + gmailResponse, + ); + logger.logInfo( `Email batch sent successfully to ${recipientEmails.length} recipients (attempt ${item.attempts})`, ); @@ -125,7 +144,19 @@ class EmailBatchProcessor { ); if (attempt >= this.maxRetries) { - await item.updateStatus('FAILED', error.message); + await item.updateStatus( + EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.FAILED, + error.message, + error.code, + ); + + // Log failed send to audit + await EmailBatchAuditService.logEmailBatchFailed(batch._id, item._id, error, { + recipientCount: item.recipients.length, + emailType: item.emailType, + finalAttempt: attempt, + }); + logger.logError( `Permanently failed to send email batch to ${item.recipients.length} recipients after ${this.maxRetries} attempts`, ); @@ -155,12 +186,12 @@ class EmailBatchProcessor { */ async retryBatchItem(itemId) { try { - const item = await EmailBatchItem.findById(itemId); + const item = await EmailBatch.findById(itemId); if (!item) { throw new Error('Batch item not found'); } - const batch = await EmailBatch.findById(item.batchId); + const batch = await Email.findById(item.batchId); if (!batch) { throw new Error('Parent batch not found'); } diff --git a/src/services/emailBatchService.js b/src/services/emailBatchService.js index 9a8d3502d..1b77dbcc6 100644 --- a/src/services/emailBatchService.js +++ b/src/services/emailBatchService.js @@ -1,11 +1,13 @@ /** - * Simplified Email Batch Service - Production Ready - * Focus: Efficient batching with email body storage + * Enhanced Email Batch Service - Production Ready with Job Queue Support + * Focus: Efficient batching with email body storage and job queue management */ const { v4: uuidv4 } = require('uuid'); +const Email = require('../models/email'); const EmailBatch = require('../models/emailBatch'); -const EmailBatchItem = require('../models/emailBatchItem'); +const EmailBatchAuditService = require('./emailBatchAuditService'); +const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); const logger = require('../startup/logger'); class EmailBatchService { @@ -14,11 +16,11 @@ class EmailBatchService { } /** - * Create a new email batch with email body + * Create a new email batch with email body and job queue support */ static async createBatch(batchData) { try { - const batch = new EmailBatch({ + const batch = new Email({ batchId: batchData.batchId || uuidv4(), subject: batchData.subject, htmlContent: batchData.htmlContent, // Store email body @@ -26,6 +28,13 @@ class EmailBatchService { }); await batch.save(); + + // Log batch creation to audit trail + await EmailBatchAuditService.logEmailCreated(batch._id, batchData.createdBy, { + batchId: batch.batchId, + subject: batch.subject, + }); + console.log('💾 Batch saved successfully:', { id: batch._id, batchId: batch.batchId, @@ -39,17 +48,17 @@ class EmailBatchService { } /** - * Add recipients to a batch with efficient batching + * Add recipients to a batch with efficient batching (uses config enums) */ static async addRecipients(batchId, recipients, batchConfig = {}) { try { - const batch = await EmailBatch.findOne({ batchId }); + const batch = await Email.findOne({ batchId }); if (!batch) { throw new Error('Batch not found'); } const batchSize = batchConfig.batchSize || 50; - const emailType = batchConfig.emailType || 'BCC'; + const emailType = batchConfig.emailType || EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC; // Create batch items with multiple recipients per item const batchItems = []; @@ -63,13 +72,13 @@ class EmailBatchService { email: recipient.email, // Only email, no name })), emailType, - status: 'PENDING', + status: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED, }; batchItems.push(batchItem); } - await EmailBatchItem.insertMany(batchItems); + await EmailBatch.insertMany(batchItems); return batch; } catch (error) { @@ -104,7 +113,10 @@ class EmailBatchService { // Add recipients with efficient batching const batchConfig = { batchSize: 50, // Use standard batch size - emailType: recipients.length === 1 ? 'TO' : 'BCC', // Single recipient uses TO, multiple use BCC + emailType: + recipients.length === 1 + ? EMAIL_JOB_CONFIG.EMAIL_TYPES.TO + : EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC, // Single recipient uses TO, multiple use BCC }; // Convert recipients to proper format @@ -124,7 +136,7 @@ class EmailBatchService { */ static async getBatchWithItems(batchId) { try { - const batch = await EmailBatch.findOne({ batchId }).populate( + const batch = await Email.findOne({ batchId }).populate( 'createdBy', 'firstName lastName email', ); @@ -133,7 +145,7 @@ class EmailBatchService { return null; } - const items = await EmailBatchItem.find({ batchId: batch._id }).sort({ createdAt: 1 }); + const items = await EmailBatch.find({ batchId: batch._id }).sort({ createdAt: 1 }); // Get dynamic counts const counts = await batch.getEmailCounts(); @@ -181,12 +193,12 @@ class EmailBatchService { const skip = (page - 1) * limit; const [batches, total] = await Promise.all([ - EmailBatch.find(query) + Email.find(query) .sort({ createdAt: -1 }) .limit(limit) .skip(skip) .populate('createdBy', 'firstName lastName email'), - EmailBatch.countDocuments(query), + Email.countDocuments(query), ]); // Add dynamic counts to each batch @@ -222,15 +234,15 @@ class EmailBatchService { try { const [totalBatches, pendingBatches, processingBatches, completedBatches, failedBatches] = await Promise.all([ - EmailBatch.countDocuments(), - EmailBatch.countDocuments({ status: 'PENDING' }), - EmailBatch.countDocuments({ status: 'PROCESSING' }), - EmailBatch.countDocuments({ status: 'COMPLETED' }), - EmailBatch.countDocuments({ status: 'FAILED' }), + Email.countDocuments(), + Email.countDocuments({ status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.QUEUED }), + Email.countDocuments({ status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING }), + Email.countDocuments({ status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT }), + Email.countDocuments({ status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED }), ]); // Calculate email stats dynamically from batch items - const emailStats = await EmailBatchItem.aggregate([ + const emailStats = await EmailBatch.aggregate([ { $group: { _id: null, @@ -240,7 +252,12 @@ class EmailBatchService { sentEmails: { $sum: { $cond: [ - { $and: [{ $eq: ['$status', 'SENT'] }, { $isArray: '$recipients' }] }, + { + $and: [ + { $eq: ['$status', EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENT] }, + { $isArray: '$recipients' }, + ], + }, { $size: '$recipients' }, 0, ], diff --git a/src/startup/routes.js b/src/startup/routes.js index 731294e38..2aae95baa 100644 --- a/src/startup/routes.js +++ b/src/startup/routes.js @@ -153,7 +153,6 @@ const ownerMessageRouter = require('../routes/ownerMessageRouter')(ownerMessage) const emailRouter = require('../routes/emailRouter')(); const emailBatchRouter = require('../routes/emailBatchRoutes'); -const emailBatchDashboardRouter = require('../routes/emailBatchDashboardRoutes'); const reasonRouter = require('../routes/reasonRouter')(reason, userProfile); const mouseoverTextRouter = require('../routes/mouseoverTextRouter')(mouseoverText); @@ -336,7 +335,6 @@ module.exports = function (app) { app.use('/api', permissionChangeLogRouter); app.use('/api', emailRouter); app.use('/api/email-batches', emailBatchRouter); - app.use('/api/email-batches', emailBatchDashboardRouter); app.use('/api', isEmailExistsRouter); app.use('/api', faqRouter); app.use('/api', mapLocationRouter); From e4b01b14c4c84aabce803606c3e9d208702360d0 Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Thu, 30 Oct 2025 10:50:10 -0400 Subject: [PATCH 03/19] feat(email): implement email processing and batch management features - Added EmailBatch and Email models for managing email batches and their recipients. - Introduced EmailBatchService and EmailService for handling batch creation, status updates, and email sending. - Implemented EmailProcessor for processing email batches with retry logic and status tracking. - Enhanced email announcement job processor to handle batch processing at scheduled intervals. - Updated email validation utilities for improved recipient handling and HTML content validation. - Added new endpoints in emailBatchController for managing email records and batch operations. --- src/config.js | 1 + src/config/emailJobConfig.js | 19 +- src/controllers/emailBatchController.js | 330 +++--- src/controllers/emailController.js | 974 ++++++++++++------ src/controllers/emailTemplateController.js | 778 ++++++++------ src/jobs/emailAnnouncementJobProcessor.js | 167 +-- src/models/email.js | 179 ---- src/models/emailBatch.js | 77 +- src/models/emailBatchAudit.js | 51 - ...EmailTemplateModel.js => emailTemplate.js} | 5 - src/routes/emailBatchRoutes.js | 20 +- src/routes/emailRouter.js | 2 + src/routes/emailTemplateRouter.js | 1 - src/services/emailAnnouncementService.js | 351 +++---- src/services/emailBatchAuditService.js | 237 +++-- src/services/emailBatchProcessor.js | 228 ---- src/services/emailBatchService.js | 385 +++---- src/services/emailProcessor.js | 429 ++++++++ src/services/emailService.js | 93 ++ src/utilities/emailSender.js | 37 +- src/utilities/emailValidators.js | 125 +++ 21 files changed, 2496 insertions(+), 1993 deletions(-) rename src/models/{EmailTemplateModel.js => emailTemplate.js} (93%) delete mode 100644 src/services/emailBatchProcessor.js create mode 100644 src/services/emailProcessor.js create mode 100644 src/services/emailService.js create mode 100644 src/utilities/emailValidators.js diff --git a/src/config.js b/src/config.js index 5f2c27d71..ac38960c3 100644 --- a/src/config.js +++ b/src/config.js @@ -11,5 +11,6 @@ config.JWT_HEADER = { alg: 'RS256', typ: 'JWT', }; +config.FRONT_END_URL = process.env.FRONT_END_URL; module.exports = config; diff --git a/src/config/emailJobConfig.js b/src/config/emailJobConfig.js index c1649ef9b..cde5fe003 100644 --- a/src/config/emailJobConfig.js +++ b/src/config/emailJobConfig.js @@ -19,7 +19,6 @@ const EMAIL_JOB_CONFIG = { SENT: 'SENT', // All emails successfully sent PROCESSED: 'PROCESSED', // Processing finished (mixed results) FAILED: 'FAILED', // Failed to send - CANCELLED: 'CANCELLED', // Cancelled by user }, EMAIL_BATCH_STATUSES: { @@ -27,25 +26,21 @@ const EMAIL_JOB_CONFIG = { SENDING: 'SENDING', // Currently sending SENT: 'SENT', // Successfully delivered FAILED: 'FAILED', // Delivery failed - RESENDING: 'RESENDING', // Resending delivery }, EMAIL_BATCH_AUDIT_ACTIONS: { // Email-level actions (main batch) - EMAIL_CREATED: 'EMAIL_CREATED', EMAIL_QUEUED: 'EMAIL_QUEUED', EMAIL_SENDING: 'EMAIL_SENDING', EMAIL_SENT: 'EMAIL_SENT', EMAIL_PROCESSED: 'EMAIL_PROCESSED', EMAIL_FAILED: 'EMAIL_FAILED', - EMAIL_CANCELLED: 'EMAIL_CANCELLED', // Email batch item-level actions EMAIL_BATCH_QUEUED: 'EMAIL_BATCH_QUEUED', EMAIL_BATCH_SENDING: 'EMAIL_BATCH_SENDING', EMAIL_BATCH_SENT: 'EMAIL_BATCH_SENT', EMAIL_BATCH_FAILED: 'EMAIL_BATCH_FAILED', - EMAIL_BATCH_RESENDING: 'EMAIL_BATCH_RESENDING', }, EMAIL_TYPES: { @@ -53,6 +48,20 @@ const EMAIL_JOB_CONFIG = { CC: 'CC', BCC: 'BCC', }, + + // Centralized limits to keep model, services, and controllers consistent + LIMITS: { + MAX_RECIPIENTS_PER_REQUEST: 1000, // Must match EmailBatch.recipients validator + MAX_HTML_BYTES: 1 * 1024 * 1024, // 1MB - Reduced since base64 media files are blocked + SUBJECT_MAX_LENGTH: 200, // Standardized subject length limit + }, + + // Announcement service runtime knobs + ANNOUNCEMENTS: { + BATCH_SIZE: 50, // recipients per SMTP send batch + CONCURRENCY: 3, // concurrent SMTP batches + RATE_LIMIT_DELAY_MS: 1000, // delay between queue cycles when more remain + }, }; module.exports = { EMAIL_JOB_CONFIG }; diff --git a/src/controllers/emailBatchController.js b/src/controllers/emailBatchController.js index 752ccedbb..38796f15f 100644 --- a/src/controllers/emailBatchController.js +++ b/src/controllers/emailBatchController.js @@ -1,53 +1,54 @@ -/** - * Simplified Email Batch Controller - Production Ready - * Focus: Essential batch management endpoints - */ - +const mongoose = require('mongoose'); const EmailBatchService = require('../services/emailBatchService'); -const emailBatchProcessor = require('../services/emailBatchProcessor'); +const EmailService = require('../services/emailService'); const EmailBatchAuditService = require('../services/emailBatchAuditService'); +const emailAnnouncementJobProcessor = require('../jobs/emailAnnouncementJobProcessor'); +const EmailBatch = require('../models/emailBatch'); +const Email = require('../models/email'); const logger = require('../startup/logger'); +const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); /** - * Get all batches with pagination and filtering + * Get all Email records (parent) */ -const getBatches = async (req, res) => { +const getEmails = async (req, res) => { try { - const { page = 1, limit = 20, status, dateFrom, dateTo } = req.query; - - const filters = { status, dateFrom, dateTo }; - const result = await EmailBatchService.getBatches( - filters, - parseInt(page, 10), - parseInt(limit, 10), - ); + const emails = await EmailBatchService.getAllEmails(); res.status(200).json({ success: true, - data: result, + data: emails, }); } catch (error) { - logger.logException(error, 'Error getting batches'); + logger.logException(error, 'Error getting emails'); res.status(500).json({ success: false, - message: 'Error getting batches', + message: 'Error getting emails', error: error.message, }); } }; /** - * Get batch details with items + * Get Email details with EmailBatch items */ -const getBatchDetails = async (req, res) => { +const getEmailDetails = async (req, res) => { try { - const { batchId } = req.params; - const result = await EmailBatchService.getBatchWithItems(batchId); + const { emailId } = req.params; // emailId is now the ObjectId of parent Email + + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + return res.status(400).json({ + success: false, + message: 'Valid Email ID is required', + }); + } + + const result = await EmailBatchService.getEmailWithBatches(emailId); if (!result) { return res.status(404).json({ success: false, - message: 'Batch not found', + message: 'Email not found', }); } @@ -56,134 +57,201 @@ const getBatchDetails = async (req, res) => { data: result, }); } catch (error) { - logger.logException(error, 'Error getting batch details'); + logger.logException(error, 'Error getting Email details with EmailBatch items'); res.status(500).json({ success: false, - message: 'Error getting batch details', + message: 'Error getting email details', error: error.message, }); } }; /** - * Get dashboard statistics + * Get worker status (minimal info for frontend) */ -const getDashboardStats = async (req, res) => { +const getWorkerStatus = async (req, res) => { try { - const stats = await EmailBatchService.getDashboardStats(); + const workerStatus = emailAnnouncementJobProcessor.getWorkerStatus(); res.status(200).json({ success: true, - data: stats, + data: workerStatus, }); } catch (error) { - logger.logException(error, 'Error getting dashboard stats'); + logger.logException(error, 'Error getting worker status'); res.status(500).json({ success: false, - message: 'Error getting dashboard stats', + message: 'Error getting worker status', error: error.message, }); } }; /** - * Get processor status + * Retry an Email by queuing all its failed EmailBatch items + * Resets failed items to QUEUED status for the cron job to process */ -const getProcessorStatus = async (req, res) => { +const retryEmail = async (req, res) => { try { - const status = emailBatchProcessor.getStatus(); + const { emailId } = req.params; - res.status(200).json({ - success: true, - data: status, - }); - } catch (error) { - logger.logException(error, 'Error getting processor status'); - res.status(500).json({ - success: false, - message: 'Error getting processor status', - error: error.message, - }); - } -}; + // Validate emailId is a valid ObjectId + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + return res.status(400).json({ + success: false, + message: 'Invalid Email ID', + }); + } -// Retry a failed batch item -const retryBatchItem = async (req, res) => { - try { - const { itemId } = req.params; + // Get requestor for audit trail + const requestorId = req?.body?.requestor?.requestorId || null; - // Find the batch item - const EmailBatch = require('../models/emailBatch'); - const item = await EmailBatch.findById(itemId); + // Find the Email + const email = await Email.findById(emailId); - if (!item) { + if (!email) { return res.status(404).json({ success: false, - message: 'Batch item not found', + message: 'Email not found', }); } - // Check if item is already being processed - if (item.status === 'SENDING') { + // Only allow retry for emails in final states (FAILED or PROCESSED) + const allowedRetryStatuses = [ + EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED, + EMAIL_JOB_CONFIG.EMAIL_STATUSES.PROCESSED, + ]; + + if (!allowedRetryStatuses.includes(email.status)) { return res.status(400).json({ success: false, - message: 'Batch item is currently being processed', + message: `Email must be in FAILED or PROCESSED status to retry. Current status: ${email.status}`, }); } - // Only allow retry for FAILED or QUEUED items - if (item.status !== 'FAILED' && item.status !== 'QUEUED') { - return res.status(400).json({ - success: false, - message: 'Only failed or pending items can be retried', + // Find all FAILED EmailBatch items + const failedItems = await EmailBatch.find({ + emailId: email._id, + status: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.FAILED, + }); + + if (failedItems.length === 0) { + logger.logInfo(`Email ${emailId} has no failed EmailBatch items to retry`); + return res.status(200).json({ + success: true, + message: 'No failed EmailBatch items to retry', + data: { + emailId: email._id, + failedItemsRetried: 0, + }, }); } - // Reset the item status to QUEUED for retry - item.status = 'QUEUED'; - item.attempts = 0; - item.error = null; - item.failedAt = null; - item.lastAttemptedAt = null; - await item.save(); + logger.logInfo(`Queuing ${failedItems.length} failed EmailBatch items for retry: ${emailId}`); + + // First, queue the parent Email so cron picks it up + await EmailService.markEmailQueued(emailId); + + // Audit Email queued for retry (with requestor) + try { + await EmailBatchAuditService.logEmailQueued( + email._id, + { reason: 'Manual retry' }, + requestorId, + ); + } catch (auditErr) { + logger.logException(auditErr, 'Audit failure: EMAIL_QUEUED (retry)'); + } + + // Reset each failed item to QUEUED + await Promise.all( + failedItems.map(async (item) => { + await EmailBatchService.resetEmailBatchForRetry(item._id); - // Use the processor's retry method - const emailBatchProcessorService = require('../services/emailBatchProcessor'); - await emailBatchProcessorService.retryBatchItem(itemId); + // Audit retry queueing + try { + await EmailBatchAuditService.logAction( + email._id, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_BATCH_QUEUED, + 'EmailBatch item queued for retry', + { reason: 'Manual retry' }, + null, + requestorId, + item._id, + ); + } catch (auditErr) { + logger.logException(auditErr, 'Audit failure: EMAIL_BATCH_QUEUED (retry)'); + } + }), + ); - res.json({ + logger.logInfo( + `Successfully queued Email ${emailId} and ${failedItems.length} failed EmailBatch items for retry`, + ); + + res.status(200).json({ success: true, - message: 'Batch item retry initiated', + message: `Successfully queued ${failedItems.length} failed EmailBatch items for retry`, data: { - itemId: item._id, - status: item.status, - attempts: item.attempts, + emailId: email._id, + failedItemsRetried: failedItems.length, }, }); } catch (error) { - logger.logException(error, 'Error retrying batch item'); + logger.logException(error, 'Error retrying Email'); res.status(500).json({ success: false, - message: 'Error retrying batch item', + message: 'Error retrying Email', error: error.message, }); } }; /** - * Get audit trail for a specific batch + * Get audit trail for a specific Email */ const getEmailAuditTrail = async (req, res) => { try { + if (!req?.body?.requestor && !req?.user) { + return res.status(401).json({ + success: false, + message: 'Missing requestor', + }); + } + + // TODO: Re-enable permission check in future + // Permission check - commented out for now + // const requestor = req.body.requestor || req.user; + // const canViewAudits = await hasPermission(requestor, 'viewEmailAudits'); + // if (!canViewAudits) { + // return res.status(403).json({ + // success: false, + // message: 'You are not authorized to view email audits', + // }); + // } + const { emailId } = req.params; - const { page = 1, limit = 50, action } = req.query; - const auditTrail = await EmailBatchAuditService.getEmailAuditTrail( - emailId, - parseInt(page, 10), - parseInt(limit, 10), - action, - ); + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + return res.status(400).json({ + success: false, + message: 'Valid Email ID is required', + }); + } + + let auditTrail; + try { + auditTrail = await EmailBatchAuditService.getEmailAuditTrail(emailId); + } catch (serviceError) { + // Handle validation errors from service + if (serviceError.message.includes('required') || serviceError.message.includes('Invalid')) { + return res.status(400).json({ + success: false, + message: serviceError.message, + }); + } + throw serviceError; + } res.status(200).json({ success: true, @@ -200,19 +268,51 @@ const getEmailAuditTrail = async (req, res) => { }; /** - * Get audit trail for a specific batch item + * Get audit trail for a specific EmailBatch item */ const getEmailBatchAuditTrail = async (req, res) => { try { + if (!req?.body?.requestor && !req?.user) { + return res.status(401).json({ + success: false, + message: 'Missing requestor', + }); + } + + // TODO: Re-enable permission check in future + // Permission check - commented out for now + // const requestor = req.body.requestor || req.user; + // const canViewAudits = await hasPermission(requestor, 'viewEmailAudits'); + // if (!canViewAudits) { + // return res.status(403).json({ + // success: false, + // message: 'You are not authorized to view email audits', + // }); + // } + const { emailBatchId } = req.params; - const { page = 1, limit = 50, action } = req.query; - const auditTrail = await EmailBatchAuditService.getEmailBatchAuditTrail( - emailBatchId, - parseInt(page, 10), - parseInt(limit, 10), - action, - ); + // Validate emailBatchId is a valid ObjectId + if (!emailBatchId || !mongoose.Types.ObjectId.isValid(emailBatchId)) { + return res.status(400).json({ + success: false, + message: 'Invalid EmailBatch ID', + }); + } + + let auditTrail; + try { + auditTrail = await EmailBatchAuditService.getEmailBatchAuditTrail(emailBatchId); + } catch (serviceError) { + // Handle validation errors from service + if (serviceError.message.includes('required') || serviceError.message.includes('Invalid')) { + return res.status(400).json({ + success: false, + message: serviceError.message, + }); + } + throw serviceError; + } res.status(200).json({ success: true, @@ -228,37 +328,11 @@ const getEmailBatchAuditTrail = async (req, res) => { } }; -/** - * Get audit statistics - */ -const getAuditStats = async (req, res) => { - try { - const { dateFrom, dateTo, action } = req.query; - - const filters = { dateFrom, dateTo, action }; - const stats = await EmailBatchAuditService.getAuditStats(filters); - - res.status(200).json({ - success: true, - data: stats, - }); - } catch (error) { - logger.logException(error, 'Error getting audit stats'); - res.status(500).json({ - success: false, - message: 'Error getting audit stats', - error: error.message, - }); - } -}; - module.exports = { - getBatches, - getBatchDetails, - getDashboardStats, - getProcessorStatus, - retryBatchItem, + getEmails, + getEmailDetails, + getWorkerStatus, + retryEmail, getEmailAuditTrail, getEmailBatchAuditTrail, - getAuditStats, }; diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index b558eb50a..43999e4d6 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -1,198 +1,453 @@ // emailController.js +const mongoose = require('mongoose'); const jwt = require('jsonwebtoken'); -const cheerio = require('cheerio'); -const emailAnnouncementService = require('../services/emailAnnouncementService'); -const { hasPermission } = require('../utilities/permissions'); +const emailSender = require('../utilities/emailSender'); +const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +const { + isValidEmailAddress, + normalizeRecipientsToArray, + ensureHtmlWithinLimit, + validateHtmlMedia, +} = require('../utilities/emailValidators'); const EmailSubcriptionList = require('../models/emailSubcriptionList'); const userProfile = require('../models/userProfile'); const EmailBatchService = require('../services/emailBatchService'); +const EmailService = require('../services/emailService'); +const EmailBatchAuditService = require('../services/emailBatchAuditService'); +const config = require('../config'); +const logger = require('../startup/logger'); -const frontEndUrl = process.env.FRONT_END_URL || 'http://localhost:3000'; -const jwtSecret = process.env.JWT_SECRET || 'EmailSecret'; - -const handleContentToOC = (htmlContent) => - ` - - - - - - ${htmlContent} - - `; - -const handleContentToNonOC = (htmlContent, email) => - ` - - - - - - ${htmlContent} -

- If you would like to unsubscribe from these emails, please click - here -

- - `; - -function extractImagesAndCreateAttachments(html) { - const $ = cheerio.load(html); - const attachments = []; - - $('img').each((i, img) => { - const src = $(img).attr('src'); - if (src.startsWith('data:image')) { - const base64Data = src.split(',')[1]; - const _cid = `image-${i}`; - attachments.push({ - filename: `image-${i}.png`, - content: Buffer.from(base64Data, 'base64'), - cid: _cid, - }); - $(img).attr('src', `cid:${_cid}`); - } - }); - return { - html: $.html(), - attachments, - }; -} +const jwtSecret = process.env.JWT_SECRET; const sendEmail = async (req, res) => { - const canSendEmail = await hasPermission(req.body.requestor, 'sendEmails'); - if (!canSendEmail) { - res.status(403).send('You are not authorized to send emails.'); - return; + // TODO: Re-enable permission check in future + // Permission check - commented out for now + // const canSendEmail = await hasPermission(req.body.requestor, 'sendEmails'); + // if (!canSendEmail) { + // return res.status(403).json({ success: false, message: 'You are not authorized to send emails.' }); + // } + + // Requestor is still required for getting user ID for audit trail + if (!req?.body?.requestor?.requestorId) { + return res.status(401).json({ success: false, message: 'Missing requestor' }); } + try { - const { to, subject, html, useBatch = true } = req.body; - if (!subject || !html || !to) { - const missingFields = []; - if (!subject) missingFields.push('Subject'); - if (!html) missingFields.push('HTML content'); - if (!to) missingFields.push('Recipient email'); + const { to, subject, html } = req.body; + + const missingFields = []; + if (!subject) missingFields.push('Subject'); + if (!html) missingFields.push('HTML content'); + if (!to) missingFields.push('Recipient email'); + if (missingFields.length) { return res .status(400) - .send(`${missingFields.join(' and ')} ${missingFields.length > 1 ? 'are' : 'is'} required`); + .json({ + success: false, + message: `${missingFields.join(' and ')} ${missingFields.length > 1 ? 'are' : 'is'} required`, + }); + } + + // Validate HTML content size + if (!ensureHtmlWithinLimit(html)) { + return res.status(413).json({ + success: false, + message: `HTML content exceeds ${EMAIL_JOB_CONFIG.LIMITS.MAX_HTML_BYTES / (1024 * 1024)}MB limit`, + }); } - const { html: processedHtml, attachments } = extractImagesAndCreateAttachments(html); + // Validate subject length against config + if (subject && subject.length > EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH) { + return res + .status(400) + .json({ + success: false, + message: `Subject cannot exceed ${EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH} characters`, + }); + } - try { - // Convert to array if it's a string - const recipientsArray = Array.isArray(to) ? to : [to]; - - if (useBatch) { - // Use new batch system for better tracking and user experience - const user = await userProfile.findById(req.body.requestor.requestorId); - if (!user) { - return res.status(400).send('User not found'); + // Validate HTML does not contain base64-encoded media + const mediaValidation = validateHtmlMedia(html); + if (!mediaValidation.isValid) { + return res.status(400).json({ + success: false, + message: 'HTML contains embedded media files. Only URLs are allowed for media.', + errors: mediaValidation.errors, + }); + } + + // Validate that all template variables have been replaced + const templateVariableRegex = /\{\{(\w+)\}\}/g; + const unmatchedVariables = []; + let match = templateVariableRegex.exec(html); + while (match !== null) { + if (!unmatchedVariables.includes(match[1])) { + unmatchedVariables.push(match[1]); + } + match = templateVariableRegex.exec(html); + } + // Check subject as well + if (subject) { + templateVariableRegex.lastIndex = 0; + match = templateVariableRegex.exec(subject); + while (match !== null) { + if (!unmatchedVariables.includes(match[1])) { + unmatchedVariables.push(match[1]); } + match = templateVariableRegex.exec(subject); + } + } + if (unmatchedVariables.length > 0) { + return res.status(400).json({ + success: false, + message: + 'Email contains unreplaced template variables. Please ensure all variables are replaced before sending.', + errors: { + unmatchedVariables: `Found unreplaced variables: ${unmatchedVariables.join(', ')}`, + }, + }); + } + + try { + // Normalize, dedupe, and validate recipients FIRST + const recipientsArray = normalizeRecipientsToArray(to); + if (recipientsArray.length === 0) { + return res + .status(400) + .json({ success: false, message: 'At least one recipient email is required' }); + } + if (recipientsArray.length > EMAIL_JOB_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST) { + return res + .status(400) + .json({ + success: false, + message: `A maximum of ${EMAIL_JOB_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST} recipients are allowed per request`, + }); + } + const invalidRecipients = recipientsArray.filter((e) => !isValidEmailAddress(e)); + if (invalidRecipients.length) { + return res + .status(400) + .json({ + success: false, + message: 'One or more recipient emails are invalid', + invalidRecipients, + }); + } - // Create batch for this email send (this already adds recipients internally) - console.log('📧 Creating batch for email send...'); - const batch = await EmailBatchService.createSingleSendBatch( + // Always use batch system for tracking and progress + const user = await userProfile.findById(req.body.requestor.requestorId); + if (!user) { + return res.status(400).json({ success: false, message: 'Requestor not found' }); + } + + // Start MongoDB transaction + const session = await mongoose.startSession(); + session.startTransaction(); + + try { + // Create parent Email within transaction + const email = await EmailService.createEmail( { - to: recipientsArray, subject, - html: processedHtml, - attachments, + htmlContent: html, + createdBy: user._id, }, - user, + session, ); - console.log('✅ Batch created with recipients:', batch.batchId); - - // REMOVED: Immediate processing - batch will be processed by cron job - // emailBatchProcessor.processBatch(batch.batchId).catch((error) => { - // console.error('❌ Error processing batch:', error); - // }); + // Create EmailBatch items with all recipients (chunked automatically) within transaction + // Always use BCC for all recipients (sender goes in 'to' field) + const recipientObjects = recipientsArray.map((emailAddr) => ({ email: emailAddr })); + const inserted = await EmailBatchService.createEmailBatches( + email._id, + recipientObjects, + { + emailType: EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC, + }, + session, + ); - // Get dynamic counts for response - const counts = await batch.getEmailCounts(); + // Commit transaction + await session.commitTransaction(); - res.status(200).json({ - success: true, - message: `Email batch created successfully for ${recipientsArray.length} recipient(s)`, - data: { - batchId: batch.batchId, - status: batch.status, - subject: batch.subject, - recipients: recipientsArray, - ...counts, - createdAt: batch.createdAt, - }, - }); - } else { - // Legacy direct sending (fallback) - using new announcement service - if (recipientsArray.length === 1) { - // Single recipient - use TO field - await emailAnnouncementService.sendAnnouncement( - to, - subject, - processedHtml, - attachments, - null, - null, - null, + // Audit logging after successful commit (outside transaction to avoid failures) + try { + await EmailBatchAuditService.logEmailQueued( + email._id, { - announcementType: 'direct_send', - priority: 'NORMAL', + subject: email.subject, }, + user._id, ); - } else { - // Multiple recipients - use BCC to hide recipient list - // Send to self (sender) as primary recipient, then BCC all actual recipients - const senderEmail = req.body.fromEmail || 'updates@onecommunityglobal.org'; - await emailAnnouncementService.sendAnnouncement( - senderEmail, - subject, - processedHtml, - attachments, - null, - null, - recipientsArray, - { - announcementType: 'direct_send', - priority: 'NORMAL', - }, + + // Audit each batch creation + await Promise.all( + inserted.map(async (item) => { + await EmailBatchAuditService.logEmailBatchQueued( + email._id, + item._id, + { + recipientCount: item.recipients?.length || 0, + emailType: item.emailType, + recipients: item.recipients?.map((r) => r.email) || [], + emailBatchId: item._id.toString(), + }, + user._id, + ); + }), ); + } catch (auditErr) { + logger.logException(auditErr, 'Audit failure after successful email creation'); + // Don't fail the request if audit fails } - res.status(200).send(`Email sent successfully to ${recipientsArray.length} recipient(s)`); + session.endSession(); + + return res.status(200).json({ + success: true, + message: `Email created successfully for ${recipientsArray.length} recipient(s)`, + }); + } catch (emailError) { + // Abort transaction on error + await session.abortTransaction(); + session.endSession(); + throw emailError; } } catch (emailError) { - console.error('Error sending email:', emailError); - res.status(500).send('Error sending email'); + logger.logException(emailError, 'Error creating email'); + return res.status(500).json({ success: false, message: 'Error creating email' }); } } catch (error) { - return res.status(500).send('Error sending email'); + return res.status(500).json({ success: false, message: 'Error creating email' }); } }; const sendEmailToAll = async (req, res) => { - const canSendEmailToAll = await hasPermission(req.body.requestor, 'sendEmailToAll'); - if (!canSendEmailToAll) { - res.status(403).send('You are not authorized to send emails to all.'); - return; + // TODO: Re-enable permission check in future + // Permission check - commented out for now + // const canSendEmailToAll = await hasPermission(req.body.requestor, 'sendEmailToAll'); + // if (!canSendEmailToAll) { + // return res.status(403).json({ success: false, message: 'You are not authorized to send emails to all.' }); + // } + + // Requestor is still required for getting user ID for audit trail + if (!req?.body?.requestor?.requestorId) { + return res.status(401).json({ success: false, message: 'Missing requestor' }); } + try { - const { subject, html, useBatch = true } = req.body; + const { subject, html } = req.body; if (!subject || !html) { - return res.status(400).send('Subject and HTML content are required'); + return res + .status(400) + .json({ success: false, message: 'Subject and HTML content are required' }); } - const { html: processedHtml, attachments } = extractImagesAndCreateAttachments(html); + if (!ensureHtmlWithinLimit(html)) { + return res + .status(413) + .json({ + success: false, + message: `HTML content exceeds ${EMAIL_JOB_CONFIG.LIMITS.MAX_HTML_BYTES / (1024 * 1024)}MB limit`, + }); + } - if (useBatch) { - // Use new batch system for broadcast emails - const user = await userProfile.findById(req.body.requestor.requestorId); - if (!user) { - return res.status(400).send('User not found'); + // Validate HTML does not contain base64-encoded media + const mediaValidation = validateHtmlMedia(html); + if (!mediaValidation.isValid) { + return res.status(400).json({ + success: false, + message: 'HTML contains embedded media files. Only URLs are allowed for media.', + errors: mediaValidation.errors, + }); + } + + // Validate that all template variables have been replaced + const templateVariableRegex = /\{\{(\w+)\}\}/g; + const unmatchedVariables = []; + let match = templateVariableRegex.exec(html); + while (match !== null) { + if (!unmatchedVariables.includes(match[1])) { + unmatchedVariables.push(match[1]); + } + match = templateVariableRegex.exec(html); + } + // Check subject as well + if (subject) { + templateVariableRegex.lastIndex = 0; + match = templateVariableRegex.exec(subject); + while (match !== null) { + if (!unmatchedVariables.includes(match[1])) { + unmatchedVariables.push(match[1]); + } + match = templateVariableRegex.exec(subject); } + } + if (unmatchedVariables.length > 0) { + return res.status(400).json({ + success: false, + message: + 'Email contains unreplaced template variables. Please ensure all variables are replaced before sending.', + errors: { + unmatchedVariables: `Found unreplaced variables: ${unmatchedVariables.join(', ')}`, + }, + }); + } + + // Always use new batch system for broadcast emails + const user = await userProfile.findById(req.body.requestor.requestorId); + if (!user) { + return res.status(400).json({ success: false, message: 'User not found' }); + } + + // Get ALL recipients FIRST (HGN users + email subscribers) + const users = await userProfile.find({ + firstName: { $ne: '' }, + email: { $ne: null }, + isActive: true, + emailSubscriptions: true, + }); + + const emailSubscribers = await EmailSubcriptionList.find({ + email: { $exists: true, $ne: '' }, + isConfirmed: true, + emailSubscriptions: true, + }); + + const totalRecipients = users.length + emailSubscribers.length; + if (totalRecipients === 0) { + return res.status(400).json({ success: false, message: 'No recipients found' }); + } - // Get all recipients + // Start MongoDB transaction + const session = await mongoose.startSession(); + session.startTransaction(); + + try { + // Create parent Email within transaction + const email = await EmailService.createEmail( + { + subject, + htmlContent: html, + createdBy: user._id, + }, + session, + ); + + // Collect all recipients into single array + const allRecipients = [ + ...users.map((hgnUser) => ({ email: hgnUser.email })), + ...emailSubscribers.map((subscriber) => ({ email: subscriber.email })), + ]; + + // Create EmailBatch items with all recipients (chunked automatically) within transaction + // Use BCC for broadcast emails to hide recipient list from each other + const inserted = await EmailBatchService.createEmailBatches( + email._id, + allRecipients, + { + emailType: EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC, + }, + session, + ); + + // Commit transaction + await session.commitTransaction(); + + // Audit logging after successful commit (outside transaction to avoid failures) + try { + await EmailBatchAuditService.logEmailQueued( + email._id, + { + subject: email.subject, + }, + user._id, + ); + + // Audit each batch creation + await Promise.all( + inserted.map(async (item) => { + await EmailBatchAuditService.logEmailBatchQueued( + email._id, + item._id, + { + recipientCount: item.recipients?.length || 0, + emailType: item.emailType, + recipients: item.recipients?.map((r) => r.email) || [], + emailBatchId: item._id.toString(), + }, + user._id, + ); + }), + ); + } catch (auditErr) { + logger.logException(auditErr, 'Audit failure after successful broadcast email creation'); + // Don't fail the request if audit fails + } + + session.endSession(); + + return res.status(200).json({ + success: true, + message: `Broadcast email created successfully for ${totalRecipients} recipient(s)`, + }); + } catch (error) { + // Abort transaction on error + await session.abortTransaction(); + session.endSession(); + throw error; + } + } catch (error) { + logger.logException(error, 'Error creating broadcast email'); + return res.status(500).json({ success: false, message: 'Error creating broadcast email' }); + } +}; + +const resendEmail = async (req, res) => { + // Requestor is required for getting user ID for audit trail + if (!req?.body?.requestor?.requestorId) { + return res.status(401).json({ success: false, message: 'Missing requestor' }); + } + + try { + const { emailId, recipientOption, specificRecipients } = req.body; + + // Validate emailId + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + return res.status(400).json({ success: false, message: 'Invalid emailId' }); + } + + // Get the original email + const originalEmail = await EmailService.getEmailById(emailId); + if (!originalEmail) { + return res.status(404).json({ success: false, message: 'Email not found' }); + } + + // Validate recipient option + if (!recipientOption) { + return res.status(400).json({ success: false, message: 'Recipient option is required' }); + } + + const validRecipientOptions = ['all', 'specific', 'same']; + if (!validRecipientOptions.includes(recipientOption)) { + return res.status(400).json({ + success: false, + message: `Invalid recipient option. Must be one of: ${validRecipientOptions.join(', ')}`, + }); + } + + // Get requestor user + const user = await userProfile.findById(req.body.requestor.requestorId); + if (!user) { + return res.status(400).json({ success: false, message: 'Requestor not found' }); + } + + let allRecipients = []; + + // Determine recipients based on option + if (recipientOption === 'all') { + // Get ALL recipients (HGN users + email subscribers) const users = await userProfile.find({ firstName: { $ne: '' }, email: { $ne: null }, @@ -206,247 +461,298 @@ const sendEmailToAll = async (req, res) => { emailSubscriptions: true, }); - const totalRecipients = users.length + emailSubscribers.length; - console.log('# sendEmailToAll total recipients:', totalRecipients); + allRecipients = [ + ...users.map((hgnUser) => ({ email: hgnUser.email })), + ...emailSubscribers.map((subscriber) => ({ email: subscriber.email })), + ]; + } else if (recipientOption === 'specific') { + // Use provided specific recipients + if ( + !specificRecipients || + !Array.isArray(specificRecipients) || + specificRecipients.length === 0 + ) { + return res + .status(400) + .json({ + success: false, + message: 'specificRecipients array is required for specific option', + }); + } - if (totalRecipients === 0) { - return res.status(400).send('No recipients found'); + // Normalize and validate recipients + const recipientsArray = normalizeRecipientsToArray(specificRecipients); + const invalidRecipients = recipientsArray.filter((e) => !isValidEmailAddress(e)); + if (invalidRecipients.length) { + return res + .status(400) + .json({ + success: false, + message: 'One or more recipient emails are invalid', + invalidRecipients, + }); } - // Create batch for broadcast - const batch = await EmailBatchService.createBatch({ - name: `Broadcast - ${subject}`, - description: `Broadcast email to all subscribers (${totalRecipients} recipients)`, - createdBy: user._id, - createdByName: `${user.firstName} ${user.lastName}`, - createdByEmail: user.email, - subject, - htmlContent: processedHtml, - attachments, - metadata: { - type: 'broadcast', - originalRequest: req.body, - priority: 'NORMAL', - }, + allRecipients = recipientsArray.map((email) => ({ email })); + } else if (recipientOption === 'same') { + // Get recipients from original email's EmailBatch items + const emailBatchItems = await EmailBatchService.getEmailBatchesByEmailId(emailId); + if (!emailBatchItems || emailBatchItems.length === 0) { + return res + .status(404) + .json({ success: false, message: 'No recipients found in original email' }); + } + + // Extract all recipients from all EmailBatch items + const batchRecipients = emailBatchItems + .filter((batch) => batch.recipients && Array.isArray(batch.recipients)) + .flatMap((batch) => batch.recipients); + allRecipients.push(...batchRecipients); + + // Deduplicate recipients by email + const seenEmails = new Set(); + allRecipients = allRecipients.filter((recipient) => { + if (!recipient || !recipient.email || seenEmails.has(recipient.email)) { + return false; + } + seenEmails.add(recipient.email); + return true; }); + } - // Add HGN users - if (users.length > 0) { - const hgnRecipients = users.map((hgnUser) => ({ - email: hgnUser.email, - name: `${hgnUser.firstName} ${hgnUser.lastName}`, - personalizedContent: handleContentToOC(processedHtml), - emailType: 'TO', - tags: ['hgn_user'], - })); - await EmailBatchService.addRecipients(batch.batchId, hgnRecipients); - } + if (allRecipients.length === 0) { + return res.status(400).json({ success: false, message: 'No recipients found' }); + } - // Add email subscribers - if (emailSubscribers.length > 0) { - const subscriberRecipients = emailSubscribers.map((subscriber) => ({ - email: subscriber.email, - personalizedContent: handleContentToNonOC(processedHtml, subscriber.email), - emailType: 'TO', - tags: ['email_subscriber'], - })); - await EmailBatchService.addRecipients(batch.batchId, subscriberRecipients); - } + // Start MongoDB transaction + const session = await mongoose.startSession(); + session.startTransaction(); - // REMOVED: Immediate processing - batch will be processed by cron job - // emailBatchProcessor.processBatch(batch.batchId).catch((error) => { - // console.error('Error processing broadcast batch:', error); - // }); + try { + // Create new Email (copy) within transaction + const newEmail = await EmailService.createEmail( + { + subject: originalEmail.subject, + htmlContent: originalEmail.htmlContent, + createdBy: user._id, + }, + session, + ); + + // Create EmailBatch items within transaction + // Always use BCC for all recipients (sender goes in 'to' field) + const inserted = await EmailBatchService.createEmailBatches( + newEmail._id, + allRecipients, + { + emailType: EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC, + }, + session, + ); + + // Commit transaction + await session.commitTransaction(); + + // Audit logging after successful commit (outside transaction) + try { + await EmailBatchAuditService.logEmailQueued( + newEmail._id, + { + subject: newEmail.subject, + resendFrom: emailId.toString(), + recipientOption, + }, + user._id, + ); + + // Audit each batch creation + await Promise.all( + inserted.map(async (item) => { + await EmailBatchAuditService.logEmailBatchQueued( + newEmail._id, + item._id, + { + recipientCount: item.recipients?.length || 0, + emailType: item.emailType, + recipients: item.recipients?.map((r) => r.email) || [], + emailBatchId: item._id.toString(), + }, + user._id, + ); + }), + ); + } catch (auditErr) { + logger.logException(auditErr, 'Audit failure after successful email resend'); + } - // Get dynamic counts for response - const counts = await batch.getEmailCounts(); + session.endSession(); return res.status(200).json({ success: true, - message: `Broadcast email batch created successfully for ${totalRecipients} recipient(s)`, + message: `Email queued for resend successfully to ${allRecipients.length} recipient(s)`, data: { - batchId: batch.batchId, - status: batch.status, - subject: batch.subject, - recipients: { - hgnUsers: users.length, - emailSubscribers: emailSubscribers.length, - total: totalRecipients, - }, - ...counts, - createdBy: batch.createdBy, - createdAt: batch.createdAt, - estimatedCompletion: new Date(Date.now() + totalRecipients * 2000), // 2 seconds per email estimate + emailId: newEmail._id, + recipientCount: allRecipients.length, }, }); + } catch (error) { + // Abort transaction on error + await session.abortTransaction(); + session.endSession(); + throw error; } - // Legacy direct sending (fallback) - using new announcement service - // HGN Users logic - const users = await userProfile.find({ - firstName: { $ne: '' }, - email: { $ne: null }, - isActive: true, - emailSubscriptions: true, - }); - - if (users.length > 0) { - const recipientEmails = users.map((user) => user.email); - console.log('# sendEmailToAll to HGN users:', recipientEmails.length); - const emailContentToOCmembers = handleContentToOC(processedHtml); - await Promise.all( - recipientEmails.map((email) => - emailAnnouncementService.sendAnnouncement( - email, - subject, - emailContentToOCmembers, - attachments, - null, - null, - null, - { - announcementType: 'broadcast_hgn', - priority: 'NORMAL', - }, - ), - ), - ); - } else { - console.log('# sendEmailToAll: No HGN users found with email subscriptions'); - } - const emailSubscribers = await EmailSubcriptionList.find({ - email: { $exists: true, $ne: '' }, - isConfirmed: true, - emailSubscriptions: true, - }); - console.log('# sendEmailToAll emailSubscribers', emailSubscribers.length); - - if (emailSubscribers.length > 0) { - await Promise.all( - emailSubscribers.map(({ email }) => { - const emailContentToNonOCmembers = handleContentToNonOC(processedHtml, email); - return emailAnnouncementService.sendAnnouncement( - email, - subject, - emailContentToNonOCmembers, - attachments, - null, - null, - null, - { - announcementType: 'broadcast_subscriber', - priority: 'NORMAL', - }, - ); - }), - ); - } else { - console.log('# sendEmailToAll: No confirmed email subscribers found'); - } - return res.status(200).send('Email sent successfully'); } catch (error) { - console.error('Error sending email:', error); - return res.status(500).send('Error sending email'); + logger.logException(error, 'Error resending email'); + return res.status(500).json({ success: false, message: 'Error resending email' }); } }; const updateEmailSubscriptions = async (req, res) => { try { + if (!req?.body?.requestor?.email) { + return res.status(401).json({ success: false, message: 'Missing requestor email' }); + } + const { emailSubscriptions } = req.body; + if (typeof emailSubscriptions !== 'boolean') { + return res + .status(400) + .json({ success: false, message: 'emailSubscriptions must be a boolean value' }); + } + const { email } = req.body.requestor; + if (!isValidEmailAddress(email)) { + return res.status(400).json({ success: false, message: 'Invalid email address' }); + } + const user = await userProfile.findOneAndUpdate( { email }, { emailSubscriptions }, { new: true }, ); - return res.status(200).send(user); + + if (!user) { + return res.status(404).json({ success: false, message: 'User not found' }); + } + + return res + .status(200) + .json({ success: true, message: 'Email subscription updated successfully' }); } catch (error) { - console.error('Error updating email subscriptions:', error); - return res.status(500).send('Error updating email subscriptions'); + logger.logException(error, 'Error updating email subscriptions'); + return res.status(500).json({ success: false, message: 'Error updating email subscriptions' }); } }; const addNonHgnEmailSubscription = async (req, res) => { try { const { email } = req.body; - if (!email) { - return res.status(400).send('Email is required'); + if (!email || typeof email !== 'string') { + return res.status(400).json({ success: false, message: 'Email is required' }); } - const emailList = await EmailSubcriptionList.find({ email: { $eq: email } }); - if (emailList.length > 0) { - return res.status(400).send('Email already exists'); + // Normalize and validate email + const normalizedEmail = email.trim().toLowerCase(); + if (!isValidEmailAddress(normalizedEmail)) { + return res.status(400).json({ success: false, message: 'Invalid email address' }); + } + + // Check if email already exists (case-insensitive) + const existingSubscription = await EmailSubcriptionList.findOne({ + email: { $regex: new RegExp(`^${normalizedEmail}$`, 'i') }, + }); + + if (existingSubscription) { + return res.status(400).json({ success: false, message: 'Email already subscribed' }); } // Save to DB immediately with confirmation pending const newEmailList = new EmailSubcriptionList({ - email, + email: normalizedEmail, isConfirmed: false, emailSubscriptions: true, }); await newEmailList.save(); - // Optional: Still send confirmation email - const payload = { email }; - const token = jwt.sign(payload, jwtSecret, { expiresIn: '360' }); + // Send confirmation email + const payload = { email: normalizedEmail }; + const token = jwt.sign(payload, jwtSecret, { expiresIn: '24h' }); // Fixed: was '360' (invalid) + + if (!config.FRONT_END_URL) { + console.error('FRONT_END_URL is not configured'); + return res + .status(500) + .json({ success: false, message: 'Server configuration error. Please contact support.' }); + } + const emailContent = ` - - - - -

Thank you for subscribing to our email updates!

-

Click here to confirm your email

- - +

Thank you for subscribing to our email updates!

+

Click here to confirm your email

`; try { - await emailAnnouncementService.sendAnnouncement( - email, + await emailSender( + normalizedEmail, 'HGN Email Subscription', emailContent, null, null, null, null, - { - announcementType: 'subscription_confirmation', - priority: 'NORMAL', - }, + { type: 'subscription_confirmation' }, ); - return res.status(200).send('Email subscribed successfully'); - } catch (emailError) { - console.error('Error sending confirmation email:', emailError); - // Still return success since the subscription was saved to DB return res .status(200) - .send('Email subscribed successfully (confirmation email failed to send)'); + .json({ + success: true, + message: 'Email subscribed successfully. Please check your inbox to confirm.', + }); + } catch (emailError) { + logger.logException(emailError, 'Error sending confirmation email'); + // Still return success since the subscription was saved to DB + return res.status(200).json({ + success: true, + message: + 'Email subscribed successfully. Confirmation email failed to send. Please contact support.', + }); } } catch (error) { - console.error('Error adding email subscription:', error); - res.status(500).send('Error adding email subscription'); + logger.logException(error, 'Error adding email subscription'); + if (error.code === 11000) { + return res.status(400).json({ success: false, message: 'Email already subscribed' }); + } + return res.status(500).json({ success: false, message: 'Error adding email subscription' }); } }; const confirmNonHgnEmailSubscription = async (req, res) => { try { const { token } = req.body; - if (!token) { - return res.status(400).send('Invalid token'); + if (!token || typeof token !== 'string') { + return res.status(400).json({ success: false, message: 'Token is required' }); } + let payload = {}; try { payload = jwt.verify(token, jwtSecret); } catch (err) { - // console.log(err); - return res.status(401).json({ errors: [{ msg: 'Token is not valid' }] }); + return res.status(401).json({ success: false, message: 'Invalid or expired token' }); } + const { email } = payload; - if (!email) { - return res.status(400).send('Invalid token'); + if (!email || !isValidEmailAddress(email)) { + return res.status(400).json({ success: false, message: 'Invalid token payload' }); } + + // Normalize email + const normalizedEmail = email.trim().toLowerCase(); + try { // Update existing subscription to confirmed, or create new one - const existingSubscription = await EmailSubcriptionList.findOne({ email }); + const existingSubscription = await EmailSubcriptionList.findOne({ + email: { $regex: new RegExp(`^${normalizedEmail}$`, 'i') }, + }); + if (existingSubscription) { existingSubscription.isConfirmed = true; existingSubscription.confirmedAt = new Date(); @@ -454,23 +760,29 @@ const confirmNonHgnEmailSubscription = async (req, res) => { await existingSubscription.save(); } else { const newEmailList = new EmailSubcriptionList({ - email, + email: normalizedEmail, isConfirmed: true, confirmedAt: new Date(), emailSubscriptions: true, }); await newEmailList.save(); } + + return res + .status(200) + .json({ success: true, message: 'Email subscription confirmed successfully' }); } catch (error) { if (error.code === 11000) { - return res.status(200).send('Email already exists'); + // Race condition - email was already confirmed/subscribed + return res + .status(200) + .json({ success: true, message: 'Email subscription already confirmed' }); } + throw error; } - // console.log('email', email); - return res.status(200).send('Email subscribed successfully'); } catch (error) { - console.error('Error updating email subscriptions:', error); - return res.status(500).send('Error updating email subscriptions'); + logger.logException(error, 'Error confirming email subscription'); + return res.status(500).json({ success: false, message: 'Error confirming email subscription' }); } }; @@ -479,29 +791,39 @@ const removeNonHgnEmailSubscription = async (req, res) => { const { email } = req.body; // Validate input - if (!email) { - return res.status(400).send('Email is required'); + if (!email || typeof email !== 'string') { + return res.status(400).json({ success: false, message: 'Email is required' }); } - // Try to delete the email subscription completely + // Normalize email + const normalizedEmail = email.trim().toLowerCase(); + if (!isValidEmailAddress(normalizedEmail)) { + return res.status(400).json({ success: false, message: 'Invalid email address' }); + } + + // Try to delete the email subscription (case-insensitive) const deletedEntry = await EmailSubcriptionList.findOneAndDelete({ - email: { $eq: email }, + email: { $regex: new RegExp(`^${normalizedEmail}$`, 'i') }, }); // If not found, respond accordingly if (!deletedEntry) { - return res.status(404).send('Email not found or already unsubscribed'); + return res + .status(404) + .json({ success: false, message: 'Email not found or already unsubscribed' }); } - return res.status(200).send('Email unsubscribed and removed from subscription list'); + return res.status(200).json({ success: true, message: 'Email unsubscribed successfully' }); } catch (error) { - return res.status(500).send('Server error while unsubscribing'); + logger.logException(error, 'Error removing email subscription'); + return res.status(500).json({ success: false, message: 'Error removing email subscription' }); } }; module.exports = { sendEmail, sendEmailToAll, + resendEmail, updateEmailSubscriptions, addNonHgnEmailSubscription, removeNonHgnEmailSubscription, diff --git a/src/controllers/emailTemplateController.js b/src/controllers/emailTemplateController.js index 35390e623..9e3cc8cd2 100644 --- a/src/controllers/emailTemplateController.js +++ b/src/controllers/emailTemplateController.js @@ -1,154 +1,220 @@ -const EmailTemplate = require('../models/EmailTemplateModel'); +const mongoose = require('mongoose'); +const EmailTemplate = require('../models/emailTemplate'); +const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +const logger = require('../startup/logger'); +const { ensureHtmlWithinLimit, validateHtmlMedia } = require('../utilities/emailValidators'); + +/** + * Validate template variables + */ +function validateTemplateVariables(variables) { + if (!variables || !Array.isArray(variables)) { + return { isValid: true }; + } + + const errors = []; + const variableNames = new Set(); + + variables.forEach((variable, index) => { + if (!variable.name || typeof variable.name !== 'string' || !variable.name.trim()) { + errors.push(`Variable ${index + 1}: name is required and must be a non-empty string`); + } else { + const varName = variable.name.trim(); + // Validate variable name format (alphanumeric and underscore only) + if (!/^[a-zA-Z0-9_]+$/.test(varName)) { + errors.push( + `Variable ${index + 1}: name must contain only alphanumeric characters and underscores`, + ); + } + // Check for duplicates + if (variableNames.has(varName.toLowerCase())) { + errors.push(`Variable ${index + 1}: duplicate variable name '${varName}'`); + } + variableNames.add(varName.toLowerCase()); + } + + if (variable.type && !['text', 'url', 'number', 'textarea', 'image'].includes(variable.type)) { + errors.push(`Variable ${index + 1}: type must be one of: text, url, number, textarea, image`); + } + }); + + return { + isValid: errors.length === 0, + errors, + }; +} + +/** + * Validate template content (HTML and subject) against defined variables + */ +function validateTemplateVariableUsage(templateVariables, htmlContent, subject) { + const errors = []; + + if (!templateVariables || templateVariables.length === 0) { + return { isValid: true, errors: [] }; + } + + // Extract variable placeholders from content (format: {{variableName}}) + const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; + const usedVariables = new Set(); + const foundPlaceholders = []; + + // Check HTML content + if (htmlContent) { + let match = variablePlaceholderRegex.exec(htmlContent); + while (match !== null) { + const varName = match[1]; + foundPlaceholders.push(varName); + usedVariables.add(varName); + match = variablePlaceholderRegex.exec(htmlContent); + } + } + + // Reset regex for subject + variablePlaceholderRegex.lastIndex = 0; + + // Check subject + if (subject) { + let match = variablePlaceholderRegex.exec(subject); + while (match !== null) { + const varName = match[1]; + foundPlaceholders.push(varName); + usedVariables.add(varName); + match = variablePlaceholderRegex.exec(subject); + } + } -// Get all email templates with pagination and optimization -exports.getAllEmailTemplates = async (req, res) => { + // Check for undefined variable placeholders in content + const definedVariableNames = templateVariables.map((v) => v.name); + foundPlaceholders.forEach((placeholder) => { + if (!definedVariableNames.includes(placeholder)) { + errors.push( + `Variable placeholder '{{${placeholder}}}' is used in content but not defined in template variables`, + ); + } + }); + + // Check for defined variables that are not used in content (treated as errors) + templateVariables.forEach((variable) => { + if (!usedVariables.has(variable.name)) { + errors.push(`Variable '{{${variable.name}}}}' is defined but not used in template content`); + } + }); + + return { + isValid: errors.length === 0, + errors, + }; +} + +/** + * Get all email templates with pagination and optimization + */ +const getAllEmailTemplates = async (req, res) => { try { - const { search, page, limit, sortBy, sortOrder, fields, includeVariables } = req.query; + // TODO: Re-enable permission check in future + // Permission check - commented out for now + // if (!req?.body?.requestor && !req?.user) { + // return res.status(401).json({ + // success: false, + // message: 'Missing requestor', + // }); + // } + + // const requestor = req.body.requestor || req.user; + // const canViewTemplates = await hasPermission(requestor, 'viewEmailTemplates'); + // if (!canViewTemplates) { + // return res.status(403).json({ + // success: false, + // message: 'You are not authorized to view email templates.', + // }); + // } + + const { search, sortBy, includeEmailContent } = req.query; const query = {}; const sort = {}; - // Parse pagination parameters - let frontend decide defaults - const pageNum = page ? Math.max(1, parseInt(page, 10)) : 1; - const limitNum = limit ? parseInt(limit, 10) : null; // No restrictions, let frontend decide - const skip = limitNum && pageNum ? (pageNum - 1) * limitNum : 0; - // Add search functionality with text index if (search && search.trim()) { - query.$or = [ - { name: { $regex: search.trim(), $options: 'i' } }, - { subject: { $regex: search.trim(), $options: 'i' } }, - ]; + query.$or = [{ name: { $regex: search.trim(), $options: 'i' } }]; } - // No filtering - removed variable filtering as requested - // Build sort object - let frontend decide sort field and order if (sortBy) { - const sortDirection = sortOrder === 'desc' ? -1 : 1; - sort[sortBy] = sortDirection; + sort[sortBy] = 1; // default ascending when sortBy provided } else { // Default sort only if frontend doesn't specify sort.created_at = -1; } - // Execute optimized query with pagination - let queryBuilder = EmailTemplate.find(query); + // Build projection based on include flags; always include audit fields + let projection = '_id name created_by updated_by created_at updated_at'; + if (includeEmailContent === 'true') projection += ' subject html_content variables'; - // Let components decide which fields to include - if (fields) { - // Parse comma-separated fields and always include _id - const fieldList = fields.split(',').map((field) => field.trim()); - if (!fieldList.includes('_id')) { - fieldList.unshift('_id'); - } - queryBuilder = queryBuilder.select(fieldList.join(' ')); - } else if (includeVariables === 'true') { - // Include all fields including variables if requested - // Don't use select('') as it excludes all fields, use no select() to include all - } else { - // Default minimal fields for list view - queryBuilder = queryBuilder.select('_id name created_at updated_at created_by updated_by'); - } - - // Populate user fields if they're in the selection - if (includeVariables === 'true' || !fields || fields.includes('created_by')) { - queryBuilder = queryBuilder.populate('created_by', 'firstName lastName'); - } - if (includeVariables === 'true' || !fields || fields.includes('updated_by')) { - queryBuilder = queryBuilder.populate('updated_by', 'firstName lastName'); - } - - queryBuilder = queryBuilder.sort(sort).skip(skip); - - // Only apply limit if specified - if (limitNum) { - queryBuilder = queryBuilder.limit(limitNum); - } - - const [templates, totalCount] = await Promise.all([ - queryBuilder.lean(), // Use lean() for better performance - EmailTemplate.countDocuments(query), - ]); - - // Transform templates based on what components requested - let processedTemplates; - if (includeVariables === 'true') { - // Return full template data including variables - processedTemplates = templates.map((template) => ({ - _id: template._id, - name: template.name, - subject: template.subject, - content: template.content, - variables: template.variables || [], - created_by: template.created_by, - updated_by: template.updated_by, - created_at: template.created_at, - updated_at: template.updated_at, - })); - } else if (fields) { - // Return only requested fields - processedTemplates = templates.map((template) => { - const fieldList = fields.split(',').map((field) => field.trim()); - const result = { _id: template._id }; - fieldList.forEach((field) => { - if (template[field] !== undefined) { - result[field] = template[field]; - } - }); - return result; - }); - } else { - // Default minimal fields for list view - processedTemplates = templates.map((template) => ({ - _id: template._id, - name: template.name, - created_by: template.created_by, - updated_by: template.updated_by, - created_at: template.created_at, - updated_at: template.updated_at, - })); - } - - // Calculate pagination info - const totalPages = limitNum ? Math.ceil(totalCount / limitNum) : 1; - const hasNextPage = limitNum ? pageNum < totalPages : false; - const hasPrevPage = pageNum > 1; + let queryBuilder = EmailTemplate.find(query).select(projection).sort(sort); + + // Always include created_by and updated_by populations + queryBuilder = queryBuilder.populate('created_by', 'firstName lastName'); + queryBuilder = queryBuilder.populate('updated_by', 'firstName lastName'); + + const templates = await queryBuilder.lean(); res.status(200).json({ success: true, - templates: processedTemplates, - pagination: { - currentPage: pageNum, - totalPages, - totalCount, - limit: limitNum, - hasNextPage, - hasPrevPage, - }, + templates, }); } catch (error) { - console.error('Error fetching email templates:', error); + logger.logException(error, 'Error fetching email templates'); res.status(500).json({ success: false, - message: 'Error fetching email templates.', + message: 'Error fetching email templates', error: error.message, }); } }; -// Get a single email template by ID -exports.getEmailTemplateById = async (req, res) => { +/** + * Get a single email template by ID + */ +const getEmailTemplateById = async (req, res) => { try { + // TODO: Re-enable permission check in future + // Permission check - commented out for now + // if (!req?.body?.requestor && !req?.user) { + // return res.status(401).json({ + // success: false, + // message: 'Missing requestor', + // }); + // } + + // const requestor = req.body.requestor || req.user; + // const canViewTemplates = await hasPermission(requestor, 'viewEmailTemplates'); + // if (!canViewTemplates) { + // return res.status(403).json({ + // success: false, + // message: 'You are not authorized to view email templates.', + // }); + // } + const { id } = req.params; + + // Validate ObjectId + if (!id || !mongoose.Types.ObjectId.isValid(id)) { + return res.status(400).json({ + success: false, + message: 'Invalid template ID', + }); + } + const template = await EmailTemplate.findById(id) - .populate('created_by', 'firstName lastName') - .populate('updated_by', 'firstName lastName'); + .populate('created_by', 'firstName lastName email') + .populate('updated_by', 'firstName lastName email'); if (!template) { return res.status(404).json({ success: false, - message: 'Email template not found.', + message: 'Email template not found', }); } @@ -157,353 +223,415 @@ exports.getEmailTemplateById = async (req, res) => { template, }); } catch (error) { - console.error('Error fetching email template:', error); + logger.logException(error, 'Error fetching email template'); res.status(500).json({ success: false, - message: 'Error fetching email template.', + message: 'Error fetching email template', error: error.message, }); } }; -// Create a new email template -exports.createEmailTemplate = async (req, res) => { +/** + * Create a new email template + */ +const createEmailTemplate = async (req, res) => { try { + // TODO: Re-enable permission check in future + // Permission check - commented out for now + // const canCreateTemplate = await hasPermission(req.body.requestor, 'createEmailTemplates'); + // if (!canCreateTemplate) { + // return res.status(403).json({ + // success: false, + // message: 'You are not authorized to create email templates.', + // }); + // } + + if (!req?.body?.requestor?.requestorId) { + return res.status(401).json({ + success: false, + message: 'Missing requestor', + }); + } + const { name, subject, html_content: htmlContent, variables } = req.body; - const userId = req.body.requestor?.requestorId; + const userId = req.body.requestor.requestorId; + + // Validate HTML content size + if (!ensureHtmlWithinLimit(htmlContent)) { + return res.status(413).json({ + success: false, + message: `HTML content exceeds ${EMAIL_JOB_CONFIG.LIMITS.MAX_HTML_BYTES / (1024 * 1024)}MB limit`, + }); + } + + // Validate HTML does not contain base64-encoded media + const mediaValidation = validateHtmlMedia(htmlContent); + if (!mediaValidation.isValid) { + return res.status(400).json({ + success: false, + message: 'HTML contains embedded media files. Only URLs are allowed for media.', + errors: mediaValidation.errors, + }); + } - // Validate required fields - if (!name || !subject || !htmlContent) { + // Validate name length + const trimmedName = name.trim(); + if (trimmedName.length > 50) { return res.status(400).json({ success: false, - message: 'Name, subject, and HTML content are required.', + message: 'Template name cannot exceed 50 characters', }); } - // Check if template with the same name already exists - const existingTemplate = await EmailTemplate.findOne({ name }); + // Validate subject length + const trimmedSubject = subject.trim(); + if (trimmedSubject.length > EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH) { + return res.status(400).json({ + success: false, + message: `Subject cannot exceed ${EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH} characters`, + }); + } + + // Check if template with the same name already exists (case-insensitive) + const existingTemplate = await EmailTemplate.findOne({ + name: { $regex: new RegExp(`^${trimmedName}$`, 'i') }, + }); if (existingTemplate) { return res.status(400).json({ success: false, - message: 'Email template with this name already exists.', + message: 'Email template with this name already exists', }); } - // Validate variables if provided + // Validate variables if (variables && variables.length > 0) { - const invalidVariable = variables.find((variable) => !variable.name || !variable.label); - if (invalidVariable) { + const variableValidation = validateTemplateVariables(variables); + if (!variableValidation.isValid) { + return res.status(400).json({ + success: false, + message: 'Invalid template variables', + errors: variableValidation.errors, + }); + } + + // Validate variable usage in content (HTML and subject) + const variableUsageValidation = validateTemplateVariableUsage( + variables, + htmlContent, + trimmedSubject, + ); + if (!variableUsageValidation.isValid) { + return res.status(400).json({ + success: false, + message: 'Invalid variable usage in template content', + errors: variableUsageValidation.errors, + }); + } + } else { + // If no variables are defined, check for any variable placeholders in content + const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; + const foundInHtml = variablePlaceholderRegex.test(htmlContent); + variablePlaceholderRegex.lastIndex = 0; + const foundInSubject = variablePlaceholderRegex.test(trimmedSubject); + + if (foundInHtml || foundInSubject) { return res.status(400).json({ success: false, - message: 'Variable name and label are required for all variables.', + message: + 'Template content contains variable placeholders ({{variableName}}) but no variables are defined. Please define variables or remove placeholders from content.', }); } } + // Validate userId is valid ObjectId + if (userId && !mongoose.Types.ObjectId.isValid(userId)) { + return res.status(400).json({ + success: false, + message: 'Invalid user ID', + }); + } + // Create new email template const template = new EmailTemplate({ - name, - subject, - html_content: htmlContent, + name: trimmedName, + subject: trimmedSubject, + html_content: htmlContent.trim(), variables: variables || [], created_by: userId, - updated_by: userId, // Set updated_by to same as created_by for new templates + updated_by: userId, }); await template.save(); // Populate created_by and updated_by fields for response - await template.populate('created_by', 'firstName lastName'); - await template.populate('updated_by', 'firstName lastName'); + await template.populate('created_by', 'firstName lastName email'); + await template.populate('updated_by', 'firstName lastName email'); + + logger.logInfo(`Email template created: ${template.name} by user ${userId}`); res.status(201).json({ success: true, - message: 'Email template created successfully.', + message: 'Email template created successfully', template, }); } catch (error) { - console.error('Error creating email template:', error); + logger.logException(error, 'Error creating email template'); res.status(500).json({ success: false, - message: 'Error creating email template.', + message: 'Error creating email template', error: error.message, }); } }; -// Update an email template -exports.updateEmailTemplate = async (req, res) => { +/** + * Update an email template + */ +const updateEmailTemplate = async (req, res) => { try { + // TODO: Re-enable permission check in future + // Permission check - commented out for now + // const canUpdateTemplate = await hasPermission(req.body.requestor, 'updateEmailTemplates'); + // if (!canUpdateTemplate) { + // return res.status(403).json({ + // success: false, + // message: 'You are not authorized to update email templates.', + // }); + // } + + if (!req?.body?.requestor?.requestorId) { + return res.status(401).json({ + success: false, + message: 'Missing requestor', + }); + } + const { id } = req.params; const { name, subject, html_content: htmlContent, variables } = req.body; - // Validate required fields - if (!name || !subject || !htmlContent) { + // Validate ObjectId + if (!id || !mongoose.Types.ObjectId.isValid(id)) { + return res.status(400).json({ + success: false, + message: 'Invalid template ID', + }); + } + + // Validate HTML content size + if (!ensureHtmlWithinLimit(htmlContent)) { + return res.status(413).json({ + success: false, + message: `HTML content exceeds ${EMAIL_JOB_CONFIG.LIMITS.MAX_HTML_BYTES / (1024 * 1024)}MB limit`, + }); + } + + // Validate HTML does not contain base64-encoded media + const mediaValidation = validateHtmlMedia(htmlContent); + if (!mediaValidation.isValid) { + return res.status(400).json({ + success: false, + message: 'HTML contains embedded media files. Only URLs are allowed for media.', + errors: mediaValidation.errors, + }); + } + + // Validate name and subject length + const trimmedName = name.trim(); + const trimmedSubject = subject.trim(); + + if (trimmedName.length > 50) { + return res.status(400).json({ + success: false, + message: 'Template name cannot exceed 50 characters', + }); + } + + if (trimmedSubject.length > EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH) { return res.status(400).json({ success: false, - message: 'Name, subject, and HTML content are required.', + message: `Subject cannot exceed ${EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH} characters`, }); } - // Get current template to check if name is actually changing + // Get current template const currentTemplate = await EmailTemplate.findById(id); if (!currentTemplate) { return res.status(404).json({ success: false, - message: 'Email template not found.', + message: 'Email template not found', }); } - // Only check for duplicate names if the name is actually changing - if (currentTemplate.name !== name) { + // Only check for duplicate names if the name is actually changing (case-insensitive) + if (currentTemplate.name.toLowerCase() !== trimmedName.toLowerCase()) { const existingTemplate = await EmailTemplate.findOne({ - name, + name: { $regex: new RegExp(`^${trimmedName}$`, 'i') }, _id: { $ne: id }, }); if (existingTemplate) { return res.status(400).json({ success: false, - message: 'Another email template with this name already exists.', + message: 'Another email template with this name already exists', }); } } - // Validate variables if provided + // Validate variables if (variables && variables.length > 0) { - const invalidVariable = variables.find((variable) => !variable.name || !variable.label); - if (invalidVariable) { + const variableValidation = validateTemplateVariables(variables); + if (!variableValidation.isValid) { + return res.status(400).json({ + success: false, + message: 'Invalid template variables', + errors: variableValidation.errors, + }); + } + + // Validate variable usage in content (HTML and subject) + const variableUsageValidation = validateTemplateVariableUsage( + variables, + htmlContent, + trimmedSubject, + ); + if (!variableUsageValidation.isValid) { + return res.status(400).json({ + success: false, + message: 'Invalid variable usage in template content', + errors: variableUsageValidation.errors, + }); + } + } else { + // If no variables are defined, check for any variable placeholders in content + const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; + const foundInHtml = variablePlaceholderRegex.test(htmlContent); + variablePlaceholderRegex.lastIndex = 0; + const foundInSubject = variablePlaceholderRegex.test(trimmedSubject); + + if (foundInHtml || foundInSubject) { return res.status(400).json({ success: false, - message: 'Variable name and label are required for all variables.', + message: + 'Template content contains variable placeholders ({{variableName}}) but no variables are defined. Please define variables or remove placeholders from content.', }); } } + // Validate userId is valid ObjectId + const userId = req.body.requestor?.requestorId; + if (userId && !mongoose.Types.ObjectId.isValid(userId)) { + return res.status(400).json({ + success: false, + message: 'Invalid user ID', + }); + } + // Update template const updateData = { - name, - subject, - html_content: htmlContent, + name: trimmedName, + subject: trimmedSubject, + html_content: htmlContent.trim(), variables: variables || [], - updated_by: req.body.requestor?.requestorId, // Set who updated the template + updated_by: userId, }; const template = await EmailTemplate.findByIdAndUpdate(id, updateData, { new: true, runValidators: true, }) - .populate('created_by', 'firstName lastName') - .populate('updated_by', 'firstName lastName'); + .populate('created_by', 'firstName lastName email') + .populate('updated_by', 'firstName lastName email'); if (!template) { return res.status(404).json({ success: false, - message: 'Email template not found.', + message: 'Email template not found', }); } + logger.logInfo(`Email template updated: ${template.name} by user ${userId}`); + res.status(200).json({ success: true, - message: 'Email template updated successfully.', + message: 'Email template updated successfully', template, }); } catch (error) { - console.error('Error updating email template:', error); + logger.logException(error, 'Error updating email template'); res.status(500).json({ success: false, - message: 'Error updating email template.', + message: 'Error updating email template', error: error.message, }); } }; -// Delete an email template -exports.deleteEmailTemplate = async (req, res) => { +/** + * Delete an email template + */ +const deleteEmailTemplate = async (req, res) => { try { - const { id } = req.params; - const template = await EmailTemplate.findByIdAndDelete(id); - - if (!template) { - return res.status(404).json({ + // TODO: Re-enable permission check in future + // Permission check - commented out for now + // const canDeleteTemplate = await hasPermission(req.body.requestor, 'deleteEmailTemplates'); + // if (!canDeleteTemplate) { + // return res.status(403).json({ + // success: false, + // message: 'You are not authorized to delete email templates.', + // }); + // } + + // Requestor is still required for audit logging + if (!req?.body?.requestor?.requestorId) { + return res.status(401).json({ success: false, - message: 'Email template not found.', + message: 'Missing requestor', }); } - res.status(200).json({ - success: true, - message: 'Email template deleted successfully.', - }); - } catch (error) { - console.error('Error deleting email template:', error); - res.status(500).json({ - success: false, - message: 'Error deleting email template.', - error: error.message, - }); - } -}; - -// Send email using template -exports.sendEmailWithTemplate = async (req, res) => { - try { const { id } = req.params; - const { recipients, variableValues, broadcastToAll } = req.body; - // Validate required fields - if (!broadcastToAll && (!recipients || !Array.isArray(recipients) || recipients.length === 0)) { + // Validate ObjectId + if (!id || !mongoose.Types.ObjectId.isValid(id)) { return res.status(400).json({ success: false, - message: 'Recipients array is required when not broadcasting to all.', + message: 'Invalid template ID', }); } - // Get template const template = await EmailTemplate.findById(id); + if (!template) { return res.status(404).json({ success: false, - message: 'Email template not found.', - }); - } - - // Validate all variables (since all are required by default) - const missingVariable = template.variables.find( - (variable) => !variableValues || !variableValues[variable.name], - ); - if (missingVariable) { - return res.status(400).json({ - success: false, - message: `Variable '${missingVariable.label}' is missing.`, - }); - } - - // Replace variables in subject and content - let processedSubject = template.subject; - let processedContent = template.html_content; - - if (variableValues) { - Object.entries(variableValues).forEach(([varName, varValue]) => { - const regex = new RegExp(`{{${varName}}}`, 'g'); - processedSubject = processedSubject.replace(regex, varValue); - processedContent = processedContent.replace(regex, varValue); + message: 'Email template not found', }); } - if (broadcastToAll) { - // Use existing broadcast functionality - const { sendEmailToAll } = require('./emailController'); - - // Create a mock request object for sendEmailToAll - const mockReq = { - body: { - requestor: req.body.requestor || req.user, // Pass the user making the request - subject: processedSubject, - html: processedContent, - }, - }; - - // Create a mock response object to capture the result - let broadcastResult = null; - const mockRes = { - status: (code) => ({ - send: (message) => { - broadcastResult = { code, message }; - }, - }), - }; - - await sendEmailToAll(mockReq, mockRes); - - if (broadcastResult && broadcastResult.code === 200) { - res.status(200).json({ - success: true, - message: 'Email template broadcasted successfully to all users.', - broadcasted: true, - }); - } else { - res.status(broadcastResult?.code || 500).json({ - success: false, - message: broadcastResult?.message || 'Error broadcasting email template.', - }); - } - } else { - // Send to specific recipients using batch system - try { - const EmailBatchService = require('../services/emailBatchService'); - const userProfile = require('../models/userProfile'); - - // Get user information - const user = await userProfile.findById(req.body.requestor.requestorId); - if (!user) { - return res.status(400).json({ - success: false, - message: 'User not found', - }); - } - - // Create batch for template email (this already adds recipients internally) - console.log('📧 Creating batch for template email...'); - const batch = await EmailBatchService.createSingleSendBatch( - { - to: recipients, - subject: processedSubject, - html: processedContent, - attachments: null, - }, - user, - ); + await EmailTemplate.findByIdAndDelete(id); - console.log('✅ Template batch created with recipients:', batch.batchId); - console.log('📊 Batch details:', { - id: batch._id, - batchId: batch.batchId, - status: batch.status, - createdBy: batch.createdBy, - }); + logger.logInfo( + `Email template deleted: ${template.name} by user ${req.body.requestor?.requestorId}`, + ); - // REMOVED: Immediate processing - batch will be processed by cron job - // emailBatchProcessor.processBatch(batch.batchId).catch((error) => { - // console.error('❌ Error processing template batch:', error); - // }); - - // Get dynamic counts for response - const counts = await batch.getEmailCounts(); - - res.status(200).json({ - success: true, - message: `Email template batch created successfully for ${recipients.length} recipient(s).`, - data: { - batchId: batch.batchId, - status: batch.status, - subject: batch.subject, - recipients, - ...counts, - template: { - id: template._id, - name: template.name, - subject: template.subject, - }, - createdBy: batch.createdBy, - createdAt: batch.createdAt, - estimatedCompletion: new Date(Date.now() + recipients.length * 2000), // 2 seconds per email estimate - }, - }); - } catch (emailError) { - console.error('Error creating template batch:', emailError); - res.status(500).json({ - success: false, - message: 'Error creating template batch.', - error: emailError.message, - }); - } - } + res.status(200).json({ + success: true, + message: 'Email template deleted successfully', + }); } catch (error) { - console.error('Error in sendEmailWithTemplate:', error); + logger.logException(error, 'Error deleting email template'); res.status(500).json({ success: false, - message: 'Error processing email template.', + message: 'Error deleting email template', error: error.message, }); } }; + +module.exports = { + getAllEmailTemplates, + getEmailTemplateById, + createEmailTemplate, + updateEmailTemplate, + deleteEmailTemplate, +}; diff --git a/src/jobs/emailAnnouncementJobProcessor.js b/src/jobs/emailAnnouncementJobProcessor.js index 00e0cdce4..155c85dc5 100644 --- a/src/jobs/emailAnnouncementJobProcessor.js +++ b/src/jobs/emailAnnouncementJobProcessor.js @@ -1,19 +1,13 @@ -/** - * Email Announcement Job Processor - * Cron-based processor for email announcement job queue - */ - const { CronJob } = require('cron'); const Email = require('../models/email'); -const emailBatchProcessor = require('../services/emailBatchProcessor'); -const EmailBatchAuditService = require('../services/emailBatchAuditService'); +const emailProcessor = require('../services/emailProcessor'); const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); const logger = require('../startup/logger'); class EmailAnnouncementJobProcessor { constructor() { this.isProcessing = false; - this.maxConcurrentBatches = EMAIL_JOB_CONFIG.MAX_CONCURRENT_BATCHES; + this.batchFetchLimit = EMAIL_JOB_CONFIG.MAX_CONCURRENT_BATCHES; this.processingInterval = EMAIL_JOB_CONFIG.CRON_INTERVAL; this.cronJob = null; } @@ -38,7 +32,20 @@ class EmailAnnouncementJobProcessor { ); this.cronJob.start(); - logger.logInfo(`Email announcement job processor started - runs every minute`); + logger.logInfo('Email announcement job processor started - runs on configured interval'); + + this.cronJob = new CronJob( + EMAIL_JOB_CONFIG.CRON_INTERVAL, + async () => { + await this.processPendingBatches(); + }, + null, + false, + 'UTC', + ); + logger.logInfo( + `Email announcement job processor started – cron=${EMAIL_JOB_CONFIG.CRON_INTERVAL}, tz=${EMAIL_JOB_CONFIG.TIMEZONE || 'UTC'}`, + ); } /** @@ -54,6 +61,7 @@ class EmailAnnouncementJobProcessor { /** * Process pending batches + * Processes ALL queued emails regardless of individual failures - ensures maximum delivery */ async processPendingBatches() { if (this.isProcessing) { @@ -64,12 +72,25 @@ class EmailAnnouncementJobProcessor { this.isProcessing = true; try { - // Get batches ready for processing + // Get batches ready for processing (QUEUED and stuck SENDING emails from previous crash/restart) + // Emails stuck in SENDING for more than 5 minutes are likely orphaned + const fiveMinutesAgo = new Date(Date.now() - 5 * 60 * 1000); + const pendingBatches = await Email.find({ - status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.QUEUED, + $or: [ + { status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.QUEUED }, + { + status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING, + $or: [ + { startedAt: { $lt: fiveMinutesAgo } }, + { startedAt: { $exists: false } }, // recover ones missing a timestamp + { startedAt: null }, + ], + }, + ], }) .sort({ createdAt: 1 }) // FIFO order - .limit(this.maxConcurrentBatches); + .limit(this.batchFetchLimit); if (pendingBatches.length === 0) { logger.logInfo('No pending email batches to process'); @@ -78,62 +99,69 @@ class EmailAnnouncementJobProcessor { logger.logInfo(`Processing ${pendingBatches.length} email batches`); - // Process each batch - const processingPromises = pendingBatches.map((batch) => - EmailAnnouncementJobProcessor.processBatchWithAuditing(batch), + // Check for and log stuck emails + const stuckEmails = pendingBatches.filter( + (email) => email.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING, ); + if (stuckEmails.length > 0) { + logger.logInfo( + `Recovering ${stuckEmails.length} emails stuck in SENDING state from previous restart/crash`, + ); + } - await Promise.allSettled(processingPromises); + // Process each email - allSettled to avoid blocking on failures + const results = await Promise.allSettled( + pendingBatches.map((email) => + EmailAnnouncementJobProcessor.processBatchWithAuditing(email), + ), + ); + + const fulfilled = results.filter((r) => r.status === 'fulfilled'); + const succeeded = fulfilled.filter((r) => r.value).length; + const failed = results.length - succeeded; + + logger.logInfo( + `Completed processing cycle: ${succeeded} email batches succeeded, ${failed} failed out of ${pendingBatches.length} total`, + ); } catch (error) { - logger.logException(error, 'Error processing announcement batches'); + logger.logException(error, 'Error in announcement batch processing cycle'); + // Continue processing - don't block other emails } finally { this.isProcessing = false; } } /** - * Process a single batch with comprehensive auditing + * Process a single Email with comprehensive auditing + * Never throws - ensures other emails continue processing even if this one fails */ - static async processBatchWithAuditing(batch) { + static async processBatchWithAuditing(email) { const startTime = Date.now(); try { - // Log batch start - await EmailBatchAuditService.logEmailStarted(batch._id, { - batchId: batch.batchId, - subject: batch.subject, - recipientCount: await batch.getEmailCounts(), - }); - - // Update batch status - batch.status = EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING; - batch.startedAt = new Date(); - await batch.save(); - - // Process using existing emailBatchProcessor - await emailBatchProcessor.processBatch(batch.batchId); + // Process using existing emailProcessor + const finalStatus = await emailProcessor.processEmail(email._id); const processingTime = Date.now() - startTime; - // Log completion - await EmailBatchAuditService.logEmailCompleted(batch._id, { - batchId: batch.batchId, - processingTime, - finalCounts: await batch.getEmailCounts(), - }); + // Completion audit is handled in the processor based on final status + logger.logInfo( + `Processed Email ${email._id} with status ${finalStatus} in ${processingTime}ms`, + ); - logger.logInfo(`Successfully processed batch ${batch.batchId} in ${processingTime}ms`); + // Return true for success, false for failure + const isSuccess = + finalStatus === EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT || + finalStatus === EMAIL_JOB_CONFIG.EMAIL_STATUSES.PROCESSED; + return isSuccess; } catch (error) { const processingTime = Date.now() - startTime; - // Log error - await EmailBatchAuditService.logEmailFailed(batch._id, error, { - batchId: batch.batchId, - processingTime, - errorMessage: error.message, - }); + // Failure audit is handled in the processor + logger.logException(error, `Failed to process Email ${email._id} after ${processingTime}ms`); - logger.logException(error, `Failed to process batch ${batch.batchId}`); + // Return false to indicate failure but don't throw - allows other emails to continue + return false; } } @@ -144,9 +172,18 @@ class EmailAnnouncementJobProcessor { return { isRunning: !!this.cronJob, isProcessing: this.isProcessing, - maxConcurrentBatches: this.maxConcurrentBatches, + batchFetchLimit: this.batchFetchLimit, cronInterval: this.processingInterval, - nextRun: this.cronJob ? this.cronJob.nextDate() : null, + nextRun: this.cronJob ? new Date(this.cronJob.nextDate().toString()) : null, + }; + } + + /** + * Get worker status (minimal info for frontend display) + */ + getWorkerStatus() { + return { + running: !!this.cronJob, }; } @@ -158,38 +195,6 @@ class EmailAnnouncementJobProcessor { status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.QUEUED, }); } - - /** - * Get processing statistics - */ - static async getProcessingStats() { - const stats = await Email.aggregate([ - { - $group: { - _id: '$status', - count: { $sum: 1 }, - avgProcessingTime: { - $avg: { - $cond: [ - { $ne: ['$processingStartedAt', null] }, - { $subtract: ['$completedAt', '$processingStartedAt'] }, - null, - ], - }, - }, - }, - }, - ]); - - return { - statusCounts: stats.reduce((acc, stat) => { - acc[stat._id] = stat.count; - return acc; - }, {}), - averageProcessingTime: stats[0]?.avgProcessingTime || 0, - lastUpdated: new Date(), - }; - } } // Create singleton instance diff --git a/src/models/email.js b/src/models/email.js index aac0bdc6b..ce6261f72 100644 --- a/src/models/email.js +++ b/src/models/email.js @@ -4,32 +4,13 @@ const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); const { Schema } = mongoose; const EmailSchema = new Schema({ - batchId: { - type: String, - required: [true, 'batchId is required'], - unique: true, - index: true, - }, subject: { type: String, required: [true, 'Subject is required'], - maxlength: [200, 'Subject cannot exceed 200 characters'], - validate: { - validator(v) { - return v && v.trim().length > 0; - }, - message: 'Subject cannot be empty or whitespace only', - }, }, htmlContent: { type: String, required: [true, 'HTML content is required'], - validate: { - validator(v) { - return v && v.trim().length > 0; - }, - message: 'HTML content cannot be empty or whitespace only', - }, }, status: { type: String, @@ -41,45 +22,20 @@ const EmailSchema = new Schema({ type: Schema.Types.ObjectId, ref: 'userProfile', required: [true, 'createdBy is required'], - validate: { - validator(v) { - return mongoose.Types.ObjectId.isValid(v); - }, - message: 'Invalid createdBy ObjectId', - }, }, createdAt: { type: Date, default: () => new Date(), index: true }, startedAt: { type: Date, - validate: { - validator(v) { - return !v || v >= this.createdAt; - }, - message: 'startedAt cannot be before createdAt', - }, }, completedAt: { type: Date, - validate: { - validator(v) { - return !v || v >= (this.startedAt || this.createdAt); - }, - message: 'completedAt cannot be before startedAt or createdAt', - }, }, updatedAt: { type: Date, default: () => new Date() }, - lastStuckFixAttempt: { type: Date }, // For preventing infinite retries }); // Update timestamps and validate basic constraints EmailSchema.pre('save', function (next) { this.updatedAt = new Date(); - - // Validate timestamp consistency - if (this.startedAt && this.completedAt && this.startedAt > this.completedAt) { - return next(new Error('startedAt cannot be after completedAt')); - } - next(); }); @@ -89,139 +45,4 @@ EmailSchema.index({ createdBy: 1, createdAt: -1 }); EmailSchema.index({ startedAt: 1 }); EmailSchema.index({ completedAt: 1 }); -// Calculate email counts dynamically from batch items with multiple recipients -EmailSchema.methods.getEmailCounts = async function () { - try { - const EmailBatch = require('./emailBatch'); - - // Validate this._id exists - if (!this._id) { - throw new Error('Email ID is required for counting'); - } - - const counts = await EmailBatch.aggregate([ - { $match: { batchId: this._id } }, // EmailBatch.batchId references Email._id (ObjectId) - { - $group: { - _id: null, - total: { $sum: { $cond: [{ $isArray: '$recipients' }, { $size: '$recipients' }, 0] } }, - sent: { - $sum: { - $cond: [ - { - $and: [ - { $eq: ['$status', EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENT] }, - { $isArray: '$recipients' }, - ], - }, - { $size: '$recipients' }, - 0, - ], - }, - }, - failed: { - $sum: { - $cond: [ - { - $and: [ - { $eq: ['$status', EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.FAILED] }, - { $isArray: '$recipients' }, - ], - }, - { $size: '$recipients' }, - 0, - ], - }, - }, - pending: { - $sum: { - $cond: [ - { - $and: [ - { - $in: [ - '$status', - [ - EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED, - EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENDING, - EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.RESENDING, - ], - ], - }, - { $isArray: '$recipients' }, - ], - }, - { $size: '$recipients' }, - 0, - ], - }, - }, - cancelled: { - $sum: { - $cond: [ - { - $and: [ - { $eq: ['$status', EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.CANCELLED] }, - { $isArray: '$recipients' }, - ], - }, - { $size: '$recipients' }, - 0, - ], - }, - }, - }, - }, - ]); - - if (counts.length > 0) { - const count = counts[0]; - - // Protect against negative counts (data corruption edge case) - const totalEmails = Math.max(0, count.total || 0); - const sentEmails = Math.max(0, Math.min(count.sent || 0, totalEmails)); - const failedEmails = Math.max(0, Math.min(count.failed || 0, totalEmails)); - const pendingEmails = Math.max(0, Math.min(count.pending || 0, totalEmails)); - const cancelledEmails = Math.max(0, Math.min(count.cancelled || 0, totalEmails)); - - // Validate counts don't exceed total - const calculatedTotal = sentEmails + failedEmails + pendingEmails + cancelledEmails; - if (calculatedTotal !== totalEmails) { - console.warn( - `Email count mismatch for ${this._id}: calculated=${calculatedTotal}, total=${totalEmails}`, - ); - } - - return { - totalEmails, - sentEmails, - failedEmails, - pendingEmails, - cancelledEmails, - progress: totalEmails > 0 ? Math.round((sentEmails / totalEmails) * 100) : 0, - }; - } - - return { - totalEmails: 0, - sentEmails: 0, - failedEmails: 0, - pendingEmails: 0, - cancelledEmails: 0, - progress: 0, - }; - } catch (error) { - // Log error and return safe defaults - console.error('Error calculating email counts:', error); - return { - totalEmails: 0, - sentEmails: 0, - failedEmails: 0, - pendingEmails: 0, - cancelledEmails: 0, - progress: 0, - }; - } -}; - module.exports = mongoose.model('Email', EmailSchema, 'emails'); diff --git a/src/models/emailBatch.js b/src/models/emailBatch.js index 7e33d10a7..a82318801 100644 --- a/src/models/emailBatch.js +++ b/src/models/emailBatch.js @@ -4,18 +4,12 @@ const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); const { Schema } = mongoose; const EmailBatchSchema = new Schema({ - // Batch reference - batchId: { + // Email reference + emailId: { type: Schema.Types.ObjectId, ref: 'Email', - required: [true, 'batchId is required'], + required: [true, 'emailId is required'], index: true, - validate: { - validator(v) { - return mongoose.Types.ObjectId.isValid(v); - }, - message: 'Invalid batchId ObjectId', - }, }, // Multiple recipients in one batch item (emails only) @@ -26,25 +20,10 @@ const EmailBatchSchema = new Schema({ email: { type: String, required: [true, 'Email is required'], - validate: { - validator(v) { - // Basic email format validation - const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; - return emailRegex.test(v); - }, - message: 'Invalid email format', - }, }, }, ], required: [true, 'Recipients array is required'], - validate: { - validator(v) { - // Ensure at least one recipient and not too many - return v && v.length > 0 && v.length <= 1000; // Max 1000 recipients per batch - }, - message: 'Recipients must have 1-1000 email addresses', - }, }, // Email type for the batch item (uses config enum) @@ -64,60 +43,30 @@ const EmailBatchSchema = new Schema({ required: [true, 'Status is required'], }, - // Processing info attempts: { type: Number, default: 0, - min: [0, 'Attempts cannot be negative'], }, lastAttemptedAt: { type: Date, - validate: { - validator(v) { - return !v || v >= this.createdAt; - }, - message: 'lastAttemptedAt cannot be before createdAt', - }, }, sentAt: { type: Date, - validate: { - validator(v) { - return !v || v >= this.createdAt; - }, - message: 'sentAt cannot be before createdAt', - }, }, failedAt: { type: Date, - validate: { - validator(v) { - return !v || v >= this.createdAt; - }, - message: 'failedAt cannot be before createdAt', - }, }, - // ERROR TRACKING lastError: { type: String, - maxlength: [500, 'Error message cannot exceed 500 characters'], }, lastErrorAt: { type: Date, - validate: { - validator(v) { - return !v || v >= this.createdAt; - }, - message: 'lastErrorAt cannot be before createdAt', - }, }, errorCode: { type: String, - maxlength: [1000, 'Error code cannot exceed 1000 characters'], }, - // Timestamps createdAt: { type: Date, default: () => new Date(), index: true }, updatedAt: { type: Date, default: () => new Date() }, }); @@ -126,11 +75,6 @@ const EmailBatchSchema = new Schema({ EmailBatchSchema.pre('save', function (next) { this.updatedAt = new Date(); - // Validate timestamp consistency - if (this.sentAt && this.failedAt) { - return next(new Error('Cannot have both sentAt and failedAt')); - } - // Validate status consistency with timestamps if (this.status === EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENT && !this.sentAt) { this.sentAt = new Date(); @@ -143,21 +87,10 @@ EmailBatchSchema.pre('save', function (next) { }); // Add indexes for better performance -EmailBatchSchema.index({ batchId: 1, status: 1 }); // For batch queries by status +EmailBatchSchema.index({ emailId: 1, status: 1 }); // For batch queries by status EmailBatchSchema.index({ status: 1, createdAt: 1 }); // For status-based queries -EmailBatchSchema.index({ batchId: 1, createdAt: -1 }); // For batch history +EmailBatchSchema.index({ emailId: 1, createdAt: -1 }); // For batch history EmailBatchSchema.index({ lastAttemptedAt: 1 }); // For retry logic EmailBatchSchema.index({ attempts: 1, status: 1 }); // For retry queries -// Get recipient count for this batch item -EmailBatchSchema.methods.getRecipientCount = function () { - return this.recipients ? this.recipients.length : 0; -}; - -// Check if this batch item is in a final state -EmailBatchSchema.methods.isFinalState = function () { - const { EMAIL_BATCH_STATUSES } = EMAIL_JOB_CONFIG; - return [EMAIL_BATCH_STATUSES.SENT, EMAIL_BATCH_STATUSES.FAILED].includes(this.status); -}; - module.exports = mongoose.model('EmailBatch', EmailBatchSchema, 'emailBatches'); diff --git a/src/models/emailBatchAudit.js b/src/models/emailBatchAudit.js index bebb0f234..dcb27bb0b 100644 --- a/src/models/emailBatchAudit.js +++ b/src/models/emailBatchAudit.js @@ -10,12 +10,6 @@ const EmailBatchAuditSchema = new Schema({ ref: 'Email', required: [true, 'emailId is required'], index: true, - validate: { - validator(v) { - return mongoose.Types.ObjectId.isValid(v); - }, - message: 'Invalid emailId ObjectId', - }, }, // Reference to specific email batch item @@ -23,12 +17,6 @@ const EmailBatchAuditSchema = new Schema({ type: Schema.Types.ObjectId, ref: 'EmailBatch', index: true, - validate: { - validator(v) { - return !v || mongoose.Types.ObjectId.isValid(v); - }, - message: 'Invalid emailBatchId ObjectId', - }, }, // Action performed (uses config enum) @@ -43,37 +31,20 @@ const EmailBatchAuditSchema = new Schema({ details: { type: String, required: [true, 'Details are required'], - maxlength: [1000, 'Details cannot exceed 1000 characters'], - validate: { - validator(v) { - return v && v.trim().length > 0; - }, - message: 'Details cannot be empty or whitespace only', - }, }, // Error information (if applicable) error: { type: String, - maxlength: [1000, 'Error message cannot exceed 1000 characters'], }, errorCode: { type: String, - maxlength: [50, 'Error code cannot exceed 50 characters'], }, // Contextual metadata (flexible object for additional data) metadata: { type: Schema.Types.Mixed, default: {}, - validate: { - validator(v) { - // Limit metadata size to prevent abuse - const sizeInBytes = Buffer.byteLength(JSON.stringify(v), 'utf8'); - return sizeInBytes <= 10000; // 10KB limit - }, - message: 'Metadata cannot exceed 10KB', - }, }, // Timestamps @@ -88,28 +59,6 @@ const EmailBatchAuditSchema = new Schema({ triggeredBy: { type: Schema.Types.ObjectId, ref: 'userProfile', - validate: { - validator(v) { - return !v || mongoose.Types.ObjectId.isValid(v); - }, - message: 'Invalid triggeredBy ObjectId', - }, - }, - - // Processing context - processingContext: { - attemptNumber: { - type: Number, - min: [0, 'Attempt number cannot be negative'], - }, - retryDelay: { - type: Number, - min: [0, 'Retry delay cannot be negative'], - }, - processingTime: { - type: Number, - min: [0, 'Processing time cannot be negative'], - }, }, }); diff --git a/src/models/EmailTemplateModel.js b/src/models/emailTemplate.js similarity index 93% rename from src/models/EmailTemplateModel.js rename to src/models/emailTemplate.js index be392b8db..2fef00154 100644 --- a/src/models/EmailTemplateModel.js +++ b/src/models/emailTemplate.js @@ -24,11 +24,6 @@ const emailTemplateSchema = new mongoose.Schema( required: true, trim: true, }, - label: { - type: String, - required: true, - trim: true, - }, type: { type: String, enum: ['text', 'url', 'number', 'textarea', 'image'], diff --git a/src/routes/emailBatchRoutes.js b/src/routes/emailBatchRoutes.js index a43bd8b0e..3218d89d9 100644 --- a/src/routes/emailBatchRoutes.js +++ b/src/routes/emailBatchRoutes.js @@ -1,25 +1,17 @@ -/** - * Simplified Email Batch Routes - Production Ready - * Focus: Essential endpoints only - */ - const express = require('express'); const router = express.Router(); + const emailBatchController = require('../controllers/emailBatchController'); -// Batch management routes -router.get('/batches', emailBatchController.getBatches); -router.get('/batches/:batchId', emailBatchController.getBatchDetails); -router.get('/dashboard', emailBatchController.getDashboardStats); -router.get('/status', emailBatchController.getProcessorStatus); +router.get('/emails', emailBatchController.getEmails); +router.get('/emails/:emailId', emailBatchController.getEmailDetails); + +router.get('/worker-status', emailBatchController.getWorkerStatus); -// Retry operations -router.post('/retry-item/:itemId', emailBatchController.retryBatchItem); +router.post('/emails/:emailId/retry', emailBatchController.retryEmail); -// Audit operations router.get('/audit/email/:emailId', emailBatchController.getEmailAuditTrail); router.get('/audit/email-batch/:emailBatchId', emailBatchController.getEmailBatchAuditTrail); -router.get('/audit/stats', emailBatchController.getAuditStats); module.exports = router; diff --git a/src/routes/emailRouter.js b/src/routes/emailRouter.js index 66b75d159..e16c4564d 100644 --- a/src/routes/emailRouter.js +++ b/src/routes/emailRouter.js @@ -2,6 +2,7 @@ const express = require('express'); const { sendEmail, sendEmailToAll, + resendEmail, updateEmailSubscriptions, addNonHgnEmailSubscription, removeNonHgnEmailSubscription, @@ -13,6 +14,7 @@ const routes = function () { emailRouter.route('/send-emails').post(sendEmail); emailRouter.route('/broadcast-emails').post(sendEmailToAll); + emailRouter.route('/resend-email').post(resendEmail); emailRouter.route('/update-email-subscriptions').post(updateEmailSubscriptions); emailRouter.route('/add-non-hgn-email-subscription').post(addNonHgnEmailSubscription); diff --git a/src/routes/emailTemplateRouter.js b/src/routes/emailTemplateRouter.js index 758b43357..7656fdafa 100644 --- a/src/routes/emailTemplateRouter.js +++ b/src/routes/emailTemplateRouter.js @@ -9,6 +9,5 @@ router.get('/email-templates/:id', emailTemplateController.getEmailTemplateById) router.post('/email-templates', emailTemplateController.createEmailTemplate); router.put('/email-templates/:id', emailTemplateController.updateEmailTemplate); router.delete('/email-templates/:id', emailTemplateController.deleteEmailTemplate); -router.post('/email-templates/:id/send', emailTemplateController.sendEmailWithTemplate); module.exports = router; diff --git a/src/services/emailAnnouncementService.js b/src/services/emailAnnouncementService.js index e9d8cc91a..c3012e0de 100644 --- a/src/services/emailAnnouncementService.js +++ b/src/services/emailAnnouncementService.js @@ -1,7 +1,7 @@ /** * Email Announcement Service - * Enhanced email service specifically tuned for announcement use cases - * Provides better tracking, analytics, and announcement-specific features + * Handles sending emails via Gmail API using OAuth2 authentication + * Provides validation, retry logic, and comprehensive error handling */ const nodemailer = require('nodemailer'); @@ -16,11 +16,15 @@ class EmailAnnouncementService { clientSecret: process.env.REACT_APP_EMAIL_CLIENT_SECRET, redirectUri: process.env.REACT_APP_EMAIL_CLIENT_REDIRECT_URI, refreshToken: process.env.REACT_APP_EMAIL_REFRESH_TOKEN, - batchSize: 50, - concurrency: 3, - rateLimitDelay: 1000, }; + // Validate configuration + const required = ['email', 'clientId', 'clientSecret', 'refreshToken', 'redirectUri']; + const missing = required.filter((k) => !this.config[k]); + if (missing.length) { + throw new Error(`Email config incomplete. Missing: ${missing.join(', ')}`); + } + this.OAuth2Client = new google.auth.OAuth2( this.config.clientId, this.config.clientSecret, @@ -29,41 +33,79 @@ class EmailAnnouncementService { this.OAuth2Client.setCredentials({ refresh_token: this.config.refreshToken }); // Create the email transporter - this.transporter = nodemailer.createTransport({ - service: 'gmail', - auth: { - type: 'OAuth2', - user: this.config.email, - clientId: this.config.clientId, - clientSecret: this.config.clientSecret, - }, - }); - - this.queue = []; - this.isProcessing = false; - } - - /** - * Normalize email field (convert to array) - */ - static normalize(field) { - if (!field) return []; - if (Array.isArray(field)) return field; - return String(field).split(','); + try { + this.transporter = nodemailer.createTransport({ + service: 'gmail', + auth: { + type: 'OAuth2', + user: this.config.email, + clientId: this.config.clientId, + clientSecret: this.config.clientSecret, + }, + }); + } catch (error) { + logger.logException(error, 'EmailAnnouncementService: Failed to create transporter'); + throw error; + } } /** * Send email with enhanced announcement tracking + * Validates input and configuration before sending + * @returns {Object} { success: boolean, response?: Object, error?: Error } */ async sendEmail(mailOptions) { + // Validation + if (!mailOptions) { + const error = new Error('INVALID_MAIL_OPTIONS: mailOptions is required'); + logger.logException(error, 'EmailAnnouncementService.sendEmail validation failed'); + return { success: false, error }; + } + + if (!mailOptions.to && !mailOptions.bcc) { + const error = new Error('INVALID_RECIPIENTS: At least one recipient (to or bcc) is required'); + logger.logException(error, 'EmailAnnouncementService.sendEmail validation failed'); + return { success: false, error }; + } + + // Validate subject and htmlContent + if (!mailOptions.subject || mailOptions.subject.trim() === '') { + const error = new Error('INVALID_SUBJECT: Subject is required and cannot be empty'); + logger.logException(error, 'EmailAnnouncementService.sendEmail validation failed'); + return { success: false, error }; + } + + if (!this.config.email || !this.config.clientId || !this.config.clientSecret) { + const error = new Error('INVALID_CONFIG: Email configuration is incomplete'); + logger.logException(error, 'EmailAnnouncementService.sendEmail configuration check failed'); + return { success: false, error }; + } + try { - const accessTokenResp = await this.OAuth2Client.getAccessToken(); - const token = typeof accessTokenResp === 'object' ? accessTokenResp?.token : accessTokenResp; + // Get access token with proper error handling + let token; + try { + const accessTokenResp = await this.OAuth2Client.getAccessToken(); + if (accessTokenResp && typeof accessTokenResp === 'object' && accessTokenResp.token) { + token = accessTokenResp.token; + } else if (typeof accessTokenResp === 'string') { + token = accessTokenResp; + } else { + throw new Error('Invalid access token response format'); + } + } catch (tokenError) { + const error = new Error(`OAUTH_TOKEN_ERROR: ${tokenError.message}`); + logger.logException(error, 'EmailAnnouncementService.sendEmail OAuth token refresh failed'); + return { success: false, error }; + } if (!token) { - throw new Error('NO_OAUTH_ACCESS_TOKEN'); + const error = new Error('NO_OAUTH_ACCESS_TOKEN: Failed to obtain access token'); + logger.logException(error, 'EmailAnnouncementService.sendEmail OAuth failed'); + return { success: false, error }; } + // Configure OAuth2 mailOptions.auth = { type: 'OAuth2', user: this.config.email, @@ -73,202 +115,100 @@ class EmailAnnouncementService { accessToken: token, }; + // Send email const result = await this.transporter.sendMail(mailOptions); // Enhanced logging for announcements - if (process.env.NODE_ENV === 'local') { - logger.logInfo(`Announcement email sent: ${JSON.stringify(result)}`); - } + logger.logInfo( + `Announcement email sent to: ${mailOptions.to || mailOptions.bcc || 'unknown'}`, + result, + ); - return result; + return { success: true, response: result }; } catch (error) { - console.error('Error sending announcement email:', error); - logger.logException(error, `Error sending announcement email: ${mailOptions.to}`); - - throw error; + logger.logException( + error, + `Error sending announcement email to: ${mailOptions.to || mailOptions.bcc || 'unknown'}`, + ); + return { success: false, error }; } } /** * Send email with retry logic and announcement-specific handling + * @param {Object} batch - Mail options batch + * @param {number} retries - Number of retry attempts + * @param {number} baseDelay - Base delay in milliseconds for exponential backoff + * @returns {Promise} { success: boolean, response?: Object, error?: Error, attemptCount: number } */ async sendWithRetry(batch, retries = 3, baseDelay = 1000) { + // Validation + if (!batch) { + const error = new Error('INVALID_BATCH: batch is required'); + logger.logException(error, 'EmailAnnouncementService.sendWithRetry validation failed'); + return { success: false, error, attemptCount: 0 }; + } + + if (!Number.isInteger(retries) || retries < 1) { + const error = new Error('INVALID_RETRIES: retries must be a positive integer'); + logger.logException(error, 'EmailAnnouncementService.sendWithRetry validation failed'); + return { success: false, error, attemptCount: 0 }; + } + + let attemptCount = 0; + /* eslint-disable no-await-in-loop */ for (let attempt = 1; attempt <= retries; attempt += 1) { + attemptCount += 1; + try { - const gmailResponse = await this.sendEmail(batch); + const result = await this.sendEmail(batch); + + if (result.success) { + // Store Gmail response for audit logging + batch.gmailResponse = result.response; + logger.logInfo( + `Email sent successfully on attempt ${attempt} to: ${batch.to || batch.bcc || 'unknown'}`, + ); + return { success: true, response: result.response, attemptCount }; + } + // result.success is false - log and try again or return + const error = result.error || new Error('Unknown error from sendEmail'); + logger.logException( + error, + `Announcement batch attempt ${attempt} failed to: ${batch.to || batch.bcc || '(empty)'}`, + ); - // Store Gmail response for audit logging - batch.gmailResponse = gmailResponse; - return true; + // If this is the last attempt, return failure info + if (attempt >= retries) { + return { success: false, error, attemptCount }; + } } catch (err) { + // Unexpected error (shouldn't happen since sendEmail now returns {success, error}) logger.logException( err, - `Announcement batch to ${batch.to || '(empty)'} attempt ${attempt}`, + `Unexpected error in announcement batch attempt ${attempt} to: ${batch.to || batch.bcc || '(empty)'}`, ); + + // If this is the last attempt, return failure info + if (attempt >= retries) { + return { success: false, error: err, attemptCount }; + } } + // Exponential backoff before retry (2^n: 1x, 2x, 4x, 8x, ...) if (attempt < retries) { - await EmailAnnouncementService.sleep(baseDelay * attempt); // Exponential backoff + const delay = baseDelay * 2 ** (attempt - 1); + await EmailAnnouncementService.sleep(delay); } } /* eslint-enable no-await-in-loop */ - return false; - } - /** - * Process email queue with announcement-specific optimizations - */ - async processQueue() { - if (this.isProcessing || this.queue.length === 0) { - return true; - } - - this.isProcessing = true; - - try { - const batches = this.queue.splice(0, this.config.concurrency); - const promises = batches.map((batch) => this.sendWithRetry(batch)); - - await Promise.all(promises); - - if (this.queue.length > 0) { - await EmailAnnouncementService.sleep(this.config.rateLimitDelay); - return this.processQueue(); - } - - // Return the last successful Gmail response for audit logging - const lastBatch = batches[batches.length - 1]; - return lastBatch?.gmailResponse || true; - } catch (error) { - logger.logException(error, 'Error processing announcement email queue'); - return false; - } finally { - this.isProcessing = false; - } - } - - /** - * Send announcement email with enhanced features - * @param {string|string[]} recipients - Email recipients - * @param {string} subject - Email subject - * @param {string} message - HTML message content - * @param {Object[]|null} attachments - Email attachments - * @param {string[]|null} cc - CC recipients - * @param {string|null} replyTo - Reply-to address - * @param {string[]|null} emailBccs - BCC recipients - * @param {Object} opts - Options including announcement-specific metadata - * @returns {Promise} Processing result - */ - async sendAnnouncement( - recipients, - subject, - message, - attachments = null, - cc = null, - replyTo = null, - emailBccs = null, - opts = {}, - ) { - const announcementType = opts.announcementType || 'general'; - const priority = opts.priority || 'NORMAL'; - const isUrgent = priority === 'URGENT'; - const isPasswordReset = announcementType === 'password_reset'; - - // Check if email sending is enabled - if ( - !process.env.sendEmail || - (String(process.env.sendEmail).toLowerCase() === 'false' && !isPasswordReset) - ) { - return Promise.resolve('EMAIL_SENDING_DISABLED'); - } - - return new Promise((resolve, reject) => { - const recipientsArray = Array.isArray(recipients) ? recipients : [recipients]; - - // Enhanced metadata for announcements - const enhancedMeta = { - ...opts, - announcementType, - priority, - timestamp: new Date(), - recipientCount: recipientsArray.length, - isUrgent, - }; - - // Process recipients in batches - for (let i = 0; i < recipientsArray.length; i += this.config.batchSize) { - const batchRecipients = recipientsArray.slice(i, i + this.config.batchSize); - - this.queue.push({ - from: this.config.email, - to: batchRecipients.length ? batchRecipients.join(',') : '', - bcc: emailBccs ? emailBccs.join(',') : '', - subject, - html: message, - attachments, - cc, - replyTo, - meta: enhancedMeta, - }); - } - - // Process queue immediately for urgent announcements - if (isUrgent) { - // Move urgent emails to front of queue - const urgentBatches = this.queue.filter((batch) => batch.meta.isUrgent); - const normalBatches = this.queue.filter((batch) => !batch.meta.isUrgent); - this.queue = [...urgentBatches, ...normalBatches]; - } - - setImmediate(async () => { - try { - const result = await this.processQueue(); - if (result === false) { - reject(new Error('Announcement email sending failed after all retries')); - } else { - // Return the last successful Gmail response for audit logging - const lastBatch = this.queue[this.queue.length - 1]; - if (lastBatch && lastBatch.gmailResponse) { - resolve(lastBatch.gmailResponse); - } else { - resolve(result); - } - } - } catch (error) { - reject(error); - } - }); - }); - } - - /** - * Send announcement summary notification - */ - async sendAnnouncementSummary(recipientEmail, summary) { - const summaryHtml = ` -

Announcement Summary

-

Total Recipients: ${summary.totalRecipients}

-

Successfully Sent: ${summary.sent}

-

Failed: ${summary.failed}

-

Success Rate: ${summary.successRate}%

-

Processing Time: ${summary.processingTime}

- ${summary.errors.length > 0 ? `

Errors:

    ${summary.errors.map((e) => `
  • ${e}
    • `).join('')}
` : ''} - `; - - return this.sendAnnouncement( - recipientEmail, - 'Announcement Summary Report', - summaryHtml, - null, - null, - null, - null, - { - announcementType: 'summary', - priority: 'LOW', - }, - ); + return { + success: false, + error: new Error('MAX_RETRIES_EXCEEDED: All retry attempts failed'), + attemptCount, + }; } /** @@ -279,21 +219,6 @@ class EmailAnnouncementService { setTimeout(resolve, ms); }); } - - /** - * Get service status - */ - getStatus() { - return { - isProcessing: this.isProcessing, - queueLength: this.queue.length, - config: { - batchSize: this.config.batchSize, - concurrency: this.config.concurrency, - rateLimitDelay: this.config.rateLimitDelay, - }, - }; - } } // Create singleton instance diff --git a/src/services/emailBatchAuditService.js b/src/services/emailBatchAuditService.js index 924b12c04..fc0c526a7 100644 --- a/src/services/emailBatchAuditService.js +++ b/src/services/emailBatchAuditService.js @@ -3,6 +3,7 @@ * Centralized audit management for email batch operations */ +const mongoose = require('mongoose'); const EmailBatchAudit = require('../models/emailBatchAudit'); const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); const logger = require('../startup/logger'); @@ -21,14 +22,51 @@ class EmailBatchAuditService { emailBatchId = null, ) { try { + // Validate emailId + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + throw new Error('Invalid emailId for audit log'); + } + + // Validate action is in enum + const validActions = Object.values(EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS); + if (!validActions.includes(action)) { + throw new Error(`Invalid audit action: ${action}`); + } + + // Normalize details (trim, limit length) + const normalizedDetails = + typeof details === 'string' + ? details.trim().slice(0, 1000) + : String(details || '').slice(0, 1000); + + // Normalize error message + const errorMessage = error?.message ? String(error.message).slice(0, 1000) : null; + const errorCode = error?.code ? String(error.code).slice(0, 50) : null; + + // Validate triggeredBy if provided + if (triggeredBy && !mongoose.Types.ObjectId.isValid(triggeredBy)) { + logger.logInfo( + `Invalid triggeredBy ObjectId in audit log: ${triggeredBy} - setting to null`, + ); + triggeredBy = null; + } + + // Validate emailBatchId if provided + if (emailBatchId && !mongoose.Types.ObjectId.isValid(emailBatchId)) { + logger.logInfo( + `Invalid emailBatchId ObjectId in audit log: ${emailBatchId} - setting to null`, + ); + emailBatchId = null; + } + const audit = new EmailBatchAudit({ emailId, emailBatchId, action, - details, - metadata, - error: error?.message, - errorCode: error?.code, + details: normalizedDetails, + metadata: metadata || {}, + error: errorMessage, + errorCode, triggeredBy, }); @@ -41,139 +79,94 @@ class EmailBatchAuditService { } /** - * Get complete audit trail for an email (main batch) + * Get complete audit trail for an Email (parent) - no pagination, no filtering + * @param {string|ObjectId} emailId - The _id (ObjectId) of the parent Email */ - static async getEmailAuditTrail(emailId, page = 1, limit = 50, action = null) { - const query = { emailId }; - - // Add action filter if provided - if (action) { - query.action = action; + static async getEmailAuditTrail(emailId) { + // Validate emailId is ObjectId + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + throw new Error('emailId is required and must be a valid ObjectId'); } - const skip = (page - 1) * limit; + const query = { emailId }; // Use ObjectId directly const auditTrail = await EmailBatchAudit.find(query) - .sort({ timestamp: 1 }) + .sort({ timestamp: -1 }) // Most recent first .populate('triggeredBy', 'firstName lastName email') - .populate('emailBatchId', 'recipients emailType') - .skip(skip) - .limit(limit); + .populate('emailBatchId', 'recipients emailType status') + .lean(); - const totalCount = await EmailBatchAudit.countDocuments(query); - const totalPages = Math.ceil(totalCount / limit); - - return { - auditTrail, - totalCount, - page, - totalPages, - limit, - }; + return auditTrail; } /** - * Get audit trail for a specific email batch item + * Get audit trail for a specific EmailBatch item - no pagination, no filtering */ - static async getEmailBatchAuditTrail(emailBatchId, page = 1, limit = 50, action = null) { - const query = { emailBatchId }; - - // Add action filter if provided - if (action) { - query.action = action; + static async getEmailBatchAuditTrail(emailBatchId) { + // Validate emailBatchId is ObjectId + if (!emailBatchId || !mongoose.Types.ObjectId.isValid(emailBatchId)) { + throw new Error('emailBatchId is required and must be a valid ObjectId'); } - const skip = (page - 1) * limit; + const query = { emailBatchId }; const auditTrail = await EmailBatchAudit.find(query) - .sort({ timestamp: 1 }) + .sort({ timestamp: -1 }) // Most recent first .populate('triggeredBy', 'firstName lastName email') - .populate('emailId', 'subject batchId') - .skip(skip) - .limit(limit); - - const totalCount = await EmailBatchAudit.countDocuments(query); - const totalPages = Math.ceil(totalCount / limit); + .populate('emailId', 'subject status') + .lean(); - return { - auditTrail, - totalCount, - page, - totalPages, - limit, - }; - } - - /** - * Get system-wide audit statistics - */ - static async getAuditStats(dateFrom = null, dateTo = null) { - const matchStage = {}; - if (dateFrom || dateTo) { - matchStage.timestamp = {}; - if (dateFrom) matchStage.timestamp.$gte = new Date(dateFrom); - if (dateTo) matchStage.timestamp.$lte = new Date(dateTo); - } - - return EmailBatchAudit.aggregate([ - { $match: matchStage }, - { - $group: { - _id: '$action', - count: { $sum: 1 }, - avgProcessingTime: { - $avg: '$processingContext.processingTime', - }, - }, - }, - ]); + return auditTrail; } /** - * Log email creation + * Log Email queued (for initial creation or retry) + * @param {string|ObjectId} emailId - The email ID + * @param {Object} metadata - Additional metadata + * @param {string|ObjectId} triggeredBy - Optional user ID who triggered this action */ - static async logEmailCreated(emailId, createdBy, metadata = {}) { + static async logEmailQueued(emailId, metadata = {}, triggeredBy = null) { return this.logAction( emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.BATCH_CREATED, - `Email created with ID: ${emailId}`, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_QUEUED, + `Email queued for processing`, metadata, null, - createdBy, + triggeredBy, ); } /** - * Log email processing start + * Log Email sending (processing start) */ - static async logEmailStarted(emailId, metadata = {}) { + static async logEmailSending(emailId, metadata = {}) { return this.logAction( emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.BATCH_STARTED, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_SENDING, `Email processing started`, metadata, ); } /** - * Log email processing completion + * Log Email processed (processing completion) */ - static async logEmailCompleted(emailId, metadata = {}) { + static async logEmailProcessed(emailId, metadata = {}) { return this.logAction( emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.BATCH_COMPLETED, - `Email processing completed successfully`, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_PROCESSED, + `Email processing completed`, metadata, ); } /** - * Log email processing failure + * Log Email processing failure */ static async logEmailFailed(emailId, error, metadata = {}) { return this.logAction( emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.BATCH_FAILED, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_FAILED, `Email processing failed`, metadata, error, @@ -181,16 +174,25 @@ class EmailBatchAuditService { } /** - * Log email batch item sent with essential delivery tracking + * Log Email sent (all batches completed successfully) */ - static async logEmailBatchSent(emailId, emailBatchId, metadata = {}, gmailResponse = null) { - const includeApiDetails = - process.env.NODE_ENV === 'development' || process.env.LOG_API_DETAILS === 'true'; + static async logEmailSent(emailId, metadata = {}) { + return this.logAction( + emailId, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_SENT, + `Email sent successfully`, + metadata, + ); + } + /** + * Log EmailBatch item sent with essential delivery tracking + */ + static async logEmailBatchSent(emailId, emailBatchId, metadata = {}, gmailResponse = null) { const enhancedMetadata = { ...metadata, // Include essential delivery tracking details - ...(includeApiDetails && gmailResponse + ...(gmailResponse ? { deliveryStatus: { messageId: gmailResponse.messageId, @@ -207,8 +209,8 @@ class EmailBatchAuditService { return this.logAction( emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.ITEM_SENT, - `Email batch item sent successfully`, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_BATCH_SENT, + `EmailBatch item sent successfully`, enhancedMetadata, null, null, @@ -217,15 +219,13 @@ class EmailBatchAuditService { } /** - * Log email batch item failure with optional Gmail API metadata + * Log EmailBatch item failure with optional Gmail API metadata */ static async logEmailBatchFailed(emailId, emailBatchId, error, metadata = {}) { - const includeApiDetails = true; - const enhancedMetadata = { ...metadata, // Include essential error tracking details - ...(includeApiDetails && error?.gmailResponse + ...(error?.gmailResponse ? { deliveryStatus: { messageId: error.gmailResponse.messageId, @@ -246,14 +246,51 @@ class EmailBatchAuditService { return this.logAction( emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.ITEM_FAILED, - `Email batch item failed to send`, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_BATCH_FAILED, + `EmailBatch item failed to send`, enhancedMetadata, error, null, emailBatchId, ); } + + /** + * Log EmailBatch item queued + * @param {string|ObjectId} emailId - The email ID + * @param {string|ObjectId} emailBatchId - The EmailBatch item ID + * @param {Object} metadata - Additional metadata + * @param {string|ObjectId} triggeredBy - Optional user ID who triggered this action + */ + static async logEmailBatchQueued(emailId, emailBatchId, metadata = {}, triggeredBy = null) { + return this.logAction( + emailId, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_BATCH_QUEUED, + `EmailBatch item queued`, + metadata, + null, + triggeredBy, + emailBatchId, + ); + } + + /** + * Log EmailBatch item sending + * @param {string|ObjectId} emailId - The email ID + * @param {string|ObjectId} emailBatchId - The EmailBatch item ID + * @param {Object} metadata - Additional metadata + */ + static async logEmailBatchSending(emailId, emailBatchId, metadata = {}) { + return this.logAction( + emailId, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_BATCH_SENDING, + `EmailBatch item sending`, + metadata, + null, + null, + emailBatchId, + ); + } } module.exports = EmailBatchAuditService; diff --git a/src/services/emailBatchProcessor.js b/src/services/emailBatchProcessor.js deleted file mode 100644 index 8d04c2783..000000000 --- a/src/services/emailBatchProcessor.js +++ /dev/null @@ -1,228 +0,0 @@ -/** - * Enhanced Email Batch Processor - Production Ready with Audit Integration - * Focus: Efficient processing with email body from batch record and comprehensive auditing - */ - -const Email = require('../models/email'); -const EmailBatch = require('../models/emailBatch'); -const emailAnnouncementService = require('./emailAnnouncementService'); -const EmailBatchAuditService = require('./emailBatchAuditService'); -const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); -const logger = require('../startup/logger'); - -class EmailBatchProcessor { - constructor() { - this.processingBatches = new Set(); - this.maxRetries = EMAIL_JOB_CONFIG.DEFAULT_MAX_RETRIES; - this.retryDelay = 2000; // 2 seconds - } - - /** - * Process a batch - */ - async processBatch(batchId) { - if (this.processingBatches.has(batchId)) { - return; // Already processing - } - - this.processingBatches.add(batchId); - - try { - console.log('🔍 Looking for batch with batchId:', batchId); - const batch = await Email.findOne({ batchId }); - if (!batch) { - console.error('❌ Batch not found with batchId:', batchId); - throw new Error('Batch not found'); - } - console.log('✅ Found batch:', batch.batchId, 'Status:', batch.status); - - if ( - batch.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT || - batch.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED - ) { - return; - } - - // Update batch status - batch.status = EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING; - batch.startedAt = new Date(); - await batch.save(); - - // Process batch items (each item contains multiple recipients) - await this.processBatchItems(batch); - - // Update final status - await batch.updateStatus(); - logger.logInfo(`Batch ${batchId} processed successfully`); - } catch (error) { - logger.logException(error, `Error processing batch ${batchId}`); - - // Mark batch as failed - try { - const batch = await Email.findOne({ batchId }); - if (batch) { - batch.status = EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED; - batch.completedAt = new Date(); - await batch.save(); - } - } catch (updateError) { - logger.logException(updateError, 'Error updating batch status to failed'); - } - } finally { - this.processingBatches.delete(batchId); - } - } - - /** - * Process all items in a batch - */ - async processBatchItems(batch) { - const items = await EmailBatch.find({ - batchId: batch._id, - status: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED, - }); - - // Process items in parallel with concurrency limit - const processPromises = items.map((item) => this.processItem(item, batch)); - await Promise.all(processPromises); - } - - /** - * Process a single batch item with multiple recipients - */ - async processItem(item, batch) { - const processWithRetry = async (attempt = 1) => { - try { - // Update to SENDING status (this increments attempts) - await item.updateStatus(EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENDING); - - // Extract recipient emails from the batch item - const recipientEmails = item.recipients.map((recipient) => recipient.email); - - // Use the new emailAnnouncementService with enhanced announcement features - const gmailResponse = await emailAnnouncementService.sendAnnouncement( - recipientEmails, // Array of emails for batching - batch.subject, - batch.htmlContent, // Use email body from batch record - null, // attachments - null, // cc - null, // replyTo - null, // bcc - { - announcementType: 'batch_send', - batchId: batch.batchId, - itemId: item._id, - emailType: item.emailType, - recipientCount: recipientEmails.length, - priority: 'NORMAL', - }, - ); - - // Mark as sent - await item.updateStatus(EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENT); - - // Log successful send to audit - await EmailBatchAuditService.logEmailBatchSent( - batch._id, - item._id, - { - recipientCount: recipientEmails.length, - emailType: item.emailType, - attempt: item.attempts, - }, - gmailResponse, - ); - - logger.logInfo( - `Email batch sent successfully to ${recipientEmails.length} recipients (attempt ${item.attempts})`, - ); - // Success - } catch (error) { - logger.logException( - error, - `Failed to send email batch to ${item.recipients.length} recipients (attempt ${attempt})`, - ); - - if (attempt >= this.maxRetries) { - await item.updateStatus( - EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.FAILED, - error.message, - error.code, - ); - - // Log failed send to audit - await EmailBatchAuditService.logEmailBatchFailed(batch._id, item._id, error, { - recipientCount: item.recipients.length, - emailType: item.emailType, - finalAttempt: attempt, - }); - - logger.logError( - `Permanently failed to send email batch to ${item.recipients.length} recipients after ${this.maxRetries} attempts`, - ); - return; - } - - // Wait before retry - await EmailBatchProcessor.sleep(this.retryDelay); - return processWithRetry(attempt + 1); - } - }; - - return processWithRetry(); - } - - /** - * Sleep utility - */ - static sleep(ms) { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); - } - - /** - * Retry a specific batch item - */ - async retryBatchItem(itemId) { - try { - const item = await EmailBatch.findById(itemId); - if (!item) { - throw new Error('Batch item not found'); - } - - const batch = await Email.findById(item.batchId); - if (!batch) { - throw new Error('Parent batch not found'); - } - - // Process the specific item - await this.processItem(item, batch); - - return { - success: true, - itemId: item._id, - status: item.status, - }; - } catch (error) { - logger.logException(error, 'Error retrying batch item'); - throw error; - } - } - - /** - * Get processor status - */ - getStatus() { - return { - isRunning: true, - processingBatches: Array.from(this.processingBatches), - maxRetries: this.maxRetries, - }; - } -} - -// Create singleton instance -const emailBatchProcessor = new EmailBatchProcessor(); - -module.exports = emailBatchProcessor; diff --git a/src/services/emailBatchService.js b/src/services/emailBatchService.js index 1b77dbcc6..2c1c1cf9b 100644 --- a/src/services/emailBatchService.js +++ b/src/services/emailBatchService.js @@ -1,302 +1,213 @@ /** - * Enhanced Email Batch Service - Production Ready with Job Queue Support - * Focus: Efficient batching with email body storage and job queue management + * Email Batch Service - Manages EmailBatch items (child records) + * Focus: Creating and managing EmailBatch items that reference parent Email records */ -const { v4: uuidv4 } = require('uuid'); +const mongoose = require('mongoose'); const Email = require('../models/email'); const EmailBatch = require('../models/emailBatch'); -const EmailBatchAuditService = require('./emailBatchAuditService'); +const EmailService = require('./emailService'); const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +const { normalizeRecipientsToObjects } = require('../utilities/emailValidators'); const logger = require('../startup/logger'); class EmailBatchService { - constructor() { - this.batchSize = 50; // Match emailSender batch size for efficiency - } - /** - * Create a new email batch with email body and job queue support + * Create EmailBatch items for an Email + * Takes all recipients, chunks them into EmailBatch items with configurable batch size + * @param {string|ObjectId} emailId - The _id (ObjectId) of the parent Email + * @param {Array} recipients - Array of recipient objects with email property + * @param {Object} config - Configuration { batchSize?, emailType? } + * @param {Object} session - MongoDB session for transaction support + * @returns {Promise} Created EmailBatch items */ - static async createBatch(batchData) { + static async createEmailBatches(emailId, recipients, config = {}, session = null) { try { - const batch = new Email({ - batchId: batchData.batchId || uuidv4(), - subject: batchData.subject, - htmlContent: batchData.htmlContent, // Store email body - createdBy: batchData.createdBy, - }); - - await batch.save(); - - // Log batch creation to audit trail - await EmailBatchAuditService.logEmailCreated(batch._id, batchData.createdBy, { - batchId: batch.batchId, - subject: batch.subject, - }); + // emailId is now the ObjectId directly - validate it + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + throw new Error(`Email not found with id: ${emailId}`); + } - console.log('💾 Batch saved successfully:', { - id: batch._id, - batchId: batch.batchId, - status: batch.status, - }); - return batch; - } catch (error) { - logger.logException(error, 'Error creating batch'); - throw error; - } - } + const batchSize = config.batchSize || EMAIL_JOB_CONFIG.ANNOUNCEMENTS.BATCH_SIZE; + const emailType = config.emailType || EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC; - /** - * Add recipients to a batch with efficient batching (uses config enums) - */ - static async addRecipients(batchId, recipients, batchConfig = {}) { - try { - const batch = await Email.findOne({ batchId }); - if (!batch) { - throw new Error('Batch not found'); + // Normalize recipients to { email } + const normalizedRecipients = normalizeRecipientsToObjects(recipients); + if (normalizedRecipients.length === 0) { + throw new Error('At least one recipient is required'); } - const batchSize = batchConfig.batchSize || 50; - const emailType = batchConfig.emailType || EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC; - - // Create batch items with multiple recipients per item - const batchItems = []; + // Chunk recipients into EmailBatch items + const emailBatchItems = []; - for (let i = 0; i < recipients.length; i += batchSize) { - const recipientChunk = recipients.slice(i, i + batchSize); + for (let i = 0; i < normalizedRecipients.length; i += batchSize) { + const recipientChunk = normalizedRecipients.slice(i, i + batchSize); - const batchItem = { - batchId: batch._id, - recipients: recipientChunk.map((recipient) => ({ - email: recipient.email, // Only email, no name - })), + const emailBatchItem = { + emailId, // emailId is now the ObjectId directly + recipients: recipientChunk.map((recipient) => ({ email: recipient.email })), emailType, status: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED, }; - batchItems.push(batchItem); + emailBatchItems.push(emailBatchItem); } - await EmailBatch.insertMany(batchItems); + // Insert with session if provided for transaction support + const inserted = await EmailBatch.insertMany(emailBatchItems, { session }); - return batch; - } catch (error) { - logger.logException(error, 'Error adding recipients to batch'); - throw error; - } - } - - /** - * Create a single send batch (most common use case) - */ - static async createSingleSendBatch(emailData, user) { - try { - // Handle both 'to' field and direct recipients array - let recipients; - if (emailData.to) { - recipients = Array.isArray(emailData.to) ? emailData.to : [emailData.to]; - } else if (Array.isArray(emailData.recipients)) { - recipients = emailData.recipients; - } else { - throw new Error('No recipients provided'); - } - - // Create batch with email body - const batch = await this.createBatch({ - batchId: uuidv4(), - subject: emailData.subject, - htmlContent: emailData.html, // Store email body in batch - createdBy: user._id || user.requestorId, - }); - - // Add recipients with efficient batching - const batchConfig = { - batchSize: 50, // Use standard batch size - emailType: - recipients.length === 1 - ? EMAIL_JOB_CONFIG.EMAIL_TYPES.TO - : EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC, // Single recipient uses TO, multiple use BCC - }; - - // Convert recipients to proper format - const recipientObjects = recipients.map((email) => ({ email })); - console.log('📧 Adding recipients to batch:', recipientObjects.length, 'recipients'); - await this.addRecipients(batch.batchId, recipientObjects, batchConfig); + logger.logInfo( + `Created ${emailBatchItems.length} EmailBatch items for Email ${emailId} with ${normalizedRecipients.length} total recipients`, + ); - return batch; + return inserted; } catch (error) { - logger.logException(error, 'Error creating single send batch'); + logger.logException(error, 'Error creating EmailBatch items'); throw error; } } /** - * Get batch with items and dynamic counts + * Get Email with its EmailBatch items and dynamic counts */ - static async getBatchWithItems(batchId) { + static async getEmailWithBatches(emailId) { try { - const batch = await Email.findOne({ batchId }).populate( - 'createdBy', - 'firstName lastName email', - ); - - if (!batch) { + const email = await EmailService.getEmailById(emailId); + if (!email) { return null; } - const items = await EmailBatch.find({ batchId: batch._id }).sort({ createdAt: 1 }); + // Populate createdBy if email exists + await email.populate('createdBy', 'firstName lastName email'); - // Get dynamic counts - const counts = await batch.getEmailCounts(); + const emailBatches = await this.getBatchesForEmail(emailId); - // Return batch items as-is (each item contains multiple recipients) - const transformedItems = items.map((item) => ({ - _id: item._id, - recipients: item.recipients || [], - status: item.status, - attempts: item.attempts || 0, - lastAttemptedAt: item.lastAttemptedAt, - sentAt: item.sentAt, - failedAt: item.failedAt, - error: item.error, - errorCode: item.errorCode, - emailType: item.emailType, - createdAt: item.createdAt, - updatedAt: item.updatedAt, + // Transform EmailBatch items + const transformedBatches = emailBatches.map((batch) => ({ + _id: batch._id, + emailId: batch.emailId, + recipients: batch.recipients || [], + status: batch.status, + attempts: batch.attempts || 0, + lastAttemptedAt: batch.lastAttemptedAt, + sentAt: batch.sentAt, + failedAt: batch.failedAt, + lastError: batch.lastError, + lastErrorAt: batch.lastErrorAt, + errorCode: batch.errorCode, + emailType: batch.emailType, + createdAt: batch.createdAt, + updatedAt: batch.updatedAt, })); return { - batch: { - ...batch.toObject(), - ...counts, - }, - items: transformedItems, + email: email.toObject(), + batches: transformedBatches, }; } catch (error) { - logger.logException(error, 'Error getting batch with items'); + logger.logException(error, 'Error getting Email with batches'); throw error; } } /** - * Get all batches with pagination and dynamic counts + * Get all Emails */ - static async getBatches(filters = {}, page = 1, limit = 20) { + static async getAllEmails() { try { - const query = {}; + const emails = await Email.find() + .sort({ createdAt: -1 }) + .populate('createdBy', 'firstName lastName email') + .lean(); - if (filters.status) query.status = filters.status; - if (filters.dateFrom) query.createdAt = { $gte: new Date(filters.dateFrom) }; - if (filters.dateTo) query.createdAt = { ...query.createdAt, $lte: new Date(filters.dateTo) }; - - const skip = (page - 1) * limit; - - const [batches, total] = await Promise.all([ - Email.find(query) - .sort({ createdAt: -1 }) - .limit(limit) - .skip(skip) - .populate('createdBy', 'firstName lastName email'), - Email.countDocuments(query), - ]); - - // Add dynamic counts to each batch - const batchesWithCounts = await Promise.all( - batches.map(async (batch) => { - const counts = await batch.getEmailCounts(); - return { - ...batch.toObject(), - ...counts, - }; - }), - ); - - return { - batches: batchesWithCounts, - pagination: { - page, - limit, - total, - pages: Math.ceil(total / limit), - }, - }; + return emails; } catch (error) { - logger.logException(error, 'Error getting batches'); + logger.logException(error, 'Error getting Emails'); throw error; } } /** - * Get dashboard statistics with dynamic calculations + * Fetch EmailBatch items for a parent emailId (ObjectId) */ - static async getDashboardStats() { - try { - const [totalBatches, pendingBatches, processingBatches, completedBatches, failedBatches] = - await Promise.all([ - Email.countDocuments(), - Email.countDocuments({ status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.QUEUED }), - Email.countDocuments({ status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING }), - Email.countDocuments({ status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT }), - Email.countDocuments({ status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED }), - ]); + static async getBatchesForEmail(emailId) { + return EmailBatch.find({ emailId }).sort({ createdAt: 1 }); + } + + /** + * Get EmailBatch items by emailId (alias for consistency) + */ + static async getEmailBatchesByEmailId(emailId) { + return this.getBatchesForEmail(emailId); + } + + /** + * Reset an EmailBatch item for retry + */ + static async resetEmailBatchForRetry(emailBatchId) { + const item = await EmailBatch.findById(emailBatchId); + if (!item) return null; + item.status = EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED; + item.attempts = 0; + item.lastError = null; + item.lastErrorAt = null; + item.errorCode = null; + item.failedAt = null; + item.lastAttemptedAt = null; + await item.save(); + return item; + } - // Calculate email stats dynamically from batch items - const emailStats = await EmailBatch.aggregate([ - { - $group: { - _id: null, - totalEmails: { - $sum: { $cond: [{ $isArray: '$recipients' }, { $size: '$recipients' }, 0] }, - }, - sentEmails: { - $sum: { - $cond: [ - { - $and: [ - { $eq: ['$status', EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENT] }, - { $isArray: '$recipients' }, - ], - }, - { $size: '$recipients' }, - 0, - ], - }, - }, - failedEmails: { - $sum: { - $cond: [ - { $and: [{ $eq: ['$status', 'FAILED'] }, { $isArray: '$recipients' }] }, - { $size: '$recipients' }, - 0, - ], - }, - }, - }, - }, - ]); + /** + * Mark a batch item as SENDING (and bump attempts/lastAttemptedAt) + */ + static async markEmailBatchSending(emailBatchId) { + const now = new Date(); + const updated = await EmailBatch.findByIdAndUpdate( + emailBatchId, + { + status: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENDING, + $inc: { attempts: 1 }, + lastAttemptedAt: now, + }, + { new: true }, + ); + return updated; + } - const stats = emailStats[0] || { totalEmails: 0, sentEmails: 0, failedEmails: 0 }; - const successRate = - stats.totalEmails > 0 ? Math.round((stats.sentEmails / stats.totalEmails) * 100) : 0; + /** + * Mark a batch item as SENT + */ + static async markEmailBatchSent(emailBatchId) { + const now = new Date(); + const updated = await EmailBatch.findByIdAndUpdate( + emailBatchId, + { + status: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENT, + sentAt: now, + }, + { new: true }, + ); + return updated; + } - return { - overview: { - totalBatches, - pendingBatches, - processingBatches, - completedBatches, - failedBatches, - }, - emailStats: { - ...stats, - successRate, - }, - }; - } catch (error) { - logger.logException(error, 'Error getting dashboard stats'); - throw error; - } + /** + * Mark a batch item as FAILED and record error info + */ + static async markEmailBatchFailed(emailBatchId, { errorCode, errorMessage }) { + const now = new Date(); + const updated = await EmailBatch.findByIdAndUpdate( + emailBatchId, + { + status: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.FAILED, + failedAt: now, + lastError: errorMessage?.slice(0, 500) || null, + lastErrorAt: now, + errorCode: errorCode?.toString().slice(0, 1000) || null, + }, + { new: true }, + ); + return updated; } } diff --git a/src/services/emailProcessor.js b/src/services/emailProcessor.js new file mode 100644 index 000000000..dc0b04463 --- /dev/null +++ b/src/services/emailProcessor.js @@ -0,0 +1,429 @@ +const mongoose = require('mongoose'); +const EmailBatch = require('../models/emailBatch'); +const EmailService = require('./emailService'); +const EmailBatchService = require('./emailBatchService'); +const emailAnnouncementService = require('./emailAnnouncementService'); +const EmailBatchAuditService = require('./emailBatchAuditService'); +const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +const logger = require('../startup/logger'); + +class EmailProcessor { + constructor() { + this.processingBatches = new Set(); + this.maxRetries = EMAIL_JOB_CONFIG.DEFAULT_MAX_RETRIES; + this.retryDelay = 1000; // 1 second + } + + /** + * Process an Email (processes all its EmailBatch items) + * @param {string|ObjectId} emailId - The _id (ObjectId) of the parent Email + */ + async processEmail(emailId) { + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + throw new Error('emailId is required and must be a valid ObjectId'); + } + + if (this.processingBatches.has(emailId)) { + logger.logInfo(`Email ${emailId} is already being processed, skipping`); + return EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING; + } + + this.processingBatches.add(emailId); + + try { + const email = await EmailService.getEmailById(emailId); + if (!email) { + throw new Error(`Email not found with id: ${emailId}`); + } + + // Skip if already in final state + if ( + email.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT || + email.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED + ) { + logger.logInfo(`Email ${emailId} is already in final state: ${email.status}`); + return email.status; + } + + // If email is already SENDING (recovery from crash), skip marking as started + // Otherwise, mark as started and audit + if (email.status !== EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING) { + await EmailService.markEmailStarted(emailId); + try { + await EmailBatchAuditService.logEmailSending(email._id, { + subject: email.subject, + }); + } catch (auditErr) { + logger.logException(auditErr, 'Audit failure: EMAIL_SENDING'); + } + } else { + logger.logInfo(`Recovering Email ${emailId} from SENDING state (from crash/restart)`); + } + + // Process all EmailBatch items + await this.processEmailBatches(email); + + // Determine final status based on batch items + const finalStatus = await EmailProcessor.determineEmailStatus(email._id); + await EmailService.markEmailCompleted(emailId, finalStatus); + // Audit completion at email level + try { + if (finalStatus === EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT) { + await EmailBatchAuditService.logEmailSent(email._id); + } else if (finalStatus === EMAIL_JOB_CONFIG.EMAIL_STATUSES.PROCESSED) { + await EmailBatchAuditService.logEmailProcessed(email._id); + } else if (finalStatus === EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED) { + await EmailBatchAuditService.logEmailFailed(email._id, new Error('Processing failed')); + } + } catch (auditErr) { + logger.logException(auditErr, 'Audit failure: EMAIL completion'); + } + + logger.logInfo(`Email ${emailId} processed with status: ${finalStatus}`); + return finalStatus; + } catch (error) { + logger.logException(error, `Error processing Email ${emailId}`); + + // Mark email as failed on error + try { + await EmailService.markEmailCompleted(emailId, EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED); + // Audit failure + try { + const failedEmail = await EmailService.getEmailById(emailId); + if (failedEmail) { + await EmailBatchAuditService.logEmailFailed(failedEmail._id, error); + } + } catch (auditErr) { + logger.logException(auditErr, 'Audit failure: EMAIL_FAILED'); + } + } catch (updateError) { + logger.logException(updateError, 'Error updating Email status to failed'); + } + return EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED; + } finally { + this.processingBatches.delete(emailId); + } + } + + /** + * Process all EmailBatch items for an Email + * Processes ALL QUEUED items regardless of individual failures - ensures maximum delivery + * @param {Object} email - The Email document + */ + async processEmailBatches(email) { + // Get ALL batches for this email first + const allBatches = await EmailBatch.find({ + emailId: email._id, + }); + + if (allBatches.length === 0) { + logger.logInfo(`No EmailBatch items found for Email ${email._id}`); + return; + } + + // Separate batches by status + // If we're processing this Email, any SENDING EmailBatch items are considered stuck + const stuckSendingBatches = allBatches.filter( + (batch) => batch.status === EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENDING, + ); + + const queuedBatches = allBatches.filter( + (batch) => batch.status === EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED, + ); + + // Reset stuck SENDING batches to QUEUED + if (stuckSendingBatches.length > 0) { + logger.logInfo( + `Resetting ${stuckSendingBatches.length} EmailBatch items stuck in SENDING state for Email ${email._id}`, + ); + await Promise.all( + stuckSendingBatches.map(async (batch) => { + await EmailBatchService.resetEmailBatchForRetry(batch._id); + + // Audit recovery + try { + await EmailBatchAuditService.logAction( + email._id, + EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_BATCH_QUEUED, + 'EmailBatch item reset from SENDING state (crash/restart recovery)', + { reason: 'Recovery from stuck SENDING state', recoveryTime: new Date() }, + null, + null, + batch._id, + ); + } catch (auditErr) { + logger.logException(auditErr, 'Audit failure: EMAIL_BATCH_QUEUED (recovery)'); + } + }), + ); + + // Add reset batches to queued list for processing + queuedBatches.push(...stuckSendingBatches); + } + + if (queuedBatches.length === 0) { + logger.logInfo(`No EmailBatch items to process for Email ${email._id}`); + return; + } + + logger.logInfo(`Processing ${queuedBatches.length} EmailBatch items for Email ${email._id}`); + + // Process items with concurrency limit, but use Promise.allSettled to ensure ALL items are attempted + const concurrency = EMAIL_JOB_CONFIG.ANNOUNCEMENTS.CONCURRENCY || 3; + const results = []; + + // eslint-disable-next-line no-await-in-loop + for (let i = 0; i < queuedBatches.length; i += concurrency) { + const batch = queuedBatches.slice(i, i + concurrency); + // eslint-disable-next-line no-await-in-loop + const batchResults = await Promise.allSettled( + batch.map((item) => this.processEmailBatch(item, email)), + ); + results.push(...batchResults); + } + + // Log summary of all processing attempts + const succeeded = results.filter((r) => r.status === 'fulfilled').length; + const failed = results.filter((r) => r.status === 'rejected').length; + + logger.logInfo( + `Completed processing ${queuedBatches.length} EmailBatch items for Email ${email._id}: ${succeeded} succeeded, ${failed} failed`, + ); + } + + /** + * Process a single EmailBatch item with multiple recipients + * @param {Object} item - The EmailBatch item + * @param {Object} email - The parent Email document + */ + async processEmailBatch(item, email) { + if (!item || !item._id) { + throw new Error('Invalid EmailBatch item'); + } + if (!email || !email._id) { + throw new Error('Invalid Email parent'); + } + + const recipientEmails = (item.recipients || []) + .map((r) => r?.email) + .filter((e) => e && typeof e === 'string'); + + if (recipientEmails.length === 0) { + logger.logException( + new Error('No valid recipients found'), + `EmailBatch item ${item._id} has no valid recipients`, + ); + const failedItem = await EmailBatchService.markEmailBatchFailed(item._id, { + errorCode: 'NO_RECIPIENTS', + errorMessage: 'No valid recipients found', + }); + + // Audit logging + try { + await EmailBatchAuditService.logEmailBatchFailed( + item.emailId, + item._id, + { message: failedItem.lastError, code: failedItem.errorCode }, + { + recipientCount: failedItem?.recipients?.length || 0, + emailType: failedItem?.emailType, + recipients: failedItem?.recipients?.map((r) => r.email) || [], + emailBatchId: failedItem?._id.toString(), + }, + ); + } catch (auditError) { + logger.logException(auditError, 'Audit failure: EMAIL_BATCH_FAILED'); + } + return; + } + + const processWithRetry = async (attempt = 1) => { + try { + // Mark as SENDING using service method + const updatedItem = await EmailBatchService.markEmailBatchSending(item._id); + + // Audit logging after successful status update + try { + await EmailBatchAuditService.logEmailBatchSending(item.emailId, item._id, { + attempt: updatedItem?.attempts || attempt, + recipientCount: updatedItem?.recipients?.length || 0, + emailType: updatedItem?.emailType, + recipients: updatedItem?.recipients?.map((r) => r.email) || [], + emailBatchId: updatedItem?._id.toString(), + }); + } catch (auditError) { + logger.logException(auditError, 'Audit failure: EMAIL_BATCH_SENDING'); + } + + // Send email directly via service (no retries here - handled by processWithRetry) + // Use the emailType stored in the EmailBatch item + const mailOptions = { + from: process.env.REACT_APP_EMAIL, + subject: email.subject, + html: email.htmlContent, + }; + + // Set recipients based on emailType + if (item.emailType === EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC) { + // For BCC, sender goes in 'to' field, all recipients in 'bcc' + mailOptions.to = process.env.REACT_APP_EMAIL; + mailOptions.bcc = recipientEmails.join(','); + } else { + // For TO/CC, recipients go in respective fields + mailOptions.to = recipientEmails.join(','); + } + + const sendResult = await emailAnnouncementService.sendEmail(mailOptions); + + // Handle result: { success, response?, error? } + if (sendResult.success) { + await EmailBatchService.markEmailBatchSent(item._id); + try { + await EmailBatchAuditService.logEmailBatchSent( + email._id, + item._id, + { + recipientCount: recipientEmails.length, + emailType: item.emailType, + attempt: updatedItem?.attempts || attempt, + }, + sendResult.response, + ); + } catch (auditError) { + logger.logException(auditError, 'Audit failure: EMAIL_BATCH_SENT'); + } + logger.logInfo( + `EmailBatch item ${item._id} sent successfully to ${recipientEmails.length} recipients (attempt ${updatedItem?.attempts || attempt})`, + ); + } else { + // Consider as failure for this attempt + throw sendResult.error || new Error('Failed to send email'); + } + } catch (error) { + logger.logException( + error, + `Failed to send EmailBatch item ${item._id} to ${recipientEmails.length} recipients (attempt ${attempt})`, + ); + + if (attempt >= this.maxRetries) { + // Mark as FAILED using service method + const failedItem = await EmailBatchService.markEmailBatchFailed(item._id, { + errorCode: error.code || 'SEND_FAILED', + errorMessage: error.message || 'Failed to send email', + }); + + // Audit logging + try { + await EmailBatchAuditService.logEmailBatchFailed( + item.emailId, + item._id, + { message: failedItem.lastError, code: failedItem.errorCode }, + { + recipientCount: failedItem?.recipients?.length || 0, + emailType: failedItem?.emailType, + recipients: failedItem?.recipients?.map((r) => r.email) || [], + emailBatchId: failedItem?._id.toString(), + }, + ); + } catch (auditError) { + logger.logException(auditError, 'Audit failure: EMAIL_BATCH_FAILED'); + } + + logger.logInfo( + `Permanently failed to send EmailBatch item ${item._id} to ${recipientEmails.length} recipients after ${this.maxRetries} attempts`, + ); + // Throw error so Promise.allSettled can distinguish failed from successful items + // This ensures accurate reporting in the summary log + throw error; + } + + // Log transient failure for this attempt (best-effort, not changing DB status) + try { + await EmailBatchAuditService.logEmailBatchFailed(email._id, item._id, error, { + recipientCount: recipientEmails.length, + emailType: item.emailType, + attempt, + transient: true, + }); + } catch (auditError) { + logger.logException(auditError, 'Audit failure: EMAIL_BATCH_FAILED (transient)'); + } + + // Wait before retry with exponential backoff (2^n: 1x, 2x, 4x, 8x, ...) + const delay = this.retryDelay * 2 ** (attempt - 1); + await EmailProcessor.sleep(delay); + return processWithRetry(attempt + 1); + } + }; + + return processWithRetry(1); + } + + /** + * Sleep utility + */ + static sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); + } + + /** + * Determine final status for an Email based on its EmailBatch items + */ + static async determineEmailStatus(emailObjectId) { + const counts = await EmailBatch.aggregate([ + { $match: { emailId: emailObjectId } }, + { + $group: { + _id: '$status', + count: { $sum: 1 }, + }, + }, + ]); + + const statusMap = counts.reduce((acc, item) => { + acc[item._id] = item.count; + return acc; + }, {}); + + const queued = statusMap[EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED] || 0; + const sending = statusMap[EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENDING] || 0; + const sent = statusMap[EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENT] || 0; + const failed = statusMap[EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.FAILED] || 0; + + // All sent = SENT + if (sent > 0 && queued === 0 && sending === 0 && failed === 0) { + return EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT; + } + + // All failed = FAILED + if (failed > 0 && queued === 0 && sending === 0 && sent === 0) { + return EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED; + } + + // Mixed results = PROCESSED + if (sent > 0 || failed > 0) { + return EMAIL_JOB_CONFIG.EMAIL_STATUSES.PROCESSED; + } + + // Still processing = keep current status + return EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING; + } + + /** + * Get processor status + */ + getStatus() { + return { + isRunning: true, + processingBatches: Array.from(this.processingBatches), + maxRetries: this.maxRetries, + }; + } +} + +// Create singleton instance +const emailProcessor = new EmailProcessor(); + +module.exports = emailProcessor; diff --git a/src/services/emailService.js b/src/services/emailService.js new file mode 100644 index 000000000..7ffdce9d9 --- /dev/null +++ b/src/services/emailService.js @@ -0,0 +1,93 @@ +const mongoose = require('mongoose'); +const Email = require('../models/email'); +const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); + +class EmailService { + static async createEmail({ subject, htmlContent, createdBy }, session = null) { + const normalizedSubject = typeof subject === 'string' ? subject.trim() : subject; + const normalizedHtml = typeof htmlContent === 'string' ? htmlContent.trim() : htmlContent; + + const email = new Email({ + subject: normalizedSubject, + htmlContent: normalizedHtml, + createdBy, + }); + + // Save with session if provided for transaction support + await email.save({ session }); + + return email; + } + + static async getEmailById(id, session = null) { + if (!id || !mongoose.Types.ObjectId.isValid(id)) return null; + return Email.findById(id).session(session); + } + + static async updateEmailStatus(emailId, status) { + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + throw new Error('Valid email ID is required'); + } + if (!Object.values(EMAIL_JOB_CONFIG.EMAIL_STATUSES).includes(status)) { + throw new Error('Invalid email status'); + } + const email = await Email.findByIdAndUpdate( + emailId, + { status, updatedAt: new Date() }, + { new: true }, + ); + return email; + } + + static async markEmailStarted(emailId) { + const now = new Date(); + const email = await Email.findByIdAndUpdate( + emailId, + { + status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING, + startedAt: now, + updatedAt: now, + }, + { new: true }, + ); + return email; + } + + static async markEmailCompleted(emailId, finalStatus) { + const now = new Date(); + const statusToSet = Object.values(EMAIL_JOB_CONFIG.EMAIL_STATUSES).includes(finalStatus) + ? finalStatus + : EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT; + + const email = await Email.findByIdAndUpdate( + emailId, + { + status: statusToSet, + completedAt: now, + updatedAt: now, + }, + { new: true }, + ); + return email; + } + + /** + * Mark an Email as QUEUED for retry (e.g., after resetting failed EmailBatch items) + */ + static async markEmailQueued(emailId) { + const now = new Date(); + const email = await Email.findByIdAndUpdate( + emailId, + { + status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.QUEUED, + startedAt: null, + completedAt: null, + updatedAt: now, + }, + { new: true }, + ); + return email; + } +} + +module.exports = EmailService; diff --git a/src/utilities/emailSender.js b/src/utilities/emailSender.js index c7112a5dc..ec571cbed 100644 --- a/src/utilities/emailSender.js +++ b/src/utilities/emailSender.js @@ -87,11 +87,9 @@ const sendWithRetry = async (batch, retries = 3, baseDelay = 1000) => { for (let attempt = 1; attempt <= retries; attempt += 1) { try { - // eslint-disable-next-line no-await-in-loop await sendEmail(batch); if (isBsAssignment) { - // eslint-disable-next-line no-await-in-loop await EmailHistory.findOneAndUpdate( { uniqueKey: key }, { @@ -115,7 +113,6 @@ const sendWithRetry = async (batch, retries = 3, baseDelay = 1000) => { logger.logException(err, `Batch to ${batch.to || '(empty)'} attempt ${attempt}`); if (attempt === retries && isBsAssignment) { - // eslint-disable-next-line no-await-in-loop await EmailHistory.findOneAndUpdate( { uniqueKey: key }, { @@ -136,46 +133,34 @@ const sendWithRetry = async (batch, retries = 3, baseDelay = 1000) => { } } - if (attempt < retries) { - // eslint-disable-next-line no-await-in-loop - await sleep(baseDelay * attempt); // backoff - } + if (attempt < retries) await sleep(baseDelay * attempt); // backoff } return false; }; const worker = async () => { - let allSuccessful = true; - // eslint-disable-next-line no-constant-condition while (true) { // atomically pull next batch const batch = queue.shift(); if (!batch) break; // queue drained for this worker - // eslint-disable-next-line no-await-in-loop - const result = await sendWithRetry(batch); - if (result === false) { - allSuccessful = false; - } - if (config.rateLimitDelay) { - // eslint-disable-next-line no-await-in-loop - await sleep(config.rateLimitDelay); // pacing + const success = await sendWithRetry(batch); + if (!success) { + throw new Error(`Failed to send email to ${batch.to} after all retry attempts`); } + if (config.rateLimitDelay) await sleep(config.rateLimitDelay); // pacing } - return allSuccessful; }; const processQueue = async () => { - if (isProcessing || queue.length === 0) return true; + if (isProcessing || queue.length === 0) return; isProcessing = true; try { const n = Math.max(1, Number(config.concurrency) || 1); const workers = Array.from({ length: n }, () => worker()); - const results = await Promise.all(workers); // drain-until-empty with N workers - // Return true if all workers succeeded, false if any failed - return results.every((result) => result !== false); + await Promise.all(workers); // drain-until-empty with N workers } finally { isProcessing = false; } @@ -251,12 +236,8 @@ const emailSender = ( setImmediate(async () => { try { - const result = await processQueue(); - if (result === false) { - reject(new Error('Email sending failed after all retries')); - } else { - resolve('Emails processed successfully'); - } + await processQueue(); + resolve('Emails processed successfully'); } catch (error) { reject(error); } diff --git a/src/utilities/emailValidators.js b/src/utilities/emailValidators.js new file mode 100644 index 000000000..ba764d884 --- /dev/null +++ b/src/utilities/emailValidators.js @@ -0,0 +1,125 @@ +const cheerio = require('cheerio'); +const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); + +/** + * Validate email address format + * @param {string} email - Email address to validate + * @returns {boolean} True if valid email format + */ +function isValidEmailAddress(email) { + if (!email || typeof email !== 'string') return false; + const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + return emailRegex.test(email.trim()); +} + +/** + * Normalize recipients input to array of email strings + * Handles both array and single value, removes duplicates (case-insensitive) + * @param {string|Array} input - Recipient(s) to normalize + * @returns {Array} Array of unique email strings + */ +function normalizeRecipientsToArray(input) { + const arr = Array.isArray(input) ? input : [input]; + const trimmed = arr + .map((e) => (typeof e === 'string' ? e.trim() : '')) + .filter((e) => e.length > 0); + // Dedupe case-insensitively + const seen = new Set(); + return trimmed.filter((e) => { + const key = e.toLowerCase(); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +/** + * Normalize recipients input to array of { email } objects + * Used by EmailBatchService for creating EmailBatch records + * @param {Array} input - Recipients array + * @returns {Array<{email: string}>} Array of recipient objects + */ +function normalizeRecipientsToObjects(input) { + if (!Array.isArray(input)) return []; + const emails = input + .filter((item) => { + if (typeof item === 'string') { + return item.trim().length > 0; + } + return item && typeof item.email === 'string' && item.email.trim().length > 0; + }) + .map((item) => ({ + email: typeof item === 'string' ? item.trim() : item.email.trim(), + })); + + // Dedupe case-insensitively + const seen = new Set(); + return emails.filter((obj) => { + const key = obj.email.toLowerCase(); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +/** + * Validate HTML content size is within limit + * @param {string} html - HTML content to validate + * @returns {boolean} True if within limit + */ +function ensureHtmlWithinLimit(html) { + const maxBytes = EMAIL_JOB_CONFIG.LIMITS.MAX_HTML_BYTES; + const size = Buffer.byteLength(html || '', 'utf8'); + return size <= maxBytes; +} + +/** + * Validate HTML content does not contain base64-encoded media (data URIs) + * Only URLs are allowed for media to keep emails light + * @param {string} html - HTML content to validate + * @returns {{isValid: boolean, errors: Array}} Validation result + */ +function validateHtmlMedia(html) { + const $ = cheerio.load(html); + const invalidMedia = []; + + // Check for base64 images in img tags + $('img').each((i, img) => { + const src = $(img).attr('src'); + if (src && src.startsWith('data:image')) { + invalidMedia.push(`Image ${i + 1}: base64-encoded image detected (use URL instead)`); + } + }); + + // Check for base64 images in CSS background-image + const htmlString = $.html(); + const base64ImageRegex = /data:image\/[^;]+;base64,[^\s"')]+/gi; + const backgroundMatches = htmlString.match(base64ImageRegex); + if (backgroundMatches) { + invalidMedia.push( + `${backgroundMatches.length} base64-encoded background image(s) detected (use URL instead)`, + ); + } + + // Check for base64 audio/video + const base64MediaRegex = /data:(audio|video)\/[^;]+;base64,[^\s"')]+/gi; + const mediaMatches = htmlString.match(base64MediaRegex); + if (mediaMatches) { + invalidMedia.push( + `${mediaMatches.length} base64-encoded media file(s) detected (use URL instead)`, + ); + } + + return { + isValid: invalidMedia.length === 0, + errors: invalidMedia, + }; +} + +module.exports = { + isValidEmailAddress, + normalizeRecipientsToArray, + normalizeRecipientsToObjects, + ensureHtmlWithinLimit, + validateHtmlMedia, +}; From be11ad4d8648406262ab10a11cfb08521d051fab Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Thu, 30 Oct 2025 12:11:57 -0400 Subject: [PATCH 04/19] feat(email): implement permission checks for email operations - Added permission checks for viewing emails, email details, and email audits in emailBatchController. - Implemented permission validation for sending emails, resending, and managing email subscriptions in emailController. - Updated emailTemplateController to enforce permission checks for creating, updating, and deleting email templates. - Refactored routes to replace deprecated sendEmailToAll with sendEmailToSubscribers. - Enhanced test cases to ensure proper handling of permission-related responses. --- src/controllers/emailBatchController.js | 80 +++++++++--- src/controllers/emailController.js | 145 +++++++++++---------- src/controllers/emailController.spec.js | 41 ++++-- src/controllers/emailTemplateController.js | 128 +++++++++--------- src/routes/emailRouter.js | 4 +- src/services/emailAnnouncementService.js | 10 +- src/test/createTestPermissions.js | 1 - src/utilities/createInitialPermissions.js | 1 - 8 files changed, 234 insertions(+), 176 deletions(-) diff --git a/src/controllers/emailBatchController.js b/src/controllers/emailBatchController.js index 38796f15f..af602d7ca 100644 --- a/src/controllers/emailBatchController.js +++ b/src/controllers/emailBatchController.js @@ -5,6 +5,7 @@ const EmailBatchAuditService = require('../services/emailBatchAuditService'); const emailAnnouncementJobProcessor = require('../jobs/emailAnnouncementJobProcessor'); const EmailBatch = require('../models/emailBatch'); const Email = require('../models/email'); +const { hasPermission } = require('../utilities/permissions'); const logger = require('../startup/logger'); const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); @@ -13,6 +14,19 @@ const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); */ const getEmails = async (req, res) => { try { + // Permission check - viewing emails requires sendEmails permission + if (!req?.body?.requestor?.requestorId && !req?.user?.userid) { + return res.status(401).json({ success: false, message: 'Missing requestor' }); + } + + const requestor = req.body.requestor || req.user; + const canViewEmails = await hasPermission(requestor, 'sendEmails'); + if (!canViewEmails) { + return res + .status(403) + .json({ success: false, message: 'You are not authorized to view emails.' }); + } + const emails = await EmailBatchService.getAllEmails(); res.status(200).json({ @@ -34,6 +48,19 @@ const getEmails = async (req, res) => { */ const getEmailDetails = async (req, res) => { try { + // Permission check - viewing email details requires sendEmails permission + if (!req?.body?.requestor?.requestorId && !req?.user?.userid) { + return res.status(401).json({ success: false, message: 'Missing requestor' }); + } + + const requestor = req.body.requestor || req.user; + const canViewEmails = await hasPermission(requestor, 'sendEmails'); + if (!canViewEmails) { + return res + .status(403) + .json({ success: false, message: 'You are not authorized to view email details.' }); + } + const { emailId } = req.params; // emailId is now the ObjectId of parent Email if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { @@ -103,8 +130,21 @@ const retryEmail = async (req, res) => { }); } + // Permission check - retrying emails requires sendEmails permission + if (!req?.body?.requestor?.requestorId && !req?.user?.userid) { + return res.status(401).json({ success: false, message: 'Missing requestor' }); + } + + const requestor = req.body.requestor || req.user; + const canRetryEmail = await hasPermission(requestor, 'sendEmails'); + if (!canRetryEmail) { + return res + .status(403) + .json({ success: false, message: 'You are not authorized to retry emails.' }); + } + // Get requestor for audit trail - const requestorId = req?.body?.requestor?.requestorId || null; + const requestorId = requestor.requestorId || requestor.userid; // Find the Email const email = await Email.findById(emailId); @@ -219,16 +259,15 @@ const getEmailAuditTrail = async (req, res) => { }); } - // TODO: Re-enable permission check in future - // Permission check - commented out for now - // const requestor = req.body.requestor || req.user; - // const canViewAudits = await hasPermission(requestor, 'viewEmailAudits'); - // if (!canViewAudits) { - // return res.status(403).json({ - // success: false, - // message: 'You are not authorized to view email audits', - // }); - // } + // Permission check - use sendEmails permission to view audits + const requestor = req.body.requestor || req.user; + const canViewAudits = await hasPermission(requestor, 'sendEmails'); + if (!canViewAudits) { + return res.status(403).json({ + success: false, + message: 'You are not authorized to view email audits', + }); + } const { emailId } = req.params; @@ -279,16 +318,15 @@ const getEmailBatchAuditTrail = async (req, res) => { }); } - // TODO: Re-enable permission check in future - // Permission check - commented out for now - // const requestor = req.body.requestor || req.user; - // const canViewAudits = await hasPermission(requestor, 'viewEmailAudits'); - // if (!canViewAudits) { - // return res.status(403).json({ - // success: false, - // message: 'You are not authorized to view email audits', - // }); - // } + // Permission check - use sendEmails permission to view audits + const requestor = req.body.requestor || req.user; + const canViewAudits = await hasPermission(requestor, 'sendEmails'); + if (!canViewAudits) { + return res.status(403).json({ + success: false, + message: 'You are not authorized to view email audits', + }); + } const { emailBatchId } = req.params; diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index 43999e4d6..5226775e4 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -14,24 +14,26 @@ const userProfile = require('../models/userProfile'); const EmailBatchService = require('../services/emailBatchService'); const EmailService = require('../services/emailService'); const EmailBatchAuditService = require('../services/emailBatchAuditService'); +const { hasPermission } = require('../utilities/permissions'); const config = require('../config'); const logger = require('../startup/logger'); const jwtSecret = process.env.JWT_SECRET; const sendEmail = async (req, res) => { - // TODO: Re-enable permission check in future - // Permission check - commented out for now - // const canSendEmail = await hasPermission(req.body.requestor, 'sendEmails'); - // if (!canSendEmail) { - // return res.status(403).json({ success: false, message: 'You are not authorized to send emails.' }); - // } - - // Requestor is still required for getting user ID for audit trail + // Requestor is required for permission check and audit trail if (!req?.body?.requestor?.requestorId) { return res.status(401).json({ success: false, message: 'Missing requestor' }); } + // Permission check + const canSendEmail = await hasPermission(req.body.requestor, 'sendEmails'); + if (!canSendEmail) { + return res + .status(403) + .json({ success: false, message: 'You are not authorized to send emails.' }); + } + try { const { to, subject, html } = req.body; @@ -40,12 +42,10 @@ const sendEmail = async (req, res) => { if (!html) missingFields.push('HTML content'); if (!to) missingFields.push('Recipient email'); if (missingFields.length) { - return res - .status(400) - .json({ - success: false, - message: `${missingFields.join(' and ')} ${missingFields.length > 1 ? 'are' : 'is'} required`, - }); + return res.status(400).json({ + success: false, + message: `${missingFields.join(' and ')} ${missingFields.length > 1 ? 'are' : 'is'} required`, + }); } // Validate HTML content size @@ -58,12 +58,10 @@ const sendEmail = async (req, res) => { // Validate subject length against config if (subject && subject.length > EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH) { - return res - .status(400) - .json({ - success: false, - message: `Subject cannot exceed ${EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH} characters`, - }); + return res.status(400).json({ + success: false, + message: `Subject cannot exceed ${EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH} characters`, + }); } // Validate HTML does not contain base64-encoded media @@ -117,22 +115,18 @@ const sendEmail = async (req, res) => { .json({ success: false, message: 'At least one recipient email is required' }); } if (recipientsArray.length > EMAIL_JOB_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST) { - return res - .status(400) - .json({ - success: false, - message: `A maximum of ${EMAIL_JOB_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST} recipients are allowed per request`, - }); + return res.status(400).json({ + success: false, + message: `A maximum of ${EMAIL_JOB_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST} recipients are allowed per request`, + }); } const invalidRecipients = recipientsArray.filter((e) => !isValidEmailAddress(e)); if (invalidRecipients.length) { - return res - .status(400) - .json({ - success: false, - message: 'One or more recipient emails are invalid', - invalidRecipients, - }); + return res.status(400).json({ + success: false, + message: 'One or more recipient emails are invalid', + invalidRecipients, + }); } // Always use batch system for tracking and progress @@ -223,19 +217,20 @@ const sendEmail = async (req, res) => { } }; -const sendEmailToAll = async (req, res) => { - // TODO: Re-enable permission check in future - // Permission check - commented out for now - // const canSendEmailToAll = await hasPermission(req.body.requestor, 'sendEmailToAll'); - // if (!canSendEmailToAll) { - // return res.status(403).json({ success: false, message: 'You are not authorized to send emails to all.' }); - // } - - // Requestor is still required for getting user ID for audit trail +const sendEmailToSubscribers = async (req, res) => { + // Requestor is required for permission check and audit trail if (!req?.body?.requestor?.requestorId) { return res.status(401).json({ success: false, message: 'Missing requestor' }); } + // Permission check - sendEmailToSubscribers requires sendEmails + const cansendEmailToSubscribers = await hasPermission(req.body.requestor, 'sendEmails'); + if (!cansendEmailToSubscribers) { + return res + .status(403) + .json({ success: false, message: 'You are not authorized to send emails to subscribers.' }); + } + try { const { subject, html } = req.body; if (!subject || !html) { @@ -245,12 +240,10 @@ const sendEmailToAll = async (req, res) => { } if (!ensureHtmlWithinLimit(html)) { - return res - .status(413) - .json({ - success: false, - message: `HTML content exceeds ${EMAIL_JOB_CONFIG.LIMITS.MAX_HTML_BYTES / (1024 * 1024)}MB limit`, - }); + return res.status(413).json({ + success: false, + message: `HTML content exceeds ${EMAIL_JOB_CONFIG.LIMITS.MAX_HTML_BYTES / (1024 * 1024)}MB limit`, + }); } // Validate HTML does not contain base64-encoded media @@ -405,11 +398,19 @@ const sendEmailToAll = async (req, res) => { }; const resendEmail = async (req, res) => { - // Requestor is required for getting user ID for audit trail + // Requestor is required for permission check and audit trail if (!req?.body?.requestor?.requestorId) { return res.status(401).json({ success: false, message: 'Missing requestor' }); } + // Permission check - resending requires sendEmails permission + const canSendEmail = await hasPermission(req.body.requestor, 'sendEmails'); + if (!canSendEmail) { + return res + .status(403) + .json({ success: false, message: 'You are not authorized to resend emails.' }); + } + try { const { emailId, recipientOption, specificRecipients } = req.body; @@ -472,25 +473,21 @@ const resendEmail = async (req, res) => { !Array.isArray(specificRecipients) || specificRecipients.length === 0 ) { - return res - .status(400) - .json({ - success: false, - message: 'specificRecipients array is required for specific option', - }); + return res.status(400).json({ + success: false, + message: 'specificRecipients array is required for specific option', + }); } // Normalize and validate recipients const recipientsArray = normalizeRecipientsToArray(specificRecipients); const invalidRecipients = recipientsArray.filter((e) => !isValidEmailAddress(e)); if (invalidRecipients.length) { - return res - .status(400) - .json({ - success: false, - message: 'One or more recipient emails are invalid', - invalidRecipients, - }); + return res.status(400).json({ + success: false, + message: 'One or more recipient emails are invalid', + invalidRecipients, + }); } allRecipients = recipientsArray.map((email) => ({ email })); @@ -666,6 +663,18 @@ const addNonHgnEmailSubscription = async (req, res) => { return res.status(400).json({ success: false, message: 'Email already subscribed' }); } + // check if this email is already in the HGN user list + const hgnUser = await userProfile.findOne({ email: normalizedEmail }); + if (hgnUser) { + return res + .status(400) + .json({ + success: false, + message: + 'You are already a member of the HGN community. Please use the HGN account profile page to subscribe to email updates.', + }); + } + // Save to DB immediately with confirmation pending const newEmailList = new EmailSubcriptionList({ email: normalizedEmail, @@ -701,12 +710,10 @@ const addNonHgnEmailSubscription = async (req, res) => { null, { type: 'subscription_confirmation' }, ); - return res - .status(200) - .json({ - success: true, - message: 'Email subscribed successfully. Please check your inbox to confirm.', - }); + return res.status(200).json({ + success: true, + message: 'Email subscribed successfully. Please check your inbox to confirm.', + }); } catch (emailError) { logger.logException(emailError, 'Error sending confirmation email'); // Still return success since the subscription was saved to DB @@ -822,7 +829,7 @@ const removeNonHgnEmailSubscription = async (req, res) => { module.exports = { sendEmail, - sendEmailToAll, + sendEmailToSubscribers, resendEmail, updateEmailSubscriptions, addNonHgnEmailSubscription, diff --git a/src/controllers/emailController.spec.js b/src/controllers/emailController.spec.js index fd6c2e38e..44e49d8a0 100644 --- a/src/controllers/emailController.spec.js +++ b/src/controllers/emailController.spec.js @@ -11,7 +11,7 @@ jest.mock('../utilities/emailSender'); const makeSut = () => { const { sendEmail, - sendEmailToAll, + sendEmailToSubscribers, updateEmailSubscriptions, addNonHgnEmailSubscription, removeNonHgnEmailSubscription, @@ -19,7 +19,7 @@ const makeSut = () => { } = emailController; return { sendEmail, - sendEmailToAll, + sendEmailToSubscribers, updateEmailSubscriptions, addNonHgnEmailSubscription, removeNonHgnEmailSubscription, @@ -75,16 +75,20 @@ describe('updateEmailSubscriptions function', () => { const updateReq = { body: { - emailSubscriptions: ['subscription1', 'subscription2'], + emailSubscriptions: true, requestor: { email: 'test@example.com', }, }, }; - const response = await updateEmailSubscriptions(updateReq, mockRes); + await updateEmailSubscriptions(updateReq, mockRes); - assertResMock(500, 'Error updating email subscriptions', response, mockRes); + expect(mockRes.status).toHaveBeenCalledWith(500); + expect(mockRes.json).toHaveBeenCalledWith({ + success: false, + message: 'Error updating email subscriptions', + }); }); }); @@ -101,9 +105,13 @@ describe('confirmNonHgnEmailSubscription function', () => { const { confirmNonHgnEmailSubscription } = makeSut(); const emptyReq = { body: {} }; - const response = await confirmNonHgnEmailSubscription(emptyReq, mockRes); + await confirmNonHgnEmailSubscription(emptyReq, mockRes); - assertResMock(400, 'Invalid token', response, mockRes); + expect(mockRes.status).toHaveBeenCalledWith(400); + expect(mockRes.json).toHaveBeenCalledWith({ + success: false, + message: 'Token is required', + }); }); test('should return 401 if token is invalid', async () => { @@ -118,7 +126,8 @@ describe('confirmNonHgnEmailSubscription function', () => { expect(mockRes.status).toHaveBeenCalledWith(401); expect(mockRes.json).toHaveBeenCalledWith({ - errors: [{ msg: 'Token is not valid' }], + success: false, + message: 'Invalid or expired token', }); }); @@ -129,9 +138,13 @@ describe('confirmNonHgnEmailSubscription function', () => { // Mocking jwt.verify to return a payload without email jwt.verify.mockReturnValue({}); - const response = await confirmNonHgnEmailSubscription(validTokenReq, mockRes); + await confirmNonHgnEmailSubscription(validTokenReq, mockRes); - assertResMock(400, 'Invalid token', response, mockRes); + expect(mockRes.status).toHaveBeenCalledWith(400); + expect(mockRes.json).toHaveBeenCalledWith({ + success: false, + message: 'Invalid token payload', + }); }); }); @@ -144,8 +157,12 @@ describe('removeNonHgnEmailSubscription function', () => { const { removeNonHgnEmailSubscription } = makeSut(); const noEmailReq = { body: {} }; - const response = await removeNonHgnEmailSubscription(noEmailReq, mockRes); + await removeNonHgnEmailSubscription(noEmailReq, mockRes); - assertResMock(400, 'Email is required', response, mockRes); + expect(mockRes.status).toHaveBeenCalledWith(400); + expect(mockRes.json).toHaveBeenCalledWith({ + success: false, + message: 'Email is required', + }); }); }); diff --git a/src/controllers/emailTemplateController.js b/src/controllers/emailTemplateController.js index 9e3cc8cd2..45a859078 100644 --- a/src/controllers/emailTemplateController.js +++ b/src/controllers/emailTemplateController.js @@ -1,6 +1,7 @@ const mongoose = require('mongoose'); const EmailTemplate = require('../models/emailTemplate'); const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +const { hasPermission } = require('../utilities/permissions'); const logger = require('../startup/logger'); const { ensureHtmlWithinLimit, validateHtmlMedia } = require('../utilities/emailValidators'); @@ -112,23 +113,22 @@ function validateTemplateVariableUsage(templateVariables, htmlContent, subject) */ const getAllEmailTemplates = async (req, res) => { try { - // TODO: Re-enable permission check in future - // Permission check - commented out for now - // if (!req?.body?.requestor && !req?.user) { - // return res.status(401).json({ - // success: false, - // message: 'Missing requestor', - // }); - // } - - // const requestor = req.body.requestor || req.user; - // const canViewTemplates = await hasPermission(requestor, 'viewEmailTemplates'); - // if (!canViewTemplates) { - // return res.status(403).json({ - // success: false, - // message: 'You are not authorized to view email templates.', - // }); - // } + // Permission check - use sendEmails permission to view templates + if (!req?.body?.requestor?.requestorId && !req?.user?.userid) { + return res.status(401).json({ + success: false, + message: 'Missing requestor', + }); + } + + const requestor = req.body.requestor || req.user; + const canViewTemplates = await hasPermission(requestor, 'sendEmails'); + if (!canViewTemplates) { + return res.status(403).json({ + success: false, + message: 'You are not authorized to view email templates.', + }); + } const { search, sortBy, includeEmailContent } = req.query; @@ -179,23 +179,22 @@ const getAllEmailTemplates = async (req, res) => { */ const getEmailTemplateById = async (req, res) => { try { - // TODO: Re-enable permission check in future - // Permission check - commented out for now - // if (!req?.body?.requestor && !req?.user) { - // return res.status(401).json({ - // success: false, - // message: 'Missing requestor', - // }); - // } - - // const requestor = req.body.requestor || req.user; - // const canViewTemplates = await hasPermission(requestor, 'viewEmailTemplates'); - // if (!canViewTemplates) { - // return res.status(403).json({ - // success: false, - // message: 'You are not authorized to view email templates.', - // }); - // } + // Permission check - use sendEmails permission to view templates + if (!req?.body?.requestor?.requestorId && !req?.user?.userid) { + return res.status(401).json({ + success: false, + message: 'Missing requestor', + }); + } + + const requestor = req.body.requestor || req.user; + const canViewTemplates = await hasPermission(requestor, 'sendEmails'); + if (!canViewTemplates) { + return res.status(403).json({ + success: false, + message: 'You are not authorized to view email templates.', + }); + } const { id } = req.params; @@ -237,16 +236,7 @@ const getEmailTemplateById = async (req, res) => { */ const createEmailTemplate = async (req, res) => { try { - // TODO: Re-enable permission check in future - // Permission check - commented out for now - // const canCreateTemplate = await hasPermission(req.body.requestor, 'createEmailTemplates'); - // if (!canCreateTemplate) { - // return res.status(403).json({ - // success: false, - // message: 'You are not authorized to create email templates.', - // }); - // } - + // Requestor is required for permission check if (!req?.body?.requestor?.requestorId) { return res.status(401).json({ success: false, @@ -254,6 +244,15 @@ const createEmailTemplate = async (req, res) => { }); } + // Permission check - use sendEmails permission to create templates + const canCreateTemplate = await hasPermission(req.body.requestor, 'sendEmails'); + if (!canCreateTemplate) { + return res.status(403).json({ + success: false, + message: 'You are not authorized to create email templates.', + }); + } + const { name, subject, html_content: htmlContent, variables } = req.body; const userId = req.body.requestor.requestorId; @@ -390,16 +389,7 @@ const createEmailTemplate = async (req, res) => { */ const updateEmailTemplate = async (req, res) => { try { - // TODO: Re-enable permission check in future - // Permission check - commented out for now - // const canUpdateTemplate = await hasPermission(req.body.requestor, 'updateEmailTemplates'); - // if (!canUpdateTemplate) { - // return res.status(403).json({ - // success: false, - // message: 'You are not authorized to update email templates.', - // }); - // } - + // Requestor is required for permission check if (!req?.body?.requestor?.requestorId) { return res.status(401).json({ success: false, @@ -407,6 +397,15 @@ const updateEmailTemplate = async (req, res) => { }); } + // Permission check - use sendEmails permission to update templates + const canUpdateTemplate = await hasPermission(req.body.requestor, 'sendEmails'); + if (!canUpdateTemplate) { + return res.status(403).json({ + success: false, + message: 'You are not authorized to update email templates.', + }); + } + const { id } = req.params; const { name, subject, html_content: htmlContent, variables } = req.body; @@ -571,17 +570,7 @@ const updateEmailTemplate = async (req, res) => { */ const deleteEmailTemplate = async (req, res) => { try { - // TODO: Re-enable permission check in future - // Permission check - commented out for now - // const canDeleteTemplate = await hasPermission(req.body.requestor, 'deleteEmailTemplates'); - // if (!canDeleteTemplate) { - // return res.status(403).json({ - // success: false, - // message: 'You are not authorized to delete email templates.', - // }); - // } - - // Requestor is still required for audit logging + // Requestor is required for permission check and audit logging if (!req?.body?.requestor?.requestorId) { return res.status(401).json({ success: false, @@ -589,6 +578,15 @@ const deleteEmailTemplate = async (req, res) => { }); } + // Permission check - use sendEmails permission to delete templates + const canDeleteTemplate = await hasPermission(req.body.requestor, 'sendEmails'); + if (!canDeleteTemplate) { + return res.status(403).json({ + success: false, + message: 'You are not authorized to delete email templates.', + }); + } + const { id } = req.params; // Validate ObjectId diff --git a/src/routes/emailRouter.js b/src/routes/emailRouter.js index e16c4564d..04ad4fb63 100644 --- a/src/routes/emailRouter.js +++ b/src/routes/emailRouter.js @@ -1,7 +1,7 @@ const express = require('express'); const { sendEmail, - sendEmailToAll, + sendEmailToSubscribers, resendEmail, updateEmailSubscriptions, addNonHgnEmailSubscription, @@ -13,7 +13,7 @@ const routes = function () { const emailRouter = express.Router(); emailRouter.route('/send-emails').post(sendEmail); - emailRouter.route('/broadcast-emails').post(sendEmailToAll); + emailRouter.route('/broadcast-emails').post(sendEmailToSubscribers); emailRouter.route('/resend-email').post(resendEmail); emailRouter.route('/update-email-subscriptions').post(updateEmailSubscriptions); diff --git a/src/services/emailAnnouncementService.js b/src/services/emailAnnouncementService.js index c3012e0de..c268aa2ad 100644 --- a/src/services/emailAnnouncementService.js +++ b/src/services/emailAnnouncementService.js @@ -11,11 +11,11 @@ const logger = require('../startup/logger'); class EmailAnnouncementService { constructor() { this.config = { - email: process.env.REACT_APP_EMAIL, - clientId: process.env.REACT_APP_EMAIL_CLIENT_ID, - clientSecret: process.env.REACT_APP_EMAIL_CLIENT_SECRET, - redirectUri: process.env.REACT_APP_EMAIL_CLIENT_REDIRECT_URI, - refreshToken: process.env.REACT_APP_EMAIL_REFRESH_TOKEN, + email: process.env.ANNOUNCEMENT_EMAIL, + clientId: process.env.ANNOUNCEMENT_EMAIL_CLIENT_ID, + clientSecret: process.env.ANNOUNCEMENT_EMAIL_CLIENT_SECRET, + redirectUri: process.env.ANNOUNCEMENT_EMAIL_CLIENT_REDIRECT_URI, + refreshToken: process.env.ANNOUNCEMENT_EMAIL_REFRESH_TOKEN, }; // Validate configuration diff --git a/src/test/createTestPermissions.js b/src/test/createTestPermissions.js index 8b32993bc..efa923a96 100644 --- a/src/test/createTestPermissions.js +++ b/src/test/createTestPermissions.js @@ -204,7 +204,6 @@ const permissionsRoles = [ 'deleteTimeEntry', 'postTimeEntry', 'sendEmails', - 'sendEmailToAll', 'updatePassword', 'resetPassword', 'getUserProfiles', diff --git a/src/utilities/createInitialPermissions.js b/src/utilities/createInitialPermissions.js index 2fcfb4174..dbd37d8d3 100644 --- a/src/utilities/createInitialPermissions.js +++ b/src/utilities/createInitialPermissions.js @@ -243,7 +243,6 @@ const permissionsRoles = [ 'deleteTimeEntry', 'postTimeEntry', 'sendEmails', - 'sendEmailToAll', 'updatePassword', 'resetPassword', 'getUserProfiles', From e07065b3bc0b83957a5e781d55f7f09d7a09e5a9 Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Thu, 30 Oct 2025 22:32:58 -0400 Subject: [PATCH 05/19] refactor(emailAnnouncementJobProcessor): streamline cron job initialization - Removed redundant cron job initialization and logging from EmailAnnouncementJobProcessor. - Updated EmailProcessor to include additional final email status check for PROCESSED, enhancing email state validation. --- src/jobs/emailAnnouncementJobProcessor.js | 13 ------------- src/services/emailProcessor.js | 3 ++- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/jobs/emailAnnouncementJobProcessor.js b/src/jobs/emailAnnouncementJobProcessor.js index 155c85dc5..ad50852bc 100644 --- a/src/jobs/emailAnnouncementJobProcessor.js +++ b/src/jobs/emailAnnouncementJobProcessor.js @@ -21,19 +21,6 @@ class EmailAnnouncementJobProcessor { return; } - this.cronJob = new CronJob( - EMAIL_JOB_CONFIG.CRON_INTERVAL, - async () => { - await this.processPendingBatches(); - }, - null, - false, // Don't start immediately - 'America/Los_Angeles', - ); - - this.cronJob.start(); - logger.logInfo('Email announcement job processor started - runs on configured interval'); - this.cronJob = new CronJob( EMAIL_JOB_CONFIG.CRON_INTERVAL, async () => { diff --git a/src/services/emailProcessor.js b/src/services/emailProcessor.js index dc0b04463..3f4f7036d 100644 --- a/src/services/emailProcessor.js +++ b/src/services/emailProcessor.js @@ -39,7 +39,8 @@ class EmailProcessor { // Skip if already in final state if ( email.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT || - email.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED + email.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED || + email.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.PROCESSED ) { logger.logInfo(`Email ${emailId} is already in final state: ${email.status}`); return email.status; From 34a27463779c27e4e61a308bbad5d94b40abb767 Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Thu, 30 Oct 2025 23:28:19 -0400 Subject: [PATCH 06/19] feat(email): refactor email processing and enhance job management --- src/config/emailJobConfig.js | 3 +- src/controllers/emailBatchController.js | 8 +- src/controllers/emailController.js | 18 +- ...obProcessor.js => announcementEmailJob.js} | 6 +- src/server.js | 2 +- .../emails/announcementEmailProcessor.js} | 200 ++++++++---------- .../emails/announcementEmailService.js} | 24 +-- .../emails}/emailBatchAuditService.js | 6 +- .../emails}/emailBatchService.js | 10 +- .../emails}/emailService.js | 4 +- 10 files changed, 124 insertions(+), 157 deletions(-) rename src/jobs/{emailAnnouncementJobProcessor.js => announcementEmailJob.js} (97%) rename src/services/{emailProcessor.js => announcements/emails/announcementEmailProcessor.js} (67%) rename src/services/{emailAnnouncementService.js => announcements/emails/announcementEmailService.js} (89%) rename src/services/{ => announcements/emails}/emailBatchAuditService.js (97%) rename src/services/{ => announcements/emails}/emailBatchService.js (95%) rename src/services/{ => announcements/emails}/emailService.js (95%) diff --git a/src/config/emailJobConfig.js b/src/config/emailJobConfig.js index cde5fe003..a3f1e2f82 100644 --- a/src/config/emailJobConfig.js +++ b/src/config/emailJobConfig.js @@ -6,11 +6,12 @@ const EMAIL_JOB_CONFIG = { // Processing intervals CRON_INTERVAL: '0 * * * * *', // Every minute at 0 seconds + TIMEZONE: 'UTC', // Cron timezone; adjust as needed (e.g., 'America/Los_Angeles') MAX_CONCURRENT_BATCHES: 3, // Retry configuration DEFAULT_MAX_RETRIES: 3, - RETRY_DELAYS: [60000, 300000, 900000], // 1min, 5min, 15min + INITIAL_RETRY_DELAY_MS: 1000, // Status enums EMAIL_STATUSES: { diff --git a/src/controllers/emailBatchController.js b/src/controllers/emailBatchController.js index af602d7ca..8edb160b7 100644 --- a/src/controllers/emailBatchController.js +++ b/src/controllers/emailBatchController.js @@ -1,8 +1,8 @@ const mongoose = require('mongoose'); -const EmailBatchService = require('../services/emailBatchService'); -const EmailService = require('../services/emailService'); -const EmailBatchAuditService = require('../services/emailBatchAuditService'); -const emailAnnouncementJobProcessor = require('../jobs/emailAnnouncementJobProcessor'); +const EmailBatchService = require('../services/announcements/emails/emailBatchService'); +const EmailService = require('../services/announcements/emails/emailService'); +const EmailBatchAuditService = require('../services/announcements/emails/emailBatchAuditService'); +const emailAnnouncementJobProcessor = require('../jobs/announcementEmailJob'); const EmailBatch = require('../models/emailBatch'); const Email = require('../models/email'); const { hasPermission } = require('../utilities/permissions'); diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index 5226775e4..5be1f7a3f 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -11,9 +11,9 @@ const { } = require('../utilities/emailValidators'); const EmailSubcriptionList = require('../models/emailSubcriptionList'); const userProfile = require('../models/userProfile'); -const EmailBatchService = require('../services/emailBatchService'); -const EmailService = require('../services/emailService'); -const EmailBatchAuditService = require('../services/emailBatchAuditService'); +const EmailBatchService = require('../services/announcements/emails/emailBatchService'); +const EmailService = require('../services/announcements/emails/emailService'); +const EmailBatchAuditService = require('../services/announcements/emails/emailBatchAuditService'); const { hasPermission } = require('../utilities/permissions'); const config = require('../config'); const logger = require('../startup/logger'); @@ -666,13 +666,11 @@ const addNonHgnEmailSubscription = async (req, res) => { // check if this email is already in the HGN user list const hgnUser = await userProfile.findOne({ email: normalizedEmail }); if (hgnUser) { - return res - .status(400) - .json({ - success: false, - message: - 'You are already a member of the HGN community. Please use the HGN account profile page to subscribe to email updates.', - }); + return res.status(400).json({ + success: false, + message: + 'You are already a member of the HGN community. Please use the HGN account profile page to subscribe to email updates.', + }); } // Save to DB immediately with confirmation pending diff --git a/src/jobs/emailAnnouncementJobProcessor.js b/src/jobs/announcementEmailJob.js similarity index 97% rename from src/jobs/emailAnnouncementJobProcessor.js rename to src/jobs/announcementEmailJob.js index ad50852bc..544926b6a 100644 --- a/src/jobs/emailAnnouncementJobProcessor.js +++ b/src/jobs/announcementEmailJob.js @@ -1,6 +1,6 @@ const { CronJob } = require('cron'); const Email = require('../models/email'); -const emailProcessor = require('../services/emailProcessor'); +const emailProcessor = require('../services/announcements/emails/announcementEmailProcessor'); const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); const logger = require('../startup/logger'); @@ -27,8 +27,8 @@ class EmailAnnouncementJobProcessor { await this.processPendingBatches(); }, null, - false, - 'UTC', + true, + EMAIL_JOB_CONFIG.TIMEZONE || 'UTC', ); logger.logInfo( `Email announcement job processor started – cron=${EMAIL_JOB_CONFIG.CRON_INTERVAL}, tz=${EMAIL_JOB_CONFIG.TIMEZONE || 'UTC'}`, diff --git a/src/server.js b/src/server.js index d2f3b447e..571f1b0a5 100644 --- a/src/server.js +++ b/src/server.js @@ -2,7 +2,7 @@ require('dotenv').config(); const http = require('http'); require('./jobs/dailyMessageEmailNotification'); -require('./jobs/emailAnnouncementJobProcessor').start(); // Start email announcement job processor +require('./jobs/announcementEmailJob').start(); // Start email announcement job processor const { app, logger } = require('./app'); const TimerWebsockets = require('./websockets').default; const MessagingWebSocket = require('./websockets/lbMessaging/messagingSocket').default; diff --git a/src/services/emailProcessor.js b/src/services/announcements/emails/announcementEmailProcessor.js similarity index 67% rename from src/services/emailProcessor.js rename to src/services/announcements/emails/announcementEmailProcessor.js index 3f4f7036d..d278546c1 100644 --- a/src/services/emailProcessor.js +++ b/src/services/announcements/emails/announcementEmailProcessor.js @@ -1,17 +1,17 @@ const mongoose = require('mongoose'); -const EmailBatch = require('../models/emailBatch'); +const EmailBatch = require('../../../models/emailBatch'); const EmailService = require('./emailService'); const EmailBatchService = require('./emailBatchService'); -const emailAnnouncementService = require('./emailAnnouncementService'); +const emailAnnouncementService = require('./announcementEmailService'); const EmailBatchAuditService = require('./emailBatchAuditService'); -const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); -const logger = require('../startup/logger'); +const { EMAIL_JOB_CONFIG } = require('../../../config/emailJobConfig'); +const logger = require('../../../startup/logger'); class EmailProcessor { constructor() { this.processingBatches = new Set(); this.maxRetries = EMAIL_JOB_CONFIG.DEFAULT_MAX_RETRIES; - this.retryDelay = 1000; // 1 second + this.retryDelay = EMAIL_JOB_CONFIG.INITIAL_RETRY_DELAY_MS; } /** @@ -238,126 +238,94 @@ class EmailProcessor { return; } - const processWithRetry = async (attempt = 1) => { - try { - // Mark as SENDING using service method - const updatedItem = await EmailBatchService.markEmailBatchSending(item._id); - - // Audit logging after successful status update - try { - await EmailBatchAuditService.logEmailBatchSending(item.emailId, item._id, { - attempt: updatedItem?.attempts || attempt, - recipientCount: updatedItem?.recipients?.length || 0, - emailType: updatedItem?.emailType, - recipients: updatedItem?.recipients?.map((r) => r.email) || [], - emailBatchId: updatedItem?._id.toString(), - }); - } catch (auditError) { - logger.logException(auditError, 'Audit failure: EMAIL_BATCH_SENDING'); - } - - // Send email directly via service (no retries here - handled by processWithRetry) - // Use the emailType stored in the EmailBatch item - const mailOptions = { - from: process.env.REACT_APP_EMAIL, - subject: email.subject, - html: email.htmlContent, - }; - - // Set recipients based on emailType - if (item.emailType === EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC) { - // For BCC, sender goes in 'to' field, all recipients in 'bcc' - mailOptions.to = process.env.REACT_APP_EMAIL; - mailOptions.bcc = recipientEmails.join(','); - } else { - // For TO/CC, recipients go in respective fields - mailOptions.to = recipientEmails.join(','); - } + // Mark as SENDING using service method (single state transition before consolidated retries) + const updatedItem = await EmailBatchService.markEmailBatchSending(item._id); - const sendResult = await emailAnnouncementService.sendEmail(mailOptions); - - // Handle result: { success, response?, error? } - if (sendResult.success) { - await EmailBatchService.markEmailBatchSent(item._id); - try { - await EmailBatchAuditService.logEmailBatchSent( - email._id, - item._id, - { - recipientCount: recipientEmails.length, - emailType: item.emailType, - attempt: updatedItem?.attempts || attempt, - }, - sendResult.response, - ); - } catch (auditError) { - logger.logException(auditError, 'Audit failure: EMAIL_BATCH_SENT'); - } - logger.logInfo( - `EmailBatch item ${item._id} sent successfully to ${recipientEmails.length} recipients (attempt ${updatedItem?.attempts || attempt})`, - ); - } else { - // Consider as failure for this attempt - throw sendResult.error || new Error('Failed to send email'); - } - } catch (error) { - logger.logException( - error, - `Failed to send EmailBatch item ${item._id} to ${recipientEmails.length} recipients (attempt ${attempt})`, - ); + // Audit logging after successful status update + try { + await EmailBatchAuditService.logEmailBatchSending(item.emailId, item._id, { + attempt: updatedItem?.attempts || 1, + recipientCount: updatedItem?.recipients?.length || 0, + emailType: updatedItem?.emailType, + recipients: updatedItem?.recipients?.map((r) => r.email) || [], + emailBatchId: updatedItem?._id.toString(), + }); + } catch (auditError) { + logger.logException(auditError, 'Audit failure: EMAIL_BATCH_SENDING'); + } - if (attempt >= this.maxRetries) { - // Mark as FAILED using service method - const failedItem = await EmailBatchService.markEmailBatchFailed(item._id, { - errorCode: error.code || 'SEND_FAILED', - errorMessage: error.message || 'Failed to send email', - }); + // Build mail options + const mailOptions = { + from: process.env.REACT_APP_EMAIL, + subject: email.subject, + html: email.htmlContent, + }; - // Audit logging - try { - await EmailBatchAuditService.logEmailBatchFailed( - item.emailId, - item._id, - { message: failedItem.lastError, code: failedItem.errorCode }, - { - recipientCount: failedItem?.recipients?.length || 0, - emailType: failedItem?.emailType, - recipients: failedItem?.recipients?.map((r) => r.email) || [], - emailBatchId: failedItem?._id.toString(), - }, - ); - } catch (auditError) { - logger.logException(auditError, 'Audit failure: EMAIL_BATCH_FAILED'); - } + if (item.emailType === EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC) { + mailOptions.to = process.env.REACT_APP_EMAIL; + mailOptions.bcc = recipientEmails.join(','); + } else { + mailOptions.to = recipientEmails.join(','); + } - logger.logInfo( - `Permanently failed to send EmailBatch item ${item._id} to ${recipientEmails.length} recipients after ${this.maxRetries} attempts`, - ); - // Throw error so Promise.allSettled can distinguish failed from successful items - // This ensures accurate reporting in the summary log - throw error; - } + // Delegate retry/backoff to the announcement service + const sendResult = await emailAnnouncementService.sendWithRetry( + mailOptions, + this.maxRetries, + this.retryDelay, + ); - // Log transient failure for this attempt (best-effort, not changing DB status) - try { - await EmailBatchAuditService.logEmailBatchFailed(email._id, item._id, error, { + if (sendResult.success) { + await EmailBatchService.markEmailBatchSent(item._id); + try { + await EmailBatchAuditService.logEmailBatchSent( + email._id, + item._id, + { recipientCount: recipientEmails.length, emailType: item.emailType, - attempt, - transient: true, - }); - } catch (auditError) { - logger.logException(auditError, 'Audit failure: EMAIL_BATCH_FAILED (transient)'); - } - - // Wait before retry with exponential backoff (2^n: 1x, 2x, 4x, 8x, ...) - const delay = this.retryDelay * 2 ** (attempt - 1); - await EmailProcessor.sleep(delay); - return processWithRetry(attempt + 1); + attempt: sendResult.attemptCount || updatedItem?.attempts || 1, + }, + sendResult.response, + ); + } catch (auditError) { + logger.logException(auditError, 'Audit failure: EMAIL_BATCH_SENT'); } - }; + logger.logInfo( + `EmailBatch item ${item._id} sent successfully to ${recipientEmails.length} recipients (attempts ${sendResult.attemptCount || updatedItem?.attempts || 1})`, + ); + return; + } + + // Final failure after retries + const finalError = sendResult.error || new Error('Failed to send email'); + const failedItem = await EmailBatchService.markEmailBatchFailed(item._id, { + errorCode: finalError.code || 'SEND_FAILED', + errorMessage: finalError.message || 'Failed to send email', + }); + + try { + await EmailBatchAuditService.logEmailBatchFailed( + item.emailId, + item._id, + { message: failedItem.lastError, code: failedItem.errorCode }, + { + recipientCount: failedItem?.recipients?.length || 0, + emailType: failedItem?.emailType, + recipients: failedItem?.recipients?.map((r) => r.email) || [], + emailBatchId: failedItem?._id.toString(), + attempts: sendResult.attemptCount || this.maxRetries, + }, + ); + } catch (auditError) { + logger.logException(auditError, 'Audit failure: EMAIL_BATCH_FAILED'); + } - return processWithRetry(1); + logger.logInfo( + `Permanently failed to send EmailBatch item ${item._id} to ${recipientEmails.length} recipients after ${sendResult.attemptCount || this.maxRetries} attempts`, + ); + // Throw to ensure Promise.allSettled records this item as failed + throw finalError; } /** diff --git a/src/services/emailAnnouncementService.js b/src/services/announcements/emails/announcementEmailService.js similarity index 89% rename from src/services/emailAnnouncementService.js rename to src/services/announcements/emails/announcementEmailService.js index c268aa2ad..6b136d654 100644 --- a/src/services/emailAnnouncementService.js +++ b/src/services/announcements/emails/announcementEmailService.js @@ -6,7 +6,7 @@ const nodemailer = require('nodemailer'); const { google } = require('googleapis'); -const logger = require('../startup/logger'); +const logger = require('../../../startup/logger'); class EmailAnnouncementService { constructor() { @@ -136,15 +136,15 @@ class EmailAnnouncementService { /** * Send email with retry logic and announcement-specific handling - * @param {Object} batch - Mail options batch + * @param {Object} mailOptions - Nodemailer-compatible mail options * @param {number} retries - Number of retry attempts - * @param {number} baseDelay - Base delay in milliseconds for exponential backoff + * @param {number} initialDelayMs - Initial delay in milliseconds for exponential backoff * @returns {Promise} { success: boolean, response?: Object, error?: Error, attemptCount: number } */ - async sendWithRetry(batch, retries = 3, baseDelay = 1000) { + async sendWithRetry(mailOptions, retries = 3, initialDelayMs = 1000) { // Validation - if (!batch) { - const error = new Error('INVALID_BATCH: batch is required'); + if (!mailOptions) { + const error = new Error('INVALID_MAIL_OPTIONS: mailOptions is required'); logger.logException(error, 'EmailAnnouncementService.sendWithRetry validation failed'); return { success: false, error, attemptCount: 0 }; } @@ -162,13 +162,13 @@ class EmailAnnouncementService { attemptCount += 1; try { - const result = await this.sendEmail(batch); + const result = await this.sendEmail(mailOptions); if (result.success) { // Store Gmail response for audit logging - batch.gmailResponse = result.response; + mailOptions.gmailResponse = result.response; logger.logInfo( - `Email sent successfully on attempt ${attempt} to: ${batch.to || batch.bcc || 'unknown'}`, + `Email sent successfully on attempt ${attempt} to: ${mailOptions.to || mailOptions.bcc || 'unknown'}`, ); return { success: true, response: result.response, attemptCount }; } @@ -176,7 +176,7 @@ class EmailAnnouncementService { const error = result.error || new Error('Unknown error from sendEmail'); logger.logException( error, - `Announcement batch attempt ${attempt} failed to: ${batch.to || batch.bcc || '(empty)'}`, + `Announcement send attempt ${attempt} failed to: ${mailOptions.to || mailOptions.bcc || '(empty)'}`, ); // If this is the last attempt, return failure info @@ -187,7 +187,7 @@ class EmailAnnouncementService { // Unexpected error (shouldn't happen since sendEmail now returns {success, error}) logger.logException( err, - `Unexpected error in announcement batch attempt ${attempt} to: ${batch.to || batch.bcc || '(empty)'}`, + `Unexpected error in announcement send attempt ${attempt} to: ${mailOptions.to || mailOptions.bcc || '(empty)'}`, ); // If this is the last attempt, return failure info @@ -198,7 +198,7 @@ class EmailAnnouncementService { // Exponential backoff before retry (2^n: 1x, 2x, 4x, 8x, ...) if (attempt < retries) { - const delay = baseDelay * 2 ** (attempt - 1); + const delay = initialDelayMs * 2 ** (attempt - 1); await EmailAnnouncementService.sleep(delay); } } diff --git a/src/services/emailBatchAuditService.js b/src/services/announcements/emails/emailBatchAuditService.js similarity index 97% rename from src/services/emailBatchAuditService.js rename to src/services/announcements/emails/emailBatchAuditService.js index fc0c526a7..a886d42a3 100644 --- a/src/services/emailBatchAuditService.js +++ b/src/services/announcements/emails/emailBatchAuditService.js @@ -4,9 +4,9 @@ */ const mongoose = require('mongoose'); -const EmailBatchAudit = require('../models/emailBatchAudit'); -const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); -const logger = require('../startup/logger'); +const EmailBatchAudit = require('../../../models/emailBatchAudit'); +const { EMAIL_JOB_CONFIG } = require('../../../config/emailJobConfig'); +const logger = require('../../../startup/logger'); class EmailBatchAuditService { /** diff --git a/src/services/emailBatchService.js b/src/services/announcements/emails/emailBatchService.js similarity index 95% rename from src/services/emailBatchService.js rename to src/services/announcements/emails/emailBatchService.js index 2c1c1cf9b..986a2a05e 100644 --- a/src/services/emailBatchService.js +++ b/src/services/announcements/emails/emailBatchService.js @@ -4,12 +4,12 @@ */ const mongoose = require('mongoose'); -const Email = require('../models/email'); -const EmailBatch = require('../models/emailBatch'); +const Email = require('../../../models/email'); +const EmailBatch = require('../../../models/emailBatch'); const EmailService = require('./emailService'); -const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); -const { normalizeRecipientsToObjects } = require('../utilities/emailValidators'); -const logger = require('../startup/logger'); +const { EMAIL_JOB_CONFIG } = require('../../../config/emailJobConfig'); +const { normalizeRecipientsToObjects } = require('../../../utilities/emailValidators'); +const logger = require('../../../startup/logger'); class EmailBatchService { /** diff --git a/src/services/emailService.js b/src/services/announcements/emails/emailService.js similarity index 95% rename from src/services/emailService.js rename to src/services/announcements/emails/emailService.js index 7ffdce9d9..64248c1b9 100644 --- a/src/services/emailService.js +++ b/src/services/announcements/emails/emailService.js @@ -1,6 +1,6 @@ const mongoose = require('mongoose'); -const Email = require('../models/email'); -const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +const Email = require('../../../models/email'); +const { EMAIL_JOB_CONFIG } = require('../../../config/emailJobConfig'); class EmailService { static async createEmail({ subject, htmlContent, createdBy }, session = null) { From 28cdf56c0d28992392db4330b8b41b2fa36e04d8 Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Thu, 30 Oct 2025 23:42:20 -0400 Subject: [PATCH 07/19] feat(email): enhance documentation and validation in email controllers and models --- src/controllers/emailBatchController.js | 26 ++++++--- src/controllers/emailController.js | 41 +++++++++++++ src/controllers/emailTemplateController.js | 34 ++++++++--- src/models/email.js | 5 ++ src/models/emailBatch.js | 4 ++ src/models/emailBatchAudit.js | 4 ++ src/models/emailTemplate.js | 6 ++ .../emails/announcementEmailProcessor.js | 48 +++++++++++---- .../emails/announcementEmailService.js | 29 +++++++--- .../emails/emailBatchAuditService.js | 58 +++++++++++-------- .../announcements/emails/emailBatchService.js | 45 +++++++++----- .../announcements/emails/emailService.js | 35 ++++++++++- 12 files changed, 262 insertions(+), 73 deletions(-) diff --git a/src/controllers/emailBatchController.js b/src/controllers/emailBatchController.js index 8edb160b7..5db32f8ac 100644 --- a/src/controllers/emailBatchController.js +++ b/src/controllers/emailBatchController.js @@ -10,7 +10,9 @@ const logger = require('../startup/logger'); const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); /** - * Get all Email records (parent) + * Get all announcement Email records (parent documents). + * @param {import('express').Request} req + * @param {import('express').Response} res */ const getEmails = async (req, res) => { try { @@ -44,7 +46,9 @@ const getEmails = async (req, res) => { }; /** - * Get Email details with EmailBatch items + * Get a parent Email and its associated EmailBatch items. + * @param {import('express').Request} req + * @param {import('express').Response} res */ const getEmailDetails = async (req, res) => { try { @@ -94,7 +98,9 @@ const getEmailDetails = async (req, res) => { }; /** - * Get worker status (minimal info for frontend) + * Get worker/cron status for the announcement email processor. + * @param {import('express').Request} req + * @param {import('express').Response} res */ const getWorkerStatus = async (req, res) => { try { @@ -115,8 +121,10 @@ const getWorkerStatus = async (req, res) => { }; /** - * Retry an Email by queuing all its failed EmailBatch items - * Resets failed items to QUEUED status for the cron job to process + * Retry a parent Email by resetting all FAILED EmailBatch items to QUEUED. + * - Queues the parent email; cron picks it up in the next cycle. + * @param {import('express').Request} req + * @param {import('express').Response} res */ const retryEmail = async (req, res) => { try { @@ -248,7 +256,9 @@ const retryEmail = async (req, res) => { }; /** - * Get audit trail for a specific Email + * Get the audit trail for a specific parent Email. + * @param {import('express').Request} req + * @param {import('express').Response} res */ const getEmailAuditTrail = async (req, res) => { try { @@ -307,7 +317,9 @@ const getEmailAuditTrail = async (req, res) => { }; /** - * Get audit trail for a specific EmailBatch item + * Get the audit trail for a specific EmailBatch item. + * @param {import('express').Request} req + * @param {import('express').Response} res */ const getEmailBatchAuditTrail = async (req, res) => { try { diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index 5be1f7a3f..f67e6a606 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -20,6 +20,13 @@ const logger = require('../startup/logger'); const jwtSecret = process.env.JWT_SECRET; +/** + * Create an announcement Email for provided recipients. + * - Validates permissions, subject/html, recipients, and template variables. + * - Creates parent Email and chunked EmailBatch items in a transaction. + * @param {import('express').Request} req + * @param {import('express').Response} res + */ const sendEmail = async (req, res) => { // Requestor is required for permission check and audit trail if (!req?.body?.requestor?.requestorId) { @@ -217,6 +224,12 @@ const sendEmail = async (req, res) => { } }; +/** + * Broadcast an announcement Email to all active HGN users and confirmed subscribers. + * - Validates permissions and content; creates Email and batches in a transaction. + * @param {import('express').Request} req + * @param {import('express').Response} res + */ const sendEmailToSubscribers = async (req, res) => { // Requestor is required for permission check and audit trail if (!req?.body?.requestor?.requestorId) { @@ -397,6 +410,12 @@ const sendEmailToSubscribers = async (req, res) => { } }; +/** + * Resend a previously created Email to a selected audience. + * - Options: 'all' (users+subscribers), 'specific' (list), 'same' (original recipients). + * @param {import('express').Request} req + * @param {import('express').Response} res + */ const resendEmail = async (req, res) => { // Requestor is required for permission check and audit trail if (!req?.body?.requestor?.requestorId) { @@ -604,6 +623,11 @@ const resendEmail = async (req, res) => { } }; +/** + * Update the current user's emailSubscriptions preference. + * @param {import('express').Request} req + * @param {import('express').Response} res + */ const updateEmailSubscriptions = async (req, res) => { try { if (!req?.body?.requestor?.email) { @@ -641,6 +665,12 @@ const updateEmailSubscriptions = async (req, res) => { } }; +/** + * Add a non-HGN user's email to the subscription list and send confirmation. + * - Rejects if already an HGN user or already subscribed. + * @param {import('express').Request} req + * @param {import('express').Response} res + */ const addNonHgnEmailSubscription = async (req, res) => { try { const { email } = req.body; @@ -730,6 +760,12 @@ const addNonHgnEmailSubscription = async (req, res) => { } }; +/** + * Confirm a non-HGN email subscription using a signed token. + * - Creates or updates the subscriber record as confirmed. + * @param {import('express').Request} req + * @param {import('express').Response} res + */ const confirmNonHgnEmailSubscription = async (req, res) => { try { const { token } = req.body; @@ -791,6 +827,11 @@ const confirmNonHgnEmailSubscription = async (req, res) => { } }; +/** + * Remove a non-HGN email from the subscription list (unsubscribe). + * @param {import('express').Request} req + * @param {import('express').Response} res + */ const removeNonHgnEmailSubscription = async (req, res) => { try { const { email } = req.body; diff --git a/src/controllers/emailTemplateController.js b/src/controllers/emailTemplateController.js index 45a859078..c6e0495c2 100644 --- a/src/controllers/emailTemplateController.js +++ b/src/controllers/emailTemplateController.js @@ -6,7 +6,10 @@ const logger = require('../startup/logger'); const { ensureHtmlWithinLimit, validateHtmlMedia } = require('../utilities/emailValidators'); /** - * Validate template variables + * Validate template variables. + * - Ensures non-empty unique names and validates allowed types. + * @param {Array<{name: string, type?: 'text'|'url'|'number'|'textarea'|'image'>} | undefined} variables + * @returns {{isValid: boolean, errors?: string[]}} */ function validateTemplateVariables(variables) { if (!variables || !Array.isArray(variables)) { @@ -46,7 +49,12 @@ function validateTemplateVariables(variables) { } /** - * Validate template content (HTML and subject) against defined variables + * Validate template content (HTML and subject) against defined variables. + * - Flags undefined placeholders and unused defined variables. + * @param {Array<{name: string}>} templateVariables + * @param {string} htmlContent + * @param {string} subject + * @returns {{isValid: boolean, errors: string[]}} */ function validateTemplateVariableUsage(templateVariables, htmlContent, subject) { const errors = []; @@ -109,7 +117,9 @@ function validateTemplateVariableUsage(templateVariables, htmlContent, subject) } /** - * Get all email templates with pagination and optimization + * Get all email templates (with basic search/sort and optional content projection). + * @param {import('express').Request} req + * @param {import('express').Response} res */ const getAllEmailTemplates = async (req, res) => { try { @@ -175,7 +185,9 @@ const getAllEmailTemplates = async (req, res) => { }; /** - * Get a single email template by ID + * Get a single email template by ID. + * @param {import('express').Request} req + * @param {import('express').Response} res */ const getEmailTemplateById = async (req, res) => { try { @@ -232,7 +244,10 @@ const getEmailTemplateById = async (req, res) => { }; /** - * Create a new email template + * Create a new email template. + * - Validates content size, media, name/subject length, variables and usage. + * @param {import('express').Request} req + * @param {import('express').Response} res */ const createEmailTemplate = async (req, res) => { try { @@ -385,7 +400,10 @@ const createEmailTemplate = async (req, res) => { }; /** - * Update an email template + * Update an existing email template by ID. + * - Validates content and ensures unique name when changed. + * @param {import('express').Request} req + * @param {import('express').Response} res */ const updateEmailTemplate = async (req, res) => { try { @@ -566,7 +584,9 @@ const updateEmailTemplate = async (req, res) => { }; /** - * Delete an email template + * Delete an email template by ID. + * @param {import('express').Request} req + * @param {import('express').Response} res */ const deleteEmailTemplate = async (req, res) => { try { diff --git a/src/models/email.js b/src/models/email.js index ce6261f72..bcfe73091 100644 --- a/src/models/email.js +++ b/src/models/email.js @@ -1,6 +1,11 @@ const mongoose = require('mongoose'); const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +/** + * Email (parent) model for announcement sending lifecycle. + * - Stores subject/html and status transitions (QUEUED → SENDING → SENT/PROCESSED/FAILED). + * - References creator and tracks timing fields for auditing. + */ const { Schema } = mongoose; const EmailSchema = new Schema({ diff --git a/src/models/emailBatch.js b/src/models/emailBatch.js index a82318801..e9d3cb009 100644 --- a/src/models/emailBatch.js +++ b/src/models/emailBatch.js @@ -1,6 +1,10 @@ const mongoose = require('mongoose'); const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +/** + * EmailBatch (child) model representing one SMTP send to a group of recipients. + * - Tracks recipients, emailType, status, attempt counters and error snapshots. + */ const { Schema } = mongoose; const EmailBatchSchema = new Schema({ diff --git a/src/models/emailBatchAudit.js b/src/models/emailBatchAudit.js index dcb27bb0b..9ee57592b 100644 --- a/src/models/emailBatchAudit.js +++ b/src/models/emailBatchAudit.js @@ -1,6 +1,10 @@ const mongoose = require('mongoose'); const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +/** + * EmailBatchAudit model for immutable audit trail of email/batch actions. + * - Captures action, details, optional error info, metadata, and actor. + */ const { Schema } = mongoose; const EmailBatchAuditSchema = new Schema({ diff --git a/src/models/emailTemplate.js b/src/models/emailTemplate.js index 2fef00154..68da27d28 100644 --- a/src/models/emailTemplate.js +++ b/src/models/emailTemplate.js @@ -1,3 +1,9 @@ +/** + * EmailTemplate model for reusable announcement email content. + * - Stores template name, subject, HTML content, and declared variables + * - Tracks creator/updater and timestamps for auditing and sorting + * - Includes helpful indexes and text search for fast lookup + */ const mongoose = require('mongoose'); const emailTemplateSchema = new mongoose.Schema( diff --git a/src/services/announcements/emails/announcementEmailProcessor.js b/src/services/announcements/emails/announcementEmailProcessor.js index d278546c1..684f3da32 100644 --- a/src/services/announcements/emails/announcementEmailProcessor.js +++ b/src/services/announcements/emails/announcementEmailProcessor.js @@ -8,6 +8,11 @@ const { EMAIL_JOB_CONFIG } = require('../../../config/emailJobConfig'); const logger = require('../../../startup/logger'); class EmailProcessor { + /** + * Initialize processor runtime configuration. + * - Tracks currently processing parent Email IDs to avoid duplicate work. + * - Loads retry settings from EMAIL_JOB_CONFIG to coordinate with sending service. + */ constructor() { this.processingBatches = new Set(); this.maxRetries = EMAIL_JOB_CONFIG.DEFAULT_MAX_RETRIES; @@ -15,8 +20,13 @@ class EmailProcessor { } /** - * Process an Email (processes all its EmailBatch items) - * @param {string|ObjectId} emailId - The _id (ObjectId) of the parent Email + * Process a single parent Email by sending all of its queued EmailBatch items. + * - Idempotent with respect to concurrent calls (skips if already processing). + * - Recovers stuck batches from previous crashes by resetting SENDING to QUEUED. + * - Audits lifecycle events (sending, sent, failed, processed). + * @param {string|ObjectId} emailId - The ObjectId of the parent Email. + * @returns {Promise} Final email status: SENT | FAILED | PROCESSED | SENDING + * @throws {Error} When emailId is invalid or the Email is not found. */ async processEmail(emailId) { if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { @@ -107,9 +117,11 @@ class EmailProcessor { } /** - * Process all EmailBatch items for an Email - * Processes ALL QUEUED items regardless of individual failures - ensures maximum delivery - * @param {Object} email - The Email document + * Process all EmailBatch child records for a given parent Email. + * - Sends batches with limited concurrency and Promise.allSettled to ensure all are attempted. + * - Resets orphaned SENDING batches to QUEUED before processing. + * @param {Object} email - The parent Email mongoose document. + * @returns {Promise} */ async processEmailBatches(email) { // Get ALL batches for this email first @@ -193,9 +205,13 @@ class EmailProcessor { } /** - * Process a single EmailBatch item with multiple recipients - * @param {Object} item - The EmailBatch item - * @param {Object} email - The parent Email document + * Send one EmailBatch item (one SMTP send for a group of recipients). + * - Marks the batch SENDING, audits, and delegates retries to the announcement service. + * - On success, marks SENT and audits delivery details; on failure after retries, marks FAILED and audits. + * @param {Object} item - The EmailBatch mongoose document. + * @param {Object} email - The parent Email mongoose document. + * @returns {Promise} + * @throws {Error} Bubbles final failure so callers can classify in allSettled results. */ async processEmailBatch(item, email) { if (!item || !item._id) { @@ -329,7 +345,9 @@ class EmailProcessor { } /** - * Sleep utility + * Sleep utility to await a given duration. + * @param {number} ms - Milliseconds to wait. + * @returns {Promise} */ static sleep(ms) { return new Promise((resolve) => { @@ -338,7 +356,14 @@ class EmailProcessor { } /** - * Determine final status for an Email based on its EmailBatch items + * Determine the final parent Email status from child EmailBatch aggregation. + * Rules: + * - All SENT => SENT + * - All FAILED => FAILED + * - Mixed (any SENT or FAILED) => PROCESSED + * - Otherwise => SENDING (still in progress) + * @param {ObjectId} emailObjectId - Parent Email ObjectId. + * @returns {Promise} Derived status constant from EMAIL_JOB_CONFIG.EMAIL_STATUSES. */ static async determineEmailStatus(emailObjectId) { const counts = await EmailBatch.aggregate([ @@ -381,7 +406,8 @@ class EmailProcessor { } /** - * Get processor status + * Get lightweight processor status for diagnostics/telemetry. + * @returns {{isRunning: boolean, processingBatches: string[], maxRetries: number}} */ getStatus() { return { diff --git a/src/services/announcements/emails/announcementEmailService.js b/src/services/announcements/emails/announcementEmailService.js index 6b136d654..fb9c2b594 100644 --- a/src/services/announcements/emails/announcementEmailService.js +++ b/src/services/announcements/emails/announcementEmailService.js @@ -9,6 +9,10 @@ const { google } = require('googleapis'); const logger = require('../../../startup/logger'); class EmailAnnouncementService { + /** + * Initialize Gmail OAuth2 transport configuration and validate required env vars. + * Throws during construction if configuration is incomplete to fail fast. + */ constructor() { this.config = { email: process.env.ANNOUNCEMENT_EMAIL, @@ -50,9 +54,12 @@ class EmailAnnouncementService { } /** - * Send email with enhanced announcement tracking - * Validates input and configuration before sending - * @returns {Object} { success: boolean, response?: Object, error?: Error } + * Send email with enhanced announcement tracking. + * - Validates recipients, subject, and service configuration. + * - Fetches OAuth2 access token and attaches OAuth credentials to the request. + * - Returns a structured result instead of throwing to simplify callers. + * @param {Object} mailOptions - Nodemailer-compatible options (to|bcc, subject, html, from?). + * @returns {Promise<{success: boolean, response?: Object, error?: Error}>} */ async sendEmail(mailOptions) { // Validation @@ -135,11 +142,13 @@ class EmailAnnouncementService { } /** - * Send email with retry logic and announcement-specific handling - * @param {Object} mailOptions - Nodemailer-compatible mail options - * @param {number} retries - Number of retry attempts - * @param {number} initialDelayMs - Initial delay in milliseconds for exponential backoff - * @returns {Promise} { success: boolean, response?: Object, error?: Error, attemptCount: number } + * Send email with retry logic and announcement-specific handling. + * - Executes exponential backoff between attempts: initialDelayMs * 2^(attempt-1). + * - Never throws; returns final success/failure and attemptCount for auditing. + * @param {Object} mailOptions - Nodemailer-compatible mail options. + * @param {number} retries - Total attempts (>=1). + * @param {number} initialDelayMs - Initial backoff delay in ms. + * @returns {Promise<{success: boolean, response?: Object, error?: Error, attemptCount: number}>} */ async sendWithRetry(mailOptions, retries = 3, initialDelayMs = 1000) { // Validation @@ -212,7 +221,9 @@ class EmailAnnouncementService { } /** - * Sleep utility + * Sleep utility for backoff timing. + * @param {number} ms - Milliseconds to wait. + * @returns {Promise} */ static sleep(ms) { return new Promise((resolve) => { diff --git a/src/services/announcements/emails/emailBatchAuditService.js b/src/services/announcements/emails/emailBatchAuditService.js index a886d42a3..070fdfcef 100644 --- a/src/services/announcements/emails/emailBatchAuditService.js +++ b/src/services/announcements/emails/emailBatchAuditService.js @@ -10,7 +10,16 @@ const logger = require('../../../startup/logger'); class EmailBatchAuditService { /** - * Log an action to the audit trail + * Log an action to the email batch audit trail. + * - Validates IDs and action enum, normalizes message fields, then persists. + * @param {string|ObjectId} emailId + * @param {string} action - One of EMAIL_BATCH_AUDIT_ACTIONS.* + * @param {string} details - Human-readable details (trimmed/limited). + * @param {Object} [metadata] + * @param {Error|null} [error] + * @param {string|ObjectId|null} [triggeredBy] + * @param {string|ObjectId|null} [emailBatchId] + * @returns {Promise} Created audit document. */ static async logAction( emailId, @@ -79,8 +88,9 @@ class EmailBatchAuditService { } /** - * Get complete audit trail for an Email (parent) - no pagination, no filtering - * @param {string|ObjectId} emailId - The _id (ObjectId) of the parent Email + * Get complete audit trail for a parent Email (no pagination). + * @param {string|ObjectId} emailId - Parent Email ObjectId. + * @returns {Promise} Sorted newest first, with basic populations. */ static async getEmailAuditTrail(emailId) { // Validate emailId is ObjectId @@ -100,7 +110,9 @@ class EmailBatchAuditService { } /** - * Get audit trail for a specific EmailBatch item - no pagination, no filtering + * Get audit trail for a specific EmailBatch item (no pagination). + * @param {string|ObjectId} emailBatchId - EmailBatch ObjectId. + * @returns {Promise} Sorted newest first, with basic populations. */ static async getEmailBatchAuditTrail(emailBatchId) { // Validate emailBatchId is ObjectId @@ -120,10 +132,10 @@ class EmailBatchAuditService { } /** - * Log Email queued (for initial creation or retry) - * @param {string|ObjectId} emailId - The email ID - * @param {Object} metadata - Additional metadata - * @param {string|ObjectId} triggeredBy - Optional user ID who triggered this action + * Log Email queued (initial creation or retry). + * @param {string|ObjectId} emailId + * @param {Object} [metadata] + * @param {string|ObjectId} [triggeredBy] */ static async logEmailQueued(emailId, metadata = {}, triggeredBy = null) { return this.logAction( @@ -137,7 +149,7 @@ class EmailBatchAuditService { } /** - * Log Email sending (processing start) + * Log Email sending (processing start). */ static async logEmailSending(emailId, metadata = {}) { return this.logAction( @@ -149,7 +161,7 @@ class EmailBatchAuditService { } /** - * Log Email processed (processing completion) + * Log Email processed (processing completion). */ static async logEmailProcessed(emailId, metadata = {}) { return this.logAction( @@ -161,7 +173,7 @@ class EmailBatchAuditService { } /** - * Log Email processing failure + * Log Email processing failure. */ static async logEmailFailed(emailId, error, metadata = {}) { return this.logAction( @@ -174,7 +186,7 @@ class EmailBatchAuditService { } /** - * Log Email sent (all batches completed successfully) + * Log Email sent (all batches completed successfully). */ static async logEmailSent(emailId, metadata = {}) { return this.logAction( @@ -186,7 +198,7 @@ class EmailBatchAuditService { } /** - * Log EmailBatch item sent with essential delivery tracking + * Log EmailBatch item sent with essential delivery tracking details. */ static async logEmailBatchSent(emailId, emailBatchId, metadata = {}, gmailResponse = null) { const enhancedMetadata = { @@ -219,7 +231,7 @@ class EmailBatchAuditService { } /** - * Log EmailBatch item failure with optional Gmail API metadata + * Log EmailBatch item failure with optional Gmail API metadata. */ static async logEmailBatchFailed(emailId, emailBatchId, error, metadata = {}) { const enhancedMetadata = { @@ -256,11 +268,11 @@ class EmailBatchAuditService { } /** - * Log EmailBatch item queued - * @param {string|ObjectId} emailId - The email ID - * @param {string|ObjectId} emailBatchId - The EmailBatch item ID - * @param {Object} metadata - Additional metadata - * @param {string|ObjectId} triggeredBy - Optional user ID who triggered this action + * Log EmailBatch item queued. + * @param {string|ObjectId} emailId + * @param {string|ObjectId} emailBatchId + * @param {Object} [metadata] + * @param {string|ObjectId} [triggeredBy] */ static async logEmailBatchQueued(emailId, emailBatchId, metadata = {}, triggeredBy = null) { return this.logAction( @@ -275,10 +287,10 @@ class EmailBatchAuditService { } /** - * Log EmailBatch item sending - * @param {string|ObjectId} emailId - The email ID - * @param {string|ObjectId} emailBatchId - The EmailBatch item ID - * @param {Object} metadata - Additional metadata + * Log EmailBatch item sending. + * @param {string|ObjectId} emailId + * @param {string|ObjectId} emailBatchId + * @param {Object} [metadata] */ static async logEmailBatchSending(emailId, emailBatchId, metadata = {}) { return this.logAction( diff --git a/src/services/announcements/emails/emailBatchService.js b/src/services/announcements/emails/emailBatchService.js index 986a2a05e..d69b05350 100644 --- a/src/services/announcements/emails/emailBatchService.js +++ b/src/services/announcements/emails/emailBatchService.js @@ -13,13 +13,14 @@ const logger = require('../../../startup/logger'); class EmailBatchService { /** - * Create EmailBatch items for an Email - * Takes all recipients, chunks them into EmailBatch items with configurable batch size - * @param {string|ObjectId} emailId - The _id (ObjectId) of the parent Email - * @param {Array} recipients - Array of recipient objects with email property - * @param {Object} config - Configuration { batchSize?, emailType? } - * @param {Object} session - MongoDB session for transaction support - * @returns {Promise} Created EmailBatch items + * Create EmailBatch items for a parent Email. + * - Validates parent Email ID, normalizes recipients and chunks by configured size. + * - Returns inserted EmailBatch documents. + * @param {string|ObjectId} emailId - Parent Email ObjectId. + * @param {Array<{email: string}|string>} recipients - Recipients (auto-normalized). + * @param {{batchSize?: number, emailType?: 'TO'|'CC'|'BCC'}} config - Optional overrides. + * @param {import('mongoose').ClientSession|null} session - Optional transaction session. + * @returns {Promise} Created EmailBatch items. */ static async createEmailBatches(emailId, recipients, config = {}, session = null) { try { @@ -68,7 +69,9 @@ class EmailBatchService { } /** - * Get Email with its EmailBatch items and dynamic counts + * Get Email with its EmailBatch items and essential metadata for UI. + * @param {string|ObjectId} emailId - Parent Email ObjectId. + * @returns {Promise<{email: Object, batches: Array}>} */ static async getEmailWithBatches(emailId) { try { @@ -111,7 +114,8 @@ class EmailBatchService { } /** - * Get all Emails + * Get all Emails ordered by creation date descending. + * @returns {Promise} Array of Email objects (lean, with createdBy populated). */ static async getAllEmails() { try { @@ -128,21 +132,25 @@ class EmailBatchService { } /** - * Fetch EmailBatch items for a parent emailId (ObjectId) + * Fetch EmailBatch items for a parent Email. + * @param {string|ObjectId} emailId - Parent Email ObjectId. + * @returns {Promise} Sorted ascending by createdAt. */ static async getBatchesForEmail(emailId) { return EmailBatch.find({ emailId }).sort({ createdAt: 1 }); } /** - * Get EmailBatch items by emailId (alias for consistency) + * Alias of getBatchesForEmail for naming consistency. */ static async getEmailBatchesByEmailId(emailId) { return this.getBatchesForEmail(emailId); } /** - * Reset an EmailBatch item for retry + * Reset an EmailBatch item for retry, clearing attempts and error fields. + * @param {string|ObjectId} emailBatchId - Batch ObjectId. + * @returns {Promise} Updated document or null if not found. */ static async resetEmailBatchForRetry(emailBatchId) { const item = await EmailBatch.findById(emailBatchId); @@ -159,7 +167,9 @@ class EmailBatchService { } /** - * Mark a batch item as SENDING (and bump attempts/lastAttemptedAt) + * Mark a batch item as SENDING, increment attempts, and set lastAttemptedAt. + * @param {string|ObjectId} emailBatchId - Batch ObjectId. + * @returns {Promise} Updated batch document. */ static async markEmailBatchSending(emailBatchId) { const now = new Date(); @@ -176,7 +186,9 @@ class EmailBatchService { } /** - * Mark a batch item as SENT + * Mark a batch item as SENT and set sentAt timestamp. + * @param {string|ObjectId} emailBatchId - Batch ObjectId. + * @returns {Promise} Updated batch document. */ static async markEmailBatchSent(emailBatchId) { const now = new Date(); @@ -192,7 +204,10 @@ class EmailBatchService { } /** - * Mark a batch item as FAILED and record error info + * Mark a batch item as FAILED and snapshot the error info. + * @param {string|ObjectId} emailBatchId - Batch ObjectId. + * @param {{errorCode?: string, errorMessage?: string}} param1 - Error details. + * @returns {Promise} Updated batch document. */ static async markEmailBatchFailed(emailBatchId, { errorCode, errorMessage }) { const now = new Date(); diff --git a/src/services/announcements/emails/emailService.js b/src/services/announcements/emails/emailService.js index 64248c1b9..bcea2c78a 100644 --- a/src/services/announcements/emails/emailService.js +++ b/src/services/announcements/emails/emailService.js @@ -3,6 +3,13 @@ const Email = require('../../../models/email'); const { EMAIL_JOB_CONFIG } = require('../../../config/emailJobConfig'); class EmailService { + /** + * Create a parent Email document for announcements. + * Trims large text fields and supports optional transaction sessions. + * @param {{subject: string, htmlContent: string, createdBy: string|ObjectId}} param0 + * @param {import('mongoose').ClientSession|null} session + * @returns {Promise} Created Email document. + */ static async createEmail({ subject, htmlContent, createdBy }, session = null) { const normalizedSubject = typeof subject === 'string' ? subject.trim() : subject; const normalizedHtml = typeof htmlContent === 'string' ? htmlContent.trim() : htmlContent; @@ -19,11 +26,23 @@ class EmailService { return email; } + /** + * Fetch a parent Email by ObjectId. + * @param {string|ObjectId} id + * @param {import('mongoose').ClientSession|null} session + * @returns {Promise} + */ static async getEmailById(id, session = null) { if (!id || !mongoose.Types.ObjectId.isValid(id)) return null; return Email.findById(id).session(session); } + /** + * Update Email status with validation against configured enum. + * @param {string|ObjectId} emailId + * @param {string} status - One of EMAIL_JOB_CONFIG.EMAIL_STATUSES.* + * @returns {Promise} Updated Email document. + */ static async updateEmailStatus(emailId, status) { if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { throw new Error('Valid email ID is required'); @@ -39,6 +58,11 @@ class EmailService { return email; } + /** + * Mark Email as SENDING and set startedAt. + * @param {string|ObjectId} emailId + * @returns {Promise} Updated Email document. + */ static async markEmailStarted(emailId) { const now = new Date(); const email = await Email.findByIdAndUpdate( @@ -53,6 +77,13 @@ class EmailService { return email; } + /** + * Mark Email as completed with final status, setting completedAt. + * Falls back to SENT if an invalid finalStatus is passed. + * @param {string|ObjectId} emailId + * @param {string} finalStatus + * @returns {Promise} Updated Email document. + */ static async markEmailCompleted(emailId, finalStatus) { const now = new Date(); const statusToSet = Object.values(EMAIL_JOB_CONFIG.EMAIL_STATUSES).includes(finalStatus) @@ -72,7 +103,9 @@ class EmailService { } /** - * Mark an Email as QUEUED for retry (e.g., after resetting failed EmailBatch items) + * Mark an Email as QUEUED for retry and clear timing fields. + * @param {string|ObjectId} emailId + * @returns {Promise} Updated Email document. */ static async markEmailQueued(emailId) { const now = new Date(); From 6297b202d59d66c53e8f362c7381d5cae9e9f293 Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Fri, 31 Oct 2025 11:18:21 -0400 Subject: [PATCH 08/19] refactor(email): remove media validation from email controllers and templates --- src/controllers/emailController.js | 40 +++++++++++++--------- src/controllers/emailTemplateController.js | 34 +++++++++--------- 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index f67e6a606..8221d9585 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -7,7 +7,6 @@ const { isValidEmailAddress, normalizeRecipientsToArray, ensureHtmlWithinLimit, - validateHtmlMedia, } = require('../utilities/emailValidators'); const EmailSubcriptionList = require('../models/emailSubcriptionList'); const userProfile = require('../models/userProfile'); @@ -72,14 +71,14 @@ const sendEmail = async (req, res) => { } // Validate HTML does not contain base64-encoded media - const mediaValidation = validateHtmlMedia(html); - if (!mediaValidation.isValid) { - return res.status(400).json({ - success: false, - message: 'HTML contains embedded media files. Only URLs are allowed for media.', - errors: mediaValidation.errors, - }); - } + // const mediaValidation = validateHtmlMedia(html); + // if (!mediaValidation.isValid) { + // return res.status(400).json({ + // success: false, + // message: 'HTML contains embedded media files. Only URLs are allowed for media.', + // errors: mediaValidation.errors, + // }); + // } // Validate that all template variables have been replaced const templateVariableRegex = /\{\{(\w+)\}\}/g; @@ -260,14 +259,14 @@ const sendEmailToSubscribers = async (req, res) => { } // Validate HTML does not contain base64-encoded media - const mediaValidation = validateHtmlMedia(html); - if (!mediaValidation.isValid) { - return res.status(400).json({ - success: false, - message: 'HTML contains embedded media files. Only URLs are allowed for media.', - errors: mediaValidation.errors, - }); - } + // const mediaValidation = validateHtmlMedia(html); + // if (!mediaValidation.isValid) { + // return res.status(400).json({ + // success: false, + // message: 'HTML contains embedded media files. Only URLs are allowed for media.', + // errors: mediaValidation.errors, + // }); + // } // Validate that all template variables have been replaced const templateVariableRegex = /\{\{(\w+)\}\}/g; @@ -509,6 +508,13 @@ const resendEmail = async (req, res) => { }); } + if (recipientsArray.length > EMAIL_JOB_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST) { + return res.status(400).json({ + success: false, + message: `A maximum of ${EMAIL_JOB_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST} recipients are allowed per request`, + }); + } + allRecipients = recipientsArray.map((email) => ({ email })); } else if (recipientOption === 'same') { // Get recipients from original email's EmailBatch items diff --git a/src/controllers/emailTemplateController.js b/src/controllers/emailTemplateController.js index c6e0495c2..0afb26241 100644 --- a/src/controllers/emailTemplateController.js +++ b/src/controllers/emailTemplateController.js @@ -3,7 +3,7 @@ const EmailTemplate = require('../models/emailTemplate'); const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); const { hasPermission } = require('../utilities/permissions'); const logger = require('../startup/logger'); -const { ensureHtmlWithinLimit, validateHtmlMedia } = require('../utilities/emailValidators'); +const { ensureHtmlWithinLimit } = require('../utilities/emailValidators'); /** * Validate template variables. @@ -280,14 +280,14 @@ const createEmailTemplate = async (req, res) => { } // Validate HTML does not contain base64-encoded media - const mediaValidation = validateHtmlMedia(htmlContent); - if (!mediaValidation.isValid) { - return res.status(400).json({ - success: false, - message: 'HTML contains embedded media files. Only URLs are allowed for media.', - errors: mediaValidation.errors, - }); - } + // const mediaValidation = validateHtmlMedia(htmlContent); + // if (!mediaValidation.isValid) { + // return res.status(400).json({ + // success: false, + // message: 'HTML contains embedded media files. Only URLs are allowed for media.', + // errors: mediaValidation.errors, + // }); + // } // Validate name length const trimmedName = name.trim(); @@ -444,14 +444,14 @@ const updateEmailTemplate = async (req, res) => { } // Validate HTML does not contain base64-encoded media - const mediaValidation = validateHtmlMedia(htmlContent); - if (!mediaValidation.isValid) { - return res.status(400).json({ - success: false, - message: 'HTML contains embedded media files. Only URLs are allowed for media.', - errors: mediaValidation.errors, - }); - } + // const mediaValidation = validateHtmlMedia(htmlContent); + // if (!mediaValidation.isValid) { + // return res.status(400).json({ + // success: false, + // message: 'HTML contains embedded media files. Only URLs are allowed for media.', + // errors: mediaValidation.errors, + // }); + // } // Validate name and subject length const trimmedName = name.trim(); From 20b2cbe87229dffb4aa83ef88e008159fbc73e42 Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Tue, 11 Nov 2025 00:45:27 -0500 Subject: [PATCH 09/19] refactor(email): remove announcement email job and related configurations - Deleted the announcementEmailJob and associated configurations from emailJobConfig. - Removed emailBatchController and its routes, streamlining email processing logic. - Updated email models and services to reflect changes in email status management. - Refactored emailController to utilize new email processing methods and improved error handling. - Enhanced email validation utilities for better recipient management and HTML content checks. --- .../{emailJobConfig.js => emailConfig.js} | 39 +- src/controllers/emailBatchController.js | 388 ------------ src/controllers/emailController.js | 518 +++++---------- src/controllers/emailOutboxController.js | 177 ++++++ src/controllers/emailTemplateController.js | 593 +++++------------- src/jobs/announcementEmailJob.js | 190 ------ src/models/email.js | 15 +- src/models/emailBatch.js | 18 +- src/models/emailBatchAudit.js | 77 --- src/models/emailTemplate.js | 30 +- src/routes/emailBatchRoutes.js | 17 - src/routes/emailOutboxRouter.js | 16 + src/routes/emailTemplateRouter.js | 2 + src/server.js | 3 - .../emails/announcementEmailProcessor.js | 424 ------------- .../emails/emailBatchAuditService.js | 308 --------- .../announcements/emails/emailBatchService.js | 246 ++++++-- .../announcements/emails/emailProcessor.js | 324 ++++++++++ ...EmailService.js => emailSendingService.js} | 89 ++- .../announcements/emails/emailService.js | 195 +++++- .../emails/emailTemplateService.js | 499 +++++++++++++++ .../emails/templateRenderingService.js | 243 +++++++ src/startup/routes.js | 4 +- src/utilities/emailValidators.js | 49 +- src/utilities/transactionHelper.js | 36 ++ 25 files changed, 2119 insertions(+), 2381 deletions(-) rename src/config/{emailJobConfig.js => emailConfig.js} (51%) delete mode 100644 src/controllers/emailBatchController.js create mode 100644 src/controllers/emailOutboxController.js delete mode 100644 src/jobs/announcementEmailJob.js delete mode 100644 src/models/emailBatchAudit.js delete mode 100644 src/routes/emailBatchRoutes.js create mode 100644 src/routes/emailOutboxRouter.js delete mode 100644 src/services/announcements/emails/announcementEmailProcessor.js delete mode 100644 src/services/announcements/emails/emailBatchAuditService.js create mode 100644 src/services/announcements/emails/emailProcessor.js rename src/services/announcements/emails/{announcementEmailService.js => emailSendingService.js} (74%) create mode 100644 src/services/announcements/emails/emailTemplateService.js create mode 100644 src/services/announcements/emails/templateRenderingService.js create mode 100644 src/utilities/transactionHelper.js diff --git a/src/config/emailJobConfig.js b/src/config/emailConfig.js similarity index 51% rename from src/config/emailJobConfig.js rename to src/config/emailConfig.js index a3f1e2f82..484124c42 100644 --- a/src/config/emailJobConfig.js +++ b/src/config/emailConfig.js @@ -1,21 +1,16 @@ /** - * Email Job Queue Configuration - * Centralized configuration for email announcement job queue system + * Email Configuration + * Centralized configuration for email announcement system */ -const EMAIL_JOB_CONFIG = { - // Processing intervals - CRON_INTERVAL: '0 * * * * *', // Every minute at 0 seconds - TIMEZONE: 'UTC', // Cron timezone; adjust as needed (e.g., 'America/Los_Angeles') - MAX_CONCURRENT_BATCHES: 3, - +const EMAIL_CONFIG = { // Retry configuration DEFAULT_MAX_RETRIES: 3, INITIAL_RETRY_DELAY_MS: 1000, // Status enums EMAIL_STATUSES: { - QUEUED: 'QUEUED', // Waiting in queue + PENDING: 'PENDING', // Created, waiting to be processed SENDING: 'SENDING', // Currently sending SENT: 'SENT', // All emails successfully sent PROCESSED: 'PROCESSED', // Processing finished (mixed results) @@ -23,27 +18,12 @@ const EMAIL_JOB_CONFIG = { }, EMAIL_BATCH_STATUSES: { - QUEUED: 'QUEUED', // Waiting to send + PENDING: 'PENDING', // Created, waiting to be processed SENDING: 'SENDING', // Currently sending SENT: 'SENT', // Successfully delivered FAILED: 'FAILED', // Delivery failed }, - EMAIL_BATCH_AUDIT_ACTIONS: { - // Email-level actions (main batch) - EMAIL_QUEUED: 'EMAIL_QUEUED', - EMAIL_SENDING: 'EMAIL_SENDING', - EMAIL_SENT: 'EMAIL_SENT', - EMAIL_PROCESSED: 'EMAIL_PROCESSED', - EMAIL_FAILED: 'EMAIL_FAILED', - - // Email batch item-level actions - EMAIL_BATCH_QUEUED: 'EMAIL_BATCH_QUEUED', - EMAIL_BATCH_SENDING: 'EMAIL_BATCH_SENDING', - EMAIL_BATCH_SENT: 'EMAIL_BATCH_SENT', - EMAIL_BATCH_FAILED: 'EMAIL_BATCH_FAILED', - }, - EMAIL_TYPES: { TO: 'TO', CC: 'CC', @@ -55,14 +35,19 @@ const EMAIL_JOB_CONFIG = { MAX_RECIPIENTS_PER_REQUEST: 1000, // Must match EmailBatch.recipients validator MAX_HTML_BYTES: 1 * 1024 * 1024, // 1MB - Reduced since base64 media files are blocked SUBJECT_MAX_LENGTH: 200, // Standardized subject length limit + TEMPLATE_NAME_MAX_LENGTH: 50, // Template name maximum length }, // Announcement service runtime knobs ANNOUNCEMENTS: { BATCH_SIZE: 50, // recipients per SMTP send batch CONCURRENCY: 3, // concurrent SMTP batches - RATE_LIMIT_DELAY_MS: 1000, // delay between queue cycles when more remain + }, + + // Email configuration + EMAIL: { + SENDER: process.env.ANNOUNCEMENT_EMAIL, }, }; -module.exports = { EMAIL_JOB_CONFIG }; +module.exports = { EMAIL_CONFIG }; diff --git a/src/controllers/emailBatchController.js b/src/controllers/emailBatchController.js deleted file mode 100644 index 5db32f8ac..000000000 --- a/src/controllers/emailBatchController.js +++ /dev/null @@ -1,388 +0,0 @@ -const mongoose = require('mongoose'); -const EmailBatchService = require('../services/announcements/emails/emailBatchService'); -const EmailService = require('../services/announcements/emails/emailService'); -const EmailBatchAuditService = require('../services/announcements/emails/emailBatchAuditService'); -const emailAnnouncementJobProcessor = require('../jobs/announcementEmailJob'); -const EmailBatch = require('../models/emailBatch'); -const Email = require('../models/email'); -const { hasPermission } = require('../utilities/permissions'); -const logger = require('../startup/logger'); -const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); - -/** - * Get all announcement Email records (parent documents). - * @param {import('express').Request} req - * @param {import('express').Response} res - */ -const getEmails = async (req, res) => { - try { - // Permission check - viewing emails requires sendEmails permission - if (!req?.body?.requestor?.requestorId && !req?.user?.userid) { - return res.status(401).json({ success: false, message: 'Missing requestor' }); - } - - const requestor = req.body.requestor || req.user; - const canViewEmails = await hasPermission(requestor, 'sendEmails'); - if (!canViewEmails) { - return res - .status(403) - .json({ success: false, message: 'You are not authorized to view emails.' }); - } - - const emails = await EmailBatchService.getAllEmails(); - - res.status(200).json({ - success: true, - data: emails, - }); - } catch (error) { - logger.logException(error, 'Error getting emails'); - res.status(500).json({ - success: false, - message: 'Error getting emails', - error: error.message, - }); - } -}; - -/** - * Get a parent Email and its associated EmailBatch items. - * @param {import('express').Request} req - * @param {import('express').Response} res - */ -const getEmailDetails = async (req, res) => { - try { - // Permission check - viewing email details requires sendEmails permission - if (!req?.body?.requestor?.requestorId && !req?.user?.userid) { - return res.status(401).json({ success: false, message: 'Missing requestor' }); - } - - const requestor = req.body.requestor || req.user; - const canViewEmails = await hasPermission(requestor, 'sendEmails'); - if (!canViewEmails) { - return res - .status(403) - .json({ success: false, message: 'You are not authorized to view email details.' }); - } - - const { emailId } = req.params; // emailId is now the ObjectId of parent Email - - if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { - return res.status(400).json({ - success: false, - message: 'Valid Email ID is required', - }); - } - - const result = await EmailBatchService.getEmailWithBatches(emailId); - - if (!result) { - return res.status(404).json({ - success: false, - message: 'Email not found', - }); - } - - res.status(200).json({ - success: true, - data: result, - }); - } catch (error) { - logger.logException(error, 'Error getting Email details with EmailBatch items'); - res.status(500).json({ - success: false, - message: 'Error getting email details', - error: error.message, - }); - } -}; - -/** - * Get worker/cron status for the announcement email processor. - * @param {import('express').Request} req - * @param {import('express').Response} res - */ -const getWorkerStatus = async (req, res) => { - try { - const workerStatus = emailAnnouncementJobProcessor.getWorkerStatus(); - - res.status(200).json({ - success: true, - data: workerStatus, - }); - } catch (error) { - logger.logException(error, 'Error getting worker status'); - res.status(500).json({ - success: false, - message: 'Error getting worker status', - error: error.message, - }); - } -}; - -/** - * Retry a parent Email by resetting all FAILED EmailBatch items to QUEUED. - * - Queues the parent email; cron picks it up in the next cycle. - * @param {import('express').Request} req - * @param {import('express').Response} res - */ -const retryEmail = async (req, res) => { - try { - const { emailId } = req.params; - - // Validate emailId is a valid ObjectId - if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { - return res.status(400).json({ - success: false, - message: 'Invalid Email ID', - }); - } - - // Permission check - retrying emails requires sendEmails permission - if (!req?.body?.requestor?.requestorId && !req?.user?.userid) { - return res.status(401).json({ success: false, message: 'Missing requestor' }); - } - - const requestor = req.body.requestor || req.user; - const canRetryEmail = await hasPermission(requestor, 'sendEmails'); - if (!canRetryEmail) { - return res - .status(403) - .json({ success: false, message: 'You are not authorized to retry emails.' }); - } - - // Get requestor for audit trail - const requestorId = requestor.requestorId || requestor.userid; - - // Find the Email - const email = await Email.findById(emailId); - - if (!email) { - return res.status(404).json({ - success: false, - message: 'Email not found', - }); - } - - // Only allow retry for emails in final states (FAILED or PROCESSED) - const allowedRetryStatuses = [ - EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED, - EMAIL_JOB_CONFIG.EMAIL_STATUSES.PROCESSED, - ]; - - if (!allowedRetryStatuses.includes(email.status)) { - return res.status(400).json({ - success: false, - message: `Email must be in FAILED or PROCESSED status to retry. Current status: ${email.status}`, - }); - } - - // Find all FAILED EmailBatch items - const failedItems = await EmailBatch.find({ - emailId: email._id, - status: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.FAILED, - }); - - if (failedItems.length === 0) { - logger.logInfo(`Email ${emailId} has no failed EmailBatch items to retry`); - return res.status(200).json({ - success: true, - message: 'No failed EmailBatch items to retry', - data: { - emailId: email._id, - failedItemsRetried: 0, - }, - }); - } - - logger.logInfo(`Queuing ${failedItems.length} failed EmailBatch items for retry: ${emailId}`); - - // First, queue the parent Email so cron picks it up - await EmailService.markEmailQueued(emailId); - - // Audit Email queued for retry (with requestor) - try { - await EmailBatchAuditService.logEmailQueued( - email._id, - { reason: 'Manual retry' }, - requestorId, - ); - } catch (auditErr) { - logger.logException(auditErr, 'Audit failure: EMAIL_QUEUED (retry)'); - } - - // Reset each failed item to QUEUED - await Promise.all( - failedItems.map(async (item) => { - await EmailBatchService.resetEmailBatchForRetry(item._id); - - // Audit retry queueing - try { - await EmailBatchAuditService.logAction( - email._id, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_BATCH_QUEUED, - 'EmailBatch item queued for retry', - { reason: 'Manual retry' }, - null, - requestorId, - item._id, - ); - } catch (auditErr) { - logger.logException(auditErr, 'Audit failure: EMAIL_BATCH_QUEUED (retry)'); - } - }), - ); - - logger.logInfo( - `Successfully queued Email ${emailId} and ${failedItems.length} failed EmailBatch items for retry`, - ); - - res.status(200).json({ - success: true, - message: `Successfully queued ${failedItems.length} failed EmailBatch items for retry`, - data: { - emailId: email._id, - failedItemsRetried: failedItems.length, - }, - }); - } catch (error) { - logger.logException(error, 'Error retrying Email'); - res.status(500).json({ - success: false, - message: 'Error retrying Email', - error: error.message, - }); - } -}; - -/** - * Get the audit trail for a specific parent Email. - * @param {import('express').Request} req - * @param {import('express').Response} res - */ -const getEmailAuditTrail = async (req, res) => { - try { - if (!req?.body?.requestor && !req?.user) { - return res.status(401).json({ - success: false, - message: 'Missing requestor', - }); - } - - // Permission check - use sendEmails permission to view audits - const requestor = req.body.requestor || req.user; - const canViewAudits = await hasPermission(requestor, 'sendEmails'); - if (!canViewAudits) { - return res.status(403).json({ - success: false, - message: 'You are not authorized to view email audits', - }); - } - - const { emailId } = req.params; - - if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { - return res.status(400).json({ - success: false, - message: 'Valid Email ID is required', - }); - } - - let auditTrail; - try { - auditTrail = await EmailBatchAuditService.getEmailAuditTrail(emailId); - } catch (serviceError) { - // Handle validation errors from service - if (serviceError.message.includes('required') || serviceError.message.includes('Invalid')) { - return res.status(400).json({ - success: false, - message: serviceError.message, - }); - } - throw serviceError; - } - - res.status(200).json({ - success: true, - data: auditTrail, - }); - } catch (error) { - logger.logException(error, 'Error getting email audit trail'); - res.status(500).json({ - success: false, - message: 'Error getting email audit trail', - error: error.message, - }); - } -}; - -/** - * Get the audit trail for a specific EmailBatch item. - * @param {import('express').Request} req - * @param {import('express').Response} res - */ -const getEmailBatchAuditTrail = async (req, res) => { - try { - if (!req?.body?.requestor && !req?.user) { - return res.status(401).json({ - success: false, - message: 'Missing requestor', - }); - } - - // Permission check - use sendEmails permission to view audits - const requestor = req.body.requestor || req.user; - const canViewAudits = await hasPermission(requestor, 'sendEmails'); - if (!canViewAudits) { - return res.status(403).json({ - success: false, - message: 'You are not authorized to view email audits', - }); - } - - const { emailBatchId } = req.params; - - // Validate emailBatchId is a valid ObjectId - if (!emailBatchId || !mongoose.Types.ObjectId.isValid(emailBatchId)) { - return res.status(400).json({ - success: false, - message: 'Invalid EmailBatch ID', - }); - } - - let auditTrail; - try { - auditTrail = await EmailBatchAuditService.getEmailBatchAuditTrail(emailBatchId); - } catch (serviceError) { - // Handle validation errors from service - if (serviceError.message.includes('required') || serviceError.message.includes('Invalid')) { - return res.status(400).json({ - success: false, - message: serviceError.message, - }); - } - throw serviceError; - } - - res.status(200).json({ - success: true, - data: auditTrail, - }); - } catch (error) { - logger.logException(error, 'Error getting email batch audit trail'); - res.status(500).json({ - success: false, - message: 'Error getting email batch audit trail', - error: error.message, - }); - } -}; - -module.exports = { - getEmails, - getEmailDetails, - getWorkerStatus, - retryEmail, - getEmailAuditTrail, - getEmailBatchAuditTrail, -}; diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index 8221d9585..d96a425e4 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -2,18 +2,16 @@ const mongoose = require('mongoose'); const jwt = require('jsonwebtoken'); const emailSender = require('../utilities/emailSender'); -const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); -const { - isValidEmailAddress, - normalizeRecipientsToArray, - ensureHtmlWithinLimit, -} = require('../utilities/emailValidators'); +const { EMAIL_CONFIG } = require('../config/emailConfig'); +const { isValidEmailAddress, normalizeRecipientsToArray } = require('../utilities/emailValidators'); +const TemplateRenderingService = require('../services/announcements/emails/templateRenderingService'); const EmailSubcriptionList = require('../models/emailSubcriptionList'); const userProfile = require('../models/userProfile'); const EmailBatchService = require('../services/announcements/emails/emailBatchService'); const EmailService = require('../services/announcements/emails/emailService'); -const EmailBatchAuditService = require('../services/announcements/emails/emailBatchAuditService'); +const emailProcessor = require('../services/announcements/emails/emailProcessor'); const { hasPermission } = require('../utilities/permissions'); +const { withTransaction } = require('../utilities/transactionHelper'); const config = require('../config'); const logger = require('../startup/logger'); @@ -27,7 +25,7 @@ const jwtSecret = process.env.JWT_SECRET; * @param {import('express').Response} res */ const sendEmail = async (req, res) => { - // Requestor is required for permission check and audit trail + // Requestor is required for permission check if (!req?.body?.requestor?.requestorId) { return res.status(401).json({ success: false, message: 'Missing requestor' }); } @@ -43,183 +41,80 @@ const sendEmail = async (req, res) => { try { const { to, subject, html } = req.body; - const missingFields = []; - if (!subject) missingFields.push('Subject'); - if (!html) missingFields.push('HTML content'); - if (!to) missingFields.push('Recipient email'); - if (missingFields.length) { + // Validate that all template variables have been replaced (business rule) + const unmatchedVariablesHtml = TemplateRenderingService.getUnreplacedVariables(html); + const unmatchedVariablesSubject = TemplateRenderingService.getUnreplacedVariables(subject); + const unmatchedVariables = [ + ...new Set([...unmatchedVariablesHtml, ...unmatchedVariablesSubject]), + ]; + if (unmatchedVariables.length > 0) { return res.status(400).json({ success: false, - message: `${missingFields.join(' and ')} ${missingFields.length > 1 ? 'are' : 'is'} required`, + message: + 'Email contains unreplaced template variables. Please ensure all variables are replaced before sending.', + unmatchedVariables, }); } - // Validate HTML content size - if (!ensureHtmlWithinLimit(html)) { - return res.status(413).json({ - success: false, - message: `HTML content exceeds ${EMAIL_JOB_CONFIG.LIMITS.MAX_HTML_BYTES / (1024 * 1024)}MB limit`, - }); + // Get user + const user = await userProfile.findById(req.body.requestor.requestorId); + if (!user) { + return res.status(404).json({ success: false, message: 'Requestor not found' }); } - // Validate subject length against config - if (subject && subject.length > EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH) { - return res.status(400).json({ - success: false, - message: `Subject cannot exceed ${EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH} characters`, - }); - } + // Normalize recipients once for service and response + const recipientsArray = normalizeRecipientsToArray(to); - // Validate HTML does not contain base64-encoded media - // const mediaValidation = validateHtmlMedia(html); - // if (!mediaValidation.isValid) { - // return res.status(400).json({ - // success: false, - // message: 'HTML contains embedded media files. Only URLs are allowed for media.', - // errors: mediaValidation.errors, - // }); - // } - - // Validate that all template variables have been replaced - const templateVariableRegex = /\{\{(\w+)\}\}/g; - const unmatchedVariables = []; - let match = templateVariableRegex.exec(html); - while (match !== null) { - if (!unmatchedVariables.includes(match[1])) { - unmatchedVariables.push(match[1]); - } - match = templateVariableRegex.exec(html); - } - // Check subject as well - if (subject) { - templateVariableRegex.lastIndex = 0; - match = templateVariableRegex.exec(subject); - while (match !== null) { - if (!unmatchedVariables.includes(match[1])) { - unmatchedVariables.push(match[1]); - } - match = templateVariableRegex.exec(subject); - } - } - if (unmatchedVariables.length > 0) { - return res.status(400).json({ - success: false, - message: - 'Email contains unreplaced template variables. Please ensure all variables are replaced before sending.', - errors: { - unmatchedVariables: `Found unreplaced variables: ${unmatchedVariables.join(', ')}`, + // Create email and batches in transaction (validation happens in services) + const email = await withTransaction(async (session) => { + // Create parent Email (validates subject, htmlContent, createdBy) + const createdEmail = await EmailService.createEmail( + { + subject, + htmlContent: html, + createdBy: user._id, }, - }); - } - - try { - // Normalize, dedupe, and validate recipients FIRST - const recipientsArray = normalizeRecipientsToArray(to); - if (recipientsArray.length === 0) { - return res - .status(400) - .json({ success: false, message: 'At least one recipient email is required' }); - } - if (recipientsArray.length > EMAIL_JOB_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST) { - return res.status(400).json({ - success: false, - message: `A maximum of ${EMAIL_JOB_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST} recipients are allowed per request`, - }); - } - const invalidRecipients = recipientsArray.filter((e) => !isValidEmailAddress(e)); - if (invalidRecipients.length) { - return res.status(400).json({ - success: false, - message: 'One or more recipient emails are invalid', - invalidRecipients, - }); - } + session, + ); - // Always use batch system for tracking and progress - const user = await userProfile.findById(req.body.requestor.requestorId); - if (!user) { - return res.status(400).json({ success: false, message: 'Requestor not found' }); - } + // Create EmailBatch items with all recipients (validates recipients, counts, email format) + const recipientObjects = recipientsArray.map((emailAddr) => ({ email: emailAddr })); + await EmailBatchService.createEmailBatches( + createdEmail._id, + recipientObjects, + { + emailType: EMAIL_CONFIG.EMAIL_TYPES.BCC, + }, + session, + ); - // Start MongoDB transaction - const session = await mongoose.startSession(); - session.startTransaction(); - - try { - // Create parent Email within transaction - const email = await EmailService.createEmail( - { - subject, - htmlContent: html, - createdBy: user._id, - }, - session, - ); - - // Create EmailBatch items with all recipients (chunked automatically) within transaction - // Always use BCC for all recipients (sender goes in 'to' field) - const recipientObjects = recipientsArray.map((emailAddr) => ({ email: emailAddr })); - const inserted = await EmailBatchService.createEmailBatches( - email._id, - recipientObjects, - { - emailType: EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC, - }, - session, - ); - - // Commit transaction - await session.commitTransaction(); - - // Audit logging after successful commit (outside transaction to avoid failures) - try { - await EmailBatchAuditService.logEmailQueued( - email._id, - { - subject: email.subject, - }, - user._id, - ); - - // Audit each batch creation - await Promise.all( - inserted.map(async (item) => { - await EmailBatchAuditService.logEmailBatchQueued( - email._id, - item._id, - { - recipientCount: item.recipients?.length || 0, - emailType: item.emailType, - recipients: item.recipients?.map((r) => r.email) || [], - emailBatchId: item._id.toString(), - }, - user._id, - ); - }), - ); - } catch (auditErr) { - logger.logException(auditErr, 'Audit failure after successful email creation'); - // Don't fail the request if audit fails - } + return createdEmail; + }); - session.endSession(); + // Process email immediately (async, fire and forget) + emailProcessor.processEmail(email._id).catch((processError) => { + logger.logException( + processError, + `Error processing email ${email._id} immediately after creation`, + ); + }); - return res.status(200).json({ - success: true, - message: `Email created successfully for ${recipientsArray.length} recipient(s)`, - }); - } catch (emailError) { - // Abort transaction on error - await session.abortTransaction(); - session.endSession(); - throw emailError; - } - } catch (emailError) { - logger.logException(emailError, 'Error creating email'); - return res.status(500).json({ success: false, message: 'Error creating email' }); - } + return res.status(200).json({ + success: true, + message: `Email created successfully for ${recipientsArray.length} recipient(s)`, + }); } catch (error) { - return res.status(500).json({ success: false, message: 'Error creating email' }); + logger.logException(error, 'Error creating email'); + const statusCode = error.statusCode || 500; + const response = { + success: false, + message: error.message || 'Error creating email', + }; + // Include invalidRecipients if present (from service validation) + if (error.invalidRecipients) { + response.invalidRecipients = error.invalidRecipients; + } + return res.status(statusCode).json(response); } }; @@ -230,7 +125,7 @@ const sendEmail = async (req, res) => { * @param {import('express').Response} res */ const sendEmailToSubscribers = async (req, res) => { - // Requestor is required for permission check and audit trail + // Requestor is required for permission check if (!req?.body?.requestor?.requestorId) { return res.status(401).json({ success: false, message: 'Missing requestor' }); } @@ -245,68 +140,29 @@ const sendEmailToSubscribers = async (req, res) => { try { const { subject, html } = req.body; - if (!subject || !html) { - return res - .status(400) - .json({ success: false, message: 'Subject and HTML content are required' }); - } - - if (!ensureHtmlWithinLimit(html)) { - return res.status(413).json({ - success: false, - message: `HTML content exceeds ${EMAIL_JOB_CONFIG.LIMITS.MAX_HTML_BYTES / (1024 * 1024)}MB limit`, - }); - } - // Validate HTML does not contain base64-encoded media - // const mediaValidation = validateHtmlMedia(html); - // if (!mediaValidation.isValid) { - // return res.status(400).json({ - // success: false, - // message: 'HTML contains embedded media files. Only URLs are allowed for media.', - // errors: mediaValidation.errors, - // }); - // } - - // Validate that all template variables have been replaced - const templateVariableRegex = /\{\{(\w+)\}\}/g; - const unmatchedVariables = []; - let match = templateVariableRegex.exec(html); - while (match !== null) { - if (!unmatchedVariables.includes(match[1])) { - unmatchedVariables.push(match[1]); - } - match = templateVariableRegex.exec(html); - } - // Check subject as well - if (subject) { - templateVariableRegex.lastIndex = 0; - match = templateVariableRegex.exec(subject); - while (match !== null) { - if (!unmatchedVariables.includes(match[1])) { - unmatchedVariables.push(match[1]); - } - match = templateVariableRegex.exec(subject); - } - } + // Validate that all template variables have been replaced (business rule) + const unmatchedVariablesHtml = TemplateRenderingService.getUnreplacedVariables(html); + const unmatchedVariablesSubject = TemplateRenderingService.getUnreplacedVariables(subject); + const unmatchedVariables = [ + ...new Set([...unmatchedVariablesHtml, ...unmatchedVariablesSubject]), + ]; if (unmatchedVariables.length > 0) { return res.status(400).json({ success: false, message: 'Email contains unreplaced template variables. Please ensure all variables are replaced before sending.', - errors: { - unmatchedVariables: `Found unreplaced variables: ${unmatchedVariables.join(', ')}`, - }, + unmatchedVariables, }); } - // Always use new batch system for broadcast emails + // Get user const user = await userProfile.findById(req.body.requestor.requestorId); if (!user) { - return res.status(400).json({ success: false, message: 'User not found' }); + return res.status(404).json({ success: false, message: 'User not found' }); } - // Get ALL recipients FIRST (HGN users + email subscribers) + // Get ALL recipients (HGN users + email subscribers) const users = await userProfile.find({ firstName: { $ne: '' }, email: { $ne: null }, @@ -325,13 +181,10 @@ const sendEmailToSubscribers = async (req, res) => { return res.status(400).json({ success: false, message: 'No recipients found' }); } - // Start MongoDB transaction - const session = await mongoose.startSession(); - session.startTransaction(); - - try { - // Create parent Email within transaction - const email = await EmailService.createEmail( + // Create email and batches in transaction (validation happens in services) + const email = await withTransaction(async (session) => { + // Create parent Email (validates subject, htmlContent, createdBy) + const createdEmail = await EmailService.createEmail( { subject, htmlContent: html, @@ -346,66 +199,43 @@ const sendEmailToSubscribers = async (req, res) => { ...emailSubscribers.map((subscriber) => ({ email: subscriber.email })), ]; - // Create EmailBatch items with all recipients (chunked automatically) within transaction - // Use BCC for broadcast emails to hide recipient list from each other - const inserted = await EmailBatchService.createEmailBatches( - email._id, + // Create EmailBatch items with all recipients (validates recipients, counts, email format) + await EmailBatchService.createEmailBatches( + createdEmail._id, allRecipients, { - emailType: EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC, + emailType: EMAIL_CONFIG.EMAIL_TYPES.BCC, }, session, ); - // Commit transaction - await session.commitTransaction(); - - // Audit logging after successful commit (outside transaction to avoid failures) - try { - await EmailBatchAuditService.logEmailQueued( - email._id, - { - subject: email.subject, - }, - user._id, - ); - - // Audit each batch creation - await Promise.all( - inserted.map(async (item) => { - await EmailBatchAuditService.logEmailBatchQueued( - email._id, - item._id, - { - recipientCount: item.recipients?.length || 0, - emailType: item.emailType, - recipients: item.recipients?.map((r) => r.email) || [], - emailBatchId: item._id.toString(), - }, - user._id, - ); - }), - ); - } catch (auditErr) { - logger.logException(auditErr, 'Audit failure after successful broadcast email creation'); - // Don't fail the request if audit fails - } + return createdEmail; + }); - session.endSession(); + // Process email immediately (async, fire and forget) + emailProcessor.processEmail(email._id).catch((processError) => { + logger.logException( + processError, + `Error processing broadcast email ${email._id} immediately after creation`, + ); + }); - return res.status(200).json({ - success: true, - message: `Broadcast email created successfully for ${totalRecipients} recipient(s)`, - }); - } catch (error) { - // Abort transaction on error - await session.abortTransaction(); - session.endSession(); - throw error; - } + return res.status(200).json({ + success: true, + message: `Broadcast email created successfully for ${totalRecipients} recipient(s)`, + }); } catch (error) { logger.logException(error, 'Error creating broadcast email'); - return res.status(500).json({ success: false, message: 'Error creating broadcast email' }); + const statusCode = error.statusCode || 500; + const response = { + success: false, + message: error.message || 'Error creating broadcast email', + }; + // Include invalidRecipients if present (from service validation) + if (error.invalidRecipients) { + response.invalidRecipients = error.invalidRecipients; + } + return res.status(statusCode).json(response); } }; @@ -416,7 +246,7 @@ const sendEmailToSubscribers = async (req, res) => { * @param {import('express').Response} res */ const resendEmail = async (req, res) => { - // Requestor is required for permission check and audit trail + // Requestor is required for permission check if (!req?.body?.requestor?.requestorId) { return res.status(401).json({ success: false, message: 'Missing requestor' }); } @@ -437,11 +267,8 @@ const resendEmail = async (req, res) => { return res.status(400).json({ success: false, message: 'Invalid emailId' }); } - // Get the original email - const originalEmail = await EmailService.getEmailById(emailId); - if (!originalEmail) { - return res.status(404).json({ success: false, message: 'Email not found' }); - } + // Get the original email (service throws error if not found) + const originalEmail = await EmailService.getEmailById(emailId, null, true); // Validate recipient option if (!recipientOption) { @@ -497,24 +324,8 @@ const resendEmail = async (req, res) => { }); } - // Normalize and validate recipients + // Normalize recipients (validation happens in service) const recipientsArray = normalizeRecipientsToArray(specificRecipients); - const invalidRecipients = recipientsArray.filter((e) => !isValidEmailAddress(e)); - if (invalidRecipients.length) { - return res.status(400).json({ - success: false, - message: 'One or more recipient emails are invalid', - invalidRecipients, - }); - } - - if (recipientsArray.length > EMAIL_JOB_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST) { - return res.status(400).json({ - success: false, - message: `A maximum of ${EMAIL_JOB_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST} recipients are allowed per request`, - }); - } - allRecipients = recipientsArray.map((email) => ({ email })); } else if (recipientOption === 'same') { // Get recipients from original email's EmailBatch items @@ -546,13 +357,10 @@ const resendEmail = async (req, res) => { return res.status(400).json({ success: false, message: 'No recipients found' }); } - // Start MongoDB transaction - const session = await mongoose.startSession(); - session.startTransaction(); - - try { - // Create new Email (copy) within transaction - const newEmail = await EmailService.createEmail( + // Create email and batches in transaction + const newEmail = await withTransaction(async (session) => { + // Create new Email (copy) - validation happens in service + const createdEmail = await EmailService.createEmail( { subject: originalEmail.subject, htmlContent: originalEmail.htmlContent, @@ -561,71 +369,47 @@ const resendEmail = async (req, res) => { session, ); - // Create EmailBatch items within transaction - // Always use BCC for all recipients (sender goes in 'to' field) - const inserted = await EmailBatchService.createEmailBatches( - newEmail._id, + // Create EmailBatch items + await EmailBatchService.createEmailBatches( + createdEmail._id, allRecipients, { - emailType: EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC, + emailType: EMAIL_CONFIG.EMAIL_TYPES.BCC, }, session, ); - // Commit transaction - await session.commitTransaction(); - - // Audit logging after successful commit (outside transaction) - try { - await EmailBatchAuditService.logEmailQueued( - newEmail._id, - { - subject: newEmail.subject, - resendFrom: emailId.toString(), - recipientOption, - }, - user._id, - ); - - // Audit each batch creation - await Promise.all( - inserted.map(async (item) => { - await EmailBatchAuditService.logEmailBatchQueued( - newEmail._id, - item._id, - { - recipientCount: item.recipients?.length || 0, - emailType: item.emailType, - recipients: item.recipients?.map((r) => r.email) || [], - emailBatchId: item._id.toString(), - }, - user._id, - ); - }), - ); - } catch (auditErr) { - logger.logException(auditErr, 'Audit failure after successful email resend'); - } + return createdEmail; + }); - session.endSession(); + // Process email immediately (async, fire and forget) + emailProcessor.processEmail(newEmail._id).catch((processError) => { + logger.logException( + processError, + `Error processing resent email ${newEmail._id} immediately after creation`, + ); + }); - return res.status(200).json({ - success: true, - message: `Email queued for resend successfully to ${allRecipients.length} recipient(s)`, - data: { - emailId: newEmail._id, - recipientCount: allRecipients.length, - }, - }); - } catch (error) { - // Abort transaction on error - await session.abortTransaction(); - session.endSession(); - throw error; - } + return res.status(200).json({ + success: true, + message: `Email created for resend successfully to ${allRecipients.length} recipient(s)`, + data: { + emailId: newEmail._id, + recipientCount: allRecipients.length, + }, + }); } catch (error) { logger.logException(error, 'Error resending email'); - return res.status(500).json({ success: false, message: 'Error resending email' }); + const statusCode = error.statusCode || 500; + const response = { + success: false, + message: error.message || 'Error resending email', + }; + // Include invalidRecipients if present (from service validation) + if (error.invalidRecipients) { + response.invalidRecipients = error.invalidRecipients; + } + return res.status(statusCode).json(response); } }; @@ -667,7 +451,11 @@ const updateEmailSubscriptions = async (req, res) => { .json({ success: true, message: 'Email subscription updated successfully' }); } catch (error) { logger.logException(error, 'Error updating email subscriptions'); - return res.status(500).json({ success: false, message: 'Error updating email subscriptions' }); + const statusCode = error.statusCode || 500; + return res.status(statusCode).json({ + success: false, + message: error.message || 'Error updating email subscriptions', + }); } }; @@ -722,7 +510,7 @@ const addNonHgnEmailSubscription = async (req, res) => { const token = jwt.sign(payload, jwtSecret, { expiresIn: '24h' }); // Fixed: was '360' (invalid) if (!config.FRONT_END_URL) { - console.error('FRONT_END_URL is not configured'); + logger.logException(new Error('FRONT_END_URL is not configured'), 'Configuration error'); return res .status(500) .json({ success: false, message: 'Server configuration error. Please contact support.' }); diff --git a/src/controllers/emailOutboxController.js b/src/controllers/emailOutboxController.js new file mode 100644 index 000000000..b45d0d2db --- /dev/null +++ b/src/controllers/emailOutboxController.js @@ -0,0 +1,177 @@ +const mongoose = require('mongoose'); +const EmailBatchService = require('../services/announcements/emails/emailBatchService'); +const EmailService = require('../services/announcements/emails/emailService'); +const emailProcessor = require('../services/announcements/emails/emailProcessor'); +const { hasPermission } = require('../utilities/permissions'); +const logger = require('../startup/logger'); +const { EMAIL_CONFIG } = require('../config/emailConfig'); + +/** + * Get all announcement Email records (parent documents) - Outbox view. + * @param {import('express').Request} req + * @param {import('express').Response} res + */ +const getEmails = async (req, res) => { + try { + // Permission check - viewing emails requires sendEmails permission + const canViewEmails = await hasPermission(req.body.requestor, 'sendEmails'); + if (!canViewEmails) { + return res + .status(403) + .json({ success: false, message: 'You are not authorized to view emails.' }); + } + + const emails = await EmailService.getAllEmails(); + + res.status(200).json({ + success: true, + data: emails, + }); + } catch (error) { + logger.logException(error, 'Error getting emails'); + const statusCode = error.statusCode || 500; + return res.status(statusCode).json({ + success: false, + message: error.message || 'Error getting emails', + }); + } +}; + +/** + * Get a parent Email and its associated EmailBatch items - Outbox detail view. + * @param {import('express').Request} req + * @param {import('express').Response} res + */ +const getEmailDetails = async (req, res) => { + try { + // Permission check - viewing email details requires sendEmails permission + const canViewEmails = await hasPermission(req.body.requestor, 'sendEmails'); + if (!canViewEmails) { + return res + .status(403) + .json({ success: false, message: 'You are not authorized to view email details.' }); + } + + const { emailId } = req.params; // emailId is now the ObjectId of parent Email + + // Service validates emailId and throws error if not found + const result = await EmailBatchService.getEmailWithBatches(emailId); + + res.status(200).json({ + success: true, + data: result, + }); + } catch (error) { + logger.logException(error, 'Error getting Email details with EmailBatch items'); + const statusCode = error.statusCode || 500; + return res.status(statusCode).json({ + success: false, + message: error.message || 'Error getting email details', + }); + } +}; + +/** + * Retry a parent Email by resetting all FAILED EmailBatch items to PENDING. + * - Processes the email immediately asynchronously. + * @param {import('express').Request} req + * @param {import('express').Response} res + */ +const retryEmail = async (req, res) => { + try { + const { emailId } = req.params; + + // Validate emailId is a valid ObjectId + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + return res.status(400).json({ + success: false, + message: 'Invalid Email ID', + }); + } + + // Permission check - retrying emails requires sendEmails permission + const canRetryEmail = await hasPermission(req.body.requestor, 'sendEmails'); + if (!canRetryEmail) { + return res + .status(403) + .json({ success: false, message: 'You are not authorized to retry emails.' }); + } + + // Get the Email (service throws error if not found) + const email = await EmailService.getEmailById(emailId, null, true); + + // Only allow retry for emails in final states (FAILED or PROCESSED) + const allowedRetryStatuses = [ + EMAIL_CONFIG.EMAIL_STATUSES.FAILED, + EMAIL_CONFIG.EMAIL_STATUSES.PROCESSED, + ]; + + if (!allowedRetryStatuses.includes(email.status)) { + return res.status(400).json({ + success: false, + message: `Email must be in FAILED or PROCESSED status to retry. Current status: ${email.status}`, + }); + } + + // Get all FAILED EmailBatch items (service validates emailId) + const failedItems = await EmailBatchService.getFailedBatchesForEmail(emailId); + + if (failedItems.length === 0) { + logger.logInfo(`Email ${emailId} has no failed EmailBatch items to retry`); + return res.status(200).json({ + success: true, + message: 'No failed EmailBatch items to retry', + data: { + emailId: email._id, + failedItemsRetried: 0, + }, + }); + } + + logger.logInfo(`Retrying ${failedItems.length} failed EmailBatch items: ${emailId}`); + + // Mark parent Email as PENDING for retry + await EmailService.markEmailPending(emailId); + + // Reset each failed item to PENDING + await Promise.all( + failedItems.map(async (item) => { + await EmailBatchService.resetEmailBatchForRetry(item._id); + }), + ); + + logger.logInfo( + `Successfully reset Email ${emailId} and ${failedItems.length} failed EmailBatch items to PENDING for retry`, + ); + + // Process email immediately (async, fire and forget) + emailProcessor.processEmail(emailId).catch((processError) => { + logger.logException( + processError, + `Error processing email ${emailId} immediately after retry`, + ); + }); + + res.status(200).json({ + success: true, + message: `Successfully reset ${failedItems.length} failed EmailBatch items for retry`, + data: { + emailId: email._id, + failedItemsRetried: failedItems.length, + }, + }); + } catch (error) { + logger.logException(error, 'Error retrying Email'); + const statusCode = error.statusCode || 500; + return res.status(statusCode).json({ + success: false, + message: error.message || 'Error retrying Email', + }); + } +}; + +module.exports = { + getEmails, + getEmailDetails, + retryEmail, +}; diff --git a/src/controllers/emailTemplateController.js b/src/controllers/emailTemplateController.js index 0afb26241..78c7d3b19 100644 --- a/src/controllers/emailTemplateController.js +++ b/src/controllers/emailTemplateController.js @@ -1,120 +1,11 @@ -const mongoose = require('mongoose'); -const EmailTemplate = require('../models/emailTemplate'); -const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); -const { hasPermission } = require('../utilities/permissions'); -const logger = require('../startup/logger'); -const { ensureHtmlWithinLimit } = require('../utilities/emailValidators'); - -/** - * Validate template variables. - * - Ensures non-empty unique names and validates allowed types. - * @param {Array<{name: string, type?: 'text'|'url'|'number'|'textarea'|'image'>} | undefined} variables - * @returns {{isValid: boolean, errors?: string[]}} - */ -function validateTemplateVariables(variables) { - if (!variables || !Array.isArray(variables)) { - return { isValid: true }; - } - - const errors = []; - const variableNames = new Set(); - - variables.forEach((variable, index) => { - if (!variable.name || typeof variable.name !== 'string' || !variable.name.trim()) { - errors.push(`Variable ${index + 1}: name is required and must be a non-empty string`); - } else { - const varName = variable.name.trim(); - // Validate variable name format (alphanumeric and underscore only) - if (!/^[a-zA-Z0-9_]+$/.test(varName)) { - errors.push( - `Variable ${index + 1}: name must contain only alphanumeric characters and underscores`, - ); - } - // Check for duplicates - if (variableNames.has(varName.toLowerCase())) { - errors.push(`Variable ${index + 1}: duplicate variable name '${varName}'`); - } - variableNames.add(varName.toLowerCase()); - } - - if (variable.type && !['text', 'url', 'number', 'textarea', 'image'].includes(variable.type)) { - errors.push(`Variable ${index + 1}: type must be one of: text, url, number, textarea, image`); - } - }); - - return { - isValid: errors.length === 0, - errors, - }; -} - /** - * Validate template content (HTML and subject) against defined variables. - * - Flags undefined placeholders and unused defined variables. - * @param {Array<{name: string}>} templateVariables - * @param {string} htmlContent - * @param {string} subject - * @returns {{isValid: boolean, errors: string[]}} + * Email Template Controller - Handles HTTP requests for email template operations */ -function validateTemplateVariableUsage(templateVariables, htmlContent, subject) { - const errors = []; - - if (!templateVariables || templateVariables.length === 0) { - return { isValid: true, errors: [] }; - } - - // Extract variable placeholders from content (format: {{variableName}}) - const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; - const usedVariables = new Set(); - const foundPlaceholders = []; - - // Check HTML content - if (htmlContent) { - let match = variablePlaceholderRegex.exec(htmlContent); - while (match !== null) { - const varName = match[1]; - foundPlaceholders.push(varName); - usedVariables.add(varName); - match = variablePlaceholderRegex.exec(htmlContent); - } - } - - // Reset regex for subject - variablePlaceholderRegex.lastIndex = 0; - - // Check subject - if (subject) { - let match = variablePlaceholderRegex.exec(subject); - while (match !== null) { - const varName = match[1]; - foundPlaceholders.push(varName); - usedVariables.add(varName); - match = variablePlaceholderRegex.exec(subject); - } - } - - // Check for undefined variable placeholders in content - const definedVariableNames = templateVariables.map((v) => v.name); - foundPlaceholders.forEach((placeholder) => { - if (!definedVariableNames.includes(placeholder)) { - errors.push( - `Variable placeholder '{{${placeholder}}}' is used in content but not defined in template variables`, - ); - } - }); - // Check for defined variables that are not used in content (treated as errors) - templateVariables.forEach((variable) => { - if (!usedVariables.has(variable.name)) { - errors.push(`Variable '{{${variable.name}}}}' is defined but not used in template content`); - } - }); - - return { - isValid: errors.length === 0, - errors, - }; -} +const EmailTemplateService = require('../services/announcements/emails/emailTemplateService'); +const TemplateRenderingService = require('../services/announcements/emails/templateRenderingService'); +const { hasPermission } = require('../utilities/permissions'); +const logger = require('../startup/logger'); /** * Get all email templates (with basic search/sort and optional content projection). @@ -145,30 +36,29 @@ const getAllEmailTemplates = async (req, res) => { const query = {}; const sort = {}; - // Add search functionality with text index + // Add search functionality if (search && search.trim()) { query.$or = [{ name: { $regex: search.trim(), $options: 'i' } }]; } - // Build sort object - let frontend decide sort field and order + // Build sort object if (sortBy) { - sort[sortBy] = 1; // default ascending when sortBy provided + sort[sortBy] = 1; } else { - // Default sort only if frontend doesn't specify sort.created_at = -1; } - // Build projection based on include flags; always include audit fields + // Build projection let projection = '_id name created_by updated_by created_at updated_at'; - if (includeEmailContent === 'true') projection += ' subject html_content variables'; - - let queryBuilder = EmailTemplate.find(query).select(projection).sort(sort); - - // Always include created_by and updated_by populations - queryBuilder = queryBuilder.populate('created_by', 'firstName lastName'); - queryBuilder = queryBuilder.populate('updated_by', 'firstName lastName'); + if (includeEmailContent === 'true') { + projection += ' subject html_content variables'; + } - const templates = await queryBuilder.lean(); + const templates = await EmailTemplateService.getAllTemplates(query, { + sort, + projection, + populate: true, + }); res.status(200).json({ success: true, @@ -176,11 +66,15 @@ const getAllEmailTemplates = async (req, res) => { }); } catch (error) { logger.logException(error, 'Error fetching email templates'); - res.status(500).json({ + const statusCode = error.statusCode || 500; + const response = { success: false, - message: 'Error fetching email templates', - error: error.message, - }); + message: error.message || 'Error fetching email templates', + }; + if (error.errors && Array.isArray(error.errors)) { + response.errors = error.errors; + } + return res.status(statusCode).json(response); } }; @@ -210,24 +104,10 @@ const getEmailTemplateById = async (req, res) => { const { id } = req.params; - // Validate ObjectId - if (!id || !mongoose.Types.ObjectId.isValid(id)) { - return res.status(400).json({ - success: false, - message: 'Invalid template ID', - }); - } - - const template = await EmailTemplate.findById(id) - .populate('created_by', 'firstName lastName email') - .populate('updated_by', 'firstName lastName email'); - - if (!template) { - return res.status(404).json({ - success: false, - message: 'Email template not found', - }); - } + // Service validates ID and throws error with statusCode if not found + const template = await EmailTemplateService.getTemplateById(id, { + populate: true, + }); res.status(200).json({ success: true, @@ -235,17 +115,16 @@ const getEmailTemplateById = async (req, res) => { }); } catch (error) { logger.logException(error, 'Error fetching email template'); - res.status(500).json({ + const statusCode = error.statusCode || 500; + return res.status(statusCode).json({ success: false, - message: 'Error fetching email template', - error: error.message, + message: error.message || 'Error fetching email template', }); } }; /** * Create a new email template. - * - Validates content size, media, name/subject length, variables and usage. * @param {import('express').Request} req * @param {import('express').Response} res */ @@ -271,118 +150,14 @@ const createEmailTemplate = async (req, res) => { const { name, subject, html_content: htmlContent, variables } = req.body; const userId = req.body.requestor.requestorId; - // Validate HTML content size - if (!ensureHtmlWithinLimit(htmlContent)) { - return res.status(413).json({ - success: false, - message: `HTML content exceeds ${EMAIL_JOB_CONFIG.LIMITS.MAX_HTML_BYTES / (1024 * 1024)}MB limit`, - }); - } - - // Validate HTML does not contain base64-encoded media - // const mediaValidation = validateHtmlMedia(htmlContent); - // if (!mediaValidation.isValid) { - // return res.status(400).json({ - // success: false, - // message: 'HTML contains embedded media files. Only URLs are allowed for media.', - // errors: mediaValidation.errors, - // }); - // } - - // Validate name length - const trimmedName = name.trim(); - if (trimmedName.length > 50) { - return res.status(400).json({ - success: false, - message: 'Template name cannot exceed 50 characters', - }); - } - - // Validate subject length - const trimmedSubject = subject.trim(); - if (trimmedSubject.length > EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH) { - return res.status(400).json({ - success: false, - message: `Subject cannot exceed ${EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH} characters`, - }); - } - - // Check if template with the same name already exists (case-insensitive) - const existingTemplate = await EmailTemplate.findOne({ - name: { $regex: new RegExp(`^${trimmedName}$`, 'i') }, - }); - if (existingTemplate) { - return res.status(400).json({ - success: false, - message: 'Email template with this name already exists', - }); - } - - // Validate variables - if (variables && variables.length > 0) { - const variableValidation = validateTemplateVariables(variables); - if (!variableValidation.isValid) { - return res.status(400).json({ - success: false, - message: 'Invalid template variables', - errors: variableValidation.errors, - }); - } - - // Validate variable usage in content (HTML and subject) - const variableUsageValidation = validateTemplateVariableUsage( - variables, - htmlContent, - trimmedSubject, - ); - if (!variableUsageValidation.isValid) { - return res.status(400).json({ - success: false, - message: 'Invalid variable usage in template content', - errors: variableUsageValidation.errors, - }); - } - } else { - // If no variables are defined, check for any variable placeholders in content - const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; - const foundInHtml = variablePlaceholderRegex.test(htmlContent); - variablePlaceholderRegex.lastIndex = 0; - const foundInSubject = variablePlaceholderRegex.test(trimmedSubject); - - if (foundInHtml || foundInSubject) { - return res.status(400).json({ - success: false, - message: - 'Template content contains variable placeholders ({{variableName}}) but no variables are defined. Please define variables or remove placeholders from content.', - }); - } - } - - // Validate userId is valid ObjectId - if (userId && !mongoose.Types.ObjectId.isValid(userId)) { - return res.status(400).json({ - success: false, - message: 'Invalid user ID', - }); - } - - // Create new email template - const template = new EmailTemplate({ - name: trimmedName, - subject: trimmedSubject, - html_content: htmlContent.trim(), - variables: variables || [], - created_by: userId, - updated_by: userId, - }); - - await template.save(); - - // Populate created_by and updated_by fields for response - await template.populate('created_by', 'firstName lastName email'); - await template.populate('updated_by', 'firstName lastName email'); + const templateData = { + name, + subject, + html_content: htmlContent, + variables, + }; - logger.logInfo(`Email template created: ${template.name} by user ${userId}`); + const template = await EmailTemplateService.createTemplate(templateData, userId); res.status(201).json({ success: true, @@ -391,17 +166,20 @@ const createEmailTemplate = async (req, res) => { }); } catch (error) { logger.logException(error, 'Error creating email template'); - res.status(500).json({ + const statusCode = error.statusCode || 500; + const response = { success: false, - message: 'Error creating email template', - error: error.message, - }); + message: error.message || 'Error creating email template', + }; + if (error.errors && Array.isArray(error.errors)) { + response.errors = error.errors; + } + return res.status(statusCode).json(response); } }; /** * Update an existing email template by ID. - * - Validates content and ensures unique name when changed. * @param {import('express').Request} req * @param {import('express').Response} res */ @@ -426,222 +204,187 @@ const updateEmailTemplate = async (req, res) => { const { id } = req.params; const { name, subject, html_content: htmlContent, variables } = req.body; + const userId = req.body.requestor.requestorId; - // Validate ObjectId - if (!id || !mongoose.Types.ObjectId.isValid(id)) { - return res.status(400).json({ - success: false, - message: 'Invalid template ID', - }); - } + const templateData = { + name, + subject, + html_content: htmlContent, + variables, + }; - // Validate HTML content size - if (!ensureHtmlWithinLimit(htmlContent)) { - return res.status(413).json({ - success: false, - message: `HTML content exceeds ${EMAIL_JOB_CONFIG.LIMITS.MAX_HTML_BYTES / (1024 * 1024)}MB limit`, - }); - } + const template = await EmailTemplateService.updateTemplate(id, templateData, userId); - // Validate HTML does not contain base64-encoded media - // const mediaValidation = validateHtmlMedia(htmlContent); - // if (!mediaValidation.isValid) { - // return res.status(400).json({ - // success: false, - // message: 'HTML contains embedded media files. Only URLs are allowed for media.', - // errors: mediaValidation.errors, - // }); - // } - - // Validate name and subject length - const trimmedName = name.trim(); - const trimmedSubject = subject.trim(); - - if (trimmedName.length > 50) { - return res.status(400).json({ - success: false, - message: 'Template name cannot exceed 50 characters', - }); + res.status(200).json({ + success: true, + message: 'Email template updated successfully', + template, + }); + } catch (error) { + logger.logException(error, 'Error updating email template'); + const statusCode = error.statusCode || 500; + const response = { + success: false, + message: error.message || 'Error updating email template', + }; + if (error.errors && Array.isArray(error.errors)) { + response.errors = error.errors; } + return res.status(statusCode).json(response); + } +}; - if (trimmedSubject.length > EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH) { - return res.status(400).json({ +/** + * Delete an email template by ID (hard delete). + * @param {import('express').Request} req + * @param {import('express').Response} res + */ +const deleteEmailTemplate = async (req, res) => { + try { + // Requestor is required for permission check + if (!req?.body?.requestor?.requestorId) { + return res.status(401).json({ success: false, - message: `Subject cannot exceed ${EMAIL_JOB_CONFIG.LIMITS.SUBJECT_MAX_LENGTH} characters`, + message: 'Missing requestor', }); } - // Get current template - const currentTemplate = await EmailTemplate.findById(id); - if (!currentTemplate) { - return res.status(404).json({ + // Permission check - use sendEmails permission to delete templates + const canDeleteTemplate = await hasPermission(req.body.requestor, 'sendEmails'); + if (!canDeleteTemplate) { + return res.status(403).json({ success: false, - message: 'Email template not found', + message: 'You are not authorized to delete email templates.', }); } - // Only check for duplicate names if the name is actually changing (case-insensitive) - if (currentTemplate.name.toLowerCase() !== trimmedName.toLowerCase()) { - const existingTemplate = await EmailTemplate.findOne({ - name: { $regex: new RegExp(`^${trimmedName}$`, 'i') }, - _id: { $ne: id }, - }); - if (existingTemplate) { - return res.status(400).json({ - success: false, - message: 'Another email template with this name already exists', - }); - } - } + const { id } = req.params; + const userId = req.body.requestor.requestorId; - // Validate variables - if (variables && variables.length > 0) { - const variableValidation = validateTemplateVariables(variables); - if (!variableValidation.isValid) { - return res.status(400).json({ - success: false, - message: 'Invalid template variables', - errors: variableValidation.errors, - }); - } - - // Validate variable usage in content (HTML and subject) - const variableUsageValidation = validateTemplateVariableUsage( - variables, - htmlContent, - trimmedSubject, - ); - if (!variableUsageValidation.isValid) { - return res.status(400).json({ - success: false, - message: 'Invalid variable usage in template content', - errors: variableUsageValidation.errors, - }); - } - } else { - // If no variables are defined, check for any variable placeholders in content - const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; - const foundInHtml = variablePlaceholderRegex.test(htmlContent); - variablePlaceholderRegex.lastIndex = 0; - const foundInSubject = variablePlaceholderRegex.test(trimmedSubject); - - if (foundInHtml || foundInSubject) { - return res.status(400).json({ - success: false, - message: - 'Template content contains variable placeholders ({{variableName}}) but no variables are defined. Please define variables or remove placeholders from content.', - }); - } + await EmailTemplateService.deleteTemplate(id, userId); + + res.status(200).json({ + success: true, + message: 'Email template deleted successfully', + }); + } catch (error) { + logger.logException(error, 'Error deleting email template'); + const statusCode = error.statusCode || 500; + return res.status(statusCode).json({ + success: false, + message: error.message || 'Error deleting email template', + }); + } +}; + +/** + * Preview template with variable values. + * @param {import('express').Request} req + * @param {import('express').Response} res + */ +const previewTemplate = async (req, res) => { + try { + // Permission check + if (!req?.body?.requestor?.requestorId && !req?.user?.userid) { + return res.status(401).json({ + success: false, + message: 'Missing requestor', + }); } - // Validate userId is valid ObjectId - const userId = req.body.requestor?.requestorId; - if (userId && !mongoose.Types.ObjectId.isValid(userId)) { - return res.status(400).json({ + const requestor = req.body.requestor || req.user; + const canViewTemplates = await hasPermission(requestor, 'sendEmails'); + if (!canViewTemplates) { + return res.status(403).json({ success: false, - message: 'Invalid user ID', + message: 'You are not authorized to preview email templates.', }); } - // Update template - const updateData = { - name: trimmedName, - subject: trimmedSubject, - html_content: htmlContent.trim(), - variables: variables || [], - updated_by: userId, - }; + const { id } = req.params; + const { variables = {} } = req.body; - const template = await EmailTemplate.findByIdAndUpdate(id, updateData, { - new: true, - runValidators: true, - }) - .populate('created_by', 'firstName lastName email') - .populate('updated_by', 'firstName lastName email'); + // Service validates ID and throws error with statusCode if not found + const template = await EmailTemplateService.getTemplateById(id, { + populate: false, + }); - if (!template) { - return res.status(404).json({ + // Validate variables + const validation = TemplateRenderingService.validateVariables(template, variables); + if (!validation.isValid) { + return res.status(400).json({ success: false, - message: 'Email template not found', + message: 'Invalid variables', + errors: validation.errors, + missing: validation.missing, }); } - logger.logInfo(`Email template updated: ${template.name} by user ${userId}`); + // Render template + const rendered = TemplateRenderingService.renderTemplate(template, variables, { + sanitize: false, // Don't sanitize for preview + strict: false, + }); res.status(200).json({ success: true, - message: 'Email template updated successfully', - template, + preview: rendered, }); } catch (error) { - logger.logException(error, 'Error updating email template'); - res.status(500).json({ + logger.logException(error, 'Error previewing email template'); + const statusCode = error.statusCode || 500; + return res.status(statusCode).json({ success: false, - message: 'Error updating email template', - error: error.message, + message: error.message || 'Error previewing email template', }); } }; /** - * Delete an email template by ID. + * Validate template structure and variables. * @param {import('express').Request} req * @param {import('express').Response} res */ -const deleteEmailTemplate = async (req, res) => { +const validateTemplate = async (req, res) => { try { - // Requestor is required for permission check and audit logging - if (!req?.body?.requestor?.requestorId) { + // Permission check + if (!req?.body?.requestor?.requestorId && !req?.user?.userid) { return res.status(401).json({ success: false, message: 'Missing requestor', }); } - // Permission check - use sendEmails permission to delete templates - const canDeleteTemplate = await hasPermission(req.body.requestor, 'sendEmails'); - if (!canDeleteTemplate) { + const requestor = req.body.requestor || req.user; + const canViewTemplates = await hasPermission(requestor, 'sendEmails'); + if (!canViewTemplates) { return res.status(403).json({ success: false, - message: 'You are not authorized to delete email templates.', + message: 'You are not authorized to validate email templates.', }); } const { id } = req.params; - // Validate ObjectId - if (!id || !mongoose.Types.ObjectId.isValid(id)) { - return res.status(400).json({ - success: false, - message: 'Invalid template ID', - }); - } - - const template = await EmailTemplate.findById(id); - - if (!template) { - return res.status(404).json({ - success: false, - message: 'Email template not found', - }); - } - - await EmailTemplate.findByIdAndDelete(id); + // Service validates ID and throws error with statusCode if not found + const template = await EmailTemplateService.getTemplateById(id, { + populate: false, + }); - logger.logInfo( - `Email template deleted: ${template.name} by user ${req.body.requestor?.requestorId}`, - ); + // Validate template data + const validation = EmailTemplateService.validateTemplateData(template); res.status(200).json({ success: true, - message: 'Email template deleted successfully', + isValid: validation.isValid, + errors: validation.errors || [], }); } catch (error) { - logger.logException(error, 'Error deleting email template'); - res.status(500).json({ + logger.logException(error, 'Error validating email template'); + const statusCode = error.statusCode || 500; + return res.status(statusCode).json({ success: false, - message: 'Error deleting email template', - error: error.message, + message: error.message || 'Error validating email template', }); } }; @@ -652,4 +395,6 @@ module.exports = { createEmailTemplate, updateEmailTemplate, deleteEmailTemplate, + previewTemplate, + validateTemplate, }; diff --git a/src/jobs/announcementEmailJob.js b/src/jobs/announcementEmailJob.js deleted file mode 100644 index 544926b6a..000000000 --- a/src/jobs/announcementEmailJob.js +++ /dev/null @@ -1,190 +0,0 @@ -const { CronJob } = require('cron'); -const Email = require('../models/email'); -const emailProcessor = require('../services/announcements/emails/announcementEmailProcessor'); -const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); -const logger = require('../startup/logger'); - -class EmailAnnouncementJobProcessor { - constructor() { - this.isProcessing = false; - this.batchFetchLimit = EMAIL_JOB_CONFIG.MAX_CONCURRENT_BATCHES; - this.processingInterval = EMAIL_JOB_CONFIG.CRON_INTERVAL; - this.cronJob = null; - } - - /** - * Start the job processor - */ - start() { - if (this.cronJob) { - logger.logInfo('Email announcement job processor is already running'); - return; - } - - this.cronJob = new CronJob( - EMAIL_JOB_CONFIG.CRON_INTERVAL, - async () => { - await this.processPendingBatches(); - }, - null, - true, - EMAIL_JOB_CONFIG.TIMEZONE || 'UTC', - ); - logger.logInfo( - `Email announcement job processor started – cron=${EMAIL_JOB_CONFIG.CRON_INTERVAL}, tz=${EMAIL_JOB_CONFIG.TIMEZONE || 'UTC'}`, - ); - } - - /** - * Stop the job processor - */ - stop() { - if (this.cronJob) { - this.cronJob.stop(); - this.cronJob = null; - logger.logInfo('Email announcement job processor stopped'); - } - } - - /** - * Process pending batches - * Processes ALL queued emails regardless of individual failures - ensures maximum delivery - */ - async processPendingBatches() { - if (this.isProcessing) { - logger.logInfo('Email job processor is already running, skipping this cycle'); - return; - } - - this.isProcessing = true; - - try { - // Get batches ready for processing (QUEUED and stuck SENDING emails from previous crash/restart) - // Emails stuck in SENDING for more than 5 minutes are likely orphaned - const fiveMinutesAgo = new Date(Date.now() - 5 * 60 * 1000); - - const pendingBatches = await Email.find({ - $or: [ - { status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.QUEUED }, - { - status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING, - $or: [ - { startedAt: { $lt: fiveMinutesAgo } }, - { startedAt: { $exists: false } }, // recover ones missing a timestamp - { startedAt: null }, - ], - }, - ], - }) - .sort({ createdAt: 1 }) // FIFO order - .limit(this.batchFetchLimit); - - if (pendingBatches.length === 0) { - logger.logInfo('No pending email batches to process'); - return; - } - - logger.logInfo(`Processing ${pendingBatches.length} email batches`); - - // Check for and log stuck emails - const stuckEmails = pendingBatches.filter( - (email) => email.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING, - ); - if (stuckEmails.length > 0) { - logger.logInfo( - `Recovering ${stuckEmails.length} emails stuck in SENDING state from previous restart/crash`, - ); - } - - // Process each email - allSettled to avoid blocking on failures - const results = await Promise.allSettled( - pendingBatches.map((email) => - EmailAnnouncementJobProcessor.processBatchWithAuditing(email), - ), - ); - - const fulfilled = results.filter((r) => r.status === 'fulfilled'); - const succeeded = fulfilled.filter((r) => r.value).length; - const failed = results.length - succeeded; - - logger.logInfo( - `Completed processing cycle: ${succeeded} email batches succeeded, ${failed} failed out of ${pendingBatches.length} total`, - ); - } catch (error) { - logger.logException(error, 'Error in announcement batch processing cycle'); - // Continue processing - don't block other emails - } finally { - this.isProcessing = false; - } - } - - /** - * Process a single Email with comprehensive auditing - * Never throws - ensures other emails continue processing even if this one fails - */ - static async processBatchWithAuditing(email) { - const startTime = Date.now(); - - try { - // Process using existing emailProcessor - const finalStatus = await emailProcessor.processEmail(email._id); - - const processingTime = Date.now() - startTime; - - // Completion audit is handled in the processor based on final status - logger.logInfo( - `Processed Email ${email._id} with status ${finalStatus} in ${processingTime}ms`, - ); - - // Return true for success, false for failure - const isSuccess = - finalStatus === EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT || - finalStatus === EMAIL_JOB_CONFIG.EMAIL_STATUSES.PROCESSED; - return isSuccess; - } catch (error) { - const processingTime = Date.now() - startTime; - - // Failure audit is handled in the processor - logger.logException(error, `Failed to process Email ${email._id} after ${processingTime}ms`); - - // Return false to indicate failure but don't throw - allows other emails to continue - return false; - } - } - - /** - * Get processor status - */ - getStatus() { - return { - isRunning: !!this.cronJob, - isProcessing: this.isProcessing, - batchFetchLimit: this.batchFetchLimit, - cronInterval: this.processingInterval, - nextRun: this.cronJob ? new Date(this.cronJob.nextDate().toString()) : null, - }; - } - - /** - * Get worker status (minimal info for frontend display) - */ - getWorkerStatus() { - return { - running: !!this.cronJob, - }; - } - - /** - * Get pending batches count - */ - static async getPendingBatchesCount() { - return Email.countDocuments({ - status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.QUEUED, - }); - } -} - -// Create singleton instance -const emailAnnouncementJobProcessor = new EmailAnnouncementJobProcessor(); - -module.exports = emailAnnouncementJobProcessor; diff --git a/src/models/email.js b/src/models/email.js index bcfe73091..5d0d7e00f 100644 --- a/src/models/email.js +++ b/src/models/email.js @@ -1,9 +1,9 @@ const mongoose = require('mongoose'); -const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +const { EMAIL_CONFIG } = require('../config/emailConfig'); /** * Email (parent) model for announcement sending lifecycle. - * - Stores subject/html and status transitions (QUEUED → SENDING → SENT/PROCESSED/FAILED). + * - Stores subject/html and status transitions (PENDING → SENDING → SENT/PROCESSED/FAILED). * - References creator and tracks timing fields for auditing. */ const { Schema } = mongoose; @@ -19,8 +19,14 @@ const EmailSchema = new Schema({ }, status: { type: String, - enum: Object.values(EMAIL_JOB_CONFIG.EMAIL_STATUSES), - default: EMAIL_JOB_CONFIG.EMAIL_STATUSES.QUEUED, + enum: Object.values(EMAIL_CONFIG.EMAIL_STATUSES), + default: EMAIL_CONFIG.EMAIL_STATUSES.PENDING, + index: true, + }, + // Optional template reference for tracking which template was used + templateId: { + type: Schema.Types.ObjectId, + ref: 'EmailTemplate', index: true, }, createdBy: { @@ -49,5 +55,6 @@ EmailSchema.index({ status: 1, createdAt: 1 }); EmailSchema.index({ createdBy: 1, createdAt: -1 }); EmailSchema.index({ startedAt: 1 }); EmailSchema.index({ completedAt: 1 }); +EmailSchema.index({ templateId: 1, createdAt: -1 }); // For template usage tracking module.exports = mongoose.model('Email', EmailSchema, 'emails'); diff --git a/src/models/emailBatch.js b/src/models/emailBatch.js index e9d3cb009..4e5f8457c 100644 --- a/src/models/emailBatch.js +++ b/src/models/emailBatch.js @@ -1,5 +1,5 @@ const mongoose = require('mongoose'); -const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +const { EMAIL_CONFIG } = require('../config/emailConfig'); /** * EmailBatch (child) model representing one SMTP send to a group of recipients. @@ -33,16 +33,16 @@ const EmailBatchSchema = new Schema({ // Email type for the batch item (uses config enum) emailType: { type: String, - enum: Object.values(EMAIL_JOB_CONFIG.EMAIL_TYPES), - default: EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC, // Use BCC for multiple recipients + enum: Object.values(EMAIL_CONFIG.EMAIL_TYPES), + default: EMAIL_CONFIG.EMAIL_TYPES.BCC, // Use BCC for multiple recipients required: [true, 'Email type is required'], }, // Status tracking (for the entire batch item) - uses config enum status: { type: String, - enum: Object.values(EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES), - default: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED, + enum: Object.values(EMAIL_CONFIG.EMAIL_BATCH_STATUSES), + default: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.PENDING, index: true, required: [true, 'Status is required'], }, @@ -80,10 +80,10 @@ EmailBatchSchema.pre('save', function (next) { this.updatedAt = new Date(); // Validate status consistency with timestamps - if (this.status === EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENT && !this.sentAt) { + if (this.status === EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENT && !this.sentAt) { this.sentAt = new Date(); } - if (this.status === EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.FAILED && !this.failedAt) { + if (this.status === EMAIL_CONFIG.EMAIL_BATCH_STATUSES.FAILED && !this.failedAt) { this.failedAt = new Date(); } @@ -96,5 +96,9 @@ EmailBatchSchema.index({ status: 1, createdAt: 1 }); // For status-based queries EmailBatchSchema.index({ emailId: 1, createdAt: -1 }); // For batch history EmailBatchSchema.index({ lastAttemptedAt: 1 }); // For retry logic EmailBatchSchema.index({ attempts: 1, status: 1 }); // For retry queries +EmailBatchSchema.index({ errorCode: 1 }); // For error queries +EmailBatchSchema.index({ failedAt: -1 }); // For failed batch queries +EmailBatchSchema.index({ status: 1, failedAt: -1 }); // Compound index for failed batches +EmailBatchSchema.index({ emailId: 1, status: 1, createdAt: -1 }); // Compound index for email batches by status module.exports = mongoose.model('EmailBatch', EmailBatchSchema, 'emailBatches'); diff --git a/src/models/emailBatchAudit.js b/src/models/emailBatchAudit.js deleted file mode 100644 index 9ee57592b..000000000 --- a/src/models/emailBatchAudit.js +++ /dev/null @@ -1,77 +0,0 @@ -const mongoose = require('mongoose'); -const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); - -/** - * EmailBatchAudit model for immutable audit trail of email/batch actions. - * - Captures action, details, optional error info, metadata, and actor. - */ -const { Schema } = mongoose; - -const EmailBatchAuditSchema = new Schema({ - // Reference to the main email - emailId: { - type: Schema.Types.ObjectId, - ref: 'Email', - required: [true, 'emailId is required'], - index: true, - }, - - // Reference to specific email batch item - emailBatchId: { - type: Schema.Types.ObjectId, - ref: 'EmailBatch', - index: true, - }, - - // Action performed (uses config enum) - action: { - type: String, - enum: Object.values(EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS), - required: [true, 'Action is required'], - index: true, - }, - - // Action details - details: { - type: String, - required: [true, 'Details are required'], - }, - - // Error information (if applicable) - error: { - type: String, - }, - errorCode: { - type: String, - }, - - // Contextual metadata (flexible object for additional data) - metadata: { - type: Schema.Types.Mixed, - default: {}, - }, - - // Timestamps - timestamp: { - type: Date, - default: () => new Date(), - index: true, - required: [true, 'Timestamp is required'], - }, - - // User who triggered the action (if applicable) - triggeredBy: { - type: Schema.Types.ObjectId, - ref: 'userProfile', - }, -}); - -// Indexes for efficient querying -EmailBatchAuditSchema.index({ emailId: 1, timestamp: 1 }); -EmailBatchAuditSchema.index({ emailBatchId: 1, timestamp: 1 }); -EmailBatchAuditSchema.index({ action: 1, timestamp: 1 }); -EmailBatchAuditSchema.index({ timestamp: -1 }); -EmailBatchAuditSchema.index({ triggeredBy: 1, timestamp: -1 }); // For user audit queries -EmailBatchAuditSchema.index({ emailId: 1, action: 1 }); // For action-specific queries - -module.exports = mongoose.model('EmailBatchAudit', EmailBatchAuditSchema, 'emailBatchAudits'); diff --git a/src/models/emailTemplate.js b/src/models/emailTemplate.js index 68da27d28..c236880d6 100644 --- a/src/models/emailTemplate.js +++ b/src/models/emailTemplate.js @@ -11,7 +11,6 @@ const emailTemplateSchema = new mongoose.Schema( name: { type: String, required: true, - unique: true, trim: true, }, subject: { @@ -54,8 +53,10 @@ const emailTemplateSchema = new mongoose.Schema( }, ); +// Unique index on name (case-insensitive) +emailTemplateSchema.index({ name: 1 }, { unique: true }); + // Indexes for better search performance -emailTemplateSchema.index({ name: 1 }); emailTemplateSchema.index({ created_at: -1 }); emailTemplateSchema.index({ updated_at: -1 }); emailTemplateSchema.index({ created_by: 1 }); @@ -71,4 +72,29 @@ emailTemplateSchema.index({ emailTemplateSchema.index({ created_by: 1, created_at: -1 }); emailTemplateSchema.index({ name: 1, created_at: -1 }); +// Virtual for camelCase compatibility (for API responses) +emailTemplateSchema.virtual('htmlContent').get(function () { + return this.html_content; +}); + +emailTemplateSchema.virtual('createdBy').get(function () { + return this.created_by; +}); + +emailTemplateSchema.virtual('updatedBy').get(function () { + return this.updated_by; +}); + +emailTemplateSchema.virtual('createdAt').get(function () { + return this.created_at; +}); + +emailTemplateSchema.virtual('updatedAt').get(function () { + return this.updated_at; +}); + +// Ensure virtuals are included in JSON output +emailTemplateSchema.set('toJSON', { virtuals: true }); +emailTemplateSchema.set('toObject', { virtuals: true }); + module.exports = mongoose.model('EmailTemplate', emailTemplateSchema); diff --git a/src/routes/emailBatchRoutes.js b/src/routes/emailBatchRoutes.js deleted file mode 100644 index 3218d89d9..000000000 --- a/src/routes/emailBatchRoutes.js +++ /dev/null @@ -1,17 +0,0 @@ -const express = require('express'); - -const router = express.Router(); - -const emailBatchController = require('../controllers/emailBatchController'); - -router.get('/emails', emailBatchController.getEmails); -router.get('/emails/:emailId', emailBatchController.getEmailDetails); - -router.get('/worker-status', emailBatchController.getWorkerStatus); - -router.post('/emails/:emailId/retry', emailBatchController.retryEmail); - -router.get('/audit/email/:emailId', emailBatchController.getEmailAuditTrail); -router.get('/audit/email-batch/:emailBatchId', emailBatchController.getEmailBatchAuditTrail); - -module.exports = router; diff --git a/src/routes/emailOutboxRouter.js b/src/routes/emailOutboxRouter.js new file mode 100644 index 000000000..4fb0422c6 --- /dev/null +++ b/src/routes/emailOutboxRouter.js @@ -0,0 +1,16 @@ +const express = require('express'); + +const router = express.Router(); + +const emailOutboxController = require('../controllers/emailOutboxController'); + +// GET /api/email-outbox - Get all sent emails (outbox list) +router.get('/', emailOutboxController.getEmails); + +// POST /api/email-outbox/:emailId/retry - Retry failed email batches +router.post('/:emailId/retry', emailOutboxController.retryEmail); + +// GET /api/email-outbox/:emailId - Get email details with batches +router.get('/:emailId', emailOutboxController.getEmailDetails); + +module.exports = router; diff --git a/src/routes/emailTemplateRouter.js b/src/routes/emailTemplateRouter.js index 7656fdafa..061998f17 100644 --- a/src/routes/emailTemplateRouter.js +++ b/src/routes/emailTemplateRouter.js @@ -9,5 +9,7 @@ router.get('/email-templates/:id', emailTemplateController.getEmailTemplateById) router.post('/email-templates', emailTemplateController.createEmailTemplate); router.put('/email-templates/:id', emailTemplateController.updateEmailTemplate); router.delete('/email-templates/:id', emailTemplateController.deleteEmailTemplate); +router.post('/email-templates/:id/preview', emailTemplateController.previewTemplate); +router.post('/email-templates/:id/validate', emailTemplateController.validateTemplate); module.exports = router; diff --git a/src/server.js b/src/server.js index 571f1b0a5..31c40b17f 100644 --- a/src/server.js +++ b/src/server.js @@ -2,7 +2,6 @@ require('dotenv').config(); const http = require('http'); require('./jobs/dailyMessageEmailNotification'); -require('./jobs/announcementEmailJob').start(); // Start email announcement job processor const { app, logger } = require('./app'); const TimerWebsockets = require('./websockets').default; const MessagingWebSocket = require('./websockets/lbMessaging/messagingSocket').default; @@ -11,8 +10,6 @@ require('./cronjobs/userProfileJobs')(); require('./jobs/analyticsAggregation').scheduleDaily(); require('./cronjobs/bidWinnerJobs')(); -// Email batch system is initialized automatically when needed - const websocketRouter = require('./websockets/webSocketRouter'); const port = process.env.PORT || 4500; diff --git a/src/services/announcements/emails/announcementEmailProcessor.js b/src/services/announcements/emails/announcementEmailProcessor.js deleted file mode 100644 index 684f3da32..000000000 --- a/src/services/announcements/emails/announcementEmailProcessor.js +++ /dev/null @@ -1,424 +0,0 @@ -const mongoose = require('mongoose'); -const EmailBatch = require('../../../models/emailBatch'); -const EmailService = require('./emailService'); -const EmailBatchService = require('./emailBatchService'); -const emailAnnouncementService = require('./announcementEmailService'); -const EmailBatchAuditService = require('./emailBatchAuditService'); -const { EMAIL_JOB_CONFIG } = require('../../../config/emailJobConfig'); -const logger = require('../../../startup/logger'); - -class EmailProcessor { - /** - * Initialize processor runtime configuration. - * - Tracks currently processing parent Email IDs to avoid duplicate work. - * - Loads retry settings from EMAIL_JOB_CONFIG to coordinate with sending service. - */ - constructor() { - this.processingBatches = new Set(); - this.maxRetries = EMAIL_JOB_CONFIG.DEFAULT_MAX_RETRIES; - this.retryDelay = EMAIL_JOB_CONFIG.INITIAL_RETRY_DELAY_MS; - } - - /** - * Process a single parent Email by sending all of its queued EmailBatch items. - * - Idempotent with respect to concurrent calls (skips if already processing). - * - Recovers stuck batches from previous crashes by resetting SENDING to QUEUED. - * - Audits lifecycle events (sending, sent, failed, processed). - * @param {string|ObjectId} emailId - The ObjectId of the parent Email. - * @returns {Promise} Final email status: SENT | FAILED | PROCESSED | SENDING - * @throws {Error} When emailId is invalid or the Email is not found. - */ - async processEmail(emailId) { - if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { - throw new Error('emailId is required and must be a valid ObjectId'); - } - - if (this.processingBatches.has(emailId)) { - logger.logInfo(`Email ${emailId} is already being processed, skipping`); - return EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING; - } - - this.processingBatches.add(emailId); - - try { - const email = await EmailService.getEmailById(emailId); - if (!email) { - throw new Error(`Email not found with id: ${emailId}`); - } - - // Skip if already in final state - if ( - email.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT || - email.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED || - email.status === EMAIL_JOB_CONFIG.EMAIL_STATUSES.PROCESSED - ) { - logger.logInfo(`Email ${emailId} is already in final state: ${email.status}`); - return email.status; - } - - // If email is already SENDING (recovery from crash), skip marking as started - // Otherwise, mark as started and audit - if (email.status !== EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING) { - await EmailService.markEmailStarted(emailId); - try { - await EmailBatchAuditService.logEmailSending(email._id, { - subject: email.subject, - }); - } catch (auditErr) { - logger.logException(auditErr, 'Audit failure: EMAIL_SENDING'); - } - } else { - logger.logInfo(`Recovering Email ${emailId} from SENDING state (from crash/restart)`); - } - - // Process all EmailBatch items - await this.processEmailBatches(email); - - // Determine final status based on batch items - const finalStatus = await EmailProcessor.determineEmailStatus(email._id); - await EmailService.markEmailCompleted(emailId, finalStatus); - // Audit completion at email level - try { - if (finalStatus === EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT) { - await EmailBatchAuditService.logEmailSent(email._id); - } else if (finalStatus === EMAIL_JOB_CONFIG.EMAIL_STATUSES.PROCESSED) { - await EmailBatchAuditService.logEmailProcessed(email._id); - } else if (finalStatus === EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED) { - await EmailBatchAuditService.logEmailFailed(email._id, new Error('Processing failed')); - } - } catch (auditErr) { - logger.logException(auditErr, 'Audit failure: EMAIL completion'); - } - - logger.logInfo(`Email ${emailId} processed with status: ${finalStatus}`); - return finalStatus; - } catch (error) { - logger.logException(error, `Error processing Email ${emailId}`); - - // Mark email as failed on error - try { - await EmailService.markEmailCompleted(emailId, EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED); - // Audit failure - try { - const failedEmail = await EmailService.getEmailById(emailId); - if (failedEmail) { - await EmailBatchAuditService.logEmailFailed(failedEmail._id, error); - } - } catch (auditErr) { - logger.logException(auditErr, 'Audit failure: EMAIL_FAILED'); - } - } catch (updateError) { - logger.logException(updateError, 'Error updating Email status to failed'); - } - return EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED; - } finally { - this.processingBatches.delete(emailId); - } - } - - /** - * Process all EmailBatch child records for a given parent Email. - * - Sends batches with limited concurrency and Promise.allSettled to ensure all are attempted. - * - Resets orphaned SENDING batches to QUEUED before processing. - * @param {Object} email - The parent Email mongoose document. - * @returns {Promise} - */ - async processEmailBatches(email) { - // Get ALL batches for this email first - const allBatches = await EmailBatch.find({ - emailId: email._id, - }); - - if (allBatches.length === 0) { - logger.logInfo(`No EmailBatch items found for Email ${email._id}`); - return; - } - - // Separate batches by status - // If we're processing this Email, any SENDING EmailBatch items are considered stuck - const stuckSendingBatches = allBatches.filter( - (batch) => batch.status === EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENDING, - ); - - const queuedBatches = allBatches.filter( - (batch) => batch.status === EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED, - ); - - // Reset stuck SENDING batches to QUEUED - if (stuckSendingBatches.length > 0) { - logger.logInfo( - `Resetting ${stuckSendingBatches.length} EmailBatch items stuck in SENDING state for Email ${email._id}`, - ); - await Promise.all( - stuckSendingBatches.map(async (batch) => { - await EmailBatchService.resetEmailBatchForRetry(batch._id); - - // Audit recovery - try { - await EmailBatchAuditService.logAction( - email._id, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_BATCH_QUEUED, - 'EmailBatch item reset from SENDING state (crash/restart recovery)', - { reason: 'Recovery from stuck SENDING state', recoveryTime: new Date() }, - null, - null, - batch._id, - ); - } catch (auditErr) { - logger.logException(auditErr, 'Audit failure: EMAIL_BATCH_QUEUED (recovery)'); - } - }), - ); - - // Add reset batches to queued list for processing - queuedBatches.push(...stuckSendingBatches); - } - - if (queuedBatches.length === 0) { - logger.logInfo(`No EmailBatch items to process for Email ${email._id}`); - return; - } - - logger.logInfo(`Processing ${queuedBatches.length} EmailBatch items for Email ${email._id}`); - - // Process items with concurrency limit, but use Promise.allSettled to ensure ALL items are attempted - const concurrency = EMAIL_JOB_CONFIG.ANNOUNCEMENTS.CONCURRENCY || 3; - const results = []; - - // eslint-disable-next-line no-await-in-loop - for (let i = 0; i < queuedBatches.length; i += concurrency) { - const batch = queuedBatches.slice(i, i + concurrency); - // eslint-disable-next-line no-await-in-loop - const batchResults = await Promise.allSettled( - batch.map((item) => this.processEmailBatch(item, email)), - ); - results.push(...batchResults); - } - - // Log summary of all processing attempts - const succeeded = results.filter((r) => r.status === 'fulfilled').length; - const failed = results.filter((r) => r.status === 'rejected').length; - - logger.logInfo( - `Completed processing ${queuedBatches.length} EmailBatch items for Email ${email._id}: ${succeeded} succeeded, ${failed} failed`, - ); - } - - /** - * Send one EmailBatch item (one SMTP send for a group of recipients). - * - Marks the batch SENDING, audits, and delegates retries to the announcement service. - * - On success, marks SENT and audits delivery details; on failure after retries, marks FAILED and audits. - * @param {Object} item - The EmailBatch mongoose document. - * @param {Object} email - The parent Email mongoose document. - * @returns {Promise} - * @throws {Error} Bubbles final failure so callers can classify in allSettled results. - */ - async processEmailBatch(item, email) { - if (!item || !item._id) { - throw new Error('Invalid EmailBatch item'); - } - if (!email || !email._id) { - throw new Error('Invalid Email parent'); - } - - const recipientEmails = (item.recipients || []) - .map((r) => r?.email) - .filter((e) => e && typeof e === 'string'); - - if (recipientEmails.length === 0) { - logger.logException( - new Error('No valid recipients found'), - `EmailBatch item ${item._id} has no valid recipients`, - ); - const failedItem = await EmailBatchService.markEmailBatchFailed(item._id, { - errorCode: 'NO_RECIPIENTS', - errorMessage: 'No valid recipients found', - }); - - // Audit logging - try { - await EmailBatchAuditService.logEmailBatchFailed( - item.emailId, - item._id, - { message: failedItem.lastError, code: failedItem.errorCode }, - { - recipientCount: failedItem?.recipients?.length || 0, - emailType: failedItem?.emailType, - recipients: failedItem?.recipients?.map((r) => r.email) || [], - emailBatchId: failedItem?._id.toString(), - }, - ); - } catch (auditError) { - logger.logException(auditError, 'Audit failure: EMAIL_BATCH_FAILED'); - } - return; - } - - // Mark as SENDING using service method (single state transition before consolidated retries) - const updatedItem = await EmailBatchService.markEmailBatchSending(item._id); - - // Audit logging after successful status update - try { - await EmailBatchAuditService.logEmailBatchSending(item.emailId, item._id, { - attempt: updatedItem?.attempts || 1, - recipientCount: updatedItem?.recipients?.length || 0, - emailType: updatedItem?.emailType, - recipients: updatedItem?.recipients?.map((r) => r.email) || [], - emailBatchId: updatedItem?._id.toString(), - }); - } catch (auditError) { - logger.logException(auditError, 'Audit failure: EMAIL_BATCH_SENDING'); - } - - // Build mail options - const mailOptions = { - from: process.env.REACT_APP_EMAIL, - subject: email.subject, - html: email.htmlContent, - }; - - if (item.emailType === EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC) { - mailOptions.to = process.env.REACT_APP_EMAIL; - mailOptions.bcc = recipientEmails.join(','); - } else { - mailOptions.to = recipientEmails.join(','); - } - - // Delegate retry/backoff to the announcement service - const sendResult = await emailAnnouncementService.sendWithRetry( - mailOptions, - this.maxRetries, - this.retryDelay, - ); - - if (sendResult.success) { - await EmailBatchService.markEmailBatchSent(item._id); - try { - await EmailBatchAuditService.logEmailBatchSent( - email._id, - item._id, - { - recipientCount: recipientEmails.length, - emailType: item.emailType, - attempt: sendResult.attemptCount || updatedItem?.attempts || 1, - }, - sendResult.response, - ); - } catch (auditError) { - logger.logException(auditError, 'Audit failure: EMAIL_BATCH_SENT'); - } - logger.logInfo( - `EmailBatch item ${item._id} sent successfully to ${recipientEmails.length} recipients (attempts ${sendResult.attemptCount || updatedItem?.attempts || 1})`, - ); - return; - } - - // Final failure after retries - const finalError = sendResult.error || new Error('Failed to send email'); - const failedItem = await EmailBatchService.markEmailBatchFailed(item._id, { - errorCode: finalError.code || 'SEND_FAILED', - errorMessage: finalError.message || 'Failed to send email', - }); - - try { - await EmailBatchAuditService.logEmailBatchFailed( - item.emailId, - item._id, - { message: failedItem.lastError, code: failedItem.errorCode }, - { - recipientCount: failedItem?.recipients?.length || 0, - emailType: failedItem?.emailType, - recipients: failedItem?.recipients?.map((r) => r.email) || [], - emailBatchId: failedItem?._id.toString(), - attempts: sendResult.attemptCount || this.maxRetries, - }, - ); - } catch (auditError) { - logger.logException(auditError, 'Audit failure: EMAIL_BATCH_FAILED'); - } - - logger.logInfo( - `Permanently failed to send EmailBatch item ${item._id} to ${recipientEmails.length} recipients after ${sendResult.attemptCount || this.maxRetries} attempts`, - ); - // Throw to ensure Promise.allSettled records this item as failed - throw finalError; - } - - /** - * Sleep utility to await a given duration. - * @param {number} ms - Milliseconds to wait. - * @returns {Promise} - */ - static sleep(ms) { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); - } - - /** - * Determine the final parent Email status from child EmailBatch aggregation. - * Rules: - * - All SENT => SENT - * - All FAILED => FAILED - * - Mixed (any SENT or FAILED) => PROCESSED - * - Otherwise => SENDING (still in progress) - * @param {ObjectId} emailObjectId - Parent Email ObjectId. - * @returns {Promise} Derived status constant from EMAIL_JOB_CONFIG.EMAIL_STATUSES. - */ - static async determineEmailStatus(emailObjectId) { - const counts = await EmailBatch.aggregate([ - { $match: { emailId: emailObjectId } }, - { - $group: { - _id: '$status', - count: { $sum: 1 }, - }, - }, - ]); - - const statusMap = counts.reduce((acc, item) => { - acc[item._id] = item.count; - return acc; - }, {}); - - const queued = statusMap[EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED] || 0; - const sending = statusMap[EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENDING] || 0; - const sent = statusMap[EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENT] || 0; - const failed = statusMap[EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.FAILED] || 0; - - // All sent = SENT - if (sent > 0 && queued === 0 && sending === 0 && failed === 0) { - return EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT; - } - - // All failed = FAILED - if (failed > 0 && queued === 0 && sending === 0 && sent === 0) { - return EMAIL_JOB_CONFIG.EMAIL_STATUSES.FAILED; - } - - // Mixed results = PROCESSED - if (sent > 0 || failed > 0) { - return EMAIL_JOB_CONFIG.EMAIL_STATUSES.PROCESSED; - } - - // Still processing = keep current status - return EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING; - } - - /** - * Get lightweight processor status for diagnostics/telemetry. - * @returns {{isRunning: boolean, processingBatches: string[], maxRetries: number}} - */ - getStatus() { - return { - isRunning: true, - processingBatches: Array.from(this.processingBatches), - maxRetries: this.maxRetries, - }; - } -} - -// Create singleton instance -const emailProcessor = new EmailProcessor(); - -module.exports = emailProcessor; diff --git a/src/services/announcements/emails/emailBatchAuditService.js b/src/services/announcements/emails/emailBatchAuditService.js deleted file mode 100644 index 070fdfcef..000000000 --- a/src/services/announcements/emails/emailBatchAuditService.js +++ /dev/null @@ -1,308 +0,0 @@ -/** - * Email Batch Audit Service - * Centralized audit management for email batch operations - */ - -const mongoose = require('mongoose'); -const EmailBatchAudit = require('../../../models/emailBatchAudit'); -const { EMAIL_JOB_CONFIG } = require('../../../config/emailJobConfig'); -const logger = require('../../../startup/logger'); - -class EmailBatchAuditService { - /** - * Log an action to the email batch audit trail. - * - Validates IDs and action enum, normalizes message fields, then persists. - * @param {string|ObjectId} emailId - * @param {string} action - One of EMAIL_BATCH_AUDIT_ACTIONS.* - * @param {string} details - Human-readable details (trimmed/limited). - * @param {Object} [metadata] - * @param {Error|null} [error] - * @param {string|ObjectId|null} [triggeredBy] - * @param {string|ObjectId|null} [emailBatchId] - * @returns {Promise} Created audit document. - */ - static async logAction( - emailId, - action, - details, - metadata = {}, - error = null, - triggeredBy = null, - emailBatchId = null, - ) { - try { - // Validate emailId - if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { - throw new Error('Invalid emailId for audit log'); - } - - // Validate action is in enum - const validActions = Object.values(EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS); - if (!validActions.includes(action)) { - throw new Error(`Invalid audit action: ${action}`); - } - - // Normalize details (trim, limit length) - const normalizedDetails = - typeof details === 'string' - ? details.trim().slice(0, 1000) - : String(details || '').slice(0, 1000); - - // Normalize error message - const errorMessage = error?.message ? String(error.message).slice(0, 1000) : null; - const errorCode = error?.code ? String(error.code).slice(0, 50) : null; - - // Validate triggeredBy if provided - if (triggeredBy && !mongoose.Types.ObjectId.isValid(triggeredBy)) { - logger.logInfo( - `Invalid triggeredBy ObjectId in audit log: ${triggeredBy} - setting to null`, - ); - triggeredBy = null; - } - - // Validate emailBatchId if provided - if (emailBatchId && !mongoose.Types.ObjectId.isValid(emailBatchId)) { - logger.logInfo( - `Invalid emailBatchId ObjectId in audit log: ${emailBatchId} - setting to null`, - ); - emailBatchId = null; - } - - const audit = new EmailBatchAudit({ - emailId, - emailBatchId, - action, - details: normalizedDetails, - metadata: metadata || {}, - error: errorMessage, - errorCode, - triggeredBy, - }); - - await audit.save(); - return audit; - } catch (err) { - logger.logException(err, 'Error logging audit action'); - throw err; - } - } - - /** - * Get complete audit trail for a parent Email (no pagination). - * @param {string|ObjectId} emailId - Parent Email ObjectId. - * @returns {Promise} Sorted newest first, with basic populations. - */ - static async getEmailAuditTrail(emailId) { - // Validate emailId is ObjectId - if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { - throw new Error('emailId is required and must be a valid ObjectId'); - } - - const query = { emailId }; // Use ObjectId directly - - const auditTrail = await EmailBatchAudit.find(query) - .sort({ timestamp: -1 }) // Most recent first - .populate('triggeredBy', 'firstName lastName email') - .populate('emailBatchId', 'recipients emailType status') - .lean(); - - return auditTrail; - } - - /** - * Get audit trail for a specific EmailBatch item (no pagination). - * @param {string|ObjectId} emailBatchId - EmailBatch ObjectId. - * @returns {Promise} Sorted newest first, with basic populations. - */ - static async getEmailBatchAuditTrail(emailBatchId) { - // Validate emailBatchId is ObjectId - if (!emailBatchId || !mongoose.Types.ObjectId.isValid(emailBatchId)) { - throw new Error('emailBatchId is required and must be a valid ObjectId'); - } - - const query = { emailBatchId }; - - const auditTrail = await EmailBatchAudit.find(query) - .sort({ timestamp: -1 }) // Most recent first - .populate('triggeredBy', 'firstName lastName email') - .populate('emailId', 'subject status') - .lean(); - - return auditTrail; - } - - /** - * Log Email queued (initial creation or retry). - * @param {string|ObjectId} emailId - * @param {Object} [metadata] - * @param {string|ObjectId} [triggeredBy] - */ - static async logEmailQueued(emailId, metadata = {}, triggeredBy = null) { - return this.logAction( - emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_QUEUED, - `Email queued for processing`, - metadata, - null, - triggeredBy, - ); - } - - /** - * Log Email sending (processing start). - */ - static async logEmailSending(emailId, metadata = {}) { - return this.logAction( - emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_SENDING, - `Email processing started`, - metadata, - ); - } - - /** - * Log Email processed (processing completion). - */ - static async logEmailProcessed(emailId, metadata = {}) { - return this.logAction( - emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_PROCESSED, - `Email processing completed`, - metadata, - ); - } - - /** - * Log Email processing failure. - */ - static async logEmailFailed(emailId, error, metadata = {}) { - return this.logAction( - emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_FAILED, - `Email processing failed`, - metadata, - error, - ); - } - - /** - * Log Email sent (all batches completed successfully). - */ - static async logEmailSent(emailId, metadata = {}) { - return this.logAction( - emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_SENT, - `Email sent successfully`, - metadata, - ); - } - - /** - * Log EmailBatch item sent with essential delivery tracking details. - */ - static async logEmailBatchSent(emailId, emailBatchId, metadata = {}, gmailResponse = null) { - const enhancedMetadata = { - ...metadata, - // Include essential delivery tracking details - ...(gmailResponse - ? { - deliveryStatus: { - messageId: gmailResponse.messageId, - accepted: gmailResponse.accepted, - rejected: gmailResponse.rejected, - }, - quotaInfo: { - quotaRemaining: gmailResponse.quotaRemaining, - quotaResetTime: gmailResponse.quotaResetTime, - }, - } - : {}), - }; - - return this.logAction( - emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_BATCH_SENT, - `EmailBatch item sent successfully`, - enhancedMetadata, - null, - null, - emailBatchId, - ); - } - - /** - * Log EmailBatch item failure with optional Gmail API metadata. - */ - static async logEmailBatchFailed(emailId, emailBatchId, error, metadata = {}) { - const enhancedMetadata = { - ...metadata, - // Include essential error tracking details - ...(error?.gmailResponse - ? { - deliveryStatus: { - messageId: error.gmailResponse.messageId, - accepted: error.gmailResponse.accepted, - rejected: error.gmailResponse.rejected, - }, - quotaInfo: { - quotaRemaining: error.gmailResponse.quotaRemaining, - quotaResetTime: error.gmailResponse.quotaResetTime, - }, - errorDetails: { - errorCode: error.gmailResponse.errorCode, - errorMessage: error.gmailResponse.errorMessage, - }, - } - : {}), - }; - - return this.logAction( - emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_BATCH_FAILED, - `EmailBatch item failed to send`, - enhancedMetadata, - error, - null, - emailBatchId, - ); - } - - /** - * Log EmailBatch item queued. - * @param {string|ObjectId} emailId - * @param {string|ObjectId} emailBatchId - * @param {Object} [metadata] - * @param {string|ObjectId} [triggeredBy] - */ - static async logEmailBatchQueued(emailId, emailBatchId, metadata = {}, triggeredBy = null) { - return this.logAction( - emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_BATCH_QUEUED, - `EmailBatch item queued`, - metadata, - null, - triggeredBy, - emailBatchId, - ); - } - - /** - * Log EmailBatch item sending. - * @param {string|ObjectId} emailId - * @param {string|ObjectId} emailBatchId - * @param {Object} [metadata] - */ - static async logEmailBatchSending(emailId, emailBatchId, metadata = {}) { - return this.logAction( - emailId, - EMAIL_JOB_CONFIG.EMAIL_BATCH_AUDIT_ACTIONS.EMAIL_BATCH_SENDING, - `EmailBatch item sending`, - metadata, - null, - null, - emailBatchId, - ); - } -} - -module.exports = EmailBatchAuditService; diff --git a/src/services/announcements/emails/emailBatchService.js b/src/services/announcements/emails/emailBatchService.js index d69b05350..3343b7f22 100644 --- a/src/services/announcements/emails/emailBatchService.js +++ b/src/services/announcements/emails/emailBatchService.js @@ -4,11 +4,13 @@ */ const mongoose = require('mongoose'); -const Email = require('../../../models/email'); const EmailBatch = require('../../../models/emailBatch'); -const EmailService = require('./emailService'); -const { EMAIL_JOB_CONFIG } = require('../../../config/emailJobConfig'); -const { normalizeRecipientsToObjects } = require('../../../utilities/emailValidators'); +const Email = require('../../../models/email'); +const { EMAIL_CONFIG } = require('../../../config/emailConfig'); +const { + normalizeRecipientsToObjects, + isValidEmailAddress, +} = require('../../../utilities/emailValidators'); const logger = require('../../../startup/logger'); class EmailBatchService { @@ -26,16 +28,40 @@ class EmailBatchService { try { // emailId is now the ObjectId directly - validate it if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { - throw new Error(`Email not found with id: ${emailId}`); + const error = new Error(`Email not found with id: ${emailId}`); + error.statusCode = 404; + throw error; } - const batchSize = config.batchSize || EMAIL_JOB_CONFIG.ANNOUNCEMENTS.BATCH_SIZE; - const emailType = config.emailType || EMAIL_JOB_CONFIG.EMAIL_TYPES.BCC; + const batchSize = config.batchSize || EMAIL_CONFIG.ANNOUNCEMENTS.BATCH_SIZE; + const emailType = config.emailType || EMAIL_CONFIG.EMAIL_TYPES.BCC; // Normalize recipients to { email } const normalizedRecipients = normalizeRecipientsToObjects(recipients); if (normalizedRecipients.length === 0) { - throw new Error('At least one recipient is required'); + const error = new Error('At least one recipient is required'); + error.statusCode = 400; + throw error; + } + + // Validate recipient count limit + if (normalizedRecipients.length > EMAIL_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST) { + const error = new Error( + `A maximum of ${EMAIL_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST} recipients are allowed per request`, + ); + error.statusCode = 400; + throw error; + } + + // Validate email format for all recipients + const invalidRecipients = normalizedRecipients.filter( + (recipient) => !isValidEmailAddress(recipient.email), + ); + if (invalidRecipients.length > 0) { + const error = new Error('One or more recipient emails are invalid'); + error.statusCode = 400; + error.invalidRecipients = invalidRecipients.map((r) => r.email); + throw error; } // Chunk recipients into EmailBatch items @@ -48,14 +74,32 @@ class EmailBatchService { emailId, // emailId is now the ObjectId directly recipients: recipientChunk.map((recipient) => ({ email: recipient.email })), emailType, - status: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED, + status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.PENDING, }; emailBatchItems.push(emailBatchItem); } // Insert with session if provided for transaction support - const inserted = await EmailBatch.insertMany(emailBatchItems, { session }); + let inserted; + try { + inserted = await EmailBatch.insertMany(emailBatchItems, { session }); + } catch (dbError) { + // Handle MongoDB errors + if (dbError.name === 'ValidationError') { + const error = new Error(`Validation error: ${dbError.message}`); + error.statusCode = 400; + throw error; + } + if (dbError.code === 11000) { + const error = new Error('Duplicate key error'); + error.statusCode = 409; + throw error; + } + // Re-throw with status code for other database errors + dbError.statusCode = 500; + throw dbError; + } logger.logInfo( `Created ${emailBatchItems.length} EmailBatch items for Email ${emailId} with ${normalizedRecipients.length} total recipients`, @@ -72,16 +116,26 @@ class EmailBatchService { * Get Email with its EmailBatch items and essential metadata for UI. * @param {string|ObjectId} emailId - Parent Email ObjectId. * @returns {Promise<{email: Object, batches: Array}>} + * @throws {Error} If email not found */ static async getEmailWithBatches(emailId) { try { - const email = await EmailService.getEmailById(emailId); - if (!email) { - return null; + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + const error = new Error('Valid email ID is required'); + error.statusCode = 400; + throw error; } - // Populate createdBy if email exists - await email.populate('createdBy', 'firstName lastName email'); + // Get email with createdBy populated using lean for consistency + const email = await Email.findById(emailId) + .populate('createdBy', 'firstName lastName email') + .lean(); + + if (!email) { + const error = new Error(`Email ${emailId} not found`); + error.statusCode = 404; + throw error; + } const emailBatches = await this.getBatchesForEmail(emailId); @@ -104,7 +158,7 @@ class EmailBatchService { })); return { - email: email.toObject(), + email, batches: transformedBatches, }; } catch (error) { @@ -114,30 +168,35 @@ class EmailBatchService { } /** - * Get all Emails ordered by creation date descending. - * @returns {Promise} Array of Email objects (lean, with createdBy populated). + * Fetch EmailBatch items for a parent Email. + * @param {string|ObjectId} emailId - Parent Email ObjectId. + * @returns {Promise} Sorted ascending by createdAt. */ - static async getAllEmails() { - try { - const emails = await Email.find() - .sort({ createdAt: -1 }) - .populate('createdBy', 'firstName lastName email') - .lean(); - - return emails; - } catch (error) { - logger.logException(error, 'Error getting Emails'); + static async getBatchesForEmail(emailId) { + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + const error = new Error('Valid email ID is required'); + error.statusCode = 400; throw error; } + return EmailBatch.find({ emailId }).sort({ createdAt: 1 }); } /** - * Fetch EmailBatch items for a parent Email. + * Get PENDING EmailBatch items for a parent Email. + * Used by email processor for processing. * @param {string|ObjectId} emailId - Parent Email ObjectId. * @returns {Promise} Sorted ascending by createdAt. */ - static async getBatchesForEmail(emailId) { - return EmailBatch.find({ emailId }).sort({ createdAt: 1 }); + static async getPendingBatchesForEmail(emailId) { + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + const error = new Error('Valid email ID is required'); + error.statusCode = 400; + throw error; + } + return EmailBatch.find({ + emailId, + status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.PENDING, + }).sort({ createdAt: 1 }); } /** @@ -147,41 +206,104 @@ class EmailBatchService { return this.getBatchesForEmail(emailId); } + /** + * Get EmailBatch by ID. + * @param {string|ObjectId} batchId - EmailBatch ObjectId. + * @returns {Promise} EmailBatch document or null if not found. + */ + static async getBatchById(batchId) { + if (!batchId || !mongoose.Types.ObjectId.isValid(batchId)) { + const error = new Error('Valid email batch ID is required'); + error.statusCode = 400; + throw error; + } + return EmailBatch.findById(batchId); + } + + /** + * Get failed EmailBatch items for a parent Email. + * @param {string|ObjectId} emailId - Parent Email ObjectId. + * @returns {Promise} Array of failed EmailBatch items. + */ + static async getFailedBatchesForEmail(emailId) { + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + const error = new Error('Valid email ID is required'); + error.statusCode = 400; + throw error; + } + return EmailBatch.find({ + emailId, + status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.FAILED, + }).sort({ createdAt: 1 }); + } + /** * Reset an EmailBatch item for retry, clearing attempts and error fields. + * Uses atomic update to prevent race conditions. * @param {string|ObjectId} emailBatchId - Batch ObjectId. - * @returns {Promise} Updated document or null if not found. + * @returns {Promise} Updated document. + * @throws {Error} If batch not found */ static async resetEmailBatchForRetry(emailBatchId) { - const item = await EmailBatch.findById(emailBatchId); - if (!item) return null; - item.status = EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.QUEUED; - item.attempts = 0; - item.lastError = null; - item.lastErrorAt = null; - item.errorCode = null; - item.failedAt = null; - item.lastAttemptedAt = null; - await item.save(); - return item; + if (!emailBatchId || !mongoose.Types.ObjectId.isValid(emailBatchId)) { + const error = new Error('Valid email batch ID is required'); + error.statusCode = 400; + throw error; + } + + const now = new Date(); + const updated = await EmailBatch.findByIdAndUpdate( + emailBatchId, + { + status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.PENDING, + attempts: 0, + lastError: null, + lastErrorAt: null, + errorCode: null, + failedAt: null, + lastAttemptedAt: null, + updatedAt: now, + }, + { new: true }, + ); + + if (!updated) { + const error = new Error(`EmailBatch ${emailBatchId} not found`); + error.statusCode = 404; + throw error; + } + + return updated; } /** * Mark a batch item as SENDING, increment attempts, and set lastAttemptedAt. + * Uses atomic update with condition to prevent race conditions. * @param {string|ObjectId} emailBatchId - Batch ObjectId. * @returns {Promise} Updated batch document. + * @throws {Error} If batch not found or not in PENDING status */ static async markEmailBatchSending(emailBatchId) { const now = new Date(); - const updated = await EmailBatch.findByIdAndUpdate( - emailBatchId, + const updated = await EmailBatch.findOneAndUpdate( { - status: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENDING, + _id: emailBatchId, + status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.PENDING, + }, + { + status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENDING, $inc: { attempts: 1 }, lastAttemptedAt: now, }, { new: true }, ); + + if (!updated) { + const error = new Error(`EmailBatch ${emailBatchId} not found or not in PENDING status`); + error.statusCode = 404; + throw error; + } + return updated; } @@ -189,17 +311,31 @@ class EmailBatchService { * Mark a batch item as SENT and set sentAt timestamp. * @param {string|ObjectId} emailBatchId - Batch ObjectId. * @returns {Promise} Updated batch document. + * @throws {Error} If batch not found */ static async markEmailBatchSent(emailBatchId) { + if (!emailBatchId || !mongoose.Types.ObjectId.isValid(emailBatchId)) { + const error = new Error('Valid email batch ID is required'); + error.statusCode = 400; + throw error; + } + const now = new Date(); const updated = await EmailBatch.findByIdAndUpdate( emailBatchId, { - status: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.SENT, + status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENT, sentAt: now, }, { new: true }, ); + + if (!updated) { + const error = new Error(`EmailBatch ${emailBatchId} not found`); + error.statusCode = 404; + throw error; + } + return updated; } @@ -208,13 +344,20 @@ class EmailBatchService { * @param {string|ObjectId} emailBatchId - Batch ObjectId. * @param {{errorCode?: string, errorMessage?: string}} param1 - Error details. * @returns {Promise} Updated batch document. + * @throws {Error} If batch not found */ static async markEmailBatchFailed(emailBatchId, { errorCode, errorMessage }) { + if (!emailBatchId || !mongoose.Types.ObjectId.isValid(emailBatchId)) { + const error = new Error('Valid email batch ID is required'); + error.statusCode = 400; + throw error; + } + const now = new Date(); const updated = await EmailBatch.findByIdAndUpdate( emailBatchId, { - status: EMAIL_JOB_CONFIG.EMAIL_BATCH_STATUSES.FAILED, + status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.FAILED, failedAt: now, lastError: errorMessage?.slice(0, 500) || null, lastErrorAt: now, @@ -222,6 +365,13 @@ class EmailBatchService { }, { new: true }, ); + + if (!updated) { + const error = new Error(`EmailBatch ${emailBatchId} not found`); + error.statusCode = 404; + throw error; + } + return updated; } } diff --git a/src/services/announcements/emails/emailProcessor.js b/src/services/announcements/emails/emailProcessor.js new file mode 100644 index 000000000..c0935fd24 --- /dev/null +++ b/src/services/announcements/emails/emailProcessor.js @@ -0,0 +1,324 @@ +const mongoose = require('mongoose'); +const EmailService = require('./emailService'); +const EmailBatchService = require('./emailBatchService'); +const emailSendingService = require('./emailSendingService'); +const { EMAIL_CONFIG } = require('../../../config/emailConfig'); +const logger = require('../../../startup/logger'); + +class EmailProcessor { + /** + * Initialize processor runtime configuration. + * - Tracks currently processing parent Email IDs to avoid duplicate work. + * - Loads retry settings from EMAIL_CONFIG to coordinate with sending service. + */ + constructor() { + this.processingBatches = new Set(); + this.maxRetries = EMAIL_CONFIG.DEFAULT_MAX_RETRIES; + this.retryDelay = EMAIL_CONFIG.INITIAL_RETRY_DELAY_MS; + } + + /** + * Process a single parent Email by sending all of its pending EmailBatch items. + * - Each request is independent - processes only the email passed to it + * - Idempotent with respect to concurrent calls (skips if already processing) + * - Simple flow: PENDING → SENDING → SENT/FAILED/PROCESSED + * @param {string|ObjectId} emailId - The ObjectId of the parent Email. + * @returns {Promise} Final email status: SENT | FAILED | PROCESSED | SENDING + * @throws {Error} When emailId is invalid or the Email is not found. + */ + async processEmail(emailId) { + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + throw new Error('emailId is required and must be a valid ObjectId'); + } + + // Prevent concurrent processing of the same email + if (this.processingBatches.has(emailId)) { + logger.logInfo(`Email ${emailId} is already being processed, skipping`); + return EMAIL_CONFIG.EMAIL_STATUSES.SENDING; + } + + this.processingBatches.add(emailId); + + try { + // Get email - don't throw if not found, handle gracefully in processor + const email = await EmailService.getEmailById(emailId); + if (!email) { + throw new Error(`Email not found with id: ${emailId}`); + } + + // Skip if already in final state + if ( + email.status === EMAIL_CONFIG.EMAIL_STATUSES.SENT || + email.status === EMAIL_CONFIG.EMAIL_STATUSES.FAILED || + email.status === EMAIL_CONFIG.EMAIL_STATUSES.PROCESSED + ) { + logger.logInfo(`Email ${emailId} is already in final state: ${email.status}`); + return email.status; + } + + // Mark email as SENDING (atomic update from PENDING to SENDING) + // If already SENDING, this will fail and we'll skip processing + try { + await EmailService.markEmailStarted(emailId); + } catch (startError) { + // If marking as started fails, email is likely already being processed + const currentEmail = await EmailService.getEmailById(emailId); + if (currentEmail && currentEmail.status === EMAIL_CONFIG.EMAIL_STATUSES.SENDING) { + logger.logInfo(`Email ${emailId} is already being processed, skipping`); + this.processingBatches.delete(emailId); + return EMAIL_CONFIG.EMAIL_STATUSES.SENDING; + } + // Re-throw if it's a different error + throw startError; + } + + // Process all PENDING EmailBatch items for this email + await this.processEmailBatches(email); + + // Determine final status based on batch items + const finalStatus = await EmailProcessor.determineEmailStatus(email._id); + await EmailService.markEmailCompleted(emailId, finalStatus); + + logger.logInfo(`Email ${emailId} processed with status: ${finalStatus}`); + return finalStatus; + } catch (error) { + logger.logException(error, `Error processing Email ${emailId}`); + + // Mark email as failed on error + try { + await EmailService.markEmailCompleted(emailId, EMAIL_CONFIG.EMAIL_STATUSES.FAILED); + } catch (updateError) { + logger.logException(updateError, 'Error updating Email status to failed'); + } + return EMAIL_CONFIG.EMAIL_STATUSES.FAILED; + } finally { + this.processingBatches.delete(emailId); + } + } + + /** + * Process all PENDING EmailBatch items for a given parent Email. + * - Only processes PENDING batches (simple and straightforward) + * - Sends batches with limited concurrency + * - Each request is independent - processes only batches for this email + * @param {Object} email - The parent Email mongoose document. + * @returns {Promise} + */ + async processEmailBatches(email) { + // Get only PENDING batches for this email (service validates emailId) + const pendingBatches = await EmailBatchService.getPendingBatchesForEmail(email._id); + + if (pendingBatches.length === 0) { + logger.logInfo(`No PENDING EmailBatch items found for Email ${email._id}`); + return; + } + + logger.logInfo( + `Processing ${pendingBatches.length} PENDING EmailBatch items for Email ${email._id}`, + ); + + // Process items with concurrency limit + const concurrency = EMAIL_CONFIG.ANNOUNCEMENTS.CONCURRENCY || 3; + const results = []; + + // Process batches in chunks with concurrency control + // eslint-disable-next-line no-await-in-loop + for (let i = 0; i < pendingBatches.length; i += concurrency) { + const batch = pendingBatches.slice(i, i + concurrency); + // eslint-disable-next-line no-await-in-loop + const batchResults = await Promise.allSettled( + batch.map((item) => this.processEmailBatch(item, email)), + ); + results.push(...batchResults); + } + + // Log summary of processing + const succeeded = results.filter((r) => r.status === 'fulfilled').length; + const failed = results.filter((r) => r.status === 'rejected').length; + + logger.logInfo( + `Completed processing ${pendingBatches.length} EmailBatch items for Email ${email._id}: ${succeeded} succeeded, ${failed} failed`, + ); + } + + /** + * Send one EmailBatch item (one SMTP send for a group of recipients). + * - Marks batch as SENDING (atomic update from PENDING) + * - Sends email with retry logic + * - Marks as SENT on success or FAILED on failure + * @param {Object} item - The EmailBatch mongoose document (should be PENDING). + * @param {Object} email - The parent Email mongoose document. + * @returns {Promise} + * @throws {Error} Bubbles final failure so callers can classify in allSettled results. + */ + async processEmailBatch(item, email) { + if (!item || !item._id) { + throw new Error('Invalid EmailBatch item'); + } + if (!email || !email._id) { + throw new Error('Invalid Email parent'); + } + + const recipientEmails = (item.recipients || []) + .map((r) => r?.email) + .filter((e) => e && typeof e === 'string'); + + if (recipientEmails.length === 0) { + logger.logException( + new Error('No valid recipients found'), + `EmailBatch item ${item._id} has no valid recipients`, + ); + await EmailBatchService.markEmailBatchFailed(item._id, { + errorCode: 'NO_RECIPIENTS', + errorMessage: 'No valid recipients found', + }); + return; + } + + // Mark as SENDING (atomic update from PENDING to SENDING) + // If this fails, batch was already processed by another thread - skip it + let updatedItem; + try { + updatedItem = await EmailBatchService.markEmailBatchSending(item._id); + } catch (markError) { + // Batch was likely already processed - check status and skip if so + // Use service method for consistency (service validates batchId) + try { + const currentBatch = await EmailBatchService.getBatchById(item._id); + if (currentBatch) { + if ( + currentBatch.status === EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENT || + currentBatch.status === EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENDING + ) { + logger.logInfo( + `EmailBatch ${item._id} is already ${currentBatch.status}, skipping duplicate processing`, + ); + return; // Skip this batch + } + } + } catch (batchError) { + // If batch not found or invalid ID, log and re-throw original error + logger.logException(batchError, `Error checking EmailBatch ${item._id} status`); + } + // Re-throw if it's a different error + throw markError; + } + + // Build mail options + const mailOptions = { + from: EMAIL_CONFIG.EMAIL.SENDER, + subject: email.subject, + html: email.htmlContent, + }; + + if (item.emailType === EMAIL_CONFIG.EMAIL_TYPES.BCC) { + mailOptions.to = EMAIL_CONFIG.EMAIL.SENDER; + mailOptions.bcc = recipientEmails.join(','); + } else { + mailOptions.to = recipientEmails.join(','); + } + + // Delegate retry/backoff to the sending service + const sendResult = await emailSendingService.sendWithRetry( + mailOptions, + this.maxRetries, + this.retryDelay, + ); + + if (sendResult.success) { + await EmailBatchService.markEmailBatchSent(item._id); + logger.logInfo( + `EmailBatch item ${item._id} sent successfully to ${recipientEmails.length} recipients (attempts ${sendResult.attemptCount || updatedItem?.attempts || 1})`, + ); + return; + } + + // Final failure after retries + const finalError = sendResult.error || new Error('Failed to send email'); + // Extract error code, or use error name if code is missing, or default to 'SEND_FAILED' + const errorCode = finalError.code || finalError.name || 'SEND_FAILED'; + const errorMessage = finalError.message || 'Failed to send email'; + await EmailBatchService.markEmailBatchFailed(item._id, { + errorCode, + errorMessage, + }); + + logger.logInfo( + `Permanently failed to send EmailBatch item ${item._id} to ${recipientEmails.length} recipients after ${sendResult.attemptCount || this.maxRetries} attempts`, + ); + // Throw to ensure Promise.allSettled records this item as failed + throw finalError; + } + + /** + * Sleep utility to await a given duration. + * @param {number} ms - Milliseconds to wait. + * @returns {Promise} + */ + static sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); + } + + /** + * Determine the final parent Email status from child EmailBatch statuses. + * Rules: + * - All SENT => SENT + * - All FAILED => FAILED + * - Mixed (some SENT and some FAILED) => PROCESSED + * - Otherwise (PENDING or SENDING batches) => SENDING (still in progress) + * @param {ObjectId} emailObjectId - Parent Email ObjectId. + * @returns {Promise} Derived status constant from EMAIL_CONFIG.EMAIL_STATUSES. + */ + static async determineEmailStatus(emailObjectId) { + // Get all batches and count statuses in memory (simpler and faster for small-medium counts) + // Use service method for consistency (service validates emailId) + const batches = await EmailBatchService.getBatchesForEmail(emailObjectId); + + const statusMap = batches.reduce((acc, batch) => { + acc[batch.status] = (acc[batch.status] || 0) + 1; + return acc; + }, {}); + + const pending = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.PENDING] || 0; + const sending = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENDING] || 0; + const sent = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENT] || 0; + const failed = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.FAILED] || 0; + + // All sent = SENT + if (sent > 0 && pending === 0 && sending === 0 && failed === 0) { + return EMAIL_CONFIG.EMAIL_STATUSES.SENT; + } + + // All failed = FAILED + if (failed > 0 && pending === 0 && sending === 0 && sent === 0) { + return EMAIL_CONFIG.EMAIL_STATUSES.FAILED; + } + + // Mixed results (some sent, some failed) = PROCESSED + if (sent > 0 || failed > 0) { + return EMAIL_CONFIG.EMAIL_STATUSES.PROCESSED; + } + + // Still processing (pending or sending batches) = keep SENDING status + return EMAIL_CONFIG.EMAIL_STATUSES.SENDING; + } + + /** + * Get lightweight processor status for diagnostics/telemetry. + * @returns {{isRunning: boolean, processingBatches: string[], maxRetries: number}} + */ + getStatus() { + return { + isRunning: true, + processingBatches: Array.from(this.processingBatches), + maxRetries: this.maxRetries, + }; + } +} + +// Create singleton instance +const emailProcessor = new EmailProcessor(); + +module.exports = emailProcessor; diff --git a/src/services/announcements/emails/announcementEmailService.js b/src/services/announcements/emails/emailSendingService.js similarity index 74% rename from src/services/announcements/emails/announcementEmailService.js rename to src/services/announcements/emails/emailSendingService.js index fb9c2b594..b68689987 100644 --- a/src/services/announcements/emails/announcementEmailService.js +++ b/src/services/announcements/emails/emailSendingService.js @@ -1,5 +1,5 @@ /** - * Email Announcement Service + * Email Sending Service * Handles sending emails via Gmail API using OAuth2 authentication * Provides validation, retry logic, and comprehensive error handling */ @@ -8,7 +8,7 @@ const nodemailer = require('nodemailer'); const { google } = require('googleapis'); const logger = require('../../../startup/logger'); -class EmailAnnouncementService { +class EmailSendingService { /** * Initialize Gmail OAuth2 transport configuration and validate required env vars. * Throws during construction if configuration is incomplete to fail fast. @@ -36,6 +36,12 @@ class EmailAnnouncementService { ); this.OAuth2Client.setCredentials({ refresh_token: this.config.refreshToken }); + // OAuth token caching + this.cachedToken = null; + this.tokenExpiryTime = null; + // Tokens typically expire in 1 hour, refresh 5 minutes before expiry + this.tokenRefreshBufferMs = 5 * 60 * 1000; // 5 minutes + // Create the email transporter try { this.transporter = nodemailer.createTransport({ @@ -48,7 +54,51 @@ class EmailAnnouncementService { }, }); } catch (error) { - logger.logException(error, 'EmailAnnouncementService: Failed to create transporter'); + logger.logException(error, 'EmailSendingService: Failed to create transporter'); + throw error; + } + } + + /** + * Get OAuth access token with caching. + * Refreshes token only if expired or about to expire. + * @returns {Promise} Access token + * @throws {Error} If token refresh fails + */ + async getAccessToken() { + const now = Date.now(); + + // Check if we have a valid cached token + if ( + this.cachedToken && + this.tokenExpiryTime && + now < this.tokenExpiryTime - this.tokenRefreshBufferMs + ) { + return this.cachedToken; + } + + // Token expired or doesn't exist, refresh it + try { + const accessTokenResp = await this.OAuth2Client.getAccessToken(); + let token; + + if (accessTokenResp && typeof accessTokenResp === 'object' && accessTokenResp.token) { + token = accessTokenResp.token; + } else if (typeof accessTokenResp === 'string') { + token = accessTokenResp; + } else { + throw new Error('Invalid access token response format'); + } + + // Cache the token with expiry time (tokens typically last 1 hour) + this.cachedToken = token; + this.tokenExpiryTime = now + 60 * 60 * 1000; // 1 hour from now + + return token; + } catch (error) { + // Clear cache on error + this.cachedToken = null; + this.tokenExpiryTime = null; throw error; } } @@ -65,50 +115,43 @@ class EmailAnnouncementService { // Validation if (!mailOptions) { const error = new Error('INVALID_MAIL_OPTIONS: mailOptions is required'); - logger.logException(error, 'EmailAnnouncementService.sendEmail validation failed'); + logger.logException(error, 'EmailSendingService.sendEmail validation failed'); return { success: false, error }; } if (!mailOptions.to && !mailOptions.bcc) { const error = new Error('INVALID_RECIPIENTS: At least one recipient (to or bcc) is required'); - logger.logException(error, 'EmailAnnouncementService.sendEmail validation failed'); + logger.logException(error, 'EmailSendingService.sendEmail validation failed'); return { success: false, error }; } // Validate subject and htmlContent if (!mailOptions.subject || mailOptions.subject.trim() === '') { const error = new Error('INVALID_SUBJECT: Subject is required and cannot be empty'); - logger.logException(error, 'EmailAnnouncementService.sendEmail validation failed'); + logger.logException(error, 'EmailSendingService.sendEmail validation failed'); return { success: false, error }; } if (!this.config.email || !this.config.clientId || !this.config.clientSecret) { const error = new Error('INVALID_CONFIG: Email configuration is incomplete'); - logger.logException(error, 'EmailAnnouncementService.sendEmail configuration check failed'); + logger.logException(error, 'EmailSendingService.sendEmail configuration check failed'); return { success: false, error }; } try { - // Get access token with proper error handling + // Get access token with caching let token; try { - const accessTokenResp = await this.OAuth2Client.getAccessToken(); - if (accessTokenResp && typeof accessTokenResp === 'object' && accessTokenResp.token) { - token = accessTokenResp.token; - } else if (typeof accessTokenResp === 'string') { - token = accessTokenResp; - } else { - throw new Error('Invalid access token response format'); - } + token = await this.getAccessToken(); } catch (tokenError) { const error = new Error(`OAUTH_TOKEN_ERROR: ${tokenError.message}`); - logger.logException(error, 'EmailAnnouncementService.sendEmail OAuth token refresh failed'); + logger.logException(error, 'EmailSendingService.sendEmail OAuth token refresh failed'); return { success: false, error }; } if (!token) { const error = new Error('NO_OAUTH_ACCESS_TOKEN: Failed to obtain access token'); - logger.logException(error, 'EmailAnnouncementService.sendEmail OAuth failed'); + logger.logException(error, 'EmailSendingService.sendEmail OAuth failed'); return { success: false, error }; } @@ -154,13 +197,13 @@ class EmailAnnouncementService { // Validation if (!mailOptions) { const error = new Error('INVALID_MAIL_OPTIONS: mailOptions is required'); - logger.logException(error, 'EmailAnnouncementService.sendWithRetry validation failed'); + logger.logException(error, 'EmailSendingService.sendWithRetry validation failed'); return { success: false, error, attemptCount: 0 }; } if (!Number.isInteger(retries) || retries < 1) { const error = new Error('INVALID_RETRIES: retries must be a positive integer'); - logger.logException(error, 'EmailAnnouncementService.sendWithRetry validation failed'); + logger.logException(error, 'EmailSendingService.sendWithRetry validation failed'); return { success: false, error, attemptCount: 0 }; } @@ -208,7 +251,7 @@ class EmailAnnouncementService { // Exponential backoff before retry (2^n: 1x, 2x, 4x, 8x, ...) if (attempt < retries) { const delay = initialDelayMs * 2 ** (attempt - 1); - await EmailAnnouncementService.sleep(delay); + await EmailSendingService.sleep(delay); } } /* eslint-enable no-await-in-loop */ @@ -233,6 +276,6 @@ class EmailAnnouncementService { } // Create singleton instance -const emailAnnouncementService = new EmailAnnouncementService(); +const emailSendingService = new EmailSendingService(); -module.exports = emailAnnouncementService; +module.exports = emailSendingService; diff --git a/src/services/announcements/emails/emailService.js b/src/services/announcements/emails/emailService.js index bcea2c78a..a57b1072f 100644 --- a/src/services/announcements/emails/emailService.js +++ b/src/services/announcements/emails/emailService.js @@ -1,27 +1,97 @@ const mongoose = require('mongoose'); const Email = require('../../../models/email'); -const { EMAIL_JOB_CONFIG } = require('../../../config/emailJobConfig'); +const { EMAIL_CONFIG } = require('../../../config/emailConfig'); +const { ensureHtmlWithinLimit } = require('../../../utilities/emailValidators'); class EmailService { /** * Create a parent Email document for announcements. - * Trims large text fields and supports optional transaction sessions. - * @param {{subject: string, htmlContent: string, createdBy: string|ObjectId}} param0 + * Validates and trims large text fields and supports optional transaction sessions. + * @param {{subject: string, htmlContent: string, createdBy: string|ObjectId, templateId?: string|ObjectId}} param0 * @param {import('mongoose').ClientSession|null} session * @returns {Promise} Created Email document. + * @throws {Error} If validation fails */ - static async createEmail({ subject, htmlContent, createdBy }, session = null) { - const normalizedSubject = typeof subject === 'string' ? subject.trim() : subject; - const normalizedHtml = typeof htmlContent === 'string' ? htmlContent.trim() : htmlContent; + static async createEmail({ subject, htmlContent, createdBy, templateId }, session = null) { + // Validate required fields + if (!subject || typeof subject !== 'string' || !subject.trim()) { + const error = new Error('Subject is required'); + error.statusCode = 400; + throw error; + } + + if (!htmlContent || typeof htmlContent !== 'string' || !htmlContent.trim()) { + const error = new Error('HTML content is required'); + error.statusCode = 400; + throw error; + } + + if (!createdBy || !mongoose.Types.ObjectId.isValid(createdBy)) { + const error = new Error('Valid createdBy is required'); + error.statusCode = 400; + throw error; + } + + // Validate subject length + const trimmedSubject = subject.trim(); + if (trimmedSubject.length > EMAIL_CONFIG.LIMITS.SUBJECT_MAX_LENGTH) { + const error = new Error( + `Subject cannot exceed ${EMAIL_CONFIG.LIMITS.SUBJECT_MAX_LENGTH} characters`, + ); + error.statusCode = 400; + throw error; + } + + // Validate HTML content size + if (!ensureHtmlWithinLimit(htmlContent)) { + const error = new Error( + `HTML content exceeds ${EMAIL_CONFIG.LIMITS.MAX_HTML_BYTES / (1024 * 1024)}MB limit`, + ); + error.statusCode = 413; + throw error; + } - const email = new Email({ - subject: normalizedSubject, + // Validate templateId if provided + if (templateId && !mongoose.Types.ObjectId.isValid(templateId)) { + const error = new Error('Invalid templateId'); + error.statusCode = 400; + throw error; + } + + const normalizedHtml = htmlContent.trim(); + + const emailData = { + subject: trimmedSubject, htmlContent: normalizedHtml, createdBy, - }); + }; + + // Add template reference if provided + if (templateId) { + emailData.templateId = templateId; + } + + const email = new Email(emailData); // Save with session if provided for transaction support - await email.save({ session }); + try { + await email.save({ session }); + } catch (dbError) { + // Handle MongoDB errors + if (dbError.name === 'ValidationError') { + const error = new Error(`Validation error: ${dbError.message}`); + error.statusCode = 400; + throw error; + } + if (dbError.code === 11000) { + const error = new Error('Duplicate key error'); + error.statusCode = 409; + throw error; + } + // Re-throw with status code for other database errors + dbError.statusCode = 500; + throw dbError; + } return email; } @@ -30,50 +100,100 @@ class EmailService { * Fetch a parent Email by ObjectId. * @param {string|ObjectId} id * @param {import('mongoose').ClientSession|null} session + * @param {boolean} throwIfNotFound - If true, throw error with statusCode 404 if not found. Default: false (returns null). + * @param {boolean} populateCreatedBy - If true, populate createdBy field. Default: false. * @returns {Promise} + * @throws {Error} If throwIfNotFound is true and email is not found */ - static async getEmailById(id, session = null) { - if (!id || !mongoose.Types.ObjectId.isValid(id)) return null; - return Email.findById(id).session(session); + static async getEmailById( + id, + session = null, + throwIfNotFound = false, + populateCreatedBy = false, + ) { + if (!id || !mongoose.Types.ObjectId.isValid(id)) { + if (throwIfNotFound) { + const error = new Error('Valid email ID is required'); + error.statusCode = 400; + throw error; + } + return null; + } + let query = Email.findById(id); + if (session) { + query = query.session(session); + } + if (populateCreatedBy) { + query = query.populate('createdBy', 'firstName lastName email'); + } + const email = await query; + if (!email && throwIfNotFound) { + const error = new Error(`Email ${id} not found`); + error.statusCode = 404; + throw error; + } + return email; } /** * Update Email status with validation against configured enum. * @param {string|ObjectId} emailId - * @param {string} status - One of EMAIL_JOB_CONFIG.EMAIL_STATUSES.* + * @param {string} status - One of EMAIL_CONFIG.EMAIL_STATUSES.* * @returns {Promise} Updated Email document. + * @throws {Error} If email not found or invalid status */ static async updateEmailStatus(emailId, status) { if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { - throw new Error('Valid email ID is required'); + const error = new Error('Valid email ID is required'); + error.statusCode = 400; + throw error; } - if (!Object.values(EMAIL_JOB_CONFIG.EMAIL_STATUSES).includes(status)) { - throw new Error('Invalid email status'); + if (!Object.values(EMAIL_CONFIG.EMAIL_STATUSES).includes(status)) { + const error = new Error('Invalid email status'); + error.statusCode = 400; + throw error; } const email = await Email.findByIdAndUpdate( emailId, { status, updatedAt: new Date() }, { new: true }, ); + if (!email) { + const error = new Error(`Email ${emailId} not found`); + error.statusCode = 404; + throw error; + } return email; } /** * Mark Email as SENDING and set startedAt. + * Uses atomic update with condition to prevent race conditions. * @param {string|ObjectId} emailId * @returns {Promise} Updated Email document. + * @throws {Error} If email not found or not in PENDING status */ static async markEmailStarted(emailId) { const now = new Date(); - const email = await Email.findByIdAndUpdate( - emailId, + const email = await Email.findOneAndUpdate( + { + _id: emailId, + status: EMAIL_CONFIG.EMAIL_STATUSES.PENDING, + }, { - status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENDING, + status: EMAIL_CONFIG.EMAIL_STATUSES.SENDING, startedAt: now, updatedAt: now, }, { new: true }, ); + + if (!email) { + const error = new Error(`Email ${emailId} not found or not in PENDING status`); + error.statusCode = 404; + throw error; + } + return email; } @@ -83,12 +203,13 @@ class EmailService { * @param {string|ObjectId} emailId * @param {string} finalStatus * @returns {Promise} Updated Email document. + * @throws {Error} If email not found */ static async markEmailCompleted(emailId, finalStatus) { const now = new Date(); - const statusToSet = Object.values(EMAIL_JOB_CONFIG.EMAIL_STATUSES).includes(finalStatus) + const statusToSet = Object.values(EMAIL_CONFIG.EMAIL_STATUSES).includes(finalStatus) ? finalStatus - : EMAIL_JOB_CONFIG.EMAIL_STATUSES.SENT; + : EMAIL_CONFIG.EMAIL_STATUSES.SENT; const email = await Email.findByIdAndUpdate( emailId, @@ -99,28 +220,52 @@ class EmailService { }, { new: true }, ); + if (!email) { + const error = new Error(`Email ${emailId} not found`); + error.statusCode = 404; + throw error; + } return email; } /** - * Mark an Email as QUEUED for retry and clear timing fields. + * Mark an Email as PENDING for retry and clear timing fields. * @param {string|ObjectId} emailId * @returns {Promise} Updated Email document. + * @throws {Error} If email not found */ - static async markEmailQueued(emailId) { + static async markEmailPending(emailId) { const now = new Date(); const email = await Email.findByIdAndUpdate( emailId, { - status: EMAIL_JOB_CONFIG.EMAIL_STATUSES.QUEUED, + status: EMAIL_CONFIG.EMAIL_STATUSES.PENDING, startedAt: null, completedAt: null, updatedAt: now, }, { new: true }, ); + if (!email) { + const error = new Error(`Email ${emailId} not found`); + error.statusCode = 404; + throw error; + } return email; } + + /** + * Get all Emails ordered by creation date descending. + * @returns {Promise} Array of Email objects (lean, with createdBy populated). + */ + static async getAllEmails() { + const emails = await Email.find() + .sort({ createdAt: -1 }) + .populate('createdBy', 'firstName lastName email') + .lean(); + + return emails; + } } module.exports = EmailService; diff --git a/src/services/announcements/emails/emailTemplateService.js b/src/services/announcements/emails/emailTemplateService.js new file mode 100644 index 000000000..5d3e56ff3 --- /dev/null +++ b/src/services/announcements/emails/emailTemplateService.js @@ -0,0 +1,499 @@ +/** + * Email Template Service - Manages EmailTemplate operations + * Provides business logic for template CRUD operations, validation, and rendering + */ + +const mongoose = require('mongoose'); +const EmailTemplate = require('../../../models/emailTemplate'); +const { EMAIL_CONFIG } = require('../../../config/emailConfig'); +const { ensureHtmlWithinLimit } = require('../../../utilities/emailValidators'); +const logger = require('../../../startup/logger'); + +class EmailTemplateService { + /** + * Validate template variables. + * - Ensures non-empty unique names and validates allowed types. + * @param {Array<{name: string, type?: 'text'|'url'|'number'|'textarea'|'image'>} | undefined} variables + * @returns {{isValid: boolean, errors?: string[]}} + */ + static validateTemplateVariables(variables) { + if (!variables || !Array.isArray(variables)) { + return { isValid: true, errors: [] }; + } + + const errors = []; + const variableNames = new Set(); + + variables.forEach((variable, index) => { + if (!variable.name || typeof variable.name !== 'string' || !variable.name.trim()) { + errors.push(`Variable ${index + 1}: name is required and must be a non-empty string`); + } else { + const varName = variable.name.trim(); + // Validate variable name format (alphanumeric and underscore only) + if (!/^[a-zA-Z0-9_]+$/.test(varName)) { + errors.push( + `Variable ${index + 1}: name must contain only alphanumeric characters and underscores`, + ); + } + // Check for duplicates (case-insensitive) + if (variableNames.has(varName.toLowerCase())) { + errors.push(`Variable ${index + 1}: duplicate variable name '${varName}'`); + } + variableNames.add(varName.toLowerCase()); + } + + if ( + variable.type && + !['text', 'url', 'number', 'textarea', 'image'].includes(variable.type) + ) { + errors.push( + `Variable ${index + 1}: type must be one of: text, url, number, textarea, image`, + ); + } + }); + + return { + isValid: errors.length === 0, + errors, + }; + } + + /** + * Validate template content (HTML and subject) against defined variables. + * - Flags undefined placeholders and unused defined variables. + * @param {Array<{name: string}>} templateVariables + * @param {string} htmlContent + * @param {string} subject + * @returns {{isValid: boolean, errors: string[]}} + */ + static validateTemplateVariableUsage(templateVariables, htmlContent, subject) { + const errors = []; + + if (!templateVariables || templateVariables.length === 0) { + return { isValid: true, errors: [] }; + } + + // Extract variable placeholders from content (format: {{variableName}}) + const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; + const usedVariables = new Set(); + const foundPlaceholders = []; + + // Check HTML content + if (htmlContent) { + let match = variablePlaceholderRegex.exec(htmlContent); + while (match !== null) { + const varName = match[1]; + foundPlaceholders.push(varName); + usedVariables.add(varName); + match = variablePlaceholderRegex.exec(htmlContent); + } + } + + // Reset regex for subject + variablePlaceholderRegex.lastIndex = 0; + + // Check subject + if (subject) { + let match = variablePlaceholderRegex.exec(subject); + while (match !== null) { + const varName = match[1]; + foundPlaceholders.push(varName); + usedVariables.add(varName); + match = variablePlaceholderRegex.exec(subject); + } + } + + // Check for undefined variable placeholders in content + const definedVariableNames = templateVariables.map((v) => v.name); + foundPlaceholders.forEach((placeholder) => { + if (!definedVariableNames.includes(placeholder)) { + errors.push( + `Variable placeholder '{{${placeholder}}}' is used in content but not defined in template variables`, + ); + } + }); + + // Check for defined variables that are not used in content (treated as errors) + templateVariables.forEach((variable) => { + if (!usedVariables.has(variable.name)) { + errors.push(`Variable '{{${variable.name}}}' is defined but not used in template content`); + } + }); + + return { + isValid: errors.length === 0, + errors, + }; + } + + /** + * Validate template data (name, subject, HTML, variables). + * @param {Object} templateData - Template data to validate + * @returns {{isValid: boolean, errors: string[]}} + */ + static validateTemplateData(templateData) { + const errors = []; + const { name, subject, html_content: htmlContent, variables } = templateData; + + // Validate name + if (!name || typeof name !== 'string' || !name.trim()) { + errors.push('Template name is required'); + } else { + const trimmedName = name.trim(); + if (trimmedName.length > EMAIL_CONFIG.LIMITS.TEMPLATE_NAME_MAX_LENGTH) { + errors.push( + `Template name cannot exceed ${EMAIL_CONFIG.LIMITS.TEMPLATE_NAME_MAX_LENGTH} characters`, + ); + } + } + + // Validate subject + if (!subject || typeof subject !== 'string' || !subject.trim()) { + errors.push('Template subject is required'); + } else { + const trimmedSubject = subject.trim(); + if (trimmedSubject.length > EMAIL_CONFIG.LIMITS.SUBJECT_MAX_LENGTH) { + errors.push(`Subject cannot exceed ${EMAIL_CONFIG.LIMITS.SUBJECT_MAX_LENGTH} characters`); + } + } + + // Validate HTML content + if (!htmlContent || typeof htmlContent !== 'string' || !htmlContent.trim()) { + errors.push('Template HTML content is required'); + } else if (!ensureHtmlWithinLimit(htmlContent)) { + errors.push( + `HTML content exceeds ${EMAIL_CONFIG.LIMITS.MAX_HTML_BYTES / (1024 * 1024)}MB limit`, + ); + } + + // Validate variables + if (variables && variables.length > 0) { + const variableValidation = this.validateTemplateVariables(variables); + if (!variableValidation.isValid) { + errors.push(...variableValidation.errors); + } + + // Validate variable usage in content + const variableUsageValidation = this.validateTemplateVariableUsage( + variables, + htmlContent, + subject, + ); + if (!variableUsageValidation.isValid) { + errors.push(...variableUsageValidation.errors); + } + } else { + // If no variables are defined, check for any variable placeholders in content + const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; + const foundInHtml = variablePlaceholderRegex.test(htmlContent); + variablePlaceholderRegex.lastIndex = 0; + const foundInSubject = variablePlaceholderRegex.test(subject); + + if (foundInHtml || foundInSubject) { + errors.push( + 'Template content contains variable placeholders ({{variableName}}) but no variables are defined. Please define variables or remove placeholders from content.', + ); + } + } + + return { + isValid: errors.length === 0, + errors, + }; + } + + /** + * Create a new email template. + * @param {Object} templateData - Template data + * @param {string|ObjectId} userId - User ID creating the template + * @returns {Promise} Created template + * @throws {Error} If validation fails or template already exists + */ + static async createTemplate(templateData, userId) { + const { name, subject, html_content: htmlContent, variables } = templateData; + + // Validate template data + const validation = this.validateTemplateData(templateData); + if (!validation.isValid) { + const error = new Error('Invalid template data'); + error.errors = validation.errors; + error.statusCode = 400; + throw error; + } + + // Validate userId + if (userId && !mongoose.Types.ObjectId.isValid(userId)) { + const error = new Error('Invalid user ID'); + error.statusCode = 400; + throw error; + } + + const trimmedName = name.trim(); + const trimmedSubject = subject.trim(); + + // Check if template with the same name already exists (case-insensitive) + const existingTemplate = await EmailTemplate.findOne({ + name: { $regex: new RegExp(`^${trimmedName}$`, 'i') }, + }); + + if (existingTemplate) { + const error = new Error('Email template with this name already exists'); + error.statusCode = 409; + throw error; + } + + // Create new email template + const template = new EmailTemplate({ + name: trimmedName, + subject: trimmedSubject, + html_content: htmlContent.trim(), + variables: variables || [], + created_by: userId, + updated_by: userId, + }); + try { + await template.save(); + } catch (dbError) { + // Handle MongoDB errors + if (dbError.name === 'ValidationError') { + const error = new Error(`Validation error: ${dbError.message}`); + error.statusCode = 400; + throw error; + } + if (dbError.code === 11000) { + const error = new Error('Email template with this name already exists'); + error.statusCode = 409; + throw error; + } + // Re-throw with status code for other database errors + dbError.statusCode = 500; + throw dbError; + } + + // Populate created_by and updated_by fields + await template.populate('created_by', 'firstName lastName email'); + await template.populate('updated_by', 'firstName lastName email'); + + logger.logInfo(`Email template created: ${template.name} by user ${userId}`); + + return template; + } + + /** + * Get template by ID. + * @param {string|ObjectId} id - Template ID + * @param {Object} options - Query options (populate) + * @returns {Promise} Template + * @throws {Error} If template not found or invalid ID + */ + static async getTemplateById(id, options = {}) { + if (!id || !mongoose.Types.ObjectId.isValid(id)) { + const error = new Error('Invalid template ID'); + error.statusCode = 400; + throw error; + } + + const { populate = true } = options; + + let template = EmailTemplate.findById(id); + + if (populate) { + template = template + .populate('created_by', 'firstName lastName email') + .populate('updated_by', 'firstName lastName email'); + } + + const result = await template; + if (!result) { + const error = new Error('Email template not found'); + error.statusCode = 404; + throw error; + } + + return result; + } + + /** + * Get all templates with optional filtering and sorting. + * @param {Object} query - MongoDB query + * @param {Object} options - Query options (sort, projection, populate) + * @returns {Promise} Array of templates + */ + static async getAllTemplates(query = {}, options = {}) { + const { sort = { created_at: -1 }, projection = null, populate = true } = options; + + let queryBuilder = EmailTemplate.find(query); + + if (projection) { + queryBuilder = queryBuilder.select(projection); + } + + if (sort) { + queryBuilder = queryBuilder.sort(sort); + } + + if (populate) { + queryBuilder = queryBuilder + .populate('created_by', 'firstName lastName') + .populate('updated_by', 'firstName lastName'); + } + + return queryBuilder.lean(); + } + + /** + * Update an existing template. + * @param {string|ObjectId} id - Template ID + * @param {Object} templateData - Updated template data + * @param {string|ObjectId} userId - User ID updating the template + * @returns {Promise} Updated template + * @throws {Error} If validation fails or template not found + */ + static async updateTemplate(id, templateData, userId) { + if (!id || !mongoose.Types.ObjectId.isValid(id)) { + const error = new Error('Invalid template ID'); + error.statusCode = 400; + throw error; + } + + // Validate template data + const validation = this.validateTemplateData(templateData); + if (!validation.isValid) { + const error = new Error('Invalid template data'); + error.errors = validation.errors; + error.statusCode = 400; + throw error; + } + + // Validate userId + if (userId && !mongoose.Types.ObjectId.isValid(userId)) { + const error = new Error('Invalid user ID'); + error.statusCode = 400; + throw error; + } + + // Get current template + const currentTemplate = await EmailTemplate.findById(id); + + if (!currentTemplate) { + const error = new Error('Email template not found'); + error.statusCode = 404; + throw error; + } + + const { name, subject, html_content: htmlContent, variables } = templateData; + const trimmedName = name.trim(); + const trimmedSubject = subject.trim(); + + // Only check for duplicate names if the name is actually changing (case-insensitive) + if (currentTemplate.name.toLowerCase() !== trimmedName.toLowerCase()) { + const existingTemplate = await EmailTemplate.findOne({ + name: { $regex: new RegExp(`^${trimmedName}$`, 'i') }, + _id: { $ne: id }, + }); + + if (existingTemplate) { + const error = new Error('Another email template with this name already exists'); + error.statusCode = 409; + throw error; + } + } + + // Update template + const updateData = { + name: trimmedName, + subject: trimmedSubject, + html_content: htmlContent.trim(), + variables: variables || [], + updated_by: userId, + }; + + let template; + try { + template = await EmailTemplate.findByIdAndUpdate(id, updateData, { + new: true, + runValidators: true, + }) + .populate('created_by', 'firstName lastName email') + .populate('updated_by', 'firstName lastName email'); + } catch (dbError) { + // Handle MongoDB errors + if (dbError.name === 'ValidationError') { + const error = new Error(`Validation error: ${dbError.message}`); + error.statusCode = 400; + throw error; + } + if (dbError.code === 11000) { + const error = new Error('Another email template with this name already exists'); + error.statusCode = 409; + throw error; + } + // Re-throw with status code for other database errors + dbError.statusCode = 500; + throw dbError; + } + + if (!template) { + const error = new Error('Email template not found'); + error.statusCode = 404; + throw error; + } + + logger.logInfo(`Email template updated: ${template.name} by user ${userId}`); + + return template; + } + + /** + * Delete a template (hard delete). + * @param {string|ObjectId} id - Template ID + * @param {string|ObjectId} userId - User ID deleting the template + * @returns {Promise} Deleted template + * @throws {Error} If template not found + */ + static async deleteTemplate(id, userId) { + if (!id || !mongoose.Types.ObjectId.isValid(id)) { + const error = new Error('Invalid template ID'); + error.statusCode = 400; + throw error; + } + + const template = await EmailTemplate.findById(id); + + if (!template) { + const error = new Error('Email template not found'); + error.statusCode = 404; + throw error; + } + + const templateName = template.name; + + // Hard delete + await EmailTemplate.findByIdAndDelete(id); + + logger.logInfo(`Email template deleted: ${templateName} by user ${userId}`); + + return template; + } + + /** + * Check if template name exists (case-insensitive). + * @param {string} name - Template name + * @param {string|ObjectId} excludeId - Template ID to exclude from check + * @returns {Promise} True if name exists + */ + static async templateNameExists(name, excludeId = null) { + const query = { + name: { $regex: new RegExp(`^${name}$`, 'i') }, + }; + + if (excludeId) { + query._id = { $ne: excludeId }; + } + + const existing = await EmailTemplate.findOne(query); + return !!existing; + } +} + +module.exports = EmailTemplateService; diff --git a/src/services/announcements/emails/templateRenderingService.js b/src/services/announcements/emails/templateRenderingService.js new file mode 100644 index 000000000..d81c13cd7 --- /dev/null +++ b/src/services/announcements/emails/templateRenderingService.js @@ -0,0 +1,243 @@ +/** + * Template Rendering Service - Handles template rendering and variable substitution + * Provides server-side template rendering with proper sanitization + */ + +const sanitizeHtmlLib = require('sanitize-html'); + +class TemplateRenderingService { + /** + * Render template with variable values. + * Replaces {{variableName}} placeholders with actual values. + * @param {Object} template - Template object with subject and html_content + * @param {Object} variables - Object mapping variable names to values + * @param {Object} options - Rendering options (sanitize, strict) + * @returns {{subject: string, htmlContent: string}} Rendered template + */ + static renderTemplate(template, variables = {}, options = {}) { + const { sanitize = true, strict = false } = options; + + if (!template) { + const error = new Error('Template is required'); + error.statusCode = 400; + throw error; + } + + let subject = template.subject || ''; + let htmlContent = template.html_content || template.htmlContent || ''; + + // Get template variables + const templateVariables = template.variables || []; + + // Replace variables in subject and HTML + templateVariables.forEach((variable) => { + if (!variable || !variable.name) return; + + const varName = variable.name; + const value = variables[varName]; + + // In strict mode, throw error if variable is missing + if (strict && value === undefined) { + const error = new Error(`Missing required variable: ${varName}`); + error.statusCode = 400; + throw error; + } + + // Skip if value is not provided + if (value === undefined || value === null) { + return; + } + + // Handle image variables + let processedValue = value; + if (variable.type === 'image') { + // Use extracted image if available + const extractedKey = `${varName}_extracted`; + if (variables[extractedKey]) { + processedValue = variables[extractedKey]; + } else if (typeof value === 'string') { + // Try to extract image URL from value + const imageMatch = + value.match(/src=["']([^"']+)["']/i) || value.match(/https?:\/\/[^\s]+/i); + if (imageMatch) { + processedValue = imageMatch[1] || imageMatch[0]; + } + } + } + + // Replace all occurrences of {{variableName}} + const regex = new RegExp(`\\{\\{${varName}\\}\\}`, 'g'); + subject = subject.replace(regex, String(processedValue)); + htmlContent = htmlContent.replace(regex, String(processedValue)); + }); + + // Sanitize HTML if requested + if (sanitize) { + htmlContent = this.sanitizeHtml(htmlContent); + } + + return { + subject: subject.trim(), + htmlContent: htmlContent.trim(), + }; + } + + /** + * Validate that all required variables are provided. + * @param {Object} template - Template object + * @param {Object} variables - Variable values + * @returns {{isValid: boolean, errors: string[], missing: string[]}} Validation result + */ + static validateVariables(template, variables = {}) { + const errors = []; + const missing = []; + const templateVariables = template.variables || []; + + // Check for missing variables + templateVariables.forEach((variable) => { + if (!variable || !variable.name) return; + + const varName = variable.name; + if ( + !(varName in variables) || + variables[varName] === undefined || + variables[varName] === null + ) { + missing.push(varName); + errors.push(`Missing required variable: ${varName}`); + } + }); + + // Check for unused variables + const templateVariableNames = new Set(templateVariables.map((v) => v.name)); + Object.keys(variables).forEach((varName) => { + if (!templateVariableNames.has(varName) && !varName.endsWith('_extracted')) { + errors.push(`Unknown variable: ${varName}`); + } + }); + + return { + isValid: errors.length === 0, + errors, + missing, + }; + } + + /** + * Check if template has unreplaced variables. + * @param {string} content - Content to check (subject or HTML) + * @returns {string[]} Array of unreplaced variable names + */ + static getUnreplacedVariables(content) { + if (!content || typeof content !== 'string') { + return []; + } + + const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; + const unreplaced = []; + let match = variablePlaceholderRegex.exec(content); + + while (match !== null) { + const varName = match[1]; + if (!unreplaced.includes(varName)) { + unreplaced.push(varName); + } + match = variablePlaceholderRegex.exec(content); + } + + return unreplaced; + } + + /** + * Sanitize HTML content to prevent XSS attacks. + * @param {string} html - HTML content to sanitize + * @param {Object} options - Sanitization options + * @returns {string} Sanitized HTML + */ + static sanitizeHtml(html, options = {}) { + if (!html || typeof html !== 'string') { + return ''; + } + + const defaultOptions = { + allowedTags: [ + 'p', + 'br', + 'strong', + 'em', + 'b', + 'i', + 'u', + 'a', + 'ul', + 'ol', + 'li', + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', + 'div', + 'span', + 'img', + 'table', + 'thead', + 'tbody', + 'tr', + 'td', + 'th', + 'blockquote', + 'hr', + ], + allowedAttributes: { + a: ['href', 'title', 'target', 'rel'], + img: ['src', 'alt', 'title'], + '*': ['style', 'class'], + }, + allowedSchemes: ['http', 'https', 'mailto'], + allowedSchemesByTag: { + img: ['http', 'https', 'data'], + }, + ...options, + }; + + return sanitizeHtmlLib(html, defaultOptions); + } + + /** + * Extract variables from template content. + * @param {Object} template - Template object + * @returns {string[]} Array of variable names found in content + */ + static extractVariablesFromContent(template) { + const variables = new Set(); + const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; + + // Check subject + if (template.subject) { + let match = variablePlaceholderRegex.exec(template.subject); + while (match !== null) { + variables.add(match[1]); + match = variablePlaceholderRegex.exec(template.subject); + } + } + + // Reset regex + variablePlaceholderRegex.lastIndex = 0; + + // Check HTML content + const htmlContent = template.html_content || template.htmlContent || ''; + if (htmlContent) { + let match = variablePlaceholderRegex.exec(htmlContent); + while (match !== null) { + variables.add(match[1]); + match = variablePlaceholderRegex.exec(htmlContent); + } + } + + return Array.from(variables); + } +} + +module.exports = TemplateRenderingService; diff --git a/src/startup/routes.js b/src/startup/routes.js index 2aae95baa..974af5e93 100644 --- a/src/startup/routes.js +++ b/src/startup/routes.js @@ -152,7 +152,7 @@ const rolePresetRouter = require('../routes/rolePresetRouter')(rolePreset); const ownerMessageRouter = require('../routes/ownerMessageRouter')(ownerMessage); const emailRouter = require('../routes/emailRouter')(); -const emailBatchRouter = require('../routes/emailBatchRoutes'); +const emailOutboxRouter = require('../routes/emailOutboxRouter'); const reasonRouter = require('../routes/reasonRouter')(reason, userProfile); const mouseoverTextRouter = require('../routes/mouseoverTextRouter')(mouseoverText); @@ -333,8 +333,8 @@ module.exports = function (app) { app.use('/api', informationRouter); app.use('/api', mouseoverTextRouter); app.use('/api', permissionChangeLogRouter); + app.use('/api/email-outbox', emailOutboxRouter); app.use('/api', emailRouter); - app.use('/api/email-batches', emailBatchRouter); app.use('/api', isEmailExistsRouter); app.use('/api', faqRouter); app.use('/api', mapLocationRouter); diff --git a/src/utilities/emailValidators.js b/src/utilities/emailValidators.js index ba764d884..00e9afb1e 100644 --- a/src/utilities/emailValidators.js +++ b/src/utilities/emailValidators.js @@ -1,5 +1,4 @@ -const cheerio = require('cheerio'); -const { EMAIL_JOB_CONFIG } = require('../config/emailJobConfig'); +const { EMAIL_CONFIG } = require('../config/emailConfig'); /** * Validate email address format @@ -68,58 +67,14 @@ function normalizeRecipientsToObjects(input) { * @returns {boolean} True if within limit */ function ensureHtmlWithinLimit(html) { - const maxBytes = EMAIL_JOB_CONFIG.LIMITS.MAX_HTML_BYTES; + const maxBytes = EMAIL_CONFIG.LIMITS.MAX_HTML_BYTES; const size = Buffer.byteLength(html || '', 'utf8'); return size <= maxBytes; } -/** - * Validate HTML content does not contain base64-encoded media (data URIs) - * Only URLs are allowed for media to keep emails light - * @param {string} html - HTML content to validate - * @returns {{isValid: boolean, errors: Array}} Validation result - */ -function validateHtmlMedia(html) { - const $ = cheerio.load(html); - const invalidMedia = []; - - // Check for base64 images in img tags - $('img').each((i, img) => { - const src = $(img).attr('src'); - if (src && src.startsWith('data:image')) { - invalidMedia.push(`Image ${i + 1}: base64-encoded image detected (use URL instead)`); - } - }); - - // Check for base64 images in CSS background-image - const htmlString = $.html(); - const base64ImageRegex = /data:image\/[^;]+;base64,[^\s"')]+/gi; - const backgroundMatches = htmlString.match(base64ImageRegex); - if (backgroundMatches) { - invalidMedia.push( - `${backgroundMatches.length} base64-encoded background image(s) detected (use URL instead)`, - ); - } - - // Check for base64 audio/video - const base64MediaRegex = /data:(audio|video)\/[^;]+;base64,[^\s"')]+/gi; - const mediaMatches = htmlString.match(base64MediaRegex); - if (mediaMatches) { - invalidMedia.push( - `${mediaMatches.length} base64-encoded media file(s) detected (use URL instead)`, - ); - } - - return { - isValid: invalidMedia.length === 0, - errors: invalidMedia, - }; -} - module.exports = { isValidEmailAddress, normalizeRecipientsToArray, normalizeRecipientsToObjects, ensureHtmlWithinLimit, - validateHtmlMedia, }; diff --git a/src/utilities/transactionHelper.js b/src/utilities/transactionHelper.js new file mode 100644 index 000000000..eacc39bf4 --- /dev/null +++ b/src/utilities/transactionHelper.js @@ -0,0 +1,36 @@ +/** + * Transaction Helper Utility + * Provides a reusable wrapper for MongoDB transactions with proper error handling + */ + +const mongoose = require('mongoose'); +const logger = require('../startup/logger'); + +/** + * Execute a callback within a MongoDB transaction. + * Handles session creation, transaction commit/abort, and cleanup automatically. + * @param {Function} callback - Async function that receives a session parameter + * @returns {Promise} Result from the callback + * @throws {Error} Re-throws any error from the callback after aborting transaction + */ +async function withTransaction(callback) { + const session = await mongoose.startSession(); + session.startTransaction(); + + try { + const result = await callback(session); + await session.commitTransaction(); + return result; + } catch (error) { + try { + await session.abortTransaction(); + } catch (abortError) { + logger.logException(abortError, 'Error aborting transaction'); + } + throw error; + } finally { + session.endSession(); + } +} + +module.exports = { withTransaction }; From 271cb8ee357e926ecdb4fec7fda08f144e4bd92b Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Tue, 11 Nov 2025 00:51:42 -0500 Subject: [PATCH 10/19] feat(email): enhance email batch status management with attempt count tracking --- .../announcements/emails/emailBatchService.js | 53 +++++++++++-------- .../announcements/emails/emailProcessor.js | 12 +++-- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/services/announcements/emails/emailBatchService.js b/src/services/announcements/emails/emailBatchService.js index 3343b7f22..e4b2ea8d3 100644 --- a/src/services/announcements/emails/emailBatchService.js +++ b/src/services/announcements/emails/emailBatchService.js @@ -310,10 +310,11 @@ class EmailBatchService { /** * Mark a batch item as SENT and set sentAt timestamp. * @param {string|ObjectId} emailBatchId - Batch ObjectId. + * @param {{attemptCount?: number}} options - Optional attempt count to update. * @returns {Promise} Updated batch document. * @throws {Error} If batch not found */ - static async markEmailBatchSent(emailBatchId) { + static async markEmailBatchSent(emailBatchId, options = {}) { if (!emailBatchId || !mongoose.Types.ObjectId.isValid(emailBatchId)) { const error = new Error('Valid email batch ID is required'); error.statusCode = 400; @@ -321,14 +322,18 @@ class EmailBatchService { } const now = new Date(); - const updated = await EmailBatch.findByIdAndUpdate( - emailBatchId, - { - status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENT, - sentAt: now, - }, - { new: true }, - ); + const updateFields = { + status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENT, + sentAt: now, + updatedAt: now, + }; + + // Update attempts count if provided (to reflect actual retry attempts) + if (options.attemptCount && options.attemptCount > 0) { + updateFields.attempts = options.attemptCount; + } + + const updated = await EmailBatch.findByIdAndUpdate(emailBatchId, updateFields, { new: true }); if (!updated) { const error = new Error(`EmailBatch ${emailBatchId} not found`); @@ -342,11 +347,11 @@ class EmailBatchService { /** * Mark a batch item as FAILED and snapshot the error info. * @param {string|ObjectId} emailBatchId - Batch ObjectId. - * @param {{errorCode?: string, errorMessage?: string}} param1 - Error details. + * @param {{errorCode?: string, errorMessage?: string, attemptCount?: number}} param1 - Error details and attempt count. * @returns {Promise} Updated batch document. * @throws {Error} If batch not found */ - static async markEmailBatchFailed(emailBatchId, { errorCode, errorMessage }) { + static async markEmailBatchFailed(emailBatchId, { errorCode, errorMessage, attemptCount }) { if (!emailBatchId || !mongoose.Types.ObjectId.isValid(emailBatchId)) { const error = new Error('Valid email batch ID is required'); error.statusCode = 400; @@ -354,17 +359,21 @@ class EmailBatchService { } const now = new Date(); - const updated = await EmailBatch.findByIdAndUpdate( - emailBatchId, - { - status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.FAILED, - failedAt: now, - lastError: errorMessage?.slice(0, 500) || null, - lastErrorAt: now, - errorCode: errorCode?.toString().slice(0, 1000) || null, - }, - { new: true }, - ); + const updateFields = { + status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.FAILED, + failedAt: now, + lastError: errorMessage?.slice(0, 500) || null, + lastErrorAt: now, + errorCode: errorCode?.toString().slice(0, 1000) || null, + updatedAt: now, + }; + + // Update attempts count if provided (to reflect actual retry attempts) + if (attemptCount && attemptCount > 0) { + updateFields.attempts = attemptCount; + } + + const updated = await EmailBatch.findByIdAndUpdate(emailBatchId, updateFields, { new: true }); if (!updated) { const error = new Error(`EmailBatch ${emailBatchId} not found`); diff --git a/src/services/announcements/emails/emailProcessor.js b/src/services/announcements/emails/emailProcessor.js index c0935fd24..842acc46c 100644 --- a/src/services/announcements/emails/emailProcessor.js +++ b/src/services/announcements/emails/emailProcessor.js @@ -226,9 +226,12 @@ class EmailProcessor { ); if (sendResult.success) { - await EmailBatchService.markEmailBatchSent(item._id); + const actualAttemptCount = sendResult.attemptCount || updatedItem?.attempts || 1; + await EmailBatchService.markEmailBatchSent(item._id, { + attemptCount: actualAttemptCount, // Persist the actual number of attempts made + }); logger.logInfo( - `EmailBatch item ${item._id} sent successfully to ${recipientEmails.length} recipients (attempts ${sendResult.attemptCount || updatedItem?.attempts || 1})`, + `EmailBatch item ${item._id} sent successfully to ${recipientEmails.length} recipients (attempts ${actualAttemptCount})`, ); return; } @@ -238,13 +241,16 @@ class EmailProcessor { // Extract error code, or use error name if code is missing, or default to 'SEND_FAILED' const errorCode = finalError.code || finalError.name || 'SEND_FAILED'; const errorMessage = finalError.message || 'Failed to send email'; + const actualAttemptCount = sendResult.attemptCount || 1; // Use actual attempt count from retry logic + await EmailBatchService.markEmailBatchFailed(item._id, { errorCode, errorMessage, + attemptCount: actualAttemptCount, // Persist the actual number of attempts made }); logger.logInfo( - `Permanently failed to send EmailBatch item ${item._id} to ${recipientEmails.length} recipients after ${sendResult.attemptCount || this.maxRetries} attempts`, + `Permanently failed to send EmailBatch item ${item._id} to ${recipientEmails.length} recipients after ${actualAttemptCount} attempts`, ); // Throw to ensure Promise.allSettled records this item as failed throw finalError; From e3742bfca08202c6ee108c2a51fcf0a648c10278 Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Tue, 11 Nov 2025 18:57:01 -0500 Subject: [PATCH 11/19] feat(email): implement email processing enhancements and startup handling - Added functionality to process pending and stuck emails on server startup after database connection. - Updated email configuration to increase maximum recipients per request and refined batch processing settings. - Refactored email processing to queue emails for non-blocking, sequential handling instead of immediate processing. - Introduced new methods for managing stuck emails and batches, ensuring robust error handling and status updates. - Enhanced email validation utilities for better normalization of email fields. --- src/config/emailConfig.js | 10 +- src/controllers/emailController.js | 27 +- src/controllers/emailOutboxController.js | 9 +- src/models/emailBatch.js | 5 + src/server.js | 9 + .../announcements/emails/emailBatchService.js | 235 +++++++++++++- .../announcements/emails/emailProcessor.js | 296 ++++++++++++++---- .../emails/emailSendingService.js | 63 +--- .../announcements/emails/emailService.js | 53 ++++ src/utilities/emailValidators.js | 21 ++ 10 files changed, 577 insertions(+), 151 deletions(-) diff --git a/src/config/emailConfig.js b/src/config/emailConfig.js index 484124c42..8d4043dc3 100644 --- a/src/config/emailConfig.js +++ b/src/config/emailConfig.js @@ -12,7 +12,7 @@ const EMAIL_CONFIG = { EMAIL_STATUSES: { PENDING: 'PENDING', // Created, waiting to be processed SENDING: 'SENDING', // Currently sending - SENT: 'SENT', // All emails successfully sent + SENT: 'SENT', // All emails successfully accepted by SMTP server PROCESSED: 'PROCESSED', // Processing finished (mixed results) FAILED: 'FAILED', // Failed to send }, @@ -32,7 +32,7 @@ const EMAIL_CONFIG = { // Centralized limits to keep model, services, and controllers consistent LIMITS: { - MAX_RECIPIENTS_PER_REQUEST: 1000, // Must match EmailBatch.recipients validator + MAX_RECIPIENTS_PER_REQUEST: 2000, // Must match EmailBatch.recipients validator MAX_HTML_BYTES: 1 * 1024 * 1024, // 1MB - Reduced since base64 media files are blocked SUBJECT_MAX_LENGTH: 200, // Standardized subject length limit TEMPLATE_NAME_MAX_LENGTH: 50, // Template name maximum length @@ -40,8 +40,10 @@ const EMAIL_CONFIG = { // Announcement service runtime knobs ANNOUNCEMENTS: { - BATCH_SIZE: 50, // recipients per SMTP send batch - CONCURRENCY: 3, // concurrent SMTP batches + BATCH_SIZE: 100, // recipients per SMTP send batch + CONCURRENCY: 3, // concurrent SMTP batches processed simultaneously + BATCH_STAGGER_START_MS: 100, // Delay between starting batches within a concurrent chunk (staggered start for rate limiting) + DELAY_BETWEEN_CHUNKS_MS: 1000, // Delay after a chunk of batches completes before starting the next chunk }, // Email configuration diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index d96a425e4..6d0a4b528 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -91,13 +91,8 @@ const sendEmail = async (req, res) => { return createdEmail; }); - // Process email immediately (async, fire and forget) - emailProcessor.processEmail(email._id).catch((processError) => { - logger.logException( - processError, - `Error processing email ${email._id} immediately after creation`, - ); - }); + // Add email to queue for processing (non-blocking, sequential processing) + emailProcessor.queueEmail(email._id); return res.status(200).json({ success: true, @@ -212,13 +207,8 @@ const sendEmailToSubscribers = async (req, res) => { return createdEmail; }); - // Process email immediately (async, fire and forget) - emailProcessor.processEmail(email._id).catch((processError) => { - logger.logException( - processError, - `Error processing broadcast email ${email._id} immediately after creation`, - ); - }); + // Add email to queue for processing (non-blocking, sequential processing) + emailProcessor.queueEmail(email._id); return res.status(200).json({ success: true, @@ -382,13 +372,8 @@ const resendEmail = async (req, res) => { return createdEmail; }); - // Process email immediately (async, fire and forget) - emailProcessor.processEmail(newEmail._id).catch((processError) => { - logger.logException( - processError, - `Error processing resent email ${newEmail._id} immediately after creation`, - ); - }); + // Add email to queue for processing (non-blocking, sequential processing) + emailProcessor.queueEmail(newEmail._id); return res.status(200).json({ success: true, diff --git a/src/controllers/emailOutboxController.js b/src/controllers/emailOutboxController.js index b45d0d2db..7e32f6f09 100644 --- a/src/controllers/emailOutboxController.js +++ b/src/controllers/emailOutboxController.js @@ -144,13 +144,8 @@ const retryEmail = async (req, res) => { `Successfully reset Email ${emailId} and ${failedItems.length} failed EmailBatch items to PENDING for retry`, ); - // Process email immediately (async, fire and forget) - emailProcessor.processEmail(emailId).catch((processError) => { - logger.logException( - processError, - `Error processing email ${emailId} immediately after retry`, - ); - }); + // Add email to queue for processing (non-blocking, sequential processing) + emailProcessor.queueEmail(emailId); res.status(200).json({ success: true, diff --git a/src/models/emailBatch.js b/src/models/emailBatch.js index 4e5f8457c..abb2326ba 100644 --- a/src/models/emailBatch.js +++ b/src/models/emailBatch.js @@ -71,6 +71,11 @@ const EmailBatchSchema = new Schema({ type: String, }, + sendResponse: { + type: Schema.Types.Mixed, + default: null, + }, + createdAt: { type: Date, default: () => new Date(), index: true }, updatedAt: { type: Date, default: () => new Date() }, }); diff --git a/src/server.js b/src/server.js index 31c40b17f..9edb642aa 100644 --- a/src/server.js +++ b/src/server.js @@ -1,15 +1,24 @@ /* eslint-disable quotes */ require('dotenv').config(); const http = require('http'); +const mongoose = require('mongoose'); require('./jobs/dailyMessageEmailNotification'); const { app, logger } = require('./app'); const TimerWebsockets = require('./websockets').default; const MessagingWebSocket = require('./websockets/lbMessaging/messagingSocket').default; +const emailProcessor = require('./services/announcements/emails/emailProcessor'); require('./startup/db')(); require('./cronjobs/userProfileJobs')(); require('./jobs/analyticsAggregation').scheduleDaily(); require('./cronjobs/bidWinnerJobs')(); +// Process pending and stuck emails on startup (only after DB is connected) +mongoose.connection.once('connected', () => { + emailProcessor.processPendingAndStuckEmails().catch((error) => { + logger.logException(error, 'Error processing pending emails on startup'); + }); +}); + const websocketRouter = require('./websockets/webSocketRouter'); const port = process.env.PORT || 4500; diff --git a/src/services/announcements/emails/emailBatchService.js b/src/services/announcements/emails/emailBatchService.js index e4b2ea8d3..4ee7e4d86 100644 --- a/src/services/announcements/emails/emailBatchService.js +++ b/src/services/announcements/emails/emailBatchService.js @@ -14,6 +14,12 @@ const { const logger = require('../../../startup/logger'); class EmailBatchService { + // Debounce map for auto-sync to prevent race conditions + // Key: emailId, Value: timeoutId + static syncDebounceMap = new Map(); + + static SYNC_DEBOUNCE_MS = 500; // Wait 500ms before syncing (allows multiple batch updates to complete) + /** * Create EmailBatch items for a parent Email. * - Validates parent Email ID, normalizes recipients and chunks by configured size. @@ -152,6 +158,7 @@ class EmailBatchService { lastError: batch.lastError, lastErrorAt: batch.lastErrorAt, errorCode: batch.errorCode, + sendResponse: batch.sendResponse || null, emailType: batch.emailType, createdAt: batch.createdAt, updatedAt: batch.updatedAt, @@ -237,6 +244,20 @@ class EmailBatchService { }).sort({ createdAt: 1 }); } + /** + * Get all stuck EmailBatch items (SENDING status). + * On server restart, any batch in SENDING status is considered stuck because + * the processing was interrupted. We reset ALL SENDING batches because the + * server restart means they're no longer being processed. + * @returns {Promise} Array of EmailBatch items with SENDING status that are stuck. + */ + static async getStuckBatches() { + // On server restart: Reset ALL batches in SENDING status (they're all stuck) + return EmailBatch.find({ + status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENDING, + }).sort({ lastAttemptedAt: 1 }); // Process oldest first + } + /** * Reset an EmailBatch item for retry, clearing attempts and error fields. * Uses atomic update to prevent race conditions. @@ -273,6 +294,10 @@ class EmailBatchService { throw error; } + // Auto-sync parent Email status with debouncing (fire-and-forget to avoid blocking) + // When batches are reset for retry, parent status might change from FAILED/PROCESSED to PENDING/SENDING + this.debouncedSyncParentEmailStatus(updated.emailId); + return updated; } @@ -309,10 +334,11 @@ class EmailBatchService { /** * Mark a batch item as SENT and set sentAt timestamp. + * Uses atomic update with status check to prevent race conditions. * @param {string|ObjectId} emailBatchId - Batch ObjectId. - * @param {{attemptCount?: number}} options - Optional attempt count to update. + * @param {{attemptCount?: number, sendResponse?: Object}} options - Optional attempt count and send response to store. * @returns {Promise} Updated batch document. - * @throws {Error} If batch not found + * @throws {Error} If batch not found or not in SENDING status */ static async markEmailBatchSent(emailBatchId, options = {}) { if (!emailBatchId || !mongoose.Types.ObjectId.isValid(emailBatchId)) { @@ -333,19 +359,46 @@ class EmailBatchService { updateFields.attempts = options.attemptCount; } - const updated = await EmailBatch.findByIdAndUpdate(emailBatchId, updateFields, { new: true }); + // Store send response if provided (contains messageId, accepted, rejected, etc.) + if (options.sendResponse) { + updateFields.sendResponse = options.sendResponse; + } + + // Atomic update: only update if batch is in SENDING status (prevents race conditions) + const updated = await EmailBatch.findOneAndUpdate( + { + _id: emailBatchId, + status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENDING, + }, + updateFields, + { new: true }, + ); if (!updated) { - const error = new Error(`EmailBatch ${emailBatchId} not found`); - error.statusCode = 404; - throw error; + // Batch not found or not in SENDING status (might already be SENT/FAILED) + // Check current status to provide better error message + const currentBatch = await EmailBatch.findById(emailBatchId); + if (!currentBatch) { + const error = new Error(`EmailBatch ${emailBatchId} not found`); + error.statusCode = 404; + throw error; + } + // Batch exists but not in SENDING status - log and return current batch (idempotent) + logger.logInfo( + `EmailBatch ${emailBatchId} is not in SENDING status (current: ${currentBatch.status}), skipping mark as SENT`, + ); + return currentBatch; } + // Auto-sync parent Email status with debouncing (fire-and-forget to avoid blocking) + this.debouncedSyncParentEmailStatus(updated.emailId); + return updated; } /** * Mark a batch item as FAILED and snapshot the error info. + * Uses atomic update with status check to prevent race conditions. * @param {string|ObjectId} emailBatchId - Batch ObjectId. * @param {{errorCode?: string, errorMessage?: string, attemptCount?: number}} param1 - Error details and attempt count. * @returns {Promise} Updated batch document. @@ -373,16 +426,178 @@ class EmailBatchService { updateFields.attempts = attemptCount; } - const updated = await EmailBatch.findByIdAndUpdate(emailBatchId, updateFields, { new: true }); + // Atomic update: only update if batch is in SENDING status (prevents race conditions) + // Allow updating from PENDING as well (in case of early failures) + const updated = await EmailBatch.findOneAndUpdate( + { + _id: emailBatchId, + status: { + $in: [ + EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENDING, + EMAIL_CONFIG.EMAIL_BATCH_STATUSES.PENDING, + ], + }, + }, + updateFields, + { new: true }, + ); if (!updated) { - const error = new Error(`EmailBatch ${emailBatchId} not found`); - error.statusCode = 404; - throw error; + // Batch not found or already in final state (SENT/FAILED) + // Check current status to provide better error message + const currentBatch = await EmailBatch.findById(emailBatchId); + if (!currentBatch) { + const error = new Error(`EmailBatch ${emailBatchId} not found`); + error.statusCode = 404; + throw error; + } + // Batch exists but already in final state - log and return current batch (idempotent) + logger.logInfo( + `EmailBatch ${emailBatchId} is already in final state (current: ${currentBatch.status}), skipping mark as FAILED`, + ); + return currentBatch; } + // Auto-sync parent Email status with debouncing (fire-and-forget to avoid blocking) + this.debouncedSyncParentEmailStatus(updated.emailId); + return updated; } + + /** + * Determine the parent Email status from child EmailBatch statuses. + * Rules: + * - All SENT => SENT + * - All FAILED => FAILED + * - Mixed (some SENT and some FAILED) => PROCESSED + * - Otherwise (PENDING or SENDING batches) => SENDING (still in progress) + * @param {string|ObjectId} emailId - Parent Email ObjectId. + * @returns {Promise} Derived status constant from EMAIL_CONFIG.EMAIL_STATUSES. + */ + static async determineEmailStatus(emailId) { + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + const error = new Error('Valid email ID is required'); + error.statusCode = 400; + throw error; + } + + // Get all batches and count statuses in memory (simpler and faster for small-medium counts) + const batches = await this.getBatchesForEmail(emailId); + + // Handle edge case: no batches + if (batches.length === 0) { + logger.logInfo(`Email ${emailId} has no batches, returning FAILED status`); + return EMAIL_CONFIG.EMAIL_STATUSES.FAILED; + } + + const statusMap = batches.reduce((acc, batch) => { + acc[batch.status] = (acc[batch.status] || 0) + 1; + return acc; + }, {}); + + const pending = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.PENDING] || 0; + const sending = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENDING] || 0; + const sent = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENT] || 0; + const failed = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.FAILED] || 0; + + // All sent = SENT + if (sent > 0 && pending === 0 && sending === 0 && failed === 0) { + return EMAIL_CONFIG.EMAIL_STATUSES.SENT; + } + + // All failed = FAILED + if (failed > 0 && pending === 0 && sending === 0 && sent === 0) { + return EMAIL_CONFIG.EMAIL_STATUSES.FAILED; + } + + // Mixed results (some sent, some failed) = PROCESSED + if (sent > 0 || failed > 0) { + return EMAIL_CONFIG.EMAIL_STATUSES.PROCESSED; + } + + // Still processing (pending or sending batches) = keep SENDING status + return EMAIL_CONFIG.EMAIL_STATUSES.SENDING; + } + + /** + * Debounced version of syncParentEmailStatus to prevent race conditions. + * Waits for a short period before syncing to allow multiple batch updates to complete. + * @param {string|ObjectId} emailId - Parent Email ObjectId. + * @returns {void} + */ + static debouncedSyncParentEmailStatus(emailId) { + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + return; + } + + const emailIdStr = emailId.toString(); + + // Clear existing timeout if any + if (this.syncDebounceMap.has(emailIdStr)) { + clearTimeout(this.syncDebounceMap.get(emailIdStr)); + } + + // Set new timeout for syncing + const timeoutId = setTimeout(() => { + this.syncDebounceMap.delete(emailIdStr); + this.syncParentEmailStatus(emailIdStr).catch((syncError) => { + logger.logException(syncError, `Error syncing parent Email status for ${emailIdStr}`); + }); + }, this.SYNC_DEBOUNCE_MS); + + this.syncDebounceMap.set(emailIdStr, timeoutId); + } + + /** + * Synchronize parent Email status based on child EmailBatch statuses. + * - Determines status from batches and updates parent Email + * - Sets completedAt timestamp only for final states (SENT, FAILED, PROCESSED) + * - This ensures Email.status stays in sync when batches are updated + * - Uses atomic update to prevent race conditions + * @param {string|ObjectId} emailId - Parent Email ObjectId. + * @returns {Promise} Updated Email document. + */ + static async syncParentEmailStatus(emailId) { + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + const error = new Error('Valid email ID is required'); + error.statusCode = 400; + throw error; + } + + // Determine the correct status from batches + const derivedStatus = await this.determineEmailStatus(emailId); + + // Check if this is a final state that should set completedAt + const finalStates = [ + EMAIL_CONFIG.EMAIL_STATUSES.SENT, + EMAIL_CONFIG.EMAIL_STATUSES.FAILED, + EMAIL_CONFIG.EMAIL_STATUSES.PROCESSED, + ]; + const isFinalState = finalStates.includes(derivedStatus); + + const now = new Date(); + const updateFields = { + status: derivedStatus, + updatedAt: now, + }; + + // Only set completedAt for final states + if (isFinalState) { + updateFields.completedAt = now; + } + + // Update Email status atomically + // Note: We don't check current status because we're recomputing from batches (source of truth) + const email = await Email.findByIdAndUpdate(emailId, updateFields, { new: true }); + + if (!email) { + // Email not found - log but don't throw (might have been deleted) + logger.logInfo(`Email ${emailId} not found when syncing status`); + return null; + } + + return email; + } } module.exports = EmailBatchService; diff --git a/src/services/announcements/emails/emailProcessor.js b/src/services/announcements/emails/emailProcessor.js index 842acc46c..20878c8dd 100644 --- a/src/services/announcements/emails/emailProcessor.js +++ b/src/services/announcements/emails/emailProcessor.js @@ -10,16 +10,117 @@ class EmailProcessor { * Initialize processor runtime configuration. * - Tracks currently processing parent Email IDs to avoid duplicate work. * - Loads retry settings from EMAIL_CONFIG to coordinate with sending service. + * - Maintains in-memory queue for sequential email processing. */ constructor() { this.processingBatches = new Set(); this.maxRetries = EMAIL_CONFIG.DEFAULT_MAX_RETRIES; this.retryDelay = EMAIL_CONFIG.INITIAL_RETRY_DELAY_MS; + this.emailQueue = []; // In-memory queue for emails to process + this.isProcessingQueue = false; // Flag to prevent multiple queue processors + this.currentlyProcessingEmailId = null; // Track which email is currently being processed + } + + /** + * Add an email to the processing queue. + * - Adds email to in-memory queue if not already queued or processing + * - Starts queue processor if not already running and DB is connected + * - Returns immediately (non-blocking) + * - Uses setImmediate pattern for asynchronous queue processing + * @param {string|ObjectId} emailId - The ObjectId of the parent Email. + * @returns {void} + */ + queueEmail(emailId) { + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + logger.logException(new Error('Invalid emailId'), 'EmailProcessor.queueEmail'); + return; + } + + const emailIdStr = emailId.toString(); + + // Skip if already in queue or currently processing + if ( + this.emailQueue.includes(emailIdStr) || + this.currentlyProcessingEmailId === emailIdStr || + this.processingBatches.has(emailIdStr) + ) { + logger.logInfo(`Email ${emailIdStr} is already queued or being processed, skipping`); + return; + } + + // Add to queue + this.emailQueue.push(emailIdStr); + logger.logInfo(`Email ${emailIdStr} added to queue. Queue length: ${this.emailQueue.length}`); + + // Start queue processor if not already running + if (!this.isProcessingQueue) { + setImmediate(() => { + this.processQueue().catch((error) => { + logger.logException(error, 'Error in queue processor'); + // Reset flag so queue can restart on next email addition + this.isProcessingQueue = false; + }); + }); + } + } + + /** + * Process the email queue sequentially. + * - Processes one email at a time + * - Once an email is done, processes the next one + * - Continues until queue is empty + * - Database connection is ensured at startup (server.js waits for DB connection) + * @returns {Promise} + */ + async processQueue() { + if (this.isProcessingQueue) { + return; // Already processing + } + + this.isProcessingQueue = true; + logger.logInfo('Email queue processor started'); + + try { + // eslint-disable-next-line no-constant-condition + while (true) { + // Get next email from queue (FIFO) + const emailId = this.emailQueue.shift(); + if (!emailId) { + break; + } + + this.currentlyProcessingEmailId = emailId; + logger.logInfo( + `Processing email ${emailId} from queue. Remaining: ${this.emailQueue.length}`, + ); + + try { + // Process the email (this processes all its batches) + // Sequential processing is required - await in loop is necessary + // eslint-disable-next-line no-await-in-loop + await this.processEmail(emailId); + logger.logInfo(`Email ${emailId} processing completed`); + } catch (error) { + logger.logException(error, `Error processing email ${emailId} from queue`); + } finally { + this.currentlyProcessingEmailId = null; + } + + // Small delay before processing next email to avoid overwhelming the system + if (this.emailQueue.length > 0) { + // eslint-disable-next-line no-await-in-loop + await EmailProcessor.sleep(100); + } + } + } finally { + this.isProcessingQueue = false; + logger.logInfo('Email queue processor stopped'); + } } /** * Process a single parent Email by sending all of its pending EmailBatch items. - * - Each request is independent - processes only the email passed to it + * - Processes all batches for the email sequentially (with concurrency within batches) * - Idempotent with respect to concurrent calls (skips if already processing) * - Simple flow: PENDING → SENDING → SENT/FAILED/PROCESSED * @param {string|ObjectId} emailId - The ObjectId of the parent Email. @@ -31,13 +132,15 @@ class EmailProcessor { throw new Error('emailId is required and must be a valid ObjectId'); } + const emailIdStr = emailId.toString(); + // Prevent concurrent processing of the same email - if (this.processingBatches.has(emailId)) { - logger.logInfo(`Email ${emailId} is already being processed, skipping`); + if (this.processingBatches.has(emailIdStr)) { + logger.logInfo(`Email ${emailIdStr} is already being processed, skipping`); return EMAIL_CONFIG.EMAIL_STATUSES.SENDING; } - this.processingBatches.add(emailId); + this.processingBatches.add(emailIdStr); try { // Get email - don't throw if not found, handle gracefully in processor @@ -64,8 +167,8 @@ class EmailProcessor { // If marking as started fails, email is likely already being processed const currentEmail = await EmailService.getEmailById(emailId); if (currentEmail && currentEmail.status === EMAIL_CONFIG.EMAIL_STATUSES.SENDING) { - logger.logInfo(`Email ${emailId} is already being processed, skipping`); - this.processingBatches.delete(emailId); + logger.logInfo(`Email ${emailIdStr} is already being processed, skipping`); + this.processingBatches.delete(emailIdStr); return EMAIL_CONFIG.EMAIL_STATUSES.SENDING; } // Re-throw if it's a different error @@ -75,24 +178,49 @@ class EmailProcessor { // Process all PENDING EmailBatch items for this email await this.processEmailBatches(email); - // Determine final status based on batch items - const finalStatus = await EmailProcessor.determineEmailStatus(email._id); - await EmailService.markEmailCompleted(emailId, finalStatus); + // Sync parent Email status based on all batch statuses + // Auto-sync from individual batch updates may have already updated status, + // but this ensures final status and completedAt timestamp are set correctly + const updatedEmail = await EmailBatchService.syncParentEmailStatus(email._id); + const finalStatus = updatedEmail ? updatedEmail.status : EMAIL_CONFIG.EMAIL_STATUSES.FAILED; - logger.logInfo(`Email ${emailId} processed with status: ${finalStatus}`); + logger.logInfo(`Email ${emailIdStr} processed with status: ${finalStatus}`); return finalStatus; } catch (error) { - logger.logException(error, `Error processing Email ${emailId}`); + logger.logException(error, `Error processing Email ${emailIdStr}`); + + // Reset any batches that were marked as SENDING back to PENDING + // This prevents batches from being stuck in SENDING status + try { + const batches = await EmailBatchService.getBatchesForEmail(emailIdStr); + const sendingBatches = batches.filter( + (batch) => batch.status === EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENDING, + ); + await Promise.allSettled( + sendingBatches.map(async (batch) => { + try { + await EmailBatchService.resetEmailBatchForRetry(batch._id); + logger.logInfo( + `Reset batch ${batch._id} from SENDING to PENDING due to email processing error`, + ); + } catch (resetError) { + logger.logException(resetError, `Error resetting batch ${batch._id} to PENDING`); + } + }), + ); + } catch (resetError) { + logger.logException(resetError, `Error resetting batches for email ${emailIdStr}`); + } // Mark email as failed on error try { - await EmailService.markEmailCompleted(emailId, EMAIL_CONFIG.EMAIL_STATUSES.FAILED); + await EmailService.markEmailCompleted(emailIdStr, EMAIL_CONFIG.EMAIL_STATUSES.FAILED); } catch (updateError) { logger.logException(updateError, 'Error updating Email status to failed'); } return EMAIL_CONFIG.EMAIL_STATUSES.FAILED; } finally { - this.processingBatches.delete(emailId); + this.processingBatches.delete(emailIdStr); } } @@ -119,17 +247,38 @@ class EmailProcessor { // Process items with concurrency limit const concurrency = EMAIL_CONFIG.ANNOUNCEMENTS.CONCURRENCY || 3; + const delayBetweenChunks = EMAIL_CONFIG.ANNOUNCEMENTS.DELAY_BETWEEN_CHUNKS_MS || 1000; + const batchStaggerStart = EMAIL_CONFIG.ANNOUNCEMENTS.BATCH_STAGGER_START_MS || 0; const results = []; // Process batches in chunks with concurrency control // eslint-disable-next-line no-await-in-loop for (let i = 0; i < pendingBatches.length; i += concurrency) { - const batch = pendingBatches.slice(i, i + concurrency); + const batchChunk = pendingBatches.slice(i, i + concurrency); + + // Process batches with optional staggered start delays within the chunk + // This staggers when each batch in the chunk starts processing (helps with rate limiting) + const batchPromises = batchChunk.map((item, index) => { + if (batchStaggerStart > 0 && index > 0) { + // Stagger the start: batch 1 starts immediately, batch 2 after staggerDelay, batch 3 after 2*staggerDelay, etc. + return EmailProcessor.sleep(batchStaggerStart * index).then(() => + this.processEmailBatch(item, email), + ); + } + // First batch in chunk starts immediately (no stagger) + return this.processEmailBatch(item, email); + }); + + // Wait for all batches in this chunk to complete // eslint-disable-next-line no-await-in-loop - const batchResults = await Promise.allSettled( - batch.map((item) => this.processEmailBatch(item, email)), - ); + const batchResults = await Promise.allSettled(batchPromises); results.push(...batchResults); + + // Add delay after this chunk completes before starting the next chunk + // This provides consistent pacing to prevent hitting Gmail rate limits + if (delayBetweenChunks > 0 && i + concurrency < pendingBatches.length) { + await EmailProcessor.sleep(delayBetweenChunks); + } } // Log summary of processing @@ -229,6 +378,7 @@ class EmailProcessor { const actualAttemptCount = sendResult.attemptCount || updatedItem?.attempts || 1; await EmailBatchService.markEmailBatchSent(item._id, { attemptCount: actualAttemptCount, // Persist the actual number of attempts made + sendResponse: sendResult.response, // Store the full response from email API }); logger.logInfo( `EmailBatch item ${item._id} sent successfully to ${recipientEmails.length} recipients (attempts ${actualAttemptCount})`, @@ -267,61 +417,83 @@ class EmailProcessor { }); } - /** - * Determine the final parent Email status from child EmailBatch statuses. - * Rules: - * - All SENT => SENT - * - All FAILED => FAILED - * - Mixed (some SENT and some FAILED) => PROCESSED - * - Otherwise (PENDING or SENDING batches) => SENDING (still in progress) - * @param {ObjectId} emailObjectId - Parent Email ObjectId. - * @returns {Promise} Derived status constant from EMAIL_CONFIG.EMAIL_STATUSES. - */ - static async determineEmailStatus(emailObjectId) { - // Get all batches and count statuses in memory (simpler and faster for small-medium counts) - // Use service method for consistency (service validates emailId) - const batches = await EmailBatchService.getBatchesForEmail(emailObjectId); - - const statusMap = batches.reduce((acc, batch) => { - acc[batch.status] = (acc[batch.status] || 0) + 1; - return acc; - }, {}); - - const pending = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.PENDING] || 0; - const sending = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENDING] || 0; - const sent = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENT] || 0; - const failed = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.FAILED] || 0; - - // All sent = SENT - if (sent > 0 && pending === 0 && sending === 0 && failed === 0) { - return EMAIL_CONFIG.EMAIL_STATUSES.SENT; - } - - // All failed = FAILED - if (failed > 0 && pending === 0 && sending === 0 && sent === 0) { - return EMAIL_CONFIG.EMAIL_STATUSES.FAILED; - } - - // Mixed results (some sent, some failed) = PROCESSED - if (sent > 0 || failed > 0) { - return EMAIL_CONFIG.EMAIL_STATUSES.PROCESSED; - } - - // Still processing (pending or sending batches) = keep SENDING status - return EMAIL_CONFIG.EMAIL_STATUSES.SENDING; - } - /** * Get lightweight processor status for diagnostics/telemetry. - * @returns {{isRunning: boolean, processingBatches: string[], maxRetries: number}} + * @returns {{isRunning: boolean, processingBatches: string[], maxRetries: number, queueLength: number, currentlyProcessing: string|null, isProcessingQueue: boolean}} */ getStatus() { return { isRunning: true, processingBatches: Array.from(this.processingBatches), maxRetries: this.maxRetries, + queueLength: this.emailQueue.length, + currentlyProcessing: this.currentlyProcessingEmailId, + isProcessingQueue: this.isProcessingQueue, }; } + + /** + * Process pending and stuck emails on system startup. + * - Called only after database connection is established (server.js uses mongoose.connection.once('connected')) + * - Resets stuck emails (SENDING status) to PENDING + * - Resets stuck batches (SENDING status) to PENDING + * - Queues all PENDING emails for processing + * @returns {Promise} + */ + async processPendingAndStuckEmails() { + try { + logger.logInfo('Starting startup processing of pending and stuck emails...'); + + // Step 1: Reset stuck emails to PENDING + const stuckEmails = await EmailService.getStuckEmails(); + if (stuckEmails.length > 0) { + logger.logInfo(`Found ${stuckEmails.length} stuck emails, resetting to PENDING...`); + await Promise.allSettled( + stuckEmails.map(async (email) => { + try { + await EmailService.resetStuckEmail(email._id); + logger.logInfo(`Reset stuck email ${email._id} to PENDING`); + } catch (error) { + logger.logException(error, `Error resetting stuck email ${email._id}`); + } + }), + ); + } + + // Step 2: Reset stuck batches to PENDING + const stuckBatches = await EmailBatchService.getStuckBatches(); + if (stuckBatches.length > 0) { + logger.logInfo(`Found ${stuckBatches.length} stuck batches, resetting to PENDING...`); + await Promise.allSettled( + stuckBatches.map(async (batch) => { + try { + await EmailBatchService.resetEmailBatchForRetry(batch._id); + logger.logInfo(`Reset stuck batch ${batch._id} to PENDING`); + } catch (error) { + logger.logException(error, `Error resetting stuck batch ${batch._id}`); + } + }), + ); + } + + // Step 3: Queue all PENDING emails for processing + const pendingEmails = await EmailService.getPendingEmails(); + if (pendingEmails.length > 0) { + logger.logInfo(`Found ${pendingEmails.length} pending emails, adding to queue...`); + // Queue all emails (non-blocking, sequential processing) + pendingEmails.forEach((email) => { + this.queueEmail(email._id); + }); + logger.logInfo(`Queued ${pendingEmails.length} pending emails for processing`); + } else { + logger.logInfo('No pending emails found on startup'); + } + + logger.logInfo('Startup processing of pending and stuck emails completed'); + } catch (error) { + logger.logException(error, 'Error during startup processing of pending and stuck emails'); + } + } } // Create singleton instance diff --git a/src/services/announcements/emails/emailSendingService.js b/src/services/announcements/emails/emailSendingService.js index b68689987..068e60ba7 100644 --- a/src/services/announcements/emails/emailSendingService.js +++ b/src/services/announcements/emails/emailSendingService.js @@ -36,12 +36,6 @@ class EmailSendingService { ); this.OAuth2Client.setCredentials({ refresh_token: this.config.refreshToken }); - // OAuth token caching - this.cachedToken = null; - this.tokenExpiryTime = null; - // Tokens typically expire in 1 hour, refresh 5 minutes before expiry - this.tokenRefreshBufferMs = 5 * 60 * 1000; // 5 minutes - // Create the email transporter try { this.transporter = nodemailer.createTransport({ @@ -60,47 +54,28 @@ class EmailSendingService { } /** - * Get OAuth access token with caching. - * Refreshes token only if expired or about to expire. + * Get OAuth access token (refreshes on each call). + * Similar to emailSender.js pattern - refreshes token for each send to avoid stale tokens. * @returns {Promise} Access token * @throws {Error} If token refresh fails */ async getAccessToken() { - const now = Date.now(); - - // Check if we have a valid cached token - if ( - this.cachedToken && - this.tokenExpiryTime && - now < this.tokenExpiryTime - this.tokenRefreshBufferMs - ) { - return this.cachedToken; + const accessTokenResp = await this.OAuth2Client.getAccessToken(); + let token; + + if (accessTokenResp && typeof accessTokenResp === 'object' && accessTokenResp.token) { + token = accessTokenResp.token; + } else if (typeof accessTokenResp === 'string') { + token = accessTokenResp; + } else { + throw new Error('Invalid access token response format'); } - // Token expired or doesn't exist, refresh it - try { - const accessTokenResp = await this.OAuth2Client.getAccessToken(); - let token; - - if (accessTokenResp && typeof accessTokenResp === 'object' && accessTokenResp.token) { - token = accessTokenResp.token; - } else if (typeof accessTokenResp === 'string') { - token = accessTokenResp; - } else { - throw new Error('Invalid access token response format'); - } - - // Cache the token with expiry time (tokens typically last 1 hour) - this.cachedToken = token; - this.tokenExpiryTime = now + 60 * 60 * 1000; // 1 hour from now - - return token; - } catch (error) { - // Clear cache on error - this.cachedToken = null; - this.tokenExpiryTime = null; - throw error; + if (!token) { + throw new Error('NO_OAUTH_ACCESS_TOKEN: Failed to obtain access token'); } + + return token; } /** @@ -139,7 +114,7 @@ class EmailSendingService { } try { - // Get access token with caching + // Get access token (refreshes on each send to avoid stale tokens) let token; try { token = await this.getAccessToken(); @@ -149,12 +124,6 @@ class EmailSendingService { return { success: false, error }; } - if (!token) { - const error = new Error('NO_OAUTH_ACCESS_TOKEN: Failed to obtain access token'); - logger.logException(error, 'EmailSendingService.sendEmail OAuth failed'); - return { success: false, error }; - } - // Configure OAuth2 mailOptions.auth = { type: 'OAuth2', diff --git a/src/services/announcements/emails/emailService.js b/src/services/announcements/emails/emailService.js index a57b1072f..eddb39ad2 100644 --- a/src/services/announcements/emails/emailService.js +++ b/src/services/announcements/emails/emailService.js @@ -266,6 +266,59 @@ class EmailService { return emails; } + + /** + * Get all PENDING emails that need to be processed. + * @returns {Promise} Array of Email objects with PENDING status. + */ + static async getPendingEmails() { + return Email.find({ + status: EMAIL_CONFIG.EMAIL_STATUSES.PENDING, + }) + .sort({ createdAt: 1 }) // Process oldest first + .lean(); + } + + /** + * Get all STUCK emails (SENDING status). + * On server restart, any email in SENDING status is considered stuck because + * the processing was interrupted. We reset ALL SENDING emails because the + * server restart means they're no longer being processed. + * @returns {Promise} Array of Email objects with SENDING status that are stuck. + */ + static async getStuckEmails() { + // On server restart: Reset ALL emails in SENDING status (they're all stuck) + return Email.find({ + status: EMAIL_CONFIG.EMAIL_STATUSES.SENDING, + }) + .sort({ startedAt: 1 }) // Process oldest first + .lean(); + } + + /** + * Reset stuck email to PENDING status so it can be reprocessed. + * @param {string|ObjectId} emailId + * @returns {Promise} Updated Email document. + * @throws {Error} If email not found + */ + static async resetStuckEmail(emailId) { + const now = new Date(); + const email = await Email.findByIdAndUpdate( + emailId, + { + status: EMAIL_CONFIG.EMAIL_STATUSES.PENDING, + startedAt: null, // Clear startedAt so it can be reprocessed + updatedAt: now, + }, + { new: true }, + ); + if (!email) { + const error = new Error(`Email ${emailId} not found`); + error.statusCode = 404; + throw error; + } + return email; + } } module.exports = EmailService; diff --git a/src/utilities/emailValidators.js b/src/utilities/emailValidators.js index 00e9afb1e..2193221bf 100644 --- a/src/utilities/emailValidators.js +++ b/src/utilities/emailValidators.js @@ -72,9 +72,30 @@ function ensureHtmlWithinLimit(html) { return size <= maxBytes; } +/** + * Normalize email field (to, cc, bcc) to array format. + * Handles arrays, comma-separated strings, single strings, or null/undefined. + * @param {string|string[]|null|undefined} field - Email field to normalize + * @returns {string[]} Array of email addresses (empty array if input is invalid) + */ +function normalizeEmailField(field) { + if (!field) { + return []; + } + if (Array.isArray(field)) { + return field.filter((e) => e && typeof e === 'string' && e.trim().length > 0); + } + // Handle comma-separated string + return String(field) + .split(',') + .map((e) => e.trim()) + .filter((e) => e.length > 0); +} + module.exports = { isValidEmailAddress, normalizeRecipientsToArray, normalizeRecipientsToObjects, ensureHtmlWithinLimit, + normalizeEmailField, }; From e27baf6c33f331474b712b0bb030ce44fdbb86f5 Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Thu, 13 Nov 2025 13:10:55 -0500 Subject: [PATCH 12/19] refactor(email): replace TemplateRenderingService with EmailTemplateService - Updated emailController and emailTemplateController to utilize EmailTemplateService for template rendering and variable validation. - Introduced TEMPLATE_VARIABLE_TYPES in emailConfig for better management of variable types. - Removed the deprecated TemplateRenderingService, streamlining the email template handling process. --- src/config/emailConfig.js | 3 + src/controllers/emailController.js | 10 +- src/controllers/emailTemplateController.js | 5 +- src/models/emailTemplate.js | 3 +- .../emails/emailTemplateService.js | 248 +++++++++++++++++- .../emails/templateRenderingService.js | 243 ----------------- 6 files changed, 251 insertions(+), 261 deletions(-) delete mode 100644 src/services/announcements/emails/templateRenderingService.js diff --git a/src/config/emailConfig.js b/src/config/emailConfig.js index 8d4043dc3..0ff5fc77c 100644 --- a/src/config/emailConfig.js +++ b/src/config/emailConfig.js @@ -38,6 +38,9 @@ const EMAIL_CONFIG = { TEMPLATE_NAME_MAX_LENGTH: 50, // Template name maximum length }, + // Template variable types + TEMPLATE_VARIABLE_TYPES: ['text', 'url', 'number', 'textarea', 'image'], + // Announcement service runtime knobs ANNOUNCEMENTS: { BATCH_SIZE: 100, // recipients per SMTP send batch diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index 6d0a4b528..ad43de951 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -4,7 +4,7 @@ const jwt = require('jsonwebtoken'); const emailSender = require('../utilities/emailSender'); const { EMAIL_CONFIG } = require('../config/emailConfig'); const { isValidEmailAddress, normalizeRecipientsToArray } = require('../utilities/emailValidators'); -const TemplateRenderingService = require('../services/announcements/emails/templateRenderingService'); +const EmailTemplateService = require('../services/announcements/emails/emailTemplateService'); const EmailSubcriptionList = require('../models/emailSubcriptionList'); const userProfile = require('../models/userProfile'); const EmailBatchService = require('../services/announcements/emails/emailBatchService'); @@ -42,8 +42,8 @@ const sendEmail = async (req, res) => { const { to, subject, html } = req.body; // Validate that all template variables have been replaced (business rule) - const unmatchedVariablesHtml = TemplateRenderingService.getUnreplacedVariables(html); - const unmatchedVariablesSubject = TemplateRenderingService.getUnreplacedVariables(subject); + const unmatchedVariablesHtml = EmailTemplateService.getUnreplacedVariables(html); + const unmatchedVariablesSubject = EmailTemplateService.getUnreplacedVariables(subject); const unmatchedVariables = [ ...new Set([...unmatchedVariablesHtml, ...unmatchedVariablesSubject]), ]; @@ -137,8 +137,8 @@ const sendEmailToSubscribers = async (req, res) => { const { subject, html } = req.body; // Validate that all template variables have been replaced (business rule) - const unmatchedVariablesHtml = TemplateRenderingService.getUnreplacedVariables(html); - const unmatchedVariablesSubject = TemplateRenderingService.getUnreplacedVariables(subject); + const unmatchedVariablesHtml = EmailTemplateService.getUnreplacedVariables(html); + const unmatchedVariablesSubject = EmailTemplateService.getUnreplacedVariables(subject); const unmatchedVariables = [ ...new Set([...unmatchedVariablesHtml, ...unmatchedVariablesSubject]), ]; diff --git a/src/controllers/emailTemplateController.js b/src/controllers/emailTemplateController.js index 78c7d3b19..124af9ca1 100644 --- a/src/controllers/emailTemplateController.js +++ b/src/controllers/emailTemplateController.js @@ -3,7 +3,6 @@ */ const EmailTemplateService = require('../services/announcements/emails/emailTemplateService'); -const TemplateRenderingService = require('../services/announcements/emails/templateRenderingService'); const { hasPermission } = require('../utilities/permissions'); const logger = require('../startup/logger'); @@ -310,7 +309,7 @@ const previewTemplate = async (req, res) => { }); // Validate variables - const validation = TemplateRenderingService.validateVariables(template, variables); + const validation = EmailTemplateService.validateVariables(template, variables); if (!validation.isValid) { return res.status(400).json({ success: false, @@ -321,7 +320,7 @@ const previewTemplate = async (req, res) => { } // Render template - const rendered = TemplateRenderingService.renderTemplate(template, variables, { + const rendered = EmailTemplateService.renderTemplate(template, variables, { sanitize: false, // Don't sanitize for preview strict: false, }); diff --git a/src/models/emailTemplate.js b/src/models/emailTemplate.js index c236880d6..7f9576187 100644 --- a/src/models/emailTemplate.js +++ b/src/models/emailTemplate.js @@ -5,6 +5,7 @@ * - Includes helpful indexes and text search for fast lookup */ const mongoose = require('mongoose'); +const { EMAIL_CONFIG } = require('../config/emailConfig'); const emailTemplateSchema = new mongoose.Schema( { @@ -31,7 +32,7 @@ const emailTemplateSchema = new mongoose.Schema( }, type: { type: String, - enum: ['text', 'url', 'number', 'textarea', 'image'], + enum: EMAIL_CONFIG.TEMPLATE_VARIABLE_TYPES, default: 'text', }, }, diff --git a/src/services/announcements/emails/emailTemplateService.js b/src/services/announcements/emails/emailTemplateService.js index 5d3e56ff3..d733aa241 100644 --- a/src/services/announcements/emails/emailTemplateService.js +++ b/src/services/announcements/emails/emailTemplateService.js @@ -4,6 +4,7 @@ */ const mongoose = require('mongoose'); +const sanitizeHtmlLib = require('sanitize-html'); const EmailTemplate = require('../../../models/emailTemplate'); const { EMAIL_CONFIG } = require('../../../config/emailConfig'); const { ensureHtmlWithinLimit } = require('../../../utilities/emailValidators'); @@ -13,7 +14,8 @@ class EmailTemplateService { /** * Validate template variables. * - Ensures non-empty unique names and validates allowed types. - * @param {Array<{name: string, type?: 'text'|'url'|'number'|'textarea'|'image'>} | undefined} variables + * - Allowed types are defined in EMAIL_CONFIG.TEMPLATE_VARIABLE_TYPES + * @param {Array<{name: string, type?: string}>} variables - Variable definitions * @returns {{isValid: boolean, errors?: string[]}} */ static validateTemplateVariables(variables) { @@ -42,12 +44,9 @@ class EmailTemplateService { variableNames.add(varName.toLowerCase()); } - if ( - variable.type && - !['text', 'url', 'number', 'textarea', 'image'].includes(variable.type) - ) { + if (variable.type && !EMAIL_CONFIG.TEMPLATE_VARIABLE_TYPES.includes(variable.type)) { errors.push( - `Variable ${index + 1}: type must be one of: text, url, number, textarea, image`, + `Variable ${index + 1}: type must be one of: ${EMAIL_CONFIG.TEMPLATE_VARIABLE_TYPES.join(', ')}`, ); } }); @@ -232,9 +231,7 @@ class EmailTemplateService { const trimmedSubject = subject.trim(); // Check if template with the same name already exists (case-insensitive) - const existingTemplate = await EmailTemplate.findOne({ - name: { $regex: new RegExp(`^${trimmedName}$`, 'i') }, - }); + const existingTemplate = await this.templateNameExists(trimmedName, null); if (existingTemplate) { const error = new Error('Email template with this name already exists'); @@ -494,6 +491,239 @@ class EmailTemplateService { const existing = await EmailTemplate.findOne(query); return !!existing; } + + /** + * Render template with variable values. + * Replaces {{variableName}} placeholders with actual values. + * @param {Object} template - Template object with subject and html_content + * @param {Object} variables - Object mapping variable names to values + * @param {Object} options - Rendering options (sanitize, strict) + * @returns {{subject: string, htmlContent: string}} Rendered template + */ + static renderTemplate(template, variables = {}, options = {}) { + const { sanitize = true, strict = false } = options; + + if (!template) { + const error = new Error('Template is required'); + error.statusCode = 400; + throw error; + } + + let subject = template.subject || ''; + let htmlContent = template.html_content || template.htmlContent || ''; + + // Get template variables + const templateVariables = template.variables || []; + + // Replace variables in subject and HTML + templateVariables.forEach((variable) => { + if (!variable || !variable.name) return; + + const varName = variable.name; + const value = variables[varName]; + + // In strict mode, throw error if variable is missing + if (strict && value === undefined) { + const error = new Error(`Missing required variable: ${varName}`); + error.statusCode = 400; + throw error; + } + + // Skip if value is not provided + if (value === undefined || value === null) { + return; + } + + // Handle image variables + let processedValue = value; + if (variable.type === 'image') { + // Use extracted image if available + const extractedKey = `${varName}_extracted`; + if (variables[extractedKey]) { + processedValue = variables[extractedKey]; + } else if (typeof value === 'string') { + // Try to extract image URL from value + const imageMatch = + value.match(/src=["']([^"']+)["']/i) || value.match(/https?:\/\/[^\s]+/i); + if (imageMatch) { + processedValue = imageMatch[1] || imageMatch[0]; + } + } + } + + // Replace all occurrences of {{variableName}} + const regex = new RegExp(`\\{\\{${varName}\\}\\}`, 'g'); + subject = subject.replace(regex, String(processedValue)); + htmlContent = htmlContent.replace(regex, String(processedValue)); + }); + + // Sanitize HTML if requested + if (sanitize) { + htmlContent = this.sanitizeHtml(htmlContent); + } + + return { + subject: subject.trim(), + htmlContent: htmlContent.trim(), + }; + } + + /** + * Validate that all required variables are provided. + * @param {Object} template - Template object + * @param {Object} variables - Variable values + * @returns {{isValid: boolean, errors: string[], missing: string[]}} Validation result + */ + static validateVariables(template, variables = {}) { + const errors = []; + const missing = []; + const templateVariables = template.variables || []; + + // Check for missing variables + templateVariables.forEach((variable) => { + if (!variable || !variable.name) return; + + const varName = variable.name; + if ( + !(varName in variables) || + variables[varName] === undefined || + variables[varName] === null + ) { + missing.push(varName); + errors.push(`Missing required variable: ${varName}`); + } + }); + + // Check for unused variables + const templateVariableNames = new Set(templateVariables.map((v) => v.name)); + Object.keys(variables).forEach((varName) => { + if (!templateVariableNames.has(varName) && !varName.endsWith('_extracted')) { + errors.push(`Unknown variable: ${varName}`); + } + }); + + return { + isValid: errors.length === 0, + errors, + missing, + }; + } + + /** + * Check if template has unreplaced variables. + * @param {string} content - Content to check (subject or HTML) + * @returns {string[]} Array of unreplaced variable names + */ + static getUnreplacedVariables(content) { + if (!content || typeof content !== 'string') { + return []; + } + + const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; + const unreplaced = []; + let match = variablePlaceholderRegex.exec(content); + + while (match !== null) { + const varName = match[1]; + if (!unreplaced.includes(varName)) { + unreplaced.push(varName); + } + match = variablePlaceholderRegex.exec(content); + } + + return unreplaced; + } + + /** + * Sanitize HTML content to prevent XSS attacks. + * @param {string} html - HTML content to sanitize + * @param {Object} options - Sanitization options + * @returns {string} Sanitized HTML + */ + static sanitizeHtml(html, options = {}) { + if (!html || typeof html !== 'string') { + return ''; + } + + const defaultOptions = { + allowedTags: [ + 'p', + 'br', + 'strong', + 'em', + 'b', + 'i', + 'u', + 'a', + 'ul', + 'ol', + 'li', + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', + 'div', + 'span', + 'img', + 'table', + 'thead', + 'tbody', + 'tr', + 'td', + 'th', + 'blockquote', + 'hr', + ], + allowedAttributes: { + a: ['href', 'title', 'target', 'rel'], + img: ['src', 'alt', 'title'], + '*': ['style', 'class'], + }, + allowedSchemes: ['http', 'https', 'mailto'], + allowedSchemesByTag: { + img: ['http', 'https', 'data'], + }, + ...options, + }; + + return sanitizeHtmlLib(html, defaultOptions); + } + + /** + * Extract variables from template content. + * @param {Object} template - Template object + * @returns {string[]} Array of variable names found in content + */ + static extractVariablesFromContent(template) { + const variables = new Set(); + const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; + + // Check subject + if (template.subject) { + let match = variablePlaceholderRegex.exec(template.subject); + while (match !== null) { + variables.add(match[1]); + match = variablePlaceholderRegex.exec(template.subject); + } + } + + // Reset regex + variablePlaceholderRegex.lastIndex = 0; + + // Check HTML content + const htmlContent = template.html_content || template.htmlContent || ''; + if (htmlContent) { + let match = variablePlaceholderRegex.exec(htmlContent); + while (match !== null) { + variables.add(match[1]); + match = variablePlaceholderRegex.exec(htmlContent); + } + } + + return Array.from(variables); + } } module.exports = EmailTemplateService; diff --git a/src/services/announcements/emails/templateRenderingService.js b/src/services/announcements/emails/templateRenderingService.js deleted file mode 100644 index d81c13cd7..000000000 --- a/src/services/announcements/emails/templateRenderingService.js +++ /dev/null @@ -1,243 +0,0 @@ -/** - * Template Rendering Service - Handles template rendering and variable substitution - * Provides server-side template rendering with proper sanitization - */ - -const sanitizeHtmlLib = require('sanitize-html'); - -class TemplateRenderingService { - /** - * Render template with variable values. - * Replaces {{variableName}} placeholders with actual values. - * @param {Object} template - Template object with subject and html_content - * @param {Object} variables - Object mapping variable names to values - * @param {Object} options - Rendering options (sanitize, strict) - * @returns {{subject: string, htmlContent: string}} Rendered template - */ - static renderTemplate(template, variables = {}, options = {}) { - const { sanitize = true, strict = false } = options; - - if (!template) { - const error = new Error('Template is required'); - error.statusCode = 400; - throw error; - } - - let subject = template.subject || ''; - let htmlContent = template.html_content || template.htmlContent || ''; - - // Get template variables - const templateVariables = template.variables || []; - - // Replace variables in subject and HTML - templateVariables.forEach((variable) => { - if (!variable || !variable.name) return; - - const varName = variable.name; - const value = variables[varName]; - - // In strict mode, throw error if variable is missing - if (strict && value === undefined) { - const error = new Error(`Missing required variable: ${varName}`); - error.statusCode = 400; - throw error; - } - - // Skip if value is not provided - if (value === undefined || value === null) { - return; - } - - // Handle image variables - let processedValue = value; - if (variable.type === 'image') { - // Use extracted image if available - const extractedKey = `${varName}_extracted`; - if (variables[extractedKey]) { - processedValue = variables[extractedKey]; - } else if (typeof value === 'string') { - // Try to extract image URL from value - const imageMatch = - value.match(/src=["']([^"']+)["']/i) || value.match(/https?:\/\/[^\s]+/i); - if (imageMatch) { - processedValue = imageMatch[1] || imageMatch[0]; - } - } - } - - // Replace all occurrences of {{variableName}} - const regex = new RegExp(`\\{\\{${varName}\\}\\}`, 'g'); - subject = subject.replace(regex, String(processedValue)); - htmlContent = htmlContent.replace(regex, String(processedValue)); - }); - - // Sanitize HTML if requested - if (sanitize) { - htmlContent = this.sanitizeHtml(htmlContent); - } - - return { - subject: subject.trim(), - htmlContent: htmlContent.trim(), - }; - } - - /** - * Validate that all required variables are provided. - * @param {Object} template - Template object - * @param {Object} variables - Variable values - * @returns {{isValid: boolean, errors: string[], missing: string[]}} Validation result - */ - static validateVariables(template, variables = {}) { - const errors = []; - const missing = []; - const templateVariables = template.variables || []; - - // Check for missing variables - templateVariables.forEach((variable) => { - if (!variable || !variable.name) return; - - const varName = variable.name; - if ( - !(varName in variables) || - variables[varName] === undefined || - variables[varName] === null - ) { - missing.push(varName); - errors.push(`Missing required variable: ${varName}`); - } - }); - - // Check for unused variables - const templateVariableNames = new Set(templateVariables.map((v) => v.name)); - Object.keys(variables).forEach((varName) => { - if (!templateVariableNames.has(varName) && !varName.endsWith('_extracted')) { - errors.push(`Unknown variable: ${varName}`); - } - }); - - return { - isValid: errors.length === 0, - errors, - missing, - }; - } - - /** - * Check if template has unreplaced variables. - * @param {string} content - Content to check (subject or HTML) - * @returns {string[]} Array of unreplaced variable names - */ - static getUnreplacedVariables(content) { - if (!content || typeof content !== 'string') { - return []; - } - - const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; - const unreplaced = []; - let match = variablePlaceholderRegex.exec(content); - - while (match !== null) { - const varName = match[1]; - if (!unreplaced.includes(varName)) { - unreplaced.push(varName); - } - match = variablePlaceholderRegex.exec(content); - } - - return unreplaced; - } - - /** - * Sanitize HTML content to prevent XSS attacks. - * @param {string} html - HTML content to sanitize - * @param {Object} options - Sanitization options - * @returns {string} Sanitized HTML - */ - static sanitizeHtml(html, options = {}) { - if (!html || typeof html !== 'string') { - return ''; - } - - const defaultOptions = { - allowedTags: [ - 'p', - 'br', - 'strong', - 'em', - 'b', - 'i', - 'u', - 'a', - 'ul', - 'ol', - 'li', - 'h1', - 'h2', - 'h3', - 'h4', - 'h5', - 'h6', - 'div', - 'span', - 'img', - 'table', - 'thead', - 'tbody', - 'tr', - 'td', - 'th', - 'blockquote', - 'hr', - ], - allowedAttributes: { - a: ['href', 'title', 'target', 'rel'], - img: ['src', 'alt', 'title'], - '*': ['style', 'class'], - }, - allowedSchemes: ['http', 'https', 'mailto'], - allowedSchemesByTag: { - img: ['http', 'https', 'data'], - }, - ...options, - }; - - return sanitizeHtmlLib(html, defaultOptions); - } - - /** - * Extract variables from template content. - * @param {Object} template - Template object - * @returns {string[]} Array of variable names found in content - */ - static extractVariablesFromContent(template) { - const variables = new Set(); - const variablePlaceholderRegex = /\{\{(\w+)\}\}/g; - - // Check subject - if (template.subject) { - let match = variablePlaceholderRegex.exec(template.subject); - while (match !== null) { - variables.add(match[1]); - match = variablePlaceholderRegex.exec(template.subject); - } - } - - // Reset regex - variablePlaceholderRegex.lastIndex = 0; - - // Check HTML content - const htmlContent = template.html_content || template.htmlContent || ''; - if (htmlContent) { - let match = variablePlaceholderRegex.exec(htmlContent); - while (match !== null) { - variables.add(match[1]); - match = variablePlaceholderRegex.exec(htmlContent); - } - } - - return Array.from(variables); - } -} - -module.exports = TemplateRenderingService; From d627919c5aabf3fc61db1f390f3ce0bf80ce3bb0 Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Thu, 13 Nov 2025 20:25:25 -0500 Subject: [PATCH 13/19] refactor(email): streamline email processing and enhance subscription management - Removed deprecated templateId from Email model to simplify email structure. - Updated email subscription list schema to enforce lowercase and trim email entries for consistency. - Introduced new methods for retrying failed emails and processing pending emails, improving error handling and user experience. - Enhanced email validation and normalization processes across controllers to ensure consistent data handling. - Refactored routes to improve clarity and organization of email-related endpoints. --- src/config.js | 1 - src/controllers/emailController.js | 283 ++++++++++++++---- src/controllers/emailOutboxController.js | 98 ------ src/controllers/emailTemplateController.js | 50 ---- src/models/email.js | 7 - src/models/emailSubcriptionList.js | 15 +- src/routes/emailOutboxRouter.js | 10 +- src/routes/emailRouter.js | 4 + src/routes/emailTemplateRouter.js | 1 - .../announcements/emails/emailBatchService.js | 65 +--- .../announcements/emails/emailService.js | 16 +- src/startup/routes.js | 2 +- 12 files changed, 264 insertions(+), 288 deletions(-) diff --git a/src/config.js b/src/config.js index ac38960c3..5f2c27d71 100644 --- a/src/config.js +++ b/src/config.js @@ -11,6 +11,5 @@ config.JWT_HEADER = { alg: 'RS256', typ: 'JWT', }; -config.FRONT_END_URL = process.env.FRONT_END_URL; module.exports = config; diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index ad43de951..0b4c9d054 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -78,12 +78,14 @@ const sendEmail = async (req, res) => { ); // Create EmailBatch items with all recipients (validates recipients, counts, email format) + // Enforce recipient limit for specific recipient requests const recipientObjects = recipientsArray.map((emailAddr) => ({ email: emailAddr })); await EmailBatchService.createEmailBatches( createdEmail._id, recipientObjects, { emailType: EMAIL_CONFIG.EMAIL_TYPES.BCC, + enforceRecipientLimit: true, // Enforce limit for specific recipients }, session, ); @@ -96,7 +98,7 @@ const sendEmail = async (req, res) => { return res.status(200).json({ success: true, - message: `Email created successfully for ${recipientsArray.length} recipient(s)`, + message: `Email created successfully for ${recipientsArray.length} recipient(s) and will be processed shortly`, }); } catch (error) { logger.logException(error, 'Error creating email'); @@ -166,7 +168,7 @@ const sendEmailToSubscribers = async (req, res) => { }); const emailSubscribers = await EmailSubcriptionList.find({ - email: { $exists: true, $ne: '' }, + email: { $exists: true, $nin: [null, ''] }, isConfirmed: true, emailSubscriptions: true, }); @@ -195,11 +197,13 @@ const sendEmailToSubscribers = async (req, res) => { ]; // Create EmailBatch items with all recipients (validates recipients, counts, email format) + // Skip recipient limit for broadcast to all subscribers await EmailBatchService.createEmailBatches( createdEmail._id, allRecipients, { emailType: EMAIL_CONFIG.EMAIL_TYPES.BCC, + enforceRecipientLimit: false, // Skip limit for broadcast }, session, ); @@ -212,7 +216,7 @@ const sendEmailToSubscribers = async (req, res) => { return res.status(200).json({ success: true, - message: `Broadcast email created successfully for ${totalRecipients} recipient(s)`, + message: `Broadcast email created successfully for ${totalRecipients} recipient(s) and will be processed shortly`, }); } catch (error) { logger.logException(error, 'Error creating broadcast email'); @@ -292,7 +296,7 @@ const resendEmail = async (req, res) => { }); const emailSubscribers = await EmailSubcriptionList.find({ - email: { $exists: true, $ne: '' }, + email: { $exists: true, $nin: [null, ''] }, isConfirmed: true, emailSubscriptions: true, }); @@ -319,7 +323,7 @@ const resendEmail = async (req, res) => { allRecipients = recipientsArray.map((email) => ({ email })); } else if (recipientOption === 'same') { // Get recipients from original email's EmailBatch items - const emailBatchItems = await EmailBatchService.getEmailBatchesByEmailId(emailId); + const emailBatchItems = await EmailBatchService.getBatchesForEmail(emailId); if (!emailBatchItems || emailBatchItems.length === 0) { return res .status(404) @@ -360,11 +364,14 @@ const resendEmail = async (req, res) => { ); // Create EmailBatch items + // Enforce limit only for 'specific' recipient option, skip for 'all' and 'same' (broadcast scenarios) + const shouldEnforceLimit = recipientOption === 'specific'; await EmailBatchService.createEmailBatches( createdEmail._id, allRecipients, { emailType: EMAIL_CONFIG.EMAIL_TYPES.BCC, + enforceRecipientLimit: shouldEnforceLimit, }, session, ); @@ -377,7 +384,7 @@ const resendEmail = async (req, res) => { return res.status(200).json({ success: true, - message: `Email created for resend successfully to ${allRecipients.length} recipient(s)`, + message: `Email created for resend successfully to ${allRecipients.length} recipient(s) and will be processed shortly`, data: { emailId: newEmail._id, recipientCount: allRecipients.length, @@ -398,8 +405,150 @@ const resendEmail = async (req, res) => { } }; +/** + * Retry a parent Email by resetting all FAILED EmailBatch items to PENDING. + * - Processes the email immediately asynchronously. + * @param {import('express').Request} req + * @param {import('express').Response} res + */ +const retryEmail = async (req, res) => { + try { + const { emailId } = req.params; + + // Requestor is required for permission check + if (!req?.body?.requestor?.requestorId) { + return res.status(401).json({ success: false, message: 'Missing requestor' }); + } + + // Validate emailId is a valid ObjectId + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + return res.status(400).json({ + success: false, + message: 'Invalid Email ID', + }); + } + + // Permission check - retrying emails requires sendEmails permission + const canRetryEmail = await hasPermission(req.body.requestor, 'sendEmails'); + if (!canRetryEmail) { + return res + .status(403) + .json({ success: false, message: 'You are not authorized to retry emails.' }); + } + + // Get the Email (service throws error if not found) + const email = await EmailService.getEmailById(emailId, null, true); + + // Only allow retry for emails in final states (FAILED or PROCESSED) + const allowedRetryStatuses = [ + EMAIL_CONFIG.EMAIL_STATUSES.FAILED, + EMAIL_CONFIG.EMAIL_STATUSES.PROCESSED, + ]; + + if (!allowedRetryStatuses.includes(email.status)) { + return res.status(400).json({ + success: false, + message: `Email must be in FAILED or PROCESSED status to retry. Current status: ${email.status}`, + }); + } + + // Get all FAILED EmailBatch items (service validates emailId) + const failedItems = await EmailBatchService.getFailedBatchesForEmail(emailId); + + if (failedItems.length === 0) { + logger.logInfo(`Email ${emailId} has no failed EmailBatch items to retry`); + return res.status(200).json({ + success: true, + message: 'No failed EmailBatch items to retry', + data: { + emailId: email._id, + failedItemsRetried: 0, + }, + }); + } + + logger.logInfo(`Retrying ${failedItems.length} failed EmailBatch items: ${emailId}`); + + // Mark parent Email as PENDING for retry + await EmailService.markEmailPending(emailId); + + // Reset each failed item to PENDING + await Promise.all( + failedItems.map(async (item) => { + await EmailBatchService.resetEmailBatchForRetry(item._id); + }), + ); + + logger.logInfo( + `Successfully reset Email ${emailId} and ${failedItems.length} failed EmailBatch items to PENDING for retry`, + ); + + // Add email to queue for processing (non-blocking, sequential processing) + emailProcessor.queueEmail(emailId); + + res.status(200).json({ + success: true, + message: `Successfully reset ${failedItems.length} failed EmailBatch items for retry`, + data: { + emailId: email._id, + failedItemsRetried: failedItems.length, + }, + }); + } catch (error) { + logger.logException(error, 'Error retrying Email'); + const statusCode = error.statusCode || 500; + return res.status(statusCode).json({ + success: false, + message: error.message || 'Error retrying Email', + }); + } +}; + +/** + * Manually trigger processing of pending and stuck emails. + * - Resets stuck emails (SENDING status) to PENDING + * - Resets stuck batches (SENDING status) to PENDING + * - Queues all PENDING emails for processing + * @param {import('express').Request} req + * @param {import('express').Response} res + */ +const processPendingAndStuckEmails = async (req, res) => { + // Requestor is required for permission check + if (!req?.body?.requestor?.requestorId) { + return res.status(401).json({ success: false, message: 'Missing requestor' }); + } + + // Permission check - processing emails requires sendEmails permission + const canProcessEmails = await hasPermission(req.body.requestor, 'sendEmails'); + if (!canProcessEmails) { + return res + .status(403) + .json({ success: false, message: 'You are not authorized to process emails.' }); + } + + try { + logger.logInfo('Manual trigger: Starting processing of pending and stuck emails...'); + + // Trigger the processor to handle pending and stuck emails + await emailProcessor.processPendingAndStuckEmails(); + + return res.status(200).json({ + success: true, + message: 'Processing of pending and stuck emails triggered successfully', + }); + } catch (error) { + logger.logException(error, 'Error triggering processing of pending and stuck emails'); + const statusCode = error.statusCode || 500; + return res.status(statusCode).json({ + success: false, + message: error.message || 'Error triggering processing of pending and stuck emails', + }); + } +}; + /** * Update the current user's emailSubscriptions preference. + * - Normalizes email to lowercase for consistent lookups. * @param {import('express').Request} req * @param {import('express').Response} res */ @@ -421,8 +570,11 @@ const updateEmailSubscriptions = async (req, res) => { return res.status(400).json({ success: false, message: 'Invalid email address' }); } + // Normalize email for consistent lookup + const normalizedEmail = email.trim().toLowerCase(); + const user = await userProfile.findOneAndUpdate( - { email }, + { email: normalizedEmail }, { emailSubscriptions }, { new: true }, ); @@ -463,9 +615,9 @@ const addNonHgnEmailSubscription = async (req, res) => { return res.status(400).json({ success: false, message: 'Invalid email address' }); } - // Check if email already exists (case-insensitive) + // Check if email already exists (direct match since schema enforces lowercase) const existingSubscription = await EmailSubcriptionList.findOne({ - email: { $regex: new RegExp(`^${normalizedEmail}$`, 'i') }, + email: normalizedEmail, }); if (existingSubscription) { @@ -477,8 +629,7 @@ const addNonHgnEmailSubscription = async (req, res) => { if (hgnUser) { return res.status(400).json({ success: false, - message: - 'You are already a member of the HGN community. Please use the HGN account profile page to subscribe to email updates.', + message: 'Please use the HGN account profile page to subscribe to email updates.', }); } @@ -494,16 +645,45 @@ const addNonHgnEmailSubscription = async (req, res) => { const payload = { email: normalizedEmail }; const token = jwt.sign(payload, jwtSecret, { expiresIn: '24h' }); // Fixed: was '360' (invalid) - if (!config.FRONT_END_URL) { - logger.logException(new Error('FRONT_END_URL is not configured'), 'Configuration error'); + // Get frontend URL from request origin + const getFrontendUrl = () => { + // Try to get from request origin header first + const origin = req.get('origin') || req.get('referer'); + if (origin) { + try { + const url = new URL(origin); + return `${url.protocol}//${url.host}`; + } catch (error) { + logger.logException(error, 'Error parsing request origin'); + } + } + // Fallback to config or construct from request + if (config.FRONT_END_URL) { + return config.FRONT_END_URL; + } + // Last resort: construct from request + const protocol = req.protocol || 'https'; + const host = req.get('host'); + if (host) { + return `${protocol}://${host}`; + } + return null; + }; + + const frontendUrl = getFrontendUrl(); + if (!frontendUrl) { + logger.logException( + new Error('Unable to determine frontend URL from request'), + 'Configuration error', + ); return res .status(500) - .json({ success: false, message: 'Server configuration error. Please contact support.' }); + .json({ success: false, message: 'Server Error. Please contact support.' }); } const emailContent = `

Thank you for subscribing to our email updates!

-

Click here to confirm your email

+

Click here to confirm your email

`; try { @@ -541,7 +721,8 @@ const addNonHgnEmailSubscription = async (req, res) => { /** * Confirm a non-HGN email subscription using a signed token. - * - Creates or updates the subscriber record as confirmed. + * - Only confirms existing unconfirmed subscriptions. + * - Returns error if subscription doesn't exist (user must subscribe first). * @param {import('express').Request} req * @param {import('express').Response} res */ @@ -564,45 +745,45 @@ const confirmNonHgnEmailSubscription = async (req, res) => { return res.status(400).json({ success: false, message: 'Invalid token payload' }); } - // Normalize email + // Normalize email (schema enforces lowercase, but normalize here for consistency) const normalizedEmail = email.trim().toLowerCase(); - try { - // Update existing subscription to confirmed, or create new one - const existingSubscription = await EmailSubcriptionList.findOne({ - email: { $regex: new RegExp(`^${normalizedEmail}$`, 'i') }, - }); + // Find existing subscription (direct match since schema enforces lowercase) + const existingSubscription = await EmailSubcriptionList.findOne({ + email: normalizedEmail, + }); - if (existingSubscription) { - existingSubscription.isConfirmed = true; - existingSubscription.confirmedAt = new Date(); - existingSubscription.emailSubscriptions = true; - await existingSubscription.save(); - } else { - const newEmailList = new EmailSubcriptionList({ - email: normalizedEmail, - isConfirmed: true, - confirmedAt: new Date(), - emailSubscriptions: true, - }); - await newEmailList.save(); - } + if (!existingSubscription) { + return res.status(404).json({ + success: false, + message: 'Subscription not found. Please subscribe first using the subscription form.', + }); + } - return res - .status(200) - .json({ success: true, message: 'Email subscription confirmed successfully' }); - } catch (error) { - if (error.code === 11000) { - // Race condition - email was already confirmed/subscribed - return res - .status(200) - .json({ success: true, message: 'Email subscription already confirmed' }); - } - throw error; + // If already confirmed, return success (idempotent) + if (existingSubscription.isConfirmed) { + return res.status(200).json({ + success: true, + message: 'Email subscription already confirmed', + }); } + + // Update subscription to confirmed + existingSubscription.isConfirmed = true; + existingSubscription.confirmedAt = new Date(); + existingSubscription.emailSubscriptions = true; + await existingSubscription.save(); + + return res + .status(200) + .json({ success: true, message: 'Email subscription confirmed successfully' }); } catch (error) { logger.logException(error, 'Error confirming email subscription'); - return res.status(500).json({ success: false, message: 'Error confirming email subscription' }); + const statusCode = error.statusCode || 500; + return res.status(statusCode).json({ + success: false, + message: error.message || 'Error confirming email subscription', + }); } }; @@ -626,9 +807,9 @@ const removeNonHgnEmailSubscription = async (req, res) => { return res.status(400).json({ success: false, message: 'Invalid email address' }); } - // Try to delete the email subscription (case-insensitive) + // Try to delete the email subscription (direct match since schema enforces lowercase) const deletedEntry = await EmailSubcriptionList.findOneAndDelete({ - email: { $regex: new RegExp(`^${normalizedEmail}$`, 'i') }, + email: normalizedEmail, }); // If not found, respond accordingly @@ -653,4 +834,6 @@ module.exports = { addNonHgnEmailSubscription, removeNonHgnEmailSubscription, confirmNonHgnEmailSubscription, + retryEmail, + processPendingAndStuckEmails, }; diff --git a/src/controllers/emailOutboxController.js b/src/controllers/emailOutboxController.js index 7e32f6f09..10726647e 100644 --- a/src/controllers/emailOutboxController.js +++ b/src/controllers/emailOutboxController.js @@ -1,10 +1,7 @@ -const mongoose = require('mongoose'); const EmailBatchService = require('../services/announcements/emails/emailBatchService'); const EmailService = require('../services/announcements/emails/emailService'); -const emailProcessor = require('../services/announcements/emails/emailProcessor'); const { hasPermission } = require('../utilities/permissions'); const logger = require('../startup/logger'); -const { EMAIL_CONFIG } = require('../config/emailConfig'); /** * Get all announcement Email records (parent documents) - Outbox view. @@ -71,102 +68,7 @@ const getEmailDetails = async (req, res) => { } }; -/** - * Retry a parent Email by resetting all FAILED EmailBatch items to PENDING. - * - Processes the email immediately asynchronously. - * @param {import('express').Request} req - * @param {import('express').Response} res - */ -const retryEmail = async (req, res) => { - try { - const { emailId } = req.params; - - // Validate emailId is a valid ObjectId - if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { - return res.status(400).json({ - success: false, - message: 'Invalid Email ID', - }); - } - - // Permission check - retrying emails requires sendEmails permission - const canRetryEmail = await hasPermission(req.body.requestor, 'sendEmails'); - if (!canRetryEmail) { - return res - .status(403) - .json({ success: false, message: 'You are not authorized to retry emails.' }); - } - - // Get the Email (service throws error if not found) - const email = await EmailService.getEmailById(emailId, null, true); - - // Only allow retry for emails in final states (FAILED or PROCESSED) - const allowedRetryStatuses = [ - EMAIL_CONFIG.EMAIL_STATUSES.FAILED, - EMAIL_CONFIG.EMAIL_STATUSES.PROCESSED, - ]; - - if (!allowedRetryStatuses.includes(email.status)) { - return res.status(400).json({ - success: false, - message: `Email must be in FAILED or PROCESSED status to retry. Current status: ${email.status}`, - }); - } - - // Get all FAILED EmailBatch items (service validates emailId) - const failedItems = await EmailBatchService.getFailedBatchesForEmail(emailId); - - if (failedItems.length === 0) { - logger.logInfo(`Email ${emailId} has no failed EmailBatch items to retry`); - return res.status(200).json({ - success: true, - message: 'No failed EmailBatch items to retry', - data: { - emailId: email._id, - failedItemsRetried: 0, - }, - }); - } - - logger.logInfo(`Retrying ${failedItems.length} failed EmailBatch items: ${emailId}`); - - // Mark parent Email as PENDING for retry - await EmailService.markEmailPending(emailId); - - // Reset each failed item to PENDING - await Promise.all( - failedItems.map(async (item) => { - await EmailBatchService.resetEmailBatchForRetry(item._id); - }), - ); - - logger.logInfo( - `Successfully reset Email ${emailId} and ${failedItems.length} failed EmailBatch items to PENDING for retry`, - ); - - // Add email to queue for processing (non-blocking, sequential processing) - emailProcessor.queueEmail(emailId); - - res.status(200).json({ - success: true, - message: `Successfully reset ${failedItems.length} failed EmailBatch items for retry`, - data: { - emailId: email._id, - failedItemsRetried: failedItems.length, - }, - }); - } catch (error) { - logger.logException(error, 'Error retrying Email'); - const statusCode = error.statusCode || 500; - return res.status(statusCode).json({ - success: false, - message: error.message || 'Error retrying Email', - }); - } -}; - module.exports = { getEmails, getEmailDetails, - retryEmail, }; diff --git a/src/controllers/emailTemplateController.js b/src/controllers/emailTemplateController.js index 124af9ca1..ccb75ac30 100644 --- a/src/controllers/emailTemplateController.js +++ b/src/controllers/emailTemplateController.js @@ -339,55 +339,6 @@ const previewTemplate = async (req, res) => { } }; -/** - * Validate template structure and variables. - * @param {import('express').Request} req - * @param {import('express').Response} res - */ -const validateTemplate = async (req, res) => { - try { - // Permission check - if (!req?.body?.requestor?.requestorId && !req?.user?.userid) { - return res.status(401).json({ - success: false, - message: 'Missing requestor', - }); - } - - const requestor = req.body.requestor || req.user; - const canViewTemplates = await hasPermission(requestor, 'sendEmails'); - if (!canViewTemplates) { - return res.status(403).json({ - success: false, - message: 'You are not authorized to validate email templates.', - }); - } - - const { id } = req.params; - - // Service validates ID and throws error with statusCode if not found - const template = await EmailTemplateService.getTemplateById(id, { - populate: false, - }); - - // Validate template data - const validation = EmailTemplateService.validateTemplateData(template); - - res.status(200).json({ - success: true, - isValid: validation.isValid, - errors: validation.errors || [], - }); - } catch (error) { - logger.logException(error, 'Error validating email template'); - const statusCode = error.statusCode || 500; - return res.status(statusCode).json({ - success: false, - message: error.message || 'Error validating email template', - }); - } -}; - module.exports = { getAllEmailTemplates, getEmailTemplateById, @@ -395,5 +346,4 @@ module.exports = { updateEmailTemplate, deleteEmailTemplate, previewTemplate, - validateTemplate, }; diff --git a/src/models/email.js b/src/models/email.js index 5d0d7e00f..63d9300e9 100644 --- a/src/models/email.js +++ b/src/models/email.js @@ -23,12 +23,6 @@ const EmailSchema = new Schema({ default: EMAIL_CONFIG.EMAIL_STATUSES.PENDING, index: true, }, - // Optional template reference for tracking which template was used - templateId: { - type: Schema.Types.ObjectId, - ref: 'EmailTemplate', - index: true, - }, createdBy: { type: Schema.Types.ObjectId, ref: 'userProfile', @@ -55,6 +49,5 @@ EmailSchema.index({ status: 1, createdAt: 1 }); EmailSchema.index({ createdBy: 1, createdAt: -1 }); EmailSchema.index({ startedAt: 1 }); EmailSchema.index({ completedAt: 1 }); -EmailSchema.index({ templateId: 1, createdAt: -1 }); // For template usage tracking module.exports = mongoose.model('Email', EmailSchema, 'emails'); diff --git a/src/models/emailSubcriptionList.js b/src/models/emailSubcriptionList.js index 263234124..e07bb3a33 100644 --- a/src/models/emailSubcriptionList.js +++ b/src/models/emailSubcriptionList.js @@ -4,7 +4,14 @@ const mongoose = require('mongoose'); const { Schema } = mongoose; const emailSubscriptionListSchema = new Schema({ - email: { type: String, required: true, unique: true }, + email: { + type: String, + required: [true, 'Email is required'], + unique: true, + lowercase: true, + trim: true, + index: true, + }, emailSubscriptions: { type: Boolean, default: true, @@ -12,10 +19,11 @@ const emailSubscriptionListSchema = new Schema({ isConfirmed: { type: Boolean, default: false, + index: true, }, subscribedAt: { type: Date, - default: Date.now, + default: () => new Date(), }, confirmedAt: { type: Date, @@ -23,6 +31,9 @@ const emailSubscriptionListSchema = new Schema({ }, }); +// Compound index for common queries (isConfirmed + emailSubscriptions) +emailSubscriptionListSchema.index({ isConfirmed: 1, emailSubscriptions: 1 }); + module.exports = mongoose.model( 'emailSubscriptions', emailSubscriptionListSchema, diff --git a/src/routes/emailOutboxRouter.js b/src/routes/emailOutboxRouter.js index 4fb0422c6..4ad1c0964 100644 --- a/src/routes/emailOutboxRouter.js +++ b/src/routes/emailOutboxRouter.js @@ -4,13 +4,7 @@ const router = express.Router(); const emailOutboxController = require('../controllers/emailOutboxController'); -// GET /api/email-outbox - Get all sent emails (outbox list) -router.get('/', emailOutboxController.getEmails); - -// POST /api/email-outbox/:emailId/retry - Retry failed email batches -router.post('/:emailId/retry', emailOutboxController.retryEmail); - -// GET /api/email-outbox/:emailId - Get email details with batches -router.get('/:emailId', emailOutboxController.getEmailDetails); +router.get('/email-outbox', emailOutboxController.getEmails); +router.get('/email-outbox/:emailId', emailOutboxController.getEmailDetails); module.exports = router; diff --git a/src/routes/emailRouter.js b/src/routes/emailRouter.js index 04ad4fb63..e47d25630 100644 --- a/src/routes/emailRouter.js +++ b/src/routes/emailRouter.js @@ -7,6 +7,8 @@ const { addNonHgnEmailSubscription, removeNonHgnEmailSubscription, confirmNonHgnEmailSubscription, + retryEmail, + processPendingAndStuckEmails, } = require('../controllers/emailController'); const routes = function () { @@ -15,6 +17,8 @@ const routes = function () { emailRouter.route('/send-emails').post(sendEmail); emailRouter.route('/broadcast-emails').post(sendEmailToSubscribers); emailRouter.route('/resend-email').post(resendEmail); + emailRouter.route('/retry-email/:emailId').post(retryEmail); + emailRouter.route('/process-pending-and-stuck-emails').post(processPendingAndStuckEmails); emailRouter.route('/update-email-subscriptions').post(updateEmailSubscriptions); emailRouter.route('/add-non-hgn-email-subscription').post(addNonHgnEmailSubscription); diff --git a/src/routes/emailTemplateRouter.js b/src/routes/emailTemplateRouter.js index 061998f17..01a0dc674 100644 --- a/src/routes/emailTemplateRouter.js +++ b/src/routes/emailTemplateRouter.js @@ -10,6 +10,5 @@ router.post('/email-templates', emailTemplateController.createEmailTemplate); router.put('/email-templates/:id', emailTemplateController.updateEmailTemplate); router.delete('/email-templates/:id', emailTemplateController.deleteEmailTemplate); router.post('/email-templates/:id/preview', emailTemplateController.previewTemplate); -router.post('/email-templates/:id/validate', emailTemplateController.validateTemplate); module.exports = router; diff --git a/src/services/announcements/emails/emailBatchService.js b/src/services/announcements/emails/emailBatchService.js index 4ee7e4d86..c508eb64c 100644 --- a/src/services/announcements/emails/emailBatchService.js +++ b/src/services/announcements/emails/emailBatchService.js @@ -14,12 +14,6 @@ const { const logger = require('../../../startup/logger'); class EmailBatchService { - // Debounce map for auto-sync to prevent race conditions - // Key: emailId, Value: timeoutId - static syncDebounceMap = new Map(); - - static SYNC_DEBOUNCE_MS = 500; // Wait 500ms before syncing (allows multiple batch updates to complete) - /** * Create EmailBatch items for a parent Email. * - Validates parent Email ID, normalizes recipients and chunks by configured size. @@ -50,8 +44,13 @@ class EmailBatchService { throw error; } - // Validate recipient count limit - if (normalizedRecipients.length > EMAIL_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST) { + // Validate recipient count limit (only enforce when enforceRecipientLimit is true) + // Default to true to enforce limit for specific recipient requests + const enforceRecipientLimit = config.enforceRecipientLimit !== false; + if ( + enforceRecipientLimit && + normalizedRecipients.length > EMAIL_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST + ) { const error = new Error( `A maximum of ${EMAIL_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST} recipients are allowed per request`, ); @@ -149,9 +148,9 @@ class EmailBatchService { const transformedBatches = emailBatches.map((batch) => ({ _id: batch._id, emailId: batch.emailId, - recipients: batch.recipients || [], + recipients: batch.recipients || null, status: batch.status, - attempts: batch.attempts || 0, + attempts: batch.attempts || null, lastAttemptedAt: batch.lastAttemptedAt, sentAt: batch.sentAt, failedAt: batch.failedAt, @@ -206,13 +205,6 @@ class EmailBatchService { }).sort({ createdAt: 1 }); } - /** - * Alias of getBatchesForEmail for naming consistency. - */ - static async getEmailBatchesByEmailId(emailId) { - return this.getBatchesForEmail(emailId); - } - /** * Get EmailBatch by ID. * @param {string|ObjectId} batchId - EmailBatch ObjectId. @@ -294,10 +286,6 @@ class EmailBatchService { throw error; } - // Auto-sync parent Email status with debouncing (fire-and-forget to avoid blocking) - // When batches are reset for retry, parent status might change from FAILED/PROCESSED to PENDING/SENDING - this.debouncedSyncParentEmailStatus(updated.emailId); - return updated; } @@ -390,9 +378,6 @@ class EmailBatchService { return currentBatch; } - // Auto-sync parent Email status with debouncing (fire-and-forget to avoid blocking) - this.debouncedSyncParentEmailStatus(updated.emailId); - return updated; } @@ -458,9 +443,6 @@ class EmailBatchService { return currentBatch; } - // Auto-sync parent Email status with debouncing (fire-and-forget to avoid blocking) - this.debouncedSyncParentEmailStatus(updated.emailId); - return updated; } @@ -519,35 +501,6 @@ class EmailBatchService { return EMAIL_CONFIG.EMAIL_STATUSES.SENDING; } - /** - * Debounced version of syncParentEmailStatus to prevent race conditions. - * Waits for a short period before syncing to allow multiple batch updates to complete. - * @param {string|ObjectId} emailId - Parent Email ObjectId. - * @returns {void} - */ - static debouncedSyncParentEmailStatus(emailId) { - if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { - return; - } - - const emailIdStr = emailId.toString(); - - // Clear existing timeout if any - if (this.syncDebounceMap.has(emailIdStr)) { - clearTimeout(this.syncDebounceMap.get(emailIdStr)); - } - - // Set new timeout for syncing - const timeoutId = setTimeout(() => { - this.syncDebounceMap.delete(emailIdStr); - this.syncParentEmailStatus(emailIdStr).catch((syncError) => { - logger.logException(syncError, `Error syncing parent Email status for ${emailIdStr}`); - }); - }, this.SYNC_DEBOUNCE_MS); - - this.syncDebounceMap.set(emailIdStr, timeoutId); - } - /** * Synchronize parent Email status based on child EmailBatch statuses. * - Determines status from batches and updates parent Email diff --git a/src/services/announcements/emails/emailService.js b/src/services/announcements/emails/emailService.js index eddb39ad2..f8863a65d 100644 --- a/src/services/announcements/emails/emailService.js +++ b/src/services/announcements/emails/emailService.js @@ -7,12 +7,12 @@ class EmailService { /** * Create a parent Email document for announcements. * Validates and trims large text fields and supports optional transaction sessions. - * @param {{subject: string, htmlContent: string, createdBy: string|ObjectId, templateId?: string|ObjectId}} param0 + * @param {{subject: string, htmlContent: string, createdBy: string|ObjectId}} param0 * @param {import('mongoose').ClientSession|null} session * @returns {Promise} Created Email document. * @throws {Error} If validation fails */ - static async createEmail({ subject, htmlContent, createdBy, templateId }, session = null) { + static async createEmail({ subject, htmlContent, createdBy }, session = null) { // Validate required fields if (!subject || typeof subject !== 'string' || !subject.trim()) { const error = new Error('Subject is required'); @@ -51,13 +51,6 @@ class EmailService { throw error; } - // Validate templateId if provided - if (templateId && !mongoose.Types.ObjectId.isValid(templateId)) { - const error = new Error('Invalid templateId'); - error.statusCode = 400; - throw error; - } - const normalizedHtml = htmlContent.trim(); const emailData = { @@ -66,11 +59,6 @@ class EmailService { createdBy, }; - // Add template reference if provided - if (templateId) { - emailData.templateId = templateId; - } - const email = new Email(emailData); // Save with session if provided for transaction support diff --git a/src/startup/routes.js b/src/startup/routes.js index 974af5e93..4aece1104 100644 --- a/src/startup/routes.js +++ b/src/startup/routes.js @@ -333,7 +333,7 @@ module.exports = function (app) { app.use('/api', informationRouter); app.use('/api', mouseoverTextRouter); app.use('/api', permissionChangeLogRouter); - app.use('/api/email-outbox', emailOutboxRouter); + app.use('/api', emailOutboxRouter); app.use('/api', emailRouter); app.use('/api', isEmailExistsRouter); app.use('/api', faqRouter); From 5090356a6d125e93e5a3968623356d37474a7176 Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Thu, 13 Nov 2025 20:30:24 -0500 Subject: [PATCH 14/19] refactor(email): update email subscription handling and validation - Enhanced validation for email subscriptions, including checks for missing or invalid email fields. - Improved error handling for subscription confirmation and removal processes, ensuring appropriate status codes are returned. - Updated positive test cases to reflect changes in subscription management and email normalization. - Removed deprecated sendEmailToAll functionality, streamlining email operations. --- .../addNonHgnEmailSubscription.md | 37 +++++++--- .../confirmNonHgnEmailSubscription.md | 31 ++++++--- .../processPendingAndStuckEmails.md | 33 +++++++++ .../removeNonHgnEmailSubscription.md | 27 ++++++-- requirements/emailController/resendEmail.md | 69 +++++++++++++++++++ requirements/emailController/retryEmail.md | 51 ++++++++++++++ requirements/emailController/sendEmail.md | 40 ++++++++++- .../emailController/sendEmailToAll.md | 26 ------- .../emailController/sendEmailToSubscribers.md | 50 ++++++++++++++ .../updateEmailSubscription.md | 29 ++++++-- .../emailOutboxController/getEmailDetails.md | 39 +++++++++++ .../emailOutboxController/getEmails.md | 30 ++++++++ .../createEmailTemplate.md | 51 ++++++++++++++ .../deleteEmailTemplate.md | 33 +++++++++ .../getAllEmailTemplates.md | 33 +++++++++ .../getEmailTemplateById.md | 30 ++++++++ .../previewTemplate.md | 45 ++++++++++++ .../updateEmailTemplate.md | 54 +++++++++++++++ 18 files changed, 650 insertions(+), 58 deletions(-) create mode 100644 requirements/emailController/processPendingAndStuckEmails.md create mode 100644 requirements/emailController/resendEmail.md create mode 100644 requirements/emailController/retryEmail.md delete mode 100644 requirements/emailController/sendEmailToAll.md create mode 100644 requirements/emailController/sendEmailToSubscribers.md create mode 100644 requirements/emailOutboxController/getEmailDetails.md create mode 100644 requirements/emailOutboxController/getEmails.md create mode 100644 requirements/emailTemplateController/createEmailTemplate.md create mode 100644 requirements/emailTemplateController/deleteEmailTemplate.md create mode 100644 requirements/emailTemplateController/getAllEmailTemplates.md create mode 100644 requirements/emailTemplateController/getEmailTemplateById.md create mode 100644 requirements/emailTemplateController/previewTemplate.md create mode 100644 requirements/emailTemplateController/updateEmailTemplate.md diff --git a/requirements/emailController/addNonHgnEmailSubscription.md b/requirements/emailController/addNonHgnEmailSubscription.md index f5748f142..e22c645e3 100644 --- a/requirements/emailController/addNonHgnEmailSubscription.md +++ b/requirements/emailController/addNonHgnEmailSubscription.md @@ -5,19 +5,40 @@ 1. ❌ **Returns error 400 if `email` field is missing from the request** - Ensures that the function checks for the presence of the `email` field in the request body and responds with a `400` status code if it's missing. -2. ❌ **Returns error 400 if the provided `email` already exists in the subscription list** +2. ❌ **Returns error 400 if the provided `email` is invalid** + - Verifies that the function validates email format using `isValidEmailAddress` and responds with a `400` status code for invalid emails. + +3. ❌ **Returns error 400 if the provided `email` already exists in the subscription list** - This case checks that the function responds with a `400` status code and a message indicating that the email is already subscribed. -3. ❌ **Returns error 500 if there is an internal error while checking the subscription list** - - Covers scenarios where there's an issue querying the `EmailSubscriptionList` collection for the provided email (e.g., database connection issues). +4. ❌ **Returns error 400 if the email is already an HGN user** + - Verifies that the function checks if the email belongs to an existing HGN user and responds with a `400` status code, directing them to use the HGN account profile page. + +5. ❌ **Returns error 500 if there is an internal error while checking the subscription list** + - Covers scenarios where there's an issue querying the `EmailSubcriptionList` collection for the provided email (e.g., database connection issues). -4. ❌ **Returns error 500 if there is an error sending the confirmation email** +6. ❌ **Returns error 500 if `FRONT_END_URL` cannot be determined from request** + - Verifies that the function handles cases where the frontend URL cannot be determined from request headers, config, or request information. + +7. ❌ **Returns error 500 if there is an error sending the confirmation email** - This case handles any issues that occur while calling the `emailSender` function, such as network errors or service unavailability. +8. ❌ **Returns error 400 if there's a duplicate key error (race condition)** + - Handles MongoDB duplicate key errors that might occur if the subscription is created simultaneously by multiple requests. + ## Positive Cases -1. ❌ **Returns status 200 when a new email is successfully subscribed** - - Ensures that the function successfully creates a JWT token, constructs the email, and sends the subscription confirmation email to the user. +1. ✅ **Returns status 200 when a new email is successfully subscribed** + - Ensures that the function successfully creates an unconfirmed subscription record, generates a JWT token, and sends the subscription confirmation email to the user. + +2. ✅ **Creates subscription with correct initial state** + - Verifies that the subscription is created with `isConfirmed: false`, `emailSubscriptions: true`, and proper normalization (lowercase email). + +3. ✅ **Successfully sends a confirmation email containing the correct link** + - Verifies that the generated JWT token is correctly included in the confirmation link, and the frontend URL is dynamically determined from the request origin. + +4. ✅ **Returns success even if confirmation email fails to send** + - Ensures that if the subscription is saved to the database but the confirmation email fails, the function still returns success (subscription is already saved). -2. ❌ **Successfully sends a confirmation email containing the correct link** - - Verifies that the generated JWT token is correctly included in the confirmation link sent to the user in the email body. +5. ❌ **Correctly normalizes email to lowercase** + - Ensures that email addresses are stored in lowercase format, matching the schema's lowercase enforcement. diff --git a/requirements/emailController/confirmNonHgnEmailSubscription.md b/requirements/emailController/confirmNonHgnEmailSubscription.md index d5e1367af..efd368f67 100644 --- a/requirements/emailController/confirmNonHgnEmailSubscription.md +++ b/requirements/emailController/confirmNonHgnEmailSubscription.md @@ -1,18 +1,29 @@ -# Confirm Non-HGN Email Subscription Function Tests +# Confirm Non-HGN Email Subscription Function ## Negative Cases -1. ✅ **Returns error 400 if `token` field is missing from the request** - - (Test: `should return 400 if token is not provided`) -2. ✅ **Returns error 401 if the provided `token` is invalid or expired** - - (Test: `should return 401 if token is invalid`) +1. ❌ **Returns error 400 if `token` field is missing from the request** + - Ensures that the function checks for the presence of the `token` field in the request body and responds with a `400` status code if it's missing. -3. ✅ **Returns error 400 if the decoded `token` does not contain a valid `email` field** - - (Test: `should return 400 if email is missing from payload`) +2. ❌ **Returns error 401 if the provided `token` is invalid or expired** + - Verifies that the function correctly handles invalid or expired JWT tokens and responds with a `401` status code. -4. ❌ **Returns error 500 if there is an internal error while saving the new email subscription** +3. ❌ **Returns error 400 if the decoded `token` does not contain a valid `email` field** + - Ensures that the function validates the token payload contains a valid email address and responds with a `400` status code if it doesn't. + +4. ❌ **Returns error 404 if subscription doesn't exist** + - Verifies that the function only confirms existing subscriptions. If no subscription exists for the email in the token, it should return a `404` status code with a message directing the user to subscribe first. + +5. ❌ **Returns error 500 if there is an internal error while updating the subscription** + - Covers scenarios where there's a database error while updating the subscription status. ## Positive Cases -1. ❌ **Returns status 200 when a new email is successfully subscribed** -2. ❌ **Returns status 200 if the email is already subscribed (duplicate email)** +1. ✅ **Returns status 200 when an existing unconfirmed subscription is successfully confirmed** + - Ensures that the function updates an existing unconfirmed subscription to confirmed status, sets `confirmedAt` timestamp, and enables `emailSubscriptions`. + +2. ✅ **Returns status 200 if the email subscription is already confirmed (idempotent)** + - Verifies that the function is idempotent - if a subscription is already confirmed, it returns success without attempting to update again. + +3. ❌ **Correctly handles email normalization (lowercase)** + - Ensures that email addresses are normalized to lowercase for consistent lookups, matching the schema's lowercase enforcement. diff --git a/requirements/emailController/processPendingAndStuckEmails.md b/requirements/emailController/processPendingAndStuckEmails.md new file mode 100644 index 000000000..ce81699ad --- /dev/null +++ b/requirements/emailController/processPendingAndStuckEmails.md @@ -0,0 +1,33 @@ +# Process Pending and Stuck Emails Function + +## Negative Cases + +1. ❌ **Returns error 401 if `requestor` is missing from the request** + - Ensures that the function checks for the presence of `requestor.requestorId` in the request body and responds with a `401` status code if it's missing. + +2. ❌ **Returns error 403 if user doesn't have `sendEmails` permission** + - Verifies that the function checks user permissions and responds with a `403` status code if the user is not authorized to process emails. + +3. ❌ **Returns error 500 if there is an internal error while processing** + - Covers scenarios where there are errors during the processing of pending and stuck emails (e.g., database errors, service failures). + +## Positive Cases + +1. ✅ **Returns status 200 when processing is triggered successfully** + - Ensures that the function triggers the email processor to handle pending and stuck emails and returns success. + +2. ✅ **Resets stuck emails (SENDING status) to PENDING** + - Verifies that emails in SENDING status are reset to PENDING so they can be reprocessed (typically after server restart). + +3. ✅ **Resets stuck batches (SENDING status) to PENDING** + - Ensures that EmailBatch items in SENDING status are reset to PENDING so they can be reprocessed. + +4. ✅ **Queues all PENDING emails for processing** + - Verifies that all emails in PENDING status are added to the processing queue for immediate processing. + +5. ✅ **Handles errors gracefully without throwing** + - Ensures that individual errors during processing (e.g., resetting a specific stuck email) are logged but don't prevent the overall process from completing. + +6. ❌ **Provides detailed logging for troubleshooting** + - Verifies that the function logs information about the number of stuck emails/batches found and processed. + diff --git a/requirements/emailController/removeNonHgnEmailSubscription.md b/requirements/emailController/removeNonHgnEmailSubscription.md index af793e2a9..dd9e92814 100644 --- a/requirements/emailController/removeNonHgnEmailSubscription.md +++ b/requirements/emailController/removeNonHgnEmailSubscription.md @@ -1,10 +1,29 @@ -# Remove Non-HGN Email Subscription Function Tests +# Remove Non-HGN Email Subscription Function ## Negative Cases + 1. ✅ **Returns error 400 if `email` field is missing from the request** - - (Test: `should return 400 if email is missing`) + - Ensures that the function checks for the presence of the `email` field in the request body and responds with a `400` status code if it's missing. + +2. ❌ **Returns error 400 if the provided `email` is invalid** + - Verifies that the function validates email format using `isValidEmailAddress` and responds with a `400` status code for invalid emails. -2. ❌ **Returns error 500 if there is an internal error while deleting the email subscription** +3. ❌ **Returns error 404 if the email subscription is not found** + - Verifies that the function handles cases where no subscription exists for the given email and responds with a `404` status code. + +4. ❌ **Returns error 500 if there is an internal error while deleting the email subscription** + - Covers scenarios where there's a database error while deleting the subscription (e.g., database connection issues). ## Positive Cases -1. ❌ **Returns status 200 when an email is successfully unsubscribed** + +1. ✅ **Returns status 200 when an email is successfully unsubscribed** + - Ensures that the function deletes the subscription record from the `EmailSubcriptionList` collection and returns success with a `200` status code. + +2. ✅ **Correctly normalizes email to lowercase for lookup** + - Verifies that the email is normalized to lowercase before querying/deleting, ensuring consistent matches with the schema's lowercase enforcement. + +3. ✅ **Uses direct email match (no regex needed)** + - Ensures that since the schema enforces lowercase emails, the function uses direct email matching instead of case-insensitive regex. + +4. ❌ **Handles concurrent unsubscribe requests gracefully** + - Ensures that if multiple unsubscribe requests are made simultaneously, the function handles race conditions appropriately. diff --git a/requirements/emailController/resendEmail.md b/requirements/emailController/resendEmail.md new file mode 100644 index 000000000..4bfee4de2 --- /dev/null +++ b/requirements/emailController/resendEmail.md @@ -0,0 +1,69 @@ +# Resend Email Function + +## Negative Cases + +1. ❌ **Returns error 401 if `requestor` is missing from the request** + - Ensures that the function checks for the presence of `requestor.requestorId` in the request body and responds with a `401` status code if it's missing. + +2. ❌ **Returns error 403 if user doesn't have `sendEmails` permission** + - Verifies that the function checks user permissions and responds with a `403` status code if the user is not authorized to resend emails. + +3. ❌ **Returns error 400 if `emailId` is missing or invalid** + - Ensures that the function validates `emailId` is a valid MongoDB ObjectId. + +4. ❌ **Returns error 404 if the original email is not found** + - Verifies that the function handles cases where the email with the provided `emailId` doesn't exist. + +5. ❌ **Returns error 400 if `recipientOption` is missing** + - Ensures that the `recipientOption` field is required in the request body. + +6. ❌ **Returns error 400 if `recipientOption` is invalid** + - Verifies that the `recipientOption` must be one of: `'all'`, `'specific'`, or `'same'`. + +7. ❌ **Returns error 400 if `specificRecipients` is required but missing for 'specific' option** + - Ensures that when `recipientOption` is `'specific'`, the `specificRecipients` array must be provided and non-empty. + +8. ❌ **Returns error 404 if no recipients found for 'same' option** + - Verifies that when `recipientOption` is `'same'`, the original email must have EmailBatch items with recipients. + +9. ❌ **Returns error 400 if recipient count exceeds maximum limit for 'specific' option** + - Ensures that when using `'specific'` option, the recipient limit (2000) is enforced. + +10. ❌ **Returns error 400 if no recipients are found** + - Verifies that after determining recipients, at least one recipient must be available. + +11. ❌ **Returns error 404 if requestor user is not found** + - Ensures that the function validates the requestor exists in the userProfile collection. + +12. ❌ **Returns error 500 if there is an internal error during email creation** + - Covers scenarios where there are database errors or service failures during email/batch creation. + +## Positive Cases + +1. ✅ **Returns status 200 when email is successfully resent with 'all' option** + - Ensures that when `recipientOption` is `'all'`, the function sends to all active HGN users and confirmed email subscribers. + +2. ✅ **Returns status 200 when email is successfully resent with 'specific' option** + - Verifies that when `recipientOption` is `'specific'`, the function sends to only the provided `specificRecipients` list. + +3. ✅ **Returns status 200 when email is successfully resent with 'same' option** + - Ensures that when `recipientOption` is `'same'`, the function extracts recipients from the original email's EmailBatch items and deduplicates them. + +4. ✅ **Creates new email copy with same subject and HTML content** + - Verifies that the function creates a new Email document with the same `subject` and `htmlContent` as the original, but with a new `createdBy` user. + +5. ✅ **Enforces recipient limit only for 'specific' option** + - Ensures that the maximum recipient limit is enforced only when `recipientOption` is `'specific'`, but skipped for `'all'` and `'same'` (broadcast scenarios). + +6. ✅ **Skips recipient limit for broadcast scenarios ('all' and 'same')** + - Verifies that when using `'all'` or `'same'` options, the recipient limit is not enforced. + +7. ✅ **Deduplicates recipients for 'same' option** + - Ensures that when using `'same'` option, duplicate email addresses are removed from the recipient list. + +8. ✅ **Creates email batches in a transaction** + - Ensures that the parent Email and all EmailBatch items are created atomically in a single transaction. + +9. ❌ **Handles transaction rollback on errors** + - Ensures that if any part of email/batch creation fails, the entire transaction is rolled back. + diff --git a/requirements/emailController/retryEmail.md b/requirements/emailController/retryEmail.md new file mode 100644 index 000000000..27012df40 --- /dev/null +++ b/requirements/emailController/retryEmail.md @@ -0,0 +1,51 @@ +# Retry Email Function + +## Negative Cases + +1. ❌ **Returns error 401 if `requestor` is missing from the request** + - Ensures that the function checks for the presence of `requestor.requestorId` in the request body and responds with a `401` status code if it's missing. + +2. ❌ **Returns error 403 if user doesn't have `sendEmails` permission** + - Verifies that the function checks user permissions and responds with a `403` status code if the user is not authorized to retry emails. + +3. ❌ **Returns error 400 if `emailId` parameter is missing or invalid** + - Ensures that the function validates `emailId` from `req.params` is a valid MongoDB ObjectId. + +4. ❌ **Returns error 404 if the email is not found** + - Verifies that the function handles cases where the email with the provided `emailId` doesn't exist. + +5. ❌ **Returns error 400 if email is not in a retryable status** + - Ensures that the function only allows retry for emails in `FAILED` or `PROCESSED` status. Returns `400` for other statuses. + +6. ❌ **Returns error 500 if there is an internal error while fetching failed batches** + - Covers scenarios where there are database errors while querying for failed EmailBatch items. + +7. ❌ **Returns error 500 if there is an internal error while resetting email status** + - Covers scenarios where there are database errors while updating the email status to PENDING. + +8. ❌ **Returns error 500 if there is an internal error while resetting batches** + - Covers scenarios where there are database errors while resetting individual EmailBatch items to PENDING. + +## Positive Cases + +1. ✅ **Returns status 200 when email is successfully retried with failed batches** + - Ensures that the function marks the parent Email as PENDING, resets all failed EmailBatch items to PENDING, queues the email for processing, and returns success with the count of failed items retried. + +2. ✅ **Returns status 200 when email has no failed batches** + - Verifies that if an email has no failed EmailBatch items, the function returns success with `failedItemsRetried: 0` without error. + +3. ✅ **Correctly resets only failed EmailBatch items** + - Ensures that only EmailBatch items with `FAILED` status are reset to PENDING for retry. + +4. ✅ **Marks parent email as PENDING** + - Verifies that the parent Email status is changed to PENDING, allowing it to be reprocessed. + +5. ✅ **Queues email for processing after reset** + - Ensures that after resetting the email and batches, the email is added to the processing queue. + +6. ✅ **Returns correct data in response** + - Verifies that the response includes `emailId` and `failedItemsRetried` count in the data field. + +7. ❌ **Handles concurrent retry requests gracefully** + - Ensures that if multiple retry requests are made simultaneously, the function handles race conditions appropriately. + diff --git a/requirements/emailController/sendEmail.md b/requirements/emailController/sendEmail.md index 7ca9a482c..4879a24a7 100644 --- a/requirements/emailController/sendEmail.md +++ b/requirements/emailController/sendEmail.md @@ -2,9 +2,43 @@ ## Negative Cases -1. ❌ **Returns error 400 if `to`, `subject`, or `html` fields are missing from the request** -2. ❌ **Returns error 500 if there is an internal error while sending the email** +1. ❌ **Returns error 401 if `requestor` is missing from the request** + - Ensures that the function checks for the presence of `requestor.requestorId` in the request body and responds with a `401` status code if it's missing. + +2. ❌ **Returns error 403 if user doesn't have `sendEmails` permission** + - Verifies that the function checks user permissions and responds with a `403` status code if the user is not authorized to send emails. + +3. ❌ **Returns error 400 if `to`, `subject`, or `html` fields are missing from the request** + - Ensures that all required fields (`to`, `subject`, `html`) are present in the request body. + +4. ❌ **Returns error 400 if email contains unreplaced template variables** + - Verifies that the function validates that all template variables in `subject` and `html` have been replaced before sending. + +5. ❌ **Returns error 404 if requestor user is not found** + - Ensures that the function validates the requestor exists in the userProfile collection. + +6. ❌ **Returns error 400 if recipient count exceeds maximum limit (2000)** + - Verifies that the function enforces the maximum recipients per request limit for specific recipient requests. + +7. ❌ **Returns error 400 if any recipient email is invalid** + - Ensures that all recipient email addresses are validated before creating batches. + +8. ❌ **Returns error 500 if there is an internal error during email creation** + - Covers scenarios where there are database errors or service failures during email/batch creation. ## Positive Cases -1. ✅ **Returns status 200 when email is successfully sent with `to`, `subject`, and `html` fields provided** +1. ✅ **Returns status 200 when email is successfully created with valid recipients** + - Ensures that the function creates the parent Email document and EmailBatch items in a transaction, queues the email for processing, and returns success. + +2. ✅ **Enforces recipient limit for specific recipient requests** + - Verifies that the maximum recipient limit (2000) is enforced when sending to specific recipients. + +3. ✅ **Creates email batches correctly** + - Ensures that recipients are properly normalized, validated, and chunked into EmailBatch items according to the configured batch size. + +4. ✅ **Validates all template variables are replaced** + - Verifies that the function checks both HTML content and subject for unreplaced template variables before allowing email creation. + +5. ❌ **Handles transaction rollback on errors** + - Ensures that if any part of email/batch creation fails, the entire transaction is rolled back. diff --git a/requirements/emailController/sendEmailToAll.md b/requirements/emailController/sendEmailToAll.md deleted file mode 100644 index 32a09fed6..000000000 --- a/requirements/emailController/sendEmailToAll.md +++ /dev/null @@ -1,26 +0,0 @@ -# Send Email to All Function - -## Negative Cases - -1. ❌ **Returns error 400 if `subject` or `html` fields are missing from the request** - - The request should be rejected if either the `subject` or `html` content is not provided in the request body. - -2. ❌ **Returns error 500 if there is an internal error while fetching users** - - This case covers scenarios where there's an error fetching users from the `userProfile` collection (e.g., database connection issues). - -3. ❌ **Returns error 500 if there is an internal error while fetching the subscription list** - - This case covers scenarios where there's an error fetching emails from the `EmailSubcriptionList` collection. - -4. ❌ **Returns error 500 if there is an error sending emails** - - This case handles any issues that occur while calling the `emailSender` function, such as network errors or service unavailability. - -## Positive Cases - -1. ❌ **Returns status 200 when emails are successfully sent to all active users** - - Ensures that the function sends emails correctly to all users meeting the criteria (`isActive` and `EmailSubcriptionList`). - -2. ❌ **Returns status 200 when emails are successfully sent to all users in the subscription list** - - Verifies that the function sends emails to all users in the `EmailSubcriptionList`, including the unsubscribe link in the email body. - -3. ❌ **Combines user and subscription list emails successfully** - - Ensures that the function correctly sends emails to both active users and the subscription list without issues. diff --git a/requirements/emailController/sendEmailToSubscribers.md b/requirements/emailController/sendEmailToSubscribers.md new file mode 100644 index 000000000..bcc2b8bbf --- /dev/null +++ b/requirements/emailController/sendEmailToSubscribers.md @@ -0,0 +1,50 @@ +# Send Email to All Subscribers Function + +## Negative Cases + +1. ❌ **Returns error 401 if `requestor` is missing from the request** + - Ensures that the function checks for the presence of `requestor.requestorId` in the request body and responds with a `401` status code if it's missing. + +2. ❌ **Returns error 403 if user doesn't have `sendEmails` permission** + - Verifies that the function checks user permissions and responds with a `403` status code if the user is not authorized to send emails to subscribers. + +3. ❌ **Returns error 400 if `subject` or `html` fields are missing from the request** + - The request should be rejected if either the `subject` or `html` content is not provided in the request body. + +4. ❌ **Returns error 400 if email contains unreplaced template variables** + - Verifies that the function validates that all template variables in `subject` and `html` have been replaced before sending. + +5. ❌ **Returns error 404 if requestor user is not found** + - Ensures that the function validates the requestor exists in the userProfile collection. + +6. ❌ **Returns error 400 if no recipients are found** + - Verifies that the function checks if there are any active HGN users or confirmed email subscribers before creating the email. + +7. ❌ **Returns error 500 if there is an internal error while fetching users** + - This case covers scenarios where there's an error fetching users from the `userProfile` collection (e.g., database connection issues). + +8. ❌ **Returns error 500 if there is an internal error while fetching the subscription list** + - This case covers scenarios where there's an error fetching emails from the `EmailSubcriptionList` collection. + +9. ❌ **Returns error 500 if there is an error creating email or batches** + - Covers scenarios where there are database errors or service failures during email/batch creation. + +## Positive Cases + +1. ✅ **Returns status 200 when emails are successfully created for all active users** + - Ensures that the function sends emails correctly to all users meeting the criteria (`isActive: true`, `emailSubscriptions: true`, non-empty `firstName`, non-null `email`). + +2. ✅ **Returns status 200 when emails are successfully created for all confirmed subscribers** + - Verifies that the function sends emails to all confirmed subscribers in the `EmailSubcriptionList` (with `isConfirmed: true` and `emailSubscriptions: true`). + +3. ✅ **Combines user and subscription list emails successfully** + - Ensures that the function correctly combines recipients from both active HGN users and confirmed email subscribers without duplicates. + +4. ✅ **Skips recipient limit for broadcast emails** + - Verifies that the maximum recipient limit is NOT enforced when broadcasting to all subscribers. + +5. ✅ **Creates email batches in a transaction** + - Ensures that the parent Email and all EmailBatch items are created atomically in a single transaction. + +6. ❌ **Handles transaction rollback on errors** + - Ensures that if any part of email/batch creation fails, the entire transaction is rolled back. diff --git a/requirements/emailController/updateEmailSubscription.md b/requirements/emailController/updateEmailSubscription.md index bcafa5a28..6f7b9fa15 100644 --- a/requirements/emailController/updateEmailSubscription.md +++ b/requirements/emailController/updateEmailSubscription.md @@ -2,19 +2,34 @@ ## Negative Cases -1. ❌ **Returns error 400 if `emailSubscriptions` field is missing from the request** +1. ❌ **Returns error 401 if `requestor.email` is missing from the request** + - Ensures that the function checks for the presence of `requestor.email` in the request body and responds with a `401` status code if it's missing. + +2. ❌ **Returns error 400 if `emailSubscriptions` field is missing from the request** - This ensures that the function checks for the presence of the `emailSubscriptions` field in the request body and responds with a `400` status code if it's missing. -2. ❌ **Returns error 400 if `email` field is missing from the requestor object** - - Ensures that the function requires an `email` field within the `requestor` object in the request body and returns `400` if it's absent. +3. ❌ **Returns error 400 if `emailSubscriptions` is not a boolean value** + - Verifies that the function validates that `emailSubscriptions` is a boolean type and returns `400` for invalid types. + +4. ❌ **Returns error 400 if the provided `email` is invalid** + - Ensures that the function validates the email format using `isValidEmailAddress` and responds with a `400` status code for invalid emails. -3. ❌ **Returns error 404 if the user with the provided `email` is not found** +5. ❌ **Returns error 404 if the user with the provided `email` is not found** - This checks that the function correctly handles cases where no user exists with the given `email` and responds with a `404` status code. -4. ✅ **Returns error 500 if there is an internal error while updating the user profile** +6. ❌ **Returns error 500 if there is an internal error while updating the user profile** - Covers scenarios where there's a database error while updating the user's email subscriptions. ## Positive Cases -1. ❌ **Returns status 200 and the updated user when email subscriptions are successfully updated** - - Ensures that the function updates the `emailSubscriptions` field for the user and returns the updated user document along with a `200` status code. +1. ✅ **Returns status 200 when email subscriptions are successfully updated** + - Ensures that the function updates the `emailSubscriptions` field for the user and returns success with a `200` status code. + +2. ✅ **Correctly normalizes email to lowercase for lookup** + - Verifies that the email is normalized to lowercase before querying the database, ensuring consistent lookups. + +3. ✅ **Updates user profile atomically** + - Ensures that the user profile update uses `findOneAndUpdate` to atomically update the subscription preference. + +4. ❌ **Handles concurrent update requests gracefully** + - Ensures that if multiple update requests are made simultaneously, the function handles race conditions appropriately. diff --git a/requirements/emailOutboxController/getEmailDetails.md b/requirements/emailOutboxController/getEmailDetails.md new file mode 100644 index 000000000..6400f4857 --- /dev/null +++ b/requirements/emailOutboxController/getEmailDetails.md @@ -0,0 +1,39 @@ +# Get Email Details (Outbox) Function + +## Negative Cases + +1. ❌ **Returns error 401 if `requestor` is missing from the request** + - Ensures that the function checks for the presence of `requestor.requestorId` in the request body and responds with a `401` status code if it's missing. + +2. ❌ **Returns error 403 if user doesn't have `sendEmails` permission** + - Verifies that the function checks user permissions and responds with a `403` status code if the user is not authorized to view email details. + +3. ❌ **Returns error 400 if email ID is invalid** + - Ensures that the function validates the email ID from `req.params.emailId` is a valid MongoDB ObjectId format. + +4. ❌ **Returns error 404 if email is not found** + - Verifies that the function handles cases where no email exists with the provided ID and responds with a `404` status code. + +5. ❌ **Returns error 500 if there is an internal error while fetching email details** + - Covers scenarios where there are database errors or service failures while fetching the email and its associated EmailBatch items. + +## Positive Cases + +1. ✅ **Returns status 200 with email and batch details** + - Ensures that the function successfully fetches the parent Email record and all associated EmailBatch items and returns them in the response. + +2. ✅ **Returns complete email information** + - Verifies that the response includes all email fields: `_id`, `subject`, `htmlContent`, `status`, `createdBy`, `createdAt`, `startedAt`, `completedAt`, `updatedAt`. + +3. ✅ **Returns all associated EmailBatch items** + - Ensures that all EmailBatch items associated with the email are included in the response, with all batch details (recipients, status, attempts, timestamps, etc.). + +4. ✅ **Returns email with populated creator information** + - Verifies that the `createdBy` field is populated with user profile information (firstName, lastName, email). + +5. ❌ **Handles emails with no batches gracefully** + - Verifies that if an email has no associated EmailBatch items, the function returns the email with an empty batches array without error. + +6. ❌ **Returns correct data structure** + - Ensures that the response follows the expected structure with email details and associated batches properly nested. + diff --git a/requirements/emailOutboxController/getEmails.md b/requirements/emailOutboxController/getEmails.md new file mode 100644 index 000000000..8121de7ab --- /dev/null +++ b/requirements/emailOutboxController/getEmails.md @@ -0,0 +1,30 @@ +# Get All Emails (Outbox) Function + +## Negative Cases + +1. ❌ **Returns error 401 if `requestor` is missing from the request** + - Ensures that the function checks for the presence of `requestor.requestorId` in the request body and responds with a `401` status code if it's missing. + +2. ❌ **Returns error 403 if user doesn't have `sendEmails` permission** + - Verifies that the function checks user permissions and responds with a `403` status code if the user is not authorized to view emails. + +3. ❌ **Returns error 500 if there is an internal error while fetching emails** + - Covers scenarios where there are database errors or service failures while fetching email records. + +## Positive Cases + +1. ✅ **Returns status 200 with all email records** + - Ensures that the function successfully fetches all Email (parent) records from the database and returns them in the response. + +2. ✅ **Returns emails ordered by creation date (descending)** + - Verifies that emails are returned sorted by `createdAt` in descending order (newest first). + +3. ✅ **Returns emails with populated creator information** + - Ensures that the `createdBy` field is populated with user profile information (firstName, lastName, email). + +4. ✅ **Returns complete email metadata** + - Verifies that the response includes all email fields: `_id`, `subject`, `htmlContent`, `status`, `createdBy`, `createdAt`, `startedAt`, `completedAt`, `updatedAt`. + +5. ❌ **Handles empty email list gracefully** + - Verifies that if no emails exist, the function returns an empty array without error. + diff --git a/requirements/emailTemplateController/createEmailTemplate.md b/requirements/emailTemplateController/createEmailTemplate.md new file mode 100644 index 000000000..ae6f41996 --- /dev/null +++ b/requirements/emailTemplateController/createEmailTemplate.md @@ -0,0 +1,51 @@ +# Create Email Template Function + +## Negative Cases + +1. ❌ **Returns error 401 if `requestor` is missing from the request** + - Ensures that the function checks for the presence of `requestor.requestorId` in the request body and responds with a `401` status code if it's missing. + +2. ❌ **Returns error 403 if user doesn't have `sendEmails` permission** + - Verifies that the function checks user permissions and responds with a `403` status code if the user is not authorized to create email templates. + +3. ❌ **Returns error 400 if required fields are missing** + - Ensures that required fields (`name`, `subject`, `html_content`) are present in the request body. + +4. ❌ **Returns error 400 if template name is invalid or empty** + - Verifies that the template name is a non-empty string and meets validation requirements. + +5. ❌ **Returns error 400 if template subject is invalid or empty** + - Ensures that the template subject is a non-empty string and meets validation requirements. + +6. ❌ **Returns error 400 if template HTML content is invalid or empty** + - Verifies that the template HTML content is a non-empty string and meets validation requirements. + +7. ❌ **Returns error 400 if template variables are invalid** + - Ensures that if `variables` are provided, they follow the correct structure and types as defined in `EMAIL_CONFIG.TEMPLATE_VARIABLE_TYPES`. + +8. ❌ **Returns error 409 if template name already exists** + - Verifies that the function checks for duplicate template names (case-insensitive) and responds with a `409` status code if a template with the same name already exists. + +9. ❌ **Returns error 500 if there is an internal error while creating the template** + - Covers scenarios where there are database errors or service failures while creating the email template. + +10. ❌ **Returns validation errors in response if template data is invalid** + - Ensures that if template validation fails, the response includes an `errors` array with specific validation error messages. + +## Positive Cases + +1. ✅ **Returns status 201 when email template is successfully created** + - Ensures that the function successfully creates a new email template and returns it with a `201` status code. + +2. ✅ **Creates template with correct creator information** + - Verifies that the `created_by` and `updated_by` fields are set to the requestor's user ID. + +3. ✅ **Stores template variables correctly** + - Ensures that if `variables` are provided, they are stored correctly with proper structure and validation. + +4. ✅ **Trims and normalizes template fields** + - Verifies that template `name` and `subject` are trimmed of whitespace before storage. + +5. ❌ **Returns created template with all fields** + - Ensures that the response includes the complete template object with all fields, timestamps, and creator information. + diff --git a/requirements/emailTemplateController/deleteEmailTemplate.md b/requirements/emailTemplateController/deleteEmailTemplate.md new file mode 100644 index 000000000..5159e1692 --- /dev/null +++ b/requirements/emailTemplateController/deleteEmailTemplate.md @@ -0,0 +1,33 @@ +# Delete Email Template Function + +## Negative Cases + +1. ❌ **Returns error 401 if `requestor` is missing from the request** + - Ensures that the function checks for the presence of `requestor.requestorId` in the request body and responds with a `401` status code if it's missing. + +2. ❌ **Returns error 403 if user doesn't have `sendEmails` permission** + - Verifies that the function checks user permissions and responds with a `403` status code if the user is not authorized to delete email templates. + +3. ❌ **Returns error 400 if template ID is invalid** + - Ensures that the function validates the template ID is a valid MongoDB ObjectId format. + +4. ❌ **Returns error 404 if template is not found** + - Verifies that the function handles cases where no template exists with the provided ID and responds with a `404` status code. + +5. ❌ **Returns error 500 if there is an internal error while deleting the template** + - Covers scenarios where there are database errors or service failures while deleting the email template. + +## Positive Cases + +1. ✅ **Returns status 200 when email template is successfully deleted** + - Ensures that the function successfully deletes the email template and returns a success message with a `200` status code. + +2. ✅ **Performs hard delete (permanently removes template)** + - Verifies that the template is permanently removed from the database, not just marked as deleted. + +3. ✅ **Records deleter information before deletion** + - Ensures that the `updated_by` field is set to the requestor's user ID before deletion (if applicable). + +4. ❌ **Handles deletion gracefully (no error if already deleted)** + - Verifies that if the template is already deleted or doesn't exist, the function handles it gracefully without error. + diff --git a/requirements/emailTemplateController/getAllEmailTemplates.md b/requirements/emailTemplateController/getAllEmailTemplates.md new file mode 100644 index 000000000..9d78fefa2 --- /dev/null +++ b/requirements/emailTemplateController/getAllEmailTemplates.md @@ -0,0 +1,33 @@ +# Get All Email Templates Function + +## Negative Cases + +1. ❌ **Returns error 401 if `requestor` is missing from the request** + - Ensures that the function checks for the presence of `requestor.requestorId` or `user.userid` in the request body/user object and responds with a `401` status code if both are missing. + +2. ❌ **Returns error 403 if user doesn't have `sendEmails` permission** + - Verifies that the function checks user permissions and responds with a `403` status code if the user is not authorized to view email templates. + +3. ❌ **Returns error 500 if there is an internal error while fetching templates** + - Covers scenarios where there are database errors or service failures while fetching email templates. + +## Positive Cases + +1. ✅ **Returns status 200 with all email templates** + - Ensures that the function successfully fetches all email templates from the database and returns them with populated creator/updater information. + +2. ✅ **Supports search functionality by template name** + - Verifies that the function filters templates by name when a `search` query parameter is provided (case-insensitive search). + +3. ✅ **Supports sorting by specified field** + - Ensures that templates can be sorted by any specified field via the `sortBy` query parameter, defaulting to `created_at` descending if not specified. + +4. ✅ **Supports optional content projection** + - Verifies that when `includeEmailContent` is set to `'true'`, the response includes `subject`, `html_content`, and `variables` fields. When not included, only basic metadata is returned. + +5. ✅ **Returns templates with populated creator and updater information** + - Ensures that `created_by` and `updated_by` fields are populated with user profile information (firstName, lastName, email). + +6. ❌ **Handles empty search results gracefully** + - Verifies that if no templates match the search criteria, the function returns an empty array without error. + diff --git a/requirements/emailTemplateController/getEmailTemplateById.md b/requirements/emailTemplateController/getEmailTemplateById.md new file mode 100644 index 000000000..f012fd067 --- /dev/null +++ b/requirements/emailTemplateController/getEmailTemplateById.md @@ -0,0 +1,30 @@ +# Get Email Template By ID Function + +## Negative Cases + +1. ❌ **Returns error 401 if `requestor` is missing from the request** + - Ensures that the function checks for the presence of `requestor.requestorId` or `user.userid` in the request body/user object and responds with a `401` status code if both are missing. + +2. ❌ **Returns error 403 if user doesn't have `sendEmails` permission** + - Verifies that the function checks user permissions and responds with a `403` status code if the user is not authorized to view email templates. + +3. ❌ **Returns error 400 if template ID is invalid** + - Ensures that the function validates the template ID is a valid MongoDB ObjectId format. + +4. ❌ **Returns error 404 if template is not found** + - Verifies that the function handles cases where no template exists with the provided ID and responds with a `404` status code. + +5. ❌ **Returns error 500 if there is an internal error while fetching the template** + - Covers scenarios where there are database errors or service failures while fetching the email template. + +## Positive Cases + +1. ✅ **Returns status 200 with the requested email template** + - Ensures that the function successfully fetches the email template with the provided ID and returns all template details. + +2. ✅ **Returns template with populated creator and updater information** + - Verifies that `created_by` and `updated_by` fields are populated with user profile information (firstName, lastName, email). + +3. ✅ **Returns complete template data including variables** + - Ensures that the response includes all template fields: `name`, `subject`, `html_content`, `variables`, `created_by`, `updated_by`, and timestamps. + diff --git a/requirements/emailTemplateController/previewTemplate.md b/requirements/emailTemplateController/previewTemplate.md new file mode 100644 index 000000000..ac96453b3 --- /dev/null +++ b/requirements/emailTemplateController/previewTemplate.md @@ -0,0 +1,45 @@ +# Preview Email Template Function + +## Negative Cases + +1. ❌ **Returns error 401 if `requestor` is missing from the request** + - Ensures that the function checks for the presence of `requestor.requestorId` or `user.userid` in the request body/user object and responds with a `401` status code if both are missing. + +2. ❌ **Returns error 403 if user doesn't have `sendEmails` permission** + - Verifies that the function checks user permissions and responds with a `403` status code if the user is not authorized to preview email templates. + +3. ❌ **Returns error 400 if template ID is invalid** + - Ensures that the function validates the template ID is a valid MongoDB ObjectId format. + +4. ❌ **Returns error 404 if template is not found** + - Verifies that the function handles cases where no template exists with the provided ID and responds with a `404` status code. + +5. ❌ **Returns error 400 if provided variables are invalid** + - Ensures that the function validates provided variables match the template's variable definitions in type and required fields. + +6. ❌ **Returns error 400 if required variables are missing** + - Verifies that all required variables for the template are provided in the request body. + +7. ❌ **Returns error 500 if there is an internal error while rendering the template** + - Covers scenarios where there are errors during template rendering (e.g., invalid template syntax, variable replacement errors). + +## Positive Cases + +1. ✅ **Returns status 200 with rendered template preview** + - Ensures that the function successfully renders the template with the provided variables and returns the rendered `subject` and `html_content`. + +2. ✅ **Replaces all template variables correctly** + - Verifies that all variables in the template are replaced with the provided values in both `subject` and `html_content`. + +3. ✅ **Validates variables before rendering** + - Ensures that the function validates all provided variables match the template's variable definitions before attempting to render. + +4. ✅ **Does not sanitize content for preview** + - Verifies that the preview is rendered without sanitization to allow full preview of the final email content. + +5. ✅ **Returns both subject and content in preview** + - Ensures that the response includes both the rendered `subject` and `html_content` (or `content`) in the preview object. + +6. ❌ **Handles missing optional variables gracefully** + - Verifies that if optional variables are not provided, they are handled appropriately (not replaced or replaced with empty strings). + diff --git a/requirements/emailTemplateController/updateEmailTemplate.md b/requirements/emailTemplateController/updateEmailTemplate.md new file mode 100644 index 000000000..9c60efda7 --- /dev/null +++ b/requirements/emailTemplateController/updateEmailTemplate.md @@ -0,0 +1,54 @@ +# Update Email Template Function + +## Negative Cases + +1. ❌ **Returns error 401 if `requestor` is missing from the request** + - Ensures that the function checks for the presence of `requestor.requestorId` in the request body and responds with a `401` status code if it's missing. + +2. ❌ **Returns error 403 if user doesn't have `sendEmails` permission** + - Verifies that the function checks user permissions and responds with a `403` status code if the user is not authorized to update email templates. + +3. ❌ **Returns error 400 if template ID is invalid** + - Ensures that the function validates the template ID is a valid MongoDB ObjectId format. + +4. ❌ **Returns error 404 if template is not found** + - Verifies that the function handles cases where no template exists with the provided ID and responds with a `404` status code. + +5. ❌ **Returns error 400 if template name is invalid or empty** + - Verifies that if `name` is provided in the update, it is a non-empty string and meets validation requirements. + +6. ❌ **Returns error 400 if template subject is invalid or empty** + - Ensures that if `subject` is provided in the update, it is a non-empty string and meets validation requirements. + +7. ❌ **Returns error 400 if template HTML content is invalid or empty** + - Verifies that if `html_content` is provided in the update, it is a non-empty string and meets validation requirements. + +8. ❌ **Returns error 400 if template variables are invalid** + - Ensures that if `variables` are provided, they follow the correct structure and types as defined in `EMAIL_CONFIG.TEMPLATE_VARIABLE_TYPES`. + +9. ❌ **Returns error 409 if template name already exists (when updating name)** + - Verifies that if the template name is being changed, the function checks for duplicate names (case-insensitive) and responds with a `409` status code if a template with the new name already exists. + +10. ❌ **Returns error 500 if there is an internal error while updating the template** + - Covers scenarios where there are database errors or service failures while updating the email template. + +11. ❌ **Returns validation errors in response if template data is invalid** + - Ensures that if template validation fails, the response includes an `errors` array with specific validation error messages. + +## Positive Cases + +1. ✅ **Returns status 200 when email template is successfully updated** + - Ensures that the function successfully updates the email template and returns the updated template with a `200` status code. + +2. ✅ **Updates template with correct updater information** + - Verifies that the `updated_by` field is set to the requestor's user ID. + +3. ✅ **Updates only provided fields (partial update support)** + - Ensures that only the fields provided in the request body are updated, leaving other fields unchanged. + +4. ✅ **Trims and normalizes updated template fields** + - Verifies that updated `name` and `subject` are trimmed of whitespace before storage. + +5. ❌ **Returns updated template with all fields** + - Ensures that the response includes the complete updated template object with all fields, timestamps, and creator/updater information. + From 5ab8a2d31586fa3a6b130d66f5550aafa58ec9be Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Sat, 6 Dec 2025 23:43:50 -0500 Subject: [PATCH 15/19] feat(email): add sender name configuration and improve logging --- src/config/emailConfig.js | 1 + src/controllers/emailController.js | 44 +-- src/controllers/emailOutboxController.js | 6 +- src/controllers/emailTemplateController.js | 17 +- .../announcements/emails/emailBatchService.js | 250 +++++++++--------- .../announcements/emails/emailProcessor.js | 127 ++++----- .../emails/emailSendingService.js | 77 +++--- .../emails/emailTemplateService.js | 13 +- 8 files changed, 261 insertions(+), 274 deletions(-) diff --git a/src/config/emailConfig.js b/src/config/emailConfig.js index 0ff5fc77c..8717e0824 100644 --- a/src/config/emailConfig.js +++ b/src/config/emailConfig.js @@ -52,6 +52,7 @@ const EMAIL_CONFIG = { // Email configuration EMAIL: { SENDER: process.env.ANNOUNCEMENT_EMAIL, + SENDER_NAME: process.env.ANNOUNCEMENT_EMAIL_SENDER_NAME, }, }; diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index 0b4c9d054..31517f010 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -13,7 +13,7 @@ const emailProcessor = require('../services/announcements/emails/emailProcessor' const { hasPermission } = require('../utilities/permissions'); const { withTransaction } = require('../utilities/transactionHelper'); const config = require('../config'); -const logger = require('../startup/logger'); +// const logger = require('../startup/logger'); const jwtSecret = process.env.JWT_SECRET; @@ -101,7 +101,7 @@ const sendEmail = async (req, res) => { message: `Email created successfully for ${recipientsArray.length} recipient(s) and will be processed shortly`, }); } catch (error) { - logger.logException(error, 'Error creating email'); + // logger.logException(error, 'Error creating email'); const statusCode = error.statusCode || 500; const response = { success: false, @@ -219,7 +219,7 @@ const sendEmailToSubscribers = async (req, res) => { message: `Broadcast email created successfully for ${totalRecipients} recipient(s) and will be processed shortly`, }); } catch (error) { - logger.logException(error, 'Error creating broadcast email'); + // logger.logException(error, 'Error creating broadcast email'); const statusCode = error.statusCode || 500; const response = { success: false, @@ -391,7 +391,7 @@ const resendEmail = async (req, res) => { }, }); } catch (error) { - logger.logException(error, 'Error resending email'); + // logger.logException(error, 'Error resending email'); const statusCode = error.statusCode || 500; const response = { success: false, @@ -456,7 +456,7 @@ const retryEmail = async (req, res) => { const failedItems = await EmailBatchService.getFailedBatchesForEmail(emailId); if (failedItems.length === 0) { - logger.logInfo(`Email ${emailId} has no failed EmailBatch items to retry`); + // logger.logInfo(`Email ${emailId} has no failed EmailBatch items to retry`); return res.status(200).json({ success: true, message: 'No failed EmailBatch items to retry', @@ -467,7 +467,7 @@ const retryEmail = async (req, res) => { }); } - logger.logInfo(`Retrying ${failedItems.length} failed EmailBatch items: ${emailId}`); + // logger.logInfo(`Retrying ${failedItems.length} failed EmailBatch items: ${emailId}`); // Mark parent Email as PENDING for retry await EmailService.markEmailPending(emailId); @@ -479,9 +479,9 @@ const retryEmail = async (req, res) => { }), ); - logger.logInfo( - `Successfully reset Email ${emailId} and ${failedItems.length} failed EmailBatch items to PENDING for retry`, - ); + // logger.logInfo( + // `Successfully reset Email ${emailId} and ${failedItems.length} failed EmailBatch items to PENDING for retry`, + // ); // Add email to queue for processing (non-blocking, sequential processing) emailProcessor.queueEmail(emailId); @@ -495,7 +495,7 @@ const retryEmail = async (req, res) => { }, }); } catch (error) { - logger.logException(error, 'Error retrying Email'); + // logger.logException(error, 'Error retrying Email'); const statusCode = error.statusCode || 500; return res.status(statusCode).json({ success: false, @@ -527,7 +527,7 @@ const processPendingAndStuckEmails = async (req, res) => { } try { - logger.logInfo('Manual trigger: Starting processing of pending and stuck emails...'); + // logger.logInfo('Manual trigger: Starting processing of pending and stuck emails...'); // Trigger the processor to handle pending and stuck emails await emailProcessor.processPendingAndStuckEmails(); @@ -537,7 +537,7 @@ const processPendingAndStuckEmails = async (req, res) => { message: 'Processing of pending and stuck emails triggered successfully', }); } catch (error) { - logger.logException(error, 'Error triggering processing of pending and stuck emails'); + // logger.logException(error, 'Error triggering processing of pending and stuck emails'); const statusCode = error.statusCode || 500; return res.status(statusCode).json({ success: false, @@ -587,7 +587,7 @@ const updateEmailSubscriptions = async (req, res) => { .status(200) .json({ success: true, message: 'Email subscription updated successfully' }); } catch (error) { - logger.logException(error, 'Error updating email subscriptions'); + // logger.logException(error, 'Error updating email subscriptions'); const statusCode = error.statusCode || 500; return res.status(statusCode).json({ success: false, @@ -654,7 +654,7 @@ const addNonHgnEmailSubscription = async (req, res) => { const url = new URL(origin); return `${url.protocol}//${url.host}`; } catch (error) { - logger.logException(error, 'Error parsing request origin'); + // logger.logException(error, 'Error parsing request origin'); } } // Fallback to config or construct from request @@ -672,10 +672,10 @@ const addNonHgnEmailSubscription = async (req, res) => { const frontendUrl = getFrontendUrl(); if (!frontendUrl) { - logger.logException( - new Error('Unable to determine frontend URL from request'), - 'Configuration error', - ); + // logger.logException( + // new Error('Unable to determine frontend URL from request'), + // 'Configuration error', + // ); return res .status(500) .json({ success: false, message: 'Server Error. Please contact support.' }); @@ -702,7 +702,7 @@ const addNonHgnEmailSubscription = async (req, res) => { message: 'Email subscribed successfully. Please check your inbox to confirm.', }); } catch (emailError) { - logger.logException(emailError, 'Error sending confirmation email'); + // logger.logException(emailError, 'Error sending confirmation email'); // Still return success since the subscription was saved to DB return res.status(200).json({ success: true, @@ -711,7 +711,7 @@ const addNonHgnEmailSubscription = async (req, res) => { }); } } catch (error) { - logger.logException(error, 'Error adding email subscription'); + // logger.logException(error, 'Error adding email subscription'); if (error.code === 11000) { return res.status(400).json({ success: false, message: 'Email already subscribed' }); } @@ -778,7 +778,7 @@ const confirmNonHgnEmailSubscription = async (req, res) => { .status(200) .json({ success: true, message: 'Email subscription confirmed successfully' }); } catch (error) { - logger.logException(error, 'Error confirming email subscription'); + // logger.logException(error, 'Error confirming email subscription'); const statusCode = error.statusCode || 500; return res.status(statusCode).json({ success: false, @@ -821,7 +821,7 @@ const removeNonHgnEmailSubscription = async (req, res) => { return res.status(200).json({ success: true, message: 'Email unsubscribed successfully' }); } catch (error) { - logger.logException(error, 'Error removing email subscription'); + // logger.logException(error, 'Error removing email subscription'); return res.status(500).json({ success: false, message: 'Error removing email subscription' }); } }; diff --git a/src/controllers/emailOutboxController.js b/src/controllers/emailOutboxController.js index 10726647e..31c37502a 100644 --- a/src/controllers/emailOutboxController.js +++ b/src/controllers/emailOutboxController.js @@ -1,7 +1,7 @@ const EmailBatchService = require('../services/announcements/emails/emailBatchService'); const EmailService = require('../services/announcements/emails/emailService'); const { hasPermission } = require('../utilities/permissions'); -const logger = require('../startup/logger'); +// const logger = require('../startup/logger'); /** * Get all announcement Email records (parent documents) - Outbox view. @@ -25,7 +25,7 @@ const getEmails = async (req, res) => { data: emails, }); } catch (error) { - logger.logException(error, 'Error getting emails'); + // logger.logException(error, 'Error getting emails'); const statusCode = error.statusCode || 500; return res.status(statusCode).json({ success: false, @@ -59,7 +59,7 @@ const getEmailDetails = async (req, res) => { data: result, }); } catch (error) { - logger.logException(error, 'Error getting Email details with EmailBatch items'); + // logger.logException(error, 'Error getting Email details with EmailBatch items'); const statusCode = error.statusCode || 500; return res.status(statusCode).json({ success: false, diff --git a/src/controllers/emailTemplateController.js b/src/controllers/emailTemplateController.js index ccb75ac30..e22d88988 100644 --- a/src/controllers/emailTemplateController.js +++ b/src/controllers/emailTemplateController.js @@ -4,7 +4,7 @@ const EmailTemplateService = require('../services/announcements/emails/emailTemplateService'); const { hasPermission } = require('../utilities/permissions'); -const logger = require('../startup/logger'); +// const logger = require('../startup/logger'); /** * Get all email templates (with basic search/sort and optional content projection). @@ -64,7 +64,7 @@ const getAllEmailTemplates = async (req, res) => { templates, }); } catch (error) { - logger.logException(error, 'Error fetching email templates'); + // logger.logException(error, 'Error fetching email templates'); const statusCode = error.statusCode || 500; const response = { success: false, @@ -113,7 +113,7 @@ const getEmailTemplateById = async (req, res) => { template, }); } catch (error) { - logger.logException(error, 'Error fetching email template'); + // logger.logException(error, 'Error fetching email template'); const statusCode = error.statusCode || 500; return res.status(statusCode).json({ success: false, @@ -164,7 +164,7 @@ const createEmailTemplate = async (req, res) => { template, }); } catch (error) { - logger.logException(error, 'Error creating email template'); + // logger.logException(error, 'Error creating email template'); const statusCode = error.statusCode || 500; const response = { success: false, @@ -220,7 +220,7 @@ const updateEmailTemplate = async (req, res) => { template, }); } catch (error) { - logger.logException(error, 'Error updating email template'); + // logger.logException(error, 'Error updating email template'); const statusCode = error.statusCode || 500; const response = { success: false, @@ -258,16 +258,15 @@ const deleteEmailTemplate = async (req, res) => { } const { id } = req.params; - const userId = req.body.requestor.requestorId; - await EmailTemplateService.deleteTemplate(id, userId); + await EmailTemplateService.deleteTemplate(id); res.status(200).json({ success: true, message: 'Email template deleted successfully', }); } catch (error) { - logger.logException(error, 'Error deleting email template'); + // logger.logException(error, 'Error deleting email template'); const statusCode = error.statusCode || 500; return res.status(statusCode).json({ success: false, @@ -330,7 +329,7 @@ const previewTemplate = async (req, res) => { preview: rendered, }); } catch (error) { - logger.logException(error, 'Error previewing email template'); + // logger.logException(error, 'Error previewing email template'); const statusCode = error.statusCode || 500; return res.status(statusCode).json({ success: false, diff --git a/src/services/announcements/emails/emailBatchService.js b/src/services/announcements/emails/emailBatchService.js index c508eb64c..766745fb0 100644 --- a/src/services/announcements/emails/emailBatchService.js +++ b/src/services/announcements/emails/emailBatchService.js @@ -11,7 +11,7 @@ const { normalizeRecipientsToObjects, isValidEmailAddress, } = require('../../../utilities/emailValidators'); -const logger = require('../../../startup/logger'); +// const logger = require('../../../startup/logger'); class EmailBatchService { /** @@ -25,96 +25,91 @@ class EmailBatchService { * @returns {Promise} Created EmailBatch items. */ static async createEmailBatches(emailId, recipients, config = {}, session = null) { - try { - // emailId is now the ObjectId directly - validate it - if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { - const error = new Error(`Email not found with id: ${emailId}`); - error.statusCode = 404; - throw error; - } - - const batchSize = config.batchSize || EMAIL_CONFIG.ANNOUNCEMENTS.BATCH_SIZE; - const emailType = config.emailType || EMAIL_CONFIG.EMAIL_TYPES.BCC; + // emailId is now the ObjectId directly - validate it + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + const error = new Error(`Email not found with id: ${emailId}`); + error.statusCode = 404; + throw error; + } - // Normalize recipients to { email } - const normalizedRecipients = normalizeRecipientsToObjects(recipients); - if (normalizedRecipients.length === 0) { - const error = new Error('At least one recipient is required'); - error.statusCode = 400; - throw error; - } + const batchSize = config.batchSize || EMAIL_CONFIG.ANNOUNCEMENTS.BATCH_SIZE; + const emailType = config.emailType || EMAIL_CONFIG.EMAIL_TYPES.BCC; - // Validate recipient count limit (only enforce when enforceRecipientLimit is true) - // Default to true to enforce limit for specific recipient requests - const enforceRecipientLimit = config.enforceRecipientLimit !== false; - if ( - enforceRecipientLimit && - normalizedRecipients.length > EMAIL_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST - ) { - const error = new Error( - `A maximum of ${EMAIL_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST} recipients are allowed per request`, - ); - error.statusCode = 400; - throw error; - } + // Normalize recipients to { email } + const normalizedRecipients = normalizeRecipientsToObjects(recipients); + if (normalizedRecipients.length === 0) { + const error = new Error('At least one recipient is required'); + error.statusCode = 400; + throw error; + } - // Validate email format for all recipients - const invalidRecipients = normalizedRecipients.filter( - (recipient) => !isValidEmailAddress(recipient.email), + // Validate recipient count limit (only enforce when enforceRecipientLimit is true) + // Default to true to enforce limit for specific recipient requests + const enforceRecipientLimit = config.enforceRecipientLimit !== false; + if ( + enforceRecipientLimit && + normalizedRecipients.length > EMAIL_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST + ) { + const error = new Error( + `A maximum of ${EMAIL_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST} recipients are allowed per request`, ); - if (invalidRecipients.length > 0) { - const error = new Error('One or more recipient emails are invalid'); - error.statusCode = 400; - error.invalidRecipients = invalidRecipients.map((r) => r.email); - throw error; - } + error.statusCode = 400; + throw error; + } - // Chunk recipients into EmailBatch items - const emailBatchItems = []; + // Validate email format for all recipients + const invalidRecipients = normalizedRecipients.filter( + (recipient) => !isValidEmailAddress(recipient.email), + ); + if (invalidRecipients.length > 0) { + const error = new Error('One or more recipient emails are invalid'); + error.statusCode = 400; + error.invalidRecipients = invalidRecipients.map((r) => r.email); + throw error; + } - for (let i = 0; i < normalizedRecipients.length; i += batchSize) { - const recipientChunk = normalizedRecipients.slice(i, i + batchSize); + // Chunk recipients into EmailBatch items + const emailBatchItems = []; - const emailBatchItem = { - emailId, // emailId is now the ObjectId directly - recipients: recipientChunk.map((recipient) => ({ email: recipient.email })), - emailType, - status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.PENDING, - }; + for (let i = 0; i < normalizedRecipients.length; i += batchSize) { + const recipientChunk = normalizedRecipients.slice(i, i + batchSize); - emailBatchItems.push(emailBatchItem); - } + const emailBatchItem = { + emailId, // emailId is now the ObjectId directly + recipients: recipientChunk.map((recipient) => ({ email: recipient.email })), + emailType, + status: EMAIL_CONFIG.EMAIL_BATCH_STATUSES.PENDING, + }; - // Insert with session if provided for transaction support - let inserted; - try { - inserted = await EmailBatch.insertMany(emailBatchItems, { session }); - } catch (dbError) { - // Handle MongoDB errors - if (dbError.name === 'ValidationError') { - const error = new Error(`Validation error: ${dbError.message}`); - error.statusCode = 400; - throw error; - } - if (dbError.code === 11000) { - const error = new Error('Duplicate key error'); - error.statusCode = 409; - throw error; - } - // Re-throw with status code for other database errors - dbError.statusCode = 500; - throw dbError; + emailBatchItems.push(emailBatchItem); + } + + // Insert with session if provided for transaction support + let inserted; + try { + inserted = await EmailBatch.insertMany(emailBatchItems, { session }); + } catch (dbError) { + // Handle MongoDB errors + if (dbError.name === 'ValidationError') { + const error = new Error(`Validation error: ${dbError.message}`); + error.statusCode = 400; + throw error; + } + if (dbError.code === 11000) { + const error = new Error('Duplicate key error'); + error.statusCode = 409; + throw error; } + // Re-throw with status code for other database errors + dbError.statusCode = 500; + throw dbError; + } - logger.logInfo( - `Created ${emailBatchItems.length} EmailBatch items for Email ${emailId} with ${normalizedRecipients.length} total recipients`, - ); + // logger.logInfo( + // `Created ${emailBatchItems.length} EmailBatch items for Email ${emailId} with ${normalizedRecipients.length} total recipients`, + // ); - return inserted; - } catch (error) { - logger.logException(error, 'Error creating EmailBatch items'); - throw error; - } + return inserted; } /** @@ -124,53 +119,48 @@ class EmailBatchService { * @throws {Error} If email not found */ static async getEmailWithBatches(emailId) { - try { - if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { - const error = new Error('Valid email ID is required'); - error.statusCode = 400; - throw error; - } - - // Get email with createdBy populated using lean for consistency - const email = await Email.findById(emailId) - .populate('createdBy', 'firstName lastName email') - .lean(); + if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { + const error = new Error('Valid email ID is required'); + error.statusCode = 400; + throw error; + } - if (!email) { - const error = new Error(`Email ${emailId} not found`); - error.statusCode = 404; - throw error; - } + // Get email with createdBy populated using lean for consistency + const email = await Email.findById(emailId) + .populate('createdBy', 'firstName lastName email') + .lean(); - const emailBatches = await this.getBatchesForEmail(emailId); - - // Transform EmailBatch items - const transformedBatches = emailBatches.map((batch) => ({ - _id: batch._id, - emailId: batch.emailId, - recipients: batch.recipients || null, - status: batch.status, - attempts: batch.attempts || null, - lastAttemptedAt: batch.lastAttemptedAt, - sentAt: batch.sentAt, - failedAt: batch.failedAt, - lastError: batch.lastError, - lastErrorAt: batch.lastErrorAt, - errorCode: batch.errorCode, - sendResponse: batch.sendResponse || null, - emailType: batch.emailType, - createdAt: batch.createdAt, - updatedAt: batch.updatedAt, - })); - - return { - email, - batches: transformedBatches, - }; - } catch (error) { - logger.logException(error, 'Error getting Email with batches'); + if (!email) { + const error = new Error(`Email ${emailId} not found`); + error.statusCode = 404; throw error; } + + const emailBatches = await this.getBatchesForEmail(emailId); + + // Transform EmailBatch items + const transformedBatches = emailBatches.map((batch) => ({ + _id: batch._id, + emailId: batch.emailId, + recipients: batch.recipients || null, + status: batch.status, + attempts: batch.attempts || null, + lastAttemptedAt: batch.lastAttemptedAt, + sentAt: batch.sentAt, + failedAt: batch.failedAt, + lastError: batch.lastError, + lastErrorAt: batch.lastErrorAt, + errorCode: batch.errorCode, + sendResponse: batch.sendResponse || null, + emailType: batch.emailType, + createdAt: batch.createdAt, + updatedAt: batch.updatedAt, + })); + + return { + email, + batches: transformedBatches, + }; } /** @@ -372,9 +362,9 @@ class EmailBatchService { throw error; } // Batch exists but not in SENDING status - log and return current batch (idempotent) - logger.logInfo( - `EmailBatch ${emailBatchId} is not in SENDING status (current: ${currentBatch.status}), skipping mark as SENT`, - ); + // logger.logInfo( + // `EmailBatch ${emailBatchId} is not in SENDING status (current: ${currentBatch.status}), skipping mark as SENT`, + // ); return currentBatch; } @@ -437,9 +427,9 @@ class EmailBatchService { throw error; } // Batch exists but already in final state - log and return current batch (idempotent) - logger.logInfo( - `EmailBatch ${emailBatchId} is already in final state (current: ${currentBatch.status}), skipping mark as FAILED`, - ); + // logger.logInfo( + // `EmailBatch ${emailBatchId} is already in final state (current: ${currentBatch.status}), skipping mark as FAILED`, + // ); return currentBatch; } @@ -468,7 +458,7 @@ class EmailBatchService { // Handle edge case: no batches if (batches.length === 0) { - logger.logInfo(`Email ${emailId} has no batches, returning FAILED status`); + // logger.logInfo(`Email ${emailId} has no batches, returning FAILED status`); return EMAIL_CONFIG.EMAIL_STATUSES.FAILED; } @@ -545,7 +535,7 @@ class EmailBatchService { if (!email) { // Email not found - log but don't throw (might have been deleted) - logger.logInfo(`Email ${emailId} not found when syncing status`); + // logger.logInfo(`Email ${emailId} not found when syncing status`); return null; } diff --git a/src/services/announcements/emails/emailProcessor.js b/src/services/announcements/emails/emailProcessor.js index 20878c8dd..725742752 100644 --- a/src/services/announcements/emails/emailProcessor.js +++ b/src/services/announcements/emails/emailProcessor.js @@ -3,7 +3,7 @@ const EmailService = require('./emailService'); const EmailBatchService = require('./emailBatchService'); const emailSendingService = require('./emailSendingService'); const { EMAIL_CONFIG } = require('../../../config/emailConfig'); -const logger = require('../../../startup/logger'); +// const logger = require('../../../startup/logger'); class EmailProcessor { /** @@ -32,7 +32,7 @@ class EmailProcessor { */ queueEmail(emailId) { if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { - logger.logException(new Error('Invalid emailId'), 'EmailProcessor.queueEmail'); + // logger.logException(new Error('Invalid emailId'), 'EmailProcessor.queueEmail'); return; } @@ -44,19 +44,19 @@ class EmailProcessor { this.currentlyProcessingEmailId === emailIdStr || this.processingBatches.has(emailIdStr) ) { - logger.logInfo(`Email ${emailIdStr} is already queued or being processed, skipping`); + // logger.logInfo(`Email ${emailIdStr} is already queued or being processed, skipping`); return; } // Add to queue this.emailQueue.push(emailIdStr); - logger.logInfo(`Email ${emailIdStr} added to queue. Queue length: ${this.emailQueue.length}`); + // logger.logInfo(`Email ${emailIdStr} added to queue. Queue length: ${this.emailQueue.length}`); // Start queue processor if not already running if (!this.isProcessingQueue) { setImmediate(() => { - this.processQueue().catch((error) => { - logger.logException(error, 'Error in queue processor'); + this.processQueue().catch(() => { + // logger.logException(error, 'Error in queue processor'); // Reset flag so queue can restart on next email addition this.isProcessingQueue = false; }); @@ -78,7 +78,7 @@ class EmailProcessor { } this.isProcessingQueue = true; - logger.logInfo('Email queue processor started'); + // logger.logInfo('Email queue processor started'); try { // eslint-disable-next-line no-constant-condition @@ -90,18 +90,18 @@ class EmailProcessor { } this.currentlyProcessingEmailId = emailId; - logger.logInfo( - `Processing email ${emailId} from queue. Remaining: ${this.emailQueue.length}`, - ); + // logger.logInfo( + // `Processing email ${emailId} from queue. Remaining: ${this.emailQueue.length}`, + // ); try { // Process the email (this processes all its batches) // Sequential processing is required - await in loop is necessary // eslint-disable-next-line no-await-in-loop await this.processEmail(emailId); - logger.logInfo(`Email ${emailId} processing completed`); + // logger.logInfo(`Email ${emailId} processing completed`); } catch (error) { - logger.logException(error, `Error processing email ${emailId} from queue`); + // logger.logException(error, `Error processing email ${emailId} from queue`); } finally { this.currentlyProcessingEmailId = null; } @@ -114,7 +114,7 @@ class EmailProcessor { } } finally { this.isProcessingQueue = false; - logger.logInfo('Email queue processor stopped'); + // logger.logInfo('Email queue processor stopped'); } } @@ -136,7 +136,7 @@ class EmailProcessor { // Prevent concurrent processing of the same email if (this.processingBatches.has(emailIdStr)) { - logger.logInfo(`Email ${emailIdStr} is already being processed, skipping`); + // logger.logInfo(`Email ${emailIdStr} is already being processed, skipping`); return EMAIL_CONFIG.EMAIL_STATUSES.SENDING; } @@ -155,7 +155,7 @@ class EmailProcessor { email.status === EMAIL_CONFIG.EMAIL_STATUSES.FAILED || email.status === EMAIL_CONFIG.EMAIL_STATUSES.PROCESSED ) { - logger.logInfo(`Email ${emailId} is already in final state: ${email.status}`); + // logger.logInfo(`Email ${emailId} is already in final state: ${email.status}`); return email.status; } @@ -167,7 +167,7 @@ class EmailProcessor { // If marking as started fails, email is likely already being processed const currentEmail = await EmailService.getEmailById(emailId); if (currentEmail && currentEmail.status === EMAIL_CONFIG.EMAIL_STATUSES.SENDING) { - logger.logInfo(`Email ${emailIdStr} is already being processed, skipping`); + // logger.logInfo(`Email ${emailIdStr} is already being processed, skipping`); this.processingBatches.delete(emailIdStr); return EMAIL_CONFIG.EMAIL_STATUSES.SENDING; } @@ -184,10 +184,10 @@ class EmailProcessor { const updatedEmail = await EmailBatchService.syncParentEmailStatus(email._id); const finalStatus = updatedEmail ? updatedEmail.status : EMAIL_CONFIG.EMAIL_STATUSES.FAILED; - logger.logInfo(`Email ${emailIdStr} processed with status: ${finalStatus}`); + // logger.logInfo(`Email ${emailIdStr} processed with status: ${finalStatus}`); return finalStatus; } catch (error) { - logger.logException(error, `Error processing Email ${emailIdStr}`); + // logger.logException(error, `Error processing Email ${emailIdStr}`); // Reset any batches that were marked as SENDING back to PENDING // This prevents batches from being stuck in SENDING status @@ -200,23 +200,23 @@ class EmailProcessor { sendingBatches.map(async (batch) => { try { await EmailBatchService.resetEmailBatchForRetry(batch._id); - logger.logInfo( - `Reset batch ${batch._id} from SENDING to PENDING due to email processing error`, - ); + // logger.logInfo( + // `Reset batch ${batch._id} from SENDING to PENDING due to email processing error`, + // ); } catch (resetError) { - logger.logException(resetError, `Error resetting batch ${batch._id} to PENDING`); + // logger.logException(resetError, `Error resetting batch ${batch._id} to PENDING`); } }), ); } catch (resetError) { - logger.logException(resetError, `Error resetting batches for email ${emailIdStr}`); + // logger.logException(resetError, `Error resetting batches for email ${emailIdStr}`); } // Mark email as failed on error try { await EmailService.markEmailCompleted(emailIdStr, EMAIL_CONFIG.EMAIL_STATUSES.FAILED); } catch (updateError) { - logger.logException(updateError, 'Error updating Email status to failed'); + // logger.logException(updateError, 'Error updating Email status to failed'); } return EMAIL_CONFIG.EMAIL_STATUSES.FAILED; } finally { @@ -237,13 +237,13 @@ class EmailProcessor { const pendingBatches = await EmailBatchService.getPendingBatchesForEmail(email._id); if (pendingBatches.length === 0) { - logger.logInfo(`No PENDING EmailBatch items found for Email ${email._id}`); + // logger.logInfo(`No PENDING EmailBatch items found for Email ${email._id}`); return; } - logger.logInfo( - `Processing ${pendingBatches.length} PENDING EmailBatch items for Email ${email._id}`, - ); + // logger.logInfo( + // `Processing ${pendingBatches.length} PENDING EmailBatch items for Email ${email._id}`, + // ); // Process items with concurrency limit const concurrency = EMAIL_CONFIG.ANNOUNCEMENTS.CONCURRENCY || 3; @@ -277,17 +277,18 @@ class EmailProcessor { // Add delay after this chunk completes before starting the next chunk // This provides consistent pacing to prevent hitting Gmail rate limits if (delayBetweenChunks > 0 && i + concurrency < pendingBatches.length) { + // eslint-disable-next-line no-await-in-loop await EmailProcessor.sleep(delayBetweenChunks); } } // Log summary of processing - const succeeded = results.filter((r) => r.status === 'fulfilled').length; - const failed = results.filter((r) => r.status === 'rejected').length; + // const succeeded = results.filter((r) => r.status === 'fulfilled').length; + // const failed = results.filter((r) => r.status === 'rejected').length; - logger.logInfo( - `Completed processing ${pendingBatches.length} EmailBatch items for Email ${email._id}: ${succeeded} succeeded, ${failed} failed`, - ); + // logger.logInfo( + // `Completed processing ${pendingBatches.length} EmailBatch items for Email ${email._id}: ${succeeded} succeeded, ${failed} failed`, + // ); } /** @@ -313,10 +314,10 @@ class EmailProcessor { .filter((e) => e && typeof e === 'string'); if (recipientEmails.length === 0) { - logger.logException( - new Error('No valid recipients found'), - `EmailBatch item ${item._id} has no valid recipients`, - ); + // logger.logException( + // new Error('No valid recipients found'), + // `EmailBatch item ${item._id} has no valid recipients`, + // ); await EmailBatchService.markEmailBatchFailed(item._id, { errorCode: 'NO_RECIPIENTS', errorMessage: 'No valid recipients found', @@ -339,23 +340,27 @@ class EmailProcessor { currentBatch.status === EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENT || currentBatch.status === EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENDING ) { - logger.logInfo( - `EmailBatch ${item._id} is already ${currentBatch.status}, skipping duplicate processing`, - ); + // logger.logInfo( + // `EmailBatch ${item._id} is already ${currentBatch.status}, skipping duplicate processing`, + // ); return; // Skip this batch } } } catch (batchError) { // If batch not found or invalid ID, log and re-throw original error - logger.logException(batchError, `Error checking EmailBatch ${item._id} status`); + // logger.logException(batchError, `Error checking EmailBatch ${item._id} status`); } // Re-throw if it's a different error throw markError; } - // Build mail options + // Build mail options with sender name + const senderName = EMAIL_CONFIG.EMAIL.SENDER_NAME; + const senderEmail = EMAIL_CONFIG.EMAIL.SENDER; + const fromField = senderName ? `${senderName} <${senderEmail}>` : senderEmail; + const mailOptions = { - from: EMAIL_CONFIG.EMAIL.SENDER, + from: fromField, subject: email.subject, html: email.htmlContent, }; @@ -380,9 +385,9 @@ class EmailProcessor { attemptCount: actualAttemptCount, // Persist the actual number of attempts made sendResponse: sendResult.response, // Store the full response from email API }); - logger.logInfo( - `EmailBatch item ${item._id} sent successfully to ${recipientEmails.length} recipients (attempts ${actualAttemptCount})`, - ); + // logger.logInfo( + // `EmailBatch item ${item._id} sent successfully to ${recipientEmails.length} recipients (attempts ${actualAttemptCount})`, + // ); return; } @@ -399,9 +404,9 @@ class EmailProcessor { attemptCount: actualAttemptCount, // Persist the actual number of attempts made }); - logger.logInfo( - `Permanently failed to send EmailBatch item ${item._id} to ${recipientEmails.length} recipients after ${actualAttemptCount} attempts`, - ); + // logger.logInfo( + // `Permanently failed to send EmailBatch item ${item._id} to ${recipientEmails.length} recipients after ${actualAttemptCount} attempts`, + // ); // Throw to ensure Promise.allSettled records this item as failed throw finalError; } @@ -442,19 +447,19 @@ class EmailProcessor { */ async processPendingAndStuckEmails() { try { - logger.logInfo('Starting startup processing of pending and stuck emails...'); + // logger.logInfo('Starting startup processing of pending and stuck emails...'); // Step 1: Reset stuck emails to PENDING const stuckEmails = await EmailService.getStuckEmails(); if (stuckEmails.length > 0) { - logger.logInfo(`Found ${stuckEmails.length} stuck emails, resetting to PENDING...`); + // logger.logInfo(`Found ${stuckEmails.length} stuck emails, resetting to PENDING...`); await Promise.allSettled( stuckEmails.map(async (email) => { try { await EmailService.resetStuckEmail(email._id); - logger.logInfo(`Reset stuck email ${email._id} to PENDING`); + // logger.logInfo(`Reset stuck email ${email._id} to PENDING`); } catch (error) { - logger.logException(error, `Error resetting stuck email ${email._id}`); + // logger.logException(error, `Error resetting stuck email ${email._id}`); } }), ); @@ -463,14 +468,14 @@ class EmailProcessor { // Step 2: Reset stuck batches to PENDING const stuckBatches = await EmailBatchService.getStuckBatches(); if (stuckBatches.length > 0) { - logger.logInfo(`Found ${stuckBatches.length} stuck batches, resetting to PENDING...`); + // logger.logInfo(`Found ${stuckBatches.length} stuck batches, resetting to PENDING...`); await Promise.allSettled( stuckBatches.map(async (batch) => { try { await EmailBatchService.resetEmailBatchForRetry(batch._id); - logger.logInfo(`Reset stuck batch ${batch._id} to PENDING`); + // logger.logInfo(`Reset stuck batch ${batch._id} to PENDING`); } catch (error) { - logger.logException(error, `Error resetting stuck batch ${batch._id}`); + // logger.logException(error, `Error resetting stuck batch ${batch._id}`); } }), ); @@ -479,19 +484,19 @@ class EmailProcessor { // Step 3: Queue all PENDING emails for processing const pendingEmails = await EmailService.getPendingEmails(); if (pendingEmails.length > 0) { - logger.logInfo(`Found ${pendingEmails.length} pending emails, adding to queue...`); + // logger.logInfo(`Found ${pendingEmails.length} pending emails, adding to queue...`); // Queue all emails (non-blocking, sequential processing) pendingEmails.forEach((email) => { this.queueEmail(email._id); }); - logger.logInfo(`Queued ${pendingEmails.length} pending emails for processing`); + // logger.logInfo(`Queued ${pendingEmails.length} pending emails for processing`); } else { - logger.logInfo('No pending emails found on startup'); + // logger.logInfo('No pending emails found on startup'); } - logger.logInfo('Startup processing of pending and stuck emails completed'); + // logger.logInfo('Startup processing of pending and stuck emails completed'); } catch (error) { - logger.logException(error, 'Error during startup processing of pending and stuck emails'); + // logger.logException(error, 'Error during startup processing of pending and stuck emails'); } } } diff --git a/src/services/announcements/emails/emailSendingService.js b/src/services/announcements/emails/emailSendingService.js index 068e60ba7..d7271a1ad 100644 --- a/src/services/announcements/emails/emailSendingService.js +++ b/src/services/announcements/emails/emailSendingService.js @@ -6,7 +6,7 @@ const nodemailer = require('nodemailer'); const { google } = require('googleapis'); -const logger = require('../../../startup/logger'); +// const logger = require('../../../startup/logger'); class EmailSendingService { /** @@ -37,20 +37,15 @@ class EmailSendingService { this.OAuth2Client.setCredentials({ refresh_token: this.config.refreshToken }); // Create the email transporter - try { - this.transporter = nodemailer.createTransport({ - service: 'gmail', - auth: { - type: 'OAuth2', - user: this.config.email, - clientId: this.config.clientId, - clientSecret: this.config.clientSecret, - }, - }); - } catch (error) { - logger.logException(error, 'EmailSendingService: Failed to create transporter'); - throw error; - } + this.transporter = nodemailer.createTransport({ + service: 'gmail', + auth: { + type: 'OAuth2', + user: this.config.email, + clientId: this.config.clientId, + clientSecret: this.config.clientSecret, + }, + }); } /** @@ -90,26 +85,26 @@ class EmailSendingService { // Validation if (!mailOptions) { const error = new Error('INVALID_MAIL_OPTIONS: mailOptions is required'); - logger.logException(error, 'EmailSendingService.sendEmail validation failed'); + // logger.logException(error, 'EmailSendingService.sendEmail validation failed'); return { success: false, error }; } if (!mailOptions.to && !mailOptions.bcc) { const error = new Error('INVALID_RECIPIENTS: At least one recipient (to or bcc) is required'); - logger.logException(error, 'EmailSendingService.sendEmail validation failed'); + // logger.logException(error, 'EmailSendingService.sendEmail validation failed'); return { success: false, error }; } // Validate subject and htmlContent if (!mailOptions.subject || mailOptions.subject.trim() === '') { const error = new Error('INVALID_SUBJECT: Subject is required and cannot be empty'); - logger.logException(error, 'EmailSendingService.sendEmail validation failed'); + // logger.logException(error, 'EmailSendingService.sendEmail validation failed'); return { success: false, error }; } if (!this.config.email || !this.config.clientId || !this.config.clientSecret) { const error = new Error('INVALID_CONFIG: Email configuration is incomplete'); - logger.logException(error, 'EmailSendingService.sendEmail configuration check failed'); + // logger.logException(error, 'EmailSendingService.sendEmail configuration check failed'); return { success: false, error }; } @@ -120,7 +115,7 @@ class EmailSendingService { token = await this.getAccessToken(); } catch (tokenError) { const error = new Error(`OAUTH_TOKEN_ERROR: ${tokenError.message}`); - logger.logException(error, 'EmailSendingService.sendEmail OAuth token refresh failed'); + // logger.logException(error, 'EmailSendingService.sendEmail OAuth token refresh failed'); return { success: false, error }; } @@ -138,17 +133,17 @@ class EmailSendingService { const result = await this.transporter.sendMail(mailOptions); // Enhanced logging for announcements - logger.logInfo( - `Announcement email sent to: ${mailOptions.to || mailOptions.bcc || 'unknown'}`, - result, - ); + // logger.logInfo( + // `Announcement email sent to: ${mailOptions.to || mailOptions.bcc || 'unknown'}`, + // result, + // ); return { success: true, response: result }; } catch (error) { - logger.logException( - error, - `Error sending announcement email to: ${mailOptions.to || mailOptions.bcc || 'unknown'}`, - ); + // logger.logException( + // error, + // `Error sending announcement email to: ${mailOptions.to || mailOptions.bcc || 'unknown'}`, + // ); return { success: false, error }; } } @@ -166,13 +161,13 @@ class EmailSendingService { // Validation if (!mailOptions) { const error = new Error('INVALID_MAIL_OPTIONS: mailOptions is required'); - logger.logException(error, 'EmailSendingService.sendWithRetry validation failed'); + // logger.logException(error, 'EmailSendingService.sendWithRetry validation failed'); return { success: false, error, attemptCount: 0 }; } if (!Number.isInteger(retries) || retries < 1) { const error = new Error('INVALID_RETRIES: retries must be a positive integer'); - logger.logException(error, 'EmailSendingService.sendWithRetry validation failed'); + // logger.logException(error, 'EmailSendingService.sendWithRetry validation failed'); return { success: false, error, attemptCount: 0 }; } @@ -188,17 +183,17 @@ class EmailSendingService { if (result.success) { // Store Gmail response for audit logging mailOptions.gmailResponse = result.response; - logger.logInfo( - `Email sent successfully on attempt ${attempt} to: ${mailOptions.to || mailOptions.bcc || 'unknown'}`, - ); + // logger.logInfo( + // `Email sent successfully on attempt ${attempt} to: ${mailOptions.to || mailOptions.bcc || 'unknown'}`, + // ); return { success: true, response: result.response, attemptCount }; } // result.success is false - log and try again or return const error = result.error || new Error('Unknown error from sendEmail'); - logger.logException( - error, - `Announcement send attempt ${attempt} failed to: ${mailOptions.to || mailOptions.bcc || '(empty)'}`, - ); + // logger.logException( + // error, + // `Announcement send attempt ${attempt} failed to: ${mailOptions.to || mailOptions.bcc || '(empty)'}`, + // ); // If this is the last attempt, return failure info if (attempt >= retries) { @@ -206,10 +201,10 @@ class EmailSendingService { } } catch (err) { // Unexpected error (shouldn't happen since sendEmail now returns {success, error}) - logger.logException( - err, - `Unexpected error in announcement send attempt ${attempt} to: ${mailOptions.to || mailOptions.bcc || '(empty)'}`, - ); + // logger.logException( + // err, + // `Unexpected error in announcement send attempt ${attempt} to: ${mailOptions.to || mailOptions.bcc || '(empty)'}`, + // ); // If this is the last attempt, return failure info if (attempt >= retries) { diff --git a/src/services/announcements/emails/emailTemplateService.js b/src/services/announcements/emails/emailTemplateService.js index d733aa241..7fc69286a 100644 --- a/src/services/announcements/emails/emailTemplateService.js +++ b/src/services/announcements/emails/emailTemplateService.js @@ -8,7 +8,7 @@ const sanitizeHtmlLib = require('sanitize-html'); const EmailTemplate = require('../../../models/emailTemplate'); const { EMAIL_CONFIG } = require('../../../config/emailConfig'); const { ensureHtmlWithinLimit } = require('../../../utilities/emailValidators'); -const logger = require('../../../startup/logger'); +// const logger = require('../../../startup/logger'); class EmailTemplateService { /** @@ -271,7 +271,7 @@ class EmailTemplateService { await template.populate('created_by', 'firstName lastName email'); await template.populate('updated_by', 'firstName lastName email'); - logger.logInfo(`Email template created: ${template.name} by user ${userId}`); + // logger.logInfo(`Email template created: ${template.name} by user ${userId}`); return template; } @@ -436,7 +436,7 @@ class EmailTemplateService { throw error; } - logger.logInfo(`Email template updated: ${template.name} by user ${userId}`); + // logger.logInfo(`Email template updated: ${template.name} by user ${userId}`); return template; } @@ -444,11 +444,10 @@ class EmailTemplateService { /** * Delete a template (hard delete). * @param {string|ObjectId} id - Template ID - * @param {string|ObjectId} userId - User ID deleting the template * @returns {Promise} Deleted template * @throws {Error} If template not found */ - static async deleteTemplate(id, userId) { + static async deleteTemplate(id) { if (!id || !mongoose.Types.ObjectId.isValid(id)) { const error = new Error('Invalid template ID'); error.statusCode = 400; @@ -463,12 +462,10 @@ class EmailTemplateService { throw error; } - const templateName = template.name; - // Hard delete await EmailTemplate.findByIdAndDelete(id); - logger.logInfo(`Email template deleted: ${templateName} by user ${userId}`); + // logger.logInfo(`Email template deleted: ${template.name} by user ${userId}`); return template; } From 9a4efd27354cd284edac1e426781cfefe6cfab03 Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Sat, 17 Jan 2026 02:10:59 -0500 Subject: [PATCH 16/19] refactor(email): update email configuration and enhance error handling. --- src/config/emailConfig.js | 3 +- src/controllers/emailController.js | 15 ++++- src/controllers/emailOutboxController.js | 10 ++++ src/controllers/emailTemplateController.js | 26 ++++++-- src/models/emailTemplate.js | 1 - .../announcements/emails/emailBatchService.js | 26 +++++--- .../announcements/emails/emailProcessor.js | 47 ++++++++++++--- .../announcements/emails/emailService.js | 59 ++++++++++++++++++- .../emails/emailTemplateService.js | 21 +++++-- 9 files changed, 178 insertions(+), 30 deletions(-) diff --git a/src/config/emailConfig.js b/src/config/emailConfig.js index 8717e0824..7ff863368 100644 --- a/src/config/emailConfig.js +++ b/src/config/emailConfig.js @@ -39,7 +39,7 @@ const EMAIL_CONFIG = { }, // Template variable types - TEMPLATE_VARIABLE_TYPES: ['text', 'url', 'number', 'textarea', 'image'], + TEMPLATE_VARIABLE_TYPES: ['text', 'number', 'image'], // Announcement service runtime knobs ANNOUNCEMENTS: { @@ -47,6 +47,7 @@ const EMAIL_CONFIG = { CONCURRENCY: 3, // concurrent SMTP batches processed simultaneously BATCH_STAGGER_START_MS: 100, // Delay between starting batches within a concurrent chunk (staggered start for rate limiting) DELAY_BETWEEN_CHUNKS_MS: 1000, // Delay after a chunk of batches completes before starting the next chunk + MAX_QUEUE_SIZE: 100, // Maximum emails in processing queue to prevent memory leak }, // Email configuration diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index 31517f010..062a11f36 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -642,8 +642,14 @@ const addNonHgnEmailSubscription = async (req, res) => { await newEmailList.save(); // Send confirmation email + if (!jwtSecret) { + return res.status(500).json({ + success: false, + message: 'Server configuration error. JWT_SECRET is not set.', + }); + } const payload = { email: normalizedEmail }; - const token = jwt.sign(payload, jwtSecret, { expiresIn: '24h' }); // Fixed: was '360' (invalid) + const token = jwt.sign(payload, jwtSecret, { expiresIn: '24h' }); // Get frontend URL from request origin const getFrontendUrl = () => { @@ -733,6 +739,13 @@ const confirmNonHgnEmailSubscription = async (req, res) => { return res.status(400).json({ success: false, message: 'Token is required' }); } + if (!jwtSecret) { + return res.status(500).json({ + success: false, + message: 'Server configuration error. JWT_SECRET is not set.', + }); + } + let payload = {}; try { payload = jwt.verify(token, jwtSecret); diff --git a/src/controllers/emailOutboxController.js b/src/controllers/emailOutboxController.js index 31c37502a..771bdf5ec 100644 --- a/src/controllers/emailOutboxController.js +++ b/src/controllers/emailOutboxController.js @@ -10,6 +10,11 @@ const { hasPermission } = require('../utilities/permissions'); */ const getEmails = async (req, res) => { try { + // Requestor validation + if (!req?.body?.requestor?.requestorId) { + return res.status(401).json({ success: false, message: 'Missing requestor' }); + } + // Permission check - viewing emails requires sendEmails permission const canViewEmails = await hasPermission(req.body.requestor, 'sendEmails'); if (!canViewEmails) { @@ -41,6 +46,11 @@ const getEmails = async (req, res) => { */ const getEmailDetails = async (req, res) => { try { + // Requestor validation + if (!req?.body?.requestor?.requestorId) { + return res.status(401).json({ success: false, message: 'Missing requestor' }); + } + // Permission check - viewing email details requires sendEmails permission const canViewEmails = await hasPermission(req.body.requestor, 'sendEmails'); if (!canViewEmails) { diff --git a/src/controllers/emailTemplateController.js b/src/controllers/emailTemplateController.js index e22d88988..edc189e67 100644 --- a/src/controllers/emailTemplateController.js +++ b/src/controllers/emailTemplateController.js @@ -30,7 +30,7 @@ const getAllEmailTemplates = async (req, res) => { }); } - const { search, sortBy, includeEmailContent } = req.query; + const { search, sortBy, sortOrder, includeEmailContent } = req.query; const query = {}; const sort = {}; @@ -42,7 +42,9 @@ const getAllEmailTemplates = async (req, res) => { // Build sort object if (sortBy) { - sort[sortBy] = 1; + // Use sortOrder if provided (asc = 1, desc = -1), otherwise default to ascending + const order = sortOrder === 'desc' ? -1 : 1; + sort[sortBy] = order; } else { sort.created_at = -1; } @@ -170,8 +172,16 @@ const createEmailTemplate = async (req, res) => { success: false, message: error.message || 'Error creating email template', }; - if (error.errors && Array.isArray(error.errors)) { + // Always include detailed errors array if available + if (error.errors && Array.isArray(error.errors) && error.errors.length > 0) { response.errors = error.errors; + // If message is vague, enhance it with error count + if ( + response.message === 'Invalid template data' || + response.message === 'Error creating email template' + ) { + response.message = `Validation failed: ${error.errors.length} error(s) found`; + } } return res.status(statusCode).json(response); } @@ -226,8 +236,16 @@ const updateEmailTemplate = async (req, res) => { success: false, message: error.message || 'Error updating email template', }; - if (error.errors && Array.isArray(error.errors)) { + // Always include detailed errors array if available + if (error.errors && Array.isArray(error.errors) && error.errors.length > 0) { response.errors = error.errors; + // If message is vague, enhance it with error count + if ( + response.message === 'Invalid template data' || + response.message === 'Error updating email template' + ) { + response.message = `Validation failed: ${error.errors.length} error(s) found`; + } } return res.status(statusCode).json(response); } diff --git a/src/models/emailTemplate.js b/src/models/emailTemplate.js index 7f9576187..929febc98 100644 --- a/src/models/emailTemplate.js +++ b/src/models/emailTemplate.js @@ -33,7 +33,6 @@ const emailTemplateSchema = new mongoose.Schema( type: { type: String, enum: EMAIL_CONFIG.TEMPLATE_VARIABLE_TYPES, - default: 'text', }, }, ], diff --git a/src/services/announcements/emails/emailBatchService.js b/src/services/announcements/emails/emailBatchService.js index 766745fb0..8c8a19bd3 100644 --- a/src/services/announcements/emails/emailBatchService.js +++ b/src/services/announcements/emails/emailBatchService.js @@ -439,9 +439,9 @@ class EmailBatchService { /** * Determine the parent Email status from child EmailBatch statuses. * Rules: - * - All SENT => SENT - * - All FAILED => FAILED - * - Mixed (some SENT and some FAILED) => PROCESSED + * - All SENT => SENT (all batches succeeded) + * - All FAILED => FAILED (all batches failed) + * - Mixed (some SENT and some FAILED, but ALL batches completed) => PROCESSED * - Otherwise (PENDING or SENDING batches) => SENDING (still in progress) * @param {string|ObjectId} emailId - Parent Email ObjectId. * @returns {Promise} Derived status constant from EMAIL_CONFIG.EMAIL_STATUSES. @@ -472,22 +472,30 @@ class EmailBatchService { const sent = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENT] || 0; const failed = statusMap[EMAIL_CONFIG.EMAIL_BATCH_STATUSES.FAILED] || 0; - // All sent = SENT - if (sent > 0 && pending === 0 && sending === 0 && failed === 0) { + // Still processing (pending or sending batches) = keep SENDING status + // This check must come FIRST to avoid returning final states when still processing + if (pending > 0 || sending > 0) { + return EMAIL_CONFIG.EMAIL_STATUSES.SENDING; + } + + // All batches completed - determine final status + // All sent = SENT (all batches succeeded) + if (sent > 0 && failed === 0) { return EMAIL_CONFIG.EMAIL_STATUSES.SENT; } - // All failed = FAILED - if (failed > 0 && pending === 0 && sending === 0 && sent === 0) { + // All failed = FAILED (all batches failed) + if (failed > 0 && sent === 0) { return EMAIL_CONFIG.EMAIL_STATUSES.FAILED; } // Mixed results (some sent, some failed) = PROCESSED - if (sent > 0 || failed > 0) { + // All batches have completed (no pending or sending), but results are mixed + if (sent > 0 && failed > 0) { return EMAIL_CONFIG.EMAIL_STATUSES.PROCESSED; } - // Still processing (pending or sending batches) = keep SENDING status + // Fallback: Should not reach here, but keep SENDING status return EMAIL_CONFIG.EMAIL_STATUSES.SENDING; } diff --git a/src/services/announcements/emails/emailProcessor.js b/src/services/announcements/emails/emailProcessor.js index 725742752..671f7b3ef 100644 --- a/src/services/announcements/emails/emailProcessor.js +++ b/src/services/announcements/emails/emailProcessor.js @@ -19,6 +19,7 @@ class EmailProcessor { this.emailQueue = []; // In-memory queue for emails to process this.isProcessingQueue = false; // Flag to prevent multiple queue processors this.currentlyProcessingEmailId = null; // Track which email is currently being processed + this.maxQueueSize = EMAIL_CONFIG.ANNOUNCEMENTS.MAX_QUEUE_SIZE || 100; // Max queue size to prevent memory leak } /** @@ -38,7 +39,8 @@ class EmailProcessor { const emailIdStr = emailId.toString(); - // Skip if already in queue or currently processing + // Atomic check and add: Skip if already in queue or currently processing + // Use includes check first for early return if ( this.emailQueue.includes(emailIdStr) || this.currentlyProcessingEmailId === emailIdStr || @@ -48,14 +50,27 @@ class EmailProcessor { return; } - // Add to queue + // Check queue size to prevent memory leak + if (this.emailQueue.length >= this.maxQueueSize) { + // logger.logException( + // new Error(`Email queue is full (${this.maxQueueSize}). Rejecting new email.`), + // 'EmailProcessor.queueEmail - Queue overflow', + // ); + // Remove oldest entries if queue is full (FIFO - keep newest) + const removeCount = Math.floor(this.maxQueueSize * 0.1); // Remove 10% of old entries + this.emailQueue.splice(0, removeCount); + // logger.logInfo(`Removed ${removeCount} old entries from queue to make room`); + } + + // Add to queue (atomic operation - push is atomic in single-threaded JS) this.emailQueue.push(emailIdStr); // logger.logInfo(`Email ${emailIdStr} added to queue. Queue length: ${this.emailQueue.length}`); // Start queue processor if not already running if (!this.isProcessingQueue) { setImmediate(() => { - this.processQueue().catch(() => { + // eslint-disable-next-line no-unused-vars + this.processQueue().catch((error) => { // logger.logException(error, 'Error in queue processor'); // Reset flag so queue can restart on next email addition this.isProcessingQueue = false; @@ -70,13 +85,17 @@ class EmailProcessor { * - Once an email is done, processes the next one * - Continues until queue is empty * - Database connection is ensured at startup (server.js waits for DB connection) + * - Uses atomic check-and-set pattern to prevent race conditions * @returns {Promise} */ async processQueue() { + // Atomic check-and-set: if already processing, return immediately + // In single-threaded Node.js, this is safe because the check and set happen synchronously if (this.isProcessingQueue) { return; // Already processing } + // Set flag synchronously before any async operations to prevent race conditions this.isProcessingQueue = true; // logger.logInfo('Email queue processor started'); @@ -134,12 +153,15 @@ class EmailProcessor { const emailIdStr = emailId.toString(); - // Prevent concurrent processing of the same email + // Atomic check-and-add to prevent race condition: prevent concurrent processing of the same email + // Check first, then add atomically (in single-threaded JS, this is safe between async operations) if (this.processingBatches.has(emailIdStr)) { // logger.logInfo(`Email ${emailIdStr} is already being processed, skipping`); return EMAIL_CONFIG.EMAIL_STATUSES.SENDING; } + // Add to processing set BEFORE any async operations to prevent race conditions + // This ensures no other processEmail call can start while this one is initializing this.processingBatches.add(emailIdStr); try { @@ -212,13 +234,22 @@ class EmailProcessor { // logger.logException(resetError, `Error resetting batches for email ${emailIdStr}`); } - // Mark email as failed on error + // Sync parent Email status based on actual batch states (not just mark as FAILED) + // This ensures status accurately reflects batches that may have succeeded before the error try { - await EmailService.markEmailCompleted(emailIdStr, EMAIL_CONFIG.EMAIL_STATUSES.FAILED); + const updatedEmail = await EmailBatchService.syncParentEmailStatus(emailIdStr); + const finalStatus = updatedEmail ? updatedEmail.status : EMAIL_CONFIG.EMAIL_STATUSES.FAILED; + return finalStatus; } catch (updateError) { - // logger.logException(updateError, 'Error updating Email status to failed'); + // If sync fails, fall back to marking as FAILED + // logger.logException(updateError, 'Error syncing Email status after error'); + try { + await EmailService.markEmailCompleted(emailIdStr, EMAIL_CONFIG.EMAIL_STATUSES.FAILED); + } catch (markError) { + // logger.logException(markError, 'Error updating Email status to failed'); + } + return EMAIL_CONFIG.EMAIL_STATUSES.FAILED; } - return EMAIL_CONFIG.EMAIL_STATUSES.FAILED; } finally { this.processingBatches.delete(emailIdStr); } diff --git a/src/services/announcements/emails/emailService.js b/src/services/announcements/emails/emailService.js index f8863a65d..d0d8db691 100644 --- a/src/services/announcements/emails/emailService.js +++ b/src/services/announcements/emails/emailService.js @@ -1,5 +1,6 @@ const mongoose = require('mongoose'); const Email = require('../../../models/email'); +const EmailBatch = require('../../../models/emailBatch'); const { EMAIL_CONFIG } = require('../../../config/emailConfig'); const { ensureHtmlWithinLimit } = require('../../../utilities/emailValidators'); @@ -244,7 +245,8 @@ class EmailService { /** * Get all Emails ordered by creation date descending. - * @returns {Promise} Array of Email objects (lean, with createdBy populated). + * Includes aggregated recipient counts from EmailBatch items. + * @returns {Promise} Array of Email objects (lean, with createdBy populated and recipient counts). */ static async getAllEmails() { const emails = await Email.find() @@ -252,7 +254,60 @@ class EmailService { .populate('createdBy', 'firstName lastName email') .lean(); - return emails; + // Aggregate recipient counts and batch counts from EmailBatch for each email + const emailsWithCounts = await Promise.all( + emails.map(async (email) => { + // Get total recipients count (sum of all recipients across all batches) + const totalRecipientsAggregation = await EmailBatch.aggregate([ + { $match: { emailId: email._id } }, + { + $group: { + _id: null, + totalRecipients: { + $sum: { $size: { $ifNull: ['$recipients', []] } }, + }, + }, + }, + ]); + + // Count batches by status (for sentEmails and failedEmails) + const batchCountsByStatus = await EmailBatch.aggregate([ + { $match: { emailId: email._id } }, + { + $group: { + _id: '$status', + batchCount: { $sum: 1 }, + }, + }, + ]); + + // Calculate totals + const totalEmails = + totalRecipientsAggregation.length > 0 + ? totalRecipientsAggregation[0].totalRecipients || 0 + : 0; + + let sentEmails = 0; // Batch count + let failedEmails = 0; // Batch count + + batchCountsByStatus.forEach((result) => { + if (result._id === EMAIL_CONFIG.EMAIL_BATCH_STATUSES.SENT) { + sentEmails = result.batchCount || 0; + } else if (result._id === EMAIL_CONFIG.EMAIL_BATCH_STATUSES.FAILED) { + failedEmails = result.batchCount || 0; + } + }); + + return { + ...email, + totalEmails, // Total recipients + sentEmails, // Count of SENT batches + failedEmails, // Count of FAILED batches + }; + }), + ); + + return emailsWithCounts; } /** diff --git a/src/services/announcements/emails/emailTemplateService.js b/src/services/announcements/emails/emailTemplateService.js index 7fc69286a..8b2f641db 100644 --- a/src/services/announcements/emails/emailTemplateService.js +++ b/src/services/announcements/emails/emailTemplateService.js @@ -44,9 +44,12 @@ class EmailTemplateService { variableNames.add(varName.toLowerCase()); } - if (variable.type && !EMAIL_CONFIG.TEMPLATE_VARIABLE_TYPES.includes(variable.type)) { + // Validate type - type is required and must be one of: text, number, image + if (!variable.type) { + errors.push(`Variable ${index + 1}: type is required`); + } else if (!EMAIL_CONFIG.TEMPLATE_VARIABLE_TYPES.includes(variable.type)) { errors.push( - `Variable ${index + 1}: type must be one of: ${EMAIL_CONFIG.TEMPLATE_VARIABLE_TYPES.join(', ')}`, + `Variable ${index + 1}: type '${variable.type}' is invalid. Type must be one of: ${EMAIL_CONFIG.TEMPLATE_VARIABLE_TYPES.join(', ')}`, ); } }); @@ -214,7 +217,12 @@ class EmailTemplateService { // Validate template data const validation = this.validateTemplateData(templateData); if (!validation.isValid) { - const error = new Error('Invalid template data'); + // Create descriptive error message from validation errors + const errorCount = validation.errors.length; + const errorMessage = + errorCount === 1 ? validation.errors[0] : `Validation failed: ${errorCount} error(s) found`; + + const error = new Error(errorMessage); error.errors = validation.errors; error.statusCode = 400; throw error; @@ -356,7 +364,12 @@ class EmailTemplateService { // Validate template data const validation = this.validateTemplateData(templateData); if (!validation.isValid) { - const error = new Error('Invalid template data'); + // Create descriptive error message from validation errors + const errorCount = validation.errors.length; + const errorMessage = + errorCount === 1 ? validation.errors[0] : `Validation failed: ${errorCount} error(s) found`; + + const error = new Error(errorMessage); error.errors = validation.errors; error.statusCode = 400; throw error; From da912f983a4efc419f56aba76b66a6c68f5a3d00 Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Sat, 17 Jan 2026 02:21:52 -0500 Subject: [PATCH 17/19] refactor(email): implement lazy initialization for email sending service and standardize error messages --- src/controllers/emailController.js | 2 +- .../emails/emailSendingService.js | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index 062a11f36..9484dbf73 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -591,7 +591,7 @@ const updateEmailSubscriptions = async (req, res) => { const statusCode = error.statusCode || 500; return res.status(statusCode).json({ success: false, - message: error.message || 'Error updating email subscriptions', + message: 'Error updating email subscriptions', }); } }; diff --git a/src/services/announcements/emails/emailSendingService.js b/src/services/announcements/emails/emailSendingService.js index d7271a1ad..6cba06b39 100644 --- a/src/services/announcements/emails/emailSendingService.js +++ b/src/services/announcements/emails/emailSendingService.js @@ -11,9 +11,26 @@ const { google } = require('googleapis'); class EmailSendingService { /** * Initialize Gmail OAuth2 transport configuration and validate required env vars. - * Throws during construction if configuration is incomplete to fail fast. + * Uses lazy initialization - only initializes when first used (not at module load). + * Throws during initialization if configuration is incomplete to fail fast. */ constructor() { + this._initialized = false; + this.config = null; + this.OAuth2Client = null; + this.transporter = null; + } + + /** + * Initialize the service if not already initialized. + * Lazy initialization allows tests to run without email config. + * @private + */ + _initialize() { + if (this._initialized) { + return; + } + this.config = { email: process.env.ANNOUNCEMENT_EMAIL, clientId: process.env.ANNOUNCEMENT_EMAIL_CLIENT_ID, @@ -46,6 +63,8 @@ class EmailSendingService { clientSecret: this.config.clientSecret, }, }); + + this._initialized = true; } /** @@ -55,6 +74,7 @@ class EmailSendingService { * @throws {Error} If token refresh fails */ async getAccessToken() { + this._initialize(); const accessTokenResp = await this.OAuth2Client.getAccessToken(); let token; @@ -82,6 +102,7 @@ class EmailSendingService { * @returns {Promise<{success: boolean, response?: Object, error?: Error}>} */ async sendEmail(mailOptions) { + this._initialize(); // Validation if (!mailOptions) { const error = new Error('INVALID_MAIL_OPTIONS: mailOptions is required'); From 66d7aedb4ca17f5b24d8f8e8a521d0f216e930de Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Sat, 17 Jan 2026 17:48:55 -0500 Subject: [PATCH 18/19] feat: Add 'url' and 'textarea' to `TEMPLATE_VARIABLE_TYPES` in email configuration. --- src/config/emailConfig.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/emailConfig.js b/src/config/emailConfig.js index 7ff863368..5964fc843 100644 --- a/src/config/emailConfig.js +++ b/src/config/emailConfig.js @@ -39,7 +39,7 @@ const EMAIL_CONFIG = { }, // Template variable types - TEMPLATE_VARIABLE_TYPES: ['text', 'number', 'image'], + TEMPLATE_VARIABLE_TYPES: ['text', 'number', 'image', 'url', 'textarea'], // Announcement service runtime knobs ANNOUNCEMENTS: { From 5d1acf87f80f3107a61c3e6936abf123d18e7fda Mon Sep 17 00:00:00 2001 From: Chaitanya Allu Date: Sat, 17 Jan 2026 22:25:13 -0500 Subject: [PATCH 19/19] feat: enhance recipient deduplication, and integrate email queuing into the transaction for atomicity. --- src/controllers/emailController.js | 201 ++++++++++++------ .../announcements/emails/emailBatchService.js | 40 ++-- .../announcements/emails/emailProcessor.js | 188 ++++++++++------ 3 files changed, 291 insertions(+), 138 deletions(-) diff --git a/src/controllers/emailController.js b/src/controllers/emailController.js index 9484dbf73..fcac88947 100644 --- a/src/controllers/emailController.js +++ b/src/controllers/emailController.js @@ -41,6 +41,18 @@ const sendEmail = async (req, res) => { try { const { to, subject, html } = req.body; + // Validate subject and html are not empty after trim + if (!subject || typeof subject !== 'string' || !subject.trim()) { + return res + .status(400) + .json({ success: false, message: 'Subject is required and cannot be empty' }); + } + if (!html || typeof html !== 'string' || !html.trim()) { + return res + .status(400) + .json({ success: false, message: 'HTML content is required and cannot be empty' }); + } + // Validate that all template variables have been replaced (business rule) const unmatchedVariablesHtml = EmailTemplateService.getUnreplacedVariables(html); const unmatchedVariablesSubject = EmailTemplateService.getUnreplacedVariables(subject); @@ -62,11 +74,16 @@ const sendEmail = async (req, res) => { return res.status(404).json({ success: false, message: 'Requestor not found' }); } - // Normalize recipients once for service and response + // Normalize and deduplicate recipients (case-insensitive) const recipientsArray = normalizeRecipientsToArray(to); + const uniqueRecipients = [ + ...new Set(recipientsArray.map((email) => email.toLowerCase().trim())), + ]; + const recipientObjects = uniqueRecipients.map((emailAddr) => ({ email: emailAddr })); // Create email and batches in transaction (validation happens in services) - const email = await withTransaction(async (session) => { + // Queue email INSIDE transaction to ensure rollback on queue failure + const _email = await withTransaction(async (session) => { // Create parent Email (validates subject, htmlContent, createdBy) const createdEmail = await EmailService.createEmail( { @@ -79,7 +96,6 @@ const sendEmail = async (req, res) => { // Create EmailBatch items with all recipients (validates recipients, counts, email format) // Enforce recipient limit for specific recipient requests - const recipientObjects = recipientsArray.map((emailAddr) => ({ email: emailAddr })); await EmailBatchService.createEmailBatches( createdEmail._id, recipientObjects, @@ -90,15 +106,23 @@ const sendEmail = async (req, res) => { session, ); + // Queue email BEFORE committing transaction + // If queue is full or queueing fails, transaction will rollback + const queued = emailProcessor.queueEmail(createdEmail._id); + if (!queued) { + const error = new Error( + 'Email queue is currently full. Please try again in a few moments or contact support if this persists.', + ); + error.statusCode = 503; // Service Unavailable + throw error; + } + return createdEmail; }); - // Add email to queue for processing (non-blocking, sequential processing) - emailProcessor.queueEmail(email._id); - return res.status(200).json({ success: true, - message: `Email created successfully for ${recipientsArray.length} recipient(s) and will be processed shortly`, + message: `Email queued for processing (${uniqueRecipients.length} recipient(s))`, }); } catch (error) { // logger.logException(error, 'Error creating email'); @@ -138,6 +162,18 @@ const sendEmailToSubscribers = async (req, res) => { try { const { subject, html } = req.body; + // Validate subject and html are not empty after trim + if (!subject || typeof subject !== 'string' || !subject.trim()) { + return res + .status(400) + .json({ success: false, message: 'Subject is required and cannot be empty' }); + } + if (!html || typeof html !== 'string' || !html.trim()) { + return res + .status(400) + .json({ success: false, message: 'HTML content is required and cannot be empty' }); + } + // Validate that all template variables have been replaced (business rule) const unmatchedVariablesHtml = EmailTemplateService.getUnreplacedVariables(html); const unmatchedVariablesSubject = EmailTemplateService.getUnreplacedVariables(subject); @@ -173,13 +209,24 @@ const sendEmailToSubscribers = async (req, res) => { emailSubscriptions: true, }); - const totalRecipients = users.length + emailSubscribers.length; - if (totalRecipients === 0) { + // Collect all recipients and deduplicate (case-insensitive) + const allRecipientEmails = [ + ...users.map((hgnUser) => hgnUser.email), + ...emailSubscribers.map((subscriber) => subscriber.email), + ]; + + const uniqueRecipients = [ + ...new Set(allRecipientEmails.map((email) => email.toLowerCase().trim())), + ]; + const recipientObjects = uniqueRecipients.map((emailAddr) => ({ email: emailAddr })); + + if (uniqueRecipients.length === 0) { return res.status(400).json({ success: false, message: 'No recipients found' }); } // Create email and batches in transaction (validation happens in services) - const email = await withTransaction(async (session) => { + // Queue email INSIDE transaction to ensure rollback on queue failure + const _email = await withTransaction(async (session) => { // Create parent Email (validates subject, htmlContent, createdBy) const createdEmail = await EmailService.createEmail( { @@ -190,17 +237,11 @@ const sendEmailToSubscribers = async (req, res) => { session, ); - // Collect all recipients into single array - const allRecipients = [ - ...users.map((hgnUser) => ({ email: hgnUser.email })), - ...emailSubscribers.map((subscriber) => ({ email: subscriber.email })), - ]; - // Create EmailBatch items with all recipients (validates recipients, counts, email format) // Skip recipient limit for broadcast to all subscribers await EmailBatchService.createEmailBatches( createdEmail._id, - allRecipients, + recipientObjects, { emailType: EMAIL_CONFIG.EMAIL_TYPES.BCC, enforceRecipientLimit: false, // Skip limit for broadcast @@ -208,15 +249,22 @@ const sendEmailToSubscribers = async (req, res) => { session, ); + // Queue email BEFORE committing transaction + const queued = emailProcessor.queueEmail(createdEmail._id); + if (!queued) { + const error = new Error( + 'Email queue is currently full. Please try again in a few moments or contact support if this persists.', + ); + error.statusCode = 503; + throw error; + } + return createdEmail; }); - // Add email to queue for processing (non-blocking, sequential processing) - emailProcessor.queueEmail(email._id); - return res.status(200).json({ success: true, - message: `Broadcast email created successfully for ${totalRecipients} recipient(s) and will be processed shortly`, + message: `Broadcast email queued for processing (${uniqueRecipients.length} recipient(s))`, }); } catch (error) { // logger.logException(error, 'Error creating broadcast email'); @@ -302,9 +350,9 @@ const resendEmail = async (req, res) => { }); allRecipients = [ - ...users.map((hgnUser) => ({ email: hgnUser.email })), - ...emailSubscribers.map((subscriber) => ({ email: subscriber.email })), - ]; + ...users.map((hgnUser) => hgnUser.email), + ...emailSubscribers.map((subscriber) => subscriber.email), + ].map((email) => ({ email })); } else if (recipientOption === 'specific') { // Use provided specific recipients if ( @@ -320,7 +368,7 @@ const resendEmail = async (req, res) => { // Normalize recipients (validation happens in service) const recipientsArray = normalizeRecipientsToArray(specificRecipients); - allRecipients = recipientsArray.map((email) => ({ email })); + allRecipients = recipientsArray.map((email) => ({ email: email.toLowerCase().trim() })); } else if (recipientOption === 'same') { // Get recipients from original email's EmailBatch items const emailBatchItems = await EmailBatchService.getBatchesForEmail(emailId); @@ -335,23 +383,21 @@ const resendEmail = async (req, res) => { .filter((batch) => batch.recipients && Array.isArray(batch.recipients)) .flatMap((batch) => batch.recipients); allRecipients.push(...batchRecipients); - - // Deduplicate recipients by email - const seenEmails = new Set(); - allRecipients = allRecipients.filter((recipient) => { - if (!recipient || !recipient.email || seenEmails.has(recipient.email)) { - return false; - } - seenEmails.add(recipient.email); - return true; - }); } - if (allRecipients.length === 0) { + // Deduplicate all recipients (case-insensitive) + const allRecipientEmails = allRecipients.map((r) => r.email).filter(Boolean); + const uniqueRecipients = [ + ...new Set(allRecipientEmails.map((email) => email.toLowerCase().trim())), + ]; + const recipientObjects = uniqueRecipients.map((emailAddr) => ({ email: emailAddr })); + + if (uniqueRecipients.length === 0) { return res.status(400).json({ success: false, message: 'No recipients found' }); } // Create email and batches in transaction + // Queue email INSIDE transaction to ensure rollback on queue failure const newEmail = await withTransaction(async (session) => { // Create new Email (copy) - validation happens in service const createdEmail = await EmailService.createEmail( @@ -368,7 +414,7 @@ const resendEmail = async (req, res) => { const shouldEnforceLimit = recipientOption === 'specific'; await EmailBatchService.createEmailBatches( createdEmail._id, - allRecipients, + recipientObjects, { emailType: EMAIL_CONFIG.EMAIL_TYPES.BCC, enforceRecipientLimit: shouldEnforceLimit, @@ -376,18 +422,25 @@ const resendEmail = async (req, res) => { session, ); + // Queue email BEFORE committing transaction + const queued = emailProcessor.queueEmail(createdEmail._id); + if (!queued) { + const error = new Error( + 'Email queue is currently full. Please try again in a few moments or contact support if this persists.', + ); + error.statusCode = 503; + throw error; + } + return createdEmail; }); - // Add email to queue for processing (non-blocking, sequential processing) - emailProcessor.queueEmail(newEmail._id); - return res.status(200).json({ success: true, - message: `Email created for resend successfully to ${allRecipients.length} recipient(s) and will be processed shortly`, + message: `Email queued for resend (${uniqueRecipients.length} recipient(s))`, data: { emailId: newEmail._id, - recipientCount: allRecipients.length, + recipientCount: uniqueRecipients.length, }, }); } catch (error) { @@ -484,7 +537,14 @@ const retryEmail = async (req, res) => { // ); // Add email to queue for processing (non-blocking, sequential processing) - emailProcessor.queueEmail(emailId); + const queued = emailProcessor.queueEmail(emailId); + if (!queued) { + return res.status(503).json({ + success: false, + message: + 'Email queue is currently full. Your email has been reset to PENDING and will be processed automatically when the server restarts, or you can use the "Process Pending Emails" button to retry manually.', + }); + } res.status(200).json({ success: true, @@ -513,35 +573,54 @@ const retryEmail = async (req, res) => { * @param {import('express').Response} res */ const processPendingAndStuckEmails = async (req, res) => { - // Requestor is required for permission check - if (!req?.body?.requestor?.requestorId) { - return res.status(401).json({ success: false, message: 'Missing requestor' }); - } + try { + // Requestor is required for permission check + if (!req?.body?.requestor?.requestorId) { + return res.status(401).json({ success: false, message: 'Missing requestor' }); + } - // Permission check - processing emails requires sendEmails permission - const canProcessEmails = await hasPermission(req.body.requestor, 'sendEmails'); - if (!canProcessEmails) { - return res - .status(403) - .json({ success: false, message: 'You are not authorized to process emails.' }); - } + // Permission check - processing stuck emails requires sendEmails permission + const canProcessEmails = await hasPermission(req.body.requestor, 'sendEmails'); + if (!canProcessEmails) { + return res + .status(403) + .json({ success: false, message: 'You are not authorized to process emails.' }); + } - try { - // logger.logInfo('Manual trigger: Starting processing of pending and stuck emails...'); + // Trigger processing and get statistics + const stats = await emailProcessor.processPendingAndStuckEmails(); + + // Build user-friendly message + const parts = []; + if (stats.stuckEmailsReset > 0) { + parts.push(`${stats.stuckEmailsReset} stuck email(s) reset`); + } + if (stats.stuckBatchesReset > 0) { + parts.push(`${stats.stuckBatchesReset} stuck batch(es) reset`); + } + if (stats.runtimeStuckBatchesReset > 0) { + parts.push(`${stats.runtimeStuckBatchesReset} timeout batch(es) reset`); + } + if (stats.pendingEmailsQueued > 0) { + parts.push(`${stats.pendingEmailsQueued} pending email(s) queued`); + } - // Trigger the processor to handle pending and stuck emails - await emailProcessor.processPendingAndStuckEmails(); + const message = + parts.length > 0 + ? `Recovery complete: ${parts.join(', ')}` + : 'No stuck or pending emails found - all clear!'; return res.status(200).json({ success: true, - message: 'Processing of pending and stuck emails triggered successfully', + message, + data: stats, }); } catch (error) { - // logger.logException(error, 'Error triggering processing of pending and stuck emails'); + // logger.logException(error, 'Error processing pending and stuck emails'); const statusCode = error.statusCode || 500; return res.status(statusCode).json({ success: false, - message: error.message || 'Error triggering processing of pending and stuck emails', + message: error.message || 'Error processing pending and stuck emails', }); } }; diff --git a/src/services/announcements/emails/emailBatchService.js b/src/services/announcements/emails/emailBatchService.js index 8c8a19bd3..451fe65cf 100644 --- a/src/services/announcements/emails/emailBatchService.js +++ b/src/services/announcements/emails/emailBatchService.js @@ -43,12 +43,35 @@ class EmailBatchService { throw error; } + // Validate email format for all recipients FIRST + const invalidRecipients = normalizedRecipients.filter( + (recipient) => !isValidEmailAddress(recipient.email), + ); + if (invalidRecipients.length > 0) { + const error = new Error('One or more recipient emails are invalid'); + error.statusCode = 400; + error.invalidRecipients = invalidRecipients.map((r) => r.email); + throw error; + } + + // Filter to only valid recipients + const validRecipients = normalizedRecipients.filter((recipient) => + isValidEmailAddress(recipient.email), + ); + + // Check if we have any valid recipients AFTER filtering + if (validRecipients.length === 0) { + const error = new Error('No valid recipients after validation'); + error.statusCode = 400; + throw error; + } + // Validate recipient count limit (only enforce when enforceRecipientLimit is true) // Default to true to enforce limit for specific recipient requests const enforceRecipientLimit = config.enforceRecipientLimit !== false; if ( enforceRecipientLimit && - normalizedRecipients.length > EMAIL_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST + validRecipients.length > EMAIL_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST ) { const error = new Error( `A maximum of ${EMAIL_CONFIG.LIMITS.MAX_RECIPIENTS_PER_REQUEST} recipients are allowed per request`, @@ -57,22 +80,11 @@ class EmailBatchService { throw error; } - // Validate email format for all recipients - const invalidRecipients = normalizedRecipients.filter( - (recipient) => !isValidEmailAddress(recipient.email), - ); - if (invalidRecipients.length > 0) { - const error = new Error('One or more recipient emails are invalid'); - error.statusCode = 400; - error.invalidRecipients = invalidRecipients.map((r) => r.email); - throw error; - } - // Chunk recipients into EmailBatch items const emailBatchItems = []; - for (let i = 0; i < normalizedRecipients.length; i += batchSize) { - const recipientChunk = normalizedRecipients.slice(i, i + batchSize); + for (let i = 0; i < validRecipients.length; i += batchSize) { + const recipientChunk = validRecipients.slice(i, i + batchSize); const emailBatchItem = { emailId, // emailId is now the ObjectId directly diff --git a/src/services/announcements/emails/emailProcessor.js b/src/services/announcements/emails/emailProcessor.js index 671f7b3ef..30691b5c6 100644 --- a/src/services/announcements/emails/emailProcessor.js +++ b/src/services/announcements/emails/emailProcessor.js @@ -1,8 +1,8 @@ const mongoose = require('mongoose'); +const { EMAIL_CONFIG } = require('../../../config/emailConfig'); const EmailService = require('./emailService'); const EmailBatchService = require('./emailBatchService'); const emailSendingService = require('./emailSendingService'); -const { EMAIL_CONFIG } = require('../../../config/emailConfig'); // const logger = require('../../../startup/logger'); class EmailProcessor { @@ -34,7 +34,7 @@ class EmailProcessor { queueEmail(emailId) { if (!emailId || !mongoose.Types.ObjectId.isValid(emailId)) { // logger.logException(new Error('Invalid emailId'), 'EmailProcessor.queueEmail'); - return; + return false; // Return false to indicate failure } const emailIdStr = emailId.toString(); @@ -47,19 +47,16 @@ class EmailProcessor { this.processingBatches.has(emailIdStr) ) { // logger.logInfo(`Email ${emailIdStr} is already queued or being processed, skipping`); - return; + return true; // Already queued, consider this success } - // Check queue size to prevent memory leak - if (this.emailQueue.length >= this.maxQueueSize) { + // Check queue size to prevent memory leak - REJECT instead of dropping old emails + if (this.emailQueue.length > this.maxQueueSize) { // logger.logException( - // new Error(`Email queue is full (${this.maxQueueSize}). Rejecting new email.`), + // new Error(`Email queue is full (${this.maxQueueSize}). Rejecting new email ${emailIdStr}.`), // 'EmailProcessor.queueEmail - Queue overflow', // ); - // Remove oldest entries if queue is full (FIFO - keep newest) - const removeCount = Math.floor(this.maxQueueSize * 0.1); // Remove 10% of old entries - this.emailQueue.splice(0, removeCount); - // logger.logInfo(`Removed ${removeCount} old entries from queue to make room`); + return false; // Return false to signal queue is full } // Add to queue (atomic operation - push is atomic in single-threaded JS) @@ -67,6 +64,7 @@ class EmailProcessor { // logger.logInfo(`Email ${emailIdStr} added to queue. Queue length: ${this.emailQueue.length}`); // Start queue processor if not already running + // Check flag and start processor synchronously to prevent race condition if (!this.isProcessingQueue) { setImmediate(() => { // eslint-disable-next-line no-unused-vars @@ -77,6 +75,8 @@ class EmailProcessor { }); }); } + + return true; // Successfully queued } /** @@ -89,13 +89,14 @@ class EmailProcessor { * @returns {Promise} */ async processQueue() { - // Atomic check-and-set: if already processing, return immediately - // In single-threaded Node.js, this is safe because the check and set happen synchronously + // Atomic check-and-set using synchronous operations + // This must be done synchronously before ANY async operations to prevent race conditions + // In Node.js event loop, synchronous code is atomic if (this.isProcessingQueue) { return; // Already processing } - // Set flag synchronously before any async operations to prevent race conditions + // Set flag IMMEDIATELY and synchronously to prevent race conditions this.isProcessingQueue = true; // logger.logInfo('Email queue processor started'); @@ -469,66 +470,127 @@ class EmailProcessor { } /** - * Process pending and stuck emails on system startup. - * - Called only after database connection is established (server.js uses mongoose.connection.once('connected')) + * Reset batches that are stuck in SENDING status during runtime. + * - Identifies batches that have been in SENDING status for more than 20 minutes + * - Resets them to PENDING so they can be retried + * - Called by cron job to handle runtime failures (not just startup) + * @returns {Promise} Number of batches reset + */ + static async resetStuckRuntimeBatches() { + try { + const STUCK_TIMEOUT_MS = 20 * 60 * 1000; // 20 minutes + const timeoutThreshold = new Date(Date.now() - STUCK_TIMEOUT_MS); + + // Find batches stuck in SENDING status for more than 15 minutes + const EmailBatch = require('../../../models/emailBatch'); + const emailConfig = require('../../../config/emailConfig').EMAIL_CONFIG; + + const stuckBatches = await EmailBatch.find({ + status: emailConfig.EMAIL_BATCH_STATUSES.SENDING, + lastAttemptedAt: { $lt: timeoutThreshold }, + }); + + if (stuckBatches.length === 0) { + return 0; + } + + // logger.logInfo( + // `Found ${stuckBatches.length} batches stuck in SENDING status for >15 minutes, resetting...`, + // ); + + let resetCount = 0; + await Promise.allSettled( + stuckBatches.map(async (batch) => { + try { + await EmailBatchService.resetEmailBatchForRetry(batch._id); + resetCount += 1; + // logger.logInfo(`Reset runtime stuck batch ${batch._id} to PENDING`); + } catch (error) { + // logger.logException(error, `Error resetting runtime stuck batch ${batch._id}`); + } + }), + ); + + return resetCount; + } catch (error) { + // logger.logException(error, 'Error in resetStuckRuntimeBatches'); + return 0; + } + } + + /** + * Process pending and stuck emails on system startup OR via cron job. + * - Called after database connection is established (server.js) + * - Called periodically by cron job (every 10 minutes) * - Resets stuck emails (SENDING status) to PENDING * - Resets stuck batches (SENDING status) to PENDING + * - Resets runtime stuck batches (SENDING > 20 minutes) * - Queues all PENDING emails for processing * @returns {Promise} */ async processPendingAndStuckEmails() { - try { - // logger.logInfo('Starting startup processing of pending and stuck emails...'); - - // Step 1: Reset stuck emails to PENDING - const stuckEmails = await EmailService.getStuckEmails(); - if (stuckEmails.length > 0) { - // logger.logInfo(`Found ${stuckEmails.length} stuck emails, resetting to PENDING...`); - await Promise.allSettled( - stuckEmails.map(async (email) => { - try { - await EmailService.resetStuckEmail(email._id); - // logger.logInfo(`Reset stuck email ${email._id} to PENDING`); - } catch (error) { - // logger.logException(error, `Error resetting stuck email ${email._id}`); - } - }), - ); - } + // logger.logInfo('Starting processing of pending and stuck emails...'); + + // Step 1: Reset stuck emails to PENDING + const stuckEmails = await EmailService.getStuckEmails(); + if (stuckEmails.length > 0) { + // logger.logInfo(`Found ${stuckEmails.length} stuck emails, resetting to PENDING...`); + await Promise.allSettled( + stuckEmails.map(async (email) => { + try { + await EmailService.resetStuckEmail(email._id); + // logger.logInfo(`Reset stuck email ${email._id} to PENDING`); + } catch (error) { + // logger.logException(error, `Error resetting stuck email ${email._id}`); + } + }), + ); + } - // Step 2: Reset stuck batches to PENDING - const stuckBatches = await EmailBatchService.getStuckBatches(); - if (stuckBatches.length > 0) { - // logger.logInfo(`Found ${stuckBatches.length} stuck batches, resetting to PENDING...`); - await Promise.allSettled( - stuckBatches.map(async (batch) => { - try { - await EmailBatchService.resetEmailBatchForRetry(batch._id); - // logger.logInfo(`Reset stuck batch ${batch._id} to PENDING`); - } catch (error) { - // logger.logException(error, `Error resetting stuck batch ${batch._id}`); - } - }), - ); - } + // Step 2: Reset stuck batches to PENDING (from server restart) + const stuckBatches = await EmailBatchService.getStuckBatches(); + if (stuckBatches.length > 0) { + // logger.logInfo(`Found ${stuckBatches.length} stuck batches, resetting to PENDING...`); + await Promise.allSettled( + stuckBatches.map(async (batch) => { + try { + await EmailBatchService.resetEmailBatchForRetry(batch._id); + // logger.logInfo(`Reset stuck batch ${batch._id} to PENDING`); + } catch (error) { + // logger.logException(error, `Error resetting stuck batch ${batch._id}`); + } + }), + ); + } - // Step 3: Queue all PENDING emails for processing - const pendingEmails = await EmailService.getPendingEmails(); - if (pendingEmails.length > 0) { - // logger.logInfo(`Found ${pendingEmails.length} pending emails, adding to queue...`); - // Queue all emails (non-blocking, sequential processing) - pendingEmails.forEach((email) => { - this.queueEmail(email._id); - }); - // logger.logInfo(`Queued ${pendingEmails.length} pending emails for processing`); - } else { - // logger.logInfo('No pending emails found on startup'); - } + // Step 3: Reset runtime stuck batches (SENDING > 20 minutes) + const runtimeResetCount = await EmailProcessor.resetStuckRuntimeBatches(); + if (runtimeResetCount > 0) { + // logger.logInfo(`Reset ${runtimeResetCount} runtime stuck batches`); + } - // logger.logInfo('Startup processing of pending and stuck emails completed'); - } catch (error) { - // logger.logException(error, 'Error during startup processing of pending and stuck emails'); + // Step 4: Queue all PENDING emails for processing + const pendingEmails = await EmailService.getPendingEmails(); + if (pendingEmails.length > 0) { + // logger.logInfo(`Found ${pendingEmails.length} pending emails, adding to queue...`); + // Queue all emails (non-blocking, sequential processing) + pendingEmails.forEach((email) => { + this.queueEmail(email._id); + }); + // logger.logInfo(`Queued ${pendingEmails.length} pending emails for processing`); + } else { + // logger.logInfo('No pending emails found'); } + + // logger.logInfo('Processing of pending and stuck emails completed'); + + // Return statistics for user feedback + return { + stuckEmailsReset: stuckEmails.length, + stuckBatchesReset: stuckBatches.length, + runtimeStuckBatchesReset: runtimeResetCount, + pendingEmailsQueued: pendingEmails.length, + }; } }