5640 - Scheduler and email

src/i18n/resources/en/email.json

@@ -1,4 +1,10 @@
 {
+  "invalidLinks": {
+    "messageBodyIntro": "The following countries have one or more invalid links:",
+    "messageFooter": "Please review and update the affected links in the FRA Platform.\n\nThank you,\nThe FRA team",
+    "messageHeader": "Dear {{recipientName}},",
+    "subject": "Action required: invalid links detected in your countries"
+  },
   "invitationAccepted": {
     "htmlMessage": "Dear {{recipientName}} {{recipientSurname}},\n<br/><br/>\nUser {{invitedUserName}} {{invitedUserSurname}} has accepted your invitation to {{assessmentName}} {{cycleName}} as {{role}} for <b><a href=\"{{- manageCollaboratorsUrl}}\">{{country}}</a></b>.\n<br/><br/>\nThe FRA team",
     "subject": "User {{invitedUserName}} {{invitedUserSurname}} has accepted your invitation",

src/server/service/mail/mail.ts

@@ -1,5 +1,6 @@
 import nodemailer from 'nodemailer'
 
+import { notifyLinksInvalid } from 'server/service/mail/notifyLinksInvalid'
 import { remindReviewers } from 'server/service/mail/remindReviewers'
 import { ProcessEnv } from 'server/utils'
 import { Logger } from 'server/utils/logger'
@@ -63,6 +64,7 @@ export const sendMail = async (email: MailServiceEmail): Promise<void> => {
 
 export const MailService = {
   assessmentNotifyUsers,
+  notifyLinksInvalid,
   oneToOneMessage,
   remindReviewers,
   resetPassword,

src/server/service/mail/notifyLinksInvalid/buildEmail.ts

@@ -0,0 +1,104 @@
+import { createI18nPromise } from 'i18n/i18nFactory'
+
+import { CountryIso } from 'meta/area/countryIso'
+import { Assessment, AssessmentName } from 'meta/assessment/assessment'
+import { Cycle, CycleName } from 'meta/assessment/cycle'
+import { Link } from 'meta/cycleData/links/link'
+import { Lang } from 'meta/lang'
+import { Routes } from 'meta/routes/routes'
+import { SectionNames } from 'meta/routes/sectionNames'
+import { User } from 'meta/user/user'
+import { Users } from 'meta/user/users'
+
+import { MailServiceEmail } from 'server/service/mail/mail'
+import { ProcessEnv } from 'server/utils'
+
+type Props = {
+  assessment: Assessment
+  cycle: Cycle
+  linksByCountry: Record<CountryIso, Array<Link>>
+  threshold: number
+  user: User
+}
+
+type CountryEntry = {
+  countryIso: CountryIso
+  countryName: string
+  countryLinksUrl: string
+  links: Array<Link>
+}
+
+type GetCountryEntriesProps = {
+  assessmentName: AssessmentName
+  cycleName: CycleName
+  linksByCountry: Record<CountryIso, Array<Link>>
+  t: (key: string) => string
+}
+
+// Return links to the linkStatus page per assessment x cycle x country
+const getCountryLinksUrl = (params: {
+  assessmentName: AssessmentName
+  cycleName: CycleName
+  countryIso: CountryIso
+}): string =>
+  `${ProcessEnv.appUri}${Routes.CountryHomeSection.generatePath({ ...params, sectionName: SectionNames.Country.Home.linksStatus })}`
+
+// Returns the text version of the body of the email
+const _getTextLines = (countryEntries: Array<CountryEntry>): string =>
+  countryEntries
+    .map(({ countryIso, countryLinksUrl, countryName, links }) => {
+      const urlLines = links.map(({ link }) => `    - ${link}`).join('\n')
+      return `  ${countryName} (${countryIso}): ${links.length} invalid link(s)\n  ${countryLinksUrl}\n${urlLines}`
+    })
+    .join('\n\n')
+
+// Returns the HTML body of the email
+const _getHtmlItems = (countryEntries: Array<CountryEntry>): string =>
+  countryEntries
+    .map(({ countryIso, countryLinksUrl, countryName, links }) => {
+      const urlItems = links.map(({ link }) => `<li><a href="${link}">${link}</a></li>`).join('')
+      return `<li><a href="${countryLinksUrl}"><strong>${countryName} (${countryIso})</strong></a>: ${links.length} invalid link(s)<ul>${urlItems}</ul></li>`
+    })
+    .join('<br>')
+
+const _getCountryEntries = (props: GetCountryEntriesProps): Array<CountryEntry> => {
+  const { assessmentName, cycleName, linksByCountry, t } = props
+
+  const entries = Object.entries(linksByCountry) as Array<[CountryIso, Array<Link>]>
+
+  return entries.map(([countryIso, links]) => ({
+    countryIso,
+    countryName: t(`area.${countryIso}.listName`),
+    countryLinksUrl: getCountryLinksUrl({ assessmentName, cycleName, countryIso }),
+    links,
+  }))
+}
+
+export const buildEmail = async (props: Props): Promise<MailServiceEmail> => {
+  const { assessment, cycle, linksByCountry, threshold, user } = props
+  const { t } = await createI18nPromise(Lang.en)
+  const to = user.email
+
+  const subject = t('email.invalidLinks.subject')
+  const messageHeader = t('email.invalidLinks.messageHeader', { recipientName: Users.getFullName(user) })
+  const messageBodyIntro = t('email.invalidLinks.messageBodyIntro', { threshold })
+  const messageFooter = t('email.invalidLinks.messageFooter')
+
+  const { name: assessmentName } = assessment.props
+  const { name: cycleName } = cycle
+
+  const countryEntries = _getCountryEntries({ assessmentName, cycleName, linksByCountry, t })
+  const textLines = _getTextLines(countryEntries)
+  const htmlItems = _getHtmlItems(countryEntries)
+
+  const text = [messageHeader, messageBodyIntro, textLines, messageFooter].join('\n\n')
+
+  const html = [
+    `<p>${messageHeader}</p>`,
+    `<p>${messageBodyIntro}</p>`,
+    `<ul>${htmlItems}</ul>`,
+    `<p style="white-space: pre-line">${messageFooter}</p>`,
+  ].join('')
+
+  return { to, subject, text, html }
+}

src/server/service/mail/notifyLinksInvalid/groupLinksByCountry.ts

@@ -0,0 +1,34 @@
+import { CountryIso } from 'meta/area/countryIso'
+import { Assessment } from 'meta/assessment/assessment'
+import { Cycle } from 'meta/assessment/cycle'
+import { Link, LinkValidationStatusCode } from 'meta/cycleData/links/link'
+
+import { LinkRepository } from 'server/db/repository/assessmentCycle/links'
+
+type Props = {
+  assessment: Assessment
+  cycle: Cycle
+  threshold: number
+}
+
+const codes = [
+  LinkValidationStatusCode.enotfound,
+  LinkValidationStatusCode.urlParsingError,
+  LinkValidationStatusCode.empty,
+]
+
+const filters = { codes, approved: false }
+
+// Return links grouped by country iso { ISO: Array<Link> }
+export const groupLinksByCountry = async (props: Props): Promise<Record<CountryIso, Array<Link>>> => {
+  const { assessment, cycle, threshold } = props
+
+  const links = await LinkRepository.getMany({ assessment, cycle, filters })
+
+  const grouped = Object.groupBy(links, (link) => link.countryIso)
+
+  // Filter out by threshold
+  return Object.fromEntries(
+    Object.entries(grouped).filter(([, countryLinks]) => (countryLinks?.length ?? 0) > threshold)
+  ) as Record<CountryIso, Array<Link>>
+}

src/server/service/mail/notifyLinksInvalid/index.ts

@@ -0,0 +1,34 @@
+import { Assessment } from 'meta/assessment/assessment'
+import { Cycle } from 'meta/assessment/cycle'
+
+import { UserRepository } from 'server/db/repository/public/user'
+import { sendMail } from 'server/service/mail/mail'
+
+import { buildEmail } from './buildEmail'
+import { groupLinksByCountry } from './groupLinksByCountry'
+
+type Props = {
+  assessment: Assessment
+  cycle: Cycle
+}
+
+const INVALID_LINK_THRESHOLD = 3
+
+export const notifyLinksInvalid = async (props: Props): Promise<void> => {
+  const { assessment, cycle } = props
+
+  const linksByCountry = await groupLinksByCountry({ assessment, cycle, threshold: INVALID_LINK_THRESHOLD })
+  // Return if no matches
+  if (Object.keys(linksByCountry).length === 0) return
+
+  // TODO: Instead, send to RFC (pt2)
+  const admins = await UserRepository.getAdmins()
+
+  await Promise.all(
+    admins.map(async (user) => {
+      const emailParams = { user, assessment, cycle, linksByCountry, threshold: INVALID_LINK_THRESHOLD }
+      const email = await buildEmail(emailParams)
+      return sendMail(email)
+    })
+  )
+}

src/server/worker/cronJobs/notifyLinksInvalid.ts

@@ -0,0 +1,49 @@
+import { QueueEvents } from 'bullmq'
+
+import { AssessmentNames } from 'meta/assessment/assessment'
+import { CycleNames } from 'meta/assessment/cycle/names'
+
+import { AssessmentController } from 'server/controller/assessment'
+import { CycleDataController } from 'server/controller/cycleData'
+import { UserController } from 'server/controller/user'
+import { MailService } from 'server/service'
+import { ProcessEnv } from 'server/utils'
+import { Job } from 'server/worker/job/job'
+import { triggerVerifyLinksWorker } from 'server/worker/tasks/verifyLinks/triggerVerifyLinksWorker'
+import { VisitCycleLinksQueueFactory } from 'server/worker/tasks/verifyLinks/visitCycleLinks/queueFactory'
+
+const name = 'Scheduler-NotifyLinksInvalid'
+// 20 min
+const VERIFY_TIMEOUT_MS = 20 * 60 * 1000
+const cycleName = CycleNames.latest
+const assessmentName = AssessmentNames.fra
+
+export class NotifyLinksInvalid extends Job {
+  constructor() {
+    super(name)
+  }
+
+  protected async execute(): Promise<void> {
+    const { assessment, cycle } = await AssessmentController.getOneWithCycle({
+      assessmentName,
+      cycleName,
+    })
+
+    const user = await UserController.getUserRobot()
+
+    // get existing job or create new job and process it
+    const existingJob = await VisitCycleLinksQueueFactory.getQueuedOrActiveJob({ assessment, cycle })
+    const job = existingJob ?? (await CycleDataController.Links.verify({ assessment, cycle, user }))
+    await triggerVerifyLinksWorker()
+
+    const connection = { url: ProcessEnv.redisQueueUrl }
+    const queueEvents = new QueueEvents(VisitCycleLinksQueueFactory.queueName, { connection })
+
+    try {
+      await job.waitUntilFinished(queueEvents, VERIFY_TIMEOUT_MS)
+    } finally {
+      await queueEvents.close()
+      await MailService.notifyLinksInvalid({ assessment, cycle })
+    }
+  }
+}

src/server/worker/index.ts

@@ -1,10 +1,12 @@
 import cron from 'node-cron'
 
 import { CleanUpFiles } from 'server/worker/cronJobs/cleanUpFiles'
+import { NotifyLinksInvalid } from 'server/worker/cronJobs/notifyLinksInvalid'
 import { RefreshMaterializedViews } from 'server/worker/cronJobs/refreshMaterializedViews'
 import { RemindReviewers } from 'server/worker/cronJobs/remindReviewers'
 
 const cleanUpFiles = new CleanUpFiles()
+const notifyLinksInvalid = new NotifyLinksInvalid()
 const refreshMaterializedViews = new RefreshMaterializedViews()
 const remindReviewers = new RemindReviewers()
 
@@ -16,3 +18,6 @@ cron.schedule('0 0 * * 0', cleanUpFiles.run.bind(cleanUpFiles))
 
 // every day at 00:01 (12:01 AM)
 cron.schedule('0 0 * * 0', remindReviewers.run.bind(remindReviewers))
+
+// every Monday at 00:05
+cron.schedule('5 0 * * 1', notifyLinksInvalid.run.bind(notifyLinksInvalid))