From ec6350f53b4f6995799ade1ba4a43eaca76d70e3 Mon Sep 17 00:00:00 2001 From: dinesh Date: Mon, 9 Feb 2026 09:48:45 -0800 Subject: [PATCH] Refactor submit-emailer to common cron job --- deployments/charts/epic-cron/.gitignore | 4 + .../epic-cron/templates/deployment.yaml | 80 +++++- .../charts/epic-cron/templates/secret.yaml | 20 +- deployments/charts/epic-cron/values.prod.yaml | 40 --- deployments/charts/epic-cron/values.test.yaml | 40 --- deployments/charts/epic-cron/values.yaml | 33 ++- jobs/epic-cron/MIGRATION_SUMMARY.md | 237 ++++++++++++++++++ jobs/epic-cron/config.py | 39 +++ jobs/epic-cron/cron/crontab | 10 +- jobs/epic-cron/invoke_jobs.py | 64 ++++- jobs/epic-cron/migrations/env.py | 90 ------- jobs/epic-cron/pre-hook-update-db.sh | 4 - jobs/epic-cron/requirements.txt | 2 - jobs/epic-cron/requirements/prod.txt | 9 +- jobs/epic-cron/run_approved_condition.sh | 3 + jobs/epic-cron/run_centre_emailer.sh | 3 + jobs/epic-cron/run_emailer.sh | 3 + jobs/epic-cron/src/compliance-api | 1 + jobs/epic-cron/src/condition-api | 1 + .../src/epic_cron/models/__init__.py | 16 ++ jobs/epic-cron/src/epic_cron/models/db.py | 56 ++++- .../src/epic_cron/models/email_job.py | 15 ++ .../epic_cron/processors/centre/__init__.py | 33 +++ .../processors/centre/access_denied.py | 46 ++++ .../processors/centre/access_granted.py | 50 ++++ .../centre/access_request_received_dst.py | 53 ++++ .../centre/access_request_submitted.py | 51 ++++ .../repositories/email_repository.py | 68 +++++ .../approved_condition_sync_service.py | 109 ++++++++ .../services/centre_email_service.py | 56 +++++ .../src/epic_cron/services/ches_service.py | 158 ++++++++++++ .../services/invitation_email_service.py | 109 ++++++++ .../src/epic_cron/services/mail_service.py | 156 ++++++++++++ .../package_submission_email_service.py | 104 ++++++++ .../services/request_update_email_service.py | 52 ++++ .../services/resubmission_email_service.py | 81 ++++++ .../src/epic_cron/services/track_service.py | 24 +- .../epic-cron/src/epic_cron/utils/__init__.py | 25 ++ .../src/epic_cron/utils/constants.py | 27 ++ .../epic-cron/src/epic_cron/utils/datetime.py | 26 ++ jobs/epic-cron/src/submit-api | 1 + jobs/epic-cron/tasks/centre_mail.py | 45 ++++ jobs/epic-cron/tasks/project_extractor.py | 168 ++++++++++--- jobs/epic-cron/tasks/submit_mail.py | 32 +++ .../tasks/sync_approved_condition.py | 32 +++ jobs/epic-cron/tasks/virus_scanner.py | 8 +- .../centre/access_denied_notification.html | 87 +++++++ .../centre/access_granted_notification.html | 95 +++++++ .../access_request_received_notification.html | 101 ++++++++ ...access_request_submitted_confirmation.html | 90 +++++++ ...nagement_plan_submission_notify_staff.html | 84 +++++++ ...nagement_plan_submission_verification.html | 87 +++++++ ...anagement_plan_update_request_created.html | 82 ++++++ .../templates/submit/new_user_invitation.html | 86 +++++++ .../submit/resubmission_request.html | 86 +++++++ jobs/epic-cron/tests/submit/__init__.py | 14 ++ jobs/epic-cron/tests/submit/jobs/__init__.py | 14 ++ 57 files changed, 2830 insertions(+), 280 deletions(-) create mode 100644 deployments/charts/epic-cron/.gitignore delete mode 100644 deployments/charts/epic-cron/values.prod.yaml delete mode 100644 deployments/charts/epic-cron/values.test.yaml create mode 100644 jobs/epic-cron/MIGRATION_SUMMARY.md delete mode 100644 jobs/epic-cron/migrations/env.py delete mode 100644 jobs/epic-cron/pre-hook-update-db.sh create mode 100644 jobs/epic-cron/run_approved_condition.sh create mode 100644 jobs/epic-cron/run_centre_emailer.sh create mode 100644 jobs/epic-cron/run_emailer.sh create mode 160000 jobs/epic-cron/src/compliance-api create mode 160000 jobs/epic-cron/src/condition-api create mode 100644 jobs/epic-cron/src/epic_cron/models/email_job.py create mode 100644 jobs/epic-cron/src/epic_cron/processors/centre/__init__.py create mode 100644 jobs/epic-cron/src/epic_cron/processors/centre/access_denied.py create mode 100644 jobs/epic-cron/src/epic_cron/processors/centre/access_granted.py create mode 100644 jobs/epic-cron/src/epic_cron/processors/centre/access_request_received_dst.py create mode 100644 jobs/epic-cron/src/epic_cron/processors/centre/access_request_submitted.py create mode 100644 jobs/epic-cron/src/epic_cron/repositories/email_repository.py create mode 100644 jobs/epic-cron/src/epic_cron/services/approved_condition_sync_service.py create mode 100644 jobs/epic-cron/src/epic_cron/services/centre_email_service.py create mode 100644 jobs/epic-cron/src/epic_cron/services/ches_service.py create mode 100644 jobs/epic-cron/src/epic_cron/services/invitation_email_service.py create mode 100644 jobs/epic-cron/src/epic_cron/services/mail_service.py create mode 100644 jobs/epic-cron/src/epic_cron/services/package_submission_email_service.py create mode 100644 jobs/epic-cron/src/epic_cron/services/request_update_email_service.py create mode 100644 jobs/epic-cron/src/epic_cron/services/resubmission_email_service.py create mode 100644 jobs/epic-cron/src/epic_cron/utils/constants.py create mode 100644 jobs/epic-cron/src/epic_cron/utils/datetime.py create mode 160000 jobs/epic-cron/src/submit-api create mode 100644 jobs/epic-cron/tasks/centre_mail.py create mode 100644 jobs/epic-cron/tasks/submit_mail.py create mode 100644 jobs/epic-cron/tasks/sync_approved_condition.py create mode 100644 jobs/epic-cron/templates/submit/centre/access_denied_notification.html create mode 100644 jobs/epic-cron/templates/submit/centre/access_granted_notification.html create mode 100644 jobs/epic-cron/templates/submit/centre/access_request_received_notification.html create mode 100644 jobs/epic-cron/templates/submit/centre/access_request_submitted_confirmation.html create mode 100644 jobs/epic-cron/templates/submit/management_plan_submission_notify_staff.html create mode 100644 jobs/epic-cron/templates/submit/management_plan_submission_verification.html create mode 100644 jobs/epic-cron/templates/submit/management_plan_update_request_created.html create mode 100644 jobs/epic-cron/templates/submit/new_user_invitation.html create mode 100644 jobs/epic-cron/templates/submit/resubmission_request.html create mode 100644 jobs/epic-cron/tests/submit/__init__.py create mode 100644 jobs/epic-cron/tests/submit/jobs/__init__.py diff --git a/deployments/charts/epic-cron/.gitignore b/deployments/charts/epic-cron/.gitignore new file mode 100644 index 0000000..1088a0d --- /dev/null +++ b/deployments/charts/epic-cron/.gitignore @@ -0,0 +1,4 @@ +# Environment-specific values files +values.dev.yaml +values.test.yaml +values.prod.yaml diff --git a/deployments/charts/epic-cron/templates/deployment.yaml b/deployments/charts/epic-cron/templates/deployment.yaml index 8c20bb6..440722c 100644 --- a/deployments/charts/epic-cron/templates/deployment.yaml +++ b/deployments/charts/epic-cron/templates/deployment.yaml @@ -36,18 +36,18 @@ spec: - name: TRACK_DATABASE_USERNAME valueFrom: secretKeyRef: - name: track-patroni + name: epictrack-patroni key: app-db-username - name: TRACK_DATABASE_PASSWORD valueFrom: secretKeyRef: - name: track-patroni + name: epictrack-patroni key: app-db-password - name: TRACK_DATABASE_NAME valueFrom: secretKeyRef: - name: track-patroni - key: app-db-username + name: epictrack-patroni + key: app-db-name - name: TRACK_DATABASE_HOST value: "{{ .Values.TRACK.database.host }}" - name: TRACK_DATABASE_PORT @@ -56,18 +56,18 @@ spec: - name: SUBMIT_DATABASE_USERNAME valueFrom: secretKeyRef: - name: submit-patroni - key: app-db-name + name: {{ .Values.name }} + key: SUBMIT_DATABASE_USERNAME - name: SUBMIT_DATABASE_PASSWORD valueFrom: secretKeyRef: - name: submit-patroni - key: app-db-password + name: {{ .Values.name }} + key: SUBMIT_DATABASE_PASSWORD - name: SUBMIT_DATABASE_NAME valueFrom: secretKeyRef: - name: submit-patroni - key: app-db-name + name: {{ .Values.name }} + key: SUBMIT_DATABASE_NAME - name: SUBMIT_DATABASE_HOST value: "{{ .Values.SUBMIT.database.host }}" - name: SUBMIT_DATABASE_PORT @@ -93,6 +93,66 @@ spec: - name: COMPLIANCE_DATABASE_PORT value: "{{ .Values.COMPLIANCE.database.port }}" + - name: CENTRE_DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.name }} + key: CENTRE_DATABASE_USERNAME + - name: CENTRE_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.name }} + key: CENTRE_DATABASE_PASSWORD + - name: CENTRE_DATABASE_NAME + valueFrom: + secretKeyRef: + name: {{ .Values.name }} + key: CENTRE_DATABASE_NAME + - name: CENTRE_DATABASE_HOST + value: "{{ .Values.CENTRE.database.host }}" + - name: CENTRE_DATABASE_PORT + value: "{{ .Values.CENTRE.database.port }}" + + - name: CHES_TOKEN_ENDPOINT + value: "{{ .Values.CHES.tokenEndpoint }}" + - name: CHES_BASE_URL + value: "{{ .Values.CHES.apiEndpoint }}" + - name: CHES_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ .Values.name }} + key: CHES_CLIENT_ID + - name: CHES_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.name }} + key: CHES_CLIENT_SECRET + + - name: WEB_URL + value: "{{ .Values.SUBMIT.web.url }}" + - name: SENDER_EMAIL + value: "{{ .Values.SUBMIT.sender.email }}" + - name: STAFF_SUPPORT_MAIL_ID + value: "{{ .Values.SUBMIT.staffSupportMailId }}" + + - name: KEYCLOAK_BASE_URL + value: "https://dev.loginproxy.gov.bc.ca/auth" + - name: KEYCLOAK_REALM_NAME + value: "eao-epic" + - name: KEYCLOAK_SERVICE_ACCOUNT_ID + valueFrom: + secretKeyRef: + name: {{ .Values.name }} + key: KEYCLOAK_SERVICE_ACCOUNT_ID + - name: KEYCLOAK_SERVICE_ACCOUNT_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.name }} + key: KEYCLOAK_SERVICE_ACCOUNT_SECRET + + - name: CONDITION_API_BASE_URL + value: "https://epic-dev.apps.silver.devops.gov.bc.ca/api/v1/condition" + resources: requests: cpu: {{ .Values.resources.requests.cpu }} diff --git a/deployments/charts/epic-cron/templates/secret.yaml b/deployments/charts/epic-cron/templates/secret.yaml index 53d4c07..e3fafd5 100644 --- a/deployments/charts/epic-cron/templates/secret.yaml +++ b/deployments/charts/epic-cron/templates/secret.yaml @@ -1,10 +1,20 @@ apiVersion: v1 kind: Secret metadata: - name: track-patroni + name: {{ .Values.name }} type: Opaque data: - # TRACK database credentials - app-db-username: {{ .Values.secrets.trackDbUsername | b64enc | quote }} - app-db-password: {{ .Values.secrets.trackDbPassword | b64enc | quote }} - app-db-name: {{ .Values.secrets.trackDbName | b64enc | quote }} + # Submit database credentials + SUBMIT_DATABASE_USERNAME: {{ .Values.secrets.submitDbUsername | b64enc | quote }} + SUBMIT_DATABASE_PASSWORD: {{ .Values.secrets.submitDbPassword | b64enc | quote }} + SUBMIT_DATABASE_NAME: {{ .Values.secrets.submitDbName | b64enc | quote }} + # Centre database credentials + CENTRE_DATABASE_USERNAME: {{ .Values.secrets.centreDbUsername | b64enc | quote }} + CENTRE_DATABASE_PASSWORD: {{ .Values.secrets.centreDbPassword | b64enc | quote }} + CENTRE_DATABASE_NAME: {{ .Values.secrets.centreDbName | b64enc | quote }} + # CHES credentials + CHES_CLIENT_ID: {{ .Values.secrets.chesClientId | b64enc | quote }} + CHES_CLIENT_SECRET: {{ .Values.secrets.chesClientSecret | b64enc | quote }} + # Keycloak service account credentials + KEYCLOAK_SERVICE_ACCOUNT_ID: {{ .Values.secrets.keycloakServiceAccountId | b64enc | quote }} + KEYCLOAK_SERVICE_ACCOUNT_SECRET: {{ .Values.secrets.keycloakServiceAccountSecret | b64enc | quote }} diff --git a/deployments/charts/epic-cron/values.prod.yaml b/deployments/charts/epic-cron/values.prod.yaml deleted file mode 100644 index df461de..0000000 --- a/deployments/charts/epic-cron/values.prod.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: epic-cron -imageNamespace: c8b80a-tools -env: prod -database: - host: submit-patroni - port: "5432" -resources: - requests: - cpu: 100m - memory: 100Mi - limits: - cpu: 250m - memory: 200Mi -replicaCount: 1 -imageTag: dev - -cronTab: | - 0 1 * * * default cd /epic-cron && ./run_project_cron_submit.sh - 0 3 * * * default cd /epic-cron && ./run_project_cron_compliance.sh - -SUBMIT: - database: - host: submit-patroni - port: "5432" - -COMPLIANCE: - database: - host: compliance-patroni - port: "5432" - -TRACK: - database: - host: - port: "5432" - - -secrets: - trackDbUsername: "" - trackDbPassword: "" - trackDbName: "" diff --git a/deployments/charts/epic-cron/values.test.yaml b/deployments/charts/epic-cron/values.test.yaml deleted file mode 100644 index 24298d9..0000000 --- a/deployments/charts/epic-cron/values.test.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: epic-cron -imageNamespace: c8b80a-tools -env: test -database: - host: submit-patroni - port: "5432" -resources: - requests: - cpu: 100m - memory: 100Mi - limits: - cpu: 250m - memory: 200Mi -replicaCount: 1 -imageTag: dev - -cronTab: | - 0 1 * * * default cd /epic-cron && ./run_project_cron_submit.sh - 0 3 * * * default cd /epic-cron && ./run_project_cron_compliance.sh - -SUBMIT: - database: - host: submit-patroni - port: "5432" - -COMPLIANCE: - database: - host: compliance-patroni - port: "5432" - -TRACK: - database: - host: - port: "5432" - - -secrets: - trackDbUsername: "" - trackDbPassword: "" - trackDbName: "" diff --git a/deployments/charts/epic-cron/values.yaml b/deployments/charts/epic-cron/values.yaml index 7562abf..35e97d4 100644 --- a/deployments/charts/epic-cron/values.yaml +++ b/deployments/charts/epic-cron/values.yaml @@ -17,11 +17,19 @@ imageTag: dev cronTab: | 0 1 * * * default cd /epic-cron && ./run_project_cron_submit.sh 0 3 * * * default cd /epic-cron && ./run_project_cron_compliance.sh + */5 * * * * default cd /epic-cron && ./run_emailer.sh + */5 * * * * default cd /epic-cron && ./run_centre_emailer.sh + 0 17 * * 1-5 default cd /epic-cron && ./run_approved_condition.sh SUBMIT: database: host: submit-patroni port: "5432" + web: + url: "https://dev.submit.eao.gov.bc.ca" + sender: + email: "EAO.ManagementPlanSupport@gov.bc.ca" + staffSupportMailId: "" COMPLIANCE: database: @@ -30,11 +38,28 @@ COMPLIANCE: TRACK: database: - host: + host: epictrack-patroni port: "5432" +CENTRE: + database: + host: submit-patroni + port: "5432" + +CHES: + tokenEndpoint: "https://dev.loginproxy.gov.bc.ca/auth/realms/comsvcauth/protocol/openid-connect/token" + apiEndpoint: "https://ches-dev.api.gov.bc.ca" + clientId: "" + clientSecret: "" secrets: - trackDbUsername: "" - trackDbPassword: "" - trackDbName: "" + submitDbUsername: "" + submitDbPassword: "" + submitDbName: "" + centreDbUsername: "" + centreDbPassword: "" + centreDbName: "" + keycloakServiceAccountId: "" + keycloakServiceAccountSecret: "" + chesClientId: "" + chesClientSecret: "" diff --git a/jobs/epic-cron/MIGRATION_SUMMARY.md b/jobs/epic-cron/MIGRATION_SUMMARY.md new file mode 100644 index 0000000..c6b38de --- /dev/null +++ b/jobs/epic-cron/MIGRATION_SUMMARY.md @@ -0,0 +1,237 @@ +# Submit-Cron to Epic-Cron Migration Summary + +**Migration Date:** February 6, 2026 +**Status:** ✅ COMPLETED + +## Overview + +Successfully migrated all submit-cron functionality from `submit-v2/submit-cron` to `common/jobs/epic-cron`. The epic-cron service now consolidates all EPIC system cron jobs in one location as part of the Epic.Common repository. + +## What Was Migrated + +### Source Code +- ✅ All submit_cron Python modules → `epic_cron/submit/` + - 9 services (mail, CHES, invitation, package submission, etc.) + - Models for Submit and Centre databases + - 5 processors for centre email handling + - Repositories and utilities +- ✅ 3 task files (submit_mail, centre_mail, sync_approved_condition) +- ✅ 9 HTML email templates → `templates/submit/` +- ✅ 3 shell scripts (run_emailer.sh, run_centre_emailer.sh, run_approved_condition.sh) + +### Configuration +- ✅ CHES email service configuration +- ✅ JWT/OIDC and Keycloak settings +- ✅ Submit web URL and sender email configuration +- ✅ Centre database configuration +- ✅ Condition API configuration + +### Cron Jobs +Now running 5 cron jobs total: +1. **Project Extractor (Submit)** - Daily at 1am +2. **Project Extractor (Compliance)** - Daily at 3am +3. **Submit Emailer** - Every 5 minutes +4. **Centre Emailer** - Every 5 minutes +5. **Approved Condition Sync** - Weekdays at 5pm + +### Helm Chart +- ✅ Updated deployment.yaml with CHES, Centre DB, and Keycloak environment variables +- ✅ Added secrets for CHES and Keycloak credentials +- ✅ Updated values.yaml, values.prod.yaml, and values.test.yaml +- ✅ Configured crontab with all 5 jobs + +### Dependencies +- ✅ Added marshmallow==3.21.3 +- ✅ Added marshmallow-enum==1.5.1 +- ✅ Added pytz +- ✅ Updated flask-jwt-oidc + +### Tests & Migrations +- ✅ Copied to `tests/submit/` +- ✅ Copied to `migrations/submit/` + +## Key Changes Made + +### Import Path Updates +All imports changed from: +```python +from submit_cron.services.mail_service import EmailService +``` +To: +```python +from epic_cron.submit.services.mail_service import EmailService +``` + +### Configuration Updates +**File:** `config.py` +- Added CHES configuration (token endpoint, client ID/secret, base URL) +- Added JWT/OIDC settings +- Added Centre database URI +- Added submit web configuration (WEB_URL, SENDER_EMAIL, STAFF_SUPPORT_MAIL_ID) +- Added SQLALCHEMY_DATABASE_URI alias for backward compatibility + +### Job Handler Updates +**File:** `invoke_jobs.py` +- Added EMAIL job handler (supports both SUBMIT and CENTRE targets) +- Added SYNC_CONDITION job handler +- Imported SubmitMailer, CentreMailer, and SyncApprovedCondition classes + +### Crontab Updates +**File:** `cron/crontab` +``` +# PROJECT EXTRACTORS +0 1 * * * default cd /epic-cron && ./run_project_cron_submit.sh +0 3 * * * default cd /epic-cron && ./run_project_cron_compliance.sh +# SUBMIT EMAILER - Runs every 5 minutes +*/5 * * * * default cd /epic-cron && ./run_emailer.sh +# CENTRE EMAILER - Runs every 5 minutes +*/5 * * * * default cd /epic-cron && ./run_centre_emailer.sh +# SYNC APPROVED CONDITION - Runs at 5pm on weekdays +0 17 * * 1-5 default cd /epic-cron && ./run_approved_condition.sh +``` + +## Directory Structure + +``` +common/jobs/epic-cron/ +├── src/ +│ └── epic_cron/ +│ ├── submit/ # NEW: All submit-cron code +│ │ ├── models/ +│ │ ├── services/ +│ │ ├── processors/ +│ │ ├── repositories/ +│ │ └── utils/ +│ ├── models/ +│ ├── services/ +│ └── utils/ +├── tasks/ +│ ├── project_extractor.py +│ ├── virus_scanner.py +│ ├── submit_mail.py # NEW +│ ├── centre_mail.py # NEW +│ └── sync_approved_condition.py # NEW +├── templates/ +│ └── submit/ # NEW: 9 HTML email templates +├── migrations/ +│ └── submit/ # NEW: Submit-specific migrations +├── tests/ +│ └── submit/ # NEW: Submit-specific tests +├── cron/ +│ └── crontab # UPDATED: 5 jobs +├── config.py # UPDATED: Added CHES, JWT, Centre DB +├── invoke_jobs.py # UPDATED: Added EMAIL, SYNC_CONDITION +├── requirements/ +│ └── prod.txt # UPDATED: Added dependencies +├── run_emailer.sh # NEW +├── run_centre_emailer.sh # NEW +└── run_approved_condition.sh # NEW +``` + +## Helm Chart Environment Variables Added + +### CHES Configuration +- `CHES_TOKEN_ENDPOINT` +- `CHES_BASE_URL` +- `CHES_CLIENT_ID` (secret) +- `CHES_CLIENT_SECRET` (secret) + +### Centre Database +- `CENTRE_DATABASE_USERNAME` +- `CENTRE_DATABASE_PASSWORD` +- `CENTRE_DATABASE_NAME` +- `CENTRE_DATABASE_HOST` +- `CENTRE_DATABASE_PORT` + +### Submit Web Configuration +- `WEB_URL` +- `SENDER_EMAIL` +- `STAFF_SUPPORT_MAIL_ID` + +### Keycloak +- `KEYCLOAK_BASE_URL` +- `KEYCLOAK_REALM_NAME` +- `KEYCLOAK_SERVICE_ACCOUNT_ID` (secret) +- `KEYCLOAK_SERVICE_ACCOUNT_SECRET` (secret) + +### Condition API +- `CONDITION_API_BASE_URL` + +## Next Steps + +### 1. Testing +- [ ] Test all 5 cron jobs in development environment +- [ ] Verify database connections (Submit, Centre, Track, Compliance, Condition) +- [ ] Test CHES email sending functionality +- [ ] Test Keycloak authentication +- [ ] Run unit tests from `tests/submit/` + +### 2. Deployment +- [ ] Build Docker image with merged code +- [ ] Update CI/CD pipeline to build epic-cron with submit functionality +- [ ] Deploy to test environment +- [ ] Monitor cron job execution logs +- [ ] Validate email delivery + +### 3. Configuration +- [ ] Set CHES credentials in secrets +- [ ] Set Keycloak service account credentials +- [ ] Configure environment-specific URLs and email addresses +- [ ] Update CONDITION_API_BASE_URL for each environment + +### 4. Documentation +- [ ] Update team documentation about consolidated cron location +- [ ] Document new environment variables +- [ ] Update deployment procedures + +### 5. Deprecation +- [ ] Plan deprecation timeline for submit-v2/submit-cron deployment +- [ ] Notify teams about migration +- [ ] Schedule removal of old submit-cron deployment + +## Important Notes + +### Backward Compatibility +- `SQLALCHEMY_DATABASE_URI` is aliased to `SUBMIT_DATABASE_URI` for compatibility +- All original submit-cron functionality is preserved +- No changes were made to submit-v2/submit-cron (as requested) + +### Database Connections +Epic-cron now connects to 5 databases: +1. **Track DB** - Project tracking data +2. **Submit DB** - Submit application data +3. **Compliance DB** - Compliance data +4. **Centre DB** - Centre-specific data (uses submit-patroni) +5. **Condition DB** - Condition API (via REST API) + +### Python Version +- Epic-cron uses Python 3.10 +- Submit-cron used Python 3.12 +- Consider upgrading to 3.12 in future for consistency + +### Dependency Considerations +- Marshmallow version pinned to 3.21.3 for compatibility +- Flask-jwt-oidc version updated (removed specific version pin) +- All submit-cron dependencies are now included + +## Verification Checklist + +Before deploying to production: +- [ ] All imports updated correctly +- [ ] All environment variables configured +- [ ] Secrets properly set in Helm values +- [ ] Crontab schedules verified +- [ ] Database connections tested +- [ ] CHES email service tested +- [ ] All 5 cron jobs execute successfully +- [ ] Logs show no errors +- [ ] Email delivery confirmed +- [ ] Approved condition sync working + +## Contact + +For questions or issues related to this migration, contact the EPIC development team. + +--- + +**Migration completed successfully on February 6, 2026** diff --git a/jobs/epic-cron/config.py b/jobs/epic-cron/config.py index 9bb7245..44d097e 100644 --- a/jobs/epic-cron/config.py +++ b/jobs/epic-cron/config.py @@ -119,6 +119,45 @@ class _Config(): # pylint: disable=too-few-public-methods CLAMAV_HOST = os.getenv('CLAMAV_HOST') CLAMAV_PORT = os.getenv('CLAMAV_PORT') + # CHES Configuration for Email Service + CHES_TOKEN_ENDPOINT = os.getenv('CHES_TOKEN_ENDPOINT') + CHES_CLIENT_ID = os.getenv('CHES_CLIENT_ID') + CHES_CLIENT_SECRET = os.getenv('CHES_CLIENT_SECRET') + CHES_BASE_URL = os.getenv('CHES_BASE_URL') + + # Submit Web Configuration + WEB_URL = os.getenv('WEB_URL') + SENDER_EMAIL = os.getenv('SENDER_EMAIL') + STAFF_SUPPORT_MAIL_ID = os.getenv('STAFF_SUPPORT_MAIL_ID', '') + SIGNUP_URL_PATH = os.getenv('SIGNUP_URL_PATH', '/proponent/registration') + + # Condition API Configuration + CONDITION_API_BASE_URL = os.getenv('CONDITION_API_BASE_URL') + + # JWT_OIDC Settings for Submit + JWT_OIDC_WELL_KNOWN_CONFIG = os.getenv('JWT_OIDC_WELL_KNOWN_CONFIG') + JWT_OIDC_ALGORITHMS = os.getenv('JWT_OIDC_ALGORITHMS', 'RS256') + JWT_OIDC_JWKS_URI = os.getenv('JWT_OIDC_JWKS_URI') + JWT_OIDC_ISSUER = os.getenv('JWT_OIDC_ISSUER') + JWT_OIDC_AUDIENCE = os.getenv('JWT_OIDC_AUDIENCE', 'account') + JWT_OIDC_CACHING_ENABLED = os.getenv('JWT_OIDC_CACHING_ENABLED', 'True') + JWT_OIDC_JWKS_CACHE_TIMEOUT = 300 + + # Centre DB Configuration (for centre emailer) + CENTRE_DB_USER = os.getenv("CENTRE_DATABASE_USERNAME", "") + CENTRE_DB_PASSWORD = os.getenv("CENTRE_DATABASE_PASSWORD", "") + CENTRE_DB_NAME = os.getenv("CENTRE_DATABASE_NAME", "") + CENTRE_DB_HOST = os.getenv("CENTRE_DATABASE_HOST", "") + CENTRE_DB_PORT = os.getenv("CENTRE_DATABASE_PORT", "5432") + CENTRE_DATABASE_URI = ( + f"postgresql://{CENTRE_DB_USER}:{CENTRE_DB_PASSWORD}@{CENTRE_DB_HOST}:{int(CENTRE_DB_PORT)}/{CENTRE_DB_NAME}" + ) + + # For backward compatibility with submit_cron code + SQLALCHEMY_DATABASE_URI = SUBMIT_DATABASE_URI + + ENVIRONMENT = os.getenv('ENVIRONMENT', os.getenv('ENV_NAME', '')) + class DevConfig(_Config): # pylint: disable=too-few-public-methods """Dev Config.""" diff --git a/jobs/epic-cron/cron/crontab b/jobs/epic-cron/cron/crontab index 0b0cb35..e32d3b9 100644 --- a/jobs/epic-cron/cron/crontab +++ b/jobs/epic-cron/cron/crontab @@ -1,2 +1,10 @@ -* * * * * default cd /epic-cron && ./run_project_cron.sh +# PROJECT EXTRACTORS +0 1 * * * default cd /epic-cron && ./run_project_cron_submit.sh +0 3 * * * default cd /epic-cron && ./run_project_cron_compliance.sh +# SUBMIT EMAILER - Runs every 5 minutes +*/5 * * * * default cd /epic-cron && ./run_emailer.sh +# CENTRE EMAILER - Runs every 5 minutes +*/5 * * * * default cd /epic-cron && ./run_centre_emailer.sh +# SYNC APPROVED CONDITION - Runs at 5pm on weekdays +0 17 * * 1-5 default cd /epic-cron && ./run_approved_condition.sh # An empty line is required at the end of this file for a valid cron file \ No newline at end of file diff --git a/jobs/epic-cron/invoke_jobs.py b/jobs/epic-cron/invoke_jobs.py index 736e588..a384f37 100644 --- a/jobs/epic-cron/invoke_jobs.py +++ b/jobs/epic-cron/invoke_jobs.py @@ -1,24 +1,31 @@ import os import sys import argparse +import logging +from datetime import datetime from flask import Flask from utils.logger import setup_logging import config from tasks.project_extractor import ProjectExtractor, TargetSystem # Import the enum from tasks.virus_scanner import VirusScanner +from tasks.submit_mail import SubmitMailer +from tasks.centre_mail import CentreMailer +from tasks.sync_approved_condition import SyncApprovedCondition setup_logging(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'logging.conf')) # important to do this first +logger = logging.getLogger(__name__) + def create_app(run_mode=os.getenv('FLASK_ENV', 'production')): """Return a configured Flask App using the Factory method.""" from epic_cron.models.db import init_db # Import the correct methods app = Flask(__name__) - print(f'>>>>> Creating app in run_mode: {run_mode}') + logger.info(f'Creating app in run_mode: {run_mode}') # Load configuration based on the run mode - app.config.from_object(config.CONFIGURATION.get(run_mode, 'production')) + app.config.from_object(config.get_named_config(run_mode)) register_shellcontext(app) @@ -34,23 +41,46 @@ def shell_context(): app.shell_context_processor(shell_context) +def email_sender(target_system='SUBMIT'): + """Send emails for submit or centre system.""" + if target_system == 'CENTRE': + logger.info(f'Starting Centre Email Sending At {datetime.now()}') + CentreMailer.send_mail() + elif target_system == 'SUBMIT' or target_system is None or target_system == '': + logger.info(f'Starting Submit Email Sending At {datetime.now()}') + SubmitMailer.send_mail() + else: + logger.error(f'Invalid target_system "{target_system}". Must be SUBMIT or CENTRE.') + raise ValueError(f'Invalid target_system: {target_system}') + + def run(job_name, target_system=None, file_path=None): """Main function to run the job.""" application = create_app() with application.app_context(): if job_name == 'EXTRACT_PROJECT': - print(f'Running Project Extractor for {target_system.value}...') + application.logger.info(f'Running Project Extractor for {target_system.value}...') ProjectExtractor.do_sync(target_system=target_system) - application.logger.info(f'<<<< Completed Project Sync for {target_system.value} >>>') + application.logger.info(f'Completed Project Sync for {target_system.value}') elif job_name == 'SCAN_VIRUS': - print(f'Running Virus Scanner on: {file_path}') + application.logger.info(f'Running Virus Scanner on: {file_path}') VirusScanner.scan_file_from_path(file_path) - application.logger.info(f'<<<< Completed Virus Scan for {file_path} >>>') + application.logger.info(f'Completed Virus Scan for {file_path}') + + elif job_name == 'EMAIL': + application.logger.info(f'Starting Email Sending At {datetime.now()}') + email_sender(target_system) + application.logger.info(f'Completed Email Task') + + elif job_name == 'SYNC_CONDITION': + application.logger.info(f'Starting Approved Condition Sync At {datetime.now()}') + SyncApprovedCondition.sync_approved_condition() + application.logger.info(f'Completed Sync Approved Condition') else: - application.logger.debug('No valid job_name passed. Exiting without running any tasks.') + application.logger.warning('No valid job_name passed. Exiting without running any tasks.') @@ -59,12 +89,22 @@ def run(job_name, target_system=None, file_path=None): args = sys.argv[1:] if not args: - print("ERROR: You must provide either a target system (SUBMIT/COMPLIANCE) or 'SCAN_VIRUS' + file path.") + logger.error("You must provide a job type: SUBMIT/COMPLIANCE/EMAIL/SYNC_CONDITION/SCAN_VIRUS") sys.exit(1) - if args[0] == "SCAN_VIRUS": + job_type = args[0] + + if job_type == "EMAIL": + # EMAIL can have optional second arg for target system (CENTRE) + target_system = args[1] if len(args) > 1 else None + run("EMAIL", target_system=target_system) + + elif job_type == "SYNC_CONDITION": + run("SYNC_CONDITION") + + elif job_type == "SCAN_VIRUS": if len(args) < 2: - print("ERROR: You must provide a file path for SCAN_VIRUS.") + logger.error("You must provide a file path for SCAN_VIRUS.") sys.exit(1) file_path = args[1] run("SCAN_VIRUS", target_system=None, file_path=file_path) @@ -72,9 +112,9 @@ def run(job_name, target_system=None, file_path=None): else: # Assume EXTRACT_PROJECT with target_system try: - target_system = TargetSystem(args[0]) + target_system = TargetSystem(job_type) run("EXTRACT_PROJECT", target_system) except ValueError: - print(f"ERROR: Invalid target system '{args[0]}'. Must be one of {[ts.value for ts in TargetSystem]}") + logger.error(f"Invalid job type '{job_type}'. Must be one of: SUBMIT, COMPLIANCE, EMAIL, SYNC_CONDITION, SCAN_VIRUS") sys.exit(1) diff --git a/jobs/epic-cron/migrations/env.py b/jobs/epic-cron/migrations/env.py deleted file mode 100644 index 42438a5..0000000 --- a/jobs/epic-cron/migrations/env.py +++ /dev/null @@ -1,90 +0,0 @@ -from __future__ import with_statement - -import logging -from logging.config import fileConfig - -from flask import current_app - -from alembic import context - -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -fileConfig(config.config_file_name) -logger = logging.getLogger('alembic.env') - -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -config.set_main_option( - 'sqlalchemy.url', - str(current_app.extensions['migrate'].db.engine.url).replace('%', '%%')) -target_metadata = current_app.extensions['migrate'].db.metadata - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def run_migrations_offline(): - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, target_metadata=target_metadata, literal_binds=True - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online(): - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - """ - - # this callback is used to prevent an auto-migration from being generated - # when there are no changes to the schema - # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html - def process_revision_directives(context, revision, directives): - if getattr(config.cmd_opts, 'autogenerate', False): - script = directives[0] - if script.upgrade_ops.is_empty(): - directives[:] = [] - logger.info('No changes in schema detected.') - - connectable = current_app.extensions['migrate'].db.engine - - with connectable.connect() as connection: - context.configure( - connection=connection, - target_metadata=target_metadata, - process_revision_directives=process_revision_directives, - **current_app.extensions['migrate'].configure_args - ) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/jobs/epic-cron/pre-hook-update-db.sh b/jobs/epic-cron/pre-hook-update-db.sh deleted file mode 100644 index c9d46d6..0000000 --- a/jobs/epic-cron/pre-hook-update-db.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh -# cd /opt/app-root -#echo 'starting upgrade' -#python3 manage.py db upgrade \ No newline at end of file diff --git a/jobs/epic-cron/requirements.txt b/jobs/epic-cron/requirements.txt index ad3b9da..d60367a 100644 --- a/jobs/epic-cron/requirements.txt +++ b/jobs/epic-cron/requirements.txt @@ -1,6 +1,5 @@ Flask-Caching==2.3.1 Flask-Mail==0.10.0 -Flask-Migrate==4.1.0 Flask-Moment==1.0.6 Flask-SQLAlchemy==3.1.1 Flask==3.1.1 @@ -11,7 +10,6 @@ SQLAlchemy-Continuum==1.4.2 SQLAlchemy-Utils==0.41.2 SQLAlchemy==2.0.43 Werkzeug==3.1.3 -alembic==1.16.4 aniso8601==10.0.1 attrs==25.3.0 backports-datetime-fromisoformat==2.0.3 diff --git a/jobs/epic-cron/requirements/prod.txt b/jobs/epic-cron/requirements/prod.txt index 5b117c3..d7ee574 100644 --- a/jobs/epic-cron/requirements/prod.txt +++ b/jobs/epic-cron/requirements/prod.txt @@ -1,17 +1,17 @@ gunicorn Flask>=3.0.0 -Flask-Migrate Flask-Mail Flask-Moment Flask-SQLAlchemy SQLAlchemy-Continuum flask-restx flask-marshmallow==1.2.1 -flask-jwt-oidc==0.7.0 +flask-jwt-oidc python-dotenv psycopg2-binary +marshmallow==3.21.3 marshmallow-sqlalchemy==1.0.0 -marshmallow_enum +marshmallow-enum==1.5.1 jsonschema requests itsdangerous @@ -25,4 +25,5 @@ requests flask_cors pyhumps importlib-resources -clamd \ No newline at end of file +clamd +pytz \ No newline at end of file diff --git a/jobs/epic-cron/run_approved_condition.sh b/jobs/epic-cron/run_approved_condition.sh new file mode 100644 index 0000000..63eb01e --- /dev/null +++ b/jobs/epic-cron/run_approved_condition.sh @@ -0,0 +1,3 @@ +#! /bin/sh +echo 'run invoke_jobs.py SYNC_CONDITION' +python3 invoke_jobs.py SYNC_CONDITION \ No newline at end of file diff --git a/jobs/epic-cron/run_centre_emailer.sh b/jobs/epic-cron/run_centre_emailer.sh new file mode 100644 index 0000000..557561b --- /dev/null +++ b/jobs/epic-cron/run_centre_emailer.sh @@ -0,0 +1,3 @@ +#! /bin/sh +echo 'run invoke_jobs.py EMAIL CENTRE' +python3 invoke_jobs.py EMAIL CENTRE \ No newline at end of file diff --git a/jobs/epic-cron/run_emailer.sh b/jobs/epic-cron/run_emailer.sh new file mode 100644 index 0000000..3d1ce36 --- /dev/null +++ b/jobs/epic-cron/run_emailer.sh @@ -0,0 +1,3 @@ +#! /bin/sh +echo 'run invoke_jobs.py EMAIL SUBMIT' +python3 invoke_jobs.py EMAIL SUBMIT \ No newline at end of file diff --git a/jobs/epic-cron/src/compliance-api b/jobs/epic-cron/src/compliance-api new file mode 160000 index 0000000..7128e2a --- /dev/null +++ b/jobs/epic-cron/src/compliance-api @@ -0,0 +1 @@ +Subproject commit 7128e2a597ea8bba44293ab3cd10ccd32b06edc4 diff --git a/jobs/epic-cron/src/condition-api b/jobs/epic-cron/src/condition-api new file mode 160000 index 0000000..2b9ee54 --- /dev/null +++ b/jobs/epic-cron/src/condition-api @@ -0,0 +1 @@ +Subproject commit 2b9ee54b716db8e1a7e2108841c4113bec4a4390 diff --git a/jobs/epic-cron/src/epic_cron/models/__init__.py b/jobs/epic-cron/src/epic_cron/models/__init__.py index 8b13789..7c011c7 100644 --- a/jobs/epic-cron/src/epic_cron/models/__init__.py +++ b/jobs/epic-cron/src/epic_cron/models/__init__.py @@ -1 +1,17 @@ +# Copyright © 2021 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""This exports all of the models and schemas used by the application.""" + +from .db import db, ma diff --git a/jobs/epic-cron/src/epic_cron/models/db.py b/jobs/epic-cron/src/epic_cron/models/db.py index d9a6ad3..f9d98f8 100644 --- a/jobs/epic-cron/src/epic_cron/models/db.py +++ b/jobs/epic-cron/src/epic_cron/models/db.py @@ -1,32 +1,66 @@ +"""Initializations for db and marshmallow.""" + +from flask_marshmallow import Marshmallow from flask_sqlalchemy import SQLAlchemy from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from flask import current_app -# DB initialization for SQLAlchemy +# DB initialize - Flask-SQLAlchemy for submit services db = SQLAlchemy() +# Marshmallow for database model schema +ma = Marshmallow() + + def create_session(engine_uri): """Create a sessionmaker for the given database engine URI.""" - engine = create_engine(engine_uri) + engine = create_engine( + engine_uri, + pool_size=5, + max_overflow=10, + pool_pre_ping=True, + pool_recycle=3600 + ) return sessionmaker(bind=engine) -def init_db(app): - """Initialize the session for the Epic Track database.""" - print("Initializing Epic Track database...") + +def init_track_db(app): + """Initialize the session for the Track database.""" + print("Initializing Track database...") return create_session(app.config['TRACK_DATABASE_URI']) + def init_compliance_db(app): """Initialize the session for the Compliance database.""" print("Initializing Compliance database...") return create_session(app.config['COMPLIANCE_DATABASE_URI']) + +def init_centre_db(app): + """Initialize the session for the Centre database.""" + print("Initializing Centre database...") + return create_session(app.config['CENTRE_DATABASE_URI']) + + +def init_condition_db(app): + """Initialize the session for the Condition database.""" + print("Initializing Condition database...") + return create_session(app.config['CONDITION_DATABASE_URI']) + + def init_submit_db(app): - """Initialize the session for the Submit database.""" + """Initialize Flask-SQLAlchemy for the Submit database.""" print("Initializing Submit database...") - return create_session(app.config['SUBMIT_DATABASE_URI']) + db.init_app(app) + return db + + +# Aliases for backward compatibility +def init_db(app): + """Initialize the session for the Track database (alias for init_track_db).""" + return init_track_db(app) + def init_conditions_db(app): - """Initialize the session for the Con Repo database.""" - print("Initializing conditions database...") - return create_session(app.config['CONDITIONS_DATABASE_URI']) \ No newline at end of file + """Initialize the session for the Conditions database (alias for init_condition_db).""" + return init_condition_db(app) diff --git a/jobs/epic-cron/src/epic_cron/models/email_job.py b/jobs/epic-cron/src/epic_cron/models/email_job.py new file mode 100644 index 0000000..c2a6db6 --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/models/email_job.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass +from datetime import datetime +from typing import Optional + + +@dataclass +class EmailJob: + id: int + template_name: str + status: str + payload: dict + error_message: Optional[str] = None + sent_at: Optional[datetime] = None + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None diff --git a/jobs/epic-cron/src/epic_cron/processors/centre/__init__.py b/jobs/epic-cron/src/epic_cron/processors/centre/__init__.py new file mode 100644 index 0000000..05a9e86 --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/processors/centre/__init__.py @@ -0,0 +1,33 @@ +# processors/centre/__init__.py +from typing import Callable, Dict + +from submit_api.data_classes.email_details import EmailDetails + +from ...models.email_job import EmailJob +from .access_denied import process_access_denied +from .access_granted import process_access_granted +from .access_request_received_dst import process_access_request_received_dst +from .access_request_submitted import process_access_request_submitted + + +# Template names (export as constants, so they’re used consistently) +TEMPLATE_ACCESS_REQUEST_SUBMITTED = "access_request_submitted_confirmation.html" +ACCESS_REQUEST_RECEIVED_NOTIFICATION = 'access_request_received_notification.html' +ACCESS_GRANTED_NOTIFICATION = "access_granted_notification.html" +ACCESS_DENIED_NOTIFICATION = "access_denied_notification.html" + +# Map: template_name -> processor function +# Each processor takes (job: EmailJob) and returns EmailDetails +PROCESSORS: Dict[str, Callable[[EmailJob], EmailDetails]] = { + TEMPLATE_ACCESS_REQUEST_SUBMITTED: process_access_request_submitted, + ACCESS_REQUEST_RECEIVED_NOTIFICATION: process_access_request_received_dst, + ACCESS_GRANTED_NOTIFICATION: process_access_granted, + ACCESS_DENIED_NOTIFICATION: process_access_denied +} + +__all__ = [ + "TEMPLATE_ACCESS_REQUEST_SUBMITTED", + "PROCESSORS", + "process_access_request_submitted", + "process_access_request_received_dst", +] diff --git a/jobs/epic-cron/src/epic_cron/processors/centre/access_denied.py b/jobs/epic-cron/src/epic_cron/processors/centre/access_denied.py new file mode 100644 index 0000000..9655275 --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/processors/centre/access_denied.py @@ -0,0 +1,46 @@ +from typing import Any, Dict, List + +from submit_api.data_classes.email_details import EmailDetails +from submit_api.exceptions import BadRequestError + +from epic_cron.models.email_job import EmailJob + + +def _require(payload: Dict[str, Any], fields: List[str]) -> None: + missing = [f for f in fields if not payload.get(f)] + if missing: + raise BadRequestError(f"Missing required payload fields: {', '.join(missing)}") + + +def process_access_denied(job: EmailJob) -> EmailDetails: + """ + Processor for the 'access request submitted' template. + + Expected job.payload: + { + "recipients": ["user@example.com"], # required + "user_name": "Jane Doe", # required + "application_name": "EPIC.centre", # required + "sender": "staff@email.com", # required (email address) + } + """ + payload = job.payload or {} + _require(payload, ["recipients", "user_name", "application_name", "sender"]) + + recipients = payload["recipients"] + if not isinstance(recipients, list) or not recipients: + raise BadRequestError("payload.recipients must be a non-empty list of email addresses") + + subject = f"Your EPIC Access Request for {payload['application_name']} Has Been Denied" + + email_details = EmailDetails( + template_name=job.template_name, + body_args={ + 'user_name': payload['user_name'], + 'application_name': payload['application_name'] + }, + subject=subject, + sender=payload['sender'], + recipients=recipients, + ) + return email_details diff --git a/jobs/epic-cron/src/epic_cron/processors/centre/access_granted.py b/jobs/epic-cron/src/epic_cron/processors/centre/access_granted.py new file mode 100644 index 0000000..9c766b2 --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/processors/centre/access_granted.py @@ -0,0 +1,50 @@ +from typing import Any, Dict, List + +from submit_api.data_classes.email_details import EmailDetails +from submit_api.exceptions import BadRequestError + +from epic_cron.models.email_job import EmailJob + + +def _require(payload: Dict[str, Any], fields: List[str]) -> None: + missing = [f for f in fields if not payload.get(f)] + if missing: + raise BadRequestError(f"Missing required payload fields: {', '.join(missing)}") + + +def process_access_granted(job: EmailJob) -> EmailDetails: + """ + Processor for the 'access request submitted' template. + + Expected job.payload: + { + "recipients": ["user@example.com"], # required + "user_name": "Jane Doe", # required + "application_name": "EPIC.centre", # required + "sender": "staff@email.com", # required (email address), + "access_level": "VIEWER", + "auth_link: "https://centre.example.com/request-access" # required + } + """ + payload = job.payload or {} + _require(payload, ["recipients", "user_name", "application_name", "sender", "auth_link", "access_level"]) + + recipients = payload["recipients"] + if not isinstance(recipients, list) or not recipients: + raise BadRequestError("payload.recipients must be a non-empty list of email addresses") + + subject = f"Your EPIC Access Request for {payload['application_name']} Has Been Granted" + + email_details = EmailDetails( + template_name=job.template_name, + body_args={ + 'user_name': payload['user_name'], + 'application_name': payload['application_name'], + 'auth_link': payload['auth_link'], + 'access_level': payload['access_level'] + }, + subject=subject, + sender=payload['sender'], + recipients=recipients, + ) + return email_details diff --git a/jobs/epic-cron/src/epic_cron/processors/centre/access_request_received_dst.py b/jobs/epic-cron/src/epic_cron/processors/centre/access_request_received_dst.py new file mode 100644 index 0000000..26187d0 --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/processors/centre/access_request_received_dst.py @@ -0,0 +1,53 @@ +from typing import Any, Dict, List + +from submit_api.data_classes.email_details import EmailDetails +from submit_api.exceptions import BadRequestError + +from epic_cron.models.email_job import EmailJob + + +def _require(payload: Dict[str, Any], fields: List[str]) -> None: + missing = [f for f in fields if not payload.get(f)] + if missing: + raise BadRequestError(f"Missing required payload fields: {', '.join(missing)}") + + +def process_access_request_received_dst(job: EmailJob) -> EmailDetails: + """ + Processor for the 'access request submitted' template. + + Expected job.payload: + { + "recipients": ["user@example.com"], # required + "user_name": "Jane Doe", # required + "user_email": "jane@email.com", # required (email address of the user requesting access) + "application_name": "EPIC.centre", # required + "requested_at": "2025-09-04 10:15 PT",# required (string already formatted) + "sender": "staff@email.com", # required (email address) + "auth_link: "https://centre.example.com/request-access" # required + } + """ + payload = job.payload or {} + _require(payload, ["recipients", "user_name", "application_name", "requested_at", "sender", "auth_link", + "user_email"]) + + recipients = payload["recipients"] + if not isinstance(recipients, list) or not recipients: + raise BadRequestError("payload.recipients must be a non-empty list of email addresses") + + subject = f"EPIC Access Request: {payload['user_name']} for {payload['application_name']}" + + email_details = EmailDetails( + template_name=job.template_name, + body_args={ + 'user_name': payload['user_name'], + 'application_name': payload['application_name'], + 'requested_at': payload['requested_at'], + 'auth_link': payload['auth_link'], + 'user_email': payload['user_email'], + }, + subject=subject, + sender=payload['sender'], + recipients=recipients, + ) + return email_details diff --git a/jobs/epic-cron/src/epic_cron/processors/centre/access_request_submitted.py b/jobs/epic-cron/src/epic_cron/processors/centre/access_request_submitted.py new file mode 100644 index 0000000..80b259e --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/processors/centre/access_request_submitted.py @@ -0,0 +1,51 @@ +from typing import Any, Dict, List + +from submit_api.data_classes.email_details import EmailDetails +from submit_api.exceptions import BadRequestError + +from epic_cron.models.email_job import EmailJob + + +def _require(payload: Dict[str, Any], fields: List[str]) -> None: + missing = [f for f in fields if not payload.get(f)] + if missing: + raise BadRequestError(f"Missing required payload fields: {', '.join(missing)}") + + +def process_access_request_submitted(job: EmailJob) -> EmailDetails: + """ + Processor for the 'access request submitted' template. + + Expected job.payload: + { + "recipients": ["user@example.com"], # required + "user_name": "Jane Doe", # required + "application_name": "EPIC.centre", # required + "requested_at": "2025-09-04 10:15 PT",# required (string already formatted) + "sender": "staff@email.com", # required (email address) + } + """ + payload = job.payload or {} + _require(payload, ["recipients", "user_name", "application_name", "requested_at", "sender", + "application_url", "epic_centre_link"]) + + recipients = payload["recipients"] + if not isinstance(recipients, list) or not recipients: + raise BadRequestError("payload.recipients must be a non-empty list of email addresses") + + subject = f"Your EPIC Access Request for {payload['application_name']} Has Been Submitted" + + email_details = EmailDetails( + template_name=job.template_name, + body_args={ + 'user_name': payload['user_name'], + 'application_name': payload['application_name'], + 'application_url': payload['application_url'], + 'requested_at': payload['requested_at'], + 'epic_centre_link': payload['epic_centre_link'], + }, + subject=subject, + sender=payload['sender'], + recipients=recipients, + ) + return email_details diff --git a/jobs/epic-cron/src/epic_cron/repositories/email_repository.py b/jobs/epic-cron/src/epic_cron/repositories/email_repository.py new file mode 100644 index 0000000..a69e594 --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/repositories/email_repository.py @@ -0,0 +1,68 @@ +from typing import List + +from sqlalchemy import Column, DateTime, Integer, MetaData, String, Table, Text, func, select +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Session + +from epic_cron.models.email_job import EmailJob + + +metadata = MetaData() + +# Local definition of the table (decoupled from submit_api.models) +email_queue_table = Table( + "email_queue", + metadata, + Column("id", Integer, primary_key=True), + Column("template_name", String(255), nullable=False), + Column("status", String(32), nullable=False, server_default="PENDING"), + Column("payload", JSONB, nullable=False), # everything template-specific lives here + Column("error_message", Text, nullable=True), + Column("sent_at", DateTime(timezone=True), nullable=True), + Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()), + Column("updated_at", DateTime(timezone=True), nullable=False, server_default=func.now()), +) + + +class EmailRepository: + def __init__(self, session: Session): + self.session = session + + def find_pending(self, limit=100) -> List[EmailJob]: + stmt = ( + select(email_queue_table) + .where(email_queue_table.c.status == "PENDING") + .limit(limit) + ) + rows = self.session.execute(stmt).fetchall() + return [ + EmailJob( + id=row.id, + template_name=row.template_name, + status=row.status, + payload=row.payload, + error_message=row.error_message, + sent_at=row.sent_at, + created_at=row.created_at, + updated_at=row.updated_at, + ) + for row in rows + ] + + def mark_sent(self, email_id: int): + stmt = ( + email_queue_table.update() + .where(email_queue_table.c.id == email_id) + .values(status="SENT", error_message=None, payload=None, sent_at=func.now()) + ) + self.session.execute(stmt) + self.session.commit() + + def mark_failed(self, email_id: int, error_message: str): + stmt = ( + email_queue_table.update() + .where(email_queue_table.c.id == email_id) + .values(status="FAILED", error_message=error_message, sent_at=func.now()) + ) + self.session.execute(stmt) + self.session.commit() diff --git a/jobs/epic-cron/src/epic_cron/services/approved_condition_sync_service.py b/jobs/epic-cron/src/epic_cron/services/approved_condition_sync_service.py new file mode 100644 index 0000000..a34a9d6 --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/services/approved_condition_sync_service.py @@ -0,0 +1,109 @@ +import requests +from flask import current_app +from submit_api.models.project import Project + +from epic_cron.models import db + + +class ApprovedConditionService: + """Service to interact with the Condition API.""" + + @staticmethod + def sync_projects_with_approved_conditions(): + """ + Fetch project data from the Condition API + + """ + # Get the Condition API base URL and endpoint + condition_api_base_url = current_app.config.get("CONDITION_API_BASE_URL") + approved_projects_endpoint = f"{condition_api_base_url}/api/projects/with-approved-conditions" + + # Fetch the Bearer token + token = ApprovedConditionService._get_admin_token() + + if not token: + raise Exception("Failed to fetch authorization token.") + + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + + current_app.logger.info(f"Fetching projects from Condition API: {approved_projects_endpoint}") + try: + # Make the GET request to the Condition API with Authorization + response = requests.get(approved_projects_endpoint, headers=headers, timeout=30) + response.raise_for_status() # Raise HTTPError for bad responses (4xx and 5xx) + projects = response.json() + + current_app.logger.info(f"Condition API returned {len(projects)} projects.") + + epic_guids = [p.get("epic_guid") for p in projects if p.get("epic_guid")] + + updated_count = 0 + + for epic_guid in epic_guids: + # Fetch the Project by epic_guid + project = db.session.query(Project).filter_by(epic_guid=epic_guid).first() + if project: + if not project.has_approved_condition: + project.has_approved_condition = True + updated_count += 1 + + db.session.commit() + + current_app.logger.info(f"Updated {updated_count} projects with has_approved_condition=True.") + return {"updated_projects": updated_count} + + except requests.RequestException as e: + db.session.rollback() + current_app.logger.error(f"Error while calling Condition API: {e}") + raise + except Exception as e: + db.session.rollback() + current_app.logger.error(f"Unexpected error: {e}") + raise + finally: + db.session.remove() + + @staticmethod + def _get_admin_token(): + """ + Fetch an admin token using client credentials from Keycloak. + + Returns: + str: Access token string. + """ + # Get Keycloak configuration from Flask app config + config = current_app.config + base_url = config.get("KEYCLOAK_BASE_URL") + realm = config.get("KEYCLOAK_REALM_NAME") + admin_client_id = config.get("KEYCLOAK_SERVICE_ACCOUNT_ID") + admin_secret = config.get("KEYCLOAK_SERVICE_ACCOUNT_SECRET") + timeout = config.get("CONNECT_TIMEOUT", 60) + + # Construct token URL and headers + token_url = f"{base_url}/auth/realms/{realm}/protocol/openid-connect/token" + + headers = { + "Content-Type": "application/x-www-form-urlencoded" + } + + # Request body for client credentials grant + data = f"client_id={admin_client_id}&grant_type=client_credentials&client_secret={admin_secret}" + + try: + current_app.logger.info(f"Fetching Keycloak token from: {token_url}") + response = requests.post(token_url, data=data, headers=headers, timeout=timeout) + + response.raise_for_status() # Raise HTTPError for bad responses (4xx and 5xx) + + # Parse and return the access token + access_token = response.json().get("access_token") + if not access_token: + raise Exception("Keycloak response did not include an access token.") + return access_token + + except requests.RequestException as e: + current_app.logger.error(f"Error while fetching Keycloak token: {e}") + raise diff --git a/jobs/epic-cron/src/epic_cron/services/centre_email_service.py b/jobs/epic-cron/src/epic_cron/services/centre_email_service.py new file mode 100644 index 0000000..a9ef18f --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/services/centre_email_service.py @@ -0,0 +1,56 @@ +from typing import Callable, Dict + +from flask import current_app +from submit_api.data_classes.email_details import EmailDetails +from submit_api.exceptions import BadRequestError + +from epic_cron.models.email_job import EmailJob +from epic_cron.repositories.email_repository import EmailRepository +from epic_cron.services.ches_service import ChesApiService + + +class CentreEmailService: + """Email service for Centre system, decoupled from submit_api models.""" + + _processors: Dict[str, Callable[[EmailJob], EmailDetails]] = {} + + @classmethod + def register_processor(cls, template_name: str, processor: Callable[[EmailJob], EmailDetails]): + """Register a template-specific processor function.""" + cls._processors[template_name] = processor + + @classmethod + def process_email_queue(cls, repository: EmailRepository, limit: int = 100): + """Fetch and process pending emails using a repository.""" + pending = repository.find_pending(limit=limit) + if not pending: + current_app.logger.info("No pending emails found.") + return + + current_app.logger.info(f"Processing {len(pending)} pending emails") + + for job in pending: + try: + processor = cls._get_processor(job) + email_details = processor(job) + cls.send_email(email_details) + repository.mark_sent(job.id) + except Exception as e: + current_app.logger.error(f"Error processing email {job.id}: {e}", exc_info=True) + repository.mark_failed(job.id, str(e)) + + @classmethod + def _get_processor(cls, job: EmailJob) -> Callable[[EmailJob], EmailDetails]: + if job.template_name not in cls._processors: + raise BadRequestError(f"Unsupported email template: {job.template_name}") + return cls._processors[job.template_name] + + @staticmethod + def send_email(email_details: EmailDetails): + """Send email via CHES.""" + try: + ches = ChesApiService() + return ches.send_email(email_details, template_sub_directory='centre') + except Exception as e: + current_app.logger.error(f"Failed to send email: {e}", exc_info=True) + raise BadRequestError(f"Failed to send email") diff --git a/jobs/epic-cron/src/epic_cron/services/ches_service.py b/jobs/epic-cron/src/epic_cron/services/ches_service.py new file mode 100644 index 0000000..bcf2b8d --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/services/ches_service.py @@ -0,0 +1,158 @@ +"""Service for integrating with the Common Hosted Email Service.""" +import base64 +import json +from datetime import datetime, timedelta + +import requests +from flask import current_app +from submit_api.data_classes.email_details import EmailDetails + +from utils.template import Template + + +class ChesApiService: + """CHES api Service class.""" + + def __init__(self): + """Initiate class.""" + self.token_endpoint = current_app.config.get('CHES_TOKEN_ENDPOINT') + self.service_client_id = current_app.config.get('CHES_CLIENT_ID') + self.service_client_secret = current_app.config.get('CHES_CLIENT_SECRET') + self.ches_base_url = current_app.config.get('CHES_BASE_URL') + current_app.logger.info(f'Initialized ChesApiService with CHES_BASE_URL: {self.ches_base_url}') + self.access_token, self.token_expiry = self._get_access_token() + + def _get_access_token(self): + """Retrieve access token from CHES.""" + basic_auth_encoded = base64.b64encode( + bytes(f'{self.service_client_id}:{self.service_client_secret}', 'utf-8') + ).decode('utf-8') + data = 'grant_type=client_credentials' + current_app.logger.info(f'Fetching access token from: {self.token_endpoint}') + + try: + response = requests.post( + self.token_endpoint, + data=data, + headers={ + 'Authorization': f'Basic {basic_auth_encoded}', + 'Content-Type': 'application/x-www-form-urlencoded' + }, + timeout=10 + ) + response.raise_for_status() + + response_json = response.json() + + expires_in = response_json['expires_in'] + expiry_time = datetime.now() + timedelta(seconds=expires_in) + + return response_json['access_token'], expiry_time + except requests.exceptions.RequestException as e: + current_app.logger.error(f'Error occurred while fetching access token: {str(e)}') + if e.response is not None: + current_app.logger.error(f'Status Code: {e.response.status_code}') + current_app.logger.error(f'Response Content: {e.response.text}') + else: + current_app.logger.error("No response received from server.") + raise # Re-raise the exception to propagate the error + + def _ensure_valid_token(self): + if datetime.now() >= self.token_expiry: + self.access_token, self.token_expiry = self._get_access_token() + + @staticmethod + def _get_email_body_from_template(template_name: str, body_args: dict, template_sub_directory: str = None): + """Get email body from a template with optional environment message for centre templates.""" + if not template_name: + raise ValueError('Template name is required') + + template = Template.get_template(template_name, template_sub_directory) + if not template: + raise ValueError('Template not found') + # logo is taken from submit UI / Web app.. + # Went for this approach since making it base64 is hard to get it working in gmail.. + # gmail strips the logo if base64 is used + # this is like submit web hosts the logo and the email uses it as a static server to get the logo image. + body_args['logo_url'] = f'{current_app.config.get("WEB_URL")}/assets/EAO_Logo-BZOR9oRj.png' + rendered_body = template.render(body_args) + + # Add environment notification for centre templates in non-production + if template_sub_directory == 'centre': + env_name = current_app.config.get('ENVIRONMENT', '') + if env_name and env_name.lower() != 'production': + env_message = ChesApiService._create_environment_banner(env_name) + rendered_body = ChesApiService._inject_environment_banner(rendered_body, env_message) + + return rendered_body + + @staticmethod + def _create_environment_banner(env_name: str) -> str: + """Create HTML banner showing the current environment.""" + return f''' +
+ You are using {env_name} environment +
+ ''' + + @staticmethod + def _inject_environment_banner(rendered_body: str, env_message: str) -> str: + """Inject environment banner into the email body.""" + if '' in rendered_body: + return rendered_body.replace('', f'{env_message}') + else: + return rendered_body + env_message + + def _get_email_body(self, email_details: EmailDetails, template_sub_directory: str = None): + """Get email body based on details or template.""" + if email_details.body: + body = email_details.body + body_type = 'text' + else: + body = self._get_email_body_from_template(email_details.template_name, + email_details.body_args, template_sub_directory) + body_type = 'html' + + return body, body_type + + def send_email(self, email_details: EmailDetails, template_sub_directory: str = None): + """Generate document based on template and data.""" + self._ensure_valid_token() + + body, body_type = self._get_email_body(email_details, template_sub_directory) + + request_body = { + 'bodyType': body_type, + 'body': body, + 'subject': email_details.subject, + 'from': email_details.sender, + 'to': email_details.recipients, + 'cc': email_details.cc, + 'bcc': email_details.bcc, + } + json_request_body = json.dumps(request_body) + + + headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {self.access_token}' + } + + url = f'{self.ches_base_url}/api/v1/email' + + try: + response = requests.post(url, data=json_request_body, headers=headers, timeout=10) + current_app.logger.info(f'Response status from CHES email endpoint: {response.status_code}') + response.raise_for_status() + + response_json = response.json() + return response_json, response.status_code + except requests.exceptions.RequestException as e: + current_app.logger.error(f'Error occurred while sending email: {str(e)}') + if e.response is not None: + current_app.logger.error(f'Status Code: {e.response.status_code}') + current_app.logger.error(f'Response Content: {e.response.text}') + else: + current_app.logger.error("No response received from server.") + raise # Re-raise the exception to propagate the error diff --git a/jobs/epic-cron/src/epic_cron/services/invitation_email_service.py b/jobs/epic-cron/src/epic_cron/services/invitation_email_service.py new file mode 100644 index 0000000..1a56278 --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/services/invitation_email_service.py @@ -0,0 +1,109 @@ +from urllib.parse import urljoin + +from flask import current_app +from submit_api.data_classes.email_details import EmailDetails +from submit_api.enums.role import RoleEnum +from submit_api.exceptions import BadRequestError +from submit_api.models.account_project import AccountProject as AccountProjectModel +from submit_api.models.invitations import Invitations as InvitationsModel +from submit_api.models.package import Package as PackageModel +from submit_api.models.project import Project as ProjectModel +from submit_api.utils.constants import NEW_USER_INVITATION_EMAIL_TEMPLATE + +from epic_cron.models import db + + +class InvitationEmailService: # pylint: disable=too-few-public-methods + """Handles sending email notifications for new user invitation.""" + + @classmethod + def prepare_invitation_email_notification(cls, invitation: InvitationsModel) -> EmailDetails: + """Prepare email details for update request creation.""" + + # Default action text + invitation_action_text = "join" + # Check role and modify invitation action text accordingly + if invitation.role and invitation.role.role_name == RoleEnum.SPECIFIC_SUBMISSION_CONTRIBUTOR.value: + invitation_action_text = "collaborate on" + + bc_service_card_url = current_app.config.get('BC_SERVICE_CARD_URL', 'https://id.gov.bc.ca') + + if invitation.project_ids: + project = cls.get_project_from_project_ids(invitation.project_ids) + elif invitation.package_ids: + project = cls.get_project_from_package_id(invitation.package_ids) + else: + raise BadRequestError("No project or package IDs provided in the invitation.") + + if not project: + raise BadRequestError(f"Project was not found for invitation id: {invitation.id}") + + invitation_url = cls.generate_signup_url(invitation.token) + + email_details = EmailDetails( + template_name=NEW_USER_INVITATION_EMAIL_TEMPLATE, + body_args={ + 'epic_submit_link': current_app.config.get('WEB_URL'), + 'invitation_url': invitation_url, + 'project_name': project.name or '', + 'bc_service_card_url': bc_service_card_url, + 'certificate_holder_name': project.proponent_name or '', + 'invitation_action_text': invitation_action_text, + }, + subject='Invitation to collaborate on EPIC.submit', + sender=current_app.config.get('SENDER_EMAIL'), + recipients=[invitation.email], + ) + + return email_details + + @staticmethod + def get_project_from_project_ids(project_ids: str) -> ProjectModel: + """Return the first matching project from a list of project IDs.""" + project_id_list = [int(pid) for pid in project_ids if isinstance(pid, (int, str)) and str(pid).isdigit()] + + # assuming one project ID is provided in one invitation + return ( + db.session.query(ProjectModel) + .filter(ProjectModel.id.in_(project_id_list)) + .first() + ) + + @staticmethod + def get_project_from_package_id(package_ids: list) -> ProjectModel: + """Return the project linked to the first package ID.""" + if not isinstance(package_ids, list) or not package_ids: + return None + + package_id = package_ids[0] + + # assuming only package ids of one project are provided in one invitation + return ( + db.session.query(ProjectModel) + .join(AccountProjectModel, ProjectModel.id == AccountProjectModel.project_id) + .join(PackageModel, AccountProjectModel.id == PackageModel.account_project_id) + .filter(PackageModel.id == package_id) + .first() + ) + + @staticmethod + def get_project_for_account_id(account_id: int) -> ProjectModel: + """Return the first project for a given account ID.""" + if not account_id: + return None + + return ( + db.session.query(ProjectModel) + .join(AccountProjectModel, ProjectModel.id == AccountProjectModel.project_id) + .filter(AccountProjectModel.account_id == account_id) + .first() + ) + + @staticmethod + def generate_signup_url(token): + """Generate a full URL with token for invitation.""" + base_url = current_app.config['WEB_URL'] + signup_path = current_app.config.get('SIGNUP_URL_PATH', '/proponent/registration') + + # Construct the URL by joining base, path, and token + return urljoin(base_url, f"{signup_path}?token={token}") diff --git a/jobs/epic-cron/src/epic_cron/services/mail_service.py b/jobs/epic-cron/src/epic_cron/services/mail_service.py new file mode 100644 index 0000000..64e1a30 --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/services/mail_service.py @@ -0,0 +1,156 @@ +from datetime import datetime +from functools import partial +from typing import List + +from flask import current_app +from submit_api.data_classes.email_details import EmailDetails +from submit_api.exceptions import BadRequestError +from submit_api.models.email_queue import EmailQueue, EmailStatus +from submit_api.models.invitations import Invitations as InvitationsModel +from submit_api.models.package import Package as PackageModel +from submit_api.utils.constants import ( + MANAGEMENT_PLAN_RESUBMISSION_REQUEST_EMAIL_TEMPLATE, MANAGEMENT_PLAN_SUBMISSION_CONFIRMATION_EMAIL_TEMPLATE, + MANAGEMENT_PLAN_SUBMISSION_NOTIFY_STAFF_EMAIL_TEMPLATE, MANAGEMENT_PLAN_UPDATE_REQUEST_CREATED_EMAIL_TEMPLATE, + NEW_USER_INVITATION_EMAIL_TEMPLATE) + +from epic_cron.models import db +from epic_cron.services.ches_service import ChesApiService +from epic_cron.services.invitation_email_service import InvitationEmailService +from epic_cron.services.package_submission_email_service import PackageSubmissionEmailService +from epic_cron.services.request_update_email_service import RequestUpdateEmailService +from epic_cron.services.resubmission_email_service import ResubmissionEmailService + + +class EmailService: # pylint: disable=too-few-public-methods + """Handles the general email sending operations.""" + + @staticmethod + def process_email_queue(): + """Process all pending emails in the email queue.""" + pending_emails = EmailService.find_pending() + if not pending_emails: + current_app.logger.info("No pending emails found.") + return + current_app.logger.info(f"Number of pending emails: {len(pending_emails)}") + for email_entry in pending_emails: + try: + email_processor = EmailService._get_email_processor(email_entry) + email_processor(email_entry) + except Exception as e: + # Log the error and update the status to FAILED + email_entry.status = EmailStatus.FAILED.value + email_entry.error_message = str(e) + db.session.commit() + + @classmethod + def _get_email_processor(cls, email_entry: EmailQueue) -> callable: + """Get the email processor based on the template name.""" + email_processors = { + MANAGEMENT_PLAN_SUBMISSION_CONFIRMATION_EMAIL_TEMPLATE: cls._process_package_submission_email, + MANAGEMENT_PLAN_UPDATE_REQUEST_CREATED_EMAIL_TEMPLATE: cls._process_request_update_creation_email, + MANAGEMENT_PLAN_RESUBMISSION_REQUEST_EMAIL_TEMPLATE: cls._process_resubmission_request_email, + # staff email uses the same content, but just a different template..so reusing the same method passing template name + MANAGEMENT_PLAN_SUBMISSION_NOTIFY_STAFF_EMAIL_TEMPLATE: partial(cls._process_package_submission_email, template_name=MANAGEMENT_PLAN_SUBMISSION_NOTIFY_STAFF_EMAIL_TEMPLATE), + NEW_USER_INVITATION_EMAIL_TEMPLATE: cls._process_new_user_invitation_email + } + template = email_entry.template_name + if template not in email_processors: + raise BadRequestError(f"Unsupported email template: {template}") + return email_processors.get(template) + + @staticmethod + def _process_package_submission_email(email_entry: EmailQueue, template_name=None ): + """Process email entry for package submission.""" + package_id = email_entry.entity_id + package: PackageModel = db.session.get(PackageModel, package_id) + if not package: + raise BadRequestError(f"Package with ID {package_id} not found.") + + email_details = PackageSubmissionEmailService.prepare_package_submission_email_confirmation(package, template_name) + + # Send the email using ChesApiService + EmailService.send_email(email_details) + + # Update the email queue status to SENT + email_entry.status = EmailStatus.SENT.value + email_entry.sent_at = datetime.utcnow() + db.session.commit() + + @staticmethod + def _process_request_update_creation_email(email_entry: EmailQueue): + """Process email entry for request update creation.""" + package_id = email_entry.entity_id + package: PackageModel = db.session.get(PackageModel, package_id) + if not package: + raise BadRequestError(f"Package with ID {package_id} not found.") + + email_details = RequestUpdateEmailService.prepare_update_request_creation_email_notification(package) + + # Send the email using ChesApiService + EmailService.send_email(email_details) + + # Update the email queue status to SENT + email_entry.status = EmailStatus.SENT.value + email_entry.sent_at = datetime.utcnow() + db.session.commit() + + @staticmethod + def _process_resubmission_request_email(email_entry: EmailQueue): + """Process email entry for resubmission invitation.""" + package_id = email_entry.entity_id + package: PackageModel = db.session.get(PackageModel, package_id) + if not package: + raise BadRequestError(f"Package with ID {package_id} not found.") + + # Get all PROJECT_ADMIN users for this account project + project_admin_users = ResubmissionEmailService.get_project_admin_users(package) + + # Send email to all project admins + email_details = ResubmissionEmailService.prepare_resubmission_request_email( + package, project_admin_users + ) + EmailService.send_email(email_details) + + # Update the original email queue status to SENT + email_entry.status = EmailStatus.SENT.value + email_entry.sent_at = datetime.utcnow() + db.session.commit() + + @staticmethod + def _process_new_user_invitation_email(email_entry: EmailQueue): + """Process email entry for new user invitation.""" + invitation_id = email_entry.entity_id + invitation: InvitationsModel = db.session.get(InvitationsModel, invitation_id) + if not invitation: + raise BadRequestError(f"Invitation with ID {invitation_id} not found.") + + email_details = InvitationEmailService.prepare_invitation_email_notification(invitation) + + # Send the email using ChesApiService + EmailService.send_email(email_details) + + # Update the email queue status to SENT + email_entry.status = EmailStatus.SENT.value + email_entry.sent_at = datetime.utcnow() + db.session.commit() + + @staticmethod + def send_email(email_details: EmailDetails): + """Send email using the ChesApiService.""" + try: + email_api_service = ChesApiService() + return email_api_service.send_email(email_details) + except Exception as e: + raise BadRequestError(f"Failed to send email: {str(e)}") + + @staticmethod + def find_pending(limit=100) -> List[EmailQueue]: + """Find all pending emails in the queue, with a limit for performance. + + Args: + limit (int): Maximum number of pending emails to return. + + Returns: + list[EmailQueue]: List of pending email queue entries. + """ + return db.session.query(EmailQueue).filter(EmailQueue.status == EmailStatus.PENDING.value).limit(limit).all() diff --git a/jobs/epic-cron/src/epic_cron/services/package_submission_email_service.py b/jobs/epic-cron/src/epic_cron/services/package_submission_email_service.py new file mode 100644 index 0000000..25be8f7 --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/services/package_submission_email_service.py @@ -0,0 +1,104 @@ + +from flask import current_app +from submit_api.data_classes.email_details import EmailDetails +from submit_api.exceptions import BadRequestError +from submit_api.models import AccountProject +from submit_api.models.account_user import AccountUser as AccountUserModel +from submit_api.models.package import Package as PackageModel +from submit_api.models.project import Project as ProjectModel +from submit_api.models.submission import SubmissionType +from submit_api.models.user import User as UserModel +from submit_api.utils.constants import ( + MANAGEMENT_PLAN_SUBMISSION_CONFIRMATION_EMAIL_TEMPLATE, MANAGEMENT_PLAN_SUBMISSION_NOTIFY_STAFF_EMAIL_TEMPLATE) + +from epic_cron.models import db +from epic_cron.utils import constants +from epic_cron.utils.datetime import convert_utc_to_local_str + + +class PackageSubmissionEmailService: # pylint: disable=too-few-public-methods + """Handles sending email notifications for package submissions.""" + + @classmethod + def prepare_package_submission_email_confirmation(cls, package: PackageModel, template_name) -> EmailDetails: + """Prepare email details for package submission confirmation.""" + submitter = cls._get_submitter(package.submitted_by) + if not submitter: + raise BadRequestError(f"Submitter with auth_guid {package.submitted_by} not found") + + sender_email = cls.get_email_sender_for_package_type(package.type.name) + + if not sender_email: + raise BadRequestError(f"Sender email not found for package type: {package.type.name}") + + account_project = cls._get_account_project_by_id(package.account_project_id) + project = cls._get_project_by_id(account_project.project_id) + if not project: + raise BadRequestError(f"Project not found for account project ID: {account_project.id}") + + document_submissions = cls._get_document_submissions_from_package(package) + email_template_name = template_name or MANAGEMENT_PLAN_SUBMISSION_CONFIRMATION_EMAIL_TEMPLATE + + if email_template_name == MANAGEMENT_PLAN_SUBMISSION_NOTIFY_STAFF_EMAIL_TEMPLATE: + staff_email = current_app.config.get('STAFF_SUPPORT_MAIL_ID') + if not staff_email: + raise BadRequestError("STAFF_SUPPORT_MAIL_ID is not configured") + + recipients = [staff_email] + subject = f"SUBMISSION - {project.name} - {package.name} - {package.submitted_on.strftime('%Y-%m-%d')}" + else: + recipients = [submitter.work_email_address] + subject = f"Confirmation of receipt for {package.name}" + + email_details = EmailDetails( + template_name=email_template_name, + body_args={ + 'project_name': project.name, + 'submitter_name': submitter.full_name, + 'submission_date': convert_utc_to_local_str(package.submitted_on), + 'certificate_holder_name': project.proponent_name, + 'package_name': package.name, + 'documents': [submission.submitted_document.name for submission in document_submissions] + }, + subject=subject, + sender=sender_email, + recipients=recipients, + ) + current_app.logger.info( + f"Sending email from {email_details.sender} to {', '.join(email_details.recipients)} for package: {email_details.body_args['package_name']}") + + return email_details + + @staticmethod + def get_email_sender_for_package_type(package_type: str) -> str: + """Get the email sender for the package type.""" + return constants.SUBMISSION_PACKAGE_TYPE_EMAIL_SENDER_MAP.get(package_type, None) + + @staticmethod + def _get_document_submissions_from_package(package: PackageModel): + """Retrieve document submissions from the package.""" + submissions = [ + submission for item in package.items for submission in item.submissions + if submission.type == SubmissionType.DOCUMENT + ] + return submissions + + @staticmethod + def _get_submitter(auth_guid: str) -> AccountUserModel: + """Retrieve the account user by their auth_guid.""" + return ( + db.session.query(AccountUserModel) + .join(UserModel) + .filter(UserModel.auth_guid == auth_guid) + .first() + ) + + @staticmethod + def _get_project_by_id(project_id: int) -> ProjectModel: + """Retrieve the project by its ID.""" + return db.session.query(ProjectModel).filter(ProjectModel.id == project_id).first() + + @staticmethod + def _get_account_project_by_id(id: int) -> AccountProject: + """Retrieve the account project by its ID.""" + return db.session.query(AccountProject).filter(AccountProject.id == id).first() diff --git a/jobs/epic-cron/src/epic_cron/services/request_update_email_service.py b/jobs/epic-cron/src/epic_cron/services/request_update_email_service.py new file mode 100644 index 0000000..a227caf --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/services/request_update_email_service.py @@ -0,0 +1,52 @@ +from flask import current_app +from submit_api.data_classes.email_details import EmailDetails +from submit_api.exceptions import BadRequestError +from submit_api.models.package import Package as PackageModel +from submit_api.utils.constants import MANAGEMENT_PLAN_UPDATE_REQUEST_CREATED_EMAIL_TEMPLATE + +from epic_cron.utils import constants + + +class RequestUpdateEmailService: # pylint: disable=too-few-public-methods + """Handles sending email notifications for package submissions.""" + + @classmethod + def prepare_update_request_creation_email_notification(cls, package: PackageModel) -> EmailDetails: + """Prepare email details for update request creation.""" + if not package.submitted_by_user or not package.submitted_by_user.account_user: + raise BadRequestError(f"Submitter with auth_guid {package.submitted_by} not found") + submitter = package.submitted_by_user.account_user + + sender_email = cls.get_email_sender_for_package_type(package.type.name) + if not sender_email: + raise BadRequestError(f"Sender email not found for package type: {package.type.name}") + + sender_name = cls.get_sender_name_for_package_type(package.type.name) + if not sender_name: + raise BadRequestError(f"Sender name not found for package type: {package.type.name}") + + web_url = current_app.config.get('WEB_URL') + email_details = EmailDetails( + template_name=MANAGEMENT_PLAN_UPDATE_REQUEST_CREATED_EMAIL_TEMPLATE, + body_args={ + 'epic_submit_link': web_url, + 'submitter_name': submitter.full_name, + 'package_name': package.name, + 'sender_name': sender_name, + }, + subject='Action Required: Update Your Submission', + sender=sender_email, + recipients=[submitter.work_email_address], + ) + + return email_details + + @staticmethod + def get_email_sender_for_package_type(package_type: str) -> str: + """Get the email sender for the package type.""" + return constants.SUBMISSION_PACKAGE_TYPE_EMAIL_SENDER_MAP.get(package_type, None) + + @staticmethod + def get_sender_name_for_package_type(package_type: str) -> str: + """Get the sender name for the package type.""" + return constants.SUBMISSION_PACKAGE_TYPE_SENDER_MAP.get(package_type, None) diff --git a/jobs/epic-cron/src/epic_cron/services/resubmission_email_service.py b/jobs/epic-cron/src/epic_cron/services/resubmission_email_service.py new file mode 100644 index 0000000..587b317 --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/services/resubmission_email_service.py @@ -0,0 +1,81 @@ +from flask import current_app +from submit_api.data_classes.email_details import EmailDetails +from submit_api.enums.role import RoleEnum +from submit_api.exceptions import BadRequestError +from submit_api.models.account_user import AccountUser as AccountUserModel +from submit_api.models.package import Package as PackageModel +from submit_api.models.role import Role as RoleModel +from submit_api.models.user_role import UserRole as UserRoleModel +from submit_api.utils.constants import MANAGEMENT_PLAN_RESUBMISSION_REQUEST_EMAIL_TEMPLATE + +from epic_cron.models import db + + +class ResubmissionEmailService: + """Handles sending email notifications for resubmission requests.""" + + @classmethod + def get_project_admin_users(cls, package: PackageModel) -> list[AccountUserModel]: + """Get all PROJECT_ADMIN users for the package's account project.""" + # Get the account_project_id from the package + account_project_id = package.account_project_id + current_app.logger.info(f"Looking for project admin users for account_project_id: {account_project_id}") + + # Get the PROJECT_ADMIN role using direct query + project_admin_role = ( + db.session.query(RoleModel) + .filter(RoleModel.role_name == RoleEnum.PROJECT_ADMIN.value) + .first() + ) + if not project_admin_role: + current_app.logger.error(f"Project admin role not found for role_name: {RoleEnum.PROJECT_ADMIN.value}") + raise BadRequestError("Project admin role not found") + + current_app.logger.info(f"Found project admin role with ID: {project_admin_role.id}") + + # Query for all project admin users for this specific account project + project_admin_users = ( + db.session.query(AccountUserModel) + .join(UserRoleModel, AccountUserModel.id == UserRoleModel.account_user_id) + .filter( + UserRoleModel.account_project_id == account_project_id, + UserRoleModel.role_id == project_admin_role.id, + UserRoleModel.active + ) + .all() + ) + + current_app.logger.info(f"Found {len(project_admin_users)} project admin users for account_project_id: {account_project_id}") + + if not project_admin_users: + current_app.logger.warning(f"No project admins found for account_project_id: {account_project_id}") + raise BadRequestError("No project admins found for this account project") + + # Log the email addresses of found users for debugging + email_addresses = [user.work_email_address for user in project_admin_users] + current_app.logger.info(f"Project admin email addresses: {email_addresses}") + + return project_admin_users + + @classmethod + def prepare_resubmission_request_email(cls, package: PackageModel, project_admin_users: list[AccountUserModel]) -> EmailDetails: + """Prepare email details for resubmission request for all project admin users.""" + if not package.submitted_by_user or not package.submitted_by_user.account_user: + raise BadRequestError(f"Submitter with auth_guid {package.submitted_by} not found") + web_url = current_app.config.get('WEB_URL') + submission_link = f"{web_url}/proponent/projects/{package.account_project_id}/submission-packages/{package.id}" + # Get all email addresses from project admin users + recipient_emails = [user.work_email_address for user in project_admin_users] + + email_details = EmailDetails( + template_name=MANAGEMENT_PLAN_RESUBMISSION_REQUEST_EMAIL_TEMPLATE, + body_args={ + 'submission_link': submission_link, + 'package_name': package.name, + }, + subject=f'Invitation to resubmit a new version of {package.name} in EPIC.submit', + sender=current_app.config.get('SENDER_EMAIL'), + recipients=recipient_emails, + ) + + return email_details diff --git a/jobs/epic-cron/src/epic_cron/services/track_service.py b/jobs/epic-cron/src/epic_cron/services/track_service.py index f604ca7..a923b82 100644 --- a/jobs/epic-cron/src/epic_cron/services/track_service.py +++ b/jobs/epic-cron/src/epic_cron/services/track_service.py @@ -11,7 +11,7 @@ class TrackService: @staticmethod def fetch_track_data(): """Fetch and log data from the track.projects table, joining with proponents.""" - print("Fetching data from track database...") + current_app.logger.info("Fetching data from track database...") required_fields = [ "id", "name", "epic_guid", "proponent_name", @@ -26,7 +26,7 @@ def fetch_track_data(): track_proponents_table = Table('proponents', track_metadata, autoload_with=session.bind) track_types_table = Table('types', track_metadata, autoload_with=session.bind) - print(f"Selecting required fields: {required_fields} and joining with proponents...") + current_app.logger.info(f"Selecting required fields: {required_fields} and joining with proponents...") # Join projects with proponents to get proponent name query = ( select( @@ -38,12 +38,10 @@ def fetch_track_data(): .outerjoin(track_types_table, track_projects_table.c.type_id == track_types_table.c.id) ) track_data = session.execute(query).fetchall() - print(f"Number of rows fetched from track.projects: {len(track_data)}") + current_app.logger.info(f"Number of rows fetched from track.projects: {len(track_data)}") - debug_logs_enabled = current_app.config.get("ENABLE_DETAILED_LOGS", False) - if debug_logs_enabled: - for row in track_data: - print(f"Fetched row: {dict(row._mapping)}") + for row in track_data: + current_app.logger.debug(f"Fetched row: {dict(row._mapping)}") return track_data @@ -73,7 +71,7 @@ def fetch_projects(): "Content-Type": "application/json" } - print(f"Fetching projects from Track API: {track_projects_endpoint}") + current_app.logger.info(f"Fetching projects from Track API: {track_projects_endpoint}") try: # Make the GET request to the Track API with Authorization response = requests.get(track_projects_endpoint, headers=headers, timeout=30) @@ -81,7 +79,7 @@ def fetch_projects(): # Parse the JSON response projects = response.json() - print(f"Track API returned {len(projects)} projects.") + current_app.logger.info(f"Track API returned {len(projects)} projects.") # Map the required fields mapped_projects = [] @@ -96,11 +94,11 @@ def fetch_projects(): } mapped_projects.append(mapped_project) - print(f"Mapped {len(mapped_projects)} projects with required fields.") + current_app.logger.info(f"Mapped {len(mapped_projects)} projects with required fields.") return mapped_projects except requests.RequestException as e: - print(f"Error while calling Track API: {e}") + current_app.logger.error(f"Error while calling Track API: {e}") raise @staticmethod @@ -129,7 +127,7 @@ def _get_admin_token(): data = f"client_id={admin_client_id}&grant_type=client_credentials&client_secret={admin_secret}" try: - print(f"Fetching Keycloak token from: {token_url}") + current_app.logger.info(f"Fetching Keycloak token from: {token_url}") response = requests.post(token_url, data=data, headers=headers, timeout=timeout) response.raise_for_status() # Raise HTTPError for bad responses (4xx and 5xx) @@ -140,5 +138,5 @@ def _get_admin_token(): return access_token except requests.RequestException as e: - print(f"Error while fetching Keycloak token: {e}") + current_app.logger.error(f"Error while fetching Keycloak token: {e}") raise diff --git a/jobs/epic-cron/src/epic_cron/utils/__init__.py b/jobs/epic-cron/src/epic_cron/utils/__init__.py index e69de29..e5bd1b9 100644 --- a/jobs/epic-cron/src/epic_cron/utils/__init__.py +++ b/jobs/epic-cron/src/epic_cron/utils/__init__.py @@ -0,0 +1,25 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Enum definitions.""" +from enum import Enum + + +class FormIoComponentType(Enum): + """FormIO Component Types.""" + + RADIO = 'radio' + CHECKBOX = 'checkbox' + TEXT = 'text' + + diff --git a/jobs/epic-cron/src/epic_cron/utils/constants.py b/jobs/epic-cron/src/epic_cron/utils/constants.py new file mode 100644 index 0000000..bf02a4c --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/utils/constants.py @@ -0,0 +1,27 @@ +# Copyright © 2024 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""This module contains constants used in the application.""" + +SUBMISSION_PACKAGE_TYPE_EMAIL_SENDER_MAP = { + 'Management Plan': 'EAO.ManagementPlanSupport@gov.bc.ca', + 'IEM': 'EAO.ManagementPlanSupport@gov.bc.ca' +} + +SUBMISSION_PACKAGE_TYPE_SENDER_MAP = { + 'Management Plan': 'The Management Plan Team at the Environmental Assessment Office', + 'IEM': 'EAO.ManagementPlanSupport@gov.bc.ca' +} + +PACKAGE_ENTITY_TYPE = 'PACKAGE' +INVITATION_ENTITY_TYPE = 'INVITATION' diff --git a/jobs/epic-cron/src/epic_cron/utils/datetime.py b/jobs/epic-cron/src/epic_cron/utils/datetime.py new file mode 100644 index 0000000..75db002 --- /dev/null +++ b/jobs/epic-cron/src/epic_cron/utils/datetime.py @@ -0,0 +1,26 @@ +"""Datetime object helper.""" + +from datetime import datetime + +import pytz +from flask import current_app + + +# Constants +PACIFIC_TZ = pytz.timezone('US/Pacific') +UTC_TZ = pytz.utc + + +def convert_utc_to_local_str(utc_dt: datetime, dt_format='%Y-%m-%d %I:%M %p %Z', timezone_override=None): + """ + Convert a UTC datetime to local timezone and format it. + """ + utc_dt = pytz.utc.localize(utc_dt) + + tz_name = timezone_override or current_app.config.get('LEGISLATIVE_TIMEZONE', 'US/Pacific') + local_tz = pytz.timezone(tz_name) + local_dt = utc_dt.astimezone(local_tz) + + # Step 3: Format + return local_dt.strftime(dt_format) + diff --git a/jobs/epic-cron/src/submit-api b/jobs/epic-cron/src/submit-api new file mode 160000 index 0000000..b0f1bed --- /dev/null +++ b/jobs/epic-cron/src/submit-api @@ -0,0 +1 @@ +Subproject commit b0f1bed071fd7a77496542c01f01cb4537672da7 diff --git a/jobs/epic-cron/tasks/centre_mail.py b/jobs/epic-cron/tasks/centre_mail.py new file mode 100644 index 0000000..59c30d0 --- /dev/null +++ b/jobs/epic-cron/tasks/centre_mail.py @@ -0,0 +1,45 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MET Publish Scheduled Engagements.""" +from datetime import datetime + +from flask import current_app + +from epic_cron.models.db import init_centre_db +from epic_cron.models.db import ma +from epic_cron.processors.centre import PROCESSORS # noqa: F401 pylint:disable=unused-import +from epic_cron.repositories.email_repository import EmailRepository +from epic_cron.services.centre_email_service import CentreEmailService + + +class CentreMailer: # pylint:disable=too-few-public-methods + """Task to publish scheduled Engagements due.""" + + @classmethod + def send_mail(cls): + """Send queued Centre emails using registered processors.""" + print('Starting Centre Email At---', datetime.now()) + _session = init_centre_db(current_app) + session = _session() + ma.init_app(current_app) + + try: + for template_name, processor in PROCESSORS.items(): + current_app.logger.debug(f'Registering processor for template: {template_name}') + CentreEmailService.register_processor(template_name, processor) + + repo = EmailRepository(session) + CentreEmailService.process_email_queue(repo, limit=100) + finally: + session.close() diff --git a/jobs/epic-cron/tasks/project_extractor.py b/jobs/epic-cron/tasks/project_extractor.py index da14635..d0209c2 100644 --- a/jobs/epic-cron/tasks/project_extractor.py +++ b/jobs/epic-cron/tasks/project_extractor.py @@ -23,22 +23,31 @@ class ProjectExtractor: @classmethod def do_sync(cls, target_system=TargetSystem.SUBMIT): """Perform the syncing.""" - print(f"Starting Project Extractor for {target_system.value} at {datetime.now()}") + current_app.logger.info(f"Starting Project Extractor for {target_system.value} at {datetime.now()}") # Initialize source and target database sessions - print("Initializing database sessions...") + current_app.logger.info("Initializing database sessions...") target_session, target_model = cls._get_target_config(target_system) - # Step 1: Fetch data from track.projects - track_data = TrackService.fetch_track_data( ) + + try: + # Step 1: Fetch data from track.projects + track_data = TrackService.fetch_track_data() - # Step 2: Clear the target database of existing records - cls._clear_target_db(target_session, target_model, target_system) + # Step 2: Clear the target database of existing records + cls._clear_target_db(target_session, target_model, target_system) - # Step 3: Insert new records into the target database - cls._insert_into_target_db(track_data, target_session, target_model, target_system) + # Step 3: Insert new records into the target database + cls._insert_into_target_db(track_data, target_session, target_model, target_system) - print(f"Project Extractor for {target_system.value} completed at {datetime.now()}") + current_app.logger.info(f"Project Extractor for {target_system.value} completed at {datetime.now()}") + finally: + # Cleanup session based on type + if target_system == TargetSystem.SUBMIT: + # Flask-SQLAlchemy cleanup + from epic_cron.models import db + db.session.remove() + # Raw SQLAlchemy sessions are cleaned up in helper methods with context managers @staticmethod def _get_target_config(target_system): @@ -60,8 +69,7 @@ def _clear_target_db(target_session, target_model, target_system): individually. Records that cannot be deleted due to dependencies are logged for further analysis. """ - print(f"Preparing to clear existing records in the {target_system.value} database...") - debug_logs_enabled = current_app.config.get("ENABLE_DETAILED_LOGS", False) + current_app.logger.info(f"Preparing to clear existing records in the {target_system.value} database...") # Initialize counters for summary total_records = 0 @@ -69,61 +77,93 @@ def _clear_target_db(target_session, target_model, target_system): successful_deletes = 0 failed_records = [] - # Iterate through each record and attempt deletion - with target_session() as session: + # Handle Flask-SQLAlchemy vs raw SQLAlchemy differently + if target_system == TargetSystem.SUBMIT: + # Flask-SQLAlchemy - use db.session directly + from epic_cron.models import db + session = db.session try: - print(f"Fetching all records from the {target_system.value} database for deletion...") + current_app.logger.info(f"Fetching all records from the {target_system.value} database for deletion...") records = session.query(target_model).all() total_records = len(records) for record in records: record_data = {col.name: getattr(record, col.name, None) for col in record.__table__.columns} - if debug_logs_enabled: - print(f"trying to delete record: {record_data}") + current_app.logger.debug(f"trying to delete record: {record_data}") try: session.delete(record) session.commit() # Commit after each successful delete successful_deletes += 1 - if debug_logs_enabled: - print(f"trying to delete record: {record_data}") + current_app.logger.debug(f"successfully deleted record: {record_data}") except Exception as delete_error: failed_deletes += 1 failed_records.append({"record": record_data, "error": str(delete_error)}) - print(f"WARNING: Could not delete record: {record_data}. Error: {delete_error}") + current_app.logger.warning(f"Could not delete record: {record_data}. Error: {delete_error}") session.rollback() # Rollback only this transaction - print(f"Finished processing deletions in the {target_system.value} database.") + current_app.logger.info(f"Finished processing deletions in the {target_system.value} database.") except Exception as fetch_error: - print(f"ERROR: Failed to fetch records from the {target_system.value} database. Error: {fetch_error}") + current_app.logger.error(f"Failed to fetch records from the {target_system.value} database. Error: {fetch_error}") session.rollback() + else: + # Raw SQLAlchemy - use context manager + with target_session() as session: + try: + current_app.logger.info(f"Fetching all records from the {target_system.value} database for deletion...") + records = session.query(target_model).all() + total_records = len(records) + + for record in records: + record_data = {col.name: getattr(record, col.name, None) for col in record.__table__.columns} + + current_app.logger.debug(f"trying to delete record: {record_data}") + try: + session.delete(record) + session.commit() # Commit after each successful delete + successful_deletes += 1 + current_app.logger.debug(f"successfully deleted record: {record_data}") + except Exception as delete_error: + failed_deletes += 1 + failed_records.append({"record": record_data, "error": str(delete_error)}) + current_app.logger.warning(f"Could not delete record: {record_data}. Error: {delete_error}") + session.rollback() # Rollback only this transaction + + current_app.logger.info(f"Finished processing deletions in the {target_system.value} database.") + except Exception as fetch_error: + current_app.logger.error(f"Failed to fetch records from the {target_system.value} database. Error: {fetch_error}") + session.rollback() # Print summary of the operation - print(f"\nSummary of target database clearing for {target_system.value}:") - print(f"- Total records found: {total_records}") - print(f"- Records successfully deleted: {successful_deletes}") - print(f"- Records failed to delete: {failed_deletes}") + current_app.logger.info(f"Summary of target database clearing for {target_system.value}:") + current_app.logger.info(f"- Total records found: {total_records}") + current_app.logger.info(f"- Records successfully deleted: {successful_deletes}") + current_app.logger.info(f"- Records failed to delete: {failed_deletes}") if failed_records: - print("\nDetails of records that failed to delete:") + current_app.logger.warning("Details of records that failed to delete:") for failed in failed_records: - print(f"Record: {failed['record']}, Error: {failed['error']}") + current_app.logger.warning(f"Record: {failed['record']}, Error: {failed['error']}") - print("Summary: Clearing operation completed.") + current_app.logger.info("Summary: Clearing operation completed.") @staticmethod def _insert_into_target_db(track_data, target_session, target_model, target_system): """Insert new records into the target database.""" - print(f"Inserting new records into the {target_system.value} database...") + current_app.logger.info(f"Inserting new records into the {target_system.value} database...") - with target_session() as session: - successful_inserts = 0 - failed_inserts = 0 + successful_inserts = 0 + failed_inserts = 0 + + # Handle Flask-SQLAlchemy vs raw SQLAlchemy differently + if target_system == TargetSystem.SUBMIT: + # Flask-SQLAlchemy - use db.session directly + from epic_cron.models import db + session = db.session + for index, row in enumerate(track_data): project_dict = dict(row._mapping) - debug_logs_enabled = current_app.config.get("ENABLE_DETAILED_LOGS", False) - if debug_logs_enabled: - print(f"Inserting project {index + 1}/{len(track_data)}: {project_dict}") + current_app.logger.debug(f"Inserting project {index + 1}/{len(track_data)}: {project_dict}") try: if target_system == TargetSystem.SUBMIT: @@ -165,13 +205,61 @@ def _insert_into_target_db(track_data, target_session, target_model, target_syst except Exception as e: failed_inserts += 1 - print(f"\n*** FAILED TO INSERT PROJECT {project_dict.get('id')} ***") - print(f"Error Details: {e}") - print(f"Failed Data: {project_dict}\n") + current_app.logger.error(f"FAILED TO INSERT PROJECT {project_dict.get('id')}") + current_app.logger.error(f"Error Details: {e}") + current_app.logger.error(f"Failed Data: {project_dict}") session.rollback() - print( + current_app.logger.info( f"Summary: Inserted {successful_inserts} records successfully into the {target_system.value} database." ) if failed_inserts > 0: - print(f"Summary: Failed to insert {failed_inserts} records into the {target_system.value} database.") + current_app.logger.warning(f"Summary: Failed to insert {failed_inserts} records into the {target_system.value} database.") + else: + # Raw SQLAlchemy - use context manager + with target_session() as session: + for index, row in enumerate(track_data): + project_dict = dict(row._mapping) + current_app.logger.debug(f"Inserting project {index + 1}/{len(track_data)}: {project_dict}") + + try: + if target_system == TargetSystem.CONDITIONS: + # con repo uses epic_guid as project_id + con_project_id = project_dict.get("epic_guid") or str(project_dict["id"]) + project_instance = target_model( + project_id=con_project_id, + project_name=project_dict["name"], + project_type=(project_dict.get("type_name") or "").strip(), + created_date=datetime.utcnow(), + updated_date=datetime.utcnow(), + created_by="cronjob", + updated_by="cronjob", + ) + else: + project_instance = target_model( + id=project_dict["id"], + name=project_dict['name'], + created_date=datetime.utcnow(), + updated_date=datetime.utcnow(), + created_by="cronjob", + updated_by="cronjob", + is_active=True, + is_deleted=False + ) + + session.add(project_instance) + session.commit() + successful_inserts += 1 + + except Exception as e: + failed_inserts += 1 + current_app.logger.error(f"FAILED TO INSERT PROJECT {project_dict.get('id')}") + current_app.logger.error(f"Error Details: {e}") + current_app.logger.error(f"Failed Data: {project_dict}") + session.rollback() + + current_app.logger.info( + f"Summary: Inserted {successful_inserts} records successfully into the {target_system.value} database." + ) + if failed_inserts > 0: + current_app.logger.warning(f"Summary: Failed to insert {failed_inserts} records into the {target_system.value} database.") diff --git a/jobs/epic-cron/tasks/submit_mail.py b/jobs/epic-cron/tasks/submit_mail.py new file mode 100644 index 0000000..d24aa9f --- /dev/null +++ b/jobs/epic-cron/tasks/submit_mail.py @@ -0,0 +1,32 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MET Publish Scheduled Engagements.""" +from datetime import datetime + +from flask import current_app + +from epic_cron.models.db import init_submit_db, ma +from epic_cron.services.mail_service import EmailService + + +class SubmitMailer: # pylint:disable=too-few-public-methods + """Task to publish scheduled Engagements due.""" + + @classmethod + def send_mail(cls): + """Publish the scheduled engagements.""" + init_submit_db(current_app) + ma.init_app(current_app) + current_app.logger.info('Starting Email At---{}'.format(datetime.now())) + EmailService.process_email_queue() diff --git a/jobs/epic-cron/tasks/sync_approved_condition.py b/jobs/epic-cron/tasks/sync_approved_condition.py new file mode 100644 index 0000000..2f714a7 --- /dev/null +++ b/jobs/epic-cron/tasks/sync_approved_condition.py @@ -0,0 +1,32 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Update projects with approved condition from condition repository.""" +from datetime import datetime + +from flask import current_app + +from epic_cron.models.db import init_submit_db, ma +from epic_cron.services.approved_condition_sync_service import ApprovedConditionService + + +class SyncApprovedCondition: # pylint:disable=too-few-public-methods + """Task to update projects having approved condition.""" + + @classmethod + def sync_approved_condition(cls): + """Update projects having approved condition.""" + init_submit_db(current_app) + ma.init_app(current_app) + current_app.logger.info('Starting Approved Condition Sync---{}'.format(datetime.now())) + ApprovedConditionService.sync_projects_with_approved_conditions() diff --git a/jobs/epic-cron/tasks/virus_scanner.py b/jobs/epic-cron/tasks/virus_scanner.py index 2bdf6ee..34399ef 100644 --- a/jobs/epic-cron/tasks/virus_scanner.py +++ b/jobs/epic-cron/tasks/virus_scanner.py @@ -15,10 +15,10 @@ def scan_file_from_path(file_path: str): infected, info = clam.scan_bytes(data) if infected is True: - print(f"Virus detected: {info}") + current_app.logger.warning(f"Virus detected: {info}") elif infected is False: - print("File is clean.") + current_app.logger.info("File is clean.") else: - print(f"Scan failed or unknown result: {info}") + current_app.logger.warning(f"Scan failed or unknown result: {info}") except Exception as e: - print(f"Error scanning file: {e}") + current_app.logger.error(f"Error scanning file: {e}") diff --git a/jobs/epic-cron/templates/submit/centre/access_denied_notification.html b/jobs/epic-cron/templates/submit/centre/access_denied_notification.html new file mode 100644 index 0000000..4119010 --- /dev/null +++ b/jobs/epic-cron/templates/submit/centre/access_denied_notification.html @@ -0,0 +1,87 @@ + + + + + EPIC Access Request: {{ user_name }} for {{ application_name }} + + + +
+ + +
+

Hello, {{user_name}}

+ +

Your access request for {{application_name}} has been denied.

+

If you have any questions or believe you should have been approved, please email EAO.EPICsystem@gov.bc.ca.

+ +

Thank you,

+ +

EPIC.centre System

+
+ +
+

This is an automated message. Please do not reply to this email.

+
+
+ + \ No newline at end of file diff --git a/jobs/epic-cron/templates/submit/centre/access_granted_notification.html b/jobs/epic-cron/templates/submit/centre/access_granted_notification.html new file mode 100644 index 0000000..ef45da9 --- /dev/null +++ b/jobs/epic-cron/templates/submit/centre/access_granted_notification.html @@ -0,0 +1,95 @@ + + + + + EPIC Access Request: {{ user_name }} for {{ application_name }} + + + +
+ + +
+

Hello, {{user_name}}

+ +

Your access request has been approved.

+ +

Access Details:

+
+

Application: {{ application_name }}

+

Access Level: {{ access_level }}

+
+ +

You can now access this application from the EPIC.centre launchpad.

+

Please note: if you are currently logged in EPIC.centre, you will need to log out, then log back in to see the changes.

+ +

Thank you,

+ +

EPIC.centre System

+
+ +
+

This is an automated message. Please do not reply to this email.

+
+
+ + diff --git a/jobs/epic-cron/templates/submit/centre/access_request_received_notification.html b/jobs/epic-cron/templates/submit/centre/access_request_received_notification.html new file mode 100644 index 0000000..1b39d97 --- /dev/null +++ b/jobs/epic-cron/templates/submit/centre/access_request_received_notification.html @@ -0,0 +1,101 @@ + + + + + EPIC Access Request: {{ user_name }} for {{ application_name }} + + + +
+ + +
+

Hello,

+ +

A user has requested access to an EPIC application that you administer.

+ +

Request Details:

+
+

Requestor: {{ user_name }}

+

Email: {{ user_email }}

+

Application: {{ application_name }}

+

Requested on: {{ requested_at }}

+
+ +

This user is requesting access to {{ application_name }}.

+

Please click the link below to review this request based on the user’s role and responsibilities.

+ +
+ Review Request in EPIC.centre +
+ +

A confirmation email will be sent to the requester when their access level is updated in EPIC.centre.

+ +

Thank you,

+
+ +
+

This is an automated message. Please do not reply to this email.

+
+
+ + diff --git a/jobs/epic-cron/templates/submit/centre/access_request_submitted_confirmation.html b/jobs/epic-cron/templates/submit/centre/access_request_submitted_confirmation.html new file mode 100644 index 0000000..5899a02 --- /dev/null +++ b/jobs/epic-cron/templates/submit/centre/access_request_submitted_confirmation.html @@ -0,0 +1,90 @@ + + + + + Your EPIC Access Request for {{ application_name }} Has Been Submitted + + + +
+ + +
+

Hello {{ user_name }},

+ +

Your request for access to {{ application_name }} has been submitted successfully.

+ +

Request Details:

+ +
+

Application: {{ application_name }}

+

Requested on: {{ requested_at }}

+
+ +

+ Super Users for this application have been notified and will review your request. + You will receive an email when your request has been processed. +

+ +

Thank you,

+ +

EPIC.centre System

+
+ +
+

This is an automated message. Please do not reply to this email.

+
+
+ + diff --git a/jobs/epic-cron/templates/submit/management_plan_submission_notify_staff.html b/jobs/epic-cron/templates/submit/management_plan_submission_notify_staff.html new file mode 100644 index 0000000..c9d568d --- /dev/null +++ b/jobs/epic-cron/templates/submit/management_plan_submission_notify_staff.html @@ -0,0 +1,84 @@ + + + + + Document Submission + + + +
+ + +
+

Hello,

+ +

We want to inform you that {{ certificate_holder_name }} submitted {{ package_name + }} for {{ project_name }} on {{ submission_date }} .

+ +
+

Log in to EPIC.submit to view the submission package.

+
+ +

Best regards,

+ +

+ EAO Management Plan Operations Team
+ B.C. Environmental Assessment Office
+ Government of British Columbia +

+
+
+ + diff --git a/jobs/epic-cron/templates/submit/management_plan_submission_verification.html b/jobs/epic-cron/templates/submit/management_plan_submission_verification.html new file mode 100644 index 0000000..5b13828 --- /dev/null +++ b/jobs/epic-cron/templates/submit/management_plan_submission_verification.html @@ -0,0 +1,87 @@ + + + + + Management Plan Submission Received + + + +
+ + +
+

Hello {{ submitter_name }},

+ +

Thank you for your submission on behalf of {{ certificate_holder_name }} through our new submission tool. We have successfully received your submission, and the details are as follows:

+ +
+

Submission Name: {{ package_name }}

+

Submission Date: {{ submission_date }}

+

Submitted Documents:

+
    + {% for document in documents %} +
  • {{ document }}
    • + {% endfor %} +
+
+ +

If you have any questions or require further assistance, please don’t hesitate to contact us by responding to this email.

+ +

Best regards,

+ +

+ EAO Management Plan Operations Team
+ B.C. Environmental Assessment Office
+ Government of British Columbia +

+
+
+ + diff --git a/jobs/epic-cron/templates/submit/management_plan_update_request_created.html b/jobs/epic-cron/templates/submit/management_plan_update_request_created.html new file mode 100644 index 0000000..b015b25 --- /dev/null +++ b/jobs/epic-cron/templates/submit/management_plan_update_request_created.html @@ -0,0 +1,82 @@ + + + + + Action Required in EPIC.submit + + + +
+ + +
+

Hello {{ submitter_name }},

+ +

You have a request regarding your submission of {{ package_name }}.

+ +
+

Please log in to the EPIC.submit application at your earliest convenience to review the details and make the necessary updates.

+
+ +

If you have any questions, feel free to reply to this email.

+ +

Thank you for your prompt attention to this matter.

+ +

Best regards,

+

+ EAO Management Plan Operations Team
+ B.C. Environmental Assessment Office
+ Government of British Columbia +

+
+
+ + diff --git a/jobs/epic-cron/templates/submit/new_user_invitation.html b/jobs/epic-cron/templates/submit/new_user_invitation.html new file mode 100644 index 0000000..d0373be --- /dev/null +++ b/jobs/epic-cron/templates/submit/new_user_invitation.html @@ -0,0 +1,86 @@ + + + + + Invitation to collaborate on EPIC.submit + + + +
+ + +
+

You have been invited to {{ invitation_action_text }} {{ project_name }} on EPIC.submit.

+ +

Click the link below and sign in with your Business BCeID or BC Services Card to access the project.

+ +
+ Join EPIC.submit +
+ +

+ If you don’t have a Business BCeID associated with this project, please contact the person in charge of the Business BCeID account for {{ certificate_holder_name }}. +

+ +

+ For more information on how to use or set up a BC Services Card account, visit {{ bc_service_card_url }}. +

+ +

Thank you,

+

+ EAO Management Plan Operations Team
+ B.C. Environmental Assessment Office
+ Government of British Columbia +

+
+
+ + diff --git a/jobs/epic-cron/templates/submit/resubmission_request.html b/jobs/epic-cron/templates/submit/resubmission_request.html new file mode 100644 index 0000000..2ea1cb7 --- /dev/null +++ b/jobs/epic-cron/templates/submit/resubmission_request.html @@ -0,0 +1,86 @@ + + + + + Invitation to Resubmit - {{ package_name }} + + + +
+ + +
+

Hello,

+ +

You can now resubmit your documents for {{ package_name }} in EPIC.submit.

+ +

Click the link below to access your submission package.

+ +
+ Access your submission in EPIC.submit +
+ +

Please note: After you submit the new version of {{ package_name }}, the Management Plan Operations Team will review it. Until the EAO reviews this latest version, the previous version finalized for implementation by the EAO (if any) remains the enforceable version.

+ +

Thank you,

+ +

+ EAO Management Plan Operations Team
+ B.C. Environmental Assessment Office
+ Government of British Columbia +

+
+
+ + \ No newline at end of file diff --git a/jobs/epic-cron/tests/submit/__init__.py b/jobs/epic-cron/tests/submit/__init__.py new file mode 100644 index 0000000..9557511 --- /dev/null +++ b/jobs/epic-cron/tests/submit/__init__.py @@ -0,0 +1,14 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The Test Suites to ensure that the service is built and operating correctly.""" diff --git a/jobs/epic-cron/tests/submit/jobs/__init__.py b/jobs/epic-cron/tests/submit/jobs/__init__.py new file mode 100644 index 0000000..e809b5c --- /dev/null +++ b/jobs/epic-cron/tests/submit/jobs/__init__.py @@ -0,0 +1,14 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""This module holds tests."""