From f575252d92b8fc18bed7db7e4cf66ac5ebfba001 Mon Sep 17 00:00:00 2001 From: Peter Burkholder Date: Mon, 10 Nov 2025 12:43:13 -0500 Subject: [PATCH 1/3] Use --wait to ensure each job is queued up and doesn't exhaust quota --- .circleci/cron.sh | 60 +++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/.circleci/cron.sh b/.circleci/cron.sh index 2dd09a68d..148257870 100755 --- a/.circleci/cron.sh +++ b/.circleci/cron.sh @@ -2,7 +2,7 @@ # Fail if anything within this script returns # a non-zero exit code -set -e +# set -e echo "Logging into cloud.gov" cf login -a $CF_API_ENDPOINT -u $CF_USERNAME -p $CF_PASSWORD -o $CF_ORG -s $CF_SPACE @@ -13,18 +13,20 @@ cf login -a $CF_API_ENDPOINT -u $CF_USERNAME -p $CF_PASSWORD -o $CF_ORG -s $CF_S echo "Running tasks in Staging..." +STAGING_TASK="cf run-task touchpoints-staging-sidekiq-worker --wait -c" + # Users -cf run-task touchpoints-staging-sidekiq-worker -c "rake scheduled_jobs:send_one_week_until_inactivation_warning" -cf run-task touchpoints-staging-sidekiq-worker -c "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" -cf run-task touchpoints-staging-sidekiq-worker -c "rake scheduled_jobs:deactivate_inactive_users" +$STAGING_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning" +$STAGING_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" +$STAGING_TASK "rake scheduled_jobs:deactivate_inactive_users" # Forms -cf run-task touchpoints-staging-sidekiq-worker -c "rake scheduled_jobs:send_daily_notifications" -cf run-task touchpoints-staging-sidekiq-worker -c "rake scheduled_jobs:send_weekly_notifications" -cf run-task touchpoints-staging-sidekiq-worker -c "rake scheduled_jobs:check_expiring_forms" -cf run-task touchpoints-staging-sidekiq-worker -c "rake scheduled_jobs:archive_forms" -cf run-task touchpoints-staging-sidekiq-worker -c "rake scheduled_jobs:notify_form_managers_of_inactive_forms" -# cf run-task touchpoints-staging-sidekiq-worker -c "rake scheduled_jobs:delete_submissions_trash" +$STAGING_TASK "rake scheduled_jobs:send_daily_notifications" +$STAGING_TASK "rake scheduled_jobs:send_weekly_notifications" +$STAGING_TASK "rake scheduled_jobs:check_expiring_forms" +$STAGING_TASK "rake scheduled_jobs:archive_forms" +$STAGING_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms" +# $STAGING_TASK "rake scheduled_jobs:delete_submissions_trash" echo "Staging tasks have completed." @@ -33,19 +35,20 @@ echo "Staging tasks have completed." # echo "Running tasks in Demo..." +DEMO_TASK="cf run-task touchpoints-demo-sidekiq-worker --wait -c" # Users -cf run-task touchpoints-demo-sidekiq-worker -c "rake scheduled_jobs:send_one_week_until_inactivation_warning" -cf run-task touchpoints-demo-sidekiq-worker -c "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" -cf run-task touchpoints-demo-sidekiq-worker -c "rake scheduled_jobs:deactivate_inactive_users" +$DEMO_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning" +$DEMO_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" +$DEMO_TASK "rake scheduled_jobs:deactivate_inactive_users" # Forms -cf run-task touchpoints-demo-sidekiq-worker -c "rake scheduled_jobs:send_daily_notifications" -cf run-task touchpoints-demo-sidekiq-worker -c "rake scheduled_jobs:send_weekly_notifications" -cf run-task touchpoints-demo-sidekiq-worker -c "rake scheduled_jobs:check_expiring_forms" -cf run-task touchpoints-demo-sidekiq-worker -c "rake scheduled_jobs:archive_forms" -cf run-task touchpoints-demo-sidekiq-worker -c "rake scheduled_jobs:notify_form_managers_of_inactive_forms" -# cf run-task touchpoints-demo-sidekiq-worker -c "rake scheduled_jobs:delete_submissions_trash" +$DEMO_TASK "rake scheduled_jobs:send_daily_notifications" +$DEMO_TASK "rake scheduled_jobs:send_weekly_notifications" +$DEMO_TASK "rake scheduled_jobs:check_expiring_forms" +$DEMO_TASK "rake scheduled_jobs:archive_forms" +$DEMO_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms" +# $DEMO_TASK "rake scheduled_jobs:delete_submissions_trash" echo "Demo tasks have completed." @@ -60,18 +63,19 @@ cf login -a $CF_API_ENDPOINT -u $CF_PRODUCTION_SPACE_DEPLOYER_USERNAME -p $CF_PR echo "Running tasks in Production..." +PROD_TASK="cf run-task touchpoints-production-sidekiq-worker --wait -c" # Users -cf run-task touchpoints-production-sidekiq-worker -c "rake scheduled_jobs:send_one_week_until_inactivation_warning" -cf run-task touchpoints-production-sidekiq-worker -c "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" -cf run-task touchpoints-production-sidekiq-worker -c "rake scheduled_jobs:deactivate_inactive_users" +$PROD_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning" +$PROD_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" +$PROD_TASK "rake scheduled_jobs:deactivate_inactive_users" # Forms -# cf run-task touchpoints-production-sidekiq-worker -c "rake scheduled_jobs:send_daily_notifications" -# cf run-task touchpoints-production-sidekiq-worker -c "rake scheduled_jobs:send_weekly_notifications" -# cf run-task touchpoints-production-sidekiq-worker -c "rake scheduled_jobs:check_expiring_forms" -# cf run-task touchpoints-production-sidekiq-worker -c "rake scheduled_jobs:archive_forms" -cf run-task touchpoints-production-sidekiq-worker -c "rake scheduled_jobs:notify_form_managers_of_inactive_forms" -# cf run-task touchpoints-production-sidekiq-worker -c "rake scheduled_jobs:delete_submissions_trash" +# $PROD_TASK "rake scheduled_jobs:send_daily_notifications" +# $PROD_TASK "rake scheduled_jobs:send_weekly_notifications" +# $PROD_TASK "rake scheduled_jobs:check_expiring_forms" +# $PROD_TASK "rake scheduled_jobs:archive_forms" +$PROD_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms" +# $PROD_TASK "rake scheduled_jobs:delete_submissions_trash" echo "Production tasks have completed." From 3eaffd6b459f97b60eb44e30abbab0ef334b7045 Mon Sep 17 00:00:00 2001 From: Peter Burkholder Date: Mon, 10 Nov 2025 12:46:37 -0500 Subject: [PATCH 2/3] Revert set -e --- .circleci/cron.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/cron.sh b/.circleci/cron.sh index 148257870..31d9f664c 100755 --- a/.circleci/cron.sh +++ b/.circleci/cron.sh @@ -2,7 +2,7 @@ # Fail if anything within this script returns # a non-zero exit code -# set -e +set -e echo "Logging into cloud.gov" cf login -a $CF_API_ENDPOINT -u $CF_USERNAME -p $CF_PASSWORD -o $CF_ORG -s $CF_SPACE From 84f1a8825a68361538dbf888b0898f1e233b62d6 Mon Sep 17 00:00:00 2001 From: Peter Burkholder Date: Mon, 10 Nov 2025 14:37:54 -0500 Subject: [PATCH 3/3] Catch failures and count 'em I non-prod I noticed that some of the tasks with --wait are failing. We don't want failing tasks to kill the entire job. We also don't want to bury failures. So this will keep track of the failues and mark the tasks as failing in CircleCI -- while still running as many as possible. --- .circleci/cron.sh | 146 ++++++++++++++++++++++++---------------------- 1 file changed, 75 insertions(+), 71 deletions(-) diff --git a/.circleci/cron.sh b/.circleci/cron.sh index 31d9f664c..d33ab8af7 100755 --- a/.circleci/cron.sh +++ b/.circleci/cron.sh @@ -4,79 +4,83 @@ # a non-zero exit code set -e -echo "Logging into cloud.gov" +# Set # of failures to 0 +F=0 + +function run_production_tasks() { + # === PRODUCTION environment =================================================== + echo "Running tasks in Production..." + + PROD_TASK="cf run-task touchpoints-production-sidekiq-worker --wait -c" + # Users + $PROD_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning" || F=$((F+=1)) + $PROD_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" || F=$((F+=1)) + $PROD_TASK "rake scheduled_jobs:deactivate_inactive_users" || F=$((F+=1)) + + # Forms + # $PROD_TASK "rake scheduled_jobs:send_daily_notifications" || F=$((F+=1)) + # $PROD_TASK "rake scheduled_jobs:send_weekly_notifications" || F=$((F+=1)) + # $PROD_TASK "rake scheduled_jobs:check_expiring_forms" || F=$((F+=1)) + # $PROD_TASK "rake scheduled_jobs:archive_forms" || F=$((F+=1)) + $PROD_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms" || F=$((F+=1)) + # $PROD_TASK "rake scheduled_jobs:delete_submissions_trash" || F=$((F+=1)) + echo "Production tasks have completed." +} + + +function run_staging_tasks() { + # === STAGING environment ====================================================== + echo "Running tasks in Staging..." + + STAGING_TASK="cf run-task touchpoints-staging-sidekiq-worker --wait -c" + + # Users + $STAGING_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning" || F=$((F+=1)) + $STAGING_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" || F=$((F+=1)) + $STAGING_TASK "rake scheduled_jobs:deactivate_inactive_users" || F=$((F+=1)) + + # Forms + $STAGING_TASK "rake scheduled_jobs:send_daily_notifications" || F=$((F+=1)) + $STAGING_TASK "rake scheduled_jobs:send_weekly_notifications" || F=$((F+=1)) + $STAGING_TASK "rake scheduled_jobs:check_expiring_forms" || F=$((F+=1)) + $STAGING_TASK "rake scheduled_jobs:archive_forms" || F=$((F+=1)) + $STAGING_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms" || F=$((F+=1)) + # $STAGING_TASK "rake scheduled_jobs:delete_submissions_trash" || F=$((F+=1)) + + echo "Staging tasks have completed." +} + +function run_demo_tasks() { + # === DEMO environment ========================================================= + echo "Running tasks in Demo..." + DEMO_TASK="cf run-task touchpoints-demo-sidekiq-worker --wait -c" + + # Users + $DEMO_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning" || F=$((F+=1)) + $DEMO_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" || F=$((F+=1)) + $DEMO_TASK "rake scheduled_jobs:deactivate_inactive_users" || F=$((F+=1)) + + # Forms + $DEMO_TASK "rake scheduled_jobs:send_daily_notifications" || F=$((F+=1)) + $DEMO_TASK "rake scheduled_jobs:send_weekly_notifications" || F=$((F+=1)) + $DEMO_TASK "rake scheduled_jobs:check_expiring_forms" || F=$((F+=1)) + $DEMO_TASK "rake scheduled_jobs:archive_forms" || F=$((F+=1)) + $DEMO_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms" || F=$((F+=1)) + # $DEMO_TASK "rake scheduled_jobs:delete_submissions_trash" || F=$((F+=1)) + + echo "Demo tasks have completed." +} + +echo "Logging into cloud.gov non-prod" cf login -a $CF_API_ENDPOINT -u $CF_USERNAME -p $CF_PASSWORD -o $CF_ORG -s $CF_SPACE - -# -# === STAGING environment ====================================================== -# - -echo "Running tasks in Staging..." - -STAGING_TASK="cf run-task touchpoints-staging-sidekiq-worker --wait -c" - -# Users -$STAGING_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning" -$STAGING_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" -$STAGING_TASK "rake scheduled_jobs:deactivate_inactive_users" - -# Forms -$STAGING_TASK "rake scheduled_jobs:send_daily_notifications" -$STAGING_TASK "rake scheduled_jobs:send_weekly_notifications" -$STAGING_TASK "rake scheduled_jobs:check_expiring_forms" -$STAGING_TASK "rake scheduled_jobs:archive_forms" -$STAGING_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms" -# $STAGING_TASK "rake scheduled_jobs:delete_submissions_trash" - -echo "Staging tasks have completed." - -# -# === DEMO environment ========================================================= -# - -echo "Running tasks in Demo..." -DEMO_TASK="cf run-task touchpoints-demo-sidekiq-worker --wait -c" - -# Users -$DEMO_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning" -$DEMO_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" -$DEMO_TASK "rake scheduled_jobs:deactivate_inactive_users" - -# Forms -$DEMO_TASK "rake scheduled_jobs:send_daily_notifications" -$DEMO_TASK "rake scheduled_jobs:send_weekly_notifications" -$DEMO_TASK "rake scheduled_jobs:check_expiring_forms" -$DEMO_TASK "rake scheduled_jobs:archive_forms" -$DEMO_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms" -# $DEMO_TASK "rake scheduled_jobs:delete_submissions_trash" - -echo "Demo tasks have completed." - +run_staging_tasks +run_demo_tasks cf logout -# -# === PRODUCTION environment =================================================== -# - -echo "Logging into cloud.gov" +echo "Logging into cloud.gov production environment" cf login -a $CF_API_ENDPOINT -u $CF_PRODUCTION_SPACE_DEPLOYER_USERNAME -p $CF_PRODUCTION_SPACE_DEPLOYER_PASSWORD -o $CF_ORG -s prod - -echo "Running tasks in Production..." - -PROD_TASK="cf run-task touchpoints-production-sidekiq-worker --wait -c" -# Users -$PROD_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning" -$PROD_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" -$PROD_TASK "rake scheduled_jobs:deactivate_inactive_users" - -# Forms -# $PROD_TASK "rake scheduled_jobs:send_daily_notifications" -# $PROD_TASK "rake scheduled_jobs:send_weekly_notifications" -# $PROD_TASK "rake scheduled_jobs:check_expiring_forms" -# $PROD_TASK "rake scheduled_jobs:archive_forms" -$PROD_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms" -# $PROD_TASK "rake scheduled_jobs:delete_submissions_trash" - -echo "Production tasks have completed." - +run_production_tasks cf logout + +echo "$0 exiting with failure count: $F" +exit $F \ No newline at end of file