Refactor workflow to use internal TF vars

.github/workflows/build-and-deploy.yml

@@ -100,7 +100,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: '24'
           cache: 'npm'
           cache-dependency-path: '**/package-lock.json'
 
@@ -181,14 +181,17 @@ jobs:
           path: code-coverage-results.md
 
   # ---------------------------
-  # REFRESH DATA
+  # REFRESH DATA - Run after environment set up via deploying app. Without prior deployment, this will fail. 
   # ---------------------------
   refresh-data:
     name: Refresh data (School Data Ingestion Pipeline)
     if: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'refresh data') }}
-    needs: [build, test]
+    needs: [build, test, deploy-review-app-with-refresh]
     uses: ./.github/workflows/data-pipeline.yml
     secrets: inherit
+    with:
+      environment: review
+      pr-number: ${{ github.event.pull_request.number }}
     permissions:
       contents: write
       id-token: write
@@ -341,17 +344,18 @@ jobs:
   # REVIEW APP DEPLOYMENT (PR) - refresh requested
   # ---------------------------
   deploy-review-app-with-refresh:
-    name: Deployment To Review (after refresh)
+    name: Deployment To Review (before refresh)
     concurrency: deploy_review_${{ github.event.pull_request.number }}
-    needs: [build, test, refresh-data]
+    needs: [build, test]
     if: ${{ github.event_name == 'pull_request'
       && (
       github.event.action == 'labeled' ||
       github.event.action == 'synchronize'
       )
       && contains(github.event.pull_request.labels.*.name, 'deploy')
       && contains(github.event.pull_request.labels.*.name, 'refresh data')
-      && needs.refresh-data.result == 'success' }}
+      }}
+
     environment:
       name: review
       url: ${{ steps.deploy_review.outputs.environment_url }}
@@ -360,12 +364,6 @@ jobs:
       id-token: write
       pull-requests: write
 
-    env:
-      AKS_RESOURCE_GROUP: ${{ secrets.AKS_RESOURCE_GROUP }}
-      AKS_CLUSTER_NAME: ${{ secrets.AKS_CLUSTER_NAME }}
-      KONDUIT_NAMESPACE: ${{ secrets.AKS_NAMESPACE }}
-      KONDUIT_APP_NAME: ${{ secrets.KONDUIT_APP_NAME }}
-
     steps:
       - name: Deploy App to Review
         id: deploy_review
@@ -384,108 +382,12 @@ jobs:
           gcp-wip: ${{ vars.GCP_WIP }}
           gcp-project-id: ${{ vars.GCP_PROJECT_ID }}
 
-      # ---------------------------
-      # REVIEW APP DATABASE DEPLOYMENT (PR)
-      # ---------------------------
-      # - name: Checkout (needed for workspace consistency)
-      #   uses: actions/checkout@v4
-
-      # - name: Install Azure CLI
-      #   shell: bash
-      #   run: |
-      #     set -euo pipefail
-      #     curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
-
-      # - name: Azure login (OIDC)
-      #   uses: azure/login@v2
-      #   with:
-      #     client-id: ${{ secrets.AZURE_CLIENT_ID }}
-      #     tenant-id: ${{ secrets.AZURE_TENANT_ID }}
-      #     subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-
-      # - name: Install kubectl (pinned)
-      #   shell: bash
-      #   run: |
-      #     set -euo pipefail
-      #     KUBECTL_VERSION="v1.29.8"
-      #     curl -fsSLo kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
-      #     sudo install -m 0755 kubectl /usr/local/bin/kubectl
-      #     kubectl version --client=true
-
-      # - name: Install kubelogin (pinned)
-      #   shell: bash
-      #   run: |
-      #     set -euo pipefail
-      #     KUBELOGIN_VERSION="v0.1.6"
-      #     curl -fsSLo kubelogin.zip "https://github.com/Azure/kubelogin/releases/download/${KUBELOGIN_VERSION}/kubelogin-linux-amd64.zip"
-      #     unzip -q kubelogin.zip
-      #     sudo install -m 0755 bin/linux_amd64/kubelogin /usr/local/bin/kubelogin
-      #     kubelogin --version
-
-      # - name: Configure AKS credentials
-      #   shell: bash
-      #   run: |
-      #     set -euo pipefail
-      #     : "${AKS_RESOURCE_GROUP:?AKS_RESOURCE_GROUP is not set}"
-      #     : "${AKS_CLUSTER_NAME:?AKS_CLUSTER_NAME is not set}"
-      #     az aks get-credentials --overwrite-existing -g "${AKS_RESOURCE_GROUP}" -n "${AKS_CLUSTER_NAME}"
-      #     kubelogin convert-kubeconfig -l azurecli
-
-      # - name: Download konduit.sh
-      #   shell: bash
-      #   run: |
-      #     set -euo pipefail
-      #     curl -fsSL https://raw.githubusercontent.com/DFE-Digital/teacher-services-cloud/main/scripts/konduit.sh \
-      #       -o "$GITHUB_WORKSPACE/konduit.sh"
-      #     chmod +x "$GITHUB_WORKSPACE/konduit.sh"
-      #     ls -la "$GITHUB_WORKSPACE/konduit.sh"
-
-      # - name: Download seed backup from Blob
-      #   shell: bash
-      #   env:
-      #     AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}
-      #     AZURE_STORAGE_CONTAINER: ${{ secrets.AZURE_STORAGE_CONTAINER }}
-      #   run: |
-      #     set -euo pipefail
-      #     BACKUP_BLOB="db-backups/sappub_review_seed_latest.sql.gz"
-      #     BACKUP_FILE="seed.sql.gz"
-
-      #     az storage blob download \
-      #       --container-name "${AZURE_STORAGE_CONTAINER}" \
-      #       --name "${BACKUP_BLOB}" \
-      #       --file "${BACKUP_FILE}" \
-      #       --connection-string "${AZURE_STORAGE_CONNECTION_STRING}" \
-      #       --overwrite true
-
-      #     ls -lh "${BACKUP_FILE}"
-
-      # - name: Restore backup into PR review DB
-      #   continue-on-error: true
-      #   shell: bash
-      #   env:
-      #     AKS_NAMESPACE: ${{ secrets.AKS_REVIEW_NAMESPACE }}
-      #   run: |
-      #     set -euo pipefail
-      #     PR_NUMBER="${{ github.event.pull_request.number }}"
-      #     APP_NAME="sap-public-pr-${PR_NUMBER}"
-      #     BACKUP_FILE="seed.sql.gz"
-      #     : "${AKS_NAMESPACE:?AKS_NAMESPACE is not set}"
-
-      #     ls -lh "${BACKUP_FILE}"
-      #     gzip -t "${BACKUP_FILE}"
-
-      #     "$GITHUB_WORKSPACE/konduit.sh" \
-      #       -n "${AKS_NAMESPACE}" \
-      #       -t 7200 \
-      #       -x \
-      #       -i "${BACKUP_FILE}" -c \
-      #       "${APP_NAME}" -- psql -v ON_ERROR_STOP=1 -X
 
   # ---------------------------
-  # DEPLOY TEST (AUTO) + PROD (MANUAL APPROVAL)
+  # DEPLOY TEST (AUTO)
   # ---------------------------
   deploy:
-    name: Deploy to Test and Production
+    name: Deploy to Test
     if: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' }}
     needs: [build, test]
     runs-on: ubuntu-latest
@@ -496,7 +398,8 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        environment: [test, production]
+        #environment: [test, production] //Commented out should we wish to reinstate auto deployment with the matrix.
+        environment: [test]
 
     environment:
       name: ${{ matrix.environment }}

.github/workflows/data-pipeline.yml

@@ -3,45 +3,80 @@ name: School Data Ingestion Pipeline
 on:
   workflow_dispatch:
   workflow_call:
+    inputs:
+      environment:
+        description: Environment to run the data pipeline in (affects source URLs and Blob paths). Default is test. (Optional)
+        required: false
+        default: test
+        type: string
+      pr-number:
+        description: The pull request number for review environment. (Optional)
+        required: false
+        default: ""
+        type: string
   schedule:
     - cron: "0 6 * * *"   # every day at 06:00 UTC
 
 permissions:
   contents: write
   id-token: write   # required if using OIDC with azure/login
 
+env:
+  SERVICE_NAME: sap-public
+  SERVICE_SHORT: sappub
+  TF_VARS_PATH: terraform/application/config
+
 jobs:
   ingest:
     name: Ingest Raw Data & Build Warehouse
     runs-on: ubuntu-latest
-    environment: test
+    environment:
+      name: ${{ inputs.environment || 'test' }}
 
     env:
+      # Terraform and service-related settings
+      DEPLOY_ENV: ${{ inputs.environment || 'test'  }}
+      PR_NUM: ${{ inputs.pr-number || ''  }}
+
       # Blob (currently from GitHub secrets)
       AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}
-      AZURE_STORAGE_CONTAINER: ${{ secrets.AZURE_STORAGE_CONTAINER }}
+      AZURE_STORAGE_CONTAINER: 'schooldata'
 
       # Sensitive source URLs (THREE endpoints in GitHub secrets)
       SENSITIVE_DATASET_URL: ${{ secrets.SENSITIVE_DATASET_URL }}
       SENSITIVE_ESTABLISHMENT_LINKS_URL: ${{ secrets.SENSITIVE_ESTABLISHMENT_LINKS_URL }}
       SENSITIVE_MAT_LINKS_URL: ${{ secrets.SENSITIVE_MAT_LINKS_URL }}
 
-
-      # Konduit / AKS settings (also in GitHub secrets)
-      AKS_RESOURCE_GROUP: ${{ secrets.AKS_RESOURCE_GROUP }}
-      AKS_CLUSTER_NAME: ${{ secrets.AKS_CLUSTER_NAME }}
-      AKS_NAMESPACE: ${{ secrets.AKS_NAMESPACE }}
-      KONDUIT_APP_NAME: ${{ secrets.KONDUIT_APP_NAME }}
-
     steps:
       # ==============================
-      # 1. Checkout repo
+      # 0. Checkout repo
       # ==============================
       - name: Checkout
         uses: actions/checkout@v4
         with:
           persist-credentials: true
 
+      # =============================
+      # 1. Set TFVars for later use
+      # =============================
+      - name: Set environment variables for ${{ env.DEPLOY_ENV }}
+        run: |
+          source global_config/${DEPLOY_ENV}.sh
+          tf_vars_file=${TF_VARS_PATH}/${DEPLOY_ENV}.tfvars.json
+          echo "AKS_NAMESPACE=$(jq -r '.namespace' ${tf_vars_file})" >> $GITHUB_ENV
+          echo "AKS_CLUSTER_NAME=$(jq -r '.cluster_name' ${tf_vars_file})" >> $GITHUB_ENV
+          echo "AKS_RESOURCE_GROUP=$(jq -r '.cluster_rg' ${tf_vars_file})" >> $GITHUB_ENV
+          echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_SHORT}dp${CONFIG_SHORT}sa" >> $GITHUB_ENV
+          echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
+          if [ "${DEPLOY_ENV}" == "review" ]; then
+            echo "KONDUIT_APP_NAME=$(jq -r '.app_prefix' ${tf_vars_file})${{env.PR_NUM}}" >> $GITHUB_ENV
+          else
+            echo "KONDUIT_APP_NAME=$(jq -r '.app_prefix' ${tf_vars_file})" >> $GITHUB_ENV
+          fi
+
+          echo "Set environment variables from ${tf_vars_file}:"
+          echo "KONDUIT_APP_NAME=${KONDUIT_APP_NAME}"
+
       # ==============================
       # 2. Install dependencies
       # ==============================
@@ -589,11 +624,11 @@ jobs:
             echo "ETL attempt $attempt/$max_attempts"
             set +e
             "$GITHUB_WORKSPACE/konduit.sh" \
-              -n "${{ secrets.AKS_NAMESPACE }}" \
+              -n "${{ env.AKS_NAMESPACE }}" \
               -t 7200 \
               -x \
               -i "$GITHUB_WORKSPACE/SAPData/Sql/run_all.sql" \
-              "${{ secrets.KONDUIT_APP_NAME }}" -- psql 2>&1 | tee "$log_file"
+              "${{ env.KONDUIT_APP_NAME }}" -- psql 2>&1 | tee "$log_file"
             exit_code=${PIPESTATUS[0]}
             set -e
             if [ $exit_code -eq 0 ]; then
@@ -622,21 +657,20 @@ jobs:
       # ==============================
       - name: Create + upload review seed DB backup
         shell: bash
+        if: ${{ env.DEPLOY_ENV == 'test' || env.DEPLOY_ENV == 'production' }}
         run: |
           set -euo pipefail
           TODAY=$(TZ=Europe/London date +"%F")
           LOCAL_FILE="sappub_review_seed_${TODAY}.sql.gz"
           # Only ever keep this single blob in storage
           LATEST_BLOB="db-backups/sappub_review_seed_latest.sql.gz"
           echo "Creating DB dump -> ${LOCAL_FILE}"
-          echo "AKS_NAMESPACE=${AKS_NAMESPACE}"
-          echo "KONDUIT_APP_NAME=${KONDUIT_APP_NAME}"
           set +e
           "$GITHUB_WORKSPACE/konduit.sh" \
-            -n "${AKS_NAMESPACE}" \
+            -n "${{ env.AKS_NAMESPACE }}" \
             -t 7200 \
             -x \
-            "${KONDUIT_APP_NAME}" -- \
+            "${{ env.KONDUIT_APP_NAME }}" -- \
             pg_dump --format=plain --no-owner --no-privileges 2> konduit_pg_dump.log | gzip -c > "${LOCAL_FILE}"
           exit_code=${PIPESTATUS[0]}
           set -e

terraform/application/config/production.tfvars.json

@@ -14,5 +14,8 @@
   "enable_monitoring": true,
   "statuscake_contact_groups": [ 282453, 350349 ],
   "replicas": 4,
-  "enable_logit": true
+  "enable_logit": true,
+  "app_prefix": "sap-public-production",
+  "cluster_name": "s189p01-tsc-production-aks",
+  "cluster_rg": "s189p01-tsc-pd-rg"
 }

terraform/application/config/review.tfvars.json

@@ -3,5 +3,8 @@
     "namespace": "sip-development",
     "deploy_azure_backing_services": false,
     "enable_postgres_ssl" : false,
+    "app_prefix": "sap-public-pr-",
+    "cluster_name":"s189t01-tsc-test-aks",
+    "cluster_rg":"s189t01-tsc-ts-rg"
     "enable_dfe_analytics_federated_auth": true
 }

terraform/application/config/test.tfvars.json

@@ -3,5 +3,8 @@
   "namespace": "sip-test",
   "enable_postgres_backup_storage": true,
   "storage_container_delete_retention_days": 7,
-  "enable_dfe_analytics_federated_auth": true
+  "enable_dfe_analytics_federated_auth": true,
+  "app_prefix": "sap-public-test",
+  "cluster_name": "s189t01-tsc-test-aks",
+  "cluster_rg": "s189t01-tsc-ts-rg"
 }