Refactor

.github/workflows/data-pipeline.yml

@@ -10,10 +10,9 @@ on:
         type: choice
         options:
         - test
-        - review
         - production
       review-app-number:
-        description: Pull request number of the review app (optional, this will generate a backup file for that review app)
+        description: Pull request number of the review app (optional, this will run against the target environment to generate a backup file for that review app to use, and then restore the target DB from backup)
         type: string
         required: false
 
@@ -40,13 +39,16 @@ jobs:
       SENSITIVE_ESTABLISHMENT_LINKS_URL: ${{ secrets.SENSITIVE_ESTABLISHMENT_LINKS_URL }}
       SENSITIVE_MAT_LINKS_URL: ${{ secrets.SENSITIVE_MAT_LINKS_URL }}
 
-
       # Konduit / AKS settings (also in GitHub secrets)
       AKS_RESOURCE_GROUP: ${{ secrets.AKS_RESOURCE_GROUP }}
       AKS_CLUSTER_NAME: ${{ secrets.AKS_CLUSTER_NAME }}
       AKS_NAMESPACE: ${{ secrets.AKS_NAMESPACE }}
       KONDUIT_APP_NAME: ${{ secrets.KONDUIT_APP_NAME }}
 
+      SERVICE_NAME: get-school-improvement-insights
+      SERVICE_SHORT: sapsec
+      TF_VARS_PATH: terraform/application/config
+      DEPLOY_ENV: ${{ inputs.environment || 'test' }}
 
     steps:
       # ==============================
@@ -75,6 +77,7 @@ jobs:
       - name: Install Azure CLI
         run: |
           curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+
       # ==============================
       # 3. Azure login (needed for az aks get-credentials)
       # ==============================
@@ -119,6 +122,7 @@ jobs:
             -o "$GITHUB_WORKSPACE/konduit.sh"
           chmod +x "$GITHUB_WORKSPACE/konduit.sh"
           ls -la "$GITHUB_WORKSPACE/konduit.sh"
+
       # ==============================
       # 5. Ensure Blob container exists (idempotent)
       # ==============================
@@ -129,6 +133,7 @@ jobs:
             --name $env:AZURE_STORAGE_CONTAINER `
             --connection-string $env:AZURE_STORAGE_CONNECTION_STRING `
             --public-access off | Out-Null
+
       # ==============================
       # 6. Download source data and store latest versions in Blob
       #
@@ -154,9 +159,9 @@ jobs:
           $conn = $env:AZURE_STORAGE_CONNECTION_STRING
           $container = $env:AZURE_STORAGE_CONTAINER
           $manifestBlobName = "versions.json"
-          $sourcesPathRepo = "SAPData/raw_sources.json"
+          $sourcesPathRepo = "Data/SAPSec.SqlGenerator/raw_sources.json"
           if (-not (Test-Path $sourcesPathRepo)) { throw "Could not find $sourcesPathRepo in the repo." }
-          $workDir = "SAPData/Work/Versioning"
+          $workDir = "Data/SAPSec.SqlGenerator/Work/Versioning"
           New-Item -ItemType Directory -Force -Path $workDir | Out-Null
           $reportPath = Join-Path $workDir "step6_report.json"
           $report = [System.Collections.Generic.List[object]]::new()
@@ -495,6 +500,7 @@ jobs:
           "changed=$anyChanged" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8
           $latestFilesJson = ($latestFiles | ConvertTo-Json -Depth 10 -Compress)
           "latest_files=$latestFilesJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8
+
       # ==============================
       # 6b. Upload the report
       # ==============================
@@ -504,7 +510,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: step6-report
-          path: SAPData/Work/Versioning/step6_report.json
+          path: Data/SAPSec.SqlGenerator/Work/Versioning/step6_report.json
           if-no-files-found: warn
 
       # ==============================
@@ -520,7 +526,7 @@ jobs:
           LATEST_FILES: ${{ steps.version.outputs.latest_files }}
         run: |
           $ErrorActionPreference = "Stop"
-          $rawDir = "SAPData/DataMap/SourceFiles"
+          $rawDir = "Data/SAPSec.SqlGenerator/DataMap/SourceFiles"
           New-Item -ItemType Directory -Force -Path $rawDir | Out-Null
           Get-ChildItem -Path $rawDir -Force | Remove-Item -Force -Recurse -ErrorAction SilentlyContinue
           # ---- Load latest managed filenames from step 6 output ----
@@ -576,29 +582,32 @@ jobs:
           Write-Host "Downloaded files:"
           $files = Get-ChildItem -Path $rawDir -File | Sort-Object Name
           $files | Select-Object Name, Length | Format-Table -AutoSize
+
       # ==============================
       # 8. Build + Run SQL generator
       # ==============================
       - name: Build SQL Generator
         run: |
           dotnet build SAPSec.sln --configuration Release
+
       - name: Generate SQL Scripts
         run: |
-          dotnet run --configuration Release --project SAPData/SAPData.csproj
+          dotnet run --configuration Release --project Data/SAPSec.SqlGenerator/SAPSec.SqlGenerator.csproj
           echo "Generated SQL scripts."
+
       # ==============================
       # 9. Run ETL via konduit (private DB) WITH RETRY on transient disconnects
       # ==============================
       - name: Run ETL pipeline via konduit (retry on transient DB disconnect)
-        working-directory: SAPData/Sql
+        working-directory: Data/SAPSec.SqlGenerator/Sql
         shell: bash
         run: |
           set -euo pipefail
           echo "Running ETL via konduit (psql) with retry on transient disconnects"
-          ls -la "$GITHUB_WORKSPACE/SAPData/Sql" || true
+          ls -la "$GITHUB_WORKSPACE/Data/SAPSec.SqlGenerator/Sql" || true
           max_attempts=5
           delay=10
-          log_file="$GITHUB_WORKSPACE/SAPData/Sql/psql.log"
+          log_file="$GITHUB_WORKSPACE/Data/SAPSec.SqlGenerator/Sql/psql.log"
           attempt=1
           while [ $attempt -le $max_attempts ]; do
             echo "ETL attempt $attempt/$max_attempts"
@@ -607,7 +616,7 @@ jobs:
               -n "${AKS_NAMESPACE}" \
               -t 28800 \
               -x \
-              -i "$GITHUB_WORKSPACE/SAPData/Sql/run_all.sql" \
+              -i "$GITHUB_WORKSPACE/Data/SAPSec.SqlGenerator/Sql/run_all.sql" \
               "${KONDUIT_APP_NAME}" -- psql 2>&1 | tee "$log_file"
             exit_code=${PIPESTATUS[0]}
             set -e
@@ -632,6 +641,7 @@ jobs:
             echo "ETL failed after $max_attempts attempts."
             exit 2
           fi
+
       # ==============================
       # 10. Create & upload DB backup (used to restore db in review app env)
       # ==============================
@@ -669,4 +679,54 @@ jobs:
             --overwrite true >/dev/null
           echo "Seed backup uploaded:"
           echo "  ${AZURE_STORAGE_CONTAINER}/${LATEST_BLOB}"
-          echo "  ${AZURE_STORAGE_CONTAINER}/${LATEST_BLOB}"
+          echo "  ${AZURE_STORAGE_CONTAINER}/${LATEST_BLOB}"
+
+      # ==============================
+      # 11. Backup target DB (if pipeline was run normally)
+      # ==============================
+      - name: Set environment variables
+        run: |
+          source global_config/${DEPLOY_ENV}.sh
+          tf_vars_file=${TF_VARS_PATH}/${DEPLOY_ENV}.tfvars.json
+          echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+          echo "NAMESPACE=$(jq -r '.namespace' ${tf_vars_file})" >> $GITHUB_ENV
+          echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+          echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_SHORT}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV
+          echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
+          TODAY=$(date +"%F")
+          BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_post_data_pipeline_${TODAY}
+          echo "BACKUP_FILE=${BACKUP_FILE}" >> $GITHUB_ENV
+          echo "KEYVAULT_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-inf-kv" >> $GITHUB_ENV
+
+      - name: Backup ${{ env.DEPLOY_ENV }} postgres (if pipeline was run normally)
+        if: ${{ inputs.environment == 'test' && inputs.review-app-number == '' }}
+        uses: DFE-Digital/github-actions/backup-postgres@master
+        with:
+          storage-account: ${{ env.STORAGE_ACCOUNT_NAME }}
+          resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+          app-name: ${{ env.SERVICE_NAME }}-${{ inputs.environment }}
+          namespace: ${{ env.NAMESPACE }}
+          cluster: ${{ env.CLUSTER }}
+          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          backup-file: ${{ env.BACKUP_FILE }}.sql
+          teams-webhook-url: ${{ secrets.TEAMS_WEBHOOK_URL }}
+          service: ${{ vars.TEAMS_MSG_SERVICE_NAME }}
+
+      # ==============================
+      # 12. Restore DB backup (if pipeline was run to to create a backup for a review app)
+      # ==============================
+      - name: Restore ${{ env.DEPLOY_ENV }} postgres (if pipeline was run to to create a backup for a review app)
+        if: ${{ inputs.environment == 'test' && inputs.review-app-number != '' }}
+        uses: DFE-Digital/github-actions/restore-postgres-backup@master
+        with:
+          storage-account: ${{ env.STORAGE_ACCOUNT_NAME }}
+          resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+          app-name: ${{ env.SERVICE_NAME }}-${{ inputs.environment }}
+          namespace: ${{ env.NAMESPACE }}
+          cluster: ${{ env.CLUSTER }}
+          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          backup-file: ${{ env.BACKUP_FILE }}.sql.gz

Data/SAPSec.Data/Files/Generated/EnglandDestinations.json

@@ -0,0 +1 @@
+[{"Id":"National","AllDest_Boy_Eng_Current_Num":"280895","AllDest_Boy_Eng_Current_Pct":"90.3","AllDest_Boy_Eng_Previous_Num":"278249","AllDest_Boy_Eng_Previous_Pct":"92.5","AllDest_Boy_Eng_Previous2_Num":"273955","AllDest_Boy_Eng_Previous2_Pct":"93.2","AllDest_Dis_Eng_Current_Num":"132612","AllDest_Dis_Eng_Current_Pct":"83.1","AllDest_Dis_Eng_Previous_Num":"133697","AllDest_Dis_Eng_Previous_Pct":"86.2","AllDest_Dis_Eng_Previous2_Num":"133641","AllDest_Dis_Eng_Previous2_Pct":"87.8","AllDest_Grl_Eng_Current_Num":"273531","AllDest_Grl_Eng_Current_Pct":"92.3","AllDest_Grl_Eng_Previous_Num":"269435","AllDest_Grl_Eng_Previous_Pct":"93.8","AllDest_Grl_Eng_Previous2_Num":"266638","AllDest_Grl_Eng_Previous2_Pct":"94.4","AllDest_NDi_Eng_Current_Num":"421814","AllDest_NDi_Eng_Current_Pct":"94.2","AllDest_NDi_Eng_Previous_Num":"413987","AllDest_NDi_Eng_Previous_Pct":"95.6","AllDest_NDi_Eng_Previous2_Num":"406952","AllDest_NDi_Eng_Previous2_Pct":"96.0","AllDest_Tot_Eng_Current_Num":"554426","AllDest_Tot_Eng_Current_Pct":"91.3","AllDest_Tot_Eng_Previous_Num":"547684","AllDest_Tot_Eng_Previous_Pct":"93.1","AllDest_Tot_Eng_Previous2_Num":"540593","AllDest_Tot_Eng_Previous2_Pct":"93.8","Apprentice_Boy_Eng_Current_Num":"12553","Apprentice_Boy_Eng_Current_Pct":"4","Apprentice_Boy_Eng_Previous_Num":"12162","Apprentice_Boy_Eng_Previous_Pct":"4.0","Apprentice_Boy_Eng_Previous2_Num":"12294","Apprentice_Boy_Eng_Previous2_Pct":"4.2","Apprentice_Dis_Eng_Current_Num":"3418","Apprentice_Dis_Eng_Current_Pct":"2.1","Apprentice_Dis_Eng_Previous_Num":"3387","Apprentice_Dis_Eng_Previous_Pct":"2.2","Apprentice_Dis_Eng_Previous2_Num":"3746","Apprentice_Dis_Eng_Previous2_Pct":"2.5","Apprentice_Grl_Eng_Current_Num":"6879","Apprentice_Grl_Eng_Current_Pct":"2.3","Apprentice_Grl_Eng_Previous_Num":"6368","Apprentice_Grl_Eng_Previous_Pct":"2.2","Apprentice_Grl_Eng_Previous2_Num":"6892","Apprentice_Grl_Eng_Previous2_Pct":"2.4","Apprentice_NDi_Eng_Current_Num":"16014","Apprentice_NDi_Eng_Current_Pct":"3.6","Apprentice_NDi_Eng_Previous_Num":"15143","Apprentice_NDi_Eng_Previous_Pct":"3.5","Apprentice_NDi_Eng_Previous2_Num":"15440","Apprentice_NDi_Eng_Previous2_Pct":"3.6","Apprentice_Tot_Eng_Current_Num":"19432","Apprentice_Tot_Eng_Current_Pct":"3.2","Apprentice_Tot_Eng_Previous_Num":"18530","Apprentice_Tot_Eng_Previous_Pct":"3.2","Apprentice_Tot_Eng_Previous2_Num":"19186","Apprentice_Tot_Eng_Previous2_Pct":"3.3","Education_Boy_Eng_Current_Num":"259982","Education_Boy_Eng_Current_Pct":"83.6","Education_Boy_Eng_Previous_Num":"253000","Education_Boy_Eng_Previous_Pct":"84.1","Education_Boy_Eng_Previous2_Num":"249261","Education_Boy_Eng_Previous2_Pct":"84.8","Education_Dis_Eng_Current_Num":"124506","Education_Dis_Eng_Current_Pct":"78","Education_Dis_Eng_Previous_Num":"121694","Education_Dis_Eng_Previous_Pct":"78.5","Education_Dis_Eng_Previous2_Num":"121626","Education_Dis_Eng_Previous2_Pct":"79.9","Education_Grl_Eng_Current_Num":"259470","Education_Grl_Eng_Current_Pct":"87.5","Education_Grl_Eng_Previous_Num":"252769","Education_Grl_Eng_Previous_Pct":"88.0","Education_Grl_Eng_Previous2_Num":"249686","Education_Grl_Eng_Previous2_Pct":"88.4","Education_NDi_Eng_Current_Num":"394946","Education_NDi_Eng_Current_Pct":"88.2","Education_NDi_Eng_Previous_Num":"384075","Education_NDi_Eng_Previous_Pct":"88.7","Education_NDi_Eng_Previous2_Num":"377321","Education_NDi_Eng_Previous2_Pct":"89.0","Education_Tot_Eng_Current_Num":"519452","Education_Tot_Eng_Current_Pct":"85.5","Education_Tot_Eng_Previous_Num":"505769","Education_Tot_Eng_Previous_Pct":"86.0","Education_Tot_Eng_Previous2_Num":"498947","Education_Tot_Eng_Previous2_Pct":"86.6","Employment_Boy_Eng_Current_Num":"8360","Employment_Boy_Eng_Current_Pct":"2.7","Employment_Boy_Eng_Previous_Num":"13087","Employment_Boy_Eng_Previous_Pct":"4.4","Employment_Boy_Eng_Previous2_Num":"12400","Employment_Boy_Eng_Previous2_Pct":"4.2","Employment_Dis_Eng_Current_Num":"4688","Employment_Dis_Eng_Current_Pct":"2.9","Employment_Dis_Eng_Previous_Num":"8616","Employment_Dis_Eng_Previous_Pct":"5.6","Employment_Dis_Eng_Previous2_Num":"8269","Employment_Dis_Eng_Previous2_Pct":"5.4","Employment_Grl_Eng_Current_Num":"7182","Employment_Grl_Eng_Current_Pct":"2.4","Employment_Grl_Eng_Previous_Num":"10298","Employment_Grl_Eng_Previous_Pct":"3.6","Employment_Grl_Eng_Previous2_Num":"10060","Employment_Grl_Eng_Previous2_Pct":"3.6","Employment_NDi_Eng_Current_Num":"10854","Employment_NDi_Eng_Current_Pct":"2.4","Employment_NDi_Eng_Previous_Num":"14769","Employment_NDi_Eng_Previous_Pct":"3.4","Employment_NDi_Eng_Previous2_Num":"14191","Employment_NDi_Eng_Previous2_Pct":"3.3","Employment_Tot_Eng_Current_Num":"15542","Employment_Tot_Eng_Current_Pct":"2.6","Employment_Tot_Eng_Previous_Num":"23385","Employment_Tot_Eng_Previous_Pct":"4.0","Employment_Tot_Eng_Previous2_Num":"22460","Employment_Tot_Eng_Previous2_Pct":"3.9"}]