update run.sh for all datasets and and publish action

Author: eveleighoj

.github/workflows/deploy.yml

@@ -0,0 +1,62 @@
+name: Publish
+
+on:
+  push:
+  workflow_dispatch:
+    inputs:
+      environment:
+        type: environment
+        description: The environment to deploy to.
+
+jobs:
+  detect-environments:
+    runs-on: ubuntu-latest
+    outputs:
+      environments: ${{ steps.environments.outputs.result }}
+    steps:
+      - uses: actions/github-script@v6
+        id: environments
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-encoding: json
+          script: |
+            if (context.payload?.inputs?.environment) return [context.payload?.inputs?.environment];
+            const {data: {environments}} =
+              await github.request(`GET /repos/${process.env.GITHUB_REPOSITORY}/environments`);
+            return environments.map(e => e.name)
+
+  deploy:
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_REPO: ${{ secrets.DEPLOY_DOCKER_REPOSITORY }}
+    needs: [detect-environments]
+    strategy:
+      matrix:
+        environment: ${{ fromJSON(needs.detect-environments.outputs.environments) }}
+    environment: ${{ matrix.environment }}
+    if: github.ref_name == 'main'
+    steps:
+      - uses: actions/checkout@v4
+
+      - id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          aws-access-key-id: ${{ secrets.DEPLOY_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.DEPLOY_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - uses: aws-actions/amazon-ecr-login@v2
+        with:
+          registry-type: public
+
+      - run: docker pull $DOCKER_REPO:${GITHUB_REF_NAME} || echo "no current latest image"
+
+      - run: docker build -t $DOCKER_REPO:${{ steps.vars.outputs.sha_short }} .
+
+      - run: docker tag $DOCKER_REPO:${{ steps.vars.outputs.sha_short }} $DOCKER_REPO:${GITHUB_REF_NAME}
+
+      - run: docker push $DOCKER_REPO:${{ steps.vars.outputs.sha_short }}
+
+      - run: docker push $DOCKER_REPO:${GITHUB_REF_NAME}

run.sh

@@ -1,7 +1,4 @@
 #!/bin/bash
 set -e
 
-make data/reporting/duplicate_entity_expectation.csv
-
-
-
+make all -j 8