data_science/.github/workflows/analytics_deployment.yml at main · prteek/data_science · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
name: Automated deployment

on:
  push:
    branches:
      - 'main'
      - 'dev/**'

jobs:

  dbt_deployment:
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/dev/dbt'
    defaults:
      run:
        working-directory: ./dbt

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.12'

      - name: Set up node
        uses: actions/setup-node@v3
        with:
          node-version: '18'

      - name: Install CDK and Python dependencies
        run: |
          npm install -g aws-cdk
          python -m pip install --upgrade pip
          python -m pip install aws-cdk-lib constructs boto3


      - name: Build and push docker
        run: |
          make all
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
          ENV: dev
          IMAGE_URI: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/dbt
          SAGEMAKER_EXECUTION_ROLE: ${{ secrets.SAGEMAKER_EXECUTION_ROLE }}
          GITHUB_USER: "sagemaker-dbt"
          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}

      - name: Deploy run-dbt lambda
        run: |
          cdk deploy RunDbt --require-approval never
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
          ENV: dev
          IMAGE_URI: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/dbt
          SAGEMAKER_EXECUTION_ROLE: ${{ secrets.SAGEMAKER_EXECUTION_ROLE }}
          GITHUB_USER: "sagemaker-dbt"
          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}


  model_triggers_deployment:
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/dev/analytics'
    defaults:
      run:
        working-directory: ./analytics

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.12'

      - name: Set up node
        uses: actions/setup-node@v3
        with:
          node-version: '18'

      - name: Install CDK and Python dependencies
        run: |
          npm install -g aws-cdk
          python -m pip install --upgrade pip
          python -m pip install aws-cdk-lib constructs boto3


      - name: Deploy model trigger rules
        run: |
          cdk deploy ModelTriggers --require-approval never
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
          ENV: dev