Skip to content
Strava Engineering and ML deployment

removed devcontainer #67

Sign in to view logs

removed devcontainer

removed devcontainer #67

Workflow file for this run

.github/workflows/strava_project_deployment.yml at cdf1071
name: Strava Engineering and ML deployment
on:
push:
branches:
- 'main'
- 'dev/**'
jobs:
check-training-pipeline:
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/dev/strava_project'
defaults:
run:
working-directory: strava_project/ml
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install dependencies
run: |
python3 -m pip install --upgrade pip
python3 -m pip install -r requirements.txt
- name: Execute pipeline as a check on different steps
run: |
guild run strava:pipeline -y
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
ENV: dev
deploy-training-docker:
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/dev/strava_project'
defaults:
run:
working-directory: strava_project/ml
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Build and push container
run: |
make all
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
IMAGE_URI: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/strava:latest
ECR_REPO: strava
run-sagemaker-jobs-offline:
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/dev/strava_project'
defaults:
run:
working-directory: strava_project/ml
needs: [check-training-pipeline, deploy-training-docker]
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install dependencies
run: |
python3 -m pip install --upgrade pip
python3 -m pip install -U sagemaker
- name: Run sagemaker jobs offline
run: |
python orchestrate_pipeline.py --execute-local-instance True
python orchestrate_fitness_training_pipeline.py --execute-local-instance True
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
SAGEMAKER_EXECUTION_ROLE: ${{ secrets.SAGEMAKER_EXECUTION_ROLE }}
AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
IMAGE_URI: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/strava:latest
ENV: dev
deploy-engineering-stack:
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/dev/strava_project'
defaults:
run:
working-directory: strava_project/engineering
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Set up node
uses: actions/setup-node@v3
with:
node-version: '18'
- name: Install CDK and Python dependencies
run: |
npm install -g aws-cdk
python3 -m pip install --upgrade pip
python3 -m pip install -Ur requirements.txt
- name: Deploy engineering stack
run: |
cdk deploy DataEngineeringStack --require-approval=never
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
SAGEMAKER_EXECUTION_ROLE: ${{ secrets.SAGEMAKER_EXECUTION_ROLE }}
IMAGE_URI: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/strava:latest
STRAVA_CLIENT_ID: ${{ secrets.STRAVA_CLIENT_ID }}
STRAVA_CLIENT_SECRET: ${{ secrets.STRAVA_CLIENT_SECRET }}
STRAVA_ACCESS_TOKEN: ${{ secrets.STRAVA_ACCESS_TOKEN }}
STRAVA_REFRESH_TOKEN: ${{ secrets.STRAVA_REFRESH_TOKEN }}
EXPIRES_AT: ${{ secrets.EXPIRES_AT }}
ENV: dev
deploy-ml-stack:
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/dev/strava_project'
defaults:
run:
working-directory: strava_project/ml
needs: [run-sagemaker-jobs-offline]
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Set up node
uses: actions/setup-node@v3
with:
node-version: '18'
- name: Install CDK and Python dependencies
run: |
npm install -g aws-cdk
python3 -m pip install --upgrade pip
python3 -m pip install -Ur requirements.txt
- name: Deploy ml stacks
run: |
python orchestrate_pipeline.py --execute-local-instance False
python orchestrate_fitness_training_pipeline.py --execute-local-instance False
cdk deploy ModelDeployerStack --require-approval=never
cdk deploy StravaTrainProdPipelineStack --require-approval=never
cdk deploy FitnessTrainProdPipelineStack --require-approval=never
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
SAGEMAKER_EXECUTION_ROLE: ${{ secrets.SAGEMAKER_EXECUTION_ROLE }}
IMAGE_URI: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-west-1.amazonaws.com/strava:latest
STRAVA_CLIENT_ID: ${{ secrets.STRAVA_CLIENT_ID }}
STRAVA_CLIENT_SECRET: ${{ secrets.STRAVA_CLIENT_SECRET }}
STRAVA_ACCESS_TOKEN: ${{ secrets.STRAVA_ACCESS_TOKEN }}
STRAVA_REFRESH_TOKEN: ${{ secrets.STRAVA_REFRESH_TOKEN }}
EXPIRES_AT: 1680976646
ENV: dev