From 72563094b005d13720295b06ca972581a1217cee Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 17:25:55 -0500 Subject: [PATCH 01/34] minor: fix variable name --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c0afd67..9e5c7a9 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -10,7 +10,7 @@ env: AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - ECR_REPO: ${{ secrets.ECR_REPO }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} CONTAINER_NAME: "pickem" ECS_SERVICE: PickemLiveStats ECS_CLUSTER: PickemLiveStats-Dev From daeb998fa605b811cbcd16859c07d9a14ebcd471 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 17:28:05 -0500 Subject: [PATCH 02/34] minor: fix branch deployment - Adds a complete copy but for production --- .../workflows/{deploy.yml => deploy_dev.yml} | 2 +- .github/workflows/deploy_prod.yml | 69 +++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) rename .github/workflows/{deploy.yml => deploy_dev.yml} (99%) create mode 100644 .github/workflows/deploy_prod.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy_dev.yml similarity index 99% rename from .github/workflows/deploy.yml rename to .github/workflows/deploy_dev.yml index 9e5c7a9..440c895 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy_dev.yml @@ -3,7 +3,7 @@ run-name: Run ${{ github.run_id }} deploying to ECR on: push: branches: - - main + - dev env: AWS_REGION: us-east-1 diff --git a/.github/workflows/deploy_prod.yml b/.github/workflows/deploy_prod.yml new file mode 100644 index 0000000..3d80407 --- /dev/null +++ b/.github/workflows/deploy_prod.yml @@ -0,0 +1,69 @@ + +name: Deploy to ECR +run-name: Run ${{ github.run_id }} deploying to ECR +on: + push: + branches: + - main + +env: + AWS_REGION: us-east-1 + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} + CONTAINER_NAME: "pickem" + ECS_SERVICE: PickemLiveStats + ECS_CLUSTER: PickemLiveStats-Prod + + +jobs: + deploy: + name: Deploy to ECR + runs-on: ubuntu-latest + environment: development + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build, tag, and push image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: ${{ secrets.ECR_REPO }} + IMAGE_TAG: ${{ github.sha }} + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT + + - name: Fill new image ID in Amazon ECS task definition + id: fill-task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: .aws/task-definition.json + container-name: my-container + image: ${{ steps.build-image.outputs.image }} + + + - name: Deploy ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + with: + task-definition: ${{ steps.fill-task-def.outputs.new-task-definition }} + service: my-service + cluster: my-cluster + wait-for-service-stability: true + + From 4e6863e7627d41368b380a39507adf4b5ca2ec18 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 17:29:48 -0500 Subject: [PATCH 03/34] minor: Fix repository variable name --- .github/workflows/deploy_dev.yml | 2 +- .github/workflows/deploy_prod.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy_dev.yml b/.github/workflows/deploy_dev.yml index 440c895..6459dbe 100644 --- a/.github/workflows/deploy_dev.yml +++ b/.github/workflows/deploy_dev.yml @@ -41,7 +41,7 @@ jobs: id: build-image env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: ${{ secrets.ECR_REPO }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} IMAGE_TAG: ${{ github.sha }} run: | docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . diff --git a/.github/workflows/deploy_prod.yml b/.github/workflows/deploy_prod.yml index 3d80407..1aeded6 100644 --- a/.github/workflows/deploy_prod.yml +++ b/.github/workflows/deploy_prod.yml @@ -42,7 +42,7 @@ jobs: id: build-image env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: ${{ secrets.ECR_REPO }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} IMAGE_TAG: ${{ github.sha }} run: | docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . From 49a12b310dc4f8e889182df0d171ec47765e6ffe Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 17:38:24 -0500 Subject: [PATCH 04/34] minor: Fix ECS variable names --- .github/workflows/deploy_dev.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy_dev.yml b/.github/workflows/deploy_dev.yml index 6459dbe..347caa2 100644 --- a/.github/workflows/deploy_dev.yml +++ b/.github/workflows/deploy_dev.yml @@ -53,7 +53,7 @@ jobs: uses: aws-actions/amazon-ecs-render-task-definition@v1 with: task-definition: .aws/task-definition.json - container-name: my-container + container-name: ${{ env.CONTAINER_NAME }} image: ${{ steps.build-image.outputs.image }} @@ -61,8 +61,8 @@ jobs: uses: aws-actions/amazon-ecs-deploy-task-definition@v1 with: task-definition: ${{ steps.fill-task-def.outputs.new-task-definition }} - service: my-service - cluster: my-cluster + service: ${{ env.ECS_SERVICE }} + cluster: ${{ env.ECS_CLUSTER }} wait-for-service-stability: true From 1482380a7d81e4f82d4ddf96e3a62dd117b70092 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 17:40:43 -0500 Subject: [PATCH 05/34] minor: fix return variable --- .github/workflows/deploy_dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_dev.yml b/.github/workflows/deploy_dev.yml index 347caa2..048e9b4 100644 --- a/.github/workflows/deploy_dev.yml +++ b/.github/workflows/deploy_dev.yml @@ -60,7 +60,7 @@ jobs: - name: Deploy ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v1 with: - task-definition: ${{ steps.fill-task-def.outputs.new-task-definition }} + task-definition: ${{ steps.fill-task-def.outputs.task-definition }} service: ${{ env.ECS_SERVICE }} cluster: ${{ env.ECS_CLUSTER }} wait-for-service-stability: true From 2f8b31a62fba4481f8d228af29087eab5aa1b898 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 19:22:07 -0500 Subject: [PATCH 06/34] minor: change task definition for .env variables --- .aws/task-definition.json | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.aws/task-definition.json b/.aws/task-definition.json index d6d3e14..19564d7 100644 --- a/.aws/task-definition.json +++ b/.aws/task-definition.json @@ -8,11 +8,13 @@ "portMappings": [], "essential": true, "environment": [], - "environmentFiles": [ - { - "value": "arn:aws:s3:::pickem-environment-bucket/dev-livestats.env", - "type": "s3" - } + "secrets": [{ + "name": "DATABASE_URL", + "valueFrom": "arn:aws:secretsmanager:us-east-1:798380260115:secret:PickemLiveStats/dev-BfVBxS" + }, { + "name": "REDIS_URL", + "valueFrom": "arn:aws:secretsmanager:us-east-1:798380260115:secret:PickemLiveStats/dev-BfVBxS" + } ], "mountPoints": [], "volumesFrom": [], From 6fa1c74f7faf927712db09ee82e289e4116d8595 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 19:53:15 -0500 Subject: [PATCH 07/34] minor: update task definition for cpus, log configuration --- .aws/task-definition.json | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.aws/task-definition.json b/.aws/task-definition.json index 19564d7..4d174e2 100644 --- a/.aws/task-definition.json +++ b/.aws/task-definition.json @@ -4,7 +4,8 @@ { "name": "pickem", "image": "798380260115.dkr.ecr.us-east-1.amazonaws.com/pickemlivestats", - "cpu": 0, + "cpu": 0.25, + "memory": 512, "portMappings": [], "essential": true, "environment": [], @@ -18,7 +19,16 @@ ], "mountPoints": [], "volumesFrom": [], - "systemControls": [] + "systemControls": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-create-group": "true", + "awslogs-group": "awslogs-wordpress", + "awslogs-region": "us-west-2", + "awslogs-stream-prefix": "awslogs-example" + } + } } ], "family": "PickemLiveStats", From a2d3fe8510b9b0e1ebe40dbdcc73ce3630384dff Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 20:01:13 -0500 Subject: [PATCH 08/34] minor: add environment variables in workflow, aws logs group --- .aws/task-definition.json | 4 ++-- .github/workflows/deploy_dev.yml | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.aws/task-definition.json b/.aws/task-definition.json index 4d174e2..94681e8 100644 --- a/.aws/task-definition.json +++ b/.aws/task-definition.json @@ -24,9 +24,9 @@ "logDriver": "awslogs", "options": { "awslogs-create-group": "true", - "awslogs-group": "awslogs-wordpress", + "awslogs-group": "pickem-livestats-dev", "awslogs-region": "us-west-2", - "awslogs-stream-prefix": "awslogs-example" + "awslogs-stream-prefix": "pickem-livestats-dev" } } } diff --git a/.github/workflows/deploy_dev.yml b/.github/workflows/deploy_dev.yml index 048e9b4..ea4f2f3 100644 --- a/.github/workflows/deploy_dev.yml +++ b/.github/workflows/deploy_dev.yml @@ -11,7 +11,9 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} - CONTAINER_NAME: "pickem" + DATABASE_URL: ${{ secrets.DATABASE_URL }} + REDIS_URL: ${{ secrets.REDIS_URL }} + CONTAINER_NAME: "pickem-livestats" ECS_SERVICE: PickemLiveStats ECS_CLUSTER: PickemLiveStats-Dev @@ -43,8 +45,10 @@ jobs: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} IMAGE_TAG: ${{ github.sha }} + REDIS_URL: ${{ secrets.REDIS_URL }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . -e REDIS_URL=$REDIS_URL DATABASE_URL=$DATABASE_URL docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT From decb214f21120c9a97ca20166d9f363a518936e8 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 20:03:15 -0500 Subject: [PATCH 09/34] minor: remove env variables --- .github/workflows/deploy_dev.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/deploy_dev.yml b/.github/workflows/deploy_dev.yml index ea4f2f3..83567f1 100644 --- a/.github/workflows/deploy_dev.yml +++ b/.github/workflows/deploy_dev.yml @@ -45,10 +45,8 @@ jobs: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} IMAGE_TAG: ${{ github.sha }} - REDIS_URL: ${{ secrets.REDIS_URL }} - DATABASE_URL: ${{ secrets.DATABASE_URL }} run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . -e REDIS_URL=$REDIS_URL DATABASE_URL=$DATABASE_URL + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT From 756a503ece57d0980e7709b8b2b8ae19385f23ec Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 20:06:09 -0500 Subject: [PATCH 10/34] minor: change container name --- .github/workflows/deploy_dev.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/deploy_dev.yml b/.github/workflows/deploy_dev.yml index 83567f1..048e9b4 100644 --- a/.github/workflows/deploy_dev.yml +++ b/.github/workflows/deploy_dev.yml @@ -11,9 +11,7 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} - DATABASE_URL: ${{ secrets.DATABASE_URL }} - REDIS_URL: ${{ secrets.REDIS_URL }} - CONTAINER_NAME: "pickem-livestats" + CONTAINER_NAME: "pickem" ECS_SERVICE: PickemLiveStats ECS_CLUSTER: PickemLiveStats-Dev From 5de4b0690a72a92f6ba5d2420a2936209f695509 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 20:31:40 -0500 Subject: [PATCH 11/34] minor: change container name --- .aws/task-definition.json | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/.aws/task-definition.json b/.aws/task-definition.json index 94681e8..d6d3e14 100644 --- a/.aws/task-definition.json +++ b/.aws/task-definition.json @@ -4,31 +4,19 @@ { "name": "pickem", "image": "798380260115.dkr.ecr.us-east-1.amazonaws.com/pickemlivestats", - "cpu": 0.25, - "memory": 512, + "cpu": 0, "portMappings": [], "essential": true, "environment": [], - "secrets": [{ - "name": "DATABASE_URL", - "valueFrom": "arn:aws:secretsmanager:us-east-1:798380260115:secret:PickemLiveStats/dev-BfVBxS" - }, { - "name": "REDIS_URL", - "valueFrom": "arn:aws:secretsmanager:us-east-1:798380260115:secret:PickemLiveStats/dev-BfVBxS" - } + "environmentFiles": [ + { + "value": "arn:aws:s3:::pickem-environment-bucket/dev-livestats.env", + "type": "s3" + } ], "mountPoints": [], "volumesFrom": [], - "systemControls": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-create-group": "true", - "awslogs-group": "pickem-livestats-dev", - "awslogs-region": "us-west-2", - "awslogs-stream-prefix": "pickem-livestats-dev" - } - } + "systemControls": [] } ], "family": "PickemLiveStats", From 76f49f5ce84fceef02156e5cb232a907160098ed Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 21:12:33 -0500 Subject: [PATCH 12/34] minor: ignore if environment variables were not retrieved --- database.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/database.go b/database.go index a340355..400ea23 100644 --- a/database.go +++ b/database.go @@ -18,11 +18,11 @@ type DatabaseClient struct { } func NewDatabaseClient() *DatabaseClient { - dotEnvErr := godotenv.Load() - if dotEnvErr != nil { - slog.Error("Error loading .env file, exiting") - os.Exit(1) - } + godotenv.Load() + //if dotEnvErr != nil { + // slog.Error("Error loading .env file, exiting") + // os.Exit(1) + //} // Connect to database connStr := os.Getenv("DATABASE_URL") From a0a403f10dd7d2feeba59917cac601dd751647a4 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 21:13:07 -0500 Subject: [PATCH 13/34] minor: update task definition --- .aws/task-definition.json | 54 +++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/.aws/task-definition.json b/.aws/task-definition.json index d6d3e14..dedeeb6 100644 --- a/.aws/task-definition.json +++ b/.aws/task-definition.json @@ -1,36 +1,65 @@ { - "taskDefinitionArn": "arn:aws:ecs:us-east-1:798380260115:task-definition/PickemLiveStats:4", + "taskDefinitionArn": "arn:aws:ecs:us-east-1:798380260115:task-definition/PickemLiveStats:19", "containerDefinitions": [ { "name": "pickem", - "image": "798380260115.dkr.ecr.us-east-1.amazonaws.com/pickemlivestats", - "cpu": 0, + "image": "798380260115.dkr.ecr.us-east-1.amazonaws.com/pickemlivestats:5de4b0690a72a92f6ba5d2420a2936209f695509", + "cpu": 256, + "memory": 512, "portMappings": [], "essential": true, "environment": [], - "environmentFiles": [ + "environmentFiles": [], + "mountPoints": [], + "volumesFrom": [], + "secrets": [ { - "value": "arn:aws:s3:::pickem-environment-bucket/dev-livestats.env", - "type": "s3" + "name": "DATABASE_URL", + "valueFrom": "arn:aws:secretsmanager:us-east-1:798380260115:secret:PickemLiveStats/dev-BfVBxS:DATABASE_URL::" + }, + { + "name": "REDIS_URL", + "valueFrom": "arn:aws:secretsmanager:us-east-1:798380260115:secret:PickemLiveStats/dev-BfVBxS:REDIS_URL::" } ], - "mountPoints": [], - "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-create-group": "true", + "awslogs-group": "pickem-livestats-dev", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "dev-stats" + }, + "secretOptions": [] + }, "systemControls": [] } ], "family": "PickemLiveStats", + "taskRoleArn": "arn:aws:iam::798380260115:role/ecsTaskExecutionRole", "executionRoleArn": "arn:aws:iam::798380260115:role/ecsTaskExecutionRole", "networkMode": "awsvpc", - "revision": 4, + "revision": 19, "volumes": [], "status": "ACTIVE", "requiresAttributes": [ + { + "name": "com.amazonaws.ecs.capability.logging-driver.awslogs" + }, + { + "name": "ecs.capability.execution-role-awslogs" + }, { "name": "com.amazonaws.ecs.capability.ecr-auth" }, { - "name": "ecs.capability.env-files.s3" + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19" + }, + { + "name": "ecs.capability.secrets.asm.environment-variables" + }, + { + "name": "com.amazonaws.ecs.capability.task-iam-role" }, { "name": "ecs.capability.execution-role-ecr-pull" @@ -40,6 +69,9 @@ }, { "name": "ecs.capability.task-eni" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.29" } ], "placementConstraints": [], @@ -56,7 +88,7 @@ "cpuArchitecture": "X86_64", "operatingSystemFamily": "LINUX" }, - "registeredAt": "2024-03-06T22:12:24.591Z", + "registeredAt": "2024-03-07T02:09:27.377Z", "registeredBy": "arn:aws:iam::798380260115:root", "tags": [] } \ No newline at end of file From c7c6c8effa1efe3f1803bbdeebd0f7ef5aed0fc9 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 22:12:16 -0500 Subject: [PATCH 14/34] feat: Remove deploying task definition from ECS - We only want to actually create a new task definition to update the latest -- it will automatically be deployed by EventBridge Scheduler every time, therefore there is no need to launch a service. --- .github/workflows/deploy_dev.yml | 11 ----------- .github/workflows/deploy_prod.yml | 13 ------------- 2 files changed, 24 deletions(-) diff --git a/.github/workflows/deploy_dev.yml b/.github/workflows/deploy_dev.yml index 048e9b4..7280d22 100644 --- a/.github/workflows/deploy_dev.yml +++ b/.github/workflows/deploy_dev.yml @@ -12,8 +12,6 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} CONTAINER_NAME: "pickem" - ECS_SERVICE: PickemLiveStats - ECS_CLUSTER: PickemLiveStats-Dev jobs: @@ -57,12 +55,3 @@ jobs: image: ${{ steps.build-image.outputs.image }} - - name: Deploy ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 - with: - task-definition: ${{ steps.fill-task-def.outputs.task-definition }} - service: ${{ env.ECS_SERVICE }} - cluster: ${{ env.ECS_CLUSTER }} - wait-for-service-stability: true - - diff --git a/.github/workflows/deploy_prod.yml b/.github/workflows/deploy_prod.yml index 1aeded6..879cc29 100644 --- a/.github/workflows/deploy_prod.yml +++ b/.github/workflows/deploy_prod.yml @@ -13,8 +13,6 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} CONTAINER_NAME: "pickem" - ECS_SERVICE: PickemLiveStats - ECS_CLUSTER: PickemLiveStats-Prod jobs: @@ -56,14 +54,3 @@ jobs: task-definition: .aws/task-definition.json container-name: my-container image: ${{ steps.build-image.outputs.image }} - - - - name: Deploy ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 - with: - task-definition: ${{ steps.fill-task-def.outputs.new-task-definition }} - service: my-service - cluster: my-cluster - wait-for-service-stability: true - - From 1ffc27139974f498cae1fa370a26c25699e5354b Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 22:18:04 -0500 Subject: [PATCH 15/34] feat: Add deploying task definition from ECS, but for scheduled tasks only - We only want to actually create a new task definition to update the latest -- it will automatically be deployed by EventBridge Scheduler every time, therefore there is no need to launch a service. --- .github/workflows/deploy_dev.yml | 7 ++++++- .github/workflows/deploy_prod.yml | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_dev.yml b/.github/workflows/deploy_dev.yml index 7280d22..cf69276 100644 --- a/.github/workflows/deploy_dev.yml +++ b/.github/workflows/deploy_dev.yml @@ -12,6 +12,7 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} CONTAINER_NAME: "pickem" + ECS_CLUSTER: PickemLiveStats-Dev jobs: @@ -54,4 +55,8 @@ jobs: container-name: ${{ env.CONTAINER_NAME }} image: ${{ steps.build-image.outputs.image }} - + - name: Deploy to Amazon ECS + uses: airfordable/ecs-deploy-task-definition-to-scheduled-task@v2.0.0 + with: + cluster: ${{ env.ECS_CLUSTER }} + task-definition: ${{ steps.fill-task-def.outputs.task-definition }} diff --git a/.github/workflows/deploy_prod.yml b/.github/workflows/deploy_prod.yml index 879cc29..e55025c 100644 --- a/.github/workflows/deploy_prod.yml +++ b/.github/workflows/deploy_prod.yml @@ -13,6 +13,7 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} CONTAINER_NAME: "pickem" + ECS_CLUSTER: PickemLiveStats-Prod jobs: From 32ec6a4b06e7155c15b52f69c5b4bc1b6a115fc8 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 6 Mar 2024 22:21:28 -0500 Subject: [PATCH 16/34] minor: Removing faulty & unneeded fields from the task definition --- .aws/task-definition.json | 45 --------------------------------------- 1 file changed, 45 deletions(-) diff --git a/.aws/task-definition.json b/.aws/task-definition.json index dedeeb6..f8c35a6 100644 --- a/.aws/task-definition.json +++ b/.aws/task-definition.json @@ -1,5 +1,4 @@ { - "taskDefinitionArn": "arn:aws:ecs:us-east-1:798380260115:task-definition/PickemLiveStats:19", "containerDefinitions": [ { "name": "pickem", @@ -39,56 +38,12 @@ "taskRoleArn": "arn:aws:iam::798380260115:role/ecsTaskExecutionRole", "executionRoleArn": "arn:aws:iam::798380260115:role/ecsTaskExecutionRole", "networkMode": "awsvpc", - "revision": 19, "volumes": [], - "status": "ACTIVE", - "requiresAttributes": [ - { - "name": "com.amazonaws.ecs.capability.logging-driver.awslogs" - }, - { - "name": "ecs.capability.execution-role-awslogs" - }, - { - "name": "com.amazonaws.ecs.capability.ecr-auth" - }, - { - "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19" - }, - { - "name": "ecs.capability.secrets.asm.environment-variables" - }, - { - "name": "com.amazonaws.ecs.capability.task-iam-role" - }, - { - "name": "ecs.capability.execution-role-ecr-pull" - }, - { - "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18" - }, - { - "name": "ecs.capability.task-eni" - }, - { - "name": "com.amazonaws.ecs.capability.docker-remote-api.1.29" - } - ], "placementConstraints": [], - "compatibilities": [ - "EC2", - "FARGATE" - ], "requiresCompatibilities": [ "FARGATE" ], "cpu": "256", "memory": "512", - "runtimePlatform": { - "cpuArchitecture": "X86_64", - "operatingSystemFamily": "LINUX" - }, - "registeredAt": "2024-03-07T02:09:27.377Z", - "registeredBy": "arn:aws:iam::798380260115:root", "tags": [] } \ No newline at end of file From 0b204a33c436492994dccd72013e4832f5578282 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Thu, 7 Mar 2024 16:42:29 -0500 Subject: [PATCH 17/34] feat: Add handling for games of unknown type - Adds handling for games of unknown type - Adds code for "preview" or "pre-game" status TODO: check if postponed or cancelled, and do accordingly --- stats.go | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/stats.go b/stats.go index e941fd8..3625c21 100644 --- a/stats.go +++ b/stats.go @@ -45,8 +45,8 @@ type CompletedGameStats struct { } type UnknownGameStats struct { - status string - gameID int + status string `redis:"status"` + gameID int `redis:"gameID"` } // Returns true or false if the game that was just handled was finished. @@ -78,6 +78,9 @@ func handleGameStats(gameID int, dbClient *DatabaseClient) (bool, error) { case InProgress: err := handleInProgressGame(gameStats, liveStats, dbClient) return false, err + case Unknown: + err := handleUnknownGame(gameStats, dbClient) + return true, err } return false, nil @@ -96,6 +99,8 @@ func getGameType(gameStats map[string]interface{}) (string, error) { switch code { case "S": // Scheduled return Scheduled, nil + case "P": + return Scheduled, nil // Pregame case "PW": // Warmup situation return Scheduled, nil case "F": // Final @@ -254,6 +259,29 @@ func handleInProgressGame(gameStats map[string]interface{}, liveStats map[string } +func handleUnknownGame(gameStats map[string]interface{}, client *DatabaseClient) error { + slog.Info("Game status unknown, setting in database and skipping") + + gameID, gameIDErr := unwrap(gameStats, "game") + if gameIDErr != nil { + return gameIDErr + } + unknownGameInfo := UnknownGameStats{ + status: Unknown, + gameID: int(gameID["pk"].(float64)), + } + + client.redisMut.Lock() + defer client.redisMut.Unlock() + hsetErr := client.redisClient.HSet(context.Background(), "game:"+strconv.Itoa(unknownGameInfo.gameID), unknownGameInfo).Err() + if hsetErr != nil { + return hsetErr + } + client.redisMut.Unlock() + + return nil +} + func getScores(liveData map[string]interface{}) (int, int, error) { lineScore, lineScoreErr := unwrap(liveData, "linescore") if lineScoreErr != nil { From 8f41fffd28f57e1acac645ca127079770ea9af38 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Thu, 7 Mar 2024 16:46:56 -0500 Subject: [PATCH 18/34] feat: Add handling for postponed/cancelled games - Adds code specifically for games that are postponed or cancelled, due to rain or other circumstances --- stats.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stats.go b/stats.go index 3625c21..13d3bdf 100644 --- a/stats.go +++ b/stats.go @@ -103,6 +103,8 @@ func getGameType(gameStats map[string]interface{}) (string, error) { return Scheduled, nil // Pregame case "PW": // Warmup situation return Scheduled, nil + case "DR": + return Completed, nil // Delayed/Postponed case "F": // Final return Completed, nil case "O": // Game Over (used as separate before decisions) @@ -277,7 +279,6 @@ func handleUnknownGame(gameStats map[string]interface{}, client *DatabaseClient) if hsetErr != nil { return hsetErr } - client.redisMut.Unlock() return nil } From 662d0c135c9e78846642bb44f8fa6261f725568d Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Fri, 8 Mar 2024 13:00:05 -0500 Subject: [PATCH 19/34] feat: Add handling for postponed/cancelled games - Adds code specifically for games that are postponed or cancelled, due to rain or other circumstances - Set homeScore and other information to null if not present --- stats.go | 60 +++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/stats.go b/stats.go index 13d3bdf..e641674 100644 --- a/stats.go +++ b/stats.go @@ -13,6 +13,7 @@ const ( Scheduled = "SCHEDULED" InProgress = "IN_PROGRESS" Completed = "COMPLETED" + Postponed = "POSTPONED" Unknown = "UNKNOWN" ) @@ -49,6 +50,13 @@ type UnknownGameStats struct { gameID int `redis:"gameID"` } +type PostponedGameStats struct { + Status string `redis:"status"` + GameID int `redis:"gameID"` + HomeScore int `redis:"homeScore"` + AwayScore int `redis:"awayScore"` +} + // Returns true or false if the game that was just handled was finished. func handleGameStats(gameID int, dbClient *DatabaseClient) (bool, error) { gameResp, err := getGameData(gameID) @@ -79,7 +87,7 @@ func handleGameStats(gameID int, dbClient *DatabaseClient) (bool, error) { err := handleInProgressGame(gameStats, liveStats, dbClient) return false, err case Unknown: - err := handleUnknownGame(gameStats, dbClient) + err := handlePostponedGame(gameStats, liveStats, dbClient) return true, err } @@ -104,7 +112,7 @@ func getGameType(gameStats map[string]interface{}) (string, error) { case "PW": // Warmup situation return Scheduled, nil case "DR": - return Completed, nil // Delayed/Postponed + return Postponed, nil // Delayed/Postponed case "F": // Final return Completed, nil case "O": // Game Over (used as separate before decisions) @@ -261,25 +269,63 @@ func handleInProgressGame(gameStats map[string]interface{}, liveStats map[string } -func handleUnknownGame(gameStats map[string]interface{}, client *DatabaseClient) error { +func handlePostponedGame(gameStats map[string]interface{}, liveStats map[string]interface{}, client *DatabaseClient) error { slog.Info("Game status unknown, setting in database and skipping") gameID, gameIDErr := unwrap(gameStats, "game") if gameIDErr != nil { return gameIDErr } - unknownGameInfo := UnknownGameStats{ - status: Unknown, - gameID: int(gameID["pk"].(float64)), + + homeScore, awayScore, scoreErr := getScores(liveStats) + if scoreErr != nil { + return scoreErr + } + + stats := PostponedGameStats{ + Status: Postponed, + GameID: int(gameID["pk"].(float64)), + HomeScore: homeScore, + AwayScore: awayScore, } client.redisMut.Lock() defer client.redisMut.Unlock() - hsetErr := client.redisClient.HSet(context.Background(), "game:"+strconv.Itoa(unknownGameInfo.gameID), unknownGameInfo).Err() + hsetErr := client.redisClient.HSet(context.Background(), "game:"+strconv.Itoa(stats.GameID), stats).Err() if hsetErr != nil { return hsetErr } + client.dbMut.Lock() + defer client.dbMut.Unlock() + cctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + tx, err := client.db.Begin(cctx) + + defer cancel() + if err != nil { + return err + } + + defer func() { + if err != nil { + tx.Rollback(context.Background()) + } else { + tx.Commit(context.Background()) + } + }() + + _, err = client.db.Exec(cctx, ` + UPDATE games SET finished=$1, home_score=$2, away_score=$3, winner = NULL + WHERE id = $4`, + true, stats.HomeScore, stats.AwayScore, stats.GameID) + + if err != nil { + return err + } + + err = tx.Commit(context.Background()) + return err + return nil } From 0f7d23b60c3f617387aad710def0b497a9c844ed Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Fri, 8 Mar 2024 20:02:38 -0500 Subject: [PATCH 20/34] feat: Cover all possible game codes for MLB - Redo getGameType to cover all possible prefixes of MLB information --- stats.go | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/stats.go b/stats.go index e641674..1afa2a3 100644 --- a/stats.go +++ b/stats.go @@ -102,23 +102,29 @@ func getGameType(gameStats map[string]interface{}) (string, error) { return "", gameStatusErr } - code := gameStatus["statusCode"].(string) + code := gameStatus["statusCode"].(string)[0] - switch code { + switch strconv.Itoa(int(code)) { case "S": // Scheduled - return Scheduled, nil + return Scheduled, nil // Scheduled case "P": - return Scheduled, nil // Pregame - case "PW": // Warmup situation - return Scheduled, nil - case "DR": - return Postponed, nil // Delayed/Postponed - case "F": // Final - return Completed, nil - case "O": // Game Over (used as separate before decisions) - return Completed, nil - case "I": // In Progress - return InProgress, nil + return Scheduled, nil // Pregame -- also accounts for delayed start + case "I": + return InProgress, nil // In Progress -- also accounts for delayed in progress + case "M": + return InProgress, nil // Manager challenge -- In progress + case "N": + return InProgress, nil // Umpire review + case "D": + return Postponed, nil // Postponed/Cancelled + case "T": + return Completed, nil // Suspended + case "Q": + return Completed, nil // Forfeit -- but is anyone ever going to use this? + case "O": + return Completed, nil // Game Over (or completed early) + case "F": + return Completed, nil // Final default: return Unknown, nil } From 3872e5b3151d590f85e663e20707e245a06e9ee6 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Fri, 8 Mar 2024 20:50:06 -0500 Subject: [PATCH 21/34] fix: First letter of code --- stats.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stats.go b/stats.go index 1afa2a3..14b162c 100644 --- a/stats.go +++ b/stats.go @@ -102,9 +102,10 @@ func getGameType(gameStats map[string]interface{}) (string, error) { return "", gameStatusErr } - code := gameStatus["statusCode"].(string)[0] + code := gameStatus["statusCode"].(string) + firstLetter := string(code[0]) - switch strconv.Itoa(int(code)) { + switch firstLetter { case "S": // Scheduled return Scheduled, nil // Scheduled case "P": From c34e32d1b755744fcbb51763502ad187fa7bbca9 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Fri, 8 Mar 2024 23:35:04 -0500 Subject: [PATCH 22/34] fix: Add expiry time for postponed games --- stats.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stats.go b/stats.go index 14b162c..d38eb05 100644 --- a/stats.go +++ b/stats.go @@ -302,6 +302,10 @@ func handlePostponedGame(gameStats map[string]interface{}, liveStats map[string] if hsetErr != nil { return hsetErr } + expireErr := client.redisClient.Expire(context.Background(), "game:"+strconv.Itoa(stats.GameID), time.Hour*24).Err() + if expireErr != nil { + return expireErr + } client.dbMut.Lock() defer client.dbMut.Unlock() From 527acd7622e8a194f973d7e34de2d17c2a96eeb3 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Thu, 28 Mar 2024 16:19:59 -0400 Subject: [PATCH 23/34] chore: Remove AWS and deploy through Google Cloud instead --- .aws/task-definition.json | 49 ------------------------ .github/workflows/deploy_dev.yml | 62 ------------------------------- .github/workflows/deploy_prod.yml | 57 ---------------------------- 3 files changed, 168 deletions(-) delete mode 100644 .aws/task-definition.json delete mode 100644 .github/workflows/deploy_dev.yml delete mode 100644 .github/workflows/deploy_prod.yml diff --git a/.aws/task-definition.json b/.aws/task-definition.json deleted file mode 100644 index f8c35a6..0000000 --- a/.aws/task-definition.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "containerDefinitions": [ - { - "name": "pickem", - "image": "798380260115.dkr.ecr.us-east-1.amazonaws.com/pickemlivestats:5de4b0690a72a92f6ba5d2420a2936209f695509", - "cpu": 256, - "memory": 512, - "portMappings": [], - "essential": true, - "environment": [], - "environmentFiles": [], - "mountPoints": [], - "volumesFrom": [], - "secrets": [ - { - "name": "DATABASE_URL", - "valueFrom": "arn:aws:secretsmanager:us-east-1:798380260115:secret:PickemLiveStats/dev-BfVBxS:DATABASE_URL::" - }, - { - "name": "REDIS_URL", - "valueFrom": "arn:aws:secretsmanager:us-east-1:798380260115:secret:PickemLiveStats/dev-BfVBxS:REDIS_URL::" - } - ], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-create-group": "true", - "awslogs-group": "pickem-livestats-dev", - "awslogs-region": "us-east-1", - "awslogs-stream-prefix": "dev-stats" - }, - "secretOptions": [] - }, - "systemControls": [] - } - ], - "family": "PickemLiveStats", - "taskRoleArn": "arn:aws:iam::798380260115:role/ecsTaskExecutionRole", - "executionRoleArn": "arn:aws:iam::798380260115:role/ecsTaskExecutionRole", - "networkMode": "awsvpc", - "volumes": [], - "placementConstraints": [], - "requiresCompatibilities": [ - "FARGATE" - ], - "cpu": "256", - "memory": "512", - "tags": [] -} \ No newline at end of file diff --git a/.github/workflows/deploy_dev.yml b/.github/workflows/deploy_dev.yml deleted file mode 100644 index cf69276..0000000 --- a/.github/workflows/deploy_dev.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: Deploy to ECR -run-name: Run ${{ github.run_id }} deploying to ECR -on: - push: - branches: - - dev - -env: - AWS_REGION: us-east-1 - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} - CONTAINER_NAME: "pickem" - ECS_CLUSTER: PickemLiveStats-Dev - - -jobs: - deploy: - name: Deploy to ECR - runs-on: ubuntu-latest - environment: development - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Build, tag, and push image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} - IMAGE_TAG: ${{ github.sha }} - run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT - - - name: Fill new image ID in Amazon ECS task definition - id: fill-task-def - uses: aws-actions/amazon-ecs-render-task-definition@v1 - with: - task-definition: .aws/task-definition.json - container-name: ${{ env.CONTAINER_NAME }} - image: ${{ steps.build-image.outputs.image }} - - - name: Deploy to Amazon ECS - uses: airfordable/ecs-deploy-task-definition-to-scheduled-task@v2.0.0 - with: - cluster: ${{ env.ECS_CLUSTER }} - task-definition: ${{ steps.fill-task-def.outputs.task-definition }} diff --git a/.github/workflows/deploy_prod.yml b/.github/workflows/deploy_prod.yml deleted file mode 100644 index e55025c..0000000 --- a/.github/workflows/deploy_prod.yml +++ /dev/null @@ -1,57 +0,0 @@ - -name: Deploy to ECR -run-name: Run ${{ github.run_id }} deploying to ECR -on: - push: - branches: - - main - -env: - AWS_REGION: us-east-1 - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} - CONTAINER_NAME: "pickem" - ECS_CLUSTER: PickemLiveStats-Prod - - -jobs: - deploy: - name: Deploy to ECR - runs-on: ubuntu-latest - environment: development - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Build, tag, and push image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} - IMAGE_TAG: ${{ github.sha }} - run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT - - - name: Fill new image ID in Amazon ECS task definition - id: fill-task-def - uses: aws-actions/amazon-ecs-render-task-definition@v1 - with: - task-definition: .aws/task-definition.json - container-name: my-container - image: ${{ steps.build-image.outputs.image }} From 5fe490ba9e56f5a3f80dfe412240f692c8e6dee6 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Fri, 29 Mar 2024 12:27:10 -0400 Subject: [PATCH 24/34] fix: Change SQL queries to correctly trigger - Modify queries to work correctly and be clearer in the DB --- stats.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/stats.go b/stats.go index d38eb05..462465c 100644 --- a/stats.go +++ b/stats.go @@ -503,8 +503,7 @@ func setFinalScoreDB(stats CompletedGameStats, client *DatabaseClient) error { UPDATE games SET finished=$1, home_score=$2, away_score=$3, winner = ( CASE WHEN home_score > away_score THEN "homeTeam_id" - WHEN home_score < away_score THEN "awayTeam_id" - ELSE NULL + ELSE "awayTeam_id" END ) WHERE id = $4`, @@ -515,11 +514,13 @@ func setFinalScoreDB(stats CompletedGameStats, client *DatabaseClient) error { } _, err = client.db.Exec(cctx, ` - UPDATE picks set correct= - (CASE WHEN "pickedHome" = - (SELECT winner = "homeTeam_id" from games where id = $1 and winner IS NOT NULL) - THEN true - ELSE false END) WHERE game_id = $1`, stats.GameID) + UPDATE picks p set correct = + (CASE + WHEN "pickedHome" = true AND (SELECT winner = "homeTeam_id" FROM games WHERE id = p.game_id AND winner is not null) THEN true + WHEN "pickedHome" = false AND (SELECT winner = "awayTeam_id" FROM games WHERE id = p.game_id and winner is not null) THEN true + ELSE false + END) + WHERE game_id = $1;`, stats.GameID) if err != nil { return err From 65da6a5ad17d9f55c5eac819f588a3311ac04a47 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 3 Apr 2024 13:47:07 -0400 Subject: [PATCH 25/34] feat: Retrieve the day's schedule from MLB instead of database - Calls MLB's statsapi to retrieve the schedule from the API for all games for the day - Includes rescheduled games from previous days - Updates database's start times when needed --- http.go | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 37 ++++++++++++++++----------------- 2 files changed, 82 insertions(+), 19 deletions(-) diff --git a/http.go b/http.go index 271183b..53a295b 100644 --- a/http.go +++ b/http.go @@ -5,9 +5,29 @@ import ( "errors" "fmt" "io" + "log/slog" "net/http" + "time" ) +type MLBScheduleResponse struct { + Dates []DateResponse `json:"dates"` +} + +type DateResponse struct { + Games []GameResponse `json:"games"` +} + +type GameResponse struct { + GameID int `json:"gamePk"` + StartTime string `json:"gameDate"` +} + +type ScheduledGame struct { + GameID int + StartTime time.Time +} + func getGameData(gameID int) (map[string]interface{}, error) { resp, respErr := http.Get(fmt.Sprintf("https://statsapi.mlb.com/api/v1.1/game/%d/feed/live", gameID)) if respErr != nil { @@ -28,6 +48,45 @@ func getGameData(gameID int) (map[string]interface{}, error) { return mlbDataJSON, nil } +func getGameSchedule(year int, month int, day int) ([]ScheduledGame, error) { + + resp, respErr := http.Get(fmt.Sprintf("https://statsapi.mlb.com/api/v1/schedule?sportId=1&date=%d-%d-%d", year, month, day)) + if respErr != nil { + return nil, respErr + } + + defer resp.Body.Close() + + body, ioErr := io.ReadAll(resp.Body) + if ioErr != nil { + return nil, ioErr + } + + var mlbDataJSON MLBScheduleResponse + if unmarshalErr := json.Unmarshal(body, &mlbDataJSON); unmarshalErr != nil { + return nil, unmarshalErr + } + + // Retrieve the gameIDs + var gameIDs []ScheduledGame + var games = mlbDataJSON.Dates[0].Games + + for i := range games { + timestamp, timeParseErr := time.Parse("2006-01-02T15:04:05Z", games[i].StartTime) + if timeParseErr != nil { + return nil, timeParseErr + } + + gameIDs = append(gameIDs, ScheduledGame{ + GameID: games[i].GameID, + StartTime: timestamp, + }) + } + + return gameIDs, nil + +} + func getGameStats(gameData map[string]interface{}) (map[string]interface{}, error) { return unwrap(gameData, "gameData") } @@ -43,3 +102,8 @@ func unwrap(jsonData map[string]interface{}, key string) (map[string]interface{} } return nestedData, nil } + +func debugJSONPrint(jsonData map[string]interface{}) { + jsonDataString, _ := json.MarshalIndent(jsonData, "", " ") + slog.Info(string(jsonDataString)) +} diff --git a/main.go b/main.go index d639f3c..15ff4ff 100644 --- a/main.go +++ b/main.go @@ -14,10 +14,11 @@ import ( var databaseClient *DatabaseClient type Game struct { - ID int `sql:"id"` - HomeTeam_ID int `sql:"home_team_id"` - AwayTeam_ID int `sql:"away_team_id"` - Date pgtype.Date `sql:"date"` + ID int `sql:"id"` + HomeTeam_ID int `sql:"home_team_id"` + AwayTeam_ID int `sql:"away_team_id"` + Date pgtype.Date `sql:"date"` + StartTimeUTC pgtype.Timestamp `sql:"startTimeUTC"` } func init() { @@ -47,31 +48,29 @@ func StatsJob(GameID int, group *sync.WaitGroup) { func main() { - // Get information about the games for today - databaseClient.dbMut.Lock() - rows, err := databaseClient.db.Query( - context.Background(), - `SELECT id, "homeTeam_id", "awayTeam_id", date FROM games WHERE date = $1`, - time.Now().Format("2006-01-02")) + // Verify from MLB about the games present for today. This will always have more games than the database itself. + gameIDs, err := getGameSchedule(time.Now().Year(), int(time.Now().Month()), time.Now().Day()) if err != nil { - slog.Error("Could not obtain games for today...") + slog.Error("Could not obtain games for today from MLB...") slog.Error(err.Error()) os.Exit(1) } - var games = make([]Game, 0) - - for rows.Next() { - var game Game - err := rows.Scan(&game.ID, &game.HomeTeam_ID, &game.AwayTeam_ID, &game.Date) + var games = make([]Game, len(gameIDs)) + // Retrieve/update games for the day in database for today + databaseClient.dbMut.Lock() + for i, gid := range gameIDs { + err := databaseClient.db.QueryRow( + context.Background(), + `UPDATE games SET "date" = $1, "startTimeUTC" = $2 WHERE id = $3 RETURNING id, "homeTeam_id", "awayTeam_id", date`, + time.Now().Format("2006-01-02"), gid.StartTime, gid.GameID).Scan(&games[i].ID, &games[i].HomeTeam_ID, &games[i].AwayTeam_ID, &games[i].Date) if err != nil { - slog.Error("Could not scan game...") + slog.Error("Could not insert/update game into database...") slog.Error(err.Error()) os.Exit(1) } - games = append(games, game) + } - rows.Close() databaseClient.dbMut.Unlock() universalWait := &sync.WaitGroup{} From 2ad77f74ba3f57b1aa7cfc0f98cfe9b41ccd23fc Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Wed, 3 Apr 2024 15:05:03 -0400 Subject: [PATCH 26/34] feat: Extra postponement checks - Adds simple batch job to set games that have been delayed already prior to job launch to be pushed onto the next day - For postponed games that occur while in progress: - Update the time present in the database - Update the time present in the Redis cache - To do (low prio): Prune all scheduled games currently in the cache and remove if the start time is less than a day away --- database.go | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 14 +++++++----- stats.go | 60 ++++++++++++++++++++++++++++++++-------------------- 3 files changed, 107 insertions(+), 28 deletions(-) diff --git a/database.go b/database.go index 400ea23..f1a66d4 100644 --- a/database.go +++ b/database.go @@ -8,6 +8,7 @@ import ( "log/slog" "os" "sync" + "time" ) type DatabaseClient struct { @@ -52,3 +53,63 @@ func NewDatabaseClient() *DatabaseClient { } } + +func BatchUpdateTime(client *DatabaseClient) { + rows, err := client.db.Query( + context.Background(), + `SELECT id, "homeTeam_id", "awayTeam_id", date FROM games WHERE date = $1`, + time.Now().Format("2006-01-02")) + if err != nil { + slog.Error("Could not obtain games for today...") + slog.Error(err.Error()) + os.Exit(1) + } + + for rows.Next() { + var game Game + err := rows.Scan(&game.ID, &game.HomeTeam_ID, &game.AwayTeam_ID, &game.Date) + if err != nil { + slog.Error("Could not scan game...") + slog.Error(err.Error()) + os.Exit(1) + } + gameData, err := getGameData(game.ID) + if err != nil { + slog.Error("Could not obtain game data...") + slog.Error(err.Error()) + os.Exit(1) + } + gameStats, err := getGameStats(gameData) + if err != nil { + slog.Error("Could not obtain game stats...") + slog.Error(err.Error()) + os.Exit(1) + } + datetime, err := unwrap(gameStats, "datetime") + if err != nil { + slog.Error("Could not unwrap datetime...") + slog.Error(err.Error()) + os.Exit(1) + } + startTime, err := time.Parse("2006-01-02T15:04:05Z", datetime["dateTime"].(string)) + if err != nil { + slog.Error("Could not parse start time...") + slog.Error(err.Error()) + os.Exit(1) + } + + localTime := startTime.In(time.Local) + client.db.Exec(context.Background(), + `UPDATE games SET date = $1, "startTimeUTC" = $2 WHERE id = $3`, + localTime.Format("2006-01-02"), startTime, game.ID) + } + rows.Close() +} + +func UpdateTime(client *DatabaseClient, gameID int, newStartTime time.Time) { + localTime := newStartTime.In(time.Local) + _, err := client.db.Exec(context.Background(), `UPDATE games SET date = $1, "startTimeUTC" = $2 WHERE id = $3`, localTime, newStartTime, gameID) + if err != nil { + slog.Error(err.Error()) + } +} diff --git a/main.go b/main.go index 15ff4ff..71f67ef 100644 --- a/main.go +++ b/main.go @@ -48,7 +48,7 @@ func StatsJob(GameID int, group *sync.WaitGroup) { func main() { - // Verify from MLB about the games present for today. This will always have more games than the database itself. + // Verify from MLB about the games present for today. This may have more games than the database itself. gameIDs, err := getGameSchedule(time.Now().Year(), int(time.Now().Month()), time.Now().Day()) if err != nil { slog.Error("Could not obtain games for today from MLB...") @@ -56,25 +56,29 @@ func main() { os.Exit(1) } - var games = make([]Game, len(gameIDs)) + var mlbGames = make([]Game, len(gameIDs)) // Retrieve/update games for the day in database for today databaseClient.dbMut.Lock() + + // Retrieve the games for today from the database (as a batch job to update issues with delays) + BatchUpdateTime(databaseClient) + + // Load games for all MLB games for today for i, gid := range gameIDs { err := databaseClient.db.QueryRow( context.Background(), `UPDATE games SET "date" = $1, "startTimeUTC" = $2 WHERE id = $3 RETURNING id, "homeTeam_id", "awayTeam_id", date`, - time.Now().Format("2006-01-02"), gid.StartTime, gid.GameID).Scan(&games[i].ID, &games[i].HomeTeam_ID, &games[i].AwayTeam_ID, &games[i].Date) + time.Now().Format("2006-01-02"), gid.StartTime, gid.GameID).Scan(&mlbGames[i].ID, &mlbGames[i].HomeTeam_ID, &mlbGames[i].AwayTeam_ID, &mlbGames[i].Date) if err != nil { slog.Error("Could not insert/update game into database...") slog.Error(err.Error()) os.Exit(1) } - } databaseClient.dbMut.Unlock() universalWait := &sync.WaitGroup{} - for _, game := range games { + for _, game := range mlbGames { universalWait.Add(1) go StatsJob(game.ID, universalWait) time.Sleep(500 * time.Millisecond) // Just to space things out a little. diff --git a/stats.go b/stats.go index 462465c..f8a867a 100644 --- a/stats.go +++ b/stats.go @@ -50,13 +50,6 @@ type UnknownGameStats struct { gameID int `redis:"gameID"` } -type PostponedGameStats struct { - Status string `redis:"status"` - GameID int `redis:"gameID"` - HomeScore int `redis:"homeScore"` - AwayScore int `redis:"awayScore"` -} - // Returns true or false if the game that was just handled was finished. func handleGameStats(gameID int, dbClient *DatabaseClient) (bool, error) { gameResp, err := getGameData(gameID) @@ -86,7 +79,7 @@ func handleGameStats(gameID int, dbClient *DatabaseClient) (bool, error) { case InProgress: err := handleInProgressGame(gameStats, liveStats, dbClient) return false, err - case Unknown: + default: err := handlePostponedGame(gameStats, liveStats, dbClient) return true, err } @@ -102,6 +95,17 @@ func getGameType(gameStats map[string]interface{}) (string, error) { return "", gameStatusErr } + // Check postponed start that's in the system + startTime, sterr := getStartTime(gameStats) + if sterr != nil { + return "", sterr + } + localStartTime := startTime.In(time.Local) + + if localStartTime.YearDay() > time.Now().YearDay() { + return Postponed, nil + } + code := gameStatus["statusCode"].(string) firstLetter := string(code[0]) @@ -284,16 +288,15 @@ func handlePostponedGame(gameStats map[string]interface{}, liveStats map[string] return gameIDErr } - homeScore, awayScore, scoreErr := getScores(liveStats) - if scoreErr != nil { - return scoreErr + newStartTime, sterr := getStartTime(gameStats) + if sterr != nil { + return sterr } - stats := PostponedGameStats{ - Status: Postponed, - GameID: int(gameID["pk"].(float64)), - HomeScore: homeScore, - AwayScore: awayScore, + stats := ScheduledGameStats{ + Status: Scheduled, + GameID: int(gameID["pk"].(float64)), + StartTimeUTC: newStartTime.Format(time.RFC3339), } client.redisMut.Lock() @@ -326,18 +329,17 @@ func handlePostponedGame(gameStats map[string]interface{}, liveStats map[string] }() _, err = client.db.Exec(cctx, ` - UPDATE games SET finished=$1, home_score=$2, away_score=$3, winner = NULL - WHERE id = $4`, - true, stats.HomeScore, stats.AwayScore, stats.GameID) - + UPDATE games SET finished=$1, winner = NULL + WHERE id = $2`, + false, stats.GameID) if err != nil { return err } - err = tx.Commit(context.Background()) - return err + UpdateTime(client, stats.GameID, newStartTime) - return nil + err = tx.Commit(cctx) + return err } func getScores(liveData map[string]interface{}) (int, int, error) { @@ -372,6 +374,18 @@ func getInnningInfo(liveData map[string]interface{}) (int, bool, int, error) { nil } +func getStartTime(gameData map[string]interface{}) (time.Time, error) { + datetime, datetimeErr := unwrap(gameData, "datetime") + if datetimeErr != nil { + return time.Now(), datetimeErr + } + startTime, timeParseErr := time.Parse("2006-01-02T15:04:05Z", datetime["dateTime"].(string)) + if timeParseErr != nil { + return time.Now(), timeParseErr + } + return startTime, nil +} + // Returns batter name, pitcher name, a list of length 3 for onFirst, onSecond, onThird, error func getAtBatInfo(gameData map[string]interface{}, liveData map[string]interface{}, isBottomInning bool) (string, string, []bool, error) { keys := [3]string{"plays", "currentPlay", "matchup"} From f1a12b5285a52885e6bb250c75c83d36fea63022 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Fri, 5 Apr 2024 16:49:29 -0400 Subject: [PATCH 27/34] feat: Database pruning on startup - Retrieves all games currently in the database, retrieves their updated status once upon startup - Allows completed games from the previous day (or instance of this job), if unresolved, to be cleaned in case of emergency - Allows postponed games -- which were postponed on the day of -- to be rechecked when they were postponed to and updated within the database(s). --- batch.go | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++ database.go | 52 --------------------------------- main.go | 3 ++ stats.go | 7 ++++- 4 files changed, 92 insertions(+), 53 deletions(-) create mode 100644 batch.go diff --git a/batch.go b/batch.go new file mode 100644 index 0000000..883cb4c --- /dev/null +++ b/batch.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "log/slog" + "os" + "strconv" + "strings" + "time" +) + +func BatchUpdateTime(client *DatabaseClient) { + rows, err := client.db.Query( + context.Background(), + `SELECT id, "homeTeam_id", "awayTeam_id", date FROM games WHERE date = $1`, + time.Now().Format("2006-01-02")) + if err != nil { + slog.Error("Could not obtain games for today...") + slog.Error(err.Error()) + os.Exit(1) + } + + for rows.Next() { + var game Game + err := rows.Scan(&game.ID, &game.HomeTeam_ID, &game.AwayTeam_ID, &game.Date) + if err != nil { + slog.Error("Could not scan game...") + slog.Error(err.Error()) + os.Exit(1) + } + gameData, err := getGameData(game.ID) + if err != nil { + slog.Error("Could not obtain game data...") + slog.Error(err.Error()) + os.Exit(1) + } + gameStats, err := getGameStats(gameData) + if err != nil { + slog.Error("Could not obtain game stats...") + slog.Error(err.Error()) + os.Exit(1) + } + datetime, err := unwrap(gameStats, "datetime") + if err != nil { + slog.Error("Could not unwrap datetime...") + slog.Error(err.Error()) + os.Exit(1) + } + startTime, err := time.Parse("2006-01-02T15:04:05Z", datetime["dateTime"].(string)) + if err != nil { + slog.Error("Could not parse start time...") + slog.Error(err.Error()) + os.Exit(1) + } + + localTime := startTime.In(time.Local) + client.db.Exec(context.Background(), + `UPDATE games SET date = $1, "startTimeUTC" = $2 WHERE id = $3`, + localTime.Format("2006-01-02"), startTime, game.ID) + } + rows.Close() +} + +func BatchResolveJobs(client *DatabaseClient) { + keys, err := client.redisClient.Keys(context.Background(), "game:*").Result() + if err != nil { + return + } + + for _, key := range keys { + gameID, err := strconv.Atoi(strings.Split(key, ":")[1]) + if err != nil { + slog.Error("Could not convert key to integer...") + slog.Error(err.Error()) + os.Exit(1) + } + + // Update all items once upon startup, if scheduled then expires within 24 hours. + handleGameStats(gameID, databaseClient) + + } + +} diff --git a/database.go b/database.go index f1a66d4..f79861a 100644 --- a/database.go +++ b/database.go @@ -54,58 +54,6 @@ func NewDatabaseClient() *DatabaseClient { } -func BatchUpdateTime(client *DatabaseClient) { - rows, err := client.db.Query( - context.Background(), - `SELECT id, "homeTeam_id", "awayTeam_id", date FROM games WHERE date = $1`, - time.Now().Format("2006-01-02")) - if err != nil { - slog.Error("Could not obtain games for today...") - slog.Error(err.Error()) - os.Exit(1) - } - - for rows.Next() { - var game Game - err := rows.Scan(&game.ID, &game.HomeTeam_ID, &game.AwayTeam_ID, &game.Date) - if err != nil { - slog.Error("Could not scan game...") - slog.Error(err.Error()) - os.Exit(1) - } - gameData, err := getGameData(game.ID) - if err != nil { - slog.Error("Could not obtain game data...") - slog.Error(err.Error()) - os.Exit(1) - } - gameStats, err := getGameStats(gameData) - if err != nil { - slog.Error("Could not obtain game stats...") - slog.Error(err.Error()) - os.Exit(1) - } - datetime, err := unwrap(gameStats, "datetime") - if err != nil { - slog.Error("Could not unwrap datetime...") - slog.Error(err.Error()) - os.Exit(1) - } - startTime, err := time.Parse("2006-01-02T15:04:05Z", datetime["dateTime"].(string)) - if err != nil { - slog.Error("Could not parse start time...") - slog.Error(err.Error()) - os.Exit(1) - } - - localTime := startTime.In(time.Local) - client.db.Exec(context.Background(), - `UPDATE games SET date = $1, "startTimeUTC" = $2 WHERE id = $3`, - localTime.Format("2006-01-02"), startTime, game.ID) - } - rows.Close() -} - func UpdateTime(client *DatabaseClient, gameID int, newStartTime time.Time) { localTime := newStartTime.In(time.Local) _, err := client.db.Exec(context.Background(), `UPDATE games SET date = $1, "startTimeUTC" = $2 WHERE id = $3`, localTime, newStartTime, gameID) diff --git a/main.go b/main.go index 71f67ef..7e6a7fb 100644 --- a/main.go +++ b/main.go @@ -48,6 +48,9 @@ func StatsJob(GameID int, group *sync.WaitGroup) { func main() { + // Resolve jobs from the previous day (or start-up). + BatchResolveJobs(databaseClient) + // Verify from MLB about the games present for today. This may have more games than the database itself. gameIDs, err := getGameSchedule(time.Now().Year(), int(time.Now().Month()), time.Now().Day()) if err != nil { diff --git a/stats.go b/stats.go index f8a867a..9d4c537 100644 --- a/stats.go +++ b/stats.go @@ -173,6 +173,10 @@ func handleScheduledGame(gameStats map[string]interface{}, client *DatabaseClien if rdErr != nil { return rdErr } + expireErr := client.redisClient.Expire(context.Background(), "game:"+strconv.Itoa(scheduledGameStats.GameID), time.Hour*24).Err() + if expireErr != nil { + return expireErr + } slog.Info("Game " + strconv.Itoa(scheduledGameStats.GameID) + " written to database") return nil } else if getRedisErr != nil { // Error, return @@ -281,7 +285,6 @@ func handleInProgressGame(gameStats map[string]interface{}, liveStats map[string } func handlePostponedGame(gameStats map[string]interface{}, liveStats map[string]interface{}, client *DatabaseClient) error { - slog.Info("Game status unknown, setting in database and skipping") gameID, gameIDErr := unwrap(gameStats, "game") if gameIDErr != nil { @@ -299,6 +302,8 @@ func handlePostponedGame(gameStats map[string]interface{}, liveStats map[string] StartTimeUTC: newStartTime.Format(time.RFC3339), } + slog.Info("Game status for game " + strconv.Itoa(stats.GameID) + " unknown, setting in database and skipping") + client.redisMut.Lock() defer client.redisMut.Unlock() hsetErr := client.redisClient.HSet(context.Background(), "game:"+strconv.Itoa(stats.GameID), stats).Err() From f84e6cf613f7fec00b52e18c09c7cd6e6e7733d5 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Sat, 6 Apr 2024 11:59:28 -0400 Subject: [PATCH 28/34] fix: Cache current status responses for longer - Cache responses for longer than 10 minutes - Will not last until next day if game is incomplete, mostly for development purposes --- stats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats.go b/stats.go index 9d4c537..4504602 100644 --- a/stats.go +++ b/stats.go @@ -271,7 +271,7 @@ func handleInProgressGame(gameStats map[string]interface{}, liveStats map[string return hsetErr } - quickDuration := time.Minute * 10 + quickDuration := time.Hour * 12 expireErr := client.redisClient.Expire( context.Background(), "game:"+strconv.Itoa(inProgressGameStats.GameID), From a547cadef7820828be32eda28ca953afc7f54d77 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Mon, 8 Apr 2024 12:00:46 -0400 Subject: [PATCH 29/34] fix: Finished games are now updated correctly - Adds a precalculation about whether the home team wins the game - Uses the precalculation in the queries, no longer relies on knowing the score or winner from other tables --- stats.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/stats.go b/stats.go index 4504602..25eafba 100644 --- a/stats.go +++ b/stats.go @@ -487,7 +487,7 @@ func setFinalScoreRedis(stats CompletedGameStats, client *DatabaseClient) error return hsetErr } - oneDayDuration := time.Hour * 24 + oneDayDuration := time.Hour * 12 expireErr := client.redisClient.Expire( context.Background(), "game:"+strconv.Itoa(stats.GameID), @@ -504,6 +504,7 @@ func setFinalScoreDB(stats CompletedGameStats, client *DatabaseClient) error { defer client.dbMut.Unlock() cctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) tx, err := client.db.Begin(cctx) + homeWins := stats.HomeScore > stats.AwayScore defer cancel() if err != nil { @@ -521,12 +522,12 @@ func setFinalScoreDB(stats CompletedGameStats, client *DatabaseClient) error { _, err = client.db.Exec(cctx, ` UPDATE games SET finished=$1, home_score=$2, away_score=$3, winner = ( - CASE WHEN home_score > away_score THEN "homeTeam_id" + CASE WHEN $4 THEN "homeTeam_id" ELSE "awayTeam_id" END ) - WHERE id = $4`, - true, stats.HomeScore, stats.AwayScore, stats.GameID) + WHERE id = $5`, + true, stats.HomeScore, stats.AwayScore, homeWins, stats.GameID) if err != nil { return err @@ -535,11 +536,11 @@ func setFinalScoreDB(stats CompletedGameStats, client *DatabaseClient) error { _, err = client.db.Exec(cctx, ` UPDATE picks p set correct = (CASE - WHEN "pickedHome" = true AND (SELECT winner = "homeTeam_id" FROM games WHERE id = p.game_id AND winner is not null) THEN true - WHEN "pickedHome" = false AND (SELECT winner = "awayTeam_id" FROM games WHERE id = p.game_id and winner is not null) THEN true + WHEN "pickedHome" = true AND $2 THEN true + WHEN "pickedHome" = false AND not $2 THEN true ELSE false END) - WHERE game_id = $1;`, stats.GameID) + WHERE game_id = $1;`, stats.GameID, homeWins) if err != nil { return err From 09e9a8dd62b01f3d4b5071f407f1288a5ffb1eb4 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Thu, 11 Apr 2024 12:56:20 -0400 Subject: [PATCH 30/34] Revert "chore: Remove AWS and deploy through Google Cloud instead" This reverts commit 527acd7622e8a194f973d7e34de2d17c2a96eeb3. --- .aws/task-definition.json | 49 ++++++++++++++++++++++++ .github/workflows/deploy_dev.yml | 62 +++++++++++++++++++++++++++++++ .github/workflows/deploy_prod.yml | 57 ++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 .aws/task-definition.json create mode 100644 .github/workflows/deploy_dev.yml create mode 100644 .github/workflows/deploy_prod.yml diff --git a/.aws/task-definition.json b/.aws/task-definition.json new file mode 100644 index 0000000..f8c35a6 --- /dev/null +++ b/.aws/task-definition.json @@ -0,0 +1,49 @@ +{ + "containerDefinitions": [ + { + "name": "pickem", + "image": "798380260115.dkr.ecr.us-east-1.amazonaws.com/pickemlivestats:5de4b0690a72a92f6ba5d2420a2936209f695509", + "cpu": 256, + "memory": 512, + "portMappings": [], + "essential": true, + "environment": [], + "environmentFiles": [], + "mountPoints": [], + "volumesFrom": [], + "secrets": [ + { + "name": "DATABASE_URL", + "valueFrom": "arn:aws:secretsmanager:us-east-1:798380260115:secret:PickemLiveStats/dev-BfVBxS:DATABASE_URL::" + }, + { + "name": "REDIS_URL", + "valueFrom": "arn:aws:secretsmanager:us-east-1:798380260115:secret:PickemLiveStats/dev-BfVBxS:REDIS_URL::" + } + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-create-group": "true", + "awslogs-group": "pickem-livestats-dev", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "dev-stats" + }, + "secretOptions": [] + }, + "systemControls": [] + } + ], + "family": "PickemLiveStats", + "taskRoleArn": "arn:aws:iam::798380260115:role/ecsTaskExecutionRole", + "executionRoleArn": "arn:aws:iam::798380260115:role/ecsTaskExecutionRole", + "networkMode": "awsvpc", + "volumes": [], + "placementConstraints": [], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "256", + "memory": "512", + "tags": [] +} \ No newline at end of file diff --git a/.github/workflows/deploy_dev.yml b/.github/workflows/deploy_dev.yml new file mode 100644 index 0000000..cf69276 --- /dev/null +++ b/.github/workflows/deploy_dev.yml @@ -0,0 +1,62 @@ +name: Deploy to ECR +run-name: Run ${{ github.run_id }} deploying to ECR +on: + push: + branches: + - dev + +env: + AWS_REGION: us-east-1 + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} + CONTAINER_NAME: "pickem" + ECS_CLUSTER: PickemLiveStats-Dev + + +jobs: + deploy: + name: Deploy to ECR + runs-on: ubuntu-latest + environment: development + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build, tag, and push image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} + IMAGE_TAG: ${{ github.sha }} + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT + + - name: Fill new image ID in Amazon ECS task definition + id: fill-task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: .aws/task-definition.json + container-name: ${{ env.CONTAINER_NAME }} + image: ${{ steps.build-image.outputs.image }} + + - name: Deploy to Amazon ECS + uses: airfordable/ecs-deploy-task-definition-to-scheduled-task@v2.0.0 + with: + cluster: ${{ env.ECS_CLUSTER }} + task-definition: ${{ steps.fill-task-def.outputs.task-definition }} diff --git a/.github/workflows/deploy_prod.yml b/.github/workflows/deploy_prod.yml new file mode 100644 index 0000000..e55025c --- /dev/null +++ b/.github/workflows/deploy_prod.yml @@ -0,0 +1,57 @@ + +name: Deploy to ECR +run-name: Run ${{ github.run_id }} deploying to ECR +on: + push: + branches: + - main + +env: + AWS_REGION: us-east-1 + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} + CONTAINER_NAME: "pickem" + ECS_CLUSTER: PickemLiveStats-Prod + + +jobs: + deploy: + name: Deploy to ECR + runs-on: ubuntu-latest + environment: development + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build, tag, and push image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} + IMAGE_TAG: ${{ github.sha }} + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT + + - name: Fill new image ID in Amazon ECS task definition + id: fill-task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: .aws/task-definition.json + container-name: my-container + image: ${{ steps.build-image.outputs.image }} From 4156261c8d37e09f99898c77d749e772d6663da9 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Sun, 14 Apr 2024 11:58:02 -0400 Subject: [PATCH 31/34] fix: time zone issues - Date should not be set for Eastern time -- previously, times had been set for UTC which caused issues when loading in AWS Resolves https://github.com/specificlanguage/PickemGoLiveStats/issues/2 --- batch.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/batch.go b/batch.go index 883cb4c..a278ec6 100644 --- a/batch.go +++ b/batch.go @@ -53,7 +53,11 @@ func BatchUpdateTime(client *DatabaseClient) { os.Exit(1) } - localTime := startTime.In(time.Local) + locTZ, err := time.LoadLocation("America/New_York") + if err != nil { + slog.Error("Could not load start time location.") + } + localTime := startTime.In(locTZ) client.db.Exec(context.Background(), `UPDATE games SET date = $1, "startTimeUTC" = $2 WHERE id = $3`, localTime.Format("2006-01-02"), startTime, game.ID) From 377c46127888e75db5fa1fc6744bcc25bcdb7dd0 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Mon, 15 Apr 2024 19:58:34 -0400 Subject: [PATCH 32/34] fix: time zone date issues - Date is now set to current date Hopefully resolves https://github.com/specificlanguage/PickemGoLiveStats/issues/2 --- batch.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/batch.go b/batch.go index a278ec6..a8c4de7 100644 --- a/batch.go +++ b/batch.go @@ -53,14 +53,8 @@ func BatchUpdateTime(client *DatabaseClient) { os.Exit(1) } - locTZ, err := time.LoadLocation("America/New_York") - if err != nil { - slog.Error("Could not load start time location.") - } - localTime := startTime.In(locTZ) client.db.Exec(context.Background(), - `UPDATE games SET date = $1, "startTimeUTC" = $2 WHERE id = $3`, - localTime.Format("2006-01-02"), startTime, game.ID) + `UPDATE games SET date = CURRENT_DATE, "startTimeUTC" = $2 WHERE id = $3`, startTime, game.ID) } rows.Close() } From c536c955cd28aeee616d9d98ea0b9532dd8b713f Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Mon, 15 Apr 2024 20:53:36 -0400 Subject: [PATCH 33/34] fix(current status): time zone date issues - Retrieving games from MLB should be correctly set to Eastern time zone specifically for this purpose Once again, I hope this resolves https://github.com/specificlanguage/PickemGoLiveStats/issues/2 --- main.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 7e6a7fb..5775cc5 100644 --- a/main.go +++ b/main.go @@ -51,6 +51,16 @@ func main() { // Resolve jobs from the previous day (or start-up). BatchResolveJobs(databaseClient) + // Load games for all MLB games for today (Eastern Time) + easternTime, err := time.LoadLocation("America/New_York") + if err != nil { + slog.Error("Could not load Eastern Time zone...") + slog.Error(err.Error()) + os.Exit(1) + + } + localTime := time.Now().In(easternTime) + // Verify from MLB about the games present for today. This may have more games than the database itself. gameIDs, err := getGameSchedule(time.Now().Year(), int(time.Now().Month()), time.Now().Day()) if err != nil { @@ -60,18 +70,19 @@ func main() { } var mlbGames = make([]Game, len(gameIDs)) + // Retrieve/update games for the day in database for today databaseClient.dbMut.Lock() // Retrieve the games for today from the database (as a batch job to update issues with delays) BatchUpdateTime(databaseClient) - // Load games for all MLB games for today + // Update dates in Database for i, gid := range gameIDs { err := databaseClient.db.QueryRow( context.Background(), `UPDATE games SET "date" = $1, "startTimeUTC" = $2 WHERE id = $3 RETURNING id, "homeTeam_id", "awayTeam_id", date`, - time.Now().Format("2006-01-02"), gid.StartTime, gid.GameID).Scan(&mlbGames[i].ID, &mlbGames[i].HomeTeam_ID, &mlbGames[i].AwayTeam_ID, &mlbGames[i].Date) + localTime.Format("2006-01-02"), gid.StartTime, gid.GameID).Scan(&mlbGames[i].ID, &mlbGames[i].HomeTeam_ID, &mlbGames[i].AwayTeam_ID, &mlbGames[i].Date) if err != nil { slog.Error("Could not insert/update game into database...") slog.Error(err.Error()) From ef7f78f8905f0744f9b4e5b42ac8f5de59f60a22 Mon Sep 17 00:00:00 2001 From: specificlanguage Date: Mon, 15 Apr 2024 22:21:23 -0400 Subject: [PATCH 34/34] fix(timezones?): Force time zone to be Eastern in docker container - Changes docker container to force Eastern time zone - Reverts previous commit --- Dockerfile | 2 ++ main.go | 10 ++-------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index ddb7592..9af18b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,4 +14,6 @@ COPY . . RUN go build -v -o /usr/local/bin/main ./... ENV DATABASE_URL=$DATABASE_URL ENV REDIS_URL=$REDIS_URL +ENV TZ="America/New_York" +RUN ls -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone CMD ["main"] \ No newline at end of file diff --git a/main.go b/main.go index 5775cc5..ea4d6a8 100644 --- a/main.go +++ b/main.go @@ -52,14 +52,6 @@ func main() { BatchResolveJobs(databaseClient) // Load games for all MLB games for today (Eastern Time) - easternTime, err := time.LoadLocation("America/New_York") - if err != nil { - slog.Error("Could not load Eastern Time zone...") - slog.Error(err.Error()) - os.Exit(1) - - } - localTime := time.Now().In(easternTime) // Verify from MLB about the games present for today. This may have more games than the database itself. gameIDs, err := getGameSchedule(time.Now().Year(), int(time.Now().Month()), time.Now().Day()) @@ -77,6 +69,8 @@ func main() { // Retrieve the games for today from the database (as a batch job to update issues with delays) BatchUpdateTime(databaseClient) + localTime := time.Now().In(time.Local) + // Update dates in Database for i, gid := range gameIDs { err := databaseClient.db.QueryRow(