diff --git a/.aws/task-definition.json b/.aws/task-definition.json index d6d3e14..f8c35a6 100644 --- a/.aws/task-definition.json +++ b/.aws/task-definition.json @@ -1,62 +1,49 @@ { - "taskDefinitionArn": "arn:aws:ecs:us-east-1:798380260115:task-definition/PickemLiveStats:4", "containerDefinitions": [ { "name": "pickem", - "image": "798380260115.dkr.ecr.us-east-1.amazonaws.com/pickemlivestats", - "cpu": 0, + "image": "798380260115.dkr.ecr.us-east-1.amazonaws.com/pickemlivestats:5de4b0690a72a92f6ba5d2420a2936209f695509", + "cpu": 256, + "memory": 512, "portMappings": [], "essential": true, "environment": [], - "environmentFiles": [ + "environmentFiles": [], + "mountPoints": [], + "volumesFrom": [], + "secrets": [ { - "value": "arn:aws:s3:::pickem-environment-bucket/dev-livestats.env", - "type": "s3" + "name": "DATABASE_URL", + "valueFrom": "arn:aws:secretsmanager:us-east-1:798380260115:secret:PickemLiveStats/dev-BfVBxS:DATABASE_URL::" + }, + { + "name": "REDIS_URL", + "valueFrom": "arn:aws:secretsmanager:us-east-1:798380260115:secret:PickemLiveStats/dev-BfVBxS:REDIS_URL::" } ], - "mountPoints": [], - "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-create-group": "true", + "awslogs-group": "pickem-livestats-dev", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "dev-stats" + }, + "secretOptions": [] + }, "systemControls": [] } ], "family": "PickemLiveStats", + "taskRoleArn": "arn:aws:iam::798380260115:role/ecsTaskExecutionRole", "executionRoleArn": "arn:aws:iam::798380260115:role/ecsTaskExecutionRole", "networkMode": "awsvpc", - "revision": 4, "volumes": [], - "status": "ACTIVE", - "requiresAttributes": [ - { - "name": "com.amazonaws.ecs.capability.ecr-auth" - }, - { - "name": "ecs.capability.env-files.s3" - }, - { - "name": "ecs.capability.execution-role-ecr-pull" - }, - { - "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18" - }, - { - "name": "ecs.capability.task-eni" - } - ], "placementConstraints": [], - "compatibilities": [ - "EC2", - "FARGATE" - ], "requiresCompatibilities": [ "FARGATE" ], "cpu": "256", "memory": "512", - "runtimePlatform": { - "cpuArchitecture": "X86_64", - "operatingSystemFamily": "LINUX" - }, - "registeredAt": "2024-03-06T22:12:24.591Z", - "registeredBy": "arn:aws:iam::798380260115:root", "tags": [] } \ No newline at end of file diff --git a/.github/workflows/deploy_dev.yml b/.github/workflows/deploy_dev.yml new file mode 100644 index 0000000..cf69276 --- /dev/null +++ b/.github/workflows/deploy_dev.yml @@ -0,0 +1,62 @@ +name: Deploy to ECR +run-name: Run ${{ github.run_id }} deploying to ECR +on: + push: + branches: + - dev + +env: + AWS_REGION: us-east-1 + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} + CONTAINER_NAME: "pickem" + ECS_CLUSTER: PickemLiveStats-Dev + + +jobs: + deploy: + name: Deploy to ECR + runs-on: ubuntu-latest + environment: development + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build, tag, and push image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} + IMAGE_TAG: ${{ github.sha }} + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT + + - name: Fill new image ID in Amazon ECS task definition + id: fill-task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: .aws/task-definition.json + container-name: ${{ env.CONTAINER_NAME }} + image: ${{ steps.build-image.outputs.image }} + + - name: Deploy to Amazon ECS + uses: airfordable/ecs-deploy-task-definition-to-scheduled-task@v2.0.0 + with: + cluster: ${{ env.ECS_CLUSTER }} + task-definition: ${{ steps.fill-task-def.outputs.task-definition }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy_prod.yml similarity index 78% rename from .github/workflows/deploy.yml rename to .github/workflows/deploy_prod.yml index c0afd67..e55025c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy_prod.yml @@ -1,3 +1,4 @@ + name: Deploy to ECR run-name: Run ${{ github.run_id }} deploying to ECR on: @@ -10,10 +11,9 @@ env: AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - ECR_REPO: ${{ secrets.ECR_REPO }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} CONTAINER_NAME: "pickem" - ECS_SERVICE: PickemLiveStats - ECS_CLUSTER: PickemLiveStats-Dev + ECS_CLUSTER: PickemLiveStats-Prod jobs: @@ -41,7 +41,7 @@ jobs: id: build-image env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: ${{ secrets.ECR_REPO }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }} IMAGE_TAG: ${{ github.sha }} run: | docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . @@ -55,14 +55,3 @@ jobs: task-definition: .aws/task-definition.json container-name: my-container image: ${{ steps.build-image.outputs.image }} - - - - name: Deploy ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 - with: - task-definition: ${{ steps.fill-task-def.outputs.new-task-definition }} - service: my-service - cluster: my-cluster - wait-for-service-stability: true - - diff --git a/Dockerfile b/Dockerfile index ddb7592..9af18b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,4 +14,6 @@ COPY . . RUN go build -v -o /usr/local/bin/main ./... ENV DATABASE_URL=$DATABASE_URL ENV REDIS_URL=$REDIS_URL +ENV TZ="America/New_York" +RUN ls -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone CMD ["main"] \ No newline at end of file diff --git a/batch.go b/batch.go new file mode 100644 index 0000000..a8c4de7 --- /dev/null +++ b/batch.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + "log/slog" + "os" + "strconv" + "strings" + "time" +) + +func BatchUpdateTime(client *DatabaseClient) { + rows, err := client.db.Query( + context.Background(), + `SELECT id, "homeTeam_id", "awayTeam_id", date FROM games WHERE date = $1`, + time.Now().Format("2006-01-02")) + if err != nil { + slog.Error("Could not obtain games for today...") + slog.Error(err.Error()) + os.Exit(1) + } + + for rows.Next() { + var game Game + err := rows.Scan(&game.ID, &game.HomeTeam_ID, &game.AwayTeam_ID, &game.Date) + if err != nil { + slog.Error("Could not scan game...") + slog.Error(err.Error()) + os.Exit(1) + } + gameData, err := getGameData(game.ID) + if err != nil { + slog.Error("Could not obtain game data...") + slog.Error(err.Error()) + os.Exit(1) + } + gameStats, err := getGameStats(gameData) + if err != nil { + slog.Error("Could not obtain game stats...") + slog.Error(err.Error()) + os.Exit(1) + } + datetime, err := unwrap(gameStats, "datetime") + if err != nil { + slog.Error("Could not unwrap datetime...") + slog.Error(err.Error()) + os.Exit(1) + } + startTime, err := time.Parse("2006-01-02T15:04:05Z", datetime["dateTime"].(string)) + if err != nil { + slog.Error("Could not parse start time...") + slog.Error(err.Error()) + os.Exit(1) + } + + client.db.Exec(context.Background(), + `UPDATE games SET date = CURRENT_DATE, "startTimeUTC" = $2 WHERE id = $3`, startTime, game.ID) + } + rows.Close() +} + +func BatchResolveJobs(client *DatabaseClient) { + keys, err := client.redisClient.Keys(context.Background(), "game:*").Result() + if err != nil { + return + } + + for _, key := range keys { + gameID, err := strconv.Atoi(strings.Split(key, ":")[1]) + if err != nil { + slog.Error("Could not convert key to integer...") + slog.Error(err.Error()) + os.Exit(1) + } + + // Update all items once upon startup, if scheduled then expires within 24 hours. + handleGameStats(gameID, databaseClient) + + } + +} diff --git a/database.go b/database.go index a340355..f79861a 100644 --- a/database.go +++ b/database.go @@ -8,6 +8,7 @@ import ( "log/slog" "os" "sync" + "time" ) type DatabaseClient struct { @@ -18,11 +19,11 @@ type DatabaseClient struct { } func NewDatabaseClient() *DatabaseClient { - dotEnvErr := godotenv.Load() - if dotEnvErr != nil { - slog.Error("Error loading .env file, exiting") - os.Exit(1) - } + godotenv.Load() + //if dotEnvErr != nil { + // slog.Error("Error loading .env file, exiting") + // os.Exit(1) + //} // Connect to database connStr := os.Getenv("DATABASE_URL") @@ -52,3 +53,11 @@ func NewDatabaseClient() *DatabaseClient { } } + +func UpdateTime(client *DatabaseClient, gameID int, newStartTime time.Time) { + localTime := newStartTime.In(time.Local) + _, err := client.db.Exec(context.Background(), `UPDATE games SET date = $1, "startTimeUTC" = $2 WHERE id = $3`, localTime, newStartTime, gameID) + if err != nil { + slog.Error(err.Error()) + } +} diff --git a/http.go b/http.go index 271183b..53a295b 100644 --- a/http.go +++ b/http.go @@ -5,9 +5,29 @@ import ( "errors" "fmt" "io" + "log/slog" "net/http" + "time" ) +type MLBScheduleResponse struct { + Dates []DateResponse `json:"dates"` +} + +type DateResponse struct { + Games []GameResponse `json:"games"` +} + +type GameResponse struct { + GameID int `json:"gamePk"` + StartTime string `json:"gameDate"` +} + +type ScheduledGame struct { + GameID int + StartTime time.Time +} + func getGameData(gameID int) (map[string]interface{}, error) { resp, respErr := http.Get(fmt.Sprintf("https://statsapi.mlb.com/api/v1.1/game/%d/feed/live", gameID)) if respErr != nil { @@ -28,6 +48,45 @@ func getGameData(gameID int) (map[string]interface{}, error) { return mlbDataJSON, nil } +func getGameSchedule(year int, month int, day int) ([]ScheduledGame, error) { + + resp, respErr := http.Get(fmt.Sprintf("https://statsapi.mlb.com/api/v1/schedule?sportId=1&date=%d-%d-%d", year, month, day)) + if respErr != nil { + return nil, respErr + } + + defer resp.Body.Close() + + body, ioErr := io.ReadAll(resp.Body) + if ioErr != nil { + return nil, ioErr + } + + var mlbDataJSON MLBScheduleResponse + if unmarshalErr := json.Unmarshal(body, &mlbDataJSON); unmarshalErr != nil { + return nil, unmarshalErr + } + + // Retrieve the gameIDs + var gameIDs []ScheduledGame + var games = mlbDataJSON.Dates[0].Games + + for i := range games { + timestamp, timeParseErr := time.Parse("2006-01-02T15:04:05Z", games[i].StartTime) + if timeParseErr != nil { + return nil, timeParseErr + } + + gameIDs = append(gameIDs, ScheduledGame{ + GameID: games[i].GameID, + StartTime: timestamp, + }) + } + + return gameIDs, nil + +} + func getGameStats(gameData map[string]interface{}) (map[string]interface{}, error) { return unwrap(gameData, "gameData") } @@ -43,3 +102,8 @@ func unwrap(jsonData map[string]interface{}, key string) (map[string]interface{} } return nestedData, nil } + +func debugJSONPrint(jsonData map[string]interface{}) { + jsonDataString, _ := json.MarshalIndent(jsonData, "", " ") + slog.Info(string(jsonDataString)) +} diff --git a/main.go b/main.go index d639f3c..ea4d6a8 100644 --- a/main.go +++ b/main.go @@ -14,10 +14,11 @@ import ( var databaseClient *DatabaseClient type Game struct { - ID int `sql:"id"` - HomeTeam_ID int `sql:"home_team_id"` - AwayTeam_ID int `sql:"away_team_id"` - Date pgtype.Date `sql:"date"` + ID int `sql:"id"` + HomeTeam_ID int `sql:"home_team_id"` + AwayTeam_ID int `sql:"away_team_id"` + Date pgtype.Date `sql:"date"` + StartTimeUTC pgtype.Timestamp `sql:"startTimeUTC"` } func init() { @@ -47,35 +48,45 @@ func StatsJob(GameID int, group *sync.WaitGroup) { func main() { - // Get information about the games for today - databaseClient.dbMut.Lock() - rows, err := databaseClient.db.Query( - context.Background(), - `SELECT id, "homeTeam_id", "awayTeam_id", date FROM games WHERE date = $1`, - time.Now().Format("2006-01-02")) + // Resolve jobs from the previous day (or start-up). + BatchResolveJobs(databaseClient) + + // Load games for all MLB games for today (Eastern Time) + + // Verify from MLB about the games present for today. This may have more games than the database itself. + gameIDs, err := getGameSchedule(time.Now().Year(), int(time.Now().Month()), time.Now().Day()) if err != nil { - slog.Error("Could not obtain games for today...") + slog.Error("Could not obtain games for today from MLB...") slog.Error(err.Error()) os.Exit(1) } - var games = make([]Game, 0) + var mlbGames = make([]Game, len(gameIDs)) + + // Retrieve/update games for the day in database for today + databaseClient.dbMut.Lock() + + // Retrieve the games for today from the database (as a batch job to update issues with delays) + BatchUpdateTime(databaseClient) + + localTime := time.Now().In(time.Local) - for rows.Next() { - var game Game - err := rows.Scan(&game.ID, &game.HomeTeam_ID, &game.AwayTeam_ID, &game.Date) + // Update dates in Database + for i, gid := range gameIDs { + err := databaseClient.db.QueryRow( + context.Background(), + `UPDATE games SET "date" = $1, "startTimeUTC" = $2 WHERE id = $3 RETURNING id, "homeTeam_id", "awayTeam_id", date`, + localTime.Format("2006-01-02"), gid.StartTime, gid.GameID).Scan(&mlbGames[i].ID, &mlbGames[i].HomeTeam_ID, &mlbGames[i].AwayTeam_ID, &mlbGames[i].Date) if err != nil { - slog.Error("Could not scan game...") + slog.Error("Could not insert/update game into database...") slog.Error(err.Error()) os.Exit(1) } - games = append(games, game) } - rows.Close() databaseClient.dbMut.Unlock() universalWait := &sync.WaitGroup{} - for _, game := range games { + for _, game := range mlbGames { universalWait.Add(1) go StatsJob(game.ID, universalWait) time.Sleep(500 * time.Millisecond) // Just to space things out a little. diff --git a/stats.go b/stats.go index e941fd8..25eafba 100644 --- a/stats.go +++ b/stats.go @@ -13,6 +13,7 @@ const ( Scheduled = "SCHEDULED" InProgress = "IN_PROGRESS" Completed = "COMPLETED" + Postponed = "POSTPONED" Unknown = "UNKNOWN" ) @@ -45,8 +46,8 @@ type CompletedGameStats struct { } type UnknownGameStats struct { - status string - gameID int + status string `redis:"status"` + gameID int `redis:"gameID"` } // Returns true or false if the game that was just handled was finished. @@ -78,6 +79,9 @@ func handleGameStats(gameID int, dbClient *DatabaseClient) (bool, error) { case InProgress: err := handleInProgressGame(gameStats, liveStats, dbClient) return false, err + default: + err := handlePostponedGame(gameStats, liveStats, dbClient) + return true, err } return false, nil @@ -91,19 +95,41 @@ func getGameType(gameStats map[string]interface{}) (string, error) { return "", gameStatusErr } + // Check postponed start that's in the system + startTime, sterr := getStartTime(gameStats) + if sterr != nil { + return "", sterr + } + localStartTime := startTime.In(time.Local) + + if localStartTime.YearDay() > time.Now().YearDay() { + return Postponed, nil + } + code := gameStatus["statusCode"].(string) + firstLetter := string(code[0]) - switch code { + switch firstLetter { case "S": // Scheduled - return Scheduled, nil - case "PW": // Warmup situation - return Scheduled, nil - case "F": // Final - return Completed, nil - case "O": // Game Over (used as separate before decisions) - return Completed, nil - case "I": // In Progress - return InProgress, nil + return Scheduled, nil // Scheduled + case "P": + return Scheduled, nil // Pregame -- also accounts for delayed start + case "I": + return InProgress, nil // In Progress -- also accounts for delayed in progress + case "M": + return InProgress, nil // Manager challenge -- In progress + case "N": + return InProgress, nil // Umpire review + case "D": + return Postponed, nil // Postponed/Cancelled + case "T": + return Completed, nil // Suspended + case "Q": + return Completed, nil // Forfeit -- but is anyone ever going to use this? + case "O": + return Completed, nil // Game Over (or completed early) + case "F": + return Completed, nil // Final default: return Unknown, nil } @@ -147,6 +173,10 @@ func handleScheduledGame(gameStats map[string]interface{}, client *DatabaseClien if rdErr != nil { return rdErr } + expireErr := client.redisClient.Expire(context.Background(), "game:"+strconv.Itoa(scheduledGameStats.GameID), time.Hour*24).Err() + if expireErr != nil { + return expireErr + } slog.Info("Game " + strconv.Itoa(scheduledGameStats.GameID) + " written to database") return nil } else if getRedisErr != nil { // Error, return @@ -241,7 +271,7 @@ func handleInProgressGame(gameStats map[string]interface{}, liveStats map[string return hsetErr } - quickDuration := time.Minute * 10 + quickDuration := time.Hour * 12 expireErr := client.redisClient.Expire( context.Background(), "game:"+strconv.Itoa(inProgressGameStats.GameID), @@ -254,6 +284,69 @@ func handleInProgressGame(gameStats map[string]interface{}, liveStats map[string } +func handlePostponedGame(gameStats map[string]interface{}, liveStats map[string]interface{}, client *DatabaseClient) error { + + gameID, gameIDErr := unwrap(gameStats, "game") + if gameIDErr != nil { + return gameIDErr + } + + newStartTime, sterr := getStartTime(gameStats) + if sterr != nil { + return sterr + } + + stats := ScheduledGameStats{ + Status: Scheduled, + GameID: int(gameID["pk"].(float64)), + StartTimeUTC: newStartTime.Format(time.RFC3339), + } + + slog.Info("Game status for game " + strconv.Itoa(stats.GameID) + " unknown, setting in database and skipping") + + client.redisMut.Lock() + defer client.redisMut.Unlock() + hsetErr := client.redisClient.HSet(context.Background(), "game:"+strconv.Itoa(stats.GameID), stats).Err() + if hsetErr != nil { + return hsetErr + } + expireErr := client.redisClient.Expire(context.Background(), "game:"+strconv.Itoa(stats.GameID), time.Hour*24).Err() + if expireErr != nil { + return expireErr + } + + client.dbMut.Lock() + defer client.dbMut.Unlock() + cctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + tx, err := client.db.Begin(cctx) + + defer cancel() + if err != nil { + return err + } + + defer func() { + if err != nil { + tx.Rollback(context.Background()) + } else { + tx.Commit(context.Background()) + } + }() + + _, err = client.db.Exec(cctx, ` + UPDATE games SET finished=$1, winner = NULL + WHERE id = $2`, + false, stats.GameID) + if err != nil { + return err + } + + UpdateTime(client, stats.GameID, newStartTime) + + err = tx.Commit(cctx) + return err +} + func getScores(liveData map[string]interface{}) (int, int, error) { lineScore, lineScoreErr := unwrap(liveData, "linescore") if lineScoreErr != nil { @@ -286,6 +379,18 @@ func getInnningInfo(liveData map[string]interface{}) (int, bool, int, error) { nil } +func getStartTime(gameData map[string]interface{}) (time.Time, error) { + datetime, datetimeErr := unwrap(gameData, "datetime") + if datetimeErr != nil { + return time.Now(), datetimeErr + } + startTime, timeParseErr := time.Parse("2006-01-02T15:04:05Z", datetime["dateTime"].(string)) + if timeParseErr != nil { + return time.Now(), timeParseErr + } + return startTime, nil +} + // Returns batter name, pitcher name, a list of length 3 for onFirst, onSecond, onThird, error func getAtBatInfo(gameData map[string]interface{}, liveData map[string]interface{}, isBottomInning bool) (string, string, []bool, error) { keys := [3]string{"plays", "currentPlay", "matchup"} @@ -382,7 +487,7 @@ func setFinalScoreRedis(stats CompletedGameStats, client *DatabaseClient) error return hsetErr } - oneDayDuration := time.Hour * 24 + oneDayDuration := time.Hour * 12 expireErr := client.redisClient.Expire( context.Background(), "game:"+strconv.Itoa(stats.GameID), @@ -399,6 +504,7 @@ func setFinalScoreDB(stats CompletedGameStats, client *DatabaseClient) error { defer client.dbMut.Unlock() cctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) tx, err := client.db.Begin(cctx) + homeWins := stats.HomeScore > stats.AwayScore defer cancel() if err != nil { @@ -416,24 +522,25 @@ func setFinalScoreDB(stats CompletedGameStats, client *DatabaseClient) error { _, err = client.db.Exec(cctx, ` UPDATE games SET finished=$1, home_score=$2, away_score=$3, winner = ( - CASE WHEN home_score > away_score THEN "homeTeam_id" - WHEN home_score < away_score THEN "awayTeam_id" - ELSE NULL + CASE WHEN $4 THEN "homeTeam_id" + ELSE "awayTeam_id" END ) - WHERE id = $4`, - true, stats.HomeScore, stats.AwayScore, stats.GameID) + WHERE id = $5`, + true, stats.HomeScore, stats.AwayScore, homeWins, stats.GameID) if err != nil { return err } _, err = client.db.Exec(cctx, ` - UPDATE picks set correct= - (CASE WHEN "pickedHome" = - (SELECT winner = "homeTeam_id" from games where id = $1 and winner IS NOT NULL) - THEN true - ELSE false END) WHERE game_id = $1`, stats.GameID) + UPDATE picks p set correct = + (CASE + WHEN "pickedHome" = true AND $2 THEN true + WHEN "pickedHome" = false AND not $2 THEN true + ELSE false + END) + WHERE game_id = $1;`, stats.GameID, homeWins) if err != nil { return err