From 594fcb73f7d646861fae5c055abe6cce1e007650 Mon Sep 17 00:00:00 2001 From: Alvin Crespo Date: Sun, 1 Oct 2023 21:33:38 -0400 Subject: [PATCH 1/8] feat(logging): standardize logging across utils and runners --- scripts/aws/ec/redis/replication-group/create | 6 +++-- scripts/aws/ec/redis/replication-group/delete | 6 +++-- scripts/aws/utils | 24 +++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/scripts/aws/ec/redis/replication-group/create b/scripts/aws/ec/redis/replication-group/create index 4be2bb8..b017672 100755 --- a/scripts/aws/ec/redis/replication-group/create +++ b/scripts/aws/ec/redis/replication-group/create @@ -9,7 +9,9 @@ check_aws_cli_installed . .env -echo "[UTILS] [EKS] Creating elasticache replication group." >&2 +set_log_context "[UTILS] [EKS]" + +info "Creating elasticache replication group." cmd="aws elasticache create-replication-group" @@ -36,4 +38,4 @@ cmd+=" --no-cli-pager" eval $cmd -echo "[UTILS] [EKS] Completed creating elasticache replication group." >&2 +info "Completed creating elasticache replication group." diff --git a/scripts/aws/ec/redis/replication-group/delete b/scripts/aws/ec/redis/replication-group/delete index 51707bc..58ba761 100755 --- a/scripts/aws/ec/redis/replication-group/delete +++ b/scripts/aws/ec/redis/replication-group/delete @@ -2,10 +2,12 @@ . .env -echo "[UTILS] [EC] [REPLICATION GROUP] Deleting elasticache replication group." >&2 +set_log_context "[UTILS] [EC] [REPLICATION GROUP]" + +info "Deleting elasticache replication group." aws elasticache delete-replication-group \ --replication-group-id "$AWS_REPLICATION_GROUP_ID" \ --no-cli-pager -echo "[UTILS] [EC] [REPLICATION GROUP] Scheduled deleting elasticache replication group." >&2 +info "[UTILS] [EC] [REPLICATION GROUP] Scheduled deleting elasticache replication group." diff --git a/scripts/aws/utils b/scripts/aws/utils index 5a77081..f0bad72 100755 --- a/scripts/aws/utils +++ b/scripts/aws/utils @@ -54,3 +54,27 @@ function check_docker_installed { echo "[UTILS] Confirmed docker is installed." >&2 } + +LOG_CONTEXT="" + +log() { + local level="$1" + shift + + local datetime=$(date +"%Y-%m-%d %H:%M:%S") + local message="$@" + + echo "[$datetime] [$level] $LOG_CONTEXT $message" >&2 +} + +info() { + log "INFO" "$@" +} + +error() { + log "ERROR" "$@" +} + +set_log_context() { + LOG_CONTEXT="$@" +} From f4943997a0f52f4c2ca872ff5b1f62e445dd18a5 Mon Sep 17 00:00:00 2001 From: Alvin Crespo Date: Sun, 1 Oct 2023 21:35:02 -0400 Subject: [PATCH 2/8] ignore .env file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 01c025d..18146d3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ *.gem +.env From 7bcbff1b4cc6ee1975db38eeb2654f34c5d61ac4 Mon Sep 17 00:00:00 2001 From: Alvin Crespo Date: Sun, 1 Oct 2023 21:50:48 -0400 Subject: [PATCH 3/8] implment AWS_SCRIPTS_BASE_DIR to define where scripts exist --- scripts/aws/ec/redis/replication-group/create | 6 +++--- scripts/aws/ec/redis/replication-group/delete | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/aws/ec/redis/replication-group/create b/scripts/aws/ec/redis/replication-group/create index b017672..46a1608 100755 --- a/scripts/aws/ec/redis/replication-group/create +++ b/scripts/aws/ec/redis/replication-group/create @@ -3,11 +3,11 @@ set -e set -o pipefail -. ./bin/aws/utils +. .env -check_aws_cli_installed +. $AWS_SCRIPTS_BASE_DIR/aws/utils -. .env +check_aws_cli_installed set_log_context "[UTILS] [EKS]" diff --git a/scripts/aws/ec/redis/replication-group/delete b/scripts/aws/ec/redis/replication-group/delete index 58ba761..8dc104b 100755 --- a/scripts/aws/ec/redis/replication-group/delete +++ b/scripts/aws/ec/redis/replication-group/delete @@ -2,6 +2,8 @@ . .env +. $AWS_SCRIPTS_BASE_DIR/aws/utils + set_log_context "[UTILS] [EC] [REPLICATION GROUP]" info "Deleting elasticache replication group." From bc5320569448a3df0ec5a2449101ac95ccad4cff Mon Sep 17 00:00:00 2001 From: Alvin Crespo Date: Sun, 1 Oct 2023 21:51:36 -0400 Subject: [PATCH 4/8] update .env.example with base dir env var with comment --- .env.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.env.example b/.env.example index 51f6330..f8a9d90 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,6 @@ +# Define relative path to aws scripts and runners +AWS_SCRIPTS_BASE_DIR="./bin" + # Redis AWS_REPLICATION_GROUP_ID="" AWS_REPLICATION_GROUP_DESCRIPTION="" From 56330e67bf96e7cd2387e5462c846789a7b9ba48 Mon Sep 17 00:00:00 2001 From: Alvin Crespo Date: Mon, 2 Oct 2023 07:19:26 -0400 Subject: [PATCH 5/8] implementing standardized logging for ed/replication-group module --- scripts/aws/ec/redis/replication-group/create | 2 +- scripts/aws/ec/redis/replication-group/delete | 5 +++++ scripts/aws/ec/redis/replication-group/describe | 11 +++++++++++ scripts/aws/ec/redis/replication-group/status | 11 +++++++++++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/scripts/aws/ec/redis/replication-group/create b/scripts/aws/ec/redis/replication-group/create index 46a1608..6751220 100755 --- a/scripts/aws/ec/redis/replication-group/create +++ b/scripts/aws/ec/redis/replication-group/create @@ -9,7 +9,7 @@ set -o pipefail check_aws_cli_installed -set_log_context "[UTILS] [EKS]" +set_log_context "[UTILS] [EC]" info "Creating elasticache replication group." diff --git a/scripts/aws/ec/redis/replication-group/delete b/scripts/aws/ec/redis/replication-group/delete index 8dc104b..847047c 100755 --- a/scripts/aws/ec/redis/replication-group/delete +++ b/scripts/aws/ec/redis/replication-group/delete @@ -1,9 +1,14 @@ #!/bin/sh +set -e +set -o pipefail + . .env . $AWS_SCRIPTS_BASE_DIR/aws/utils +check_aws_cli_installed + set_log_context "[UTILS] [EC] [REPLICATION GROUP]" info "Deleting elasticache replication group." diff --git a/scripts/aws/ec/redis/replication-group/describe b/scripts/aws/ec/redis/replication-group/describe index 9a08705..8ea97b0 100755 --- a/scripts/aws/ec/redis/replication-group/describe +++ b/scripts/aws/ec/redis/replication-group/describe @@ -1,7 +1,18 @@ #!/bin/sh +set -e +set -o pipefail + . .env +. $AWS_SCRIPTS_BASE_DIR/aws/utils + +check_aws_cli_installed + +set_log_context "[UTILS] [EC]" + +info "Describing elasticache replication group." + aws elasticache describe-replication-groups \ --replication-group-id "$AWS_REPLICATION_GROUP_ID" \ --query "ReplicationGroups[0].ConfigurationEndpoint.Address" \ diff --git a/scripts/aws/ec/redis/replication-group/status b/scripts/aws/ec/redis/replication-group/status index 50114ff..3ab1f1e 100755 --- a/scripts/aws/ec/redis/replication-group/status +++ b/scripts/aws/ec/redis/replication-group/status @@ -1,7 +1,18 @@ #!/bin/sh +set -e +set -o pipefail + . .env +. $AWS_SCRIPTS_BASE_DIR/aws/utils + +check_aws_cli_installed + +set_log_context "[UTILS] [EC]" + +info "Fetching status of elasticache replication group." + aws elasticache describe-replication-groups \ --replication-group-id "$AWS_REPLICATION_GROUP_ID" \ --query "ReplicationGroups[0].Status" \ From c1ec5e8b15ab12ae2e0358a6a6888279e1d932f2 Mon Sep 17 00:00:00 2001 From: Alvin Crespo Date: Mon, 2 Oct 2023 07:38:35 -0400 Subject: [PATCH 6/8] implement standardized logging to utils --- scripts/aws/utils | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/scripts/aws/utils b/scripts/aws/utils index f0bad72..79b4468 100755 --- a/scripts/aws/utils +++ b/scripts/aws/utils @@ -1,58 +1,68 @@ #!/bin/bash function check_aws_cli_installed { - echo "[UTILS] Checking AWS CLI is installed." >&2 + set_log_context "[UTILS]" + + info "Checking AWS CLI is installed." if ! command -v aws &> /dev/null; then - echo "AWS CLI is not installed. Please install it first. https://aws.amazon.com/cli/" >&2 + error "AWS CLI is not installed. Please install it first. https://aws.amazon.com/cli/" exit 1 fi - echo "[UTILS] Confirmed AWS CLI is installed." >&2 + info "Confirmed AWS CLI is installed." } function check_helm_installed { - echo "[UTILS] Checking Helm is installed." >&2 + set_log_context "[UTILS]" + + info "Checking Helm is installed." if ! command -v helm &> /dev/null; then - echo "hl could not be found. Please install it first. https://helm.sh/" >&2 + error "hl could not be found. Please install it first. https://helm.sh/" exit 1 fi - echo "[UTILS] Confirmed Helm is installed." >&2 + info "Confirmed Helm is installed." } function check_kubectl_installed { - echo "[UTILS] Checking kubectl is installed." >&2 + set_log_context "[UTILS]" + + info "Checking kubectl is installed." if ! command -v helm &> /dev/null; then - echo "kubectl could not be found. Please install it first. https://kubernetes.io/docs/reference/kubectl/" >&2 + error "kubectl could not be found. Please install it first. https://kubernetes.io/docs/reference/kubectl/" exit 1 fi - echo "[UTILS] Confirmed kubectl is installed." >&2 + info "Confirmed kubectl is installed." } function check_eksctl_installed { - echo "[UTILS] Checking eksctl is installed." >&2 + set_log_context "[UTILS]" + + info "Checking eksctl is installed." if ! command -v eksctl &> /dev/null; then - echo "eksctl could not be found. Please install it first. https://eksctl.io/" >&2 + error "eksctl could not be found. Please install it first. https://eksctl.io/" exit 1 fi - echo "[UTILS] Confirmed eksctl is installed." >&2 + info "Confirmed eksctl is installed." } function check_docker_installed { - echo "[UTILS] Checking docker is installed." >&2 + set_log_context "[UTILS]" + + info "Checking docker is installed." if ! command -v docker &> /dev/null; then - echo "Docker could not be found. Please install it first. https://www.docker.com/" >&2 + error "Docker could not be found. Please install it first. https://www.docker.com/" exit 1 fi - echo "[UTILS] Confirmed docker is installed." >&2 + info "Confirmed docker is installed." } LOG_CONTEXT="" From 5317705073b6b77ce7a586cff739a9115ab0078b Mon Sep 17 00:00:00 2001 From: Alvin Crespo Date: Mon, 2 Oct 2023 07:44:58 -0400 Subject: [PATCH 7/8] implement standardize logging for ec/subnet-groups --- .../subnet-group => ec/subnet-groups}/create | 16 +++++++++------- scripts/aws/ec/subnet-groups/delete | 10 ++++++---- 2 files changed, 15 insertions(+), 11 deletions(-) rename scripts/aws/{eks/subnet-group => ec/subnet-groups}/create (57%) diff --git a/scripts/aws/eks/subnet-group/create b/scripts/aws/ec/subnet-groups/create similarity index 57% rename from scripts/aws/eks/subnet-group/create rename to scripts/aws/ec/subnet-groups/create index 6aa5f03..5c06461 100755 --- a/scripts/aws/eks/subnet-group/create +++ b/scripts/aws/ec/subnet-groups/create @@ -3,19 +3,21 @@ set -e set -o pipefail -. ./bin/aws/utils +. .env + +. $AWS_SCRIPTS_BASE_DIR/aws/utils check_aws_cli_installed -. .env +set_log_context "[UTILS] [EC]" -echo "[UTILS] [EKS] Creating elasticache subnet group." >&2 +info "Creating elasticache subnet group." cmd="aws elasticache create-cache-subnet-group" -cache_subnet_group_name="${EC_SUBNET_GROUP_NAME:-}" -cache_subnet_group_description="${EC_SUBNET_GROUP_DESCRIPTION:-}" -subnet_ids="${SUBNET_IDS:-}" +cache_subnet_group_name="${AWS_EC_SUBNET_GROUP_NAME:-}" +cache_subnet_group_description="${AWS_EC_SUBNET_GROUP_DESCRIPTION:-}" +subnet_ids="${AWS_VPC_SUBNET_IDS:-}" [[ -n "$cache_subnet_group_name" ]] && cmd+=" --cache-subnet-group-name \"$cache_subnet_group_name\"" [[ -n "$cache_subnet_group_description" ]] && cmd+=" --cache-subnet-group-description \"$cache_subnet_group_description\"" @@ -25,4 +27,4 @@ cmd+=" --no-cli-pager" eval $cmd -echo "[UTILS] [EKS] Completed creating elasticache subnet group." >&2 +info "Completed creating elasticache subnet group." diff --git a/scripts/aws/ec/subnet-groups/delete b/scripts/aws/ec/subnet-groups/delete index 68d205c..9505906 100755 --- a/scripts/aws/ec/subnet-groups/delete +++ b/scripts/aws/ec/subnet-groups/delete @@ -3,16 +3,18 @@ set -e set -o pipefail -. ./bin/aws/utils +. .env + +. $AWS_SCRIPTS_BASE_DIR/aws/utils check_aws_cli_installed -. .env +set_log_context "[UTILS] [EC] [SUBNET GROUP]" -echo "[UTILS] [EC] [SUBNET GROUP] Delete elasticache subnet group. ${AWS_ELASTICACHE_SUBNET_GROUP_ID}" +info "Delete elasticache subnet group. ${AWS_ELASTICACHE_SUBNET_GROUP_ID}" aws elasticache delete-cache-subnet-group \ --cache-subnet-group-name "$AWS_ELASTICACHE_SUBNET_GROUP_ID" \ --no-cli-pager -echo "[UTILS] [EC] [SUBNET GROUP] Deleted elasticache subnet group. ${AWS_ELASTICACHE_SUBNET_GROUP_ID}" +info "Deleted elasticache subnet group. ${AWS_ELASTICACHE_SUBNET_GROUP_ID}" From 634225d48febb98c70e3ebe8e1bdb5b7494126c9 Mon Sep 17 00:00:00 2001 From: Alvin Crespo Date: Mon, 2 Oct 2023 07:46:11 -0400 Subject: [PATCH 8/8] add new env vars for subnet group to .env.example --- .env.example | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.env.example b/.env.example index f8a9d90..1bfa77e 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,9 @@ # Define relative path to aws scripts and runners AWS_SCRIPTS_BASE_DIR="./bin" +# VPC +AWS_VPC_SUBNET_IDS="" + # Redis AWS_REPLICATION_GROUP_ID="" AWS_REPLICATION_GROUP_DESCRIPTION="" @@ -9,6 +12,8 @@ AWS_REPLICATION_ENGINE="" AWS_REPLICATION_CACHE_PARAMETER_GROUP_NAME="" AWS_REPLICATION_CACHE_NUMBER_OF_CLUSTERS="" AWS_ELASTICACHE_SUBNET_GROUP_ID="" +AWS_EC_SUBNET_GROUP_NAME="" +AWS_EC_SUBNET_GROUP_DESCRIPTION="" # Database AWS_DB_INSTANCE_ID=""