From 7fb96632159cd30c55d6a02b52cd99de492ad54c Mon Sep 17 00:00:00 2001 From: Calvin Hendryx-Parker Date: Tue, 6 May 2025 18:06:54 -0400 Subject: [PATCH 1/3] feat: enhance Makefile to manage DynamoDB local container and table creation - Added conditional logic in the `dynamo-up` target to start the DynamoDB local container only if it is not already running, using a container ID file to store the ID. - Updated `dynamo-down` target to stop the running DynamoDB local container using the saved container ID and clear the ID file afterward, improving cleanup. - Modified `create-table` target to create the table with parameterized attributes and provide feedback using `jq` for better JSON output formatting if it is installed. - Eliminated the previous hardcoded table creation parameters for better flexibility and future scalability. --- {{copier__project_name}}/Makefile | 47 +++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/{{copier__project_name}}/Makefile b/{{copier__project_name}}/Makefile index daa1025..8fa39d7 100644 --- a/{{copier__project_name}}/Makefile +++ b/{{copier__project_name}}/Makefile @@ -29,22 +29,45 @@ invoke: {% if copier__dynamo_db %} dynamo-up: - docker-compose up -d dynamodb-local + @if [ -f .dynamodb-container-id ] && [ -n "$$(docker ps -q -f id=$$(cat .dynamodb-container-id 2>/dev/null))" ]; then \ + echo "DynamoDB local container is already running with ID: $$(cat .dynamodb-container-id)"; \ + else \ + echo "Starting DynamoDB local container..."; \ + rm -f .dynamodb-container-id 2>/dev/null || true; \ + docker run -d --rm -p 8000:8000 --name dynamodb-local amazon/dynamodb-local -jar DynamoDBLocal.jar -sharedDb > .dynamodb-container-id; \ + echo "DynamoDB local container started with ID: $$(cat .dynamodb-container-id)"; \ + fi dynamo-down: - dokcer-compose down + @if [ -f .dynamodb-container-id ]; then \ + if [ -n "$$(docker ps -q -f id=$$(cat .dynamodb-container-id 2>/dev/null))" ]; then \ + echo "Stopping DynamoDB local container with ID: $$(cat .dynamodb-container-id)"; \ + docker stop $$(cat .dynamodb-container-id); \ + else \ + echo "No running container found with saved ID"; \ + fi; \ + rm -f .dynamodb-container-id; \ + else \ + echo "No container ID file found"; \ + fi create-table: - aws dynamodb create-table \ - --table-name RequestsTable \ - --attribute-definitions \ - AttributeName=ip_address,AttributeType=S \ - AttributeName=timestamp,AttributeType=S \ - --key-schema \ - AttributeName=ip_address,KeyType=HASH \ - AttributeName=timestamp,KeyType=RANGE \ - --billing-mode PAY_PER_REQUEST \ - --endpoint-url $(DYNAMO_ENDPOINT) + @# Store the AWS command output in a variable + $(eval AWS_OUTPUT := $(shell aws dynamodb create-table \ + --table-name $(TABLE_NAME) \ + --attribute-definitions AttributeName=id,AttributeType=S \ + --key-schema AttributeName=id,KeyType=HASH \ + --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5 \ + --endpoint-url $(DYNAMO_ENDPOINT) \ + --no-cli-pager)) + + @# Check if jq exists and use it if available + @if command -v jq > /dev/null 2>&1; then \ + echo '$(AWS_OUTPUT)' | jq; \ + else \ + echo '$(AWS_OUTPUT)'; \ + echo "\033[33mNote: Install jq for colorized JSON output\033[0m"; \ + fi setup: dynamo-up create-table{% else %} setup: From 0d097b9a06f9653220b8aab46d042311ea53e8b5 Mon Sep 17 00:00:00 2001 From: Calvin Hendryx-Parker Date: Tue, 6 May 2025 18:06:54 -0400 Subject: [PATCH 2/3] feat: enhance Makefile to manage DynamoDB local container and table creation - Added conditional logic in the `dynamo-up` target to start the DynamoDB local container only if it is not already running, using a container ID file to store the ID. - Updated `dynamo-down` target to stop the running DynamoDB local container using the saved container ID and clear the ID file afterward, improving cleanup. - Modified `create-table` target to create the table with parameterized attributes and provide feedback using `jq` for better JSON output formatting if it is installed. - Eliminated the previous hardcoded table creation parameters for better flexibility and future scalability. --- {{copier__project_name}}/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/{{copier__project_name}}/Makefile b/{{copier__project_name}}/Makefile index 8fa39d7..1c14f1d 100644 --- a/{{copier__project_name}}/Makefile +++ b/{{copier__project_name}}/Makefile @@ -5,7 +5,8 @@ STACK_NAME ?= {{ copier__project_name }} REGION ?= us-east-1 TEMPLATE_FILE ?= template.yaml BUILD_DIR ?= .aws-sam/build -{% if copier__dynamo_db %}DYNAMO_ENDPOINT ?= http://localhost:8000{% endif %} +{% if copier__dynamo_db %}DYNAMO_ENDPOINT ?= http://localhost:8000 +TABLE_NAME = RequestsTable{% endif %} # Default target all: build From be45ecc8dc781aec1f9fb72def9bcd28711a54ee Mon Sep 17 00:00:00 2001 From: Muhammad Rehan Date: Thu, 8 May 2025 20:48:21 +0500 Subject: [PATCH 3/3] Add initial commit on project creation Remove docker-compose dependency Make Docker python image dynamic based on runtime and architecture Update project structure to match scaf-leaderboard repo --- aws-lambda-template.test-data.yaml | 7 +-- copier.yml | 22 +++++++--- tasks.py | 43 ++++++++++++++++--- {{copier__project_name}}/.envrc | 3 ++ {{copier__project_name}}/.gitignore | 12 +++++- {{copier__project_name}}/Makefile | 31 ++++++++++--- {{copier__project_name}}/samconfig.toml | 36 ++++++++++++++++ {{copier__project_name}}/{ => src}/app.py | 0 .../{ => src}/requirements.txt | 0 ...if copier__auth %}authorizer.py{% endif %} | 0 ...type == \"image\" %}Dockerfile{% endif %}" | 17 ++++++++ {{copier__project_name}}/template.yaml | 14 +++--- ...pier__auth %}docker-compose.yml{% endif %} | 17 -------- ...copier__auth %}env_example.json{% endif %} | 4 +- ...type == \"image\" %}Dockerfile{% endif %}" | 8 ---- 15 files changed, 157 insertions(+), 57 deletions(-) create mode 100644 {{copier__project_name}}/.envrc create mode 100644 {{copier__project_name}}/samconfig.toml rename {{copier__project_name}}/{ => src}/app.py (100%) rename {{copier__project_name}}/{ => src}/requirements.txt (100%) rename {{copier__project_name}}/{ => src}/{% if copier__auth %}authorizer.py{% endif %} (100%) create mode 100644 "{{copier__project_name}}/src/{% if copier__package_type == \"image\" %}Dockerfile{% endif %}" delete mode 100644 {{copier__project_name}}/{% if copier__auth %}docker-compose.yml{% endif %} delete mode 100644 "{{copier__project_name}}/{% if copier__package_type == \"image\" %}Dockerfile{% endif %}" diff --git a/aws-lambda-template.test-data.yaml b/aws-lambda-template.test-data.yaml index 76917dc..247cc12 100644 --- a/aws-lambda-template.test-data.yaml +++ b/aws-lambda-template.test-data.yaml @@ -1,9 +1,10 @@ copier__project_name: test_project copier__runtime: "python3.11" -copier__architectures: ["x86_64"] -copier__package_type: "zip" +copier__architectures: "x86_64" +copier__package_type: "image" copier__xray_tracing: y copier__cloudwatch_monitor: y copier__enable_logs: y copier__auth: y -copier__dynamo_db: y \ No newline at end of file +copier__dynamo_db: y +copier__repo_url: git@github.com:organization_name/test_project.git \ No newline at end of file diff --git a/copier.yml b/copier.yml index 51db33c..b18c862 100644 --- a/copier.yml +++ b/copier.yml @@ -30,17 +30,15 @@ copier__runtime: default: "python3.11" help: "The python version of your project." choices: - python3.8: "python3.8" - python3.9: "python3.9" - python3.10: "python3.10" - python3.11: "python3.11" - python3.12: "python3.12" + "3.9": "python3.9" + "3.10": "python3.10" + "3.11": "python3.11" + "3.12": "python3.12" copier__architectures: type: str help: "The architecture of your project." - default: ["arm64"] - multiselect: true + default: "arm64" choices: x86_64: "x86_64" arm64: "arm64" @@ -82,3 +80,13 @@ copier__stack_name: type: str default: "{{ copier__project_name.lower().replace(' ', '-') }}" when: false + +copier__repo_url: + type: str + default: "git@github.com:organization_name/{{ copier__stack_name }}.git" + help: "The URL of the repository." + validator: >- + {% if not copier__repo_url.startswith("git@") or not ":" in copier__repo_url or not "/" in copier__repo_url.split(":")[1] or not ".git" in copier__repo_url %} + Value must follow the format git@provider:orgname/repo.git + {% endif %} + diff --git a/tasks.py b/tasks.py index e1751d7..40749a4 100644 --- a/tasks.py +++ b/tasks.py @@ -1,23 +1,56 @@ import os -import random import shlex import shutil -import string import subprocess +TERMINATOR = "\x1b[0m" +WARNING = "\x1b[1;33m [WARNING]: " +INFO = "\x1b[1;33m [INFO]: " +HINT = "\x1b[3;33m" +SUCCESS = "\x1b[1;32m [SUCCESS]: " + + def run_setup(): + print("Performing initial commit.") + subprocess.run(shlex.split("git add .")) + subprocess.run(shlex.split("git commit -m 'Initial commit' --quiet")) + if not shutil.which("sam"): print("Error: AWS SAM CLI is not installed. Please install it and try again.") exit(1) - + print("Running AWS SAM build and validate...") - subprocess.run(shlex.split("sam validate --lint")) - subprocess.run(shlex.split("sam build")) + subprocess.run(shlex.split("make validate")) + subprocess.run(shlex.split("make build")) print("AWS Lambda template setup complete.") +def init_git_repo(): + print(INFO + "Initializing git repository..." + TERMINATOR) + print(INFO + f"Current working directory: {os.getcwd()}" + TERMINATOR) + subprocess.run(shlex.split("git -c init.defaultBranch=main init . --quiet")) + print(SUCCESS + "Git repository initialized." + TERMINATOR) + + +def configure_git_remote(): + repo_url = "{{ copier__repo_url }}" + if repo_url: + print(INFO + f"repo_url: {repo_url}" + TERMINATOR) + command = f"git remote add origin {repo_url}" + subprocess.run(shlex.split(command)) + print(SUCCESS + f"Remote origin={repo_url} added." + TERMINATOR) + else: + print( + WARNING + + "No repo_url provided. Skipping git remote configuration." + + TERMINATOR + ) + + def main(): + init_git_repo() + configure_git_remote() run_setup() diff --git a/{{copier__project_name}}/.envrc b/{{copier__project_name}}/.envrc new file mode 100644 index 0000000..2d14708 --- /dev/null +++ b/{{copier__project_name}}/.envrc @@ -0,0 +1,3 @@ +export AWS_ACCESS_KEY_ID=dummyAccessKeyId +export AWS_SECRET_ACCESS_KEY=dummySecretAccessKey +export AWS_REGION=us-east-1 diff --git a/{{copier__project_name}}/.gitignore b/{{copier__project_name}}/.gitignore index 4808264..6b7d724 100644 --- a/{{copier__project_name}}/.gitignore +++ b/{{copier__project_name}}/.gitignore @@ -1,3 +1,7 @@ +# Project specific ignores + +env.json +.dynamodb-container-id # Created by https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode @@ -241,4 +245,10 @@ $RECYCLE.BIN/ */build/* -# End of https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode \ No newline at end of file +# End of https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode + +# gitignore template for AWS Serverless Application Model project +# website: https://docs.aws.amazon.com/serverless-application-model + +# Ignore build folder +.aws-sam/ diff --git a/{{copier__project_name}}/Makefile b/{{copier__project_name}}/Makefile index 1c14f1d..9467600 100644 --- a/{{copier__project_name}}/Makefile +++ b/{{copier__project_name}}/Makefile @@ -14,6 +14,9 @@ all: build build: sam build {% if copier__package_type != "image" %}--use-container{% endif %} +validate: + sam validate --lint + deploy: sam deploy --guided @@ -35,7 +38,7 @@ dynamo-up: else \ echo "Starting DynamoDB local container..."; \ rm -f .dynamodb-container-id 2>/dev/null || true; \ - docker run -d --rm -p 8000:8000 --name dynamodb-local amazon/dynamodb-local -jar DynamoDBLocal.jar -sharedDb > .dynamodb-container-id; \ + docker run -d --rm --network lambda-local -p 8000:8000 --name dynamodb-local amazon/dynamodb-local:latest -jar DynamoDBLocal.jar -sharedDb > .dynamodb-container-id; \ echo "DynamoDB local container started with ID: $$(cat .dynamodb-container-id)"; \ fi @@ -52,13 +55,31 @@ dynamo-down: echo "No container ID file found"; \ fi +wait-for-dynamodb: + @echo "Waiting for DynamoDB Local to be ready..." + @attempts=0; \ + until curl -s http://localhost:8000 >/dev/null; do \ + if [ $$attempts -ge 10 ]; then \ + echo "DynamoDB Local did not become ready in time"; \ + exit 1; \ + fi; \ + echo "Waiting..."; \ + sleep 1; \ + attempts=$$((attempts + 1)); \ + done; \ + echo "DynamoDB Local is ready." + create-table: @# Store the AWS command output in a variable $(eval AWS_OUTPUT := $(shell aws dynamodb create-table \ --table-name $(TABLE_NAME) \ - --attribute-definitions AttributeName=id,AttributeType=S \ - --key-schema AttributeName=id,KeyType=HASH \ - --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5 \ + --attribute-definitions \ + AttributeName=ip_address,AttributeType=S \ + AttributeName=timestamp,AttributeType=S \ + --key-schema \ + AttributeName=ip_address,KeyType=HASH \ + AttributeName=timestamp,KeyType=RANGE \ + --billing-mode PAY_PER_REQUEST \ --endpoint-url $(DYNAMO_ENDPOINT) \ --no-cli-pager)) @@ -70,7 +91,7 @@ create-table: echo "\033[33mNote: Install jq for colorized JSON output\033[0m"; \ fi -setup: dynamo-up create-table{% else %} +setup: dynamo-up wait-for-dynamodb create-table{% else %} setup: pip install -r requirements.txt{% endif %} diff --git a/{{copier__project_name}}/samconfig.toml b/{{copier__project_name}}/samconfig.toml new file mode 100644 index 0000000..4f37994 --- /dev/null +++ b/{{copier__project_name}}/samconfig.toml @@ -0,0 +1,36 @@ +# More information about the configuration file can be found here: +# https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-config.html +version = 0.1 + +[default] +[default.global.parameters] +stack_name = "{{copier__stack_name}}" + +[default.build.parameters] +cached = true +parallel = true + +[default.validate.parameters] +lint = true + +[default.deploy.parameters] +capabilities = "CAPABILITY_IAM" +confirm_changeset = true +resolve_s3 = true +s3_prefix = "{{copier__stack_name}}" +region = "us-east-1" +profile = "{{copier__stack_name}}" +image_repositories = [] +parameter_overrides = "DynamoDbEndpoint=\"https://dynamodb.us-east-1.amazonaws.com\"" + +[default.package.parameters] +resolve_s3 = true + +[default.sync.parameters] +watch = true + +[default.local_start_api.parameters] +warm_containers = "EAGER" + +[default.local_start_lambda.parameters] +warm_containers = "EAGER" diff --git a/{{copier__project_name}}/app.py b/{{copier__project_name}}/src/app.py similarity index 100% rename from {{copier__project_name}}/app.py rename to {{copier__project_name}}/src/app.py diff --git a/{{copier__project_name}}/requirements.txt b/{{copier__project_name}}/src/requirements.txt similarity index 100% rename from {{copier__project_name}}/requirements.txt rename to {{copier__project_name}}/src/requirements.txt diff --git a/{{copier__project_name}}/{% if copier__auth %}authorizer.py{% endif %} b/{{copier__project_name}}/src/{% if copier__auth %}authorizer.py{% endif %} similarity index 100% rename from {{copier__project_name}}/{% if copier__auth %}authorizer.py{% endif %} rename to {{copier__project_name}}/src/{% if copier__auth %}authorizer.py{% endif %} diff --git "a/{{copier__project_name}}/src/{% if copier__package_type == \"image\" %}Dockerfile{% endif %}" "b/{{copier__project_name}}/src/{% if copier__package_type == \"image\" %}Dockerfile{% endif %}" new file mode 100644 index 0000000..1500497 --- /dev/null +++ "b/{{copier__project_name}}/src/{% if copier__package_type == \"image\" %}Dockerfile{% endif %}" @@ -0,0 +1,17 @@ +{% if copier__runtime == "python3.8" %} +FROM public.ecr.aws/lambda/python:3.8-{{copier__architectures}} +{% elif copier__runtime == "python3.9" %} +FROM public.ecr.aws/lambda/python:3.9-{{copier__architectures}} +{% elif copier__runtime == "python3.10" %} +FROM public.ecr.aws/lambda/python:3.10-{{copier__architectures}} +{% elif copier__runtime == "python3.11" %} +FROM public.ecr.aws/lambda/python:3.11-{{copier__architectures}} +{% elif copier__runtime == "python3.12" %} +FROM public.ecr.aws/lambda/python:3.12-{{copier__architectures}} +{% else %} +FROM public.ecr.aws/lambda/python:3.10-{{copier__architectures}} +{% endif %} +COPY app.py ${LAMBDA_TASK_ROOT} + +# Command can be overwritten by providing a different command in the template directly. +CMD ["app.lambda_handler"] diff --git a/{{copier__project_name}}/template.yaml b/{{copier__project_name}}/template.yaml index db7ad9e..7be1007 100644 --- a/{{copier__project_name}}/template.yaml +++ b/{{copier__project_name}}/template.yaml @@ -1,8 +1,6 @@ AWSTemplateFormatVersion: '2010-09-09' Transform: AWS::Serverless-2016-10-31 Description: > - {{ copier__project_name }} - Sample SAM Template for {{ copier__project_name }} # More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst @@ -57,14 +55,12 @@ Resources: {% if copier__package_type == "image" %} PackageType: Image {% else %} - CodeUri: ./ + CodeUri: ./src Handler: app.lambda_handler Runtime: {{ copier__runtime }} {% endif %} Architectures: - {%- for arch in copier__architectures %} - - {{arch}} - {%- endfor %} + - {{copier__architectures}} {% if copier__dynamo_db %} Environment: Variables: @@ -86,8 +82,8 @@ Resources: {% if copier__package_type == "image" %} Metadata: Dockerfile: Dockerfile - DockerContext: ./ - DockerTag: python3.13-v1 + DockerContext: ./src + DockerTag: {{copier__runtime}}-v1 {% endif %} {% if copier__cloudwatch_monitor %} ApplicationResourceGroup: @@ -108,7 +104,7 @@ Resources: PassageAuthFunction: Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction Properties: - CodeUri: ./ + CodeUri: ./src Handler: authorizer.handler Runtime: {{ copier__runtime }} Timeout: 10 diff --git a/{{copier__project_name}}/{% if copier__auth %}docker-compose.yml{% endif %} b/{{copier__project_name}}/{% if copier__auth %}docker-compose.yml{% endif %} deleted file mode 100644 index 3600086..0000000 --- a/{{copier__project_name}}/{% if copier__auth %}docker-compose.yml{% endif %} +++ /dev/null @@ -1,17 +0,0 @@ -version: '3.8' - -services: - dynamodb-local: - container_name: dynamodb-local - image: amazon/dynamodb-local:latest - ports: - - "8000:8000" - command: "-jar DynamoDBLocal.jar -sharedDb -dbPath /home/dynamodblocal/data" - volumes: - - "./dynamodb-data:/home/dynamodblocal/data" - networks: - - lambda-local - -networks: - lambda-local: - name: lambda-local \ No newline at end of file diff --git a/{{copier__project_name}}/{% if copier__auth %}env_example.json{% endif %} b/{{copier__project_name}}/{% if copier__auth %}env_example.json{% endif %} index 416bde6..b2fc11b 100644 --- a/{{copier__project_name}}/{% if copier__auth %}env_example.json{% endif %} +++ b/{{copier__project_name}}/{% if copier__auth %}env_example.json{% endif %} @@ -1,12 +1,12 @@ { "HelloWorldFunction": { - "DYNAMODB_ENDPOINT": "http://dynamo-local:8000", + "DYNAMODB_ENDPOINT": "http://dynamodb-local:8000", "DYNAMODB_TABLE_NAME": "RequestsTable" }, "PassageAuthFunction": { "PASSAGE_APP_ID": "CHANGEME", "PASSAGE_API_KEY": "CHANGEME", - "DYNAMODB_ENDPOINT": "http://dynamo-local:8000", + "DYNAMODB_ENDPOINT": "http://dynamodb-local:8000", "DYNAMODB_TABLE_NAME": "RequestsTable" } } \ No newline at end of file diff --git "a/{{copier__project_name}}/{% if copier__package_type == \"image\" %}Dockerfile{% endif %}" "b/{{copier__project_name}}/{% if copier__package_type == \"image\" %}Dockerfile{% endif %}" deleted file mode 100644 index c37d94f..0000000 --- "a/{{copier__project_name}}/{% if copier__package_type == \"image\" %}Dockerfile{% endif %}" +++ /dev/null @@ -1,8 +0,0 @@ -FROM public.ecr.aws/lambda/python:3.13 - -COPY app.py requirements.txt ./ - -RUN python3.13 -m pip install -r requirements.txt -t . - -# Command can be overwritten by providing a different command in the template directly. -CMD ["app.lambda_handler"]