From 55bf4ea2b6b31ced3295d5f0f936499e082416dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Wed, 5 Nov 2025 17:25:19 +0100 Subject: [PATCH 01/32] feat: workflow for s3 added --- .../cloudformation-s3cli-iam.template.json | 84 +++++++ ...rmation-s3cli-private-bucket.template.json | 14 ++ ...ormation-s3cli-public-bucket.template.json | 30 +++ .github/scripts/s3/assets/lambda_function.py | 22 ++ .../scripts/s3/run-integration-aws-assume.sh | 34 +++ .github/scripts/s3/run-integration-aws-iam.sh | 107 +++++++++ .github/scripts/s3/run-integration-aws.sh | 49 ++++ .../scripts/s3/run-integration-s3-compat.sh | 26 +++ .../scripts/s3/setup-aws-infrastructure.sh | 52 +++++ .github/scripts/s3/teardown-infrastructure.sh | 44 ++++ .github/scripts/s3/utils.sh | 20 ++ .github/workflows/s3-integration.yml | 209 ++++++++++++++++++ 12 files changed, 691 insertions(+) create mode 100644 .github/scripts/s3/assets/cloudformation-s3cli-iam.template.json create mode 100644 .github/scripts/s3/assets/cloudformation-s3cli-private-bucket.template.json create mode 100644 .github/scripts/s3/assets/cloudformation-s3cli-public-bucket.template.json create mode 100644 .github/scripts/s3/assets/lambda_function.py create mode 100755 .github/scripts/s3/run-integration-aws-assume.sh create mode 100755 .github/scripts/s3/run-integration-aws-iam.sh create mode 100755 .github/scripts/s3/run-integration-aws.sh create mode 100755 .github/scripts/s3/run-integration-s3-compat.sh create mode 100755 .github/scripts/s3/setup-aws-infrastructure.sh create mode 100755 .github/scripts/s3/teardown-infrastructure.sh create mode 100755 .github/scripts/s3/utils.sh create mode 100644 .github/workflows/s3-integration.yml diff --git a/.github/scripts/s3/assets/cloudformation-s3cli-iam.template.json b/.github/scripts/s3/assets/cloudformation-s3cli-iam.template.json new file mode 100644 index 0000000..e836b65 --- /dev/null +++ b/.github/scripts/s3/assets/cloudformation-s3cli-iam.template.json @@ -0,0 +1,84 @@ +{ + "Resources": { + "S3Bucket": { + "Type": "AWS::S3::Bucket", + "DeletionPolicy": "Delete", + "Properties": { + "AccessControl": "Private" + } + }, + "Role": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + }, + "Action": [ + "sts:AssumeRole" + ] + } + ] + }, + "Path": "/", + "Policies": [ + { + "PolicyName": "S3CLIPermissions", + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": "arn:aws:logs:*:*:*" + }, + { + "Action": [ + "s3:GetObject*", + "s3:PutObject*", + "s3:List*", + "s3:DeleteObject*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:aws:s3:::", + { "Ref": "S3Bucket" } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:aws:s3:::", + { "Ref": "S3Bucket" }, + "/*" + ] + ] + } + ] + } + ] + } + } + ] + } + } + }, + "Outputs": { + "BucketName": { "Value": { "Ref": "S3Bucket" }}, + "IamRoleArn": { "Value": {"Fn::GetAtt" : ["Role", "Arn"] }} + } +} diff --git a/.github/scripts/s3/assets/cloudformation-s3cli-private-bucket.template.json b/.github/scripts/s3/assets/cloudformation-s3cli-private-bucket.template.json new file mode 100644 index 0000000..f9bf0a3 --- /dev/null +++ b/.github/scripts/s3/assets/cloudformation-s3cli-private-bucket.template.json @@ -0,0 +1,14 @@ +{ + "Resources": { + "S3Bucket": { + "Type": "AWS::S3::Bucket", + "DeletionPolicy": "Delete", + "Properties": { + "AccessControl": "Private" + } + } + }, + "Outputs": { + "BucketName": { "Value": { "Ref": "S3Bucket" }} + } +} diff --git a/.github/scripts/s3/assets/cloudformation-s3cli-public-bucket.template.json b/.github/scripts/s3/assets/cloudformation-s3cli-public-bucket.template.json new file mode 100644 index 0000000..1e8e17b --- /dev/null +++ b/.github/scripts/s3/assets/cloudformation-s3cli-public-bucket.template.json @@ -0,0 +1,30 @@ +{ + "Resources": { + "S3PublicReadBucket": { + "Type": "AWS::S3::Bucket", + "DeletionPolicy": "Delete", + "Properties": { + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": false, + "BlockPublicPolicy": false, + "IgnorePublicAcls": false, + "RestrictPublicBuckets": false + }, + "OwnershipControls": { + "Rules": [ + { + "ObjectOwnership": "ObjectWriter" + } + ] + } + } + } + }, + "Outputs": { + "BucketName": { + "Value": { + "Ref": "S3PublicReadBucket" + } + } + } +} \ No newline at end of file diff --git a/.github/scripts/s3/assets/lambda_function.py b/.github/scripts/s3/assets/lambda_function.py new file mode 100644 index 0000000..bcbbcf5 --- /dev/null +++ b/.github/scripts/s3/assets/lambda_function.py @@ -0,0 +1,22 @@ +import os +import logging +import subprocess + +def test_runner_handler(event, context): + os.environ['S3_CLI_PATH'] = './s3cli' + os.environ['BUCKET_NAME'] = event['bucket_name'] + os.environ['REGION'] = event['region'] + os.environ['S3_HOST'] = event['s3_host'] + + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + + try: + output = subprocess.check_output(['./integration.test', '-ginkgo.focus', 'AWS STANDARD IAM ROLE'], + env=os.environ, stderr=subprocess.STDOUT) + logger.debug("INTEGRATION TEST OUTPUT:") + logger.debug(output) + except subprocess.CalledProcessError as e: + logger.debug("INTEGRATION TEST EXITED WITH STATUS: " + str(e.returncode)) + logger.debug(e.output) + raise diff --git a/.github/scripts/s3/run-integration-aws-assume.sh b/.github/scripts/s3/run-integration-aws-assume.sh new file mode 100755 index 0000000..0fb4a93 --- /dev/null +++ b/.github/scripts/s3/run-integration-aws-assume.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + + +# Get the directory where this script is located +script_dir="$( cd "$(dirname "${0}")" && pwd )" + +# Source utils from the same directory +source "${script_dir}/utils.sh" + +: "${access_key_id:?}" +: "${secret_access_key:?}" +: "${region_name:=unset}" +: "${focus_regex:?}" +: "${assume_role_arn:=unset}" +: "${s3_endpoint_host:=unset}" + + +# Just need these to get the stack info +export AWS_ACCESS_KEY_ID=${access_key_id} +export AWS_SECRET_ACCESS_KEY=${secret_access_key} +export AWS_DEFAULT_REGION=${region_name} +export ASSUME_ROLE_ARN=${assume_role_arn} + +# Some of these are optional +export ACCESS_KEY_ID=${access_key_id} +export SECRET_ACCESS_KEY=${secret_access_key} +export REGION=${region_name} +export S3_HOST=${s3_endpoint_host} + +pushd "${release_dir}" > /dev/null + echo -e "\n running tests with $(go version)..." + scripts/ginkgo -r --focus="${focus_regex}" s3/integration/ +popd > /dev/null diff --git a/.github/scripts/s3/run-integration-aws-iam.sh b/.github/scripts/s3/run-integration-aws-iam.sh new file mode 100755 index 0000000..22e4b52 --- /dev/null +++ b/.github/scripts/s3/run-integration-aws-iam.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +set -euo pipefail + + +# Get the directory where this script is located +script_dir="$( cd "$(dirname "${0}")" && pwd )" + +# Source utils from the same directory +source "${script_dir}/utils.sh" + +: "${access_key_id:?}" +: "${secret_access_key:?}" +: "${region_name:?}" +: "${stack_name:?}" + +# Just need these to get the stack info and to create/invoke the Lambda function +export AWS_ACCESS_KEY_ID=${access_key_id} +export AWS_SECRET_ACCESS_KEY=${secret_access_key} +export AWS_DEFAULT_REGION=${region_name} + +stack_info=$(get_stack_info "${stack_name}") +bucket_name=$(get_stack_info_of "${stack_info}" "BucketName") +iam_role_arn=$(get_stack_info_of "${stack_info}" "IamRoleArn") +lambda_payload="{\"region\": \"${region_name}\", \"bucket_name\": \"${bucket_name}\", \"s3_host\": \"s3.amazonaws.com\"}" + +lambda_log=$(mktemp -t "XXXXXX-lambda.log") +trap "cat ${lambda_log}" EXIT + +pushd "${release_dir}" > /dev/null + echo -e "\n building artifact with $(go version)..." +# TODO change repo in here later + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o out/s3cli \ + github.com/cloudfoundry/bosh-s3cli + CGO_ENABLED=0 scripts/ginkgo build s3/integration + + zip -j payload.zip s3/integration/integration.test out/s3cli ${script_dir}/assets/lambda_function.py + + lambda_function_name=s3cli-integration-$(date +%s) + + aws lambda create-function \ + --region "${region_name}" \ + --function-name "${lambda_function_name}" \ + --zip-file fileb://payload.zip \ + --role "${iam_role_arn}" \ + --timeout 300 \ + --handler lambda_function.test_runner_handler \ + --runtime python3.9 + + set +e + tries=0 + get_function_status_command="aws lambda get-function --region ${region_name} --function-name ${lambda_function_name}" + function_status=$(${get_function_status_command}) + while [[ ( $(echo "${function_status}" | jq -r ".Configuration.State") != "Active" ) && ( $tries -ne 5 ) ]] ; do + sleep 2 + echo "Checking for function readiness; attempt: $tries" + tries=$((tries + 1)) + function_status=$(${get_function_status_command}) + done + set -e + + aws lambda invoke \ + --invocation-type RequestResponse \ + --function-name "${lambda_function_name}" \ + --region "${region_name}" \ + --log-type Tail \ + --payload "${lambda_payload}" \ + "${lambda_log}" | tee lambda_output.json + + set +e + log_group_name="/aws/lambda/${lambda_function_name}" + + logs_command="aws logs describe-log-streams --log-group-name=${log_group_name}" + tries=0 + + log_streams_json=$(${logs_command}) + while [[ ( $? -ne 0 ) && ( $tries -ne 5 ) ]] ; do + sleep 2 + echo "Retrieving CloudWatch logs; attempt: $tries" + tries=$((tries + 1)) + log_streams_json=$(${logs_command}) + done + set -e + + log_stream_name=$(echo "${log_streams_json}" | jq -r ".logStreams[0].logStreamName") + + echo "Lambda execution log output for ${log_stream_name}" + + tries=0 + > lambda_output.log + while [[ ( "$(du lambda_output.log | cut -f 1)" -eq "0" ) && ( $tries -ne 20 ) ]] ; do + sleep 2 + tries=$((tries + 1)) + echo "Retrieving CloudWatch events; attempt: $tries" + + aws logs get-log-events \ + --log-group-name="${log_group_name}" \ + --log-stream-name="${log_stream_name}" \ + | jq -r ".events | map(.message) | .[]" | tee lambda_output.log + done + + aws lambda delete-function \ + --function-name "${lambda_function_name}" + + aws logs delete-log-group --log-group-name="${log_group_name}" + + jq -r ".FunctionError" < lambda_output.json | grep -v -e "Handled" -e "Unhandled" +popd > /dev/null diff --git a/.github/scripts/s3/run-integration-aws.sh b/.github/scripts/s3/run-integration-aws.sh new file mode 100755 index 0000000..bff200e --- /dev/null +++ b/.github/scripts/s3/run-integration-aws.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + + +# Get the directory where this script is located +script_dir="$( cd "$(dirname "${0}")" && pwd )" + +# Source utils from the same directory +source "${script_dir}/utils.sh" + +: "${access_key_id:?}" +: "${secret_access_key:?}" +: "${region_name:?}" +: "${stack_name:?}" +: "${focus_regex:?}" +: "${s3_endpoint_host:=unset}" + + +# Just need these to get the stack info +export AWS_ACCESS_KEY_ID=${access_key_id} +export AWS_SECRET_ACCESS_KEY=${secret_access_key} +export AWS_DEFAULT_REGION=${region_name} +export AWS_ROLE_ARN=${role_arn} +stack_info=$(get_stack_info "${stack_name}") + +if [ -n "${AWS_ROLE_ARN}" ]; then + aws configure --profile creds_account set aws_access_key_id "${AWS_ACCESS_KEY_ID}" + aws configure --profile creds_account set aws_secret_access_key "${AWS_SECRET_ACCESS_KEY}" + aws configure --profile resource_account set source_profile "creds_account" + aws configure --profile resource_account set role_arn "${AWS_ROLE_ARN}" + aws configure --profile resource_account set region "${AWS_DEFAULT_REGION}" + unset AWS_ACCESS_KEY_ID + unset AWS_SECRET_ACCESS_KEY + unset AWS_DEFAULT_REGION + export AWS_PROFILE=resource_account +fi + +# Some of these are optional +export ACCESS_KEY_ID=${access_key_id} +export SECRET_ACCESS_KEY=${secret_access_key} +export REGION=${region_name} +export BUCKET_NAME +BUCKET_NAME=$(get_stack_info_of "${stack_info}" "BucketName") +export S3_HOST=${s3_endpoint_host} + +pushd "${release_dir}" > /dev/null + echo -e "\n running tests with $(go version)..." + scripts/ginkgo -r --focus="${focus_regex}" s3/integration/ +popd > /dev/null diff --git a/.github/scripts/s3/run-integration-s3-compat.sh b/.github/scripts/s3/run-integration-s3-compat.sh new file mode 100755 index 0000000..83c8c51 --- /dev/null +++ b/.github/scripts/s3/run-integration-s3-compat.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + + +# Get the directory where this script is located +script_dir="$( cd "$(dirname "${0}")" && pwd )" + +# Source utils from the same directory +source "${script_dir}/utils.sh" + +: "${access_key_id:?}" +: "${secret_access_key:?}" +: "${bucket_name:?}" +: "${s3_endpoint_host:?}" +: "${s3_endpoint_port:?}" + +export ACCESS_KEY_ID=${access_key_id} +export SECRET_ACCESS_KEY=${secret_access_key} +export BUCKET_NAME=${bucket_name} +export S3_HOST=${s3_endpoint_host} +export S3_PORT=${s3_endpoint_port} + +pushd "${release_dir}" > /dev/null + echo -e "\n running tests with $(go version)..." + scripts/ginkgo -r --focus="S3 COMPATIBLE" s3/integration/ +popd > /dev/null diff --git a/.github/scripts/s3/setup-aws-infrastructure.sh b/.github/scripts/s3/setup-aws-infrastructure.sh new file mode 100755 index 0000000..e60b40e --- /dev/null +++ b/.github/scripts/s3/setup-aws-infrastructure.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Get the directory where this script is located +script_dir="$( cd "$(dirname "${0}")" && pwd )" + +# Source utils from the same directory +source "${script_dir}/utils.sh" + +: "${access_key_id:?}" +: "${secret_access_key:?}" +: "${region_name:?}" +: "${stack_name:?}" + +export AWS_ACCESS_KEY_ID=${access_key_id} +export AWS_SECRET_ACCESS_KEY=${secret_access_key} +export AWS_DEFAULT_REGION=${region_name} + +if [ -n "${role_arn:-}" ]; then + export AWS_ROLE_ARN=${role_arn} + aws configure --profile creds_account set aws_access_key_id "${AWS_ACCESS_KEY_ID}" + aws configure --profile creds_account set aws_secret_access_key "${AWS_SECRET_ACCESS_KEY}" + aws configure --profile resource_account set source_profile "creds_account" + aws configure --profile resource_account set role_arn "${AWS_ROLE_ARN}" + aws configure --profile resource_account set region "${AWS_DEFAULT_REGION}" + unset AWS_ACCESS_KEY_ID + unset AWS_SECRET_ACCESS_KEY + unset AWS_DEFAULT_REGION + export AWS_PROFILE=resource_account +fi + +cmd="aws cloudformation create-stack \ + --stack-name ${stack_name} \ + --template-body file://${script_dir}/assets/cloudformation-${stack_name}.template.json \ + --capabilities CAPABILITY_IAM" +echo "Running: ${cmd}"; ${cmd} + +while true; do + stack_status=$(get_stack_status "${stack_name}") + echo "StackStatus ${stack_status}" + if [ "${stack_status}" == 'CREATE_IN_PROGRESS' ]; then + echo "sleeping 5s"; sleep 5s + else + break + fi +done + +if [ "${stack_status}" != 'CREATE_COMPLETE' ]; then + echo "cloudformation failed stack info:" + get_stack_info "${stack_name}" + exit 1 +fi diff --git a/.github/scripts/s3/teardown-infrastructure.sh b/.github/scripts/s3/teardown-infrastructure.sh new file mode 100755 index 0000000..0054cbb --- /dev/null +++ b/.github/scripts/s3/teardown-infrastructure.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Get the directory where this script is located +script_dir="$( cd "$(dirname "${0}")" && pwd )" + +# Source utils from the same directory +source "${script_dir}/utils.sh" + +: "${access_key_id:?}" +: "${secret_access_key:?}" +: "${region_name:?}" +: "${stack_name:?}" + +export AWS_ACCESS_KEY_ID=${access_key_id} +export AWS_SECRET_ACCESS_KEY=${secret_access_key} +export AWS_DEFAULT_REGION=${region_name} + +stack_info=$(get_stack_info "${stack_name}") +bucket_name=$(get_stack_info_of "${stack_info}" "BucketName") +aws s3 rm "s3://${bucket_name}" --recursive + +cmd="aws cloudformation delete-stack --stack-name ${stack_name}" +echo "Running: ${cmd}"; ${cmd} + +while true; do + stack_status=$(get_stack_status "${stack_name}") + echo "StackStatus ${stack_status}" + if [[ -z "${stack_status}" ]]; then #get empty status due to stack not existed on aws + echo "No stack found"; break + break + elif [ "${stack_status}" == 'DELETE_IN_PROGRESS' ]; then + echo "${stack_status}: sleeping 5s"; sleep 5s + else + echo "Expecting the stack to either be deleted or in the process of being deleted but was ${stack_status}" + get_stack_info "${stack_name}" + exit 1 + fi +done + +echo "Deleting lambda functions" +aws lambda list-functions \ + | jq -r '.Functions[].FunctionName' \ + | xargs -n1 -I{} aws lambda delete-function --function-name {} diff --git a/.github/scripts/s3/utils.sh b/.github/scripts/s3/utils.sh new file mode 100755 index 0000000..a58b073 --- /dev/null +++ b/.github/scripts/s3/utils.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +get_stack_info() { + local stack_name=$1 + + aws cloudformation describe-stacks \ + | jq --arg stack_name "${stack_name}" '.Stacks[] | select(.StackName=="\($stack_name)")' +} + +get_stack_info_of() { + local stack_info=$1 + local key=$2 + echo "${stack_info}" | jq -r --arg key "${key}" '.Outputs[] | select(.OutputKey=="\($key)").OutputValue' +} + +get_stack_status() { + local stack_name=$1 + + get_stack_info "${stack_name}" | jq -r '.StackStatus' +} diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml new file mode 100644 index 0000000..8f93444 --- /dev/null +++ b/.github/workflows/s3-integration.yml @@ -0,0 +1,209 @@ +name: S3 Integration Tests + +on: + push: + branches: [ main, feature/aws-integration-tests ] + pull_request: + branches: [ main ] + +jobs: + # AWS S3 US Integration Tests + aws-s3-us-integration: + name: AWS S3 US Integration + runs-on: ubuntu-latest + environment: aws-integration + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Install Ginkgo + run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + + - name: Setup AWS infrastructure + run: | + export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export region_name="us-east-1" + export stack_name="s3cli-iam" + export role_arn="${{ secrets.AWS_ROLE_ARN }}" + ./.github/scripts/s3/setup-aws-infrastructure.sh + + - name: Run AWS integration tests (parallel) + run: | + set -e + + # Export common variables + export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export region_name="us-east-1" + export stack_name="s3cli-iam" + export s3_endpoint_host="s3.amazonaws.com" + + # Test static credentials + export focus_regex="GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1" + ./.github/scripts/s3/run-integration-aws.sh & + PID1=$! + + # Test assume roles + export assume_role_arn="${{ secrets.ASSUME_ROLE_ARN }}" + export focus_regex="AWS ASSUME ROLE" + ./.github/scripts/s3/run-integration-aws-assume.sh & + PID2=$! + + # Test IAM roles + export focus_regex="" + ./.github/scripts/s3/run-integration-aws-iam.sh & + PID3=$! + + # Wait for all tests to complete + wait $PID1 && echo "Static credentials tests passed" || exit 1 + wait $PID2 && echo "Assume role tests passed" || exit 1 + wait $PID3 && echo "IAM role tests passed" || exit 1 + + - name: Teardown AWS infrastructure + if: always() + run: | + export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export region_name="us-east-1" + export stack_name="s3cli-iam" + export role_arn="${{ secrets.AWS_ROLE_ARN }}" + ./.github/scripts/s3/teardown-infrastructure.sh + +# # AWS S3 Public Read Integration +# aws-s3-public-read-integration: +# name: AWS S3 Public Read Integration +# runs-on: ubuntu-latest +# if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'integration-test')) +# environment: aws-integration +# steps: +# - name: Checkout code +# uses: actions/checkout@v4 + +# - name: Set up Go +# uses: actions/setup-go@v5 +# with: +# go-version-file: 'go.mod' + +# - name: Install Ginkgo +# run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + +# - name: Configure AWS credentials +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# aws-region: us-east-1 + +# - name: Setup AWS infrastructure +# run: | +# export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" +# export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" +# export region_name="us-east-1" +# export stack_name="s3cli-public-bucket" +# export role_arn="${{ secrets.AWS_ROLE_ARN }}" +# ./.github/scripts/s3/setup-aws-infrastructure.sh + +# - name: Run public read tests +# run: | +# export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" +# export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" +# export region_name="us-east-1" +# export stack_name="s3cli-public-bucket" +# export focus_regex="PUBLIC READ ONLY" +# ./.github/scripts/s3/run-integration-aws.sh + +# - name: Teardown AWS infrastructure +# if: always() +# run: | +# export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" +# export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" +# export region_name="us-east-1" +# export stack_name="s3cli-public-bucket" +# export role_arn="${{ secrets.AWS_ROLE_ARN }}" +# ./.github/scripts/s3/teardown-infrastructure.sh + +# # AWS S3 Frankfurt Integration +# aws-s3-frankfurt-integration: +# name: AWS S3 Frankfurt Integration +# runs-on: ubuntu-latest +# if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'integration-test')) +# environment: aws-integration +# steps: +# - name: Checkout code +# uses: actions/checkout@v4 + +# - name: Set up Go +# uses: actions/setup-go@v5 +# with: +# go-version-file: 'go.mod' + +# - name: Install Ginkgo +# run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + +# - name: Configure AWS credentials +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# aws-region: eu-central-1 + +# - name: Setup AWS infrastructure (Frankfurt) +# run: | +# export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" +# export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" +# export region_name="eu-central-1" +# export stack_name="s3cli-iam" +# export role_arn="${{ secrets.AWS_ROLE_ARN }}" +# ./.github/scripts/s3/setup-aws-infrastructure.sh + +# - name: Run Frankfurt region tests +# run: | +# export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" +# export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" +# export region_name="eu-central-1" +# export stack_name="s3cli-iam" +# export focus_regex="GENERAL AWS|AWS V4 REGION" +# export s3_endpoint_host="s3.amazonaws.com" +# ./.github/scripts/s3/run-integration-aws.sh + +# - name: Teardown AWS infrastructure +# if: always() +# run: | +# export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" +# export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" +# export region_name="eu-central-1" +# export stack_name="s3cli-iam" +# export role_arn="${{ secrets.AWS_ROLE_ARN }}" +# ./.github/scripts/s3/teardown-infrastructure.sh + +# # S3 Compatible Integration (optional - only if MinIO is available) +# s3-compatible-integration: +# name: S3 Compatible Integration +# runs-on: ubuntu-latest +# environment: gcp-integration +# steps: +# - name: Checkout code +# uses: actions/checkout@v4 + +# - name: Set up Go +# uses: actions/setup-go@v5 +# with: +# go-version-file: 'go.mod' + +# - name: Install Ginkgo +# run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + +# - name: Run GCS S3 compatible tests +# run: | +# export access_key_id="${{ secrets.GCP_ACCESS_KEY_ID }}" +# export secret_access_key="${{ secrets.GCP_SECRET_ACCESS_KEY }}" +# export bucket_name="${{ secrets.GCS_BUCKET_NAME }}" +# export s3_endpoint_host="storage.googleapis.com" +# export s3_endpoint_port="443" +# ./ci/tasks/run-integration-s3-compat.sh From 88d7f9f7f469c5ced470a6279fbc8c1470ea89c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Wed, 5 Nov 2025 17:55:25 +0100 Subject: [PATCH 02/32] fix: repo root added --- .github/scripts/s3/run-integration-aws-assume.sh | 6 ++++-- .github/scripts/s3/run-integration-aws-iam.sh | 8 ++++++-- .github/scripts/s3/run-integration-aws.sh | 7 +++++-- .github/workflows/s3-integration.yml | 1 + 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/scripts/s3/run-integration-aws-assume.sh b/.github/scripts/s3/run-integration-aws-assume.sh index 0fb4a93..2d42337 100755 --- a/.github/scripts/s3/run-integration-aws-assume.sh +++ b/.github/scripts/s3/run-integration-aws-assume.sh @@ -4,6 +4,7 @@ set -euo pipefail # Get the directory where this script is located script_dir="$( cd "$(dirname "${0}")" && pwd )" +repo_root="$(cd "${script_dir}/../../.." && pwd)" # Source utils from the same directory source "${script_dir}/utils.sh" @@ -28,7 +29,8 @@ export SECRET_ACCESS_KEY=${secret_access_key} export REGION=${region_name} export S3_HOST=${s3_endpoint_host} -pushd "${release_dir}" > /dev/null + +pushd "${repo_root}" > /dev/null echo -e "\n running tests with $(go version)..." - scripts/ginkgo -r --focus="${focus_regex}" s3/integration/ + ginkgo -r --focus="${focus_regex}" s3/integration/ popd > /dev/null diff --git a/.github/scripts/s3/run-integration-aws-iam.sh b/.github/scripts/s3/run-integration-aws-iam.sh index 22e4b52..8f1e5cb 100755 --- a/.github/scripts/s3/run-integration-aws-iam.sh +++ b/.github/scripts/s3/run-integration-aws-iam.sh @@ -4,6 +4,7 @@ set -euo pipefail # Get the directory where this script is located script_dir="$( cd "$(dirname "${0}")" && pwd )" +repo_root="$(cd "${script_dir}/../../.." && pwd)" # Source utils from the same directory source "${script_dir}/utils.sh" @@ -26,12 +27,15 @@ lambda_payload="{\"region\": \"${region_name}\", \"bucket_name\": \"${bucket_nam lambda_log=$(mktemp -t "XXXXXX-lambda.log") trap "cat ${lambda_log}" EXIT -pushd "${release_dir}" > /dev/null +# Go to the repository root (3 levels up from script directory) + +pushd "${repo_root}" > /dev/null + echo -e "\n building artifact with $(go version)..." # TODO change repo in here later CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o out/s3cli \ github.com/cloudfoundry/bosh-s3cli - CGO_ENABLED=0 scripts/ginkgo build s3/integration + CGO_ENABLED=0 ginkgo build s3/integration zip -j payload.zip s3/integration/integration.test out/s3cli ${script_dir}/assets/lambda_function.py diff --git a/.github/scripts/s3/run-integration-aws.sh b/.github/scripts/s3/run-integration-aws.sh index bff200e..747719c 100755 --- a/.github/scripts/s3/run-integration-aws.sh +++ b/.github/scripts/s3/run-integration-aws.sh @@ -5,6 +5,8 @@ set -euo pipefail # Get the directory where this script is located script_dir="$( cd "$(dirname "${0}")" && pwd )" +repo_root="$(cd "${script_dir}/../../.." && pwd)" + # Source utils from the same directory source "${script_dir}/utils.sh" @@ -43,7 +45,8 @@ export BUCKET_NAME BUCKET_NAME=$(get_stack_info_of "${stack_info}" "BucketName") export S3_HOST=${s3_endpoint_host} -pushd "${release_dir}" > /dev/null + +pushd "${repo_root}" > /dev/null echo -e "\n running tests with $(go version)..." - scripts/ginkgo -r --focus="${focus_regex}" s3/integration/ + ginkgo -r --focus="${focus_regex}" s3/integration/ popd > /dev/null diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 8f93444..88249f8 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -42,6 +42,7 @@ jobs: export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" export region_name="us-east-1" export stack_name="s3cli-iam" + export role_arn="${{ secrets.AWS_ROLE_ARN }}" export s3_endpoint_host="s3.amazonaws.com" # Test static credentials From 90d245062c9862e9d631cf07dbabb50099b6efb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 6 Nov 2025 08:37:41 +0100 Subject: [PATCH 03/32] fix: build from current repo --- .github/scripts/s3/run-integration-aws-iam.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/scripts/s3/run-integration-aws-iam.sh b/.github/scripts/s3/run-integration-aws-iam.sh index 8f1e5cb..daafd1c 100755 --- a/.github/scripts/s3/run-integration-aws-iam.sh +++ b/.github/scripts/s3/run-integration-aws-iam.sh @@ -32,9 +32,8 @@ trap "cat ${lambda_log}" EXIT pushd "${repo_root}" > /dev/null echo -e "\n building artifact with $(go version)..." -# TODO change repo in here later - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o out/s3cli \ - github.com/cloudfoundry/bosh-s3cli +# Build the S3 CLI from the current repository + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o out/s3cli ./s3 CGO_ENABLED=0 ginkgo build s3/integration zip -j payload.zip s3/integration/integration.test out/s3cli ${script_dir}/assets/lambda_function.py From 39bfa695f1ded5c27e22649ceb0a6439819ea36f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 6 Nov 2025 08:54:53 +0100 Subject: [PATCH 04/32] fix: path changed --- .github/scripts/s3/run-integration-aws-iam.sh | 1 - s3/integration/integration_suite_test.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/scripts/s3/run-integration-aws-iam.sh b/.github/scripts/s3/run-integration-aws-iam.sh index daafd1c..7aa2d0f 100755 --- a/.github/scripts/s3/run-integration-aws-iam.sh +++ b/.github/scripts/s3/run-integration-aws-iam.sh @@ -32,7 +32,6 @@ trap "cat ${lambda_log}" EXIT pushd "${repo_root}" > /dev/null echo -e "\n building artifact with $(go version)..." -# Build the S3 CLI from the current repository CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o out/s3cli ./s3 CGO_ENABLED=0 ginkgo build s3/integration diff --git a/s3/integration/integration_suite_test.go b/s3/integration/integration_suite_test.go index a3494e2..2abd83d 100644 --- a/s3/integration/integration_suite_test.go +++ b/s3/integration/integration_suite_test.go @@ -27,7 +27,7 @@ var _ = BeforeSuite(func() { if len(s3CLIPath) == 0 { var err error - s3CLIPath, err = gexec.Build("github.com/cloudfoundry/bosh-s3cli") + s3CLIPath, err = gexec.Build("github.com/cloudfoundry/storage-cli/s3") Expect(err).ShouldNot(HaveOccurred()) } }) From 0b76cbb90260d15a7cd52b2465bbed3f38fff3ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 6 Nov 2025 10:19:39 +0100 Subject: [PATCH 05/32] fix: AWS_ROLE_ARN is used --- .github/workflows/s3-integration.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 88249f8..2711dd7 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -44,18 +44,19 @@ jobs: export stack_name="s3cli-iam" export role_arn="${{ secrets.AWS_ROLE_ARN }}" export s3_endpoint_host="s3.amazonaws.com" + export bucket_name="s3cli-pipeline" # Test static credentials export focus_regex="GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1" ./.github/scripts/s3/run-integration-aws.sh & PID1=$! - + # Test assume roles - export assume_role_arn="${{ secrets.ASSUME_ROLE_ARN }}" + export assume_role_arn="${{ secrets.AWS_ROLE_ARN }}" export focus_regex="AWS ASSUME ROLE" ./.github/scripts/s3/run-integration-aws-assume.sh & PID2=$! - + # Test IAM roles export focus_regex="" ./.github/scripts/s3/run-integration-aws-iam.sh & From 6af21a3ffa6bc02b2d51986d891b0623db81c6de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 6 Nov 2025 10:48:44 +0100 Subject: [PATCH 06/32] feat: each parallel test converted to steps --- .github/scripts/s3/run-integration-aws-iam.sh | 5 +- .github/workflows/s3-integration.yml | 49 +++++++++++-------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/.github/scripts/s3/run-integration-aws-iam.sh b/.github/scripts/s3/run-integration-aws-iam.sh index 7aa2d0f..632df35 100755 --- a/.github/scripts/s3/run-integration-aws-iam.sh +++ b/.github/scripts/s3/run-integration-aws-iam.sh @@ -22,7 +22,10 @@ export AWS_DEFAULT_REGION=${region_name} stack_info=$(get_stack_info "${stack_name}") bucket_name=$(get_stack_info_of "${stack_info}" "BucketName") iam_role_arn=$(get_stack_info_of "${stack_info}" "IamRoleArn") -lambda_payload="{\"region\": \"${region_name}\", \"bucket_name\": \"${bucket_name}\", \"s3_host\": \"s3.amazonaws.com\"}" + +# Create JSON payload and base64 encode it +lambda_payload_json="{\"region\": \"${region_name}\", \"bucket_name\": \"${bucket_name}\", \"s3_host\": \"s3.amazonaws.com\"}" +lambda_payload_base64=$(echo -n "${lambda_payload_json}" | base64) lambda_log=$(mktemp -t "XXXXXX-lambda.log") trap "cat ${lambda_log}" EXIT diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 2711dd7..4fe12cd 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -31,10 +31,12 @@ jobs: export region_name="us-east-1" export stack_name="s3cli-iam" export role_arn="${{ secrets.AWS_ROLE_ARN }}" - ./.github/scripts/s3/setup-aws-infrastructure.sh - - name: Run AWS integration tests (parallel) - run: | + + ./.github/scripts/s3/setup-aws-infrastructure.sh + + - name: Test Static Credentials + run : | set -e # Export common variables @@ -42,30 +44,35 @@ jobs: export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" export region_name="us-east-1" export stack_name="s3cli-iam" - export role_arn="${{ secrets.AWS_ROLE_ARN }}" export s3_endpoint_host="s3.amazonaws.com" - export bucket_name="s3cli-pipeline" - - # Test static credentials export focus_regex="GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1" - ./.github/scripts/s3/run-integration-aws.sh & - PID1=$! - # Test assume roles + ./.github/scripts/s3/run-integration-aws.sh + + - name: Test IAM Roles + run : | + set -e + + # Export common variables + export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export region_name="us-east-1" + export stack_name="s3cli-iam" + + ./.github/scripts/s3/run-integration-aws-iam.sh + + - name: Test Assume Roles + run: | + set -e + + # Export common variables + export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export region_name="us-east-1" export assume_role_arn="${{ secrets.AWS_ROLE_ARN }}" export focus_regex="AWS ASSUME ROLE" - ./.github/scripts/s3/run-integration-aws-assume.sh & - PID2=$! - # Test IAM roles - export focus_regex="" - ./.github/scripts/s3/run-integration-aws-iam.sh & - PID3=$! - - # Wait for all tests to complete - wait $PID1 && echo "Static credentials tests passed" || exit 1 - wait $PID2 && echo "Assume role tests passed" || exit 1 - wait $PID3 && echo "IAM role tests passed" || exit 1 + ./.github/scripts/s3/run-integration-aws-assume.sh - name: Teardown AWS infrastructure if: always() From d25af469646f97cc30856f1dc5af6673fda9878a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 6 Nov 2025 12:09:31 +0100 Subject: [PATCH 07/32] fix: role_arn added --- .github/workflows/s3-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 4fe12cd..619f92a 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -32,7 +32,6 @@ jobs: export stack_name="s3cli-iam" export role_arn="${{ secrets.AWS_ROLE_ARN }}" - ./.github/scripts/s3/setup-aws-infrastructure.sh - name: Test Static Credentials @@ -42,6 +41,7 @@ jobs: # Export common variables export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export role_arn="${{ secrets.AWS_ROLE_ARN }}" export region_name="us-east-1" export stack_name="s3cli-iam" export s3_endpoint_host="s3.amazonaws.com" From 4629501c43f82aceac762a1fd40591c6dfa590cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 6 Nov 2025 12:15:07 +0100 Subject: [PATCH 08/32] fix: s3 link updated --- .github/workflows/s3-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 619f92a..f6d2b28 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -44,7 +44,7 @@ jobs: export role_arn="${{ secrets.AWS_ROLE_ARN }}" export region_name="us-east-1" export stack_name="s3cli-iam" - export s3_endpoint_host="s3.amazonaws.com" + export s3_endpoint_host="https://s3.amazonaws.com" export focus_regex="GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1" ./.github/scripts/s3/run-integration-aws.sh From 5c9b7a9e838dc07883a6821eb345f5e49ace87f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 6 Nov 2025 12:24:18 +0100 Subject: [PATCH 09/32] fix: unnecessary bas64 removed --- .github/scripts/s3/run-integration-aws-iam.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/scripts/s3/run-integration-aws-iam.sh b/.github/scripts/s3/run-integration-aws-iam.sh index 632df35..c4b4660 100755 --- a/.github/scripts/s3/run-integration-aws-iam.sh +++ b/.github/scripts/s3/run-integration-aws-iam.sh @@ -24,8 +24,7 @@ bucket_name=$(get_stack_info_of "${stack_info}" "BucketName") iam_role_arn=$(get_stack_info_of "${stack_info}" "IamRoleArn") # Create JSON payload and base64 encode it -lambda_payload_json="{\"region\": \"${region_name}\", \"bucket_name\": \"${bucket_name}\", \"s3_host\": \"s3.amazonaws.com\"}" -lambda_payload_base64=$(echo -n "${lambda_payload_json}" | base64) +lambda_payload="{\"region\": \"${region_name}\", \"bucket_name\": \"${bucket_name}\", \"s3_host\": \"s3.amazonaws.com\"}" lambda_log=$(mktemp -t "XXXXXX-lambda.log") trap "cat ${lambda_log}" EXIT From 6ddff767e37b92d8c975dad0c45e47a383458479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 6 Nov 2025 12:27:57 +0100 Subject: [PATCH 10/32] fix: bas64 is needed actually --- .github/scripts/s3/run-integration-aws-iam.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/scripts/s3/run-integration-aws-iam.sh b/.github/scripts/s3/run-integration-aws-iam.sh index c4b4660..a6e2064 100755 --- a/.github/scripts/s3/run-integration-aws-iam.sh +++ b/.github/scripts/s3/run-integration-aws-iam.sh @@ -24,7 +24,8 @@ bucket_name=$(get_stack_info_of "${stack_info}" "BucketName") iam_role_arn=$(get_stack_info_of "${stack_info}" "IamRoleArn") # Create JSON payload and base64 encode it -lambda_payload="{\"region\": \"${region_name}\", \"bucket_name\": \"${bucket_name}\", \"s3_host\": \"s3.amazonaws.com\"}" +lambda_payload_json="{\"region\": \"${region_name}\", \"bucket_name\": \"${bucket_name}\", \"s3_host\": \"s3.amazonaws.com\"}" +lambda_payload_base64=$(echo -n "${lambda_payload_json}" | base64) lambda_log=$(mktemp -t "XXXXXX-lambda.log") trap "cat ${lambda_log}" EXIT @@ -67,7 +68,7 @@ pushd "${repo_root}" > /dev/null --function-name "${lambda_function_name}" \ --region "${region_name}" \ --log-type Tail \ - --payload "${lambda_payload}" \ + --payload "${lambda_payload_base64}" \ "${lambda_log}" | tee lambda_output.json set +e From ccdf19f9cf3373cdf8a26488044ed42a96789e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 7 Nov 2025 10:38:50 +0100 Subject: [PATCH 11/32] feat: bucket with new name created --- s3/integration/aws_assume_role_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3/integration/aws_assume_role_test.go b/s3/integration/aws_assume_role_test.go index 0b43f2b..c436701 100644 --- a/s3/integration/aws_assume_role_test.go +++ b/s3/integration/aws_assume_role_test.go @@ -22,7 +22,7 @@ var _ = Describe("Testing AWS assume role ", func() { assumeRoleArn := os.Getenv("ASSUME_ROLE_ARN") Expect(assumeRoleArn).ToNot(BeEmpty(), "ASSUME_ROLE_ARN must be set") - bucketName := "bosh-s3cli-assume-role-integration-test" + bucketName := "bosh-s3cli-assume-role-integration-test-1" region := "us-east-1" nonAssumedRoleCfg := &config.S3Cli{ From 0f21850e172a087236d6b90ce86dbee9c040dd76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 7 Nov 2025 10:54:54 +0100 Subject: [PATCH 12/32] feat: testing public read integration --- .github/workflows/s3-integration.yml | 186 +++++++++++++-------------- 1 file changed, 89 insertions(+), 97 deletions(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index f6d2b28..c5b7c04 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -7,17 +7,94 @@ on: branches: [ main ] jobs: - # AWS S3 US Integration Tests - aws-s3-us-integration: - name: AWS S3 US Integration + # # AWS S3 US Integration Tests + # aws-s3-us-integration: + # name: AWS S3 US Integration + # runs-on: ubuntu-latest + # environment: aws-integration + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + + # - name: Set up Go + # uses: actions/setup-go@v5 + # with: + # go-version-file: 'go.mod' + + # - name: Install Ginkgo + # run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + + # - name: Setup AWS infrastructure + # run: | + # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + # export region_name="us-east-1" + # export stack_name="s3cli-iam" + # export role_arn="${{ secrets.AWS_ROLE_ARN }}" + + # ./.github/scripts/s3/setup-aws-infrastructure.sh + + # - name: Test Static Credentials + # run : | + # set -e + + # # Export common variables + # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + # export role_arn="${{ secrets.AWS_ROLE_ARN }}" + # export region_name="us-east-1" + # export stack_name="s3cli-iam" + # export s3_endpoint_host="https://s3.amazonaws.com" + # export focus_regex="GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1" + + # ./.github/scripts/s3/run-integration-aws.sh + + # - name: Test IAM Roles + # run : | + # set -e + + # # Export common variables + # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + # export region_name="us-east-1" + # export stack_name="s3cli-iam" + + # ./.github/scripts/s3/run-integration-aws-iam.sh + + # - name: Test Assume Roles + # run: | + # set -e + + # # Export common variables + # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + # export region_name="us-east-1" + # export assume_role_arn="${{ secrets.AWS_ROLE_ARN }}" + # export focus_regex="AWS ASSUME ROLE" + + # ./.github/scripts/s3/run-integration-aws-assume.sh + + # - name: Teardown AWS infrastructure + # if: always() + # run: | + # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + # export region_name="us-east-1" + # export stack_name="s3cli-iam" + # export role_arn="${{ secrets.AWS_ROLE_ARN }}" + # ./.github/scripts/s3/teardown-infrastructure.sh + + # AWS S3 Public Read Integration + aws-s3-public-read-integration: + name: AWS S3 Public Read Integration runs-on: ubuntu-latest environment: aws-integration steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version-file: 'go.mod' @@ -29,50 +106,18 @@ jobs: export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" export region_name="us-east-1" - export stack_name="s3cli-iam" + export stack_name="s3cli-public-bucket" export role_arn="${{ secrets.AWS_ROLE_ARN }}" - ./.github/scripts/s3/setup-aws-infrastructure.sh - - - name: Test Static Credentials - run : | - set -e - - # Export common variables - export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export role_arn="${{ secrets.AWS_ROLE_ARN }}" - export region_name="us-east-1" - export stack_name="s3cli-iam" - export s3_endpoint_host="https://s3.amazonaws.com" - export focus_regex="GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1" - - ./.github/scripts/s3/run-integration-aws.sh - - - name: Test IAM Roles - run : | - set -e - - # Export common variables - export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export region_name="us-east-1" - export stack_name="s3cli-iam" - ./.github/scripts/s3/run-integration-aws-iam.sh - - - name: Test Assume Roles + - name: Run public read tests run: | - set -e - - # Export common variables export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" export region_name="us-east-1" - export assume_role_arn="${{ secrets.AWS_ROLE_ARN }}" - export focus_regex="AWS ASSUME ROLE" - - ./.github/scripts/s3/run-integration-aws-assume.sh + export stack_name="s3cli-public-bucket" + export focus_regex="PUBLIC READ ONLY" + ./.github/scripts/s3/run-integration-aws.sh - name: Teardown AWS infrastructure if: always() @@ -80,63 +125,10 @@ jobs: export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" export region_name="us-east-1" - export stack_name="s3cli-iam" + export stack_name="s3cli-public-bucket" export role_arn="${{ secrets.AWS_ROLE_ARN }}" ./.github/scripts/s3/teardown-infrastructure.sh -# # AWS S3 Public Read Integration -# aws-s3-public-read-integration: -# name: AWS S3 Public Read Integration -# runs-on: ubuntu-latest -# if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'integration-test')) -# environment: aws-integration -# steps: -# - name: Checkout code -# uses: actions/checkout@v4 - -# - name: Set up Go -# uses: actions/setup-go@v5 -# with: -# go-version-file: 'go.mod' - -# - name: Install Ginkgo -# run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - -# - name: Configure AWS credentials -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# aws-region: us-east-1 - -# - name: Setup AWS infrastructure -# run: | -# export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" -# export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" -# export region_name="us-east-1" -# export stack_name="s3cli-public-bucket" -# export role_arn="${{ secrets.AWS_ROLE_ARN }}" -# ./.github/scripts/s3/setup-aws-infrastructure.sh - -# - name: Run public read tests -# run: | -# export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" -# export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" -# export region_name="us-east-1" -# export stack_name="s3cli-public-bucket" -# export focus_regex="PUBLIC READ ONLY" -# ./.github/scripts/s3/run-integration-aws.sh - -# - name: Teardown AWS infrastructure -# if: always() -# run: | -# export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" -# export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" -# export region_name="us-east-1" -# export stack_name="s3cli-public-bucket" -# export role_arn="${{ secrets.AWS_ROLE_ARN }}" -# ./.github/scripts/s3/teardown-infrastructure.sh - # # AWS S3 Frankfurt Integration # aws-s3-frankfurt-integration: # name: AWS S3 Frankfurt Integration @@ -191,7 +183,7 @@ jobs: # export role_arn="${{ secrets.AWS_ROLE_ARN }}" # ./.github/scripts/s3/teardown-infrastructure.sh -# # S3 Compatible Integration (optional - only if MinIO is available) + # s3-compatible-integration: # name: S3 Compatible Integration # runs-on: ubuntu-latest From a89bd49b64f750d3175c491361b073952c3c5001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 7 Nov 2025 10:58:18 +0100 Subject: [PATCH 13/32] fix: role arn added --- .github/workflows/s3-integration.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index c5b7c04..bebe80a 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -105,15 +105,16 @@ jobs: run: | export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export role_arn="${{ secrets.AWS_ROLE_ARN }}" export region_name="us-east-1" export stack_name="s3cli-public-bucket" - export role_arn="${{ secrets.AWS_ROLE_ARN }}" ./.github/scripts/s3/setup-aws-infrastructure.sh - name: Run public read tests run: | export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export role_arn="${{ secrets.AWS_ROLE_ARN }}" export region_name="us-east-1" export stack_name="s3cli-public-bucket" export focus_regex="PUBLIC READ ONLY" @@ -124,9 +125,9 @@ jobs: run: | export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export role_arn="${{ secrets.AWS_ROLE_ARN }}" export region_name="us-east-1" export stack_name="s3cli-public-bucket" - export role_arn="${{ secrets.AWS_ROLE_ARN }}" ./.github/scripts/s3/teardown-infrastructure.sh # # AWS S3 Frankfurt Integration From 7375ccb141642aa1bb0a0357d0afd15cf0438ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 7 Nov 2025 11:00:47 +0100 Subject: [PATCH 14/32] fix: role arn with empty string --- .github/workflows/s3-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index bebe80a..648b91d 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -125,7 +125,7 @@ jobs: run: | export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export role_arn="${{ secrets.AWS_ROLE_ARN }}" + export role_arn="" export region_name="us-east-1" export stack_name="s3cli-public-bucket" ./.github/scripts/s3/teardown-infrastructure.sh From 01c87594fcb2e3217e3c1cbbadac02b0e0b77356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 7 Nov 2025 11:28:44 +0100 Subject: [PATCH 15/32] feat: test frankfurt integration --- .github/workflows/s3-integration.yml | 130 +++++++++++++-------------- 1 file changed, 61 insertions(+), 69 deletions(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 648b91d..b86629a 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -84,9 +84,55 @@ jobs: # export role_arn="${{ secrets.AWS_ROLE_ARN }}" # ./.github/scripts/s3/teardown-infrastructure.sh - # AWS S3 Public Read Integration - aws-s3-public-read-integration: - name: AWS S3 Public Read Integration + # # AWS S3 Public Read Integration + # aws-s3-public-read-integration: + # name: AWS S3 Public Read Integration + # runs-on: ubuntu-latest + # environment: aws-integration + # steps: + # - name: Checkout code + # uses: actions/checkout@v5 + + # - name: Set up Go + # uses: actions/setup-go@v6 + # with: + # go-version-file: 'go.mod' + + # - name: Install Ginkgo + # run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + + # - name: Setup AWS infrastructure + # run: | + # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + # export role_arn="${{ secrets.AWS_ROLE_ARN }}" + # export region_name="us-east-1" + # export stack_name="s3cli-public-bucket" + # ./.github/scripts/s3/setup-aws-infrastructure.sh + + # - name: Run public read tests + # run: | + # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + # export role_arn="${{ secrets.AWS_ROLE_ARN }}" + # export region_name="us-east-1" + # export stack_name="s3cli-public-bucket" + # export focus_regex="PUBLIC READ ONLY" + # ./.github/scripts/s3/run-integration-aws.sh + + # - name: Teardown AWS infrastructure + # if: always() + # run: | + # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + # export role_arn="" + # export region_name="us-east-1" + # export stack_name="s3cli-public-bucket" + # ./.github/scripts/s3/teardown-infrastructure.sh + + # AWS S3 Frankfurt Integration + aws-s3-frankfurt-integration: + name: AWS S3 Frankfurt Integration runs-on: ubuntu-latest environment: aws-integration steps: @@ -101,23 +147,24 @@ jobs: - name: Install Ginkgo run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - - name: Setup AWS infrastructure + - name: Setup AWS infrastructure (Frankfurt) run: | export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export role_arn="${{ secrets.AWS_ROLE_ARN }}" - export region_name="us-east-1" - export stack_name="s3cli-public-bucket" + export role_arn="" + export region_name="eu-central-1" + export stack_name="s3cli-private-bucket" ./.github/scripts/s3/setup-aws-infrastructure.sh - - name: Run public read tests + - name: Run Frankfurt region tests run: | export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export role_arn="${{ secrets.AWS_ROLE_ARN }}" - export region_name="us-east-1" - export stack_name="s3cli-public-bucket" - export focus_regex="PUBLIC READ ONLY" + export role_arn="" + export region_name="eu-central-1" + export stack_name="s3cli-private-bucket" + export focus_regex="GENERAL AWS|AWS V4 REGION|AWS V4 ONLY REGION" + export s3_endpoint_host="https://s3.amazonaws.com" ./.github/scripts/s3/run-integration-aws.sh - name: Teardown AWS infrastructure @@ -125,65 +172,10 @@ jobs: run: | export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export role_arn="" - export region_name="us-east-1" - export stack_name="s3cli-public-bucket" + export region_name="eu-central-1" + export stack_name="s3cli-private-bucket" ./.github/scripts/s3/teardown-infrastructure.sh -# # AWS S3 Frankfurt Integration -# aws-s3-frankfurt-integration: -# name: AWS S3 Frankfurt Integration -# runs-on: ubuntu-latest -# if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'integration-test')) -# environment: aws-integration -# steps: -# - name: Checkout code -# uses: actions/checkout@v4 - -# - name: Set up Go -# uses: actions/setup-go@v5 -# with: -# go-version-file: 'go.mod' - -# - name: Install Ginkgo -# run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - -# - name: Configure AWS credentials -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# aws-region: eu-central-1 - -# - name: Setup AWS infrastructure (Frankfurt) -# run: | -# export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" -# export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" -# export region_name="eu-central-1" -# export stack_name="s3cli-iam" -# export role_arn="${{ secrets.AWS_ROLE_ARN }}" -# ./.github/scripts/s3/setup-aws-infrastructure.sh - -# - name: Run Frankfurt region tests -# run: | -# export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" -# export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" -# export region_name="eu-central-1" -# export stack_name="s3cli-iam" -# export focus_regex="GENERAL AWS|AWS V4 REGION" -# export s3_endpoint_host="s3.amazonaws.com" -# ./.github/scripts/s3/run-integration-aws.sh - -# - name: Teardown AWS infrastructure -# if: always() -# run: | -# export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" -# export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" -# export region_name="eu-central-1" -# export stack_name="s3cli-iam" -# export role_arn="${{ secrets.AWS_ROLE_ARN }}" -# ./.github/scripts/s3/teardown-infrastructure.sh - # s3-compatible-integration: # name: S3 Compatible Integration From e48f65adbb18a58f33db28784a67fe62ebe6d8ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 7 Nov 2025 12:37:39 +0100 Subject: [PATCH 16/32] fix: s3 endpoint changed with region --- .github/workflows/s3-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index b86629a..4b04842 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -164,7 +164,7 @@ jobs: export region_name="eu-central-1" export stack_name="s3cli-private-bucket" export focus_regex="GENERAL AWS|AWS V4 REGION|AWS V4 ONLY REGION" - export s3_endpoint_host="https://s3.amazonaws.com" + export s3_endpoint_host="https://s3.eu-central-1.amazonaws.com" ./.github/scripts/s3/run-integration-aws.sh - name: Teardown AWS infrastructure From d600bedf8351f485a319b131219dcd749d65555a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 7 Nov 2025 15:05:16 +0100 Subject: [PATCH 17/32] fix: AWS V4 ONLY REGION tests are removed - check this issue https://github.com/cloudfoundry/bosh-s3cli/issues/55 --- .github/workflows/s3-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 4b04842..ce44425 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -163,7 +163,7 @@ jobs: export role_arn="" export region_name="eu-central-1" export stack_name="s3cli-private-bucket" - export focus_regex="GENERAL AWS|AWS V4 REGION|AWS V4 ONLY REGION" + export focus_regex="GENERAL AWS|AWS V4 REGION" export s3_endpoint_host="https://s3.eu-central-1.amazonaws.com" ./.github/scripts/s3/run-integration-aws.sh From d17d57aaa013accc121967dec24120b36311d471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 7 Nov 2025 15:23:33 +0100 Subject: [PATCH 18/32] fix: remove unnecessary role_arn --- .github/workflows/s3-integration.yml | 238 +++++++++++++-------------- 1 file changed, 118 insertions(+), 120 deletions(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index ce44425..9c93d47 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -7,128 +7,123 @@ on: branches: [ main ] jobs: - # # AWS S3 US Integration Tests - # aws-s3-us-integration: - # name: AWS S3 US Integration - # runs-on: ubuntu-latest - # environment: aws-integration - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 - - # - name: Set up Go - # uses: actions/setup-go@v5 - # with: - # go-version-file: 'go.mod' - - # - name: Install Ginkgo - # run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - - # - name: Setup AWS infrastructure - # run: | - # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - # export region_name="us-east-1" - # export stack_name="s3cli-iam" - # export role_arn="${{ secrets.AWS_ROLE_ARN }}" - - # ./.github/scripts/s3/setup-aws-infrastructure.sh - - # - name: Test Static Credentials - # run : | - # set -e - - # # Export common variables - # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - # export role_arn="${{ secrets.AWS_ROLE_ARN }}" - # export region_name="us-east-1" - # export stack_name="s3cli-iam" - # export s3_endpoint_host="https://s3.amazonaws.com" - # export focus_regex="GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1" - - # ./.github/scripts/s3/run-integration-aws.sh - - # - name: Test IAM Roles - # run : | - # set -e + # AWS S3 US Integration Tests + aws-s3-us-integration: + name: AWS S3 US Integration + runs-on: ubuntu-latest + environment: aws-integration + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Install Ginkgo + run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + + - name: Setup AWS infrastructure + run: | + set -e + export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export role_arn="" + export region_name="us-east-1" + export stack_name="s3cli-iam" + ./.github/scripts/s3/setup-aws-infrastructure.sh - # # Export common variables - # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - # export region_name="us-east-1" - # export stack_name="s3cli-iam" + - name: Test Static Credentials + run : | + set -e + export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export role_arn="" + export region_name="us-east-1" + export stack_name="s3cli-iam" + export s3_endpoint_host="https://s3.amazonaws.com" + export focus_regex="GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1" + ./.github/scripts/s3/run-integration-aws.sh - # ./.github/scripts/s3/run-integration-aws-iam.sh + - name: Test IAM Roles + run : | + set -e + export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export region_name="us-east-1" + export stack_name="s3cli-iam" + ./.github/scripts/s3/run-integration-aws-iam.sh - # - name: Test Assume Roles - # run: | - # set -e - - # # Export common variables - # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - # export region_name="us-east-1" - # export assume_role_arn="${{ secrets.AWS_ROLE_ARN }}" - # export focus_regex="AWS ASSUME ROLE" - - # ./.github/scripts/s3/run-integration-aws-assume.sh - - # - name: Teardown AWS infrastructure - # if: always() - # run: | - # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - # export region_name="us-east-1" - # export stack_name="s3cli-iam" - # export role_arn="${{ secrets.AWS_ROLE_ARN }}" - # ./.github/scripts/s3/teardown-infrastructure.sh - - # # AWS S3 Public Read Integration - # aws-s3-public-read-integration: - # name: AWS S3 Public Read Integration - # runs-on: ubuntu-latest - # environment: aws-integration - # steps: - # - name: Checkout code - # uses: actions/checkout@v5 - - # - name: Set up Go - # uses: actions/setup-go@v6 - # with: - # go-version-file: 'go.mod' - - # - name: Install Ginkgo - # run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - - # - name: Setup AWS infrastructure - # run: | - # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - # export role_arn="${{ secrets.AWS_ROLE_ARN }}" - # export region_name="us-east-1" - # export stack_name="s3cli-public-bucket" - # ./.github/scripts/s3/setup-aws-infrastructure.sh - - # - name: Run public read tests - # run: | - # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - # export role_arn="${{ secrets.AWS_ROLE_ARN }}" - # export region_name="us-east-1" - # export stack_name="s3cli-public-bucket" - # export focus_regex="PUBLIC READ ONLY" - # ./.github/scripts/s3/run-integration-aws.sh - - # - name: Teardown AWS infrastructure - # if: always() - # run: | - # export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - # export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - # export role_arn="" - # export region_name="us-east-1" - # export stack_name="s3cli-public-bucket" - # ./.github/scripts/s3/teardown-infrastructure.sh + - name: Test Assume Roles + run: | + set -e + export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export assume_role_arn="${{ secrets.AWS_ROLE_ARN }}" + export region_name="us-east-1" + export focus_regex="AWS ASSUME ROLE" + ./.github/scripts/s3/run-integration-aws-assume.sh + + - name: Teardown AWS infrastructure + if: always() + run: | + set -e + export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export role_arn="" + export region_name="us-east-1" + export stack_name="s3cli-iam" + ./.github/scripts/s3/teardown-infrastructure.sh + + # AWS S3 Public Read Integration + aws-s3-public-read-integration: + name: AWS S3 Public Read Integration + runs-on: ubuntu-latest + environment: aws-integration + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + + - name: Install Ginkgo + run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + + - name: Setup AWS infrastructure + run: | + set -e + export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export role_arn="" + export region_name="us-east-1" + export stack_name="s3cli-public-bucket" + ./.github/scripts/s3/setup-aws-infrastructure.sh + + - name: Run public read tests + run: | + set -e + export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export role_arn="" + export region_name="us-east-1" + export stack_name="s3cli-public-bucket" + export focus_regex="PUBLIC READ ONLY" + ./.github/scripts/s3/run-integration-aws.sh + + - name: Teardown AWS infrastructure + if: always() + run: | + set -e + export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export role_arn="" + export region_name="us-east-1" + export stack_name="s3cli-public-bucket" + ./.github/scripts/s3/teardown-infrastructure.sh # AWS S3 Frankfurt Integration aws-s3-frankfurt-integration: @@ -147,8 +142,9 @@ jobs: - name: Install Ginkgo run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - - name: Setup AWS infrastructure (Frankfurt) + - name: Setup AWS infrastructure run: | + set -e export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" export role_arn="" @@ -158,6 +154,7 @@ jobs: - name: Run Frankfurt region tests run: | + set -e export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" export role_arn="" @@ -170,6 +167,7 @@ jobs: - name: Teardown AWS infrastructure if: always() run: | + set -e export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" export region_name="eu-central-1" From 2349ffa5402c8ad6bcbb92a0eee8df92198f5068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Mon, 10 Nov 2025 14:06:16 +0100 Subject: [PATCH 19/32] feat: action for setup, run and teardown added --- .github/actions/s3-integration-run/action.yml | 61 ++++++ .../actions/s3-integration-setup/action.yml | 34 ++++ .../s3-integration-teardown/action.yml | 28 +++ .github/workflows/s3-integration.yml | 192 +++++++++--------- 4 files changed, 219 insertions(+), 96 deletions(-) create mode 100644 .github/actions/s3-integration-run/action.yml create mode 100644 .github/actions/s3-integration-setup/action.yml create mode 100644 .github/actions/s3-integration-teardown/action.yml diff --git a/.github/actions/s3-integration-run/action.yml b/.github/actions/s3-integration-run/action.yml new file mode 100644 index 0000000..92750d9 --- /dev/null +++ b/.github/actions/s3-integration-run/action.yml @@ -0,0 +1,61 @@ +name: Run AWS S3 Integration Tests +description: Runs integration tests against to aws infrastructure. + +inputs: + access_key_id: + description: 'AWS Access Key ID' + required: true + secret_access_key: + description: 'AWS Secret Access Key' + required: true + region_name: + description: 'AWS Region Name' + required: true + stack_name: + description: 'CloudFormation Stack Name (required for IAM tests)' + required: true + test_type: + description: 'Type of test to run (e.g.,aws, aws-iam, aws-assume)' + required: true + focus_regex: + description: 'Ginkgo Focus Regex for tests to run' + required: false + s3_endpoint_host: + description: 'Custom S3 Endpoint Host' + required: false + role_arn: + description: 'AWS Role ARN to test assume role functionality' + required: false + default: '' + +runs: + using: 'composite' + steps: + - name: Run AWS S3 Integration Tests + shell: bash + run: | + set -e + export access_key_id="${{inputs.access_key_id}}" + export secret_access_key="${{inputs.secret_access_key}}" + export region_name="${{inputs.region_name}}" + export stack_name="${{inputs.stack_name}}" + + if [[ "${{inputs.test_type}}" == "aws" ]]; then + export role_arn="${{inputs.role_arn}}" + export s3_endpoint_host="${{inputs.s3_endpoint_host}}" + export focus_regex="${{inputs.focus_regex}}" + echo "Running standard AWS integration tests..." + ./.github/scripts/s3/run-integration-aws.sh + elif [[ "${{inputs.test_type}}" == "aws-iam" ]]; then + echo "Running AWS IAM role tests..." + ./.github/scripts/s3/run-integration-aws-iam.sh + elif [[ "${{inputs.test_type}}" == "aws-assume" ]]; then + export assume_role_arn="${{inputs.role_arn}}" + export focus_regex="${{inputs.focus_regex}}" + echo "Running AWS assume role tests..." + ./.github/scripts/s3/run-integration-aws-assume.sh + else + echo "Error: Unknown test_type '${{inputs.test_type}}'" + echo "Valid options are: aws, aws-iam, aws-assume" + exit 1 + fi \ No newline at end of file diff --git a/.github/actions/s3-integration-setup/action.yml b/.github/actions/s3-integration-setup/action.yml new file mode 100644 index 0000000..f4ff1c6 --- /dev/null +++ b/.github/actions/s3-integration-setup/action.yml @@ -0,0 +1,34 @@ +name: Set up AWS S3 Integration Infrastructure +description: Sets up AWS S3 Integration Infrastructure for testing purposes. + +inputs: + access_key_id: + description: 'AWS Access Key ID' + required: true + secret_access_key: + description: 'AWS Secret Access Key' + required: true + region_name: + description: 'AWS Region Name' + required: true + stack_name: + description: 'CloudFormation Stack Name' + required: true + role_arn: + description: 'AWS Role ARN' + required: false + default: '' + +runs: + using: 'composite' + steps: + - name: Set up AWS Infrastructure + shell: bash + run: | + set -e + export access_key_id="${{inputs.access_key_id}}" + export secret_access_key="${{inputs.secret_access_key}}" + export role_arn="${{inputs.role_arn}}" + export region_name="${{inputs.region_name}}" + export stack_name="${{inputs.stack_name}}" + ./.github/scripts/s3/setup-aws-infrastructure.sh diff --git a/.github/actions/s3-integration-teardown/action.yml b/.github/actions/s3-integration-teardown/action.yml new file mode 100644 index 0000000..af3a606 --- /dev/null +++ b/.github/actions/s3-integration-teardown/action.yml @@ -0,0 +1,28 @@ +name: 'Tear down AWS S3 Integration Infrastructure' +description: 'Tears down AWS S3 Integration Infrastructure used for testing purposes.' +inputs: + access_key_id: + description: 'AWS Access Key ID' + required: true + secret_access_key: + description: 'AWS Secret Access Key' + required: true + region_name: + description: 'AWS Region Name' + required: true + stack_name: + description: 'CloudFormation Stack Name' + required: true + +runs: + using: 'composite' + steps: + - name: Teardown AWS Infrastructure + shell: bash + run: | + set -e + export access_key_id="${{inputs.access_key_id}}" + export secret_access_key="${{inputs.secret_access_key}}" + export region_name="${{inputs.region_name}}" + export stack_name="${{inputs.stack_name}}" + ./.github/scripts/s3/teardown-infrastructure.sh \ No newline at end of file diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 9c93d47..8a30d34 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -12,6 +12,10 @@ jobs: name: AWS S3 US Integration runs-on: ubuntu-latest environment: aws-integration + env: + REGION_NAME: us-east-1 + STACK_NAME: s3cli-iam + S3_ENDPOINT_HOST: https://s3.amazonaws.com steps: - name: Checkout code uses: actions/checkout@v4 @@ -19,68 +23,67 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version-file: go.mod - name: Install Ginkgo run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - name: Setup AWS infrastructure - run: | - set -e - export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export role_arn="" - export region_name="us-east-1" - export stack_name="s3cli-iam" - ./.github/scripts/s3/setup-aws-infrastructure.sh + uses: ./.github/actions/s3-integration-setup + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} - name: Test Static Credentials - run : | - set -e - export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export role_arn="" - export region_name="us-east-1" - export stack_name="s3cli-iam" - export s3_endpoint_host="https://s3.amazonaws.com" - export focus_regex="GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1" - ./.github/scripts/s3/run-integration-aws.sh + uses: ./.github/actions/s3-integration-run + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} + s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} + focus_regex: 'GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1' + test_type: 'aws' - name: Test IAM Roles - run : | - set -e - export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export region_name="us-east-1" - export stack_name="s3cli-iam" - ./.github/scripts/s3/run-integration-aws-iam.sh - + uses: ./.github/actions/s3-integration-run + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} + test_type: 'aws-iam' + - name: Test Assume Roles - run: | - set -e - export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export assume_role_arn="${{ secrets.AWS_ROLE_ARN }}" - export region_name="us-east-1" - export focus_regex="AWS ASSUME ROLE" - ./.github/scripts/s3/run-integration-aws-assume.sh + uses: ./.github/actions/s3-integration-run + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + role_arn: ${{ secrets.AWS_ROLE_ARN }} + focus_regex: 'AWS ASSUME ROLE' + test_type: 'aws-assume' - name: Teardown AWS infrastructure if: always() - run: | - set -e - export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export role_arn="" - export region_name="us-east-1" - export stack_name="s3cli-iam" - ./.github/scripts/s3/teardown-infrastructure.sh + uses: ./.github/actions/s3-integration-teardown + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} # AWS S3 Public Read Integration aws-s3-public-read-integration: name: AWS S3 Public Read Integration runs-on: ubuntu-latest environment: aws-integration + env: + REGION_NAME: us-east-1 + STACK_NAME: s3cli-public-bucket + S3_ENDPOINT_HOST: https://s3.amazonaws.com steps: - name: Checkout code uses: actions/checkout@v5 @@ -88,48 +91,49 @@ jobs: - name: Set up Go uses: actions/setup-go@v6 with: - go-version-file: 'go.mod' + go-version-file: go.mod - name: Install Ginkgo run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - name: Setup AWS infrastructure - run: | - set -e - export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export role_arn="" - export region_name="us-east-1" - export stack_name="s3cli-public-bucket" - ./.github/scripts/s3/setup-aws-infrastructure.sh + uses: ./.github/actions/s3-integration-setup + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} - name: Run public read tests - run: | - set -e - export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export role_arn="" - export region_name="us-east-1" - export stack_name="s3cli-public-bucket" - export focus_regex="PUBLIC READ ONLY" - ./.github/scripts/s3/run-integration-aws.sh + uses: ./.github/actions/s3-integration-run + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} + s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} + focus_regex: 'PUBLIC READ ONLY' + test_type: 'aws' - name: Teardown AWS infrastructure if: always() - run: | - set -e - export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export role_arn="" - export region_name="us-east-1" - export stack_name="s3cli-public-bucket" - ./.github/scripts/s3/teardown-infrastructure.sh + uses: ./.github/actions/s3-integration-teardown + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} # AWS S3 Frankfurt Integration aws-s3-frankfurt-integration: name: AWS S3 Frankfurt Integration runs-on: ubuntu-latest environment: aws-integration + env: + REGION_NAME: eu-central-1 + STACK_NAME: s3cli-private-bucket + S3_ENDPOINT_HOST: https://s3.eu-central-1.amazonaws.com + steps: - name: Checkout code uses: actions/checkout@v5 @@ -137,42 +141,38 @@ jobs: - name: Set up Go uses: actions/setup-go@v6 with: - go-version-file: 'go.mod' + go-version-file: go.mod - name: Install Ginkgo run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - name: Setup AWS infrastructure - run: | - set -e - export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export role_arn="" - export region_name="eu-central-1" - export stack_name="s3cli-private-bucket" - ./.github/scripts/s3/setup-aws-infrastructure.sh + uses: ./.github/actions/s3-integration-setup + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} - name: Run Frankfurt region tests - run: | - set -e - export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export role_arn="" - export region_name="eu-central-1" - export stack_name="s3cli-private-bucket" - export focus_regex="GENERAL AWS|AWS V4 REGION" - export s3_endpoint_host="https://s3.eu-central-1.amazonaws.com" - ./.github/scripts/s3/run-integration-aws.sh + uses: ./.github/actions/s3-integration-run + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} + s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} + focus_regex: 'GENERAL AWS|AWS V4 REGION' + test_type: 'aws' - name: Teardown AWS infrastructure if: always() - run: | - set -e - export access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" - export region_name="eu-central-1" - export stack_name="s3cli-private-bucket" - ./.github/scripts/s3/teardown-infrastructure.sh + uses: ./.github/actions/s3-integration-teardown + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} # s3-compatible-integration: @@ -186,7 +186,7 @@ jobs: # - name: Set up Go # uses: actions/setup-go@v5 # with: -# go-version-file: 'go.mod' +# go-version-file: go.mod # - name: Install Ginkgo # run: go install github.com/onsi/ginkgo/v2/ginkgo@latest From f20fffe753b3e411fd057744e5dc1f0d20c48af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 13 Nov 2025 08:56:27 +0100 Subject: [PATCH 20/32] feat: run only s3 compatible tests --- .github/workflows/s3-integration.yml | 358 +++++++++++++-------------- 1 file changed, 179 insertions(+), 179 deletions(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 8a30d34..2f90de6 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -7,195 +7,195 @@ on: branches: [ main ] jobs: - # AWS S3 US Integration Tests - aws-s3-us-integration: - name: AWS S3 US Integration - runs-on: ubuntu-latest - environment: aws-integration - env: - REGION_NAME: us-east-1 - STACK_NAME: s3cli-iam - S3_ENDPOINT_HOST: https://s3.amazonaws.com - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - - name: Install Ginkgo - run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - - - name: Setup AWS infrastructure - uses: ./.github/actions/s3-integration-setup - with: - access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - region_name: ${{ env.REGION_NAME }} - stack_name: ${{ env.STACK_NAME }} + # # AWS S3 US Integration Tests + # aws-s3-us-integration: + # name: AWS S3 US Integration + # runs-on: ubuntu-latest + # environment: aws-integration + # env: + # REGION_NAME: us-east-1 + # STACK_NAME: s3cli-iam + # S3_ENDPOINT_HOST: https://s3.amazonaws.com + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + + # - name: Set up Go + # uses: actions/setup-go@v5 + # with: + # go-version-file: go.mod + + # - name: Install Ginkgo + # run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + + # - name: Setup AWS infrastructure + # uses: ./.github/actions/s3-integration-setup + # with: + # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # region_name: ${{ env.REGION_NAME }} + # stack_name: ${{ env.STACK_NAME }} - - name: Test Static Credentials - uses: ./.github/actions/s3-integration-run - with: - access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - region_name: ${{ env.REGION_NAME }} - stack_name: ${{ env.STACK_NAME }} - s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} - focus_regex: 'GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1' - test_type: 'aws' - - - name: Test IAM Roles - uses: ./.github/actions/s3-integration-run - with: - access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - region_name: ${{ env.REGION_NAME }} - stack_name: ${{ env.STACK_NAME }} - test_type: 'aws-iam' - - - name: Test Assume Roles - uses: ./.github/actions/s3-integration-run - with: - access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - region_name: ${{ env.REGION_NAME }} - role_arn: ${{ secrets.AWS_ROLE_ARN }} - focus_regex: 'AWS ASSUME ROLE' - test_type: 'aws-assume' - - - name: Teardown AWS infrastructure - if: always() - uses: ./.github/actions/s3-integration-teardown - with: - access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - region_name: ${{ env.REGION_NAME }} - stack_name: ${{ env.STACK_NAME }} - - # AWS S3 Public Read Integration - aws-s3-public-read-integration: - name: AWS S3 Public Read Integration - runs-on: ubuntu-latest - environment: aws-integration - env: - REGION_NAME: us-east-1 - STACK_NAME: s3cli-public-bucket - S3_ENDPOINT_HOST: https://s3.amazonaws.com - steps: - - name: Checkout code - uses: actions/checkout@v5 - - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - - - name: Install Ginkgo - run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - - - name: Setup AWS infrastructure - uses: ./.github/actions/s3-integration-setup - with: - access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - region_name: ${{ env.REGION_NAME }} - stack_name: ${{ env.STACK_NAME }} - - - name: Run public read tests - uses: ./.github/actions/s3-integration-run - with: - access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - region_name: ${{ env.REGION_NAME }} - stack_name: ${{ env.STACK_NAME }} - s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} - focus_regex: 'PUBLIC READ ONLY' - test_type: 'aws' - - - name: Teardown AWS infrastructure - if: always() - uses: ./.github/actions/s3-integration-teardown - with: - access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - region_name: ${{ env.REGION_NAME }} - stack_name: ${{ env.STACK_NAME }} - - # AWS S3 Frankfurt Integration - aws-s3-frankfurt-integration: - name: AWS S3 Frankfurt Integration + # - name: Test Static Credentials + # uses: ./.github/actions/s3-integration-run + # with: + # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # region_name: ${{ env.REGION_NAME }} + # stack_name: ${{ env.STACK_NAME }} + # s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} + # focus_regex: 'GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1' + # test_type: 'aws' + + # - name: Test IAM Roles + # uses: ./.github/actions/s3-integration-run + # with: + # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # region_name: ${{ env.REGION_NAME }} + # stack_name: ${{ env.STACK_NAME }} + # test_type: 'aws-iam' + + # - name: Test Assume Roles + # uses: ./.github/actions/s3-integration-run + # with: + # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # region_name: ${{ env.REGION_NAME }} + # role_arn: ${{ secrets.AWS_ROLE_ARN }} + # focus_regex: 'AWS ASSUME ROLE' + # test_type: 'aws-assume' + + # - name: Teardown AWS infrastructure + # if: always() + # uses: ./.github/actions/s3-integration-teardown + # with: + # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # region_name: ${{ env.REGION_NAME }} + # stack_name: ${{ env.STACK_NAME }} + + # # AWS S3 Public Read Integration + # aws-s3-public-read-integration: + # name: AWS S3 Public Read Integration + # runs-on: ubuntu-latest + # environment: aws-integration + # env: + # REGION_NAME: us-east-1 + # STACK_NAME: s3cli-public-bucket + # S3_ENDPOINT_HOST: https://s3.amazonaws.com + # steps: + # - name: Checkout code + # uses: actions/checkout@v5 + + # - name: Set up Go + # uses: actions/setup-go@v6 + # with: + # go-version-file: go.mod + + # - name: Install Ginkgo + # run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + + # - name: Setup AWS infrastructure + # uses: ./.github/actions/s3-integration-setup + # with: + # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # region_name: ${{ env.REGION_NAME }} + # stack_name: ${{ env.STACK_NAME }} + + # - name: Run public read tests + # uses: ./.github/actions/s3-integration-run + # with: + # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # region_name: ${{ env.REGION_NAME }} + # stack_name: ${{ env.STACK_NAME }} + # s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} + # focus_regex: 'PUBLIC READ ONLY' + # test_type: 'aws' + + # - name: Teardown AWS infrastructure + # if: always() + # uses: ./.github/actions/s3-integration-teardown + # with: + # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # region_name: ${{ env.REGION_NAME }} + # stack_name: ${{ env.STACK_NAME }} + + # # AWS S3 Frankfurt Integration + # aws-s3-frankfurt-integration: + # name: AWS S3 Frankfurt Integration + # runs-on: ubuntu-latest + # environment: aws-integration + # env: + # REGION_NAME: eu-central-1 + # STACK_NAME: s3cli-private-bucket + # S3_ENDPOINT_HOST: https://s3.eu-central-1.amazonaws.com + + # steps: + # - name: Checkout code + # uses: actions/checkout@v5 + + # - name: Set up Go + # uses: actions/setup-go@v6 + # with: + # go-version-file: go.mod + + # - name: Install Ginkgo + # run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + + # - name: Setup AWS infrastructure + # uses: ./.github/actions/s3-integration-setup + # with: + # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # region_name: ${{ env.REGION_NAME }} + # stack_name: ${{ env.STACK_NAME }} + + # - name: Run Frankfurt region tests + # uses: ./.github/actions/s3-integration-run + # with: + # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # region_name: ${{ env.REGION_NAME }} + # stack_name: ${{ env.STACK_NAME }} + # s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} + # focus_regex: 'GENERAL AWS|AWS V4 REGION' + # test_type: 'aws' + + # - name: Teardown AWS infrastructure + # if: always() + # uses: ./.github/actions/s3-integration-teardown + # with: + # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # region_name: ${{ env.REGION_NAME }} + # stack_name: ${{ env.STACK_NAME }} + + + s3-compatible-integration: + name: S3 Compatible Integration runs-on: ubuntu-latest environment: aws-integration - env: - REGION_NAME: eu-central-1 - STACK_NAME: s3cli-private-bucket - S3_ENDPOINT_HOST: https://s3.eu-central-1.amazonaws.com - steps: - name: Checkout code uses: actions/checkout@v5 - + - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: go.mod - + - name: Install Ginkgo run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - - name: Setup AWS infrastructure - uses: ./.github/actions/s3-integration-setup - with: - access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - region_name: ${{ env.REGION_NAME }} - stack_name: ${{ env.STACK_NAME }} - - - name: Run Frankfurt region tests - uses: ./.github/actions/s3-integration-run - with: - access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - region_name: ${{ env.REGION_NAME }} - stack_name: ${{ env.STACK_NAME }} - s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} - focus_regex: 'GENERAL AWS|AWS V4 REGION' - test_type: 'aws' - - - name: Teardown AWS infrastructure - if: always() - uses: ./.github/actions/s3-integration-teardown - with: - access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - region_name: ${{ env.REGION_NAME }} - stack_name: ${{ env.STACK_NAME }} - - -# s3-compatible-integration: -# name: S3 Compatible Integration -# runs-on: ubuntu-latest -# environment: gcp-integration -# steps: -# - name: Checkout code -# uses: actions/checkout@v4 - -# - name: Set up Go -# uses: actions/setup-go@v5 -# with: -# go-version-file: go.mod - -# - name: Install Ginkgo -# run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - -# - name: Run GCS S3 compatible tests -# run: | -# export access_key_id="${{ secrets.GCP_ACCESS_KEY_ID }}" -# export secret_access_key="${{ secrets.GCP_SECRET_ACCESS_KEY }}" -# export bucket_name="${{ secrets.GCS_BUCKET_NAME }}" -# export s3_endpoint_host="storage.googleapis.com" -# export s3_endpoint_port="443" -# ./ci/tasks/run-integration-s3-compat.sh + - name: Run GCS S3 compatible tests + run: | + export access_key_id="${{ secrets.GCP_ACCESS_KEY_ID }}" + export secret_access_key="${{ secrets.GCP_SECRET_ACCESS_KEY }}" + export bucket_name="${{ secrets.GCS_BUCKET_NAME }}" + export s3_endpoint_host=https://storage.googleapis.com + export s3_endpoint_port=443 + ./ci/tasks/run-integration-s3-compat.sh \ No newline at end of file From 3301028a27fa26e5db7b767a41bc55fe880a3a12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 13 Nov 2025 08:58:25 +0100 Subject: [PATCH 21/32] fix: file location fixed --- .github/workflows/s3-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 2f90de6..5d7b398 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -198,4 +198,4 @@ jobs: export bucket_name="${{ secrets.GCS_BUCKET_NAME }}" export s3_endpoint_host=https://storage.googleapis.com export s3_endpoint_port=443 - ./ci/tasks/run-integration-s3-compat.sh \ No newline at end of file + ./.github/scripts/s3/run-integration-s3-compat.sh \ No newline at end of file From 1cd96e849ceca34a59bc7fbd5bc5e8a90a70d920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 13 Nov 2025 09:01:59 +0100 Subject: [PATCH 22/32] fix: unbound var release_dir fixed --- .github/scripts/s3/run-integration-s3-compat.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/s3/run-integration-s3-compat.sh b/.github/scripts/s3/run-integration-s3-compat.sh index 83c8c51..63d452e 100755 --- a/.github/scripts/s3/run-integration-s3-compat.sh +++ b/.github/scripts/s3/run-integration-s3-compat.sh @@ -4,6 +4,8 @@ set -euo pipefail # Get the directory where this script is located script_dir="$( cd "$(dirname "${0}")" && pwd )" +repo_root="$(cd "${script_dir}/../../.." && pwd)" + # Source utils from the same directory source "${script_dir}/utils.sh" @@ -20,7 +22,7 @@ export BUCKET_NAME=${bucket_name} export S3_HOST=${s3_endpoint_host} export S3_PORT=${s3_endpoint_port} -pushd "${release_dir}" > /dev/null +pushd "${repo_root}" > /dev/null echo -e "\n running tests with $(go version)..." scripts/ginkgo -r --focus="S3 COMPATIBLE" s3/integration/ popd > /dev/null From ac988b6b87a440065a210a68d494868efc59d5d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 13 Nov 2025 09:05:51 +0100 Subject: [PATCH 23/32] fix: ginkgo used directly --- .github/scripts/s3/run-integration-s3-compat.sh | 2 +- .github/workflows/s3-integration.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/s3/run-integration-s3-compat.sh b/.github/scripts/s3/run-integration-s3-compat.sh index 63d452e..e63b954 100755 --- a/.github/scripts/s3/run-integration-s3-compat.sh +++ b/.github/scripts/s3/run-integration-s3-compat.sh @@ -24,5 +24,5 @@ export S3_PORT=${s3_endpoint_port} pushd "${repo_root}" > /dev/null echo -e "\n running tests with $(go version)..." - scripts/ginkgo -r --focus="S3 COMPATIBLE" s3/integration/ + ginkgo -r --focus="S3 COMPATIBLE" s3/integration/ popd > /dev/null diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 5d7b398..962a4f6 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -196,6 +196,6 @@ jobs: export access_key_id="${{ secrets.GCP_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.GCP_SECRET_ACCESS_KEY }}" export bucket_name="${{ secrets.GCS_BUCKET_NAME }}" - export s3_endpoint_host=https://storage.googleapis.com - export s3_endpoint_port=443 + export s3_endpoint_host="storage.googleapis.com" + export s3_endpoint_port="443" ./.github/scripts/s3/run-integration-s3-compat.sh \ No newline at end of file From 03498451c71214e5a8c89597a9e0d1071a44e59a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 13 Nov 2025 09:08:18 +0100 Subject: [PATCH 24/32] fix: https protocol added --- .github/workflows/s3-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 962a4f6..6362e41 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -196,6 +196,6 @@ jobs: export access_key_id="${{ secrets.GCP_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.GCP_SECRET_ACCESS_KEY }}" export bucket_name="${{ secrets.GCS_BUCKET_NAME }}" - export s3_endpoint_host="storage.googleapis.com" + export s3_endpoint_host="https://storage.googleapis.com" export s3_endpoint_port="443" ./.github/scripts/s3/run-integration-s3-compat.sh \ No newline at end of file From c1214ed8cd072eef9112a5bbcb348363547314d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 13 Nov 2025 09:24:06 +0100 Subject: [PATCH 25/32] fix: bucket name added as env var --- .github/workflows/s3-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 6362e41..57384fe 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -195,7 +195,7 @@ jobs: run: | export access_key_id="${{ secrets.GCP_ACCESS_KEY_ID }}" export secret_access_key="${{ secrets.GCP_SECRET_ACCESS_KEY }}" - export bucket_name="${{ secrets.GCS_BUCKET_NAME }}" + export bucket_name="storage-cli-test-aws" export s3_endpoint_host="https://storage.googleapis.com" export s3_endpoint_port="443" ./.github/scripts/s3/run-integration-s3-compat.sh \ No newline at end of file From 4bf9ce5567000a1ab11a3a47e76d6f8585c86676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 13 Nov 2025 13:10:02 +0100 Subject: [PATCH 26/32] fix: s3-compatible-integration job needs deep dive investigation not working --- .github/workflows/s3-integration.yml | 344 +++++++++++++-------------- 1 file changed, 172 insertions(+), 172 deletions(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 57384fe..26cc6dd 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -7,195 +7,195 @@ on: branches: [ main ] jobs: - # # AWS S3 US Integration Tests - # aws-s3-us-integration: - # name: AWS S3 US Integration - # runs-on: ubuntu-latest - # environment: aws-integration - # env: - # REGION_NAME: us-east-1 - # STACK_NAME: s3cli-iam - # S3_ENDPOINT_HOST: https://s3.amazonaws.com - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 + # AWS S3 US Integration Tests + aws-s3-us-integration: + name: AWS S3 US Integration + runs-on: ubuntu-latest + environment: aws-integration + env: + REGION_NAME: us-east-1 + STACK_NAME: s3cli-iam + S3_ENDPOINT_HOST: https://s3.amazonaws.com + steps: + - name: Checkout code + uses: actions/checkout@v4 - # - name: Set up Go - # uses: actions/setup-go@v5 - # with: - # go-version-file: go.mod + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod - # - name: Install Ginkgo - # run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + - name: Install Ginkgo + run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - # - name: Setup AWS infrastructure - # uses: ./.github/actions/s3-integration-setup - # with: - # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # region_name: ${{ env.REGION_NAME }} - # stack_name: ${{ env.STACK_NAME }} + - name: Setup AWS infrastructure + uses: ./.github/actions/s3-integration-setup + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} - # - name: Test Static Credentials - # uses: ./.github/actions/s3-integration-run - # with: - # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # region_name: ${{ env.REGION_NAME }} - # stack_name: ${{ env.STACK_NAME }} - # s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} - # focus_regex: 'GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1' - # test_type: 'aws' - - # - name: Test IAM Roles - # uses: ./.github/actions/s3-integration-run - # with: - # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # region_name: ${{ env.REGION_NAME }} - # stack_name: ${{ env.STACK_NAME }} - # test_type: 'aws-iam' - - # - name: Test Assume Roles - # uses: ./.github/actions/s3-integration-run - # with: - # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # region_name: ${{ env.REGION_NAME }} - # role_arn: ${{ secrets.AWS_ROLE_ARN }} - # focus_regex: 'AWS ASSUME ROLE' - # test_type: 'aws-assume' - - # - name: Teardown AWS infrastructure - # if: always() - # uses: ./.github/actions/s3-integration-teardown - # with: - # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # region_name: ${{ env.REGION_NAME }} - # stack_name: ${{ env.STACK_NAME }} - - # # AWS S3 Public Read Integration - # aws-s3-public-read-integration: - # name: AWS S3 Public Read Integration - # runs-on: ubuntu-latest - # environment: aws-integration - # env: - # REGION_NAME: us-east-1 - # STACK_NAME: s3cli-public-bucket - # S3_ENDPOINT_HOST: https://s3.amazonaws.com - # steps: - # - name: Checkout code - # uses: actions/checkout@v5 - - # - name: Set up Go - # uses: actions/setup-go@v6 - # with: - # go-version-file: go.mod - - # - name: Install Ginkgo - # run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - - # - name: Setup AWS infrastructure - # uses: ./.github/actions/s3-integration-setup - # with: - # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # region_name: ${{ env.REGION_NAME }} - # stack_name: ${{ env.STACK_NAME }} - - # - name: Run public read tests - # uses: ./.github/actions/s3-integration-run - # with: - # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # region_name: ${{ env.REGION_NAME }} - # stack_name: ${{ env.STACK_NAME }} - # s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} - # focus_regex: 'PUBLIC READ ONLY' - # test_type: 'aws' - - # - name: Teardown AWS infrastructure - # if: always() - # uses: ./.github/actions/s3-integration-teardown - # with: - # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # region_name: ${{ env.REGION_NAME }} - # stack_name: ${{ env.STACK_NAME }} - - # # AWS S3 Frankfurt Integration - # aws-s3-frankfurt-integration: - # name: AWS S3 Frankfurt Integration - # runs-on: ubuntu-latest - # environment: aws-integration - # env: - # REGION_NAME: eu-central-1 - # STACK_NAME: s3cli-private-bucket - # S3_ENDPOINT_HOST: https://s3.eu-central-1.amazonaws.com - - # steps: - # - name: Checkout code - # uses: actions/checkout@v5 - - # - name: Set up Go - # uses: actions/setup-go@v6 - # with: - # go-version-file: go.mod - - # - name: Install Ginkgo - # run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + - name: Test Static Credentials + uses: ./.github/actions/s3-integration-run + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} + s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} + focus_regex: 'GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1' + test_type: 'aws' + + - name: Test IAM Roles + uses: ./.github/actions/s3-integration-run + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} + test_type: 'aws-iam' + + - name: Test Assume Roles + uses: ./.github/actions/s3-integration-run + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + role_arn: ${{ secrets.AWS_ROLE_ARN }} + focus_regex: 'AWS ASSUME ROLE' + test_type: 'aws-assume' + + - name: Teardown AWS infrastructure + if: always() + uses: ./.github/actions/s3-integration-teardown + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} + + # AWS S3 Public Read Integration + aws-s3-public-read-integration: + name: AWS S3 Public Read Integration + runs-on: ubuntu-latest + environment: aws-integration + env: + REGION_NAME: us-east-1 + STACK_NAME: s3cli-public-bucket + S3_ENDPOINT_HOST: https://s3.amazonaws.com + steps: + - name: Checkout code + uses: actions/checkout@v5 - # - name: Setup AWS infrastructure - # uses: ./.github/actions/s3-integration-setup - # with: - # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # region_name: ${{ env.REGION_NAME }} - # stack_name: ${{ env.STACK_NAME }} + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod - # - name: Run Frankfurt region tests - # uses: ./.github/actions/s3-integration-run - # with: - # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # region_name: ${{ env.REGION_NAME }} - # stack_name: ${{ env.STACK_NAME }} - # s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} - # focus_regex: 'GENERAL AWS|AWS V4 REGION' - # test_type: 'aws' - - # - name: Teardown AWS infrastructure - # if: always() - # uses: ./.github/actions/s3-integration-teardown - # with: - # access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # region_name: ${{ env.REGION_NAME }} - # stack_name: ${{ env.STACK_NAME }} + - name: Install Ginkgo + run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + - name: Setup AWS infrastructure + uses: ./.github/actions/s3-integration-setup + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} - s3-compatible-integration: - name: S3 Compatible Integration + - name: Run public read tests + uses: ./.github/actions/s3-integration-run + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} + s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} + focus_regex: 'PUBLIC READ ONLY' + test_type: 'aws' + + - name: Teardown AWS infrastructure + if: always() + uses: ./.github/actions/s3-integration-teardown + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} + + # AWS S3 Frankfurt Integration + aws-s3-frankfurt-integration: + name: AWS S3 Frankfurt Integration runs-on: ubuntu-latest environment: aws-integration + env: + REGION_NAME: eu-central-1 + STACK_NAME: s3cli-private-bucket + S3_ENDPOINT_HOST: https://s3.eu-central-1.amazonaws.com + steps: - name: Checkout code uses: actions/checkout@v5 - + - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: go.mod - + - name: Install Ginkgo run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - - name: Run GCS S3 compatible tests - run: | - export access_key_id="${{ secrets.GCP_ACCESS_KEY_ID }}" - export secret_access_key="${{ secrets.GCP_SECRET_ACCESS_KEY }}" - export bucket_name="storage-cli-test-aws" - export s3_endpoint_host="https://storage.googleapis.com" - export s3_endpoint_port="443" - ./.github/scripts/s3/run-integration-s3-compat.sh \ No newline at end of file + - name: Setup AWS infrastructure + uses: ./.github/actions/s3-integration-setup + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} + + - name: Run Frankfurt region tests + uses: ./.github/actions/s3-integration-run + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} + s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} + focus_regex: 'GENERAL AWS|AWS V4 REGION' + test_type: 'aws' + + - name: Teardown AWS infrastructure + if: always() + uses: ./.github/actions/s3-integration-teardown + with: + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region_name: ${{ env.REGION_NAME }} + stack_name: ${{ env.STACK_NAME }} + + + # s3-compatible-integration: + # name: S3 Compatible Integration + # runs-on: ubuntu-latest + # environment: aws-integration + # steps: + # - name: Checkout code + # uses: actions/checkout@v5 + + # - name: Set up Go + # uses: actions/setup-go@v6 + # with: + # go-version-file: go.mod + + # - name: Install Ginkgo + # run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + + # - name: Run GCS S3 compatible tests + # run: | + # export access_key_id="${{ secrets.GCP_ACCESS_KEY_ID }}" + # export secret_access_key="${{ secrets.GCP_SECRET_ACCESS_KEY }}" + # export bucket_name="storage-cli-test-aws" + # export s3_endpoint_host="https://storage.googleapis.com" + # export s3_endpoint_port="443" + # ./.github/scripts/s3/run-integration-s3-compat.sh \ No newline at end of file From 7312ee6ccdef45e58390f892560cded9cba7d2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Mon, 17 Nov 2025 11:31:27 +0100 Subject: [PATCH 27/32] feat: path for pr added --- .github/workflows/s3-integration.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 26cc6dd..44f9146 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -1,10 +1,10 @@ name: S3 Integration Tests on: - push: - branches: [ main, feature/aws-integration-tests ] pull_request: - branches: [ main ] + paths: + - ".github/workflows/s3-integration.yml" + - "s3/**" jobs: # AWS S3 US Integration Tests @@ -18,10 +18,10 @@ jobs: S3_ENDPOINT_HOST: https://s3.amazonaws.com steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -162,7 +162,7 @@ jobs: region_name: ${{ env.REGION_NAME }} stack_name: ${{ env.STACK_NAME }} s3_endpoint_host: ${{ env.S3_ENDPOINT_HOST }} - focus_regex: 'GENERAL AWS|AWS V4 REGION' + focus_regex: 'GENERAL AWS|AWS V4 REGION' # TODO: after aws-sdk-go-v2 migration, AWS V4 ONLY REGION test is failling, Removed temporarily from focus. test_type: 'aws' - name: Teardown AWS infrastructure @@ -174,7 +174,7 @@ jobs: region_name: ${{ env.REGION_NAME }} stack_name: ${{ env.STACK_NAME }} - + # TODO: after aws-sdk-go-v2 migration, not working properly. Disabled for now. # s3-compatible-integration: # name: S3 Compatible Integration # runs-on: ubuntu-latest From d4969a58c9d5e7796b1bb64e208224c23b2f309f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Mon, 17 Nov 2025 12:57:55 +0100 Subject: [PATCH 28/32] fix: bucket name reverted --- s3/integration/aws_assume_role_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3/integration/aws_assume_role_test.go b/s3/integration/aws_assume_role_test.go index c436701..0b43f2b 100644 --- a/s3/integration/aws_assume_role_test.go +++ b/s3/integration/aws_assume_role_test.go @@ -22,7 +22,7 @@ var _ = Describe("Testing AWS assume role ", func() { assumeRoleArn := os.Getenv("ASSUME_ROLE_ARN") Expect(assumeRoleArn).ToNot(BeEmpty(), "ASSUME_ROLE_ARN must be set") - bucketName := "bosh-s3cli-assume-role-integration-test-1" + bucketName := "bosh-s3cli-assume-role-integration-test" region := "us-east-1" nonAssumedRoleCfg := &config.S3Cli{ From 160eb6a07c31bc68c05c06f7912e0c37e5c338d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Tue, 18 Nov 2025 12:16:35 +0100 Subject: [PATCH 29/32] fix: environment names is changed --- .github/workflows/s3-integration.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index 44f9146..ac228bf 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -11,7 +11,7 @@ jobs: aws-s3-us-integration: name: AWS S3 US Integration runs-on: ubuntu-latest - environment: aws-integration + environment: s3-integration env: REGION_NAME: us-east-1 STACK_NAME: s3cli-iam @@ -79,7 +79,7 @@ jobs: aws-s3-public-read-integration: name: AWS S3 Public Read Integration runs-on: ubuntu-latest - environment: aws-integration + environment: s3-integration env: REGION_NAME: us-east-1 STACK_NAME: s3cli-public-bucket @@ -128,7 +128,7 @@ jobs: aws-s3-frankfurt-integration: name: AWS S3 Frankfurt Integration runs-on: ubuntu-latest - environment: aws-integration + environment: s3-integration env: REGION_NAME: eu-central-1 STACK_NAME: s3cli-private-bucket @@ -178,7 +178,7 @@ jobs: # s3-compatible-integration: # name: S3 Compatible Integration # runs-on: ubuntu-latest - # environment: aws-integration + # environment: s3-integration # steps: # - name: Checkout code # uses: actions/checkout@v5 From cb8ccf6de079ceb0c6a87ef3ce650c0f479f1ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Thu, 20 Nov 2025 09:20:18 +0100 Subject: [PATCH 30/32] fix: environment tag in workflow is removed --- .github/workflows/s3-integration.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index ac228bf..e9cfe68 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -11,7 +11,6 @@ jobs: aws-s3-us-integration: name: AWS S3 US Integration runs-on: ubuntu-latest - environment: s3-integration env: REGION_NAME: us-east-1 STACK_NAME: s3cli-iam @@ -79,7 +78,6 @@ jobs: aws-s3-public-read-integration: name: AWS S3 Public Read Integration runs-on: ubuntu-latest - environment: s3-integration env: REGION_NAME: us-east-1 STACK_NAME: s3cli-public-bucket @@ -128,7 +126,6 @@ jobs: aws-s3-frankfurt-integration: name: AWS S3 Frankfurt Integration runs-on: ubuntu-latest - environment: s3-integration env: REGION_NAME: eu-central-1 STACK_NAME: s3cli-private-bucket @@ -178,7 +175,6 @@ jobs: # s3-compatible-integration: # name: S3 Compatible Integration # runs-on: ubuntu-latest - # environment: s3-integration # steps: # - name: Checkout code # uses: actions/checkout@v5 From 9184adbdc64daa347fcd0106a55fd0de0d44abfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 21 Nov 2025 10:52:10 +0100 Subject: [PATCH 31/32] docs: paths are changed based on new repo --- s3/README.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/s3/README.md b/s3/README.md index 59e822e..bc567be 100644 --- a/s3/README.md +++ b/s3/README.md @@ -3,7 +3,7 @@ A CLI for uploading, fetching and deleting content to/from an S3-compatible blobstore. -Continuous integration: +Continuous integration: Releases can be found in `https://s3.amazonaws.com/bosh-s3cli-artifacts`. The Linux binaries follow the regex `s3cli-(\d+\.\d+\.\d+)-linux-amd64` and the windows binaries `s3cli-(\d+\.\d+\.\d+)-windows-amd64`. @@ -69,12 +69,12 @@ Follow these steps to make a contribution to the project: ``` - Run tests to check your development environment setup ``` bash - scripts/ginkgo -r -race --skip-package=integration ./ + ginkgo --race --skip-package=integration --randomize-all --cover -v -r ./s3/... ``` - Make your changes (*be sure to add/update tests*) - Run tests to check your changes ``` bash - scripts/ginkgo -r -race --skip-package=integration ./ + ginkgo --race --skip-package=integration --randomize-all --cover -v -r ./s3/... ``` - Push changes to your fork ``` bash @@ -84,7 +84,17 @@ Follow these steps to make a contribution to the project: ``` - Create a GitHub pull request, selecting `main` as the target branch -## Running integration tests +## Testing + +### Unit Tests +**Note:** Run the following commands from the repository root directory. + ``` bash + go install github.com/onsi/ginkgo/v2/ginkgo + + ginkgo --skip-package=integration --randomize-all --cover -v -r ./s3/... + ``` + +### Integration Tests To run the integration tests, export the following variables into your environment: @@ -98,4 +108,4 @@ export stack_name=s3cli-iam export bucket_name=s3cli-pipeline ``` -Run `ci/tasks/setup-aws-infrastructure.sh` and `ci/tasks/teardown-infrastructure.sh` before and after the `run-integration-*` tests in `ci/tasks`. +Run `./.github/scripts/s3/setup-aws-infrastructure.sh` and `./.github/scripts/s3/teardown-infrastructure.sh` before and after the `./.github/scripts/s3/run-integration-*` in repo's root folder. From 5c79e0dbc2338b573cd1aaf230c9dd66cebfaf75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 21 Nov 2025 13:37:33 +0100 Subject: [PATCH 32/32] feat: push main and manual trigger are added --- .github/workflows/s3-integration.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/s3-integration.yml b/.github/workflows/s3-integration.yml index e9cfe68..eff8736 100644 --- a/.github/workflows/s3-integration.yml +++ b/.github/workflows/s3-integration.yml @@ -1,10 +1,14 @@ name: S3 Integration Tests on: + workflow_dispatch: pull_request: paths: - ".github/workflows/s3-integration.yml" - "s3/**" + push: + branches: + - main jobs: # AWS S3 US Integration Tests