diff --git a/.github/workflows/rhiza_release.yml b/.github/workflows/rhiza_release.yml index 24b93597..973a56d2 100644 --- a/.github/workflows/rhiza_release.yml +++ b/.github/workflows/rhiza_release.yml @@ -32,14 +32,17 @@ # # 🚀 PyPI Publishing: # - Skipped if no dist/ artifacts exist -# - Skipped if pyproject.toml contains "Private :: Do Not Upload" +# - Skipped if pyproject.toml contains "Private :: Do Not Upload" (unless custom feed) # - Uses Trusted Publishing (OIDC) for PyPI (no stored credentials) -# - For custom feeds, use PYPI_REPOSITORY_URL and PYPI_TOKEN secrets +# - For custom feeds, use PYPI_REPOSITORY_URL variable and PYPI_TOKEN secret +# - For AWS CodeArtifact, set PYPI_REPOSITORY_URL to the CodeArtifact endpoint +# and provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY secrets # - Adds PyPI/custom feed link to GitHub release notes # # 🔐 Security: # - No PyPI credentials stored; relies on Trusted Publishing via GitHub OIDC # - For custom feeds, PYPI_TOKEN secret is used with default username __token__ +# - For AWS CodeArtifact, uses AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY secrets # - Container registry uses GITHUB_TOKEN for authentication # - SLSA provenance attestations generated for build artifacts (public repos only) # - SBOM attestations generated for supply chain transparency (public repos only) @@ -47,6 +50,8 @@ # 📄 Requirements: # - pyproject.toml with top-level version field (for Python packages) # - Package registered on PyPI as Trusted Publisher (for PyPI publishing) +# - PYPI_REPOSITORY_URL variable + AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY secrets +# (for CodeArtifact publishing) # - PUBLISH_DEVCONTAINER variable set to "true" (for devcontainer publishing) # - .devcontainer/devcontainer.json file (for devcontainer publishing) # @@ -61,6 +66,11 @@ # 🎯 For repos without devcontainers: # The workflow gracefully skips devcontainer steps if PUBLISH_DEVCONTAINER is not # set to "true" or .devcontainer directory doesn't exist. +# +# 🎯 For repos with AWS CodeArtifact: +# Set PYPI_REPOSITORY_URL to the CodeArtifact endpoint and provide +# AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY secrets. The workflow +# automatically detects CodeArtifact URLs and handles token exchange. name: (RHIZA) RELEASE @@ -267,20 +277,61 @@ jobs: - name: Check if dist contains artifacts and not marked as private id: check_dist + env: + PYPI_REPOSITORY_URL: ${{ vars.PYPI_REPOSITORY_URL }} run: | if [[ ! -d dist ]]; then echo "::warning::No folder dist/. Skipping PyPI publish." echo "should_publish=false" >> "$GITHUB_OUTPUT" + elif [[ -z "$PYPI_REPOSITORY_URL" ]] && grep -q "Private :: Do Not Upload" pyproject.toml; then + echo "::notice::Package marked as private. Skipping PyPI publish." + echo "should_publish=false" >> "$GITHUB_OUTPUT" else - if grep -R "Private :: Do Not Upload" pyproject.toml; then - echo "should_publish=false" >> "$GITHUB_OUTPUT" + echo "should_publish=true" >> "$GITHUB_OUTPUT" + fi + cat "$GITHUB_OUTPUT" + + - name: Get AWS CodeArtifact token + if: ${{ steps.check_dist.outputs.should_publish == 'true' }} + id: aws_token + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + PYPI_REPOSITORY_URL: ${{ vars.PYPI_REPOSITORY_URL }} + run: | + if [[ -n "$AWS_ACCESS_KEY_ID" && "$PYPI_REPOSITORY_URL" == *".codeartifact."* ]]; then + # Parse domain and region from CodeArtifact URL + # Format: https://{domain}-{owner}.d.codeartifact.{region}.amazonaws.com/pypi/{repo}/ + STRIPPED="${PYPI_REPOSITORY_URL#https://}" + HOSTNAME="${STRIPPED%%/*}" + REGION="${HOSTNAME#*.d.codeartifact.}" + REGION="${REGION%.amazonaws.com}" + DOMAIN_OWNER="${HOSTNAME%%.d.codeartifact.*}" + OWNER=$(echo "$DOMAIN_OWNER" | grep -oE '[0-9]{12}$' || true) + if [ -n "$OWNER" ]; then + DOMAIN="${DOMAIN_OWNER%-"$OWNER"}" else - echo "should_publish=true" >> "$GITHUB_OUTPUT" + DOMAIN="$DOMAIN_OWNER" + fi + + OWNER_ARGS=() + if [ -n "$OWNER" ]; then + OWNER_ARGS+=(--domain-owner "$OWNER") fi + + TOKEN=$(aws codeartifact get-authorization-token \ + --domain "$DOMAIN" \ + --region "$REGION" \ + "${OWNER_ARGS[@]}" \ + --query authorizationToken --output text) || { + echo "::error::Failed to get CodeArtifact token. Check AWS credentials and permissions." + exit 1 + } + echo "::add-mask::$TOKEN" + echo "token=$TOKEN" >> "$GITHUB_OUTPUT" + echo "username=aws" >> "$GITHUB_OUTPUT" fi - cat "$GITHUB_OUTPUT" - # this should not take place, as "Private :: Do Not Upload" set in pyproject.toml # repository-url and password only used for custom feeds, not for PyPI with OIDC - name: Publish to PyPI if: ${{ steps.check_dist.outputs.should_publish == 'true' }} @@ -289,7 +340,8 @@ jobs: packages-dir: dist/ skip-existing: true repository-url: ${{ vars.PYPI_REPOSITORY_URL }} - password: ${{ secrets.PYPI_TOKEN }} + user: ${{ steps.aws_token.outputs.username || '__token__' }} + password: ${{ steps.aws_token.outputs.token || secrets.PYPI_TOKEN }} devcontainer: name: Publish Devcontainer Image diff --git a/.gitlab/README.md b/.gitlab/README.md index 884f5c79..abe9e09b 100644 --- a/.gitlab/README.md +++ b/.gitlab/README.md @@ -141,7 +141,7 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun --- ### 8. Release (`rhiza_release.yml`) -**Purpose:** Create releases and publish packages to PyPI. +**Purpose:** Create releases and publish packages to PyPI or custom feeds (including AWS CodeArtifact). **Trigger:** - On version tags (e.g., `v1.2.3`) @@ -150,6 +150,7 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun - Version validation - Python package building with Hatch - PyPI publishing with twine +- AWS CodeArtifact support via `PYPI_REPOSITORY_URL` + AWS credentials - GitLab release creation **Equivalent GitHub Action:** `.github/workflows/rhiza_release.yml` @@ -157,6 +158,7 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun **GitLab-specific:** - Uses GitLab Releases API instead of GitHub Releases - Uses PYPI_TOKEN instead of OIDC Trusted Publishing +- Uses AWS CI/CD variables for CodeArtifact credentials --- diff --git a/.gitlab/TESTING.md b/.gitlab/TESTING.md index be6abbfd..4cd410fa 100644 --- a/.gitlab/TESTING.md +++ b/.gitlab/TESTING.md @@ -197,6 +197,7 @@ ls -la _book/ - Builds Python package - Creates GitLab release - Publishes to PyPI (if configured) +- Publishes to AWS CodeArtifact (if configured) - Finalizes release with links **Manual test:** @@ -211,10 +212,12 @@ git push origin v0.0.1-test - Package builds successfully - GitLab release created - PyPI upload succeeds (if PYPI_TOKEN set) +- CodeArtifact upload succeeds (if AWS credentials + PYPI_REPOSITORY_URL set) **Configuration needed:** - Set `PYPI_TOKEN` for PyPI publishing - Optionally set `PYPI_REPOSITORY_URL` for custom feed +- For CodeArtifact, set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `PYPI_REPOSITORY_URL` --- @@ -225,10 +228,12 @@ Set these in GitLab project settings (Settings > CI/CD > Variables): ### Secrets (Protected & Masked) - `PYPI_TOKEN` - PyPI authentication token (for releases) - `PAT_TOKEN` - Project/Group Access Token (for sync workflow) +- `AWS_ACCESS_KEY_ID` - AWS access key (for CodeArtifact publishing) +- `AWS_SECRET_ACCESS_KEY` - AWS secret key (for CodeArtifact publishing) ### Configuration Variables - `UV_EXTRA_INDEX_URL` - Extra index URL for UV (optional) -- `PYPI_REPOSITORY_URL` - Custom PyPI URL (optional) +- `PYPI_REPOSITORY_URL` - Custom PyPI URL or CodeArtifact endpoint (optional) - `PUBLISH_COMPANION_BOOK` - Publish documentation (default: true) - `CREATE_MR` - Auto-create merge requests (default: true) diff --git a/.gitlab/workflows/rhiza_release.yml b/.gitlab/workflows/rhiza_release.yml index ec9a634b..13c570e9 100644 --- a/.gitlab/workflows/rhiza_release.yml +++ b/.gitlab/workflows/rhiza_release.yml @@ -14,13 +14,17 @@ # # 🚀 PyPI Publishing: # - Skipped if no dist/ artifacts exist -# - Skipped if pyproject.toml contains "Private :: Do Not Upload" +# - Skipped if pyproject.toml contains "Private :: Do Not Upload" (unless custom feed) # - Uses PYPI_TOKEN for authentication # - For custom feeds, use PYPI_REPOSITORY_URL and PYPI_TOKEN variables +# - For AWS CodeArtifact, set PYPI_REPOSITORY_URL to the CodeArtifact endpoint +# and provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY variables # # 📄 Requirements: # - pyproject.toml with top-level version field (for Python packages) # - PYPI_TOKEN variable for PyPI publishing +# - PYPI_REPOSITORY_URL + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY +# (for CodeArtifact publishing) # # ✅ To Trigger: # Create and push a version tag: @@ -132,30 +136,64 @@ release:pypi: - uv pip install twine script: - | - # Check if dist contains artifacts and not marked as private + # Check if dist contains artifacts if [ ! -d "dist" ] || [ -z "$(ls -A dist)" ]; then echo "⚠️ No dist/ artifacts found. Skipping PyPI publish." exit 0 fi - if grep -R "Private :: Do Not Upload" pyproject.toml; then + # Private classifier only blocks publishing to default PyPI + if [ -z "$PYPI_REPOSITORY_URL" ] && grep -q "Private :: Do Not Upload" pyproject.toml; then echo "⏭️ Package marked as private. Skipping PyPI publish." exit 0 fi - # Publish to PyPI + USERNAME="__token__" + PASSWORD="$PYPI_TOKEN" + + # Auto-detect CodeArtifact URL and get token if AWS credentials are available + if [ -n "$AWS_ACCESS_KEY_ID" ] && case "$PYPI_REPOSITORY_URL" in *.codeartifact.*) true;; *) false;; esac; then + apt-get update -qq && apt-get install -y -qq awscli > /dev/null 2>&1 + HOSTNAME=$(echo "$PYPI_REPOSITORY_URL" | sed 's|https://||; s|/.*||') + REGION=$(echo "$HOSTNAME" | sed -n 's/.*\.d\.codeartifact\.\(.*\)\.amazonaws\.com/\1/p') + DOMAIN_OWNER=$(echo "$HOSTNAME" | sed 's/\.d\.codeartifact\..*//') + OWNER=$(echo "$DOMAIN_OWNER" | grep -oE '[0-9]{12}$' || true) + if [ -n "$OWNER" ]; then + DOMAIN=$(echo "$DOMAIN_OWNER" | sed "s/-${OWNER}$//") + else + DOMAIN="$DOMAIN_OWNER" + fi + + OWNER_ARGS="" + if [ -n "$OWNER" ]; then + OWNER_ARGS="--domain-owner $OWNER" + fi + + # OWNER_ARGS is intentionally unquoted to allow word splitting into separate arguments + PASSWORD=$(aws codeartifact get-authorization-token \ + --domain "$DOMAIN" \ + --region "$REGION" \ + $OWNER_ARGS \ + --query authorizationToken --output text) || { + echo "❌ Failed to get CodeArtifact token. Check AWS credentials and permissions." + exit 1 + } + USERNAME="aws" + fi + + # Publish if [ -n "$PYPI_REPOSITORY_URL" ]; then - echo "📦 Publishing to custom repository: $PYPI_REPOSITORY_URL" + echo "📦 Publishing to: $PYPI_REPOSITORY_URL" twine upload --repository-url "$PYPI_REPOSITORY_URL" \ - --username __token__ --password "$PYPI_TOKEN" \ + --username "$USERNAME" --password "$PASSWORD" \ --skip-existing dist/* else echo "📦 Publishing to PyPI" - twine upload --username __token__ --password "$PYPI_TOKEN" \ + twine upload --username "$USERNAME" --password "$PASSWORD" \ --skip-existing dist/* fi - echo "✅ Published to PyPI" + echo "✅ Published successfully" rules: - if: $CI_COMMIT_TAG =~ /^v.*/ when: on_success diff --git a/.rhiza/docs/RELEASING.md b/.rhiza/docs/RELEASING.md index 5f4c51f6..34c1dcc4 100644 --- a/.rhiza/docs/RELEASING.md +++ b/.rhiza/docs/RELEASING.md @@ -79,7 +79,7 @@ The release workflow (`.github/workflows/rhiza_release.yml`) triggers on the tag 1. **Validates** - Checks the tag format and ensures no duplicate releases 2. **Builds** - Builds the Python package (if `pyproject.toml` exists) 3. **Drafts** - Creates a draft GitHub release with artifacts -4. **PyPI** - Publishes to PyPI (if not marked private) +4. **PyPI** - Publishes to PyPI or custom feed such as CodeArtifact (if not marked private) 5. **Devcontainer** - Publishes devcontainer image (if `PUBLISH_DEVCONTAINER=true`) 6. **Finalizes** - Publishes the GitHub release with links to PyPI and container images @@ -91,6 +91,80 @@ The release workflow (`.github/workflows/rhiza_release.yml`) triggers on the tag - Use `PYPI_REPOSITORY_URL` and `PYPI_TOKEN` for custom feeds - Mark as private with `Private :: Do Not Upload` in `pyproject.toml` +### AWS CodeArtifact Publishing + +Publish Python packages to an AWS CodeArtifact repository. This is useful for private packages +or organisations that use CodeArtifact as their internal package registry. + +CodeArtifact publishing is handled through the same `pypi` job by setting `PYPI_REPOSITORY_URL` +to the CodeArtifact endpoint and providing AWS credentials. The workflow automatically detects +CodeArtifact URLs and handles token exchange. + +> **Note:** The `Private :: Do Not Upload` classifier only blocks publishing to the default +> PyPI. When `PYPI_REPOSITORY_URL` is set (e.g. to a CodeArtifact endpoint), private packages +> will still be published. + +#### Setup Instructions + +1. **Get your CodeArtifact repository endpoint** (the `PYPI_REPOSITORY_URL`): + ```bash + aws codeartifact get-repository-endpoint \ + --domain my-domain \ + --repository my-repo \ + --format pypi \ + --query repositoryEndpoint --output text + ``` + This returns a URL like: + `https://my-domain-123456789012.d.codeartifact.us-east-1.amazonaws.com/pypi/my-repo/` + +2. **Create an IAM user** (or use an existing one) with the following permissions: + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "codeartifact:GetAuthorizationToken", + "codeartifact:GetRepositoryEndpoint", + "codeartifact:PublishPackageVersion", + "codeartifact:PutPackageMetadata" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": "sts:GetServiceBearerToken", + "Resource": "*", + "Condition": { + "StringEquals": { "sts:AWSServiceName": "codeartifact.amazonaws.com" } + } + } + ] + } + ``` + +3. **Set repository secrets and variables:** + + **GitHub** (Settings → Secrets and variables → Actions): + | Type | Name | Value | + |---|---|---| + | Secret | `AWS_ACCESS_KEY_ID` | IAM user access key | + | Secret | `AWS_SECRET_ACCESS_KEY` | IAM user secret key | + | Variable | `PYPI_REPOSITORY_URL` | CodeArtifact endpoint URL from step 1 | + + **GitLab** (Settings → CI/CD → Variables): + | Name | Value | Protected | Masked | + |---|---|---|---| + | `AWS_ACCESS_KEY_ID` | IAM user access key | ✅ | ✅ | + | `AWS_SECRET_ACCESS_KEY` | IAM user secret key | ✅ | ✅ | + | `PYPI_REPOSITORY_URL` | CodeArtifact endpoint URL from step 1 | ✅ | ❌ | + +That's it — no additional configuration needed. The release workflow automatically: +- Detects the CodeArtifact URL from `PYPI_REPOSITORY_URL` +- Uses `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to obtain a temporary auth token +- Publishes the package with `twine` + ### Devcontainer Publishing - Set repository variable `PUBLISH_DEVCONTAINER=true` to enable diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 76d971fb..f80f8733 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -108,7 +108,7 @@ flowchart TD tag[Push Tag v*] --> validate[Validate Tag] validate --> build[Build Package] build --> draft[Draft GitHub Release] - draft --> pypi[Publish to PyPI] + draft --> pypi[Publish to PyPI / Custom Feed] draft --> devcontainer[Publish Devcontainer] pypi --> finalize[Finalize Release] devcontainer --> finalize diff --git a/docs/CHANGELOG_GUIDE.md b/docs/CHANGELOG_GUIDE.md index 9d403f40..a0f476a9 100644 --- a/docs/CHANGELOG_GUIDE.md +++ b/docs/CHANGELOG_GUIDE.md @@ -19,7 +19,7 @@ The release workflow: 1. Validates the tag format 2. Builds distribution artifacts 3. Creates a draft GitHub release with `generate_release_notes: true` -4. Publishes to PyPI (if applicable) +4. Publishes to PyPI or custom feed such as CodeArtifact (if applicable) 5. Publishes devcontainer (if configured) 6. Finalizes the release diff --git a/docs/PRESENTATION.md b/docs/PRESENTATION.md index 259a6ed3..5ed41ba4 100644 --- a/docs/PRESENTATION.md +++ b/docs/PRESENTATION.md @@ -265,6 +265,7 @@ make release-status ✅ Builds Python package ✅ Creates GitHub release ✅ Publishes to PyPI (if public) +✅ Publishes to AWS CodeArtifact (if configured) ✅ Publishes devcontainer image (optional) --- @@ -419,6 +420,14 @@ Automatic if configured as **Trusted Publisher**: 2. Add GitHub Actions as trusted publisher 3. Release workflow publishes automatically +### AWS CodeArtifact Publication + +Automatic if repository secrets and variables are configured: + +1. Set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` as secrets +2. Set `PYPI_REPOSITORY_URL` to the CodeArtifact endpoint +3. Release workflow publishes automatically via `twine` + ### Private Packages Add to `pyproject.toml`: @@ -428,6 +437,8 @@ classifiers = [ ] ``` +> Private packages skip PyPI but still publish to CodeArtifact (if configured). + --- ## 🎯 Real-World Usage