From 8e9fcede27e4da2cd5bf8160803212a545e4bf62 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 21 Mar 2026 21:50:16 +0000 Subject: [PATCH 1/4] Initial plan From 53e4261971b2de33aef42f2e64886e0458c3fae4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 21 Mar 2026 21:59:16 +0000 Subject: [PATCH 2/4] feat: add AWS CodeArtifact publishing support to release workflows Add a new CodeArtifact job to both GitHub Actions and GitLab CI release workflows. The job is conditional on AWS_CODEARTIFACT_DOMAIN being set and uses OIDC-based role assumption (GitHub) or CI/CD variables (GitLab) for credential-free authentication. - Add `codeartifact` job to `.github/workflows/rhiza_release.yml` - Add `release:codeartifact` job to `.gitlab/workflows/rhiza_release.yml` - Update finalise-release jobs to include CodeArtifact links in release notes - Add comprehensive setup instructions to RELEASING.md - Update architecture diagrams and documentation Co-authored-by: markrichardson <5681211+markrichardson@users.noreply.github.com> Agent-Logs-Url: https://github.com/Jebel-Quant/rhiza/sessions/ecb241b0-b8bf-4bb7-a48b-7467c69e6fec --- .github/workflows/rhiza_release.yml | 147 +++++++++++++++++++++++++++- .gitlab/README.md | 4 +- .gitlab/TESTING.md | 9 ++ .gitlab/workflows/rhiza_release.yml | 83 +++++++++++++++- .rhiza/docs/RELEASING.md | 99 ++++++++++++++++++- docs/ARCHITECTURE.md | 6 ++ docs/CHANGELOG_GUIDE.md | 5 +- docs/PRESENTATION.md | 11 +++ 8 files changed, 354 insertions(+), 10 deletions(-) diff --git a/.github/workflows/rhiza_release.yml b/.github/workflows/rhiza_release.yml index 24b93597..8a37765b 100644 --- a/.github/workflows/rhiza_release.yml +++ b/.github/workflows/rhiza_release.yml @@ -11,8 +11,9 @@ # 3. 📦 Generate SBOM - Create Software Bill of Materials (CycloneDX format) # 4. 📝 Draft Release - Create draft GitHub release with build artifacts and SBOM # 5. 🚀 Publish to PyPI - Publish package using OIDC or custom feed -# 6. 🐳 Publish Devcontainer - Build and publish devcontainer image (conditional) -# 7. ✅ Finalize Release - Publish the GitHub release with links +# 6. 🏭 Publish to AWS CodeArtifact - Publish package to CodeArtifact (conditional) +# 7. 🐳 Publish Devcontainer - Build and publish devcontainer image (conditional) +# 8. ✅ Finalize Release - Publish the GitHub release with links # # 📦 SBOM Generation: # - Generated using CycloneDX format (industry standard for software supply chain security) @@ -37,9 +38,18 @@ # - For custom feeds, use PYPI_REPOSITORY_URL and PYPI_TOKEN secrets # - Adds PyPI/custom feed link to GitHub release notes # +# 🏭 AWS CodeArtifact Publishing: +# - Skipped if AWS_CODEARTIFACT_DOMAIN variable is not set +# - Skipped if no dist/ artifacts exist +# - Uses GitHub OIDC to assume an AWS IAM role (via AWS_ROLE_ARN) +# - Dynamically obtains a CodeArtifact authorization token +# - Publishes with twine using the CodeArtifact repository endpoint +# - Adds CodeArtifact link to GitHub release notes +# # 🔐 Security: # - No PyPI credentials stored; relies on Trusted Publishing via GitHub OIDC # - For custom feeds, PYPI_TOKEN secret is used with default username __token__ +# - AWS CodeArtifact uses OIDC role assumption (no static AWS credentials) # - Container registry uses GITHUB_TOKEN for authentication # - SLSA provenance attestations generated for build artifacts (public repos only) # - SBOM attestations generated for supply chain transparency (public repos only) @@ -47,6 +57,8 @@ # 📄 Requirements: # - pyproject.toml with top-level version field (for Python packages) # - Package registered on PyPI as Trusted Publisher (for PyPI publishing) +# - AWS_CODEARTIFACT_DOMAIN, AWS_CODEARTIFACT_REPOSITORY, AWS_ROLE_ARN variables +# (for CodeArtifact publishing) # - PUBLISH_DEVCONTAINER variable set to "true" (for devcontainer publishing) # - .devcontainer/devcontainer.json file (for devcontainer publishing) # @@ -61,6 +73,10 @@ # 🎯 For repos without devcontainers: # The workflow gracefully skips devcontainer steps if PUBLISH_DEVCONTAINER is not # set to "true" or .devcontainer directory doesn't exist. +# +# 🎯 For repos without CodeArtifact: +# The workflow gracefully skips CodeArtifact steps if AWS_CODEARTIFACT_DOMAIN +# is not set. name: (RHIZA) RELEASE @@ -379,11 +395,110 @@ jobs: imageName: ${{ steps.image_name.outputs.image_name }} imageTag: ${{ needs.tag.outputs.tag }} + codeartifact: + name: Publish to AWS CodeArtifact + runs-on: ubuntu-latest + environment: release + needs: [tag, build, draft-release] + outputs: + should_publish: ${{ steps.check_dist.outputs.should_publish }} + + steps: + - name: Checkout Code + uses: actions/checkout@v6.0.2 + with: + fetch-depth: 0 + + - name: Download dist artifact + uses: actions/download-artifact@v8.0.1 + with: + name: dist + path: dist + continue-on-error: true + + - name: Check if CodeArtifact is configured and dist contains artifacts + id: check_dist + env: + AWS_CODEARTIFACT_DOMAIN: ${{ vars.AWS_CODEARTIFACT_DOMAIN }} + run: | + if [[ -z "$AWS_CODEARTIFACT_DOMAIN" ]]; then + echo "⏭️ Skipping CodeArtifact publish (AWS_CODEARTIFACT_DOMAIN not configured)" + echo "should_publish=false" >> "$GITHUB_OUTPUT" + elif [[ ! -d dist ]] || [[ -z "$(ls -A dist)" ]]; then + echo "::warning::No dist/ artifacts. Skipping CodeArtifact publish." + echo "should_publish=false" >> "$GITHUB_OUTPUT" + else + echo "should_publish=true" >> "$GITHUB_OUTPUT" + fi + cat "$GITHUB_OUTPUT" + + - name: Configure AWS credentials + if: ${{ steps.check_dist.outputs.should_publish == 'true' }} + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} + + - name: Get CodeArtifact authorization token + if: ${{ steps.check_dist.outputs.should_publish == 'true' }} + id: codeartifact_token + env: + DOMAIN: ${{ vars.AWS_CODEARTIFACT_DOMAIN }} + OWNER: ${{ vars.AWS_CODEARTIFACT_OWNER }} + REGION: ${{ vars.AWS_REGION || 'us-east-1' }} + run: | + OWNER_ARGS=() + if [ -n "$OWNER" ]; then + OWNER_ARGS+=(--domain-owner "$OWNER") + fi + + TOKEN=$(aws codeartifact get-authorization-token \ + --domain "$DOMAIN" \ + --region "$REGION" \ + "${OWNER_ARGS[@]}" \ + --query authorizationToken --output text) + echo "::add-mask::$TOKEN" + echo "token=$TOKEN" >> "$GITHUB_OUTPUT" + + - name: Get CodeArtifact repository URL + if: ${{ steps.check_dist.outputs.should_publish == 'true' }} + id: repo_url + env: + DOMAIN: ${{ vars.AWS_CODEARTIFACT_DOMAIN }} + REPO: ${{ vars.AWS_CODEARTIFACT_REPOSITORY }} + OWNER: ${{ vars.AWS_CODEARTIFACT_OWNER }} + REGION: ${{ vars.AWS_REGION || 'us-east-1' }} + run: | + OWNER_ARGS=() + if [ -n "$OWNER" ]; then + OWNER_ARGS+=(--domain-owner "$OWNER") + fi + + URL=$(aws codeartifact get-repository-endpoint \ + --domain "$DOMAIN" \ + --repository "$REPO" \ + --format pypi \ + --region "$REGION" \ + "${OWNER_ARGS[@]}" \ + --query repositoryEndpoint --output text) + echo "url=${URL}" >> "$GITHUB_OUTPUT" + + - name: Publish to CodeArtifact + if: ${{ steps.check_dist.outputs.should_publish == 'true' }} + run: | + pip install twine + twine upload \ + --repository-url "${{ steps.repo_url.outputs.url }}" \ + --username aws \ + --password "${{ steps.codeartifact_token.outputs.token }}" \ + --skip-existing \ + dist/* + finalise-release: name: Finalise Release runs-on: ubuntu-latest - needs: [tag, pypi, devcontainer] - if: needs.pypi.result == 'success' || needs.devcontainer.result == 'success' + needs: [tag, pypi, devcontainer, codeartifact] + if: needs.pypi.result == 'success' || needs.devcontainer.result == 'success' || needs.codeartifact.result == 'success' steps: - name: Checkout Code uses: actions/checkout@v6.0.2 @@ -443,12 +558,35 @@ jobs: echo "EOF" } >> "$GITHUB_OUTPUT" + - name: Generate CodeArtifact Link + id: codeartifact_link + if: needs.codeartifact.outputs.should_publish == 'true' && needs.codeartifact.result == 'success' + env: + DOMAIN: ${{ vars.AWS_CODEARTIFACT_DOMAIN }} + REPO: ${{ vars.AWS_CODEARTIFACT_REPOSITORY }} + REGION: ${{ vars.AWS_REGION || 'us-east-1' }} + run: | + PACKAGE_NAME=$(uv run python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['name'])") + VERSION="${{ needs.tag.outputs.tag }}" + VERSION=${VERSION#v} + + CONSOLE_URL="https://${REGION}.console.aws.amazon.com/codesuite/codeartifact/d/${DOMAIN}/r/${REPO}/p/pypi/p/${PACKAGE_NAME}/versions?region=${REGION}" + + { + echo "message<> "$GITHUB_OUTPUT" + - name: Publish Release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ needs.tag.outputs.tag }} DEVCONTAINER_MSG: ${{ steps.devcontainer_link.outputs.message }} PYPI_MSG: ${{ steps.pypi_link.outputs.message }} + CODEARTIFACT_MSG: ${{ steps.codeartifact_link.outputs.message }} run: | # Get existing auto-generated release notes (gracefully handle missing release) EXISTING=$(gh release view "$TAG" --json body --jq '.body // ""' 2>/dev/null || echo "") @@ -458,6 +596,7 @@ jobs: printf '%s' "$EXISTING" [ -n "$DEVCONTAINER_MSG" ] && printf '\n\n%s' "$DEVCONTAINER_MSG" [ -n "$PYPI_MSG" ] && printf '\n\n%s' "$PYPI_MSG" + [ -n "$CODEARTIFACT_MSG" ] && printf '\n\n%s' "$CODEARTIFACT_MSG" } > /tmp/notes.md gh release edit "$TAG" --draft=false --notes-file /tmp/notes.md diff --git a/.gitlab/README.md b/.gitlab/README.md index 884f5c79..ff5b5716 100644 --- a/.gitlab/README.md +++ b/.gitlab/README.md @@ -141,7 +141,7 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun --- ### 8. Release (`rhiza_release.yml`) -**Purpose:** Create releases and publish packages to PyPI. +**Purpose:** Create releases and publish packages to PyPI and/or AWS CodeArtifact. **Trigger:** - On version tags (e.g., `v1.2.3`) @@ -150,6 +150,7 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun - Version validation - Python package building with Hatch - PyPI publishing with twine +- AWS CodeArtifact publishing (if configured) - GitLab release creation **Equivalent GitHub Action:** `.github/workflows/rhiza_release.yml` @@ -157,6 +158,7 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun **GitLab-specific:** - Uses GitLab Releases API instead of GitHub Releases - Uses PYPI_TOKEN instead of OIDC Trusted Publishing +- Uses AWS CI/CD variables for CodeArtifact (instead of GitHub OIDC role assumption) --- diff --git a/.gitlab/TESTING.md b/.gitlab/TESTING.md index be6abbfd..10dea54c 100644 --- a/.gitlab/TESTING.md +++ b/.gitlab/TESTING.md @@ -197,6 +197,7 @@ ls -la _book/ - Builds Python package - Creates GitLab release - Publishes to PyPI (if configured) +- Publishes to AWS CodeArtifact (if configured) - Finalizes release with links **Manual test:** @@ -211,10 +212,12 @@ git push origin v0.0.1-test - Package builds successfully - GitLab release created - PyPI upload succeeds (if PYPI_TOKEN set) +- CodeArtifact upload succeeds (if AWS_CODEARTIFACT_DOMAIN set) **Configuration needed:** - Set `PYPI_TOKEN` for PyPI publishing - Optionally set `PYPI_REPOSITORY_URL` for custom feed +- Set `AWS_CODEARTIFACT_DOMAIN`, `AWS_CODEARTIFACT_REPOSITORY`, and AWS credentials for CodeArtifact --- @@ -225,9 +228,15 @@ Set these in GitLab project settings (Settings > CI/CD > Variables): ### Secrets (Protected & Masked) - `PYPI_TOKEN` - PyPI authentication token (for releases) - `PAT_TOKEN` - Project/Group Access Token (for sync workflow) +- `AWS_ACCESS_KEY_ID` - AWS access key (for CodeArtifact, if not using OIDC) +- `AWS_SECRET_ACCESS_KEY` - AWS secret key (for CodeArtifact, if not using OIDC) ### Configuration Variables - `UV_EXTRA_INDEX_URL` - Extra index URL for UV (optional) +- `AWS_CODEARTIFACT_DOMAIN` - CodeArtifact domain name (optional) +- `AWS_CODEARTIFACT_REPOSITORY` - CodeArtifact repository name (optional) +- `AWS_REGION` - AWS region for CodeArtifact (optional, defaults to us-east-1) +- `AWS_CODEARTIFACT_OWNER` - Domain owner account ID for cross-account access (optional) - `PYPI_REPOSITORY_URL` - Custom PyPI URL (optional) - `PUBLISH_COMPANION_BOOK` - Publish documentation (default: true) - `CREATE_MR` - Auto-create merge requests (default: true) diff --git a/.gitlab/workflows/rhiza_release.yml b/.gitlab/workflows/rhiza_release.yml index ec9a634b..642879af 100644 --- a/.gitlab/workflows/rhiza_release.yml +++ b/.gitlab/workflows/rhiza_release.yml @@ -10,7 +10,8 @@ # 2. 🏗️ Build - Build Python package with Hatch (if [build-system] is defined in pyproject.toml # 3. 📝 Draft Release - Create draft GitLab release with build artifacts # 4. 🚀 Publish to PyPI - Publish package using token or custom feed -# 5. ✅ Finalize Release - Publish the GitLab release with links +# 5. 🏭 Publish to AWS CodeArtifact - Publish package to CodeArtifact (conditional) +# 6. ✅ Finalize Release - Publish the GitLab release with links # # 🚀 PyPI Publishing: # - Skipped if no dist/ artifacts exist @@ -18,9 +19,18 @@ # - Uses PYPI_TOKEN for authentication # - For custom feeds, use PYPI_REPOSITORY_URL and PYPI_TOKEN variables # +# 🏭 AWS CodeArtifact Publishing: +# - Skipped if AWS_CODEARTIFACT_DOMAIN variable is not set +# - Skipped if no dist/ artifacts exist +# - Uses AWS credentials from CI/CD variables (or OIDC via AWS_ROLE_ARN) +# - Dynamically obtains a CodeArtifact authorization token +# - Publishes with twine using the CodeArtifact repository endpoint +# # 📄 Requirements: # - pyproject.toml with top-level version field (for Python packages) # - PYPI_TOKEN variable for PyPI publishing +# - AWS_CODEARTIFACT_DOMAIN, AWS_CODEARTIFACT_REPOSITORY, and AWS credentials +# (for CodeArtifact publishing) # # ✅ To Trigger: # Create and push a version tag: @@ -161,12 +171,76 @@ release:pypi: when: on_success allow_failure: false +release:codeartifact: + stage: deploy + needs: + - job: release:build + artifacts: true + - job: release:draft + image: ghcr.io/astral-sh/uv:0.9.30-bookworm + before_script: + - apt-get update -qq && apt-get install -y -qq awscli > /dev/null 2>&1 + - uv python install $(cat .python-version) + - uv pip install twine + script: + - | + # Check if CodeArtifact is configured + if [ -z "$AWS_CODEARTIFACT_DOMAIN" ]; then + echo "⏭️ Skipping: AWS_CODEARTIFACT_DOMAIN not configured" + exit 0 + fi + + # Check if dist contains artifacts + if [ ! -d "dist" ] || [ -z "$(ls -A dist)" ]; then + echo "⚠️ No dist/ artifacts found. Skipping CodeArtifact publish." + exit 0 + fi + + REGION="${AWS_REGION:-us-east-1}" + + OWNER_ARGS="" + if [ -n "$AWS_CODEARTIFACT_OWNER" ]; then + OWNER_ARGS="--domain-owner $AWS_CODEARTIFACT_OWNER" + fi + + # Get CodeArtifact authorization token + TOKEN=$(aws codeartifact get-authorization-token \ + --domain "$AWS_CODEARTIFACT_DOMAIN" \ + --region "$REGION" \ + $OWNER_ARGS \ + --query authorizationToken --output text) + + # Get CodeArtifact repository endpoint + REPO_URL=$(aws codeartifact get-repository-endpoint \ + --domain "$AWS_CODEARTIFACT_DOMAIN" \ + --repository "$AWS_CODEARTIFACT_REPOSITORY" \ + --format pypi \ + --region "$REGION" \ + $OWNER_ARGS \ + --query repositoryEndpoint --output text) + + echo "📦 Publishing to CodeArtifact: $AWS_CODEARTIFACT_DOMAIN/$AWS_CODEARTIFACT_REPOSITORY" + twine upload \ + --repository-url "$REPO_URL" \ + --username aws \ + --password "$TOKEN" \ + --skip-existing \ + dist/* + + echo "✅ Published to CodeArtifact" + rules: + - if: $CI_COMMIT_TAG =~ /^v.*/ + when: on_success + allow_failure: false + release:finalize: stage: .post needs: - job: release:build - job: release:pypi optional: true + - job: release:codeartifact + optional: true image: alpine:latest before_script: - apk add --no-cache curl jq python3 @@ -185,6 +259,13 @@ release:finalize: else RELEASE_BODY="${RELEASE_BODY}\n\n### Custom Feed Package\n\n[$PACKAGE_NAME]($PYPI_REPOSITORY_URL)" fi + + # Add CodeArtifact link if published + if [ -n "$AWS_CODEARTIFACT_DOMAIN" ] && [ -n "$AWS_CODEARTIFACT_REPOSITORY" ]; then + REGION="${AWS_REGION:-us-east-1}" + CONSOLE_URL="https://${REGION}.console.aws.amazon.com/codesuite/codeartifact/d/${AWS_CODEARTIFACT_DOMAIN}/r/${AWS_CODEARTIFACT_REPOSITORY}/p/pypi/p/${PACKAGE_NAME}/versions?region=${REGION}" + RELEASE_BODY="${RELEASE_BODY}\n\n### AWS CodeArtifact Package\n\n[$PACKAGE_NAME@$VERSION]($CONSOLE_URL) (\`${AWS_CODEARTIFACT_DOMAIN}/${AWS_CODEARTIFACT_REPOSITORY}\`)" + fi fi fi diff --git a/.rhiza/docs/RELEASING.md b/.rhiza/docs/RELEASING.md index 5f4c51f6..e919f2a0 100644 --- a/.rhiza/docs/RELEASING.md +++ b/.rhiza/docs/RELEASING.md @@ -80,8 +80,9 @@ The release workflow (`.github/workflows/rhiza_release.yml`) triggers on the tag 2. **Builds** - Builds the Python package (if `pyproject.toml` exists) 3. **Drafts** - Creates a draft GitHub release with artifacts 4. **PyPI** - Publishes to PyPI (if not marked private) -5. **Devcontainer** - Publishes devcontainer image (if `PUBLISH_DEVCONTAINER=true`) -6. **Finalizes** - Publishes the GitHub release with links to PyPI and container images +5. **CodeArtifact** - Publishes to AWS CodeArtifact (if `AWS_CODEARTIFACT_DOMAIN` is set) +6. **Devcontainer** - Publishes devcontainer image (if `PUBLISH_DEVCONTAINER=true`) +7. **Finalizes** - Publishes the GitHub release with links to PyPI, CodeArtifact, and container images ## Configuration Options @@ -91,6 +92,100 @@ The release workflow (`.github/workflows/rhiza_release.yml`) triggers on the tag - Use `PYPI_REPOSITORY_URL` and `PYPI_TOKEN` for custom feeds - Mark as private with `Private :: Do Not Upload` in `pyproject.toml` +### AWS CodeArtifact Publishing + +Publish Python packages to an AWS CodeArtifact repository. This is useful for private packages +or organisations that use CodeArtifact as their internal package registry. + +> **Note:** The `Private :: Do Not Upload` classifier only affects PyPI publishing. Packages +> marked as private can still be published to CodeArtifact, which is the typical use case +> for internal packages. + +#### Required Repository Variables + +| Variable | Description | Example | +|---|---|---| +| `AWS_CODEARTIFACT_DOMAIN` | CodeArtifact domain name | `my-company` | +| `AWS_CODEARTIFACT_REPOSITORY` | CodeArtifact repository name | `python-packages` | +| `AWS_ROLE_ARN` | IAM role ARN for GitHub OIDC (**GitHub only**) | `arn:aws:iam::123456789012:role/github-release` | + +#### Optional Repository Variables + +| Variable | Default | Description | +|---|---|---| +| `AWS_REGION` | `us-east-1` | AWS region for CodeArtifact | +| `AWS_CODEARTIFACT_OWNER` | *(current account)* | Domain owner AWS account ID (for cross-account access) | + +#### Setup Instructions (GitHub) + +1. **Create an IAM OIDC identity provider** for GitHub Actions in your AWS account: + - Provider URL: `https://token.actions.githubusercontent.com` + - Audience: `sts.amazonaws.com` + +2. **Create an IAM role** with a trust policy allowing GitHub OIDC: + ```json + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": { "Federated": "arn:aws:iam::123456789012:oidc-provider/token.actions.githubusercontent.com" }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" + }, + "StringLike": { + "token.actions.githubusercontent.com:sub": "repo:YOUR-ORG/YOUR-REPO:*" + } + } + }] + } + ``` + +3. **Attach a policy** granting CodeArtifact access: + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "codeartifact:GetAuthorizationToken", + "codeartifact:GetRepositoryEndpoint", + "codeartifact:PublishPackageVersion", + "codeartifact:PutPackageMetadata" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": "sts:GetServiceBearerToken", + "Resource": "*", + "Condition": { + "StringEquals": { "sts:AWSServiceName": "codeartifact.amazonaws.com" } + } + } + ] + } + ``` + +4. **Set repository variables** in GitHub (Settings → Secrets and variables → Actions → Variables): + - `AWS_CODEARTIFACT_DOMAIN` = your domain name + - `AWS_CODEARTIFACT_REPOSITORY` = your repository name + - `AWS_ROLE_ARN` = the IAM role ARN from step 2 + - `AWS_REGION` = your AWS region (e.g., `eu-west-2`) + +#### Setup Instructions (GitLab) + +1. **Configure AWS credentials** as CI/CD variables: + - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (or configure GitLab OIDC with AWS) + +2. **Set CI/CD variables**: + - `AWS_CODEARTIFACT_DOMAIN` = your domain name + - `AWS_CODEARTIFACT_REPOSITORY` = your repository name + - `AWS_REGION` = your AWS region + - `AWS_CODEARTIFACT_OWNER` = domain owner account ID (if cross-account) + ### Devcontainer Publishing - Set repository variable `PUBLISH_DEVCONTAINER=true` to enable diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 76d971fb..26024a02 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -109,18 +109,24 @@ flowchart TD validate --> build[Build Package] build --> draft[Draft GitHub Release] draft --> pypi[Publish to PyPI] + draft --> codeartifact[Publish to CodeArtifact] draft --> devcontainer[Publish Devcontainer] pypi --> finalize[Finalize Release] + codeartifact --> finalize devcontainer --> finalize subgraph Conditions pypi_cond{Has dist/ &
not Private?} + ca_cond{Has dist/ &
AWS_CODEARTIFACT_DOMAIN?} dev_cond{PUBLISH_DEVCONTAINER
= true?} end draft --> pypi_cond pypi_cond -->|yes| pypi pypi_cond -->|no| finalize + draft --> ca_cond + ca_cond -->|yes| codeartifact + ca_cond -->|no| finalize draft --> dev_cond dev_cond -->|yes| devcontainer dev_cond -->|no| finalize diff --git a/docs/CHANGELOG_GUIDE.md b/docs/CHANGELOG_GUIDE.md index 9d403f40..93b37128 100644 --- a/docs/CHANGELOG_GUIDE.md +++ b/docs/CHANGELOG_GUIDE.md @@ -20,8 +20,9 @@ The release workflow: 2. Builds distribution artifacts 3. Creates a draft GitHub release with `generate_release_notes: true` 4. Publishes to PyPI (if applicable) -5. Publishes devcontainer (if configured) -6. Finalizes the release +5. Publishes to AWS CodeArtifact (if configured) +6. Publishes devcontainer (if configured) +7. Finalizes the release GitHub automatically generates release notes by: - Listing all PRs merged since the last release diff --git a/docs/PRESENTATION.md b/docs/PRESENTATION.md index 259a6ed3..b3184021 100644 --- a/docs/PRESENTATION.md +++ b/docs/PRESENTATION.md @@ -265,6 +265,7 @@ make release-status ✅ Builds Python package ✅ Creates GitHub release ✅ Publishes to PyPI (if public) +✅ Publishes to AWS CodeArtifact (if configured) ✅ Publishes devcontainer image (optional) --- @@ -419,6 +420,14 @@ Automatic if configured as **Trusted Publisher**: 2. Add GitHub Actions as trusted publisher 3. Release workflow publishes automatically +### AWS CodeArtifact Publication + +Automatic if repository variables are configured: + +1. Set `AWS_CODEARTIFACT_DOMAIN` and `AWS_CODEARTIFACT_REPOSITORY` +2. Configure OIDC role (`AWS_ROLE_ARN`) for credential-free auth +3. Release workflow publishes automatically via `twine` + ### Private Packages Add to `pyproject.toml`: @@ -428,6 +437,8 @@ classifiers = [ ] ``` +> Private packages skip PyPI but still publish to CodeArtifact (if configured). + --- ## 🎯 Real-World Usage From 58b52f7b572d633811b798a97f66b4bbe97a7986 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 23 Mar 2026 19:50:33 +0000 Subject: [PATCH 3/4] refactor: simplify CodeArtifact to use AWS secrets + PYPI_REPOSITORY_URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the separate codeartifact job from both GitHub Actions and GitLab CI workflows. Instead, integrate CodeArtifact support directly into the existing pypi job — auto-detect CodeArtifact URLs and fetch a temporary auth token using AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY secrets. Users configure just 3 things: - AWS_ACCESS_KEY_ID (secret) - AWS_SECRET_ACCESS_KEY (secret) - PYPI_REPOSITORY_URL (variable, set to CodeArtifact endpoint) Co-authored-by: markrichardson <5681211+markrichardson@users.noreply.github.com> Agent-Logs-Url: https://github.com/Jebel-Quant/rhiza/sessions/d31294fb-ccb9-44fc-8f54-4720cee99ca3 --- .github/workflows/rhiza_release.yml | 208 ++++++++-------------------- .gitlab/README.md | 6 +- .gitlab/TESTING.md | 14 +- .gitlab/workflows/rhiza_release.yml | 131 ++++++------------ .rhiza/docs/RELEASING.md | 101 ++++++-------- docs/ARCHITECTURE.md | 8 +- docs/CHANGELOG_GUIDE.md | 7 +- docs/PRESENTATION.md | 6 +- 8 files changed, 156 insertions(+), 325 deletions(-) diff --git a/.github/workflows/rhiza_release.yml b/.github/workflows/rhiza_release.yml index 8a37765b..0e1ee1ae 100644 --- a/.github/workflows/rhiza_release.yml +++ b/.github/workflows/rhiza_release.yml @@ -11,9 +11,8 @@ # 3. 📦 Generate SBOM - Create Software Bill of Materials (CycloneDX format) # 4. 📝 Draft Release - Create draft GitHub release with build artifacts and SBOM # 5. 🚀 Publish to PyPI - Publish package using OIDC or custom feed -# 6. 🏭 Publish to AWS CodeArtifact - Publish package to CodeArtifact (conditional) -# 7. 🐳 Publish Devcontainer - Build and publish devcontainer image (conditional) -# 8. ✅ Finalize Release - Publish the GitHub release with links +# 6. 🐳 Publish Devcontainer - Build and publish devcontainer image (conditional) +# 7. ✅ Finalize Release - Publish the GitHub release with links # # 📦 SBOM Generation: # - Generated using CycloneDX format (industry standard for software supply chain security) @@ -33,23 +32,17 @@ # # 🚀 PyPI Publishing: # - Skipped if no dist/ artifacts exist -# - Skipped if pyproject.toml contains "Private :: Do Not Upload" +# - Skipped if pyproject.toml contains "Private :: Do Not Upload" (unless custom feed) # - Uses Trusted Publishing (OIDC) for PyPI (no stored credentials) -# - For custom feeds, use PYPI_REPOSITORY_URL and PYPI_TOKEN secrets +# - For custom feeds, use PYPI_REPOSITORY_URL variable and PYPI_TOKEN secret +# - For AWS CodeArtifact, set PYPI_REPOSITORY_URL to the CodeArtifact endpoint +# and provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY secrets # - Adds PyPI/custom feed link to GitHub release notes # -# 🏭 AWS CodeArtifact Publishing: -# - Skipped if AWS_CODEARTIFACT_DOMAIN variable is not set -# - Skipped if no dist/ artifacts exist -# - Uses GitHub OIDC to assume an AWS IAM role (via AWS_ROLE_ARN) -# - Dynamically obtains a CodeArtifact authorization token -# - Publishes with twine using the CodeArtifact repository endpoint -# - Adds CodeArtifact link to GitHub release notes -# # 🔐 Security: # - No PyPI credentials stored; relies on Trusted Publishing via GitHub OIDC # - For custom feeds, PYPI_TOKEN secret is used with default username __token__ -# - AWS CodeArtifact uses OIDC role assumption (no static AWS credentials) +# - For AWS CodeArtifact, uses AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY secrets # - Container registry uses GITHUB_TOKEN for authentication # - SLSA provenance attestations generated for build artifacts (public repos only) # - SBOM attestations generated for supply chain transparency (public repos only) @@ -57,7 +50,7 @@ # 📄 Requirements: # - pyproject.toml with top-level version field (for Python packages) # - Package registered on PyPI as Trusted Publisher (for PyPI publishing) -# - AWS_CODEARTIFACT_DOMAIN, AWS_CODEARTIFACT_REPOSITORY, AWS_ROLE_ARN variables +# - PYPI_REPOSITORY_URL variable + AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY secrets # (for CodeArtifact publishing) # - PUBLISH_DEVCONTAINER variable set to "true" (for devcontainer publishing) # - .devcontainer/devcontainer.json file (for devcontainer publishing) @@ -74,9 +67,10 @@ # The workflow gracefully skips devcontainer steps if PUBLISH_DEVCONTAINER is not # set to "true" or .devcontainer directory doesn't exist. # -# 🎯 For repos without CodeArtifact: -# The workflow gracefully skips CodeArtifact steps if AWS_CODEARTIFACT_DOMAIN -# is not set. +# 🎯 For repos with AWS CodeArtifact: +# Set PYPI_REPOSITORY_URL to the CodeArtifact endpoint and provide +# AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY secrets. The workflow +# automatically detects CodeArtifact URLs and handles token exchange. name: (RHIZA) RELEASE @@ -283,20 +277,58 @@ jobs: - name: Check if dist contains artifacts and not marked as private id: check_dist + env: + PYPI_REPOSITORY_URL: ${{ vars.PYPI_REPOSITORY_URL }} run: | if [[ ! -d dist ]]; then echo "::warning::No folder dist/. Skipping PyPI publish." echo "should_publish=false" >> "$GITHUB_OUTPUT" + elif [[ -z "$PYPI_REPOSITORY_URL" ]] && grep -q "Private :: Do Not Upload" pyproject.toml; then + echo "::notice::Package marked as private. Skipping PyPI publish." + echo "should_publish=false" >> "$GITHUB_OUTPUT" else - if grep -R "Private :: Do Not Upload" pyproject.toml; then - echo "should_publish=false" >> "$GITHUB_OUTPUT" + echo "should_publish=true" >> "$GITHUB_OUTPUT" + fi + cat "$GITHUB_OUTPUT" + + - name: Get AWS CodeArtifact token + if: ${{ steps.check_dist.outputs.should_publish == 'true' }} + id: aws_token + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + PYPI_REPOSITORY_URL: ${{ vars.PYPI_REPOSITORY_URL }} + run: | + if [[ -n "$AWS_ACCESS_KEY_ID" && "$PYPI_REPOSITORY_URL" == *".codeartifact."* ]]; then + # Parse domain and region from CodeArtifact URL + # Format: https://{domain}-{owner}.d.codeartifact.{region}.amazonaws.com/pypi/{repo}/ + STRIPPED="${PYPI_REPOSITORY_URL#https://}" + HOSTNAME="${STRIPPED%%/*}" + REGION="${HOSTNAME#*.d.codeartifact.}" + REGION="${REGION%.amazonaws.com}" + DOMAIN_OWNER="${HOSTNAME%%.d.codeartifact.*}" + OWNER=$(echo "$DOMAIN_OWNER" | grep -oE '[0-9]{12}$' || true) + if [ -n "$OWNER" ]; then + DOMAIN="${DOMAIN_OWNER%-"$OWNER"}" else - echo "should_publish=true" >> "$GITHUB_OUTPUT" + DOMAIN="$DOMAIN_OWNER" + fi + + OWNER_ARGS=() + if [ -n "$OWNER" ]; then + OWNER_ARGS+=(--domain-owner "$OWNER") fi + + TOKEN=$(aws codeartifact get-authorization-token \ + --domain "$DOMAIN" \ + --region "$REGION" \ + "${OWNER_ARGS[@]}" \ + --query authorizationToken --output text) + echo "::add-mask::$TOKEN" + echo "token=$TOKEN" >> "$GITHUB_OUTPUT" + echo "username=aws" >> "$GITHUB_OUTPUT" fi - cat "$GITHUB_OUTPUT" - # this should not take place, as "Private :: Do Not Upload" set in pyproject.toml # repository-url and password only used for custom feeds, not for PyPI with OIDC - name: Publish to PyPI if: ${{ steps.check_dist.outputs.should_publish == 'true' }} @@ -305,7 +337,8 @@ jobs: packages-dir: dist/ skip-existing: true repository-url: ${{ vars.PYPI_REPOSITORY_URL }} - password: ${{ secrets.PYPI_TOKEN }} + user: ${{ steps.aws_token.outputs.username || '__token__' }} + password: ${{ steps.aws_token.outputs.token || secrets.PYPI_TOKEN }} devcontainer: name: Publish Devcontainer Image @@ -395,110 +428,11 @@ jobs: imageName: ${{ steps.image_name.outputs.image_name }} imageTag: ${{ needs.tag.outputs.tag }} - codeartifact: - name: Publish to AWS CodeArtifact - runs-on: ubuntu-latest - environment: release - needs: [tag, build, draft-release] - outputs: - should_publish: ${{ steps.check_dist.outputs.should_publish }} - - steps: - - name: Checkout Code - uses: actions/checkout@v6.0.2 - with: - fetch-depth: 0 - - - name: Download dist artifact - uses: actions/download-artifact@v8.0.1 - with: - name: dist - path: dist - continue-on-error: true - - - name: Check if CodeArtifact is configured and dist contains artifacts - id: check_dist - env: - AWS_CODEARTIFACT_DOMAIN: ${{ vars.AWS_CODEARTIFACT_DOMAIN }} - run: | - if [[ -z "$AWS_CODEARTIFACT_DOMAIN" ]]; then - echo "⏭️ Skipping CodeArtifact publish (AWS_CODEARTIFACT_DOMAIN not configured)" - echo "should_publish=false" >> "$GITHUB_OUTPUT" - elif [[ ! -d dist ]] || [[ -z "$(ls -A dist)" ]]; then - echo "::warning::No dist/ artifacts. Skipping CodeArtifact publish." - echo "should_publish=false" >> "$GITHUB_OUTPUT" - else - echo "should_publish=true" >> "$GITHUB_OUTPUT" - fi - cat "$GITHUB_OUTPUT" - - - name: Configure AWS credentials - if: ${{ steps.check_dist.outputs.should_publish == 'true' }} - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ vars.AWS_ROLE_ARN }} - aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} - - - name: Get CodeArtifact authorization token - if: ${{ steps.check_dist.outputs.should_publish == 'true' }} - id: codeartifact_token - env: - DOMAIN: ${{ vars.AWS_CODEARTIFACT_DOMAIN }} - OWNER: ${{ vars.AWS_CODEARTIFACT_OWNER }} - REGION: ${{ vars.AWS_REGION || 'us-east-1' }} - run: | - OWNER_ARGS=() - if [ -n "$OWNER" ]; then - OWNER_ARGS+=(--domain-owner "$OWNER") - fi - - TOKEN=$(aws codeartifact get-authorization-token \ - --domain "$DOMAIN" \ - --region "$REGION" \ - "${OWNER_ARGS[@]}" \ - --query authorizationToken --output text) - echo "::add-mask::$TOKEN" - echo "token=$TOKEN" >> "$GITHUB_OUTPUT" - - - name: Get CodeArtifact repository URL - if: ${{ steps.check_dist.outputs.should_publish == 'true' }} - id: repo_url - env: - DOMAIN: ${{ vars.AWS_CODEARTIFACT_DOMAIN }} - REPO: ${{ vars.AWS_CODEARTIFACT_REPOSITORY }} - OWNER: ${{ vars.AWS_CODEARTIFACT_OWNER }} - REGION: ${{ vars.AWS_REGION || 'us-east-1' }} - run: | - OWNER_ARGS=() - if [ -n "$OWNER" ]; then - OWNER_ARGS+=(--domain-owner "$OWNER") - fi - - URL=$(aws codeartifact get-repository-endpoint \ - --domain "$DOMAIN" \ - --repository "$REPO" \ - --format pypi \ - --region "$REGION" \ - "${OWNER_ARGS[@]}" \ - --query repositoryEndpoint --output text) - echo "url=${URL}" >> "$GITHUB_OUTPUT" - - - name: Publish to CodeArtifact - if: ${{ steps.check_dist.outputs.should_publish == 'true' }} - run: | - pip install twine - twine upload \ - --repository-url "${{ steps.repo_url.outputs.url }}" \ - --username aws \ - --password "${{ steps.codeartifact_token.outputs.token }}" \ - --skip-existing \ - dist/* - finalise-release: name: Finalise Release runs-on: ubuntu-latest - needs: [tag, pypi, devcontainer, codeartifact] - if: needs.pypi.result == 'success' || needs.devcontainer.result == 'success' || needs.codeartifact.result == 'success' + needs: [tag, pypi, devcontainer] + if: needs.pypi.result == 'success' || needs.devcontainer.result == 'success' steps: - name: Checkout Code uses: actions/checkout@v6.0.2 @@ -558,35 +492,12 @@ jobs: echo "EOF" } >> "$GITHUB_OUTPUT" - - name: Generate CodeArtifact Link - id: codeartifact_link - if: needs.codeartifact.outputs.should_publish == 'true' && needs.codeartifact.result == 'success' - env: - DOMAIN: ${{ vars.AWS_CODEARTIFACT_DOMAIN }} - REPO: ${{ vars.AWS_CODEARTIFACT_REPOSITORY }} - REGION: ${{ vars.AWS_REGION || 'us-east-1' }} - run: | - PACKAGE_NAME=$(uv run python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['name'])") - VERSION="${{ needs.tag.outputs.tag }}" - VERSION=${VERSION#v} - - CONSOLE_URL="https://${REGION}.console.aws.amazon.com/codesuite/codeartifact/d/${DOMAIN}/r/${REPO}/p/pypi/p/${PACKAGE_NAME}/versions?region=${REGION}" - - { - echo "message<> "$GITHUB_OUTPUT" - - name: Publish Release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ needs.tag.outputs.tag }} DEVCONTAINER_MSG: ${{ steps.devcontainer_link.outputs.message }} PYPI_MSG: ${{ steps.pypi_link.outputs.message }} - CODEARTIFACT_MSG: ${{ steps.codeartifact_link.outputs.message }} run: | # Get existing auto-generated release notes (gracefully handle missing release) EXISTING=$(gh release view "$TAG" --json body --jq '.body // ""' 2>/dev/null || echo "") @@ -596,7 +507,6 @@ jobs: printf '%s' "$EXISTING" [ -n "$DEVCONTAINER_MSG" ] && printf '\n\n%s' "$DEVCONTAINER_MSG" [ -n "$PYPI_MSG" ] && printf '\n\n%s' "$PYPI_MSG" - [ -n "$CODEARTIFACT_MSG" ] && printf '\n\n%s' "$CODEARTIFACT_MSG" } > /tmp/notes.md gh release edit "$TAG" --draft=false --notes-file /tmp/notes.md diff --git a/.gitlab/README.md b/.gitlab/README.md index ff5b5716..abe9e09b 100644 --- a/.gitlab/README.md +++ b/.gitlab/README.md @@ -141,7 +141,7 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun --- ### 8. Release (`rhiza_release.yml`) -**Purpose:** Create releases and publish packages to PyPI and/or AWS CodeArtifact. +**Purpose:** Create releases and publish packages to PyPI or custom feeds (including AWS CodeArtifact). **Trigger:** - On version tags (e.g., `v1.2.3`) @@ -150,7 +150,7 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun - Version validation - Python package building with Hatch - PyPI publishing with twine -- AWS CodeArtifact publishing (if configured) +- AWS CodeArtifact support via `PYPI_REPOSITORY_URL` + AWS credentials - GitLab release creation **Equivalent GitHub Action:** `.github/workflows/rhiza_release.yml` @@ -158,7 +158,7 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun **GitLab-specific:** - Uses GitLab Releases API instead of GitHub Releases - Uses PYPI_TOKEN instead of OIDC Trusted Publishing -- Uses AWS CI/CD variables for CodeArtifact (instead of GitHub OIDC role assumption) +- Uses AWS CI/CD variables for CodeArtifact credentials --- diff --git a/.gitlab/TESTING.md b/.gitlab/TESTING.md index 10dea54c..4cd410fa 100644 --- a/.gitlab/TESTING.md +++ b/.gitlab/TESTING.md @@ -212,12 +212,12 @@ git push origin v0.0.1-test - Package builds successfully - GitLab release created - PyPI upload succeeds (if PYPI_TOKEN set) -- CodeArtifact upload succeeds (if AWS_CODEARTIFACT_DOMAIN set) +- CodeArtifact upload succeeds (if AWS credentials + PYPI_REPOSITORY_URL set) **Configuration needed:** - Set `PYPI_TOKEN` for PyPI publishing - Optionally set `PYPI_REPOSITORY_URL` for custom feed -- Set `AWS_CODEARTIFACT_DOMAIN`, `AWS_CODEARTIFACT_REPOSITORY`, and AWS credentials for CodeArtifact +- For CodeArtifact, set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `PYPI_REPOSITORY_URL` --- @@ -228,16 +228,12 @@ Set these in GitLab project settings (Settings > CI/CD > Variables): ### Secrets (Protected & Masked) - `PYPI_TOKEN` - PyPI authentication token (for releases) - `PAT_TOKEN` - Project/Group Access Token (for sync workflow) -- `AWS_ACCESS_KEY_ID` - AWS access key (for CodeArtifact, if not using OIDC) -- `AWS_SECRET_ACCESS_KEY` - AWS secret key (for CodeArtifact, if not using OIDC) +- `AWS_ACCESS_KEY_ID` - AWS access key (for CodeArtifact publishing) +- `AWS_SECRET_ACCESS_KEY` - AWS secret key (for CodeArtifact publishing) ### Configuration Variables - `UV_EXTRA_INDEX_URL` - Extra index URL for UV (optional) -- `AWS_CODEARTIFACT_DOMAIN` - CodeArtifact domain name (optional) -- `AWS_CODEARTIFACT_REPOSITORY` - CodeArtifact repository name (optional) -- `AWS_REGION` - AWS region for CodeArtifact (optional, defaults to us-east-1) -- `AWS_CODEARTIFACT_OWNER` - Domain owner account ID for cross-account access (optional) -- `PYPI_REPOSITORY_URL` - Custom PyPI URL (optional) +- `PYPI_REPOSITORY_URL` - Custom PyPI URL or CodeArtifact endpoint (optional) - `PUBLISH_COMPANION_BOOK` - Publish documentation (default: true) - `CREATE_MR` - Auto-create merge requests (default: true) diff --git a/.gitlab/workflows/rhiza_release.yml b/.gitlab/workflows/rhiza_release.yml index 642879af..633cb60c 100644 --- a/.gitlab/workflows/rhiza_release.yml +++ b/.gitlab/workflows/rhiza_release.yml @@ -10,26 +10,20 @@ # 2. 🏗️ Build - Build Python package with Hatch (if [build-system] is defined in pyproject.toml # 3. 📝 Draft Release - Create draft GitLab release with build artifacts # 4. 🚀 Publish to PyPI - Publish package using token or custom feed -# 5. 🏭 Publish to AWS CodeArtifact - Publish package to CodeArtifact (conditional) -# 6. ✅ Finalize Release - Publish the GitLab release with links +# 5. ✅ Finalize Release - Publish the GitLab release with links # # 🚀 PyPI Publishing: # - Skipped if no dist/ artifacts exist -# - Skipped if pyproject.toml contains "Private :: Do Not Upload" +# - Skipped if pyproject.toml contains "Private :: Do Not Upload" (unless custom feed) # - Uses PYPI_TOKEN for authentication # - For custom feeds, use PYPI_REPOSITORY_URL and PYPI_TOKEN variables -# -# 🏭 AWS CodeArtifact Publishing: -# - Skipped if AWS_CODEARTIFACT_DOMAIN variable is not set -# - Skipped if no dist/ artifacts exist -# - Uses AWS credentials from CI/CD variables (or OIDC via AWS_ROLE_ARN) -# - Dynamically obtains a CodeArtifact authorization token -# - Publishes with twine using the CodeArtifact repository endpoint +# - For AWS CodeArtifact, set PYPI_REPOSITORY_URL to the CodeArtifact endpoint +# and provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY variables # # 📄 Requirements: # - pyproject.toml with top-level version field (for Python packages) # - PYPI_TOKEN variable for PyPI publishing -# - AWS_CODEARTIFACT_DOMAIN, AWS_CODEARTIFACT_REPOSITORY, and AWS credentials +# - PYPI_REPOSITORY_URL + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY # (for CodeArtifact publishing) # # ✅ To Trigger: @@ -142,92 +136,60 @@ release:pypi: - uv pip install twine script: - | - # Check if dist contains artifacts and not marked as private + # Check if dist contains artifacts if [ ! -d "dist" ] || [ -z "$(ls -A dist)" ]; then echo "⚠️ No dist/ artifacts found. Skipping PyPI publish." exit 0 fi - if grep -R "Private :: Do Not Upload" pyproject.toml; then + # Private classifier only blocks publishing to default PyPI + if [ -z "$PYPI_REPOSITORY_URL" ] && grep -q "Private :: Do Not Upload" pyproject.toml; then echo "⏭️ Package marked as private. Skipping PyPI publish." exit 0 fi - # Publish to PyPI + USERNAME="__token__" + PASSWORD="$PYPI_TOKEN" + + # Auto-detect CodeArtifact URL and get token if AWS credentials are available + if [ -n "$AWS_ACCESS_KEY_ID" ] && echo "$PYPI_REPOSITORY_URL" | grep -q '\.codeartifact\.'; then + apt-get update -qq && apt-get install -y -qq awscli > /dev/null 2>&1 + HOSTNAME=$(echo "$PYPI_REPOSITORY_URL" | sed 's|https://||; s|/.*||') + REGION=$(echo "$HOSTNAME" | sed -n 's/.*\.d\.codeartifact\.\(.*\)\.amazonaws\.com/\1/p') + DOMAIN_OWNER=$(echo "$HOSTNAME" | sed 's/\.d\.codeartifact\..*//') + OWNER=$(echo "$DOMAIN_OWNER" | grep -oE '[0-9]{12}$' || true) + if [ -n "$OWNER" ]; then + DOMAIN=$(echo "$DOMAIN_OWNER" | sed "s/-${OWNER}$//") + else + DOMAIN="$DOMAIN_OWNER" + fi + + OWNER_ARGS="" + if [ -n "$OWNER" ]; then + OWNER_ARGS="--domain-owner $OWNER" + fi + + PASSWORD=$(aws codeartifact get-authorization-token \ + --domain "$DOMAIN" \ + --region "$REGION" \ + $OWNER_ARGS \ + --query authorizationToken --output text) + USERNAME="aws" + fi + + # Publish if [ -n "$PYPI_REPOSITORY_URL" ]; then - echo "📦 Publishing to custom repository: $PYPI_REPOSITORY_URL" + echo "📦 Publishing to: $PYPI_REPOSITORY_URL" twine upload --repository-url "$PYPI_REPOSITORY_URL" \ - --username __token__ --password "$PYPI_TOKEN" \ + --username "$USERNAME" --password "$PASSWORD" \ --skip-existing dist/* else echo "📦 Publishing to PyPI" - twine upload --username __token__ --password "$PYPI_TOKEN" \ + twine upload --username "$USERNAME" --password "$PASSWORD" \ --skip-existing dist/* fi - echo "✅ Published to PyPI" - rules: - - if: $CI_COMMIT_TAG =~ /^v.*/ - when: on_success - allow_failure: false - -release:codeartifact: - stage: deploy - needs: - - job: release:build - artifacts: true - - job: release:draft - image: ghcr.io/astral-sh/uv:0.9.30-bookworm - before_script: - - apt-get update -qq && apt-get install -y -qq awscli > /dev/null 2>&1 - - uv python install $(cat .python-version) - - uv pip install twine - script: - - | - # Check if CodeArtifact is configured - if [ -z "$AWS_CODEARTIFACT_DOMAIN" ]; then - echo "⏭️ Skipping: AWS_CODEARTIFACT_DOMAIN not configured" - exit 0 - fi - - # Check if dist contains artifacts - if [ ! -d "dist" ] || [ -z "$(ls -A dist)" ]; then - echo "⚠️ No dist/ artifacts found. Skipping CodeArtifact publish." - exit 0 - fi - - REGION="${AWS_REGION:-us-east-1}" - - OWNER_ARGS="" - if [ -n "$AWS_CODEARTIFACT_OWNER" ]; then - OWNER_ARGS="--domain-owner $AWS_CODEARTIFACT_OWNER" - fi - - # Get CodeArtifact authorization token - TOKEN=$(aws codeartifact get-authorization-token \ - --domain "$AWS_CODEARTIFACT_DOMAIN" \ - --region "$REGION" \ - $OWNER_ARGS \ - --query authorizationToken --output text) - - # Get CodeArtifact repository endpoint - REPO_URL=$(aws codeartifact get-repository-endpoint \ - --domain "$AWS_CODEARTIFACT_DOMAIN" \ - --repository "$AWS_CODEARTIFACT_REPOSITORY" \ - --format pypi \ - --region "$REGION" \ - $OWNER_ARGS \ - --query repositoryEndpoint --output text) - - echo "📦 Publishing to CodeArtifact: $AWS_CODEARTIFACT_DOMAIN/$AWS_CODEARTIFACT_REPOSITORY" - twine upload \ - --repository-url "$REPO_URL" \ - --username aws \ - --password "$TOKEN" \ - --skip-existing \ - dist/* - - echo "✅ Published to CodeArtifact" + echo "✅ Published successfully" rules: - if: $CI_COMMIT_TAG =~ /^v.*/ when: on_success @@ -239,8 +201,6 @@ release:finalize: - job: release:build - job: release:pypi optional: true - - job: release:codeartifact - optional: true image: alpine:latest before_script: - apk add --no-cache curl jq python3 @@ -259,13 +219,6 @@ release:finalize: else RELEASE_BODY="${RELEASE_BODY}\n\n### Custom Feed Package\n\n[$PACKAGE_NAME]($PYPI_REPOSITORY_URL)" fi - - # Add CodeArtifact link if published - if [ -n "$AWS_CODEARTIFACT_DOMAIN" ] && [ -n "$AWS_CODEARTIFACT_REPOSITORY" ]; then - REGION="${AWS_REGION:-us-east-1}" - CONSOLE_URL="https://${REGION}.console.aws.amazon.com/codesuite/codeartifact/d/${AWS_CODEARTIFACT_DOMAIN}/r/${AWS_CODEARTIFACT_REPOSITORY}/p/pypi/p/${PACKAGE_NAME}/versions?region=${REGION}" - RELEASE_BODY="${RELEASE_BODY}\n\n### AWS CodeArtifact Package\n\n[$PACKAGE_NAME@$VERSION]($CONSOLE_URL) (\`${AWS_CODEARTIFACT_DOMAIN}/${AWS_CODEARTIFACT_REPOSITORY}\`)" - fi fi fi diff --git a/.rhiza/docs/RELEASING.md b/.rhiza/docs/RELEASING.md index e919f2a0..34c1dcc4 100644 --- a/.rhiza/docs/RELEASING.md +++ b/.rhiza/docs/RELEASING.md @@ -79,10 +79,9 @@ The release workflow (`.github/workflows/rhiza_release.yml`) triggers on the tag 1. **Validates** - Checks the tag format and ensures no duplicate releases 2. **Builds** - Builds the Python package (if `pyproject.toml` exists) 3. **Drafts** - Creates a draft GitHub release with artifacts -4. **PyPI** - Publishes to PyPI (if not marked private) -5. **CodeArtifact** - Publishes to AWS CodeArtifact (if `AWS_CODEARTIFACT_DOMAIN` is set) -6. **Devcontainer** - Publishes devcontainer image (if `PUBLISH_DEVCONTAINER=true`) -7. **Finalizes** - Publishes the GitHub release with links to PyPI, CodeArtifact, and container images +4. **PyPI** - Publishes to PyPI or custom feed such as CodeArtifact (if not marked private) +5. **Devcontainer** - Publishes devcontainer image (if `PUBLISH_DEVCONTAINER=true`) +6. **Finalizes** - Publishes the GitHub release with links to PyPI and container images ## Configuration Options @@ -97,52 +96,28 @@ The release workflow (`.github/workflows/rhiza_release.yml`) triggers on the tag Publish Python packages to an AWS CodeArtifact repository. This is useful for private packages or organisations that use CodeArtifact as their internal package registry. -> **Note:** The `Private :: Do Not Upload` classifier only affects PyPI publishing. Packages -> marked as private can still be published to CodeArtifact, which is the typical use case -> for internal packages. +CodeArtifact publishing is handled through the same `pypi` job by setting `PYPI_REPOSITORY_URL` +to the CodeArtifact endpoint and providing AWS credentials. The workflow automatically detects +CodeArtifact URLs and handles token exchange. -#### Required Repository Variables +> **Note:** The `Private :: Do Not Upload` classifier only blocks publishing to the default +> PyPI. When `PYPI_REPOSITORY_URL` is set (e.g. to a CodeArtifact endpoint), private packages +> will still be published. -| Variable | Description | Example | -|---|---|---| -| `AWS_CODEARTIFACT_DOMAIN` | CodeArtifact domain name | `my-company` | -| `AWS_CODEARTIFACT_REPOSITORY` | CodeArtifact repository name | `python-packages` | -| `AWS_ROLE_ARN` | IAM role ARN for GitHub OIDC (**GitHub only**) | `arn:aws:iam::123456789012:role/github-release` | +#### Setup Instructions -#### Optional Repository Variables - -| Variable | Default | Description | -|---|---|---| -| `AWS_REGION` | `us-east-1` | AWS region for CodeArtifact | -| `AWS_CODEARTIFACT_OWNER` | *(current account)* | Domain owner AWS account ID (for cross-account access) | - -#### Setup Instructions (GitHub) - -1. **Create an IAM OIDC identity provider** for GitHub Actions in your AWS account: - - Provider URL: `https://token.actions.githubusercontent.com` - - Audience: `sts.amazonaws.com` - -2. **Create an IAM role** with a trust policy allowing GitHub OIDC: - ```json - { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Principal": { "Federated": "arn:aws:iam::123456789012:oidc-provider/token.actions.githubusercontent.com" }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" - }, - "StringLike": { - "token.actions.githubusercontent.com:sub": "repo:YOUR-ORG/YOUR-REPO:*" - } - } - }] - } +1. **Get your CodeArtifact repository endpoint** (the `PYPI_REPOSITORY_URL`): + ```bash + aws codeartifact get-repository-endpoint \ + --domain my-domain \ + --repository my-repo \ + --format pypi \ + --query repositoryEndpoint --output text ``` + This returns a URL like: + `https://my-domain-123456789012.d.codeartifact.us-east-1.amazonaws.com/pypi/my-repo/` -3. **Attach a policy** granting CodeArtifact access: +2. **Create an IAM user** (or use an existing one) with the following permissions: ```json { "Version": "2012-10-17", @@ -169,22 +144,26 @@ or organisations that use CodeArtifact as their internal package registry. } ``` -4. **Set repository variables** in GitHub (Settings → Secrets and variables → Actions → Variables): - - `AWS_CODEARTIFACT_DOMAIN` = your domain name - - `AWS_CODEARTIFACT_REPOSITORY` = your repository name - - `AWS_ROLE_ARN` = the IAM role ARN from step 2 - - `AWS_REGION` = your AWS region (e.g., `eu-west-2`) - -#### Setup Instructions (GitLab) - -1. **Configure AWS credentials** as CI/CD variables: - - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (or configure GitLab OIDC with AWS) - -2. **Set CI/CD variables**: - - `AWS_CODEARTIFACT_DOMAIN` = your domain name - - `AWS_CODEARTIFACT_REPOSITORY` = your repository name - - `AWS_REGION` = your AWS region - - `AWS_CODEARTIFACT_OWNER` = domain owner account ID (if cross-account) +3. **Set repository secrets and variables:** + + **GitHub** (Settings → Secrets and variables → Actions): + | Type | Name | Value | + |---|---|---| + | Secret | `AWS_ACCESS_KEY_ID` | IAM user access key | + | Secret | `AWS_SECRET_ACCESS_KEY` | IAM user secret key | + | Variable | `PYPI_REPOSITORY_URL` | CodeArtifact endpoint URL from step 1 | + + **GitLab** (Settings → CI/CD → Variables): + | Name | Value | Protected | Masked | + |---|---|---|---| + | `AWS_ACCESS_KEY_ID` | IAM user access key | ✅ | ✅ | + | `AWS_SECRET_ACCESS_KEY` | IAM user secret key | ✅ | ✅ | + | `PYPI_REPOSITORY_URL` | CodeArtifact endpoint URL from step 1 | ✅ | ❌ | + +That's it — no additional configuration needed. The release workflow automatically: +- Detects the CodeArtifact URL from `PYPI_REPOSITORY_URL` +- Uses `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to obtain a temporary auth token +- Publishes the package with `twine` ### Devcontainer Publishing diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 26024a02..f80f8733 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -108,25 +108,19 @@ flowchart TD tag[Push Tag v*] --> validate[Validate Tag] validate --> build[Build Package] build --> draft[Draft GitHub Release] - draft --> pypi[Publish to PyPI] - draft --> codeartifact[Publish to CodeArtifact] + draft --> pypi[Publish to PyPI / Custom Feed] draft --> devcontainer[Publish Devcontainer] pypi --> finalize[Finalize Release] - codeartifact --> finalize devcontainer --> finalize subgraph Conditions pypi_cond{Has dist/ &
not Private?} - ca_cond{Has dist/ &
AWS_CODEARTIFACT_DOMAIN?} dev_cond{PUBLISH_DEVCONTAINER
= true?} end draft --> pypi_cond pypi_cond -->|yes| pypi pypi_cond -->|no| finalize - draft --> ca_cond - ca_cond -->|yes| codeartifact - ca_cond -->|no| finalize draft --> dev_cond dev_cond -->|yes| devcontainer dev_cond -->|no| finalize diff --git a/docs/CHANGELOG_GUIDE.md b/docs/CHANGELOG_GUIDE.md index 93b37128..a0f476a9 100644 --- a/docs/CHANGELOG_GUIDE.md +++ b/docs/CHANGELOG_GUIDE.md @@ -19,10 +19,9 @@ The release workflow: 1. Validates the tag format 2. Builds distribution artifacts 3. Creates a draft GitHub release with `generate_release_notes: true` -4. Publishes to PyPI (if applicable) -5. Publishes to AWS CodeArtifact (if configured) -6. Publishes devcontainer (if configured) -7. Finalizes the release +4. Publishes to PyPI or custom feed such as CodeArtifact (if applicable) +5. Publishes devcontainer (if configured) +6. Finalizes the release GitHub automatically generates release notes by: - Listing all PRs merged since the last release diff --git a/docs/PRESENTATION.md b/docs/PRESENTATION.md index b3184021..5ed41ba4 100644 --- a/docs/PRESENTATION.md +++ b/docs/PRESENTATION.md @@ -422,10 +422,10 @@ Automatic if configured as **Trusted Publisher**: ### AWS CodeArtifact Publication -Automatic if repository variables are configured: +Automatic if repository secrets and variables are configured: -1. Set `AWS_CODEARTIFACT_DOMAIN` and `AWS_CODEARTIFACT_REPOSITORY` -2. Configure OIDC role (`AWS_ROLE_ARN`) for credential-free auth +1. Set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` as secrets +2. Set `PYPI_REPOSITORY_URL` to the CodeArtifact endpoint 3. Release workflow publishes automatically via `twine` ### Private Packages From f501825ba821d421cc757315b3d11ae7a0ead600 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 23 Mar 2026 19:52:05 +0000 Subject: [PATCH 4/4] fix: add error handling for CodeArtifact token fetch and use POSIX-safe pattern matching Co-authored-by: markrichardson <5681211+markrichardson@users.noreply.github.com> Agent-Logs-Url: https://github.com/Jebel-Quant/rhiza/sessions/d31294fb-ccb9-44fc-8f54-4720cee99ca3 --- .github/workflows/rhiza_release.yml | 5 ++++- .gitlab/workflows/rhiza_release.yml | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rhiza_release.yml b/.github/workflows/rhiza_release.yml index 0e1ee1ae..973a56d2 100644 --- a/.github/workflows/rhiza_release.yml +++ b/.github/workflows/rhiza_release.yml @@ -323,7 +323,10 @@ jobs: --domain "$DOMAIN" \ --region "$REGION" \ "${OWNER_ARGS[@]}" \ - --query authorizationToken --output text) + --query authorizationToken --output text) || { + echo "::error::Failed to get CodeArtifact token. Check AWS credentials and permissions." + exit 1 + } echo "::add-mask::$TOKEN" echo "token=$TOKEN" >> "$GITHUB_OUTPUT" echo "username=aws" >> "$GITHUB_OUTPUT" diff --git a/.gitlab/workflows/rhiza_release.yml b/.gitlab/workflows/rhiza_release.yml index 633cb60c..13c570e9 100644 --- a/.gitlab/workflows/rhiza_release.yml +++ b/.gitlab/workflows/rhiza_release.yml @@ -152,7 +152,7 @@ release:pypi: PASSWORD="$PYPI_TOKEN" # Auto-detect CodeArtifact URL and get token if AWS credentials are available - if [ -n "$AWS_ACCESS_KEY_ID" ] && echo "$PYPI_REPOSITORY_URL" | grep -q '\.codeartifact\.'; then + if [ -n "$AWS_ACCESS_KEY_ID" ] && case "$PYPI_REPOSITORY_URL" in *.codeartifact.*) true;; *) false;; esac; then apt-get update -qq && apt-get install -y -qq awscli > /dev/null 2>&1 HOSTNAME=$(echo "$PYPI_REPOSITORY_URL" | sed 's|https://||; s|/.*||') REGION=$(echo "$HOSTNAME" | sed -n 's/.*\.d\.codeartifact\.\(.*\)\.amazonaws\.com/\1/p') @@ -169,11 +169,15 @@ release:pypi: OWNER_ARGS="--domain-owner $OWNER" fi + # OWNER_ARGS is intentionally unquoted to allow word splitting into separate arguments PASSWORD=$(aws codeartifact get-authorization-token \ --domain "$DOMAIN" \ --region "$REGION" \ $OWNER_ARGS \ - --query authorizationToken --output text) + --query authorizationToken --output text) || { + echo "❌ Failed to get CodeArtifact token. Check AWS credentials and permissions." + exit 1 + } USERNAME="aws" fi