diff --git a/containers/debian/Dockerfile b/containers/debian/Dockerfile index 29bf371f..e1d8c1a2 100644 --- a/containers/debian/Dockerfile +++ b/containers/debian/Dockerfile @@ -258,13 +258,68 @@ spack compiler find --scope site spack config blame compilers EOF +## Setup GPG public key for buildcache verification +## REQUIRED: A persistent GPG key MUST be provided via SPACK_SIGNING_KEY secret. +## Only the PUBLIC key is stored in container layers for verification. +## The PRIVATE key is mounted as a secret during build steps that require signing. +## +## SECURITY MODEL: +## - Private key: Stored in GitHub Secrets, mounted during builds, NEVER in container layers +## - Public key: Embedded in container at ${SPACK_ROOT}/spack-public-key.pub for verification +## - Signing: Happens during builder install steps, using --mount=type=secret for private key +## - Verification: Happens during runtime install steps, using embedded public key +RUN --mount=type=secret,id=SPACK_SIGNING_KEY,target=/tmp/spack-signing-key,required=true \ + <" + echo "See docs/BUILDCACHE_SIGNING.md for instructions on generating a key" + exit 1 +fi + +# Initialize Spack's GPG environment +spack gpg init + +# Extract and store ONLY the public key (not the private key) +# Import to temporary GPG home to extract public key +export GNUPGHOME=/tmp/gnupg-temp +mkdir -p ${GNUPGHOME} +chmod 700 ${GNUPGHOME} +gpg --batch --import /tmp/spack-signing-key +# Export only the public key +gpg --batch --export > ${SPACK_ROOT}/spack-public-key.pub +# Import public key into Spack's keyring for verification +spack gpg trust ${SPACK_ROOT}/spack-public-key.pub +# Clean up temporary GPG home (private key never persisted in container) +rm -rf ${GNUPGHOME} +unset GNUPGHOME + +# Verify we have the public key +if [ -f ${SPACK_ROOT}/spack-public-key.pub ]; then + echo "Public key successfully exported to ${SPACK_ROOT}/spack-public-key.pub" + echo "Private key was NOT stored in container - will be mounted during build steps" +else + echo "ERROR: Failed to export public key" + exit 1 +fi + +spack gpg list +EOF + ## Setup buildcache mirrors ## - this always adds the read-only mirror to the container ## - the write-enabled mirror is provided later as a secret mount +## - ghcr mirror is configured as --signed (changed from --unsigned) +## The transition strategy in runtime stages (eic/Dockerfile) handles both signed and unsigned +## artifacts during migration period, attempting signed first and falling back to unsigned +## - New buildcaches will be automatically signed via autopush (mirrors.yaml.in has signed: true) +## - Deploy order: This change is safe as long as runtime stages have transition logic in place RUN --mount=type=cache,target=/var/cache/spack <&1 | tee /tmp/install.log; then + # Check if the failure was due to signature verification issues + # Look for specific Spack error patterns related to signatures and GPG + if grep -qE "(NoVerifyException|signature.*verif|gpg.*trust|gpg.*key|buildcache.*sign)" /tmp/install.log; then + echo "==> Signature verification issue detected, retrying with --no-check-signature for transition compatibility" + echo "==> This fallback allows installation of unsigned buildcaches created before signature enforcement" + spack ${SPACK_FLAGS} install ${SPACK_INSTALL_FLAGS} --no-check-signature --use-buildcache only + else + # Different error (not signature-related), fail the build with full log + echo "==> Installation failed with non-signature error:" + cat /tmp/install.log + exit 1 + fi +fi spack gc --yes-to-all go go-bootstrap rust rust-bootstrap py-setuptools-rust py-maturin EOF @@ -216,6 +262,7 @@ ARG TARGETPLATFORM LABEL org.opencontainers.image.title="Electron-Ion Collider build installation image (custom configuration, $TARGETPLATFORM)" # Installation (custom environment) +# Mount SPACK_SIGNING_KEY secret for signing during autopush RUN --mount=type=cache,target=/ccache,id=ccache-${TARGETPLATFORM} \ --mount=type=cache,target=/var/cache/spack \ --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \ @@ -223,14 +270,28 @@ RUN --mount=type=cache,target=/ccache,id=ccache-${TARGETPLATFORM} \ --mount=type=secret,id=CI_REGISTRY_PASSWORD,env=CI_REGISTRY_PASSWORD \ --mount=type=secret,id=GITHUB_REGISTRY_USER,env=GITHUB_REGISTRY_USER \ --mount=type=secret,id=GITHUB_REGISTRY_TOKEN,env=GITHUB_REGISTRY_TOKEN \ + --mount=type=secret,id=SPACK_SIGNING_KEY,target=/tmp/spack-signing-key,required=true \ <&1 | tee /tmp/install.log; then + # Check if the failure was due to signature verification issues + # Look for specific Spack error patterns related to signatures and GPG + if grep -qE "(NoVerifyException|signature.*verif|gpg.*trust|gpg.*key|buildcache.*sign)" /tmp/install.log; then + echo "==> Signature verification issue detected, retrying with --no-check-signature for transition compatibility" + echo "==> This fallback allows installation of unsigned buildcaches created before signature enforcement" + spack ${SPACK_FLAGS} install ${SPACK_INSTALL_FLAGS} --no-check-signature --use-buildcache only + else + # Different error (not signature-related), fail the build with full log + echo "==> Installation failed with non-signature error:" + cat /tmp/install.log + exit 1 + fi +fi spack gc --yes-to-all go go-bootstrap rust rust-bootstrap py-setuptools-rust py-maturin EOF diff --git a/docs/BUILDCACHE_SIGNING.md b/docs/BUILDCACHE_SIGNING.md new file mode 100644 index 00000000..9c549dde --- /dev/null +++ b/docs/BUILDCACHE_SIGNING.md @@ -0,0 +1,180 @@ +# Buildcache Signature Verification + +## Overview + +The container build process uses Spack buildcaches to distribute pre-built binary packages. These buildcaches are signed using GPG keys to ensure integrity and authenticity. + +**SECURITY MODEL**: The GPG private key is **NEVER** stored in container layers. It is provided as a Docker secret during build and mounted only when needed for signing operations. Only the public key is embedded in the container for verification purposes. + +## How It Works + +### Build Process + +1. **Base Layer Setup**: + - GPG private key is provided via SPACK_SIGNING_KEY secret (required) + - Public key is extracted and stored at `${SPACK_ROOT}/spack-public-key.pub` + - Private key is NOT persisted in any container layer + +2. **Builder Stage**: + - During package compilation, the private key is mounted as a secret + - Packages are compiled from source and automatically pushed to OCI buildcache + - Spack autopush signs packages using the mounted private key + - Private key is only available during the RUN step, not in final layer + +3. **Runtime Stage**: + - Only the public key (from base layer) is available + - Signatures are verified when installing from buildcache + - Private key is never present in runtime containers + +### Security Layers + +The buildcache security model provides: + +1. **GPG Signatures**: Cryptographic signing with private key stored externally (GitHub Secrets) +2. **Key Isolation**: Private key never embedded in container layers, only mounted during build +3. **Authenticity Verification**: Signatures prove packages were signed by holder of private key +4. **OCI Registry Integrity**: SHA256 digest verification for all artifacts +5. **HTTPS/TLS**: Encrypted communication with the registry +6. **Access Control**: Write access to the registry requires authentication + +## Production Setup (Required) + +A persistent GPG key **MUST** be provided via GitHub Secrets. There is no fallback - builds will fail without this secret. + +### Generate GPG Key + +```bash +# Generate a new GPG key (non-interactively) +gpg --batch --gen-key < spack-signing-key.asc +``` + +### Add to GitHub Secrets + +1. Go to repository Settings → Secrets and variables → Actions +2. Create a new secret named `SPACK_SIGNING_KEY` +3. Paste the contents of `spack-signing-key.asc` +4. Save and delete the local `spack-signing-key.asc` file securely + +### Update Workflow + +Add the secret to the build step in `.github/workflows/build-push.yml`: + +```yaml +secrets: | + "CI_REGISTRY_USER=${{ secrets.GHCR_REGISTRY_USER }}" + "CI_REGISTRY_PASSWORD=${{ secrets.GHCR_REGISTRY_TOKEN }}" + "GITHUB_REGISTRY_USER=${{ secrets.GHCR_REGISTRY_USER }}" + "GITHUB_REGISTRY_TOKEN=${{ secrets.GHCR_REGISTRY_TOKEN }}" + "SPACK_SIGNING_KEY=${{ secrets.SPACK_SIGNING_KEY }}" +``` + +**NOTE**: The SPACK_SIGNING_KEY secret is now **REQUIRED**. Builds will fail if not provided. + +## Security Considerations + +### Security Model + +**✅ SECURE IMPLEMENTATION**: The private key is **NEVER** stored in container layers. + +**Key Security Features:** +- **Private Key Isolation**: Stored only in GitHub Secrets, never in container filesystem +- **Secret Mounting**: Private key is mounted during build steps that need signing, then discarded +- **Public Key Only**: Container layers contain only the public key for verification +- **No Extraction Risk**: Private key cannot be extracted from published containers +- **Strong Authenticity**: Signatures prove packages were signed by holder of the secret private key + +### Trust Model + +The GPG signature model provides: +- ✅ **Integrity verification**: Packages haven't been modified after signing +- ✅ **Authenticity verification**: Packages signed by holder of private key in GitHub Secrets +- ✅ **Non-repudiation**: Only authorized CI/CD with access to secret can sign packages +- ✅ **Key Protection**: Private key never exposed in public container layers +- ✅ **Transition compatibility**: Graceful migration from unsigned to signed buildcaches + +### How It Works + +1. **Private Key Storage**: Stored securely in GitHub Secrets (SPACK_SIGNING_KEY) +2. **Build-Time Mounting**: + - Private key mounted as Docker secret during builder RUN steps + - Used for signing packages during autopush + - Exists only in memory during RUN, never written to filesystem layer +3. **Public Key Embedding**: + - Public key extracted and stored at `${SPACK_ROOT}/spack-public-key.pub` + - Safe to include in public containers (used only for verification) +4. **Verification**: Runtime containers use embedded public key to verify signatures + +### Key Protection + +- **GitHub Secrets**: Private key encrypted at rest in GitHub's secure storage +- **Limited Access**: Only authorized repository collaborators can modify secrets +- **Audit Trail**: GitHub tracks who accesses and modifies secrets +- **No Container Exposure**: Private key never persisted in any Docker layer +- **Temporary Usage**: Private key exists only during RUN execution, then removed + +### Key Rotation + +When rotating keys: + +1. Generate new GPG key (see instructions above) +2. Update SPACK_SIGNING_KEY secret in GitHub +3. Clear old buildcaches (signed with old key) +4. Rebuild containers to generate new signed buildcaches +5. Securely delete old private key + +Recommended rotation frequency: Every 6-12 months or when: +- Key may have been compromised +- Team member with key access leaves +- As part of regular security hygiene + +### Best Practices + +- **Key Protection**: Store private key only in GitHub Secrets, never commit to repository or share +- **Access Control**: Limit who can view/modify SPACK_SIGNING_KEY secret in repository settings +- **Key Rotation**: Rotate GPG keys periodically (recommended: every 6-12 months) +- **Buildcache Clearing**: When rotating keys, clear old buildcaches signed with previous key +- **Monitoring**: Monitor for suspicious packages or signature verification failures +- **Audit**: Review GitHub Actions logs for unauthorized build attempts + +## Troubleshooting + +### Build Fails: SPACK_SIGNING_KEY Required + +If you see: +``` +ERROR: SPACK_SIGNING_KEY secret is required but not provided +``` + +Solution: +- The secret is mandatory - generate a GPG key and add it to GitHub Secrets +- Follow the "Production Setup" instructions above +- Ensure the secret is named exactly `SPACK_SIGNING_KEY` + +### Signature Verification Failures + +If you see errors like: +``` +==> Error: Unable to verify buildcache package +``` + +Possible causes: +- The buildcache was signed with a different key +- The public key wasn't properly imported +- The buildcache is from before signing was enabled + +Solutions: +- Clear the buildcache and rebuild +- Ensure SPACK_SIGNING_KEY secret is configured correctly +- Check that the public key is being exported and imported correctly +- Verify key consistency across builds diff --git a/mirrors.yaml.in b/mirrors.yaml.in index f0d218fe..96ac3f2c 100644 --- a/mirrors.yaml.in +++ b/mirrors.yaml.in @@ -2,7 +2,7 @@ mirrors: eicweb: autopush: true url: oci://${CI_REGISTRY}/${CI_PROJECT_PATH}/spack-${SPACKPACKAGES_VERSION} - signed: false + signed: true fetch: access_pair: id_variable: CI_REGISTRY_USER @@ -14,7 +14,7 @@ mirrors: ghcr: autopush: true url: oci://ghcr.io/eic/spack-${SPACKPACKAGES_VERSION} - signed: false + signed: true fetch: access_pair: id_variable: GITHUB_REGISTRY_USER