From 686f9aa13f21b1ab9e268e5f554f285fc913b27d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 30 Dec 2025 14:05:34 +0000 Subject: [PATCH 01/10] Initial plan From d1f929c34a2d3ed6f6f30319dff89a093344d16d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 30 Dec 2025 14:16:42 +0000 Subject: [PATCH 02/10] Enable GPG signature verification for buildcache installations - Change mirrors.yaml to use signed: true for both eicweb and ghcr mirrors - Add GPG key setup in builder stages with support for persistent keys via secret - Export public key from builder stages for runtime verification - Import public key in runtime stages before installing from buildcache - Remove --no-check-signature from runtime SPACK_INSTALL_FLAGS - Keep --no-check-signature in builder stages (building from source) Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com> --- containers/eic/Dockerfile | 36 +++++++++++++++++++++++++++++++++++- mirrors.yaml.in | 4 ++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile index ff3aa4e0..e5eba53e 100644 --- a/containers/eic/Dockerfile +++ b/containers/eic/Dockerfile @@ -89,6 +89,25 @@ ARG TARGETPLATFORM # Open Container Initiative labels LABEL org.opencontainers.image.title="Electron-Ion Collider build installation image (default configuration, $TARGETPLATFORM)" +# Set up GPG key for buildcache signing +RUN --mount=type=secret,id=SPACK_SIGNING_KEY,target=/tmp/spack-signing-key \ + < /tmp/spack-public-key.pub EOF @@ -124,12 +145,23 @@ COPY --from=spack-environment . /opt/spack-environment/ ARG ENV=xl ENV SPACK_ENV=/opt/spack-environment/${ENV} ARG SPACK_FLAGS="--backtrace" -ARG SPACK_INSTALL_FLAGS="--no-check-signature --show-log-on-error --yes-to-all" +ARG SPACK_INSTALL_FLAGS="--show-log-on-error --yes-to-all" ENV SPACK_COLOR="always" COPY --from=builder_installation_default \ /opt/spack-environment/${ENV}/spack.* \ /opt/spack-environment/${ENV}/ +COPY --from=builder_installation_default \ + /tmp/spack-public-key.pub \ + /tmp/spack-public-key.pub + +# Import public key for buildcache verification +RUN < /tmp/spack-public-key.pub EOF diff --git a/mirrors.yaml.in b/mirrors.yaml.in index f0d218fe..96ac3f2c 100644 --- a/mirrors.yaml.in +++ b/mirrors.yaml.in @@ -2,7 +2,7 @@ mirrors: eicweb: autopush: true url: oci://${CI_REGISTRY}/${CI_PROJECT_PATH}/spack-${SPACKPACKAGES_VERSION} - signed: false + signed: true fetch: access_pair: id_variable: CI_REGISTRY_USER @@ -14,7 +14,7 @@ mirrors: ghcr: autopush: true url: oci://ghcr.io/eic/spack-${SPACKPACKAGES_VERSION} - signed: false + signed: true fetch: access_pair: id_variable: GITHUB_REGISTRY_USER From 6788c73a16d40fc1760def7599b7af9f6e99d6ff Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 30 Dec 2025 14:18:48 +0000 Subject: [PATCH 03/10] docs: add buildcache signing documentation and improve comments - Add comprehensive documentation in docs/BUILDCACHE_SIGNING.md - Explain GPG key setup for production use - Document security layers and troubleshooting - Add comment clarifying temporary vs persistent key usage Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com> --- containers/eic/Dockerfile | 3 + docs/BUILDCACHE_SIGNING.md | 109 +++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 docs/BUILDCACHE_SIGNING.md diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile index e5eba53e..e399f86c 100644 --- a/containers/eic/Dockerfile +++ b/containers/eic/Dockerfile @@ -90,6 +90,9 @@ ARG TARGETPLATFORM LABEL org.opencontainers.image.title="Electron-Ion Collider build installation image (default configuration, $TARGETPLATFORM)" # Set up GPG key for buildcache signing +# For production use, provide a persistent GPG key via SPACK_SIGNING_KEY secret to ensure +# buildcaches can be verified across builds. Without this secret, a temporary key is created +# for each build, which works for single-build scenarios but prevents buildcache reuse. RUN --mount=type=secret,id=SPACK_SIGNING_KEY,target=/tmp/spack-signing-key \ < spack-signing-key.asc +``` + +### Add to GitHub Secrets + +1. Go to repository Settings → Secrets and variables → Actions +2. Create a new secret named `SPACK_SIGNING_KEY` +3. Paste the contents of `spack-signing-key.asc` +4. Save and delete the local `spack-signing-key.asc` file securely + +### Update Workflow + +Add the secret to the build step in `.github/workflows/build-push.yml`: + +```yaml +secrets: | + "CI_REGISTRY_USER=${{ secrets.GHCR_REGISTRY_USER }}" + "CI_REGISTRY_PASSWORD=${{ secrets.GHCR_REGISTRY_TOKEN }}" + "GITHUB_REGISTRY_USER=${{ secrets.GHCR_REGISTRY_USER }}" + "GITHUB_REGISTRY_TOKEN=${{ secrets.GHCR_REGISTRY_TOKEN }}" + "SPACK_SIGNING_KEY=${{ secrets.SPACK_SIGNING_KEY }}" +``` + +## Development/Testing Setup + +For local development and testing, the build will automatically create a temporary GPG key if `SPACK_SIGNING_KEY` is not provided. This works for single builds but won't allow buildcache reuse across builds. + +## Migration Notes + +When enabling signature verification for existing buildcaches: + +1. **Clear Old Buildcaches**: Old unsigned buildcaches may need to be cleared +2. **Transition Period**: During the transition, some builds may fail if they try to use old unsigned buildcaches +3. **Key Rotation**: If you need to rotate keys, clear the buildcache and create a new key + +## Security Considerations + +- **Key Protection**: The private key should be stored securely in GitHub Secrets and never committed to the repository +- **Key Rotation**: Consider rotating the GPG key periodically (e.g., annually) +- **Buildcache Clearing**: When rotating keys, clear old buildcaches to prevent signature verification failures +- **Trust Model**: The GPG signature verifies that packages were signed by someone with access to the private key, which should be limited to the CI/CD system + +## Troubleshooting + +### Signature Verification Failures + +If you see errors like: +``` +==> Error: Unable to verify buildcache package +``` + +Possible causes: +- The buildcache was signed with a different key +- The public key wasn't properly imported +- The buildcache is from before signing was enabled + +Solutions: +- Clear the buildcache and rebuild +- Ensure SPACK_SIGNING_KEY secret is configured +- Check that the public key is being exported and imported correctly + +### Temporary Key Issues + +If builds are failing because they can't reuse buildcaches: +- Add the SPACK_SIGNING_KEY secret with a persistent key +- Or, clear buildcaches between builds From fd079186d82230a0644ebaf226624191d62e7796 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 30 Dec 2025 14:21:22 +0000 Subject: [PATCH 04/10] fix: add GPG setup to builder_installation_custom and runtime_custom - Add GPG key setup to builder_installation_custom stage - Copy and import public key from builder_installation_custom to runtime_custom - This ensures custom builds have the same security as default builds - Fix documentation placeholder Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com> --- containers/eic/Dockerfile | 34 ++++++++++++++++++++++++++++++++++ docs/BUILDCACHE_SIGNING.md | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile index e399f86c..b58eebc2 100644 --- a/containers/eic/Dockerfile +++ b/containers/eic/Dockerfile @@ -250,6 +250,28 @@ ARG TARGETPLATFORM # Open Container Initiative labels LABEL org.opencontainers.image.title="Electron-Ion Collider build installation image (custom configuration, $TARGETPLATFORM)" +# Set up GPG key for buildcache signing +# For production use, provide a persistent GPG key via SPACK_SIGNING_KEY secret to ensure +# buildcaches can be verified across builds. Without this secret, a temporary key is created +# for each build, which works for single-build scenarios but prevents buildcache reuse. +RUN --mount=type=secret,id=SPACK_SIGNING_KEY,target=/tmp/spack-signing-key \ + < Date: Tue, 30 Dec 2025 14:24:42 +0000 Subject: [PATCH 05/10] refactor: improve GPG key import and add clarifying comments - Fix GPG private key import to use gpg --import instead of spack gpg trust - Add clearer comments explaining key management strategy - Document stage inheritance relationships - Acknowledge code duplication and explain why it's necessary Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com> --- containers/eic/Dockerfile | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile index b58eebc2..237a4d7d 100644 --- a/containers/eic/Dockerfile +++ b/containers/eic/Dockerfile @@ -100,8 +100,11 @@ set -e spack gpg init # Import signing key if provided, otherwise create a new one if [ -f /tmp/spack-signing-key ] && [ -s /tmp/spack-signing-key ]; then - # Import the GPG key (both public and private) - spack gpg trust /tmp/spack-signing-key + # Import the GPG key (expects ASCII-armored key with both public and private parts) + # First import into GPG, then Spack will use it from there + cat /tmp/spack-signing-key | gpg --batch --import + # Also trust the public key in Spack's keyring for verification + cat /tmp/spack-signing-key | gpg --batch --export | spack gpg trust else # Create a temporary GPG key for signing (non-interactive) # Note: For production, a persistent key should be provided via SPACK_SIGNING_KEY secret @@ -251,6 +254,8 @@ ARG TARGETPLATFORM LABEL org.opencontainers.image.title="Electron-Ion Collider build installation image (custom configuration, $TARGETPLATFORM)" # Set up GPG key for buildcache signing +# Note: This setup is duplicated from builder_installation_default due to Dockerfile limitations. +# Both builder stages need independent GPG setup as they don't share a common parent with GPG configured. # For production use, provide a persistent GPG key via SPACK_SIGNING_KEY secret to ensure # buildcaches can be verified across builds. Without this secret, a temporary key is created # for each build, which works for single-build scenarios but prevents buildcache reuse. @@ -261,8 +266,11 @@ set -e spack gpg init # Import signing key if provided, otherwise create a new one if [ -f /tmp/spack-signing-key ] && [ -s /tmp/spack-signing-key ]; then - # Import the GPG key (both public and private) - spack gpg trust /tmp/spack-signing-key + # Import the GPG key (expects ASCII-armored key with both public and private parts) + # First import into GPG, then Spack will use it from there + cat /tmp/spack-signing-key | gpg --batch --import + # Also trust the public key in Spack's keyring for verification + cat /tmp/spack-signing-key | gpg --batch --export | spack gpg trust else # Create a temporary GPG key for signing (non-interactive) # Note: For production, a persistent key should be provided via SPACK_SIGNING_KEY secret @@ -320,8 +328,11 @@ COPY --from=builder_installation_custom \ /tmp/spack-public-key-custom.pub # Import public key from custom builder for buildcache verification -# Note: runtime_custom inherits the key from runtime_default, but we also need -# the key from builder_installation_custom in case temporary keys are used +# Note: This stage is based on runtime_default (FROM runtime_default AS runtime_custom at line 300), +# which already has the public key from builder_installation_default imported. +# However, we also import the key from builder_installation_custom to handle the case +# where temporary keys are used (each builder stage creates a different temporary key). +# With a persistent SPACK_SIGNING_KEY secret, both builders use the same key, making this redundant but harmless. RUN < Date: Tue, 30 Dec 2025 14:31:55 +0000 Subject: [PATCH 06/10] refactor: move GPG key setup to base layer and add transition strategy - Move GPG key initialization from eic/Dockerfile to debian/Dockerfile base layer - Export public key to ${SPACK_ROOT}/spack-public-key.pub instead of /tmp - Update ghcr mirror to use --signed instead of --unsigned - Remove duplicate GPG setup from builder stages (now inherited from base) - Simplify runtime stages to use GPG key from base layer - Add transition strategy: try signed buildcache first, fall back to unsigned - This allows graceful handling of unsigned artifacts from before this PR Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com> --- containers/debian/Dockerfile | 31 +++++++++- containers/eic/Dockerfile | 111 ++++++++++------------------------- 2 files changed, 62 insertions(+), 80 deletions(-) diff --git a/containers/debian/Dockerfile b/containers/debian/Dockerfile index 29bf371f..1aab3393 100644 --- a/containers/debian/Dockerfile +++ b/containers/debian/Dockerfile @@ -258,13 +258,42 @@ spack compiler find --scope site spack config blame compilers EOF +## Setup GPG key for buildcache signing +## For production use, provide a persistent GPG key via SPACK_SIGNING_KEY secret to ensure +## buildcaches can be verified across builds. Without this secret, a temporary key is created +## which works for single-build scenarios but prevents buildcache reuse across builds. +RUN --mount=type=secret,id=SPACK_SIGNING_KEY,target=/tmp/spack-signing-key \ + < ${SPACK_ROOT}/spack-public-key.pub +# List keys to verify setup +spack gpg list +EOF + ## Setup buildcache mirrors ## - this always adds the read-only mirror to the container ## - the write-enabled mirror is provided later as a secret mount +## - ghcr mirror is configured as signed, but installation will attempt both signed and unsigned +## during transition period (before all artifacts are signed) RUN --mount=type=cache,target=/var/cache/spack < /tmp/spack-public-key.pub EOF @@ -157,19 +130,17 @@ ENV SPACK_COLOR="always" COPY --from=builder_installation_default \ /opt/spack-environment/${ENV}/spack.* \ /opt/spack-environment/${ENV}/ -COPY --from=builder_installation_default \ - /tmp/spack-public-key.pub \ - /tmp/spack-public-key.pub -# Import public key for buildcache verification +# Public key is already available from base layer at ${SPACK_ROOT}/spack-public-key.pub +# Trust it in case the base layer GPG was created with a temporary key RUN <&1 | tee /tmp/install.log; then + # Check if the failure was due to signature verification + if grep -q "signature" /tmp/install.log || grep -q "gpg" /tmp/install.log; then + echo "Signature verification failed, retrying with --no-check-signature for transition compatibility" + spack ${SPACK_FLAGS} install ${SPACK_INSTALL_FLAGS} --no-check-signature --use-buildcache only + else + # Different error, fail the build + cat /tmp/install.log + exit 1 + fi +fi spack gc --yes-to-all go go-bootstrap rust rust-bootstrap py-setuptools-rust py-maturin EOF @@ -253,33 +235,6 @@ ARG TARGETPLATFORM # Open Container Initiative labels LABEL org.opencontainers.image.title="Electron-Ion Collider build installation image (custom configuration, $TARGETPLATFORM)" -# Set up GPG key for buildcache signing -# Note: This setup is duplicated from builder_installation_default due to Dockerfile limitations. -# Both builder stages need independent GPG setup as they don't share a common parent with GPG configured. -# For production use, provide a persistent GPG key via SPACK_SIGNING_KEY secret to ensure -# buildcaches can be verified across builds. Without this secret, a temporary key is created -# for each build, which works for single-build scenarios but prevents buildcache reuse. -RUN --mount=type=secret,id=SPACK_SIGNING_KEY,target=/tmp/spack-signing-key \ - < /tmp/spack-public-key.pub EOF @@ -323,29 +276,29 @@ COPY --from=builder_installation_custom \ COPY --from=builder_installation_custom \ /opt/spack-environment/packages.yaml \ /opt/spack-environment/ -COPY --from=builder_installation_custom \ - /tmp/spack-public-key.pub \ - /tmp/spack-public-key-custom.pub - -# Import public key from custom builder for buildcache verification -# Note: This stage is based on runtime_default (FROM runtime_default AS runtime_custom at line 300), -# which already has the public key from builder_installation_default imported. -# However, we also import the key from builder_installation_custom to handle the case -# where temporary keys are used (each builder stage creates a different temporary key). -# With a persistent SPACK_SIGNING_KEY secret, both builders use the same key, making this redundant but harmless. -RUN <&1 | tee /tmp/install.log; then + # Check if the failure was due to signature verification + if grep -q "signature" /tmp/install.log || grep -q "gpg" /tmp/install.log; then + echo "Signature verification failed, retrying with --no-check-signature for transition compatibility" + spack ${SPACK_FLAGS} install ${SPACK_INSTALL_FLAGS} --no-check-signature --use-buildcache only + else + # Different error, fail the build + cat /tmp/install.log + exit 1 + fi +fi spack gc --yes-to-all go go-bootstrap rust rust-bootstrap py-setuptools-rust py-maturin EOF From fb96ddc6be4102ed5ea4226c4a89a3d4482f981f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 30 Dec 2025 14:34:00 +0000 Subject: [PATCH 07/10] refactor: improve transition strategy error detection - Use more specific regex pattern for signature verification errors - Look for NoVerifyException and other Spack-specific error patterns - Add clearer log messages explaining the fallback behavior - Reduce false positives from generic keyword matches - Apply consistent logic to both runtime stages Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com> --- containers/eic/Dockerfile | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile index 5de74dfe..ae6f94c1 100644 --- a/containers/eic/Dockerfile +++ b/containers/eic/Dockerfile @@ -141,6 +141,7 @@ EOF # Installation (default environment, from buildcache) # Transition strategy: Try with signature verification first, fall back to unsigned if needed +# This allows graceful migration from unsigned buildcaches (pre-signing) to signed ones RUN --mount=type=cache,target=/var/cache/spack \ --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \ --mount=type=secret,id=CI_REGISTRY_USER,env=CI_REGISTRY_USER \ @@ -151,12 +152,15 @@ RUN --mount=type=cache,target=/var/cache/spack \ set -e # Try to install with signature verification (signed buildcaches) if ! spack ${SPACK_FLAGS} install ${SPACK_INSTALL_FLAGS} --use-buildcache only 2>&1 | tee /tmp/install.log; then - # Check if the failure was due to signature verification - if grep -q "signature" /tmp/install.log || grep -q "gpg" /tmp/install.log; then - echo "Signature verification failed, retrying with --no-check-signature for transition compatibility" + # Check if the failure was due to signature verification issues + # Look for specific Spack error patterns related to signatures and GPG + if grep -qE "(NoVerifyException|signature.*verif|gpg.*trust|gpg.*key|buildcache.*sign)" /tmp/install.log; then + echo "==> Signature verification issue detected, retrying with --no-check-signature for transition compatibility" + echo "==> This fallback allows installation of unsigned buildcaches created before signature enforcement" spack ${SPACK_FLAGS} install ${SPACK_INSTALL_FLAGS} --no-check-signature --use-buildcache only else - # Different error, fail the build + # Different error (not signature-related), fail the build with full log + echo "==> Installation failed with non-signature error:" cat /tmp/install.log exit 1 fi @@ -282,6 +286,7 @@ COPY --from=builder_installation_custom \ # Installation (custom environment, from buildcache) # Transition strategy: Try with signature verification first, fall back to unsigned if needed +# This allows graceful migration from unsigned buildcaches (pre-signing) to signed ones RUN --mount=type=cache,target=/var/cache/spack \ --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \ <&1 | tee /tmp/install.log; then - # Check if the failure was due to signature verification - if grep -q "signature" /tmp/install.log || grep -q "gpg" /tmp/install.log; then - echo "Signature verification failed, retrying with --no-check-signature for transition compatibility" + # Check if the failure was due to signature verification issues + # Look for specific Spack error patterns related to signatures and GPG + if grep -qE "(NoVerifyException|signature.*verif|gpg.*trust|gpg.*key|buildcache.*sign)" /tmp/install.log; then + echo "==> Signature verification issue detected, retrying with --no-check-signature for transition compatibility" + echo "==> This fallback allows installation of unsigned buildcaches created before signature enforcement" spack ${SPACK_FLAGS} install ${SPACK_INSTALL_FLAGS} --no-check-signature --use-buildcache only else - # Different error, fail the build + # Different error (not signature-related), fail the build with full log + echo "==> Installation failed with non-signature error:" cat /tmp/install.log exit 1 fi From 96baf51130cb78bc5bbce4e010121abcfdfc9ded Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 30 Dec 2025 14:36:27 +0000 Subject: [PATCH 08/10] refactor: add error handling and improve documentation - Add GPG public key existence check before trusting - Document regex pattern duplication and need to keep in sync - Expand mirror configuration comments with deployment guidance - Clarify transition strategy and safe deploy order Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com> --- containers/debian/Dockerfile | 7 +++++-- containers/eic/Dockerfile | 9 ++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/containers/debian/Dockerfile b/containers/debian/Dockerfile index 1aab3393..f6438c31 100644 --- a/containers/debian/Dockerfile +++ b/containers/debian/Dockerfile @@ -288,8 +288,11 @@ EOF ## Setup buildcache mirrors ## - this always adds the read-only mirror to the container ## - the write-enabled mirror is provided later as a secret mount -## - ghcr mirror is configured as signed, but installation will attempt both signed and unsigned -## during transition period (before all artifacts are signed) +## - ghcr mirror is configured as --signed (changed from --unsigned) +## The transition strategy in runtime stages (eic/Dockerfile) handles both signed and unsigned +## artifacts during migration period, attempting signed first and falling back to unsigned +## - New buildcaches will be automatically signed via autopush (mirrors.yaml.in has signed: true) +## - Deploy order: This change is safe as long as runtime stages have transition logic in place RUN --mount=type=cache,target=/var/cache/spack < Date: Tue, 30 Dec 2025 14:45:10 +0000 Subject: [PATCH 09/10] docs: add comprehensive security evaluation and warnings - Document security limitations of embedding private keys in public containers - Explain that signatures provide integrity but not authenticity verification - Add security notice in Dockerfile about key extraction risk - Provide recommendations for production-grade security improvements - Clarify current implementation is suitable for transition/internal use - Suggest alternative approaches (CI-based signing, OCI-native signing) Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com> --- containers/debian/Dockerfile | 7 ++++ docs/BUILDCACHE_SIGNING.md | 62 ++++++++++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/containers/debian/Dockerfile b/containers/debian/Dockerfile index f6438c31..1a0801c0 100644 --- a/containers/debian/Dockerfile +++ b/containers/debian/Dockerfile @@ -262,6 +262,13 @@ EOF ## For production use, provide a persistent GPG key via SPACK_SIGNING_KEY secret to ensure ## buildcaches can be verified across builds. Without this secret, a temporary key is created ## which works for single-build scenarios but prevents buildcache reuse across builds. +## +## SECURITY NOTICE: The private key (whether temporary or imported) is embedded in this +## public container layer and can be extracted by anyone. This provides integrity verification +## but NOT authenticity (anyone can sign packages with the extracted key). For production +## environments requiring strong security, consider signing buildcaches in CI/CD before +## pushing to registry, and only embed the public key in the container. +## See docs/BUILDCACHE_SIGNING.md for detailed security considerations. RUN --mount=type=secret,id=SPACK_SIGNING_KEY,target=/tmp/spack-signing-key \ < Date: Tue, 30 Dec 2025 15:02:19 +0000 Subject: [PATCH 10/10] security: require secret key and never store in container layers - Make SPACK_SIGNING_KEY secret mandatory (no fallback to temporary keys) - Use separate GNUPGHOME to avoid persisting private key in GPG keyring - Mount private key as secret during all build steps that need signing - Only embed public key in container layers for verification - Private key exists only during RUN execution, never in final layers - Update documentation to reflect secure key management model - Remove security warnings about key exposure (no longer applicable) Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com> --- containers/debian/Dockerfile | 64 ++++++++------ containers/eic/Dockerfile | 30 +++++++ docs/BUILDCACHE_SIGNING.md | 157 +++++++++++++++++++---------------- 3 files changed, 156 insertions(+), 95 deletions(-) diff --git a/containers/debian/Dockerfile b/containers/debian/Dockerfile index 1a0801c0..e1d8c1a2 100644 --- a/containers/debian/Dockerfile +++ b/containers/debian/Dockerfile @@ -258,37 +258,53 @@ spack compiler find --scope site spack config blame compilers EOF -## Setup GPG key for buildcache signing -## For production use, provide a persistent GPG key via SPACK_SIGNING_KEY secret to ensure -## buildcaches can be verified across builds. Without this secret, a temporary key is created -## which works for single-build scenarios but prevents buildcache reuse across builds. +## Setup GPG public key for buildcache verification +## REQUIRED: A persistent GPG key MUST be provided via SPACK_SIGNING_KEY secret. +## Only the PUBLIC key is stored in container layers for verification. +## The PRIVATE key is mounted as a secret during build steps that require signing. ## -## SECURITY NOTICE: The private key (whether temporary or imported) is embedded in this -## public container layer and can be extracted by anyone. This provides integrity verification -## but NOT authenticity (anyone can sign packages with the extracted key). For production -## environments requiring strong security, consider signing buildcaches in CI/CD before -## pushing to registry, and only embed the public key in the container. -## See docs/BUILDCACHE_SIGNING.md for detailed security considerations. -RUN --mount=type=secret,id=SPACK_SIGNING_KEY,target=/tmp/spack-signing-key \ +## SECURITY MODEL: +## - Private key: Stored in GitHub Secrets, mounted during builds, NEVER in container layers +## - Public key: Embedded in container at ${SPACK_ROOT}/spack-public-key.pub for verification +## - Signing: Happens during builder install steps, using --mount=type=secret for private key +## - Verification: Happens during runtime install steps, using embedded public key +RUN --mount=type=secret,id=SPACK_SIGNING_KEY,target=/tmp/spack-signing-key,required=true \ <" + echo "See docs/BUILDCACHE_SIGNING.md for instructions on generating a key" + exit 1 +fi + # Initialize Spack's GPG environment spack gpg init -# Import signing key if provided, otherwise create a new one -if [ -f /tmp/spack-signing-key ] && [ -s /tmp/spack-signing-key ]; then - # Import the GPG key (expects ASCII-armored key with both public and private parts) - # First import into GPG, then Spack will use it from there - cat /tmp/spack-signing-key | gpg --batch --import - # Also trust the public key in Spack's keyring for verification - cat /tmp/spack-signing-key | gpg --batch --export | spack gpg trust + +# Extract and store ONLY the public key (not the private key) +# Import to temporary GPG home to extract public key +export GNUPGHOME=/tmp/gnupg-temp +mkdir -p ${GNUPGHOME} +chmod 700 ${GNUPGHOME} +gpg --batch --import /tmp/spack-signing-key +# Export only the public key +gpg --batch --export > ${SPACK_ROOT}/spack-public-key.pub +# Import public key into Spack's keyring for verification +spack gpg trust ${SPACK_ROOT}/spack-public-key.pub +# Clean up temporary GPG home (private key never persisted in container) +rm -rf ${GNUPGHOME} +unset GNUPGHOME + +# Verify we have the public key +if [ -f ${SPACK_ROOT}/spack-public-key.pub ]; then + echo "Public key successfully exported to ${SPACK_ROOT}/spack-public-key.pub" + echo "Private key was NOT stored in container - will be mounted during build steps" else - # Create a temporary GPG key for signing (non-interactive) - # Note: For production, a persistent key should be provided via SPACK_SIGNING_KEY secret - spack gpg create "EIC Containers" eic-containers@github.com + echo "ERROR: Failed to export public key" + exit 1 fi -# Export public key to SPACK_ROOT for use by derived images -spack gpg export > ${SPACK_ROOT}/spack-public-key.pub -# List keys to verify setup + spack gpg list EOF diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile index 32e51cea..e987b172 100644 --- a/containers/eic/Dockerfile +++ b/containers/eic/Dockerfile @@ -90,6 +90,7 @@ ARG TARGETPLATFORM LABEL org.opencontainers.image.title="Electron-Ion Collider build installation image (default configuration, $TARGETPLATFORM)" # Installation (default environment) +# Mount SPACK_SIGNING_KEY secret for signing during autopush RUN --mount=type=cache,target=/ccache,id=ccache-${TARGETPLATFORM} \ --mount=type=cache,target=/var/cache/spack \ --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \ @@ -97,15 +98,29 @@ RUN --mount=type=cache,target=/ccache,id=ccache-${TARGETPLATFORM} \ --mount=type=secret,id=CI_REGISTRY_PASSWORD,env=CI_REGISTRY_PASSWORD \ --mount=type=secret,id=GITHUB_REGISTRY_USER,env=GITHUB_REGISTRY_USER \ --mount=type=secret,id=GITHUB_REGISTRY_TOKEN,env=GITHUB_REGISTRY_TOKEN \ + --mount=type=secret,id=SPACK_SIGNING_KEY,target=/tmp/spack-signing-key,required=true \ <