build-sys: Rework sealing to be one build step

.github/workflows/ci.yml

@@ -198,6 +198,10 @@ jobs:
       - name: Unit and container integration tests
         run: just test-container
 
+      - name: Validate composefs digest (sealed UKI only)
+        if: matrix.variant == 'composefs-sealeduki-sdboot'
+        run: just validate-composefs-digest
+
       - name: Run TMT integration tests
         run: |
           if [ "${{ matrix.variant }}" = "composefs-sealeduki-sdboot" ]; then

CONTRIBUTING.md

@@ -41,11 +41,20 @@ accepted!
 Worth stating: before you start diving into the code you should understand using
 the system as a user and how it works.  See the user documentation for that.
 
-## Understanding the Justfile
+## The Justfile
+
+The [Justfile](Justfile) is the primary interface for building and testing bootc.
+
+```bash
+just --list           # Show all targets organized by group
+just list-variants    # Show available build variants and current config
+```
+
+### Building from source
 
 Edit the source code; a simple thing to do is add e.g.
-`eprintln!("hello world");` into `run_from_opt` in [crates/lib/src/cli.rs](cli.rs).
-You can run `make` or `cargo build` to build that locally.  However, a key
+`eprintln!("hello world");` into `run_from_opt` in [crates/lib/src/cli.rs](crates/lib/src/cli.rs).
+You can run `make` or `cargo build` to build that locally. However, a key
 next step is to get that binary into a bootc container image.
 
 Running `just` defaults to `just build` which will build a container
@@ -58,14 +67,24 @@ and try running `bootc`.
 
 ### Running container-oriented integration tests
 
-`just test-container`
+```bash
+just test-container
+```
 
 ### Running (TMT) integration tests
 
 A common cycle here is you'll edit e.g. `deploy.rs` and want to run the
 tests that perform an upgrade:
 
-`just test-tmt-one test-20-local-upgrade`
+```bash
+just test-tmt local-upgrade-reboot
+```
+
+To run a specific test:
+
+```bash
+just test-tmt readonly
+```
 
 ### Faster iteration cycles
 
@@ -96,6 +115,26 @@ then you can `cargo b --release` directly in a Fedora 42 container
 or even on your host system, and then directly run e.g. `./target/release/bootc upgrade`
 etc.
 
+### Testing with composefs (sealed images)
+
+To build and test with the experimental composefs backend:
+
+```bash
+# Build a sealed image with auto-generated test Secure Boot keys
+just build-sealed
+
+# Run composefs-specific tests
+just test-composefs
+
+# Validate that composefs digests match between build and install views
+# (useful for debugging mtime/metadata issues)
+just validate-composefs-digest
+```
+
+The `build-sealed` target generates test Secure Boot keys in `target/test-secureboot/`
+and builds a complete sealed image with UKI. See [experimental-composefs.md](docs/src/experimental-composefs.md)
+for more information on sealed images.
+
 
 ### Debugging via lldb
 

Cargo.toml

@@ -42,9 +42,17 @@ clap = "4.5.4"
 clap_mangen = { version = "0.2.20" }
 # Reviewers (including AI tools): The composefs-rs git revision is duplicated for each crate.
 # If adding/removing crates here, also update docs/Dockerfile.mdbook and docs/src/internals.md.
-composefs = { git = "https://github.com/containers/composefs-rs", rev = "e9008489375044022e90d26656960725a76f4620", package = "composefs", features = ["rhel9"] }
-composefs-boot = { git = "https://github.com/containers/composefs-rs", rev = "e9008489375044022e90d26656960725a76f4620", package = "composefs-boot" }
-composefs-oci = { git = "https://github.com/containers/composefs-rs", rev = "e9008489375044022e90d26656960725a76f4620", package = "composefs-oci" }
+#
+# To develop against a local composefs-rs checkout:
+# 1. Set BOOTC_extra_src to your composefs-rs path when building:
+#      BOOTC_extra_src=$HOME/src/composefs-rs just build
+# 2. Comment out the git refs below and uncomment the path refs:
+composefs = { git = "https://github.com/containers/composefs-rs", rev = "4a060161e0122bd2727e639437c61e05ecc7cab3", package = "composefs", features = ["rhel9"] }
+composefs-boot = { git = "https://github.com/containers/composefs-rs", rev = "4a060161e0122bd2727e639437c61e05ecc7cab3", package = "composefs-boot" }
+composefs-oci = { git = "https://github.com/containers/composefs-rs", rev = "4a060161e0122bd2727e639437c61e05ecc7cab3", package = "composefs-oci" }
+# composefs = { path = "/run/extra-src/crates/composefs", package = "composefs", features = ["rhel9"] }
+# composefs-boot = { path = "/run/extra-src/crates/composefs-boot", package = "composefs-boot" }
+# composefs-oci = { path = "/run/extra-src/crates/composefs-oci", package = "composefs-oci" }
 fn-error-context = "0.2.1"
 hex = "0.4.3"
 indicatif = "0.18.0"

Dockerfile

@@ -23,6 +23,7 @@ FROM $base as buildroot
 ARG initramfs=1
 # This installs our buildroot, and we want to cache it independently of the rest.
 # Basically we don't want changing a .rs file to blow out the cache of packages.
+# Use tmpfs,target=/run with bind mounts inside to avoid leaking mount stubs into the image
 RUN --mount=type=tmpfs,target=/run \
     --mount=type=bind,from=packaging,src=/,target=/run/packaging \
     /run/packaging/install-buildroot
@@ -32,12 +33,9 @@ WORKDIR /src
 # See https://www.reddit.com/r/rust/comments/126xeyx/exploring_the_problem_of_faster_cargo_docker/
 # We aren't using the full recommendations there, just the simple bits.
 # First we download all of our Rust dependencies
+# Note: /run/extra-src is optionally bind-mounted via BOOTC_extra_src for local composefs-rs development
 RUN --mount=type=tmpfs,target=/run --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome cargo fetch
 
-FROM buildroot as sdboot-content
-# Writes to /out
-RUN --mount=type=tmpfs,target=/run /src/contrib/packaging/configure-systemdboot download
-
 # We always do a "from scratch" build
 # https://docs.fedoraproject.org/en-US/bootc/building-from-scratch/
 # because this fixes https://github.com/containers/composefs-rs/issues/132
@@ -47,6 +45,11 @@ RUN --mount=type=tmpfs,target=/run /src/contrib/packaging/configure-systemdboot
 #       local sources. We'll override it later.
 # NOTE: All your base belong to me.
 FROM $base as target-base
+# Handle version skew between base image and mirrors for CentOS Stream
+# xref https://gitlab.com/redhat/centos-stream/containers/bootc/-/issues/1174
+RUN --mount=type=tmpfs,target=/run \
+    --mount=type=bind,from=packaging,src=/,target=/run/packaging \
+    /run/packaging/enable-compose-repos
 RUN --mount=type=tmpfs,target=/run /usr/libexec/bootc-base-imagectl build-rootfs --manifest=standard /target-rootfs
 
 FROM scratch as base
@@ -69,6 +72,13 @@ ENV container=oci
 STOPSIGNAL SIGRTMIN+3
 CMD ["/sbin/init"]
 
+# This layer contains things which aren't in the default image and may
+# be used for sealing images in particular.
+FROM base as tools
+RUN --mount=type=tmpfs,target=/run \
+    --mount=type=bind,from=packaging,src=/,target=/run/packaging \
+    /run/packaging/initialize-sealing-tools
+
 # -------------
 # external dependency cutoff point:
 # NOTE: Every RUN instruction past this point should use `--network=none`; we want to ensure
@@ -85,14 +95,35 @@ ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
 # Build RPM directly from source, using cached target directory
 RUN --network=none --mount=type=tmpfs,target=/run --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome RPM_VERSION="${pkgversion}" /src/contrib/packaging/build-rpm
 
-FROM buildroot as sdboot-signed
+# This image signs systemd-boot using our key, and writes the resulting binary into /out
+FROM tools as sdboot-signed
 # The secureboot key and cert are passed via Justfile
 # We write the signed binary into /out
+# Note: /out already contains systemd-boot-unsigned RPM from initialize-sealing-tools
 RUN --network=none --mount=type=tmpfs,target=/run \
-    --mount=type=bind,from=sdboot-content,src=/,target=/run/sdboot-package \
     --mount=type=secret,id=secureboot_key \
-    --mount=type=secret,id=secureboot_cert \
-    /src/contrib/packaging/configure-systemdboot sign
+    --mount=type=secret,id=secureboot_cert <<EORUN
+set -xeuo pipefail
+
+# Extract the unsigned systemd-boot binary from the downloaded RPM
+cd /tmp
+rpm2cpio /out/*.rpm | cpio -idmv
+# Find the extracted unsigned binary
+sdboot_unsigned=$(ls ./usr/lib/systemd/boot/efi/systemd-boot*.efi)
+sdboot_bn=$(basename ${sdboot_unsigned})
+# Sign with sbsign using db certificate and key
+sbsign --key /run/secrets/secureboot_key \
+   --cert /run/secrets/secureboot_cert \
+   --output /out/${sdboot_bn} \
+   ${sdboot_unsigned}
+ls -al /out/${sdboot_bn}
+EORUN
+
+# ----
+# Unit and integration tests
+# The section here (up until the last `FROM` line which acts as the default target)
+# is non-default images for unit and source code validation.
+# ----
 
 # This "build" includes our unit tests
 FROM build as units
@@ -105,28 +136,66 @@ RUN --network=none --mount=type=tmpfs,target=/run --mount=type=cache,target=/src
 FROM buildroot as validate
 RUN --network=none --mount=type=tmpfs,target=/run --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome make validate
 
-# Common base for final images: configures variant, rootfs, and injects extra content
-FROM base as final-common
+# ----
+# Stages for the final image
+# ----
+
+# Perform all filesystem transformations except generating the sealed UKI (if configured)
+FROM base as base-penultimate
 ARG variant
+# Switch to a signed systemd-boot, if configured
 RUN --network=none --mount=type=tmpfs,target=/run \
     --mount=type=bind,from=packaging,src=/,target=/run/packaging \
-    --mount=type=bind,from=sdboot-content,src=/,target=/run/sdboot-content \
-    --mount=type=bind,from=sdboot-signed,src=/,target=/run/sdboot-signed \
-    /run/packaging/configure-variant "${variant}"
+    --mount=type=bind,from=sdboot-signed,src=/,target=/run/sdboot-signed <<EORUN
+set -xeuo pipefail
+if test "${variant}" = "composefs-sealeduki-sdboot"; then
+  /run/packaging/switch-to-sdboot /run/sdboot-signed
+fi
+EORUN
+# Configure the rootfs
 ARG rootfs=""
 RUN --network=none --mount=type=tmpfs,target=/run \
     --mount=type=bind,from=packaging,src=/,target=/run/packaging \
     /run/packaging/configure-rootfs "${variant}" "${rootfs}"
-COPY --from=packaging /usr-extras/ /usr/
-
-# Final target: installs pre-built packages from the 'packages' build context.
-# Use with: podman build --target=final --build-context packages=path/to/packages
-# We use --build-context instead of -v to avoid volume mount stubs leaking into /run.
-FROM final-common as final
+# Override with our built package
 RUN --network=none --mount=type=tmpfs,target=/run \
     --mount=type=bind,from=packaging,src=/,target=/run/packaging \
     --mount=type=bind,from=packages,src=/,target=/run/packages \
     /run/packaging/install-rpm-and-setup /run/packages
-# Use tmpfs on /run to hide any content created by podman for DNS resolution
-# (e.g., /run/systemd/resolve/stub-resolv.conf on Ubuntu hosts)
+# Inject some other configuration
+COPY --from=packaging /usr-extras/ /usr/
+
+# Generate the sealed UKI in a separate stage
+# This computes the composefs digest from base-penultimate and creates a signed UKI
+# We need our newly-built bootc for the compute-composefs-digest command
+FROM tools as sealed-uki
+ARG variant
+# Install our bootc package (only needed for the compute-composefs-digest command)
+RUN --network=none --mount=type=tmpfs,target=/run \
+    --mount=type=bind,from=packages,src=/,target=/run/packages \
+    rpm -Uvh --oldpackage /run/packages/bootc-*.rpm
+RUN --network=none --mount=type=tmpfs,target=/run \
+    --mount=type=secret,id=secureboot_key \
+    --mount=type=secret,id=secureboot_cert \
+    --mount=type=bind,from=packaging,src=/,target=/run/packaging \
+    --mount=type=bind,from=base-penultimate,src=/,target=/run/target <<EORUN
+set -xeuo pipefail
+if test "${variant}" = "composefs-sealeduki-sdboot"; then
+  /run/packaging/seal-uki /run/target /out /run/secrets
+fi
+EORUN
+
+# And now the final image
+FROM base-penultimate
+ARG variant
+# Copy the sealed UKI and finalize the image (remove raw kernel, create symlinks)
+RUN --network=none --mount=type=tmpfs,target=/run \
+    --mount=type=bind,from=packaging,src=/,target=/run/packaging \
+    --mount=type=bind,from=sealed-uki,src=/,target=/run/sealed-uki <<EORUN
+set -xeuo pipefail
+if test "${variant}" = "composefs-sealeduki-sdboot"; then
+  /run/packaging/finalize-uki /run/sealed-uki/out
+fi
+EORUN
+# And finally, test our linting
 RUN --network=none --mount=type=tmpfs,target=/run bootc container lint --fatal-warnings