From aa38036cea6e37351d78a7e6cb5014218d015668 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Mon, 19 Jan 2026 11:49:11 -0500 Subject: [PATCH 1/3] podstorage: Fallback to running system's auth.json This fixes a bug where running bootc upgrade/switch to an image that does not have an auth.json results in failing to pull new LBIs that are stored in an authenticated registry. Signed-off-by: ckyrouac --- crates/lib/src/podstorage.rs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/crates/lib/src/podstorage.rs b/crates/lib/src/podstorage.rs index 74880a7c5..60885306b 100644 --- a/crates/lib/src/podstorage.rs +++ b/crates/lib/src/podstorage.rs @@ -119,6 +119,15 @@ fn bind_storage_roots(cmd: &mut Command, storage_root: &Dir, run_root: &Dir) -> Ok(()) } +/// Get the global authfile from the host root filesystem. +/// This is used as a fallback when the authfile is not found in the sysroot, +/// such as during upgrades where the new image may not have auth.json +/// but the running system does. +fn get_host_authfile() -> Result> { + let host_root = Dir::open_ambient_dir("/", cap_std::ambient_authority())?; + ostree_ext::globals::get_global_authfile(&host_root) +} + // Initialize a `podman` subprocess with: // - storage overridden to point to to storage_root // - Authentication (auth.json) using the bootc/ostree owned auth @@ -133,10 +142,22 @@ fn new_podman_cmd_in(sysroot: &Dir, storage_root: &Dir, run_root: &Dir) -> Resul // Keep this in sync with https://github.com/bootc-dev/containers-image-proxy-rs/blob/b5e0861ad5065f47eaf9cda0d48da3529cc1bc43/src/imageproxy.rs#L310 // We always override the auth to match the bootc setup. - let authfile_fd = ostree_ext::globals::get_global_authfile(sysroot)?.map(|v| v.1); - if let Some(mut fd) = authfile_fd { + // First try to get the authfile from the sysroot (e.g., upgrade image), and if not found, + // fall back to the host root. This handles the case where during an upgrade, the new image + // may not have auth.json but the running system does. + let authfile = if let Some((path, file)) = ostree_ext::globals::get_global_authfile(sysroot)? { + tracing::debug!("Using authfile from sysroot: {path}"); + Some(file) + } else if let Some((path, file)) = get_host_authfile()? { + tracing::debug!("Using authfile from host root: {path}"); + Some(file) + } else { + None + }; + if let Some(mut fd) = authfile { std::io::copy(&mut fd, &mut tempfile)?; } else { + tracing::debug!("No authfile found, using empty auth"); // Note that if there's no bootc-owned auth, then we force an empty authfile to ensure // that podman doesn't fall back to searching the user-owned paths. tempfile.write_all(b"{}")?; From 4f83c2c407d21a43ba4fe02d6eba5b64f6326db5 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Mon, 19 Jan 2026 12:01:01 -0500 Subject: [PATCH 2/3] tests: Add cstor-dist registry infrastructure to bootc_testlib.nu Add reusable functions for running cstor-dist, a tool that serves images from containers-storage via an authenticated OCI registry endpoint. This enables testing authenticated logically bound images. The following functions are exported: - start_cstor_dist: Starts cstor-dist container with basic auth - get_cstor_auth: Returns registry address and base64-encoded credentials - setup_insecure_registry: Configures registries.conf.d for non-TLS access - setup_system_auth: Sets up /run/ostree/auth.json with credentials Also update test-logically-bound-switch.nu to: - Import get_cstor_auth from bootc_testlib.nu - Add --with-auth flag to build_image for baking auth.json into images - Fix verify_images to check if image name is IN the Names array (not exact match) to support images with multiple tags Assisted-by: Claude Code (claude-opus-4-5@20251101) Signed-off-by: ckyrouac --- tmt/tests/booted/bootc_testlib.nu | 85 +++++++++++++++++++ .../booted/test-logically-bound-switch.nu | 36 +++++--- 2 files changed, 107 insertions(+), 14 deletions(-) diff --git a/tmt/tests/booted/bootc_testlib.nu b/tmt/tests/booted/bootc_testlib.nu index f3d1fa013..dc041d6c8 100644 --- a/tmt/tests/booted/bootc_testlib.nu +++ b/tmt/tests/booted/bootc_testlib.nu @@ -23,3 +23,88 @@ export def have_hostexports [] { export def parse_cmdline [] { open /proc/cmdline | str trim | split row " " } + +# cstor-dist configuration for authenticated registry testing +# cstor-dist serves images from containers-storage via an authenticated OCI registry endpoint +# https://github.com/ckyrouac/cstor-dist +const CSTOR_DIST_IMAGE = "ghcr.io/ckyrouac/cstor-dist:latest" +const CSTOR_DIST_USER = "testuser" +const CSTOR_DIST_PASS = "testpass" +const CSTOR_DIST_PORT = 8000 + +# The registry address for cstor-dist +export const CSTOR_DIST_REGISTRY = $"localhost:($CSTOR_DIST_PORT)" + +# Start cstor-dist with basic auth on localhost +# Fails if cstor-dist cannot be started +export def start_cstor_dist [] { + print "Starting cstor-dist with basic auth..." + + # Pull test images that cstor-dist will serve + print "Pulling test images for cstor-dist to serve..." + podman pull docker.io/library/alpine:latest + podman pull docker.io/library/busybox:latest + + # Run cstor-dist container with auth enabled + # Mount the local containers storage so cstor-dist can serve images from it + let storage_path = if ("/var/lib/containers/storage" | path exists) { + "/var/lib/containers/storage" + } else { + $"($env.HOME)/.local/share/containers/storage" + } + + (podman run --privileged --rm -d --name cstor-dist-auth + -p $"($CSTOR_DIST_PORT):8000" + -v $"($storage_path):/var/lib/containers/storage" + $CSTOR_DIST_IMAGE --username $CSTOR_DIST_USER --password $CSTOR_DIST_PASS) + + # Wait for cstor-dist to be ready by testing HTTP connection + # Loop for up to 20 seconds + print "Waiting for cstor-dist to be ready..." + let auth_header = $"($CSTOR_DIST_USER):($CSTOR_DIST_PASS)" | encode base64 + mut ready = false + for i in 1..20 { + let result = do { curl -sf -H $"Authorization: Basic ($auth_header)" $"http://($CSTOR_DIST_REGISTRY)/v2/" } | complete + if $result.exit_code == 0 { + $ready = true + break + } + print $"Attempt ($i)/20: cstor-dist not ready yet..." + sleep 1sec + } + + if not $ready { + # Show container logs for debugging + print "cstor-dist failed to start. Container logs:" + podman logs cstor-dist-auth + error make { msg: "cstor-dist failed to become ready within 20 seconds" } + } + + print $"cstor-dist running on ($CSTOR_DIST_REGISTRY)" +} + +# Get cstor-dist auth config +export def get_cstor_auth [] { + # Base64 encode the credentials for auth.json + let auth_b64 = $"($CSTOR_DIST_USER):($CSTOR_DIST_PASS)" | encode base64 + { + registry: $CSTOR_DIST_REGISTRY, + auth_b64: $auth_b64 + } +} + +# Configure insecure registry for cstor-dist (no TLS) +export def setup_insecure_registry [] { + mkdir /etc/containers/registries.conf.d + (echo $"[[registry]]\nlocation=\"($CSTOR_DIST_REGISTRY)\"\ninsecure=true" + | save -f /etc/containers/registries.conf.d/99-cstor-dist.conf) +} + +# Set up auth.json on the running system with cstor-dist credentials +export def setup_system_auth [] { + mkdir /run/ostree + let cstor = get_cstor_auth + print $"Setting up system auth for cstor-dist at ($cstor.registry)" + let auth_json = $'{"auths": {"($cstor.registry)": {"auth": "($cstor.auth_b64)"}}}' + echo $auth_json | save -f /run/ostree/auth.json +} diff --git a/tmt/tests/booted/test-logically-bound-switch.nu b/tmt/tests/booted/test-logically-bound-switch.nu index fbe69a9da..99115e27a 100644 --- a/tmt/tests/booted/test-logically-bound-switch.nu +++ b/tmt/tests/booted/test-logically-bound-switch.nu @@ -15,24 +15,21 @@ use std assert use tap.nu +use bootc_testlib.nu [get_cstor_auth] # This code runs on *each* boot. bootc status let st = bootc status --json | from json let booted = $st.status.booted.image -# The tests here aren't fetching from a registry which requires auth by default, -# but we can replicate the failure in https://github.com/bootc-dev/bootc/pull/1852 -# by just injecting any auth file. -echo '{}' | save -f /run/ostree/auth.json - def initial_setup [] { bootc image copy-to-storage podman images podman image inspect localhost/bootc | from json } -def build_image [name images containers] { +# Build an image with optional auth.json baked in +def build_image [name images containers --with-auth] { let td = mktemp -d cd $td mkdir usr/share/containers/systemd @@ -59,6 +56,16 @@ RUN echo sanity check > /usr/share/bound-image-sanity-check.txt } } + # Optionally bake auth.json into the image + if $with_auth { + let cstor = get_cstor_auth + print "Baking auth.json into the image" + mkdir etc/ostree + let auth_json = $'{"auths": {"($cstor.registry)": {"auth": "($cstor.auth_b64)"}}}' + echo $auth_json | save etc/ostree/auth.json + echo "COPY etc/ /etc/\n" | save Dockerfile --append + } + # Build it podman build -t $name . # Just sanity check it @@ -74,12 +81,13 @@ def verify_images [images containers] { let image_names = podman --storage-opt=additionalimagestore=/usr/lib/bootc/storage images --format json | from json | select -i Names for $image in $bound_images { - let found = $image_names | where Names == [$image.image] + # Check if the expected image name is IN the Names array (not exact match) + let found = $image_names | where { |row| $image.image in $row.Names } assert (($found | length) > 0) $"($image.image) not found" } for $container in $bound_containers { - let found = $image_names | where Names == [$container.image] + let found = $image_names | where { |row| $container.image in $row.Names } assert (($found | length) > 0) $"($container.image) not found" } } @@ -89,14 +97,14 @@ def first_boot [] { initial_setup - # build a bootc image that includes bound images + # Build a bootc image that includes bound images let images = [ { "bound": true, "image": "registry.access.redhat.com/ubi9/ubi-minimal:9.4", "name": "ubi-minimal" }, { "bound": false, "image": "quay.io/centos-bootc/centos-bootc:stream9", "name": "centos-bootc" } ] let containers = [{ - "bound": true, "image": "docker.io/library/alpine:latest", "name": "alpine" + "bound": true, "image": "docker.io/library/alpine:latest", "name": "alpine" }] let image_name = "localhost/bootc-bound" @@ -111,18 +119,18 @@ def second_boot [] { assert equal $booted.image.transport containers-storage assert equal $booted.image.image localhost/bootc-bound - # verify images are still there after boot + # Verify images are still there after boot let images = [ { "bound": true, "image": "registry.access.redhat.com/ubi9/ubi-minimal:9.4", "name": "ubi-minimal" }, { "bound": false, "image": "quay.io/centos-bootc/centos-bootc:stream9", "name": "centos-bootc" } ] let containers = [{ - "bound": true, "image": "docker.io/library/alpine:latest", "name": "alpine" + "bound": true, "image": "docker.io/library/alpine:latest", "name": "alpine" }] verify_images $images $containers - # build a new bootc image with an additional bound image + # Build a new bootc image with an additional bound image print "bootc upgrade with another bound image" let image_name = "localhost/bootc-bound" let more_images = $images | append [{ "bound": true, "image": "registry.access.redhat.com/ubi9/ubi-minimal:9.3", "name": "ubi-minimal-9-3" }] @@ -144,7 +152,7 @@ def third_boot [] { ] let containers = [{ - "bound": true, "image": "docker.io/library/alpine:latest", "name": "alpine" + "bound": true, "image": "docker.io/library/alpine:latest", "name": "alpine" }] verify_images $images $containers From 66fa1c28d2e1f1635666fc0206939df53a80d5d1 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Mon, 19 Jan 2026 12:02:18 -0500 Subject: [PATCH 3/3] tests: Add authenticated LBI testing via cstor-dist Extend the logically bound images switch test to verify that authenticated bound images work correctly in two scenarios: 1. Auth credentials baked into the image itself (/etc/ostree/auth.json) 2. Auth credentials on the running system (/run/ostree/auth.json) The test uses cstor-dist to serve images from containers-storage via an authenticated OCI registry endpoint. On first boot, an image with auth baked in is switched to. On second boot, the test upgrades to a new image WITHOUT baked-in auth, forcing the use of the fallback system auth file. This validates the auth fallback fix. Test images: - cstor-alpine: authenticated LBI pulled on first switch - cstor-busybox: authenticated LBI added on upgrade (tests fallback) Assisted-by: Claude Code (claude-opus-4-5@20251101) Signed-off-by: ckyrouac --- .../booted/test-logically-bound-switch.nu | 55 ++++++++++++++++--- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/tmt/tests/booted/test-logically-bound-switch.nu b/tmt/tests/booted/test-logically-bound-switch.nu index 99115e27a..feda79c8b 100644 --- a/tmt/tests/booted/test-logically-bound-switch.nu +++ b/tmt/tests/booted/test-logically-bound-switch.nu @@ -12,10 +12,17 @@ # # # +# +# This test also verifies that authenticated LBIs work in two scenarios: +# 1. Auth credentials stored in the image itself +# 2. Auth credentials stored on the running system +# +# The test uses cstor-dist to serve images from containers-storage via an +# authenticated OCI registry endpoint. use std assert use tap.nu -use bootc_testlib.nu [get_cstor_auth] +use bootc_testlib.nu [CSTOR_DIST_REGISTRY, start_cstor_dist, get_cstor_auth, setup_insecure_registry, setup_system_auth] # This code runs on *each* boot. bootc status @@ -97,10 +104,20 @@ def first_boot [] { initial_setup + # Start cstor-dist for authenticated LBI testing + start_cstor_dist + setup_insecure_registry + + # Set up auth on running system for the switch operation + # The image will also have auth baked in - both should work + setup_system_auth + # Build a bootc image that includes bound images + # Include an authenticated LBI from cstor-dist with auth baked into the image let images = [ { "bound": true, "image": "registry.access.redhat.com/ubi9/ubi-minimal:9.4", "name": "ubi-minimal" }, - { "bound": false, "image": "quay.io/centos-bootc/centos-bootc:stream9", "name": "centos-bootc" } + { "bound": false, "image": "quay.io/centos-bootc/centos-bootc:stream9", "name": "centos-bootc" }, + { "bound": true, "image": $"($CSTOR_DIST_REGISTRY)/docker.io/library/alpine:latest", "name": "cstor-alpine" } ] let containers = [{ @@ -108,7 +125,8 @@ def first_boot [] { }] let image_name = "localhost/bootc-bound" - build_image $image_name $images $containers + print "Building image WITH auth.json baked in (tests auth from image)" + build_image $image_name $images $containers --with-auth bootc switch --transport containers-storage $image_name verify_images $images $containers tmt-reboot @@ -119,10 +137,19 @@ def second_boot [] { assert equal $booted.image.transport containers-storage assert equal $booted.image.image localhost/bootc-bound - # Verify images are still there after boot + # Start cstor-dist again (container doesn't survive reboot) + start_cstor_dist + setup_insecure_registry + + # Set up auth on the RUNNING SYSTEM for the upgrade + # The new image will NOT have auth baked in, so the fallback to system auth is needed + setup_system_auth + + # Verify images from first switch are still there let images = [ { "bound": true, "image": "registry.access.redhat.com/ubi9/ubi-minimal:9.4", "name": "ubi-minimal" }, - { "bound": false, "image": "quay.io/centos-bootc/centos-bootc:stream9", "name": "centos-bootc" } + { "bound": false, "image": "quay.io/centos-bootc/centos-bootc:stream9", "name": "centos-bootc" }, + { "bound": true, "image": $"($CSTOR_DIST_REGISTRY)/docker.io/library/alpine:latest", "name": "cstor-alpine" } ] let containers = [{ @@ -130,10 +157,19 @@ def second_boot [] { }] verify_images $images $containers - # Build a new bootc image with an additional bound image + # Build a NEW bootc image WITHOUT auth baked in + # Add a DIFFERENT authenticated LBI (busybox instead of alpine) + # This tests that auth from the running system works (the fallback fix) print "bootc upgrade with another bound image" let image_name = "localhost/bootc-bound" - let more_images = $images | append [{ "bound": true, "image": "registry.access.redhat.com/ubi9/ubi-minimal:9.3", "name": "ubi-minimal-9-3" }] + let more_images = [ + { "bound": true, "image": "registry.access.redhat.com/ubi9/ubi-minimal:9.4", "name": "ubi-minimal" }, + { "bound": true, "image": "registry.access.redhat.com/ubi9/ubi-minimal:9.3", "name": "ubi-minimal-9-3" }, + { "bound": false, "image": "quay.io/centos-bootc/centos-bootc:stream9", "name": "centos-bootc" }, + { "bound": true, "image": $"($CSTOR_DIST_REGISTRY)/docker.io/library/alpine:latest", "name": "cstor-alpine" }, + { "bound": true, "image": $"($CSTOR_DIST_REGISTRY)/docker.io/library/busybox:latest", "name": "cstor-busybox" } + ] + print "Building image WITHOUT auth.json (tests auth fallback from running system)" build_image $image_name $more_images $containers bootc upgrade verify_images $more_images $containers @@ -145,10 +181,13 @@ def third_boot [] { assert equal $booted.image.transport containers-storage assert equal $booted.image.image localhost/bootc-bound + # No need to start cstor-dist - we're just verifying the images are in storage let images = [ { "bound": true, "image": "registry.access.redhat.com/ubi9/ubi-minimal:9.4", "name": "ubi-minimal" }, { "bound": true, "image": "registry.access.redhat.com/ubi9/ubi-minimal:9.3", "name": "ubi-minimal-9-3" }, - { "bound": false, "image": "quay.io/centos-bootc/centos-bootc:stream9", "name": "centos-bootc" } + { "bound": false, "image": "quay.io/centos-bootc/centos-bootc:stream9", "name": "centos-bootc" }, + { "bound": true, "image": $"($CSTOR_DIST_REGISTRY)/docker.io/library/alpine:latest", "name": "cstor-alpine" }, + { "bound": true, "image": $"($CSTOR_DIST_REGISTRY)/docker.io/library/busybox:latest", "name": "cstor-busybox" } ] let containers = [{