From d03c6fa285da352380c91e661b5f3a8da87102bd Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Tue, 13 Jan 2026 14:41:13 -0500 Subject: [PATCH] install: Enable installing to multi device parents When the root filesystem spans multiple backing devices (e.g., LVM across multiple disks), discover all parent devices and find ESP partitions on each. For bootupd/GRUB, install the bootloader to all devices with an ESP partition, enabling boot from any disk in a multi-disk setup. systemd-boot and zipl only support single-device configurations. This adds a new integration test validating both single-ESP and dual-ESP multi-device scenarios. Fixes: #481 Assisted-by: Claude Code (Opus 4.5) Signed-off-by: ckyrouac --- crates/lib/src/bootc_composefs/boot.rs | 22 +- crates/lib/src/bootloader.rs | 69 ++++-- crates/lib/src/install.rs | 103 ++++++--- crates/lib/src/install/baseline.rs | 2 +- tmt/plans/integration.fmf | 7 + tmt/tests/booted/test-multi-device-esp.nu | 246 ++++++++++++++++++++++ tmt/tests/test-32-multi-device-esp.fmf | 3 + 7 files changed, 408 insertions(+), 44 deletions(-) create mode 100644 tmt/tests/booted/test-multi-device-esp.nu create mode 100644 tmt/tests/test-32-multi-device-esp.fmf diff --git a/crates/lib/src/bootc_composefs/boot.rs b/crates/lib/src/bootc_composefs/boot.rs index 1a295d071..1e7556111 100644 --- a/crates/lib/src/bootc_composefs/boot.rs +++ b/crates/lib/src/bootc_composefs/boot.rs @@ -520,8 +520,13 @@ pub(crate) fn setup_composefs_bls_boot( cmdline_options.extend(&Cmdline::from(&composefs_cmdline)); - // Locate ESP partition device - let esp_part = esp_in(&root_setup.device_info)?; + // Locate ESP partition device (use first device) + // TODO: Handle multiple devices (RAID, LVM, etc) + let device_info = root_setup + .device_info + .first() + .ok_or_else(|| anyhow!("Cannot locate ESP: no backing device found"))?; + let esp_part = esp_in(device_info)?; ( root_setup.physical_root_path.clone(), @@ -1063,7 +1068,12 @@ pub(crate) fn setup_composefs_uki_boot( BootSetupType::Setup((root_setup, state, postfetch, ..)) => { state.require_no_kargs_for_uki()?; - let esp_part = esp_in(&root_setup.device_info)?; + //TODO: Handle multiple devices (RAID, LVM, etc) + let device_info = root_setup + .device_info + .first() + .ok_or_else(|| anyhow!("Cannot locate ESP: no backing device found"))?; + let esp_part = esp_in(device_info)?; ( root_setup.physical_root_path.clone(), @@ -1233,7 +1243,8 @@ pub(crate) async fn setup_composefs_boot( if cfg!(target_arch = "s390x") { // TODO: Integrate s390x support into install_via_bootupd - crate::bootloader::install_via_zipl(&root_setup.device_info, boot_uuid)?; + // zipl only supports single device + crate::bootloader::install_via_zipl(root_setup.device_info.first(), boot_uuid)?; } else if postfetch.detected_bootloader == Bootloader::Grub { crate::bootloader::install_via_bootupd( &root_setup.device_info, @@ -1242,8 +1253,9 @@ pub(crate) async fn setup_composefs_boot( None, )?; } else { + // systemd-boot only supports single device crate::bootloader::install_systemd_boot( - &root_setup.device_info, + root_setup.device_info.first(), &root_setup.physical_root_path, &state.config_opts, None, diff --git a/crates/lib/src/bootloader.rs b/crates/lib/src/bootloader.rs index 7fce2888a..9bef886ad 100644 --- a/crates/lib/src/bootloader.rs +++ b/crates/lib/src/bootloader.rs @@ -82,7 +82,7 @@ pub(crate) fn supports_bootupd(root: &Dir) -> Result { #[context("Installing bootloader")] pub(crate) fn install_via_bootupd( - device: &PartitionTable, + devices: &[PartitionTable], rootfs: &Utf8Path, configopts: &crate::install::InstallConfigOpts, deployment_path: Option<&str>, @@ -97,26 +97,61 @@ pub(crate) fn install_via_bootupd( } else { vec![] }; - let devpath = device.path(); - println!("Installing bootloader via bootupd"); - Command::new("bootupctl") - .args(["backend", "install", "--write-uuid"]) - .args(verbose) - .args(bootupd_opts.iter().copied().flatten()) - .args(src_root_arg) - .args(["--device", devpath.as_str(), rootfs.as_str()]) - .log_debug() - .run_inherited_with_cmd_context() + + // No backing devices with ESP found. Run bootupd without --device and let it + // try to auto-detect. This works for: + // - BIOS boot (uses MBR, not ESP) + // - Systems where bootupd can find ESP via mounted /boot/efi + // UEFI boot will fail if bootupd cannot locate the ESP. + if devices.is_empty() { + tracing::warn!( + "No backing device with ESP found; UEFI boot may fail if ESP cannot be auto-detected" + ); + println!("Installing bootloader via bootupd (no target device specified)"); + return Command::new("bootupctl") + .args(["backend", "install", "--write-uuid"]) + .args(verbose) + .args(bootupd_opts.iter().copied().flatten()) + .args(&src_root_arg) + .arg(rootfs.as_str()) + .log_debug() + .run_inherited_with_cmd_context(); + } + + // Install bootloader to each device + for dev in devices { + let devpath = dev.path(); + println!("Installing bootloader via bootupd to {devpath}"); + Command::new("bootupctl") + .args(["backend", "install", "--write-uuid"]) + .args(verbose) + .args(bootupd_opts.iter().copied().flatten()) + .args(&src_root_arg) + .args(["--device", devpath.as_str()]) + .arg(rootfs.as_str()) + .log_debug() + .run_inherited_with_cmd_context()?; + } + + Ok(()) } #[context("Installing bootloader")] pub(crate) fn install_systemd_boot( - device: &PartitionTable, + device: Option<&PartitionTable>, _rootfs: &Utf8Path, _configopts: &crate::install::InstallConfigOpts, _deployment_path: Option<&str>, autoenroll: Option, ) -> Result<()> { + // systemd-boot requires the backing device to locate the ESP partition + let device = device.ok_or_else(|| { + anyhow!( + "Cannot install systemd-boot: no single backing device found \ + (root may span multiple devices such as LVM across multiple disks)" + ) + })?; + let esp_part = device .find_partition_of_type(discoverable_partition_specification::ESP) .ok_or_else(|| anyhow::anyhow!("ESP partition not found"))?; @@ -161,7 +196,15 @@ pub(crate) fn install_systemd_boot( } #[context("Installing bootloader using zipl")] -pub(crate) fn install_via_zipl(device: &PartitionTable, boot_uuid: &str) -> Result<()> { +pub(crate) fn install_via_zipl(device: Option<&PartitionTable>, boot_uuid: &str) -> Result<()> { + // zipl requires the backing device information to install the bootloader + let device = device.ok_or_else(|| { + anyhow!( + "Cannot install zipl bootloader: no single backing device found \ + (root may span multiple devices such as LVM across multiple disks)" + ) + })?; + // Identify the target boot partition from UUID let fs = mount::inspect_filesystem_by_uuid(boot_uuid)?; let boot_dir = Utf8Path::new(&fs.target); diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index 105e2b3c1..a7806c696 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -1127,7 +1127,10 @@ pub(crate) fn exec_in_host_mountns(args: &[std::ffi::OsString]) -> Result<()> { pub(crate) struct RootSetup { #[cfg(feature = "install-to-disk")] luks_device: Option, - pub(crate) device_info: bootc_blockdev::PartitionTable, + /// Information about the backing block device partition tables. + /// Contains all devices that have an ESP partition when the root filesystem + /// spans multiple backing devices (e.g., LVM across multiple disks). + pub(crate) device_info: Vec, /// Absolute path to the location where we've mounted the physical /// root filesystem for the system we're installing. pub(crate) physical_root_path: Utf8PathBuf, @@ -1588,7 +1591,9 @@ async fn install_with_sysroot( if cfg!(target_arch = "s390x") { // TODO: Integrate s390x support into install_via_bootupd - crate::bootloader::install_via_zipl(&rootfs.device_info, boot_uuid)?; + // zipl only supports single device + let device = rootfs.device_info.first(); + crate::bootloader::install_via_zipl(device, boot_uuid)?; } else { match postfetch.detected_bootloader { Bootloader::Grub => { @@ -1719,15 +1724,21 @@ async fn install_to_filesystem_impl( // Drop exclusive ownership since we're done with mutation let rootfs = &*rootfs; - match &rootfs.device_info.label { - bootc_blockdev::PartitionType::Dos => crate::utils::medium_visibility_warning( - "Installing to `dos` format partitions is not recommended", - ), - bootc_blockdev::PartitionType::Gpt => { - // The only thing we should be using in general - } - bootc_blockdev::PartitionType::Unknown(o) => { - crate::utils::medium_visibility_warning(&format!("Unknown partition label {o}")) + // Check partition type of all backing devices + for device_info in &rootfs.device_info { + match &device_info.label { + bootc_blockdev::PartitionType::Dos => { + crate::utils::medium_visibility_warning(&format!( + "Installing to `dos` format partitions is not recommended: {}", + device_info.path() + )) + } + bootc_blockdev::PartitionType::Gpt => { + // The only thing we should be using in general + } + bootc_blockdev::PartitionType::Unknown(o) => crate::utils::medium_visibility_warning( + &format!("Unknown partition label {o}: {}", device_info.path()), + ), } } @@ -2277,27 +2288,69 @@ pub(crate) async fn install_to_filesystem( }; tracing::debug!("boot UUID: {boot_uuid:?}"); - // Find the real underlying backing device for the root. This is currently just required - // for GRUB (BIOS) and in the future zipl (I think). - let backing_device = { + // Walk up the block device hierarchy to find physical backing device(s). + // Examples: + // /dev/sda3 -> /dev/sda (single disk) + // /dev/mapper/vg-lv -> /dev/sda2, /dev/sdb2 (LVM across two disks) + let backing_devices: Vec = { let mut dev = inspect.source; loop { tracing::debug!("Finding parents for {dev}"); - let mut parents = bootc_blockdev::find_parent_devices(&dev)?.into_iter(); - let Some(parent) = parents.next() else { - break; - }; - if let Some(next) = parents.next() { - anyhow::bail!( - "Found multiple parent devices {parent} and {next}; not currently supported" + let parents = bootc_blockdev::find_parent_devices(&dev)?; + if parents.is_empty() { + // Reached a physical disk + break vec![dev]; + } + if parents.len() > 1 { + // Multi-device (e.g., LVM across disks) - return all + tracing::debug!( + "Found multiple parent devices: {:?}; will search for ESP", + parents ); + break parents; + } + // Single parent (e.g. LVM LV -> VG -> PV) - keep walking up + dev = parents.into_iter().next().unwrap(); + } + }; + tracing::debug!("Backing devices: {backing_devices:?}"); + + // Determine the device and partition info to use for bootloader installation. + // If there are multiple backing devices, we search for all that contain an ESP. + let device_info: Vec = if backing_devices.len() == 1 { + // Single backing device - use it directly + let dev = &backing_devices[0]; + vec![bootc_blockdev::partitions_of(Utf8Path::new(dev))?] + } else { + // Multiple backing devices - find all with ESP + let mut esp_devices = Vec::new(); + for dev in &backing_devices { + match bootc_blockdev::partitions_of(Utf8Path::new(dev)) { + Ok(table) => { + if table.find_partition_of_esp()?.is_some() { + tracing::info!("Found ESP on device {dev}"); + esp_devices.push(table); + } + } + Err(e) => { + // Some backing devices may not have partition tables (e.g., raw LVM PVs + // or whole-disk filesystems). These can't have an ESP, so skip them. + tracing::debug!("Failed to read partition table from {dev}: {e}"); + } } - dev = parent; } - dev + if esp_devices.is_empty() { + // No ESP found on any backing device. This is not fatal because: + // - BIOS boot uses MBR, not ESP + // - bootupd may auto-detect ESP via mounted /boot/efi + // However, UEFI boot without a detectable ESP will fail. + tracing::warn!( + "No ESP found on any backing device ({:?}); UEFI boot may fail", + backing_devices + ); + } + esp_devices }; - tracing::debug!("Backing device: {backing_device}"); - let device_info = bootc_blockdev::partitions_of(Utf8Path::new(&backing_device))?; let rootarg = format!("root={}", root_info.mount_spec); let mut boot = if let Some(spec) = fsopts.boot_mount_spec { diff --git a/crates/lib/src/install/baseline.rs b/crates/lib/src/install/baseline.rs index d05604ed5..2e5208c41 100644 --- a/crates/lib/src/install/baseline.rs +++ b/crates/lib/src/install/baseline.rs @@ -485,7 +485,7 @@ pub(crate) fn install_create_rootfs( BlockSetup::Direct => None, BlockSetup::Tpm2Luks => Some(luks_name.to_string()), }; - let device_info = bootc_blockdev::partitions_of(&devpath)?; + let device_info = vec![bootc_blockdev::partitions_of(&devpath)?]; Ok(RootSetup { luks_device, device_info, diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index 365e6618c..01e18f1ea 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -159,4 +159,11 @@ execute: how: fmf test: - /tmt/tests/tests/test-32-install-to-filesystem-var-mount + +/plan-33-multi-device-esp: + summary: Test multi-device ESP detection for to-existing-root + discover: + how: fmf + test: + - /tmt/tests/test-32-multi-device-esp # END GENERATED PLANS diff --git a/tmt/tests/booted/test-multi-device-esp.nu b/tmt/tests/booted/test-multi-device-esp.nu new file mode 100644 index 000000000..5d696b52f --- /dev/null +++ b/tmt/tests/booted/test-multi-device-esp.nu @@ -0,0 +1,246 @@ +# number: 32 +# tmt: +# summary: Test multi-device ESP detection for to-existing-root +# duration: 45m +# +# Test that bootc install to-existing-root can find and use ESP partitions +# when the root filesystem spans multiple backing devices (e.g., LVM across disks). +# +# Two scenarios are tested: +# 1. Single ESP: Only one backing device has an ESP partition +# 2. Dual ESP: Both backing devices have ESP partitions +# +# This validates the fix for https://github.com/bootc-dev/bootc/issues/481 + +use std assert +use tap.nu + +# Use the currently booted image (copied to container storage) +const target_image = "localhost/bootc" + +# ESP partition type GUID +const ESP_TYPE = "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" +# Linux LVM partition type GUID +const LVM_TYPE = "E6D6D379-F507-44C2-A23C-238F2A3DF928" + +# Cleanup function for LVM and loop devices +def cleanup [vg_name: string, loop1: string, loop2: string, mountpoint: string] { + # Unmount if mounted + do { umount $mountpoint } | complete | ignore + do { rmdir $mountpoint } | complete | ignore + + # Deactivate and remove LVM + do { lvchange -an $"($vg_name)/test_lv" } | complete | ignore + do { lvremove -f $"($vg_name)/test_lv" } | complete | ignore + do { vgchange -an $vg_name } | complete | ignore + do { vgremove -f $vg_name } | complete | ignore + + # Remove PVs and detach loop devices + if ($loop1 | path exists) { + do { pvremove -f $loop1 } | complete | ignore + do { losetup -d $loop1 } | complete | ignore + } + if ($loop2 | path exists) { + do { pvremove -f $loop2 } | complete | ignore + do { losetup -d $loop2 } | complete | ignore + } +} + +# Create a disk with GPT, optional ESP, and LVM partition +# Returns the loop device path +def setup_disk_with_partitions [ + disk_path: string, + with_esp: bool, + disk_size: string = "5G" +] { + # Create disk image + truncate -s $disk_size $disk_path + + # Setup loop device + let loop = (losetup -f --show $disk_path | str trim) + + # Create partition table + if $with_esp { + # GPT with ESP (512MB) + LVM partition + $"label: gpt\nsize=512M, type=($ESP_TYPE)\ntype=($LVM_TYPE)\n" | sfdisk $loop + + # Reload partition table (partx is part of util-linux) + partx -u $loop + sleep 1sec + + # Format ESP + mkfs.vfat -F 32 $"($loop)p1" + } else { + # GPT with only LVM partition (full disk) + $"label: gpt\ntype=($LVM_TYPE)\n" | sfdisk $loop + + # Reload partition table (partx is part of util-linux) + partx -u $loop + sleep 1sec + } + + $loop +} + +# Validate that an ESP partition has bootloader files installed +def validate_esp [esp_partition: string] { + let esp_mount = "/var/mnt/esp_check" + mkdir $esp_mount + mount $esp_partition $esp_mount + + # Check for EFI directory with bootloader files + let efi_dir = $"($esp_mount)/EFI" + if not ($efi_dir | path exists) { + umount $esp_mount + rmdir $esp_mount + error make {msg: $"ESP validation failed: EFI directory not found on ($esp_partition)"} + } + + # Verify there's actual content in EFI (not just empty) + let efi_contents = (ls $efi_dir | length) + umount $esp_mount + rmdir $esp_mount + + if $efi_contents == 0 { + error make {msg: $"ESP validation failed: EFI directory is empty on ($esp_partition)"} + } +} + +# Run bootc install to-existing-root from within the container image under test +def run_install [mountpoint: string] { + (podman run + --rm + --privileged + -v $"($mountpoint):/target" + -v /dev:/dev + -v /usr/share/empty:/usr/lib/bootc/bound-images.d + --pid=host + --security-opt label=type:unconfined_t + $target_image + bootc install to-existing-root + --disable-selinux + --acknowledge-destructive + --target-no-signature-verification + /target) +} + +# Test scenario 1: Single ESP on first device +def test_single_esp [] { + tap begin "multi-device ESP detection tests" + + # Copy the currently booted image to container storage for podman to use + bootc image copy-to-storage + + print "Starting single ESP test" + + let vg_name = "test_single_esp_vg" + let mountpoint = "/var/mnt/test_single_esp" + let disk1 = "/var/tmp/disk1_single.img" + let disk2 = "/var/tmp/disk2_single.img" + + # Setup disks + # DISK1: ESP + LVM partition + # DISK2: Full LVM partition (no ESP) + let loop1 = (setup_disk_with_partitions $disk1 true) + let loop2 = (setup_disk_with_partitions $disk2 false) + + try { + # Create LVM spanning both devices + # Use partition 2 from disk1 (after ESP) and partition 1 from disk2 (full disk) + pvcreate $"($loop1)p2" $"($loop2)p1" + vgcreate $vg_name $"($loop1)p2" $"($loop2)p1" + lvcreate -l "100%FREE" -n test_lv $vg_name + + let lv_path = $"/dev/($vg_name)/test_lv" + + # Create filesystem and mount + mkfs.ext4 -q $lv_path + mkdir $mountpoint + mount $lv_path $mountpoint + + # Create boot directory + mkdir $"($mountpoint)/boot" + + # Show block device hierarchy + lsblk --pairs --paths --inverse --output NAME,TYPE $lv_path + + run_install $mountpoint + + # Validate ESP was installed correctly + validate_esp $"($loop1)p1" + } catch {|e| + cleanup $vg_name $loop1 $loop2 $mountpoint + rm -f $disk1 $disk2 + error make {msg: $"Single ESP test failed: ($e)"} + } + + # Cleanup + cleanup $vg_name $loop1 $loop2 $mountpoint + rm -f $disk1 $disk2 + + print "Single ESP test completed successfully" + tmt-reboot +} + +# Test scenario 2: ESP on both devices +def test_dual_esp [] { + print "Starting dual ESP test" + + let vg_name = "test_dual_esp_vg" + let mountpoint = "/var/mnt/test_dual_esp" + let disk1 = "/var/tmp/disk1_dual.img" + let disk2 = "/var/tmp/disk2_dual.img" + + # Setup disks + # DISK1: ESP + LVM partition + # DISK2: ESP + LVM partition + let loop1 = (setup_disk_with_partitions $disk1 true) + let loop2 = (setup_disk_with_partitions $disk2 true) + + try { + # Create LVM spanning both devices + # Use partition 2 from both disks (after ESP) + pvcreate $"($loop1)p2" $"($loop2)p2" + vgcreate $vg_name $"($loop1)p2" $"($loop2)p2" + lvcreate -l "100%FREE" -n test_lv $vg_name + + let lv_path = $"/dev/($vg_name)/test_lv" + + # Create filesystem and mount + mkfs.ext4 -q $lv_path + mkdir $mountpoint + mount $lv_path $mountpoint + + # Create boot directory + mkdir $"($mountpoint)/boot" + + # Show block device hierarchy + lsblk --pairs --paths --inverse --output NAME,TYPE $lv_path + + run_install $mountpoint + + # Validate both ESPs were installed correctly + validate_esp $"($loop1)p1" + validate_esp $"($loop2)p1" + } catch {|e| + cleanup $vg_name $loop1 $loop2 $mountpoint + rm -f $disk1 $disk2 + error make {msg: $"Dual ESP test failed: ($e)"} + } + + # Cleanup + cleanup $vg_name $loop1 $loop2 $mountpoint + rm -f $disk1 $disk2 + + print "Dual ESP test completed successfully" + tap ok +} + +def main [] { + # See https://tmt.readthedocs.io/en/stable/stories/features.html#reboot-during-test + match $env.TMT_REBOOT_COUNT? { + null | "0" => test_single_esp, + "1" => test_dual_esp, + $o => { error make { msg: $"Invalid TMT_REBOOT_COUNT ($o)" } }, + } +} diff --git a/tmt/tests/test-32-multi-device-esp.fmf b/tmt/tests/test-32-multi-device-esp.fmf new file mode 100644 index 000000000..6436fffca --- /dev/null +++ b/tmt/tests/test-32-multi-device-esp.fmf @@ -0,0 +1,3 @@ +summary: Test multi-device ESP detection for to-existing-root +test: nu booted/test-multi-device-esp.nu +duration: 45m