diff --git a/README.md b/README.md index 6bdd7f8d..71fad58c 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,9 @@ https://github.com/confidential-containers) project. SC2 currently supports AMD SEV-SNP and Intel TDX as underlying TEE, and requires deployment on a bare-metal host. Before moving forward, make sure you have a -correct installation. For SEV-SNP you may use [`snphost ok`]( -https://github.com/virtee/snphost.git). - -Lastly, make sure you are using the exact host kernel: - -| **SEV-SNP** | **TDX** | -|---|---| -| [6.8.0-rc5-next-20240221-snp-host-cc2568386](https://github.com/confidential-containers/linux/tree/amd-snp-host-202402240000) | [6.8.0-1004-intel](https://git.launchpad.net/~kobuk-team/ubuntu/+source/linux-intel/tree/?h=noble-main-next) | +correct host installation. For SEV-SNP you may use [`snphost ok`]( +https://github.com/virtee/snphost.git). Also make sure you have the [right +host kernel](./docs/host_kernel.md). ## Quick Start diff --git a/bin/launch_svsm.sh b/bin/launch_svsm.sh new file mode 100755 index 00000000..f078e3a6 --- /dev/null +++ b/bin/launch_svsm.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +SVSM_ROOT=/opt/sc2/svsm + +# C-bit pos may be obtained by running coconut-svsm/svsm/utils/cbit +CBIT_POS=51 + +IGVM=${SVSM_ROOT}/share/igvm/coconut-qemu.igvm +KERNEL=${SVSM_ROOT}/share/sc2/vmlinuz-kata-containers-sc2 +INITRD=/opt/sc2/svsm/share/sc2/initrd-kata.img + +# Ensure terminal settings are restored on exit +orig_stty=$(stty -g) +trap "stty '$orig_stty'" EXIT + +# Remap Ctrl-C to Ctrl-] to allow the guest to handle Ctrl-C. +stty intr ^] + +sudo ${SVSM_ROOT}/bin/qemu-system-x86_64 \ + -cpu EPYC-v4 \ + -machine q35,confidential-guest-support=sev0,memory-backend=ram1,igvm-cfg=igvm0,accel=kvm \ + -object memory-backend-memfd,id=ram1,size=8G,share=true,prealloc=false,reserve=false \ + -object sev-snp-guest,id=sev0,cbitpos=${CBIT_POS},reduced-phys-bits=1 \ + -object igvm-cfg,id=igvm0,file=$IGVM \ + -smp 8 \ + -no-reboot \ + -netdev user,id=vmnic -device e1000,netdev=vmnic,romfile= \ + -kernel ${KERNEL} \ + -initrd ${INITRD} \ + -append "console=ttyS0 loglevel=8 earlyprintk=serial rdinit=/bin/sh" \ + -monitor none \ + -nographic \ + -serial stdio \ + -serial pty diff --git a/docker/base.dockerfile b/docker/base.dockerfile index 320849e1..afe90c1d 100644 --- a/docker/base.dockerfile +++ b/docker/base.dockerfile @@ -10,10 +10,15 @@ RUN apt update \ && apt upgrade -y \ && apt install -y \ clang \ + cmake \ curl \ + g++ \ + gcc \ git \ + gopls \ libclang-dev \ libdevmapper-dev \ + make \ wget # Clone the dotfiles repo diff --git a/docker/coconut/ovmf.dockerfile b/docker/coconut/ovmf.dockerfile deleted file mode 100644 index a1f73cee..00000000 --- a/docker/coconut/ovmf.dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM ubuntu:22.04 - -RUN sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list -RUN apt update \ - && apt upgrade -y \ - && apt install -y \ - git \ - && apt build-dep ovmf -y - -RUN git clone https://github.com/coconut-svsm/edk2.git ~/edk2\ - && cd ~/edk2/ \ - && git checkout svsm \ - && git submodule init \ - && git submodule update \ - && export PYTHON3_ENABLE=TRUE \ - && export PYTHON_COMMAND=python3 \ - && make -j16 -C BaseTools/ \ - && . ./edksetup.sh --reconfig \ - && build -a X64 -b DEBUG -t GCC5 -D DEBUG_ON_SERIAL_PORT -D DEBUG_VERBOSE -DTPM2_ENABLE -p OvmfPkg/OvmfPkgX64.dsc diff --git a/docker/coconut/qemu.dockerfile b/docker/coconut/qemu.dockerfile deleted file mode 100644 index 8d197f6a..00000000 --- a/docker/coconut/qemu.dockerfile +++ /dev/null @@ -1,57 +0,0 @@ -FROM ubuntu:24.04 - -RUN apt update \ - && apt upgrade -y \ - && apt install -y \ - git \ - make \ - cbindgen \ - curl \ - gcc \ - libcunit1 \ - libcunit1-doc \ - libcunit1-dev \ - gettext \ - python3-venv \ - ninja-build \ - bzip2 \ - libglib2.0-dev \ - rustc \ - iasl \ - build-essential \ - libglib2.0-dev \ - libfdt-dev \ - libpixman-1-dev \ - zlib1g-dev \ - libudev-dev \ - libvdeplug-dev \ - libslirp-dev \ - seabios - -# Clone and build IGVM -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \ - && . "$HOME/.cargo/env" \ - && git clone --branch igvm-v0.1.6 https://github.com/microsoft/igvm/ ~/igvm \ - && cd ~/igvm \ - && make -f igvm_c/Makefile \ - && make -f igvm_c/Makefile install - -# Clone and build IGVM-enabled Qemu -ARG QEMU_DATADIR -RUN git clone https://github.com/coconut-svsm/qemu ~/qemu \ - && cd ~/qemu \ - && git checkout svsm-igvm \ - && export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/lib64/pkgconfig/ \ - && ./configure \ - --datadir=${QEMU_DATADIR} \ - --prefix=$HOME/bin/qemu-svsm/ \ - --target-list=x86_64-softmmu \ - --enable-igvm \ - --static \ - --disable-gio \ - --disable-libudev \ - --enable-kvm \ - --enable-trace-backends=log,simple \ - --enable-slirp \ - && ninja -C build/ \ - && make install -j $(nproc) diff --git a/docker/coconut/svsm.dockerfile b/docker/coconut/svsm.dockerfile deleted file mode 100644 index 87e4c9b9..00000000 --- a/docker/coconut/svsm.dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM ubuntu:24.04 - -RUN apt update \ - && apt upgrade -y \ - && apt install -y \ - curl \ - git \ - gcc \ - make \ - automake \ - libssl-dev \ - autoconf \ - autoconf-archive \ - build-essential \ - pkg-config \ - libclang-dev - -ARG OVMF_DIR -COPY ${OVMF_DIR}/ovmf-svsm.fd /bin/ovmf-svsm.fd - -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \ - && . "$HOME/.cargo/env" \ - && rustup target add x86_64-unknown-none \ - && git clone https://github.com/sc2-sys/svsm.git ~/svsm \ - && cd ~/svsm \ - && git submodule update --init \ - && cargo install bindgen-cli \ - && FW_FILE=/bin/ovmf-svsm.fd make diff --git a/docker/containerd.dockerfile b/docker/containerd.dockerfile index afd30e88..bb5a61fb 100644 --- a/docker/containerd.dockerfile +++ b/docker/containerd.dockerfile @@ -8,9 +8,7 @@ FROM ghcr.io/sc2-sys/base:0.10.0 RUN apt update \ && apt upgrade -y \ && apt install -y \ - libbtrfs-dev \ - gopls \ - make + libbtrfs-dev # Clone and build containerd ARG CODE_DIR=/go/src/github.com/sc2-sys/containerd diff --git a/docker/kata.dockerfile b/docker/kata.dockerfile index ab2c584b..6ebaf544 100644 --- a/docker/kata.dockerfile +++ b/docker/kata.dockerfile @@ -6,10 +6,7 @@ FROM ghcr.io/sc2-sys/base:0.10.0 # Install APT dependencies RUN apt install -y \ - gcc \ - gopls \ libseccomp-dev \ - make \ musl-tools \ wget diff --git a/docker/nydus.dockerfile b/docker/nydus.dockerfile index 709a22c5..c7a53ab3 100644 --- a/docker/nydus.dockerfile +++ b/docker/nydus.dockerfile @@ -4,13 +4,6 @@ FROM ghcr.io/sc2-sys/base:0.10.0 # Nydus daemon set-up # --------------------------- -# Install APT dependencies -RUN apt-get update \ - && apt-get install -y \ - cmake \ - gopls \ - make - # Build the daemon and other tools like nydusify ARG CODE_DIR=/go/src/github.com/sc2-sys/nydus RUN mkdir -p ${CODE_DIR} \ diff --git a/docker/nydus_snapshotter.dockerfile b/docker/nydus_snapshotter.dockerfile index 57745e89..9933094c 100644 --- a/docker/nydus_snapshotter.dockerfile +++ b/docker/nydus_snapshotter.dockerfile @@ -7,8 +7,6 @@ FROM ghcr.io/sc2-sys/base:0.10.0 # Install APT dependencies RUN apt-get update \ && apt-get install -y \ - gopls \ - make \ protobuf-compiler ARG CODE_DIR=/go/src/github.com/sc2-sys/nydus-snapshotter diff --git a/docker/ovmf.dockerfile b/docker/ovmf.dockerfile index fd56f05a..b32e501f 100644 --- a/docker/ovmf.dockerfile +++ b/docker/ovmf.dockerfile @@ -1,37 +1,30 @@ -FROM ubuntu:22.04 +FROM ghcr.io/sc2-sys/base:0.10.0 RUN apt update \ && apt upgrade -y \ && apt install -y \ - g++ \ - gcc \ - git \ + dosfstools \ + grub2-common \ + grub-efi \ iasl \ - make \ + mtools \ nasm \ - python3 \ - python-is-python3 \ - uuid-dev \ - vim + uuid-dev -ARG OVMF_PATCH -COPY ${OVMF_PATCH} /tmp/ovmf_profile.patch -ARG TARGET -RUN mkdir -p /usr/src/edk2 \ +ARG OVMF_VERSION +ARG CODE_DIR=/git/sc2-sys/edk2 +RUN mkdir -p ${CODE_DIR} \ && git clone \ - -b edk2-stable202302 \ - --single-branch --depth 1 \ + --branch ${OVMF_VERSION} \ + --depth 1 \ https://github.com/tianocore/edk2.git \ - /usr/src/edk2 \ - && cd /usr/src/edk2 \ + ${CODE_DIR} \ + && cd ${CODE_DIR} \ && git submodule update --init \ - && make -C BaseTools/ \ - && touch OvmfPkg/AmdSev/Grub/grub.efi \ - && git apply /tmp/ovmf_profile.patch \ - && cd OvmfPkg \ - && ./build.sh \ - -b ${TARGET} \ - -D DEBUG_ON_SERIAL_PORT \ - -p OvmfPkg/AmdSev/AmdSevX64.dsc - -WORKDIR /usr/src/edk2 + && export PYTHON3_ENABLE=TRUE \ + && export PYTHON_COMMAND=python3 \ + && make -j $(nproc) -C BaseTools/ \ + && . ./edksetup.sh --reconfig \ + && build -a X64 -b RELEASE -t GCC5 -p OvmfPkg/OvmfPkgX64.dsc \ + && touch OvmfPkg/AmdSev/Grub/grub.efi \ + && build -a X64 -b RELEASE -t GCC5 -p OvmfPkg/AmdSev/AmdSevX64.dsc diff --git a/docker/qemu.dockerfile b/docker/qemu.dockerfile index 5b338ecb..e987ada0 100644 --- a/docker/qemu.dockerfile +++ b/docker/qemu.dockerfile @@ -47,7 +47,6 @@ RUN mkdir -p /usr/src \ --disable-guest-agent-msi \ --disable-libiscsi \ --disable-libudev \ - --disable-linux-user \ --disable-live-block-migration \ --disable-lzo \ --disable-opengl \ diff --git a/docker/svsm.dockerfile b/docker/svsm.dockerfile new file mode 100644 index 00000000..7fcd027f --- /dev/null +++ b/docker/svsm.dockerfile @@ -0,0 +1,22 @@ +FROM ghcr.io/sc2-sys/base:0.10.0 + +RUN apt update \ + && apt upgrade -y \ + && apt install -y \ + autoconf \ + autoconf-archive \ + libclang-dev \ + libssl-dev \ + pkg-config + +ARG OVMF_FILE +COPY ./${OVMF_FILE} /bin/ovmf-svsm.fd + +ARG CODE_DIR=/git/coconut-svsm/svsm +RUN mkdir -p ${CODE_DIR} \ + && git clone https://github.com/coconut-svsm/svsm ${CODE_DIR} \ + && cd ${CODE_DIR} \ + && git submodule update --init \ + && rustup target add x86_64-unknown-none \ + && cargo install bindgen-cli \ + && FW_FILE=/bin/ovmf-svsm.fd ./build --release configs/qemu-target.json diff --git a/docker/svsm_kernel.dockerfile b/docker/svsm_kernel.dockerfile new file mode 100644 index 00000000..133a358f --- /dev/null +++ b/docker/svsm_kernel.dockerfile @@ -0,0 +1,31 @@ +FROM ghcr.io/sc2-sys/base:0.10.0 + +RUN apt update \ + && apt upgrade -y \ + && apt install -y \ + bc \ + bison \ + cpio \ + flex \ + kmod \ + libelf-dev \ + libssl-dev \ + xz-utils \ + zstd + +# Clone kernel source tree +ARG CODE_DIR=/git/coconut-svsm/linux +RUN mkdir -p ${CODE_DIR} \ + && git clone \ + --branch svsm \ + --depth=1 https://github.com/coconut-svsm/linux \ + ${CODE_DIR} + +# Copy generated config file. The filename and path are hardcoded in ./tasks/svsm.py +COPY ./svsm_kernel_config ${CODE_DIR}/.config + +ARG MODULES_OUTDIR +RUN cd ${CODE_DIR} \ + && make olddefconfig \ + && make -j $(nproc) \ + && make modules_install INSTALL_MOD_PATH=${MODULES_OUTDIR} diff --git a/docker/svsm_qemu.dockerfile b/docker/svsm_qemu.dockerfile new file mode 100644 index 00000000..45aaa24a --- /dev/null +++ b/docker/svsm_qemu.dockerfile @@ -0,0 +1,110 @@ +FROM ghcr.io/sc2-sys/base:0.10.0 + +RUN apt update \ + && apt upgrade -y \ + && apt install -y \ + bzip2 \ + cbindgen \ + gettext \ + iasl \ + libcunit1-dev \ + libfdt-dev \ + libglib2.0-dev \ + libpixman-1-dev \ + libudev-dev \ + libvdeplug-dev \ + nasm \ + ninja-build \ + python3-tomli \ + python3-venv \ + seabios \ + zlib1g-dev + +# Clone and build IGVM +ARG IGVM_VERSION +ARG CODE_DIR=/git/microsoft/igvm +RUN mkdir -p ${CODE_DIR} \ + && git clone --branch igvm-v${IGVM_VERSION} https://github.com/microsoft/igvm/ ${CODE_DIR} \ + && cd ${CODE_DIR} \ + && make -f igvm_c/Makefile \ + && make -f igvm_c/Makefile install + +# Clone and build SVSM's OVMF +ARG CODE_DIR=/git/coconut-svsm/edk2 +RUN mkdir -p ${CODE_DIR} \ + && git clone https://github.com/coconut-svsm/edk2 ${CODE_DIR} \ + && cd ${CODE_DIR} \ + && git checkout svsm \ + && git submodule init \ + && git submodule update \ + && export PYTHON3_ENABLE=TRUE \ + && export PYTHON_COMMAND=python3 \ + && make -j $(nproc) -C BaseTools/ \ + && . ./edksetup.sh --reconfig \ + && build -a X64 -b RELEASE -t GCC5 -DTPM2_ENABLE -p OvmfPkg/OvmfPkgX64.dsc \ + && build -a X64 -b DEBUG -t GCC5 -DTPM2_ENABLE -p OvmfPkg/OvmfPkgX64.dsc + +# Clone and build IGVM-enabled Qemu +ARG QEMU_DATADIR +ARG QEMU_PREFIX +ARG CODE_DIR=/git/coconut-svsm/qemu +RUN mkdir -p ${CODE_DIR} \ + && git clone https://github.com/coconut-svsm/qemu ${CODE_DIR} \ + && cd ${CODE_DIR} \ + && git checkout svsm-igvm \ + && export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/lib64/pkgconfig/ \ + && ./configure \ + --cpu=x86_64 \ + # The `--datadir` flag is the path where QEMU will look for firmware + # images. The default `--datadir` path when using a system provisioned + # by the operator is: `/opt/confidential-containers/share/kata-qemu`. + # For our QEMU fork we use `/opt/sc2/svsm/share/qemu + --datadir=${QEMU_DATADIR} \ + --prefix=${QEMU_PREFIX} \ + --target-list=x86_64-softmmu \ + # Must enable IGVM + --enable-igvm \ + --enable-kvm \ + --enable-slirp \ + --enable-trace-backends=log,simple \ + # As a reference we use Kata's --disable-x flags when building QEMU: + # https://github.com/kata-containers/kata-containers/blob/main/tools/packaging/scripts/configure-hypervisor.sh + --disable-auth-pam \ + --disable-brlapi \ + --disable-bsd-user \ + --disable-capstone \ + --disable-curl \ + --disable-curses \ + --disable-debug-tcg \ + --disable-docs \ + --disable-gio \ + --disable-glusterfs \ + --disable-gtk \ + --disable-guest-agent \ + --disable-guest-agent-msi \ + --disable-libiscsi \ + --disable-libudev \ + --disable-libnfs \ + --disable-libusb \ + --disable-linux-user \ + --disable-lzo \ + --disable-opengl \ + --disable-rdma \ + --disable-replication \ + --disable-sdl \ + --disable-snappy \ + --disable-spice \ + --disable-tcg-interpreter \ + --disable-tools \ + --disable-tpm \ + --disable-usb-redir \ + --disable-vde \ + --disable-vte \ + --disable-virglrenderer \ + --disable-vnc \ + --disable-vnc-jpeg \ + --disable-vnc-sasl \ + --disable-vte \ + --disable-xen \ + && make -j $(nproc) \ + && make install -j $(nproc) diff --git a/docs/coconut_svsm.md b/docs/coconut_svsm.md index 9e3cd0f4..4063d03f 100644 --- a/docs/coconut_svsm.md +++ b/docs/coconut_svsm.md @@ -1,152 +1,50 @@ # Coconut-SVSM -## Installation +SC2 uses the [Coconut-SVSM](https://github.com/coconut-svsm/svsm) as an SVSM for +its confidential pods. The SVSM requires its own forked versions of the kernel, +QEMU, and OVMF. -For more details, please refer to [Coconut SVSM](https://github.com/coconut-svsm/svsm). The following instructions are intended to simplify the installation, especially for our usecase. +> [!WARNING] +> Currently we only support running SNP guests in SVSM using a manual script +> with QEMU. In the future we will integrate it with Kata pods (#148). -### 1. Preparing the Host +## Quick Start -The host kernel needs support for SVSM. -Install necessary dependencies: +After installing SC2, you can run: ```bash -$ sudo apt install git fakeroot build-essential ncurses-dev xz-utils libssl-dev bc flex libelf-dev bison +inv svsm.install [--clean] ``` -Clone the linux kernel: +and then: ```bash -$ git clone https://github.com/coconut-svsm/linux -$ cd linux -$ git checkout svsm +./bin/launch_svsm.sh ``` -To use your current kernel configuration, execute: +to start an SNP guest in the SVSM. -```bash -$ cp /boot/config-$(uname -r) .config -``` +## Guest Kernel -Use `menuconfig` and ensure that `CONFIG_KVM_AMD_SEV` is enabled. +We have an automated script to build the guest kernel which, for some reason, +we still cannot use from the host (the `initramfs` seems to be corrupted, but +the kernel is fine to run in the guest). -Next, compile and install the kernel: +You can trigger the build by running: ```bash -$ make -j $(nproc) -$ sudo make modules_install -$ sudo make install +inv svsm.build-guest-kernel ``` -### 2. Building QEMU - -You will need a QEMU build that supports launching guests using IGVM: - -```bash -$ inv coconut.qemu.build -``` - -This will create a file called `qemu-system-x86_64-igvm` in the `BIN_DIR`. - -### 3. Building the guest firmware - -A special OVMF build is needed to launch a guest on top of the COCONUT-SVSM: - -```bash -$ inv coconut.ovmf.build -``` - -A file named `ovmf-svsm.fd` will be created in the `BIN_DIR`. - +## QEMU/OVMF -### 4. Preparing the guest image - -In our case, we are creating a Ubuntu guest image. - -Possible instructions: - -```bash -$ wget http://releases.ubuntu.com/jammy/ubuntu-22.04.4-live-server-amd64.iso -$ qemu-img create -f qcow2 ubuntu.qcow2 40G -$ qemu-system-x86_64 \ - -cdrom ubuntu-22.04.4-live-server-amd64.iso \ - -drive "file=ubuntu.qcow2,format=qcow2" \ - -bios /usr/share/ovmf/OVMF.fd \ - -enable-kvm \ - -m 4G \ - -smp 4 -``` - -Follow the instructions to install Ubuntu. -After the installation, start the VM. We are going to use a shared directory to copy the kernel configuration (see step 1, `CONFIG_KVM_AMD_SEV` has to be enabled) into the VM. - -```bash -$ mkdir -p /path/to/shared/directory -$ cp /path/to/kernel-config /path/to/shared/directory/ -$ qemu-system-x86_64 \ - -drive "file=ubuntu-snapshot.qcow2,format=qcow2" \ - -enable-kvm -m 2G -smp 2 \ - -virtfs local,path=/path/to/shared/directory,mount_tag=hostshare,security_model=passthrough,id=hostshare -``` - -Within the VM, execute: - -```bash -$ sudo apt-get install git fakeroot build-essential ncurses-dev xz-utils libssl-dev bc flex libelf-dev bison -$ git clone https://github.com/coconut-svsm/linux -$ cd linux -$ git checkout svsm -``` - -Next, mount the shared directory and copy the kernel configuration file: - -```bash -$ sudo mount -t 9p -o trans=virtio hostshare /mnt -$ cp /mnt/kernel-config ./.config -``` - -Then, install the kernel: - -```bash -$ make -j $(nproc) -$ sudo make modules_install -$ sudo make install -``` - -To compile the kernel on Arch Linux, you can use and adapt the `bin/compile_guest_kernel_arch.sh` script. - -### 5. Building the COCONUT-SVSM - -To build the SVSM itself using the special OVMF built in step 3, run - -``` bash -$ inv coconut.svsm.build -``` - -This command creates the file `coconut-qemu.igvm` in the `BIN_DIR`. - -### 6. Putting it all together - -To start the guest VM, execute - -```bash -$ inv coconut.qemu.guest -``` - -Modify the QEMU command in this file according to your needs. -`-d`/ `--detach` starts qemu in detached mode. - -If you are executing QEMU on a remote server, -you can use TigerVNC: - -```bash -$ inv coconut.qemu.guest -v -$ ssh -L 5901:localhost:5901 user@server -$ vncviewer localhost:5901 -``` +The SVSM uses the [IGVM](https://github.com/microsoft/igvm) image format to +package the initial guest image containing the virtual firmware (OVMF) as well +as the SVSM itself. -If you want to connect to the VM using ssh, you can use the following commands: +We need a patched version of QEMU and OVMF to support using IGVM files. You +can build both by running: ```bash -$ ssh-copy-id -i ~/.ssh/YOUR_SSH_KEY -p 2222 user@localhost -$ ssh -i ~/.ssh/YOUR_SSH_KEY -p 2222 user@localhost" +inv svsm.build-qemu ``` diff --git a/docs/host_kernel.md b/docs/host_kernel.md index 21be47dc..34e27839 100644 --- a/docs/host_kernel.md +++ b/docs/host_kernel.md @@ -1,26 +1,42 @@ -## Host Kernel Set-Up +# Host Kernel Set-Up -Frequently, until the SNP patches are upstreamed, we need to bump the host -kernel. +Some of the SC2 features require specific host-kernel patches. You can see +the expected host-kernel versions in [`./tasks/util/versions.py`]( +./tasks/util/versions.py), which we also summarise in the following table. -After re-building the right kernel from SNP, we need to pick it for our host. -We can do so doing the following: +| **SEV-SNP** | **TDX** | +|---|---| +| `6.11.0-snp-host-cc2568386` | `6.8.0-1004-intel` | + +## SNP + +For SNP hosts, we need kernel patches to (i) enable SNP, and (ii) enable the +SVSM. The former were upstreamed in `6.11`, but we still need the latter. To +build the host-kernel from source you may use: + +```bash +git clone -b svsm --depth=1 https://github.com/coconut-svsm/linux ../svsm-linux +``` + +and then just run: ```bash -grep -E "menuentry '.*'" /boot/grub/grub.cfg | cut -d"'" -f2 - -# Will output something like: -# Ubuntu -# Advanced options for Ubuntu -# Ubuntu, with Linux 5.15.0-50-generic -# Ubuntu, with Linux 5.15.0-50-generic (recovery mode) -# Ubuntu, with Linux 5.13.0-48-generic -# Ubuntu, with Linux 5.13.0-48-generic (recovery mode) - -# Now, pick the default kernel by setting the GRUB boot index to "1" -# (for Advanced options), and then pick your kernel as 0-indexed. -sudo vi /etc/default/grub - -# For "Ubuntu, with Linux 5.13.0-48-generic" -GRUB_DEFAULT="1>2" +cd ../svsm-linux +cp /boot/config-$(uname -r) .config +# Make sure CONFIG_KVM_AMD_SEV=y and CONFIG_TCG_PLATFORM=y +make olddefconfig +make -j $(nproc) + +sudo make modules_install +sudo make install +``` + +Lastly, update the `GRUB_DEFAULT` variable in `/etc/default/grub` to: + +``` +GRUB_DEFAULT="Advanced options for Ubuntu>Ubuntu, with Linux 6.11.0-......" +sudo update-grub +sudo reboot now ``` + +which you can copy and paste from `ls -lart /boot`. diff --git a/tasks/__init__.py b/tasks/__init__.py index f9a17f25..9024b708 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -23,8 +23,7 @@ from . import sc2 from . import sev from . import skopeo - -from tasks.coconut import ns as coconut_ns +from . import svsm ns = Collection( base, @@ -50,6 +49,5 @@ sc2, sev, skopeo, + svsm, ) - -ns.add_collection(coconut_ns, name="coconut") diff --git a/tasks/coconut/__init__.py b/tasks/coconut/__init__.py deleted file mode 100644 index 574398b9..00000000 --- a/tasks/coconut/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from invoke import Collection - -from . import qemu -from . import ovmf -from . import svsm - -ns = Collection(qemu, ovmf, svsm) diff --git a/tasks/coconut/ovmf.py b/tasks/coconut/ovmf.py deleted file mode 100644 index 673c2a4d..00000000 --- a/tasks/coconut/ovmf.py +++ /dev/null @@ -1,16 +0,0 @@ -from invoke import task -from os.path import join -from tasks.util.env import BIN_DIR -from tasks.util.docker import copy_from_ctr_image - -# refer to -# https://github.com/coconut-svsm/svsm/blob/main/Documentation/docs/installation/INSTALL.md - -OVMF_IMAGE_TAG = "ovmf-svsm-build" - - -@task -def build(ctx): - ctr_path = "/root/edk2/Build/OvmfX64/DEBUG_GCC5/FV/OVMF.fd" - host_path = join(BIN_DIR, "ovmf-svsm.fd") - copy_from_ctr_image(OVMF_IMAGE_TAG, ctr_path, host_path) diff --git a/tasks/coconut/qemu.py b/tasks/coconut/qemu.py deleted file mode 100644 index 85bdc317..00000000 --- a/tasks/coconut/qemu.py +++ /dev/null @@ -1,75 +0,0 @@ -from invoke import task -from os.path import join -from subprocess import run -from tasks.util.env import BIN_DIR, PROJ_ROOT, KATA_ROOT -from tasks.util.docker import copy_from_ctr_image, build_image_and_run - -# refer to -# https://github.com/coconut-svsm/svsm/blob/main/Documentation/docs/installation/INSTALL.md - -QEMU_IMAGE_TAG = "qemu-igvm-build" -DATA_DIR = join(KATA_ROOT, "coconut", "qemu-svsm", "share") - - -@task -def build(ctx): - tmp_ctr_name = "tmp-qemu-igvm-run" - - # TODO: fix me - build_image_and_run( - QEMU_IMAGE_TAG, - join(PROJ_ROOT, "docker", "coconut", "qemu.dockerfile"), - tmp_ctr_name, - {"QEMU_DATADIR": DATA_DIR}, - ) - - copy_from_ctr_image( - QEMU_IMAGE_TAG, - "/root/bin/qemu-svsm/bin/qemu-system-x86_64", - join(BIN_DIR, "qemu-system-x86_64-igvm"), - ) - # copy_from_container(tmp_ctr_name, f"{DATA_DIR}/.", DATA_DIR) - - -@task -def guest(ctx, guest_img_path=join(PROJ_ROOT, "arch_7.qcow2"), detach=False, vnc=False): - qemu_path = join(BIN_DIR, "qemu-system-x86_64-igvm") - igvm_path = join(BIN_DIR, "coconut-qemu.igvm") - - qemu_cmd = [ - "sudo", - qemu_path, - "-enable-kvm", - "-cpu EPYC-v4", - "-machine q35,confidential-guest-support=sev0,memory-backend=ram1", - ( - "-object memory-backend-memfd,id=ram1,size=8G,share=true," - "prealloc=false,reserve=false" - ), - ( - "-object sev-snp-guest,id=sev0,cbitpos=51," - "reduced-phys-bits=1,igvm-file={}" - ).format(igvm_path), - "-smp 8", - "-no-reboot", - ( - "-netdev user,id=vmnic,hostfwd=tcp::2222-:22," - "hostfwd=tcp::8080-:80 -device e1000,netdev=vmnic,romfile=" - ), - "-device virtio-scsi-pci,id=scsi0,disable-legacy=on,iommu_platform=on", - "-device scsi-hd,drive=disk0,bootindex=0", - "-drive file={},if=none,id=disk0,format=qcow2,snapshot=off".format( - guest_img_path - ), - "--serial file:tmp.log", - "-display none", - ] - if detach: - qemu_cmd.append("-daemonize") - - if vnc: - qemu_cmd.append("-vnc :1") - - qemu_cmd = " ".join(qemu_cmd) - print(qemu_cmd) - run(qemu_cmd, shell=True, check=True) diff --git a/tasks/coconut/svsm.py b/tasks/coconut/svsm.py deleted file mode 100644 index c07f1321..00000000 --- a/tasks/coconut/svsm.py +++ /dev/null @@ -1,36 +0,0 @@ -from invoke import task -from os.path import join -from tasks.util.env import BIN_DIR, PROJ_ROOT -from tasks.util.docker import build_image_and_run, copy_from_ctr_image, stop_container - -# refer to -# https://github.com/coconut-svsm/svsm/blob/main/Documentation/docs/installation/INSTALL.md - -QEMU_IMAGE_TAG = "svsm-build" - - -@task -def build(ctx): - tmp_ctr_name = "tmp-svsm-run" - - build_image_and_run( - QEMU_IMAGE_TAG, - join(PROJ_ROOT, "docker", "coconut", "svsm.dockerfile"), - tmp_ctr_name, - {"OVMF_DIR": "bin"}, - ) - - ctr_path = "/root/svsm/bin" - host_path = BIN_DIR - files_to_copy = [ - "svsm.bin", - "coconut-qemu.igvm", - "../target/x86_64-unknown-none/debug/svsm", - ] - # FIXME: sure it is the right tag? - for file_name in files_to_copy: - copy_from_ctr_image( - QEMU_IMAGE_TAG, join(ctr_path, file_name), join(host_path, file_name) - ) - - stop_container(tmp_ctr_name) diff --git a/tasks/kernel.py b/tasks/kernel.py index e02327ab..5e090f93 100644 --- a/tasks/kernel.py +++ b/tasks/kernel.py @@ -1,8 +1,10 @@ from invoke import task from os import makedirs from os.path import exists, join +from shutil import rmtree from tasks.util.env import KATA_CONFIG_DIR, KATA_IMG_DIR, KATA_RUNTIMES, SC2_RUNTIMES from tasks.util.kata import KATA_SOURCE_DIR, copy_from_kata_workon_ctr +from tasks.util.kernel import grub_update_default_kernel from tasks.util.toml import update_toml from tasks.util.versions import GUEST_KERNEL_VERSION from subprocess import run @@ -28,9 +30,9 @@ def build_guest(debug=False, hot_replace=False): script_files = [ "kernel/build-kernel.sh", - "kernel/configs/", + "kernel/configs", "kernel/kata_config_version", - "kernel/patches/", + "kernel/patches", "scripts/apply_patches.sh", "scripts/lib.sh", ] @@ -123,3 +125,37 @@ def hot_replace_guest(ctx, debug=False): Hot-replace guest kernel """ build_guest(debug=debug, hot_replace=True) + + +@task +def install_host_kernel_from_upstream(ctx): + # TODO: find a way to automate the grepping of deb files and kernel name + kernel_ver = "6.13" + kernel_name = f"{kernel_ver}.0-061300-generic" + + tmp_dir = f"/tmp/kernel-{kernel_ver}" + if exists(tmp_dir): + rmtree(tmp_dir) + makedirs(tmp_dir) + + base_url = f"https://kernel.ubuntu.com/mainline/v{kernel_ver}/amd64/" + # The order of this files in the array _matters_ + deb_files = [ + "linux-headers-6.13.0-061300_6.13.0-061300.202501302155_all.deb", + f"linux-headers-{kernel_name}_6.13.0-061300.202501302155_amd64.deb", + f"linux-modules-{kernel_name}_6.13.0-061300.202501302155_amd64.deb", + f"linux-image-unsigned-{kernel_name}_6.13.0-061300.202501302155_amd64.deb", + ] + for deb_file in deb_files: + result = run( + f"wget {base_url}/{deb_file}", shell=True, capture_output=True, cwd=tmp_dir + ) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + for deb_file in deb_files: + result = run( + f"sudo dpkg -i {deb_file}", shell=True, capture_output=True, cwd=tmp_dir + ) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + grub_update_default_kernel(kernel_name) diff --git a/tasks/ovmf.py b/tasks/ovmf.py index da03a0e0..c4af35d0 100644 --- a/tasks/ovmf.py +++ b/tasks/ovmf.py @@ -1,104 +1,42 @@ from invoke import task from os.path import join from subprocess import run -from tasks.util.env import BIN_DIR, KATA_CONFIG_DIR, PROJ_ROOT -from tasks.util.toml import update_toml +from tasks.util.docker import copy_from_ctr_image +from tasks.util.env import GHCR_URL, GITHUB_ORG, KATA_ROOT, PROJ_ROOT +from tasks.util.versions import OVMF_VERSION -OVMF_IMAGE_TAG = "ovmf-build" +OVMF_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, f"ovmf:{OVMF_VERSION}") -def do_ovmf_build(target, patch): +def do_ovmf_build(nocache=False, push=False): docker_cmd = [ "docker build", - "--build-arg TARGET={}".format(target), - "--build-arg OVMF_PATCH={}".format(patch), - "-t {}".format(OVMF_IMAGE_TAG), + f"--build-arg OVMF_VERSION={OVMF_VERSION}", + f"-t {OVMF_IMAGE_TAG}", + "--nocache" if nocache else "", "-f {} .".format(join(PROJ_ROOT, "docker", "ovmf.dockerfile")), ] docker_cmd = " ".join(docker_cmd) - run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) + result = run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + if push: + result = run(f"docker push {OVMF_IMAGE_TAG}", shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) -def copy_ovmf_from_src(dst_path, target, patch): + +def install(): """ Copy a custom build of OVMF into the destination path """ - do_ovmf_build(target, patch) - - # Copy the debug-built OVMF into the destiantion path - tmp_ctr_name = "tmp_ovmf" - docker_cmd = "docker run -td --name {} {}".format(tmp_ctr_name, OVMF_IMAGE_TAG) - run(docker_cmd, shell=True, check=True) - - ctr_fd_path = "/usr/src/edk2/Build/AmdSev/{}_GCC5/FV/OVMF.fd".format(target) - docker_cmd = "docker cp {}:{} {}".format( - tmp_ctr_name, - ctr_fd_path, - dst_path, - ) - run(docker_cmd, shell=True, check=True) - - run("docker rm -f {}".format(tmp_ctr_name), shell=True, check=True) + ctr_paths = ["/git/sc2-sys/edk2/Build/AmdSev/RELEASE_GCC5/FV/OVMF.fd"] + host_paths = [join(KATA_ROOT, "share", "ovmf", "AMDSEV.fd")] + copy_from_ctr_image(OVMF_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True) @task -def build( - ctx, target="RELEASE", patch=join(PROJ_ROOT, "patches", "ovmf_profile.patch") -): +def build(ctx, nocache=False, push=False): """ Build the OVMF work-on container image """ - do_ovmf_build(target, patch) - - -@task -def set_log_level(ctx, log_level): - """ - Set OVMF's log level, must be one in: info, debug, very-debug - - In order to toggle debug logging in OVMF, we need to update the QEMU - command line to include a couple of OVMF flags. To change the QEMU command - line, we use a bash wrapper with the extra flags, and point Kata to the - wrapper script. - - In addition, we need to re-compile OVMF from scratch with some additional - DEBUG statements (which we apply using a patch in `./patches`). - - Note that using a verbose version of OVMF is only supported with the - qemu-sev runtime class. Also note that the DEBUG log level still builds - a RELEASE target which introduces around 0.5 s of overhead to the boot - process. - """ - allowed_log_levels = ["info", "debug", "very-debug"] - if log_level not in allowed_log_levels: - print( - "Unsupported log level '{}'. Must be one in: {}".format( - log_level, allowed_log_levels - ) - ) - return - - default_qemu_path = "/opt/confidential-containers/bin/qemu-system-x86_64" - wrapper_qemu_path = join(BIN_DIR, "qemu_wrapper_ovmf_logging.sh") - qemu_path = default_qemu_path if log_level == "info" else wrapper_qemu_path - - default_fw_path = "/opt/confidential-containers/share/ovmf/AMDSEV.fd" - debug_fw_path = "/opt/confidential-containers/share/ovmf/AMDSEV_CSG.fd" - if log_level != "info": - if log_level == "debug": - patch = join(PROJ_ROOT, "patches", "ovmf_profile.patch") - else: - patch = join(PROJ_ROOT, "patches", "ovmf_overhead_wip.patch") - copy_ovmf_from_src(debug_fw_path, "RELEASE", patch) - fw_path = default_fw_path if log_level == "info" else debug_fw_path - - updated_toml_str = """ - [hypervisor.qemu] - path = "{qemu_path}" - firmware = "{fw_path}" - """.format( - qemu_path=qemu_path, fw_path=fw_path - ) - - conf_file_path = join(KATA_CONFIG_DIR, "configuration-qemu-sev.toml") - update_toml(conf_file_path, updated_toml_str) + do_ovmf_build(nocache=nocache, push=push) diff --git a/tasks/sc2.py b/tasks/sc2.py index bab92fc9..328d2bae 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -2,6 +2,7 @@ from os import environ, makedirs from os.path import exists, join from subprocess import run +from sys import exit from tasks.containerd import ( install as containerd_install, install_bbolt as bbolt_install, @@ -25,7 +26,9 @@ install as operator_install, install_cc_runtime as operator_install_cc_runtime, ) +from tasks.ovmf import install as ovmf_install from tasks.util.containerd import restart_containerd +from tasks.util.docker import pull_artifact_images from tasks.util.env import ( COCO_ROOT, CONF_FILES_DIR, @@ -33,7 +36,6 @@ CONTAINERD_CONFIG_ROOT, KATA_CONFIG_DIR, KATA_ROOT, - KATA_IMAGE_TAG, KATA_IMG_DIR, PROJ_ROOT, SC2_CONFIG_DIR, @@ -46,6 +48,7 @@ replace_agent as replace_kata_agent, replace_shim as replace_kata_shim, ) +from tasks.util.kernel import get_host_kernel_expected_version, get_host_kernel_version from tasks.util.kubeadm import run_kubectl_command from tasks.util.registry import ( HOST_CERT_DIR, @@ -53,7 +56,11 @@ stop as stop_local_registry, ) from tasks.util.toml import update_toml -from tasks.util.versions import COCO_VERSION, GUEST_KERNEL_VERSION, KATA_VERSION +from tasks.util.versions import ( + COCO_VERSION, + GUEST_KERNEL_VERSION, + OVMF_VERSION, +) from time import sleep @@ -212,7 +219,18 @@ def deploy(ctx, debug=False, clean=False): if exists(SC2_DEPLOYMENT_FILE): print(f"ERROR: SC2 already deployed (file {SC2_DEPLOYMENT_FILE} exists)") print("ERROR: only remove deployment file if you know what you are doing!") - raise RuntimeError("SC2 already deployed!") + exit(1) + + # Fail-fast if we are not using the expected host kernel + host_kernel_version = get_host_kernel_version() + host_kernel_expected_version = get_host_kernel_expected_version() + if host_kernel_version != host_kernel_expected_version: + print( + f"ERROR: wrong host kernel: expected {host_kernel_expected_version} " + f"- got {host_kernel_version}" + ) + print("ERROR: install the right host kernel (./docs/host_kernel.md)") + exit(1) if clean: # Remove all directories that we populate and modify @@ -258,6 +276,9 @@ def deploy(ctx, debug=False, clean=False): # Disable swap run("sudo swapoff -a", shell=True, check=True) + # Pull all artifact container images necessary + pull_artifact_images(debug=debug) + # Build and install containerd containerd_install(debug=debug, clean=clean) bbolt_install(debug=debug, clean=clean) @@ -286,13 +307,37 @@ def deploy(ctx, debug=False, clean=False): # Install Knative knative_install(debug=debug) - # Apply general patches to the Kata Agent (and initrd), making sure we - # have the latest patched version - print_dotted_line(f"Pulling latest Kata image (v{KATA_VERSION})") - result = run(f"docker pull {KATA_IMAGE_TAG}", shell=True, capture_output=True) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) - if debug: - print(result.stdout.decode("utf-8").strip()) + # Install an up-to-date version of OVMF (the one currently shipped with + # CoCo is not enough to run on 6.11 and QEMU 9.1) + print_dotted_line(f"Installing OVMF ({OVMF_VERSION})") + ovmf_install() + print("Success!") + + # Update SNP class to use default QEMU (we use host kernel 6.11, so we + # can use upstream QEMU 9.1) + # TODO: remove when bumping to a new CoCo release + qemu_path = join(KATA_ROOT, "bin", "qemu-system-x86_64") + updated_toml_str = """ + [hypervisor.qemu] + path = "{qemu_path}" + valid_hypervisor_paths = [ "{qemu_path}" ] + """.format( + qemu_path=qemu_path + ) + update_toml( + join(KATA_CONFIG_DIR, "configuration-qemu-snp.toml"), + updated_toml_str, + requires_root=True, + ) + + # Apply general patches to the Kata runtime + replace_kata_shim( + dst_shim_binary=join(KATA_ROOT, "bin", "containerd-shim-kata-v2"), + dst_runtime_binary=join(KATA_ROOT, "bin", "kata-runtime"), + sc2=False, + ) + + # Apply general patches to the Kata Agent (and initrd) replace_kata_agent( dst_initrd_path=join( KATA_IMG_DIR, "kata-containers-initrd-confidential-sc2-baseline.img" @@ -300,7 +345,6 @@ def deploy(ctx, debug=False, clean=False): debug=debug, sc2=False, ) - print("Success!") # Install sc2 runtime with patches print_dotted_line(f"Installing SC2 (v{COCO_VERSION})") diff --git a/tasks/svsm.py b/tasks/svsm.py new file mode 100644 index 00000000..d49ac975 --- /dev/null +++ b/tasks/svsm.py @@ -0,0 +1,265 @@ +from invoke import task +from os.path import basename, exists, join +from subprocess import run +from tasks.util.docker import copy_from_ctr_image +from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT, SC2_ROOT +from tasks.util.kata import KATA_AGENT_SOURCE_DIR, KATA_IMAGE_TAG, KATA_SOURCE_DIR +from tasks.util.kernel import get_host_kernel_version +from tasks.util.versions import IGVM_VERSION + +SVSM_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "svsm:main") +SVSM_KERNEL_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "linux:svsm") +SVSM_QEMU_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "qemu:svsm") + +SVSM_ROOT = join(SC2_ROOT, "svsm") +SVSM_QEMU_DATA_DIR = join(SVSM_ROOT, "share") + +SVSM_GUEST_INITRD = join(SVSM_ROOT, "share", "sc2", "initrd-kata.img") + + +def get_kernel_version_from_ctr_image(): + tmp_file = "/tmp/sc2_kernel_release" + copy_from_ctr_image( + SVSM_KERNEL_IMAGE_TAG, + ["/git/coconut-svsm/linux/include/config/kernel.release"], + [tmp_file], + ) + with open(tmp_file, "r") as fh: + kernel_version = fh.read().strip() + kernel_version_trimmed = ( + kernel_version if not kernel_version.endswith("+") else kernel_version[:-1] + ) + + return kernel_version, kernel_version_trimmed + + +def do_build_initrd(clean=False): + if clean and exists(SVSM_GUEST_INITRD): + run(f"sudo rm -f {SVSM_GUEST_INITRD}", shell=True, check=True) + + if exists(SVSM_GUEST_INITRD): + return + + # TODO: this initrd is built using dracut, which is different to how we + # normally build initrd's for SC2 in Kata. Whenever we incorportate the + # SVSM into SC2, we will have to converge this method with the regular + # initrd preparation for SC2. + + # Prepare our rootfs with the kata agent and co. + initrd_base_dir = "/tmp/svsm_initrd_base_dir" + run(f"sudo rm -rf {initrd_base_dir}", shell=True, check=True) + run(f"sudo mkdir -p {initrd_base_dir}", shell=True, check=True) + + host_paths = [ + join(initrd_base_dir, "VERSION"), + join(initrd_base_dir, "kata-agent"), + join(initrd_base_dir, "tools", "osbuilder", "Makefile"), + join(initrd_base_dir, "tools", "osbuilder", "dracut"), + join(initrd_base_dir, "tools", "osbuilder", "initrd-builder"), + join(initrd_base_dir, "tools", "osbuilder", "rootfs-builder"), + join(initrd_base_dir, "tools", "osbuilder", "scripts"), + ] + ctr_paths = [ + f"{KATA_SOURCE_DIR}/VERSION", + f"{KATA_AGENT_SOURCE_DIR}/target/x86_64-unknown-linux-musl/release/kata-agent", + f"{KATA_SOURCE_DIR}/tools/osbuilder/Makefile", + f"{KATA_SOURCE_DIR}/tools/osbuilder/dracut", + f"{KATA_SOURCE_DIR}/tools/osbuilder/initrd-builder", + f"{KATA_SOURCE_DIR}/tools/osbuilder/rootfs-builder", + f"{KATA_SOURCE_DIR}/tools/osbuilder/scripts", + ] + copy_from_ctr_image(KATA_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True) + + # This initrd must contain our agent, but also the kernel modules + # corresponding to an SVSM-enlightened kernel + initrd_cmd = [ + "sudo -E make", + "BUILD_METHOD=dracut", + f"TARGET_INITRD={SVSM_GUEST_INITRD}", + "AGENT_SOURCE_BIN={}".format(join(initrd_base_dir, "kata-agent")), + "DRACUT_KVERSION={}".format(get_host_kernel_version()), + "initrd", + ] + initrd_cmd = " ".join(initrd_cmd) + result = run( + initrd_cmd, + shell=True, + capture_output=True, + cwd=join(initrd_base_dir, "tools", "osbuilder"), + ) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + +def do_build_kernel(nocache=False): + """ + This method builds the forked kernel needed in the SVSM. It is used for + the __guest__ kernel only. + """ + # First, generate the right config file: we start from our current one, + # and make sure the following are set: + # - CONFIG_KVM_AMD_SEV: for general SNP support in KVM + # - CONFIG_TCG_PLATFORM: for vTPM support in the SVSM + current_kernel_name = get_host_kernel_version() + + tmp_file = "/tmp/svsm_kernel_config" + run(f"cp /boot/config-{current_kernel_name} {tmp_file}", shell=True, check=True) + with open(tmp_file, "r") as fh: + kernel_config = fh.read().strip().split("\n") + + snp_set = False + tpm_set = False + for line in kernel_config: + if line.startswith("CONFIG_KVM_AMD_SEV="): + line = "CONFIG_KVM_AMD_SEV=y" + snp_set = True + + if line.startswith("CONFIG_TCG_PLATFORM="): + line = "CONFIG_TCG_PLATFORM=y" + tpm_set = True + + # Cover for the case where the entries are not there at all + if not snp_set: + kernel_config += ["CONFIG_KVM_AMD_SEV=y"] + if not tpm_set: + kernel_config += ["CONFIG_TCG_PLATFORM=y"] + + with open(tmp_file, "w") as fh: + fh.write("\n".join(kernel_config) + "\n") + + build_args = { + "KERNEL_CONFIG_FILE": basename(tmp_file), + "MODULES_OUTDIR": join(SVSM_ROOT, "share", "linux", "modules"), + } + build_args_str = [ + "--build-arg {}={}".format(key, build_args[key]) for key in build_args + ] + build_args_str = " ".join(build_args_str) + + docker_cmd = "docker build{} {} -t {} -f {} /tmp".format( + " --no-cache" if nocache else "", + build_args_str, + # f"{tmp_file}:/tmp/kernel_config", + SVSM_KERNEL_IMAGE_TAG, + join(PROJ_ROOT, "docker", "svsm_kernel.dockerfile"), + ) + run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) + + +def do_build_qemu(nocache=False): + build_args = { + "IGVM_VERSION": IGVM_VERSION, + "QEMU_DATADIR": SVSM_QEMU_DATA_DIR, + "QEMU_PREFIX": SVSM_ROOT, + } + build_args_str = [ + "--build-arg {}={}".format(key, build_args[key]) for key in build_args + ] + build_args_str = " ".join(build_args_str) + + docker_cmd = "docker build{} {} -t {} -f {} .".format( + " --no-cache" if nocache else "", + build_args_str, + SVSM_QEMU_IMAGE_TAG, + join(PROJ_ROOT, "docker", "svsm_qemu.dockerfile"), + ) + run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) + + +def do_install(debug, clean): + if clean and exists(SVSM_ROOT): + result = run(f"sudo rm -rf {SVSM_ROOT}", shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + run(f"sudo mkdir -p ${SVSM_ROOT}", shell=True, check=True) + + # Install guest kernel + copy_from_ctr_image( + SVSM_KERNEL_IMAGE_TAG, + ["/git/coconut-svsm/linux/arch/x86/boot/bzImage"], + [join(SVSM_ROOT, "share", "sc2", "vmlinuz-kata-containers-sc2")], + requires_sudo=True, + ) + + # Install QEMU and OVMF + ctr_paths = [ + join(SVSM_ROOT, "bin", "qemu-system-x86_64"), + join(SVSM_QEMU_DATA_DIR, "qemu"), + "/git/coconut-svsm/edk2/Build/OvmfX64/RELEASE_GCC5/FV/OVMF.fd", + ] + host_paths = [ + join(SVSM_ROOT, "bin", "qemu-system-x86_64"), + join(SVSM_QEMU_DATA_DIR, "qemu"), + join(SVSM_ROOT, "share", "ovmf", "OVMF.fd"), + ] + copy_from_ctr_image(SVSM_QEMU_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True) + + # Prepare the guest's initrd + do_build_initrd(clean=clean) + + # Install SVSM's IGVM image + copy_from_ctr_image( + SVSM_IMAGE_TAG, + ["/git/coconut-svsm/svsm/bin/coconut-qemu.igvm"], + [join(SVSM_ROOT, "share", "igvm", "coconut-qemu.igvm")], + requires_sudo=True, + ) + + +# ------------------------------------------------------------------------------ +# Entry-point tasks +# ------------------------------------------------------------------------------ + + +@task +def build_guest_kernel(ctx, nocache=False, push=False): + """ + Build the host/guest kernel fork to use with the SVSM + """ + do_build_kernel(nocache=nocache) + + +@task +def build_initrd(ctx, clean=False): + """ + Generate an initrd with the kata agent and the different kernel modules. + """ + do_build_initrd(clean=clean) + + +@task +def build_qemu(ctx, nocache=False, push=False): + """ + Build the QEMU fork for its use with the SVSM + """ + do_build_qemu(nocache=nocache) + + if push: + run(f"docker push {SVSM_QEMU_IMAGE_TAG}", shell=True, check=True) + + +@task +def build_svsm(ctx, nocache=False): + build_args = { + "OVMF_FILE": "OVMF.fd", + } + build_args_str = [ + "--build-arg {}={}".format(key, build_args[key]) for key in build_args + ] + build_args_str = " ".join(build_args_str) + + docker_cmd = "docker build{} {} -t {} -f {} {}".format( + " --no-cache" if nocache else "", + build_args_str, + SVSM_IMAGE_TAG, + join(PROJ_ROOT, "docker", "svsm.dockerfile"), + join(SVSM_ROOT, "share", "ovmf"), + ) + run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) + + +@task +def install(ctx, clean=False): + """ + Install guest kernel, QEMU, OVMF, and SVSM IGVM image + """ + do_install(debug=False, clean=clean) diff --git a/tasks/util/docker.py b/tasks/util/docker.py index fba2d7f6..ebfa2e41 100644 --- a/tasks/util/docker.py +++ b/tasks/util/docker.py @@ -1,5 +1,13 @@ +from os.path import dirname, exists from subprocess import run -from tasks.util.env import PROJ_ROOT +from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT, print_dotted_line +from tasks.util.versions import ( + CONTAINERD_VERSION, + KATA_VERSION, + NYDUS_SNAPSHOTTER_VERSION, + NYDUS_VERSION, + OVMF_VERSION, +) def is_ctr_running(ctr_name): @@ -28,7 +36,17 @@ def copy_from_ctr_image(ctr_image, ctr_paths, host_paths, requires_sudo=False): ) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + def cleanup(): + result = run(f"docker rm -f {tmp_ctr_name}", shell=True, capture_output=True) + assert result.returncode == 0 + for ctr_path, host_path in zip(ctr_paths, host_paths): + host_dir = dirname(host_path) + if not exists(host_dir): + mkdir = "sudo mkdir" if requires_sudo else "mkdir" + result = run(f"{mkdir} -p {host_dir}", shell=True, capture_output=True) + assert result.returncode == 0 + try: prefix = "sudo " if requires_sudo else "" result = run( @@ -40,9 +58,10 @@ def copy_from_ctr_image(ctr_image, ctr_paths, host_paths, requires_sudo=False): except AssertionError: stderr = result.stderr.decode("utf-8").strip() print(f"Error copying {ctr_image}:{ctr_path} to {host_path}: {stderr}") - break + cleanup() + raise RuntimeError("Error copying from container!") - result = run(f"docker rm -f {tmp_ctr_name}", shell=True, capture_output=True) + cleanup() def build_image(image_tag, dockerfile, build_args=None): @@ -73,3 +92,23 @@ def stop_container(ctr_name): def build_image_and_run(image_tag, dockerfile, ctr_name, build_args=None): build_image(image_tag, dockerfile, build_args) run_container(image_tag, ctr_name) + + +def pull_artifact_images(debug=False): + print_dotted_line("Pulling artifact container images") + components = ["containerd", "kata-containers", "nydus", "nydus-snapshotter", "ovmf"] + versions = [ + CONTAINERD_VERSION, + KATA_VERSION, + NYDUS_VERSION, + NYDUS_SNAPSHOTTER_VERSION, + OVMF_VERSION, + ] + for component, version in zip(components, versions): + docker_cmd = f"docker pull {GHCR_URL}/{GITHUB_ORG}/{component}:{version}" + result = run(docker_cmd, shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + if debug: + print(result.stdout.decode("utf-8").strip()) + + print("Success!") diff --git a/tasks/util/env.py b/tasks/util/env.py index 4843319c..6cdfc669 100644 --- a/tasks/util/env.py +++ b/tasks/util/env.py @@ -57,6 +57,7 @@ # ---------- SC2 config ---------- +SC2_ROOT = "/opt/sc2" SC2_CONFIG_DIR = join(expanduser("~"), ".config", "sc2") SC2_DEPLOYMENT_FILE = join(SC2_CONFIG_DIR, "DEPLOYED") SC2_RUNTIMES = ["qemu-snp-sc2", "qemu-tdx-sc2"] diff --git a/tasks/util/kata.py b/tasks/util/kata.py index cc565394..050a12d3 100644 --- a/tasks/util/kata.py +++ b/tasks/util/kata.py @@ -108,7 +108,7 @@ def build_pause_image(sc2, debug, hot_replace): makedirs(join(pause_image_build_dir, "static-build")) makedirs(join(pause_image_build_dir, "scripts")) - script_files = ["static-build/pause-image/", "scripts/lib.sh"] + script_files = ["static-build/pause-image", "scripts/lib.sh"] for ctr_path, host_path in zip( [ join( @@ -157,22 +157,15 @@ def build_pause_image(sc2, debug, hot_replace): ) -def replace_agent( - dst_initrd_path=join(KATA_IMG_DIR, "kata-containers-initrd-confidential-sc2.img"), - debug=False, - sc2=False, - hot_replace=False, -): +def prepare_rootfs(tmp_rootfs_base_dir, debug=False, sc2=False, hot_replace=False): """ - Replace the kata-agent with a custom-built one - - We use Kata's `rootfs-builder` to prepare a `rootfs` based on an Ubuntu - image with custom packages, then copy into the rootfs additional files - that we may need, and finally package it using Kata's `initrd-builder`. + This function takes a directory as input, and generates the root-filesystem + needed in SC2 at /rootfs. The result can be consumed + to pack an `initrd` or a .qcow2 image. """ + # ----- Prepare temporary rootfs directory ----- - tmp_rootfs_base_dir = "/tmp/sc2-rootfs-build-dir" tmp_rootfs_dir = join(tmp_rootfs_base_dir, "rootfs") tmp_rootfs_scripts_dir = join(tmp_rootfs_base_dir, "osbuilder") @@ -194,7 +187,7 @@ def replace_agent( script_files = [ "initrd-builder/initrd_builder.sh", "rootfs-builder/rootfs.sh", - "rootfs-builder/nvidia/", + "rootfs-builder/nvidia", "rootfs-builder/ubuntu/config.sh", "rootfs-builder/ubuntu/Dockerfile.in", "rootfs-builder/ubuntu/rootfs_lib.sh", @@ -320,6 +313,26 @@ def replace_agent( check=True, ) + +def replace_agent( + dst_initrd_path=join(KATA_IMG_DIR, "kata-containers-initrd-confidential-sc2.img"), + debug=False, + sc2=False, + hot_replace=False, +): + """ + Replace the kata-agent with a custom-built one + + We use Kata's `rootfs-builder` to prepare a `rootfs` based on an Ubuntu + image with custom packages, then copy into the rootfs additional files + that we may need, and finally package it using Kata's `initrd-builder`. + """ + # Generate rootfs + tmp_rootfs_base_dir = "/tmp/sc2-rootfs-build-dir" + tmp_rootfs_dir = join(tmp_rootfs_base_dir, "rootfs") + tmp_rootfs_scripts_dir = join(tmp_rootfs_base_dir, "osbuilder") + prepare_rootfs(tmp_rootfs_base_dir, debug=debug, sc2=sc2, hot_replace=hot_replace) + # ----- Pack rootfs into initrd using Kata's script ----- work_env = {"AGENT_INIT": "yes"} diff --git a/tasks/util/kernel.py b/tasks/util/kernel.py new file mode 100644 index 00000000..bd4b1179 --- /dev/null +++ b/tasks/util/kernel.py @@ -0,0 +1,42 @@ +from os import environ +from subprocess import run +from tasks.util.versions import HOST_KERNEL_VERSION_SNP, HOST_KERNEL_VERSION_TDX + + +def get_host_kernel_expected_version(): + sc2_runtime_class = environ["SC2_RUNTIME_CLASS"] + if "snp" in sc2_runtime_class: + return HOST_KERNEL_VERSION_SNP + + if "tdx" in sc2_runtime_class: + return HOST_KERNEL_VERSION_TDX + + print("ERROR: neither 'snp' nor 'tdx' detected!") + raise RuntimeError("Error detecting expected host kernel") + + +def get_host_kernel_version(): + # Use a tmp file to not rely on being able to capture our stdout + tmp_file = "/tmp/svsm_kernel_config" + run(f"uname -r > {tmp_file}", shell=True, check=True) + with open(tmp_file, "r") as fh: + current_kernel_name = fh.read().strip() + + return current_kernel_name + + +def grub_update_default_kernel(kernel_version): + """ + This method replaces the GRUB_DEFAULT value + """ + grub_default = f"Advanced options for Ubuntu>Ubuntu, with Linux {kernel_version}" + result = run( + f"sudo sed -i 's/^GRUB_DEFAULT=.*/GRUB_DEFAULT=\"{grub_default}\"/' " + "/etc/default/grub", + shell=True, + capture_output=True, + ) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + result = run("sudo update-grub", shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) diff --git a/tasks/util/versions.py b/tasks/util/versions.py index e0de07f9..8079a1df 100644 --- a/tasks/util/versions.py +++ b/tasks/util/versions.py @@ -28,4 +28,14 @@ KNATIVE_VERSION = "1.15.0" # Kernel versions +# WARNING: if we update the host kernel version, make sure to update it in the +# table in ./docs/host_kernel.md +HOST_KERNEL_VERSION_SNP = "6.11.0-snp-host-cc2568386+" +HOST_KERNEL_VERSION_TDX = "6.8.0-1013-intel" GUEST_KERNEL_VERSION = "6.12.8" + +# Coconut SVSM versions +IGVM_VERSION = "0.3.4" + +# Firmware +OVMF_VERSION = "edk2-stable202411"