From 0b43619d60d1a59bc5585a96c669b764e5c1cf0b Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 18 Feb 2025 16:47:04 +0000 Subject: [PATCH 01/15] more svsm work --- docker/base.dockerfile | 5 + docker/coconut/ovmf.dockerfile | 4 +- docker/coconut/qemu.dockerfile | 57 ------- docker/containerd.dockerfile | 4 +- docker/kata.dockerfile | 3 - docker/nydus.dockerfile | 7 - docker/nydus_snapshotter.dockerfile | 2 - docker/svsm_kernel.dockerfile | 31 ++++ docker/svsm_qemu.dockerfile | 110 ++++++++++++++ tasks/__init__.py | 2 + tasks/sc2.py | 6 + tasks/svsm.py | 220 ++++++++++++++++++++++++++++ tasks/util/docker.py | 24 ++- tasks/util/env.py | 1 + tasks/util/kata.py | 37 +++-- tasks/util/versions.py | 3 + 16 files changed, 429 insertions(+), 87 deletions(-) delete mode 100644 docker/coconut/qemu.dockerfile create mode 100644 docker/svsm_kernel.dockerfile create mode 100644 docker/svsm_qemu.dockerfile create mode 100644 tasks/svsm.py diff --git a/docker/base.dockerfile b/docker/base.dockerfile index 320849e1..afe90c1d 100644 --- a/docker/base.dockerfile +++ b/docker/base.dockerfile @@ -10,10 +10,15 @@ RUN apt update \ && apt upgrade -y \ && apt install -y \ clang \ + cmake \ curl \ + g++ \ + gcc \ git \ + gopls \ libclang-dev \ libdevmapper-dev \ + make \ wget # Clone the dotfiles repo diff --git a/docker/coconut/ovmf.dockerfile b/docker/coconut/ovmf.dockerfile index a1f73cee..dd2b78d6 100644 --- a/docker/coconut/ovmf.dockerfile +++ b/docker/coconut/ovmf.dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:22.04 -RUN sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list +RUN sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list RUN apt update \ && apt upgrade -y \ && apt install -y \ @@ -14,6 +14,6 @@ RUN git clone https://github.com/coconut-svsm/edk2.git ~/edk2\ && git submodule update \ && export PYTHON3_ENABLE=TRUE \ && export PYTHON_COMMAND=python3 \ - && make -j16 -C BaseTools/ \ + && make -j16 -C BaseTools/ \ && . ./edksetup.sh --reconfig \ && build -a X64 -b DEBUG -t GCC5 -D DEBUG_ON_SERIAL_PORT -D DEBUG_VERBOSE -DTPM2_ENABLE -p OvmfPkg/OvmfPkgX64.dsc diff --git a/docker/coconut/qemu.dockerfile b/docker/coconut/qemu.dockerfile deleted file mode 100644 index 8d197f6a..00000000 --- a/docker/coconut/qemu.dockerfile +++ /dev/null @@ -1,57 +0,0 @@ -FROM ubuntu:24.04 - -RUN apt update \ - && apt upgrade -y \ - && apt install -y \ - git \ - make \ - cbindgen \ - curl \ - gcc \ - libcunit1 \ - libcunit1-doc \ - libcunit1-dev \ - gettext \ - python3-venv \ - ninja-build \ - bzip2 \ - libglib2.0-dev \ - rustc \ - iasl \ - build-essential \ - libglib2.0-dev \ - libfdt-dev \ - libpixman-1-dev \ - zlib1g-dev \ - libudev-dev \ - libvdeplug-dev \ - libslirp-dev \ - seabios - -# Clone and build IGVM -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \ - && . "$HOME/.cargo/env" \ - && git clone --branch igvm-v0.1.6 https://github.com/microsoft/igvm/ ~/igvm \ - && cd ~/igvm \ - && make -f igvm_c/Makefile \ - && make -f igvm_c/Makefile install - -# Clone and build IGVM-enabled Qemu -ARG QEMU_DATADIR -RUN git clone https://github.com/coconut-svsm/qemu ~/qemu \ - && cd ~/qemu \ - && git checkout svsm-igvm \ - && export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/lib64/pkgconfig/ \ - && ./configure \ - --datadir=${QEMU_DATADIR} \ - --prefix=$HOME/bin/qemu-svsm/ \ - --target-list=x86_64-softmmu \ - --enable-igvm \ - --static \ - --disable-gio \ - --disable-libudev \ - --enable-kvm \ - --enable-trace-backends=log,simple \ - --enable-slirp \ - && ninja -C build/ \ - && make install -j $(nproc) diff --git a/docker/containerd.dockerfile b/docker/containerd.dockerfile index afd30e88..bb5a61fb 100644 --- a/docker/containerd.dockerfile +++ b/docker/containerd.dockerfile @@ -8,9 +8,7 @@ FROM ghcr.io/sc2-sys/base:0.10.0 RUN apt update \ && apt upgrade -y \ && apt install -y \ - libbtrfs-dev \ - gopls \ - make + libbtrfs-dev # Clone and build containerd ARG CODE_DIR=/go/src/github.com/sc2-sys/containerd diff --git a/docker/kata.dockerfile b/docker/kata.dockerfile index ab2c584b..6ebaf544 100644 --- a/docker/kata.dockerfile +++ b/docker/kata.dockerfile @@ -6,10 +6,7 @@ FROM ghcr.io/sc2-sys/base:0.10.0 # Install APT dependencies RUN apt install -y \ - gcc \ - gopls \ libseccomp-dev \ - make \ musl-tools \ wget diff --git a/docker/nydus.dockerfile b/docker/nydus.dockerfile index 709a22c5..c7a53ab3 100644 --- a/docker/nydus.dockerfile +++ b/docker/nydus.dockerfile @@ -4,13 +4,6 @@ FROM ghcr.io/sc2-sys/base:0.10.0 # Nydus daemon set-up # --------------------------- -# Install APT dependencies -RUN apt-get update \ - && apt-get install -y \ - cmake \ - gopls \ - make - # Build the daemon and other tools like nydusify ARG CODE_DIR=/go/src/github.com/sc2-sys/nydus RUN mkdir -p ${CODE_DIR} \ diff --git a/docker/nydus_snapshotter.dockerfile b/docker/nydus_snapshotter.dockerfile index 57745e89..9933094c 100644 --- a/docker/nydus_snapshotter.dockerfile +++ b/docker/nydus_snapshotter.dockerfile @@ -7,8 +7,6 @@ FROM ghcr.io/sc2-sys/base:0.10.0 # Install APT dependencies RUN apt-get update \ && apt-get install -y \ - gopls \ - make \ protobuf-compiler ARG CODE_DIR=/go/src/github.com/sc2-sys/nydus-snapshotter diff --git a/docker/svsm_kernel.dockerfile b/docker/svsm_kernel.dockerfile new file mode 100644 index 00000000..133a358f --- /dev/null +++ b/docker/svsm_kernel.dockerfile @@ -0,0 +1,31 @@ +FROM ghcr.io/sc2-sys/base:0.10.0 + +RUN apt update \ + && apt upgrade -y \ + && apt install -y \ + bc \ + bison \ + cpio \ + flex \ + kmod \ + libelf-dev \ + libssl-dev \ + xz-utils \ + zstd + +# Clone kernel source tree +ARG CODE_DIR=/git/coconut-svsm/linux +RUN mkdir -p ${CODE_DIR} \ + && git clone \ + --branch svsm \ + --depth=1 https://github.com/coconut-svsm/linux \ + ${CODE_DIR} + +# Copy generated config file. The filename and path are hardcoded in ./tasks/svsm.py +COPY ./svsm_kernel_config ${CODE_DIR}/.config + +ARG MODULES_OUTDIR +RUN cd ${CODE_DIR} \ + && make olddefconfig \ + && make -j $(nproc) \ + && make modules_install INSTALL_MOD_PATH=${MODULES_OUTDIR} diff --git a/docker/svsm_qemu.dockerfile b/docker/svsm_qemu.dockerfile new file mode 100644 index 00000000..d58e8b1e --- /dev/null +++ b/docker/svsm_qemu.dockerfile @@ -0,0 +1,110 @@ +FROM ghcr.io/sc2-sys/base:0.10.0 + +RUN apt update \ + && apt upgrade -y \ + && apt install -y \ + bzip2 \ + cbindgen \ + gettext \ + iasl \ + libcunit1-dev \ + libfdt-dev \ + libglib2.0-dev \ + libpixman-1-dev \ + libudev-dev \ + libvdeplug-dev \ + nasm \ + ninja-build \ + python3-tomli \ + python3-venv \ + seabios \ + zlib1g-dev + +# Clone and build IGVM +ARG IGVM_VERSION +ARG CODE_DIR=/git/microsoft/igvm +RUN mkdir -p ${CODE_DIR} \ + && git clone --branch igvm-v${IGVM_VERSION} https://github.com/microsoft/igvm/ ${CODE_DIR} \ + && cd ${CODE_DIR} \ + && make -f igvm_c/Makefile \ + && make -f igvm_c/Makefile install + +# Clone and build IGVM-enabled Qemu +ARG QEMU_DATADIR +ARG QEMU_PREFIX +ARG CODE_DIR=/git/coconut-svsm/qemu +RUN mkdir -p ${CODE_DIR} \ + && git clone https://github.com/coconut-svsm/qemu ${CODE_DIR} \ + && cd ${CODE_DIR} \ + && git checkout svsm-igvm \ + && export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/lib64/pkgconfig/ \ + && ./configure \ + --cpu=x86_64 \ + # The `--datadir` flag is the path where QEMU will look for firmware + # images. The default `--datadir` path when using a system provisioned + # by the operator is: `/opt/confidential-containers/share/kata-qemu`. + # For our QEMu fork we use `/opt/sc2/svsm/share/qemu + --datadir=${QEMU_DATADIR} \ + --prefix=${QEMU_PREFIX} \ + --target-list=x86_64-softmmu \ + # Must enable IGVM + --enable-igvm \ + --enable-kvm \ + --enable-trace-backends=log,simple \ + # As a reference we use Kata's --disable-x flags when building QEMU: + # https://github.com/kata-containers/kata-containers/blob/main/tools/packaging/scripts/configure-hypervisor.sh + --disable-auth-pam \ + --disable-brlapi \ + --disable-bsd-user \ + --disable-capstone \ + --disable-curl \ + --disable-curses \ + --disable-debug-tcg \ + --disable-docs \ + --disable-gio \ + --disable-glusterfs \ + --disable-gtk \ + --disable-guest-agent \ + --disable-guest-agent-msi \ + --disable-libiscsi \ + --disable-libudev \ + --disable-libnfs \ + --disable-libusb \ + --disable-linux-user \ + --disable-lzo \ + --disable-opengl \ + --disable-rdma \ + --disable-replication \ + --disable-sdl \ + --disable-slirp \ + --disable-snappy \ + --disable-spice \ + --disable-tcg-interpreter \ + --disable-tools \ + --disable-tpm \ + --disable-usb-redir \ + --disable-vde \ + --disable-vte \ + --disable-virglrenderer \ + --disable-vnc \ + --disable-vnc-jpeg \ + --disable-vnc-sasl \ + --disable-vte \ + --disable-xen \ + --static \ + && make -j $(nproc) \ + && make install -j $(nproc) + +# Clone and build SVSM's OVMF +ARG CODE_DIR=/git/coconut-svsm/edk2 +RUN mkdir -p ${CODE_DIR} \ + && git clone https://github.com/coconut-svsm/edk2 ${CODE_DIR} \ + && cd ${CODE_DIR} \ + && git checkout svsm \ + && git submodule init \ + && git submodule update \ + && export PYTHON3_ENABLE=TRUE \ + && export PYTHON_COMMAND=python3 \ + && make -j $(nproc) -C BaseTools/ \ + && . ./edksetup.sh --reconfig \ + && build -a X64 -b RELEASE -t GCC5 -DTPM2_ENABLE -p OvmfPkg/OvmfPkgX64.dsc diff --git a/tasks/__init__.py b/tasks/__init__.py index f9a17f25..89323acc 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -23,6 +23,7 @@ from . import sc2 from . import sev from . import skopeo +from . import svsm from tasks.coconut import ns as coconut_ns @@ -50,6 +51,7 @@ sc2, sev, skopeo, + svsm, ) ns.add_collection(coconut_ns, name="coconut") diff --git a/tasks/sc2.py b/tasks/sc2.py index bab92fc9..8845b4af 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -26,6 +26,7 @@ install_cc_runtime as operator_install_cc_runtime, ) from tasks.util.containerd import restart_containerd +from tasks.util.docker import pull_artifact_images from tasks.util.env import ( COCO_ROOT, CONF_FILES_DIR, @@ -214,6 +215,8 @@ def deploy(ctx, debug=False, clean=False): print("ERROR: only remove deployment file if you know what you are doing!") raise RuntimeError("SC2 already deployed!") + # TODO: Fail-fast if we are not using the expected host kernel + if clean: # Remove all directories that we populate and modify for nuked_dir in [ @@ -258,6 +261,9 @@ def deploy(ctx, debug=False, clean=False): # Disable swap run("sudo swapoff -a", shell=True, check=True) + # Pull all artifact container images necessary + pull_artifact_images(debug=debug) + # Build and install containerd containerd_install(debug=debug, clean=clean) bbolt_install(debug=debug, clean=clean) diff --git a/tasks/svsm.py b/tasks/svsm.py new file mode 100644 index 00000000..87509cb2 --- /dev/null +++ b/tasks/svsm.py @@ -0,0 +1,220 @@ +from invoke import task +from os.path import basename, exists, join +from subprocess import run +from tasks.util.docker import copy_from_ctr_image +from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT, SC2_ROOT +from tasks.util.kata import prepare_rootfs +from tasks.util.versions import IGVM_VERSION + +SVSM_KERNEL_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "linux:svsm") +SVSM_QEMU_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "qemu:svsm") + +SVSM_ROOT = join(SC2_ROOT, "svsm") +SVSM_QEMU_DATA_DIR = join(SVSM_ROOT, "share", "qemu") + +SVSM_GUEST_IMAGE = join(SVSM_QEMU_DATA_DIR, "sc2.qcow2") +# Can we do with less? +SVSM_GUEST_IMAGE_SIZE = "10G" + + +@task +def build_guest_image(ctx, clean=False): + if clean and exists(SVSM_GUEST_IMAGE): + run(f"sudo rm -f {SVSM_GUEST_IMAGE}", shell=True, check=True) + + # Get the kernel version we will install in the guest image + tmp_file = "/tmp/sc2_kernel_release" + copy_from_ctr_image(SVSM_KERNEL_IMAGE_TAG, ["/git/coconut-svsm/linux/include/config/kernel.release"], [tmp_file]) + with open(tmp_file, "r") as fh: + kernel_version = fh.read().strip() + kernel_version_trimmed = kernel_version if not kernel_version.endswith("+") else kernel_version[:-1] + + # Prepare our rootfs with the kata agent and co. + rootfs_base_dir = "/tmp/svsm_rootfs_base_dir" + prepare_rootfs(rootfs_base_dir, debug=False, sc2=True, hot_replace=False) + rootfs_dir = join(rootfs_base_dir, "rootfs") + + # Install deps + result = run("sudo DEBIAN_FRONTEND=noninteractive apt install -y qemu-utils libguestfs-tools", shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + # Create qcow image + result = run(f"qemu-img create -f qcow2 {SVSM_GUEST_IMAGE} {SVSM_GUEST_IMAGE_SIZE}", shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + # Format it as an ext4 fielsystem + result = run(f"mkfs.ext4 {SVSM_GUEST_IMAGE}", shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + # Mount the qcow image to copy in our rootfs + mount_dir = "/tmp/svsm_guest_image" + result = run(f"guestmount -a {SVSM_GUEST_IMAGE} -m /dev/sda1 {mount_dir}", shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + # Copy our rootfs into the qcow image + result = run(f"cp -a {rootfs_dir}/* {mount_dir}", shell=True, check=True) + + # Install the SVSM guest kernel into the image + run(f"mkdir -p {mount_dir}/boot", shell=True, check=True) + ctr_paths = [ + "/git/coconut-svsm/linux/arch/x86/boot/bzImage", + f"/opt/sc2/svsm/share/linux/modules/{kernel_version}", + ] + host_paths = [ + f"{mount_dir}/boot/vmlinuz-{kernel_version_trimmed}", + f"{mount_dir}/lib/modules/{kernel_version_trimmed}", + ] + copy_from_ctr_image(SVSM_KERNEL_IMAGE_TAG, ctr_paths, host_paths) + + # Configure GRUB inside the guest image + subsystems = ["dev", "proc", "sys"] + for subsystem in subsystems: + run(f"mount --bind /{subsystem} {mount_dir}/{subsystem}", shell=True, check=True) + cmd = """ +chroot {mount_dir} /bin/bash <> /etc/default/grub +update-grub +EOF +""".format(mount_dir=mount_dir, kernel_version=kernel_version_trimmed) + + # Clean-up + for subsystem in subsystems: + run(f"umount --bind /{subsystem} {mount_dir}/{subsystem}", shell=True, check=True) + # result = run(f"guestunmount {mount_dir}", shell=True, capture_output=True) + # assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + +def do_build_kernel(nocache=False): + # First, generate the right config file: we start from our current one, + # and make sure the following are set: + # - CONFIG_KVM_AMD_SEV: for general SNP support in KVM + # - CONFIG_TCG_PLATFORM: for vTPM support in the SVSM + + # TODO: move to tasks/util/kernel.py + # Use a tmp file to not rely on being able to capture our stdout + tmp_file = "/tmp/svsm_kernel_config" + run(f"uname -r > {tmp_file}", shell=True, check=True) + with open(tmp_file, "r") as fh: + current_kernel_name = fh.read().strip() + + run(f"cp /boot/config-{current_kernel_name} {tmp_file}", shell=True, check=True) + with open(tmp_file, "r") as fh: + kernel_config = fh.read().strip().split("\n") + + snp_set = False + tpm_set = False + for line in kernel_config: + if line.startswith("CONFIG_KVM_AMD_SEV="): + line = "CONFIG_KVM_AMD_SEV=y" + snp_set = True + + if line.startswith("CONFIG_TCG_PLATFORM="): + line = "CONFIG_TCG_PLATFORM=y" + tpm_set = True + + # Cover for the case where the entries are not there at all + if not snp_set: + kernel_config += ["CONFIG_KVM_AMD_SEV=y"] + if not tpm_set: + kernel_config += ["CONFIG_TCG_PLATFORM=y"] + + with open(tmp_file, "w") as fh: + fh.write("\n".join(kernel_config) + "\n") + + build_args = { + "KERNEL_CONFIG_FILE": basename(tmp_file), + "MODULES_OUTDIR": join(SVSM_ROOT, "share", "linux", "modules") + } + build_args_str = [ + "--build-arg {}={}".format(key, build_args[key]) for key in build_args + ] + build_args_str = " ".join(build_args_str) + + docker_cmd = "docker build{} {} -t {} -f {} /tmp".format( + " --no-cache" if nocache else "", + build_args_str, + # f"{tmp_file}:/tmp/kernel_config", + SVSM_KERNEL_IMAGE_TAG, + join(PROJ_ROOT, "docker", "svsm_kernel.dockerfile"), + ) + run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) + + +@task +def build_kernel(ctx, nocache=False, push=False): + """ + Build the host/guest kernel fork to use with the SVSM + """ + do_build_kernel(nocache=nocache) + + +def do_build_qemu(nocache=False): + build_args = { + "IGVM_VERSION": IGVM_VERSION, + "QEMU_DATADIR": SVSM_QEMU_DATA_DIR, + "QEMU_PREFIX": SVSM_ROOT, + } + build_args_str = [ + "--build-arg {}={}".format(key, build_args[key]) for key in build_args + ] + build_args_str = " ".join(build_args_str) + + docker_cmd = "docker build{} {} -t {} -f {} .".format( + " --no-cache" if nocache else "", + build_args_str, + SVSM_QEMU_IMAGE_TAG, + join(PROJ_ROOT, "docker", "svsm_qemu.dockerfile"), + ) + run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) + + +@task +def build_qemu(ctx, nocache=False, push=False): + """ + Build the QEMU fork for its use with the SVSM + """ + do_build_qemu(nocache=nocache) + + if push: + run(f"docker push {SVSM_QEMU_IMAGE_TAG}", shell=True, check=True) + + +def install(debug, clean): + if clean and exists(SVSM_ROOT): + result = run(f"sudo rm -rf {SVSM_ROOT}", shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + run(f"sudo mkdir -p ${SVSM_ROOT}", shell=True, check=True) + + # TODO: install guest kernel + + # Install QEMU and OVMF + ctr_paths = [ + join(SVSM_ROOT, "bin", "qemu-system-x86_64"), + join(SVSM_ROOT, "share", "qemu", "qemu/"), + "/git/coconut-svsm/edk2/Build/OvmfX64/RELEASE_GCC5/FV/OVMF.fd" + ] + host_paths = [ + join(SVSM_ROOT, "bin", "qemu-system-x86_64"), + join(SVSM_ROOT, "share", "qemu"), + join(SVSM_ROOT, "share", "ovmf", "OVMF.fd"), + ] + copy_from_ctr_image(SVSM_QEMU_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True) + + +@task +def foo(ctx): + install(debug=False, clean=False) diff --git a/tasks/util/docker.py b/tasks/util/docker.py index fba2d7f6..9ff6b3b4 100644 --- a/tasks/util/docker.py +++ b/tasks/util/docker.py @@ -1,5 +1,7 @@ +from os.path import dirname, exists from subprocess import run -from tasks.util.env import PROJ_ROOT +from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT, print_dotted_line +from tasks.util.versions import CONTAINERD_VERSION, KATA_VERSION, NYDUS_SNAPSHOTTER_VERSION, NYDUS_VERSION def is_ctr_running(ctr_name): @@ -29,6 +31,12 @@ def copy_from_ctr_image(ctr_image, ctr_paths, host_paths, requires_sudo=False): assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) for ctr_path, host_path in zip(ctr_paths, host_paths): + host_dir = dirname(host_path) + if not exists(host_dir): + mkdir = "sudo mkdir" if requires_sudo else "mkdir" + result = run(f"{mkdir} -p {host_dir}", shell=True, capture_output=True) + assert result.returncode == 0 + try: prefix = "sudo " if requires_sudo else "" result = run( @@ -73,3 +81,17 @@ def stop_container(ctr_name): def build_image_and_run(image_tag, dockerfile, ctr_name, build_args=None): build_image(image_tag, dockerfile, build_args) run_container(image_tag, ctr_name) + + +def pull_artifact_images(debug=False): + print_dotted_line("Pulling artifact container images") + components = ["containerd", "kata-containers", "nydus", "nydus-snapshotter"] + versions = [CONTAINERD_VERSION, KATA_VERSION, NYDUS_VERSION, NYDUS_SNAPSHOTTER_VERSION] + for component, version in zip(components, versions): + docker_cmd = f"docker pull {GHCR_URL}/{GITHUB_ORG}/{component}:{version}" + result = run(docker_cmd, shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + if debug: + print(result.stdout.decode("utf-8").strip()) + + print("Success!") diff --git a/tasks/util/env.py b/tasks/util/env.py index 4843319c..6cdfc669 100644 --- a/tasks/util/env.py +++ b/tasks/util/env.py @@ -57,6 +57,7 @@ # ---------- SC2 config ---------- +SC2_ROOT = "/opt/sc2" SC2_CONFIG_DIR = join(expanduser("~"), ".config", "sc2") SC2_DEPLOYMENT_FILE = join(SC2_CONFIG_DIR, "DEPLOYED") SC2_RUNTIMES = ["qemu-snp-sc2", "qemu-tdx-sc2"] diff --git a/tasks/util/kata.py b/tasks/util/kata.py index cc565394..caea06b3 100644 --- a/tasks/util/kata.py +++ b/tasks/util/kata.py @@ -157,22 +157,15 @@ def build_pause_image(sc2, debug, hot_replace): ) -def replace_agent( - dst_initrd_path=join(KATA_IMG_DIR, "kata-containers-initrd-confidential-sc2.img"), - debug=False, - sc2=False, - hot_replace=False, -): +def prepare_rootfs(tmp_rootfs_base_dir, debug=False, sc2=False, hot_replace=False): """ - Replace the kata-agent with a custom-built one - - We use Kata's `rootfs-builder` to prepare a `rootfs` based on an Ubuntu - image with custom packages, then copy into the rootfs additional files - that we may need, and finally package it using Kata's `initrd-builder`. + This function takes a directory as input, and generates the root-filesystem + needed in SC2 at /rootfs. The result can be consumed + to pack an `initrd` or a .qcow2 image. """ + # ----- Prepare temporary rootfs directory ----- - tmp_rootfs_base_dir = "/tmp/sc2-rootfs-build-dir" tmp_rootfs_dir = join(tmp_rootfs_base_dir, "rootfs") tmp_rootfs_scripts_dir = join(tmp_rootfs_base_dir, "osbuilder") @@ -320,6 +313,26 @@ def replace_agent( check=True, ) + +def replace_agent( + dst_initrd_path=join(KATA_IMG_DIR, "kata-containers-initrd-confidential-sc2.img"), + debug=False, + sc2=False, + hot_replace=False, +): + """ + Replace the kata-agent with a custom-built one + + We use Kata's `rootfs-builder` to prepare a `rootfs` based on an Ubuntu + image with custom packages, then copy into the rootfs additional files + that we may need, and finally package it using Kata's `initrd-builder`. + """ + # Generate rootfs + tmp_rootfs_base_dir = "/tmp/sc2-rootfs-build-dir" + tmp_rootfs_dir = join(tmp_rootfs_base_dir, "rootfs") + tmp_rootfs_scripts_dir = join(tmp_rootfs_base_dir, "osbuilder") + prepare_rootfs(tmp_rootfs_base_dir, debug=debug, sc2=sc2, hot_replace=hot_replace) + # ----- Pack rootfs into initrd using Kata's script ----- work_env = {"AGENT_INIT": "yes"} diff --git a/tasks/util/versions.py b/tasks/util/versions.py index e0de07f9..54056b49 100644 --- a/tasks/util/versions.py +++ b/tasks/util/versions.py @@ -29,3 +29,6 @@ # Kernel versions GUEST_KERNEL_VERSION = "6.12.8" + +# Coconut SVSM versions +IGVM_VERSION = "0.3.4" From 41f3cddee6f28821413b838b88d1791cebb08a6a Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 18 Feb 2025 18:55:16 +0000 Subject: [PATCH 02/15] more wip --- docker/coconut/svsm.dockerfile | 2 +- docker/svsm.dockerfile | 21 ++++++++ tasks/svsm.py | 95 ++++++++++++++++++++++++++++------ tasks/util/kata.py | 4 +- 4 files changed, 103 insertions(+), 19 deletions(-) create mode 100644 docker/svsm.dockerfile diff --git a/docker/coconut/svsm.dockerfile b/docker/coconut/svsm.dockerfile index 87e4c9b9..71bb4a87 100644 --- a/docker/coconut/svsm.dockerfile +++ b/docker/coconut/svsm.dockerfile @@ -25,4 +25,4 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \ && cd ~/svsm \ && git submodule update --init \ && cargo install bindgen-cli \ - && FW_FILE=/bin/ovmf-svsm.fd make + && FW_FILE=/bin/ovmf-svsm.fd ./build --release configs/qemu-target.json diff --git a/docker/svsm.dockerfile b/docker/svsm.dockerfile new file mode 100644 index 00000000..2d02c4c5 --- /dev/null +++ b/docker/svsm.dockerfile @@ -0,0 +1,21 @@ +FROM ghcr.io/sc2-sys/base:0.10.0 + +RUN apt update \ + && apt upgrade -y \ + && apt install -y \ + autoconf \ + autoconf-archive \ + libssl-dev \ + pkg-config + +ARG OVMF_FILE +COPY ./${OVMF_FILE} /bin/ovmf-svsm.fd + +ARG CODE_DIR=/git/coconut-svsm/svsm +RUN mkdir -p ${CODE_DIR} \ + && git clone https://github.com/coconut-svsm/svsm ${CODE_DIR} \ + && cd ${CODE_DIR} \ + && git submodule update --init \ + && rustup target add x86_64-unknown-none \ + && cargo install bindgen-cli + # && FW_FILE=/bin/ovmf-svsm.fd make diff --git a/tasks/svsm.py b/tasks/svsm.py index 87509cb2..ed069c10 100644 --- a/tasks/svsm.py +++ b/tasks/svsm.py @@ -6,6 +6,7 @@ from tasks.util.kata import prepare_rootfs from tasks.util.versions import IGVM_VERSION +SVSM_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "svsm:main") SVSM_KERNEL_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "linux:svsm") SVSM_QEMU_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "qemu:svsm") @@ -19,6 +20,13 @@ @task def build_guest_image(ctx, clean=False): + """ + This function generates a qcow2 guest image with the SVSM-enlightened + guest kernel. + + TODO: check if we really need a qcow image or we can get away with an + initrd. + """ if clean and exists(SVSM_GUEST_IMAGE): run(f"sudo rm -f {SVSM_GUEST_IMAGE}", shell=True, check=True) @@ -39,60 +47,95 @@ def build_guest_image(ctx, clean=False): assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) # Create qcow image - result = run(f"qemu-img create -f qcow2 {SVSM_GUEST_IMAGE} {SVSM_GUEST_IMAGE_SIZE}", shell=True, capture_output=True) + result = run(f"sudo qemu-img create -f qcow2 -o preallocation=metadata {SVSM_GUEST_IMAGE} {SVSM_GUEST_IMAGE_SIZE}", shell=True, capture_output=True) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) # Format it as an ext4 fielsystem - result = run(f"mkfs.ext4 {SVSM_GUEST_IMAGE}", shell=True, capture_output=True) + result = run(f"sudo mkfs.ext4 {SVSM_GUEST_IMAGE}", shell=True, capture_output=True) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) # Mount the qcow image to copy in our rootfs mount_dir = "/tmp/svsm_guest_image" - result = run(f"guestmount -a {SVSM_GUEST_IMAGE} -m /dev/sda1 {mount_dir}", shell=True, capture_output=True) + if exists(mount_dir): + run(f"sudo rm -rf {mount_dir}", shell=True, check=True) + run(f"sudo mkdir -p {mount_dir}", shell=True, check=True) + result = run(f"sudo guestmount -a {SVSM_GUEST_IMAGE} -m /dev/sda {mount_dir}", shell=True, capture_output=True) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) # Copy our rootfs into the qcow image - result = run(f"cp -a {rootfs_dir}/* {mount_dir}", shell=True, check=True) + result = run(f"sudo cp -a {rootfs_dir}/* {mount_dir}", shell=True, check=True) # Install the SVSM guest kernel into the image - run(f"mkdir -p {mount_dir}/boot", shell=True, check=True) + run(f"sudo mkdir -p {mount_dir}/boot", shell=True, check=True) ctr_paths = [ "/git/coconut-svsm/linux/arch/x86/boot/bzImage", - f"/opt/sc2/svsm/share/linux/modules/{kernel_version}", + f"/opt/sc2/svsm/share/linux/modules/lib/modules/{kernel_version}", ] host_paths = [ f"{mount_dir}/boot/vmlinuz-{kernel_version_trimmed}", f"{mount_dir}/lib/modules/{kernel_version_trimmed}", ] - copy_from_ctr_image(SVSM_KERNEL_IMAGE_TAG, ctr_paths, host_paths) + copy_from_ctr_image(SVSM_KERNEL_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True) # Configure GRUB inside the guest image subsystems = ["dev", "proc", "sys"] + + def unmount_subsys(): + for subsystem in subsystems: + run(f"sudo umount {mount_dir}/{subsystem}", shell=True, check=True) + for subsystem in subsystems: - run(f"mount --bind /{subsystem} {mount_dir}/{subsystem}", shell=True, check=True) + run(f"sudo mount --bind /{subsystem} {mount_dir}/{subsystem}", shell=True, check=True) + + # First, install manually install apt in the rootfs + for apt_dir in ["/usr/bin/apt", "/usr/lib/apt", "/var/lib/apt", "/var/lib/dpkg"]: + if apt_dir == "/usr/bin/apt": + result = run(f"sudo cp -r {apt_dir}* {mount_dir}/usr/bin/", capture_output=True, shell=True) + else: + result = run(f"sudo cp -r {apt_dir} {mount_dir}{apt_dir}", capture_output=True, shell=True) + + if result.returncode != 0: + print(result.stderr.decode("utf-8").strip()) + unmount_subsys() + raise RuntimeError("Error installing APT") + + for shared_lib in ["libapt-private.so.0.0", "libapt-pkg.so.6.0", "libstdc++.so.6", "libxxhash.so.0"]: + result = run(f"sudo cp /lib/x86_64-linux-gnu/{shared_lib} {mount_dir}/lib/x86_64-linux-gnu/{shared_lib}", capture_output=True, shell=True) + if result.returncode != 0: + print(result.stderr.decode("utf-8").strip()) + unmount_subsys() + raise RuntimeError("Error installing APT shared libs") + cmd = """ -chroot {mount_dir} /bin/bash <> /etc/default/grub update-grub EOF """.format(mount_dir=mount_dir, kernel_version=kernel_version_trimmed) + result = run(cmd, shell=True, capture_output=True) + if result.returncode != 0: + print(result.stderr.decode("utf-8").strip()) + unmount_subsys() + raise RuntimeError("Error setting default kernel") # Clean-up - for subsystem in subsystems: - run(f"umount --bind /{subsystem} {mount_dir}/{subsystem}", shell=True, check=True) + unmount_subsys() # result = run(f"guestunmount {mount_dir}", shell=True, capture_output=True) # assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) @@ -181,6 +224,26 @@ def do_build_qemu(nocache=False): run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) +@task +def do_build_svsm(ctx, nocache=False): + build_args = { + "OVMF_FILE": "OVMF.fd", + } + build_args_str = [ + "--build-arg {}={}".format(key, build_args[key]) for key in build_args + ] + build_args_str = " ".join(build_args_str) + + docker_cmd = "docker build{} {} -t {} -f {} {}".format( + " --no-cache" if nocache else "", + build_args_str, + SVSM_IMAGE_TAG, + join(PROJ_ROOT, "docker", "svsm.dockerfile"), + join(SVSM_ROOT, "share", "ovmf") + ) + run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) + + @task def build_qemu(ctx, nocache=False, push=False): """ diff --git a/tasks/util/kata.py b/tasks/util/kata.py index caea06b3..050a12d3 100644 --- a/tasks/util/kata.py +++ b/tasks/util/kata.py @@ -108,7 +108,7 @@ def build_pause_image(sc2, debug, hot_replace): makedirs(join(pause_image_build_dir, "static-build")) makedirs(join(pause_image_build_dir, "scripts")) - script_files = ["static-build/pause-image/", "scripts/lib.sh"] + script_files = ["static-build/pause-image", "scripts/lib.sh"] for ctr_path, host_path in zip( [ join( @@ -187,7 +187,7 @@ def prepare_rootfs(tmp_rootfs_base_dir, debug=False, sc2=False, hot_replace=Fals script_files = [ "initrd-builder/initrd_builder.sh", "rootfs-builder/rootfs.sh", - "rootfs-builder/nvidia/", + "rootfs-builder/nvidia", "rootfs-builder/ubuntu/config.sh", "rootfs-builder/ubuntu/Dockerfile.in", "rootfs-builder/ubuntu/rootfs_lib.sh", From a6e8ada63ed838fc16fab63501773b88fa8f54e4 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 19 Feb 2025 10:36:18 +0000 Subject: [PATCH 03/15] nits: run clang format --- tasks/svsm.py | 90 +++++++++++++++++++++++++++++++++++--------- tasks/util/docker.py | 14 ++++++- 2 files changed, 84 insertions(+), 20 deletions(-) diff --git a/tasks/svsm.py b/tasks/svsm.py index ed069c10..6a009e36 100644 --- a/tasks/svsm.py +++ b/tasks/svsm.py @@ -32,22 +32,39 @@ def build_guest_image(ctx, clean=False): # Get the kernel version we will install in the guest image tmp_file = "/tmp/sc2_kernel_release" - copy_from_ctr_image(SVSM_KERNEL_IMAGE_TAG, ["/git/coconut-svsm/linux/include/config/kernel.release"], [tmp_file]) + copy_from_ctr_image( + SVSM_KERNEL_IMAGE_TAG, + ["/git/coconut-svsm/linux/include/config/kernel.release"], + [tmp_file], + ) with open(tmp_file, "r") as fh: kernel_version = fh.read().strip() - kernel_version_trimmed = kernel_version if not kernel_version.endswith("+") else kernel_version[:-1] + kernel_version_trimmed = ( + kernel_version if not kernel_version.endswith("+") else kernel_version[:-1] + ) # Prepare our rootfs with the kata agent and co. rootfs_base_dir = "/tmp/svsm_rootfs_base_dir" - prepare_rootfs(rootfs_base_dir, debug=False, sc2=True, hot_replace=False) + # TODO: move hot_replace to False when done experimenting + prepare_rootfs(rootfs_base_dir, debug=False, sc2=True, hot_replace=True) rootfs_dir = join(rootfs_base_dir, "rootfs") # Install deps - result = run("sudo DEBIAN_FRONTEND=noninteractive apt install -y qemu-utils libguestfs-tools", shell=True, capture_output=True) + result = run( + "sudo DEBIAN_FRONTEND=noninteractive apt install -y " + "qemu-utils libguestfs-tools", + shell=True, + capture_output=True, + ) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) # Create qcow image - result = run(f"sudo qemu-img create -f qcow2 -o preallocation=metadata {SVSM_GUEST_IMAGE} {SVSM_GUEST_IMAGE_SIZE}", shell=True, capture_output=True) + result = run( + f"sudo qemu-img create -f qcow2 -o preallocation=metadata {SVSM_GUEST_IMAGE} " + f"{SVSM_GUEST_IMAGE_SIZE}", + shell=True, + capture_output=True, + ) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) # Format it as an ext4 fielsystem @@ -59,7 +76,11 @@ def build_guest_image(ctx, clean=False): if exists(mount_dir): run(f"sudo rm -rf {mount_dir}", shell=True, check=True) run(f"sudo mkdir -p {mount_dir}", shell=True, check=True) - result = run(f"sudo guestmount -a {SVSM_GUEST_IMAGE} -m /dev/sda {mount_dir}", shell=True, capture_output=True) + result = run( + f"sudo guestmount -a {SVSM_GUEST_IMAGE} -m /dev/sda {mount_dir}", + shell=True, + capture_output=True, + ) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) # Copy our rootfs into the qcow image @@ -75,7 +96,9 @@ def build_guest_image(ctx, clean=False): f"{mount_dir}/boot/vmlinuz-{kernel_version_trimmed}", f"{mount_dir}/lib/modules/{kernel_version_trimmed}", ] - copy_from_ctr_image(SVSM_KERNEL_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True) + copy_from_ctr_image( + SVSM_KERNEL_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True + ) # Configure GRUB inside the guest image subsystems = ["dev", "proc", "sys"] @@ -85,26 +108,52 @@ def unmount_subsys(): run(f"sudo umount {mount_dir}/{subsystem}", shell=True, check=True) for subsystem in subsystems: - run(f"sudo mount --bind /{subsystem} {mount_dir}/{subsystem}", shell=True, check=True) + run( + f"sudo mount --bind /{subsystem} {mount_dir}/{subsystem}", + shell=True, + check=True, + ) # First, install manually install apt in the rootfs + # TODO: probably get rid if this mess + """ for apt_dir in ["/usr/bin/apt", "/usr/lib/apt", "/var/lib/apt", "/var/lib/dpkg"]: if apt_dir == "/usr/bin/apt": - result = run(f"sudo cp -r {apt_dir}* {mount_dir}/usr/bin/", capture_output=True, shell=True) + result = run( + f"sudo cp -r {apt_dir}* {mount_dir}/usr/bin/", + capture_output=True, + shell=True, + ) else: - result = run(f"sudo cp -r {apt_dir} {mount_dir}{apt_dir}", capture_output=True, shell=True) + result = run( + f"sudo cp -r {apt_dir} {mount_dir}{apt_dir}", + capture_output=True, + shell=True, + ) if result.returncode != 0: print(result.stderr.decode("utf-8").strip()) unmount_subsys() raise RuntimeError("Error installing APT") - for shared_lib in ["libapt-private.so.0.0", "libapt-pkg.so.6.0", "libstdc++.so.6", "libxxhash.so.0"]: - result = run(f"sudo cp /lib/x86_64-linux-gnu/{shared_lib} {mount_dir}/lib/x86_64-linux-gnu/{shared_lib}", capture_output=True, shell=True) + for shared_lib in [ + "libapt-private.so.0.0", + "libapt-pkg.so.6.0", + "libstdc++.so.6", + "libxxhash.so.0", + ]: + result = run( + f"sudo cp /lib/x86_64-linux-gnu/{shared_lib} " + f"{mount_dir}/lib/x86_64-linux-gnu/{shared_lib}", + capture_output=True, + shell=True, + ) if result.returncode != 0: print(result.stderr.decode("utf-8").strip()) unmount_subsys() raise RuntimeError("Error installing APT shared libs") + # TODO: end of the mess to be deleted + """ cmd = """ sudo chroot {mount_dir} /usr/bin/bash <> /etc/default/grub +echo "GRUB_DEFAULT='Advanced options for Ubuntu>vmlinuz-{kernel_version}'" \ + >> /etc/default/grub update-grub EOF -""".format(mount_dir=mount_dir, kernel_version=kernel_version_trimmed) +""".format( + mount_dir=mount_dir, kernel_version=kernel_version_trimmed + ) result = run(cmd, shell=True, capture_output=True) if result.returncode != 0: print(result.stderr.decode("utf-8").strip()) @@ -179,7 +233,7 @@ def do_build_kernel(nocache=False): build_args = { "KERNEL_CONFIG_FILE": basename(tmp_file), - "MODULES_OUTDIR": join(SVSM_ROOT, "share", "linux", "modules") + "MODULES_OUTDIR": join(SVSM_ROOT, "share", "linux", "modules"), } build_args_str = [ "--build-arg {}={}".format(key, build_args[key]) for key in build_args @@ -239,7 +293,7 @@ def do_build_svsm(ctx, nocache=False): build_args_str, SVSM_IMAGE_TAG, join(PROJ_ROOT, "docker", "svsm.dockerfile"), - join(SVSM_ROOT, "share", "ovmf") + join(SVSM_ROOT, "share", "ovmf"), ) run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) @@ -268,7 +322,7 @@ def install(debug, clean): ctr_paths = [ join(SVSM_ROOT, "bin", "qemu-system-x86_64"), join(SVSM_ROOT, "share", "qemu", "qemu/"), - "/git/coconut-svsm/edk2/Build/OvmfX64/RELEASE_GCC5/FV/OVMF.fd" + "/git/coconut-svsm/edk2/Build/OvmfX64/RELEASE_GCC5/FV/OVMF.fd", ] host_paths = [ join(SVSM_ROOT, "bin", "qemu-system-x86_64"), diff --git a/tasks/util/docker.py b/tasks/util/docker.py index 9ff6b3b4..d7cfb951 100644 --- a/tasks/util/docker.py +++ b/tasks/util/docker.py @@ -1,7 +1,12 @@ from os.path import dirname, exists from subprocess import run from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT, print_dotted_line -from tasks.util.versions import CONTAINERD_VERSION, KATA_VERSION, NYDUS_SNAPSHOTTER_VERSION, NYDUS_VERSION +from tasks.util.versions import ( + CONTAINERD_VERSION, + KATA_VERSION, + NYDUS_SNAPSHOTTER_VERSION, + NYDUS_VERSION, +) def is_ctr_running(ctr_name): @@ -86,7 +91,12 @@ def build_image_and_run(image_tag, dockerfile, ctr_name, build_args=None): def pull_artifact_images(debug=False): print_dotted_line("Pulling artifact container images") components = ["containerd", "kata-containers", "nydus", "nydus-snapshotter"] - versions = [CONTAINERD_VERSION, KATA_VERSION, NYDUS_VERSION, NYDUS_SNAPSHOTTER_VERSION] + versions = [ + CONTAINERD_VERSION, + KATA_VERSION, + NYDUS_VERSION, + NYDUS_SNAPSHOTTER_VERSION, + ] for component, version in zip(components, versions): docker_cmd = f"docker pull {GHCR_URL}/{GITHUB_ORG}/{component}:{version}" result = run(docker_cmd, shell=True, capture_output=True) From 8d9d370a5c049e27c9afc7b4f8953b997b4d7605 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 19 Feb 2025 16:18:28 +0000 Subject: [PATCH 04/15] more wip --- tasks/svsm.py | 187 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 120 insertions(+), 67 deletions(-) diff --git a/tasks/svsm.py b/tasks/svsm.py index 6a009e36..4cd8cb2d 100644 --- a/tasks/svsm.py +++ b/tasks/svsm.py @@ -18,6 +18,22 @@ SVSM_GUEST_IMAGE_SIZE = "10G" +def get_kernel_version_from_ctr_image(): + tmp_file = "/tmp/sc2_kernel_release" + copy_from_ctr_image( + SVSM_KERNEL_IMAGE_TAG, + ["/git/coconut-svsm/linux/include/config/kernel.release"], + [tmp_file], + ) + with open(tmp_file, "r") as fh: + kernel_version = fh.read().strip() + kernel_version_trimmed = ( + kernel_version if not kernel_version.endswith("+") else kernel_version[:-1] + ) + + return kernel_version, kernel_version_trimmed + + @task def build_guest_image(ctx, clean=False): """ @@ -31,17 +47,7 @@ def build_guest_image(ctx, clean=False): run(f"sudo rm -f {SVSM_GUEST_IMAGE}", shell=True, check=True) # Get the kernel version we will install in the guest image - tmp_file = "/tmp/sc2_kernel_release" - copy_from_ctr_image( - SVSM_KERNEL_IMAGE_TAG, - ["/git/coconut-svsm/linux/include/config/kernel.release"], - [tmp_file], - ) - with open(tmp_file, "r") as fh: - kernel_version = fh.read().strip() - kernel_version_trimmed = ( - kernel_version if not kernel_version.endswith("+") else kernel_version[:-1] - ) + kernel_version, kernel_version_trimmed = get_kernel_version_from_ctr_image() # Prepare our rootfs with the kata agent and co. rootfs_base_dir = "/tmp/svsm_rootfs_base_dir" @@ -51,8 +57,7 @@ def build_guest_image(ctx, clean=False): # Install deps result = run( - "sudo DEBIAN_FRONTEND=noninteractive apt install -y " - "qemu-utils libguestfs-tools", + "sudo DEBIAN_FRONTEND=noninteractive apt install -y qemu-utils", shell=True, capture_output=True, ) @@ -67,22 +72,57 @@ def build_guest_image(ctx, clean=False): ) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) - # Format it as an ext4 fielsystem - result = run(f"sudo mkfs.ext4 {SVSM_GUEST_IMAGE}", shell=True, capture_output=True) + # Attach a loop device to the qcow image + loop_device_file = "/tmp/svsm_loop_device" + run( + f"sudo losetup --find --show {SVSM_GUEST_IMAGE} > {loop_device_file}", + shell=True, + check=True, + ) + with open(loop_device_file, "r") as fh: + loop_device = fh.read().strip() + + # Create a partition in the image + run(f"sudo parted -s {loop_device} mklabel msdos", shell=True, check=True) + run( + f"sudo parted -s {loop_device} mkpart primary ext4 1MiB 100%", + shell=True, + check=True, + ) + + # Detach and reattach loop device to pick up new partition + run(f"sudo losetup -d {loop_device}", shell=True, check=True) + run( + f"sudo losetup --find --show -P {SVSM_GUEST_IMAGE} > {loop_device_file}", + shell=True, + check=True, + ) + with open(loop_device_file, "r") as fh: + loop_device = fh.read().strip() + loop_device_part = loop_device + "p1" + + # Format partition as ext4 filesystem + result = run(f"sudo mkfs.ext4 {loop_device_part}", shell=True, capture_output=True) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) - # Mount the qcow image to copy in our rootfs + # Create mount dir mount_dir = "/tmp/svsm_guest_image" if exists(mount_dir): run(f"sudo rm -rf {mount_dir}", shell=True, check=True) run(f"sudo mkdir -p {mount_dir}", shell=True, check=True) + + # Mount loop device partition to mount dir result = run( - f"sudo guestmount -a {SVSM_GUEST_IMAGE} -m /dev/sda {mount_dir}", + f"sudo mount {loop_device_part} {mount_dir}", shell=True, capture_output=True, ) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + def cleanup_loop_device(): + run(f"sudo umount {mount_dir}", shell=True, check=True) + run(f"sudo losetup -d {loop_device}", shell=True, check=True) + # Copy our rootfs into the qcow image result = run(f"sudo cp -a {rootfs_dir}/* {mount_dir}", shell=True, check=True) @@ -91,10 +131,12 @@ def build_guest_image(ctx, clean=False): ctr_paths = [ "/git/coconut-svsm/linux/arch/x86/boot/bzImage", f"/opt/sc2/svsm/share/linux/modules/lib/modules/{kernel_version}", + "/git/coconut-svsm/linux/.config", ] host_paths = [ f"{mount_dir}/boot/vmlinuz-{kernel_version_trimmed}", f"{mount_dir}/lib/modules/{kernel_version_trimmed}", + f"{mount_dir}/boot/config-{kernel_version_trimmed}", ] copy_from_ctr_image( SVSM_KERNEL_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True @@ -108,62 +150,28 @@ def unmount_subsys(): run(f"sudo umount {mount_dir}/{subsystem}", shell=True, check=True) for subsystem in subsystems: - run( + result = run( f"sudo mount --bind /{subsystem} {mount_dir}/{subsystem}", shell=True, - check=True, - ) - - # First, install manually install apt in the rootfs - # TODO: probably get rid if this mess - """ - for apt_dir in ["/usr/bin/apt", "/usr/lib/apt", "/var/lib/apt", "/var/lib/dpkg"]: - if apt_dir == "/usr/bin/apt": - result = run( - f"sudo cp -r {apt_dir}* {mount_dir}/usr/bin/", - capture_output=True, - shell=True, - ) - else: - result = run( - f"sudo cp -r {apt_dir} {mount_dir}{apt_dir}", - capture_output=True, - shell=True, - ) - - if result.returncode != 0: - print(result.stderr.decode("utf-8").strip()) - unmount_subsys() - raise RuntimeError("Error installing APT") - - for shared_lib in [ - "libapt-private.so.0.0", - "libapt-pkg.so.6.0", - "libstdc++.so.6", - "libxxhash.so.0", - ]: - result = run( - f"sudo cp /lib/x86_64-linux-gnu/{shared_lib} " - f"{mount_dir}/lib/x86_64-linux-gnu/{shared_lib}", capture_output=True, - shell=True, ) + if result.returncode != 0: print(result.stderr.decode("utf-8").strip()) - unmount_subsys() - raise RuntimeError("Error installing APT shared libs") - # TODO: end of the mess to be deleted - """ + cleanup_loop_device() + raise RuntimeError("Error mounting /proc and /sys") + run(f"sudo mkdir -p {mount_dir}/usr/share/locale", shell=True, check=True) + # Make sure to soft-link /bin/sh to the right binary, as it is used + # by update-grub cmd = """ -sudo chroot {mount_dir} /usr/bin/bash < /tmp/mk_log echo "GRUB_DEFAULT='Advanced options for Ubuntu>vmlinuz-{kernel_version}'" \ >> /etc/default/grub update-grub @@ -186,12 +195,12 @@ def unmount_subsys(): if result.returncode != 0: print(result.stderr.decode("utf-8").strip()) unmount_subsys() + cleanup_loop_device() raise RuntimeError("Error setting default kernel") # Clean-up unmount_subsys() - # result = run(f"guestunmount {mount_dir}", shell=True, capture_output=True) - # assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + cleanup_loop_device() def do_build_kernel(nocache=False): @@ -316,7 +325,7 @@ def install(debug, clean): run(f"sudo mkdir -p ${SVSM_ROOT}", shell=True, check=True) - # TODO: install guest kernel + # TODO: install guest qcow2 image # Install QEMU and OVMF ctr_paths = [ @@ -335,3 +344,47 @@ def install(debug, clean): @task def foo(ctx): install(debug=False, clean=False) + + +@task +def install_host_kernel(ctx): + """ + Install the SVSM kernel in the host system + """ + kernel_version, kernel_version_trimmed = get_kernel_version_from_ctr_image() + + # Install the SVSM guest kernel into the host + ctr_paths = [ + "/git/coconut-svsm/linux/arch/x86/boot/bzImage", + f"/opt/sc2/svsm/share/linux/modules/lib/modules/{kernel_version}", + "/git/coconut-svsm/linux/.config", + ] + host_paths = [ + f"/boot/vmlinuz-{kernel_version_trimmed}", + f"/lib/modules/{kernel_version_trimmed}", + f"/boot/config-{kernel_version_trimmed}", + ] + copy_from_ctr_image( + SVSM_KERNEL_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True + ) + + # Generate the corresponding kernel image + result = run( + f"sudo mkinitramfs -o /boot/initrd.img-{kernel_version_trimmed} " + f"{kernel_version_trimmed}", + shell=True, + capture_output=True, + ) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + # Replace the GRUB_DEFAULT value + grub_default = ( + "Advanced options for Ubuntu>Ubuntu, with Linux " f"{kernel_version_trimmed}" + ) + result = run( + f"sudo sed -i 's/^GRUB_DEFAULT=.*/GRUB_DEFAULT=\"{grub_default}\"/' " + "/etc/default/grub", + shell=True, + capture_output=True, + ) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) From ff78f0e02a29bd23ac22cb310df2f0b101a1d79b Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 21 Feb 2025 17:18:51 +0000 Subject: [PATCH 05/15] svsm working with kata's initrd --- bin/launch_svsm.sh | 33 +++++++ docker/qemu.dockerfile | 1 - docker/svsm.dockerfile | 7 +- docker/svsm_qemu.dockerfile | 32 +++---- tasks/svsm.py | 170 +++++++++++++++++++++++++++--------- tasks/util/kernel.py | 11 +++ 6 files changed, 195 insertions(+), 59 deletions(-) create mode 100755 bin/launch_svsm.sh create mode 100644 tasks/util/kernel.py diff --git a/bin/launch_svsm.sh b/bin/launch_svsm.sh new file mode 100755 index 00000000..33b7c49f --- /dev/null +++ b/bin/launch_svsm.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e + +SVSM_ROOT=/opt/sc2/svsm + +# C-bit pos may be obtained by running coconut-svsm/svsm/utils/cbit +CBIT_POS=51 +IGVM=${SVSM_ROOT}/share/igvm/coconut-qemu.igvm +QCOW2=${SVSM_ROOT}/share/qemu/sc2.qcow2 +KERNEL=${SVSM_ROOT}/share/sc2/vmlinuz-kata-containers-sc2 +INITRD=${SVSM_ROOT}/share/sc2/kata-containers-sc2.img + +# Remap Ctrl-C to Ctrl-] to allow the guest to handle Ctrl-C. +stty intr ^] + +sudo ${SVSM_ROOT}/bin/qemu-system-x86_64 \ + -enable-kvm \ + -cpu EPYC-v4 \ + -machine q35,confidential-guest-support=sev0,memory-backend=ram1,igvm-cfg=igvm0 \ + -object memory-backend-memfd,id=ram1,size=8G,share=true,prealloc=false,reserve=false \ + -object sev-snp-guest,id=sev0,cbitpos=${CBIT_POS},reduced-phys-bits=1 \ + -object igvm-cfg,id=igvm0,file=$IGVM \ + -smp 8 \ + -no-reboot \ + -netdev user,id=vmnic -device e1000,netdev=vmnic,romfile= \ + -kernel ${KERNEL} \ + -initrd ${INITRD} \ + -append "console=ttyS0 loglevel=8 rdinit=/bin/sh" \ + -monitor none \ + -nographic \ + -serial stdio \ + -serial pty diff --git a/docker/qemu.dockerfile b/docker/qemu.dockerfile index 5b338ecb..e987ada0 100644 --- a/docker/qemu.dockerfile +++ b/docker/qemu.dockerfile @@ -47,7 +47,6 @@ RUN mkdir -p /usr/src \ --disable-guest-agent-msi \ --disable-libiscsi \ --disable-libudev \ - --disable-linux-user \ --disable-live-block-migration \ --disable-lzo \ --disable-opengl \ diff --git a/docker/svsm.dockerfile b/docker/svsm.dockerfile index 2d02c4c5..672e8c8c 100644 --- a/docker/svsm.dockerfile +++ b/docker/svsm.dockerfile @@ -5,6 +5,7 @@ RUN apt update \ && apt install -y \ autoconf \ autoconf-archive \ + libclang-dev \ libssl-dev \ pkg-config @@ -17,5 +18,7 @@ RUN mkdir -p ${CODE_DIR} \ && cd ${CODE_DIR} \ && git submodule update --init \ && rustup target add x86_64-unknown-none \ - && cargo install bindgen-cli - # && FW_FILE=/bin/ovmf-svsm.fd make + && cargo install bindgen-cli \ + # TODO: we may not want a releae build for the time being, as we cannot see + # the SVSM logs we care about + && FW_FILE=/bin/ovmf-svsm.fd ./build --release configs/qemu-target.json diff --git a/docker/svsm_qemu.dockerfile b/docker/svsm_qemu.dockerfile index d58e8b1e..2e65a3a7 100644 --- a/docker/svsm_qemu.dockerfile +++ b/docker/svsm_qemu.dockerfile @@ -29,6 +29,21 @@ RUN mkdir -p ${CODE_DIR} \ && make -f igvm_c/Makefile \ && make -f igvm_c/Makefile install +# Clone and build SVSM's OVMF +ARG CODE_DIR=/git/coconut-svsm/edk2 +RUN mkdir -p ${CODE_DIR} \ + && git clone https://github.com/coconut-svsm/edk2 ${CODE_DIR} \ + && cd ${CODE_DIR} \ + && git checkout svsm \ + && git submodule init \ + && git submodule update \ + && export PYTHON3_ENABLE=TRUE \ + && export PYTHON_COMMAND=python3 \ + && make -j $(nproc) -C BaseTools/ \ + && . ./edksetup.sh --reconfig \ + && build -a X64 -b RELEASE -t GCC5 -DTPM2_ENABLE -p OvmfPkg/OvmfPkgX64.dsc \ + && build -a X64 -b DEBUG -t GCC5 -DTPM2_ENABLE -p OvmfPkg/OvmfPkgX64.dsc + # Clone and build IGVM-enabled Qemu ARG QEMU_DATADIR ARG QEMU_PREFIX @@ -50,6 +65,7 @@ RUN mkdir -p ${CODE_DIR} \ # Must enable IGVM --enable-igvm \ --enable-kvm \ + --enable-slirp \ --enable-trace-backends=log,simple \ # As a reference we use Kata's --disable-x flags when building QEMU: # https://github.com/kata-containers/kata-containers/blob/main/tools/packaging/scripts/configure-hypervisor.sh @@ -76,7 +92,6 @@ RUN mkdir -p ${CODE_DIR} \ --disable-rdma \ --disable-replication \ --disable-sdl \ - --disable-slirp \ --disable-snappy \ --disable-spice \ --disable-tcg-interpreter \ @@ -91,20 +106,5 @@ RUN mkdir -p ${CODE_DIR} \ --disable-vnc-sasl \ --disable-vte \ --disable-xen \ - --static \ && make -j $(nproc) \ && make install -j $(nproc) - -# Clone and build SVSM's OVMF -ARG CODE_DIR=/git/coconut-svsm/edk2 -RUN mkdir -p ${CODE_DIR} \ - && git clone https://github.com/coconut-svsm/edk2 ${CODE_DIR} \ - && cd ${CODE_DIR} \ - && git checkout svsm \ - && git submodule init \ - && git submodule update \ - && export PYTHON3_ENABLE=TRUE \ - && export PYTHON_COMMAND=python3 \ - && make -j $(nproc) -C BaseTools/ \ - && . ./edksetup.sh --reconfig \ - && build -a X64 -b RELEASE -t GCC5 -DTPM2_ENABLE -p OvmfPkg/OvmfPkgX64.dsc diff --git a/tasks/svsm.py b/tasks/svsm.py index 4cd8cb2d..d2f50cfc 100644 --- a/tasks/svsm.py +++ b/tasks/svsm.py @@ -1,9 +1,12 @@ from invoke import task +from os import makedirs from os.path import basename, exists, join +from shutil import rmtree from subprocess import run from tasks.util.docker import copy_from_ctr_image from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT, SC2_ROOT from tasks.util.kata import prepare_rootfs +from tasks.util.kernel import get_host_kernel_version from tasks.util.versions import IGVM_VERSION SVSM_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "svsm:main") @@ -18,6 +21,25 @@ SVSM_GUEST_IMAGE_SIZE = "10G" +def grub_update_default_kernel(kernel_version): + """ + This method replaces the GRUB_DEFAULT value + """ + grub_default = ( + f"Advanced options for Ubuntu>Ubuntu, with Linux {kernel_version}" + ) + result = run( + f"sudo sed -i 's/^GRUB_DEFAULT=.*/GRUB_DEFAULT=\"{grub_default}\"/' " + "/etc/default/grub", + shell=True, + capture_output=True, + ) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + result = run("sudo update-grub", shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + def get_kernel_version_from_ctr_image(): tmp_file = "/tmp/sc2_kernel_release" copy_from_ctr_image( @@ -46,8 +68,11 @@ def build_guest_image(ctx, clean=False): if clean and exists(SVSM_GUEST_IMAGE): run(f"sudo rm -f {SVSM_GUEST_IMAGE}", shell=True, check=True) - # Get the kernel version we will install in the guest image - kernel_version, kernel_version_trimmed = get_kernel_version_from_ctr_image() + # -------------------------------------------------------------------------- + # Create raw disk image with our rootfs + # -------------------------------------------------------------------------- + + guest_raw_image = "/tmp/svsm_guest.raw" # Prepare our rootfs with the kata agent and co. rootfs_base_dir = "/tmp/svsm_rootfs_base_dir" @@ -65,7 +90,7 @@ def build_guest_image(ctx, clean=False): # Create qcow image result = run( - f"sudo qemu-img create -f qcow2 -o preallocation=metadata {SVSM_GUEST_IMAGE} " + f"sudo qemu-img create -f raw {guest_raw_image} " f"{SVSM_GUEST_IMAGE_SIZE}", shell=True, capture_output=True, @@ -75,7 +100,7 @@ def build_guest_image(ctx, clean=False): # Attach a loop device to the qcow image loop_device_file = "/tmp/svsm_loop_device" run( - f"sudo losetup --find --show {SVSM_GUEST_IMAGE} > {loop_device_file}", + f"sudo losetup --find --show {guest_raw_image} > {loop_device_file}", shell=True, check=True, ) @@ -93,7 +118,7 @@ def build_guest_image(ctx, clean=False): # Detach and reattach loop device to pick up new partition run(f"sudo losetup -d {loop_device}", shell=True, check=True) run( - f"sudo losetup --find --show -P {SVSM_GUEST_IMAGE} > {loop_device_file}", + f"sudo losetup --find --show -P {guest_raw_image} > {loop_device_file}", shell=True, check=True, ) @@ -126,21 +151,52 @@ def cleanup_loop_device(): # Copy our rootfs into the qcow image result = run(f"sudo cp -a {rootfs_dir}/* {mount_dir}", shell=True, check=True) - # Install the SVSM guest kernel into the image + # -------------------------------------------------------------------------- + # Install guest kernel + # -------------------------------------------------------------------------- + run(f"sudo mkdir -p {mount_dir}/boot", shell=True, check=True) - ctr_paths = [ - "/git/coconut-svsm/linux/arch/x86/boot/bzImage", - f"/opt/sc2/svsm/share/linux/modules/lib/modules/{kernel_version}", - "/git/coconut-svsm/linux/.config", - ] - host_paths = [ - f"{mount_dir}/boot/vmlinuz-{kernel_version_trimmed}", - f"{mount_dir}/lib/modules/{kernel_version_trimmed}", - f"{mount_dir}/boot/config-{kernel_version_trimmed}", - ] - copy_from_ctr_image( - SVSM_KERNEL_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True - ) + run(f"sudo mkdir -p {mount_dir}/lib/modules", shell=True, check=True) + + use_host_kernel = True + if use_host_kernel: + kernel_version = get_host_kernel_version() + + host_src_paths = [ + f"/boot/vmlinuz-{kernel_version}", + f"/boot/config-{kernel_version}", + f"/boot/initrd.img-{kernel_version}", + f"/lib/modules/{kernel_version}", + ] + host_dst_paths = [ + f"{mount_dir}/boot/vmlinuz-{kernel_version}", + f"{mount_dir}/boot/config-{kernel_version}", + f"{mount_dir}/boot/initrd.img-{kernel_version}", + f"{mount_dir}/lib/modules/{kernel_version}", + ] + for host_src_path, host_dst_path in zip(host_src_paths, host_dst_paths): + cp_cmd = "sudo cp -r " if "modules" in host_src_path else "sudo cp" + result = run(f"{cp_cmd} {host_src_path} {host_dst_path}", shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + else: + # Copy from our custom built kernel + + # Get the kernel version we will install in the guest image + kernel_version_long, kernel_version = get_kernel_version_from_ctr_image() + + ctr_paths = [ + "/git/coconut-svsm/linux/arch/x86/boot/bzImage", + "/git/coconut-svsm/linux/.config", + f"/opt/sc2/svsm/share/linux/modules/lib/modules/{kernel_version_long}", + ] + host_paths = [ + f"{mount_dir}/boot/vmlinuz-{kernel_version}", + f"{mount_dir}/boot/config-{kernel_version}", + f"{mount_dir}/lib/modules/{kernel_version}", + ] + copy_from_ctr_image( + SVSM_KERNEL_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True + ) # Configure GRUB inside the guest image subsystems = ["dev", "proc", "sys"] @@ -183,13 +239,13 @@ def unmount_subsys(): cmd = """ sudo chroot {mount_dir} /usr/bin/sh < /tmp/mk_log +# mkinitramfs -o /boot/initrd.img-{kernel_version} {kernel_version} 2> /tmp/mk_log echo "GRUB_DEFAULT='Advanced options for Ubuntu>vmlinuz-{kernel_version}'" \ >> /etc/default/grub update-grub EOF """.format( - mount_dir=mount_dir, kernel_version=kernel_version_trimmed + mount_dir=mount_dir, kernel_version=kernel_version ) result = run(cmd, shell=True, capture_output=True) if result.returncode != 0: @@ -202,20 +258,22 @@ def unmount_subsys(): unmount_subsys() cleanup_loop_device() + # -------------------------------------------------------------------------- + # Generate qcow2 image + # -------------------------------------------------------------------------- + + result = run(f"sudo qemu-img convert -f raw -O qcow2 {guest_raw_image} {SVSM_GUEST_IMAGE}", shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + def do_build_kernel(nocache=False): # First, generate the right config file: we start from our current one, # and make sure the following are set: # - CONFIG_KVM_AMD_SEV: for general SNP support in KVM # - CONFIG_TCG_PLATFORM: for vTPM support in the SVSM + current_kernel_name = get_host_kernel_version() - # TODO: move to tasks/util/kernel.py - # Use a tmp file to not rely on being able to capture our stdout tmp_file = "/tmp/svsm_kernel_config" - run(f"uname -r > {tmp_file}", shell=True, check=True) - with open(tmp_file, "r") as fh: - current_kernel_name = fh.read().strip() - run(f"cp /boot/config-{current_kernel_name} {tmp_file}", shell=True, check=True) with open(tmp_file, "r") as fh: kernel_config = fh.read().strip().split("\n") @@ -340,6 +398,14 @@ def install(debug, clean): ] copy_from_ctr_image(SVSM_QEMU_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True) + # Install SVSM's IGVM image + copy_from_ctr_image( + SVSM_IMAGE_TAG, + ["/git/coconut-svsm/svsm/bin/coconut-qemu.igvm"], + [join(SVSM_ROOT, "share", "igvm", "coconut-qemu.igvm")], + requires_sudo=True + ) + @task def foo(ctx): @@ -369,22 +435,46 @@ def install_host_kernel(ctx): ) # Generate the corresponding kernel image - result = run( - f"sudo mkinitramfs -o /boot/initrd.img-{kernel_version_trimmed} " - f"{kernel_version_trimmed}", - shell=True, - capture_output=True, - ) + # echo "LVM=yes" > /user/share/initramfs-tools/conf.d/lvm + result = run("sudo DEBIAN_FRONTEND=noninteractive apt install -y initramfs-tools", shell=True, capture_output=True) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) - - # Replace the GRUB_DEFAULT value - grub_default = ( - "Advanced options for Ubuntu>Ubuntu, with Linux " f"{kernel_version_trimmed}" - ) result = run( - f"sudo sed -i 's/^GRUB_DEFAULT=.*/GRUB_DEFAULT=\"{grub_default}\"/' " - "/etc/default/grub", + # f"sudo mkinitramfs -o /boot/initrd.img-{kernel_version_trimmed} " + # f"{kernel_version_trimmed}", + f"sudo update-initramfs -c -k {kernel_version_trimmed}", shell=True, capture_output=True, ) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + grub_update_default_kernel(kernel_version_trimmed) + + +@task +def install_upstream_kernel(ctx): + # TODO: find a way to automate this + kernel_ver = "6.13" + kernel_name = f"6.13.0-061300-generic" + + tmp_dir = f"/tmp/kernel-{kernel_ver}" + if exists(tmp_dir): + rmtree(tmp_dir) + makedirs(tmp_dir) + + base_url = f"https://kernel.ubuntu.com/mainline/v{kernel_ver}/amd64/" + # The order of this files in the array _matters_ + deb_files = [ + "linux-headers-6.13.0-061300_6.13.0-061300.202501302155_all.deb", + "linux-headers-6.13.0-061300-generic_6.13.0-061300.202501302155_amd64.deb", + "linux-modules-6.13.0-061300-generic_6.13.0-061300.202501302155_amd64.deb", + "linux-image-unsigned-6.13.0-061300-generic_6.13.0-061300.202501302155_amd64.deb", + ] + for deb_file in deb_files: + result = run(f"wget {base_url}/{deb_file}", shell=True, capture_output=True, cwd=tmp_dir) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + for deb_file in deb_files: + result = run(f"sudo dpkg -i {deb_file}", shell=True, capture_output=True, cwd=tmp_dir) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + grub_update_default_kernel(kernel_name) diff --git a/tasks/util/kernel.py b/tasks/util/kernel.py new file mode 100644 index 00000000..0a7cdbf3 --- /dev/null +++ b/tasks/util/kernel.py @@ -0,0 +1,11 @@ +from subprocess import run + + +def get_host_kernel_version(): + # Use a tmp file to not rely on being able to capture our stdout + tmp_file = "/tmp/svsm_kernel_config" + run(f"uname -r > {tmp_file}", shell=True, check=True) + with open(tmp_file, "r") as fh: + current_kernel_name = fh.read().strip() + + return current_kernel_name From 4a995bc3986814005571c18cf682f0e694dfb813 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 24 Feb 2025 13:44:43 +0000 Subject: [PATCH 06/15] cleanup --- README.md | 11 +- bin/launch_svsm.sh | 13 +- docker/svsm.dockerfile | 2 - docker/svsm_qemu.dockerfile | 2 +- docs/coconut_svsm.md | 152 +++---------- docs/host_kernel.md | 58 +++-- tasks/kernel.py | 36 +++ tasks/svsm.py | 433 +++++++++--------------------------- tasks/util/docker.py | 9 +- tasks/util/kernel.py | 17 ++ 10 files changed, 243 insertions(+), 490 deletions(-) diff --git a/README.md b/README.md index 6bdd7f8d..71fad58c 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,9 @@ https://github.com/confidential-containers) project. SC2 currently supports AMD SEV-SNP and Intel TDX as underlying TEE, and requires deployment on a bare-metal host. Before moving forward, make sure you have a -correct installation. For SEV-SNP you may use [`snphost ok`]( -https://github.com/virtee/snphost.git). - -Lastly, make sure you are using the exact host kernel: - -| **SEV-SNP** | **TDX** | -|---|---| -| [6.8.0-rc5-next-20240221-snp-host-cc2568386](https://github.com/confidential-containers/linux/tree/amd-snp-host-202402240000) | [6.8.0-1004-intel](https://git.launchpad.net/~kobuk-team/ubuntu/+source/linux-intel/tree/?h=noble-main-next) | +correct host installation. For SEV-SNP you may use [`snphost ok`]( +https://github.com/virtee/snphost.git). Also make sure you have the [right +host kernel](./docs/host_kernel.md). ## Quick Start diff --git a/bin/launch_svsm.sh b/bin/launch_svsm.sh index 33b7c49f..27bb6c0c 100755 --- a/bin/launch_svsm.sh +++ b/bin/launch_svsm.sh @@ -1,15 +1,18 @@ #!/bin/bash -set -e - SVSM_ROOT=/opt/sc2/svsm # C-bit pos may be obtained by running coconut-svsm/svsm/utils/cbit CBIT_POS=51 + IGVM=${SVSM_ROOT}/share/igvm/coconut-qemu.igvm -QCOW2=${SVSM_ROOT}/share/qemu/sc2.qcow2 KERNEL=${SVSM_ROOT}/share/sc2/vmlinuz-kata-containers-sc2 -INITRD=${SVSM_ROOT}/share/sc2/kata-containers-sc2.img +# KERNEL=/home/csegarra/git/sc2-sys/svsm-linux/arch/x86/boot/bzImage +INITRD=/opt/sc2/svsm/share/sc2/initrd-kata.img + +# Ensure terminal settings are restored on exit +orig_stty=$(stty -g) +trap "stty '$orig_stty'" EXIT # Remap Ctrl-C to Ctrl-] to allow the guest to handle Ctrl-C. stty intr ^] @@ -26,7 +29,7 @@ sudo ${SVSM_ROOT}/bin/qemu-system-x86_64 \ -netdev user,id=vmnic -device e1000,netdev=vmnic,romfile= \ -kernel ${KERNEL} \ -initrd ${INITRD} \ - -append "console=ttyS0 loglevel=8 rdinit=/bin/sh" \ + -append "console=ttyS0 loglevel=8 earlyprintk=serial rdinit=/bin/sh" \ -monitor none \ -nographic \ -serial stdio \ diff --git a/docker/svsm.dockerfile b/docker/svsm.dockerfile index 672e8c8c..7fcd027f 100644 --- a/docker/svsm.dockerfile +++ b/docker/svsm.dockerfile @@ -19,6 +19,4 @@ RUN mkdir -p ${CODE_DIR} \ && git submodule update --init \ && rustup target add x86_64-unknown-none \ && cargo install bindgen-cli \ - # TODO: we may not want a releae build for the time being, as we cannot see - # the SVSM logs we care about && FW_FILE=/bin/ovmf-svsm.fd ./build --release configs/qemu-target.json diff --git a/docker/svsm_qemu.dockerfile b/docker/svsm_qemu.dockerfile index 2e65a3a7..45aaa24a 100644 --- a/docker/svsm_qemu.dockerfile +++ b/docker/svsm_qemu.dockerfile @@ -58,7 +58,7 @@ RUN mkdir -p ${CODE_DIR} \ # The `--datadir` flag is the path where QEMU will look for firmware # images. The default `--datadir` path when using a system provisioned # by the operator is: `/opt/confidential-containers/share/kata-qemu`. - # For our QEMu fork we use `/opt/sc2/svsm/share/qemu + # For our QEMU fork we use `/opt/sc2/svsm/share/qemu --datadir=${QEMU_DATADIR} \ --prefix=${QEMU_PREFIX} \ --target-list=x86_64-softmmu \ diff --git a/docs/coconut_svsm.md b/docs/coconut_svsm.md index 9e3cd0f4..4063d03f 100644 --- a/docs/coconut_svsm.md +++ b/docs/coconut_svsm.md @@ -1,152 +1,50 @@ # Coconut-SVSM -## Installation +SC2 uses the [Coconut-SVSM](https://github.com/coconut-svsm/svsm) as an SVSM for +its confidential pods. The SVSM requires its own forked versions of the kernel, +QEMU, and OVMF. -For more details, please refer to [Coconut SVSM](https://github.com/coconut-svsm/svsm). The following instructions are intended to simplify the installation, especially for our usecase. +> [!WARNING] +> Currently we only support running SNP guests in SVSM using a manual script +> with QEMU. In the future we will integrate it with Kata pods (#148). -### 1. Preparing the Host +## Quick Start -The host kernel needs support for SVSM. -Install necessary dependencies: +After installing SC2, you can run: ```bash -$ sudo apt install git fakeroot build-essential ncurses-dev xz-utils libssl-dev bc flex libelf-dev bison +inv svsm.install [--clean] ``` -Clone the linux kernel: +and then: ```bash -$ git clone https://github.com/coconut-svsm/linux -$ cd linux -$ git checkout svsm +./bin/launch_svsm.sh ``` -To use your current kernel configuration, execute: +to start an SNP guest in the SVSM. -```bash -$ cp /boot/config-$(uname -r) .config -``` +## Guest Kernel -Use `menuconfig` and ensure that `CONFIG_KVM_AMD_SEV` is enabled. +We have an automated script to build the guest kernel which, for some reason, +we still cannot use from the host (the `initramfs` seems to be corrupted, but +the kernel is fine to run in the guest). -Next, compile and install the kernel: +You can trigger the build by running: ```bash -$ make -j $(nproc) -$ sudo make modules_install -$ sudo make install +inv svsm.build-guest-kernel ``` -### 2. Building QEMU - -You will need a QEMU build that supports launching guests using IGVM: - -```bash -$ inv coconut.qemu.build -``` - -This will create a file called `qemu-system-x86_64-igvm` in the `BIN_DIR`. - -### 3. Building the guest firmware - -A special OVMF build is needed to launch a guest on top of the COCONUT-SVSM: - -```bash -$ inv coconut.ovmf.build -``` - -A file named `ovmf-svsm.fd` will be created in the `BIN_DIR`. - +## QEMU/OVMF -### 4. Preparing the guest image - -In our case, we are creating a Ubuntu guest image. - -Possible instructions: - -```bash -$ wget http://releases.ubuntu.com/jammy/ubuntu-22.04.4-live-server-amd64.iso -$ qemu-img create -f qcow2 ubuntu.qcow2 40G -$ qemu-system-x86_64 \ - -cdrom ubuntu-22.04.4-live-server-amd64.iso \ - -drive "file=ubuntu.qcow2,format=qcow2" \ - -bios /usr/share/ovmf/OVMF.fd \ - -enable-kvm \ - -m 4G \ - -smp 4 -``` - -Follow the instructions to install Ubuntu. -After the installation, start the VM. We are going to use a shared directory to copy the kernel configuration (see step 1, `CONFIG_KVM_AMD_SEV` has to be enabled) into the VM. - -```bash -$ mkdir -p /path/to/shared/directory -$ cp /path/to/kernel-config /path/to/shared/directory/ -$ qemu-system-x86_64 \ - -drive "file=ubuntu-snapshot.qcow2,format=qcow2" \ - -enable-kvm -m 2G -smp 2 \ - -virtfs local,path=/path/to/shared/directory,mount_tag=hostshare,security_model=passthrough,id=hostshare -``` - -Within the VM, execute: - -```bash -$ sudo apt-get install git fakeroot build-essential ncurses-dev xz-utils libssl-dev bc flex libelf-dev bison -$ git clone https://github.com/coconut-svsm/linux -$ cd linux -$ git checkout svsm -``` - -Next, mount the shared directory and copy the kernel configuration file: - -```bash -$ sudo mount -t 9p -o trans=virtio hostshare /mnt -$ cp /mnt/kernel-config ./.config -``` - -Then, install the kernel: - -```bash -$ make -j $(nproc) -$ sudo make modules_install -$ sudo make install -``` - -To compile the kernel on Arch Linux, you can use and adapt the `bin/compile_guest_kernel_arch.sh` script. - -### 5. Building the COCONUT-SVSM - -To build the SVSM itself using the special OVMF built in step 3, run - -``` bash -$ inv coconut.svsm.build -``` - -This command creates the file `coconut-qemu.igvm` in the `BIN_DIR`. - -### 6. Putting it all together - -To start the guest VM, execute - -```bash -$ inv coconut.qemu.guest -``` - -Modify the QEMU command in this file according to your needs. -`-d`/ `--detach` starts qemu in detached mode. - -If you are executing QEMU on a remote server, -you can use TigerVNC: - -```bash -$ inv coconut.qemu.guest -v -$ ssh -L 5901:localhost:5901 user@server -$ vncviewer localhost:5901 -``` +The SVSM uses the [IGVM](https://github.com/microsoft/igvm) image format to +package the initial guest image containing the virtual firmware (OVMF) as well +as the SVSM itself. -If you want to connect to the VM using ssh, you can use the following commands: +We need a patched version of QEMU and OVMF to support using IGVM files. You +can build both by running: ```bash -$ ssh-copy-id -i ~/.ssh/YOUR_SSH_KEY -p 2222 user@localhost -$ ssh -i ~/.ssh/YOUR_SSH_KEY -p 2222 user@localhost" +inv svsm.build-qemu ``` diff --git a/docs/host_kernel.md b/docs/host_kernel.md index 21be47dc..34e27839 100644 --- a/docs/host_kernel.md +++ b/docs/host_kernel.md @@ -1,26 +1,42 @@ -## Host Kernel Set-Up +# Host Kernel Set-Up -Frequently, until the SNP patches are upstreamed, we need to bump the host -kernel. +Some of the SC2 features require specific host-kernel patches. You can see +the expected host-kernel versions in [`./tasks/util/versions.py`]( +./tasks/util/versions.py), which we also summarise in the following table. -After re-building the right kernel from SNP, we need to pick it for our host. -We can do so doing the following: +| **SEV-SNP** | **TDX** | +|---|---| +| `6.11.0-snp-host-cc2568386` | `6.8.0-1004-intel` | + +## SNP + +For SNP hosts, we need kernel patches to (i) enable SNP, and (ii) enable the +SVSM. The former were upstreamed in `6.11`, but we still need the latter. To +build the host-kernel from source you may use: + +```bash +git clone -b svsm --depth=1 https://github.com/coconut-svsm/linux ../svsm-linux +``` + +and then just run: ```bash -grep -E "menuentry '.*'" /boot/grub/grub.cfg | cut -d"'" -f2 - -# Will output something like: -# Ubuntu -# Advanced options for Ubuntu -# Ubuntu, with Linux 5.15.0-50-generic -# Ubuntu, with Linux 5.15.0-50-generic (recovery mode) -# Ubuntu, with Linux 5.13.0-48-generic -# Ubuntu, with Linux 5.13.0-48-generic (recovery mode) - -# Now, pick the default kernel by setting the GRUB boot index to "1" -# (for Advanced options), and then pick your kernel as 0-indexed. -sudo vi /etc/default/grub - -# For "Ubuntu, with Linux 5.13.0-48-generic" -GRUB_DEFAULT="1>2" +cd ../svsm-linux +cp /boot/config-$(uname -r) .config +# Make sure CONFIG_KVM_AMD_SEV=y and CONFIG_TCG_PLATFORM=y +make olddefconfig +make -j $(nproc) + +sudo make modules_install +sudo make install +``` + +Lastly, update the `GRUB_DEFAULT` variable in `/etc/default/grub` to: + +``` +GRUB_DEFAULT="Advanced options for Ubuntu>Ubuntu, with Linux 6.11.0-......" +sudo update-grub +sudo reboot now ``` + +which you can copy and paste from `ls -lart /boot`. diff --git a/tasks/kernel.py b/tasks/kernel.py index e02327ab..3340ea15 100644 --- a/tasks/kernel.py +++ b/tasks/kernel.py @@ -1,8 +1,10 @@ from invoke import task from os import makedirs from os.path import exists, join +from shutil import rmtree from tasks.util.env import KATA_CONFIG_DIR, KATA_IMG_DIR, KATA_RUNTIMES, SC2_RUNTIMES from tasks.util.kata import KATA_SOURCE_DIR, copy_from_kata_workon_ctr +from tasks.util.kernel import grub_update_default_kernel from tasks.util.toml import update_toml from tasks.util.versions import GUEST_KERNEL_VERSION from subprocess import run @@ -123,3 +125,37 @@ def hot_replace_guest(ctx, debug=False): Hot-replace guest kernel """ build_guest(debug=debug, hot_replace=True) + + +@task +def install_host_kernel_from_upstream(ctx): + # TODO: find a way to automate the grepping of deb files and kernel name + kernel_ver = "6.13" + kernel_name = f"{kernel_ver}.0-061300-generic" + + tmp_dir = f"/tmp/kernel-{kernel_ver}" + if exists(tmp_dir): + rmtree(tmp_dir) + makedirs(tmp_dir) + + base_url = f"https://kernel.ubuntu.com/mainline/v{kernel_ver}/amd64/" + # The order of this files in the array _matters_ + deb_files = [ + "linux-headers-6.13.0-061300_6.13.0-061300.202501302155_all.deb", + f"linux-headers-{kernel_name}_6.13.0-061300.202501302155_amd64.deb", + f"linux-modules-{kernel_name}_6.13.0-061300.202501302155_amd64.deb", + f"linux-image-unsigned-{kernel_name}_6.13.0-061300.202501302155_amd64.deb", + ] + for deb_file in deb_files: + result = run( + f"wget {base_url}/{deb_file}", shell=True, capture_output=True, cwd=tmp_dir + ) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + for deb_file in deb_files: + result = run( + f"sudo dpkg -i {deb_file}", shell=True, capture_output=True, cwd=tmp_dir + ) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + grub_update_default_kernel(kernel_name) diff --git a/tasks/svsm.py b/tasks/svsm.py index d2f50cfc..d49ac975 100644 --- a/tasks/svsm.py +++ b/tasks/svsm.py @@ -1,11 +1,9 @@ from invoke import task -from os import makedirs from os.path import basename, exists, join -from shutil import rmtree from subprocess import run from tasks.util.docker import copy_from_ctr_image from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT, SC2_ROOT -from tasks.util.kata import prepare_rootfs +from tasks.util.kata import KATA_AGENT_SOURCE_DIR, KATA_IMAGE_TAG, KATA_SOURCE_DIR from tasks.util.kernel import get_host_kernel_version from tasks.util.versions import IGVM_VERSION @@ -14,30 +12,9 @@ SVSM_QEMU_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "qemu:svsm") SVSM_ROOT = join(SC2_ROOT, "svsm") -SVSM_QEMU_DATA_DIR = join(SVSM_ROOT, "share", "qemu") +SVSM_QEMU_DATA_DIR = join(SVSM_ROOT, "share") -SVSM_GUEST_IMAGE = join(SVSM_QEMU_DATA_DIR, "sc2.qcow2") -# Can we do with less? -SVSM_GUEST_IMAGE_SIZE = "10G" - - -def grub_update_default_kernel(kernel_version): - """ - This method replaces the GRUB_DEFAULT value - """ - grub_default = ( - f"Advanced options for Ubuntu>Ubuntu, with Linux {kernel_version}" - ) - result = run( - f"sudo sed -i 's/^GRUB_DEFAULT=.*/GRUB_DEFAULT=\"{grub_default}\"/' " - "/etc/default/grub", - shell=True, - capture_output=True, - ) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) - - result = run("sudo update-grub", shell=True, capture_output=True) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) +SVSM_GUEST_INITRD = join(SVSM_ROOT, "share", "sc2", "initrd-kata.img") def get_kernel_version_from_ctr_image(): @@ -56,217 +33,68 @@ def get_kernel_version_from_ctr_image(): return kernel_version, kernel_version_trimmed -@task -def build_guest_image(ctx, clean=False): - """ - This function generates a qcow2 guest image with the SVSM-enlightened - guest kernel. +def do_build_initrd(clean=False): + if clean and exists(SVSM_GUEST_INITRD): + run(f"sudo rm -f {SVSM_GUEST_INITRD}", shell=True, check=True) - TODO: check if we really need a qcow image or we can get away with an - initrd. - """ - if clean and exists(SVSM_GUEST_IMAGE): - run(f"sudo rm -f {SVSM_GUEST_IMAGE}", shell=True, check=True) - - # -------------------------------------------------------------------------- - # Create raw disk image with our rootfs - # -------------------------------------------------------------------------- + if exists(SVSM_GUEST_INITRD): + return - guest_raw_image = "/tmp/svsm_guest.raw" + # TODO: this initrd is built using dracut, which is different to how we + # normally build initrd's for SC2 in Kata. Whenever we incorportate the + # SVSM into SC2, we will have to converge this method with the regular + # initrd preparation for SC2. # Prepare our rootfs with the kata agent and co. - rootfs_base_dir = "/tmp/svsm_rootfs_base_dir" - # TODO: move hot_replace to False when done experimenting - prepare_rootfs(rootfs_base_dir, debug=False, sc2=True, hot_replace=True) - rootfs_dir = join(rootfs_base_dir, "rootfs") - - # Install deps - result = run( - "sudo DEBIAN_FRONTEND=noninteractive apt install -y qemu-utils", - shell=True, - capture_output=True, - ) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) - - # Create qcow image - result = run( - f"sudo qemu-img create -f raw {guest_raw_image} " - f"{SVSM_GUEST_IMAGE_SIZE}", - shell=True, - capture_output=True, - ) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + initrd_base_dir = "/tmp/svsm_initrd_base_dir" + run(f"sudo rm -rf {initrd_base_dir}", shell=True, check=True) + run(f"sudo mkdir -p {initrd_base_dir}", shell=True, check=True) - # Attach a loop device to the qcow image - loop_device_file = "/tmp/svsm_loop_device" - run( - f"sudo losetup --find --show {guest_raw_image} > {loop_device_file}", - shell=True, - check=True, - ) - with open(loop_device_file, "r") as fh: - loop_device = fh.read().strip() - - # Create a partition in the image - run(f"sudo parted -s {loop_device} mklabel msdos", shell=True, check=True) - run( - f"sudo parted -s {loop_device} mkpart primary ext4 1MiB 100%", - shell=True, - check=True, - ) - - # Detach and reattach loop device to pick up new partition - run(f"sudo losetup -d {loop_device}", shell=True, check=True) - run( - f"sudo losetup --find --show -P {guest_raw_image} > {loop_device_file}", - shell=True, - check=True, - ) - with open(loop_device_file, "r") as fh: - loop_device = fh.read().strip() - loop_device_part = loop_device + "p1" - - # Format partition as ext4 filesystem - result = run(f"sudo mkfs.ext4 {loop_device_part}", shell=True, capture_output=True) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) - - # Create mount dir - mount_dir = "/tmp/svsm_guest_image" - if exists(mount_dir): - run(f"sudo rm -rf {mount_dir}", shell=True, check=True) - run(f"sudo mkdir -p {mount_dir}", shell=True, check=True) - - # Mount loop device partition to mount dir + host_paths = [ + join(initrd_base_dir, "VERSION"), + join(initrd_base_dir, "kata-agent"), + join(initrd_base_dir, "tools", "osbuilder", "Makefile"), + join(initrd_base_dir, "tools", "osbuilder", "dracut"), + join(initrd_base_dir, "tools", "osbuilder", "initrd-builder"), + join(initrd_base_dir, "tools", "osbuilder", "rootfs-builder"), + join(initrd_base_dir, "tools", "osbuilder", "scripts"), + ] + ctr_paths = [ + f"{KATA_SOURCE_DIR}/VERSION", + f"{KATA_AGENT_SOURCE_DIR}/target/x86_64-unknown-linux-musl/release/kata-agent", + f"{KATA_SOURCE_DIR}/tools/osbuilder/Makefile", + f"{KATA_SOURCE_DIR}/tools/osbuilder/dracut", + f"{KATA_SOURCE_DIR}/tools/osbuilder/initrd-builder", + f"{KATA_SOURCE_DIR}/tools/osbuilder/rootfs-builder", + f"{KATA_SOURCE_DIR}/tools/osbuilder/scripts", + ] + copy_from_ctr_image(KATA_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True) + + # This initrd must contain our agent, but also the kernel modules + # corresponding to an SVSM-enlightened kernel + initrd_cmd = [ + "sudo -E make", + "BUILD_METHOD=dracut", + f"TARGET_INITRD={SVSM_GUEST_INITRD}", + "AGENT_SOURCE_BIN={}".format(join(initrd_base_dir, "kata-agent")), + "DRACUT_KVERSION={}".format(get_host_kernel_version()), + "initrd", + ] + initrd_cmd = " ".join(initrd_cmd) result = run( - f"sudo mount {loop_device_part} {mount_dir}", + initrd_cmd, shell=True, capture_output=True, + cwd=join(initrd_base_dir, "tools", "osbuilder"), ) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) - def cleanup_loop_device(): - run(f"sudo umount {mount_dir}", shell=True, check=True) - run(f"sudo losetup -d {loop_device}", shell=True, check=True) - - # Copy our rootfs into the qcow image - result = run(f"sudo cp -a {rootfs_dir}/* {mount_dir}", shell=True, check=True) - - # -------------------------------------------------------------------------- - # Install guest kernel - # -------------------------------------------------------------------------- - - run(f"sudo mkdir -p {mount_dir}/boot", shell=True, check=True) - run(f"sudo mkdir -p {mount_dir}/lib/modules", shell=True, check=True) - - use_host_kernel = True - if use_host_kernel: - kernel_version = get_host_kernel_version() - - host_src_paths = [ - f"/boot/vmlinuz-{kernel_version}", - f"/boot/config-{kernel_version}", - f"/boot/initrd.img-{kernel_version}", - f"/lib/modules/{kernel_version}", - ] - host_dst_paths = [ - f"{mount_dir}/boot/vmlinuz-{kernel_version}", - f"{mount_dir}/boot/config-{kernel_version}", - f"{mount_dir}/boot/initrd.img-{kernel_version}", - f"{mount_dir}/lib/modules/{kernel_version}", - ] - for host_src_path, host_dst_path in zip(host_src_paths, host_dst_paths): - cp_cmd = "sudo cp -r " if "modules" in host_src_path else "sudo cp" - result = run(f"{cp_cmd} {host_src_path} {host_dst_path}", shell=True, capture_output=True) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) - else: - # Copy from our custom built kernel - - # Get the kernel version we will install in the guest image - kernel_version_long, kernel_version = get_kernel_version_from_ctr_image() - - ctr_paths = [ - "/git/coconut-svsm/linux/arch/x86/boot/bzImage", - "/git/coconut-svsm/linux/.config", - f"/opt/sc2/svsm/share/linux/modules/lib/modules/{kernel_version_long}", - ] - host_paths = [ - f"{mount_dir}/boot/vmlinuz-{kernel_version}", - f"{mount_dir}/boot/config-{kernel_version}", - f"{mount_dir}/lib/modules/{kernel_version}", - ] - copy_from_ctr_image( - SVSM_KERNEL_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True - ) - - # Configure GRUB inside the guest image - subsystems = ["dev", "proc", "sys"] - - def unmount_subsys(): - for subsystem in subsystems: - run(f"sudo umount {mount_dir}/{subsystem}", shell=True, check=True) - - for subsystem in subsystems: - result = run( - f"sudo mount --bind /{subsystem} {mount_dir}/{subsystem}", - shell=True, - capture_output=True, - ) - - if result.returncode != 0: - print(result.stderr.decode("utf-8").strip()) - cleanup_loop_device() - raise RuntimeError("Error mounting /proc and /sys") - - run(f"sudo mkdir -p {mount_dir}/usr/share/locale", shell=True, check=True) - # Make sure to soft-link /bin/sh to the right binary, as it is used - # by update-grub - cmd = """ -sudo chroot {mount_dir} /usr/bin/sh < /tmp/mk_log -echo "GRUB_DEFAULT='Advanced options for Ubuntu>vmlinuz-{kernel_version}'" \ - >> /etc/default/grub -update-grub -EOF -""".format( - mount_dir=mount_dir, kernel_version=kernel_version - ) - result = run(cmd, shell=True, capture_output=True) - if result.returncode != 0: - print(result.stderr.decode("utf-8").strip()) - unmount_subsys() - cleanup_loop_device() - raise RuntimeError("Error setting default kernel") - - # Clean-up - unmount_subsys() - cleanup_loop_device() - - # -------------------------------------------------------------------------- - # Generate qcow2 image - # -------------------------------------------------------------------------- - - result = run(f"sudo qemu-img convert -f raw -O qcow2 {guest_raw_image} {SVSM_GUEST_IMAGE}", shell=True, capture_output=True) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) - def do_build_kernel(nocache=False): + """ + This method builds the forked kernel needed in the SVSM. It is used for + the __guest__ kernel only. + """ # First, generate the right config file: we start from our current one, # and make sure the following are set: # - CONFIG_KVM_AMD_SEV: for general SNP support in KVM @@ -317,14 +145,6 @@ def do_build_kernel(nocache=False): run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) -@task -def build_kernel(ctx, nocache=False, push=False): - """ - Build the host/guest kernel fork to use with the SVSM - """ - do_build_kernel(nocache=nocache) - - def do_build_qemu(nocache=False): build_args = { "IGVM_VERSION": IGVM_VERSION, @@ -345,136 +165,101 @@ def do_build_qemu(nocache=False): run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) -@task -def do_build_svsm(ctx, nocache=False): - build_args = { - "OVMF_FILE": "OVMF.fd", - } - build_args_str = [ - "--build-arg {}={}".format(key, build_args[key]) for key in build_args - ] - build_args_str = " ".join(build_args_str) - - docker_cmd = "docker build{} {} -t {} -f {} {}".format( - " --no-cache" if nocache else "", - build_args_str, - SVSM_IMAGE_TAG, - join(PROJ_ROOT, "docker", "svsm.dockerfile"), - join(SVSM_ROOT, "share", "ovmf"), - ) - run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) - - -@task -def build_qemu(ctx, nocache=False, push=False): - """ - Build the QEMU fork for its use with the SVSM - """ - do_build_qemu(nocache=nocache) - - if push: - run(f"docker push {SVSM_QEMU_IMAGE_TAG}", shell=True, check=True) - - -def install(debug, clean): +def do_install(debug, clean): if clean and exists(SVSM_ROOT): result = run(f"sudo rm -rf {SVSM_ROOT}", shell=True, capture_output=True) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) run(f"sudo mkdir -p ${SVSM_ROOT}", shell=True, check=True) - # TODO: install guest qcow2 image + # Install guest kernel + copy_from_ctr_image( + SVSM_KERNEL_IMAGE_TAG, + ["/git/coconut-svsm/linux/arch/x86/boot/bzImage"], + [join(SVSM_ROOT, "share", "sc2", "vmlinuz-kata-containers-sc2")], + requires_sudo=True, + ) # Install QEMU and OVMF ctr_paths = [ join(SVSM_ROOT, "bin", "qemu-system-x86_64"), - join(SVSM_ROOT, "share", "qemu", "qemu/"), + join(SVSM_QEMU_DATA_DIR, "qemu"), "/git/coconut-svsm/edk2/Build/OvmfX64/RELEASE_GCC5/FV/OVMF.fd", ] host_paths = [ join(SVSM_ROOT, "bin", "qemu-system-x86_64"), - join(SVSM_ROOT, "share", "qemu"), + join(SVSM_QEMU_DATA_DIR, "qemu"), join(SVSM_ROOT, "share", "ovmf", "OVMF.fd"), ] copy_from_ctr_image(SVSM_QEMU_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True) + # Prepare the guest's initrd + do_build_initrd(clean=clean) + # Install SVSM's IGVM image copy_from_ctr_image( SVSM_IMAGE_TAG, ["/git/coconut-svsm/svsm/bin/coconut-qemu.igvm"], [join(SVSM_ROOT, "share", "igvm", "coconut-qemu.igvm")], - requires_sudo=True + requires_sudo=True, ) +# ------------------------------------------------------------------------------ +# Entry-point tasks +# ------------------------------------------------------------------------------ + + @task -def foo(ctx): - install(debug=False, clean=False) +def build_guest_kernel(ctx, nocache=False, push=False): + """ + Build the host/guest kernel fork to use with the SVSM + """ + do_build_kernel(nocache=nocache) @task -def install_host_kernel(ctx): +def build_initrd(ctx, clean=False): """ - Install the SVSM kernel in the host system + Generate an initrd with the kata agent and the different kernel modules. """ - kernel_version, kernel_version_trimmed = get_kernel_version_from_ctr_image() + do_build_initrd(clean=clean) - # Install the SVSM guest kernel into the host - ctr_paths = [ - "/git/coconut-svsm/linux/arch/x86/boot/bzImage", - f"/opt/sc2/svsm/share/linux/modules/lib/modules/{kernel_version}", - "/git/coconut-svsm/linux/.config", - ] - host_paths = [ - f"/boot/vmlinuz-{kernel_version_trimmed}", - f"/lib/modules/{kernel_version_trimmed}", - f"/boot/config-{kernel_version_trimmed}", - ] - copy_from_ctr_image( - SVSM_KERNEL_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True - ) - # Generate the corresponding kernel image - # echo "LVM=yes" > /user/share/initramfs-tools/conf.d/lvm - result = run("sudo DEBIAN_FRONTEND=noninteractive apt install -y initramfs-tools", shell=True, capture_output=True) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) - result = run( - # f"sudo mkinitramfs -o /boot/initrd.img-{kernel_version_trimmed} " - # f"{kernel_version_trimmed}", - f"sudo update-initramfs -c -k {kernel_version_trimmed}", - shell=True, - capture_output=True, - ) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) +@task +def build_qemu(ctx, nocache=False, push=False): + """ + Build the QEMU fork for its use with the SVSM + """ + do_build_qemu(nocache=nocache) - grub_update_default_kernel(kernel_version_trimmed) + if push: + run(f"docker push {SVSM_QEMU_IMAGE_TAG}", shell=True, check=True) @task -def install_upstream_kernel(ctx): - # TODO: find a way to automate this - kernel_ver = "6.13" - kernel_name = f"6.13.0-061300-generic" - - tmp_dir = f"/tmp/kernel-{kernel_ver}" - if exists(tmp_dir): - rmtree(tmp_dir) - makedirs(tmp_dir) - - base_url = f"https://kernel.ubuntu.com/mainline/v{kernel_ver}/amd64/" - # The order of this files in the array _matters_ - deb_files = [ - "linux-headers-6.13.0-061300_6.13.0-061300.202501302155_all.deb", - "linux-headers-6.13.0-061300-generic_6.13.0-061300.202501302155_amd64.deb", - "linux-modules-6.13.0-061300-generic_6.13.0-061300.202501302155_amd64.deb", - "linux-image-unsigned-6.13.0-061300-generic_6.13.0-061300.202501302155_amd64.deb", +def build_svsm(ctx, nocache=False): + build_args = { + "OVMF_FILE": "OVMF.fd", + } + build_args_str = [ + "--build-arg {}={}".format(key, build_args[key]) for key in build_args ] - for deb_file in deb_files: - result = run(f"wget {base_url}/{deb_file}", shell=True, capture_output=True, cwd=tmp_dir) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + build_args_str = " ".join(build_args_str) - for deb_file in deb_files: - result = run(f"sudo dpkg -i {deb_file}", shell=True, capture_output=True, cwd=tmp_dir) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + docker_cmd = "docker build{} {} -t {} -f {} {}".format( + " --no-cache" if nocache else "", + build_args_str, + SVSM_IMAGE_TAG, + join(PROJ_ROOT, "docker", "svsm.dockerfile"), + join(SVSM_ROOT, "share", "ovmf"), + ) + run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) - grub_update_default_kernel(kernel_name) + +@task +def install(ctx, clean=False): + """ + Install guest kernel, QEMU, OVMF, and SVSM IGVM image + """ + do_install(debug=False, clean=clean) diff --git a/tasks/util/docker.py b/tasks/util/docker.py index d7cfb951..f8125acc 100644 --- a/tasks/util/docker.py +++ b/tasks/util/docker.py @@ -35,6 +35,10 @@ def copy_from_ctr_image(ctr_image, ctr_paths, host_paths, requires_sudo=False): ) assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + def cleanup(): + result = run(f"docker rm -f {tmp_ctr_name}", shell=True, capture_output=True) + assert result.returncode == 0 + for ctr_path, host_path in zip(ctr_paths, host_paths): host_dir = dirname(host_path) if not exists(host_dir): @@ -53,9 +57,10 @@ def copy_from_ctr_image(ctr_image, ctr_paths, host_paths, requires_sudo=False): except AssertionError: stderr = result.stderr.decode("utf-8").strip() print(f"Error copying {ctr_image}:{ctr_path} to {host_path}: {stderr}") - break + cleanup() + raise RuntimeError("Error copying from container!") - result = run(f"docker rm -f {tmp_ctr_name}", shell=True, capture_output=True) + cleanup() def build_image(image_tag, dockerfile, build_args=None): diff --git a/tasks/util/kernel.py b/tasks/util/kernel.py index 0a7cdbf3..3eb1420d 100644 --- a/tasks/util/kernel.py +++ b/tasks/util/kernel.py @@ -9,3 +9,20 @@ def get_host_kernel_version(): current_kernel_name = fh.read().strip() return current_kernel_name + + +def grub_update_default_kernel(kernel_version): + """ + This method replaces the GRUB_DEFAULT value + """ + grub_default = f"Advanced options for Ubuntu>Ubuntu, with Linux {kernel_version}" + result = run( + f"sudo sed -i 's/^GRUB_DEFAULT=.*/GRUB_DEFAULT=\"{grub_default}\"/' " + "/etc/default/grub", + shell=True, + capture_output=True, + ) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + + result = run("sudo update-grub", shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) From e49e8f051ec2f7b77b4df086f3a100463269e840 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 24 Feb 2025 18:04:29 +0000 Subject: [PATCH 07/15] snp pods with kata working (default qemu) --- bin/launch_svsm.sh | 4 +- docker/ovmf.dockerfile | 47 +++++++++----------- tasks/kernel.py | 4 +- tasks/ovmf.py | 98 ++++++++++-------------------------------- tasks/sc2.py | 36 +++++++++++----- tasks/util/kernel.py | 14 ++++++ tasks/util/versions.py | 7 +++ 7 files changed, 93 insertions(+), 117 deletions(-) diff --git a/bin/launch_svsm.sh b/bin/launch_svsm.sh index 27bb6c0c..a338b1b1 100755 --- a/bin/launch_svsm.sh +++ b/bin/launch_svsm.sh @@ -17,10 +17,10 @@ trap "stty '$orig_stty'" EXIT # Remap Ctrl-C to Ctrl-] to allow the guest to handle Ctrl-C. stty intr ^] + # -enable-kvm \ sudo ${SVSM_ROOT}/bin/qemu-system-x86_64 \ - -enable-kvm \ -cpu EPYC-v4 \ - -machine q35,confidential-guest-support=sev0,memory-backend=ram1,igvm-cfg=igvm0 \ + -machine q35,confidential-guest-support=sev0,memory-backend=ram1,igvm-cfg=igvm0,accel=kvm \ -object memory-backend-memfd,id=ram1,size=8G,share=true,prealloc=false,reserve=false \ -object sev-snp-guest,id=sev0,cbitpos=${CBIT_POS},reduced-phys-bits=1 \ -object igvm-cfg,id=igvm0,file=$IGVM \ diff --git a/docker/ovmf.dockerfile b/docker/ovmf.dockerfile index fd56f05a..b32e501f 100644 --- a/docker/ovmf.dockerfile +++ b/docker/ovmf.dockerfile @@ -1,37 +1,30 @@ -FROM ubuntu:22.04 +FROM ghcr.io/sc2-sys/base:0.10.0 RUN apt update \ && apt upgrade -y \ && apt install -y \ - g++ \ - gcc \ - git \ + dosfstools \ + grub2-common \ + grub-efi \ iasl \ - make \ + mtools \ nasm \ - python3 \ - python-is-python3 \ - uuid-dev \ - vim + uuid-dev -ARG OVMF_PATCH -COPY ${OVMF_PATCH} /tmp/ovmf_profile.patch -ARG TARGET -RUN mkdir -p /usr/src/edk2 \ +ARG OVMF_VERSION +ARG CODE_DIR=/git/sc2-sys/edk2 +RUN mkdir -p ${CODE_DIR} \ && git clone \ - -b edk2-stable202302 \ - --single-branch --depth 1 \ + --branch ${OVMF_VERSION} \ + --depth 1 \ https://github.com/tianocore/edk2.git \ - /usr/src/edk2 \ - && cd /usr/src/edk2 \ + ${CODE_DIR} \ + && cd ${CODE_DIR} \ && git submodule update --init \ - && make -C BaseTools/ \ - && touch OvmfPkg/AmdSev/Grub/grub.efi \ - && git apply /tmp/ovmf_profile.patch \ - && cd OvmfPkg \ - && ./build.sh \ - -b ${TARGET} \ - -D DEBUG_ON_SERIAL_PORT \ - -p OvmfPkg/AmdSev/AmdSevX64.dsc - -WORKDIR /usr/src/edk2 + && export PYTHON3_ENABLE=TRUE \ + && export PYTHON_COMMAND=python3 \ + && make -j $(nproc) -C BaseTools/ \ + && . ./edksetup.sh --reconfig \ + && build -a X64 -b RELEASE -t GCC5 -p OvmfPkg/OvmfPkgX64.dsc \ + && touch OvmfPkg/AmdSev/Grub/grub.efi \ + && build -a X64 -b RELEASE -t GCC5 -p OvmfPkg/AmdSev/AmdSevX64.dsc diff --git a/tasks/kernel.py b/tasks/kernel.py index 3340ea15..5e090f93 100644 --- a/tasks/kernel.py +++ b/tasks/kernel.py @@ -30,9 +30,9 @@ def build_guest(debug=False, hot_replace=False): script_files = [ "kernel/build-kernel.sh", - "kernel/configs/", + "kernel/configs", "kernel/kata_config_version", - "kernel/patches/", + "kernel/patches", "scripts/apply_patches.sh", "scripts/lib.sh", ] diff --git a/tasks/ovmf.py b/tasks/ovmf.py index da03a0e0..318b0986 100644 --- a/tasks/ovmf.py +++ b/tasks/ovmf.py @@ -1,104 +1,52 @@ from invoke import task from os.path import join from subprocess import run -from tasks.util.env import BIN_DIR, KATA_CONFIG_DIR, PROJ_ROOT -from tasks.util.toml import update_toml +from tasks.util.docker import copy_from_ctr_image +from tasks.util.env import GHCR_URL, GITHUB_ORG, KATA_ROOT, PROJ_ROOT +from tasks.util.versions import OVMF_VERSION -OVMF_IMAGE_TAG = "ovmf-build" +OVMF_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, f"ovmf:{OVMF_VERSION}") -def do_ovmf_build(target, patch): +def do_ovmf_build(nocache=False, push=False): docker_cmd = [ "docker build", - "--build-arg TARGET={}".format(target), - "--build-arg OVMF_PATCH={}".format(patch), - "-t {}".format(OVMF_IMAGE_TAG), + f"--build-arg OVMF_VERSION={OVMF_VERSION}", + f"-t {OVMF_IMAGE_TAG}", + "--nocache" if nocache else "", "-f {} .".format(join(PROJ_ROOT, "docker", "ovmf.dockerfile")), ] docker_cmd = " ".join(docker_cmd) - run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) + result = run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) + # assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + if push: + result = run(f"docker push {OVMF_IMAGE_TAG}", shell=True, capture_output=True) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) -def copy_ovmf_from_src(dst_path, target, patch): + +def install(): """ Copy a custom build of OVMF into the destination path """ - do_ovmf_build(target, patch) - # Copy the debug-built OVMF into the destiantion path tmp_ctr_name = "tmp_ovmf" docker_cmd = "docker run -td --name {} {}".format(tmp_ctr_name, OVMF_IMAGE_TAG) run(docker_cmd, shell=True, check=True) - ctr_fd_path = "/usr/src/edk2/Build/AmdSev/{}_GCC5/FV/OVMF.fd".format(target) - docker_cmd = "docker cp {}:{} {}".format( - tmp_ctr_name, - ctr_fd_path, - dst_path, - ) - run(docker_cmd, shell=True, check=True) - - run("docker rm -f {}".format(tmp_ctr_name), shell=True, check=True) + ctr_paths = ["/git/sc2-sys/edk2/Build/AmdSev/RELEASE_GCC5/FV/OVMF.fd"] + host_paths = [join(KATA_ROOT, "share", "ovmf", "AMDSEV.fd")] + copy_from_ctr_image(OVMF_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True) @task -def build( - ctx, target="RELEASE", patch=join(PROJ_ROOT, "patches", "ovmf_profile.patch") -): - """ - Build the OVMF work-on container image - """ - do_ovmf_build(target, patch) +def foo(ctx): + install() @task -def set_log_level(ctx, log_level): +def build(ctx): """ - Set OVMF's log level, must be one in: info, debug, very-debug - - In order to toggle debug logging in OVMF, we need to update the QEMU - command line to include a couple of OVMF flags. To change the QEMU command - line, we use a bash wrapper with the extra flags, and point Kata to the - wrapper script. - - In addition, we need to re-compile OVMF from scratch with some additional - DEBUG statements (which we apply using a patch in `./patches`). - - Note that using a verbose version of OVMF is only supported with the - qemu-sev runtime class. Also note that the DEBUG log level still builds - a RELEASE target which introduces around 0.5 s of overhead to the boot - process. + Build the OVMF work-on container image """ - allowed_log_levels = ["info", "debug", "very-debug"] - if log_level not in allowed_log_levels: - print( - "Unsupported log level '{}'. Must be one in: {}".format( - log_level, allowed_log_levels - ) - ) - return - - default_qemu_path = "/opt/confidential-containers/bin/qemu-system-x86_64" - wrapper_qemu_path = join(BIN_DIR, "qemu_wrapper_ovmf_logging.sh") - qemu_path = default_qemu_path if log_level == "info" else wrapper_qemu_path - - default_fw_path = "/opt/confidential-containers/share/ovmf/AMDSEV.fd" - debug_fw_path = "/opt/confidential-containers/share/ovmf/AMDSEV_CSG.fd" - if log_level != "info": - if log_level == "debug": - patch = join(PROJ_ROOT, "patches", "ovmf_profile.patch") - else: - patch = join(PROJ_ROOT, "patches", "ovmf_overhead_wip.patch") - copy_ovmf_from_src(debug_fw_path, "RELEASE", patch) - fw_path = default_fw_path if log_level == "info" else debug_fw_path - - updated_toml_str = """ - [hypervisor.qemu] - path = "{qemu_path}" - firmware = "{fw_path}" - """.format( - qemu_path=qemu_path, fw_path=fw_path - ) - - conf_file_path = join(KATA_CONFIG_DIR, "configuration-qemu-sev.toml") - update_toml(conf_file_path, updated_toml_str) + do_ovmf_build() diff --git a/tasks/sc2.py b/tasks/sc2.py index 8845b4af..6f84b765 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -25,6 +25,7 @@ install as operator_install, install_cc_runtime as operator_install_cc_runtime, ) +from tasks.ovmf import install as ovmf_install from tasks.util.containerd import restart_containerd from tasks.util.docker import pull_artifact_images from tasks.util.env import ( @@ -34,7 +35,6 @@ CONTAINERD_CONFIG_ROOT, KATA_CONFIG_DIR, KATA_ROOT, - KATA_IMAGE_TAG, KATA_IMG_DIR, PROJ_ROOT, SC2_CONFIG_DIR, @@ -47,6 +47,7 @@ replace_agent as replace_kata_agent, replace_shim as replace_kata_shim, ) +from tasks.util.kernel import get_host_kernel_expected_version, get_host_kernel_version from tasks.util.kubeadm import run_kubectl_command from tasks.util.registry import ( HOST_CERT_DIR, @@ -54,7 +55,11 @@ stop as stop_local_registry, ) from tasks.util.toml import update_toml -from tasks.util.versions import COCO_VERSION, GUEST_KERNEL_VERSION, KATA_VERSION +from tasks.util.versions import ( + COCO_VERSION, + GUEST_KERNEL_VERSION, + OVMF_VERSION, +) from time import sleep @@ -213,9 +218,18 @@ def deploy(ctx, debug=False, clean=False): if exists(SC2_DEPLOYMENT_FILE): print(f"ERROR: SC2 already deployed (file {SC2_DEPLOYMENT_FILE} exists)") print("ERROR: only remove deployment file if you know what you are doing!") - raise RuntimeError("SC2 already deployed!") + return - # TODO: Fail-fast if we are not using the expected host kernel + # Fail-fast if we are not using the expected host kernel + host_kernel_version = get_host_kernel_version() + host_kernel_expected_version = get_host_kernel_expected_version() + if host_kernel_version != host_kernel_expected_version: + print( + f"ERROR: wrong host kernel: expected {host_kernel_expected_version} " + f"- got {host_kernel_version}" + ) + print("ERROR: install the right host kernel (./docs/host_kernel.md)") + return if clean: # Remove all directories that we populate and modify @@ -292,13 +306,13 @@ def deploy(ctx, debug=False, clean=False): # Install Knative knative_install(debug=debug) - # Apply general patches to the Kata Agent (and initrd), making sure we - # have the latest patched version - print_dotted_line(f"Pulling latest Kata image (v{KATA_VERSION})") - result = run(f"docker pull {KATA_IMAGE_TAG}", shell=True, capture_output=True) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) - if debug: - print(result.stdout.decode("utf-8").strip()) + print_dotted_line(f"Installing OVMF (v{OVMF_VERSION})") + ovmf_install() + print("Success!") + + # TODO: update SNP classes to use default QEMU + + # Apply general patches to the Kata Agent (and initrd) replace_kata_agent( dst_initrd_path=join( KATA_IMG_DIR, "kata-containers-initrd-confidential-sc2-baseline.img" diff --git a/tasks/util/kernel.py b/tasks/util/kernel.py index 3eb1420d..bd4b1179 100644 --- a/tasks/util/kernel.py +++ b/tasks/util/kernel.py @@ -1,4 +1,18 @@ +from os import environ from subprocess import run +from tasks.util.versions import HOST_KERNEL_VERSION_SNP, HOST_KERNEL_VERSION_TDX + + +def get_host_kernel_expected_version(): + sc2_runtime_class = environ["SC2_RUNTIME_CLASS"] + if "snp" in sc2_runtime_class: + return HOST_KERNEL_VERSION_SNP + + if "tdx" in sc2_runtime_class: + return HOST_KERNEL_VERSION_TDX + + print("ERROR: neither 'snp' nor 'tdx' detected!") + raise RuntimeError("Error detecting expected host kernel") def get_host_kernel_version(): diff --git a/tasks/util/versions.py b/tasks/util/versions.py index 54056b49..8cefe87f 100644 --- a/tasks/util/versions.py +++ b/tasks/util/versions.py @@ -28,7 +28,14 @@ KNATIVE_VERSION = "1.15.0" # Kernel versions +# WARNING: if we update the host kernel version, make sure to update it in the +# table in ./docs/host_kernel.md +HOST_KERNEL_VERSION_SNP = "6.11.0-snp-host-cc2568386+" +HOST_KERNEL_VERSION_TDX = "6.8.0-1004-intel" GUEST_KERNEL_VERSION = "6.12.8" # Coconut SVSM versions IGVM_VERSION = "0.3.4" + +# Firmware +OVMF_VERSION = "edk2-stable202411" From b97335b79ff3913306029dc7b00e68b05a848860 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Feb 2025 10:20:29 +0000 Subject: [PATCH 08/15] docker: also pull ovmf image --- tasks/sc2.py | 19 ++++++++++++++++++- tasks/util/docker.py | 4 +++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tasks/sc2.py b/tasks/sc2.py index 6f84b765..cb8db5bb 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -306,11 +306,28 @@ def deploy(ctx, debug=False, clean=False): # Install Knative knative_install(debug=debug) + # Install an up-to-date version of OVMF (the one currently shipped with + # CoCo is not enough to run on 6.11 and QEMU 9.1) print_dotted_line(f"Installing OVMF (v{OVMF_VERSION})") ovmf_install() print("Success!") - # TODO: update SNP classes to use default QEMU + # Update SNP class to use default QEMU (we use host kernel 6.11, so we + # can use upstream QEMU 9.1) + # TODO: remove when bumping to a new CoCo release + qemu_path = join(KATA_ROOT, "bin", "qemu-system-x86_64") + updated_toml_str = """ + [hypervisor.qemu] + path = "{qemu_path}" + valid_hypervisor_paths = [ "{qemu_path}" ] + """.format( + qemu_path=qemu_path + ) + update_toml( + join(KATA_CONFIG_DIR, "configuration-qemu-snp.toml"), + updated_toml_str, + requires_sudo=True, + ) # Apply general patches to the Kata Agent (and initrd) replace_kata_agent( diff --git a/tasks/util/docker.py b/tasks/util/docker.py index f8125acc..ebfa2e41 100644 --- a/tasks/util/docker.py +++ b/tasks/util/docker.py @@ -6,6 +6,7 @@ KATA_VERSION, NYDUS_SNAPSHOTTER_VERSION, NYDUS_VERSION, + OVMF_VERSION, ) @@ -95,12 +96,13 @@ def build_image_and_run(image_tag, dockerfile, ctr_name, build_args=None): def pull_artifact_images(debug=False): print_dotted_line("Pulling artifact container images") - components = ["containerd", "kata-containers", "nydus", "nydus-snapshotter"] + components = ["containerd", "kata-containers", "nydus", "nydus-snapshotter", "ovmf"] versions = [ CONTAINERD_VERSION, KATA_VERSION, NYDUS_VERSION, NYDUS_SNAPSHOTTER_VERSION, + OVMF_VERSION, ] for component, version in zip(components, versions): docker_cmd = f"docker pull {GHCR_URL}/{GITHUB_ORG}/{component}:{version}" From 81754b347ca4da8b908be81c7c778439c6a513c9 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Feb 2025 10:27:33 +0000 Subject: [PATCH 09/15] ovmf: fix installation --- tasks/ovmf.py | 10 ---------- tasks/sc2.py | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/tasks/ovmf.py b/tasks/ovmf.py index 318b0986..f84ef564 100644 --- a/tasks/ovmf.py +++ b/tasks/ovmf.py @@ -29,21 +29,11 @@ def install(): """ Copy a custom build of OVMF into the destination path """ - # Copy the debug-built OVMF into the destiantion path - tmp_ctr_name = "tmp_ovmf" - docker_cmd = "docker run -td --name {} {}".format(tmp_ctr_name, OVMF_IMAGE_TAG) - run(docker_cmd, shell=True, check=True) - ctr_paths = ["/git/sc2-sys/edk2/Build/AmdSev/RELEASE_GCC5/FV/OVMF.fd"] host_paths = [join(KATA_ROOT, "share", "ovmf", "AMDSEV.fd")] copy_from_ctr_image(OVMF_IMAGE_TAG, ctr_paths, host_paths, requires_sudo=True) -@task -def foo(ctx): - install() - - @task def build(ctx): """ diff --git a/tasks/sc2.py b/tasks/sc2.py index cb8db5bb..a95fac03 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -308,7 +308,7 @@ def deploy(ctx, debug=False, clean=False): # Install an up-to-date version of OVMF (the one currently shipped with # CoCo is not enough to run on 6.11 and QEMU 9.1) - print_dotted_line(f"Installing OVMF (v{OVMF_VERSION})") + print_dotted_line(f"Installing OVMF ({OVMF_VERSION})") ovmf_install() print("Success!") From 17872b0025576c34a4b75d3c6421bd2e8128b209 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Feb 2025 10:29:44 +0000 Subject: [PATCH 10/15] ovmf: nits --- tasks/ovmf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/ovmf.py b/tasks/ovmf.py index f84ef564..39f88754 100644 --- a/tasks/ovmf.py +++ b/tasks/ovmf.py @@ -35,8 +35,8 @@ def install(): @task -def build(ctx): +def build(ctx, nocache=False, push=False): """ Build the OVMF work-on container image """ - do_ovmf_build() + do_ovmf_build(nocache=nocache, push=push) From b7cdbf0aa3ac7a3b17d5c689d32dd153639448ef Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Feb 2025 10:33:06 +0000 Subject: [PATCH 11/15] svsm: clean-up old coconut folders --- docker/coconut/ovmf.dockerfile | 19 --------- docker/coconut/svsm.dockerfile | 28 ------------- tasks/__init__.py | 4 -- tasks/coconut/__init__.py | 7 ---- tasks/coconut/ovmf.py | 16 -------- tasks/coconut/qemu.py | 75 ---------------------------------- tasks/coconut/svsm.py | 36 ---------------- tasks/ovmf.py | 2 +- 8 files changed, 1 insertion(+), 186 deletions(-) delete mode 100644 docker/coconut/ovmf.dockerfile delete mode 100644 docker/coconut/svsm.dockerfile delete mode 100644 tasks/coconut/__init__.py delete mode 100644 tasks/coconut/ovmf.py delete mode 100644 tasks/coconut/qemu.py delete mode 100644 tasks/coconut/svsm.py diff --git a/docker/coconut/ovmf.dockerfile b/docker/coconut/ovmf.dockerfile deleted file mode 100644 index dd2b78d6..00000000 --- a/docker/coconut/ovmf.dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM ubuntu:22.04 - -RUN sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list -RUN apt update \ - && apt upgrade -y \ - && apt install -y \ - git \ - && apt build-dep ovmf -y - -RUN git clone https://github.com/coconut-svsm/edk2.git ~/edk2\ - && cd ~/edk2/ \ - && git checkout svsm \ - && git submodule init \ - && git submodule update \ - && export PYTHON3_ENABLE=TRUE \ - && export PYTHON_COMMAND=python3 \ - && make -j16 -C BaseTools/ \ - && . ./edksetup.sh --reconfig \ - && build -a X64 -b DEBUG -t GCC5 -D DEBUG_ON_SERIAL_PORT -D DEBUG_VERBOSE -DTPM2_ENABLE -p OvmfPkg/OvmfPkgX64.dsc diff --git a/docker/coconut/svsm.dockerfile b/docker/coconut/svsm.dockerfile deleted file mode 100644 index 71bb4a87..00000000 --- a/docker/coconut/svsm.dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM ubuntu:24.04 - -RUN apt update \ - && apt upgrade -y \ - && apt install -y \ - curl \ - git \ - gcc \ - make \ - automake \ - libssl-dev \ - autoconf \ - autoconf-archive \ - build-essential \ - pkg-config \ - libclang-dev - -ARG OVMF_DIR -COPY ${OVMF_DIR}/ovmf-svsm.fd /bin/ovmf-svsm.fd - -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \ - && . "$HOME/.cargo/env" \ - && rustup target add x86_64-unknown-none \ - && git clone https://github.com/sc2-sys/svsm.git ~/svsm \ - && cd ~/svsm \ - && git submodule update --init \ - && cargo install bindgen-cli \ - && FW_FILE=/bin/ovmf-svsm.fd ./build --release configs/qemu-target.json diff --git a/tasks/__init__.py b/tasks/__init__.py index 89323acc..9024b708 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -25,8 +25,6 @@ from . import skopeo from . import svsm -from tasks.coconut import ns as coconut_ns - ns = Collection( base, coco, @@ -53,5 +51,3 @@ skopeo, svsm, ) - -ns.add_collection(coconut_ns, name="coconut") diff --git a/tasks/coconut/__init__.py b/tasks/coconut/__init__.py deleted file mode 100644 index 574398b9..00000000 --- a/tasks/coconut/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from invoke import Collection - -from . import qemu -from . import ovmf -from . import svsm - -ns = Collection(qemu, ovmf, svsm) diff --git a/tasks/coconut/ovmf.py b/tasks/coconut/ovmf.py deleted file mode 100644 index 673c2a4d..00000000 --- a/tasks/coconut/ovmf.py +++ /dev/null @@ -1,16 +0,0 @@ -from invoke import task -from os.path import join -from tasks.util.env import BIN_DIR -from tasks.util.docker import copy_from_ctr_image - -# refer to -# https://github.com/coconut-svsm/svsm/blob/main/Documentation/docs/installation/INSTALL.md - -OVMF_IMAGE_TAG = "ovmf-svsm-build" - - -@task -def build(ctx): - ctr_path = "/root/edk2/Build/OvmfX64/DEBUG_GCC5/FV/OVMF.fd" - host_path = join(BIN_DIR, "ovmf-svsm.fd") - copy_from_ctr_image(OVMF_IMAGE_TAG, ctr_path, host_path) diff --git a/tasks/coconut/qemu.py b/tasks/coconut/qemu.py deleted file mode 100644 index 85bdc317..00000000 --- a/tasks/coconut/qemu.py +++ /dev/null @@ -1,75 +0,0 @@ -from invoke import task -from os.path import join -from subprocess import run -from tasks.util.env import BIN_DIR, PROJ_ROOT, KATA_ROOT -from tasks.util.docker import copy_from_ctr_image, build_image_and_run - -# refer to -# https://github.com/coconut-svsm/svsm/blob/main/Documentation/docs/installation/INSTALL.md - -QEMU_IMAGE_TAG = "qemu-igvm-build" -DATA_DIR = join(KATA_ROOT, "coconut", "qemu-svsm", "share") - - -@task -def build(ctx): - tmp_ctr_name = "tmp-qemu-igvm-run" - - # TODO: fix me - build_image_and_run( - QEMU_IMAGE_TAG, - join(PROJ_ROOT, "docker", "coconut", "qemu.dockerfile"), - tmp_ctr_name, - {"QEMU_DATADIR": DATA_DIR}, - ) - - copy_from_ctr_image( - QEMU_IMAGE_TAG, - "/root/bin/qemu-svsm/bin/qemu-system-x86_64", - join(BIN_DIR, "qemu-system-x86_64-igvm"), - ) - # copy_from_container(tmp_ctr_name, f"{DATA_DIR}/.", DATA_DIR) - - -@task -def guest(ctx, guest_img_path=join(PROJ_ROOT, "arch_7.qcow2"), detach=False, vnc=False): - qemu_path = join(BIN_DIR, "qemu-system-x86_64-igvm") - igvm_path = join(BIN_DIR, "coconut-qemu.igvm") - - qemu_cmd = [ - "sudo", - qemu_path, - "-enable-kvm", - "-cpu EPYC-v4", - "-machine q35,confidential-guest-support=sev0,memory-backend=ram1", - ( - "-object memory-backend-memfd,id=ram1,size=8G,share=true," - "prealloc=false,reserve=false" - ), - ( - "-object sev-snp-guest,id=sev0,cbitpos=51," - "reduced-phys-bits=1,igvm-file={}" - ).format(igvm_path), - "-smp 8", - "-no-reboot", - ( - "-netdev user,id=vmnic,hostfwd=tcp::2222-:22," - "hostfwd=tcp::8080-:80 -device e1000,netdev=vmnic,romfile=" - ), - "-device virtio-scsi-pci,id=scsi0,disable-legacy=on,iommu_platform=on", - "-device scsi-hd,drive=disk0,bootindex=0", - "-drive file={},if=none,id=disk0,format=qcow2,snapshot=off".format( - guest_img_path - ), - "--serial file:tmp.log", - "-display none", - ] - if detach: - qemu_cmd.append("-daemonize") - - if vnc: - qemu_cmd.append("-vnc :1") - - qemu_cmd = " ".join(qemu_cmd) - print(qemu_cmd) - run(qemu_cmd, shell=True, check=True) diff --git a/tasks/coconut/svsm.py b/tasks/coconut/svsm.py deleted file mode 100644 index c07f1321..00000000 --- a/tasks/coconut/svsm.py +++ /dev/null @@ -1,36 +0,0 @@ -from invoke import task -from os.path import join -from tasks.util.env import BIN_DIR, PROJ_ROOT -from tasks.util.docker import build_image_and_run, copy_from_ctr_image, stop_container - -# refer to -# https://github.com/coconut-svsm/svsm/blob/main/Documentation/docs/installation/INSTALL.md - -QEMU_IMAGE_TAG = "svsm-build" - - -@task -def build(ctx): - tmp_ctr_name = "tmp-svsm-run" - - build_image_and_run( - QEMU_IMAGE_TAG, - join(PROJ_ROOT, "docker", "coconut", "svsm.dockerfile"), - tmp_ctr_name, - {"OVMF_DIR": "bin"}, - ) - - ctr_path = "/root/svsm/bin" - host_path = BIN_DIR - files_to_copy = [ - "svsm.bin", - "coconut-qemu.igvm", - "../target/x86_64-unknown-none/debug/svsm", - ] - # FIXME: sure it is the right tag? - for file_name in files_to_copy: - copy_from_ctr_image( - QEMU_IMAGE_TAG, join(ctr_path, file_name), join(host_path, file_name) - ) - - stop_container(tmp_ctr_name) diff --git a/tasks/ovmf.py b/tasks/ovmf.py index 39f88754..c4af35d0 100644 --- a/tasks/ovmf.py +++ b/tasks/ovmf.py @@ -18,7 +18,7 @@ def do_ovmf_build(nocache=False, push=False): ] docker_cmd = " ".join(docker_cmd) result = run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT) - # assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) if push: result = run(f"docker push {OVMF_IMAGE_TAG}", shell=True, capture_output=True) From ea629a49d35f82eeefc01ca0a1be1a934de06d4e Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Feb 2025 16:05:02 +0000 Subject: [PATCH 12/15] nits: self-review --- bin/launch_svsm.sh | 2 -- tasks/sc2.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/bin/launch_svsm.sh b/bin/launch_svsm.sh index a338b1b1..f078e3a6 100755 --- a/bin/launch_svsm.sh +++ b/bin/launch_svsm.sh @@ -7,7 +7,6 @@ CBIT_POS=51 IGVM=${SVSM_ROOT}/share/igvm/coconut-qemu.igvm KERNEL=${SVSM_ROOT}/share/sc2/vmlinuz-kata-containers-sc2 -# KERNEL=/home/csegarra/git/sc2-sys/svsm-linux/arch/x86/boot/bzImage INITRD=/opt/sc2/svsm/share/sc2/initrd-kata.img # Ensure terminal settings are restored on exit @@ -17,7 +16,6 @@ trap "stty '$orig_stty'" EXIT # Remap Ctrl-C to Ctrl-] to allow the guest to handle Ctrl-C. stty intr ^] - # -enable-kvm \ sudo ${SVSM_ROOT}/bin/qemu-system-x86_64 \ -cpu EPYC-v4 \ -machine q35,confidential-guest-support=sev0,memory-backend=ram1,igvm-cfg=igvm0,accel=kvm \ diff --git a/tasks/sc2.py b/tasks/sc2.py index a95fac03..58856c1b 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -326,7 +326,7 @@ def deploy(ctx, debug=False, clean=False): update_toml( join(KATA_CONFIG_DIR, "configuration-qemu-snp.toml"), updated_toml_str, - requires_sudo=True, + requires_root=True, ) # Apply general patches to the Kata Agent (and initrd) @@ -337,7 +337,6 @@ def deploy(ctx, debug=False, clean=False): debug=debug, sc2=False, ) - print("Success!") # Install sc2 runtime with patches print_dotted_line(f"Installing SC2 (v{COCO_VERSION})") From b8c376320936ff8b7ede63678ead150ee3a32637 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Feb 2025 16:35:04 +0000 Subject: [PATCH 13/15] sc2: exit with an error code when error-ing --- tasks/sc2.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks/sc2.py b/tasks/sc2.py index 58856c1b..c56055b5 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -2,6 +2,7 @@ from os import environ, makedirs from os.path import exists, join from subprocess import run +from sys import exit from tasks.containerd import ( install as containerd_install, install_bbolt as bbolt_install, @@ -218,7 +219,7 @@ def deploy(ctx, debug=False, clean=False): if exists(SC2_DEPLOYMENT_FILE): print(f"ERROR: SC2 already deployed (file {SC2_DEPLOYMENT_FILE} exists)") print("ERROR: only remove deployment file if you know what you are doing!") - return + exit(1) # Fail-fast if we are not using the expected host kernel host_kernel_version = get_host_kernel_version() @@ -229,7 +230,7 @@ def deploy(ctx, debug=False, clean=False): f"- got {host_kernel_version}" ) print("ERROR: install the right host kernel (./docs/host_kernel.md)") - return + exit(1) if clean: # Remove all directories that we populate and modify From 47f7071b55e76e1051207e39ee63a2516e53274c Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Feb 2025 17:09:22 +0000 Subject: [PATCH 14/15] sc2: patch kata-runtime to fix qemu behaviour in 6.11 --- tasks/sc2.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tasks/sc2.py b/tasks/sc2.py index c56055b5..328d2bae 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -330,6 +330,13 @@ def deploy(ctx, debug=False, clean=False): requires_root=True, ) + # Apply general patches to the Kata runtime + replace_kata_shim( + dst_shim_binary=join(KATA_ROOT, "bin", "containerd-shim-kata-v2"), + dst_runtime_binary=join(KATA_ROOT, "bin", "kata-runtime"), + sc2=False, + ) + # Apply general patches to the Kata Agent (and initrd) replace_kata_agent( dst_initrd_path=join( From 699d2e5032af495da711a5681d7dc7dcaf0437e1 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Feb 2025 17:09:36 +0000 Subject: [PATCH 15/15] util(versions): update intel tdx host kernel version --- tasks/util/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/util/versions.py b/tasks/util/versions.py index 8cefe87f..8079a1df 100644 --- a/tasks/util/versions.py +++ b/tasks/util/versions.py @@ -31,7 +31,7 @@ # WARNING: if we update the host kernel version, make sure to update it in the # table in ./docs/host_kernel.md HOST_KERNEL_VERSION_SNP = "6.11.0-snp-host-cc2568386+" -HOST_KERNEL_VERSION_TDX = "6.8.0-1004-intel" +HOST_KERNEL_VERSION_TDX = "6.8.0-1013-intel" GUEST_KERNEL_VERSION = "6.12.8" # Coconut SVSM versions