From a58ba6b925dfd375e7eba4f034fa668408ae81a3 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 22 Jan 2025 19:00:29 +0000 Subject: [PATCH 1/7] kata: use rootfs-builder to build rootfs --- tasks/util/kata.py | 180 ++++++++++++++++++++++++++------------------- 1 file changed, 105 insertions(+), 75 deletions(-) diff --git a/tasks/util/kata.py b/tasks/util/kata.py index 755fa098..559be720 100644 --- a/tasks/util/kata.py +++ b/tasks/util/kata.py @@ -74,9 +74,16 @@ def copy_from_kata_workon_ctr( ctr_path, host_path, ) + if sudo: docker_cmd = "sudo {}".format(docker_cmd) + if debug: + print(docker_cmd) + result = run(docker_cmd, shell=True, capture_output=True) + assert result.returncode == 0, "Error copying from container: {}".format( + result.stderr.decode("utf-8") + ) if debug: print(result.stdout.decode("utf-8").strip()) else: @@ -94,40 +101,68 @@ def replace_agent( """ Replace the kata-agent with a custom-built one - Replacing the kata-agent is a bit fiddly, as the kata-agent binary lives - inside the initrd guest image that we load to the VM. The replacement - includes the following steps: - 1. Find the initrd file - should be pointed in the Kata config file - 2. Unpack the initrd - 3. Replace the init process by the new kata agent - 4. Re-build the initrd - 5. Update the kata config to point to the new initrd - - By using the extra_flags optional argument, you can pass a dictionary of - host_path: guest_path pairs of files you want to be included in the initrd. + We use Kata's `rootfs-builder` to prepare a `rootfs` based on an Ubuntu + image with custom packages, then copy into the rootfs additional files + that we may need, and finally package it using Kata's `initrd-builder`. """ - # This is a list of files that we want to _always_ include in our custom - # agent builds - extra_files = { - "/etc/hosts": {"path": "/etc/hosts", "mode": "w"}, - HOST_CERT_PATH: {"path": "/etc/ssl/certs/ca-certificates.crt", "mode": "a"}, - } + # ----- Prepare temporary rootfs directory ----- + + tmp_rootfs_base_dir = "/tmp/sc2-rootfs-build-dir" + tmp_rootfs_dir = join(tmp_rootfs_base_dir, "rootfs") + tmp_rootfs_scripts_dir = join(tmp_rootfs_base_dir, "osbuilder") - # Use a hardcoded path, as we want to always start from a _clean_ initrd - initrd_path = join(KATA_IMG_DIR, "kata-containers-initrd-confidential.img") + if exists(tmp_rootfs_base_dir): + out = run(f"sudo rm -rf {tmp_rootfs_base_dir}", shell=True, capture_output=True) + assert out.returncode == 0, "Error removing previous rootfs: {}".format( + out.stderr.decode("utf-8") + ) - # Make empty temporary dir to expand the initrd filesystem - workdir = "/tmp/qemu-sev-initrd" - run("sudo rm -rf {}".format(workdir), shell=True, check=True) - makedirs(workdir) + makedirs(tmp_rootfs_base_dir) + makedirs(tmp_rootfs_dir) + makedirs(tmp_rootfs_scripts_dir) + makedirs(join(tmp_rootfs_scripts_dir, "initrd-builder")) + makedirs(join(tmp_rootfs_scripts_dir, "rootfs-builder")) + makedirs(join(tmp_rootfs_scripts_dir, "rootfs-builder", "ubuntu")) + makedirs(join(tmp_rootfs_scripts_dir, "scripts")) - # sudo unpack the initrd filesystem - zcat_cmd = "sudo bash -c 'zcat {} | cpio -idmv'".format(initrd_path) - out = run(zcat_cmd, shell=True, capture_output=True, cwd=workdir) - assert out.returncode == 0, "Error unpacking initrd: {}".format(out.stderr) + # Copy all the tooling/script files we need from the container + script_files = [ + "initrd-builder/initrd_builder.sh", + "rootfs-builder/rootfs.sh", + "rootfs-builder/ubuntu/config.sh", + "rootfs-builder/ubuntu/Dockerfile.in", + "rootfs-builder/ubuntu/rootfs_lib.sh", + "scripts/lib.sh", + ] - # Copy the kata-agent in our docker image into `/usr/bin/kata-agent` as - # this is the path expected by the kata initrd_builder.sh script + for ctr_path, host_path in zip( + [ + join( + KATA_SOURCE_DIR if sc2 else KATA_BASELINE_SOURCE_DIR, + "tools/osbuilder", + script, + ) + for script in script_files + ], + [join(tmp_rootfs_scripts_dir, script) for script in script_files], + ): + copy_from_kata_workon_ctr( + ctr_path, host_path, sudo=True, debug=debug, hot_replace=hot_replace + ) + + # Also copy a policy file needed to build the rootfs + copy_from_kata_workon_ctr( + join( + KATA_SOURCE_DIR if sc2 else KATA_BASELINE_SOURCE_DIR, + "src/kata-opa/allow-all.rego", + ), + join(tmp_rootfs_base_dir, "allow-all.rego"), + sudo=True, + debug=debug, + hot_replace=hot_replace, + ) + + # Finally, also copy our kata agent agent_host_path = join( KATA_AGENT_SOURCE_DIR if sc2 else KATA_BASELINE_AGENT_SOURCE_DIR, "target", @@ -135,27 +170,50 @@ def replace_agent( "release", "kata-agent", ) - agent_initrd_path = join(workdir, "usr/bin/kata-agent") copy_from_kata_workon_ctr( agent_host_path, - agent_initrd_path, + join(tmp_rootfs_base_dir, "kata-agent"), sudo=True, debug=debug, hot_replace=hot_replace, ) - # We also need to manually copy the agent to /sbin/init (note that - # /init is a symlink to /sbin/init) - alt_agent_initrd_path = join(workdir, "sbin", "init") - run("sudo rm {}".format(alt_agent_initrd_path), shell=True, check=True) - copy_from_kata_workon_ctr( - agent_host_path, - alt_agent_initrd_path, - sudo=True, - debug=debug, - hot_replace=hot_replace, + # ----- Populate rootfs with base ubuntu using Kata's scripts ----- + + out = run("sudo apt install -y makedev multistrap", shell=True, capture_output=True) + assert out.returncode == 0, "Error preparing rootfs: {}".format( + out.stderr.decode("utf-8") ) + rootfs_builder_dir = join(tmp_rootfs_scripts_dir, "rootfs-builder") + work_env = { + "AGENT_INIT": "yes", + "AGENT_POLICY_FILE": join(tmp_rootfs_base_dir, "allow-all.rego"), + "AGENT_SOURCE_BIN": join(tmp_rootfs_base_dir, "kata-agent"), + "DMVERITY_SUPPORT": "yes", + "ROOTFS_DIR": tmp_rootfs_dir, + } + rootfs_builder_cmd = f"sudo -E {rootfs_builder_dir}/rootfs.sh ubuntu > /dev/null" + out = run( + rootfs_builder_cmd, + shell=True, + env=work_env, + cwd=rootfs_builder_dir, + capture_output=True, + ) + assert out.returncode == 0, "Error preparing rootfs: {}".format( + out.stderr.decode("utf-8") + ) + if debug: + print(out.stdout.decode("utf-8").strip()) + + # ----- Add extra files to the rootfs ----- + + extra_files = { + "/etc/hosts": {"path": "/etc/hosts", "mode": "w"}, + HOST_CERT_PATH: {"path": "/etc/ssl/certs/ca-certificates.crt", "mode": "a"}, + } + # Include any extra files that the caller may have provided if extra_files is not None: for host_path in extra_files: @@ -165,7 +223,7 @@ def replace_agent( if rel_guest_path.startswith("/"): rel_guest_path = rel_guest_path[1:] - guest_path = join(workdir, rel_guest_path) + guest_path = join(tmp_rootfs_dir, rel_guest_path) if not exists(dirname(guest_path)): run( "sudo mkdir -p {}".format(dirname(guest_path)), @@ -186,41 +244,13 @@ def replace_agent( check=True, ) - # Pack the initrd again (copy the script from the container into a - # temporarly location). Annoyingly, we also need to copy a bash script in - # the same relative directory structure (cuz bash). - kata_tmp_scripts = "/tmp/osbuilder" - run( - "rm -rf {} && mkdir -p {} {}".format( - kata_tmp_scripts, - join(kata_tmp_scripts, "scripts"), - join(kata_tmp_scripts, "initrd-builder"), - ), - shell=True, - check=True, - ) - ctr_initrd_builder_path = join( - KATA_SOURCE_DIR, "tools", "osbuilder", "initrd-builder", "initrd_builder.sh" - ) - ctr_lib_path = join(KATA_SOURCE_DIR, "tools", "osbuilder", "scripts", "lib.sh") - initrd_builder_path = join(kata_tmp_scripts, "initrd-builder", "initrd_builder.sh") - copy_from_kata_workon_ctr( - ctr_initrd_builder_path, - initrd_builder_path, - debug=debug, - hot_replace=hot_replace, - ) - copy_from_kata_workon_ctr( - ctr_lib_path, - join(kata_tmp_scripts, "scripts", "lib.sh"), - debug=debug, - hot_replace=hot_replace, - ) + # ----- Pack rootfs into initrd using Kata's script ----- + work_env = {"AGENT_INIT": "yes"} - initrd_pack_cmd = "sudo {} -o {} {}".format( - initrd_builder_path, + initrd_pack_cmd = "sudo -E {} -o {} {}".format( + join(tmp_rootfs_scripts_dir, "initrd-builder", "initrd_builder.sh"), dst_initrd_path, - workdir, + tmp_rootfs_dir, ) out = run(initrd_pack_cmd, shell=True, env=work_env, capture_output=True) assert out.returncode == 0, "Error packing initrd: {}".format( From 0f5039c273c5581198485102ef55e26eb756a27b Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 23 Jan 2025 18:06:35 +0000 Subject: [PATCH 2/7] kernel: add task to build our guest kernel --- tasks/__init__.py | 2 + tasks/kernel.py | 112 +++++++++++++++++++++++++++++++++++++++++ tasks/sc2.py | 8 ++- tasks/util/env.py | 4 +- tasks/util/kata.py | 81 ++++++++++++++++++++++++++++- tasks/util/versions.py | 4 ++ 6 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 tasks/kernel.py diff --git a/tasks/__init__.py b/tasks/__init__.py index 4fbccff0..f9a17f25 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -7,6 +7,7 @@ from . import demo_apps from . import format_code from . import gc +from . import kernel from . import k8s from . import k9s from . import kata @@ -37,6 +38,7 @@ k9s, kata, kbs, + kernel, knative, kubeadm, nydus, diff --git a/tasks/kernel.py b/tasks/kernel.py new file mode 100644 index 00000000..721d2a2a --- /dev/null +++ b/tasks/kernel.py @@ -0,0 +1,112 @@ +from invoke import task +from os import makedirs +from os.path import exists, join +from tasks.util.env import KATA_CONFIG_DIR, KATA_IMG_DIR, KATA_RUNTIMES, SC2_RUNTIMES +from tasks.util.kata import KATA_SOURCE_DIR, copy_from_kata_workon_ctr +from tasks.util.toml import update_toml +from tasks.util.versions import GUEST_KERNEL_VERSION +from subprocess import run + + +def build_guest(debug=False, hot_replace=False): + """ + Build the guest kernel. + + We use Kata's build-kernel.sh to build the guest kernel. Note that, for + the time being, there is no difference between SC2 and non-SC2 guest + kernels. We still need to update them all, because our manual rootfs + build requires a manual kernel build too (for some reason). + """ + kernel_build_dir = "/tmp/sc2-guest-kernel-build-dir" + + if exists(kernel_build_dir): + run(f"sudo rm -rf {kernel_build_dir}", shell=True, check=True) + + makedirs(kernel_build_dir) + makedirs(join(kernel_build_dir, "kernel")) + makedirs(join(kernel_build_dir, "scripts")) + + # Prefix: tools/packaging + script_files = [ + "kernel/build-kernel.sh", + "kernel/configs/", + "kernel/kata_config_version", + "kernel/patches/", + "scripts/apply_patches.sh", + "scripts/lib.sh", + ] + + for ctr_path, host_path in zip( + [ + join( + # WARNING: for the time being it is OK to copy from the SC2 + # Kata source dir because there is no difference between + # SC2 and non-SC2 guest kernels, but this is something we + # should keep in mind. + KATA_SOURCE_DIR, + "tools/packaging", + script, + ) + for script in script_files + ], + [join(kernel_build_dir, script) for script in script_files], + ): + copy_from_kata_workon_ctr( + ctr_path, host_path, sudo=False, debug=debug, hot_replace=hot_replace + ) + + build_kernel_base_cmd = [ + "./build-kernel.sh -x -f" f" -v {GUEST_KERNEL_VERSION}", + "-u 'https://cdn.kernel.org/pub/linux/kernel/v{}.x/'".format( + GUEST_KERNEL_VERSION.split(".")[0] + ), + ] + build_kernel_base_cmd = " ".join(build_kernel_base_cmd) + + for step in ["setup", "build"]: + out = run( + f"{build_kernel_base_cmd} {step}", + shell=True, + capture_output=True, + cwd=join(kernel_build_dir, "kernel"), + ) + assert out.returncode == 0, "Error building guest kernel: {}\n{}".format( + out.stdout.decode("utf-8"), out.stderr.decode("utf-8") + ) + if debug: + print(out.stdout.decode("utf-8")) + + # Copy the built kernel into the desired path + # "./build-kernel.sh -x -f -v 6.7 setup" + with open(join(kernel_build_dir, "kernel", "kata_config_version"), "r") as fh: + kata_config_version = fh.read() + kata_config_version = kata_config_version.strip() + + sc2_kernel_name = "vmlinuz-confidential-sc2.container" + bzimage_src_path = join( + kernel_build_dir, + "kernel", + f"kata-linux-{GUEST_KERNEL_VERSION}-{kata_config_version}", + "arch", + "x86", + "boot", + "bzImage", + ) + bzimage_dst_path = join(KATA_IMG_DIR, sc2_kernel_name) + run(f"sudo cp {bzimage_src_path} {bzimage_dst_path}", shell=True, check=True) + + # Update the paths in the config files + for runtime in KATA_RUNTIMES + SC2_RUNTIMES: + conf_file_path = join(KATA_CONFIG_DIR, "configuration-{}.toml".format(runtime)) + updated_toml_str = """ + [hypervisor.qemu] + kernel = "{new_kernel_path}" + """.format( + new_kernel_path=bzimage_dst_path + ) + update_toml(conf_file_path, updated_toml_str) + + +@task +def hot_replace_guest(ctx, debug=False): + build_guest(debug=debug, hot_replace=True) diff --git a/tasks/sc2.py b/tasks/sc2.py index bd9f524f..f9a8f4ae 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -8,6 +8,7 @@ ) from tasks.k8s import install as k8s_tooling_install from tasks.k9s import install as k9s_install +from tasks.kernel import build_guest as build_guest_kernel from tasks.knative import install as knative_install from tasks.kubeadm import create as k8s_create, destroy as k8s_destroy from tasks.nydus_snapshotter import install as nydus_snapshotter_install @@ -44,7 +45,7 @@ stop as stop_local_registry, ) from tasks.util.toml import update_toml -from tasks.util.versions import COCO_VERSION, KATA_VERSION +from tasks.util.versions import COCO_VERSION, GUEST_KERNEL_VERSION, KATA_VERSION from time import sleep @@ -263,6 +264,11 @@ def deploy(ctx, debug=False, clean=False): # Install the nydusify tool nydus_install() + # Build and install the guest VM kernel + print_dotted_line(f"Build and install guest VM kernel (v{GUEST_KERNEL_VERSION})") + build_guest_kernel() + print("Success!") + # Start a local docker registry (must happen before knative installation, # as we rely on it to host our sidecar image) start_local_registry(debug=debug, clean=clean) diff --git a/tasks/util/env.py b/tasks/util/env.py index dd76329f..4843319c 100644 --- a/tasks/util/env.py +++ b/tasks/util/env.py @@ -1,6 +1,6 @@ from os.path import dirname, expanduser, realpath, join from subprocess import run -from tasks.util.versions import KATA_VERSION +from tasks.util.versions import KATA_VERSION, PAUSE_IMAGE_VERSION PROJ_ROOT = dirname(dirname(dirname(realpath(__file__)))) @@ -18,6 +18,8 @@ K8S_CONFIG_FILE = "/etc/kubernetes/admin.conf" # This value is hardcoded in ./.config/kubeadm.conf CRI_RUNTIME_SOCKET = "unix:///run/containerd/containerd.sock" +PAUSE_IMAGE_REPO = "docker://registry.k8s.io/pause" +PAUSE_IMAGE = f"{PAUSE_IMAGE_REPO}:{PAUSE_IMAGE_VERSION}" # Containerd diff --git a/tasks/util/kata.py b/tasks/util/kata.py index 559be720..b8f5d23d 100644 --- a/tasks/util/kata.py +++ b/tasks/util/kata.py @@ -1,4 +1,4 @@ -from os import makedirs +from os import environ, makedirs from os.path import dirname, exists, join from subprocess import run from tasks.util.docker import copy_from_ctr_image, is_ctr_running @@ -10,9 +10,11 @@ KATA_RUNTIMES, KATA_WORKON_CTR_NAME, KATA_IMAGE_TAG, + PAUSE_IMAGE_REPO, SC2_RUNTIMES, ) from tasks.util.registry import HOST_CERT_PATH +from tasks.util.versions import PAUSE_IMAGE_VERSION from tasks.util.toml import remove_entry_from_toml, update_toml # These paths are hardcoded in the docker image: ./docker/kata.dockerfile @@ -92,6 +94,69 @@ def copy_from_kata_workon_ctr( copy_from_ctr_image(KATA_IMAGE_TAG, [ctr_path], [host_path], requires_sudo=sudo) +def build_pause_image(sc2, debug, hot_replace): + """ + When we create a rootfs for CoCo, we need to embed the pause image into + it. As a consequence, we need to build the tarball first. + """ + pause_image_build_dir = "/tmp/sc2-pause-image-build-dir" + + if exists(pause_image_build_dir): + run(f"sudo rm -rf {pause_image_build_dir}", shell=True, check=True) + + makedirs(pause_image_build_dir) + makedirs(join(pause_image_build_dir, "static-build")) + makedirs(join(pause_image_build_dir, "scripts")) + + script_files = ["static-build/pause-image/", "scripts/lib.sh"] + for ctr_path, host_path in zip( + [ + join( + KATA_SOURCE_DIR if sc2 else KATA_BASELINE_SOURCE_DIR, + "tools/packaging", + script, + ) + for script in script_files + ], + [join(pause_image_build_dir, script) for script in script_files], + ): + copy_from_kata_workon_ctr( + ctr_path, host_path, sudo=False, debug=debug, hot_replace=hot_replace + ) + + # Build pause image + work_env = environ.update( + { + "pause_image_repo": PAUSE_IMAGE_REPO, + "pause_image_version": PAUSE_IMAGE_VERSION, + } + ) + out = run( + "./build.sh", + shell=True, + cwd=join(pause_image_build_dir, "static-build", "pause-image"), + env=work_env, + capture_output=True, + ) + assert out.returncode == 0, "Error building pause image: {}".format( + out.stderr.decode("utf-8") + ) + + # Generate tarball of pause bundle + pause_bundle_tarball_name = "pause_bundle_sc2.tar.xz" + tar_cmd = f"tar -cJf {pause_bundle_tarball_name} pause_bundle" + run( + tar_cmd, + shell=True, + check=True, + cwd=join(pause_image_build_dir, "static-build", "pause-image"), + ) + + return join( + pause_image_build_dir, "static-build", "pause-image", pause_bundle_tarball_name + ) + + def replace_agent( dst_initrd_path=join(KATA_IMG_DIR, "kata-containers-initrd-confidential-sc2.img"), debug=False, @@ -185,15 +250,25 @@ def replace_agent( out.stderr.decode("utf-8") ) + # WARNING: for the time being we are building the same rootfs for + # confidential and non-confidential rootfs_builder_dir = join(tmp_rootfs_scripts_dir, "rootfs-builder") work_env = { "AGENT_INIT": "yes", "AGENT_POLICY_FILE": join(tmp_rootfs_base_dir, "allow-all.rego"), "AGENT_SOURCE_BIN": join(tmp_rootfs_base_dir, "kata-agent"), + "CONFIDENTIAL_GUEST": "yes", "DMVERITY_SUPPORT": "yes", + "MEASURED_ROOTFS": "yes", + # WARNING: even though only confidential rootfs-es need the pause + # image bundle, we also need it to run `qemu-coco-dev` + "PAUSE_IMAGE_TARBALL": build_pause_image( + sc2=sc2, debug=debug, hot_replace=hot_replace + ), + "PULL_TYPE": "default", "ROOTFS_DIR": tmp_rootfs_dir, } - rootfs_builder_cmd = f"sudo -E {rootfs_builder_dir}/rootfs.sh ubuntu > /dev/null" + rootfs_builder_cmd = f"sudo -E {rootfs_builder_dir}/rootfs.sh ubuntu" out = run( rootfs_builder_cmd, shell=True, @@ -256,6 +331,8 @@ def replace_agent( assert out.returncode == 0, "Error packing initrd: {}".format( out.stderr.decode("utf-8") ) + if debug: + print(out.stdout.decode("utf-8").strip()) # Lastly, update the Kata config to point to the new initrd target_runtimes = SC2_RUNTIMES if sc2 else KATA_RUNTIMES diff --git a/tasks/util/versions.py b/tasks/util/versions.py index 0b881082..65506f0b 100644 --- a/tasks/util/versions.py +++ b/tasks/util/versions.py @@ -13,6 +13,7 @@ CNI_VERSION = "1.3.0" CRICTL_VERSION = "1.28.0" K9S_VERSION = "0.32.5" +PAUSE_IMAGE_VERSION = "3.9" # Container image managament versions REGISTRY_VERSION = "2.8" @@ -25,3 +26,6 @@ # Knative versions KNATIVE_VERSION = "1.15.0" + +# Kernel versions +GUEST_KERNEL_VERSION = "6.7" From ac05f4fff19a98c19aadbe730c5afbfbcd171f1f Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 23 Jan 2025 18:15:53 +0000 Subject: [PATCH 3/7] nits: self-review --- tasks/kernel.py | 4 +--- tasks/util/kata.py | 4 ---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/tasks/kernel.py b/tasks/kernel.py index 721d2a2a..bf3b55e1 100644 --- a/tasks/kernel.py +++ b/tasks/kernel.py @@ -26,7 +26,6 @@ def build_guest(debug=False, hot_replace=False): makedirs(join(kernel_build_dir, "kernel")) makedirs(join(kernel_build_dir, "scripts")) - # Prefix: tools/packaging script_files = [ "kernel/build-kernel.sh", "kernel/configs/", @@ -56,7 +55,7 @@ def build_guest(debug=False, hot_replace=False): ) build_kernel_base_cmd = [ - "./build-kernel.sh -x -f" f" -v {GUEST_KERNEL_VERSION}", + f"./build-kernel.sh -x -f -v {GUEST_KERNEL_VERSION}", "-u 'https://cdn.kernel.org/pub/linux/kernel/v{}.x/'".format( GUEST_KERNEL_VERSION.split(".")[0] ), @@ -77,7 +76,6 @@ def build_guest(debug=False, hot_replace=False): print(out.stdout.decode("utf-8")) # Copy the built kernel into the desired path - # "./build-kernel.sh -x -f -v 6.7 setup" with open(join(kernel_build_dir, "kernel", "kata_config_version"), "r") as fh: kata_config_version = fh.read() kata_config_version = kata_config_version.strip() diff --git a/tasks/util/kata.py b/tasks/util/kata.py index b8f5d23d..73f7447d 100644 --- a/tasks/util/kata.py +++ b/tasks/util/kata.py @@ -250,8 +250,6 @@ def replace_agent( out.stderr.decode("utf-8") ) - # WARNING: for the time being we are building the same rootfs for - # confidential and non-confidential rootfs_builder_dir = join(tmp_rootfs_scripts_dir, "rootfs-builder") work_env = { "AGENT_INIT": "yes", @@ -260,8 +258,6 @@ def replace_agent( "CONFIDENTIAL_GUEST": "yes", "DMVERITY_SUPPORT": "yes", "MEASURED_ROOTFS": "yes", - # WARNING: even though only confidential rootfs-es need the pause - # image bundle, we also need it to run `qemu-coco-dev` "PAUSE_IMAGE_TARBALL": build_pause_image( sc2=sc2, debug=debug, hot_replace=hot_replace ), From 721028db92af20892301d3e3c2afeb48cfdd991d Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 23 Jan 2025 18:26:27 +0000 Subject: [PATCH 4/7] sc2: install guest kernel after sc2 --- tasks/sc2.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tasks/sc2.py b/tasks/sc2.py index f9a8f4ae..53076b18 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -264,11 +264,6 @@ def deploy(ctx, debug=False, clean=False): # Install the nydusify tool nydus_install() - # Build and install the guest VM kernel - print_dotted_line(f"Build and install guest VM kernel (v{GUEST_KERNEL_VERSION})") - build_guest_kernel() - print("Success!") - # Start a local docker registry (must happen before knative installation, # as we rely on it to host our sidecar image) start_local_registry(debug=debug, clean=clean) @@ -297,6 +292,12 @@ def deploy(ctx, debug=False, clean=False): install_sc2_runtime(debug=debug) print("Success!") + # Build and install the guest VM kernel (must be after installing SC2, so + # that we can patch all config files) + print_dotted_line(f"Build and install guest VM kernel (v{GUEST_KERNEL_VERSION})") + build_guest_kernel() + print("Success!") + # Once we are done with installing components, restart containerd restart_containerd(debug=debug) From d9ea434c44a4f25f4657f49845a39d0abf1f6768 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 24 Jan 2025 09:19:53 +0000 Subject: [PATCH 5/7] sc2: start vm_cache after we are done making all modifications --- tasks/sc2.py | 57 +++++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/tasks/sc2.py b/tasks/sc2.py index 53076b18..21611edc 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -1,5 +1,5 @@ from invoke import task -from os import makedirs +from os import environ, makedirs from os.path import exists, join from subprocess import run from tasks.containerd import install as containerd_install @@ -49,6 +49,31 @@ from time import sleep +def start_vm_cache(debug=False): + vm_cache_dir = join(PROJ_ROOT, "vm-cache") + + # Build the VM cache server + if debug: + print("Building VM cache wrapper...") + result = run( + "cargo build --release", cwd=vm_cache_dir, shell=True, capture_output=True + ) + assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) + if debug: + print(result.stdout.decode("utf-8").strip()) + + # Run the VM cache server in the background + if debug: + print("Running VM cache wrapper in background mode...") + run( + "sudo -E target/release/vm-cache background > /dev/null 2>&1", + cwd=vm_cache_dir, + env=environ, + shell=True, + check=True, + ) + + def install_sc2_runtime(debug=False): """ This script installs SC2 as a different runtime class @@ -164,30 +189,6 @@ def install_sc2_runtime(debug=False): sc2=True, ) - # ---------- VM Cache --------- - - vm_cache_dir = join(PROJ_ROOT, "vm-cache") - - # Build the VM cache server - if debug: - print("Building VM cache wrapper...") - result = run( - "cargo build --release", cwd=vm_cache_dir, shell=True, capture_output=True - ) - assert result.returncode == 0, print(result.stderr.decode("utf-8").strip()) - if debug: - print(result.stdout.decode("utf-8").strip()) - - # Run the VM cache server in the background - if debug: - print("Running VM cache wrapper in background mode...") - run( - "sudo -E target/release/vm-cache background > /dev/null 2>&1", - cwd=vm_cache_dir, - shell=True, - check=True, - ) - @task(default=True) def deploy(ctx, debug=False, clean=False): @@ -301,6 +302,12 @@ def deploy(ctx, debug=False, clean=False): # Once we are done with installing components, restart containerd restart_containerd(debug=debug) + # Start the VM cache at the end so that we can pick up the latest config + # changes + print_dotted_line(f"Starting cVM cache...") + start_vm_cache(debug=debug) + print_dotted_line(f"Success!") + # Push demo apps to local registry for easy testing push_demo_apps_to_local_registry(debug=debug) From 5e7d0b09b72309f170a87b615eee63918413a17c Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 24 Jan 2025 09:46:29 +0000 Subject: [PATCH 6/7] kernel: add apt deps --- tasks/kernel.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tasks/kernel.py b/tasks/kernel.py index bf3b55e1..22cf3161 100644 --- a/tasks/kernel.py +++ b/tasks/kernel.py @@ -62,6 +62,12 @@ def build_guest(debug=False, hot_replace=False): ] build_kernel_base_cmd = " ".join(build_kernel_base_cmd) + # Install APT deps needed to build guest kernel + out = run("sudo apt install -y bison flex libelf-dev libssl-dev make", shell=True, capture_output=True) + assert out.returncode == 0, "Error installing deps: {}".format( + out.stderr.decode("utf-8") + ) + for step in ["setup", "build"]: out = run( f"{build_kernel_base_cmd} {step}", From df163597ae724b23efb9b148edad3cd935d3ac52 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 24 Jan 2025 09:47:20 +0000 Subject: [PATCH 7/7] nits: run python formatting --- tasks/kernel.py | 6 +++++- tasks/sc2.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tasks/kernel.py b/tasks/kernel.py index 22cf3161..d8af7980 100644 --- a/tasks/kernel.py +++ b/tasks/kernel.py @@ -63,7 +63,11 @@ def build_guest(debug=False, hot_replace=False): build_kernel_base_cmd = " ".join(build_kernel_base_cmd) # Install APT deps needed to build guest kernel - out = run("sudo apt install -y bison flex libelf-dev libssl-dev make", shell=True, capture_output=True) + out = run( + "sudo apt install -y bison flex libelf-dev libssl-dev make", + shell=True, + capture_output=True, + ) assert out.returncode == 0, "Error installing deps: {}".format( out.stderr.decode("utf-8") ) diff --git a/tasks/sc2.py b/tasks/sc2.py index 21611edc..7c620d31 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -304,9 +304,9 @@ def deploy(ctx, debug=False, clean=False): # Start the VM cache at the end so that we can pick up the latest config # changes - print_dotted_line(f"Starting cVM cache...") + print_dotted_line("Starting cVM cache...") start_vm_cache(debug=debug) - print_dotted_line(f"Success!") + print("Success!") # Push demo apps to local registry for easy testing push_demo_apps_to_local_registry(debug=debug)