From fc4e1ddef1a29aeea88142901cd110b1fd84719e Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 6 Mar 2025 15:51:25 +0000 Subject: [PATCH 1/5] kata: bump to 3.14.0 + include gc in workon --- docker/kata.dockerfile | 50 +++++++++++++++++++++++++++++++++++++----- tasks/kata.py | 7 ++++-- tasks/util/gc.py | 1 + tasks/util/kata.py | 4 +++- tasks/util/versions.py | 4 ++-- 5 files changed, 55 insertions(+), 11 deletions(-) create mode 100644 tasks/util/gc.py diff --git a/docker/kata.dockerfile b/docker/kata.dockerfile index f12932bc..cec22710 100644 --- a/docker/kata.dockerfile +++ b/docker/kata.dockerfile @@ -7,9 +7,12 @@ FROM ghcr.io/sc2-sys/base:0.12.0 # Install APT dependencies RUN apt install -y \ libseccomp-dev \ - musl-tools + musl-tools \ + pkg-config \ + protobuf-compiler \ + tss2 -# --------------------------- +# ------------------------------------------------------------------------------ # Build Kata # # We need a few patches to get Knative and our baseline experiments to work @@ -18,12 +21,27 @@ RUN apt install -y \ # SC2 (branch sc2-main). This introduces a bit of duplication but, at the same # time, allows us to have clearly differentiated targets to, e.g., build # different initrds. -# --------------------------- +# ------------------------------------------------------------------------------ -# Kata does not build with Rust > 1.78, so we pin to an older version -# https://github.com/kata-containers/kata-containers/pull/10320 +# We must pin Kata to the rust version in kata-containers/versions.yml ARG RUST_VERSION +# To build the Kata Agent from source we need to statically link libseccomp +# https://github.com/kata-containers/kata-containers/issues/5044 +ARG LIBSECCOMP_VERSION="2.5.5" +ARG LIBSECCOMP_URL="https://github.com/seccomp/libseccomp" +ARG LIBSECCOMP_PATH="/usr/local/kata-libseccomp" +ARG GPERF_VERSION="3.1" +ARG GPERF_URL="http://ftp.gnu.org/pub/gnu/gperf/" +ARG GPERF_PATH="/usr/local/kata-gperf" + +# After building libseccomp, we need to set these two env. variables to make +# sure successive builds succeed +ARG LIBSECCOMP_LINK_TYPE=static +ARG LIBSECCOMP_LIB_PATH=${LIBSECCOMP_PATH}/lib +ENV LIBSECCOMP_LINK_TYPE=${LIBSECCOMP_LINK_TYPE} +ENV LIBSECCOMP_LIB_PATH=${LIBSECCOMP_LIB_PATH} + # Fetch code and build the runtime and the agent for our baselines ARG CODE_DIR=/go/src/github.com/kata-containers/kata-containers-baseline RUN mkdir -p ${CODE_DIR} \ @@ -34,6 +52,8 @@ RUN mkdir -p ${CODE_DIR} \ && git config --global --add safe.directory ${CODE_DIR} \ && cd ${CODE_DIR}/src/runtime \ && make \ + && cd ${CODE_DIR} \ + && ./ci/install_libseccomp.sh ${LIBSECCOMP_PATH} ${GPERF_PATH} \ && cd ${CODE_DIR}/src/agent \ && rustup default ${RUST_VERSION} \ && rustup component add rust-analyzer \ @@ -51,9 +71,27 @@ RUN mkdir -p ${CODE_DIR} \ && cd ${CODE_DIR}/src/runtime \ && make \ && cd ${CODE_DIR}/src/agent \ - && rustup default ${RUST_VERSION} \ && rustup component add rust-analyzer \ && rustup target add x86_64-unknown-linux-musl \ && make +# ------------------------------------------------------------------------------ +# Build Guest Components +# +# The agent is very tightly-coupled with guest-components, so it makes sense +# to modify both in the same work-on container +# ------------------------------------------------------------------------------ + +ARG CODE_DIR_GC=/git/sc2-sys/guest-components +ARG RUST_VERSION_GC=1.81 +RUN mkdir -p ${CODE_DIR_GC} \ + && git clone\ + -b sc2-main \ + https://github.com/sc2-sys/guest-components \ + ${CODE_DIR_GC} \ + && git config --global --add safe.directory ${CODE_DIR_GC} \ + && cd ${CODE_DIR_GC}/image-rs \ + && rustup override set ${RUST_VERSION_GC} \ + && cargo build --release --features "nydus" + WORKDIR ${CODE_DIR} diff --git a/tasks/kata.py b/tasks/kata.py index 244f0d53..639ce455 100644 --- a/tasks/kata.py +++ b/tasks/kata.py @@ -54,14 +54,17 @@ def build(ctx, nocache=False, push=False): @task -def cli(ctx, mount_path=join(PROJ_ROOT, "..", "kata-containers")): +def cli(ctx, mount_path=join(PROJ_ROOT, "..", "kata-containers"), gc_mount_path=join(PROJ_ROOT, "..", "guest-components")): """ Get a working environemnt to develop Kata """ if mount_path is not None: mount_path = abspath(mount_path) - run_kata_workon_ctr(mount_path=mount_path) + if gc_mount_path is not None: + gc_mount_path = abspath(gc_mount_path) + + run_kata_workon_ctr(mount_path=mount_path, gc_mount_path=gc_mount_path) run("docker exec -it {} bash".format(KATA_WORKON_CTR_NAME), shell=True, check=True) diff --git a/tasks/util/gc.py b/tasks/util/gc.py new file mode 100644 index 00000000..6a51ad4d --- /dev/null +++ b/tasks/util/gc.py @@ -0,0 +1 @@ +GC_SOURCE_DIR = "/git/sc2-sys/guest-components" diff --git a/tasks/util/kata.py b/tasks/util/kata.py index 98b3a709..09328470 100644 --- a/tasks/util/kata.py +++ b/tasks/util/kata.py @@ -15,6 +15,7 @@ PROJ_ROOT, SC2_RUNTIMES, ) +from tasks.util.gc import GC_SOURCE_DIR from tasks.util.registry import HOST_CERT_PATH from tasks.util.versions import KATA_VERSION, PAUSE_IMAGE_VERSION, RUST_VERSION from tasks.util.toml import remove_entry_from_toml, update_toml @@ -104,7 +105,7 @@ def build_pause_image(sc2, debug, hot_replace): ) -def run_kata_workon_ctr(mount_path=None): +def run_kata_workon_ctr(mount_path=None, gc_mount_path=None): """ Start Kata workon container image if it is not running. Return `True` if we actually did start the container @@ -116,6 +117,7 @@ def run_kata_workon_ctr(mount_path=None): "docker run", "-d -t", (f"-v {mount_path}:{KATA_SOURCE_DIR}" if mount_path else ""), + (f"-v {gc_mount_path}:{GC_SOURCE_DIR}" if gc_mount_path else ""), "--name {}".format(KATA_WORKON_CTR_NAME), KATA_IMAGE_TAG, "bash", diff --git a/tasks/util/versions.py b/tasks/util/versions.py index acda86e4..ee7278c2 100644 --- a/tasks/util/versions.py +++ b/tasks/util/versions.py @@ -1,10 +1,10 @@ # CoCo versions (note that the CoCo release pins the Kata Version) COCO_VERSION = "0.12.0" -KATA_VERSION = "3.13.0" +KATA_VERSION = "3.14.0" # Base software versions GO_VERSION = "1.23.0" -RUST_VERSION = "1.78.0" +RUST_VERSION = "1.80.0" # Kubernetes versions CONTAINERD_VERSION = "1.7.19" From 9aecd3cd5aef9b20a2938bd248979ea1dd5b1c37 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 13 Mar 2025 10:24:36 +0000 Subject: [PATCH 2/5] bump to match fix in kata --- tasks/sc2.py | 18 ------------------ tasks/util/kata.py | 4 ++-- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/tasks/sc2.py b/tasks/sc2.py index 6fd32b89..5baf7ca6 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -313,24 +313,6 @@ def deploy(ctx, debug=False, clean=False): ovmf_install() print("Success!") - # Update SNP class to use default QEMU (we use host kernel 6.11, so we - # can use upstream QEMU 9.1). We do this update before generating the SC2 - # runtime classes, so they will inherit the QEMU value - # TODO: remove when bumping to a new CoCo release - qemu_path = join(KATA_ROOT, "bin", "qemu-system-x86_64") - updated_toml_str = """ - [hypervisor.qemu] - path = "{qemu_path}" - valid_hypervisor_paths = [ "{qemu_path}" ] - """.format( - qemu_path=qemu_path - ) - update_toml( - join(KATA_CONFIG_DIR, "configuration-qemu-snp.toml"), - updated_toml_str, - requires_root=True, - ) - # Apply general patches to the Kata runtime replace_kata_shim( dst_shim_binary=join(KATA_ROOT, "bin", "containerd-shim-kata-v2"), diff --git a/tasks/util/kata.py b/tasks/util/kata.py index 09328470..8d14af36 100644 --- a/tasks/util/kata.py +++ b/tasks/util/kata.py @@ -146,7 +146,7 @@ def copy_from_kata_workon_ctr( ): if hot_replace and not is_ctr_running(KATA_WORKON_CTR_NAME): print("Must have the work-on container running to hot replace!") - print("Consider running: inv containerd.cli ") + print("Consider running: inv kata.cli ") raise RuntimeError("Hot-replace without work-on running!") if hot_replace: @@ -277,7 +277,7 @@ def prepare_rootfs(tmp_rootfs_base_dir, debug=False, sc2=False, hot_replace=Fals # host OS versions from introducing subtle changes in the rootfs "USE_DOCKER": "yes", "OS_VERSION": "jammy", - "RUST_VERSION": "1.75.0", + "RUST_VERSION": RUST_VERSION, "GO_VERSION": "1.22.2", "PAUSE_IMAGE_TARBALL": build_pause_image( sc2=sc2, debug=debug, hot_replace=hot_replace From 4ec7ac40fbac00528b9bfd03fe560ac4b9aa58db Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 13 Mar 2025 10:40:06 +0000 Subject: [PATCH 3/5] nit: run formatting --- tasks/kata.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tasks/kata.py b/tasks/kata.py index 639ce455..a8dcb305 100644 --- a/tasks/kata.py +++ b/tasks/kata.py @@ -54,7 +54,11 @@ def build(ctx, nocache=False, push=False): @task -def cli(ctx, mount_path=join(PROJ_ROOT, "..", "kata-containers"), gc_mount_path=join(PROJ_ROOT, "..", "guest-components")): +def cli( + ctx, + mount_path=join(PROJ_ROOT, "..", "kata-containers"), + gc_mount_path=join(PROJ_ROOT, "..", "guest-components"), +): """ Get a working environemnt to develop Kata """ From b365998eadebe0d1a66d929c48557af25194f3aa Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 13 Mar 2025 12:12:06 +0000 Subject: [PATCH 4/5] deploy: undo the patches. we only updated kata not coco --- tasks/sc2.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tasks/sc2.py b/tasks/sc2.py index 5baf7ca6..6fd32b89 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -313,6 +313,24 @@ def deploy(ctx, debug=False, clean=False): ovmf_install() print("Success!") + # Update SNP class to use default QEMU (we use host kernel 6.11, so we + # can use upstream QEMU 9.1). We do this update before generating the SC2 + # runtime classes, so they will inherit the QEMU value + # TODO: remove when bumping to a new CoCo release + qemu_path = join(KATA_ROOT, "bin", "qemu-system-x86_64") + updated_toml_str = """ + [hypervisor.qemu] + path = "{qemu_path}" + valid_hypervisor_paths = [ "{qemu_path}" ] + """.format( + qemu_path=qemu_path + ) + update_toml( + join(KATA_CONFIG_DIR, "configuration-qemu-snp.toml"), + updated_toml_str, + requires_root=True, + ) + # Apply general patches to the Kata runtime replace_kata_shim( dst_shim_binary=join(KATA_ROOT, "bin", "containerd-shim-kata-v2"), From 38a477a2656ab8103f4026f959be1b3d02ed28ee Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 19 Mar 2025 12:23:37 +0000 Subject: [PATCH 5/5] kata: install systemd files together with our custom agent --- docker/kata.dockerfile | 3 ++- tasks/util/kata.py | 43 +++++++++++++++++++++++++++++++++++++----- tasks/util/versions.py | 2 +- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/docker/kata.dockerfile b/docker/kata.dockerfile index cec22710..b9587a5d 100644 --- a/docker/kata.dockerfile +++ b/docker/kata.dockerfile @@ -73,7 +73,8 @@ RUN mkdir -p ${CODE_DIR} \ && cd ${CODE_DIR}/src/agent \ && rustup component add rust-analyzer \ && rustup target add x86_64-unknown-linux-musl \ - && make + && make \ + && make install-services # ------------------------------------------------------------------------------ # Build Guest Components diff --git a/tasks/util/kata.py b/tasks/util/kata.py index 8d14af36..1ef5f9c8 100644 --- a/tasks/util/kata.py +++ b/tasks/util/kata.py @@ -30,6 +30,8 @@ KATA_BASELINE_AGENT_SOURCE_DIR = join(KATA_BASELINE_SOURCE_DIR, "src", "agent") KATA_BASELINE_SHIM_SOURCE_DIR = join(KATA_BASELINE_SOURCE_DIR, "src", "runtime") +KATA_AGENT_INIT = "no" + def build_kata_image(nocache, push, debug=True): build_image( @@ -238,7 +240,17 @@ def prepare_rootfs(tmp_rootfs_base_dir, debug=False, sc2=False, hot_replace=Fals hot_replace=hot_replace, ) - # Finally, also copy our kata agent + # ----- Prepare kata agent tarball ----- + + tmp_rootfs_agent_tarball_dir = join(tmp_rootfs_base_dir, "agent-tarball") + makedirs(tmp_rootfs_agent_tarball_dir) + makedirs(join(tmp_rootfs_agent_tarball_dir, "usr")) + makedirs(join(tmp_rootfs_agent_tarball_dir, "usr", "bin")) + makedirs(join(tmp_rootfs_agent_tarball_dir, "usr", "lib")) + makedirs(join(tmp_rootfs_agent_tarball_dir, "usr", "lib", "systemd")) + makedirs(join(tmp_rootfs_agent_tarball_dir, "usr", "lib", "systemd", "system")) + + # Copy our kata agent agent_host_path = join( KATA_AGENT_SOURCE_DIR if sc2 else KATA_BASELINE_AGENT_SOURCE_DIR, "target", @@ -248,12 +260,33 @@ def prepare_rootfs(tmp_rootfs_base_dir, debug=False, sc2=False, hot_replace=Fals ) copy_from_kata_workon_ctr( agent_host_path, - join(tmp_rootfs_base_dir, "kata-agent"), + join(tmp_rootfs_agent_tarball_dir, "usr", "bin", "kata-agent"), sudo=True, debug=debug, hot_replace=hot_replace, ) + # Copy Kata's systemd files + unit_src_dir = "/usr/lib/systemd/system" + unit_files = ["kata-agent.service", "kata-containers.target"] + for unit_file in unit_files: + copy_from_kata_workon_ctr( + join(unit_src_dir, unit_file), + join(tmp_rootfs_agent_tarball_dir, unit_src_dir[1:], unit_file), + sudo=True, + debug=debug, + hot_replace=hot_replace, + ) + + agent_tarball = join(tmp_rootfs_base_dir, "kata_agent.tar.xz") + result = run( + f"tar cvJf {agent_tarball} ./usr", + shell=True, + capture_output=True, + cwd=tmp_rootfs_agent_tarball_dir, + ) + assert result.returncode == 0, result.stderr.decode("utf-8") + # ----- Populate rootfs with base ubuntu using Kata's scripts ----- out = run( @@ -267,9 +300,9 @@ def prepare_rootfs(tmp_rootfs_base_dir, debug=False, sc2=False, hot_replace=Fals rootfs_builder_dir = join(tmp_rootfs_scripts_dir, "rootfs-builder") work_env = { - "AGENT_INIT": "yes", + "AGENT_INIT": KATA_AGENT_INIT, "AGENT_POLICY_FILE": join(tmp_rootfs_base_dir, "allow-all.rego"), - "AGENT_SOURCE_BIN": join(tmp_rootfs_base_dir, "kata-agent"), + "AGENT_TARBALL": agent_tarball, "CONFIDENTIAL_GUEST": "yes", "DMVERITY_SUPPORT": "yes", "MEASURED_ROOTFS": "no", @@ -358,7 +391,7 @@ def replace_agent( # ----- Pack rootfs into initrd using Kata's script ----- - work_env = {"AGENT_INIT": "yes"} + work_env = {"AGENT_INIT": KATA_AGENT_INIT} initrd_pack_cmd = "sudo -E {} -o {} {}".format( join(tmp_rootfs_scripts_dir, "initrd-builder", "initrd_builder.sh"), dst_initrd_path, diff --git a/tasks/util/versions.py b/tasks/util/versions.py index ee7278c2..6c479992 100644 --- a/tasks/util/versions.py +++ b/tasks/util/versions.py @@ -32,7 +32,7 @@ # table in ./docs/host_kernel.md HOST_KERNEL_VERSION_SNP = "6.11.0-snp-host-cc2568386" HOST_KERNEL_VERSION_TDX = "6.8.0-1013-intel" -GUEST_KERNEL_VERSION = "6.12.8" +GUEST_KERNEL_VERSION = "6.12.13" # Coconut SVSM versions IGVM_VERSION = "0.3.4"