From e72c814fbac0deee33eccd90cbccc815208fb36b Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 10 Dec 2024 14:49:58 +0000 Subject: [PATCH 01/10] sc2(nydus): enable lazy pulling in the guest --- tasks/demo_apps.py | 67 +++++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/tasks/demo_apps.py b/tasks/demo_apps.py index 8136fbdd..7888d2dc 100644 --- a/tasks/demo_apps.py +++ b/tasks/demo_apps.py @@ -8,6 +8,7 @@ LOCAL_REGISTRY_URL, print_dotted_line, ) +from tasks.util.nydus import nydusify APP_LIST = { "helloworld-py": join(APPS_SOURCE_DIR, "helloworld-py"), @@ -15,15 +16,23 @@ } -def get_docker_tag_for_app(app_name): +def get_docker_tag_for_app(app_name, nydus=False): docker_tag = join(GHCR_URL, GITHUB_ORG, "applications", app_name) docker_tag += ":unencrypted" + + if nydus: + docker_tag += "-nydus" + return docker_tag -def get_local_registry_tag_for_app(app_name): +def get_local_registry_tag_for_app(app_name, nydus=False): docker_tag = join(LOCAL_REGISTRY_URL, "applications", app_name) docker_tag += ":unencrypted" + + if nydus: + docker_tag += "-nydus" + return docker_tag @@ -58,34 +67,44 @@ def build(ctx, app=None, nocache=False): docker_cmd = "docker push {}".format(get_docker_tag_for_app(app_name)) run(docker_cmd, shell=True, check=True) + # Now, convert it to a nydus image, and push again + nydusify(get_docker_tag_for_app(app_name), get_docker_tag_for_app(app_name, nydus=True)) + def do_push_to_local_registry(debug=False): print_dotted_line("Pushing {} demo apps to local regsitry".format(len(APP_LIST))) for app_name in APP_LIST: - docker_tag = get_docker_tag_for_app(app_name) - local_registry_tag = get_local_registry_tag_for_app(app_name) - - result = run(f"docker pull {docker_tag}", shell=True, capture_output=True) - assert result.returncode == 0, result.stderr.decode("utf-8").strip() - if debug: - print(result.stdout.decode("utf-8").strip()) - - result = run( - f"docker tag {docker_tag} {local_registry_tag}", - shell=True, - capture_output=True, - ) - assert result.returncode == 0, result.stderr.decode("utf-8").strip() - if debug: - print(result.stdout.decode("utf-8").strip()) + docker_tags = [ + get_docker_tag_for_app(app_name), + get_docker_tag_for_app(app_name, nydus=True), + ] + local_registry_tags = [ + get_local_registry_tag_for_app(app_name), + get_local_registry_tag_for_app(app_name, nydus=True), + ] - result = run( - f"docker push {local_registry_tag}", shell=True, capture_output=True - ) - assert result.returncode == 0, result.stderr.decode("utf-8").strip() - if debug: - print(result.stdout.decode("utf-8").strip()) + for docker_tag, local_registry_tag in zip(docker_tags, local_registry_tags): + result = run(f"docker pull {docker_tag}", shell=True, capture_output=True) + assert result.returncode == 0, result.stderr.decode("utf-8").strip() + if debug: + print(result.stdout.decode("utf-8").strip()) + + result = run( + f"docker tag {docker_tag} {local_registry_tag}", + shell=True, + capture_output=True, + ) + assert result.returncode == 0, result.stderr.decode("utf-8").strip() + if debug: + print(result.stdout.decode("utf-8").strip()) + + result = run( + f"docker push {local_registry_tag}", shell=True, capture_output=True + ) + assert result.returncode == 0, result.stderr.decode("utf-8").strip() + if debug: + print(result.stdout.decode("utf-8").strip()) print("Success!") From bcffff83038dedb696add83594d42f29859dec06 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 10 Dec 2024 16:38:35 +0000 Subject: [PATCH 02/10] feat: basic lazy guest pull working --- demo-apps/helloworld-py-nydus/Dockerfile | 16 +++++ demo-apps/helloworld-py-nydus/app.py | 14 ++++ demo-apps/helloworld-py-nydus/deployment.yaml | 37 ++++++++++ tasks/demo_apps.py | 72 +++++++++++-------- tasks/nydus.py | 7 ++ tools/check-fork-hashes/src/main.rs | 7 ++ 6 files changed, 122 insertions(+), 31 deletions(-) create mode 100644 demo-apps/helloworld-py-nydus/Dockerfile create mode 100644 demo-apps/helloworld-py-nydus/app.py create mode 100644 demo-apps/helloworld-py-nydus/deployment.yaml diff --git a/demo-apps/helloworld-py-nydus/Dockerfile b/demo-apps/helloworld-py-nydus/Dockerfile new file mode 100644 index 00000000..5b5538e5 --- /dev/null +++ b/demo-apps/helloworld-py-nydus/Dockerfile @@ -0,0 +1,16 @@ +# Adapted from https://github.com/knative/docs/tree/main/code-samples/serving/hello-world/helloworld-python +FROM python:3.10-slim + +# Allow statements and log messages to immediately appear in the Knative logs +ENV PYTHONUNBUFFERED True + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . ./ + +# Install production dependencies. +RUN pip install Flask + +# Run the web service on container startup. +CMD ["python", "app.py"] diff --git a/demo-apps/helloworld-py-nydus/app.py b/demo-apps/helloworld-py-nydus/app.py new file mode 100644 index 00000000..792a036d --- /dev/null +++ b/demo-apps/helloworld-py-nydus/app.py @@ -0,0 +1,14 @@ +from flask import Flask +from os import environ + +app = Flask(__name__) + + +@app.route("/") +def hello_world(): + target = environ.get("TARGET", "World") + return "Hello {}!\n".format(target) + + +if __name__ == "__main__": + app.run(debug=True, host="0.0.0.0", port=int(environ.get("PORT", 8080))) diff --git a/demo-apps/helloworld-py-nydus/deployment.yaml b/demo-apps/helloworld-py-nydus/deployment.yaml new file mode 100644 index 00000000..12d18256 --- /dev/null +++ b/demo-apps/helloworld-py-nydus/deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: coco-helloworld-py-node-port +spec: + type: NodePort + selector: + apps.sc2.io/name: helloworld-py + ports: + - name: http + port: 8080 + targetPort: 8080 + protocol: TCP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coco-helloworld-py + labels: + apps.sc2.io/name: helloworld-py +spec: + replicas: 1 + selector: + matchLabels: + apps.sc2.io/name: helloworld-py + template: + metadata: + labels: + apps.sc2.io/name: helloworld-py + spec: + runtimeClassName: kata-${SC2_RUNTIME_CLASS} + containers: + - name: helloworld-py + image: sc2cr.io/applications/helloworld-py:unencrypted-nydus + imagePullPolicy: Always + ports: + - containerPort: 8080 diff --git a/tasks/demo_apps.py b/tasks/demo_apps.py index 7888d2dc..ecccf2a4 100644 --- a/tasks/demo_apps.py +++ b/tasks/demo_apps.py @@ -8,7 +8,7 @@ LOCAL_REGISTRY_URL, print_dotted_line, ) -from tasks.util.nydus import nydusify +from tasks.util.nydus import NYDUSIFY_PATH, nydusify APP_LIST = { "helloworld-py": join(APPS_SOURCE_DIR, "helloworld-py"), @@ -68,43 +68,53 @@ def build(ctx, app=None, nocache=False): run(docker_cmd, shell=True, check=True) # Now, convert it to a nydus image, and push again - nydusify(get_docker_tag_for_app(app_name), get_docker_tag_for_app(app_name, nydus=True)) + nydusify( + get_docker_tag_for_app(app_name), + get_docker_tag_for_app(app_name, nydus=True), + ) def do_push_to_local_registry(debug=False): print_dotted_line("Pushing {} demo apps to local regsitry".format(len(APP_LIST))) for app_name in APP_LIST: - docker_tags = [ - get_docker_tag_for_app(app_name), - get_docker_tag_for_app(app_name, nydus=True), - ] - local_registry_tags = [ - get_local_registry_tag_for_app(app_name), - get_local_registry_tag_for_app(app_name, nydus=True), - ] + docker_tag = get_docker_tag_for_app(app_name) + local_registry_tag = get_local_registry_tag_for_app(app_name) - for docker_tag, local_registry_tag in zip(docker_tags, local_registry_tags): - result = run(f"docker pull {docker_tag}", shell=True, capture_output=True) - assert result.returncode == 0, result.stderr.decode("utf-8").strip() - if debug: - print(result.stdout.decode("utf-8").strip()) - - result = run( - f"docker tag {docker_tag} {local_registry_tag}", - shell=True, - capture_output=True, - ) - assert result.returncode == 0, result.stderr.decode("utf-8").strip() - if debug: - print(result.stdout.decode("utf-8").strip()) - - result = run( - f"docker push {local_registry_tag}", shell=True, capture_output=True - ) - assert result.returncode == 0, result.stderr.decode("utf-8").strip() - if debug: - print(result.stdout.decode("utf-8").strip()) + if debug: + print(f"Pushing {docker_tag} to {local_registry_tag}...") + + result = run(f"docker pull {docker_tag}", shell=True, capture_output=True) + assert result.returncode == 0, result.stderr.decode("utf-8").strip() + if debug: + print(result.stdout.decode("utf-8").strip()) + + result = run( + f"docker tag {docker_tag} {local_registry_tag}", + shell=True, + capture_output=True, + ) + assert result.returncode == 0, result.stderr.decode("utf-8").strip() + if debug: + print(result.stdout.decode("utf-8").strip()) + + result = run( + f"docker push {local_registry_tag}", shell=True, capture_output=True + ) + assert result.returncode == 0, result.stderr.decode("utf-8").strip() + if debug: + print(result.stdout.decode("utf-8").strip()) + + # For nydus, we directly use `nydusify copy` as we cannot `docker pull` + # a nydus image + result = run("{} copy --source {} --target {} --target-insecure".format( + NYDUSIFY_PATH, + get_docker_tag_for_app(app_name, nydus=True), + get_local_registry_tag_for_app(app_name, nydus=True)), + shell=True, + capture_output=True + ) + assert result.returncode == 0, result.stderr.decode("utf-8").strip() print("Success!") diff --git a/tasks/nydus.py b/tasks/nydus.py index 53a21fd7..57eb5078 100644 --- a/tasks/nydus.py +++ b/tasks/nydus.py @@ -9,6 +9,13 @@ NYDUS_CTR_NAME = "nydus-workon" NYDUS_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "nydus") + f":{NYDUS_VERSION}" +# You can see all options to configure the nydus-snapshotter here: +# https://github.com/containerd/nydus-snapshotter/blob/main/misc/snapshotter/config.toml + + +def restart_nydus(): + run("sudo service nydus-snapshotter restart", shell=True, check=True) + @task def build(ctx, nocache=False, push=False): diff --git a/tools/check-fork-hashes/src/main.rs b/tools/check-fork-hashes/src/main.rs index cba6f144..9952d700 100644 --- a/tools/check-fork-hashes/src/main.rs +++ b/tools/check-fork-hashes/src/main.rs @@ -93,6 +93,13 @@ fn get_local_hash(container: &str, path: &str, branch: &str) -> Result>>>>>> 0e45580 (feat: basic lazy guest pull working) let forks = vec![ { let mut dict = HashMap::new(); From f09945f0a1cee2ac0644b7e1ee52245fafb2efe4 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 10 Dec 2024 17:33:14 +0000 Subject: [PATCH 03/10] gha: add test --- .github/workflows/tests.yml | 42 +++++++++++++++++++++++++++++++++++++ tasks/demo_apps.py | 12 ++++++----- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8ca1adb3..eec8c9ba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -128,6 +128,48 @@ jobs: env: POD_LABEL: apps.sc2.io/name=helloworld-py + - name: "Run nydus lazy guest-pulling test" + run: | + export SC2_RUNTIME_CLASS=qemu-${{ matrix.tee }}-sc2 + export POD_LABEL="apps.sc2.io/name=helloworld-py" + + # ----- Python Test ---- + + echo "Running python test..." + envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ./bin/kubectl apply -f - + + # Wait for pod to be ready + until [ "$(./bin/kubectl get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done + sleep 1 + + # Get the pod's IP + service_ip=$(./bin/kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') + [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] + envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ./bin/kubectl delete -f - + + # Wait for pod to be deleted + ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=30s + + # Extra cautionary sleep + sleep 5 + echo "Python test succesful!" + + # ----- Knative Test ---- + envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ./bin/kubectl apply -f - + sleep 1 + + # Get the service URL + service_url=$(./bin/kubectl get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) + [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] + + # Wait for pod to be deleted + envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ./bin/kubectl delete -f - + ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=60s + + # Extra cautionary sleep + sleep 5 + echo "Knative test succesful!" + - name: "Enable default-memory annotation" run: | for runtime_class in ${{ matrix.runtime_classes }}; do diff --git a/tasks/demo_apps.py b/tasks/demo_apps.py index ecccf2a4..236d3dc5 100644 --- a/tasks/demo_apps.py +++ b/tasks/demo_apps.py @@ -107,12 +107,14 @@ def do_push_to_local_registry(debug=False): # For nydus, we directly use `nydusify copy` as we cannot `docker pull` # a nydus image - result = run("{} copy --source {} --target {} --target-insecure".format( - NYDUSIFY_PATH, - get_docker_tag_for_app(app_name, nydus=True), - get_local_registry_tag_for_app(app_name, nydus=True)), + result = run( + "{} copy --source {} --target {} --target-insecure".format( + NYDUSIFY_PATH, + get_docker_tag_for_app(app_name, nydus=True), + get_local_registry_tag_for_app(app_name, nydus=True), + ), shell=True, - capture_output=True + capture_output=True, ) assert result.returncode == 0, result.stderr.decode("utf-8").strip() From 60fbba08fd5afbef4516847aa6a778671a5b3fcc Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 19 Dec 2024 14:08:45 +0000 Subject: [PATCH 04/10] more wip --- .../helloworld-knative-nydus/service.yaml | 21 +++++++++++++++++++ demo-apps/helloworld-py-nydus/Dockerfile | 16 -------------- demo-apps/helloworld-py-nydus/app.py | 14 ------------- 3 files changed, 21 insertions(+), 30 deletions(-) create mode 100644 demo-apps/helloworld-knative-nydus/service.yaml delete mode 100644 demo-apps/helloworld-py-nydus/Dockerfile delete mode 100644 demo-apps/helloworld-py-nydus/app.py diff --git a/demo-apps/helloworld-knative-nydus/service.yaml b/demo-apps/helloworld-knative-nydus/service.yaml new file mode 100644 index 00000000..81d75acd --- /dev/null +++ b/demo-apps/helloworld-knative-nydus/service.yaml @@ -0,0 +1,21 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: helloworld-knative +spec: + template: + metadata: + labels: + apps.sc2.io/name: helloworld-py + spec: + runtimeClassName: kata-${SC2_RUNTIME_CLASS} + # coco-knative: need to run user container as root + securityContext: + runAsUser: 1000 + containers: + - image: sc2cr.io/applications/helloworld-py:unencrypted-nydus + ports: + - containerPort: 8080 + env: + - name: TARGET + value: "World" diff --git a/demo-apps/helloworld-py-nydus/Dockerfile b/demo-apps/helloworld-py-nydus/Dockerfile deleted file mode 100644 index 5b5538e5..00000000 --- a/demo-apps/helloworld-py-nydus/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Adapted from https://github.com/knative/docs/tree/main/code-samples/serving/hello-world/helloworld-python -FROM python:3.10-slim - -# Allow statements and log messages to immediately appear in the Knative logs -ENV PYTHONUNBUFFERED True - -# Copy local code to the container image. -ENV APP_HOME /app -WORKDIR $APP_HOME -COPY . ./ - -# Install production dependencies. -RUN pip install Flask - -# Run the web service on container startup. -CMD ["python", "app.py"] diff --git a/demo-apps/helloworld-py-nydus/app.py b/demo-apps/helloworld-py-nydus/app.py deleted file mode 100644 index 792a036d..00000000 --- a/demo-apps/helloworld-py-nydus/app.py +++ /dev/null @@ -1,14 +0,0 @@ -from flask import Flask -from os import environ - -app = Flask(__name__) - - -@app.route("/") -def hello_world(): - target = environ.get("TARGET", "World") - return "Hello {}!\n".format(target) - - -if __name__ == "__main__": - app.run(debug=True, host="0.0.0.0", port=int(environ.get("PORT", 8080))) From 998751d3b4aeb76a443b4cba5186e6562fe28ccb Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 19 Dec 2024 14:32:57 +0000 Subject: [PATCH 05/10] nits: self-review --- tools/check-fork-hashes/src/main.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tools/check-fork-hashes/src/main.rs b/tools/check-fork-hashes/src/main.rs index 9952d700..cba6f144 100644 --- a/tools/check-fork-hashes/src/main.rs +++ b/tools/check-fork-hashes/src/main.rs @@ -93,13 +93,6 @@ fn get_local_hash(container: &str, path: &str, branch: &str) -> Result>>>>>> 0e45580 (feat: basic lazy guest pull working) let forks = vec![ { let mut dict = HashMap::new(); From 1dc747e64980e91284678a429b128472f53700bd Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 16 Jan 2025 18:57:29 +0000 Subject: [PATCH 06/10] cleanup diff after rebase --- tasks/nydus.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tasks/nydus.py b/tasks/nydus.py index 57eb5078..53a21fd7 100644 --- a/tasks/nydus.py +++ b/tasks/nydus.py @@ -9,13 +9,6 @@ NYDUS_CTR_NAME = "nydus-workon" NYDUS_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "nydus") + f":{NYDUS_VERSION}" -# You can see all options to configure the nydus-snapshotter here: -# https://github.com/containerd/nydus-snapshotter/blob/main/misc/snapshotter/config.toml - - -def restart_nydus(): - run("sudo service nydus-snapshotter restart", shell=True, check=True) - @task def build(ctx, nocache=False, push=False): From 907d9e5978ec5b3e35a78e44cf66335d563ca488 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 16 Jan 2025 19:33:01 +0000 Subject: [PATCH 07/10] demo-apps: add annotation to nydus app --- demo-apps/helloworld-knative-nydus/service.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/demo-apps/helloworld-knative-nydus/service.yaml b/demo-apps/helloworld-knative-nydus/service.yaml index 81d75acd..fa84bf5b 100644 --- a/demo-apps/helloworld-knative-nydus/service.yaml +++ b/demo-apps/helloworld-knative-nydus/service.yaml @@ -7,6 +7,15 @@ spec: metadata: labels: apps.sc2.io/name: helloworld-py + # WARNING: this annotation is crucial, as otherwise containerd will + # revert to pulling images with the default snapshotter, which in turn + # will mean that: + # 1. The image is fully pulled, as well, on the host. + # 2. The nydus-snapshotter prepares the snapshots on the host, too. + # These two things include an additional runtime of roughly 10 seconds + # for cold starts. + annotations: + io.containerd.cri.runtime-handler: kata-${SC2_RUNTIME_CLASS} spec: runtimeClassName: kata-${SC2_RUNTIME_CLASS} # coco-knative: need to run user container as root From 9c974a1dcda7771f844b9a8948563420dbd62418 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 17 Jan 2025 10:56:26 +0000 Subject: [PATCH 08/10] kata: add hot-replace feature --- tasks/containerd.py | 1 + tasks/kata.py | 5 +-- tasks/util/kata.py | 81 +++++++++++++++++++++++++++++---------------- 3 files changed, 57 insertions(+), 30 deletions(-) diff --git a/tasks/containerd.py b/tasks/containerd.py index 83c2e88b..1abf3eed 100644 --- a/tasks/containerd.py +++ b/tasks/containerd.py @@ -105,6 +105,7 @@ def hot_replace(ctx): if not is_ctr_running(CONTAINERD_CTR_NAME): print("Must have the work-on container running to hot replace!") print("Consider running: inv containerd.cli ") + return for binary in CONTAINERD_BINARY_NAMES: print( diff --git a/tasks/kata.py b/tasks/kata.py index 2972c55e..1128fb67 100644 --- a/tasks/kata.py +++ b/tasks/kata.py @@ -122,18 +122,19 @@ def enable_annotation(ctx, annotation, runtime="qemu-snp-sc2"): @task -def replace_agent(ctx, debug=False, runtime="qemu-snp-sc2"): +def hot_replace_agent(ctx, debug=False, runtime="qemu-snp-sc2"): replace_kata_agent( dst_initrd_path=join( KATA_IMG_DIR, "kata-containers-initrd-confidential-sc2.img" ), debug=debug, sc2=runtime in SC2_RUNTIMES, + hot_replace=True, ) @task -def replace_shim(ctx, runtime="qemu-snp-sc2"): +def hot_replace_shim(ctx, runtime="qemu-snp-sc2"): replace_kata_shim( dst_shim_binary=join( KATA_ROOT, diff --git a/tasks/util/kata.py b/tasks/util/kata.py index 71b666b4..755fa098 100644 --- a/tasks/util/kata.py +++ b/tasks/util/kata.py @@ -1,7 +1,7 @@ from os import makedirs from os.path import dirname, exists, join from subprocess import run -from tasks.util.docker import is_ctr_running +from tasks.util.docker import copy_from_ctr_image, is_ctr_running from tasks.util.env import ( CONTAINERD_CONFIG_FILE, KATA_CONFIG_DIR, @@ -59,33 +59,37 @@ def stop_kata_workon_ctr(): assert result.returncode == 0 -# TODO: differentiate between a hot-replace and a regular replace -def copy_from_kata_workon_ctr(ctr_path, host_path, sudo=False, debug=False): - ctr_started = run_kata_workon_ctr() - - if not ctr_started: - print("Copying files from running Kata container...") - - docker_cmd = "docker cp {}:{} {}".format( - KATA_WORKON_CTR_NAME, - ctr_path, - host_path, - ) - if sudo: - docker_cmd = "sudo {}".format(docker_cmd) - result = run(docker_cmd, shell=True, capture_output=True) - if debug: - print(result.stdout.decode("utf-8").strip()) - - # If the Kata workon ctr was not running before, make sure we delete it - if ctr_started: - stop_kata_workon_ctr() +def copy_from_kata_workon_ctr( + ctr_path, host_path, sudo=False, debug=False, hot_replace=False +): + if hot_replace and not is_ctr_running(KATA_WORKON_CTR_NAME): + print("Must have the work-on container running to hot replace!") + print("Consider running: inv containerd.cli ") + raise RuntimeError("Hot-replace without work-on running!") + + if hot_replace: + # If hot-replacing, manually copy from the work-on image + docker_cmd = "docker cp {}:{} {}".format( + KATA_WORKON_CTR_NAME, + ctr_path, + host_path, + ) + if sudo: + docker_cmd = "sudo {}".format(docker_cmd) + result = run(docker_cmd, shell=True, capture_output=True) + if debug: + print(result.stdout.decode("utf-8").strip()) + else: + # If not hot-replacing, use the built-in method to copy from a + # container rootfs without initializing it + copy_from_ctr_image(KATA_IMAGE_TAG, [ctr_path], [host_path], requires_sudo=sudo) def replace_agent( dst_initrd_path=join(KATA_IMG_DIR, "kata-containers-initrd-confidential-sc2.img"), debug=False, sc2=False, + hot_replace=False, ): """ Replace the kata-agent with a custom-built one @@ -133,7 +137,11 @@ def replace_agent( ) agent_initrd_path = join(workdir, "usr/bin/kata-agent") copy_from_kata_workon_ctr( - agent_host_path, agent_initrd_path, sudo=True, debug=debug + agent_host_path, + agent_initrd_path, + sudo=True, + debug=debug, + hot_replace=hot_replace, ) # We also need to manually copy the agent to /sbin/init (note that @@ -141,7 +149,11 @@ def replace_agent( alt_agent_initrd_path = join(workdir, "sbin", "init") run("sudo rm {}".format(alt_agent_initrd_path), shell=True, check=True) copy_from_kata_workon_ctr( - agent_host_path, alt_agent_initrd_path, sudo=True, debug=debug + agent_host_path, + alt_agent_initrd_path, + sudo=True, + debug=debug, + hot_replace=hot_replace, ) # Include any extra files that the caller may have provided @@ -192,9 +204,17 @@ def replace_agent( ) ctr_lib_path = join(KATA_SOURCE_DIR, "tools", "osbuilder", "scripts", "lib.sh") initrd_builder_path = join(kata_tmp_scripts, "initrd-builder", "initrd_builder.sh") - copy_from_kata_workon_ctr(ctr_initrd_builder_path, initrd_builder_path, debug=debug) copy_from_kata_workon_ctr( - ctr_lib_path, join(kata_tmp_scripts, "scripts", "lib.sh"), debug=debug + ctr_initrd_builder_path, + initrd_builder_path, + debug=debug, + hot_replace=hot_replace, + ) + copy_from_kata_workon_ctr( + ctr_lib_path, + join(kata_tmp_scripts, "scripts", "lib.sh"), + debug=debug, + hot_replace=hot_replace, ) work_env = {"AGENT_INIT": "yes"} initrd_pack_cmd = "sudo {} -o {} {}".format( @@ -232,6 +252,7 @@ def replace_shim( dst_shim_binary=join(KATA_ROOT, "bin", "containerd-shim-kata-sc2-v2"), dst_runtime_binary=join(KATA_ROOT, "bin", "kata-runtime-sc2"), sc2=True, + hot_replace=False, ): """ Replace the containerd-kata-shim with a custom one @@ -244,14 +265,18 @@ def replace_shim( KATA_SHIM_SOURCE_DIR if sc2 else KATA_BASELINE_SHIM_SOURCE_DIR, "containerd-shim-kata-v2", ) - copy_from_kata_workon_ctr(src_shim_binary, dst_shim_binary, sudo=True) + copy_from_kata_workon_ctr( + src_shim_binary, dst_shim_binary, sudo=True, hot_replace=hot_replace + ) # Also copy the kata-runtime binary src_runtime_binary = join( KATA_SHIM_SOURCE_DIR if sc2 else KATA_BASELINE_SHIM_SOURCE_DIR, "kata-runtime", ) - copy_from_kata_workon_ctr(src_runtime_binary, dst_runtime_binary, sudo=True) + copy_from_kata_workon_ctr( + src_runtime_binary, dst_runtime_binary, sudo=True, hot_replace=hot_replace + ) target_runtimes = SC2_RUNTIMES if sc2 else KATA_RUNTIMES for runtime in target_runtimes: From e7e6534f255c317752eb4cfe5dcf94525cb5ae87 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 17 Jan 2025 14:33:21 +0000 Subject: [PATCH 09/10] demo-apps: add runtime for non-knative demo too --- demo-apps/helloworld-py-nydus/deployment.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/demo-apps/helloworld-py-nydus/deployment.yaml b/demo-apps/helloworld-py-nydus/deployment.yaml index 12d18256..3e06a2cf 100644 --- a/demo-apps/helloworld-py-nydus/deployment.yaml +++ b/demo-apps/helloworld-py-nydus/deployment.yaml @@ -27,6 +27,15 @@ spec: metadata: labels: apps.sc2.io/name: helloworld-py + # WARNING: this annotation is crucial, as otherwise containerd will + # revert to pulling images with the default snapshotter, which in turn + # will mean that: + # 1. The image is fully pulled, as well, on the host. + # 2. The nydus-snapshotter prepares the snapshots on the host, too. + # These two things include an additional runtime of roughly 10 seconds + # for cold starts. + annotations: + io.containerd.cri.runtime-handler: kata-${SC2_RUNTIME_CLASS} spec: runtimeClassName: kata-${SC2_RUNTIME_CLASS} containers: From 151a0f4550381a6b4eb81c86009a0c9f8466b1db Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 17 Jan 2025 15:13:13 +0000 Subject: [PATCH 10/10] nydus: install nydusify when installing sc2 --- .github/workflows/tests.yml | 1 + tasks/nydus.py | 20 ++++++++++++++------ tasks/sc2.py | 4 ++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eec8c9ba..93d35d7a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -63,6 +63,7 @@ jobs: run: | docker pull ghcr.io/sc2-sys/containerd:$(grep -oP 'CONTAINERD_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) docker pull ghcr.io/sc2-sys/kata-containers:$(grep -oP 'KATA_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) + docker pull ghcr.io/sc2-sys/nydus:$(grep -oP 'NYDUS_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) docker pull ghcr.io/sc2-sys/nydus-snapshotter:$(grep -oP 'NYDUS_SNAPSHOTTER_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) - name: "Install SC2" diff --git a/tasks/nydus.py b/tasks/nydus.py index 53a21fd7..63d70647 100644 --- a/tasks/nydus.py +++ b/tasks/nydus.py @@ -2,7 +2,7 @@ from os.path import join from subprocess import run from tasks.util.docker import copy_from_ctr_image -from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT +from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT, print_dotted_line from tasks.util.nydus import NYDUSIFY_PATH from tasks.util.versions import NYDUS_VERSION @@ -26,11 +26,19 @@ def build(ctx, nocache=False, push=False): run(f"docker push {NYDUS_IMAGE_TAG}", shell=True, check=True) -@task -def install(ctx, debug=False, clean=False): - """ - Install the nydus snapshotter binaries and the nydusify CLI tool - """ +def do_install(): + print_dotted_line(f"Installing nydusify (v{NYDUS_VERSION})") + ctr_bin = ["/go/src/github.com/sc2-sys/nydus/contrib/nydusify/cmd/nydusify"] host_bin = [NYDUSIFY_PATH] copy_from_ctr_image(NYDUS_IMAGE_TAG, ctr_bin, host_bin, requires_sudo=False) + + print("Success!") + + +@task +def install(ctx): + """ + Install the nydusify CLI tool + """ + do_install() diff --git a/tasks/sc2.py b/tasks/sc2.py index f7fdba4e..bd9f524f 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -11,6 +11,7 @@ from tasks.knative import install as knative_install from tasks.kubeadm import create as k8s_create, destroy as k8s_destroy from tasks.nydus_snapshotter import install as nydus_snapshotter_install +from tasks.nydus import do_install as nydus_install from tasks.operator import ( install as operator_install, install_cc_runtime as operator_install_cc_runtime, @@ -259,6 +260,9 @@ def deploy(ctx, debug=False, clean=False): # Install the nydus-snapshotter (must happen after we install CoCo) nydus_snapshotter_install(debug=debug, clean=clean) + # Install the nydusify tool + nydus_install() + # Start a local docker registry (must happen before knative installation, # as we rely on it to host our sidecar image) start_local_registry(debug=debug, clean=clean)