diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8ca1adb3..93d35d7a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -63,6 +63,7 @@ jobs: run: | docker pull ghcr.io/sc2-sys/containerd:$(grep -oP 'CONTAINERD_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) docker pull ghcr.io/sc2-sys/kata-containers:$(grep -oP 'KATA_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) + docker pull ghcr.io/sc2-sys/nydus:$(grep -oP 'NYDUS_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) docker pull ghcr.io/sc2-sys/nydus-snapshotter:$(grep -oP 'NYDUS_SNAPSHOTTER_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) - name: "Install SC2" @@ -128,6 +129,48 @@ jobs: env: POD_LABEL: apps.sc2.io/name=helloworld-py + - name: "Run nydus lazy guest-pulling test" + run: | + export SC2_RUNTIME_CLASS=qemu-${{ matrix.tee }}-sc2 + export POD_LABEL="apps.sc2.io/name=helloworld-py" + + # ----- Python Test ---- + + echo "Running python test..." + envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ./bin/kubectl apply -f - + + # Wait for pod to be ready + until [ "$(./bin/kubectl get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done + sleep 1 + + # Get the pod's IP + service_ip=$(./bin/kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') + [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] + envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ./bin/kubectl delete -f - + + # Wait for pod to be deleted + ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=30s + + # Extra cautionary sleep + sleep 5 + echo "Python test succesful!" + + # ----- Knative Test ---- + envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ./bin/kubectl apply -f - + sleep 1 + + # Get the service URL + service_url=$(./bin/kubectl get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) + [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] + + # Wait for pod to be deleted + envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ./bin/kubectl delete -f - + ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=60s + + # Extra cautionary sleep + sleep 5 + echo "Knative test succesful!" + - name: "Enable default-memory annotation" run: | for runtime_class in ${{ matrix.runtime_classes }}; do diff --git a/demo-apps/helloworld-knative-nydus/service.yaml b/demo-apps/helloworld-knative-nydus/service.yaml new file mode 100644 index 00000000..fa84bf5b --- /dev/null +++ b/demo-apps/helloworld-knative-nydus/service.yaml @@ -0,0 +1,30 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: helloworld-knative +spec: + template: + metadata: + labels: + apps.sc2.io/name: helloworld-py + # WARNING: this annotation is crucial, as otherwise containerd will + # revert to pulling images with the default snapshotter, which in turn + # will mean that: + # 1. The image is fully pulled, as well, on the host. + # 2. The nydus-snapshotter prepares the snapshots on the host, too. + # These two things include an additional runtime of roughly 10 seconds + # for cold starts. + annotations: + io.containerd.cri.runtime-handler: kata-${SC2_RUNTIME_CLASS} + spec: + runtimeClassName: kata-${SC2_RUNTIME_CLASS} + # coco-knative: need to run user container as root + securityContext: + runAsUser: 1000 + containers: + - image: sc2cr.io/applications/helloworld-py:unencrypted-nydus + ports: + - containerPort: 8080 + env: + - name: TARGET + value: "World" diff --git a/demo-apps/helloworld-py-nydus/deployment.yaml b/demo-apps/helloworld-py-nydus/deployment.yaml new file mode 100644 index 00000000..3e06a2cf --- /dev/null +++ b/demo-apps/helloworld-py-nydus/deployment.yaml @@ -0,0 +1,46 @@ +apiVersion: v1 +kind: Service +metadata: + name: coco-helloworld-py-node-port +spec: + type: NodePort + selector: + apps.sc2.io/name: helloworld-py + ports: + - name: http + port: 8080 + targetPort: 8080 + protocol: TCP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coco-helloworld-py + labels: + apps.sc2.io/name: helloworld-py +spec: + replicas: 1 + selector: + matchLabels: + apps.sc2.io/name: helloworld-py + template: + metadata: + labels: + apps.sc2.io/name: helloworld-py + # WARNING: this annotation is crucial, as otherwise containerd will + # revert to pulling images with the default snapshotter, which in turn + # will mean that: + # 1. The image is fully pulled, as well, on the host. + # 2. The nydus-snapshotter prepares the snapshots on the host, too. + # These two things include an additional runtime of roughly 10 seconds + # for cold starts. + annotations: + io.containerd.cri.runtime-handler: kata-${SC2_RUNTIME_CLASS} + spec: + runtimeClassName: kata-${SC2_RUNTIME_CLASS} + containers: + - name: helloworld-py + image: sc2cr.io/applications/helloworld-py:unencrypted-nydus + imagePullPolicy: Always + ports: + - containerPort: 8080 diff --git a/tasks/containerd.py b/tasks/containerd.py index 83c2e88b..1abf3eed 100644 --- a/tasks/containerd.py +++ b/tasks/containerd.py @@ -105,6 +105,7 @@ def hot_replace(ctx): if not is_ctr_running(CONTAINERD_CTR_NAME): print("Must have the work-on container running to hot replace!") print("Consider running: inv containerd.cli ") + return for binary in CONTAINERD_BINARY_NAMES: print( diff --git a/tasks/demo_apps.py b/tasks/demo_apps.py index 8136fbdd..236d3dc5 100644 --- a/tasks/demo_apps.py +++ b/tasks/demo_apps.py @@ -8,6 +8,7 @@ LOCAL_REGISTRY_URL, print_dotted_line, ) +from tasks.util.nydus import NYDUSIFY_PATH, nydusify APP_LIST = { "helloworld-py": join(APPS_SOURCE_DIR, "helloworld-py"), @@ -15,15 +16,23 @@ } -def get_docker_tag_for_app(app_name): +def get_docker_tag_for_app(app_name, nydus=False): docker_tag = join(GHCR_URL, GITHUB_ORG, "applications", app_name) docker_tag += ":unencrypted" + + if nydus: + docker_tag += "-nydus" + return docker_tag -def get_local_registry_tag_for_app(app_name): +def get_local_registry_tag_for_app(app_name, nydus=False): docker_tag = join(LOCAL_REGISTRY_URL, "applications", app_name) docker_tag += ":unencrypted" + + if nydus: + docker_tag += "-nydus" + return docker_tag @@ -58,6 +67,12 @@ def build(ctx, app=None, nocache=False): docker_cmd = "docker push {}".format(get_docker_tag_for_app(app_name)) run(docker_cmd, shell=True, check=True) + # Now, convert it to a nydus image, and push again + nydusify( + get_docker_tag_for_app(app_name), + get_docker_tag_for_app(app_name, nydus=True), + ) + def do_push_to_local_registry(debug=False): print_dotted_line("Pushing {} demo apps to local regsitry".format(len(APP_LIST))) @@ -66,6 +81,9 @@ def do_push_to_local_registry(debug=False): docker_tag = get_docker_tag_for_app(app_name) local_registry_tag = get_local_registry_tag_for_app(app_name) + if debug: + print(f"Pushing {docker_tag} to {local_registry_tag}...") + result = run(f"docker pull {docker_tag}", shell=True, capture_output=True) assert result.returncode == 0, result.stderr.decode("utf-8").strip() if debug: @@ -87,6 +105,19 @@ def do_push_to_local_registry(debug=False): if debug: print(result.stdout.decode("utf-8").strip()) + # For nydus, we directly use `nydusify copy` as we cannot `docker pull` + # a nydus image + result = run( + "{} copy --source {} --target {} --target-insecure".format( + NYDUSIFY_PATH, + get_docker_tag_for_app(app_name, nydus=True), + get_local_registry_tag_for_app(app_name, nydus=True), + ), + shell=True, + capture_output=True, + ) + assert result.returncode == 0, result.stderr.decode("utf-8").strip() + print("Success!") diff --git a/tasks/kata.py b/tasks/kata.py index 2972c55e..1128fb67 100644 --- a/tasks/kata.py +++ b/tasks/kata.py @@ -122,18 +122,19 @@ def enable_annotation(ctx, annotation, runtime="qemu-snp-sc2"): @task -def replace_agent(ctx, debug=False, runtime="qemu-snp-sc2"): +def hot_replace_agent(ctx, debug=False, runtime="qemu-snp-sc2"): replace_kata_agent( dst_initrd_path=join( KATA_IMG_DIR, "kata-containers-initrd-confidential-sc2.img" ), debug=debug, sc2=runtime in SC2_RUNTIMES, + hot_replace=True, ) @task -def replace_shim(ctx, runtime="qemu-snp-sc2"): +def hot_replace_shim(ctx, runtime="qemu-snp-sc2"): replace_kata_shim( dst_shim_binary=join( KATA_ROOT, diff --git a/tasks/nydus.py b/tasks/nydus.py index 53a21fd7..63d70647 100644 --- a/tasks/nydus.py +++ b/tasks/nydus.py @@ -2,7 +2,7 @@ from os.path import join from subprocess import run from tasks.util.docker import copy_from_ctr_image -from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT +from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT, print_dotted_line from tasks.util.nydus import NYDUSIFY_PATH from tasks.util.versions import NYDUS_VERSION @@ -26,11 +26,19 @@ def build(ctx, nocache=False, push=False): run(f"docker push {NYDUS_IMAGE_TAG}", shell=True, check=True) -@task -def install(ctx, debug=False, clean=False): - """ - Install the nydus snapshotter binaries and the nydusify CLI tool - """ +def do_install(): + print_dotted_line(f"Installing nydusify (v{NYDUS_VERSION})") + ctr_bin = ["/go/src/github.com/sc2-sys/nydus/contrib/nydusify/cmd/nydusify"] host_bin = [NYDUSIFY_PATH] copy_from_ctr_image(NYDUS_IMAGE_TAG, ctr_bin, host_bin, requires_sudo=False) + + print("Success!") + + +@task +def install(ctx): + """ + Install the nydusify CLI tool + """ + do_install() diff --git a/tasks/sc2.py b/tasks/sc2.py index f7fdba4e..bd9f524f 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -11,6 +11,7 @@ from tasks.knative import install as knative_install from tasks.kubeadm import create as k8s_create, destroy as k8s_destroy from tasks.nydus_snapshotter import install as nydus_snapshotter_install +from tasks.nydus import do_install as nydus_install from tasks.operator import ( install as operator_install, install_cc_runtime as operator_install_cc_runtime, @@ -259,6 +260,9 @@ def deploy(ctx, debug=False, clean=False): # Install the nydus-snapshotter (must happen after we install CoCo) nydus_snapshotter_install(debug=debug, clean=clean) + # Install the nydusify tool + nydus_install() + # Start a local docker registry (must happen before knative installation, # as we rely on it to host our sidecar image) start_local_registry(debug=debug, clean=clean) diff --git a/tasks/util/kata.py b/tasks/util/kata.py index 71b666b4..755fa098 100644 --- a/tasks/util/kata.py +++ b/tasks/util/kata.py @@ -1,7 +1,7 @@ from os import makedirs from os.path import dirname, exists, join from subprocess import run -from tasks.util.docker import is_ctr_running +from tasks.util.docker import copy_from_ctr_image, is_ctr_running from tasks.util.env import ( CONTAINERD_CONFIG_FILE, KATA_CONFIG_DIR, @@ -59,33 +59,37 @@ def stop_kata_workon_ctr(): assert result.returncode == 0 -# TODO: differentiate between a hot-replace and a regular replace -def copy_from_kata_workon_ctr(ctr_path, host_path, sudo=False, debug=False): - ctr_started = run_kata_workon_ctr() - - if not ctr_started: - print("Copying files from running Kata container...") - - docker_cmd = "docker cp {}:{} {}".format( - KATA_WORKON_CTR_NAME, - ctr_path, - host_path, - ) - if sudo: - docker_cmd = "sudo {}".format(docker_cmd) - result = run(docker_cmd, shell=True, capture_output=True) - if debug: - print(result.stdout.decode("utf-8").strip()) - - # If the Kata workon ctr was not running before, make sure we delete it - if ctr_started: - stop_kata_workon_ctr() +def copy_from_kata_workon_ctr( + ctr_path, host_path, sudo=False, debug=False, hot_replace=False +): + if hot_replace and not is_ctr_running(KATA_WORKON_CTR_NAME): + print("Must have the work-on container running to hot replace!") + print("Consider running: inv containerd.cli ") + raise RuntimeError("Hot-replace without work-on running!") + + if hot_replace: + # If hot-replacing, manually copy from the work-on image + docker_cmd = "docker cp {}:{} {}".format( + KATA_WORKON_CTR_NAME, + ctr_path, + host_path, + ) + if sudo: + docker_cmd = "sudo {}".format(docker_cmd) + result = run(docker_cmd, shell=True, capture_output=True) + if debug: + print(result.stdout.decode("utf-8").strip()) + else: + # If not hot-replacing, use the built-in method to copy from a + # container rootfs without initializing it + copy_from_ctr_image(KATA_IMAGE_TAG, [ctr_path], [host_path], requires_sudo=sudo) def replace_agent( dst_initrd_path=join(KATA_IMG_DIR, "kata-containers-initrd-confidential-sc2.img"), debug=False, sc2=False, + hot_replace=False, ): """ Replace the kata-agent with a custom-built one @@ -133,7 +137,11 @@ def replace_agent( ) agent_initrd_path = join(workdir, "usr/bin/kata-agent") copy_from_kata_workon_ctr( - agent_host_path, agent_initrd_path, sudo=True, debug=debug + agent_host_path, + agent_initrd_path, + sudo=True, + debug=debug, + hot_replace=hot_replace, ) # We also need to manually copy the agent to /sbin/init (note that @@ -141,7 +149,11 @@ def replace_agent( alt_agent_initrd_path = join(workdir, "sbin", "init") run("sudo rm {}".format(alt_agent_initrd_path), shell=True, check=True) copy_from_kata_workon_ctr( - agent_host_path, alt_agent_initrd_path, sudo=True, debug=debug + agent_host_path, + alt_agent_initrd_path, + sudo=True, + debug=debug, + hot_replace=hot_replace, ) # Include any extra files that the caller may have provided @@ -192,9 +204,17 @@ def replace_agent( ) ctr_lib_path = join(KATA_SOURCE_DIR, "tools", "osbuilder", "scripts", "lib.sh") initrd_builder_path = join(kata_tmp_scripts, "initrd-builder", "initrd_builder.sh") - copy_from_kata_workon_ctr(ctr_initrd_builder_path, initrd_builder_path, debug=debug) copy_from_kata_workon_ctr( - ctr_lib_path, join(kata_tmp_scripts, "scripts", "lib.sh"), debug=debug + ctr_initrd_builder_path, + initrd_builder_path, + debug=debug, + hot_replace=hot_replace, + ) + copy_from_kata_workon_ctr( + ctr_lib_path, + join(kata_tmp_scripts, "scripts", "lib.sh"), + debug=debug, + hot_replace=hot_replace, ) work_env = {"AGENT_INIT": "yes"} initrd_pack_cmd = "sudo {} -o {} {}".format( @@ -232,6 +252,7 @@ def replace_shim( dst_shim_binary=join(KATA_ROOT, "bin", "containerd-shim-kata-sc2-v2"), dst_runtime_binary=join(KATA_ROOT, "bin", "kata-runtime-sc2"), sc2=True, + hot_replace=False, ): """ Replace the containerd-kata-shim with a custom one @@ -244,14 +265,18 @@ def replace_shim( KATA_SHIM_SOURCE_DIR if sc2 else KATA_BASELINE_SHIM_SOURCE_DIR, "containerd-shim-kata-v2", ) - copy_from_kata_workon_ctr(src_shim_binary, dst_shim_binary, sudo=True) + copy_from_kata_workon_ctr( + src_shim_binary, dst_shim_binary, sudo=True, hot_replace=hot_replace + ) # Also copy the kata-runtime binary src_runtime_binary = join( KATA_SHIM_SOURCE_DIR if sc2 else KATA_BASELINE_SHIM_SOURCE_DIR, "kata-runtime", ) - copy_from_kata_workon_ctr(src_runtime_binary, dst_runtime_binary, sudo=True) + copy_from_kata_workon_ctr( + src_runtime_binary, dst_runtime_binary, sudo=True, hot_replace=hot_replace + ) target_runtimes = SC2_RUNTIMES if sc2 else KATA_RUNTIMES for runtime in target_runtimes: