sc2(nydus): enable lazy pulling in the guest

.github/workflows/tests.yml

@@ -63,6 +63,7 @@ jobs:
         run: |
           docker pull ghcr.io/sc2-sys/containerd:$(grep -oP 'CONTAINERD_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py)
           docker pull ghcr.io/sc2-sys/kata-containers:$(grep -oP 'KATA_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py)
+          docker pull ghcr.io/sc2-sys/nydus:$(grep -oP 'NYDUS_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py)
           docker pull ghcr.io/sc2-sys/nydus-snapshotter:$(grep -oP 'NYDUS_SNAPSHOTTER_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py)
 
       - name: "Install SC2"
@@ -128,6 +129,48 @@ jobs:
         env:
           POD_LABEL: apps.sc2.io/name=helloworld-py
 
+      - name: "Run nydus lazy guest-pulling test"
+        run: |
+          export SC2_RUNTIME_CLASS=qemu-${{ matrix.tee }}-sc2
+          export POD_LABEL="apps.sc2.io/name=helloworld-py"
+
+          # ----- Python Test ----
+
+          echo "Running python test..."
+          envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ./bin/kubectl apply -f -
+
+          # Wait for pod to be ready
+          until [ "$(./bin/kubectl get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done
+          sleep 1
+
+          # Get the pod's IP
+          service_ip=$(./bin/kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}')
+          [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ]
+          envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ./bin/kubectl delete -f -
+
+          # Wait for pod to be deleted
+          ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=30s
+
+          # Extra cautionary sleep
+          sleep 5
+          echo "Python test succesful!"
+
+          # ----- Knative Test ----
+          envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ./bin/kubectl apply -f -
+          sleep 1
+
+          # Get the service URL
+          service_url=$(./bin/kubectl get ksvc helloworld-knative  --output=custom-columns=URL:.status.url --no-headers)
+          [ "$(curl --retry 3 ${service_url})" = "Hello World!" ]
+
+          # Wait for pod to be deleted
+          envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ./bin/kubectl delete -f -
+          ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=60s
+
+          # Extra cautionary sleep
+          sleep 5
+          echo "Knative test succesful!"
+
       - name: "Enable default-memory annotation"
         run: |
           for runtime_class in ${{ matrix.runtime_classes }}; do

demo-apps/helloworld-knative-nydus/service.yaml

@@ -0,0 +1,30 @@
+apiVersion: serving.knative.dev/v1
+kind: Service
+metadata:
+  name: helloworld-knative
+spec:
+  template:
+    metadata:
+      labels:
+        apps.sc2.io/name: helloworld-py
+      # WARNING: this annotation is crucial, as otherwise containerd will
+      # revert to pulling images with the default snapshotter, which in turn
+      # will mean that:
+      # 1. The image is fully pulled, as well, on the host.
+      # 2. The nydus-snapshotter prepares the snapshots on the host, too.
+      # These two things include an additional runtime of roughly 10 seconds
+      # for cold starts.
+      annotations:
+        io.containerd.cri.runtime-handler: kata-${SC2_RUNTIME_CLASS}
+    spec:
+      runtimeClassName: kata-${SC2_RUNTIME_CLASS}
+      # coco-knative: need to run user container as root
+      securityContext:
+        runAsUser: 1000
+      containers:
+        - image: sc2cr.io/applications/helloworld-py:unencrypted-nydus
+          ports:
+            - containerPort: 8080
+          env:
+            - name: TARGET
+              value: "World"

demo-apps/helloworld-py-nydus/deployment.yaml

@@ -0,0 +1,46 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: coco-helloworld-py-node-port
+spec:
+  type: NodePort
+  selector:
+    apps.sc2.io/name: helloworld-py
+  ports:
+  - name: http
+    port: 8080
+    targetPort: 8080
+    protocol: TCP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coco-helloworld-py
+  labels:
+    apps.sc2.io/name: helloworld-py
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      apps.sc2.io/name: helloworld-py
+  template:
+    metadata:
+      labels:
+        apps.sc2.io/name: helloworld-py
+      # WARNING: this annotation is crucial, as otherwise containerd will
+      # revert to pulling images with the default snapshotter, which in turn
+      # will mean that:
+      # 1. The image is fully pulled, as well, on the host.
+      # 2. The nydus-snapshotter prepares the snapshots on the host, too.
+      # These two things include an additional runtime of roughly 10 seconds
+      # for cold starts.
+      annotations:
+        io.containerd.cri.runtime-handler: kata-${SC2_RUNTIME_CLASS}
+    spec:
+      runtimeClassName: kata-${SC2_RUNTIME_CLASS}
+      containers:
+      - name: helloworld-py
+        image: sc2cr.io/applications/helloworld-py:unencrypted-nydus
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8080

tasks/containerd.py

@@ -105,6 +105,7 @@ def hot_replace(ctx):
     if not is_ctr_running(CONTAINERD_CTR_NAME):
         print("Must have the work-on container running to hot replace!")
         print("Consider running: inv containerd.cli ")
+        return
 
     for binary in CONTAINERD_BINARY_NAMES:
         print(

tasks/demo_apps.py

@@ -8,22 +8,31 @@
     LOCAL_REGISTRY_URL,
     print_dotted_line,
 )
+from tasks.util.nydus import NYDUSIFY_PATH, nydusify
 
 APP_LIST = {
     "helloworld-py": join(APPS_SOURCE_DIR, "helloworld-py"),
     "knative-chaining": join(APPS_SOURCE_DIR, "knative-chaining"),
 }
 
 
-def get_docker_tag_for_app(app_name):
+def get_docker_tag_for_app(app_name, nydus=False):
     docker_tag = join(GHCR_URL, GITHUB_ORG, "applications", app_name)
     docker_tag += ":unencrypted"
+
+    if nydus:
+        docker_tag += "-nydus"
+
     return docker_tag
 
 
-def get_local_registry_tag_for_app(app_name):
+def get_local_registry_tag_for_app(app_name, nydus=False):
     docker_tag = join(LOCAL_REGISTRY_URL, "applications", app_name)
     docker_tag += ":unencrypted"
+
+    if nydus:
+        docker_tag += "-nydus"
+
     return docker_tag
 
 
@@ -58,6 +67,12 @@ def build(ctx, app=None, nocache=False):
         docker_cmd = "docker push {}".format(get_docker_tag_for_app(app_name))
         run(docker_cmd, shell=True, check=True)
 
+        # Now, convert it to a nydus image, and push again
+        nydusify(
+            get_docker_tag_for_app(app_name),
+            get_docker_tag_for_app(app_name, nydus=True),
+        )
+
 
 def do_push_to_local_registry(debug=False):
     print_dotted_line("Pushing {} demo apps to local regsitry".format(len(APP_LIST)))
@@ -66,6 +81,9 @@ def do_push_to_local_registry(debug=False):
         docker_tag = get_docker_tag_for_app(app_name)
         local_registry_tag = get_local_registry_tag_for_app(app_name)
 
+        if debug:
+            print(f"Pushing {docker_tag} to {local_registry_tag}...")
+
         result = run(f"docker pull {docker_tag}", shell=True, capture_output=True)
         assert result.returncode == 0, result.stderr.decode("utf-8").strip()
         if debug:
@@ -87,6 +105,19 @@ def do_push_to_local_registry(debug=False):
         if debug:
             print(result.stdout.decode("utf-8").strip())
 
+        # For nydus, we directly use `nydusify copy` as we cannot `docker pull`
+        # a nydus image
+        result = run(
+            "{} copy --source {} --target {} --target-insecure".format(
+                NYDUSIFY_PATH,
+                get_docker_tag_for_app(app_name, nydus=True),
+                get_local_registry_tag_for_app(app_name, nydus=True),
+            ),
+            shell=True,
+            capture_output=True,
+        )
+        assert result.returncode == 0, result.stderr.decode("utf-8").strip()
+
     print("Success!")
 
 

tasks/kata.py

@@ -122,18 +122,19 @@ def enable_annotation(ctx, annotation, runtime="qemu-snp-sc2"):
 
 
 @task
-def replace_agent(ctx, debug=False, runtime="qemu-snp-sc2"):
+def hot_replace_agent(ctx, debug=False, runtime="qemu-snp-sc2"):
     replace_kata_agent(
         dst_initrd_path=join(
             KATA_IMG_DIR, "kata-containers-initrd-confidential-sc2.img"
         ),
         debug=debug,
         sc2=runtime in SC2_RUNTIMES,
+        hot_replace=True,
     )
 
 
 @task
-def replace_shim(ctx, runtime="qemu-snp-sc2"):
+def hot_replace_shim(ctx, runtime="qemu-snp-sc2"):
     replace_kata_shim(
         dst_shim_binary=join(
             KATA_ROOT,

tasks/nydus.py

@@ -2,7 +2,7 @@
 from os.path import join
 from subprocess import run
 from tasks.util.docker import copy_from_ctr_image
-from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT
+from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT, print_dotted_line
 from tasks.util.nydus import NYDUSIFY_PATH
 from tasks.util.versions import NYDUS_VERSION
 
@@ -26,11 +26,19 @@ def build(ctx, nocache=False, push=False):
         run(f"docker push {NYDUS_IMAGE_TAG}", shell=True, check=True)
 
 
-@task
-def install(ctx, debug=False, clean=False):
-    """
-    Install the nydus snapshotter binaries and the nydusify CLI tool
-    """
+def do_install():
+    print_dotted_line(f"Installing nydusify (v{NYDUS_VERSION})")
+
     ctr_bin = ["/go/src/github.com/sc2-sys/nydus/contrib/nydusify/cmd/nydusify"]
     host_bin = [NYDUSIFY_PATH]
     copy_from_ctr_image(NYDUS_IMAGE_TAG, ctr_bin, host_bin, requires_sudo=False)
+
+    print("Success!")
+
+
+@task
+def install(ctx):
+    """
+    Install the nydusify CLI tool
+    """
+    do_install()

tasks/sc2.py

@@ -11,6 +11,7 @@
 from tasks.knative import install as knative_install
 from tasks.kubeadm import create as k8s_create, destroy as k8s_destroy
 from tasks.nydus_snapshotter import install as nydus_snapshotter_install
+from tasks.nydus import do_install as nydus_install
 from tasks.operator import (
     install as operator_install,
     install_cc_runtime as operator_install_cc_runtime,
@@ -259,6 +260,9 @@ def deploy(ctx, debug=False, clean=False):
     # Install the nydus-snapshotter (must happen after we install CoCo)
     nydus_snapshotter_install(debug=debug, clean=clean)
 
+    # Install the nydusify tool
+    nydus_install()
+
     # Start a local docker registry (must happen before knative installation,
     # as we rely on it to host our sidecar image)
     start_local_registry(debug=debug, clean=clean)