diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5fba7e21..de0b8fae 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,223 +57,18 @@ jobs: - name: "Checkout code" uses: actions/checkout@v4 - # Make sure we have the up-to-date version of all images we need as - # part of the deployment process - - name: "Pull all images" - run: | - docker pull ghcr.io/sc2-sys/containerd:$(grep -oP 'CONTAINERD_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) - docker pull ghcr.io/sc2-sys/kata-containers:$(grep -oP 'KATA_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) - docker pull ghcr.io/sc2-sys/nydus:$(grep -oP 'NYDUS_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) - docker pull ghcr.io/sc2-sys/nydus-snapshotter:$(grep -oP 'NYDUS_SNAPSHOTTER_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) - - name: "Install SC2" run: ./bin/inv_wrapper.sh sc2.deploy --clean - - name: "Run python hello world (cold and warm starts)" - run: | - for runtime_class in ${{ matrix.runtime_classes }}; do - for flavour in "cold" "warm"; do - echo "Running test for ${runtime_class} (${flavour})..." - - # If running a cold-start test, make sure to remove the image first - if [ "$flavour" == "cold" ]; then - sudo crictl rmi $(sudo crictl --runtime-endpoint unix:///run/containerd/containerd.sock images | grep sc2cr.io/applications/helloworld-py | awk '{print $3}') > /dev/null 2>&1 - fi - - # Deploy python pod - export SC2_RUNTIME_CLASS=${runtime_class} - envsubst < ./demo-apps/helloworld-py/deployment.yaml | ./bin/kubectl apply -f - - - # Wait for pod to be ready - until [ "$(./bin/kubectl get pods -l ${{ env.POD_LABEL }} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done - sleep 1 - - # Get the pod's IP - service_ip=$(./bin/kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') - [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] - envsubst < ./demo-apps/helloworld-py/deployment.yaml | ./bin/kubectl delete -f - - - # Wait for pod to be deleted - ./bin/kubectl wait --for=delete -l ${{ env.POD_LABEL }} pod --timeout=30s - # Extra cautionary sleep - sleep 5 - echo "Test for ${runtime_class} (${flavour}) successful!" - done - done - env: - POD_LABEL: apps.sc2.io/name=helloworld-py - - - name: "Fetch content (see #130)" - run: sudo ctr -n k8s.io content fetch -k sc2cr.io/applications/helloworld-py:unencrypted - - - name: "Run Knative hello world" - run: | - for runtime_class in ${{ matrix.runtime_classes }}; do - echo "Running test for ${runtime_class}..." - export SC2_RUNTIME_CLASS=${runtime_class} - envsubst < ./demo-apps/helloworld-knative/service.yaml | ./bin/kubectl apply -f - - sleep 1 - - # Get the service URL - service_url=$(./bin/kubectl get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) - [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] - - # Wait for pod to be deleted - envsubst < ./demo-apps/helloworld-knative/service.yaml | ./bin/kubectl delete -f - - ./bin/kubectl wait --for=delete -l ${{ env.POD_LABEL }} pod --timeout=60s - - # Extra cautionary sleep - sleep 5 - echo "Test for ${runtime_class} successful!" - done + - name: "Install BATS" + run: sudo -E apt install -y bats env: - POD_LABEL: apps.sc2.io/name=helloworld-py - - - name: "Run nydus lazy guest-pulling test" - run: | - export SC2_RUNTIME_CLASS=qemu-${{ matrix.tee }}-sc2 - export POD_LABEL="apps.sc2.io/name=helloworld-py" - - # ----- Python Test ---- - - echo "Running python test..." - envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ./bin/kubectl apply -f - - - # Wait for pod to be ready - until [ "$(./bin/kubectl get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done - sleep 1 - - # Get the pod's IP - service_ip=$(./bin/kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') - [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] - envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ./bin/kubectl delete -f - - - # Wait for pod to be deleted - ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=30s - - # Extra cautionary sleep - sleep 5 - echo "Python test succesful!" - - # ----- Knative Test ---- - envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ./bin/kubectl apply -f - - sleep 1 - - # Get the service URL - service_url=$(./bin/kubectl get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) - [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] - - # Wait for pod to be deleted - envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ./bin/kubectl delete -f - - ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=60s + DEBIAN_FRONTEND: noninteractive - # Extra cautionary sleep - sleep 5 - echo "Knative test succesful!" - - - name: "Run nydus host-share test" - # Host-share mechanisms seem not to work with TDX - if: ${{ matrix.tee != 'tdx' }} - run: | - # Change the snapshotter mode and purge (necessary to clear - # containred's content store) - ./bin/inv_wrapper.sh nydus-snapshotter.set-mode host-share - sleep 2 - ./bin/inv_wrapper.sh nydus-snapshotter.purge - - export SC2_RUNTIME_CLASS=qemu-${{ matrix.tee }}-sc2 - export POD_LABEL="apps.sc2.io/name=helloworld-py" - - # When updating the runtime we update all the config files, so we - # need to re-start the cache - sudo -E ./vm-cache/target/release/vm-cache restart - - # ----- Python Test ---- - - echo "Running python test..." - envsubst < ./demo-apps/helloworld-py/deployment.yaml | ./bin/kubectl apply -f - - - # Wait for pod to be ready - until [ "$(./bin/kubectl get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done - sleep 1 - - # Get the pod's IP - service_ip=$(./bin/kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') - [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] - envsubst < ./demo-apps/helloworld-py/deployment.yaml | ./bin/kubectl delete -f - - - # Wait for pod to be deleted - ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=30s - - # Extra cautionary sleep - sleep 5 - echo "Python test succesful!" - - # ----- Knative Test ---- - envsubst < ./demo-apps/helloworld-knative/service.yaml | ./bin/kubectl apply -f - - sleep 1 - - # Get the service URL - service_url=$(./bin/kubectl get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) - [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] - - # Wait for pod to be deleted - envsubst < ./demo-apps/helloworld-knative/service.yaml | ./bin/kubectl delete -f - - ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=60s - - # Extra cautionary sleep - sleep 5 - echo "Knative test succesful!" - - # Change the snapshotter mode back again (and purge) - # - ./bin/inv_wrapper.sh nydus-snapshotter.set-mode guest-pull - sleep 2 - ./bin/inv_wrapper.sh nydus-snapshotter.purge - - - name: "Enable default-memory annotation" - run: | - for runtime_class in ${{ matrix.runtime_classes }}; do - ./bin/inv_wrapper.sh kata.enable-annotation default_memory --runtime ${runtime_class} - # Here we benefit that the last variable is the one we want to use - # for vm-cache - export SC2_RUNTIME_CLASS=${runtime_class} - done - - # Aftre changing the annotation of the qemu-snp-sc2 runtime class we - # need to restart the VM cache - sudo -E ./vm-cache/target/release/vm-cache restart - - - name: "Fetch content (see #130)" - run: | - sudo ctr -n k8s.io content fetch -k sc2cr.io/system/knative-sidecar@sha256:79d5f6031f308cee209c4c32eeab9113b29a1ed4096c5d657504096734ca3b1d - sudo ctr -n k8s.io content fetch registry.k8s.io/pause:3.8 - - - name: "Run knative chaining demo" - run: | - for runtime_class in ${{ matrix.runtime_classes }}; do - echo "Running test for ${runtime_class}..." - export SC2_RUNTIME_CLASS=${runtime_class} - envsubst < ./demo-apps/knative-chaining/chaining.yaml | ./bin/kubectl apply -f - - sleep 1 - - # Curl the channel URL - ./demo-apps/knative-chaining/curl_cmd.sh - - # Wait for pod 3 to be scaled down - until [ "$(kubectl -n ${{ env.NAMESPACE }} logs -l ${{ env.POD_LABEL_THREE }} | grep 'cloudevent(s3): done!' | wc -l)" = "1" ]; do echo "Waiting for chain to finish..."; sleep 2; done - - # Finally, clear-up - envsubst < ./demo-apps/knative-chaining/chaining.yaml | ./bin/kubectl delete -f - - - # Extra cautionary sleep - sleep 5 - echo "Test for ${runtime_class} successful!" - done + - name: "Run integration tests" + run: bats ./tests env: - NAMESPACE: chaining-test - POD_LABEL_ONE: apps.sc2.io/name=knative-chaining-one - POD_LABEL_THREE: apps.sc2.io/name=knative-chaining-three + SC2_TEE: ${{ matrix.tee }} - name: "Clean-up" if: always() diff --git a/README.md b/README.md index 71fad58c..2b8e0895 100644 --- a/README.md +++ b/README.md @@ -51,11 +51,11 @@ export SC2_RUNTIME_CLASS=qemu-snp-sc2 # Knative demo envsubst < ./demo-apps/helloworld-knative/service.yaml | kubectl apply -f - -curl $(kubectl get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) +curl $(kubectl -n sc2-demo get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) # Non-Knative demo envsubst < ./demo-apps/helloworld-py/deployment.yaml | kubectl apply -f - -curl $(kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}'):8080 +curl $(kubectl -n sc2-demo get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}'):8080 ``` for more complex applications and workloads, please check our [applications]( diff --git a/bin/workon.sh b/bin/workon.sh index 7a780b64..994bbdf5 100644 --- a/bin/workon.sh +++ b/bin/workon.sh @@ -39,13 +39,13 @@ TEE_DETECT_BINARY=${TEE_DETECT_ROOT}/target/release/tee-detect cargo build -q --release --manifest-path ${TEE_DETECT_ROOT}/Cargo.toml if "${TEE_DETECT_BINARY}" snp; then - TEE=snp + export SC2_TEE=snp export SC2_RUNTIME_CLASS=qemu-snp-sc2 elif "${TEE_DETECT_BINARY}" tdx; then - TEE=tdx + export SC2_TEE=tdx export SC2_RUNTIME_CLASS=qemu-tdx-sc2 else - TEE=none + export SC2_TEE=none echo "sc2-deploy: WARN: neither SNP nor TDX is enabled" fi @@ -75,7 +75,7 @@ echo "" echo "----------------------------------" echo "CLI for SC2 Deployment Scripts" echo "CoCo Version: ${COCO_VERSION}" -echo "TEE: ${TEE}" +echo "TEE: ${SC2_TEE}" echo "----------------------------------" echo "" diff --git a/demo-apps/helloworld-knative-nydus/service.yaml b/demo-apps/helloworld-knative-nydus/service.yaml index fa84bf5b..6ac371f7 100644 --- a/demo-apps/helloworld-knative-nydus/service.yaml +++ b/demo-apps/helloworld-knative-nydus/service.yaml @@ -1,7 +1,13 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: sc2-demo +--- apiVersion: serving.knative.dev/v1 kind: Service metadata: name: helloworld-knative + namespace: sc2-demo spec: template: metadata: diff --git a/demo-apps/helloworld-knative/service.yaml b/demo-apps/helloworld-knative/service.yaml index fd5c658b..15e1635f 100644 --- a/demo-apps/helloworld-knative/service.yaml +++ b/demo-apps/helloworld-knative/service.yaml @@ -1,7 +1,13 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: sc2-demo +--- apiVersion: serving.knative.dev/v1 kind: Service metadata: name: helloworld-knative + namespace: sc2-demo spec: template: metadata: diff --git a/demo-apps/helloworld-py-nydus/deployment.yaml b/demo-apps/helloworld-py-nydus/deployment.yaml index 3e06a2cf..7c56f03a 100644 --- a/demo-apps/helloworld-py-nydus/deployment.yaml +++ b/demo-apps/helloworld-py-nydus/deployment.yaml @@ -1,7 +1,13 @@ apiVersion: v1 +kind: Namespace +metadata: + name: sc2-demo +--- +apiVersion: v1 kind: Service metadata: name: coco-helloworld-py-node-port + namespace: sc2-demo spec: type: NodePort selector: @@ -16,6 +22,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: coco-helloworld-py + namespace: sc2-demo labels: apps.sc2.io/name: helloworld-py spec: diff --git a/demo-apps/helloworld-py/Dockerfile b/demo-apps/helloworld-py/Dockerfile index 5b5538e5..0a6e1451 100644 --- a/demo-apps/helloworld-py/Dockerfile +++ b/demo-apps/helloworld-py/Dockerfile @@ -2,10 +2,10 @@ FROM python:3.10-slim # Allow statements and log messages to immediately appear in the Knative logs -ENV PYTHONUNBUFFERED True +ENV PYTHONUNBUFFERED=True # Copy local code to the container image. -ENV APP_HOME /app +ENV APP_HOME=/app WORKDIR $APP_HOME COPY . ./ diff --git a/demo-apps/helloworld-py/deployment.yaml b/demo-apps/helloworld-py/deployment.yaml index f55e269b..7e5461fb 100644 --- a/demo-apps/helloworld-py/deployment.yaml +++ b/demo-apps/helloworld-py/deployment.yaml @@ -1,7 +1,13 @@ apiVersion: v1 +kind: Namespace +metadata: + name: sc2-demo +--- +apiVersion: v1 kind: Service metadata: name: coco-helloworld-py-node-port + namespace: sc2-demo spec: type: NodePort selector: @@ -16,6 +22,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: coco-helloworld-py + namespace: sc2-demo labels: apps.sc2.io/name: helloworld-py spec: diff --git a/demo-apps/knative-chaining/Dockerfile b/demo-apps/knative-chaining/Dockerfile index 08ba9f4c..276ff322 100644 --- a/demo-apps/knative-chaining/Dockerfile +++ b/demo-apps/knative-chaining/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80 +FROM rust:1.84 WORKDIR /app COPY . ./ diff --git a/demo-apps/knative-chaining/chaining.yaml b/demo-apps/knative-chaining/chaining.yaml index d629a713..8d37a17c 100644 --- a/demo-apps/knative-chaining/chaining.yaml +++ b/demo-apps/knative-chaining/chaining.yaml @@ -18,7 +18,7 @@ apiVersion: v1 kind: Namespace metadata: - name: chaining-test + name: sc2-demo --- # We need to have as many channels as edges in our workflow DAG. Alternatively, # we could enforce edges by using a Broker/Trigger pattern and filtering on @@ -27,7 +27,7 @@ apiVersion: messaging.knative.dev/v1 kind: Channel metadata: name: ingress-to-one - namespace: chaining-test + namespace: sc2-demo spec: channelTemplate: apiVersion: messaging.knative.dev/v1 @@ -37,7 +37,7 @@ apiVersion: messaging.knative.dev/v1 kind: Channel metadata: name: one-to-two - namespace: chaining-test + namespace: sc2-demo spec: channelTemplate: apiVersion: messaging.knative.dev/v1 @@ -47,7 +47,7 @@ apiVersion: messaging.knative.dev/v1 kind: Channel metadata: name: two-to-three - namespace: chaining-test + namespace: sc2-demo spec: channelTemplate: apiVersion: messaging.knative.dev/v1 @@ -60,7 +60,7 @@ apiVersion: serving.knative.dev/v1 kind: Service metadata: name: coco-knative-chaining-one - namespace: chaining-test + namespace: sc2-demo spec: template: spec: @@ -74,6 +74,7 @@ spec: labels: apps.sc2.io/name: knative-chaining-one annotations: + io.containerd.cri.runtime-handler: kata-${SC2_RUNTIME_CLASS} # NOTE: we may have to enable this annotation in Kata's config file # under hypervisor.qemu.enable_annotations (add 'default_memory') io.katacontainers.config.hypervisor.default_memory: "6144" @@ -85,7 +86,7 @@ apiVersion: sinks.knative.dev/v1alpha1 kind: JobSink metadata: name: coco-knative-chaining-two - namespace: chaining-test + namespace: sc2-demo spec: job: spec: @@ -108,6 +109,7 @@ spec: labels: apps.sc2.io/name: knative-chaining-two annotations: + io.containerd.cri.runtime-handler: kata-${SC2_RUNTIME_CLASS} # NOTE: we may have to enable this annotation in Kata's config file # under hypervisor.qemu.enable_annotations (add 'default_memory') io.katacontainers.config.hypervisor.default_memory: "6144" @@ -118,7 +120,7 @@ apiVersion: serving.knative.dev/v1 kind: Service metadata: name: coco-knative-chaining-three - namespace: chaining-test + namespace: sc2-demo spec: template: spec: @@ -133,6 +135,7 @@ spec: apps.sc2.io/name: knative-chaining-three annotations: autoscaling.knative.dev/scale-to-zero-pod-retention-period: "1m" + io.containerd.cri.runtime-handler: kata-${SC2_RUNTIME_CLASS} # NOTE: we may have to enable this annotation in Kata's config file # under hypervisor.qemu.enable_annotations (add 'default_memory') io.katacontainers.config.hypervisor.default_memory: "6144" @@ -141,7 +144,7 @@ apiVersion: messaging.knative.dev/v1 kind: Subscription metadata: name: edge-one-subscription - namespace: chaining-test + namespace: sc2-demo spec: channel: apiVersion: messaging.knative.dev/v1 @@ -162,7 +165,7 @@ apiVersion: messaging.knative.dev/v1 kind: Subscription metadata: name: edge-two-subscription - namespace: chaining-test + namespace: sc2-demo spec: channel: apiVersion: messaging.knative.dev/v1 @@ -178,7 +181,7 @@ apiVersion: messaging.knative.dev/v1 kind: Subscription metadata: name: edge-three-subscription - namespace: chaining-test + namespace: sc2-demo spec: channel: apiVersion: messaging.knative.dev/v1 diff --git a/demo-apps/knative-chaining/curl_cmd.sh b/demo-apps/knative-chaining/curl_cmd.sh index b9a3cf3d..93c1cb3e 100755 --- a/demo-apps/knative-chaining/curl_cmd.sh +++ b/demo-apps/knative-chaining/curl_cmd.sh @@ -1,10 +1,11 @@ #!/bin/bash -./bin/kubectl run curl --image=curlimages/curl --rm=true --restart=Never -i -- -X POST -v \ +./bin/kubectl -n sc2-demo run curl --image=curlimages/curl --restart=Never -- -X POST -v \ -H "content-type: application/json" \ -H "ce-specversion: 1.0" \ -H "ce-source: cli" \ - -H "ce-type: http://one-to-two-kn-channel.chaining-test.svc.cluster.local" \ + -H "ce-type: http://one-to-two-kn-channel.sc2-demo.svc.cluster.local" \ -H "ce-id: 1" \ -d '{"details":"ChannelDemo"}' \ - http://ingress-to-one-kn-channel.chaining-test.svc.cluster.local + http://ingress-to-one-kn-channel.sc2-demo.svc.cluster.local + diff --git a/demo-apps/knative-chaining/src/main.rs b/demo-apps/knative-chaining/src/main.rs index c261b2f1..0d70bcae 100644 --- a/demo-apps/knative-chaining/src/main.rs +++ b/demo-apps/knative-chaining/src/main.rs @@ -91,7 +91,7 @@ pub fn process_event(mut event: Event) -> Event { // Write the new destination channel // FIXME: this is hardcoded in chaining.yaml - scaled_event.set_type("http://two-to-three-kn-channel.chaining-test.svc.cluster.local"); + scaled_event.set_type("http://two-to-three-kn-channel.sc2-demo.svc.cluster.local"); println!("cloudevent(s1): fanning out by a factor of {FAN_OUT_SCALE}"); diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index a6cca897..391536de 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -141,16 +141,19 @@ lines of: failed to extract layer sha256:1e604deea57dbda554a168861cff1238f93b8c6c69c863c43aed37d9d99c5fed: failed to get reader from content store: content digest sha256:9fa9226be034e47923c0457d916aa68474cdfb23af8d4525e9baeebc4760977a: not found ``` -you need to manually fetch the image contents on the host. This is a once-per- -host fix: +you may not be setting the right annotation in the pod: + +``` +io.containerd.cri.runtime-handler: kata-qemu-... +``` + +A temporary fix to get the pod running is to manually fetch the content but, as +said above, the annotation should take care of this. ```bash ctr -n k8s.io content fetch ${IMAGE_NAME} ``` -the image name is the image tag appearing right before the error message in -the pod logs. - ### Rootfs Mount Issue Sometimes, if we are mixing and matching different snapshotters, we may run diff --git a/tasks/nydus.py b/tasks/nydus.py index a8a61f25..8609a53b 100644 --- a/tasks/nydus.py +++ b/tasks/nydus.py @@ -2,15 +2,14 @@ from os.path import join from subprocess import run from tasks.util.docker import copy_from_ctr_image, is_ctr_running -from tasks.util.env import COCO_ROOT, GHCR_URL, GITHUB_ORG, PROJ_ROOT, print_dotted_line -from tasks.util.nydus import NYDUSIFY_PATH +from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT, print_dotted_line +from tasks.util.nydus import NYDUS_IMAGE_HOST_PATH, NYDUSIFY_PATH from tasks.util.versions import NYDUS_VERSION NYDUS_CTR_NAME = "nydus-workon" NYDUS_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "nydus") + f":{NYDUS_VERSION}" NYDUS_IMAGE_CTR_PATH = "/go/src/github.com/sc2-sys/nydus/target/release/nydus-image" -NYDUS_IMAGE_HOST_PATH = join(COCO_ROOT, "bin", "nydus-image") @task diff --git a/tasks/nydus_snapshotter.py b/tasks/nydus_snapshotter.py index f62c6b25..7abd96f0 100644 --- a/tasks/nydus_snapshotter.py +++ b/tasks/nydus_snapshotter.py @@ -3,6 +3,7 @@ from os import getgid, getuid from os.path import exists, join from subprocess import run +from tasks.util.containerd import wait_for_containerd_socket from tasks.util.docker import copy_from_ctr_image, is_ctr_running from tasks.util.env import ( BIN_DIR, @@ -140,13 +141,16 @@ def do_purge(debug=False): # Clear all possibly used images (only images in our registry, or the # pause container images) + tmp_out = "/tmp/cmd_output" cmd = ( "sudo crictl --runtime-endpoint unix:///run/containerd/containerd.sock" - " images -o json" + f" images -o json > {tmp_out} 2> /dev/null" ) rm_cmd = "sudo crictl --runtime-endpoint unix:///run/containerd/containerd.sock rmi" try: - stdout = run(cmd, shell=True, capture_output=True).stdout.decode("utf-8") + run(cmd, shell=True, check=True) + with open(tmp_out, "r") as fh: + stdout = fh.read().strip() data = json_loads(stdout) except JSONDecodeError as e: stderr = run(cmd, shell=True, capture_output=True).stderr.decode("utf-8") @@ -202,6 +206,7 @@ def purge(ctx): """ Remove all cached snapshots in the snapshotter cache """ + wait_for_containerd_socket() do_purge(debug=True) diff --git a/tasks/sc2.py b/tasks/sc2.py index 328d2bae..ee102c13 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -314,7 +314,8 @@ def deploy(ctx, debug=False, clean=False): print("Success!") # Update SNP class to use default QEMU (we use host kernel 6.11, so we - # can use upstream QEMU 9.1) + # can use upstream QEMU 9.1). We do this update before generating the SC2 + # runtime classes, so they will inherit the QEMU value # TODO: remove when bumping to a new CoCo release qemu_path = join(KATA_ROOT, "bin", "qemu-system-x86_64") updated_toml_str = """ diff --git a/tasks/util/kata.py b/tasks/util/kata.py index 050a12d3..8d896d6f 100644 --- a/tasks/util/kata.py +++ b/tasks/util/kata.py @@ -161,7 +161,7 @@ def prepare_rootfs(tmp_rootfs_base_dir, debug=False, sc2=False, hot_replace=Fals """ This function takes a directory as input, and generates the root-filesystem needed in SC2 at /rootfs. The result can be consumed - to pack an `initrd` or a .qcow2 image. + to pack an `initrd` """ # ----- Prepare temporary rootfs directory ----- @@ -255,7 +255,13 @@ def prepare_rootfs(tmp_rootfs_base_dir, debug=False, sc2=False, hot_replace=Fals "AGENT_SOURCE_BIN": join(tmp_rootfs_base_dir, "kata-agent"), "CONFIDENTIAL_GUEST": "yes", "DMVERITY_SUPPORT": "yes", - "MEASURED_ROOTFS": "yes", + "MEASURED_ROOTFS": "no", + # We build the `initrd` inside a container image to prevent different + # host OS versions from introducing subtle changes in the rootfs + "USE_DOCKER": "yes", + "OS_VERSION": "jammy", + "RUST_VERSION": "1.75.0", + "GO_VERSION": "1.22.2", "PAUSE_IMAGE_TARBALL": build_pause_image( sc2=sc2, debug=debug, hot_replace=hot_replace ), diff --git a/tasks/util/nydus.py b/tasks/util/nydus.py index eab2c8b8..bb6f5e8a 100644 --- a/tasks/util/nydus.py +++ b/tasks/util/nydus.py @@ -1,15 +1,23 @@ -from os.path import join +from os import environ +from os.path import dirname, join from subprocess import run -from tasks.util.env import PROJ_ROOT +from tasks.util.env import COCO_ROOT, PROJ_ROOT NYDUSIFY_PATH = join(PROJ_ROOT, "bin", "nydusify") +NYDUS_IMAGE_HOST_PATH = join(COCO_ROOT, "bin", "nydus-image") + def nydusify(src_tag, dst_tag): + # Add nydus-image to path + work_env = environ.copy() + work_env["PATH"] = work_env.get("PATH", "") + ":" + dirname(NYDUS_IMAGE_HOST_PATH) + # Note that nydusify automatically pushes the image result = run( f"{NYDUSIFY_PATH} convert --source {src_tag} --target {dst_tag}", shell=True, capture_output=True, + env=work_env, ) assert result.returncode == 0, result.stderr.decode("utf-8").strip() diff --git a/tests/cold_starts_guest_pull.bats b/tests/cold_starts_guest_pull.bats new file mode 100644 index 00000000..0f1e8306 --- /dev/null +++ b/tests/cold_starts_guest_pull.bats @@ -0,0 +1,104 @@ +#!/usr/bin/env bats + +load utils/env.sh + +setup_file() { + load utils/helpers.sh + + set_snapshotter_mode "guest-pull" +} + +# Make sure we purge before each test so that we have a cold start +setup() { + load utils/helpers.sh + + ${INV} nydus-snapshotter.purge +} + +teardown() { + load utils/helpers.sh + + common_teardown +} + +snapshotter="guest-pull" + +# ------------------------------------------------------------------------------ +# Python cold starts +# ------------------------------------------------------------------------------ + +@test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[0]}" + ' + + cleanup_python_hello_world +} + +@test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[1]}" + ' + + cleanup_python_hello_world +} + +@test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[2]}" + ' + + cleanup_python_hello_world +} + +@test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_python_hello_world +} + +# ------------------------------------------------------------------------------ +# Knative cold starts +# ------------------------------------------------------------------------------ + +@test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[0]}" + ' + + cleanup_knative_hello_world +} + +@test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[1]}" + ' + + cleanup_knative_hello_world +} + +@test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[2]}" + ' + + cleanup_knative_hello_world +} + +@test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_hello_world +} diff --git a/tests/cold_starts_host_share.bats b/tests/cold_starts_host_share.bats new file mode 100644 index 00000000..719f0799 --- /dev/null +++ b/tests/cold_starts_host_share.bats @@ -0,0 +1,54 @@ +#!/usr/bin/env bats + +load utils/env.sh + +setup_file() { + load utils/helpers.sh + + set_snapshotter_mode "host-share" +} + +# Make sure we purge before each test so that we have a cold start +setup() { + load utils/helpers.sh + + ${INV} nydus-snapshotter.purge +} + +teardown() { + load utils/helpers.sh + + common_teardown +} + +snapshotter="host-share" + +# ------------------------------------------------------------------------------ +# Python cold starts +# ------------------------------------------------------------------------------ + +@test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + [[ "$SC2_TEE" == "tdx" ]] && skip "#142" + + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_python_hello_world +} + +# ------------------------------------------------------------------------------ +# Knative cold starts +# ------------------------------------------------------------------------------ + +@test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + [[ "$SC2_TEE" == "tdx" ]] && skip "#142" + + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_hello_world +} diff --git a/tests/helloworld_knative_guest_pull.bats b/tests/helloworld_knative_guest_pull.bats new file mode 100644 index 00000000..28a46b09 --- /dev/null +++ b/tests/helloworld_knative_guest_pull.bats @@ -0,0 +1,62 @@ +#!/usr/bin/env bats + +load utils/env.sh + +setup_file() { + load utils/helpers.sh + + set_snapshotter_mode "guest-pull" +} + +setup() { + load utils/helpers.sh +} + +teardown() { + load utils/helpers.sh + + common_teardown +} + +TEST_NAME="Test knative hello world" +snapshotter="guest-pull" + +# ------------------------------------------------------------------------------ +# Nydus Snapshotter in Guest Pull mode +# ------------------------------------------------------------------------------ + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[0]}" + ' + + cleanup_knative_hello_world +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[1]}" + ' + + cleanup_knative_hello_world +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[2]}" + ' + + cleanup_knative_hello_world +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_hello_world +} diff --git a/tests/helloworld_knative_host_share.bats b/tests/helloworld_knative_host_share.bats new file mode 100644 index 00000000..8a3879a7 --- /dev/null +++ b/tests/helloworld_knative_host_share.bats @@ -0,0 +1,41 @@ +#!/usr/bin/env bats + +load utils/env.sh + +setup_file() { + load utils/helpers.sh + + set_snapshotter_mode "host-share" +} + +setup() { + load utils/helpers.sh +} + +teardown() { + load utils/helpers.sh + + common_teardown +} + +TEST_NAME="Test knative hello world" +snapshotter="host-share" + +# ------------------------------------------------------------------------------ +# Nydus Snapshotter in Host Share mode +# +# Using the snapshotter in host-share mode is only supported for SC2 runtimes, +# as we have only implemented the patches in our forked branches. +# ------------------------------------------------------------------------------ + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + [[ "$SC2_TEE" == "tdx" ]] && skip "#142" + + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_hello_world +} + diff --git a/tests/helloworld_python_guest_pull.bats b/tests/helloworld_python_guest_pull.bats new file mode 100644 index 00000000..a0893481 --- /dev/null +++ b/tests/helloworld_python_guest_pull.bats @@ -0,0 +1,62 @@ +#!/usr/bin/env bats + +load utils/env.sh + +setup_file() { + load utils/helpers.sh + + set_snapshotter_mode "guest-pull" +} + +setup() { + load utils/helpers.sh +} + +teardown() { + load utils/helpers.sh + + common_teardown +} + +TEST_NAME="Test python hello world" +snapshotter="guest-pull" + +# ------------------------------------------------------------------------------ +# Nydus Snapshotter in Guest Pull mode +# ------------------------------------------------------------------------------ + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[0]}" + ' + + cleanup_python_hello_world +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[1]}" + ' + + cleanup_python_hello_world +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[2]}" + ' + + cleanup_python_hello_world +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_python_hello_world +} diff --git a/tests/helloworld_python_host_share.bats b/tests/helloworld_python_host_share.bats new file mode 100644 index 00000000..69924ee5 --- /dev/null +++ b/tests/helloworld_python_host_share.bats @@ -0,0 +1,41 @@ +#!/usr/bin/env bats + +load utils/env.sh + +setup_file() { + load utils/helpers.sh + + set_snapshotter_mode "host-share" +} + +setup() { + load utils/helpers.sh +} + +teardown() { + load utils/helpers.sh + + common_teardown +} + +TEST_NAME="Test python hello world" +snapshotter="host-share" + +# ------------------------------------------------------------------------------ +# Nydus Snapshotter in Host Share mode +# +# Using the snapshotter in host-share mode is only supported for SC2 runtimes, +# as we have only implemented the patches in our forked branches. +# ------------------------------------------------------------------------------ + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + [[ "$SC2_TEE" == "tdx" ]] && skip "#142" + + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_python_hello_world +} + diff --git a/tests/knative_chaining_guest_pull.bats b/tests/knative_chaining_guest_pull.bats new file mode 100644 index 00000000..d7486ac5 --- /dev/null +++ b/tests/knative_chaining_guest_pull.bats @@ -0,0 +1,74 @@ +#!/usr/bin/env bats + +load utils/env.sh + +setup_file() { + load utils/helpers.sh + + set_snapshotter_mode "guest-pull" +} + +setup() { + load utils/helpers.sh + + # Longer timeout for chaining tests + export SC2_TEST_TIMEOUT=180 +} + +teardown() { + load utils/helpers.sh + + common_teardown +} + +TEST_NAME="Test knative chaining" +snapshotter="guest-pull" + +# ------------------------------------------------------------------------------ +# Nydus Snapshotter in Guest Pull mode +# ------------------------------------------------------------------------------ + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { + enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[0]}" + + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_chaining "${SC2_RUNTIME_CLASSES[0]}" + ' + + cleanup_knative_chaining +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { + enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[1]}" + + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_chaining "${SC2_RUNTIME_CLASSES[1]}" + ' + + cleanup_knative_chaining +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { + enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[2]}" + + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_chaining "${SC2_RUNTIME_CLASSES[2]}" + ' + + cleanup_knative_chaining +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[3]}" + restart_vm_cache + + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_chaining "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_chaining +} diff --git a/tests/knative_chaining_host_share.bats b/tests/knative_chaining_host_share.bats new file mode 100644 index 00000000..09fe6f34 --- /dev/null +++ b/tests/knative_chaining_host_share.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats + +load utils/env.sh + +setup_file() { + load utils/helpers.sh + + set_snapshotter_mode "host-share" +} + +setup() { + load utils/helpers.sh + + # Longer timeout for chaining tests + export SC2_TEST_TIMEOUT=120 +} + +teardown() { + load utils/helpers.sh + + common_teardown +} + +TEST_NAME="Test knative chaining" +snapshotter="host-share" + +# ------------------------------------------------------------------------------ +# Nydus Snapshotter in Host Share mode +# +# Using the snapshotter in host-share mode is only supported for SC2 runtimes, +# as we have only implemented the patches in our forked branches. +# ------------------------------------------------------------------------------ + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + [[ "$SC2_TEE" == "tdx" ]] && skip "#142" + + enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[3]}" + restart_vm_cache + + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_chaining "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_chaining +} + diff --git a/tests/lazy_loading.bats b/tests/lazy_loading.bats new file mode 100644 index 00000000..3c9b5aa4 --- /dev/null +++ b/tests/lazy_loading.bats @@ -0,0 +1,39 @@ +#!/usr/bin/env bats + +load utils/env.sh + +setup_file() { + load utils/helpers.sh + + set_snapshotter_mode "guest-pull" +} + +setup() { + load utils/helpers.sh +} + +teardown() { + load utils/helpers.sh + + common_teardown +} + +snapshotter="guest-pull" + +@test "Test python lazy loading: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_lazy_loading "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_python_hello_world +} + +@test "Test knative lazy loading: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_lazy_loading "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_hello_world +} diff --git a/tests/utils/env.sh b/tests/utils/env.sh new file mode 100644 index 00000000..d0d117a6 --- /dev/null +++ b/tests/utils/env.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +export PROJ_ROOT=${THIS_DIR}/../.. +export KUBECTL=${PROJ_ROOT}/bin/kubectl +export INV=${PROJ_ROOT}/bin/inv_wrapper.sh + +# Work out the runtime classes from the env. var +if [ -n "$SC2_TEE" ]; then + if [ "$SC2_TEE" == "snp" ]; then + export SC2_RUNTIME_CLASSES=("qemu" "qemu-coco-dev" "qemu-snp" "qemu-snp-sc2") + elif [ "$SC2_TEE" == "tdx" ]; then + export SC2_RUNTIME_CLASSES=("qemu" "qemu-coco-dev" "qemu-tdx" "qemu-tdx-sc2") + else + echo "ERROR: SC2_TEE env. var must be one in: 'snp', 'tdx'" + exit 1 + fi +else + echo "ERROR: SC2_TEE env. var must be set" + exit 1 +fi + +# Useful constants +INTERTEST_SLEEP_SECS=1 +SC2_DEMO_NAMESPACE=sc2-demo +SC2_TEST_TIMEOUT=60 + +# Images +PAUSE_IMAGE="registry.k8s.io/pause:3.8" +SIDECAR_IMAGE="sc2cr.io/system/knative-sidecar@sha256:79d5f6031f308cee209c4c32eeab9113b29a1ed4096c5d657504096734ca3b1d" diff --git a/tests/utils/helpers.sh b/tests/utils/helpers.sh new file mode 100644 index 00000000..f0eff804 --- /dev/null +++ b/tests/utils/helpers.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +source ./tests/utils/env.sh + +# ------------------------------------------------------------------------------ +# Helper functions to set-up the environment +# ------------------------------------------------------------------------------ + +common_teardown() { + ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + + # Cautionary inter-test sleep + sleep ${INTERTEST_SLEEP_SECS} +} + +enable_kata_annotation() { + local annotation="$1" + local runtime="$2" + ${INV} kata.enable-annotation ${annotation} --runtime ${runtime} +} + +restart_vm_cache() { + export SC2_RUNTIME_CLASS="qemu-${SC2_TEE}-sc2" + + # Need to pass the environment because the vm-cache script not only uses + # the SC2_RUNTIME_CLASS env. var, but also others like $HOME or $USER + sudo -E ${PROJ_ROOT}/vm-cache/target/release/vm-cache restart +} + +set_snapshotter_mode() { + local snapshotter_mode="$1" + ${INV} nydus-snapshotter.set-mode ${snapshotter_mode} + ${INV} nydus-snapshotter.purge + + export SC2_SNAPSHOTTER=${snapshotter_mode} + + sleep 1 + + # Setting the snapshotter mode changes the config file, so we must restart + # the vm cache + restart_vm_cache +} + +# ------------------------------------------------------------------------------ +# Helper functions clean-up after executions +# ------------------------------------------------------------------------------ + +cleanup_knative_chaining() { + # Curl service + kubectl -n sc2-demo delete -l run=curl pod + + # First service: fan-out + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete service.serving.knative.dev coco-knative-chaining-one + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete -l apps.sc2.io/name=knative-chaining-one pod + + # Second service: job sink + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete JobSink.sinks.knative.dev coco-knative-chaining-two + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete -l apps.sc2.io/name=knative-chaining-two pod + + # Third service: fan-in + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete service.serving.knative.dev coco-knative-chaining-three + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete -l apps.sc2.io/name=knative-chaining-three pod +} + +cleanup_knative_hello_world() { + # Delete resources in order, as it seems to prevent deletion from being stuck + SERVICE_NAME="helloworld-knative" + POD_LABEL="apps.sc2.io/name=helloworld-py" + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete service.serving.knative.dev ${SERVICE_NAME} + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete -l ${POD_LABEL} pod +} + +cleanup_python_hello_world() { + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete deployment coco-helloworld-py +} + +# ------------------------------------------------------------------------------ +# Helper functions to run specific workloads +# ------------------------------------------------------------------------------ + +run_knative_chaining() { + local runtime_class="$1" + export SC2_RUNTIME_CLASS="$runtime_class" + + envsubst < ./demo-apps/knative-chaining/chaining.yaml | ${KUBECTL} apply -f - + sleep 1 + + # Curl the channel URL + ./demo-apps/knative-chaining/curl_cmd.sh + + POD_LABEL="apps.sc2.io/name=knative-chaining-three" + + # Wait for pod 3 to be scaled down + until [ "$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} logs -l ${POD_LABEL} | grep 'cloudevent(s3): done!' | wc -l)" = "1" ]; do echo "Waiting for chain to finish..." ; sleep 2; done +} + +run_knative_hello_world() { + local runtime_class="$1" + export SC2_RUNTIME_CLASS="$runtime_class" + + envsubst < ./demo-apps/helloworld-knative/service.yaml | ${KUBECTL} apply -f - + sleep 1 + + # Get the service URL + service_url=$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) + [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] +} + +run_knative_lazy_loading() { + local runtime_class="$1" + export SC2_RUNTIME_CLASS="$runtime_class" + + envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ${KUBECTL} apply -f - + sleep 1 + + # Get the service URL + service_url=$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) + [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] +} + +run_python_hello_world() { + local runtime_class="$1" + export SC2_RUNTIME_CLASS="$runtime_class" + + envsubst < ./demo-apps/helloworld-py/deployment.yaml | ${KUBECTL} apply -f - + + export POD_LABEL="apps.sc2.io/name=helloworld-py" + + # Wait for pod to be ready + until [ "$(${KUBECTL} get pods -n ${SC2_DEMO_NAMESPACE} -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done + sleep 1 + + # Get the pod's IP + service_ip=$(${KUBECTL} get services -n ${SC2_DEMO_NAMESPACE} -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') + [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] +} + +run_python_lazy_loading() { + local runtime_class="$1" + export SC2_RUNTIME_CLASS="$runtime_class" + + envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ${KUBECTL} apply -f - + + export POD_LABEL="apps.sc2.io/name=helloworld-py" + + # Wait for pod to be ready + until [ "$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done + sleep 1 + + # Get the pod's IP + service_ip=$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') + [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] +} diff --git a/vm-cache/src/main.rs b/vm-cache/src/main.rs index 781a6430..0b70ab18 100644 --- a/vm-cache/src/main.rs +++ b/vm-cache/src/main.rs @@ -134,15 +134,22 @@ fn read_pid() -> std::io::Result { fn stop_background_process() -> std::io::Result<()> { if let Ok(pid) = read_pid() { - kill(Pid::from_raw(pid as i32), Signal::SIGTERM).map_err(|e| { - error!("failed to kill process: {e}"); - std::io::Error::new( - std::io::ErrorKind::Other, - "sc2(vm-cache): failed to kill process", - ) - })?; - - info!("stopped background process with PID {pid}"); + match kill(Pid::from_raw(pid as i32), Signal::SIGTERM) { + Ok(_) => { + info!("stopped background process with PID {pid}"); + } + Err(nix::errno::Errno::ESRCH) => { + info!("process with PID {pid} not found (already stopped?)"); + } + Err(e) => { + error!("failed to kill process: {e}"); + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "sc2(vm-cache): failed to kill process", + )); + } + } + std::fs::remove_file(get_pid_file())?; std::fs::remove_file(get_log_file())?; } else {