From 5a5c1ef934139408661ce45cd577e2d3de5aa973 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 11 Feb 2025 18:41:13 +0000 Subject: [PATCH 01/24] tests: move to bats and simplify gha script --- tests/e2e.bats | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/e2e.bats diff --git a/tests/e2e.bats b/tests/e2e.bats new file mode 100644 index 00000000..baabb251 --- /dev/null +++ b/tests/e2e.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats + +# Define an array of test values +VALUES=("one" "two" "three") + +setup() { + if [ -n "$SC2_TEE" ]; then + if [ "$SC2_TEE" == "snp" ]; then + SC2_RUNTIME_CLASSES=("qemu" "qemu-coco-dev" "qemu-snp" "qemu-snp-sc2") + elif [ "$SC2_TEE" == "tdx" ]; then + SC2_RUNTIME_CLASSES=("qemu" "qemu-coco-dev" "qemu-tdx" "qemu-tdx-sc2") + else + echo "ERROR: SC2_TEE env. var must be one in: 'snp', 'tdx'" + exit 1 + fi + else + echo "ERROR: SC2_TEE env. var must be set" + exit 1 + fi + + # Set the TEST_VALUE based on the array and current test number + export SC2_RUNTIME_CLASS="${SC2_RUNTIME_CLASSES[$((bats_test_number - 1))]}" +} + +# TODO: cold/warm starts and guest-pull host-share +@test "Check python hello world" { + # TODO: move a lot to helper functions + echo "Running test for $SC2_RUNTIME_CLASS" + envsubst < ./demo-apps/helloworld-py/deployment.yaml | ./bin/kubectl apply -f - + + export POD_LABEL="apps.sc2.io/name=helloworld-py" + # Wait for pod to be ready + until [ "$(./bin/kubectl get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done + sleep 1 + + # Get the pod's IP + service_ip=$(./bin/kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') + [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] + + envsubst < ./demo-apps/helloworld-py/deployment.yaml | ./bin/kubectl delete -f - + + # Wait for pod to be deleted + ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=30s + # Extra cautionary sleep + sleep 5 + echo "Test for ${runtime_class} (${flavour}) successful!" +} From a41e9623c7c5bdc08753fe227f776085a901f34d Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 12 Feb 2025 19:36:35 +0000 Subject: [PATCH 02/24] more wip --- demo-apps/knative-chaining/chaining.yaml | 26 ++++---- tasks/nydus_snapshotter.py | 9 ++- tests/e2e.bats | 47 -------------- tests/helloworld_knative_guest_pull.bats | 36 +++++++++++ tests/helloworld_knative_host_share.bats | 29 +++++++++ tests/helloworld_python_guest_pull.bats | 36 +++++++++++ tests/helloworld_python_host_share.bats | 29 +++++++++ tests/knative_chaining_guest_pull.bats | 36 +++++++++++ tests/knative_chaining_host_share.bats | 29 +++++++++ tests/utils/env.sh | 23 +++++++ tests/utils/helpers.sh | 80 ++++++++++++++++++++++++ 11 files changed, 320 insertions(+), 60 deletions(-) delete mode 100644 tests/e2e.bats create mode 100644 tests/helloworld_knative_guest_pull.bats create mode 100644 tests/helloworld_knative_host_share.bats create mode 100644 tests/helloworld_python_guest_pull.bats create mode 100644 tests/helloworld_python_host_share.bats create mode 100644 tests/knative_chaining_guest_pull.bats create mode 100644 tests/knative_chaining_host_share.bats create mode 100644 tests/utils/env.sh create mode 100644 tests/utils/helpers.sh diff --git a/demo-apps/knative-chaining/chaining.yaml b/demo-apps/knative-chaining/chaining.yaml index d629a713..54eb0364 100644 --- a/demo-apps/knative-chaining/chaining.yaml +++ b/demo-apps/knative-chaining/chaining.yaml @@ -70,13 +70,14 @@ spec: ports: - containerPort: 8080 command: [ "cargo", "run", "--release" ] + resources: + limits: + memory: "6144Mi" + requests: + memory: "6144Mi" metadata: labels: apps.sc2.io/name: knative-chaining-one - annotations: - # NOTE: we may have to enable this annotation in Kata's config file - # under hypervisor.qemu.enable_annotations (add 'default_memory') - io.katacontainers.config.hypervisor.default_memory: "6144" --- # JobSink guarantees one Job per CloudEvent, satisfying our dynamic scale-up # requirements. However, JobSink's propagate CloudEvents through a volume @@ -104,13 +105,14 @@ spec: env: - name: CE_FROM_FILE value: "on" + resources: + limits: + memory: "6144Mi" + requests: + memory: "6144Mi" metadata: labels: apps.sc2.io/name: knative-chaining-two - annotations: - # NOTE: we may have to enable this annotation in Kata's config file - # under hypervisor.qemu.enable_annotations (add 'default_memory') - io.katacontainers.config.hypervisor.default_memory: "6144" --- # For this last service, we want to give it a high-grace period to make sure # that the same instance processes all of the events @@ -128,14 +130,16 @@ spec: ports: - containerPort: 8080 command: [ "cargo", "run", "--release" ] + resources: + limits: + memory: "6144Mi" + requests: + memory: "6144Mi" metadata: labels: apps.sc2.io/name: knative-chaining-three annotations: autoscaling.knative.dev/scale-to-zero-pod-retention-period: "1m" - # NOTE: we may have to enable this annotation in Kata's config file - # under hypervisor.qemu.enable_annotations (add 'default_memory') - io.katacontainers.config.hypervisor.default_memory: "6144" --- apiVersion: messaging.knative.dev/v1 kind: Subscription diff --git a/tasks/nydus_snapshotter.py b/tasks/nydus_snapshotter.py index f62c6b25..8a16da4c 100644 --- a/tasks/nydus_snapshotter.py +++ b/tasks/nydus_snapshotter.py @@ -3,6 +3,7 @@ from os import getgid, getuid from os.path import exists, join from subprocess import run +from tasks.util.containerd import wait_for_containerd_socket from tasks.util.docker import copy_from_ctr_image, is_ctr_running from tasks.util.env import ( BIN_DIR, @@ -140,13 +141,16 @@ def do_purge(debug=False): # Clear all possibly used images (only images in our registry, or the # pause container images) + tmp_out = "/tmp/cmd_output" cmd = ( "sudo crictl --runtime-endpoint unix:///run/containerd/containerd.sock" - " images -o json" + f" images -o json > {tmp_out} 2> /tmp/jeje" ) rm_cmd = "sudo crictl --runtime-endpoint unix:///run/containerd/containerd.sock rmi" try: - stdout = run(cmd, shell=True, capture_output=True).stdout.decode("utf-8") + run(cmd, shell=True, check=True) + with open(tmp_out, "r") as fh: + stdout = fh.read().strip() data = json_loads(stdout) except JSONDecodeError as e: stderr = run(cmd, shell=True, capture_output=True).stderr.decode("utf-8") @@ -202,6 +206,7 @@ def purge(ctx): """ Remove all cached snapshots in the snapshotter cache """ + wait_for_containerd_socket() do_purge(debug=True) diff --git a/tests/e2e.bats b/tests/e2e.bats deleted file mode 100644 index baabb251..00000000 --- a/tests/e2e.bats +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bats - -# Define an array of test values -VALUES=("one" "two" "three") - -setup() { - if [ -n "$SC2_TEE" ]; then - if [ "$SC2_TEE" == "snp" ]; then - SC2_RUNTIME_CLASSES=("qemu" "qemu-coco-dev" "qemu-snp" "qemu-snp-sc2") - elif [ "$SC2_TEE" == "tdx" ]; then - SC2_RUNTIME_CLASSES=("qemu" "qemu-coco-dev" "qemu-tdx" "qemu-tdx-sc2") - else - echo "ERROR: SC2_TEE env. var must be one in: 'snp', 'tdx'" - exit 1 - fi - else - echo "ERROR: SC2_TEE env. var must be set" - exit 1 - fi - - # Set the TEST_VALUE based on the array and current test number - export SC2_RUNTIME_CLASS="${SC2_RUNTIME_CLASSES[$((bats_test_number - 1))]}" -} - -# TODO: cold/warm starts and guest-pull host-share -@test "Check python hello world" { - # TODO: move a lot to helper functions - echo "Running test for $SC2_RUNTIME_CLASS" - envsubst < ./demo-apps/helloworld-py/deployment.yaml | ./bin/kubectl apply -f - - - export POD_LABEL="apps.sc2.io/name=helloworld-py" - # Wait for pod to be ready - until [ "$(./bin/kubectl get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done - sleep 1 - - # Get the pod's IP - service_ip=$(./bin/kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') - [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] - - envsubst < ./demo-apps/helloworld-py/deployment.yaml | ./bin/kubectl delete -f - - - # Wait for pod to be deleted - ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=30s - # Extra cautionary sleep - sleep 5 - echo "Test for ${runtime_class} (${flavour}) successful!" -} diff --git a/tests/helloworld_knative_guest_pull.bats b/tests/helloworld_knative_guest_pull.bats new file mode 100644 index 00000000..69f4a05d --- /dev/null +++ b/tests/helloworld_knative_guest_pull.bats @@ -0,0 +1,36 @@ +#!/usr/bin/env bats + +source ./tests/utils/env.sh +source ./tests/utils/helpers.sh + +setup_file() { + set_snapshotter_mode "guest-pull" +} + +teardown() { + # Cautionary inter-test sleep + sleep 5 +} + +TEST_NAME="Test knative hello world" +snapshotter="guest-pull" + +# ------------------------------------------------------------------------------ +# Nydus Snapshotter in Guest Pull mode +# ------------------------------------------------------------------------------ + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { + run_knative_hello_world "${SC2_RUNTIME_CLASSES[0]}" +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { + run_knative_hello_world "${SC2_RUNTIME_CLASSES[1]}" +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { + run_knative_hello_world "${SC2_RUNTIME_CLASSES[2]}" +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" +} diff --git a/tests/helloworld_knative_host_share.bats b/tests/helloworld_knative_host_share.bats new file mode 100644 index 00000000..170e7a20 --- /dev/null +++ b/tests/helloworld_knative_host_share.bats @@ -0,0 +1,29 @@ +#!/usr/bin/env bats + +source ./tests/utils/env.sh +source ./tests/utils/helpers.sh + +setup_file() { + set_snapshotter_mode "host-share" +} + +teardown() { + # Cautionary inter-test sleep + sleep 5 +} + +TEST_NAME="Test knative hello world" +snapshotter="host-share" + +# ------------------------------------------------------------------------------ +# Nydus Snapshotter in Host Share mode +# +# Using the snapshotter in host-share mode is only supported for SC2 runtimes, +# as we have only implemented the patches in our forked branches. +# ------------------------------------------------------------------------------ + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + [[ "$SC2_TEE" == "tdx" ]] && skip "Host-share not supported for TDX (#142)" + run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" +} + diff --git a/tests/helloworld_python_guest_pull.bats b/tests/helloworld_python_guest_pull.bats new file mode 100644 index 00000000..33cf885b --- /dev/null +++ b/tests/helloworld_python_guest_pull.bats @@ -0,0 +1,36 @@ +#!/usr/bin/env bats + +source ./tests/utils/env.sh +source ./tests/utils/helpers.sh + +setup_file() { + set_snapshotter_mode "guest-pull" +} + +teardown() { + # Cautionary inter-test sleep + sleep 5 +} + +TEST_NAME="Test python hello world" +snapshotter="guest-pull" + +# ------------------------------------------------------------------------------ +# Nydus Snapshotter in Guest Pull mode +# ------------------------------------------------------------------------------ + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { + run_python_hello_world "${SC2_RUNTIME_CLASSES[0]}" +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { + run_python_hello_world "${SC2_RUNTIME_CLASSES[1]}" +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { + run_python_hello_world "${SC2_RUNTIME_CLASSES[2]}" +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" +} diff --git a/tests/helloworld_python_host_share.bats b/tests/helloworld_python_host_share.bats new file mode 100644 index 00000000..f5ae0af2 --- /dev/null +++ b/tests/helloworld_python_host_share.bats @@ -0,0 +1,29 @@ +#!/usr/bin/env bats + +source ./tests/utils/env.sh +source ./tests/utils/helpers.sh + +setup_file() { + set_snapshotter_mode "host-share" +} + +teardown() { + # Cautionary inter-test sleep + sleep 5 +} + +TEST_NAME="Test python hello world" +snapshotter="host-share" + +# ------------------------------------------------------------------------------ +# Nydus Snapshotter in Host Share mode +# +# Using the snapshotter in host-share mode is only supported for SC2 runtimes, +# as we have only implemented the patches in our forked branches. +# ------------------------------------------------------------------------------ + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + [[ "$SC2_TEE" == "tdx" ]] && skip "Host-share not supported for TDX (#142)" + run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" +} + diff --git a/tests/knative_chaining_guest_pull.bats b/tests/knative_chaining_guest_pull.bats new file mode 100644 index 00000000..d79daf73 --- /dev/null +++ b/tests/knative_chaining_guest_pull.bats @@ -0,0 +1,36 @@ +#!/usr/bin/env bats + +source ./tests/utils/env.sh +source ./tests/utils/helpers.sh + +setup_file() { + set_snapshotter_mode "guest-pull" +} + +teardown() { + # Cautionary inter-test sleep + sleep 5 +} + +TEST_NAME="Test knative chaining" +snapshotter="guest-pull" + +# ------------------------------------------------------------------------------ +# Nydus Snapshotter in Guest Pull mode +# ------------------------------------------------------------------------------ + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { + run_knative_chaining "${SC2_RUNTIME_CLASSES[0]}" +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { + run_knative_chaining "${SC2_RUNTIME_CLASSES[1]}" +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { + run_knative_chaining "${SC2_RUNTIME_CLASSES[2]}" +} + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + run_knative_chaining "${SC2_RUNTIME_CLASSES[3]}" +} diff --git a/tests/knative_chaining_host_share.bats b/tests/knative_chaining_host_share.bats new file mode 100644 index 00000000..19f74a08 --- /dev/null +++ b/tests/knative_chaining_host_share.bats @@ -0,0 +1,29 @@ +#!/usr/bin/env bats + +source ./tests/utils/env.sh +source ./tests/utils/helpers.sh + +setup_file() { + set_snapshotter_mode "host-share" +} + +teardown() { + # Cautionary inter-test sleep + sleep 5 +} + +TEST_NAME="Test knative chaining" +snapshotter="host-share" + +# ------------------------------------------------------------------------------ +# Nydus Snapshotter in Host Share mode +# +# Using the snapshotter in host-share mode is only supported for SC2 runtimes, +# as we have only implemented the patches in our forked branches. +# ------------------------------------------------------------------------------ + +@test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + [[ "$SC2_TEE" == "tdx" ]] && skip "Host-share not supported for TDX (#142)" + run_knative_chaining "${SC2_RUNTIME_CLASSES[3]}" +} + diff --git a/tests/utils/env.sh b/tests/utils/env.sh new file mode 100644 index 00000000..3631db2b --- /dev/null +++ b/tests/utils/env.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +export PROJ_ROOT=${THIS_DIR}/../.. +export KUBECTL=${PROJ_ROOT}/bin/kubectl +export INV=${PROJ_ROOT}/bin/inv_wrapper.sh + +# Work out the runtime classes from the env. var +if [ -n "$SC2_TEE" ]; then + if [ "$SC2_TEE" == "snp" ]; then + export SC2_RUNTIME_CLASSES=("qemu" "qemu-coco-dev" "qemu-snp" "qemu-snp-sc2") + elif [ "$SC2_TEE" == "tdx" ]; then + export SC2_RUNTIME_CLASSES=("qemu" "qemu-coco-dev" "qemu-tdx" "qemu-tdx-sc2") + else + echo "ERROR: SC2_TEE env. var must be one in: 'snp', 'tdx'" + exit 1 + fi +else + echo "ERROR: SC2_TEE env. var must be set" + exit 1 +fi + diff --git a/tests/utils/helpers.sh b/tests/utils/helpers.sh new file mode 100644 index 00000000..1e8950af --- /dev/null +++ b/tests/utils/helpers.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +source ./tests/utils/env.sh + +# ------------------------------------------------------------------------------ +# Helper functions to set-up the environment +# ------------------------------------------------------------------------------ + +set_snapshotter_mode() { + local snapshotter_mode="$1" + ${INV} nydus-snapshotter.set-mode ${snapshotter_mode} + ${INV} nydus-snapshotter.purge + + export SC2_SNAPSHOTTER=${snapshotter_mode} + + sleep 1 +} + +# ------------------------------------------------------------------------------ +# Helper functions to run specific workloads +# ------------------------------------------------------------------------------ + +run_python_hello_world() { + local runtime_class="$1" + export SC2_RUNTIME_CLASS="$runtime_class" + + envsubst < ./demo-apps/helloworld-py/deployment.yaml | ${KUBECTL} apply -f - + + export POD_LABEL="apps.sc2.io/name=helloworld-py" + + # Wait for pod to be ready + until [ "$(${KUBECTL} get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done + sleep 1 + + # Get the pod's IP + service_ip=$(${KUBECTL} get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') + [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] + + envsubst < ./demo-apps/helloworld-py/deployment.yaml | ${KUBECTL} delete -f - + + # Wait for pod to be deleted + ${KUBECTL} wait --for=delete -l ${POD_LABEL} pod --timeout=30s +} + +run_knative_hello_world() { + local runtime_class="$1" + export SC2_RUNTIME_CLASS="$runtime_class" + + envsubst < ./demo-apps/helloworld-knative/service.yaml | ${KUBECTL} apply -f - + sleep 1 + + # Get the service URL + service_url=$(${KUBECTL} get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) + [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] + + # Wait for pod to be deleted + POD_LABEL="apps.sc2.io/name=helloworld-py" + envsubst < ./demo-apps/helloworld-knative/service.yaml | ${KUBECTL} delete -f - + ${KUBECTL} wait --for=delete -l ${POD_LABEL} pod --timeout=60s +} + +run_knative_chaining() { + local runtime_class="$1" + export SC2_RUNTIME_CLASS="$runtime_class" + + envsubst < ./demo-apps/knative-chaining/chaining.yaml | ${KUBECTL} apply -f - + sleep 1 + + # Curl the channel URL + ./demo-apps/knative-chaining/curl_cmd.sh + + NAMESPACE="chaining-test" + POD_LABEL="apps.sc2.io/name=knative-chaining-three" + + # Wait for pod 3 to be scaled down + until [ "$(${KUBECTL} -n ${NAMESPACE} logs -l ${POD_LABEL_THREE} | grep 'cloudevent(s3): done!' | wc -l)" = "1" ]; do echo "Waiting for chain to finish..."; sleep 2; done + + # Finally, clear-up + envsubst < ./demo-apps/knative-chaining/chaining.yaml | ${KUBECTL} delete -f - +} From 7bd1a0a8b9f759f801199d9834b3b287e8f20869 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 13 Feb 2025 10:19:43 +0000 Subject: [PATCH 03/24] tests: finish adding test scripts --- bin/workon.sh | 8 +- demo-apps/knative-chaining/chaining.yaml | 26 +++---- tests/cold_starts_guest_pull.bats | 60 ++++++++++++++ tests/cold_starts_host_share.bats | 36 +++++++++ tests/knative_chaining_guest_pull.bats | 9 +++ tests/knative_chaining_host_share.bats | 10 +++ tests/lazy_loading.bats | 23 ++++++ tests/utils/env.sh | 2 + tests/utils/helpers.sh | 99 ++++++++++++++++++------ 9 files changed, 231 insertions(+), 42 deletions(-) create mode 100644 tests/cold_starts_guest_pull.bats create mode 100644 tests/cold_starts_host_share.bats create mode 100644 tests/lazy_loading.bats diff --git a/bin/workon.sh b/bin/workon.sh index 7a780b64..994bbdf5 100644 --- a/bin/workon.sh +++ b/bin/workon.sh @@ -39,13 +39,13 @@ TEE_DETECT_BINARY=${TEE_DETECT_ROOT}/target/release/tee-detect cargo build -q --release --manifest-path ${TEE_DETECT_ROOT}/Cargo.toml if "${TEE_DETECT_BINARY}" snp; then - TEE=snp + export SC2_TEE=snp export SC2_RUNTIME_CLASS=qemu-snp-sc2 elif "${TEE_DETECT_BINARY}" tdx; then - TEE=tdx + export SC2_TEE=tdx export SC2_RUNTIME_CLASS=qemu-tdx-sc2 else - TEE=none + export SC2_TEE=none echo "sc2-deploy: WARN: neither SNP nor TDX is enabled" fi @@ -75,7 +75,7 @@ echo "" echo "----------------------------------" echo "CLI for SC2 Deployment Scripts" echo "CoCo Version: ${COCO_VERSION}" -echo "TEE: ${TEE}" +echo "TEE: ${SC2_TEE}" echo "----------------------------------" echo "" diff --git a/demo-apps/knative-chaining/chaining.yaml b/demo-apps/knative-chaining/chaining.yaml index 54eb0364..d629a713 100644 --- a/demo-apps/knative-chaining/chaining.yaml +++ b/demo-apps/knative-chaining/chaining.yaml @@ -70,14 +70,13 @@ spec: ports: - containerPort: 8080 command: [ "cargo", "run", "--release" ] - resources: - limits: - memory: "6144Mi" - requests: - memory: "6144Mi" metadata: labels: apps.sc2.io/name: knative-chaining-one + annotations: + # NOTE: we may have to enable this annotation in Kata's config file + # under hypervisor.qemu.enable_annotations (add 'default_memory') + io.katacontainers.config.hypervisor.default_memory: "6144" --- # JobSink guarantees one Job per CloudEvent, satisfying our dynamic scale-up # requirements. However, JobSink's propagate CloudEvents through a volume @@ -105,14 +104,13 @@ spec: env: - name: CE_FROM_FILE value: "on" - resources: - limits: - memory: "6144Mi" - requests: - memory: "6144Mi" metadata: labels: apps.sc2.io/name: knative-chaining-two + annotations: + # NOTE: we may have to enable this annotation in Kata's config file + # under hypervisor.qemu.enable_annotations (add 'default_memory') + io.katacontainers.config.hypervisor.default_memory: "6144" --- # For this last service, we want to give it a high-grace period to make sure # that the same instance processes all of the events @@ -130,16 +128,14 @@ spec: ports: - containerPort: 8080 command: [ "cargo", "run", "--release" ] - resources: - limits: - memory: "6144Mi" - requests: - memory: "6144Mi" metadata: labels: apps.sc2.io/name: knative-chaining-three annotations: autoscaling.knative.dev/scale-to-zero-pod-retention-period: "1m" + # NOTE: we may have to enable this annotation in Kata's config file + # under hypervisor.qemu.enable_annotations (add 'default_memory') + io.katacontainers.config.hypervisor.default_memory: "6144" --- apiVersion: messaging.knative.dev/v1 kind: Subscription diff --git a/tests/cold_starts_guest_pull.bats b/tests/cold_starts_guest_pull.bats new file mode 100644 index 00000000..4680524c --- /dev/null +++ b/tests/cold_starts_guest_pull.bats @@ -0,0 +1,60 @@ +#!/usr/bin/env bats + +source ./tests/utils/env.sh +source ./tests/utils/helpers.sh + +setup_file() { + set_snapshotter_mode "guest-pull" +} + +# Make sure we purge before each test so that we have a cold start +setup() { + ${INV} nydus-snapshotter.purge +} + +teardown() { + # Cautionary inter-test sleep + sleep 5 +} + +snapshotter="guest-pull" + +# ------------------------------------------------------------------------------ +# Python cold starts +# ------------------------------------------------------------------------------ + +@test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { + run_python_hello_world "${SC2_RUNTIME_CLASSES[0]}" +} + +@test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { + run_python_hello_world "${SC2_RUNTIME_CLASSES[1]}" +} + +@test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { + run_python_hello_world "${SC2_RUNTIME_CLASSES[2]}" +} + +@test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" +} + +# ------------------------------------------------------------------------------ +# Knative cold starts +# ------------------------------------------------------------------------------ + +@test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { + run_knative_hello_world "${SC2_RUNTIME_CLASSES[0]}" +} + +@test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { + run_knative_hello_world "${SC2_RUNTIME_CLASSES[1]}" +} + +@test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { + run_knative_hello_world "${SC2_RUNTIME_CLASSES[2]}" +} + +@test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" +} diff --git a/tests/cold_starts_host_share.bats b/tests/cold_starts_host_share.bats new file mode 100644 index 00000000..f30802d1 --- /dev/null +++ b/tests/cold_starts_host_share.bats @@ -0,0 +1,36 @@ +#!/usr/bin/env bats + +source ./tests/utils/env.sh +source ./tests/utils/helpers.sh + +setup_file() { + set_snapshotter_mode "host-share" +} + +# Make sure we purge before each test so that we have a cold start +setup() { + ${INV} nydus-snapshotter.purge +} + +teardown() { + # Cautionary inter-test sleep + sleep 5 +} + +snapshotter="host-share" + +# ------------------------------------------------------------------------------ +# Python cold starts +# ------------------------------------------------------------------------------ + +@test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" +} + +# ------------------------------------------------------------------------------ +# Knative cold starts +# ------------------------------------------------------------------------------ + +@test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" +} diff --git a/tests/knative_chaining_guest_pull.bats b/tests/knative_chaining_guest_pull.bats index d79daf73..aa05dd9c 100644 --- a/tests/knative_chaining_guest_pull.bats +++ b/tests/knative_chaining_guest_pull.bats @@ -5,6 +5,15 @@ source ./tests/utils/helpers.sh setup_file() { set_snapshotter_mode "guest-pull" + + # The chaining tests need more memory and we set it using the + # default_memory annotation + enable_kata_annotation "default_memory" + restart_vm_cache + + # May have to fetch content here + k8s_content_fetch ${PAUSE_IMAGE} + k8s_content_fetch ${SIDECAR_IMAGE} } teardown() { diff --git a/tests/knative_chaining_host_share.bats b/tests/knative_chaining_host_share.bats index 19f74a08..dfae7971 100644 --- a/tests/knative_chaining_host_share.bats +++ b/tests/knative_chaining_host_share.bats @@ -5,6 +5,15 @@ source ./tests/utils/helpers.sh setup_file() { set_snapshotter_mode "host-share" + + # The chaining tests need more memory and we set it using the + # default_memory annotation + enable_kata_annotation "default_memory" + restart_vm_cache + + # May have to fetch content here + k8s_content_fetch ${PAUSE_IMAGE} + k8s_content_fetch ${SIDECAR_IMAGE} } teardown() { @@ -24,6 +33,7 @@ snapshotter="host-share" @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { [[ "$SC2_TEE" == "tdx" ]] && skip "Host-share not supported for TDX (#142)" + skip "Knative chaining with host-share not supported (#145)" run_knative_chaining "${SC2_RUNTIME_CLASSES[3]}" } diff --git a/tests/lazy_loading.bats b/tests/lazy_loading.bats new file mode 100644 index 00000000..045ec479 --- /dev/null +++ b/tests/lazy_loading.bats @@ -0,0 +1,23 @@ +#!/usr/bin/env bats + +source ./tests/utils/env.sh +source ./tests/utils/helpers.sh + +setup_file() { + set_snapshotter_mode "guest-pull" +} + +teardown() { + # Cautionary inter-test sleep + sleep 5 +} + +snapshotter="guest-pull" + +@test "Test python lazy loading: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + run_python_lazy_loading "${SC2_RUNTIME_CLASSES[3]}" +} + +@test "Test knative lazy loading: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + run_knative_lazy_loading "${SC2_RUNTIME_CLASSES[3]}" +} diff --git a/tests/utils/env.sh b/tests/utils/env.sh index 3631db2b..0c5aeb6e 100644 --- a/tests/utils/env.sh +++ b/tests/utils/env.sh @@ -21,3 +21,5 @@ else exit 1 fi +PAUSE_IMAGE="registry.k8s.io/pause:3.8" +SIDECAR_IMAGE="sc2cr.io/system/knative-sidecar@sha256:79d5f6031f308cee209c4c32eeab9113b29a1ed4096c5d657504096734ca3b1d" diff --git a/tests/utils/helpers.sh b/tests/utils/helpers.sh index 1e8950af..370aca47 100644 --- a/tests/utils/helpers.sh +++ b/tests/utils/helpers.sh @@ -6,6 +6,20 @@ source ./tests/utils/env.sh # Helper functions to set-up the environment # ------------------------------------------------------------------------------ +enable_kata_annotation() { + local annotation="$1" + ${INV} kata.enable-annotation ${annotation} --runtime "qemu-${SC2_TEE}-sc2" +} + +k8s_content_fetch() { + local image=$1 + sudo ctr -n k8s.io content fetch -k ${image} +} + +restart_vm_cache() { + sudo -E ${PROJ_ROOT}/vm-cache/target/release/vm-cache restart +} + set_snapshotter_mode() { local snapshotter_mode="$1" ${INV} nydus-snapshotter.set-mode ${snapshotter_mode} @@ -20,26 +34,24 @@ set_snapshotter_mode() { # Helper functions to run specific workloads # ------------------------------------------------------------------------------ -run_python_hello_world() { +run_knative_chaining() { local runtime_class="$1" export SC2_RUNTIME_CLASS="$runtime_class" - envsubst < ./demo-apps/helloworld-py/deployment.yaml | ${KUBECTL} apply -f - - - export POD_LABEL="apps.sc2.io/name=helloworld-py" - - # Wait for pod to be ready - until [ "$(${KUBECTL} get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done + envsubst < ./demo-apps/knative-chaining/chaining.yaml | ${KUBECTL} apply -f - sleep 1 - # Get the pod's IP - service_ip=$(${KUBECTL} get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') - [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] + # Curl the channel URL + ./demo-apps/knative-chaining/curl_cmd.sh - envsubst < ./demo-apps/helloworld-py/deployment.yaml | ${KUBECTL} delete -f - + NAMESPACE="chaining-test" + POD_LABEL="apps.sc2.io/name=knative-chaining-three" - # Wait for pod to be deleted - ${KUBECTL} wait --for=delete -l ${POD_LABEL} pod --timeout=30s + # Wait for pod 3 to be scaled down + until [ "$(${KUBECTL} -n ${NAMESPACE} logs -l ${POD_LABEL} | grep 'cloudevent(s3): done!' | wc -l)" = "1" ]; do echo "Waiting for chain to finish..."; sleep 2; done + + # Finally, clear-up + ${KUBECTL} delete namespace ${NAMESPACE} } run_knative_hello_world() { @@ -59,22 +71,63 @@ run_knative_hello_world() { ${KUBECTL} wait --for=delete -l ${POD_LABEL} pod --timeout=60s } -run_knative_chaining() { +run_knative_lazy_loading() { local runtime_class="$1" export SC2_RUNTIME_CLASS="$runtime_class" - envsubst < ./demo-apps/knative-chaining/chaining.yaml | ${KUBECTL} apply -f - + envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ${KUBECTL} apply -f - sleep 1 - # Curl the channel URL - ./demo-apps/knative-chaining/curl_cmd.sh + # Get the service URL + service_url=$(${KUBECTL} get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) + [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] - NAMESPACE="chaining-test" - POD_LABEL="apps.sc2.io/name=knative-chaining-three" + # Wait for pod to be deleted + POD_LABEL="apps.sc2.io/name=helloworld-py" + envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ${KUBECTL} delete -f - + ${KUBECTL} wait --for=delete -l ${POD_LABEL} pod --timeout=60s +} - # Wait for pod 3 to be scaled down - until [ "$(${KUBECTL} -n ${NAMESPACE} logs -l ${POD_LABEL_THREE} | grep 'cloudevent(s3): done!' | wc -l)" = "1" ]; do echo "Waiting for chain to finish..."; sleep 2; done +run_python_hello_world() { + local runtime_class="$1" + export SC2_RUNTIME_CLASS="$runtime_class" - # Finally, clear-up - envsubst < ./demo-apps/knative-chaining/chaining.yaml | ${KUBECTL} delete -f - + envsubst < ./demo-apps/helloworld-py/deployment.yaml | ${KUBECTL} apply -f - + + export POD_LABEL="apps.sc2.io/name=helloworld-py" + + # Wait for pod to be ready + until [ "$(${KUBECTL} get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done + sleep 1 + + # Get the pod's IP + service_ip=$(${KUBECTL} get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') + [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] + + envsubst < ./demo-apps/helloworld-py/deployment.yaml | ${KUBECTL} delete -f - + + # Wait for pod to be deleted + ${KUBECTL} wait --for=delete -l ${POD_LABEL} pod --timeout=30s +} + +run_python_lazy_loading() { + local runtime_class="$1" + export SC2_RUNTIME_CLASS="$runtime_class" + + envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ${KUBECTL} apply -f - + + export POD_LABEL="apps.sc2.io/name=helloworld-py" + + # Wait for pod to be ready + until [ "$(${KUBECTL} get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done + sleep 1 + + # Get the pod's IP + service_ip=$(${KUBECTL} get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') + [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] + + envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ${KUBECTL} delete -f - + + # Wait for pod to be deleted + ${KUBECTL} wait --for=delete -l ${POD_LABEL} pod --timeout=30s } From 1788037b1a883b740b49003e9a1372f86a045845 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 13 Feb 2025 10:21:55 +0000 Subject: [PATCH 04/24] gha: move to using bats --- .github/workflows/tests.yml | 208 +----------------------------------- 1 file changed, 4 insertions(+), 204 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5fba7e21..38baa745 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -69,211 +69,11 @@ jobs: - name: "Install SC2" run: ./bin/inv_wrapper.sh sc2.deploy --clean - - name: "Run python hello world (cold and warm starts)" - run: | - for runtime_class in ${{ matrix.runtime_classes }}; do - for flavour in "cold" "warm"; do - echo "Running test for ${runtime_class} (${flavour})..." - - # If running a cold-start test, make sure to remove the image first - if [ "$flavour" == "cold" ]; then - sudo crictl rmi $(sudo crictl --runtime-endpoint unix:///run/containerd/containerd.sock images | grep sc2cr.io/applications/helloworld-py | awk '{print $3}') > /dev/null 2>&1 - fi - - # Deploy python pod - export SC2_RUNTIME_CLASS=${runtime_class} - envsubst < ./demo-apps/helloworld-py/deployment.yaml | ./bin/kubectl apply -f - - - # Wait for pod to be ready - until [ "$(./bin/kubectl get pods -l ${{ env.POD_LABEL }} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done - sleep 1 - - # Get the pod's IP - service_ip=$(./bin/kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') - [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] - envsubst < ./demo-apps/helloworld-py/deployment.yaml | ./bin/kubectl delete -f - - - # Wait for pod to be deleted - ./bin/kubectl wait --for=delete -l ${{ env.POD_LABEL }} pod --timeout=30s - # Extra cautionary sleep - sleep 5 - echo "Test for ${runtime_class} (${flavour}) successful!" - done - done - env: - POD_LABEL: apps.sc2.io/name=helloworld-py - - - name: "Fetch content (see #130)" - run: sudo ctr -n k8s.io content fetch -k sc2cr.io/applications/helloworld-py:unencrypted - - - name: "Run Knative hello world" - run: | - for runtime_class in ${{ matrix.runtime_classes }}; do - echo "Running test for ${runtime_class}..." - export SC2_RUNTIME_CLASS=${runtime_class} - envsubst < ./demo-apps/helloworld-knative/service.yaml | ./bin/kubectl apply -f - - sleep 1 - - # Get the service URL - service_url=$(./bin/kubectl get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) - [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] - - # Wait for pod to be deleted - envsubst < ./demo-apps/helloworld-knative/service.yaml | ./bin/kubectl delete -f - - ./bin/kubectl wait --for=delete -l ${{ env.POD_LABEL }} pod --timeout=60s - - # Extra cautionary sleep - sleep 5 - echo "Test for ${runtime_class} successful!" - done - env: - POD_LABEL: apps.sc2.io/name=helloworld-py + - name: "Install BATS" + run: sudo apt install -y bats - - name: "Run nydus lazy guest-pulling test" - run: | - export SC2_RUNTIME_CLASS=qemu-${{ matrix.tee }}-sc2 - export POD_LABEL="apps.sc2.io/name=helloworld-py" - - # ----- Python Test ---- - - echo "Running python test..." - envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ./bin/kubectl apply -f - - - # Wait for pod to be ready - until [ "$(./bin/kubectl get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done - sleep 1 - - # Get the pod's IP - service_ip=$(./bin/kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') - [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] - envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ./bin/kubectl delete -f - - - # Wait for pod to be deleted - ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=30s - - # Extra cautionary sleep - sleep 5 - echo "Python test succesful!" - - # ----- Knative Test ---- - envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ./bin/kubectl apply -f - - sleep 1 - - # Get the service URL - service_url=$(./bin/kubectl get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) - [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] - - # Wait for pod to be deleted - envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ./bin/kubectl delete -f - - ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=60s - - # Extra cautionary sleep - sleep 5 - echo "Knative test succesful!" - - - name: "Run nydus host-share test" - # Host-share mechanisms seem not to work with TDX - if: ${{ matrix.tee != 'tdx' }} - run: | - # Change the snapshotter mode and purge (necessary to clear - # containred's content store) - ./bin/inv_wrapper.sh nydus-snapshotter.set-mode host-share - sleep 2 - ./bin/inv_wrapper.sh nydus-snapshotter.purge - - export SC2_RUNTIME_CLASS=qemu-${{ matrix.tee }}-sc2 - export POD_LABEL="apps.sc2.io/name=helloworld-py" - - # When updating the runtime we update all the config files, so we - # need to re-start the cache - sudo -E ./vm-cache/target/release/vm-cache restart - - # ----- Python Test ---- - - echo "Running python test..." - envsubst < ./demo-apps/helloworld-py/deployment.yaml | ./bin/kubectl apply -f - - - # Wait for pod to be ready - until [ "$(./bin/kubectl get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done - sleep 1 - - # Get the pod's IP - service_ip=$(./bin/kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') - [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] - envsubst < ./demo-apps/helloworld-py/deployment.yaml | ./bin/kubectl delete -f - - - # Wait for pod to be deleted - ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=30s - - # Extra cautionary sleep - sleep 5 - echo "Python test succesful!" - - # ----- Knative Test ---- - envsubst < ./demo-apps/helloworld-knative/service.yaml | ./bin/kubectl apply -f - - sleep 1 - - # Get the service URL - service_url=$(./bin/kubectl get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) - [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] - - # Wait for pod to be deleted - envsubst < ./demo-apps/helloworld-knative/service.yaml | ./bin/kubectl delete -f - - ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=60s - - # Extra cautionary sleep - sleep 5 - echo "Knative test succesful!" - - # Change the snapshotter mode back again (and purge) - # - ./bin/inv_wrapper.sh nydus-snapshotter.set-mode guest-pull - sleep 2 - ./bin/inv_wrapper.sh nydus-snapshotter.purge - - - name: "Enable default-memory annotation" - run: | - for runtime_class in ${{ matrix.runtime_classes }}; do - ./bin/inv_wrapper.sh kata.enable-annotation default_memory --runtime ${runtime_class} - # Here we benefit that the last variable is the one we want to use - # for vm-cache - export SC2_RUNTIME_CLASS=${runtime_class} - done - - # Aftre changing the annotation of the qemu-snp-sc2 runtime class we - # need to restart the VM cache - sudo -E ./vm-cache/target/release/vm-cache restart - - - name: "Fetch content (see #130)" - run: | - sudo ctr -n k8s.io content fetch -k sc2cr.io/system/knative-sidecar@sha256:79d5f6031f308cee209c4c32eeab9113b29a1ed4096c5d657504096734ca3b1d - sudo ctr -n k8s.io content fetch registry.k8s.io/pause:3.8 - - - name: "Run knative chaining demo" - run: | - for runtime_class in ${{ matrix.runtime_classes }}; do - echo "Running test for ${runtime_class}..." - export SC2_RUNTIME_CLASS=${runtime_class} - envsubst < ./demo-apps/knative-chaining/chaining.yaml | ./bin/kubectl apply -f - - sleep 1 - - # Curl the channel URL - ./demo-apps/knative-chaining/curl_cmd.sh - - # Wait for pod 3 to be scaled down - until [ "$(kubectl -n ${{ env.NAMESPACE }} logs -l ${{ env.POD_LABEL_THREE }} | grep 'cloudevent(s3): done!' | wc -l)" = "1" ]; do echo "Waiting for chain to finish..."; sleep 2; done - - # Finally, clear-up - envsubst < ./demo-apps/knative-chaining/chaining.yaml | ./bin/kubectl delete -f - - - # Extra cautionary sleep - sleep 5 - echo "Test for ${runtime_class} successful!" - done - env: - NAMESPACE: chaining-test - POD_LABEL_ONE: apps.sc2.io/name=knative-chaining-one - POD_LABEL_THREE: apps.sc2.io/name=knative-chaining-three + - name: "Run integration tests" + run: bats ./tests - name: "Clean-up" if: always() From 26f736865888debfef30b77f5e073b24c85fb99a Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 13 Feb 2025 10:25:59 +0000 Subject: [PATCH 05/24] nits: self-review --- tasks/nydus_snapshotter.py | 2 +- tests/cold_starts_guest_pull.bats | 2 +- tests/cold_starts_host_share.bats | 2 +- tests/helloworld_knative_guest_pull.bats | 2 +- tests/helloworld_knative_host_share.bats | 2 +- tests/helloworld_python_guest_pull.bats | 2 +- tests/helloworld_python_host_share.bats | 2 +- tests/knative_chaining_guest_pull.bats | 2 +- tests/knative_chaining_host_share.bats | 2 +- tests/lazy_loading.bats | 2 +- tests/utils/env.sh | 4 ++++ 11 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tasks/nydus_snapshotter.py b/tasks/nydus_snapshotter.py index 8a16da4c..9d6516a6 100644 --- a/tasks/nydus_snapshotter.py +++ b/tasks/nydus_snapshotter.py @@ -144,7 +144,7 @@ def do_purge(debug=False): tmp_out = "/tmp/cmd_output" cmd = ( "sudo crictl --runtime-endpoint unix:///run/containerd/containerd.sock" - f" images -o json > {tmp_out} 2> /tmp/jeje" + f" images -o json > {tmp_out}" ) rm_cmd = "sudo crictl --runtime-endpoint unix:///run/containerd/containerd.sock rmi" try: diff --git a/tests/cold_starts_guest_pull.bats b/tests/cold_starts_guest_pull.bats index 4680524c..15465bf1 100644 --- a/tests/cold_starts_guest_pull.bats +++ b/tests/cold_starts_guest_pull.bats @@ -14,7 +14,7 @@ setup() { teardown() { # Cautionary inter-test sleep - sleep 5 + sleep ${INTERTEST_SLEEP_SECS} } snapshotter="guest-pull" diff --git a/tests/cold_starts_host_share.bats b/tests/cold_starts_host_share.bats index f30802d1..6ec4c189 100644 --- a/tests/cold_starts_host_share.bats +++ b/tests/cold_starts_host_share.bats @@ -14,7 +14,7 @@ setup() { teardown() { # Cautionary inter-test sleep - sleep 5 + sleep ${INTERTEST_SLEEP_SECS} } snapshotter="host-share" diff --git a/tests/helloworld_knative_guest_pull.bats b/tests/helloworld_knative_guest_pull.bats index 69f4a05d..20695520 100644 --- a/tests/helloworld_knative_guest_pull.bats +++ b/tests/helloworld_knative_guest_pull.bats @@ -9,7 +9,7 @@ setup_file() { teardown() { # Cautionary inter-test sleep - sleep 5 + sleep ${INTERTEST_SLEEP_SECS} } TEST_NAME="Test knative hello world" diff --git a/tests/helloworld_knative_host_share.bats b/tests/helloworld_knative_host_share.bats index 170e7a20..321247a1 100644 --- a/tests/helloworld_knative_host_share.bats +++ b/tests/helloworld_knative_host_share.bats @@ -9,7 +9,7 @@ setup_file() { teardown() { # Cautionary inter-test sleep - sleep 5 + sleep ${INTERTEST_SLEEP_SECS} } TEST_NAME="Test knative hello world" diff --git a/tests/helloworld_python_guest_pull.bats b/tests/helloworld_python_guest_pull.bats index 33cf885b..b5893f65 100644 --- a/tests/helloworld_python_guest_pull.bats +++ b/tests/helloworld_python_guest_pull.bats @@ -9,7 +9,7 @@ setup_file() { teardown() { # Cautionary inter-test sleep - sleep 5 + sleep ${INTERTEST_SLEEP_SECS} } TEST_NAME="Test python hello world" diff --git a/tests/helloworld_python_host_share.bats b/tests/helloworld_python_host_share.bats index f5ae0af2..73a8949e 100644 --- a/tests/helloworld_python_host_share.bats +++ b/tests/helloworld_python_host_share.bats @@ -9,7 +9,7 @@ setup_file() { teardown() { # Cautionary inter-test sleep - sleep 5 + sleep ${INTERTEST_SLEEP_SECS} } TEST_NAME="Test python hello world" diff --git a/tests/knative_chaining_guest_pull.bats b/tests/knative_chaining_guest_pull.bats index aa05dd9c..1d031386 100644 --- a/tests/knative_chaining_guest_pull.bats +++ b/tests/knative_chaining_guest_pull.bats @@ -18,7 +18,7 @@ setup_file() { teardown() { # Cautionary inter-test sleep - sleep 5 + sleep ${INTERTEST_SLEEP_SECS} } TEST_NAME="Test knative chaining" diff --git a/tests/knative_chaining_host_share.bats b/tests/knative_chaining_host_share.bats index dfae7971..c57386fe 100644 --- a/tests/knative_chaining_host_share.bats +++ b/tests/knative_chaining_host_share.bats @@ -18,7 +18,7 @@ setup_file() { teardown() { # Cautionary inter-test sleep - sleep 5 + sleep ${INTERTEST_SLEEP_SECS} } TEST_NAME="Test knative chaining" diff --git a/tests/lazy_loading.bats b/tests/lazy_loading.bats index 045ec479..b9ca4af4 100644 --- a/tests/lazy_loading.bats +++ b/tests/lazy_loading.bats @@ -9,7 +9,7 @@ setup_file() { teardown() { # Cautionary inter-test sleep - sleep 5 + sleep ${INTERTEST_SLEEP_SECS} } snapshotter="guest-pull" diff --git a/tests/utils/env.sh b/tests/utils/env.sh index 0c5aeb6e..2c74c3aa 100644 --- a/tests/utils/env.sh +++ b/tests/utils/env.sh @@ -21,5 +21,9 @@ else exit 1 fi +# Useful constants +INTERTEST_SLEEP_SECS=5 + +# Images PAUSE_IMAGE="registry.k8s.io/pause:3.8" SIDECAR_IMAGE="sc2cr.io/system/knative-sidecar@sha256:79d5f6031f308cee209c4c32eeab9113b29a1ed4096c5d657504096734ca3b1d" From f988a2017c10c78bbe0c68d8ff35d6cd2a476fe9 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 13 Feb 2025 10:45:08 +0000 Subject: [PATCH 06/24] gha: make installation non-interactive --- .github/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 38baa745..1fbafc22 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -70,7 +70,9 @@ jobs: run: ./bin/inv_wrapper.sh sc2.deploy --clean - name: "Install BATS" - run: sudo apt install -y bats + run: sudo -E apt install -y bats + env: + DEBIAN_FRONTEND: noninteractive - name: "Run integration tests" run: bats ./tests From 7d46b0eb6f5faba1d1be24284ab0c0ae8635eea0 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 13 Feb 2025 10:55:47 +0000 Subject: [PATCH 07/24] ns: devnull stderr --- tasks/nydus_snapshotter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/nydus_snapshotter.py b/tasks/nydus_snapshotter.py index 9d6516a6..7abd96f0 100644 --- a/tasks/nydus_snapshotter.py +++ b/tasks/nydus_snapshotter.py @@ -144,7 +144,7 @@ def do_purge(debug=False): tmp_out = "/tmp/cmd_output" cmd = ( "sudo crictl --runtime-endpoint unix:///run/containerd/containerd.sock" - f" images -o json > {tmp_out}" + f" images -o json > {tmp_out} 2> /dev/null" ) rm_cmd = "sudo crictl --runtime-endpoint unix:///run/containerd/containerd.sock rmi" try: From 56384a85defd0907b5bf482c7383459904f3200e Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 13 Feb 2025 13:10:29 +0000 Subject: [PATCH 08/24] gha: set env. var --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1fbafc22..1b3f3ae8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -76,6 +76,8 @@ jobs: - name: "Run integration tests" run: bats ./tests + env: + SC2_TEE: ${{ matrix.tee }} - name: "Clean-up" if: always() From 7fe6a7d03363e62b8ba02903a192f3fd4eb8ccf3 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 17 Feb 2025 17:34:26 +0000 Subject: [PATCH 09/24] tests: fix setting annotation in chaining test --- tests/knative_chaining_guest_pull.bats | 14 +++++++++----- tests/utils/helpers.sh | 3 ++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/knative_chaining_guest_pull.bats b/tests/knative_chaining_guest_pull.bats index 1d031386..b1257b57 100644 --- a/tests/knative_chaining_guest_pull.bats +++ b/tests/knative_chaining_guest_pull.bats @@ -6,11 +6,6 @@ source ./tests/utils/helpers.sh setup_file() { set_snapshotter_mode "guest-pull" - # The chaining tests need more memory and we set it using the - # default_memory annotation - enable_kata_annotation "default_memory" - restart_vm_cache - # May have to fetch content here k8s_content_fetch ${PAUSE_IMAGE} k8s_content_fetch ${SIDECAR_IMAGE} @@ -29,17 +24,26 @@ snapshotter="guest-pull" # ------------------------------------------------------------------------------ @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { + enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[0]}" + run_knative_chaining "${SC2_RUNTIME_CLASSES[0]}" } @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { + enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[1]}" + run_knative_chaining "${SC2_RUNTIME_CLASSES[1]}" } @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { + enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[2]}" + run_knative_chaining "${SC2_RUNTIME_CLASSES[2]}" } @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[3]}" + restart_vm_cache + run_knative_chaining "${SC2_RUNTIME_CLASSES[3]}" } diff --git a/tests/utils/helpers.sh b/tests/utils/helpers.sh index 370aca47..7a822d80 100644 --- a/tests/utils/helpers.sh +++ b/tests/utils/helpers.sh @@ -8,7 +8,8 @@ source ./tests/utils/env.sh enable_kata_annotation() { local annotation="$1" - ${INV} kata.enable-annotation ${annotation} --runtime "qemu-${SC2_TEE}-sc2" + local runtime="$2" + ${INV} kata.enable-annotation ${annotation} --runtime ${runtime} } k8s_content_fetch() { From e116e03d33fe7844f318b56e3383ee1c12331783 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 17 Feb 2025 18:04:22 +0000 Subject: [PATCH 10/24] tests: reset vm cache when setting the snapshotter mode --- tests/utils/helpers.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/utils/helpers.sh b/tests/utils/helpers.sh index 7a822d80..d7b0d67d 100644 --- a/tests/utils/helpers.sh +++ b/tests/utils/helpers.sh @@ -29,6 +29,10 @@ set_snapshotter_mode() { export SC2_SNAPSHOTTER=${snapshotter_mode} sleep 1 + + # Setting the snapshotter mode changes the config file, so we must restart + # the vm cache + restart_vm_cache } # ------------------------------------------------------------------------------ From 4f54fe2e17d99d1dab447a6c4f40177069e4d233 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 17 Feb 2025 18:23:49 +0000 Subject: [PATCH 11/24] tests: set right env. var when restarting vm cache --- tests/utils/helpers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/helpers.sh b/tests/utils/helpers.sh index d7b0d67d..c7d224a8 100644 --- a/tests/utils/helpers.sh +++ b/tests/utils/helpers.sh @@ -18,7 +18,7 @@ k8s_content_fetch() { } restart_vm_cache() { - sudo -E ${PROJ_ROOT}/vm-cache/target/release/vm-cache restart + sudo SC2_RUNTIME_CLASS="qemu-${SC2_TEE}-sc2" ${PROJ_ROOT}/vm-cache/target/release/vm-cache restart } set_snapshotter_mode() { From ae4ade2e2b5ae53205f5f4d31e1b08733b51775f Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 18 Feb 2025 10:38:25 +0000 Subject: [PATCH 12/24] tests: fix env. vars when restarting vm-cache --- tests/utils/helpers.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/utils/helpers.sh b/tests/utils/helpers.sh index c7d224a8..3689555d 100644 --- a/tests/utils/helpers.sh +++ b/tests/utils/helpers.sh @@ -18,7 +18,11 @@ k8s_content_fetch() { } restart_vm_cache() { - sudo SC2_RUNTIME_CLASS="qemu-${SC2_TEE}-sc2" ${PROJ_ROOT}/vm-cache/target/release/vm-cache restart + export SC2_RUNTIME_CLASS="qemu-${SC2_TEE}-sc2" + + # Need to pass the environment because the vm-cache script not only uses + # the SC2_RUNTIME_CLASS env. var, but also others like $HOME or $USER + sudo -E ${PROJ_ROOT}/vm-cache/target/release/vm-cache restart } set_snapshotter_mode() { From 0503ef79b9011ad38c54fe63ada293d722b66cda Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 19 Feb 2025 16:31:11 +0000 Subject: [PATCH 13/24] tests: fix knative host share --- tests/knative_chaining_host_share.bats | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/knative_chaining_host_share.bats b/tests/knative_chaining_host_share.bats index c57386fe..2dbc416a 100644 --- a/tests/knative_chaining_host_share.bats +++ b/tests/knative_chaining_host_share.bats @@ -6,11 +6,6 @@ source ./tests/utils/helpers.sh setup_file() { set_snapshotter_mode "host-share" - # The chaining tests need more memory and we set it using the - # default_memory annotation - enable_kata_annotation "default_memory" - restart_vm_cache - # May have to fetch content here k8s_content_fetch ${PAUSE_IMAGE} k8s_content_fetch ${SIDECAR_IMAGE} @@ -32,8 +27,12 @@ snapshotter="host-share" # ------------------------------------------------------------------------------ @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { - [[ "$SC2_TEE" == "tdx" ]] && skip "Host-share not supported for TDX (#142)" - skip "Knative chaining with host-share not supported (#145)" + [[ "$SC2_TEE" == "tdx" ]] && skip "#142" + skip "#145" + + enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[3]}" + restart_vm_cache + run_knative_chaining "${SC2_RUNTIME_CLASSES[3]}" } From 02e0020bdf507ce37ab47d98811b5075f49eac88 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 19 Feb 2025 16:36:12 +0000 Subject: [PATCH 14/24] tests: make skip syntax homogeneous --- tests/helloworld_knative_host_share.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helloworld_knative_host_share.bats b/tests/helloworld_knative_host_share.bats index 321247a1..18c63076 100644 --- a/tests/helloworld_knative_host_share.bats +++ b/tests/helloworld_knative_host_share.bats @@ -23,7 +23,7 @@ snapshotter="host-share" # ------------------------------------------------------------------------------ @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { - [[ "$SC2_TEE" == "tdx" ]] && skip "Host-share not supported for TDX (#142)" + [[ "$SC2_TEE" == "tdx" ]] && skip "#142" run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" } From 65b1e7a56ff2afe696494b188bf7b47eff4b1046 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 19 Feb 2025 17:24:11 +0000 Subject: [PATCH 15/24] tests: fix knative chaining test --- demo-apps/knative-chaining/chaining.yaml | 3 +++ demo-apps/knative-chaining/curl_cmd.sh | 2 +- docs/troubleshooting.md | 13 ++++++++----- tests/knative_chaining_guest_pull.bats | 4 ---- tests/knative_chaining_host_share.bats | 5 ----- tests/utils/helpers.sh | 5 ----- 6 files changed, 12 insertions(+), 20 deletions(-) diff --git a/demo-apps/knative-chaining/chaining.yaml b/demo-apps/knative-chaining/chaining.yaml index d629a713..f8cc307a 100644 --- a/demo-apps/knative-chaining/chaining.yaml +++ b/demo-apps/knative-chaining/chaining.yaml @@ -74,6 +74,7 @@ spec: labels: apps.sc2.io/name: knative-chaining-one annotations: + io.containerd.cri.runtime-handler: kata-${SC2_RUNTIME_CLASS} # NOTE: we may have to enable this annotation in Kata's config file # under hypervisor.qemu.enable_annotations (add 'default_memory') io.katacontainers.config.hypervisor.default_memory: "6144" @@ -108,6 +109,7 @@ spec: labels: apps.sc2.io/name: knative-chaining-two annotations: + io.containerd.cri.runtime-handler: kata-${SC2_RUNTIME_CLASS} # NOTE: we may have to enable this annotation in Kata's config file # under hypervisor.qemu.enable_annotations (add 'default_memory') io.katacontainers.config.hypervisor.default_memory: "6144" @@ -133,6 +135,7 @@ spec: apps.sc2.io/name: knative-chaining-three annotations: autoscaling.knative.dev/scale-to-zero-pod-retention-period: "1m" + io.containerd.cri.runtime-handler: kata-${SC2_RUNTIME_CLASS} # NOTE: we may have to enable this annotation in Kata's config file # under hypervisor.qemu.enable_annotations (add 'default_memory') io.katacontainers.config.hypervisor.default_memory: "6144" diff --git a/demo-apps/knative-chaining/curl_cmd.sh b/demo-apps/knative-chaining/curl_cmd.sh index b9a3cf3d..9d5029c3 100755 --- a/demo-apps/knative-chaining/curl_cmd.sh +++ b/demo-apps/knative-chaining/curl_cmd.sh @@ -1,6 +1,6 @@ #!/bin/bash -./bin/kubectl run curl --image=curlimages/curl --rm=true --restart=Never -i -- -X POST -v \ +./bin/kubectl -n chaining-test run curl --image=curlimages/curl --rm=true --restart=Never -i -- -X POST -v \ -H "content-type: application/json" \ -H "ce-specversion: 1.0" \ -H "ce-source: cli" \ diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index a6cca897..391536de 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -141,16 +141,19 @@ lines of: failed to extract layer sha256:1e604deea57dbda554a168861cff1238f93b8c6c69c863c43aed37d9d99c5fed: failed to get reader from content store: content digest sha256:9fa9226be034e47923c0457d916aa68474cdfb23af8d4525e9baeebc4760977a: not found ``` -you need to manually fetch the image contents on the host. This is a once-per- -host fix: +you may not be setting the right annotation in the pod: + +``` +io.containerd.cri.runtime-handler: kata-qemu-... +``` + +A temporary fix to get the pod running is to manually fetch the content but, as +said above, the annotation should take care of this. ```bash ctr -n k8s.io content fetch ${IMAGE_NAME} ``` -the image name is the image tag appearing right before the error message in -the pod logs. - ### Rootfs Mount Issue Sometimes, if we are mixing and matching different snapshotters, we may run diff --git a/tests/knative_chaining_guest_pull.bats b/tests/knative_chaining_guest_pull.bats index b1257b57..2884b663 100644 --- a/tests/knative_chaining_guest_pull.bats +++ b/tests/knative_chaining_guest_pull.bats @@ -5,10 +5,6 @@ source ./tests/utils/helpers.sh setup_file() { set_snapshotter_mode "guest-pull" - - # May have to fetch content here - k8s_content_fetch ${PAUSE_IMAGE} - k8s_content_fetch ${SIDECAR_IMAGE} } teardown() { diff --git a/tests/knative_chaining_host_share.bats b/tests/knative_chaining_host_share.bats index 2dbc416a..5c3f4fae 100644 --- a/tests/knative_chaining_host_share.bats +++ b/tests/knative_chaining_host_share.bats @@ -5,10 +5,6 @@ source ./tests/utils/helpers.sh setup_file() { set_snapshotter_mode "host-share" - - # May have to fetch content here - k8s_content_fetch ${PAUSE_IMAGE} - k8s_content_fetch ${SIDECAR_IMAGE} } teardown() { @@ -28,7 +24,6 @@ snapshotter="host-share" @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { [[ "$SC2_TEE" == "tdx" ]] && skip "#142" - skip "#145" enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[3]}" restart_vm_cache diff --git a/tests/utils/helpers.sh b/tests/utils/helpers.sh index 3689555d..5a3ffbbb 100644 --- a/tests/utils/helpers.sh +++ b/tests/utils/helpers.sh @@ -12,11 +12,6 @@ enable_kata_annotation() { ${INV} kata.enable-annotation ${annotation} --runtime ${runtime} } -k8s_content_fetch() { - local image=$1 - sudo ctr -n k8s.io content fetch -k ${image} -} - restart_vm_cache() { export SC2_RUNTIME_CLASS="qemu-${SC2_TEE}-sc2" From 73f67de7b4105656b394dcc7e581d0f617b79cf3 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Feb 2025 19:00:06 +0000 Subject: [PATCH 16/24] tests: set timeout --- tests/cold_starts_guest_pull.bats | 9 ++++++--- tests/cold_starts_host_share.bats | 9 ++++++--- tests/helloworld_knative_guest_pull.bats | 11 ++++++++--- tests/helloworld_knative_host_share.bats | 11 ++++++++--- tests/helloworld_python_guest_pull.bats | 11 ++++++++--- tests/helloworld_python_host_share.bats | 11 ++++++++--- tests/knative_chaining_guest_pull.bats | 11 ++++++++--- tests/lazy_loading.bats | 11 ++++++++--- tests/setup_suite.bash | 3 +++ 9 files changed, 63 insertions(+), 24 deletions(-) create mode 100644 tests/setup_suite.bash diff --git a/tests/cold_starts_guest_pull.bats b/tests/cold_starts_guest_pull.bats index 15465bf1..4f6439e4 100644 --- a/tests/cold_starts_guest_pull.bats +++ b/tests/cold_starts_guest_pull.bats @@ -1,14 +1,17 @@ #!/usr/bin/env bats -source ./tests/utils/env.sh -source ./tests/utils/helpers.sh - setup_file() { + load utils/env.sh + load utils/helpers.sh + set_snapshotter_mode "guest-pull" } # Make sure we purge before each test so that we have a cold start setup() { + load utils/env.sh + load utils/helpers.sh + ${INV} nydus-snapshotter.purge } diff --git a/tests/cold_starts_host_share.bats b/tests/cold_starts_host_share.bats index 6ec4c189..a3a61802 100644 --- a/tests/cold_starts_host_share.bats +++ b/tests/cold_starts_host_share.bats @@ -1,14 +1,17 @@ #!/usr/bin/env bats -source ./tests/utils/env.sh -source ./tests/utils/helpers.sh - setup_file() { + load utils/env.sh + load utils/helpers.sh + set_snapshotter_mode "host-share" } # Make sure we purge before each test so that we have a cold start setup() { + load utils/env.sh + load utils/helpers.sh + ${INV} nydus-snapshotter.purge } diff --git a/tests/helloworld_knative_guest_pull.bats b/tests/helloworld_knative_guest_pull.bats index 20695520..9c249db9 100644 --- a/tests/helloworld_knative_guest_pull.bats +++ b/tests/helloworld_knative_guest_pull.bats @@ -1,12 +1,17 @@ #!/usr/bin/env bats -source ./tests/utils/env.sh -source ./tests/utils/helpers.sh - setup_file() { + load utils/env.sh + load utils/helpers.sh + set_snapshotter_mode "guest-pull" } +setup() { + load utils/env.sh + load utils/helpers.sh +} + teardown() { # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} diff --git a/tests/helloworld_knative_host_share.bats b/tests/helloworld_knative_host_share.bats index 18c63076..9b8d9c24 100644 --- a/tests/helloworld_knative_host_share.bats +++ b/tests/helloworld_knative_host_share.bats @@ -1,12 +1,17 @@ #!/usr/bin/env bats -source ./tests/utils/env.sh -source ./tests/utils/helpers.sh - setup_file() { + load utils/env.sh + load utils/helpers.sh + set_snapshotter_mode "host-share" } +setup() { + load utils/env.sh + load utils/helpers.sh +} + teardown() { # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} diff --git a/tests/helloworld_python_guest_pull.bats b/tests/helloworld_python_guest_pull.bats index b5893f65..77cd5406 100644 --- a/tests/helloworld_python_guest_pull.bats +++ b/tests/helloworld_python_guest_pull.bats @@ -1,12 +1,17 @@ #!/usr/bin/env bats -source ./tests/utils/env.sh -source ./tests/utils/helpers.sh - setup_file() { + load utils/env.sh + load utils/helpers.sh + set_snapshotter_mode "guest-pull" } +setup() { + load utils/env.sh + load utils/helpers.sh +} + teardown() { # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} diff --git a/tests/helloworld_python_host_share.bats b/tests/helloworld_python_host_share.bats index 73a8949e..cfb1d1f7 100644 --- a/tests/helloworld_python_host_share.bats +++ b/tests/helloworld_python_host_share.bats @@ -1,12 +1,17 @@ #!/usr/bin/env bats -source ./tests/utils/env.sh -source ./tests/utils/helpers.sh - setup_file() { + load utils/env.sh + load utils/helpers.sh + set_snapshotter_mode "host-share" } +setup() { + load utils/env.sh + load utils/helpers.sh +} + teardown() { # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} diff --git a/tests/knative_chaining_guest_pull.bats b/tests/knative_chaining_guest_pull.bats index 2884b663..2afe856d 100644 --- a/tests/knative_chaining_guest_pull.bats +++ b/tests/knative_chaining_guest_pull.bats @@ -1,12 +1,17 @@ #!/usr/bin/env bats -source ./tests/utils/env.sh -source ./tests/utils/helpers.sh - setup_file() { + load utils/env.sh + load utils/helpers.sh + set_snapshotter_mode "guest-pull" } +setup() { + load utils/env.sh + load utils/helpers.sh +} + teardown() { # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} diff --git a/tests/lazy_loading.bats b/tests/lazy_loading.bats index b9ca4af4..b4bce32f 100644 --- a/tests/lazy_loading.bats +++ b/tests/lazy_loading.bats @@ -1,12 +1,17 @@ #!/usr/bin/env bats -source ./tests/utils/env.sh -source ./tests/utils/helpers.sh - setup_file() { + load utils/env.sh + load utils/helpers.sh + set_snapshotter_mode "guest-pull" } +setup() { + load utils/env.sh + load utils/helpers.sh +} + teardown() { # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} diff --git a/tests/setup_suite.bash b/tests/setup_suite.bash new file mode 100644 index 00000000..37b2e4e0 --- /dev/null +++ b/tests/setup_suite.bash @@ -0,0 +1,3 @@ +setup_suite() { + export BATS_TEST_TIMEOUT=${SC2_TEST_TIMEOUT:-60} +} From 99e108422196c95939bbc9360613f1e635e32fc4 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Feb 2025 10:48:11 +0000 Subject: [PATCH 17/24] gha: skip pulling images --- .github/workflows/tests.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1b3f3ae8..de0b8fae 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,15 +57,6 @@ jobs: - name: "Checkout code" uses: actions/checkout@v4 - # Make sure we have the up-to-date version of all images we need as - # part of the deployment process - - name: "Pull all images" - run: | - docker pull ghcr.io/sc2-sys/containerd:$(grep -oP 'CONTAINERD_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) - docker pull ghcr.io/sc2-sys/kata-containers:$(grep -oP 'KATA_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) - docker pull ghcr.io/sc2-sys/nydus:$(grep -oP 'NYDUS_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) - docker pull ghcr.io/sc2-sys/nydus-snapshotter:$(grep -oP 'NYDUS_SNAPSHOTTER_VERSION\s*=\s*"\K[^"]+' ./tasks/util/versions.py) - - name: "Install SC2" run: ./bin/inv_wrapper.sh sc2.deploy --clean From f7636eecafcd5860b18d9155a0e6b1ff2bb74769 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Feb 2025 10:49:34 +0000 Subject: [PATCH 18/24] tests: use sc2-demo namespace for safer clean-up --- .../helloworld-knative-nydus/service.yaml | 6 ++++++ demo-apps/helloworld-knative/service.yaml | 6 ++++++ demo-apps/helloworld-py-nydus/deployment.yaml | 7 +++++++ demo-apps/helloworld-py/deployment.yaml | 7 +++++++ demo-apps/knative-chaining/chaining.yaml | 20 +++++++++---------- tests/cold_starts_guest_pull.bats | 2 ++ tests/cold_starts_host_share.bats | 2 ++ tests/helloworld_knative_guest_pull.bats | 2 ++ tests/helloworld_knative_host_share.bats | 2 ++ tests/helloworld_python_guest_pull.bats | 2 ++ tests/helloworld_python_host_share.bats | 6 ++++-- tests/knative_chaining_guest_pull.bats | 2 ++ tests/knative_chaining_host_share.bats | 13 +++++++++--- tests/lazy_loading.bats | 2 ++ tests/setup_suite.bash | 2 +- tests/utils/env.sh | 3 ++- tests/utils/helpers.sh | 10 +++------- 17 files changed, 70 insertions(+), 24 deletions(-) diff --git a/demo-apps/helloworld-knative-nydus/service.yaml b/demo-apps/helloworld-knative-nydus/service.yaml index fa84bf5b..6ac371f7 100644 --- a/demo-apps/helloworld-knative-nydus/service.yaml +++ b/demo-apps/helloworld-knative-nydus/service.yaml @@ -1,7 +1,13 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: sc2-demo +--- apiVersion: serving.knative.dev/v1 kind: Service metadata: name: helloworld-knative + namespace: sc2-demo spec: template: metadata: diff --git a/demo-apps/helloworld-knative/service.yaml b/demo-apps/helloworld-knative/service.yaml index fd5c658b..15e1635f 100644 --- a/demo-apps/helloworld-knative/service.yaml +++ b/demo-apps/helloworld-knative/service.yaml @@ -1,7 +1,13 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: sc2-demo +--- apiVersion: serving.knative.dev/v1 kind: Service metadata: name: helloworld-knative + namespace: sc2-demo spec: template: metadata: diff --git a/demo-apps/helloworld-py-nydus/deployment.yaml b/demo-apps/helloworld-py-nydus/deployment.yaml index 3e06a2cf..7c56f03a 100644 --- a/demo-apps/helloworld-py-nydus/deployment.yaml +++ b/demo-apps/helloworld-py-nydus/deployment.yaml @@ -1,7 +1,13 @@ apiVersion: v1 +kind: Namespace +metadata: + name: sc2-demo +--- +apiVersion: v1 kind: Service metadata: name: coco-helloworld-py-node-port + namespace: sc2-demo spec: type: NodePort selector: @@ -16,6 +22,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: coco-helloworld-py + namespace: sc2-demo labels: apps.sc2.io/name: helloworld-py spec: diff --git a/demo-apps/helloworld-py/deployment.yaml b/demo-apps/helloworld-py/deployment.yaml index f55e269b..7e5461fb 100644 --- a/demo-apps/helloworld-py/deployment.yaml +++ b/demo-apps/helloworld-py/deployment.yaml @@ -1,7 +1,13 @@ apiVersion: v1 +kind: Namespace +metadata: + name: sc2-demo +--- +apiVersion: v1 kind: Service metadata: name: coco-helloworld-py-node-port + namespace: sc2-demo spec: type: NodePort selector: @@ -16,6 +22,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: coco-helloworld-py + namespace: sc2-demo labels: apps.sc2.io/name: helloworld-py spec: diff --git a/demo-apps/knative-chaining/chaining.yaml b/demo-apps/knative-chaining/chaining.yaml index f8cc307a..8d37a17c 100644 --- a/demo-apps/knative-chaining/chaining.yaml +++ b/demo-apps/knative-chaining/chaining.yaml @@ -18,7 +18,7 @@ apiVersion: v1 kind: Namespace metadata: - name: chaining-test + name: sc2-demo --- # We need to have as many channels as edges in our workflow DAG. Alternatively, # we could enforce edges by using a Broker/Trigger pattern and filtering on @@ -27,7 +27,7 @@ apiVersion: messaging.knative.dev/v1 kind: Channel metadata: name: ingress-to-one - namespace: chaining-test + namespace: sc2-demo spec: channelTemplate: apiVersion: messaging.knative.dev/v1 @@ -37,7 +37,7 @@ apiVersion: messaging.knative.dev/v1 kind: Channel metadata: name: one-to-two - namespace: chaining-test + namespace: sc2-demo spec: channelTemplate: apiVersion: messaging.knative.dev/v1 @@ -47,7 +47,7 @@ apiVersion: messaging.knative.dev/v1 kind: Channel metadata: name: two-to-three - namespace: chaining-test + namespace: sc2-demo spec: channelTemplate: apiVersion: messaging.knative.dev/v1 @@ -60,7 +60,7 @@ apiVersion: serving.knative.dev/v1 kind: Service metadata: name: coco-knative-chaining-one - namespace: chaining-test + namespace: sc2-demo spec: template: spec: @@ -86,7 +86,7 @@ apiVersion: sinks.knative.dev/v1alpha1 kind: JobSink metadata: name: coco-knative-chaining-two - namespace: chaining-test + namespace: sc2-demo spec: job: spec: @@ -120,7 +120,7 @@ apiVersion: serving.knative.dev/v1 kind: Service metadata: name: coco-knative-chaining-three - namespace: chaining-test + namespace: sc2-demo spec: template: spec: @@ -144,7 +144,7 @@ apiVersion: messaging.knative.dev/v1 kind: Subscription metadata: name: edge-one-subscription - namespace: chaining-test + namespace: sc2-demo spec: channel: apiVersion: messaging.knative.dev/v1 @@ -165,7 +165,7 @@ apiVersion: messaging.knative.dev/v1 kind: Subscription metadata: name: edge-two-subscription - namespace: chaining-test + namespace: sc2-demo spec: channel: apiVersion: messaging.knative.dev/v1 @@ -181,7 +181,7 @@ apiVersion: messaging.knative.dev/v1 kind: Subscription metadata: name: edge-three-subscription - namespace: chaining-test + namespace: sc2-demo spec: channel: apiVersion: messaging.knative.dev/v1 diff --git a/tests/cold_starts_guest_pull.bats b/tests/cold_starts_guest_pull.bats index 4f6439e4..540b548b 100644 --- a/tests/cold_starts_guest_pull.bats +++ b/tests/cold_starts_guest_pull.bats @@ -16,6 +16,8 @@ setup() { } teardown() { + ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} } diff --git a/tests/cold_starts_host_share.bats b/tests/cold_starts_host_share.bats index a3a61802..05070370 100644 --- a/tests/cold_starts_host_share.bats +++ b/tests/cold_starts_host_share.bats @@ -16,6 +16,8 @@ setup() { } teardown() { + ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} } diff --git a/tests/helloworld_knative_guest_pull.bats b/tests/helloworld_knative_guest_pull.bats index 9c249db9..6af22941 100644 --- a/tests/helloworld_knative_guest_pull.bats +++ b/tests/helloworld_knative_guest_pull.bats @@ -13,6 +13,8 @@ setup() { } teardown() { + ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} } diff --git a/tests/helloworld_knative_host_share.bats b/tests/helloworld_knative_host_share.bats index 9b8d9c24..6e8b65c7 100644 --- a/tests/helloworld_knative_host_share.bats +++ b/tests/helloworld_knative_host_share.bats @@ -13,6 +13,8 @@ setup() { } teardown() { + ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} } diff --git a/tests/helloworld_python_guest_pull.bats b/tests/helloworld_python_guest_pull.bats index 77cd5406..435ce04c 100644 --- a/tests/helloworld_python_guest_pull.bats +++ b/tests/helloworld_python_guest_pull.bats @@ -13,6 +13,8 @@ setup() { } teardown() { + ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} } diff --git a/tests/helloworld_python_host_share.bats b/tests/helloworld_python_host_share.bats index cfb1d1f7..2b40d4cd 100644 --- a/tests/helloworld_python_host_share.bats +++ b/tests/helloworld_python_host_share.bats @@ -1,18 +1,20 @@ #!/usr/bin/env bats +load utils/env.sh + setup_file() { - load utils/env.sh load utils/helpers.sh set_snapshotter_mode "host-share" } setup() { - load utils/env.sh load utils/helpers.sh } teardown() { + ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} } diff --git a/tests/knative_chaining_guest_pull.bats b/tests/knative_chaining_guest_pull.bats index 2afe856d..7dcff231 100644 --- a/tests/knative_chaining_guest_pull.bats +++ b/tests/knative_chaining_guest_pull.bats @@ -13,6 +13,8 @@ setup() { } teardown() { + ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} } diff --git a/tests/knative_chaining_host_share.bats b/tests/knative_chaining_host_share.bats index 5c3f4fae..c042447c 100644 --- a/tests/knative_chaining_host_share.bats +++ b/tests/knative_chaining_host_share.bats @@ -1,13 +1,20 @@ #!/usr/bin/env bats -source ./tests/utils/env.sh -source ./tests/utils/helpers.sh - setup_file() { + load utils/env.sh + load utils/helpers.sh + set_snapshotter_mode "host-share" } +setup() { + load utils/env.sh + load utils/helpers.sh +} + teardown() { + ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} } diff --git a/tests/lazy_loading.bats b/tests/lazy_loading.bats index b4bce32f..78f278c3 100644 --- a/tests/lazy_loading.bats +++ b/tests/lazy_loading.bats @@ -13,6 +13,8 @@ setup() { } teardown() { + ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} } diff --git a/tests/setup_suite.bash b/tests/setup_suite.bash index 37b2e4e0..fb73aea8 100644 --- a/tests/setup_suite.bash +++ b/tests/setup_suite.bash @@ -1,3 +1,3 @@ setup_suite() { - export BATS_TEST_TIMEOUT=${SC2_TEST_TIMEOUT:-60} + export BATS_TEST_TIMEOUT=${SC2_TEST_TIMEOUT:-90} } diff --git a/tests/utils/env.sh b/tests/utils/env.sh index 2c74c3aa..ee78236e 100644 --- a/tests/utils/env.sh +++ b/tests/utils/env.sh @@ -22,7 +22,8 @@ else fi # Useful constants -INTERTEST_SLEEP_SECS=5 +INTERTEST_SLEEP_SECS=1 +SC2_DEMO_NAMESPACE=sc2-demo # Images PAUSE_IMAGE="registry.k8s.io/pause:3.8" diff --git a/tests/utils/helpers.sh b/tests/utils/helpers.sh index 5a3ffbbb..2036080c 100644 --- a/tests/utils/helpers.sh +++ b/tests/utils/helpers.sh @@ -48,14 +48,10 @@ run_knative_chaining() { # Curl the channel URL ./demo-apps/knative-chaining/curl_cmd.sh - NAMESPACE="chaining-test" POD_LABEL="apps.sc2.io/name=knative-chaining-three" # Wait for pod 3 to be scaled down until [ "$(${KUBECTL} -n ${NAMESPACE} logs -l ${POD_LABEL} | grep 'cloudevent(s3): done!' | wc -l)" = "1" ]; do echo "Waiting for chain to finish..."; sleep 2; done - - # Finally, clear-up - ${KUBECTL} delete namespace ${NAMESPACE} } run_knative_hello_world() { @@ -101,17 +97,17 @@ run_python_hello_world() { export POD_LABEL="apps.sc2.io/name=helloworld-py" # Wait for pod to be ready - until [ "$(${KUBECTL} get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done + until [ "$(${KUBECTL} get pods -n ${SC2_DEMO_NAMESPACE} -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done sleep 1 # Get the pod's IP - service_ip=$(${KUBECTL} get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') + service_ip=$(${KUBECTL} get services -n ${SC2_DEMO_NAMESPACE} -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] envsubst < ./demo-apps/helloworld-py/deployment.yaml | ${KUBECTL} delete -f - # Wait for pod to be deleted - ${KUBECTL} wait --for=delete -l ${POD_LABEL} pod --timeout=30s + ${KUBECTL} wait --for=delete -n ${SC2_DEMO_NAMESPACE} -l ${POD_LABEL} pod --timeout=30s } run_python_lazy_loading() { From 71bc5535a91cd154a423cbc55e8037330ef38a80 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Feb 2025 11:51:28 +0000 Subject: [PATCH 19/24] sc2: add comment --- tasks/sc2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tasks/sc2.py b/tasks/sc2.py index 328d2bae..ee102c13 100644 --- a/tasks/sc2.py +++ b/tasks/sc2.py @@ -314,7 +314,8 @@ def deploy(ctx, debug=False, clean=False): print("Success!") # Update SNP class to use default QEMU (we use host kernel 6.11, so we - # can use upstream QEMU 9.1) + # can use upstream QEMU 9.1). We do this update before generating the SC2 + # runtime classes, so they will inherit the QEMU value # TODO: remove when bumping to a new CoCo release qemu_path = join(KATA_ROOT, "bin", "qemu-system-x86_64") updated_toml_str = """ From 215dabed0d08559f46758e26fcf46d049b82c637 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 28 Feb 2025 18:13:54 +0000 Subject: [PATCH 20/24] tests: fix host-share in 24.04 + faster cleanup --- README.md | 4 ++-- tasks/util/kata.py | 10 ++++++++-- tests/cold_starts_guest_pull.bats | 4 ++-- tests/helloworld_knative_guest_pull.bats | 2 -- tests/helloworld_knative_host_share.bats | 2 -- tests/utils/helpers.sh | 20 +++++++++++-------- vm-cache/src/main.rs | 25 +++++++++++++++--------- 7 files changed, 40 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 71fad58c..2b8e0895 100644 --- a/README.md +++ b/README.md @@ -51,11 +51,11 @@ export SC2_RUNTIME_CLASS=qemu-snp-sc2 # Knative demo envsubst < ./demo-apps/helloworld-knative/service.yaml | kubectl apply -f - -curl $(kubectl get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) +curl $(kubectl -n sc2-demo get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) # Non-Knative demo envsubst < ./demo-apps/helloworld-py/deployment.yaml | kubectl apply -f - -curl $(kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}'):8080 +curl $(kubectl -n sc2-demo get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}'):8080 ``` for more complex applications and workloads, please check our [applications]( diff --git a/tasks/util/kata.py b/tasks/util/kata.py index 050a12d3..8d896d6f 100644 --- a/tasks/util/kata.py +++ b/tasks/util/kata.py @@ -161,7 +161,7 @@ def prepare_rootfs(tmp_rootfs_base_dir, debug=False, sc2=False, hot_replace=Fals """ This function takes a directory as input, and generates the root-filesystem needed in SC2 at /rootfs. The result can be consumed - to pack an `initrd` or a .qcow2 image. + to pack an `initrd` """ # ----- Prepare temporary rootfs directory ----- @@ -255,7 +255,13 @@ def prepare_rootfs(tmp_rootfs_base_dir, debug=False, sc2=False, hot_replace=Fals "AGENT_SOURCE_BIN": join(tmp_rootfs_base_dir, "kata-agent"), "CONFIDENTIAL_GUEST": "yes", "DMVERITY_SUPPORT": "yes", - "MEASURED_ROOTFS": "yes", + "MEASURED_ROOTFS": "no", + # We build the `initrd` inside a container image to prevent different + # host OS versions from introducing subtle changes in the rootfs + "USE_DOCKER": "yes", + "OS_VERSION": "jammy", + "RUST_VERSION": "1.75.0", + "GO_VERSION": "1.22.2", "PAUSE_IMAGE_TARBALL": build_pause_image( sc2=sc2, debug=debug, hot_replace=hot_replace ), diff --git a/tests/cold_starts_guest_pull.bats b/tests/cold_starts_guest_pull.bats index 540b548b..bdef1237 100644 --- a/tests/cold_starts_guest_pull.bats +++ b/tests/cold_starts_guest_pull.bats @@ -1,5 +1,7 @@ #!/usr/bin/env bats +load utils/env.sh + setup_file() { load utils/env.sh load utils/helpers.sh @@ -16,8 +18,6 @@ setup() { } teardown() { - ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found - # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} } diff --git a/tests/helloworld_knative_guest_pull.bats b/tests/helloworld_knative_guest_pull.bats index 6af22941..9c249db9 100644 --- a/tests/helloworld_knative_guest_pull.bats +++ b/tests/helloworld_knative_guest_pull.bats @@ -13,8 +13,6 @@ setup() { } teardown() { - ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found - # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} } diff --git a/tests/helloworld_knative_host_share.bats b/tests/helloworld_knative_host_share.bats index 6e8b65c7..9b8d9c24 100644 --- a/tests/helloworld_knative_host_share.bats +++ b/tests/helloworld_knative_host_share.bats @@ -13,8 +13,6 @@ setup() { } teardown() { - ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found - # Cautionary inter-test sleep sleep ${INTERTEST_SLEEP_SECS} } diff --git a/tests/utils/helpers.sh b/tests/utils/helpers.sh index 2036080c..309c2969 100644 --- a/tests/utils/helpers.sh +++ b/tests/utils/helpers.sh @@ -62,13 +62,15 @@ run_knative_hello_world() { sleep 1 # Get the service URL - service_url=$(${KUBECTL} get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) + service_url=$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] - # Wait for pod to be deleted + # Delete resources in order, as it seems to prevent deletion from being stuck + SERVICE_NAME="helloworld-knative" POD_LABEL="apps.sc2.io/name=helloworld-py" - envsubst < ./demo-apps/helloworld-knative/service.yaml | ${KUBECTL} delete -f - - ${KUBECTL} wait --for=delete -l ${POD_LABEL} pod --timeout=60s + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete service.serving.knative.dev ${SERVICE_NAME} + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} wait --for=delete -l ${POD_LABEL} pod --timeout=60s + ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} } run_knative_lazy_loading() { @@ -79,13 +81,15 @@ run_knative_lazy_loading() { sleep 1 # Get the service URL - service_url=$(${KUBECTL} get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) + service_url=$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] - # Wait for pod to be deleted + # Delete resources in order, as it seems to prevent deletion from being stuck + SERVICE_NAME="helloworld-knative" POD_LABEL="apps.sc2.io/name=helloworld-py" - envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ${KUBECTL} delete -f - - ${KUBECTL} wait --for=delete -l ${POD_LABEL} pod --timeout=60s + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete service.serving.knative.dev ${SERVICE_NAME} + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} wait --for=delete -l ${POD_LABEL} pod --timeout=60s + ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} } run_python_hello_world() { diff --git a/vm-cache/src/main.rs b/vm-cache/src/main.rs index 781a6430..0b70ab18 100644 --- a/vm-cache/src/main.rs +++ b/vm-cache/src/main.rs @@ -134,15 +134,22 @@ fn read_pid() -> std::io::Result { fn stop_background_process() -> std::io::Result<()> { if let Ok(pid) = read_pid() { - kill(Pid::from_raw(pid as i32), Signal::SIGTERM).map_err(|e| { - error!("failed to kill process: {e}"); - std::io::Error::new( - std::io::ErrorKind::Other, - "sc2(vm-cache): failed to kill process", - ) - })?; - - info!("stopped background process with PID {pid}"); + match kill(Pid::from_raw(pid as i32), Signal::SIGTERM) { + Ok(_) => { + info!("stopped background process with PID {pid}"); + } + Err(nix::errno::Errno::ESRCH) => { + info!("process with PID {pid} not found (already stopped?)"); + } + Err(e) => { + error!("failed to kill process: {e}"); + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "sc2(vm-cache): failed to kill process", + )); + } + } + std::fs::remove_file(get_pid_file())?; std::fs::remove_file(get_log_file())?; } else { From f0caeaec361bd8fd6b42f59222a0d0f15d630fd8 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Mar 2025 10:24:41 +0000 Subject: [PATCH 21/24] tests: fix knative tests --- demo-apps/helloworld-py/Dockerfile | 4 ++-- demo-apps/knative-chaining/Dockerfile | 2 +- demo-apps/knative-chaining/curl_cmd.sh | 6 +++--- demo-apps/knative-chaining/src/main.rs | 2 +- tasks/nydus.py | 5 ++--- tasks/util/nydus.py | 12 ++++++++++-- tests/cold_starts_guest_pull.bats | 7 +++---- tests/cold_starts_host_share.bats | 9 ++++----- tests/helloworld_knative_guest_pull.bats | 9 +++++---- tests/helloworld_knative_host_share.bats | 9 +++++---- tests/helloworld_python_guest_pull.bats | 9 ++++----- tests/helloworld_python_host_share.bats | 5 ++--- tests/knative_chaining_guest_pull.bats | 9 ++++----- tests/knative_chaining_host_share.bats | 9 ++++----- tests/lazy_loading.bats | 9 ++++----- tests/utils/helpers.sh | 9 ++++++++- 16 files changed, 62 insertions(+), 53 deletions(-) diff --git a/demo-apps/helloworld-py/Dockerfile b/demo-apps/helloworld-py/Dockerfile index 5b5538e5..0a6e1451 100644 --- a/demo-apps/helloworld-py/Dockerfile +++ b/demo-apps/helloworld-py/Dockerfile @@ -2,10 +2,10 @@ FROM python:3.10-slim # Allow statements and log messages to immediately appear in the Knative logs -ENV PYTHONUNBUFFERED True +ENV PYTHONUNBUFFERED=True # Copy local code to the container image. -ENV APP_HOME /app +ENV APP_HOME=/app WORKDIR $APP_HOME COPY . ./ diff --git a/demo-apps/knative-chaining/Dockerfile b/demo-apps/knative-chaining/Dockerfile index 08ba9f4c..276ff322 100644 --- a/demo-apps/knative-chaining/Dockerfile +++ b/demo-apps/knative-chaining/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80 +FROM rust:1.84 WORKDIR /app COPY . ./ diff --git a/demo-apps/knative-chaining/curl_cmd.sh b/demo-apps/knative-chaining/curl_cmd.sh index 9d5029c3..ead627d9 100755 --- a/demo-apps/knative-chaining/curl_cmd.sh +++ b/demo-apps/knative-chaining/curl_cmd.sh @@ -1,10 +1,10 @@ #!/bin/bash -./bin/kubectl -n chaining-test run curl --image=curlimages/curl --rm=true --restart=Never -i -- -X POST -v \ +./bin/kubectl -n sc2-demo run curl --image=curlimages/curl --rm=true --restart=Never -i -- -X POST -v \ -H "content-type: application/json" \ -H "ce-specversion: 1.0" \ -H "ce-source: cli" \ - -H "ce-type: http://one-to-two-kn-channel.chaining-test.svc.cluster.local" \ + -H "ce-type: http://one-to-two-kn-channel.sc2-demo.svc.cluster.local" \ -H "ce-id: 1" \ -d '{"details":"ChannelDemo"}' \ - http://ingress-to-one-kn-channel.chaining-test.svc.cluster.local + http://ingress-to-one-kn-channel.sc2-demo.svc.cluster.local diff --git a/demo-apps/knative-chaining/src/main.rs b/demo-apps/knative-chaining/src/main.rs index c261b2f1..0d70bcae 100644 --- a/demo-apps/knative-chaining/src/main.rs +++ b/demo-apps/knative-chaining/src/main.rs @@ -91,7 +91,7 @@ pub fn process_event(mut event: Event) -> Event { // Write the new destination channel // FIXME: this is hardcoded in chaining.yaml - scaled_event.set_type("http://two-to-three-kn-channel.chaining-test.svc.cluster.local"); + scaled_event.set_type("http://two-to-three-kn-channel.sc2-demo.svc.cluster.local"); println!("cloudevent(s1): fanning out by a factor of {FAN_OUT_SCALE}"); diff --git a/tasks/nydus.py b/tasks/nydus.py index a8a61f25..8609a53b 100644 --- a/tasks/nydus.py +++ b/tasks/nydus.py @@ -2,15 +2,14 @@ from os.path import join from subprocess import run from tasks.util.docker import copy_from_ctr_image, is_ctr_running -from tasks.util.env import COCO_ROOT, GHCR_URL, GITHUB_ORG, PROJ_ROOT, print_dotted_line -from tasks.util.nydus import NYDUSIFY_PATH +from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT, print_dotted_line +from tasks.util.nydus import NYDUS_IMAGE_HOST_PATH, NYDUSIFY_PATH from tasks.util.versions import NYDUS_VERSION NYDUS_CTR_NAME = "nydus-workon" NYDUS_IMAGE_TAG = join(GHCR_URL, GITHUB_ORG, "nydus") + f":{NYDUS_VERSION}" NYDUS_IMAGE_CTR_PATH = "/go/src/github.com/sc2-sys/nydus/target/release/nydus-image" -NYDUS_IMAGE_HOST_PATH = join(COCO_ROOT, "bin", "nydus-image") @task diff --git a/tasks/util/nydus.py b/tasks/util/nydus.py index eab2c8b8..bb6f5e8a 100644 --- a/tasks/util/nydus.py +++ b/tasks/util/nydus.py @@ -1,15 +1,23 @@ -from os.path import join +from os import environ +from os.path import dirname, join from subprocess import run -from tasks.util.env import PROJ_ROOT +from tasks.util.env import COCO_ROOT, PROJ_ROOT NYDUSIFY_PATH = join(PROJ_ROOT, "bin", "nydusify") +NYDUS_IMAGE_HOST_PATH = join(COCO_ROOT, "bin", "nydus-image") + def nydusify(src_tag, dst_tag): + # Add nydus-image to path + work_env = environ.copy() + work_env["PATH"] = work_env.get("PATH", "") + ":" + dirname(NYDUS_IMAGE_HOST_PATH) + # Note that nydusify automatically pushes the image result = run( f"{NYDUSIFY_PATH} convert --source {src_tag} --target {dst_tag}", shell=True, capture_output=True, + env=work_env, ) assert result.returncode == 0, result.stderr.decode("utf-8").strip() diff --git a/tests/cold_starts_guest_pull.bats b/tests/cold_starts_guest_pull.bats index bdef1237..43a19091 100644 --- a/tests/cold_starts_guest_pull.bats +++ b/tests/cold_starts_guest_pull.bats @@ -3,7 +3,6 @@ load utils/env.sh setup_file() { - load utils/env.sh load utils/helpers.sh set_snapshotter_mode "guest-pull" @@ -11,15 +10,15 @@ setup_file() { # Make sure we purge before each test so that we have a cold start setup() { - load utils/env.sh load utils/helpers.sh ${INV} nydus-snapshotter.purge } teardown() { - # Cautionary inter-test sleep - sleep ${INTERTEST_SLEEP_SECS} + load utils/helpers.sh + + common_teardown } snapshotter="guest-pull" diff --git a/tests/cold_starts_host_share.bats b/tests/cold_starts_host_share.bats index 05070370..89189201 100644 --- a/tests/cold_starts_host_share.bats +++ b/tests/cold_starts_host_share.bats @@ -1,7 +1,8 @@ #!/usr/bin/env bats +load utils/env.sh + setup_file() { - load utils/env.sh load utils/helpers.sh set_snapshotter_mode "host-share" @@ -9,17 +10,15 @@ setup_file() { # Make sure we purge before each test so that we have a cold start setup() { - load utils/env.sh load utils/helpers.sh ${INV} nydus-snapshotter.purge } teardown() { - ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + load utils/helpers.sh - # Cautionary inter-test sleep - sleep ${INTERTEST_SLEEP_SECS} + common_teardown } snapshotter="host-share" diff --git a/tests/helloworld_knative_guest_pull.bats b/tests/helloworld_knative_guest_pull.bats index 9c249db9..3d767e35 100644 --- a/tests/helloworld_knative_guest_pull.bats +++ b/tests/helloworld_knative_guest_pull.bats @@ -1,20 +1,21 @@ #!/usr/bin/env bats +load utils/env.sh + setup_file() { - load utils/env.sh load utils/helpers.sh set_snapshotter_mode "guest-pull" } setup() { - load utils/env.sh load utils/helpers.sh } teardown() { - # Cautionary inter-test sleep - sleep ${INTERTEST_SLEEP_SECS} + load utils/helpers.sh + + common_teardown } TEST_NAME="Test knative hello world" diff --git a/tests/helloworld_knative_host_share.bats b/tests/helloworld_knative_host_share.bats index 9b8d9c24..1b764e5e 100644 --- a/tests/helloworld_knative_host_share.bats +++ b/tests/helloworld_knative_host_share.bats @@ -1,20 +1,21 @@ #!/usr/bin/env bats +load utils/env.sh + setup_file() { - load utils/env.sh load utils/helpers.sh set_snapshotter_mode "host-share" } setup() { - load utils/env.sh load utils/helpers.sh } teardown() { - # Cautionary inter-test sleep - sleep ${INTERTEST_SLEEP_SECS} + load utils/helpers.sh + + common_teardown } TEST_NAME="Test knative hello world" diff --git a/tests/helloworld_python_guest_pull.bats b/tests/helloworld_python_guest_pull.bats index 435ce04c..ab9948c2 100644 --- a/tests/helloworld_python_guest_pull.bats +++ b/tests/helloworld_python_guest_pull.bats @@ -1,22 +1,21 @@ #!/usr/bin/env bats +load utils/env.sh + setup_file() { - load utils/env.sh load utils/helpers.sh set_snapshotter_mode "guest-pull" } setup() { - load utils/env.sh load utils/helpers.sh } teardown() { - ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + load utils/helpers.sh - # Cautionary inter-test sleep - sleep ${INTERTEST_SLEEP_SECS} + common_teardown } TEST_NAME="Test python hello world" diff --git a/tests/helloworld_python_host_share.bats b/tests/helloworld_python_host_share.bats index 2b40d4cd..9bf9bb48 100644 --- a/tests/helloworld_python_host_share.bats +++ b/tests/helloworld_python_host_share.bats @@ -13,10 +13,9 @@ setup() { } teardown() { - ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + load utils/helpers.sh - # Cautionary inter-test sleep - sleep ${INTERTEST_SLEEP_SECS} + common_teardown } TEST_NAME="Test python hello world" diff --git a/tests/knative_chaining_guest_pull.bats b/tests/knative_chaining_guest_pull.bats index 7dcff231..0e8d6b4d 100644 --- a/tests/knative_chaining_guest_pull.bats +++ b/tests/knative_chaining_guest_pull.bats @@ -1,22 +1,21 @@ #!/usr/bin/env bats +load utils/env.sh + setup_file() { - load utils/env.sh load utils/helpers.sh set_snapshotter_mode "guest-pull" } setup() { - load utils/env.sh load utils/helpers.sh } teardown() { - ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + load utils/helpers.sh - # Cautionary inter-test sleep - sleep ${INTERTEST_SLEEP_SECS} + common_teardown } TEST_NAME="Test knative chaining" diff --git a/tests/knative_chaining_host_share.bats b/tests/knative_chaining_host_share.bats index c042447c..57b256b8 100644 --- a/tests/knative_chaining_host_share.bats +++ b/tests/knative_chaining_host_share.bats @@ -1,22 +1,21 @@ #!/usr/bin/env bats +load utils/env.sh + setup_file() { - load utils/env.sh load utils/helpers.sh set_snapshotter_mode "host-share" } setup() { - load utils/env.sh load utils/helpers.sh } teardown() { - ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + load utils/helpers.sh - # Cautionary inter-test sleep - sleep ${INTERTEST_SLEEP_SECS} + common_teardown } TEST_NAME="Test knative chaining" diff --git a/tests/lazy_loading.bats b/tests/lazy_loading.bats index 78f278c3..c6fd0e2a 100644 --- a/tests/lazy_loading.bats +++ b/tests/lazy_loading.bats @@ -1,22 +1,21 @@ #!/usr/bin/env bats +load utils/env.sh + setup_file() { - load utils/env.sh load utils/helpers.sh set_snapshotter_mode "guest-pull" } setup() { - load utils/env.sh load utils/helpers.sh } teardown() { - ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + load utils/helpers.sh - # Cautionary inter-test sleep - sleep ${INTERTEST_SLEEP_SECS} + common_teardown } snapshotter="guest-pull" diff --git a/tests/utils/helpers.sh b/tests/utils/helpers.sh index 309c2969..aa360a70 100644 --- a/tests/utils/helpers.sh +++ b/tests/utils/helpers.sh @@ -6,6 +6,13 @@ source ./tests/utils/env.sh # Helper functions to set-up the environment # ------------------------------------------------------------------------------ +common_teardown() { + ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} --ignore-not-found + + # Cautionary inter-test sleep + sleep ${INTERTEST_SLEEP_SECS} +} + enable_kata_annotation() { local annotation="$1" local runtime="$2" @@ -51,7 +58,7 @@ run_knative_chaining() { POD_LABEL="apps.sc2.io/name=knative-chaining-three" # Wait for pod 3 to be scaled down - until [ "$(${KUBECTL} -n ${NAMESPACE} logs -l ${POD_LABEL} | grep 'cloudevent(s3): done!' | wc -l)" = "1" ]; do echo "Waiting for chain to finish..."; sleep 2; done + until [ "$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} logs -l ${POD_LABEL} | grep 'cloudevent(s3): done!' | wc -l)" = "1" ]; do echo "Waiting for chain to finish..."; sleep 2; done } run_knative_hello_world() { From 35b28e671cb690452d2f3e93d989e86b83c2288b Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Mar 2025 13:40:59 +0000 Subject: [PATCH 22/24] tests: consistent clean-up + home-made timeout --- demo-apps/knative-chaining/curl_cmd.sh | 3 +- tests/cold_starts_guest_pull.bats | 56 ++++++++++++++++++---- tests/cold_starts_host_share.bats | 14 +++++- tests/helloworld_knative_guest_pull.bats | 28 +++++++++-- tests/helloworld_knative_host_share.bats | 8 +++- tests/helloworld_python_guest_pull.bats | 28 +++++++++-- tests/helloworld_python_host_share.bats | 10 +++- tests/knative_chaining_guest_pull.bats | 31 +++++++++++-- tests/knative_chaining_host_share.bats | 10 +++- tests/lazy_loading.bats | 14 +++++- tests/setup_suite.bash | 3 -- tests/utils/env.sh | 1 + tests/utils/helpers.sh | 59 ++++++++++++++---------- 13 files changed, 208 insertions(+), 57 deletions(-) delete mode 100644 tests/setup_suite.bash diff --git a/demo-apps/knative-chaining/curl_cmd.sh b/demo-apps/knative-chaining/curl_cmd.sh index ead627d9..93c1cb3e 100755 --- a/demo-apps/knative-chaining/curl_cmd.sh +++ b/demo-apps/knative-chaining/curl_cmd.sh @@ -1,6 +1,6 @@ #!/bin/bash -./bin/kubectl -n sc2-demo run curl --image=curlimages/curl --rm=true --restart=Never -i -- -X POST -v \ +./bin/kubectl -n sc2-demo run curl --image=curlimages/curl --restart=Never -- -X POST -v \ -H "content-type: application/json" \ -H "ce-specversion: 1.0" \ -H "ce-source: cli" \ @@ -8,3 +8,4 @@ -H "ce-id: 1" \ -d '{"details":"ChannelDemo"}' \ http://ingress-to-one-kn-channel.sc2-demo.svc.cluster.local + diff --git a/tests/cold_starts_guest_pull.bats b/tests/cold_starts_guest_pull.bats index 43a19091..0f1e8306 100644 --- a/tests/cold_starts_guest_pull.bats +++ b/tests/cold_starts_guest_pull.bats @@ -28,19 +28,39 @@ snapshotter="guest-pull" # ------------------------------------------------------------------------------ @test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { - run_python_hello_world "${SC2_RUNTIME_CLASSES[0]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[0]}" + ' + + cleanup_python_hello_world } @test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { - run_python_hello_world "${SC2_RUNTIME_CLASSES[1]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[1]}" + ' + + cleanup_python_hello_world } @test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { - run_python_hello_world "${SC2_RUNTIME_CLASSES[2]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[2]}" + ' + + cleanup_python_hello_world } @test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { - run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_python_hello_world } # ------------------------------------------------------------------------------ @@ -48,17 +68,37 @@ snapshotter="guest-pull" # ------------------------------------------------------------------------------ @test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { - run_knative_hello_world "${SC2_RUNTIME_CLASSES[0]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[0]}" + ' + + cleanup_knative_hello_world } @test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { - run_knative_hello_world "${SC2_RUNTIME_CLASSES[1]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[1]}" + ' + + cleanup_knative_hello_world } @test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { - run_knative_hello_world "${SC2_RUNTIME_CLASSES[2]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[2]}" + ' + + cleanup_knative_hello_world } @test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { - run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_hello_world } diff --git a/tests/cold_starts_host_share.bats b/tests/cold_starts_host_share.bats index 89189201..55366214 100644 --- a/tests/cold_starts_host_share.bats +++ b/tests/cold_starts_host_share.bats @@ -28,7 +28,12 @@ snapshotter="host-share" # ------------------------------------------------------------------------------ @test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { - run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_python_hello_world } # ------------------------------------------------------------------------------ @@ -36,5 +41,10 @@ snapshotter="host-share" # ------------------------------------------------------------------------------ @test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { - run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_hello_world } diff --git a/tests/helloworld_knative_guest_pull.bats b/tests/helloworld_knative_guest_pull.bats index 3d767e35..28a46b09 100644 --- a/tests/helloworld_knative_guest_pull.bats +++ b/tests/helloworld_knative_guest_pull.bats @@ -26,17 +26,37 @@ snapshotter="guest-pull" # ------------------------------------------------------------------------------ @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { - run_knative_hello_world "${SC2_RUNTIME_CLASSES[0]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[0]}" + ' + + cleanup_knative_hello_world } @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { - run_knative_hello_world "${SC2_RUNTIME_CLASSES[1]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[1]}" + ' + + cleanup_knative_hello_world } @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { - run_knative_hello_world "${SC2_RUNTIME_CLASSES[2]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[2]}" + ' + + cleanup_knative_hello_world } @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { - run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_hello_world } diff --git a/tests/helloworld_knative_host_share.bats b/tests/helloworld_knative_host_share.bats index 1b764e5e..8a3879a7 100644 --- a/tests/helloworld_knative_host_share.bats +++ b/tests/helloworld_knative_host_share.bats @@ -30,6 +30,12 @@ snapshotter="host-share" @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { [[ "$SC2_TEE" == "tdx" ]] && skip "#142" - run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" + + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_hello_world } diff --git a/tests/helloworld_python_guest_pull.bats b/tests/helloworld_python_guest_pull.bats index ab9948c2..a0893481 100644 --- a/tests/helloworld_python_guest_pull.bats +++ b/tests/helloworld_python_guest_pull.bats @@ -26,17 +26,37 @@ snapshotter="guest-pull" # ------------------------------------------------------------------------------ @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { - run_python_hello_world "${SC2_RUNTIME_CLASSES[0]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[0]}" + ' + + cleanup_python_hello_world } @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { - run_python_hello_world "${SC2_RUNTIME_CLASSES[1]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[1]}" + ' + + cleanup_python_hello_world } @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { - run_python_hello_world "${SC2_RUNTIME_CLASSES[2]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[2]}" + ' + + cleanup_python_hello_world } @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { - run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_python_hello_world } diff --git a/tests/helloworld_python_host_share.bats b/tests/helloworld_python_host_share.bats index 9bf9bb48..69924ee5 100644 --- a/tests/helloworld_python_host_share.bats +++ b/tests/helloworld_python_host_share.bats @@ -29,7 +29,13 @@ snapshotter="host-share" # ------------------------------------------------------------------------------ @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { - [[ "$SC2_TEE" == "tdx" ]] && skip "Host-share not supported for TDX (#142)" - run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" + [[ "$SC2_TEE" == "tdx" ]] && skip "#142" + + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_python_hello_world } diff --git a/tests/knative_chaining_guest_pull.bats b/tests/knative_chaining_guest_pull.bats index 0e8d6b4d..d7486ac5 100644 --- a/tests/knative_chaining_guest_pull.bats +++ b/tests/knative_chaining_guest_pull.bats @@ -10,6 +10,9 @@ setup_file() { setup() { load utils/helpers.sh + + # Longer timeout for chaining tests + export SC2_TEST_TIMEOUT=180 } teardown() { @@ -28,24 +31,44 @@ snapshotter="guest-pull" @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[0]} snapshotter=${snapshotter}" { enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[0]}" - run_knative_chaining "${SC2_RUNTIME_CLASSES[0]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_chaining "${SC2_RUNTIME_CLASSES[0]}" + ' + + cleanup_knative_chaining } @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[1]} snapshotter=${snapshotter}" { enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[1]}" - run_knative_chaining "${SC2_RUNTIME_CLASSES[1]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_chaining "${SC2_RUNTIME_CLASSES[1]}" + ' + + cleanup_knative_chaining } @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[2]} snapshotter=${snapshotter}" { enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[2]}" - run_knative_chaining "${SC2_RUNTIME_CLASSES[2]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_chaining "${SC2_RUNTIME_CLASSES[2]}" + ' + + cleanup_knative_chaining } @test "${TEST_NAME}: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[3]}" restart_vm_cache - run_knative_chaining "${SC2_RUNTIME_CLASSES[3]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_chaining "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_chaining } diff --git a/tests/knative_chaining_host_share.bats b/tests/knative_chaining_host_share.bats index 57b256b8..09fe6f34 100644 --- a/tests/knative_chaining_host_share.bats +++ b/tests/knative_chaining_host_share.bats @@ -10,6 +10,9 @@ setup_file() { setup() { load utils/helpers.sh + + # Longer timeout for chaining tests + export SC2_TEST_TIMEOUT=120 } teardown() { @@ -34,6 +37,11 @@ snapshotter="host-share" enable_kata_annotation "default_memory" "${SC2_RUNTIME_CLASSES[3]}" restart_vm_cache - run_knative_chaining "${SC2_RUNTIME_CLASSES[3]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_chaining "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_chaining } diff --git a/tests/lazy_loading.bats b/tests/lazy_loading.bats index c6fd0e2a..3c9b5aa4 100644 --- a/tests/lazy_loading.bats +++ b/tests/lazy_loading.bats @@ -21,9 +21,19 @@ teardown() { snapshotter="guest-pull" @test "Test python lazy loading: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { - run_python_lazy_loading "${SC2_RUNTIME_CLASSES[3]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_python_lazy_loading "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_python_hello_world } @test "Test knative lazy loading: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { - run_knative_lazy_loading "${SC2_RUNTIME_CLASSES[3]}" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' + source ./tests/utils/helpers.sh + run_knative_lazy_loading "${SC2_RUNTIME_CLASSES[3]}" + ' + + cleanup_knative_hello_world } diff --git a/tests/setup_suite.bash b/tests/setup_suite.bash deleted file mode 100644 index fb73aea8..00000000 --- a/tests/setup_suite.bash +++ /dev/null @@ -1,3 +0,0 @@ -setup_suite() { - export BATS_TEST_TIMEOUT=${SC2_TEST_TIMEOUT:-90} -} diff --git a/tests/utils/env.sh b/tests/utils/env.sh index ee78236e..d0d117a6 100644 --- a/tests/utils/env.sh +++ b/tests/utils/env.sh @@ -24,6 +24,7 @@ fi # Useful constants INTERTEST_SLEEP_SECS=1 SC2_DEMO_NAMESPACE=sc2-demo +SC2_TEST_TIMEOUT=60 # Images PAUSE_IMAGE="registry.k8s.io/pause:3.8" diff --git a/tests/utils/helpers.sh b/tests/utils/helpers.sh index aa360a70..5cc0e156 100644 --- a/tests/utils/helpers.sh +++ b/tests/utils/helpers.sh @@ -41,6 +41,39 @@ set_snapshotter_mode() { restart_vm_cache } +# ------------------------------------------------------------------------------ +# Helper functions clean-up after executions +# ------------------------------------------------------------------------------ + +cleanup_knative_chaining() { + # Curl service + kubectl -n sc2-demo delete -l run=curl pod + + # First service: fan-out + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete service.serving.knative.dev coco-knative-chaining-one + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete -l apps.sc2.io/name=knative-chaining-one pod + + # Second service: job sink + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete JobSink.sinks.knative.dev coco-knative-chaining-two + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete -l apps.sc2.io/name=knative-chaining-two pod + + # Third service: fan-in + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete service.serving.knative.dev coco-knative-chaining-three + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete -l apps.sc2.io/name=knative-chaining-three pod +} + +cleanup_knative_hello_world() { + # Delete resources in order, as it seems to prevent deletion from being stuck + SERVICE_NAME="helloworld-knative" + POD_LABEL="apps.sc2.io/name=helloworld-py" + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete service.serving.knative.dev ${SERVICE_NAME} + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete -l ${POD_LABEL} pod +} + +cleanup_python_hello_world() { + ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete deployment coco-helloworld-py +} + # ------------------------------------------------------------------------------ # Helper functions to run specific workloads # ------------------------------------------------------------------------------ @@ -58,7 +91,7 @@ run_knative_chaining() { POD_LABEL="apps.sc2.io/name=knative-chaining-three" # Wait for pod 3 to be scaled down - until [ "$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} logs -l ${POD_LABEL} | grep 'cloudevent(s3): done!' | wc -l)" = "1" ]; do echo "Waiting for chain to finish..."; sleep 2; done + until [ "$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} logs -l ${POD_LABEL} | grep 'cloudevent(s3): done!' | wc -l)" = "1" ]; do echo "Waiting for chain to finish..." ; sleep 2; done } run_knative_hello_world() { @@ -71,13 +104,6 @@ run_knative_hello_world() { # Get the service URL service_url=$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] - - # Delete resources in order, as it seems to prevent deletion from being stuck - SERVICE_NAME="helloworld-knative" - POD_LABEL="apps.sc2.io/name=helloworld-py" - ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete service.serving.knative.dev ${SERVICE_NAME} - ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} wait --for=delete -l ${POD_LABEL} pod --timeout=60s - ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} } run_knative_lazy_loading() { @@ -90,13 +116,6 @@ run_knative_lazy_loading() { # Get the service URL service_url=$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} get ksvc helloworld-knative --output=custom-columns=URL:.status.url --no-headers) [ "$(curl --retry 3 ${service_url})" = "Hello World!" ] - - # Delete resources in order, as it seems to prevent deletion from being stuck - SERVICE_NAME="helloworld-knative" - POD_LABEL="apps.sc2.io/name=helloworld-py" - ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} delete service.serving.knative.dev ${SERVICE_NAME} - ${KUBECTL} -n ${SC2_DEMO_NAMESPACE} wait --for=delete -l ${POD_LABEL} pod --timeout=60s - ${KUBECTL} delete namespace ${SC2_DEMO_NAMESPACE} } run_python_hello_world() { @@ -114,11 +133,6 @@ run_python_hello_world() { # Get the pod's IP service_ip=$(${KUBECTL} get services -n ${SC2_DEMO_NAMESPACE} -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] - - envsubst < ./demo-apps/helloworld-py/deployment.yaml | ${KUBECTL} delete -f - - - # Wait for pod to be deleted - ${KUBECTL} wait --for=delete -n ${SC2_DEMO_NAMESPACE} -l ${POD_LABEL} pod --timeout=30s } run_python_lazy_loading() { @@ -136,9 +150,4 @@ run_python_lazy_loading() { # Get the pod's IP service_ip=$(${KUBECTL} get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] - - envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ${KUBECTL} delete -f - - - # Wait for pod to be deleted - ${KUBECTL} wait --for=delete -l ${POD_LABEL} pod --timeout=30s } From 6efa442e9b3c74aef95cd5ea89dbf696bb83cb0a Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Mar 2025 14:57:22 +0000 Subject: [PATCH 23/24] tests: fix python lazy loading --- tests/utils/helpers.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils/helpers.sh b/tests/utils/helpers.sh index 5cc0e156..f0eff804 100644 --- a/tests/utils/helpers.sh +++ b/tests/utils/helpers.sh @@ -144,10 +144,10 @@ run_python_lazy_loading() { export POD_LABEL="apps.sc2.io/name=helloworld-py" # Wait for pod to be ready - until [ "$(${KUBECTL} get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done + until [ "$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done sleep 1 # Get the pod's IP - service_ip=$(${KUBECTL} get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') + service_ip=$(${KUBECTL} -n ${SC2_DEMO_NAMESPACE} get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}') [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ] } From 33bb89c9ed08d0190190081a9047f02ee7eb81ff Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Mar 2025 17:08:27 +0000 Subject: [PATCH 24/24] tests: skip tdx for all host share --- tests/cold_starts_host_share.bats | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/cold_starts_host_share.bats b/tests/cold_starts_host_share.bats index 55366214..719f0799 100644 --- a/tests/cold_starts_host_share.bats +++ b/tests/cold_starts_host_share.bats @@ -28,6 +28,8 @@ snapshotter="host-share" # ------------------------------------------------------------------------------ @test "Test python cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + [[ "$SC2_TEE" == "tdx" ]] && skip "#142" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' source ./tests/utils/helpers.sh run_python_hello_world "${SC2_RUNTIME_CLASSES[3]}" @@ -41,6 +43,8 @@ snapshotter="host-share" # ------------------------------------------------------------------------------ @test "Test knative cold starts: runtime=${SC2_RUNTIME_CLASSES[3]} snapshotter=${snapshotter}" { + [[ "$SC2_TEE" == "tdx" ]] && skip "#142" + timeout "${SC2_TEST_TIMEOUT}" bash -c ' source ./tests/utils/helpers.sh run_knative_hello_world "${SC2_RUNTIME_CLASSES[3]}"