diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6ff568c9..47889f2a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -11,7 +11,7 @@ on: jobs: prepare: - runs-on: [self-hosted] + runs-on: [ram] outputs: vaccelrt_sha: ${{ steps.vars.outputs.vaccelrt_sha }} firecracker_sha: ${{ steps.vars.outputs.firecracker_sha }} @@ -53,7 +53,7 @@ jobs: VSOCK_PLUGIN_SHA: ${{ needs.prepare.outputs.vsock_plugin_sha }} AGENT_SHA: ${{ needs.prepare.outputs.agent_sha }} JOB_TYPE: ${{ matrix.build_type }} - ARCH: ${{ matrix.arch }} + ARCHITECTURE: ${{ matrix.arch }} strategy: matrix: @@ -62,12 +62,17 @@ jobs: fail-fast: false steps: + - name: Cleanup previous runs + run: | + sudo rm -rf ${{ github.workspace }}/* + sudo rm -rf ${{ github.workspace }}/.??* + - uses: actions/checkout@v2 - name: Install packages run: | sudo apt update && sudo apt install -y \ - subversion coreutils rsync iproute2 libfdt-dev + subversion coreutils rsync iproute2 libfdt-dev qemu-system-x86-64 parted debootstrap bc udev libssl-dev libelf-dev - name: Setup vars id: vars @@ -84,11 +89,84 @@ jobs: access-key: ${{ env.NBFC_S3_ACCESS }} secret-key: ${{ env.NBFC_S3_SECRET }} local-path: /github/workspace/artifacts/opt/ - remote-path: nbfc-assets/github/vaccelrt/${{env.VACCELRT_SHA}}/${{env.ARCH}}/${{ env.JOB_TYPE }}/opt/ + remote-path: nbfc-assets/github/vaccelrt/${{env.VACCELRT_SHA}}/${{env.ARCHITECTURE}}/${{ env.JOB_TYPE }}/opt/ env: ACTION_UID: ${{ steps.vars.outputs.uid }} ACTION_GID: ${{ steps.vars.outputs.gid }} + - name: Download go & kata containers + run: | + if [[ $ARCHITECTURE == "x86_64" ]] + then + export ARCH=amd64 + else + export ARCH=arm64 + fi + #wget https://golang.org/dl/go1.15.11.linux-$ARCH.tar.gz + #sudo tar -zxf go1.15.11.linux-$ARCH.tar.gz -C /usr/local/ + git submodule update --init kata-containers + git submodule update --init virtio-accel + + - name: Checkout private tools + uses: actions/checkout@v2 + with: + repository: nubificus/vaccel-go-runtime + ref: main + token: ${{ secrets.NBFC_BUILDER_TOKEN }} # `GitHub_PAT` is a secret that contains your PAT + path: kata-containers/src/runtime/vendor/github.com/nubificus/vaccel-go-runtime + + - name: Build and install kata-containers + env: + GOROOT: /usr/local/go + GOPATH: /home/runner/go + run: | + export PATH=$GOROOT/bin:$PATH + export PATH=$GOPATH/bin:$PATH + if [[ $ARCHITECTURE == "x86_64" ]] + then + export ARCH=amd64 + export ARCH_KERNEL=x86_64 + else + export ARCH_KERNEL=arm64 + fi + cd kata-containers + make -C src/runtime + sudo cp src/runtime/containerd-shim-kata-v2 ${{ github.workspace }}/artifacts/opt/bin/ + sudo cp src/runtime/cli/config/configuration-fc.toml ${{ github.workspace }}/artifacts/opt/share/ + cd tools/packaging/kernel + echo "CONFIG_MODULES=y" > configs/fragments/$ARCH_KERNEL/vaccel.conf + echo "CONFIG_MODULE_UNLOAD=y" >> configs/fragments/$ARCH_KERNEL/vaccel.conf + echo "CONFIG_MODULE_SIG=y" >> configs/fragments/$ARCH_KERNEL/vaccel.conf + VERSION=`./build-kernel.sh setup 2>&1 |grep Kernel\ version\: | awk '{print $4}'` + ./build-kernel.sh build + if [[ $ARCHITECTURE == "x86_64" ]] + then + sudo cp kata-linux*/vmlinux ${{ github.workspace }}/artifacts/opt/share/vmlinux-kata-fc + else + sudo cp kata-linux*/arch/arm64/boot/Image ${{ github.workspace }}/artifacts/opt/share/vmlinux-kata-fc + fi + cd ${{ github.workspace }}/virtio-accel + git checkout fix_kzfree + KDIR=${{ github.workspace }}/kata-containers/tools/packaging/kernel/kata-linux*/ make ZC=0 ARCH=$ARCH_KERNEL + sudo cp virtio_accel.ko ${{ github.workspace }}/artifacts/opt/share/virtio_accel-kata.ko + if [[ $ARCHITECTURE != "blah" ]] + then + cd ${{ github.workspace }}/kata-containers/tools/osbuilder/rootfs-builder/ + export ROOTFS_DIR=$PWD/rootfs + rm -rf $ROOTFS_DIR + script -fec 'sudo su root -c ". /opt/cargo/env && GOPATH=$GOPATH GOROOT=$GOROOT PATH=$GOROOT/bin:$PATH RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/cargo PATH=/opt/cargo/bin:$PATH ./rootfs.sh -r $ROOTFS_DIR ubuntu"' + MODULES_DIR=${ROOTFS_DIR}/lib/modules/$VERSION + sudo mkdir -p ${MODULES_DIR} + sudo cp ${{ github.workspace }}/virtio-accel/virtio_accel.ko $MODULES_DIR + sudo touch ${MODULES_DIR}/modules.builtin + sudo touch ${MODULES_DIR}/modules.order + echo "virtio_accel" > virtio_accel.conf + sudo cp virtio_accel.conf ${ROOTFS_DIR}/etc/modules-load.d/ + sudo chroot ${ROOTFS_DIR} /sbin/depmod $VERSION + cd ../image-builder + script -fec 'sudo bash -x ./image_builder.sh $ROOTFS_DIR' + sudo cp kata-containers.img ${{ github.workspace }}/artifacts/opt/share/ + fi - name: Fetch Firecracker uses: cloudkernels/minio-download@master @@ -96,7 +174,7 @@ jobs: url: https://s3.nubificus.co.uk access-key: ${{ env.NBFC_S3_ACCESS }} secret-key: ${{ env.NBFC_S3_SECRET }} - remote-path: nbfc-assets/github/firecracker/${{ env.FIRECRACKER_SHA }}/${{env.ARCH}}/${{ steps.vars.outputs.firecracker_build_type }}/firecracker + remote-path: nbfc-assets/github/firecracker/${{ env.FIRECRACKER_SHA }}/${{env.ARCHITECTURE}}/${{ steps.vars.outputs.firecracker_build_type }}/firecracker local-path: /github/workspace/artifacts/opt/bin/ env: ACTION_UID: ${{ steps.vars.outputs.uid }} @@ -108,7 +186,7 @@ jobs: url: https://s3.nubificus.co.uk access-key: ${{ env.NBFC_S3_ACCESS }} secret-key: ${{ env.NBFC_S3_SECRET }} - remote-path: nbfc-assets/github/virtio-accel/${{ env.VIRTIO_SHA }}/${{env.ARCH}}/linux/vmlinux + remote-path: nbfc-assets/github/virtio-accel/${{ env.VIRTIO_SHA }}/${{env.ARCHITECTURE}}/linux/vmlinux local-path: /github/workspace/artifacts/opt/share/ env: ACTION_UID: ${{ steps.vars.outputs.uid }} @@ -120,7 +198,7 @@ jobs: url: https://s3.nubificus.co.uk access-key: ${{ env.NBFC_S3_ACCESS }} secret-key: ${{ env.NBFC_S3_SECRET }} - remote-path: nbfc-assets/github/virtio-accel/${{ env.VIRTIO_SHA }}/${{env.ARCH}}/linux/virtio_accel.ko + remote-path: nbfc-assets/github/virtio-accel/${{ env.VIRTIO_SHA }}/${{env.ARCHITECTURE}}/linux/virtio_accel.ko local-path: /github/workspace/artifacts/opt/share/ env: ACTION_UID: ${{ steps.vars.outputs.uid }} @@ -133,7 +211,7 @@ jobs: access-key: ${{ env.NBFC_S3_ACCESS }} secret-key: ${{ env.NBFC_S3_SECRET }} local-path: /github/workspace/artifacts/opt/lib/ - remote-path: nbfc-assets/github/virtio-plugin/${{env.VIRTIO_PLUGIN_SHA}}/${{env.ARCH}}/${{ env.JOB_TYPE }}/libvaccel-virtio.so + remote-path: nbfc-assets/github/virtio-plugin/${{env.VIRTIO_PLUGIN_SHA}}/${{env.ARCHITECTURE}}/${{ env.JOB_TYPE }}/libvaccel-virtio.so env: ACTION_UID: ${{ steps.vars.outputs.uid }} ACTION_GID: ${{ steps.vars.outputs.gid }} @@ -145,7 +223,7 @@ jobs: access-key: ${{ env.NBFC_S3_ACCESS }} secret-key: ${{ env.NBFC_S3_SECRET }} local-path: /github/workspace/artifacts/opt/lib/ - remote-path: nbfc-assets/github/vsock-plugin/${{env.VSOCK_PLUGIN_SHA}}/${{env.ARCH}}/${{ env.JOB_TYPE }}/libvaccel-vsock.so + remote-path: nbfc-assets/github/vsock-plugin/${{env.VSOCK_PLUGIN_SHA}}/${{env.ARCHITECTURE}}/${{ env.JOB_TYPE }}/libvaccel-vsock.so env: ACTION_UID: ${{ steps.vars.outputs.uid }} ACTION_GID: ${{ steps.vars.outputs.gid }} @@ -157,7 +235,7 @@ jobs: access-key: ${{ env.NBFC_S3_ACCESS }} secret-key: ${{ env.NBFC_S3_SECRET }} local-path: /github/workspace/artifacts/opt/bin/ - remote-path: nbfc-assets/github/vaccel-agent/${{env.AGENT_SHA}}/${{env.ARCH}}/${{ steps.vars.outputs.agent_build_type }}/vaccelrt-agent + remote-path: nbfc-assets/github/vaccel-agent/${{env.AGENT_SHA}}/${{env.ARCHITECTURE}}/${{ steps.vars.outputs.agent_build_type }}/vaccelrt-agent env: ACTION_UID: ${{ steps.vars.outputs.uid }} ACTION_GID: ${{ steps.vars.outputs.gid }} @@ -180,6 +258,10 @@ jobs: working-directory: ${{ github.workspace }}/artifacts/opt run: | sudo rm -f fc.sock + sudo rm -f /usr/local/share/imagenet-models/networks fc.sock + sudo mkdir -p /usr/local/share/imagenet-models + sudo chmod a+rw /usr/local/share/imagenet-models + ln -s ${{ github.workspace }}/artifacts/opt/share/networks /usr/local/share/imagenet-models/networks sudo VACCEL_BACKENDS=${{ github.workspace }}/artifacts/opt/lib/libvaccel-noop.so \ LD_LIBRARY_PATH=${{ github.workspace }}/artifacts/opt/lib:/usr/local/lib:/usr/lib \ VACCEL_DEBUG_LEVEL=4 \ @@ -189,7 +271,7 @@ jobs: --seccomp-level 0 & - name: Test Firecracker is up - run: ping -c 20 172.42.0.2 + run: sudo ping -c 20 172.42.0.2 - name: Test Firecracker ssh timeout-minutes: 2 @@ -214,28 +296,42 @@ jobs: --agent-prefix ${{ github.workspace }}/artifacts/opt/bin - name: Pack latest release - if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/master' }} working-directory: ${{ github.workspace }}/artifacts/opt run: | cp ${{github.workspace}}/conf/{config_virtio_accel.json,config_vsock.json} share/ + if [[ $ARCHITECTURE == "x86_64" ]] + then + cp /opt/cargo/bin/vaccelrt-agent bin/ + fi zip -r ${{github.workspace}}/vaccel_${{matrix.arch}}_${{matrix.build_type}}.zip bin/ include/ lib/ \ share/config_virtio_accel.json share/config_vsock.json \ - share/fc_test share/fc_test.pub \ share/rootfs.img share/virtio_accel.ko share/vmlinux \ + share/kata-containers.img share/virtio_accel-kata.ko share/vmlinux-kata-fc \ + share/configuration-fc.toml \ share/vaccel.pc + - name: Extract branch name + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + shell: bash + run: echo "##[set-output name=branch;]$(echo ${{ github.head_ref }})" + id: extract_branch + + - name: Upload latest master release to s3 - if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: cloudkernels/minio-upload@master with: url: https://s3.nubificus.co.uk access-key: ${{ env.NBFC_S3_ACCESS }} secret-key: ${{ env.NBFC_S3_SECRET }} - remote-path: nbfc-assets/github/vaccel/master/${{env.ARCH}}/${{env.JOB_TYPE}}/ + remote-path: nbfc-assets/github/vaccel/${{ steps.extract_branch.outputs.branch }}/${{env.ARCHITECTURE}}/${{env.JOB_TYPE}}/ local-path: /github/workspace/vaccel_${{matrix.arch}}_${{matrix.build_type}}.zip - name: Cleanup run if: ${{ always() }} + #if: ${{ github.ref == 'refs/heads/main' }} run: | sudo rm -rf ${{ github.workspace }}/* sudo rm -rf ${{ github.workspace }}/.??* @@ -267,27 +363,35 @@ jobs: echo "::set-output name=uid::$(id -u)" echo "::set-output name=gid::$(id -g)" + - name: Extract branch name + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + shell: bash + run: echo "##[set-output name=branch;]$(echo ${{ github.head_ref }})" + id: extract_branch + + - name: Download artifacts - if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: cloudkernels/minio-download@master with: url: https://s3.nubificus.co.uk access-key: ${{ env.NBFC_S3_ACCESS }} secret-key: ${{ env.NBFC_S3_SECRET }} - remote-path: nbfc-assets/github/vaccel/master/ + remote-path: nbfc-assets/github/vaccel/${{ steps.extract_branch.outputs.branch }}/ local-path: /github/workspace/master/ env: ACTION_UID: ${{ steps.vars.outputs.uid }} ACTION_GID: ${{ steps.vars.outputs.gid }} - + - name: Update the release - if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: marvinpinto/action-automatic-releases@latest with: repo_token: ${{ secrets.GITHUB_TOKEN }} - automatic_release_tag: "latest" + automatic_release_tag: ${{ steps.extract_branch.outputs.branch }} prerelease: true - title: "Latest master build" + title: "Latest ${{ steps.extract_branch.outputs.branch }} build" files: | master/vaccel_x86_64_Debug.zip master/vaccel_x86_64_Release.zip diff --git a/.gitmodules b/.gitmodules index 3da9121c..1e70834b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,9 +4,6 @@ [submodule "firecracker"] path = firecracker url = git@github.com:cloudkernels/firecracker.git -[submodule "virtio-accel"] - path = virtio-accel - url = git@github.com:cloudkernels/virtio-accel.git [submodule "vaccel-grpc"] path = vaccel-grpc url = git@github.com:cloudkernels/vaccel-grpc.git @@ -28,3 +25,10 @@ [submodule "bindings/rust-bindings"] path = bindings/rust-bindings url = git@github.com:cloudkernels/vaccel-bindings.git +[submodule "kata-containers"] + path = kata-containers + url = https://github.com/nubificus/kata-containers + branch = vaccel-release +[submodule "virtio-accel"] + path = virtio-accel + url = https://github.com/cloudkernels/virtio-accel diff --git a/dockerfiles/ubuntu/latest/Dockerfile b/dockerfiles/ubuntu/latest/Dockerfile index 3fb1028d..3635463b 100644 --- a/dockerfiles/ubuntu/latest/Dockerfile +++ b/dockerfiles/ubuntu/latest/Dockerfile @@ -31,12 +31,14 @@ RUN echo "export VACCEL_BACKENDS=/opt/vaccel/lib/libvaccel-virtio.so" >> /root/. # Enable ssh server RUN systemctl enable ssh -COPY fc_test.pub id_rsa.pub -RUN mkdir /root/.ssh/ && \ - cat id_rsa.pub >> /root/.ssh/authorized_keys && \ - chmod 0700 /root/.ssh && \ - chmod 0600 /root/.ssh/authorized_keys && \ - chown -R root:root /root/.ssh +#COPY fc_test.pub id_rsa.pub +RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config +RUN echo "PermitEmptyPasswords yes" >> /etc/ssh/sshd_config +#RUN mkdir /root/.ssh/ && \ +# cat id_rsa.pub >> /root/.ssh/authorized_keys && \ +# chmod 0700 /root/.ssh && \ +# chmod 0600 /root/.ssh/authorized_keys && \ +# chown -R root:root /root/.ssh # Disable root password RUN sed s/root\:x\:/root\:\:/ -i /etc/passwd diff --git a/kata-containers b/kata-containers new file mode 160000 index 00000000..8d27480b --- /dev/null +++ b/kata-containers @@ -0,0 +1 @@ +Subproject commit 8d27480b1fb243633b658344e8dbfe4b15472de3 diff --git a/scripts/build_rootfs.sh b/scripts/build_rootfs.sh index 1e58c42a..87894d86 100755 --- a/scripts/build_rootfs.sh +++ b/scripts/build_rootfs.sh @@ -77,10 +77,10 @@ build() { cd ${BUILD_DIR}/rootfs # Create RSA key to rootfs - ssh-keygen -t rsa -f fc_test -N "" + #ssh-keygen -t rsa -f fc_test -N "" # Create root filesystem - DOCKER_BUILDKIT=1 docker build \ + DOCKER_BUILDKIT=1 docker build --no-cache \ --network=host \ -t vaccel-rootfs \ --build-arg "KERNEL_VERSION=4.20.0" \ @@ -96,7 +96,7 @@ build() { sudo mount rootfs.img $mnt ok_or_die "Could not mount rootfs" - sudo rsync -aogxvPH rootfs/* $mnt + sudo rsync -aogxPH rootfs/* $mnt sudo chown -R root:root $mnt/root ok_or_die "Could not populate rootfs" @@ -110,7 +110,7 @@ build() { sudo rmdir $mnt cp rootfs.img ${INSTALL_PREFIX}/share/ - cp fc_test* ${INSTALL_PREFIX}/share/ + #cp fc_test* ${INSTALL_PREFIX}/share/ cp -r imagenet/{networks,images} ${INSTALL_PREFIX}/share/ } diff --git a/scripts/test_virtio.sh b/scripts/test_virtio.sh index 62d331e3..de4b9ef8 100755 --- a/scripts/test_virtio.sh +++ b/scripts/test_virtio.sh @@ -13,7 +13,7 @@ SSH_TIMEOUT=300 FC_IP="172.42.0.2" # Path to ssh private key -SSH_KEY=$(pwd)/opt/share/fc_test +#SSH_KEY=$(pwd)/opt/share/fc_test # script name for logging LOG_NAME="$(basename $0)" @@ -28,7 +28,7 @@ print_help() { echo " -v|--vaccel Directory of vAccel installation (default: '/opt/vaccel')" echo " -t|--timeout Timeout in seconds to wait response from Firecracker (default: 300)" echo " -a|--ip-address Address of Firecracker VM" - echo " -i|--ssh-key RSA key to use for SSHing inside the VM" + echo " -i|--ssh-key RSA key to use for SSHing inside the VM (not currently used)" echo "" } @@ -37,7 +37,8 @@ run_test() { in_fc_cmd="$in_fc_cmd VACCEL_BACKENDS=$VACCEL_PATH/lib/libvaccel-virtio.so" in_fc_cmd="$in_fc_cmd $VACCEL_PATH/bin/classify /root/images/dog_0.jpg 1" - ssh -o StrictHostKeyChecking=no -i $SSH_KEY root@$FC_IP $in_fc_cmd + ssh -o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null \ + -o UserKnownHostsFile=/dev/null root@$FC_IP $in_fc_cmd } main() { diff --git a/scripts/test_vsock.sh b/scripts/test_vsock.sh index 8a1f07ab..815ee7ff 100755 --- a/scripts/test_vsock.sh +++ b/scripts/test_vsock.sh @@ -13,7 +13,7 @@ SSH_TIMEOUT=300 FC_IP="172.42.0.2" # Path to ssh private key -SSH_KEY=$(pwd)/opt/share/fc_test +#SSH_KEY=$(pwd)/opt/share/fc_test # vsock socket to use inside the VM VACCEL_VSOCK="vsock://2:2048" @@ -43,7 +43,7 @@ print_help() { echo " -v|--vaccel Directory of vAccel installation (default: '/opt/vaccel')" echo " -t|--timeout Timeout in seconds to wait response from Firecracker (default: 300)" echo " -a|--ip-address Address of Firecracker VM" - echo " -i|--ssh-key RSA key to use for SSHing inside the VM" + echo " -i|--ssh-key RSA key to use for SSHing inside the VM (not currently used)" echo " -p|--plugin Plugin to use for agent" echo " --agent-prefix Location of the agent binary" echo " --vsock Vsock socket to use inside the VM" @@ -74,7 +74,8 @@ run_test() { launch_agent ok_or_die "Could not launch agent" - ssh -o StrictHostKeyChecking=no -i $SSH_KEY root@$FC_IP $in_fc_cmd + ssh -o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null \ + -o UserKnownHostsFile=/dev/null root@$FC_IP $in_fc_cmd retval=$? kill_agent diff --git a/virtio-accel b/virtio-accel index 05b5922b..9bf3a041 160000 --- a/virtio-accel +++ b/virtio-accel @@ -1 +1 @@ -Subproject commit 05b5922bd2d1d32a50b19cbf7589b57aed880578 +Subproject commit 9bf3a0412130bcc24d9517c4554f41da4ce8cadc