Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/linters/urunc-dict.txt
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,7 @@ DEFROUTE
blockfile
thinpool
vcpus
cpus
Virtiofs
virtiofs
virtiofsd
Expand Down
50 changes: 48 additions & 2 deletions docs/hypervisor-support.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@ somewhere in the `$PATH`.
VMMs use hardware-assisted virtualization technologies in order to create a
Virtual Machine (VM) where a guest OS will execute. It is one of the most
widely used technology for providing strong isolation in multi-tenant
environments. For the time being `urunc` supports 3 types of such VMMs: 1)
environments. For the time being `urunc` supports 4 types of such VMMs: 1)
[Qemu](https://www.qemu.org/), 2)
[Firecracker](https://firecracker-microvm.github.io/) and 3) [Solo5-hvt](https://github.com/Solo5/solo5).
[Firecracker](https://firecracker-microvm.github.io/), 3)
[Cloud Hypervisor](https://www.cloudhypervisor.org/) and 4) [Solo5-hvt](https://github.com/Solo5/solo5).

### Qemu

Expand Down Expand Up @@ -139,6 +140,51 @@ An example unikernel:
sudo nerdctl run --rm -ti --runtime io.containerd.urunc.v2 harbor.nbfc.io/nubificus/urunc/nginx-firecracker-unikraft-initrd:latest
```

### Cloud Hypervisor

[Cloud Hypervisor](https://www.cloudhypervisor.org/) is an open-source Virtual
Machine Monitor (VMM) that runs on top of the KVM hypervisor. It is part of the
rust-vmm project and works in a similar way to Firecracker. Cloud Hypervisor
provides a modern, secure, and efficient VMM with a focus on cloud workloads.
It supports virtio devices and offers fast boot times with minimal overhead.

#### Installing Cloud Hypervisor

Cloud Hypervisor can be installed by downloading a pre-built binary from the
[releases page](https://github.com/cloud-hypervisor/cloud-hypervisor/releases).

```bash
ARCH="$(uname -m)"
VERSION="v43.0"
release_url="https://github.com/cloud-hypervisor/cloud-hypervisor/releases"
curl -L ${release_url}/download/${VERSION}/cloud-hypervisor-static-${ARCH} -o cloud-hypervisor
chmod +x cloud-hypervisor
sudo mv cloud-hypervisor /usr/local/bin/
```

It is important to note that `urunc` expects to find the `cloud-hypervisor`
binary located in the `$PATH` and named `cloud-hypervisor`.

#### Cloud Hypervisor and `urunc`

In the case of [Cloud Hypervisor](https://www.cloudhypervisor.org/), `urunc`
makes use of its `virtio-net` device to provide network support for the
unikernel through a tap device. `urunc` can also leverage Cloud Hypervisor's
initramfs option to provide the unikernel with an initial RamFS. Cloud
Hypervisor supports virtio-block for storage and virtiofs for shared
filesystems between the host and guest.

Supported unikernel frameworks with `urunc`:

- [Unikraft](../unikernel-support#unikraft)
- [Linux](../unikernel-support#linux)

An example unikernel:

```bash
sudo nerdctl run --rm -ti --runtime io.containerd.urunc.v2 harbor.nbfc.io/nubificus/urunc/nginx-cloud-hypervisor-unikraft-initrd:latest
```

### Solo5-hvt

[Solo5-hvt](https://github.com/Solo5/solo5) is a lightweight, high-performance
Expand Down
147 changes: 147 additions & 0 deletions pkg/unikontainers/hypervisors/cloud_hypervisor.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
// Copyright (c) 2023-2025, Nubificus LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hypervisors

import (
"fmt"
"strings"
"syscall"

"github.com/urunc-dev/urunc/pkg/unikontainers/types"
)

const (
CloudHypervisorVmm VmmType = "cloud-hypervisor"
CloudHypervisorBinary string = "cloud-hypervisor"
)

type CloudHypervisor struct {
binaryPath string
binary string
}

func (ch *CloudHypervisor) Stop(pid int) error {
return killProcess(pid)
}

func (ch *CloudHypervisor) Ok() error {
return nil
}

// UsesKVM returns true as Cloud Hypervisor is a KVM-based VMM
func (ch *CloudHypervisor) UsesKVM() bool {
return true
}

// SupportsSharedfs returns true as Cloud Hypervisor supports virtiofs
func (ch *CloudHypervisor) SupportsSharedfs(fsType string) bool {
switch fsType {
case "virtio", "virtiofs":
return true
default:
return false
}
}

func (ch *CloudHypervisor) Path() string {
return ch.binaryPath
}

func (ch *CloudHypervisor) Execve(args types.ExecArgs, ukernel types.Unikernel) error {
chMem := BytesToStringMB(args.MemSizeB)

// Start building the command
exArgs := []string{ch.binaryPath}

// Memory configuration
exArgs = append(exArgs, "--memory", fmt.Sprintf("size=%sM,shared=on", chMem))

// CPU configuration
if args.VCPUs > 0 {
exArgs = append(exArgs, "--cpus", fmt.Sprintf("boot=%d", args.VCPUs))
}

// Kernel path
exArgs = append(exArgs, "--kernel", args.UnikernelPath)

// Console configuration - disable graphical output
exArgs = append(exArgs, "--console", "off", "--serial", "tty")

// Seccomp configuration
if args.Seccomp {
exArgs = append(exArgs, "--seccomp", "true")
} else {
exArgs = append(exArgs, "--seccomp", "false")
}

// Network configuration
if args.Net.TapDev != "" {
netCli := ukernel.MonitorNetCli(args.Net.TapDev, args.Net.MAC)
if netCli == "" {
// Default network configuration for Cloud Hypervisor
exArgs = append(exArgs, "--net", fmt.Sprintf("tap=%s,mac=%s", args.Net.TapDev, args.Net.MAC))
} else {
exArgs = append(exArgs, strings.Split(strings.TrimSpace(netCli), " ")...)
}
}

// Block device configuration
blockArgs := ukernel.MonitorBlockCli()
for _, blockArg := range blockArgs {
if blockArg.ExactArgs != "" {
exArgs = append(exArgs, strings.Split(strings.TrimSpace(blockArg.ExactArgs), " ")...)
} else if blockArg.Path != "" {
diskArg := fmt.Sprintf("path=%s", blockArg.Path)
if blockArg.ID != "" {
diskArg += fmt.Sprintf(",id=%s", blockArg.ID)
}
exArgs = append(exArgs, "--disk", diskArg)
}
}

// Initrd configuration
if args.InitrdPath != "" {
exArgs = append(exArgs, "--initramfs", args.InitrdPath)
}

// Check for extra initrd from unikernel monitor args
extraMonArgs := ukernel.MonitorCli()
if extraMonArgs.ExtraInitrd != "" && args.InitrdPath == "" {
exArgs = append(exArgs, "--initramfs", extraMonArgs.ExtraInitrd)
}

switch args.Sharedfs.Type {
case "virtiofs":
exArgs = append(exArgs, "--fs", "tag=fs0,socket=/tmp/vhostqemu")
default:
// No shared filesystem
}

if args.VAccelType == "vsock" {
exArgs = append(exArgs, "--vsock", fmt.Sprintf("cid=%d,socket=%s/vaccel.sock",
args.VSockDevID, args.VSockDevPath))
}

if extraMonArgs.OtherArgs != "" {
exArgs = append(exArgs, strings.Split(strings.TrimSpace(extraMonArgs.OtherArgs), " ")...)
}

// Add the command line arguments for the kernel
exArgs = append(exArgs, "--cmdline", args.Command)

vmmLog.WithField("cloud-hypervisor command", exArgs).Debug("Ready to execve cloud-hypervisor")

return syscall.Exec(ch.Path(), exArgs, args.Environment) //nolint: gosec
}
6 changes: 6 additions & 0 deletions pkg/unikontainers/hypervisors/vmm.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ var vmmFactories = map[VmmType]VMMFactory{
binary: FirecrackerBinary,
createFunc: func(binary, binaryPath string) types.VMM { return &Firecracker{binary: binary, binaryPath: binaryPath} },
},
CloudHypervisorVmm: {
binary: CloudHypervisorBinary,
createFunc: func(binary, binaryPath string) types.VMM {
return &CloudHypervisor{binary: binary, binaryPath: binaryPath}
},
},
}

func NewVMM(vmmType VmmType, monitors map[string]types.MonitorConfig) (vmm types.VMM, err error) {
Expand Down
2 changes: 1 addition & 1 deletion pkg/unikontainers/ipc.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ func AwaitMessage(listener *net.UnixListener, expectedMessage IPCMessage) error
}
msg := string(buf[0:n])
if msg != string(expectedMessage) {
return fmt.Errorf("received unexpected message: %s", msg)
return fmt.Errorf("received unexpected message: %s (expected %s)", msg, expectedMessage)
}
return nil
}
37 changes: 28 additions & 9 deletions pkg/unikontainers/mount.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func createTmpfs(monRootfs string, path string, flags uint64, mode string, size

if mode == "1777" {
// sonarcloud:go:S2612 -- This is a tmpfs mount point, sticky bit 1777 is required (like /tmp), controlled path, safe by design
err := os.Chmod(path, 01777) // NOSONAR
err := os.Chmod(dstPath, 01777) // NOSONAR
if err != nil {
return fmt.Errorf("failed to chmod %s: %w", path, err)
}
Expand Down Expand Up @@ -116,6 +116,23 @@ func setupDev(monRootfs string, devPath string) error {
// Create the new device node
err = unix.Mknod(dstPath, devStat.Mode, int(newDev)) //nolint: gosec
if err != nil {
// If mknod fails because of permissions (e.g. in a container), try to bind mount it
if errors.Is(err, unix.EPERM) || errors.Is(err, unix.EACCES) {
uniklog.Warnf("failed to make device node %s: %v. Fallback to bind mount.", dstPath, err)
// Create an empty file to be used as mount point
f, err1 := os.Create(dstPath)
if err1 != nil && !os.IsExist(err1) {
return fmt.Errorf("failed to create mount point for device %s: %w", dstPath, err1)
}
if f != nil {
f.Close()
}
err = unix.Mount(devPath, dstPath, "", unix.MS_BIND, "")
if err != nil {
return fmt.Errorf("failed to bind mount device %s: %w", devPath, err)
}
return nil
}
return fmt.Errorf("failed to make device node %s: %w", dstPath, err)
}

Expand Down Expand Up @@ -188,15 +205,17 @@ func fileFromHost(monRootfs string, hostPath string, target string, mFlags int,
}
}

// Set up the permissions and ownership of the original file.
err = unix.Chmod(dstPath, fileInfo.Mode)
if err != nil {
return fmt.Errorf("failed to chmod %s: %w", dstPath, err)
}
if withCopy {
// Set up the permissions and ownership of the original file.
err = unix.Chmod(dstPath, fileInfo.Mode)
if err != nil {
return fmt.Errorf("failed to chmod %s: %w", dstPath, err)
}

err = os.Chown(dstPath, int(fileInfo.Uid), int(fileInfo.Gid))
if err != nil {
return fmt.Errorf("failed to chown %s: %w", dstPath, err)
err = os.Chown(dstPath, int(fileInfo.Uid), int(fileInfo.Gid))
if err != nil {
return fmt.Errorf("failed to chown %s: %w", dstPath, err)
}
}

// The initial MS_BIND won't change the mount options, we need to do a
Expand Down
9 changes: 5 additions & 4 deletions pkg/unikontainers/urunc_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,11 @@ func defaultTimestampsConfig() UruncTimestamps {

func defaultMonitorsConfig() map[string]types.MonitorConfig {
return map[string]types.MonitorConfig{
"qemu": {DefaultMemoryMB: 256, DefaultVCPUs: 1},
"hvt": {DefaultMemoryMB: 256, DefaultVCPUs: 1},
"spt": {DefaultMemoryMB: 256, DefaultVCPUs: 1},
"firecracker": {DefaultMemoryMB: 256, DefaultVCPUs: 1},
"qemu": {DefaultMemoryMB: 256, DefaultVCPUs: 1},
"hvt": {DefaultMemoryMB: 256, DefaultVCPUs: 1},
"spt": {DefaultMemoryMB: 256, DefaultVCPUs: 1},
"firecracker": {DefaultMemoryMB: 256, DefaultVCPUs: 1},
"cloud-hypervisor": {DefaultMemoryMB: 256, DefaultVCPUs: 1},
}
}

Expand Down
5 changes: 3 additions & 2 deletions pkg/unikontainers/urunc_config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -438,11 +438,12 @@ func TestDefaultConfigs(t *testing.T) {
t.Parallel()
config := defaultMonitorsConfig()

assert.Len(t, config, 4)
assert.Len(t, config, 5)
assert.Contains(t, config, "qemu")
assert.Contains(t, config, "hvt")
assert.Contains(t, config, "spt")
assert.Contains(t, config, "firecracker")
assert.Contains(t, config, "cloud-hypervisor")

// Check default values for each monitor
for _, hvConfig := range config {
Expand Down Expand Up @@ -472,7 +473,7 @@ func TestDefaultConfigs(t *testing.T) {
assert.False(t, config.Log.Syslog)
assert.False(t, config.Timestamps.Enabled)
assert.Equal(t, testTimestampsPath, config.Timestamps.Destination)
assert.Len(t, config.Monitors, 4)
assert.Len(t, config.Monitors, 5)
assert.Len(t, config.ExtraBins, 1)
})

Expand Down
7 changes: 7 additions & 0 deletions tests/e2e/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ func commonCmdExec(command string) (output string, err error) {
var stderrBuf bytes.Buffer

params := strings.Fields(command)
fmt.Printf("Executing command: %s\n", command)
cmd := exec.Command(params[0], params[1:]...) //nolint:gosec
cmd.Stderr = &stderrBuf
outBytes, err := cmd.Output()
Expand Down Expand Up @@ -224,7 +225,13 @@ func findValOfKey(searchArea string, key string) (string, error) {
return "", err
}
match := r.FindString(searchArea)
if match == "" {
return "", fmt.Errorf("key %s not found in search area", key)
}
keyValMatch := strings.Split(match, ":")
if len(keyValMatch) < 2 {
return "", fmt.Errorf("invalid format for key %s: %s", key, match)
}
val := strings.ReplaceAll(keyValMatch[1], "\"", "")
return strings.TrimSpace(val), nil
}
5 changes: 4 additions & 1 deletion tests/e2e/nerdctl.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,10 @@ func (i *nerdctlInfo) inspectCAndGet(key string) (string, error) {
return commonInspectCAndGet(nerdctlName, i.containerID, key)
}

func (i *nerdctlInfo) inspectPAndGet(string) (string, error) {
func (i *nerdctlInfo) inspectPAndGet(key string) (string, error) {
if key == "pid" || key == "Pid" {
return i.inspectCAndGet("Pid")
}
// Not supported by nerdctl
return "", errToolDoesNotSupport
}
Loading