Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on:

jobs:
test:
runs-on: ubuntu-latest
runs-on: [self-hosted, linux, x64, kvm]
steps:
- uses: actions/checkout@v4

Expand All @@ -17,7 +17,10 @@ jobs:
- name: Install dependencies
run: |
set -xe
sudo apt-get install -y erofs-utils
if ! command -v mkfs.erofs &> /dev/null; then
sudo apt-get update
sudo apt-get install -y erofs-utils
fi
go mod download

# Avoids rate limits when running the tests
Expand All @@ -33,4 +36,3 @@ jobs:

- name: Build
run: make build

3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,6 @@ bin/**
tmp
tmp/**
.datadir

# Cloud Hypervisor binaries (embedded at build time)
lib/vmm/binaries/cloud-hypervisor/*/*/cloud-hypervisor
50 changes: 46 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
SHELL := /bin/bash
.PHONY: oapi-generate generate-wire generate-all dev build test install-tools gen-jwt
.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build test install-tools gen-jwt download-ch-binaries download-ch-spec ensure-ch-binaries

# Directory where local binaries will be installed
BIN_DIR ?= $(CURDIR)/bin
Expand Down Expand Up @@ -31,31 +31,72 @@ $(GODOTENV): | $(BIN_DIR)

install-tools: $(OAPI_CODEGEN) $(AIR) $(WIRE) $(GODOTENV)

# Download Cloud Hypervisor binaries
download-ch-binaries:
@echo "Downloading Cloud Hypervisor binaries..."
@mkdir -p lib/vmm/binaries/cloud-hypervisor/v48.0/{x86_64,aarch64}
@mkdir -p lib/vmm/binaries/cloud-hypervisor/v49.0/{x86_64,aarch64}
@echo "Downloading v48.0..."
@curl -L -o lib/vmm/binaries/cloud-hypervisor/v48.0/x86_64/cloud-hypervisor \
https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v48.0/cloud-hypervisor-static
@curl -L -o lib/vmm/binaries/cloud-hypervisor/v48.0/aarch64/cloud-hypervisor \
https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v48.0/cloud-hypervisor-static-aarch64
@echo "Downloading v49.0..."
@curl -L -o lib/vmm/binaries/cloud-hypervisor/v49.0/x86_64/cloud-hypervisor \
https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v49.0/cloud-hypervisor-static
@curl -L -o lib/vmm/binaries/cloud-hypervisor/v49.0/aarch64/cloud-hypervisor \
https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v49.0/cloud-hypervisor-static-aarch64
@chmod +x lib/vmm/binaries/cloud-hypervisor/v*/*/cloud-hypervisor
@echo "Binaries downloaded successfully"

# Download Cloud Hypervisor API spec
download-ch-spec:
@echo "Downloading Cloud Hypervisor API spec..."
@mkdir -p specs/cloud-hypervisor/api-v0.3.0
@curl -L -o specs/cloud-hypervisor/api-v0.3.0/cloud-hypervisor.yaml \
https://raw.githubusercontent.com/cloud-hypervisor/cloud-hypervisor/refs/tags/v48.0/vmm/src/api/openapi/cloud-hypervisor.yaml
@echo "API spec downloaded"

# Generate Go code from OpenAPI spec
oapi-generate: $(OAPI_CODEGEN)
@echo "Generating Go code from OpenAPI spec..."
$(OAPI_CODEGEN) -config ./oapi-codegen.yaml ./openapi.yaml
@echo "Formatting generated code..."
go fmt ./lib/oapi/oapi.go

# Generate Cloud Hypervisor client from their OpenAPI spec
generate-vmm-client: $(OAPI_CODEGEN)
@echo "Generating Cloud Hypervisor client from spec..."
$(OAPI_CODEGEN) -config ./oapi-codegen-vmm.yaml ./specs/cloud-hypervisor/api-v0.3.0/cloud-hypervisor.yaml
@echo "Formatting generated code..."
go fmt ./lib/vmm/vmm.go

# Generate wire dependency injection code
generate-wire: $(WIRE)
@echo "Generating wire code..."
cd ./cmd/api && $(WIRE)

# Generate all code
generate-all: oapi-generate generate-wire
generate-all: oapi-generate generate-vmm-client generate-wire

# Check if binaries exist, download if missing
.PHONY: ensure-ch-binaries
ensure-ch-binaries:
@if [ ! -f lib/vmm/binaries/cloud-hypervisor/v48.0/x86_64/cloud-hypervisor ]; then \
echo "Cloud Hypervisor binaries not found, downloading..."; \
$(MAKE) download-ch-binaries; \
fi

# Build the binary
build: | $(BIN_DIR)
build: ensure-ch-binaries | $(BIN_DIR)
go build -tags containers_image_openpgp -o $(BIN_DIR)/hypeman ./cmd/api

# Run in development mode with hot reload
dev: $(AIR)
$(AIR) -c .air.toml

# Run tests
test:
test: ensure-ch-binaries
go test -tags containers_image_openpgp -v -timeout 30s ./...

# Generate JWT token for testing
Expand All @@ -67,4 +108,5 @@ gen-jwt: $(GODOTENV)
clean:
rm -rf $(BIN_DIR)
rm -f lib/oapi/oapi.go
rm -f lib/vmm/vmm.go

9 changes: 7 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,18 @@ Run containerized workloads in VMs, powered by [Cloud Hypervisor](https://github

### Prerequisites

**Go 1.25.4+**, **Cloud Hypervisor**, **KVM**, **erofs-utils**
**Go 1.25.4+**, **KVM**, **erofs-utils**

```bash
cloud-hypervisor --version
mkfs.erofs --version
```

**KVM Access:** User must be in `kvm` group for VM access:
```bash
sudo usermod -aG kvm $USER
# Log out and back in, or use: newgrp kvm
```

### Configuration

#### Environment variables
Expand Down
138 changes: 138 additions & 0 deletions lib/vmm/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
# Cloud Hypervisor VMM Client

Thin Go wrapper around Cloud Hypervisor's HTTP API with embedded binaries.

Supports multiple versions of the Cloud Hypervisor VMM because instances are version-locked to cloud hypervisor if they are in standby. We can switch them to the latest version if we shutdown / reboot the VM.

Embeds the binaries to make it easier to install, instead of managing multiple versions of the Cloud Hypervisor CLI externally + configuring it.

## Features

- Embedded Cloud Hypervisor binaries of multiple versions
- Generated HTTP client from official OpenAPI spec
- Automatic binary extraction to data directory
- Unix socket communication
- Version detection and validation

## Requirements

**System:**
- Linux with KVM support (`/dev/kvm`)
- User must be in `kvm` group or run as root

**Add user to kvm group:**
```bash
sudo usermod -aG kvm $USER
# Then log out and back in, or use: sg kvm -c "your-command"
```

## Usage

### Start a VMM Process

```go
import "github.com/onkernel/hypeman/lib/vmm"

ctx := context.Background()
dataDir := "/var/lib/hypeman"
socketPath := "/tmp/vmm.sock"

// Start Cloud Hypervisor VMM (extracts binary if needed)
err := vmm.StartProcess(ctx, dataDir, vmm.V48_0, socketPath)
if err != nil {
log.Fatal(err)
}
```

### Connect to Existing VMM

```go
// Create client for existing VMM socket
client, err := vmm.NewVMM(socketPath)
if err != nil {
log.Fatal(err)
}

// All generated API methods available directly
resp, err := client.GetVmmPingWithResponse(ctx)
```

### Check Binary Version

```go
binaryPath, _ := vmm.GetBinaryPath(dataDir, vmm.V48_0)
version, err := vmm.ParseVersion(binaryPath)
fmt.Println(version) // "v48.0"
```

## Architecture

```
lib/vmm/
├── vmm.go # Generated from OpenAPI spec (DO NOT EDIT)
├── client.go # Thin wrapper: NewVMM, StartProcess
├── binaries.go # Binary embedding and extraction
├── version.go # Version parsing utilities
├── binaries/ # Embedded Cloud Hypervisor binaries
│ └── cloud-hypervisor/
│ ├── v48.0/
│ │ ├── x86_64/cloud-hypervisor (4.5MB)
│ │ └── aarch64/cloud-hypervisor (3.3MB)
│ └── v49.0/
│ ├── x86_64/cloud-hypervisor (4.5MB)
│ └── aarch64/cloud-hypervisor (3.3MB)
| # There will be additional versions in the future...
└── client_test.go # Tests with real Cloud Hypervisor
```

## Supported Versions

- Cloud Hypervisor v48.0 (API v0.3.0)
- Cloud Hypervisor v49.0 (API v0.3.0)

There may be additional versions in the future. Cloud hypervisor versions may update frequently, while the API updates less frequently.

## Regenerating Client

```bash
# Download latest spec (if needed)
make download-ch-spec

# Regenerate client from spec
make generate-vmm-client
```

## Testing

Tests run against real Cloud Hypervisor binaries (not mocked).

```bash
# Must be in kvm group
sg kvm -c "go test ./lib/vmm/..."
```

## Binary Extraction

Binaries are automatically extracted from embedded FS to:
```
{dataDir}/system/binaries/{version}/{arch}/cloud-hypervisor
```

Extraction happens once per version. Subsequent calls reuse the extracted binary.

## API Methods

All Cloud Hypervisor API methods available via embedded `*ClientWithResponses`:

- `GetVmmPingWithResponse()` - Check VMM health
- `ShutdownVMMWithResponse()` - Shutdown VMM
- `CreateVMWithResponse()` - Create VM configuration
- `BootVMWithResponse()` - Boot configured VM
- `PauseVMWithResponse()` - Pause running VM
- `ResumeVMWithResponse()` - Resume paused VM
- `VmSnapshotPutWithResponse()` - Create VM snapshot
- `VmRestorePutWithResponse()` - Restore from snapshot
- `VmInfoGetWithResponse()` - Get VM info
- And many more...

See generated `vmm.go` for full API surface.
63 changes: 63 additions & 0 deletions lib/vmm/binaries.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package vmm

import (
"embed"
"fmt"
"os"
"path/filepath"
"runtime"
)

//go:embed binaries/cloud-hypervisor/v48.0/x86_64/cloud-hypervisor
//go:embed binaries/cloud-hypervisor/v48.0/aarch64/cloud-hypervisor
//go:embed binaries/cloud-hypervisor/v49.0/x86_64/cloud-hypervisor
//go:embed binaries/cloud-hypervisor/v49.0/aarch64/cloud-hypervisor
var binaryFS embed.FS

type CHVersion string

const (
V48_0 CHVersion = "v48.0"
V49_0 CHVersion = "v49.0"
)

var SupportedVersions = []CHVersion{V48_0, V49_0}

// ExtractBinary extracts the embedded Cloud Hypervisor binary to the data directory
func ExtractBinary(dataDir string, version CHVersion) (string, error) {
arch := runtime.GOARCH
if arch == "amd64" {
arch = "x86_64"
}

embeddedPath := fmt.Sprintf("binaries/cloud-hypervisor/%s/%s/cloud-hypervisor", version, arch)
extractPath := filepath.Join(dataDir, "system", "binaries", string(version), arch, "cloud-hypervisor")

// Check if already extracted
if _, err := os.Stat(extractPath); err == nil {
return extractPath, nil
}

// Create directory
if err := os.MkdirAll(filepath.Dir(extractPath), 0755); err != nil {
return "", fmt.Errorf("create binaries dir: %w", err)
}

// Read embedded binary
data, err := binaryFS.ReadFile(embeddedPath)
if err != nil {
return "", fmt.Errorf("read embedded binary: %w", err)
}

// Write to filesystem
if err := os.WriteFile(extractPath, data, 0755); err != nil {
return "", fmt.Errorf("write binary: %w", err)
}

return extractPath, nil
}

// GetBinaryPath returns path to extracted binary, extracting if needed
func GetBinaryPath(dataDir string, version CHVersion) (string, error) {
return ExtractBinary(dataDir, version)
}
3 changes: 3 additions & 0 deletions lib/vmm/binaries/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Ignore all binaries
cloud-hypervisor

Loading