From 5b402dfc18c3d45c04fbae3f5a68bc25860c7e6f Mon Sep 17 00:00:00 2001 From: gangwgr Date: Wed, 28 Jan 2026 12:12:09 +0530 Subject: [PATCH] Add Vault KMSv2 plugin and update socket path to kms.sock - Add Vault KMSv2 plugin for e2e encryption tests - main.go: KMS v2 gRPC server connecting to Vault Transit - Dockerfile: Multi-stage build for the plugin image - go.mod/go.sum: Go module dependencies - build-image.sh: Script to build the container image - deploy-vault-kms-plugin.sh: Script to deploy on OpenShift - README.md: Documentation and usage instructions - Update socket path from socket.sock to kms.sock - vault-plugin files updated - k8s_mock_kms_plugin_daemonset.yaml updated --- .../kms/k8s_mock_kms_plugin_deployer_test.go | 74 ++++ .../encryption/kms/vault-plugin/Dockerfile | 31 ++ .../encryption/kms/vault-plugin/README.md | 139 +++++++ .../kms/vault-plugin/build-image.sh | 134 ++++++ .../kms/vault-plugin/cleanup-vault-kms.sh | 27 ++ .../vault-plugin/deploy-vault-kms-plugin.sh | 387 ++++++++++++++++++ .../encryption/kms/vault-plugin/go.mod | 17 + .../encryption/kms/vault-plugin/go.sum | 13 + .../encryption/kms/vault-plugin/main.go | 353 ++++++++++++++++ .../kms/vault-plugin/setup-vault-kms.sh | 78 ++++ .../kms/vault-plugin/status-vault-kms.sh | 27 ++ 11 files changed, 1280 insertions(+) create mode 100644 test/library/encryption/kms/k8s_mock_kms_plugin_deployer_test.go create mode 100644 test/library/encryption/kms/vault-plugin/Dockerfile create mode 100644 test/library/encryption/kms/vault-plugin/README.md create mode 100755 test/library/encryption/kms/vault-plugin/build-image.sh create mode 100755 test/library/encryption/kms/vault-plugin/cleanup-vault-kms.sh create mode 100755 test/library/encryption/kms/vault-plugin/deploy-vault-kms-plugin.sh create mode 100644 test/library/encryption/kms/vault-plugin/go.mod create mode 100644 test/library/encryption/kms/vault-plugin/go.sum create mode 100644 test/library/encryption/kms/vault-plugin/main.go create mode 100755 test/library/encryption/kms/vault-plugin/setup-vault-kms.sh create mode 100755 test/library/encryption/kms/vault-plugin/status-vault-kms.sh diff --git a/test/library/encryption/kms/k8s_mock_kms_plugin_deployer_test.go b/test/library/encryption/kms/k8s_mock_kms_plugin_deployer_test.go new file mode 100644 index 0000000000..86d0a16c00 --- /dev/null +++ b/test/library/encryption/kms/k8s_mock_kms_plugin_deployer_test.go @@ -0,0 +1,74 @@ +package kms + +import ( + "context" + "testing" + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +// getKubeClient returns a Kubernetes client for testing. +func getKubeClient(t *testing.T) kubernetes.Interface { + t.Helper() + + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + + config, err := kubeConfig.ClientConfig() + if err != nil { + t.Fatalf("Failed to get kubeconfig: %v", err) + } + + client, err := kubernetes.NewForConfig(config) + if err != nil { + t.Fatalf("Failed to create kubernetes client: %v", err) + } + + return client +} + +// TestDeployUpstreamMockKMSPlugin tests deploying the KMS plugin. +func TestDeployUpstreamMockKMSPlugin(t *testing.T) { + if testing.Short() { + t.Skip("Skipping KMS deploy test in short mode") + } + + kubeClient := getKubeClient(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + namespace := WellKnownUpstreamMockKMSPluginNamespace + image := WellKnownUpstreamMockKMSPluginImage + + t.Logf("Deploying KMS plugin with namespace=%s, image=%s", namespace, image) + + // Deploy and get cleanup function + cleanup := DeployUpstreamMockKMSPlugin(ctx, t, kubeClient, namespace, image) + defer cleanup() + + t.Log("KMS plugin deployed successfully!") +} + +// TestDeployUpstreamMockKMSPluginNoCleanup deploys the KMS plugin without cleanup. +func TestDeployUpstreamMockKMSPluginNoCleanup(t *testing.T) { + if testing.Short() { + t.Skip("Skipping KMS deploy test in short mode") + } + + kubeClient := getKubeClient(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + namespace := WellKnownUpstreamMockKMSPluginNamespace + image := WellKnownUpstreamMockKMSPluginImage + + t.Logf("Deploying KMS plugin with namespace=%s, image=%s (no cleanup)", namespace, image) + + // Deploy without calling cleanup + _ = DeployUpstreamMockKMSPlugin(ctx, t, kubeClient, namespace, image) + + t.Log("KMS plugin deployed successfully! Resources left in cluster.") +} diff --git a/test/library/encryption/kms/vault-plugin/Dockerfile b/test/library/encryption/kms/vault-plugin/Dockerfile new file mode 100644 index 0000000000..fb197afc0d --- /dev/null +++ b/test/library/encryption/kms/vault-plugin/Dockerfile @@ -0,0 +1,31 @@ +# Build stage +FROM golang:1.22-alpine AS builder + +WORKDIR /app + +# Install git for go mod download +RUN apk add --no-cache git ca-certificates + +# Copy source files +COPY go.mod main.go ./ + +# Download dependencies and generate go.sum +RUN go mod tidy + +# Build the binary +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ + -ldflags="-w -s" \ + -o /vault-kms-plugin . + +# Runtime stage +FROM alpine:3.19 + +RUN apk add --no-cache ca-certificates + +COPY --from=builder /vault-kms-plugin /usr/local/bin/vault-kms-plugin + +# Create socket directory +RUN mkdir -p /var/run/kmsplugin + +ENTRYPOINT ["/usr/local/bin/vault-kms-plugin"] +CMD ["-listen-addr=unix:///var/run/kmsplugin/kms.sock"] diff --git a/test/library/encryption/kms/vault-plugin/README.md b/test/library/encryption/kms/vault-plugin/README.md new file mode 100644 index 0000000000..1bea4daf21 --- /dev/null +++ b/test/library/encryption/kms/vault-plugin/README.md @@ -0,0 +1,139 @@ +# Vault KMS v2 Plugin Deployer + +This directory contains scripts for deploying a Vault-based KMS v2 plugin on OpenShift. + +## Architecture + +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ API Server │────▶│ Vault KMS │────▶│ HashiCorp │ +│ (kube-api) │ │ Plugin │ │ Vault │ +│ │ │ (DaemonSet) │ │ (Transit) │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ │ │ + │ │ │ + Encryption Unix Socket AppRole Auth + Config /var/run/kmsplugin Transit Engine +``` + +## Prerequisites + +1. **HashiCorp Vault** deployed with Transit engine enabled +2. **AppRole credentials** (Role ID and Secret ID) +3. **oc CLI** logged into OpenShift cluster + +## Quick Start + +### Step 1: Setup Vault (if not already done) + +Use the Vault setup script from [gangwgr/kms-setup](https://github.com/gangwgr/kms-setup): + +```bash +curl -sL https://raw.githubusercontent.com/gangwgr/kms-setup/main/vault-kms-setup/setup-vault-transit-kms.sh | bash +``` + +This will output the AppRole credentials needed for the KMS plugin. + +### Step 2: Deploy KMS Plugin + +```bash +./deploy-vault-kms-plugin.sh \ + --vault-addr http://vault.vault.svc.cluster.local:8200 \ + --role-id \ + --secret-id +``` + +Or using environment variables: + +```bash +export VAULT_ADDR="http://vault.vault.svc.cluster.local:8200" +export VAULT_ROLE_ID="abc123..." +export VAULT_SECRET_ID="xyz789..." +./deploy-vault-kms-plugin.sh +``` + +### Step 3: Check Status + +```bash +./deploy-vault-kms-plugin.sh --status +``` + +### Step 4: Cleanup + +```bash +./deploy-vault-kms-plugin.sh --cleanup +``` + +## Configuration Options + +| Option | Environment Variable | Default | Description | +|--------|---------------------|---------|-------------| +| `--vault-addr` | `VAULT_ADDR` | Auto-detect | Vault server address | +| `--vault-key` | `VAULT_TRANSIT_KEY` | `kubernetes-encryption` | Transit key name | +| `--role-id` | `VAULT_ROLE_ID` | Required | AppRole Role ID | +| `--secret-id` | `VAULT_SECRET_ID` | Required | AppRole Secret ID | +| `--namespace` | `KMS_NAMESPACE` | `openshift-kms-plugin` | Plugin namespace | +| `--image` | `KMS_PLUGIN_IMAGE` | `quay.io/openshifttest/vault-kms-plugin:latest` | Plugin image | + +## What Gets Deployed + +1. **Namespace**: `openshift-kms-plugin` (with privileged pod security) +2. **ServiceAccount**: `vault-kms-plugin` +3. **RoleBinding**: Grants privileged SCC access +4. **Secret**: Vault AppRole credentials +5. **ConfigMap**: Plugin and Vault configuration +6. **DaemonSet**: KMS plugin running on control-plane nodes + +## Socket Path + +The KMS plugin listens on: `unix:///var/run/kmsplugin/kms.sock` + +This path is mounted as a hostPath volume so the API server can access it. + +## Differences from Mock KMS Plugin + +| Feature | Mock KMS Plugin | Vault KMS Plugin | +|---------|-----------------|------------------| +| Backend | SoftHSM (local) | HashiCorp Vault | +| External Dependencies | None | Vault server | +| Key Management | Local PKCS11 | Vault Transit | +| Use Case | Testing | Production-like testing | +| Authentication | None | AppRole | + +## Troubleshooting + +### Check plugin logs + +```bash +oc logs -n openshift-kms-plugin -l app=vault-kms-plugin +``` + +### Verify socket exists + +```bash +oc exec -n openshift-kms-plugin -- ls -la /var/run/kmsplugin/ +``` + +### Test Vault connectivity from plugin pod + +```bash +oc exec -n openshift-kms-plugin -- curl -s $VAULT_ADDR/v1/sys/health +``` + +### Check AppRole authentication + +```bash +oc exec -n openshift-kms-plugin -- \ + curl -s -X POST $VAULT_ADDR/v1/auth/approle/login \ + -d '{"role_id":"","secret_id":""}' +``` + +## Files + +- `deploy-vault-kms-plugin.sh` - Main deployment script +- `README.md` - This documentation + +## Related + +- [Mock KMS Plugin](../k8s-mock-plugin/) - Simple mock plugin using SoftHSM +- [Vault Setup Script](https://github.com/gangwgr/kms-setup) - Sets up Vault Transit engine diff --git a/test/library/encryption/kms/vault-plugin/build-image.sh b/test/library/encryption/kms/vault-plugin/build-image.sh new file mode 100755 index 0000000000..2a76b81ee8 --- /dev/null +++ b/test/library/encryption/kms/vault-plugin/build-image.sh @@ -0,0 +1,134 @@ +#!/bin/bash + +# Build Vault KMS Plugin Image +# This script builds the Vault KMS v2 plugin container image + +set -euo pipefail + +# Configuration +IMAGE_NAME="${IMAGE_NAME:-vault-kms-plugin}" +IMAGE_TAG="${IMAGE_TAG:-quay.io/openshifttest/vault-kms-plugin:latest}" +CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-podman}" + +# Colors +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +usage() { + cat < /dev/null; then + log_error "${CONTAINER_RUNTIME} is not installed" + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${SCRIPT_DIR}" + +log_info "Building Vault KMS plugin image..." +log_info "Image tag: ${IMAGE_TAG}" +log_info "Container runtime: ${CONTAINER_RUNTIME}" + +# Download dependencies +log_info "Downloading Go dependencies..." +go mod download 2>/dev/null || go mod tidy + +# Build image +log_info "Building container image..." +${CONTAINER_RUNTIME} build \ + --platform linux/amd64 \ + -t "${IMAGE_TAG}" \ + -f Dockerfile \ + . + +log_info "Image built successfully: ${IMAGE_TAG}" + +# Push if requested +if [ "$PUSH" = true ]; then + log_info "Pushing image to registry..." + ${CONTAINER_RUNTIME} push "${IMAGE_TAG}" + log_info "Image pushed successfully!" +fi + +echo "" +log_info "==========================================" +log_info "Build Complete!" +log_info "==========================================" +echo "" +echo "Image: ${IMAGE_TAG}" +echo "" +echo "To push manually:" +echo " ${CONTAINER_RUNTIME} push ${IMAGE_TAG}" +echo "" +echo "To run locally (for testing):" +echo " ${CONTAINER_RUNTIME} run -it --rm ${IMAGE_TAG} --help" +echo "" +echo "To deploy on OpenShift:" +echo " ./deploy-vault-kms-plugin.sh --image ${IMAGE_TAG} \\" +echo " --vault-addr http://vault.vault.svc:8200 \\" +echo " --role-id --secret-id " diff --git a/test/library/encryption/kms/vault-plugin/cleanup-vault-kms.sh b/test/library/encryption/kms/vault-plugin/cleanup-vault-kms.sh new file mode 100755 index 0000000000..386669e429 --- /dev/null +++ b/test/library/encryption/kms/vault-plugin/cleanup-vault-kms.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Cleanup Vault KMS v2 Setup + +set -e + +echo "=== Cleaning up Vault KMS v2 ===" + +# Cleanup KMS plugin +echo "[1/2] Removing KMS plugin..." +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" +./deploy-vault-kms-plugin.sh --cleanup 2>/dev/null || true + +# Cleanup Vault (optional) +read -p "Also remove Vault? (y/N) " -n 1 -r +echo +if [[ $REPLY =~ ^[Yy]$ ]]; then + echo "[2/2] Removing Vault..." + helm uninstall vault -n vault 2>/dev/null || true + oc delete namespace vault --wait=false 2>/dev/null || true + echo "Vault removed." +else + echo "[2/2] Keeping Vault." +fi + +echo "" +echo "=== Cleanup Complete ===" diff --git a/test/library/encryption/kms/vault-plugin/deploy-vault-kms-plugin.sh b/test/library/encryption/kms/vault-plugin/deploy-vault-kms-plugin.sh new file mode 100755 index 0000000000..cfe88d69f2 --- /dev/null +++ b/test/library/encryption/kms/vault-plugin/deploy-vault-kms-plugin.sh @@ -0,0 +1,387 @@ +#!/bin/bash + +# Vault KMS v2 Plugin Deployer for OpenShift +# This deploys a KMS v2 plugin that connects to HashiCorp Vault Transit engine +# +# Prerequisites: +# - Vault Transit engine setup (run setup-vault-transit-kms.sh first) +# - AppRole credentials (Role ID and Secret ID) +# - oc CLI logged into OpenShift cluster + +set -e + +# Colors +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +log_step() { + echo -e "${BLUE}[STEP]${NC} $1" +} + +# Configuration - can be overridden by environment variables +KMS_NAMESPACE="${KMS_NAMESPACE:-openshift-kms-plugin}" +KMS_PLUGIN_IMAGE="${KMS_PLUGIN_IMAGE:-quay.io/openshifttest/vault-kms-plugin:latest}" +KMS_SOCKET_PATH="${KMS_SOCKET_PATH:-/var/run/kmsplugin/kms.sock}" + +# Vault configuration - MUST be provided +VAULT_ADDR="${VAULT_ADDR:-}" +VAULT_TRANSIT_KEY="${VAULT_TRANSIT_KEY:-kubernetes-encryption}" +VAULT_ROLE_ID="${VAULT_ROLE_ID:-}" +VAULT_SECRET_ID="${VAULT_SECRET_ID:-}" +VAULT_NAMESPACE="${VAULT_NAMESPACE:-vault}" + +usage() { + cat < /dev/null; then + log_error "oc CLI is not installed" + exit 1 + fi + + if ! oc whoami &> /dev/null; then + log_error "Not logged into OpenShift cluster" + exit 1 + fi + + log_info "Prerequisites satisfied" +} + +# Cleanup function +cleanup() { + log_step "Cleaning up Vault KMS plugin..." + + # Delete DaemonSet + oc delete daemonset vault-kms-plugin -n ${KMS_NAMESPACE} 2>/dev/null || true + + # Delete Secret + oc delete secret vault-kms-plugin-credentials -n ${KMS_NAMESPACE} 2>/dev/null || true + + # Delete ServiceAccount + oc delete serviceaccount vault-kms-plugin -n ${KMS_NAMESPACE} 2>/dev/null || true + + # Delete RoleBinding + oc delete rolebinding vault-kms-plugin -n ${KMS_NAMESPACE} 2>/dev/null || true + + # Delete Namespace + oc delete namespace ${KMS_NAMESPACE} --wait=false 2>/dev/null || true + + log_info "Cleanup complete" +} + +# Status function +status() { + log_step "Checking Vault KMS plugin status..." + + echo "" + log_info "Namespace: ${KMS_NAMESPACE}" + + if ! oc get namespace ${KMS_NAMESPACE} &>/dev/null; then + log_warn "Namespace ${KMS_NAMESPACE} does not exist" + return + fi + + echo "" + log_info "DaemonSet:" + oc get daemonset vault-kms-plugin -n ${KMS_NAMESPACE} 2>/dev/null || log_warn "DaemonSet not found" + + echo "" + log_info "Pods:" + oc get pods -n ${KMS_NAMESPACE} -l app=vault-kms-plugin 2>/dev/null || log_warn "No pods found" + + echo "" + log_info "Pod Logs (last 20 lines from first pod):" + FIRST_POD=$(oc get pods -n ${KMS_NAMESPACE} -l app=vault-kms-plugin -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + if [ -n "$FIRST_POD" ]; then + oc logs -n ${KMS_NAMESPACE} ${FIRST_POD} --tail=20 2>/dev/null || log_warn "Could not get logs" + fi + + echo "" + log_info "Socket check on first pod:" + if [ -n "$FIRST_POD" ]; then + oc exec -n ${KMS_NAMESPACE} ${FIRST_POD} -- ls -la /var/run/kmsplugin/ 2>/dev/null || log_warn "Could not check socket" + fi +} + +# Deploy function +deploy() { + # Validate required parameters + if [ -z "$VAULT_ADDR" ]; then + # Try to auto-detect from Vault namespace + if oc get namespace ${VAULT_NAMESPACE} &>/dev/null; then + VAULT_ADDR="http://vault.${VAULT_NAMESPACE}.svc.cluster.local:8200" + log_info "Auto-detected Vault address: ${VAULT_ADDR}" + else + log_error "VAULT_ADDR is required. Use --vault-addr or set VAULT_ADDR env var" + exit 1 + fi + fi + + if [ -z "$VAULT_ROLE_ID" ] || [ -z "$VAULT_SECRET_ID" ]; then + log_error "VAULT_ROLE_ID and VAULT_SECRET_ID are required" + log_info "Run setup-vault-transit-kms.sh first to get credentials" + exit 1 + fi + + log_step "Deploying Vault KMS v2 plugin..." + log_info "Namespace: ${KMS_NAMESPACE}" + log_info "Vault Address: ${VAULT_ADDR}" + log_info "Transit Key: ${VAULT_TRANSIT_KEY}" + log_info "Image: ${KMS_PLUGIN_IMAGE}" + + # Step 1: Create Namespace + log_step "Creating namespace..." + cat </dev/null || echo "0") + READY=$(oc get daemonset vault-kms-plugin -n ${KMS_NAMESPACE} -o jsonpath='{.status.numberReady}' 2>/dev/null || echo "0") + + if [ "$DESIRED" -gt 0 ] && [ "$READY" -eq "$DESIRED" ]; then + log_info "DaemonSet ready: ${READY}/${DESIRED} pods" + break + fi + + log_info "Waiting... ${READY}/${DESIRED} pods ready" + sleep 5 + done + + # Final status + echo "" + log_info "==========================================" + log_info "Vault KMS Plugin Deployment Complete!" + log_info "==========================================" + + status + + echo "" + log_info "Next Steps:" + echo " 1. Verify plugin is connecting to Vault (check logs above)" + echo " 2. Configure API server to use KMS encryption" + echo " 3. Test encryption with: oc create secret generic test-secret --from-literal=key=value" + echo "" + log_info "Socket path for API server config: unix://${KMS_SOCKET_PATH}" +} + +# Main +check_prerequisites + +if [ "$CLEANUP" = true ]; then + cleanup + exit 0 +fi + +if [ "$STATUS" = true ]; then + status + exit 0 +fi + +deploy diff --git a/test/library/encryption/kms/vault-plugin/go.mod b/test/library/encryption/kms/vault-plugin/go.mod new file mode 100644 index 0000000000..33a1c07216 --- /dev/null +++ b/test/library/encryption/kms/vault-plugin/go.mod @@ -0,0 +1,17 @@ +module vault-kms-plugin + +go 1.22 + +require ( + google.golang.org/grpc v1.60.1 + k8s.io/kms v0.29.0 +) + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect + google.golang.org/protobuf v1.32.0 // indirect +) diff --git a/test/library/encryption/kms/vault-plugin/go.sum b/test/library/encryption/kms/vault-plugin/go.sum new file mode 100644 index 0000000000..ec2b3c6de7 --- /dev/null +++ b/test/library/encryption/kms/vault-plugin/go.sum @@ -0,0 +1,13 @@ +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0/go.mod h1:FUoWkonphQm3RhTS+kOEhF8h0iDpm4tdXolVCeZ9KKA= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +k8s.io/kms v0.29.0/go.mod h1:mB0f9HLxRXeXUfHfn1A7rpwOlzXI1gIWu86z6buNoYA= diff --git a/test/library/encryption/kms/vault-plugin/main.go b/test/library/encryption/kms/vault-plugin/main.go new file mode 100644 index 0000000000..b18e9e2712 --- /dev/null +++ b/test/library/encryption/kms/vault-plugin/main.go @@ -0,0 +1,353 @@ +// Vault KMS v2 Plugin for Kubernetes +// This plugin implements the KMS v2 gRPC interface and uses HashiCorp Vault Transit engine +// for encryption/decryption operations. + +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "flag" + "fmt" + "io" + "net" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "google.golang.org/grpc" + kmsapi "k8s.io/kms/apis/v2" +) + +const ( + apiVersion = "v2" + pluginName = "vault-kms-plugin" +) + +// Config holds the Vault KMS plugin configuration +type Config struct { + VaultAddress string `json:"vaultAddress"` + TransitKeyName string `json:"transitKeyName"` + AuthMethod string `json:"authMethod"` + RoleIDFile string `json:"roleIdFile"` + SecretIDFile string `json:"secretIdFile"` + Token string `json:"token"` + TLSSkipVerify bool `json:"tlsSkipVerify"` +} + +// VaultKMSPlugin implements the KMS v2 plugin interface +type VaultKMSPlugin struct { + kmsapi.UnimplementedKeyManagementServiceServer + config Config + httpClient *http.Client + token string +} + +func main() { + listenAddr := flag.String("listen-addr", "unix:///var/run/kmsplugin/kms.sock", "gRPC listen address") + configFile := flag.String("config-file", "/etc/vault-config/vault-config.json", "Path to config file") + flag.Parse() + + // Load configuration + config, err := loadConfig(*configFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to load config: %v\n", err) + os.Exit(1) + } + + // Create plugin + plugin, err := NewVaultKMSPlugin(config) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to create plugin: %v\n", err) + os.Exit(1) + } + + // Parse listen address + network, address := parseListenAddr(*listenAddr) + + // Remove existing socket if it exists + if network == "unix" { + os.Remove(address) + } + + // Create listener + listener, err := net.Listen(network, address) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to listen on %s: %v\n", *listenAddr, err) + os.Exit(1) + } + defer listener.Close() + + fmt.Printf("Vault KMS plugin listening on %s\n", *listenAddr) + fmt.Printf("Vault address: %s\n", config.VaultAddress) + fmt.Printf("Transit key: %s\n", config.TransitKeyName) + + // Create gRPC server + server := grpc.NewServer() + kmsapi.RegisterKeyManagementServiceServer(server, plugin) + + // Handle shutdown + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + + go func() { + <-stop + fmt.Println("Shutting down...") + server.GracefulStop() + }() + + // Serve + if err := server.Serve(listener); err != nil { + fmt.Fprintf(os.Stderr, "Failed to serve: %v\n", err) + os.Exit(1) + } +} + +func loadConfig(path string) (Config, error) { + var config Config + + // Try to load from file + data, err := os.ReadFile(path) + if err != nil { + // Fall back to environment variables + config.VaultAddress = os.Getenv("VAULT_ADDR") + config.TransitKeyName = os.Getenv("VAULT_TRANSIT_KEY") + if config.TransitKeyName == "" { + config.TransitKeyName = "kubernetes-encryption" + } + config.AuthMethod = "approle" + config.RoleIDFile = "/etc/vault-credentials/role-id" + config.SecretIDFile = "/etc/vault-credentials/secret-id" + config.TLSSkipVerify = true + + if config.VaultAddress == "" { + return config, fmt.Errorf("VAULT_ADDR not set and config file not found: %v", err) + } + return config, nil + } + + if err := json.Unmarshal(data, &config); err != nil { + return config, fmt.Errorf("failed to parse config: %v", err) + } + + return config, nil +} + +func parseListenAddr(addr string) (network, address string) { + if strings.HasPrefix(addr, "unix://") { + return "unix", strings.TrimPrefix(addr, "unix://") + } + return "tcp", addr +} + +// NewVaultKMSPlugin creates a new Vault KMS plugin +func NewVaultKMSPlugin(config Config) (*VaultKMSPlugin, error) { + plugin := &VaultKMSPlugin{ + config: config, + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + } + + // Authenticate with Vault + if err := plugin.authenticate(); err != nil { + return nil, fmt.Errorf("failed to authenticate with Vault: %v", err) + } + + return plugin, nil +} + +func (p *VaultKMSPlugin) authenticate() error { + switch p.config.AuthMethod { + case "approle": + return p.authenticateAppRole() + case "token": + p.token = p.config.Token + return nil + default: + // Try to read token from environment + p.token = os.Getenv("VAULT_TOKEN") + if p.token == "" { + return p.authenticateAppRole() + } + return nil + } +} + +func (p *VaultKMSPlugin) authenticateAppRole() error { + roleID, err := os.ReadFile(p.config.RoleIDFile) + if err != nil { + return fmt.Errorf("failed to read role ID: %v", err) + } + + secretID, err := os.ReadFile(p.config.SecretIDFile) + if err != nil { + return fmt.Errorf("failed to read secret ID: %v", err) + } + + // Login with AppRole + loginData := map[string]string{ + "role_id": strings.TrimSpace(string(roleID)), + "secret_id": strings.TrimSpace(string(secretID)), + } + + jsonData, _ := json.Marshal(loginData) + url := fmt.Sprintf("%s/v1/auth/approle/login", p.config.VaultAddress) + + resp, err := p.httpClient.Post(url, "application/json", strings.NewReader(string(jsonData))) + if err != nil { + return fmt.Errorf("failed to login: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("login failed: %s - %s", resp.Status, string(body)) + } + + var result struct { + Auth struct { + ClientToken string `json:"client_token"` + } `json:"auth"` + } + + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return fmt.Errorf("failed to decode login response: %v", err) + } + + p.token = result.Auth.ClientToken + fmt.Println("Successfully authenticated with Vault using AppRole") + return nil +} + +// Status returns the status of the KMS plugin +func (p *VaultKMSPlugin) Status(ctx context.Context, req *kmsapi.StatusRequest) (*kmsapi.StatusResponse, error) { + // Check Vault health + url := fmt.Sprintf("%s/v1/sys/health", p.config.VaultAddress) + resp, err := p.httpClient.Get(url) + if err != nil { + return &kmsapi.StatusResponse{ + Version: apiVersion, + Healthz: "error", + KeyId: p.config.TransitKeyName, + }, nil + } + defer resp.Body.Close() + + healthz := "ok" + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusTooManyRequests { + healthz = "error" + } + + return &kmsapi.StatusResponse{ + Version: apiVersion, + Healthz: healthz, + KeyId: p.config.TransitKeyName, + }, nil +} + +// Encrypt encrypts the plaintext using Vault Transit engine +func (p *VaultKMSPlugin) Encrypt(ctx context.Context, req *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) { + // Base64 encode the plaintext + b64Plaintext := base64.StdEncoding.EncodeToString(req.Plaintext) + + // Call Vault Transit encrypt + encryptData := map[string]string{ + "plaintext": b64Plaintext, + } + + jsonData, _ := json.Marshal(encryptData) + url := fmt.Sprintf("%s/v1/transit/encrypt/%s", p.config.VaultAddress, p.config.TransitKeyName) + + httpReq, err := http.NewRequestWithContext(ctx, "POST", url, strings.NewReader(string(jsonData))) + if err != nil { + return nil, fmt.Errorf("failed to create request: %v", err) + } + + httpReq.Header.Set("X-Vault-Token", p.token) + httpReq.Header.Set("Content-Type", "application/json") + + resp, err := p.httpClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("failed to encrypt: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("encrypt failed: %s - %s", resp.Status, string(body)) + } + + var result struct { + Data struct { + Ciphertext string `json:"ciphertext"` + KeyVersion int `json:"key_version"` + } `json:"data"` + } + + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode encrypt response: %v", err) + } + + return &kmsapi.EncryptResponse{ + KeyId: fmt.Sprintf("%s:v%d", p.config.TransitKeyName, result.Data.KeyVersion), + Ciphertext: []byte(result.Data.Ciphertext), + Annotations: map[string][]byte{ + "vault.hashicorp.com/transit-key": []byte(p.config.TransitKeyName), + }, + }, nil +} + +// Decrypt decrypts the ciphertext using Vault Transit engine +func (p *VaultKMSPlugin) Decrypt(ctx context.Context, req *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) { + // Call Vault Transit decrypt + decryptData := map[string]string{ + "ciphertext": string(req.Ciphertext), + } + + jsonData, _ := json.Marshal(decryptData) + url := fmt.Sprintf("%s/v1/transit/decrypt/%s", p.config.VaultAddress, p.config.TransitKeyName) + + httpReq, err := http.NewRequestWithContext(ctx, "POST", url, strings.NewReader(string(jsonData))) + if err != nil { + return nil, fmt.Errorf("failed to create request: %v", err) + } + + httpReq.Header.Set("X-Vault-Token", p.token) + httpReq.Header.Set("Content-Type", "application/json") + + resp, err := p.httpClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("failed to decrypt: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("decrypt failed: %s - %s", resp.Status, string(body)) + } + + var result struct { + Data struct { + Plaintext string `json:"plaintext"` + } `json:"data"` + } + + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode decrypt response: %v", err) + } + + // Base64 decode the plaintext + plaintext, err := base64.StdEncoding.DecodeString(result.Data.Plaintext) + if err != nil { + return nil, fmt.Errorf("failed to decode plaintext: %v", err) + } + + return &kmsapi.DecryptResponse{ + Plaintext: plaintext, + }, nil +} diff --git a/test/library/encryption/kms/vault-plugin/setup-vault-kms.sh b/test/library/encryption/kms/vault-plugin/setup-vault-kms.sh new file mode 100755 index 0000000000..840ed916b3 --- /dev/null +++ b/test/library/encryption/kms/vault-plugin/setup-vault-kms.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Simple Vault KMS v2 Setup Script +# This script sets up Vault and deploys the KMS v2 plugin for testing. + +set -e + +echo "=== Vault KMS v2 Setup ===" + +# Step 1: Deploy Vault +echo "" +echo "[1/5] Deploying Vault..." +helm repo add hashicorp https://helm.releases.hashicorp.com 2>/dev/null || true +helm repo update + +if helm status vault -n vault >/dev/null 2>&1; then + echo "Vault already installed, skipping..." +else + oc create namespace vault --dry-run=client -o yaml | oc apply -f - + helm install vault hashicorp/vault \ + --namespace vault \ + --set "global.openshift=true" \ + --set "server.image.repository=docker.io/hashicorp/vault" \ + --set "server.image.tag=1.15.4" \ + --set "server.dev.enabled=true" \ + --set "server.dev.devRootToken=root" \ + --set "injector.enabled=false" \ + --wait --timeout 5m +fi + +echo "Waiting for Vault pod..." +oc wait --for=condition=Ready pod -l app.kubernetes.io/name=vault -n vault --timeout=120s + +# Step 2: Configure Transit Engine +echo "" +echo "[2/5] Configuring Vault Transit Engine..." +oc exec -n vault vault-0 -- vault secrets enable transit 2>/dev/null || echo "Transit already enabled" +oc exec -n vault vault-0 -- vault write -f transit/keys/kubernetes-encryption 2>/dev/null || echo "Key already exists" + +# Step 3: Configure AppRole +echo "" +echo "[3/5] Configuring AppRole authentication..." +oc exec -n vault vault-0 -- vault auth enable approle 2>/dev/null || echo "AppRole already enabled" + +# Create policy +oc exec -n vault vault-0 -- sh -c 'vault policy write kms-policy - </dev/null || echo "Vault namespace not found" + +echo "" +echo "[KMS Plugin]" +oc get pods -n openshift-kms-plugin -l app=vault-kms-plugin 2>/dev/null || echo "KMS plugin not found" + +echo "" +echo "[KMS Plugin Logs]" +FIRST_POD=$(oc get pods -n openshift-kms-plugin -l app=vault-kms-plugin -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) +if [ -n "$FIRST_POD" ]; then + oc logs -n openshift-kms-plugin "$FIRST_POD" --tail=10 2>/dev/null +else + echo "No KMS plugin pods found" +fi + +echo "" +echo "[Socket Check]" +if [ -n "$FIRST_POD" ]; then + oc exec -n openshift-kms-plugin "$FIRST_POD" -- ls -la /var/run/kmsplugin/ 2>/dev/null || echo "Could not check socket" +fi