From 5a99e4d0d7f8dc344e4c25b75e348f8430cccf2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?M=C3=A1rton=20Rasek?= <rasekmarci@gmail.com>
Date: Mon, 19 Dec 2022 20:10:30 +0100
Subject: [PATCH 01/11] add docker-slim runnable

---
 env_docker_slim.go | 412 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 412 insertions(+)
 create mode 100644 env_docker_slim.go

diff --git a/env_docker_slim.go b/env_docker_slim.go
new file mode 100644
index 0000000..e9cda29
--- /dev/null
+++ b/env_docker_slim.go
@@ -0,0 +1,412 @@
+// Copyright (c) The EfficientGo Authors.
+// Licensed under the Apache License 2.0.
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/efficientgo/core/backoff"
+	"github.com/efficientgo/core/errors"
+)
+
+type DockerSlimOptions struct {
+	Name           string
+	ExtraBuildArgs []string
+}
+
+func (e *DockerEnvironment) RunnableSlim(opts DockerSlimOptions) RunnableBuilder {
+	if e.closed {
+		return errorer{name: opts.Name, err: errors.New("environment close was invoked already.")}
+	}
+
+	if e.isRegistered(opts.Name) {
+		return errorer{name: opts.Name, err: errors.Newf("there is already one runnable created with the same name %v", opts.Name)}
+	}
+
+	d := &dockerSlimRunnable{
+		container: &dockerRunnable{
+			env:        e,
+			name:       opts.Name,
+			logger:     e.logger,
+			ports:      map[string]int{},
+			hostPorts:  map[string]int{},
+			extensions: map[any]any{},
+		},
+		containerName:  dockerNetworkContainerHost(e.networkName, opts.Name),
+		extraBuildArgs: opts.ExtraBuildArgs,
+	}
+	if err := os.MkdirAll(d.Dir(), 0750); err != nil {
+		return errorer{name: opts.Name, err: err}
+	}
+	e.register(opts.Name)
+	return d
+}
+
+func (e *DockerEnvironment) buildDockerSlimRunArgs(name string, ports map[string]int, extraBuildArgs []string, opts StartOptions) []string {
+	args := []string{"--show-clogs", "--show-blogs", "--label", "application=" + dockerNetworkContainerHost(e.networkName, name),
+		"--http-probe=false", "--target", opts.Image, "--tag", fmt.Sprintf("%s-slim", opts.Image),
+		"--network=" + e.networkName, "--hostname=" + name, "--continue-after", "signal"}
+
+	if len(extraBuildArgs) > 0 {
+		args = append(args, extraBuildArgs...)
+	}
+
+	// Mount the shared/ directory into the container. We share all containers dir to each other to allow easier scenarios.
+	args = append(args, "--mount", fmt.Sprintf("%s:%s:z", e.dir, dockerLocalSharedDir))
+
+	for _, v := range opts.Volumes {
+		args = append(args, "--mount", v)
+	}
+
+	// Environment variables
+	for name, value := range opts.EnvVars {
+		args = append(args, "--env", name+"="+value)
+	}
+
+	if opts.User != "" {
+		args = append(args, "--user", opts.User)
+	}
+
+	// Published ports.
+	for _, port := range ports {
+		args = append(args, "--expose", strconv.Itoa(port), "--publish-port", strconv.Itoa(port))
+	}
+
+	// Disable entrypoint if required.
+	if opts.Command.EntrypointDisabled {
+		args = append(args, "--entrypoint", "")
+	}
+
+	var cmdArgs string
+
+	if opts.Command.Cmd != "" {
+		cmdArgs += opts.Command.Cmd
+	}
+	if len(opts.Command.Args) > 0 {
+		for _, arg := range opts.Command.Args {
+			if cmdArgs != "" {
+				cmdArgs += fmt.Sprintf(" %s", arg)
+			} else {
+				cmdArgs += arg
+			}
+		}
+	}
+
+	if cmdArgs != "" {
+		args = append(args, "--cmd", cmdArgs)
+	}
+	return args
+}
+
+type dockerSlimRunnable struct {
+	container      *dockerRunnable
+	containerName  string
+	extraBuildArgs []string
+	pid            int
+}
+
+func (d *dockerSlimRunnable) Name() string {
+	return d.containerName
+}
+
+func (d *dockerSlimRunnable) BuildErr() error {
+	return nil
+}
+
+func (d *dockerSlimRunnable) Dir() string {
+	return filepath.Join(d.container.env.dir, "data", d.Name())
+}
+
+func (d *dockerSlimRunnable) InternalDir() string {
+	return filepath.Join(dockerLocalSharedDir, "data", d.Name())
+}
+
+func (d *dockerSlimRunnable) Init(opts StartOptions) Runnable {
+	if opts.WaitReadyBackoff == nil {
+		opts.WaitReadyBackoff = &backoff.Config{
+			Min:        300 * time.Millisecond,
+			Max:        600 * time.Millisecond,
+			MaxRetries: 50, // Sometimes the CI is slow ¯\_(ツ)_/¯.
+		}
+	}
+
+	d.container.opts = opts
+	d.container.waitBackoffReady = backoff.New(context.Background(), *opts.WaitReadyBackoff)
+	return d
+}
+
+func (d *dockerSlimRunnable) WithPorts(ports map[string]int) RunnableBuilder {
+	d.container.ports = ports
+	return d
+}
+
+func (d *dockerSlimRunnable) SetMetadata(key, value any) {
+	d.container.extensions[key] = value
+}
+
+func (d *dockerSlimRunnable) GetMetadata(key any) (any, bool) {
+	v, ok := d.container.extensions[key]
+	return v, ok
+}
+
+func (d *dockerSlimRunnable) Future() FutureRunnable {
+	return d
+}
+
+func (d *dockerSlimRunnable) IsRunning() bool {
+	return d.container.usedNetworkName != ""
+}
+
+// Start starts runnable.
+func (d *dockerSlimRunnable) Start() (err error) {
+	if d.IsRunning() {
+		return errors.Newf("%v is running. Stop or kill it first to restart.", d.Name())
+	}
+
+	d.container.logger.Log("Starting", d.Name())
+
+	// In case of any error, if the container was already created, we
+	// have to cleanup removing it. We ignore the error of the "docker rm"
+	// because we don't know if the container was created or not.
+	defer func() {
+		if err != nil {
+			_, _ = d.container.env.exec("docker", "rm", "--force", d.container.name).CombinedOutput()
+		}
+	}()
+
+	// Make sure the image is available locally; if not wait for it to download.
+	if err := d.container.prePullImage(context.TODO()); err != nil {
+		return err
+	}
+
+	cmd := d.container.env.exec("docker-slim", append([]string{"build"}, d.container.env.buildDockerSlimRunArgs(d.container.name, d.container.ports,
+		d.extraBuildArgs, d.container.opts)...)...)
+	l := &LinePrefixLogger{prefix: d.Name() + ": ", logger: d.container.logger}
+	cmd.Stdout = l
+	cmd.Stderr = l
+	cmd.Stdin = os.Stdin
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	d.pid = cmd.Process.Pid
+	d.container.usedNetworkName = d.container.env.networkName
+
+	if err := d.addName(); err != nil {
+		return err
+	}
+
+	// Wait until the container has been started.
+	if err := d.waitForRunning(); err != nil {
+		return err
+	}
+
+	if err := d.container.env.registerStarted(d); err != nil {
+		return err
+	}
+
+	// Get the dynamic local ports mapped to the container.
+	for portName, containerPort := range d.container.ports {
+		d.container.hostPorts[portName] = containerPort
+	}
+
+	d.container.logger.Log("Ports for container", d.container.containerName(), ">> Local ports:", d.container.ports, "Ports available from host:", d.container.hostPorts)
+	return nil
+}
+
+func (d *dockerSlimRunnable) Stop() error {
+	if d.pid == 0 {
+		return nil
+	}
+
+	if err := syscall.Kill(d.pid, syscall.SIGUSR1); err != nil {
+		return err
+	}
+
+	process, err := os.FindProcess(d.pid)
+
+	if err != nil {
+		return err
+	}
+
+	if _, err := process.Wait(); err != nil {
+		return err
+	}
+
+	d.container.usedNetworkName = ""
+	return d.container.env.registerStopped(d.Name())
+}
+
+func (d *dockerSlimRunnable) Kill() error {
+	if d.pid == 0 {
+		return nil
+	}
+
+	if err := syscall.Kill(d.pid, syscall.SIGUSR1); err != nil {
+		return d.container.Kill()
+	}
+
+	process, err := os.FindProcess(d.pid)
+
+	if err != nil {
+		return d.container.Kill()
+	}
+
+	if _, err := process.Wait(); err != nil {
+		return d.container.Kill()
+	}
+
+	d.container.usedNetworkName = ""
+	return d.container.Kill()
+}
+
+// Endpoint returns external (from host perspective) service endpoint (host:port) for given port name.
+// External means that it will be accessible only from host, but not from docker containers.
+//
+// If your service is not running, this method returns incorrect `stopped` endpoint.
+func (d *dockerSlimRunnable) Endpoint(portName string) string {
+	return d.container.Endpoint(portName)
+}
+
+// InternalEndpoint returns internal service endpoint (host:port) for given internal port.
+// Internal means that it will be accessible only from docker containers within the network that this
+// service is running in. If you configure your local resolver with docker DNS namespace you can access it from host
+// as well. Use `Endpoint` for host access.
+func (d *dockerSlimRunnable) InternalEndpoint(portName string) string {
+	return d.container.InternalEndpoint(portName)
+}
+
+func (d *dockerSlimRunnable) Ready() error {
+	return d.container.Ready()
+}
+
+func (d *dockerSlimRunnable) WaitReady() error {
+	err := d.container.WaitReady()
+	return err
+}
+
+// Exec runs the provided command against the docker container specified by this
+// service.
+func (d *dockerSlimRunnable) Exec(command Command, opts ...ExecOption) error {
+	if !d.IsRunning() {
+		return errors.Newf("service %s is stopped", d.Name())
+	}
+
+	l := &LinePrefixLogger{prefix: d.Name() + "-exec: ", logger: d.container.logger}
+	o := ExecOptions{Stdout: l, Stderr: l}
+	for _, opt := range opts {
+		opt(&o)
+	}
+
+	args := []string{"exec", d.container.name}
+	args = append(args, command.Cmd)
+	args = append(args, command.Args...)
+	if o.Stdin != nil {
+		args = append(args[:1], append([]string{"-i"}, args[1:]...)...)
+	}
+	cmd := d.container.env.exec("docker", args...)
+	if o.Stdin != nil {
+		cmd.Stdin = o.Stdin
+	}
+	cmd.Stdout = o.Stdout
+	cmd.Stderr = o.Stderr
+	return cmd.Run()
+}
+
+func (d *dockerSlimRunnable) addName() error {
+	var out []byte
+	var err error
+	for d.container.waitBackoffReady.Reset(); d.container.waitBackoffReady.Ongoing(); {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		out, err = d.container.env.execContext(
+			ctx,
+			"docker",
+			"ps",
+			"-a",
+			"-f",
+			"label=application="+d.containerName,
+			"--format",
+			"{{.Names}}",
+		).CombinedOutput()
+
+		if err != nil {
+			d.container.waitBackoffReady.Wait()
+			continue
+		}
+		if out == nil {
+			err = errors.Newf("nil output")
+			d.container.waitBackoffReady.Wait()
+			continue
+		}
+
+		name := strings.TrimSpace(string(out))
+		if strings.Contains(name, "\n") || !strings.HasPrefix(name, "dockerslimk_") {
+			err = errors.Newf("unexpected output: %q", name)
+			d.container.waitBackoffReady.Wait()
+			continue
+		}
+
+		d.container.name = name
+		return nil
+	}
+
+	if len(out) > 0 {
+		d.container.logger.Log(string(out))
+	}
+	return errors.Wrapf(err, "docker container %s failed to start", d.Name())
+}
+
+func (d *dockerSlimRunnable) waitForRunning() (err error) {
+	if !d.IsRunning() {
+		return errors.Newf("service %s is stopped", d.Name())
+	}
+
+	var out []byte
+	for d.container.waitBackoffReady.Reset(); d.container.waitBackoffReady.Ongoing(); {
+		// Enforce a timeout on the command execution because we've seen some flaky tests
+		// stuck here.
+
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		out, err = d.container.env.execContext(
+			ctx,
+			"docker",
+			"inspect",
+			"--format={{json .State.Running}}",
+			d.container.name,
+		).CombinedOutput()
+
+		if err != nil {
+			d.container.waitBackoffReady.Wait()
+			continue
+		}
+
+		if out == nil {
+			err = errors.Newf("nil output")
+			d.container.waitBackoffReady.Wait()
+			continue
+		}
+
+		str := strings.TrimSpace(string(out))
+		if str != "true" {
+			err = errors.Newf("unexpected output: %q", str)
+			d.container.waitBackoffReady.Wait()
+			continue
+		}
+
+		return nil
+	}
+
+	if len(out) > 0 {
+		d.container.logger.Log(string(out))
+	}
+	return errors.Wrapf(err, "docker container %s failed to start", d.Name())
+}

From 3ac84d97cfb793c93bff5170854f8f5a07e895b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?M=C3=A1rton=20Rasek?= <rasekmarci@gmail.com>
Date: Wed, 21 Dec 2022 20:50:13 +0100
Subject: [PATCH 02/11] solve conflicts

---
 env_docker_slim.go | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/env_docker_slim.go b/env_docker_slim.go
index e9cda29..1de6a4b 100644
--- a/env_docker_slim.go
+++ b/env_docker_slim.go
@@ -60,7 +60,7 @@ func (e *DockerEnvironment) buildDockerSlimRunArgs(name string, ports map[string
 	}
 
 	// Mount the shared/ directory into the container. We share all containers dir to each other to allow easier scenarios.
-	args = append(args, "--mount", fmt.Sprintf("%s:%s:z", e.dir, dockerLocalSharedDir))
+	args = append(args, "--mount", fmt.Sprintf("%s:%s:z", e.dir, e.dir))
 
 	for _, v := range opts.Volumes {
 		args = append(args, "--mount", v)
@@ -126,7 +126,7 @@ func (d *dockerSlimRunnable) Dir() string {
 }
 
 func (d *dockerSlimRunnable) InternalDir() string {
-	return filepath.Join(dockerLocalSharedDir, "data", d.Name())
+	return d.Dir()
 }
 
 func (d *dockerSlimRunnable) Init(opts StartOptions) Runnable {
@@ -308,13 +308,7 @@ func (d *dockerSlimRunnable) Exec(command Command, opts ...ExecOption) error {
 	args := []string{"exec", d.container.name}
 	args = append(args, command.Cmd)
 	args = append(args, command.Args...)
-	if o.Stdin != nil {
-		args = append(args[:1], append([]string{"-i"}, args[1:]...)...)
-	}
 	cmd := d.container.env.exec("docker", args...)
-	if o.Stdin != nil {
-		cmd.Stdin = o.Stdin
-	}
 	cmd.Stdout = o.Stdout
 	cmd.Stderr = o.Stderr
 	return cmd.Run()

From cf445642c7a3e28bfed406546d6289d181b7b67a Mon Sep 17 00:00:00 2001
From: Douglas Camata <159076+douglascamata@users.noreply.github.com>
Date: Wed, 7 Dec 2022 13:28:20 +0100
Subject: [PATCH 03/11] Improve WSL 2 experience (#57)

* Detect WSL2 and open in browser from Windows

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Fix hostAddr in WSL2

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Fix host-local endpoint test in WSL2

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Fail fast to start in WSL2 with cadvisor

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Inline some conditionals

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Detect WSL 2 without relying on OS binaries/tools

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Fix var name

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Drop environment.WSL2() and use func from package

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Improve how OS platform is checked

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Document what is WSL

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Fix WSL2 check for network test

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Fix networkType deletion

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Separate OS platform function into its own package

* Reorder imports

* Add TODO to new `host` package

* Please the copyright linter

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>
---
 env_docker.go              | 12 +++++++-----
 host/host.go               | 32 ++++++++++++++++++++++++++++++++
 interactive/interactive.go |  7 +++++--
 monitoring/monitoring.go   | 17 +++++++++++++++--
 4 files changed, 59 insertions(+), 9 deletions(-)
 create mode 100644 host/host.go

diff --git a/env_docker.go b/env_docker.go
index b72f3d2..03074be 100644
--- a/env_docker.go
+++ b/env_docker.go
@@ -18,12 +18,14 @@ import (
 	"strings"
 	"time"
 
+	"github.com/efficientgo/e2e/host"
+
 	"github.com/efficientgo/core/backoff"
 	"github.com/efficientgo/core/errors"
 )
 
 const (
-	dockerMacOSGatewayAddr = "host.docker.internal"
+	dockerGatewayAddr = "host.docker.internal"
 )
 
 var (
@@ -129,10 +131,10 @@ func New(opts ...EnvironmentOption) (_ *DockerEnvironment, err error) {
 		return nil, errors.Wrapf(err, "create docker network '%s'", d.networkName)
 	}
 
-	switch runtime.GOOS {
-	case "darwin":
-		d.hostAddr = dockerMacOSGatewayAddr
-	default:
+	switch host.OSPlatform() {
+	case "darwin", "WSL2":
+		d.hostAddr = dockerGatewayAddr
+	default: // the "linux" behavior is default
 		out, err := d.exec("docker", "network", "inspect", d.networkName).CombinedOutput()
 		if err != nil {
 			e.logger.Log(string(out))
diff --git a/host/host.go b/host/host.go
new file mode 100644
index 0000000..6334440
--- /dev/null
+++ b/host/host.go
@@ -0,0 +1,32 @@
+// Copyright (c) The EfficientGo Authors.
+// Licensed under the Apache License 2.0.
+
+package host
+
+import (
+	"bytes"
+	"os"
+	"runtime"
+)
+
+// OSPlatform returns the host's OS platform akin to `runtime.GOOS`, with
+// added awareness of Windows Subsystem for Linux (WSL) 2 environments.
+// The possible values are the same as `runtime.GOOS`, plus "WSL2".
+// TODO: move this to a new home, potentially github.com/efficientgo/core.
+func OSPlatform() string {
+	if isWSL2() {
+		return "WSL2"
+	}
+	return runtime.GOOS
+}
+
+func isWSL2() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+	version, err := os.ReadFile("/proc/version")
+	if err != nil {
+		return false
+	}
+	return bytes.Contains(version, []byte("WSL2"))
+}
diff --git a/interactive/interactive.go b/interactive/interactive.go
index 303e0da..e5802c8 100644
--- a/interactive/interactive.go
+++ b/interactive/interactive.go
@@ -11,10 +11,11 @@ import (
 	"os"
 	"os/exec"
 	"os/signal"
-	"runtime"
 	"sync"
 	"syscall"
 
+	"github.com/efficientgo/e2e/host"
+
 	"github.com/efficientgo/core/errors"
 )
 
@@ -22,7 +23,9 @@ import (
 func OpenInBrowser(url string) error {
 	fmt.Println("Opening", url, "in browser.")
 	var err error
-	switch runtime.GOOS {
+	switch host.OSPlatform() {
+	case "WSL2":
+		err = exec.Command("rundll32.exe", "url.dll,FileProtocolHandler", url).Run()
 	case "linux":
 		err = exec.Command("xdg-open", url).Run()
 	case "windows":
diff --git a/monitoring/monitoring.go b/monitoring/monitoring.go
index 69631a2..373cb7f 100644
--- a/monitoring/monitoring.go
+++ b/monitoring/monitoring.go
@@ -15,6 +15,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/efficientgo/e2e/host"
+
 	"github.com/efficientgo/core/errcapture"
 	"github.com/efficientgo/core/errors"
 	"github.com/efficientgo/e2e"
@@ -259,8 +261,16 @@ func Start(env e2e.Environment, opts ...Option) (_ *Service, err error) {
 		h.ServeHTTP(w, req)
 	}))
 
-	// Listen on all addresses, since we need to connect to it from docker container.
-	list, err := net.Listen("tcp", "0.0.0.0:0")
+	// Listen on all tcp4 addresses, since we need to connect to it from Docker container.
+	// For unknown reasons, when using WSL 2, if the network type is "tcp" it will
+	// end up only binding to the IPv6 in the WSL host, which later cannot be acessed
+	// via IPv4 to confirm Prometheus can scrape the local endpoint.
+	// Explicitly asking for an IPv4 listener works.
+	networkType := "tcp"
+	if host.OSPlatform() == "WSL2" {
+		networkType = "tcp4"
+	}
+	list, err := net.Listen(networkType, "0.0.0.0:0")
 	if err != nil {
 		return nil, err
 	}
@@ -282,6 +292,9 @@ func Start(env e2e.Environment, opts ...Option) (_ *Service, err error) {
 	env.AddListener(l)
 
 	if opt.useCadvisor {
+		if host.OSPlatform() == "WSL2" {
+			return nil, errors.New("cadvisor is not supported in WSL 2 environments")
+		}
 		c := newCadvisor(env, "cadvisor")
 		if err := e2e.StartAndWaitReady(c); err != nil {
 			return nil, errors.Wrap(err, "starting cadvisor and waiting until ready")

From 7baa1955fbffdde60dbfa6af3f131425af35162f Mon Sep 17 00:00:00 2001
From: Douglas Camata <159076+douglascamata@users.noreply.github.com>
Date: Mon, 12 Dec 2022 17:20:46 +0100
Subject: [PATCH 04/11] Use minio console access as readiness check (#58)

* Use minio console access as readiness check

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Update etcd example lines

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Please the copyright linter

* Test minio readiness by making a bucket

* Rebuild

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Adding comment linking to issue about minio readiness check

* Go fmt file

* Update example go.sum file

* Update mdox entry

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>
---
 README.md              |  2 +-
 db/db.go               | 27 +++++++++++++++++++------
 db/db_test.go          | 40 ++++++++++++++++++++++++++++++++++++
 examples/thanos/go.sum | 46 +++++++++++++++++++++++++++++++++++++++---
 go.mod                 | 21 +++++++++++++++----
 go.sum                 | 38 +++++++++++++++++++++++++++++++---
 6 files changed, 157 insertions(+), 17 deletions(-)
 create mode 100644 db/db_test.go

diff --git a/README.md b/README.md
index 8f2ef59..61dfd9c 100644
--- a/README.md
+++ b/README.md
@@ -48,7 +48,7 @@ Let's go through an example leveraging the `go test` flow:
 
 4. Use `e2emon.AsInstrumented` if you want to be able to query your service for metrics, which is a great way to assess it's internal state in tests! For example see following Etcd definition:
 
-   ```go mdox-exec="sed -n '216,228p' db/db.go"
+   ```go mdox-exec="sed -n '231,243p' db/db.go"
    	return e2emon.AsInstrumented(env.Runnable(name).WithPorts(map[string]int{AccessPortName: 2379, "metrics": 9000}).Init(
    		e2e.StartOptions{
    			Image: o.image,
diff --git a/db/db.go b/db/db.go
index 9a6a5dd..2a27dcf 100644
--- a/db/db.go
+++ b/db/db.go
@@ -80,11 +80,12 @@ func NewMinio(env e2e.Environment, name, bktName string, opts ...Option) *e2emon
 	}
 
 	userID := strconv.Itoa(os.Getuid())
-	ports := map[string]int{AccessPortName: 8090}
+	const consolePortName = "console"
+	ports := map[string]int{AccessPortName: 8090, consolePortName: 8080}
 	envVars := []string{
 		"MINIO_ROOT_USER=" + MinioAccessKey,
 		"MINIO_ROOT_PASSWORD=" + MinioSecretKey,
-		"MINIO_BROWSER=" + "off",
+		"MINIO_BROWSER=" + "on",
 		"ENABLE_HTTPS=" + "0",
 	}
 
@@ -109,11 +110,25 @@ func NewMinio(env e2e.Environment, name, bktName string, opts ...Option) *e2emon
 		e2e.StartOptions{
 			Image: o.image,
 			// Create the required bucket before starting minio.
-			Command: e2e.NewCommandWithoutEntrypoint("sh", "-c", command+fmt.Sprintf(
-				"su - me -s /bin/sh -c 'mkdir -p %s && %s /opt/bin/minio server --address :%v --quiet %v'",
-				filepath.Join(f.Dir(), bktName), strings.Join(envVars, " "), ports[AccessPortName], f.Dir()),
+			Command: e2e.NewCommandWithoutEntrypoint(
+				"sh",
+				"-c",
+				command+fmt.Sprintf(
+					"su - me -s /bin/sh -c 'mkdir -p %s && %s /opt/bin/minio server --address :%v --console-address :%v --quiet %v'",
+					filepath.Join(f.Dir(), bktName),
+					strings.Join(envVars, " "),
+					ports[AccessPortName],
+					ports[consolePortName],
+					f.Dir(),
+				),
+			),
+			// Using console as readiness check until MinIO fixes https://github.com/minio/minio/issues/16213.
+			Readiness: e2e.NewHTTPReadinessProbe(
+				consolePortName,
+				"/",
+				200,
+				200,
 			),
-			Readiness: e2e.NewHTTPReadinessProbe(AccessPortName, "/minio/health/live", 200, 200),
 		},
 	), AccessPortName)
 }
diff --git a/db/db_test.go b/db/db_test.go
new file mode 100644
index 0000000..37d0639
--- /dev/null
+++ b/db/db_test.go
@@ -0,0 +1,40 @@
+// Copyright (c) The EfficientGo Authors.
+// Licensed under the Apache License 2.0.
+
+package e2edb
+
+import (
+	"context"
+	"testing"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+
+	"github.com/efficientgo/core/testutil"
+	"github.com/efficientgo/e2e"
+)
+
+func TestMinio(t *testing.T) {
+	t.Parallel()
+
+	e, err := e2e.New()
+	testutil.Ok(t, err)
+	t.Cleanup(e.Close)
+
+	minioContainer := NewMinio(e, "mintest", "bkt")
+	testutil.Ok(t, e2e.StartAndWaitReady(minioContainer))
+
+	endpoint := minioContainer.Endpoint("http")
+	accessKeyID := MinioAccessKey
+	secretAccessKey := MinioSecretKey
+	minioClient, err := minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure: false,
+	})
+	testutil.Ok(t, err)
+
+	testutil.Ok(
+		t,
+		minioClient.MakeBucket(context.Background(), "test-bucket", minio.MakeBucketOptions{}),
+	)
+}
diff --git a/examples/thanos/go.sum b/examples/thanos/go.sum
index c24aaac..beaebca 100644
--- a/examples/thanos/go.sum
+++ b/examples/thanos/go.sum
@@ -54,6 +54,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/efficientgo/core v1.0.0-rc.0 h1:jJoA0N+C4/knWYVZ6GrdHOtDyrg8Y/TR4vFpTaqTsqs=
 github.com/efficientgo/core v1.0.0-rc.0/go.mod h1:kQa0V74HNYMfuJH6jiPiwNdpWXl4xd/K4tzlrcvYDQI=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -124,6 +126,8 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -141,6 +145,12 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0=
+github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -152,6 +162,12 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.45 h1:g4IeM9M9pW/Lo8AGGNOjBZYlvmtlE1N5TQEYWXRWzIs=
+github.com/minio/minio-go/v7 v7.0.45/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -193,9 +209,13 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -206,6 +226,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -217,6 +238,9 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -247,6 +271,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -275,10 +300,14 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -296,6 +325,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -333,18 +363,25 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -388,6 +425,7 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -476,6 +514,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI=
+gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/go.mod b/go.mod
index 6993aca..5193c5c 100644
--- a/go.mod
+++ b/go.mod
@@ -4,6 +4,7 @@ go 1.18
 
 require (
 	github.com/efficientgo/core v1.0.0-rc.0
+	github.com/minio/minio-go/v7 v7.0.45
 	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.36.0
@@ -14,17 +15,29 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.15.9 // indirect
+	github.com/klauspost/cpuid/v2 v2.1.0 // indirect
 	github.com/kr/pretty v0.2.1 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/stretchr/testify v1.7.0 // indirect
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
+	github.com/rs/xid v1.4.0 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
+	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
+	golang.org/x/net v0.1.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/ini.v1 v1.66.6 // indirect
 )
diff --git a/go.sum b/go.sum
index b6bbdca..4f029d6 100644
--- a/go.sum
+++ b/go.sum
@@ -54,6 +54,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/efficientgo/core v1.0.0-rc.0 h1:jJoA0N+C4/knWYVZ6GrdHOtDyrg8Y/TR4vFpTaqTsqs=
 github.com/efficientgo/core v1.0.0-rc.0/go.mod h1:kQa0V74HNYMfuJH6jiPiwNdpWXl4xd/K4tzlrcvYDQI=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -124,6 +126,8 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -134,12 +138,19 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0=
+github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -151,10 +162,18 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.45 h1:g4IeM9M9pW/Lo8AGGNOjBZYlvmtlE1N5TQEYWXRWzIs=
+github.com/minio/minio-go/v7 v7.0.45/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
@@ -188,9 +207,13 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -212,6 +235,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -272,8 +297,9 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -328,8 +354,11 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -338,8 +367,9 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -471,6 +501,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI=
+gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

From 9cce7a5382ad02f4f45c69121b5d486c24f8d4e6 Mon Sep 17 00:00:00 2001
From: Douglas Camata <159076+douglascamata@users.noreply.github.com>
Date: Fri, 13 Jan 2023 11:30:53 +0100
Subject: [PATCH 05/11] Use cluster healthcheck as readiness probe for MinIO.
 (#61)

* Improve minio readiness check

It uses now the cluster health endpoint, so we don't need the console
anymore.

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Make xargs macOS friendly

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Remove now unused console port name variable

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

* Update README

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>
---
 Makefile  |  6 ++++--
 README.md | 25 +++++++++++++------------
 db/db.go  | 13 +++++--------
 3 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/Makefile b/Makefile
index f588b35..89e137e 100644
--- a/Makefile
+++ b/Makefile
@@ -13,6 +13,8 @@ GIT ?= $(shell which git)
 # Support gsed on OSX (installed via brew), falling back to sed. On Linux
 # systems gsed won't be installed, so will use sed as expected.
 SED ?= $(shell which gsed 2>/dev/null || which sed)
+# Same as above, but for xargs.
+XARGS ?= $(shell which gxargs 2>/dev/null || which xargs)
 
 define require_clean_work_tree
 	@git update-index -q --ignore-submodules --refresh
@@ -119,7 +121,7 @@ lint: $(FAILLINT) $(GOLANGCI_LINT) $(MISSPELL) $(COPYRIGHT) build format docs ch
 		cd $${dir} && $(GOLANGCI_LINT) run; \
 	done
 	@echo ">> detecting misspells"
-	@find . -type f | grep -v vendor/ | grep -vE '\./\..*' | xargs $(MISSPELL) -error
+	@find . -type f | grep -v vendor/ | grep -vE '\./\..*' | $(XARGS) $(MISSPELL) -error
 	@echo ">> ensuring Copyright headers"
-	@$(COPYRIGHT) $(shell go list -f "{{.Dir}}" ./... | xargs -i find "{}" -name "*.go")
+	@$(COPYRIGHT) $(shell go list -f "{{.Dir}}" ./... | $(XARGS) -i find "{}" -name "*.go")
 	$(call require_clean_work_tree,"detected files without copyright - run make lint and commit changes.")
diff --git a/README.md b/README.md
index 61dfd9c..d13fd8a 100644
--- a/README.md
+++ b/README.md
@@ -48,20 +48,21 @@ Let's go through an example leveraging the `go test` flow:
 
 4. Use `e2emon.AsInstrumented` if you want to be able to query your service for metrics, which is a great way to assess it's internal state in tests! For example see following Etcd definition:
 
-   ```go mdox-exec="sed -n '231,243p' db/db.go"
+   ```go mdox-exec="sed -n '228,243p' db/db.go"
    	return e2emon.AsInstrumented(env.Runnable(name).WithPorts(map[string]int{AccessPortName: 2379, "metrics": 9000}).Init(
-   		e2e.StartOptions{
-   			Image: o.image,
-   			Command: e2e.NewCommand(
-   				"/usr/local/bin/etcd",
-   				"--listen-client-urls=http://0.0.0.0:2379",
-   				"--advertise-client-urls=http://0.0.0.0:2379",
-   				"--listen-metrics-urls=http://0.0.0.0:9000",
-   				"--log-level=error",
-   			),
-   			Readiness: e2e.NewHTTPReadinessProbe("metrics", "/health", 200, 204),
-   		},
+        e2e.StartOptions{
+            Image: o.image,
+            Command: e2e.NewCommand(
+                "/usr/local/bin/etcd",
+                "--listen-client-urls=http://0.0.0.0:2379",
+                "--advertise-client-urls=http://0.0.0.0:2379",
+                "--listen-metrics-urls=http://0.0.0.0:9000",
+                "--log-level=error",
+            ),
+            Readiness: e2e.NewHTTPReadinessProbe("metrics", "/health", 200, 204),
+        },
    	), "metrics")
+   }
    ```
 
 5. Program your scenario as you want. You can start, wait for their readiness, stop, check their metrics and use their network endpoints from both unit test (`Endpoint`) as well as within each workload (`InternalEndpoint`). You can also access workload directory. There is a shared directory across all workloads. Check `Dir` and `InternalDir` runnable methods.
diff --git a/db/db.go b/db/db.go
index 2a27dcf..398d76f 100644
--- a/db/db.go
+++ b/db/db.go
@@ -80,12 +80,11 @@ func NewMinio(env e2e.Environment, name, bktName string, opts ...Option) *e2emon
 	}
 
 	userID := strconv.Itoa(os.Getuid())
-	const consolePortName = "console"
-	ports := map[string]int{AccessPortName: 8090, consolePortName: 8080}
+	ports := map[string]int{AccessPortName: 8090}
 	envVars := []string{
 		"MINIO_ROOT_USER=" + MinioAccessKey,
 		"MINIO_ROOT_PASSWORD=" + MinioSecretKey,
-		"MINIO_BROWSER=" + "on",
+		"MINIO_BROWSER=" + "off",
 		"ENABLE_HTTPS=" + "0",
 	}
 
@@ -114,18 +113,16 @@ func NewMinio(env e2e.Environment, name, bktName string, opts ...Option) *e2emon
 				"sh",
 				"-c",
 				command+fmt.Sprintf(
-					"su - me -s /bin/sh -c 'mkdir -p %s && %s /opt/bin/minio server --address :%v --console-address :%v --quiet %v'",
+					"su - me -s /bin/sh -c 'mkdir -p %s && %s /opt/bin/minio server --address :%v --quiet %v'",
 					filepath.Join(f.Dir(), bktName),
 					strings.Join(envVars, " "),
 					ports[AccessPortName],
-					ports[consolePortName],
 					f.Dir(),
 				),
 			),
-			// Using console as readiness check until MinIO fixes https://github.com/minio/minio/issues/16213.
 			Readiness: e2e.NewHTTPReadinessProbe(
-				consolePortName,
-				"/",
+				AccessPortName,
+				"/minio/health/cluster",
 				200,
 				200,
 			),

From 07dc68913fd27f92ac19300fed9937af34652832 Mon Sep 17 00:00:00 2001
From: Saswata Mukherjee <saswataminsta@yahoo.com>
Date: Thu, 19 Jan 2023 14:39:47 +0530
Subject: [PATCH 06/11] Add option to enable TLS for minio (#62)

* Add option to enable TLS for minio

Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>

* Fix readiness probe

Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>

* Fix lint

Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>

Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>
---
 README.md |  25 +++++----
 db/db.go  | 151 +++++++++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 149 insertions(+), 27 deletions(-)

diff --git a/README.md b/README.md
index d13fd8a..5c06808 100644
--- a/README.md
+++ b/README.md
@@ -48,21 +48,20 @@ Let's go through an example leveraging the `go test` flow:
 
 4. Use `e2emon.AsInstrumented` if you want to be able to query your service for metrics, which is a great way to assess it's internal state in tests! For example see following Etcd definition:
 
-   ```go mdox-exec="sed -n '228,243p' db/db.go"
+   ```go mdox-exec="sed -n '351,363p' db/db.go"
    	return e2emon.AsInstrumented(env.Runnable(name).WithPorts(map[string]int{AccessPortName: 2379, "metrics": 9000}).Init(
-        e2e.StartOptions{
-            Image: o.image,
-            Command: e2e.NewCommand(
-                "/usr/local/bin/etcd",
-                "--listen-client-urls=http://0.0.0.0:2379",
-                "--advertise-client-urls=http://0.0.0.0:2379",
-                "--listen-metrics-urls=http://0.0.0.0:9000",
-                "--log-level=error",
-            ),
-            Readiness: e2e.NewHTTPReadinessProbe("metrics", "/health", 200, 204),
-        },
+   		e2e.StartOptions{
+   			Image: o.image,
+   			Command: e2e.NewCommand(
+   				"/usr/local/bin/etcd",
+   				"--listen-client-urls=http://0.0.0.0:2379",
+   				"--advertise-client-urls=http://0.0.0.0:2379",
+   				"--listen-metrics-urls=http://0.0.0.0:9000",
+   				"--log-level=error",
+   			),
+   			Readiness: e2e.NewHTTPReadinessProbe("metrics", "/health", 200, 204),
+   		},
    	), "metrics")
-   }
    ```
 
 5. Program your scenario as you want. You can start, wait for their readiness, stop, check their metrics and use their network endpoints from both unit test (`Endpoint`) as well as within each workload (`InternalEndpoint`). You can also access workload directory. There is a shared directory across all workloads. Check `Dir` and `InternalDir` runnable methods.
diff --git a/db/db.go b/db/db.go
index 398d76f..9aad647 100644
--- a/db/db.go
+++ b/db/db.go
@@ -6,12 +6,20 @@
 package e2edb
 
 import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
+	"math/big"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
+	"github.com/efficientgo/core/errors"
 	"github.com/efficientgo/e2e"
 	e2emon "github.com/efficientgo/e2e/monitoring"
 	e2eprof "github.com/efficientgo/e2e/profiling"
@@ -32,6 +40,7 @@ type options struct {
 
 type minioOptions struct {
 	enableSSE bool
+	enableTLS bool
 }
 
 func WithImage(image string) Option {
@@ -52,6 +61,12 @@ func WithMinioSSE() Option {
 	}
 }
 
+func WithMinioTLS() Option {
+	return func(o *options) {
+		o.minioOptions.enableTLS = true
+	}
+}
+
 const AccessPortName = "http"
 
 func NewPrometheus(env e2e.Environment, name string, opts ...Option) *e2emon.Prometheus {
@@ -85,7 +100,6 @@ func NewMinio(env e2e.Environment, name, bktName string, opts ...Option) *e2emon
 		"MINIO_ROOT_USER=" + MinioAccessKey,
 		"MINIO_ROOT_PASSWORD=" + MinioSecretKey,
 		"MINIO_BROWSER=" + "off",
-		"ENABLE_HTTPS=" + "0",
 	}
 
 	f := env.Runnable(name).WithPorts(ports).Future()
@@ -105,6 +119,56 @@ func NewMinio(env e2e.Environment, name, bktName string, opts ...Option) *e2emon
 		command += "curl -sSL --tlsv1.3 -O 'https://raw.githubusercontent.com/minio/kes/master/root.key' -O 'https://raw.githubusercontent.com/minio/kes/master/root.cert' && cp root.* /home/me/ && "
 	}
 
+	var readiness e2e.ReadinessProbe
+
+	if o.minioOptions.enableTLS {
+		if err := os.MkdirAll(filepath.Join(f.Dir(), "certs", "CAs"), 0750); err != nil {
+			return &e2emon.InstrumentedRunnable{Runnable: e2e.NewFailedRunnable(name, errors.Wrap(err, "create certs dir"))}
+		}
+
+		if err := genCerts(
+			filepath.Join(f.Dir(), "certs", "public.crt"),
+			filepath.Join(f.Dir(), "certs", "private.key"),
+			filepath.Join(f.Dir(), "certs", "CAs", "ca.crt"),
+			fmt.Sprintf("%s-%s", env.Name(), name),
+		); err != nil {
+			return &e2emon.InstrumentedRunnable{Runnable: e2e.NewFailedRunnable(name, errors.Wrap(err, "fail to generate certs"))}
+		}
+
+		envVars = append(envVars, "ENABLE_HTTPS="+"1")
+		command = command + fmt.Sprintf(
+			"su - me -s /bin/sh -c 'mkdir -p %s && %s /opt/bin/minio server --certs-dir %s/certs --address :%v --quiet %v'",
+			filepath.Join(f.Dir(), bktName),
+			strings.Join(envVars, " "),
+			f.Dir(),
+			ports[AccessPortName],
+			f.Dir(),
+		)
+
+		readiness = e2e.NewHTTPSReadinessProbe(
+			AccessPortName,
+			"/minio/health/cluster",
+			200,
+			200,
+		)
+	} else {
+		envVars = append(envVars, "ENABLE_HTTPS="+"0")
+		command = command + fmt.Sprintf(
+			"su - me -s /bin/sh -c 'mkdir -p %s && %s /opt/bin/minio server --address :%v --quiet %v'",
+			filepath.Join(f.Dir(), bktName),
+			strings.Join(envVars, " "),
+			ports[AccessPortName],
+			f.Dir(),
+		)
+
+		readiness = e2e.NewHTTPReadinessProbe(
+			AccessPortName,
+			"/minio/health/cluster",
+			200,
+			200,
+		)
+	}
+
 	return e2emon.AsInstrumented(f.Init(
 		e2e.StartOptions{
 			Image: o.image,
@@ -112,24 +176,83 @@ func NewMinio(env e2e.Environment, name, bktName string, opts ...Option) *e2emon
 			Command: e2e.NewCommandWithoutEntrypoint(
 				"sh",
 				"-c",
-				command+fmt.Sprintf(
-					"su - me -s /bin/sh -c 'mkdir -p %s && %s /opt/bin/minio server --address :%v --quiet %v'",
-					filepath.Join(f.Dir(), bktName),
-					strings.Join(envVars, " "),
-					ports[AccessPortName],
-					f.Dir(),
-				),
-			),
-			Readiness: e2e.NewHTTPReadinessProbe(
-				AccessPortName,
-				"/minio/health/cluster",
-				200,
-				200,
+				command,
 			),
+			Readiness: readiness,
 		},
 	), AccessPortName)
 }
 
+// genCerts generates certificates and writes those to the provided paths.
+func genCerts(certPath, privkeyPath, caPath, serverName string) error {
+	var caRoot = &x509.Certificate{
+		SerialNumber:          big.NewInt(2019),
+		NotAfter:              time.Now().AddDate(10, 0, 0),
+		IsCA:                  true,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+	}
+
+	var cert = &x509.Certificate{
+		SerialNumber: big.NewInt(1658),
+		DNSNames:     []string{serverName},
+		IPAddresses:  []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
+		NotAfter:     time.Now().AddDate(10, 0, 0),
+		SubjectKeyId: []byte{1, 2, 3},
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+	}
+
+	caPrivKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return err
+	}
+
+	certPrivKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return err
+	}
+	// Generate CA cert.
+	caBytes, err := x509.CreateCertificate(rand.Reader, caRoot, caRoot, &caPrivKey.PublicKey, caPrivKey)
+	if err != nil {
+		return err
+	}
+	caPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: caBytes,
+	})
+	err = os.WriteFile(caPath, caPEM, 0644)
+	if err != nil {
+		return err
+	}
+
+	// Sign the cert with the CA private key.
+	certBytes, err := x509.CreateCertificate(rand.Reader, cert, caRoot, &certPrivKey.PublicKey, caPrivKey)
+	if err != nil {
+		return err
+	}
+	certPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: certBytes,
+	})
+	err = os.WriteFile(certPath, certPEM, 0644)
+	if err != nil {
+		return err
+	}
+
+	certPrivKeyPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
+	})
+	err = os.WriteFile(privkeyPath, certPrivKeyPEM, 0644)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func NewConsul(env e2e.Environment, name string, opts ...Option) *e2emon.InstrumentedRunnable {
 	o := options{image: "consul:1.8.4"}
 	for _, opt := range opts {

From 8d3ab061c4984646e1ee2cfecd450965f8f5d9cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= <giedriuswork@gmail.com>
Date: Fri, 24 Mar 2023 12:06:59 +0200
Subject: [PATCH 07/11] env_docker: support docker-in-docker (#63)

* env_docker: support docker-in-docker

If we are running e2e tests inside Docker then it's not so
straightforward to access host's network. Let's use the special
`host.docker.internal` for that.

* env_docker: try resolve before using docker gateway
---
 env_docker.go | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/env_docker.go b/env_docker.go
index 03074be..fc4a440 100644
--- a/env_docker.go
+++ b/env_docker.go
@@ -9,6 +9,7 @@ import (
 	"crypto/sha256"
 	"encoding/json"
 	"fmt"
+	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -523,7 +524,18 @@ func (d *dockerRunnable) Endpoint(portName string) string {
 	}
 
 	// Do not use "localhost", because it doesn't work with the AWS DynamoDB client.
-	return fmt.Sprintf("127.0.0.1:%d", localPort)
+	addr := "127.0.0.1"
+	// If we are running inside Docker then 127.0.0.1 is not accessible.
+	// To access the host's network, let's use host.docker.internal.
+	// See: https://docs.docker.com/desktop/networking/#i-want-to-connect-from-a-container-to-a-service-on-the-host.
+	if _, err := os.Stat("/.dockerenv"); err == nil {
+		dockerAddrs, err := net.LookupIP(dockerGatewayAddr)
+		if err == nil && len(dockerAddrs) > 0 {
+			addr = dockerGatewayAddr
+		}
+	}
+
+	return fmt.Sprintf("%s:%d", addr, localPort)
 }
 
 // InternalEndpoint returns internal service endpoint (host:port) for given internal port.

From 04d66327f410a501e7cc305f6d8251a404a75d8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lucas=20Serv=C3=A9n=20Mar=C3=ADn?= <lserven@gmail.com>
Date: Wed, 29 Mar 2023 09:38:54 +0200
Subject: [PATCH 08/11] e2e: add environment for kind (#64)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* CI: ensure macOS can connect to containers in Lima

This commit allows the macOS host in CI to route packets directly to the
IP addresses of containers running inside the Lima guest VM, without
needing to explicitly forward ports via SSH. This allows macs to behave
the same way that Linux hosts do.

Signed-off-by: Lucas Servén Marín <lserven@gmail.com>

* e2e: add environment for kind

This commit adds a new implementation of the Environment interface
implemented using kind, ie Kubernetes in Docker. The motivation is that
you might want to manually run some complex configurations in the e2e
tests using Kubernetes resources and still manage the tests simply using
the e2e package.

Users who need the escape hatch of deploying things to a Kubernetes
cluster manually, e.g. workloads that are not a simple deployment, can
use the kubeconfig file located in the environment's shared directory.

Signed-off-by: Lucas Servén Marín <lserven@gmail.com>

---------

Signed-off-by: Lucas Servén Marín <lserven@gmail.com>
---
 .github/workflows/go.yaml |  11 +-
 env_docker_ext_test.go    |  76 +---
 env_ext_test.go           |  86 ++++
 env_kind.go               | 826 ++++++++++++++++++++++++++++++++++++++
 env_kind_ext_test.go      |  20 +
 env_kind_test.go          | 343 ++++++++++++++++
 6 files changed, 1285 insertions(+), 77 deletions(-)
 create mode 100644 env_ext_test.go
 create mode 100644 env_kind.go
 create mode 100644 env_kind_ext_test.go
 create mode 100644 env_kind_test.go

diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml
index 924db4d..71bdd66 100644
--- a/.github/workflows/go.yaml
+++ b/.github/workflows/go.yaml
@@ -50,8 +50,15 @@ jobs:
       - name: Install docker - MacOS
         if: runner.os == 'macOS'
         run: |
-          brew install docker colima
-          colima start
+          brew install docker colima kind kubectl
+          # Ensure Lima guest is accessible from mac host.
+          colima start --network-address
+          # Find the IP address of the Lima guest.
+          IP=$(limactl shell colima -- ip a show dev col0 | head -n3 | tail -n1 | cut -d' ' -f6 | rev | cut -c4- | rev)
+          # Route packets destined for 172/8 through the Lima guest.
+          sudo route -nv add -net 172 $IP
+          # Configure the Lima guest to forward packets from the col0 bridge to Docker containers.
+          limactl shell colima -- sudo iptables -t filter -A FORWARD -s $IP/24 -d 172.0.0.0/8 -j ACCEPT
 
       - uses: actions/cache@v1
         with:
diff --git a/env_docker_ext_test.go b/env_docker_ext_test.go
index 378aada..0c7c617 100644
--- a/env_docker_ext_test.go
+++ b/env_docker_ext_test.go
@@ -4,91 +4,17 @@
 package e2e_test
 
 import (
-	"bytes"
-	"io"
-	"net/http"
-	"path/filepath"
 	"testing"
 
 	"github.com/efficientgo/core/testutil"
 	"github.com/efficientgo/e2e"
-	e2edb "github.com/efficientgo/e2e/db"
-	e2emon "github.com/efficientgo/e2e/monitoring"
 )
 
-func wgetFlagsCmd(hostPort string) e2e.Command {
-	return e2e.NewCommandWithoutEntrypoint("/bin/sh", "-c", "wget http://"+hostPort+"/api/v1/status/flags -O /tmp/flags && cat /tmp/flags")
-}
-
 func TestDockerEnvironment(t *testing.T) {
 	t.Parallel()
 
 	e, err := e2e.New()
 	testutil.Ok(t, err)
 	t.Cleanup(e.Close)
-
-	p1 := e2edb.NewPrometheus(e, "prometheus-1")
-	testutil.Equals(t, "prometheus-1", p1.Name())
-	testutil.Equals(t, filepath.Join(e.SharedDir(), "data", p1.Name()), p1.Dir())
-	testutil.Equals(t, "e2e-80D5E99C3992-prometheus-1:9090", p1.InternalEndpoint("http"))
-	testutil.Equals(t, "", p1.InternalEndpoint("not-existing"))
-	testutil.Assert(t, !p1.IsRunning())
-
-	// Errors as p1 was not started yet.
-	testutil.NotOk(t, p1.WaitReady())
-	testutil.Ok(t, p1.Stop())
-	testutil.Ok(t, p1.Kill())
-
-	testutil.NotOk(t, p1.Exec(wgetFlagsCmd("localhost:9090")))
-	testutil.Equals(t, "stopped", p1.Endpoint("http"))
-	testutil.Equals(t, "stopped", p1.Endpoint("not-existing"))
-
-	p2 := e2edb.NewPrometheus(e, "prometheus-2")
-	testutil.Ok(t, e2e.StartAndWaitReady(p1, p2))
-	testutil.Ok(t, p1.WaitReady())
-	testutil.Ok(t, p1.WaitReady())
-
-	testutil.Ok(t, p1.WaitSumMetrics(e2emon.Greater(50), "prometheus_tsdb_head_samples_appended_total"))
-
-	testutil.Equals(t, "prometheus-1", p1.Name())
-	testutil.Equals(t, filepath.Join(e.SharedDir(), "data", p1.Name()), p1.Dir())
-	testutil.Equals(t, "e2e-80D5E99C3992-prometheus-1:9090", p1.InternalEndpoint("http"))
-	testutil.Equals(t, "", p1.InternalEndpoint("not-existing"))
-	testutil.Assert(t, p1.IsRunning())
-
-	expectedFlagsOutputProm1 := "{\"status\":\"success\",\"data\":{\"alertmanager.notification-queue-capacity\":\"10000\",\"alertmanager.timeout\":\"\",\"config.file\":\"" + filepath.Join(e.SharedDir(), "/data/prometheus-1/prometheus.yml") + "\",\"enable-feature\":\"\",\"log.format\":\"logfmt\",\"log.level\":\"info\",\"query.lookback-delta\":\"5m\",\"query.max-concurrency\":\"20\",\"query.max-samples\":\"50000000\",\"query.timeout\":\"2m\",\"rules.alert.for-grace-period\":\"10m\",\"rules.alert.for-outage-tolerance\":\"1h\",\"rules.alert.resend-delay\":\"1m\",\"scrape.adjust-timestamps\":\"true\",\"scrape.discovery-reload-interval\":\"5s\",\"scrape.timestamp-tolerance\":\"2ms\",\"storage.agent.no-lockfile\":\"false\",\"storage.agent.path\":\"data-agent/\",\"storage.agent.retention.max-time\":\"0s\",\"storage.agent.retention.min-time\":\"0s\",\"storage.agent.wal-compression\":\"true\",\"storage.agent.wal-segment-size\":\"0B\",\"storage.agent.wal-truncate-frequency\":\"0s\",\"storage.remote.flush-deadline\":\"1m\",\"storage.remote.read-concurrent-limit\":\"10\",\"storage.remote.read-max-bytes-in-frame\":\"1048576\",\"storage.remote.read-sample-limit\":\"50000000\",\"storage.tsdb.allow-overlapping-blocks\":\"false\",\"storage.tsdb.head-chunks-write-queue-size\":\"0\",\"storage.tsdb.max-block-chunk-segment-size\":\"0B\",\"storage.tsdb.max-block-duration\":\"2h\",\"storage.tsdb.min-block-duration\":\"2h\",\"storage.tsdb.no-lockfile\":\"false\",\"storage.tsdb.path\":\"" + filepath.Join(e.SharedDir(), "/data/prometheus-1/") + "\",\"storage.tsdb.retention\":\"0s\",\"storage.tsdb.retention.size\":\"0B\",\"storage.tsdb.retention.time\":\"0s\",\"storage.tsdb.wal-compression\":\"true\",\"storage.tsdb.wal-segment-size\":\"0B\",\"web.config.file\":\"\",\"web.console.libraries\":\"console_libraries\",\"web.console.templates\":\"consoles\",\"web.cors.origin\":\".*\",\"web.enable-admin-api\":\"false\",\"web.enable-lifecycle\":\"false\",\"web.enable-remote-write-receiver\":\"false\",\"web.external-url\":\"\",\"web.listen-address\":\":9090\",\"web.max-connections\":\"512\",\"web.page-title\":\"Prometheus Time Series Collection and Processing Server\",\"web.read-timeout\":\"5m\",\"web.route-prefix\":\"/\",\"web.user-assets\":\"\"}}"
-	expectedFlagsOutputProm2 := "{\"status\":\"success\",\"data\":{\"alertmanager.notification-queue-capacity\":\"10000\",\"alertmanager.timeout\":\"\",\"config.file\":\"" + filepath.Join(e.SharedDir(), "/data/prometheus-2/prometheus.yml") + "\",\"enable-feature\":\"\",\"log.format\":\"logfmt\",\"log.level\":\"info\",\"query.lookback-delta\":\"5m\",\"query.max-concurrency\":\"20\",\"query.max-samples\":\"50000000\",\"query.timeout\":\"2m\",\"rules.alert.for-grace-period\":\"10m\",\"rules.alert.for-outage-tolerance\":\"1h\",\"rules.alert.resend-delay\":\"1m\",\"scrape.adjust-timestamps\":\"true\",\"scrape.discovery-reload-interval\":\"5s\",\"scrape.timestamp-tolerance\":\"2ms\",\"storage.agent.no-lockfile\":\"false\",\"storage.agent.path\":\"data-agent/\",\"storage.agent.retention.max-time\":\"0s\",\"storage.agent.retention.min-time\":\"0s\",\"storage.agent.wal-compression\":\"true\",\"storage.agent.wal-segment-size\":\"0B\",\"storage.agent.wal-truncate-frequency\":\"0s\",\"storage.remote.flush-deadline\":\"1m\",\"storage.remote.read-concurrent-limit\":\"10\",\"storage.remote.read-max-bytes-in-frame\":\"1048576\",\"storage.remote.read-sample-limit\":\"50000000\",\"storage.tsdb.allow-overlapping-blocks\":\"false\",\"storage.tsdb.head-chunks-write-queue-size\":\"0\",\"storage.tsdb.max-block-chunk-segment-size\":\"0B\",\"storage.tsdb.max-block-duration\":\"2h\",\"storage.tsdb.min-block-duration\":\"2h\",\"storage.tsdb.no-lockfile\":\"false\",\"storage.tsdb.path\":\"" + filepath.Join(e.SharedDir(), "/data/prometheus-2/") + "\",\"storage.tsdb.retention\":\"0s\",\"storage.tsdb.retention.size\":\"0B\",\"storage.tsdb.retention.time\":\"0s\",\"storage.tsdb.wal-compression\":\"true\",\"storage.tsdb.wal-segment-size\":\"0B\",\"web.config.file\":\"\",\"web.console.libraries\":\"console_libraries\",\"web.console.templates\":\"consoles\",\"web.cors.origin\":\".*\",\"web.enable-admin-api\":\"false\",\"web.enable-lifecycle\":\"false\",\"web.enable-remote-write-receiver\":\"false\",\"web.external-url\":\"\",\"web.listen-address\":\":9090\",\"web.max-connections\":\"512\",\"web.page-title\":\"Prometheus Time Series Collection and Processing Server\",\"web.read-timeout\":\"5m\",\"web.route-prefix\":\"/\",\"web.user-assets\":\"\"}}"
-
-	var out bytes.Buffer
-	testutil.Ok(t, p1.Exec(wgetFlagsCmd("localhost:9090"), e2e.WithExecOptionStdout(&out)))
-	testutil.Equals(t, expectedFlagsOutputProm1, out.String())
-
-	resp, err := http.Get("http://" + p1.Endpoint("http") + "/api/v1/status/flags")
-	testutil.Ok(t, err)
-	defer resp.Body.Close()
-
-	b, err := io.ReadAll(resp.Body)
-	testutil.Ok(t, err)
-	testutil.Equals(t, expectedFlagsOutputProm1, string(b))
-	testutil.Equals(t, "", p1.Endpoint("not-existing"))
-
-	// Now try the same but cross containers.
-	out.Reset()
-	testutil.Ok(t, p1.Exec(wgetFlagsCmd(p2.InternalEndpoint("http")), e2e.WithExecOptionStdout(&out)))
-	testutil.Equals(t, expectedFlagsOutputProm2, out.String())
-
-	testutil.NotOk(t, p1.Start()) // Starting ok, should fail.
-
-	// Batch job example and test.
-	batch := e.Runnable("batch").Init(e2e.StartOptions{Image: "ubuntu:20.04", Command: e2e.NewCommandRunUntilStop()})
-	testutil.Ok(t, batch.Start())
-	for i := 0; i < 3; i++ {
-		out.Reset()
-		testutil.Ok(t, batch.Exec(e2e.NewCommand("echo", "yolo"), e2e.WithExecOptionStdout(&out)))
-		testutil.Equals(t, "yolo\n", out.String())
-	}
-
-	e.Close()
-	afterClose := e2edb.NewPrometheus(e, "prometheus-3") // Should fail.
-	testutil.NotOk(t, afterClose.Start())
+	testEnvironment(t, e)
 }
diff --git a/env_ext_test.go b/env_ext_test.go
new file mode 100644
index 0000000..ef0890f
--- /dev/null
+++ b/env_ext_test.go
@@ -0,0 +1,86 @@
+// Copyright (c) The EfficientGo Authors.
+// Licensed under the Apache License 2.0.
+
+package e2e_test
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"path/filepath"
+	"testing"
+
+	"github.com/efficientgo/core/testutil"
+	"github.com/efficientgo/e2e"
+	e2edb "github.com/efficientgo/e2e/db"
+	e2emon "github.com/efficientgo/e2e/monitoring"
+)
+
+func wgetFlagsCmd(hostPort string) e2e.Command {
+	return e2e.NewCommandWithoutEntrypoint("/bin/sh", "-c", "wget http://"+hostPort+"/api/v1/status/flags -O /tmp/flags && cat /tmp/flags")
+}
+
+func testEnvironment(t *testing.T, e e2e.Environment) {
+	p1 := e2edb.NewPrometheus(e, "prometheus-1")
+	testutil.Equals(t, "prometheus-1", p1.Name())
+	testutil.Equals(t, filepath.Join(e.SharedDir(), "data", p1.Name()), p1.Dir())
+	testutil.Equals(t, "", p1.InternalEndpoint("not-existing"))
+	testutil.Assert(t, !p1.IsRunning())
+
+	// Errors as p1 was not started yet.
+	testutil.NotOk(t, p1.WaitReady())
+	testutil.Ok(t, p1.Stop())
+	testutil.Ok(t, p1.Kill())
+
+	testutil.NotOk(t, p1.Exec(wgetFlagsCmd("localhost:9090")))
+	testutil.Equals(t, "stopped", p1.Endpoint("http"))
+	testutil.Equals(t, "stopped", p1.Endpoint("not-existing"))
+
+	p2 := e2edb.NewPrometheus(e, "prometheus-2")
+	testutil.Ok(t, e2e.StartAndWaitReady(p1, p2))
+	testutil.Ok(t, p1.WaitReady())
+	testutil.Ok(t, p1.WaitReady())
+
+	testutil.Ok(t, p1.WaitSumMetrics(e2emon.Greater(50), "prometheus_tsdb_head_samples_appended_total"))
+
+	testutil.Equals(t, "prometheus-1", p1.Name())
+	testutil.Equals(t, filepath.Join(e.SharedDir(), "data", p1.Name()), p1.Dir())
+	testutil.Equals(t, "", p1.InternalEndpoint("not-existing"))
+	testutil.Assert(t, p1.IsRunning())
+
+	expectedFlagsOutputProm1 := "{\"status\":\"success\",\"data\":{\"alertmanager.notification-queue-capacity\":\"10000\",\"alertmanager.timeout\":\"\",\"config.file\":\"" + filepath.Join(e.SharedDir(), "/data/prometheus-1/prometheus.yml") + "\",\"enable-feature\":\"\",\"log.format\":\"logfmt\",\"log.level\":\"info\",\"query.lookback-delta\":\"5m\",\"query.max-concurrency\":\"20\",\"query.max-samples\":\"50000000\",\"query.timeout\":\"2m\",\"rules.alert.for-grace-period\":\"10m\",\"rules.alert.for-outage-tolerance\":\"1h\",\"rules.alert.resend-delay\":\"1m\",\"scrape.adjust-timestamps\":\"true\",\"scrape.discovery-reload-interval\":\"5s\",\"scrape.timestamp-tolerance\":\"2ms\",\"storage.agent.no-lockfile\":\"false\",\"storage.agent.path\":\"data-agent/\",\"storage.agent.retention.max-time\":\"0s\",\"storage.agent.retention.min-time\":\"0s\",\"storage.agent.wal-compression\":\"true\",\"storage.agent.wal-segment-size\":\"0B\",\"storage.agent.wal-truncate-frequency\":\"0s\",\"storage.remote.flush-deadline\":\"1m\",\"storage.remote.read-concurrent-limit\":\"10\",\"storage.remote.read-max-bytes-in-frame\":\"1048576\",\"storage.remote.read-sample-limit\":\"50000000\",\"storage.tsdb.allow-overlapping-blocks\":\"false\",\"storage.tsdb.head-chunks-write-queue-size\":\"0\",\"storage.tsdb.max-block-chunk-segment-size\":\"0B\",\"storage.tsdb.max-block-duration\":\"2h\",\"storage.tsdb.min-block-duration\":\"2h\",\"storage.tsdb.no-lockfile\":\"false\",\"storage.tsdb.path\":\"" + filepath.Join(e.SharedDir(), "/data/prometheus-1/") + "\",\"storage.tsdb.retention\":\"0s\",\"storage.tsdb.retention.size\":\"0B\",\"storage.tsdb.retention.time\":\"0s\",\"storage.tsdb.wal-compression\":\"true\",\"storage.tsdb.wal-segment-size\":\"0B\",\"web.config.file\":\"\",\"web.console.libraries\":\"console_libraries\",\"web.console.templates\":\"consoles\",\"web.cors.origin\":\".*\",\"web.enable-admin-api\":\"false\",\"web.enable-lifecycle\":\"false\",\"web.enable-remote-write-receiver\":\"false\",\"web.external-url\":\"\",\"web.listen-address\":\":9090\",\"web.max-connections\":\"512\",\"web.page-title\":\"Prometheus Time Series Collection and Processing Server\",\"web.read-timeout\":\"5m\",\"web.route-prefix\":\"/\",\"web.user-assets\":\"\"}}"
+	expectedFlagsOutputProm2 := "{\"status\":\"success\",\"data\":{\"alertmanager.notification-queue-capacity\":\"10000\",\"alertmanager.timeout\":\"\",\"config.file\":\"" + filepath.Join(e.SharedDir(), "/data/prometheus-2/prometheus.yml") + "\",\"enable-feature\":\"\",\"log.format\":\"logfmt\",\"log.level\":\"info\",\"query.lookback-delta\":\"5m\",\"query.max-concurrency\":\"20\",\"query.max-samples\":\"50000000\",\"query.timeout\":\"2m\",\"rules.alert.for-grace-period\":\"10m\",\"rules.alert.for-outage-tolerance\":\"1h\",\"rules.alert.resend-delay\":\"1m\",\"scrape.adjust-timestamps\":\"true\",\"scrape.discovery-reload-interval\":\"5s\",\"scrape.timestamp-tolerance\":\"2ms\",\"storage.agent.no-lockfile\":\"false\",\"storage.agent.path\":\"data-agent/\",\"storage.agent.retention.max-time\":\"0s\",\"storage.agent.retention.min-time\":\"0s\",\"storage.agent.wal-compression\":\"true\",\"storage.agent.wal-segment-size\":\"0B\",\"storage.agent.wal-truncate-frequency\":\"0s\",\"storage.remote.flush-deadline\":\"1m\",\"storage.remote.read-concurrent-limit\":\"10\",\"storage.remote.read-max-bytes-in-frame\":\"1048576\",\"storage.remote.read-sample-limit\":\"50000000\",\"storage.tsdb.allow-overlapping-blocks\":\"false\",\"storage.tsdb.head-chunks-write-queue-size\":\"0\",\"storage.tsdb.max-block-chunk-segment-size\":\"0B\",\"storage.tsdb.max-block-duration\":\"2h\",\"storage.tsdb.min-block-duration\":\"2h\",\"storage.tsdb.no-lockfile\":\"false\",\"storage.tsdb.path\":\"" + filepath.Join(e.SharedDir(), "/data/prometheus-2/") + "\",\"storage.tsdb.retention\":\"0s\",\"storage.tsdb.retention.size\":\"0B\",\"storage.tsdb.retention.time\":\"0s\",\"storage.tsdb.wal-compression\":\"true\",\"storage.tsdb.wal-segment-size\":\"0B\",\"web.config.file\":\"\",\"web.console.libraries\":\"console_libraries\",\"web.console.templates\":\"consoles\",\"web.cors.origin\":\".*\",\"web.enable-admin-api\":\"false\",\"web.enable-lifecycle\":\"false\",\"web.enable-remote-write-receiver\":\"false\",\"web.external-url\":\"\",\"web.listen-address\":\":9090\",\"web.max-connections\":\"512\",\"web.page-title\":\"Prometheus Time Series Collection and Processing Server\",\"web.read-timeout\":\"5m\",\"web.route-prefix\":\"/\",\"web.user-assets\":\"\"}}"
+
+	var out bytes.Buffer
+	testutil.Ok(t, p1.Exec(wgetFlagsCmd("localhost:9090"), e2e.WithExecOptionStdout(&out)))
+	testutil.Equals(t, expectedFlagsOutputProm1, out.String())
+
+	resp, err := http.Get("http://" + p1.Endpoint("http") + "/api/v1/status/flags")
+	testutil.Ok(t, err)
+	defer resp.Body.Close()
+
+	b, err := io.ReadAll(resp.Body)
+	testutil.Ok(t, err)
+	testutil.Equals(t, expectedFlagsOutputProm1, string(b))
+	testutil.Equals(t, "", p1.Endpoint("not-existing"))
+
+	// Now try the same but cross containers.
+	out.Reset()
+	testutil.Ok(t, p1.Exec(wgetFlagsCmd(p2.InternalEndpoint("http")), e2e.WithExecOptionStdout(&out)))
+	testutil.Equals(t, expectedFlagsOutputProm2, out.String())
+
+	testutil.NotOk(t, p1.Start()) // Starting ok, should fail.
+
+	// Batch job example and test.
+	batch := e.Runnable("batch").Init(e2e.StartOptions{Image: "ubuntu:20.04", Command: e2e.NewCommandRunUntilStop()})
+	testutil.Ok(t, batch.Start())
+	for i := 0; i < 3; i++ {
+		out.Reset()
+		testutil.Ok(t, batch.Exec(e2e.NewCommand("echo", "yolo"), e2e.WithExecOptionStdout(&out)))
+		testutil.Equals(t, "yolo\n", out.String())
+	}
+
+	e.Close()
+	afterClose := e2edb.NewPrometheus(e, "prometheus-3") // Should fail.
+	testutil.NotOk(t, afterClose.Start())
+}
diff --git a/env_kind.go b/env_kind.go
new file mode 100644
index 0000000..f70add3
--- /dev/null
+++ b/env_kind.go
@@ -0,0 +1,826 @@
+// Copyright (c) The EfficientGo Authors.
+// Licensed under the Apache License 2.0.
+
+package e2e
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/efficientgo/core/backoff"
+	"github.com/efficientgo/core/errors"
+)
+
+var (
+	kindNamePattern = regexp.MustCompile(`^[a-z0-9.-]+$`)
+
+	_ Environment = &KindEnvironment{}
+)
+
+// KindEnvironment defines single node kind cluster that allows running services.
+type KindEnvironment struct {
+	dir         string
+	logger      Logger
+	clusterName string
+
+	nodeIP  net.IP
+	volumes []string
+	verbose bool
+
+	mutex sync.Mutex
+	// Access to the following fields must be guarded
+	// by a mutex.
+	registered map[string]struct{}
+	listeners  []EnvironmentListener
+	started    []Runnable
+	closers    []func()
+	closed     bool
+}
+
+func validateKindName(name string) error {
+	if len(name) == 0 {
+		return errors.New("name can't be empty")
+	}
+	if !kindNamePattern.MatchString(name) {
+		return errors.Newf("name can have only %v characters due to kind cluster name constraints, got: %v", kindNamePattern.String(), name)
+
+	}
+	return nil
+}
+
+func unwrapQuotes(b []byte) []byte {
+	return []byte(strings.Trim(string(b), "'"))
+}
+
+var kindConfig = template.Must(template.New("config").Parse(`kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  {{- with .}}
+  extraMounts:
+  {{- range .}}
+  - hostPath: {{.}}
+    containerPath: {{.}}
+  {{- end}}
+  {{- end}}
+`))
+
+// NewKindEnvironment creates a new, isolated kind environment.
+func NewKindEnvironment(opts ...EnvironmentOption) (_ *KindEnvironment, err error) {
+	e := environmentOptions{}
+	for _, o := range opts {
+		o(&e)
+	}
+	if e.name == "" {
+		e.name, err = generateName()
+		if err != nil {
+			return nil, err
+		}
+		e.name = strings.ToLower(e.name)
+	}
+	if err := validateKindName(e.name); err != nil {
+		return nil, err
+	}
+
+	if e.logger == nil {
+		e.logger = NewLogger(os.Stdout)
+	}
+
+	k := &KindEnvironment{
+		logger:      e.logger,
+		clusterName: e.name,
+		verbose:     e.verbose,
+		registered:  map[string]struct{}{},
+		volumes:     e.volumes,
+	}
+
+	// Force a shutdown in order to cleanup from a spurious situation in case
+	// the previous tests run didn't cleanup correctly.
+	k.close()
+
+	dir, err := getTmpDirectory()
+	if err != nil {
+		return nil, err
+	}
+	k.dir = dir
+	var buf bytes.Buffer
+	if err := kindConfig.Execute(&buf, append([]string{k.dir}, k.volumes...)); err != nil {
+		k.Close()
+		return nil, errors.Wrap(err, "generate cluster kind configuration")
+	}
+	kindConfigPath := filepath.Join(k.dir, "kind.yaml")
+	if err := os.WriteFile(kindConfigPath, buf.Bytes(), 0644); err != nil {
+		k.Close()
+		return nil, errors.Wrap(err, "write cluster kind configuration")
+	}
+
+	// Setup the kind cluster.
+	if out, err := k.exec("kind", "create", "cluster", "--kubeconfig", k.kubeconfig(), "--config", kindConfigPath, "--name", k.clusterName).CombinedOutput(); err != nil {
+		e.logger.Log(string(out))
+		k.Close()
+		return nil, errors.Wrapf(err, "create kind cluster %q", k.clusterName)
+	}
+
+	out, err := k.exec("kubectl", "--kubeconfig", k.kubeconfig(), "get", "nodes", fmt.Sprintf("%s-control-plane", k.clusterName), "--output", `jsonpath='{.status.addresses}'`).CombinedOutput()
+	if err != nil {
+		e.logger.Log(string(out))
+		k.Close()
+		return nil, errors.Wrapf(err, "get details of kind cluster node '%s-control-plane'", k.clusterName)
+	}
+
+	type address struct {
+		Address string
+		Type    string
+	}
+	var addresses []address
+	out = unwrapQuotes(out)
+	if err := json.Unmarshal(out, &addresses); err != nil {
+		e.logger.Log("my string without quotes")
+		e.logger.Log(len(out))
+		e.logger.Log(string(out))
+		k.Close()
+		return nil, errors.Wrap(err, "unmarshal kubectl output to get node IP")
+	}
+
+	var internalIPs []address
+	for _, a := range addresses {
+		if a.Type == "InternalIP" {
+			internalIPs = append(internalIPs, a)
+		}
+	}
+	if len(internalIPs) != 1 {
+		k.Close()
+		return nil, errors.Newf("unexpected output of kubectl get node; expected exactly one internal IP, got %d", len(internalIPs))
+	}
+	k.nodeIP = net.ParseIP(internalIPs[0].Address)
+
+	return k, e.logger.Log("msg", "started kind environment", "name", k.clusterName)
+}
+
+func (e *KindEnvironment) kubeconfig() string { return filepath.Join(e.dir, "kubeconfig") }
+
+func (e *KindEnvironment) HostAddr() string { return e.nodeIP.String() }
+func (e *KindEnvironment) Name() string     { return e.clusterName }
+
+func (e *KindEnvironment) AddCloser(f func()) {
+	defer e.mutex.Unlock()
+	e.mutex.Lock()
+	e.closers = append(e.closers, f)
+}
+
+// Runnable returns a new RunnableBuilder for the KindEnvironment.
+// Note: runnables are modeled as Kubernetes Deployments with a single replica.
+// If a runnable of a different Kubernetes kind is needed, then it must be
+// manually deployed using the kubeconfig in the environment's shared directory.
+func (e *KindEnvironment) Runnable(name string) RunnableBuilder {
+	defer e.mutex.Unlock()
+	e.mutex.Lock()
+	if e.closed {
+		return errorer{name: name, err: errors.New("environment close was already invoked")}
+	}
+
+	if e.isRegistered(name) {
+		return errorer{name: name, err: errors.Newf("there is already one runnable created with the same name %q", name)}
+	}
+
+	r := &kindRunnable{
+		env:        e,
+		name:       name,
+		logger:     e.logger,
+		ports:      map[string]int{},
+		hostPorts:  map[string]int{},
+		extensions: map[any]any{},
+	}
+	if err := os.MkdirAll(r.Dir(), 0750); err != nil {
+		return errorer{name: name, err: err}
+	}
+	e.register(name)
+	return r
+}
+
+// AddListener registers the given listener to be notified on environment runnable changes.
+func (e *KindEnvironment) AddListener(listener EnvironmentListener) {
+	defer e.mutex.Unlock()
+	e.mutex.Lock()
+	e.listeners = append(e.listeners, listener)
+}
+
+func (e *KindEnvironment) isRegistered(name string) bool {
+	_, ok := e.registered[name]
+	return ok
+}
+
+func (e *KindEnvironment) register(name string) {
+	e.registered[name] = struct{}{}
+}
+
+func (e *KindEnvironment) registerStarted(r Runnable) error {
+	e.started = append(e.started, r)
+
+	for _, l := range e.listeners {
+		if err := l.OnRunnableChange(e.started); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (e *KindEnvironment) registerStopped(name string) error {
+	for i, r := range e.started {
+		if r.Name() == name {
+			e.started = append(e.started[:i], e.started[i+1:]...)
+			for _, l := range e.listeners {
+				if err := l.OnRunnableChange(e.started); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+	}
+	return nil
+}
+
+func (e *KindEnvironment) SharedDir() string {
+	return e.dir
+}
+
+func (e *KindEnvironment) buildManifest(name string, ports map[string]int, opts StartOptions) (io.Reader, error) {
+	values := kindManifestValues{
+		Name:         name,
+		Image:        opts.Image,
+		Command:      opts.Command.Cmd,
+		Args:         opts.Command.Args,
+		Ports:        ports,
+		Envs:         opts.EnvVars,
+		Bytes:        opts.LimitMemoryBytes,
+		CPUs:         opts.LimitCPUs,
+		Privileged:   opts.Privileged,
+		Capabilities: opts.Capabilities,
+		Volumes: map[string]string{
+			// Mount the working directory into the container. It's shared across all containers to allow easier scenarios.
+			"working-directory": e.dir,
+		},
+		User:   opts.User,
+		UserNs: opts.UserNs,
+	}
+	for i, v := range e.volumes {
+		values.Volumes[fmt.Sprintf("volume%d", i)] = v
+	}
+	for i, v := range opts.Volumes {
+		values.Volumes[fmt.Sprintf("volume%d", i+len(e.volumes))] = v
+	}
+	var buf bytes.Buffer
+	if err := kindManifest.Execute(&buf, values); err != nil {
+		return nil, err
+	}
+	return &buf, nil
+}
+
+var kindManifest = template.Must(template.New("manifest").Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: "{{.Name}}"
+  name: "{{.Name}}"
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: "{{.Name}}"
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: "{{.Name}}"
+    spec:
+      containers:
+      - name: "{{.Name}}"
+        image: "{{.Image}}"
+        {{- with .Command}}
+        command:
+        - "{{.}}"
+        {{- end}}
+        {{- with .Args}}
+        args:
+        {{- range .}}
+        - "{{.}}"
+        {{- end}}
+        {{- end}}
+        {{- with .Ports}}
+        ports:
+        {{- range $k, $v := .}}
+        - name: "{{$k}}"
+          containerPort: {{$v}}
+        {{- end}}
+        {{- end}}
+        {{- with .Envs}}
+        env:
+        {{- range $k, $v := .}}
+        - name: "{{$k}}"
+          value: "{{$v}}"
+        {{- end}}
+        {{- end}}
+        {{- if or .Bytes .CPUs}}
+        resources:
+          limits:
+            {{- with .Bytes}}
+            memory: {{.}}
+            {{- end}}
+            {{- with .CPUs}}
+            cpu: {{.}}
+            {{- end}}
+          requests:
+            {{- with .Bytes}}
+            memory: {{.}}
+            {{- end}}
+            {{- with .CPUs}}
+            cpu: {{.}}
+            {{- end}}
+        {{- end}}
+        {{- if or .Privileged .Capabilities .User}}
+        securityContext:
+          {{- with .User}}
+          runAsUser: {{.}}
+          {{- end}}
+          {{- if .Privileged}}
+          privileged: true
+          {{- end}}
+          {{- with .Capabilities}}
+          capabilities:
+            add:
+            {{- range .}}
+            - {{.}}
+            {{- end}}
+          {{- end}}
+        {{- end}}
+        {{- with .Volumes}}
+        volumeMounts:
+        {{- range $k, $v := .}}
+        - name: "{{$k}}"
+          mountPath: {{$v}}
+        {{- end}}
+        {{- end}}
+      {{- with .Volumes}}
+      volumes:
+      {{- range $k, $v := .}}
+      - name: "{{$k}}"
+        hostPath:
+          path: {{$v}}
+      {{- end}}
+      {{- end}}
+{{- if .Ports}}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: "{{.Name}}"
+  name: "{{.Name}}"
+spec:
+  type: NodePort
+  selector:
+    app.kubernetes.io/name: "{{.Name}}"
+  {{- with .Ports}}
+  ports:
+  {{- range $k, $v := .}}
+  - name: "{{$k}}"
+    port: {{$v}}
+  {{- end}}
+  {{- end}}
+{{- end}}
+`))
+
+type kindManifestValues struct {
+	Name         string
+	Image        string
+	Command      string
+	Args         []string
+	Ports        map[string]int
+	Envs         map[string]string
+	Bytes        uint
+	CPUs         float64
+	Privileged   bool
+	Capabilities []RunnableCapabilities
+	Volumes      map[string]string
+	User         string
+	UserNs       string
+}
+
+func (e *KindEnvironment) Close() {
+	defer e.mutex.Unlock()
+	e.mutex.Lock()
+	for _, c := range e.closers {
+		c()
+	}
+	e.close()
+	e.closed = true
+}
+
+func (e *KindEnvironment) exec(cmd string, args ...string) *exec.Cmd {
+	return e.execContext(context.Background(), cmd, args...)
+}
+
+func (e *KindEnvironment) execContext(ctx context.Context, cmd string, args ...string) *exec.Cmd {
+	c := NewCommand(cmd, args...)
+	if e.verbose {
+		e.logger.Log("kindEnv:", c.toString())
+	}
+	return c.exec(ctx)
+}
+
+func (e *KindEnvironment) close() {
+	if e == nil || e.closed {
+		return
+	}
+
+	// Teardown the kind cluter.
+	// Kind is idempotent and doesn't care if the cluster doesn't exist, it won't throw an error.
+	if out, err := e.exec("kind", "delete", "cluster", "--name", e.clusterName).CombinedOutput(); err != nil {
+		e.logger.Log(string(out))
+		e.logger.Log("Unable to delete kind cluster", e.clusterName, ":", err.Error())
+	}
+
+	if e.dir != "" {
+		if err := e.exec("chmod", "-R", "777", e.dir).Run(); err != nil {
+			e.logger.Log("Error while chmod sharedDir", e.dir, "err:", err)
+		}
+		if err := os.RemoveAll(e.dir); err != nil {
+			e.logger.Log("Error while removing sharedDir", e.dir, "err:", err)
+		}
+	}
+}
+
+type kindRunnable struct {
+	env    *KindEnvironment
+	name   string
+	logger Logger
+
+	mutex sync.Mutex
+	// Access to the following fields must be guarded
+	// by a mutex.
+	ports            map[string]int
+	opts             StartOptions
+	running          bool
+	hostPorts        map[string]int
+	extensions       map[any]any
+	waitBackoffReady *backoff.Backoff
+}
+
+func (r *kindRunnable) Name() string {
+	return r.name
+}
+
+func (r *kindRunnable) BuildErr() error {
+	return nil
+}
+
+func (r *kindRunnable) Dir() string {
+	return filepath.Join(r.env.SharedDir(), "data", r.Name())
+}
+
+func (r *kindRunnable) InternalDir() string {
+	return r.Dir()
+}
+
+func (r *kindRunnable) Init(opts StartOptions) Runnable {
+	defer r.mutex.Unlock()
+	r.mutex.Lock()
+
+	if opts.WaitReadyBackoff == nil {
+		opts.WaitReadyBackoff = &backoff.Config{
+			Min:        300 * time.Millisecond,
+			Max:        600 * time.Millisecond,
+			MaxRetries: 50, // Sometimes the CI is slow ¯\_(ツ)_/¯.
+		}
+	}
+
+	r.opts = opts
+	r.waitBackoffReady = backoff.New(context.Background(), *opts.WaitReadyBackoff)
+	return r
+}
+
+func (r *kindRunnable) WithPorts(ports map[string]int) RunnableBuilder {
+	defer r.mutex.Unlock()
+	r.mutex.Lock()
+
+	r.ports = ports
+	return r
+}
+
+func (r *kindRunnable) SetMetadata(key, value any) {
+	defer r.mutex.Unlock()
+	r.mutex.Lock()
+
+	r.extensions[key] = value
+}
+
+func (r *kindRunnable) GetMetadata(key any) (any, bool) {
+	defer r.mutex.Unlock()
+	r.mutex.Lock()
+
+	v, ok := r.extensions[key]
+	return v, ok
+}
+
+func (r *kindRunnable) Future() FutureRunnable {
+	return r
+}
+
+func (r *kindRunnable) IsRunning() bool {
+	defer r.mutex.Unlock()
+	r.mutex.Lock()
+
+	return r.running
+}
+
+// Start starts the runnable.
+func (r *kindRunnable) Start() (err error) {
+	if r.IsRunning() {
+		return errors.Newf("%q is running; stop or kill it first to restart", r.Name())
+	}
+
+	r.logger.Log("Starting", r.Name())
+
+	// In case of any error, if the container was already created, we
+	// have to cleanup removing it.
+	defer func() {
+		if err != nil {
+			if out, err := r.env.exec("kubernetes", "delete", "deployment", r.Name(), "--ignore-not-found", "--grace-period", "0", "--force").CombinedOutput(); err != nil {
+				r.logger.Log(string(out))
+				return
+			}
+			if out, err := r.env.exec("kubernetes", "delete", "service", r.Name(), "--ignore-not-found", "--grace-period", "0", "--force").CombinedOutput(); err != nil {
+				r.logger.Log(string(out))
+				return
+			}
+		}
+	}()
+
+	// Make sure the image is available locally; if not wait for it to download.
+	if err := r.prePullImage(context.TODO()); err != nil {
+		return err
+	}
+
+	defer r.mutex.Unlock()
+	r.mutex.Lock()
+	manifest, err := r.env.buildManifest(r.name, r.ports, r.opts)
+	if err != nil {
+		return errors.Wrap(err, "building manifest")
+	}
+	cmd := r.env.exec("kubectl", "--kubeconfig", r.env.kubeconfig(), "apply", "--filename", "-")
+	l := &LinePrefixLogger{prefix: r.Name() + ": ", logger: r.logger}
+	cmd.Stdout = l
+	cmd.Stderr = l
+	cmd.Stdin = manifest
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	r.running = true
+
+	// Wait until the container has been started.
+	if err := r.waitForRunning(); err != nil {
+		return err
+	}
+
+	if err := r.env.registerStarted(r); err != nil {
+		return err
+	}
+
+	if len(r.ports) > 0 {
+		// Get the dynamic local ports mapped to the container.
+		out, err := r.env.exec("kubectl", "--kubeconfig", r.env.kubeconfig(), "get", "service", r.Name(), "--output", `jsonpath='{.spec.ports}'`).CombinedOutput()
+		if err != nil {
+			return errors.Wrapf(err, "unable to get mapping for ports for service %q; output: %q", r.Name(), out)
+		}
+		var ports []struct {
+			Name     string
+			NodePort int
+		}
+		out = unwrapQuotes(out)
+		if err := json.Unmarshal(out, &ports); err != nil {
+			return errors.Wrap(err, "unmarshal kubectl output to get ports")
+		}
+		if len(ports) != len(r.ports) {
+			return errors.Newf("found inconsistent ports: the running service %q has a different number of ports than declared", r.Name())
+		}
+		for _, port := range ports {
+			if _, ok := r.ports[port.Name]; !ok {
+				return errors.Newf("found inconsistent ports: port %q is not declared in service %q", port.Name, r.Name())
+			}
+			r.hostPorts[port.Name] = port.NodePort
+		}
+
+		r.logger.Log("Ports for container", r.Name(), ">> Local ports:", r.ports, "Ports available from host:", r.hostPorts)
+	}
+
+	return nil
+}
+
+func (r *kindRunnable) Stop() error {
+	if !r.IsRunning() {
+		return nil
+	}
+
+	r.logger.Log("Stopping", r.Name())
+	if out, err := r.env.exec("kubernetes", "delete", "deployment", r.Name(), "--ignore-not-found", "--grace-period", "30").CombinedOutput(); err != nil {
+		r.logger.Log(string(out))
+		return err
+	}
+	if out, err := r.env.exec("kubernetes", "delete", "service", r.Name(), "--ignore-not-found", "--grace-period", "30").CombinedOutput(); err != nil {
+		r.logger.Log(string(out))
+		return err
+	}
+	defer r.mutex.Unlock()
+	r.mutex.Lock()
+	r.running = false
+	return r.env.registerStopped(r.Name())
+}
+
+func (r *kindRunnable) Kill() error {
+	if !r.IsRunning() {
+		return nil
+	}
+
+	r.logger.Log("Killing", r.Name())
+	if out, err := r.env.exec("kubernetes", "delete", "deployment", r.Name(), "--ignore-not-found", "--grace-period", "0", "--force").CombinedOutput(); err != nil {
+		r.logger.Log(string(out))
+		return err
+	}
+	if out, err := r.env.exec("kubernetes", "delete", "service", r.Name(), "--ignore-not-found", "--grace-period", "0", "--force").CombinedOutput(); err != nil {
+		r.logger.Log(string(out))
+		return err
+	}
+
+	defer r.mutex.Unlock()
+	r.mutex.Lock()
+	r.running = false
+	return r.env.registerStopped(r.Name())
+}
+
+// Endpoint returns the external service endpoint (host:port) for a given port name.
+// External means that it will be accessible from the host.
+// If the service is not running, this method returns the incorrect `stopped` endpoint.
+func (r *kindRunnable) Endpoint(portName string) string {
+	if !r.IsRunning() {
+		return "stopped"
+	}
+
+	defer r.mutex.Unlock()
+	r.mutex.Lock()
+
+	// Map the container port to the local port.
+	localPort, ok := r.hostPorts[portName]
+	if !ok {
+		return ""
+	}
+
+	return fmt.Sprintf("%s:%d", r.env.nodeIP, localPort)
+}
+
+// InternalEndpoint returns the internal service endpoint (host:port) for a given internal port.
+// Internal means that it will be accessible only from containers in the environment that this
+// service is running in. Use `Endpoint` for host access.
+func (r *kindRunnable) InternalEndpoint(portName string) string {
+	defer r.mutex.Unlock()
+	r.mutex.Lock()
+
+	// Map the port name to the container port.
+	port, ok := r.ports[portName]
+	if !ok {
+		return ""
+	}
+
+	return fmt.Sprintf("%s:%d", r.Name(), port)
+}
+
+func (r *kindRunnable) Ready() error {
+	if !r.IsRunning() {
+		return errors.Newf("service %s is stopped", r.Name())
+	}
+
+	r.mutex.Lock()
+	readiness := r.opts.Readiness
+	r.mutex.Unlock()
+	// Ensure the service has a readiness probe configured.
+	if readiness == nil {
+		return nil
+	}
+
+	return readiness.Ready(r)
+}
+
+func (r *kindRunnable) waitForRunning() (err error) {
+	if !r.running {
+		return errors.Newf("service %s is stopped", r.Name())
+	}
+
+	var out []byte
+	for r.waitBackoffReady.Reset(); r.waitBackoffReady.Ongoing(); {
+		// Enforce a timeout on the command execution because we've seen some flaky tests
+		// stuck here.
+
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		out, err = r.env.execContext(
+			ctx,
+			"kubectl",
+			"--kubeconfig",
+			r.env.kubeconfig(),
+			"wait",
+			"pod",
+			"--for",
+			"condition=Ready",
+			"--selector",
+			fmt.Sprintf("app.kubernetes.io/name=%s", r.Name()),
+			"--timeout",
+			"5s",
+		).CombinedOutput()
+		if err != nil {
+			r.waitBackoffReady.Wait()
+			continue
+		}
+
+		return nil
+	}
+
+	if len(out) > 0 {
+		r.logger.Log(string(out))
+	}
+	return errors.Wrapf(err, "pod %q failed to start", r.Name())
+}
+
+// We want to pre-pull all images using Docker and then load them into the cluster.
+// This ensures that the cluster will always have access to any locally built images.
+func (r *kindRunnable) prePullImage(ctx context.Context) (err error) {
+	if r.running {
+		return errors.Newf("service %q is running; expected stopped", r.Name())
+	}
+
+	if _, err = r.env.execContext(ctx, "docker", "image", "inspect", r.opts.Image).CombinedOutput(); err == nil {
+		return nil
+	}
+
+	// Assuming Error: No such image: <image>.
+	cmd := r.env.execContext(ctx, "docker", "pull", r.opts.Image)
+	l := &LinePrefixLogger{prefix: r.Name() + ": ", logger: r.logger}
+	cmd.Stdout = l
+	cmd.Stderr = l
+	if err = cmd.Run(); err != nil {
+		return errors.Wrapf(err, "docker image %q failed to download", r.opts.Image)
+	}
+
+	if err := r.env.execContext(ctx, "docker", "image", "inspect", r.opts.Image).Run(); err == nil {
+		return errors.Wrapf(err, "load image %q into cluster", r.opts.Image)
+
+	}
+
+	return nil
+}
+
+func (r *kindRunnable) WaitReady() (err error) {
+	if !r.IsRunning() {
+		return errors.Newf("service %s is stopped", r.Name())
+	}
+
+	for r.waitBackoffReady.Reset(); r.waitBackoffReady.Ongoing(); {
+		err = r.Ready()
+		if err == nil {
+			return nil
+		}
+
+		r.waitBackoffReady.Wait()
+	}
+	return errors.Wrapf(err, "service %q is not ready", r.Name())
+}
+
+// Exec runs the provided command in the container specified by this
+// service.
+func (r *kindRunnable) Exec(command Command, opts ...ExecOption) error {
+	if !r.IsRunning() {
+		return errors.Newf("service %q is stopped", r.Name())
+	}
+
+	l := &LinePrefixLogger{prefix: r.Name() + "-exec: ", logger: r.logger}
+	o := ExecOptions{Stdout: l, Stderr: l}
+	for _, opt := range opts {
+		opt(&o)
+	}
+
+	args := []string{"kubectl", "--kubeconfig", r.env.kubeconfig(), "exec", "deployment/" + r.name, "--"}
+	args = append(args, command.Cmd)
+	args = append(args, command.Args...)
+	cmd := r.env.exec(args[0], args[1:]...)
+	cmd.Stdout = o.Stdout
+	cmd.Stderr = o.Stderr
+	return cmd.Run()
+}
diff --git a/env_kind_ext_test.go b/env_kind_ext_test.go
new file mode 100644
index 0000000..250af22
--- /dev/null
+++ b/env_kind_ext_test.go
@@ -0,0 +1,20 @@
+// Copyright (c) The EfficientGo Authors.
+// Licensed under the Apache License 2.0.
+
+package e2e_test
+
+import (
+	"testing"
+
+	"github.com/efficientgo/core/testutil"
+	"github.com/efficientgo/e2e"
+)
+
+func TestKindEnvironment(t *testing.T) {
+	t.Parallel()
+
+	e, err := e2e.NewKindEnvironment()
+	testutil.Ok(t, err)
+	t.Cleanup(e.Close)
+	testEnvironment(t, e)
+}
diff --git a/env_kind_test.go b/env_kind_test.go
new file mode 100644
index 0000000..6ec40c9
--- /dev/null
+++ b/env_kind_test.go
@@ -0,0 +1,343 @@
+// Copyright (c) The EfficientGo Authors.
+// Licensed under the Apache License 2.0.
+
+package e2e
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestKindManifest(t *testing.T) {
+	for _, tc := range []struct {
+		values kindManifestValues
+		out    string
+	}{
+		{
+			values: kindManifestValues{
+				Name:  "simple",
+				Image: "alpine",
+			},
+			out: `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: "simple"
+  name: "simple"
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: "simple"
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: "simple"
+    spec:
+      containers:
+      - name: "simple"
+        image: "alpine"
+`,
+		},
+		{
+			values: kindManifestValues{
+				Name:    "command-and-args",
+				Image:   "debian",
+				Command: "bash",
+				Args:    []string{"-c", "tail", "-f", "/dev/null"},
+			},
+			out: `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: "command-and-args"
+  name: "command-and-args"
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: "command-and-args"
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: "command-and-args"
+    spec:
+      containers:
+      - name: "command-and-args"
+        image: "debian"
+        command:
+        - "bash"
+        args:
+        - "-c"
+        - "tail"
+        - "-f"
+        - "/dev/null"
+`,
+		},
+		{
+			values: kindManifestValues{
+				Name:    "command-and-args-and-env-and-ports",
+				Image:   "debian",
+				Command: "bash",
+				Args:    []string{"-c", "tail", "-f", "/dev/null"},
+				Ports: map[string]int{
+					"http":    8080,
+					"metrics": 9090,
+				},
+				Envs: map[string]string{
+					"FOO": "bar",
+					"BAZ": "qux",
+				},
+			},
+			out: `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: "command-and-args-and-env-and-ports"
+  name: "command-and-args-and-env-and-ports"
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: "command-and-args-and-env-and-ports"
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: "command-and-args-and-env-and-ports"
+    spec:
+      containers:
+      - name: "command-and-args-and-env-and-ports"
+        image: "debian"
+        command:
+        - "bash"
+        args:
+        - "-c"
+        - "tail"
+        - "-f"
+        - "/dev/null"
+        ports:
+        - name: "http"
+          containerPort: 8080
+        - name: "metrics"
+          containerPort: 9090
+        env:
+        - name: "BAZ"
+          value: "qux"
+        - name: "FOO"
+          value: "bar"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: "command-and-args-and-env-and-ports"
+  name: "command-and-args-and-env-and-ports"
+spec:
+  type: NodePort
+  selector:
+    app.kubernetes.io/name: "command-and-args-and-env-and-ports"
+  ports:
+  - name: "http"
+    port: 8080
+  - name: "metrics"
+    port: 9090
+`,
+		},
+		{
+			values: kindManifestValues{
+				Name:    "command-and-args-and-resources-and-security-context",
+				Image:   "debian",
+				Command: "bash",
+				Args:    []string{"-c", "tail", "-f", "/dev/null"},
+				Bytes:   1024,
+				CPUs:    2.5,
+				Capabilities: []RunnableCapabilities{
+					RunnableCapabilitiesSysAdmin,
+				},
+			},
+			out: `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: "command-and-args-and-resources-and-security-context"
+  name: "command-and-args-and-resources-and-security-context"
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: "command-and-args-and-resources-and-security-context"
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: "command-and-args-and-resources-and-security-context"
+    spec:
+      containers:
+      - name: "command-and-args-and-resources-and-security-context"
+        image: "debian"
+        command:
+        - "bash"
+        args:
+        - "-c"
+        - "tail"
+        - "-f"
+        - "/dev/null"
+        resources:
+          limits:
+            memory: 1024
+            cpu: 2.5
+          requests:
+            memory: 1024
+            cpu: 2.5
+        securityContext:
+          capabilities:
+            add:
+            - SYS_ADMIN
+`,
+		},
+		{
+			values: kindManifestValues{
+				Name:    "command-and-args-and-resources-and-security-context2",
+				Image:   "debian",
+				Command: "bash",
+				Args:    []string{"-c", "tail", "-f", "/dev/null"},
+				Bytes:   1024,
+				CPUs:    2.5,
+				Capabilities: []RunnableCapabilities{
+					RunnableCapabilitiesSysAdmin,
+				},
+				Privileged: true,
+			},
+			out: `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: "command-and-args-and-resources-and-security-context2"
+  name: "command-and-args-and-resources-and-security-context2"
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: "command-and-args-and-resources-and-security-context2"
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: "command-and-args-and-resources-and-security-context2"
+    spec:
+      containers:
+      - name: "command-and-args-and-resources-and-security-context2"
+        image: "debian"
+        command:
+        - "bash"
+        args:
+        - "-c"
+        - "tail"
+        - "-f"
+        - "/dev/null"
+        resources:
+          limits:
+            memory: 1024
+            cpu: 2.5
+          requests:
+            memory: 1024
+            cpu: 2.5
+        securityContext:
+          privileged: true
+          capabilities:
+            add:
+            - SYS_ADMIN
+`,
+		},
+		{
+			values: kindManifestValues{
+				Name:    "command-and-args-and-resources-and-security-context3",
+				Image:   "debian",
+				Command: "bash",
+				Args:    []string{"-c", "tail", "-f", "/dev/null"},
+				Bytes:   1024,
+				CPUs:    2.5,
+				User:    "1000",
+			},
+			out: `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: "command-and-args-and-resources-and-security-context3"
+  name: "command-and-args-and-resources-and-security-context3"
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: "command-and-args-and-resources-and-security-context3"
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: "command-and-args-and-resources-and-security-context3"
+    spec:
+      containers:
+      - name: "command-and-args-and-resources-and-security-context3"
+        image: "debian"
+        command:
+        - "bash"
+        args:
+        - "-c"
+        - "tail"
+        - "-f"
+        - "/dev/null"
+        resources:
+          limits:
+            memory: 1024
+            cpu: 2.5
+          requests:
+            memory: 1024
+            cpu: 2.5
+        securityContext:
+          runAsUser: 1000
+`,
+		},
+		{
+			values: kindManifestValues{
+				Name:  "volumes",
+				Image: "debian",
+				Volumes: map[string]string{
+					"foo": "/bar",
+					"baz": "/qux",
+				},
+			},
+			out: `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: "volumes"
+  name: "volumes"
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: "volumes"
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: "volumes"
+    spec:
+      containers:
+      - name: "volumes"
+        image: "debian"
+        volumeMounts:
+        - name: "baz"
+          mountPath: /qux
+        - name: "foo"
+          mountPath: /bar
+      volumes:
+      - name: "baz"
+        hostPath:
+          path: /qux
+      - name: "foo"
+        hostPath:
+          path: /bar
+`,
+		},
+	} {
+		t.Run(tc.values.Name, func(t *testing.T) {
+			var buf bytes.Buffer
+			if err := kindManifest.Execute(&buf, tc.values); err != nil {
+				t.Fatal(err)
+			}
+			if buf.String() != tc.out {
+				t.Errorf("got\n%s\nexpected\n%s", buf.String(), tc.out)
+			}
+		})
+	}
+}

From e8731fc063f94b7f6f2599d5731bfc39b025d8e8 Mon Sep 17 00:00:00 2001
From: Douglas Camata <159076+douglascamata@users.noreply.github.com>
Date: Thu, 13 Apr 2023 18:29:04 +0200
Subject: [PATCH 09/11] Create a Between MetricValueExpectation (#65)

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>
---
 monitoring/metrics.go | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/monitoring/metrics.go b/monitoring/metrics.go
index ba4b7af..7471c90 100644
--- a/monitoring/metrics.go
+++ b/monitoring/metrics.go
@@ -201,6 +201,17 @@ func Less(value float64) MetricValueExpectation {
 	}
 }
 
+// Between is a MetricValueExpectation function for WaitSumMetrics that returns true if given single sum is between
+// the lower and upper bounds (non-inclusive, as in `lower < x < upper`).
+func Between(lower, upper float64) MetricValueExpectation {
+	return func(sums ...float64) bool {
+		if len(sums) != 1 {
+			panic("between: expected one value")
+		}
+		return sums[0] > lower && sums[0] < upper
+	}
+}
+
 // EqualsAmongTwo is an isExpected function for WaitSumMetrics that returns true if first sum is equal to the second.
 // NOTE: Be careful on scrapes in between of process that changes two metrics. Those are
 // usually not atomic.

From 465da2b04fc1d4a0009363cd39ede7ea095e5c59 Mon Sep 17 00:00:00 2001
From: Rasek91 <47672951+Rasek91@users.noreply.github.com>
Date: Thu, 13 Apr 2023 18:30:36 +0200
Subject: [PATCH 10/11] add stdin as exec option (#60)

---
 env.go        | 9 +++++++++
 env_docker.go | 6 ++++++
 2 files changed, 15 insertions(+)

diff --git a/env.go b/env.go
index 389294c..bb2af0f 100644
--- a/env.go
+++ b/env.go
@@ -191,10 +191,19 @@ type runnable interface {
 type ExecOption func(o *ExecOptions)
 
 type ExecOptions struct {
+	Stdin  io.Reader
 	Stdout io.Writer
 	Stderr io.Writer
 }
 
+// WithExecOptionStdin sets stdin reader to be used when exec is performed.
+// By default, it is nil.
+func WithExecOptionStdin(stdin io.Reader) ExecOption {
+	return func(o *ExecOptions) {
+		o.Stdin = stdin
+	}
+}
+
 // WithExecOptionStdout sets stdout writer to be used when exec is performed.
 // By default, it is streaming to the env logger.
 func WithExecOptionStdout(stdout io.Writer) ExecOption {
diff --git a/env_docker.go b/env_docker.go
index fc4a440..f6419e0 100644
--- a/env_docker.go
+++ b/env_docker.go
@@ -677,7 +677,13 @@ func (d *dockerRunnable) Exec(command Command, opts ...ExecOption) error {
 	args := []string{"exec", d.containerName()}
 	args = append(args, command.Cmd)
 	args = append(args, command.Args...)
+	if o.Stdin != nil {
+		args = append(args[:1], append([]string{"-i"}, args[1:]...)...)
+	}
 	cmd := d.env.exec("docker", args...)
+	if o.Stdin != nil {
+		cmd.Stdin = o.Stdin
+	}
 	cmd.Stdout = o.Stdout
 	cmd.Stderr = o.Stderr
 	return cmd.Run()

From 7b7b104fccbaa7b6d676a694cc6680e44d761e38 Mon Sep 17 00:00:00 2001
From: Bartlomiej Plotka <bwplotka@gmail.com>
Date: Mon, 24 Apr 2023 11:20:21 +0200
Subject: [PATCH 11/11] Added run until endpoint hit with port. (#66)

Signed-off-by: bwplotka <bwplotka@gmail.com>
---
 interactive/interactive.go      | 11 ++++++++---
 interactive/interactive_test.go | 26 ++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/interactive/interactive.go b/interactive/interactive.go
index e5802c8..94d7bec 100644
--- a/interactive/interactive.go
+++ b/interactive/interactive.go
@@ -45,13 +45,18 @@ func OpenInBrowser(url string) error {
 // This function is useful when you want to interact with e2e tests and manually decide when to finish. Use this function
 // as opposed to RunUntilSignal for certain IDEs that does not send correct signal on stop (e.g. Goland pre 2022.3,
 // see https://youtrack.jetbrains.com/issue/GO-5982).
-func RunUntilEndpointHit() (err error) {
+func RunUntilEndpointHit() error {
+	return RunUntilEndpointHitWithPort(0)
+}
+
+// RunUntilEndpointHitWithPort is like RunUntilEndpointHit, but it allows specifying static port.
+func RunUntilEndpointHitWithPort(port int) (err error) {
 	once := sync.Once{}
 	wg := sync.WaitGroup{}
 	stopWG := sync.WaitGroup{}
 	stopWG.Add(1)
 
-	l, err := net.Listen("tcp", "localhost:0")
+	l, err := net.Listen("tcp", fmt.Sprintf("localhost:%v", port))
 	if err != nil {
 		return err
 	}
@@ -81,7 +86,7 @@ func RunUntilEndpointHit() (err error) {
 		wg.Done()
 	}()
 
-	fmt.Println("Waiting for user HTTP request on ", "http://"+l.Addr().String(), " or SIGINT, SIGKILL or SIGHUP signal...")
+	fmt.Println("Waiting for user HTTP request on", "http://"+l.Addr().String(), " or SIGINT, SIGKILL or SIGHUP signal...")
 	stopWG.Wait()
 
 	// Cleanup.
diff --git a/interactive/interactive_test.go b/interactive/interactive_test.go
index 1990a2a..d62f8d1 100644
--- a/interactive/interactive_test.go
+++ b/interactive/interactive_test.go
@@ -5,6 +5,8 @@ package e2einteractive
 
 import (
 	"context"
+	"fmt"
+	"net/http"
 	"os"
 	"syscall"
 	"testing"
@@ -15,6 +17,30 @@ import (
 	"github.com/efficientgo/core/testutil"
 )
 
+func TestRunUntilEndpointHitWithPort(t *testing.T) {
+	const port = 12131
+
+	s := make(chan struct{})
+	go func() {
+		testutil.Ok(t, RunUntilEndpointHitWithPort(port))
+		s <- struct{}{}
+		close(s)
+	}()
+
+	r, err := http.Get(fmt.Sprintf("http://localhost:%v", port))
+	testutil.Ok(t, err)
+	testutil.Equals(t, 200, r.StatusCode)
+
+	testutil.Ok(t, runutil.Retry(200*time.Millisecond, context.Background().Done(), func() error {
+		select {
+		case <-s:
+			return nil
+		default:
+			return errors.New("execution is still stopped")
+		}
+	}))
+}
+
 func TestRunUntilEndpointHit_Signal(t *testing.T) {
 	s := make(chan struct{})
 	go func() {