no more docker

README.md

@@ -53,7 +53,6 @@ All available options and their values are in [testdata/default.toml](./testdata
 * A recent version of `qemu` (8.1.3 is known to work)
 * A Linux kernel with the necessary configuration (>= 4.9 is known to work)
 * KVM (optional, see [VIMTO_DISABLE_KVM](docs/tips.md))
-* Docker (optional, to fetch kernels from OCI registries)
 
 Here is a non-exhaustive list of required Linux options:
 

docs/tips.md

@@ -23,3 +23,17 @@ GOTRACEBACK=crash vimto -- go test ...
 [`GOTRACEBACK`](https://pkg.go.dev/runtime) is interpreted by the Go runtime.
 `vimto` will preserve the test binaries if it detects a core dump. This allows
 you to collect the binary and the core dump in CI for later debugging.
+
+## Debug the kernel using GDB
+
+You can debug the kernel by passing the `-gdb` flag:
+
+```
+$ go test -c .
+$ vimto -gdb -kernel :4.9 exec -- pkg.test
+Starting GDB server with CPU halted, connect using:
+        gdb -ex 'target remote localhost:1234' -ex '[...]'
+```
+
+This works best if the image you are using contains an uncompressed `vmlinux`
+which includes debug symbols.

go.mod

@@ -1,41 +1,53 @@
 module lmb.io/vimto
 
-go 1.21
+go 1.23.0
 
 require (
 	github.com/BurntSushi/toml v1.3.2
 	github.com/creack/pty/v2 v2.0.1
-	github.com/docker/docker v24.0.7+incompatible
+	github.com/docker/docker v27.5.0+incompatible
 	github.com/go-quicktest/qt v1.101.0
+	github.com/google/go-containerregistry v0.20.3
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
+	github.com/schollz/progressbar/v3 v3.18.0
 	github.com/u-root/u-root v0.11.0
-	golang.org/x/sync v0.6.0
-	golang.org/x/sys v0.13.0
+	golang.org/x/sync v0.10.0
+	golang.org/x/sys v0.29.0
 	rsc.io/script v0.0.2-0.20231205190631-334f6c18cff3
 )
 
 require (
-	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
-	github.com/Microsoft/go-winio v0.4.14 // indirect
-	github.com/distribution/reference v0.5.0 // indirect
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect
+	github.com/adrg/xdg v0.5.3 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect
+	github.com/docker/cli v27.5.0+incompatible // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
-	github.com/docker/go-connections v0.5.0 // indirect
-	github.com/docker/go-units v0.5.0 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/docker/docker-credential-helpers v0.8.2 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/goexpect v0.0.0-20191001010744-5b6988669ffa // indirect
 	github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/moby/term v0.5.0 // indirect
-	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/user v0.3.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/stretchr/testify v1.10.0 // indirect
+	github.com/vbatts/tar-split v0.11.6 // indirect
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
-	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.14.0 // indirect
-	google.golang.org/grpc v1.29.1 // indirect
+	golang.org/x/term v0.28.0 // indirect
+	golang.org/x/tools v0.29.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d // indirect
+	google.golang.org/grpc v1.69.4 // indirect
 	gotest.tools/v3 v3.5.1 // indirect
 )

image.go

@@ -1,20 +1,24 @@
 package main
 
 import (
-	"archive/tar"
 	"context"
-	"encoding/json"
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
-	"strings"
-
-	"github.com/docker/docker/api/types"
-	docker "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/jsonmessage"
+	"runtime"
+	"slices"
+	"time"
+
+	"github.com/adrg/xdg"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/schollz/progressbar/v3"
 	"golang.org/x/sys/unix"
 )
 
@@ -23,46 +27,45 @@ import (
 //
 // The main concern is startup speed of vimto.
 type imageCache struct {
-	cli     *docker.Client
 	baseDir string
 }
 
-func newImageCache(cli *docker.Client) (*imageCache, error) {
-	allCache := filepath.Join(os.TempDir(), "vimto")
-	if err := os.MkdirAll(allCache, 0777); err != nil {
-		return nil, err
-	}
+var userCacheDir = filepath.Join(xdg.CacheHome, "vimto")
 
-	uid := os.Getuid()
-	userCache := filepath.Join(allCache, fmt.Sprint(uid))
-	if err := os.Mkdir(userCache, 0755); err != nil && !errors.Is(err, os.ErrExist) {
+func newImageCache() (*imageCache, error) {
+	if err := os.MkdirAll(userCacheDir, 0700); err != nil && !errors.Is(err, os.ErrExist) {
 		return nil, fmt.Errorf("create cache directory: %w", err)
 	}
 
-	return &imageCache{cli, userCache}, nil
+	return &imageCache{userCacheDir}, nil
 }
 
 // Acquire an image from the cache.
 //
 // The image remains valid even after closing the cache.
-func (ic *imageCache) Acquire(ctx context.Context, img string, status io.Writer) (_ *image, err error) {
-	refStr, digest, err := fetchImage(ctx, ic.cli, img, status)
+func (ic *imageCache) Acquire(ctx context.Context, refStr string, status io.Writer) (_ *image, err error) {
+	ref, err := name.ParseReference(refStr)
 	if err != nil {
-		return nil, fmt.Errorf("fetch image: %w", err)
+		return nil, fmt.Errorf("parsing reference %q: %w", refStr, err)
 	}
 
-	// Replace sha256:deadbeef with sha256@deadbeef to avoid colon being
-	// interpreted as a path separator.
-	digest = strings.Replace(digest, ":", "@", 1)
+	// Use the sha256 of the canonical reference as the cache key. This means
+	// that images / tags pointing at the blob will have separate cache entries.
+	dir := fmt.Sprintf("%x", sha256.Sum256([]byte(ref.Name())))
+
+	lock, path, err := populateDirectoryOnce(filepath.Join(ic.baseDir, dir), func(path string) error {
+		err := fetchImage(ctx, refStr, path, status)
+		if err != nil {
+			return fmt.Errorf("fetch image: %w", err)
+		}
 
-	lock, path, err := populateDirectoryOnce(filepath.Join(ic.baseDir, digest), func(path string) error {
-		return extractImage(ctx, ic.cli, refStr, path)
+		return nil
 	})
 	if err != nil {
 		return nil, err
 	}
 
-	return &image{img, path, lock}, nil
+	return &image{refStr, path, lock}, nil
 }
 
 func populateDirectoryOnce(path string, fn func(string) error) (lock *os.File, _ string, err error) {
@@ -187,143 +190,68 @@ func newBootFilesFromImage(img *image) (*bootFiles, error) {
 	return bf, nil
 }
 
-func fetchImage(ctx context.Context, cli *docker.Client, refStr string, status io.Writer) (string, string, error) {
-	if refStr, digest, err := imageID(ctx, cli, refStr); err == nil {
-		// Found a cached image, use that.
-		// TODO: We don't notice if the tag changes since we don't pull
-		// again if we can resolve refStr to id locally.
-		return refStr, digest, nil
-	}
+var remoteOptions = []remote.Option{
+	remote.WithUserAgent("vimto"),
+	remote.WithPlatform(v1.Platform{
+		OS:           "linux",
+		Architecture: runtime.GOARCH,
+	}),
+}
 
-	remotePullReader, err := cli.ImagePull(ctx, refStr, types.ImagePullOptions{})
+func fetchImage(ctx context.Context, refStr, dst string, status io.Writer) error {
+	ref, err := name.ParseReference(refStr)
 	if err != nil {
-		return "", "", fmt.Errorf("cannot pull image %s: %w", refStr, err)
-	}
-	defer remotePullReader.Close()
-
-	isTTY := false
-	if f, ok := status.(*os.File); ok {
-		isTTY, err = fileIsTTY(f)
-		if err != nil {
-			return "", "", fmt.Errorf("check whether output is tty: %w", err)
-		}
-	}
-
-	decoder := json.NewDecoder(remotePullReader)
-	for {
-		var pullResponse jsonmessage.JSONMessage
-		if err := decoder.Decode(&pullResponse); errors.Is(err, io.EOF) {
-			break
-		} else if err != nil {
-			return "", "", err
-		}
-
-		if err := pullResponse.Display(status, isTTY); err != nil {
-			return "", "", fmt.Errorf("docker response: %w", pullResponse.Error)
-		}
+		return fmt.Errorf("parsing reference %q: %w", refStr, err)
 	}
 
-	return imageID(ctx, cli, refStr)
-}
-
-func imageID(ctx context.Context, cli *docker.Client, refStr string) (string, string, error) {
-	image, _, err := cli.ImageInspectWithRaw(ctx, refStr)
+	bar := progressbar.NewOptions64(
+		-1,
+		progressbar.OptionSetDescription(fmt.Sprintf("Caching %s", ref.Name())),
+		progressbar.OptionSetWriter(status),
+		progressbar.OptionShowBytes(true),
+		progressbar.OptionShowTotalBytes(false),
+		progressbar.OptionThrottle(65*time.Millisecond),
+		progressbar.OptionShowCount(),
+		progressbar.OptionSpinnerType(14),
+		progressbar.OptionFullWidth(),
+		progressbar.OptionSetRenderBlankState(true),
+		progressbar.OptionClearOnFinish(),
+	)
+	defer bar.Finish()
+
+	options := append(slices.Clone(remoteOptions),
+		remote.WithContext(ctx),
+	)
+
+	rmt, err := remote.Get(ref, options...)
 	if err != nil {
-		return "", "", fmt.Errorf("inspect image: %w", err)
+		return fmt.Errorf("get from remote: %w", err)
 	}
 
-	if len(image.RepoDigests) < 1 {
-		return "", "", fmt.Errorf("no digest for %q", refStr)
+	image, err := rmt.Image()
+	if err != nil {
+		return err
 	}
 
-	return image.RepoDigests[0], image.ID, nil
-}
-
-func extractImage(ctx context.Context, cli *docker.Client, image, dst string) error {
-	cmd := exec.CommandContext(ctx, "docker", "buildx", "build", "--quiet", "--output", dst, "-")
-	cmd.Stdin = strings.NewReader(fmt.Sprintf("FROM %s\n", image))
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("%w: %s", err, string(out))
+	if err := os.MkdirAll(dst, 0755); err != nil {
+		return fmt.Errorf("create destination directory: %w", err)
 	}
-	return nil
-}
 
-func extractTar(r io.Reader, path string) error {
-	tr := tar.NewReader(r)
-	for {
-		hdr, err := tr.Next()
-		if err == io.EOF {
-			return nil
-		}
-		if err != nil {
-			return fmt.Errorf("read tar header: %w", err)
-		}
+	rc := mutate.Extract(image)
+	defer rc.Close()
 
-		dstPath, err := secureJoin(path, path, hdr.Name)
-		if err != nil {
-			return err
-		}
+	reader := readProxy{rc, bar}
 
-		switch hdr.Typeflag {
-		case tar.TypeDir:
-			if err := os.MkdirAll(dstPath, 0755); err != nil {
-				return err
-			}
-
-		case tar.TypeReg:
-			dst, err := os.Create(dstPath)
-			if err != nil {
-				return err
-			}
-			defer dst.Close()
-
-			_, err = io.Copy(dst, tr)
-			if err != nil {
-				return err
-			}
-
-		case tar.TypeLink:
-			srcPath, err := secureJoin(path, path, hdr.Linkname)
-			if err != nil {
-				return fmt.Errorf("hard link: %w", err)
-			}
-
-			if err := os.Link(srcPath, dstPath); err != nil {
-				return err
-			}
-
-		case tar.TypeSymlink:
-			// Relative symlinks start from the location of the symlink.
-			srcPath, err := secureJoin(path, filepath.Dir(dstPath), hdr.Linkname)
-			if err != nil {
-				return fmt.Errorf("sym link: %w (dst: %s)", err, dstPath)
-			}
-
-			if err := os.Symlink(srcPath, dstPath); err != nil {
-				return err
-			}
-
-		default:
-			return fmt.Errorf("unexpected tar header type %d", hdr.Typeflag)
-		}
-	}
+	return archive.UntarUncompressed(reader, dst, &archive.TarOptions{NoLchown: true})
 }
 
-func secureJoin(base string, parts ...string) (string, error) {
-	base, err := filepath.Abs(base)
-	if err != nil {
-		return "", err
-	}
-
-	path := filepath.Join(parts...)
-	if !filepath.IsAbs(path) {
-		path = filepath.Join(base, path)
-	}
-
-	if !strings.HasPrefix(path, base) {
-		return "", fmt.Errorf("invalid path %q (escapes %s)", path, base)
-	}
+type readProxy struct {
+	io.Reader
+	*progressbar.ProgressBar
+}
 
-	return path, nil
+func (rp readProxy) Read(p []byte) (int, error) {
+	n, err := rp.Reader.Read(p)
+	rp.ProgressBar.Add(n)
+	return n, err
 }