From c8a6b5ac0f8b7f350649ca0da40307a44a980d8a Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Wed, 22 Jan 2025 11:16:27 +0000 Subject: [PATCH 1/3] don't depend on docker anymore Use crane to pull images directly from the registry, without going via the docker daemon. --- README.md | 1 - go.mod | 47 ++++++----- go.sum | 192 +++++++++++++++++--------------------------- image.go | 218 +++++++++++++++++--------------------------------- image_test.go | 66 +-------------- main.go | 9 +-- main_test.go | 9 +-- 7 files changed, 182 insertions(+), 360 deletions(-) diff --git a/README.md b/README.md index b7918d8..efb492d 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,6 @@ All available options and their values are in [testdata/default.toml](./testdata * A recent version of `qemu` (8.1.3 is known to work) * A Linux kernel with the necessary configuration (>= 4.9 is known to work) * KVM (optional, see [VIMTO_DISABLE_KVM](docs/tips.md)) -* Docker (optional, to fetch kernels from OCI registries) Here is a non-exhaustive list of required Linux options: diff --git a/go.mod b/go.mod index 93cb261..c288362 100644 --- a/go.mod +++ b/go.mod @@ -1,41 +1,52 @@ module lmb.io/vimto -go 1.21 +go 1.23.0 require ( github.com/BurntSushi/toml v1.3.2 github.com/creack/pty/v2 v2.0.1 - github.com/docker/docker v24.0.7+incompatible + github.com/docker/docker v27.5.0+incompatible github.com/go-quicktest/qt v1.101.0 + github.com/google/go-containerregistry v0.20.3 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 + github.com/schollz/progressbar/v3 v3.18.0 github.com/u-root/u-root v0.11.0 - golang.org/x/sync v0.6.0 - golang.org/x/sys v0.13.0 + golang.org/x/sync v0.10.0 + golang.org/x/sys v0.29.0 rsc.io/script v0.0.2-0.20231205190631-334f6c18cff3 ) require ( - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Microsoft/go-winio v0.4.14 // indirect - github.com/distribution/reference v0.5.0 // indirect + github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect + github.com/docker/cli v27.5.0+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/go-connections v0.5.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/docker/docker-credential-helpers v0.8.2 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/goexpect v0.0.0-20191001010744-5b6988669ffa // indirect github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/moby/term v0.5.0 // indirect - github.com/morikuni/aec v1.0.0 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.3.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.2 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/stretchr/testify v1.10.0 // indirect + github.com/vbatts/tar-split v0.11.6 // indirect golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect - golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.14.0 // indirect - google.golang.org/grpc v1.29.1 // indirect + golang.org/x/term v0.28.0 // indirect + golang.org/x/tools v0.29.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d // indirect + google.golang.org/grpc v1.69.4 // indirect gotest.tools/v3 v3.5.1 // indirect ) diff --git a/go.sum b/go.sum index 7bab460..4aaf219 100644 --- a/go.sum +++ b/go.sum @@ -1,153 +1,109 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8= +github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty/v2 v2.0.1 h1:RDY1VY5b+7m2mfPsugucOYPIxMp+xal5ZheSyVzUA+k= github.com/creack/pty/v2 v2.0.1/go.mod h1:2dSssKp3b86qYEMwA/FPwc3ff+kYpDdQI8osU8J7gxQ= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= -github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/cli v27.5.0+incompatible h1:aMphQkcGtpHixwwhAXJT1rrK/detk2JIvDaFkLctbGM= +github.com/docker/cli v27.5.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= -github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/docker/docker v27.5.0+incompatible h1:um++2NcQtGRTz5eEgO6aJimo6/JxrTXC941hd05JO6U= +github.com/docker/docker v27.5.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= +github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= +github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.20.3 h1:oNx7IdTI936V8CQRveCjaxOiegWwvM7kqkbXTpyiovI= +github.com/google/go-containerregistry v0.20.3/go.mod h1:w00pIgBRDVUDFM6bq+Qx8lwNWK+cxgCuX1vd3PIBDNI= github.com/google/goexpect v0.0.0-20191001010744-5b6988669ffa h1:PMkmJA8ju9DjqAJjIzrBdrmhuuPsoNnNLYgKQBopWL0= github.com/google/goexpect v0.0.0-20191001010744-5b6988669ffa/go.mod h1:qtE5aAEkt0vOSA84DBh8aJsz6riL8ONfqfULY7lBjqc= github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2 h1:CVuJwN34x4xM2aT4sIKhmeib40NeBPhRihNjQmpJsA4= github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= +github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= -github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= +github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/u-root/u-root v0.11.0 h1:6gCZLOeRyevw7gbTwMj3fKxnr9+yHFlgF3N7udUVNO8= github.com/u-root/u-root v0.11.0/go.mod h1:DBkDtiZyONk9hzVEdB/PWI9B4TxDkElWlVTHseglrZY= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +github.com/vbatts/tar-split v0.11.6 h1:4SjTW5+PU11n6fZenf2IPoV8/tz3AaYHMWjf23envGs= +github.com/vbatts/tar-split v0.11.6/go.mod h1:dqKNtesIOr2j2Qv3W/cHjnvk9I8+G7oAkFDFN6TCBEI= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d h1:xJJRGY7TJcvIlpSrN3K6LAWgNFUILlO+OMAqtg9aqnw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4= +google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= +google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= +google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= rsc.io/script v0.0.2-0.20231205190631-334f6c18cff3 h1:2vM6uMBq2/Dou/Wzu2p+yUFkuI3lgMbX0UYfVnzh0ck= rsc.io/script v0.0.2-0.20231205190631-334f6c18cff3/go.mod h1:cKBjCtFBBeZ0cbYFRXkRoxP+xGqhArPa9t3VWhtXfzU= diff --git a/image.go b/image.go index 8da9221..933e09a 100644 --- a/image.go +++ b/image.go @@ -1,20 +1,23 @@ package main import ( - "archive/tar" "context" - "encoding/json" + "crypto/sha256" "errors" "fmt" "io" "os" - "os/exec" "path/filepath" - "strings" - - "github.com/docker/docker/api/types" - docker "github.com/docker/docker/client" - "github.com/docker/docker/pkg/jsonmessage" + "runtime" + "slices" + "time" + + "github.com/docker/docker/pkg/archive" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/schollz/progressbar/v3" "golang.org/x/sys/unix" ) @@ -23,11 +26,10 @@ import ( // // The main concern is startup speed of vimto. type imageCache struct { - cli *docker.Client baseDir string } -func newImageCache(cli *docker.Client) (*imageCache, error) { +func newImageCache() (*imageCache, error) { allCache := filepath.Join(os.TempDir(), "vimto") if err := os.MkdirAll(allCache, 0777); err != nil { return nil, err @@ -39,30 +41,35 @@ func newImageCache(cli *docker.Client) (*imageCache, error) { return nil, fmt.Errorf("create cache directory: %w", err) } - return &imageCache{cli, userCache}, nil + return &imageCache{userCache}, nil } // Acquire an image from the cache. // // The image remains valid even after closing the cache. -func (ic *imageCache) Acquire(ctx context.Context, img string, status io.Writer) (_ *image, err error) { - refStr, digest, err := fetchImage(ctx, ic.cli, img, status) +func (ic *imageCache) Acquire(ctx context.Context, refStr string, status io.Writer) (_ *image, err error) { + ref, err := name.ParseReference(refStr) if err != nil { - return nil, fmt.Errorf("fetch image: %w", err) + return nil, fmt.Errorf("parsing reference %q: %w", refStr, err) } - // Replace sha256:deadbeef with sha256@deadbeef to avoid colon being - // interpreted as a path separator. - digest = strings.Replace(digest, ":", "@", 1) + // Use the sha256 of the canonical reference as the cache key. This means + // that images / tags pointing at the blob will have separate cache entries. + dir := fmt.Sprintf("%x", sha256.Sum256([]byte(ref.Name()))) - lock, path, err := populateDirectoryOnce(filepath.Join(ic.baseDir, digest), func(path string) error { - return extractImage(ctx, ic.cli, refStr, path) + lock, path, err := populateDirectoryOnce(filepath.Join(ic.baseDir, dir), func(path string) error { + err := fetchImage(ctx, refStr, path, status) + if err != nil { + return fmt.Errorf("fetch image: %w", err) + } + + return nil }) if err != nil { return nil, err } - return &image{img, path, lock}, nil + return &image{refStr, path, lock}, nil } func populateDirectoryOnce(path string, fn func(string) error) (lock *os.File, _ string, err error) { @@ -187,143 +194,68 @@ func newBootFilesFromImage(img *image) (*bootFiles, error) { return bf, nil } -func fetchImage(ctx context.Context, cli *docker.Client, refStr string, status io.Writer) (string, string, error) { - if refStr, digest, err := imageID(ctx, cli, refStr); err == nil { - // Found a cached image, use that. - // TODO: We don't notice if the tag changes since we don't pull - // again if we can resolve refStr to id locally. - return refStr, digest, nil - } +var remoteOptions = []remote.Option{ + remote.WithUserAgent("vimto"), + remote.WithPlatform(v1.Platform{ + OS: "linux", + Architecture: runtime.GOARCH, + }), +} - remotePullReader, err := cli.ImagePull(ctx, refStr, types.ImagePullOptions{}) +func fetchImage(ctx context.Context, refStr, dst string, status io.Writer) error { + ref, err := name.ParseReference(refStr) if err != nil { - return "", "", fmt.Errorf("cannot pull image %s: %w", refStr, err) - } - defer remotePullReader.Close() - - isTTY := false - if f, ok := status.(*os.File); ok { - isTTY, err = fileIsTTY(f) - if err != nil { - return "", "", fmt.Errorf("check whether output is tty: %w", err) - } + return fmt.Errorf("parsing reference %q: %w", refStr, err) } - decoder := json.NewDecoder(remotePullReader) - for { - var pullResponse jsonmessage.JSONMessage - if err := decoder.Decode(&pullResponse); errors.Is(err, io.EOF) { - break - } else if err != nil { - return "", "", err - } - - if err := pullResponse.Display(status, isTTY); err != nil { - return "", "", fmt.Errorf("docker response: %w", pullResponse.Error) - } - } - - return imageID(ctx, cli, refStr) -} - -func imageID(ctx context.Context, cli *docker.Client, refStr string) (string, string, error) { - image, _, err := cli.ImageInspectWithRaw(ctx, refStr) + bar := progressbar.NewOptions64( + -1, + progressbar.OptionSetDescription(fmt.Sprintf("Caching %s", ref.Name())), + progressbar.OptionSetWriter(status), + progressbar.OptionShowBytes(true), + progressbar.OptionShowTotalBytes(false), + progressbar.OptionThrottle(65*time.Millisecond), + progressbar.OptionShowCount(), + progressbar.OptionSpinnerType(14), + progressbar.OptionFullWidth(), + progressbar.OptionSetRenderBlankState(true), + progressbar.OptionClearOnFinish(), + ) + defer bar.Finish() + + options := append(slices.Clone(remoteOptions), + remote.WithContext(ctx), + ) + + rmt, err := remote.Get(ref, options...) if err != nil { - return "", "", fmt.Errorf("inspect image: %w", err) + return fmt.Errorf("get from remote: %w", err) } - if len(image.RepoDigests) < 1 { - return "", "", fmt.Errorf("no digest for %q", refStr) + image, err := rmt.Image() + if err != nil { + return err } - return image.RepoDigests[0], image.ID, nil -} - -func extractImage(ctx context.Context, cli *docker.Client, image, dst string) error { - cmd := exec.CommandContext(ctx, "docker", "buildx", "build", "--quiet", "--output", dst, "-") - cmd.Stdin = strings.NewReader(fmt.Sprintf("FROM %s\n", image)) - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("%w: %s", err, string(out)) + if err := os.MkdirAll(dst, 0755); err != nil { + return fmt.Errorf("create destination directory: %w", err) } - return nil -} -func extractTar(r io.Reader, path string) error { - tr := tar.NewReader(r) - for { - hdr, err := tr.Next() - if err == io.EOF { - return nil - } - if err != nil { - return fmt.Errorf("read tar header: %w", err) - } + rc := mutate.Extract(image) + defer rc.Close() - dstPath, err := secureJoin(path, path, hdr.Name) - if err != nil { - return err - } + reader := readProxy{rc, bar} - switch hdr.Typeflag { - case tar.TypeDir: - if err := os.MkdirAll(dstPath, 0755); err != nil { - return err - } - - case tar.TypeReg: - dst, err := os.Create(dstPath) - if err != nil { - return err - } - defer dst.Close() - - _, err = io.Copy(dst, tr) - if err != nil { - return err - } - - case tar.TypeLink: - srcPath, err := secureJoin(path, path, hdr.Linkname) - if err != nil { - return fmt.Errorf("hard link: %w", err) - } - - if err := os.Link(srcPath, dstPath); err != nil { - return err - } - - case tar.TypeSymlink: - // Relative symlinks start from the location of the symlink. - srcPath, err := secureJoin(path, filepath.Dir(dstPath), hdr.Linkname) - if err != nil { - return fmt.Errorf("sym link: %w (dst: %s)", err, dstPath) - } - - if err := os.Symlink(srcPath, dstPath); err != nil { - return err - } - - default: - return fmt.Errorf("unexpected tar header type %d", hdr.Typeflag) - } - } + return archive.UntarUncompressed(reader, dst, &archive.TarOptions{NoLchown: true}) } -func secureJoin(base string, parts ...string) (string, error) { - base, err := filepath.Abs(base) - if err != nil { - return "", err - } - - path := filepath.Join(parts...) - if !filepath.IsAbs(path) { - path = filepath.Join(base, path) - } - - if !strings.HasPrefix(path, base) { - return "", fmt.Errorf("invalid path %q (escapes %s)", path, base) - } +type readProxy struct { + io.Reader + *progressbar.ProgressBar +} - return path, nil +func (rp readProxy) Read(p []byte) (int, error) { + n, err := rp.Reader.Read(p) + rp.ProgressBar.Add(n) + return n, err } diff --git a/image_test.go b/image_test.go index 7ccce71..007948e 100644 --- a/image_test.go +++ b/image_test.go @@ -10,14 +10,11 @@ import ( "testing" "time" - "github.com/docker/docker/api/types" - docker "github.com/docker/docker/client" "github.com/go-quicktest/qt" ) func TestCacheAcquire(t *testing.T) { - cli := mustNewDockerClient(t) - cache := imageCache{cli, t.TempDir()} + cache := imageCache{t.TempDir()} img1, err := cache.Acquire(context.Background(), "busybox", io.Discard) qt.Assert(t, qt.IsNil(err)) @@ -35,21 +32,8 @@ func TestCacheAcquire(t *testing.T) { func TestFetchAndExtractImage(t *testing.T) { tmp := t.TempDir() - cli := mustNewDockerClient(t) - cli.ImageRemove(context.Background(), "busybox", types.ImageRemoveOptions{Force: true}) - - refStr, digest, err := fetchImage(context.Background(), cli, "busybox", io.Discard) - qt.Assert(t, qt.IsNil(err)) - - refStr2, digest2, err := imageID(context.Background(), cli, refStr) - qt.Assert(t, qt.IsNil(err)) - qt.Assert(t, qt.Equals(refStr, refStr2)) - qt.Assert(t, qt.Equals(digest, digest2)) - - t.Log("digest is", digest) - - err = extractImage(context.Background(), cli, refStr, tmp) + err := fetchImage(context.Background(), "busybox", tmp, io.Discard) qt.Assert(t, qt.IsNil(err)) _, err = os.Stat(filepath.Join(tmp, "bin", "sh")) @@ -96,49 +80,3 @@ func TestPopulateDirectoryOnce(t *testing.T) { qt.Assert(t, qt.IsNil(<-errs)) qt.Assert(t, qt.IsNil(<-errs)) } - -func TestSecureJoin(t *testing.T) { - for _, tc := range []struct { - base string - parts []string - result string - }{ - { - "/foo", - []string{"/foo/usr/bin", "../../bin/env"}, - "/foo/bin/env", - }, - } { - path, err := secureJoin(tc.base, tc.parts...) - qt.Assert(t, qt.IsNil(err)) - qt.Assert(t, qt.Equals(path, tc.result)) - } -} - -func BenchmarkExtractImage(b *testing.B) { - cli := mustNewDockerClient(b) - refStr, _, err := fetchImage(context.Background(), cli, "busybox", io.Discard) - qt.Assert(b, qt.IsNil(err)) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - tmp := b.TempDir() - b.StartTimer() - - extractImage(context.Background(), cli, refStr, tmp) - - b.StopTimer() - os.RemoveAll(tmp) - b.StartTimer() - } -} - -func mustNewDockerClient(tb testing.TB) *docker.Client { - tb.Helper() - - cli, err := docker.NewClientWithOpts(docker.FromEnv, docker.WithAPIVersionNegotiation()) - qt.Assert(tb, qt.IsNil(err)) - tb.Cleanup(func() { cli.Close() }) - return cli -} diff --git a/main.go b/main.go index 59c0117..cbf71c6 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,6 @@ import ( "slices" "strings" - docker "github.com/docker/docker/client" "github.com/kballard/go-shellquote" "golang.org/x/sys/unix" ) @@ -269,13 +268,7 @@ func findBootFiles(kernel string) (_ *bootFiles, err error) { info, err := os.Stat(kernel) if errors.Is(err, os.ErrNotExist) { // Assume that kernel is a reference to an image. - cli, err := docker.NewClientWithOpts(docker.FromEnv, docker.WithAPIVersionNegotiation()) - if err != nil { - return nil, fmt.Errorf("create docker client: %w", err) - } - defer cli.Close() - - cache, err := newImageCache(cli) + cache, err := newImageCache() if err != nil { return nil, fmt.Errorf("image cache: %w", err) } diff --git a/main_test.go b/main_test.go index da4fc4b..5f7600d 100644 --- a/main_test.go +++ b/main_test.go @@ -16,7 +16,6 @@ import ( "testing" "time" - docker "github.com/docker/docker/client" "github.com/go-quicktest/qt" "golang.org/x/sys/unix" "rsc.io/script" @@ -104,13 +103,7 @@ func kernelImage() string { } var fetchKernelImage = sync.OnceValues(func() (*bootFiles, error) { - cli, err := docker.NewClientWithOpts(docker.FromEnv, docker.WithAPIVersionNegotiation()) - if err != nil { - return nil, err - } - defer cli.Close() - - cache, err := newImageCache(cli) + cache, err := newImageCache() if err != nil { return nil, err } From 4dc9b225becf83d3bdb3756a3ddc657da687ab27 Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Wed, 22 Jan 2025 11:40:33 +0000 Subject: [PATCH 2/3] store cache in XDG cache directory Store the cache in the users cache directory instead of in a temporary directory. This means that the cache will persist across reboots. vimto flush-cache will remove the entire cache. --- go.mod | 1 + go.sum | 2 ++ image.go | 14 +++++--------- main.go | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index c288362..d81b62a 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect + github.com/adrg/xdg v0.5.3 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect github.com/docker/cli v27.5.0+incompatible // indirect diff --git a/go.sum b/go.sum index 4aaf219..a95ff02 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8af github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78= +github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ= github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= diff --git a/image.go b/image.go index 933e09a..092d351 100644 --- a/image.go +++ b/image.go @@ -12,6 +12,7 @@ import ( "slices" "time" + "github.com/adrg/xdg" "github.com/docker/docker/pkg/archive" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -29,19 +30,14 @@ type imageCache struct { baseDir string } -func newImageCache() (*imageCache, error) { - allCache := filepath.Join(os.TempDir(), "vimto") - if err := os.MkdirAll(allCache, 0777); err != nil { - return nil, err - } +var userCacheDir = filepath.Join(xdg.CacheHome, "vimto") - uid := os.Getuid() - userCache := filepath.Join(allCache, fmt.Sprint(uid)) - if err := os.Mkdir(userCache, 0755); err != nil && !errors.Is(err, os.ErrExist) { +func newImageCache() (*imageCache, error) { + if err := os.MkdirAll(userCacheDir, 0700); err != nil && !errors.Is(err, os.ErrExist) { return nil, fmt.Errorf("create cache directory: %w", err) } - return &imageCache{userCache}, nil + return &imageCache{userCacheDir}, nil } // Acquire an image from the cache. diff --git a/main.go b/main.go index cbf71c6..2dd3af7 100644 --- a/main.go +++ b/main.go @@ -45,6 +45,7 @@ Usage: %s [flags] [command] [--] ... Available commands: exec Execute a command inside a VM + flush-cache Clear the image cache Flags: ` @@ -73,6 +74,9 @@ func run(args []string) error { case cmd == "exec": err = execCmd(&cfg, fs.Args()[1:]) + case cmd == "flush-cache": + err = flushCacheCmd(fs.Args()[1:]) + case strings.HasPrefix(cmd, "go"): // This is an invocation of go test, possibly via a pre-relase binary // like go1.21rc2. @@ -353,3 +357,34 @@ func preserveTestBinary(path, wd, corePrefix string) error { _, err = io.Copy(dst, src) return err } + +// flushCacheCmd deletes the image cache directory by first renaming it +// and then removing it to handle concurrent access. +func flushCacheCmd(args []string) error { + if len(args) > 0 { + return fmt.Errorf("flush-cache command takes no arguments") + } + + // Generate a random suffix for the temporary directory + tmp := make([]byte, 4) + if _, err := rand.Read(tmp); err != nil { + return fmt.Errorf("generate random suffix: %w", err) + } + tmpDir := userCacheDir + fmt.Sprintf(".%x", tmp) + + // Rename the cache directory to prevent another process from seeing + // a partially delete cache. + if err := os.Rename(userCacheDir, tmpDir); err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("rename cache directory: %w", err) + } + + // Remove the renamed directory + if err := os.RemoveAll(tmpDir); err != nil { + return fmt.Errorf("remove cache directory: %w", err) + } + + return nil +} From 043fdf7dea14fb7dcc04c07acd75cdec84f06784 Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Wed, 22 Jan 2025 11:53:12 +0000 Subject: [PATCH 3/3] document the -gdb flag Add some documentation around the gdb flag, which I seem to have forgotten. --- docs/tips.md | 14 ++++++++++++++ main.go | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/tips.md b/docs/tips.md index d466405..5486b57 100644 --- a/docs/tips.md +++ b/docs/tips.md @@ -23,3 +23,17 @@ GOTRACEBACK=crash vimto -- go test ... [`GOTRACEBACK`](https://pkg.go.dev/runtime) is interpreted by the Go runtime. `vimto` will preserve the test binaries if it detects a core dump. This allows you to collect the binary and the core dump in CI for later debugging. + +## Debug the kernel using GDB + +You can debug the kernel by passing the `-gdb` flag: + +``` +$ go test -c . +$ vimto -gdb -kernel :4.9 exec -- pkg.test +Starting GDB server with CPU halted, connect using: + gdb -ex 'target remote localhost:1234' -ex '[...]' +``` + +This works best if the image you are using contains an uncompressed `vmlinux` +which includes debug symbols. diff --git a/main.go b/main.go index 2dd3af7..8aa0f45 100644 --- a/main.go +++ b/main.go @@ -113,7 +113,7 @@ func goTestCmd(cfg *config, flags []string, goBinary string, goArgs []string) er } if cfg.GDB != "" { - return fmt.Errorf("can't enable gdb integration when running multiple tests") + return fmt.Errorf("can't enable gdb integration when invoking via go toolchain") } exe, err := findExecutable()